{
  "schema_version": 1,
  "channel": "homelab-runtime",
  "command_id": "BACKUP-OFFHOST-CURRENT-STATE-212",
  "status": "OK",
  "rc": 0,
  "host": "pve01",
  "mode": "read-only",
  "component": "backup-audit",
  "started_at_utc": "2026-07-22T14:53:51Z",
  "finished_at_utc": "2026-07-22T14:53:55Z",
  "reference_register_checked": true,
  "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
  "error_register_checked": true,
  "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752",
  "command_sha256": "5b9f985ab6daceaad18e56d2da3dde0d2dba7eefdfe7f75e83eee6bef0a0ad8a",
  "duplicate_failed_command_blocked": false,
  "block_reason": null,
  "execution_started": true,
  "changes_made": false,
  "sanitized": true,
  "secrets_included": false,
  "private_addresses_included": false,
  "raw_evidence_retained_locally": true,
  "raw_evidence_sha256": "9cb937926db26fa8365e08f136b4e2db887816e2c6d2da5a13bb76acfb311ff2",
  "sanitized_output_sha256": "9cb937926db26fa8365e08f136b4e2db887816e2c6d2da5a13bb76acfb311ff2",
  "output_truncated_in_json": false,
  "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
  "output": "validation=PASS\nPREVIOUS_RESULT_204=VERIFIED_OK\nHISTORICAL_BACKUP_BASELINE_122=VERIFIED\nREFERENCE_CHECK=OK\nERROR_REGISTER_CHECK=OK\nAUDIT_GENERATED_AT=2026-07-22T14:53:51Z\n=== CURRENT_REFERENCE_STATE ===\n{\"status\":\"AUTO_REFRESH_READY\",\"next_action\":null,\"updated_at\":\"2026-07-22T14:22:14+00:00\"}\n=== CLUSTER_NODES ===\n{\"node\":\"pve01\",\"status\":\"online\",\"uptime\":540902}\n{\"node\":\"pve02\",\"status\":\"online\",\"uptime\":540921}\n{\"node\":\"pve03\",\"status\":\"online\",\"uptime\":540774}\n=== PVE_BACKUP_JOBS ===\n[{\"compress\":\"zstd\",\"enabled\":1,\"id\":\"homelab-nightly-all\",\"mailnotification\":\"failure\",\"mode\":\"snapshot\",\"next-run\":1784766600,\"prune-backups\":{\"keep-daily\":\"4\"},\"schedule\":\"03:30\",\"storage\":\"local\",\"type\":\"vzdump\",\"vmid\":\"110,111,112,113,130,150,190\"}]\n=== STORAGE_STATUS ===\nName             Type     Status     Total (KiB)      Used (KiB) Available (KiB)        %\nlocal             dir     active       102626232        42074800        55292168   41.00%\nlocal-lvm     lvmthin     active       848797696       305906689       542891006   36.04%\n========== NODE=pve01 ==========\nSSH_STATUS=OK\nHOST=pve01\nUTC_NOW=2026-07-22T14:53:53Z\n=== FAILED_UNITS ===\nFAILED_UNIT_COUNT=3\nhomelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state\nhomelab-cluster-admin-pull.service    loaded failed failed Pull cluster-admin observer health into pve01\nhomelab-vm160-remote-backup.service   loaded failed failed VM160 full-image backup streamed from pve02 to pve03\n=== TARGET_UNITS ===\nResult=exit-code\nExecMainStatus=1\nExecStart={ path=/usr/local/sbin/homelab-backup-matrix-refresh ; argv[]=/usr/local/sbin/homelab-backup-matrix-refresh ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-backup-matrix-refresh.service\nLoadState=loaded\nActiveState=failed\nSubState=failed\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Wed 2026-07-22 10:11:15 MSK\nResult=success\nId=homelab-backup-matrix-refresh.timer\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=disabled\nActiveEnterTimestamp=\nInactiveEnterTimestamp=\nResult=exit-code\nExecMainStatus=1\nExecStart={ path=/usr/local/sbin/homelab-vm160-remote-backup ; argv[]=/usr/local/sbin/homelab-vm160-remote-backup ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-vm160-remote-backup.service\nLoadState=loaded\nActiveState=failed\nSubState=failed\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Tue 2026-07-21 02:39:14 MSK\nResult=success\nId=homelab-vm160-remote-backup.timer\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=disabled\nActiveEnterTimestamp=\nInactiveEnterTimestamp=\nResult=exit-code\nExecMainStatus=1\nExecStart={ path=/usr/local/sbin/homelab-cluster-admin-pull ; argv[]=/usr/local/sbin/homelab-cluster-admin-pull ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-cluster-admin-pull.service\nLoadState=loaded\nActiveState=failed\nSubState=failed\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Tue 2026-07-21 02:37:04 MSK\nResult=success\nId=homelab-cluster-admin-pull.timer\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=disabled\nActiveEnterTimestamp=\nInactiveEnterTimestamp=\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; argv[]=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-vm-local-dumps-2cloud.service\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Sun 2026-07-19 10:13:40 MSK\nResult=success\nId=homelab-vm-local-dumps-2cloud.timer\nLoadState=loaded\nActiveState=active\nSubState=waiting\nUnitFileState=enabled\nActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK\nInactiveEnterTimestamp=\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-mail-cloud-critical-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-critical-upload ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-mail-cloud-critical-upload.service\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Wed 2026-07-22 06:46:47 MSK\nResult=success\nId=homelab-mail-cloud-critical-upload.timer\nLoadState=loaded\nActiveState=active\nSubState=waiting\nUnitFileState=enabled\nActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK\nInactiveEnterTimestamp=\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-mail-cloud-edge-vm-upload.service\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Sun 2026-07-19 09:41:56 MSK\nResult=success\nId=homelab-mail-cloud-edge-vm-upload.timer\nLoadState=loaded\nActiveState=active\nSubState=waiting\nUnitFileState=enabled\nActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK\nInactiveEnterTimestamp=\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-mail-cloud-nextcloud-vm-upload.service\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Wed 2026-07-22 07:45:01 MSK\nResult=success\nId=homelab-mail-cloud-nextcloud-vm-upload.timer\nLoadState=loaded\nActiveState=active\nSubState=waiting\nUnitFileState=enabled\nActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK\nInactiveEnterTimestamp=\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok homelab-mail-cloud-restore-drill.service superseded_by_safety_net_restore_check ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-mail-cloud-restore-drill.service\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Mon 2026-07-20 07:25:14 MSK\nResult=success\nId=homelab-mail-cloud-restore-drill.timer\nLoadState=loaded\nActiveState=active\nSubState=waiting\nUnitFileState=enabled\nActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK\nInactiveEnterTimestamp=\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-backup-health-evidence ; argv[]=/usr/local/sbin/homelab-backup-health-evidence ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-verified-backup-health.service\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Wed 2026-07-22 06:40:29 MSK\nResult=success\nId=homelab-verified-backup-health.timer\nLoadState=loaded\nActiveState=active\nSubState=waiting\nUnitFileState=enabled\nActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK\nInactiveEnterTimestamp=\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-backup-sla-health ; argv[]=/usr/local/sbin/homelab-backup-sla-health ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-backup-sla-health.service\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Wed 2026-07-22 08:10:42 MSK\nResult=success\nId=homelab-backup-sla-health.timer\nLoadState=loaded\nActiveState=active\nSubState=waiting\nUnitFileState=enabled\nActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK\nInactiveEnterTimestamp=\n=== RELEVANT_TIMERS ===\nThu 2026-07-23 06:38:07 MSK              12h Wed 2026-07-22 06:39:05 MSK      11h ago homelab-verified-backup-health.timer                   homelab-verified-backup-health.service\nThu 2026-07-23 06:44:08 MSK              12h Wed 2026-07-22 06:44:25 MSK      11h ago homelab-mail-cloud-critical-upload.timer               homelab-mail-cloud-critical-upload.service\nThu 2026-07-23 07:58:35 MSK              14h Wed 2026-07-22 07:42:14 MSK      10h ago homelab-mail-cloud-nextcloud-vm-upload.timer           homelab-mail-cloud-nextcloud-vm-upload.service\nThu 2026-07-23 08:12:51 MSK              14h Wed 2026-07-22 08:10:41 MSK       9h ago homelab-backup-sla-health.timer                        homelab-backup-sla-health.service\nSun 2026-07-26 09:46:53 MSK           3 days Sun 2026-07-19 09:32:29 MSK   3 days ago homelab-mail-cloud-edge-vm-upload.timer                homelab-mail-cloud-edge-vm-upload.service\nSun 2026-07-26 10:23:39 MSK           3 days Sun 2026-07-19 10:06:07 MSK   3 days ago homelab-vm-local-dumps-2cloud.timer                    homelab-vm-local-dumps-2cloud.service\nMon 2026-07-27 07:34:27 MSK           4 days Mon 2026-07-20 07:25:14 MSK   2 days ago homelab-mail-cloud-restore-drill.timer                 homelab-mail-cloud-restore-drill.service\n=== TARGET_SCRIPT_STATE ===\nFILE=/usr/local/sbin/homelab-backup-matrix-refresh TYPE=regular file MODE=755 OWNER=root:root SIZE=8714 MTIME=2026-07-10 13:02:04.846868245 +0300\nSCRIPT_SYNTAX=/usr/local/sbin/homelab-backup-matrix-refresh:FAIL\nFILE=/usr/local/sbin/homelab-vm160-remote-backup TYPE=regular file MODE=755 OWNER=root:root SIZE=3140 MTIME=2026-07-18 01:38:18.170629793 +0300\nSCRIPT_SYNTAX=/usr/local/sbin/homelab-vm160-remote-backup:OK\nFILE=/usr/local/sbin/homelab-cluster-admin-pull TYPE=regular file MODE=755 OWNER=root:root SIZE=2501 MTIME=2026-07-13 01:15:06.431144817 +0300\nSCRIPT_SYNTAX=/usr/local/sbin/homelab-cluster-admin-pull:OK\nFILE=/usr/local/sbin/homelab-vm-local-dumps-2cloud TYPE=regular file MODE=700 OWNER=root:root SIZE=7272 MTIME=2026-07-06 18:49:49.740645050 +0300\nSCRIPT_SYNTAX=/usr/local/sbin/homelab-vm-local-dumps-2cloud:OK\nFILE=/usr/local/sbin/homelab-mail-cloud-critical-upload TYPE=regular file MODE=700 OWNER=root:root SIZE=7836 MTIME=2026-07-17 14:03:50.373696143 +0300\nSCRIPT_SYNTAX=/usr/local/sbin/homelab-mail-cloud-critical-upload:OK\nFILE=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload TYPE=regular file MODE=750 OWNER=root:root SIZE=12899 MTIME=2026-07-13 21:27:22.164609073 +0300\nSCRIPT_SYNTAX=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:OK\nFILE=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload TYPE=regular file MODE=750 OWNER=root:root SIZE=6542 MTIME=2026-06-30 15:00:33.588620116 +0300\nSCRIPT_SYNTAX=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:OK\nFILE=/usr/local/sbin/homelab-mail-cloud-restore-drill TYPE=regular file MODE=700 OWNER=root:root SIZE=7995 MTIME=2026-07-10 04:14:10.054957455 +0300\nSCRIPT_SYNTAX=/usr/local/sbin/homelab-mail-cloud-restore-drill:OK\n=== CURRENT_BACKUP_HEALTH ===\nHEALTH_FILE_MISSING=/var/lib/homelab-health/unified-backup-matrix.txt\nHEALTH_FILE=/var/lib/homelab-health/backup-framework.txt\nHEALTH_MTIME=2026-07-22 10:11:15.588809944 +0300 HEALTH_SIZE=803\nSTATUS=OK\nHEALTH_FILE=/var/lib/homelab-health/backup-sla.txt\nHEALTH_MTIME=2026-07-22 17:45:04.576070554 +0300 HEALTH_SIZE=154\nSTATUS=WARN TS=2026-07-22T14:45:04Z TYPE=backup-sla BAD=0 WARN=2 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nHEALTH_FILE=/var/lib/homelab-health/backup-coverage-matrix.txt\nHEALTH_MTIME=2026-07-22 10:11:15.236509358 +0300 HEALTH_SIZE=808\nSTATUS=OK\nHEALTH_FILE=/var/lib/homelab-health/vm-local-dumps-cloud.txt\nHEALTH_MTIME=2026-07-22 10:11:15.694007276 +0300 HEALTH_SIZE=800\nSTATUS=OK\nHEALTH_FILE=/var/lib/homelab-health/vm-local-dumps-retention.txt\nHEALTH_MTIME=2026-07-22 10:11:15.711074323 +0300 HEALTH_SIZE=826\nSTATUS=OK\nHEALTH_FILE=/var/lib/homelab-health/vm-backup.txt\nHEALTH_MTIME=2026-07-22 07:35:14.956502473 +0300 HEALTH_SIZE=170\nSTATUS=OK TS=2026-07-22T04:35:14Z TYPE=vm-backup BAD=0 auto-vmct=OK edge-vm-backup=OK edge-vm-offhost=OK edge-vm-restore=OK auto-timer=active edge-freshness-timer=active\nHEALTH_FILE=/var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt\nHEALTH_MTIME=2026-07-09 08:07:47.213452046 +0300 HEALTH_SIZE=217\nSTATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md\nHEALTH_FILE=/var/lib/homelab-health/edge-vm-offhost-freshness.txt\nHEALTH_MTIME=2026-07-22 04:46:23.189237991 +0300 HEALTH_SIZE=409\nSTATUS=OK TS=2026-07-22T01:39:30Z TYPE=edge-vm-offhost-freshness BAD=0 VMID=130 SOURCE_NODE=pve03 DEST_NODE=pve02 SRC=/var/lib/vz/dump/vzdump-qemu-130-2026_07_22-03_30_02.vma.zst DEST=/mnt/staging/offhost/edge-vm-vzdump-from-pve03/vzdump-qemu-130-2026_07_22-03_30_02.vma.zst SIZE=11072513783 SHA256=a8e5179614e02e6933617adb30cb95a35a1b5d17b30c9f00a2626cfa06b14710 COPY_STATUS=copied ZSTD_RC=0 VMA_RC=0 KEEP=4\nHEALTH_FILE=/var/lib/homelab-health/mail-cloud-restore-drill.txt\nHEALTH_MTIME=2026-07-22 10:11:15.491775958 +0300 HEALTH_SIZE=830\nSTATUS=STALE\n=== RCLONE_REMOTE_NAMES ===\npve01-mail-01:\npve01-mail-01-crypt:\npve01-mail-02:\npve01-mail-02-crypt:\npve01-mail-03:\npve01-mail-03-crypt:\npve01-mail-04:\npve01-mail-04-crypt:\npve01-mail-05:\npve01-mail-05-crypt:\npve02-mail-01:\npve02-mail-01-crypt:\npve02-mail-02:\npve02-mail-02-crypt:\npve02-mail-03:\npve02-mail-03-crypt:\npve02-mail-04:\npve02-mail-04-crypt:\npve03-mail-01:\npve03-mail-01-crypt:\npve03-mail-02:\npve03-mail-02-crypt:\npve03-mail-03:\npve03-mail-03-crypt:\npve03-mail-04:\npve03-mail-04-crypt:\n=== ACTIVE_VZDUMP ===\n1599768 bash /usr/local/sbin/homelab-chat-run BACKUP-OFFHOST-CURRENT-STATE-212 read-only backup-audit -- /usr/bin/bash -c  set -eu  test \"$(hostname -s)\" = \"pve01\" test -s /etc/pve/31_HOMELAB_REFERENCE.md test -s /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md command -v curl >/dev/null command -v jq >/dev/null command -v ssh >/dev/null  /usr/local/sbin/homelab-reference validate  W=\"$(mktemp -d /var/tmp/backup-current-state-212.XXXXXX)\" trap \"rm -rf -- \\\"$W\\\"\" EXIT  BASE=\"https://git.gram1.ru/homelab-admin/homelab-public-context/raw/branch/assistant-runtime/runtime/history\"  curl -fsSL \\     \"$BASE/HOMELAB-TAILS-FINAL-CLOSEOUT-204.json\" \\     -o \"$W/204.json\"  jq -e \"     .command_id == \\\"HOMELAB-TAILS-FINAL-CLOSEOUT-204\\\" and     .status == \\\"OK\\\" and     ((.rc | tonumber) == 0) and     .reference_register_checked and     .error_register_checked and     ((.output // \\\"\\\") | contains(\\\"ALL_TAILS_CLOSED=YES\\\")) and     ((.output // \\\"\\\") | contains(\\\"TAILS_FINAL_CLOSEOUT=COMPLETE\\\")) \" \"$W/204.json\" >/dev/null  curl -fsSL \\     \"$BASE/HOMELAB-INDEPENDENT-FINAL-STATE-122.json\" \\     -o \"$W/122.json\"  jq -e \"     .command_id == \\\"HOMELAB-INDEPENDENT-FINAL-STATE-122\\\" and     .status == \\\"OK\\\" and     ((.rc | tonumber) == 0) and     ((.output // \\\"\\\") | contains(\\\"HOMELAB_FINAL_STATE=BLOCKED\\\")) and     ((.output // \\\"\\\") | contains(\\\"NEXT_ACTION=RECONCILE_DUAL_MAIL_CLOUD_BACKUPS\\\")) \" \"$W/122.json\" >/dev/null  echo \"PREVIOUS_RESULT_204=VERIFIED_OK\" echo \"HISTORICAL_BACKUP_BASELINE_122=VERIFIED\" echo \"REFERENCE_CHECK=OK\" echo \"ERROR_REGISTER_CHECK=OK\" echo \"AUDIT_GENERATED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)\"  echo \"=== CURRENT_REFERENCE_STATE ===\"  STATE=\"/etc/homelab/reference/state/current.json\"  if test -s \"$STATE\" && jq -e \".\" \"$STATE\" >/dev/null; then     jq -c \"{status,next_action,updated_at}\" \"$STATE\" else     echo \"REFERENCE_STATE=INVALID_OR_MISSING\" fi  echo \"=== CLUSTER_NODES ===\" pvesh get /nodes --output-format json | jq -c \"sort_by(.node)[] | {node,status,uptime}\"  echo \"=== PVE_BACKUP_JOBS ===\" pvesh get /cluster/backup --output-format json | jq -c \"sort_by(.id)\"  echo \"=== STORAGE_STATUS ===\" pvesm status || true  SSH_OPTS=(     -o BatchMode=yes     -o ConnectTimeout=10     -o ConnectionAttempts=1     -o StrictHostKeyChecking=yes )  for NODE in pve01 pve02 pve03 do     echo \"========== NODE=$NODE ==========\"      if ssh \"${SSH_OPTS[@]}\" root@\"$NODE\" true     then         echo \"SSH_STATUS=OK\"          ssh \"${SSH_OPTS[@]}\" root@\"$NODE\" \"             set -u              echo HOST=\\$(hostname -s)             echo UTC_NOW=\\$(date -u +%Y-%m-%dT%H:%M:%SZ)              echo === FAILED_UNITS ===             echo -n FAILED_UNIT_COUNT=             systemctl --failed --no-legend --plain 2>/dev/null | wc -l             systemctl --failed --no-legend --plain 2>/dev/null || true              echo === TARGET_UNITS ===             for U in \\                 homelab-backup-matrix-refresh.service \\                 homelab-backup-matrix-refresh.timer \\                 homelab-vm160-remote-backup.service \\                 homelab-vm160-remote-backup.timer \\                 homelab-cluster-admin-pull.service \\                 homelab-cluster-admin-pull.timer \\                 homelab-vm-local-dumps-2cloud.service \\                 homelab-vm-local-dumps-2cloud.timer \\                 homelab-mail-cloud-critical-upload.service \\                 homelab-mail-cloud-critical-upload.timer \\                 homelab-mail-cloud-edge-vm-upload.service \\                 homelab-mail-cloud-edge-vm-upload.timer \\                 homelab-mail-cloud-nextcloud-vm-upload.service \\                 homelab-mail-cloud-nextcloud-vm-upload.timer \\                 homelab-mail-cloud-restore-drill.service \\                 homelab-mail-cloud-restore-drill.timer \\                 homelab-verified-backup-health.service \\                 homelab-verified-backup-health.timer \\                 homelab-backup-sla-health.service \\                 homelab-backup-sla-health.timer             do                 if systemctl cat \\\"\\$U\\\" >/dev/null 2>&1                 then                     systemctl show \\\"\\$U\\\" \\                         -p Id \\                         -p LoadState \\                         -p ActiveState \\                         -p SubState \\                         -p Result \\                         -p ExecMainStatus \\                         -p UnitFileState \\                         -p ActiveEnterTimestamp \\                         -p InactiveEnterTimestamp \\                         -p ExecStart \\                         --no-pager                 else                     echo UNIT_MISSING=\\$U                 fi             done              echo === RELEVANT_TIMERS ===             systemctl list-timers --all --no-pager --no-legend 2>/dev/null |                 grep -E \\\"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\\\" ||                 true              echo === TARGET_SCRIPT_STATE ===             for F in \\                 /usr/local/sbin/homelab-backup-matrix-refresh \\                 /usr/local/sbin/homelab-vm160-remote-backup \\                 /usr/local/sbin/homelab-cluster-admin-pull \\                 /usr/local/sbin/homelab-vm-local-dumps-2cloud \\                 /usr/local/sbin/homelab-mail-cloud-critical-upload \\                 /usr/local/sbin/homelab-mail-cloud-edge-vm-upload \\                 /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload \\                 /usr/local/sbin/homelab-mail-cloud-restore-drill             do                 if test -e \\\"\\$F\\\"                 then                     stat -c \\\"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\\\" \\\"\\$F\\\"                      if /usr/bin/bash -n \\\"\\$F\\\" 2>/dev/null                     then                         echo SCRIPT_SYNTAX=\\$F:OK                     else                         echo SCRIPT_SYNTAX=\\$F:FAIL                     fi                 else                     echo FILE_MISSING=\\$F                 fi             done              echo === CURRENT_BACKUP_HEALTH ===             for H in \\                 /var/lib/homelab-health/unified-backup-matrix.txt \\                 /var/lib/homelab-health/backup-framework.txt \\                 /var/lib/homelab-health/backup-sla.txt \\                 /var/lib/homelab-health/backup-coverage-matrix.txt \\                 /var/lib/homelab-health/vm-local-dumps-cloud.txt \\                 /var/lib/homelab-health/vm-local-dumps-retention.txt \\                 /var/lib/homelab-health/vm-backup.txt \\                 /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt \\                 /var/lib/homelab-health/edge-vm-offhost-freshness.txt \\                 /var/lib/homelab-health/mail-cloud-restore-drill.txt             do                 if test -f \\\"\\$H\\\"                 then                     echo HEALTH_FILE=\\$H                     stat -c \\\"HEALTH_MTIME=%y HEALTH_SIZE=%s\\\" \\\"\\$H\\\"                     sed -n \\\"1p\\\" \\\"\\$H\\\"                 else                     echo HEALTH_FILE_MISSING=\\$H                 fi             done              echo === RCLONE_REMOTE_NAMES ===             if command -v rclone >/dev/null 2>&1             then                 rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL             else                 echo RCLONE=NOT_INSTALLED             fi              echo === ACTIVE_VZDUMP ===             pgrep -af \\\"[v]zdump\\\" || echo ACTIVE_VZDUMP=NO              echo === LOCAL_ARCHIVES ===             echo -n ARCHIVE_COUNT=             find /var/lib/vz/dump /mnt/pve \\                 -type f \\                 \\( -name \\\"vzdump-*.zst\\\" \\                    -o -name \\\"vzdump-*.lzo\\\" \\                    -o -name \\\"vzdump-*.gz\\\" \\) \\                 -printf x \\                 2>/dev/null |                 wc -c              find /var/lib/vz/dump /mnt/pve \\                 -type f \\                 \\( -name \\\"vzdump-*.zst\\\" \\                    -o -name \\\"vzdump-*.lzo\\\" \\                    -o -name \\\"vzdump-*.gz\\\" \\) \\                 -printf \\\"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\\n\\\" \\                 2>/dev/null |                 sort              echo === TARGET_UNIT_LOGS ===             journalctl \\                 -u homelab-backup-matrix-refresh.service \\                 -u homelab-vm160-remote-backup.service \\                 -u homelab-cluster-admin-pull.service \\                 -n 120 \\                 --no-pager \\                 -o short-iso \\                 2>/dev/null ||                 true         \"     else         echo \"SSH_STATUS=FAIL\"     fi done  echo \"CHANGES_MADE=NO\" echo \"SECRET_PRINTED=NO\" echo \"PRIVATE_ADDRESS_PRINTED=NO\" echo \"RESULT=OK\" echo \"BACKUP_OFFHOST_CURRENT_STATE=COMPLETE\" \n1599834 timeout --foreground --preserve-status 300 /usr/bin/bash -c  set -eu  test \"$(hostname -s)\" = \"pve01\" test -s /etc/pve/31_HOMELAB_REFERENCE.md test -s /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md command -v curl >/dev/null command -v jq >/dev/null command -v ssh >/dev/null  /usr/local/sbin/homelab-reference validate  W=\"$(mktemp -d /var/tmp/backup-current-state-212.XXXXXX)\" trap \"rm -rf -- \\\"$W\\\"\" EXIT  BASE=\"https://git.gram1.ru/homelab-admin/homelab-public-context/raw/branch/assistant-runtime/runtime/history\"  curl -fsSL \\     \"$BASE/HOMELAB-TAILS-FINAL-CLOSEOUT-204.json\" \\     -o \"$W/204.json\"  jq -e \"     .command_id == \\\"HOMELAB-TAILS-FINAL-CLOSEOUT-204\\\" and     .status == \\\"OK\\\" and     ((.rc | tonumber) == 0) and     .reference_register_checked and     .error_register_checked and     ((.output // \\\"\\\") | contains(\\\"ALL_TAILS_CLOSED=YES\\\")) and     ((.output // \\\"\\\") | contains(\\\"TAILS_FINAL_CLOSEOUT=COMPLETE\\\")) \" \"$W/204.json\" >/dev/null  curl -fsSL \\     \"$BASE/HOMELAB-INDEPENDENT-FINAL-STATE-122.json\" \\     -o \"$W/122.json\"  jq -e \"     .command_id == \\\"HOMELAB-INDEPENDENT-FINAL-STATE-122\\\" and     .status == \\\"OK\\\" and     ((.rc | tonumber) == 0) and     ((.output // \\\"\\\") | contains(\\\"HOMELAB_FINAL_STATE=BLOCKED\\\")) and     ((.output // \\\"\\\") | contains(\\\"NEXT_ACTION=RECONCILE_DUAL_MAIL_CLOUD_BACKUPS\\\")) \" \"$W/122.json\" >/dev/null  echo \"PREVIOUS_RESULT_204=VERIFIED_OK\" echo \"HISTORICAL_BACKUP_BASELINE_122=VERIFIED\" echo \"REFERENCE_CHECK=OK\" echo \"ERROR_REGISTER_CHECK=OK\" echo \"AUDIT_GENERATED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)\"  echo \"=== CURRENT_REFERENCE_STATE ===\"  STATE=\"/etc/homelab/reference/state/current.json\"  if test -s \"$STATE\" && jq -e \".\" \"$STATE\" >/dev/null; then     jq -c \"{status,next_action,updated_at}\" \"$STATE\" else     echo \"REFERENCE_STATE=INVALID_OR_MISSING\" fi  echo \"=== CLUSTER_NODES ===\" pvesh get /nodes --output-format json | jq -c \"sort_by(.node)[] | {node,status,uptime}\"  echo \"=== PVE_BACKUP_JOBS ===\" pvesh get /cluster/backup --output-format json | jq -c \"sort_by(.id)\"  echo \"=== STORAGE_STATUS ===\" pvesm status || true  SSH_OPTS=(     -o BatchMode=yes     -o ConnectTimeout=10     -o ConnectionAttempts=1     -o StrictHostKeyChecking=yes )  for NODE in pve01 pve02 pve03 do     echo \"========== NODE=$NODE ==========\"      if ssh \"${SSH_OPTS[@]}\" root@\"$NODE\" true     then         echo \"SSH_STATUS=OK\"          ssh \"${SSH_OPTS[@]}\" root@\"$NODE\" \"             set -u              echo HOST=\\$(hostname -s)             echo UTC_NOW=\\$(date -u +%Y-%m-%dT%H:%M:%SZ)              echo === FAILED_UNITS ===             echo -n FAILED_UNIT_COUNT=             systemctl --failed --no-legend --plain 2>/dev/null | wc -l             systemctl --failed --no-legend --plain 2>/dev/null || true              echo === TARGET_UNITS ===             for U in \\                 homelab-backup-matrix-refresh.service \\                 homelab-backup-matrix-refresh.timer \\                 homelab-vm160-remote-backup.service \\                 homelab-vm160-remote-backup.timer \\                 homelab-cluster-admin-pull.service \\                 homelab-cluster-admin-pull.timer \\                 homelab-vm-local-dumps-2cloud.service \\                 homelab-vm-local-dumps-2cloud.timer \\                 homelab-mail-cloud-critical-upload.service \\                 homelab-mail-cloud-critical-upload.timer \\                 homelab-mail-cloud-edge-vm-upload.service \\                 homelab-mail-cloud-edge-vm-upload.timer \\                 homelab-mail-cloud-nextcloud-vm-upload.service \\                 homelab-mail-cloud-nextcloud-vm-upload.timer \\                 homelab-mail-cloud-restore-drill.service \\                 homelab-mail-cloud-restore-drill.timer \\                 homelab-verified-backup-health.service \\                 homelab-verified-backup-health.timer \\                 homelab-backup-sla-health.service \\                 homelab-backup-sla-health.timer             do                 if systemctl cat \\\"\\$U\\\" >/dev/null 2>&1                 then                     systemctl show \\\"\\$U\\\" \\                         -p Id \\                         -p LoadState \\                         -p ActiveState \\                         -p SubState \\                         -p Result \\                         -p ExecMainStatus \\                         -p UnitFileState \\                         -p ActiveEnterTimestamp \\                         -p InactiveEnterTimestamp \\                         -p ExecStart \\                         --no-pager                 else                     echo UNIT_MISSING=\\$U                 fi             done              echo === RELEVANT_TIMERS ===             systemctl list-timers --all --no-pager --no-legend 2>/dev/null |                 grep -E \\\"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\\\" ||                 true              echo === TARGET_SCRIPT_STATE ===             for F in \\                 /usr/local/sbin/homelab-backup-matrix-refresh \\                 /usr/local/sbin/homelab-vm160-remote-backup \\                 /usr/local/sbin/homelab-cluster-admin-pull \\                 /usr/local/sbin/homelab-vm-local-dumps-2cloud \\                 /usr/local/sbin/homelab-mail-cloud-critical-upload \\                 /usr/local/sbin/homelab-mail-cloud-edge-vm-upload \\                 /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload \\                 /usr/local/sbin/homelab-mail-cloud-restore-drill             do                 if test -e \\\"\\$F\\\"                 then                     stat -c \\\"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\\\" \\\"\\$F\\\"                      if /usr/bin/bash -n \\\"\\$F\\\" 2>/dev/null                     then                         echo SCRIPT_SYNTAX=\\$F:OK                     else                         echo SCRIPT_SYNTAX=\\$F:FAIL                     fi                 else                     echo FILE_MISSING=\\$F                 fi             done              echo === CURRENT_BACKUP_HEALTH ===             for H in \\                 /var/lib/homelab-health/unified-backup-matrix.txt \\                 /var/lib/homelab-health/backup-framework.txt \\                 /var/lib/homelab-health/backup-sla.txt \\                 /var/lib/homelab-health/backup-coverage-matrix.txt \\                 /var/lib/homelab-health/vm-local-dumps-cloud.txt \\                 /var/lib/homelab-health/vm-local-dumps-retention.txt \\                 /var/lib/homelab-health/vm-backup.txt \\                 /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt \\                 /var/lib/homelab-health/edge-vm-offhost-freshness.txt \\                 /var/lib/homelab-health/mail-cloud-restore-drill.txt             do                 if test -f \\\"\\$H\\\"                 then                     echo HEALTH_FILE=\\$H                     stat -c \\\"HEALTH_MTIME=%y HEALTH_SIZE=%s\\\" \\\"\\$H\\\"                     sed -n \\\"1p\\\" \\\"\\$H\\\"                 else                     echo HEALTH_FILE_MISSING=\\$H                 fi             done              echo === RCLONE_REMOTE_NAMES ===             if command -v rclone >/dev/null 2>&1             then                 rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL             else                 echo RCLONE=NOT_INSTALLED             fi              echo === ACTIVE_VZDUMP ===             pgrep -af \\\"[v]zdump\\\" || echo ACTIVE_VZDUMP=NO              echo === LOCAL_ARCHIVES ===             echo -n ARCHIVE_COUNT=             find /var/lib/vz/dump /mnt/pve \\                 -type f \\                 \\( -name \\\"vzdump-*.zst\\\" \\                    -o -name \\\"vzdump-*.lzo\\\" \\                    -o -name \\\"vzdump-*.gz\\\" \\) \\                 -printf x \\                 2>/dev/null |                 wc -c              find /var/lib/vz/dump /mnt/pve \\                 -type f \\                 \\( -name \\\"vzdump-*.zst\\\" \\                    -o -name \\\"vzdump-*.lzo\\\" \\                    -o -name \\\"vzdump-*.gz\\\" \\) \\                 -printf \\\"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\\n\\\" \\                 2>/dev/null |                 sort              echo === TARGET_UNIT_LOGS ===             journalctl \\                 -u homelab-backup-matrix-refresh.service \\                 -u homelab-vm160-remote-backup.service \\                 -u homelab-cluster-admin-pull.service \\                 -n 120 \\                 --no-pager \\                 -o short-iso \\                 2>/dev/null ||                 true         \"     else         echo \"SSH_STATUS=FAIL\"     fi done  echo \"CHANGES_MADE=NO\" echo \"SECRET_PRINTED=NO\" echo \"PRIVATE_ADDRESS_PRINTED=NO\" echo \"RESULT=OK\" echo \"BACKUP_OFFHOST_CURRENT_STATE=COMPLETE\" \n1599836 /usr/bin/bash -c  set -eu  test \"$(hostname -s)\" = \"pve01\" test -s /etc/pve/31_HOMELAB_REFERENCE.md test -s /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md command -v curl >/dev/null command -v jq >/dev/null command -v ssh >/dev/null  /usr/local/sbin/homelab-reference validate  W=\"$(mktemp -d /var/tmp/backup-current-state-212.XXXXXX)\" trap \"rm -rf -- \\\"$W\\\"\" EXIT  BASE=\"https://git.gram1.ru/homelab-admin/homelab-public-context/raw/branch/assistant-runtime/runtime/history\"  curl -fsSL \\     \"$BASE/HOMELAB-TAILS-FINAL-CLOSEOUT-204.json\" \\     -o \"$W/204.json\"  jq -e \"     .command_id == \\\"HOMELAB-TAILS-FINAL-CLOSEOUT-204\\\" and     .status == \\\"OK\\\" and     ((.rc | tonumber) == 0) and     .reference_register_checked and     .error_register_checked and     ((.output // \\\"\\\") | contains(\\\"ALL_TAILS_CLOSED=YES\\\")) and     ((.output // \\\"\\\") | contains(\\\"TAILS_FINAL_CLOSEOUT=COMPLETE\\\")) \" \"$W/204.json\" >/dev/null  curl -fsSL \\     \"$BASE/HOMELAB-INDEPENDENT-FINAL-STATE-122.json\" \\     -o \"$W/122.json\"  jq -e \"     .command_id == \\\"HOMELAB-INDEPENDENT-FINAL-STATE-122\\\" and     .status == \\\"OK\\\" and     ((.rc | tonumber) == 0) and     ((.output // \\\"\\\") | contains(\\\"HOMELAB_FINAL_STATE=BLOCKED\\\")) and     ((.output // \\\"\\\") | contains(\\\"NEXT_ACTION=RECONCILE_DUAL_MAIL_CLOUD_BACKUPS\\\")) \" \"$W/122.json\" >/dev/null  echo \"PREVIOUS_RESULT_204=VERIFIED_OK\" echo \"HISTORICAL_BACKUP_BASELINE_122=VERIFIED\" echo \"REFERENCE_CHECK=OK\" echo \"ERROR_REGISTER_CHECK=OK\" echo \"AUDIT_GENERATED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)\"  echo \"=== CURRENT_REFERENCE_STATE ===\"  STATE=\"/etc/homelab/reference/state/current.json\"  if test -s \"$STATE\" && jq -e \".\" \"$STATE\" >/dev/null; then     jq -c \"{status,next_action,updated_at}\" \"$STATE\" else     echo \"REFERENCE_STATE=INVALID_OR_MISSING\" fi  echo \"=== CLUSTER_NODES ===\" pvesh get /nodes --output-format json | jq -c \"sort_by(.node)[] | {node,status,uptime}\"  echo \"=== PVE_BACKUP_JOBS ===\" pvesh get /cluster/backup --output-format json | jq -c \"sort_by(.id)\"  echo \"=== STORAGE_STATUS ===\" pvesm status || true  SSH_OPTS=(     -o BatchMode=yes     -o ConnectTimeout=10     -o ConnectionAttempts=1     -o StrictHostKeyChecking=yes )  for NODE in pve01 pve02 pve03 do     echo \"========== NODE=$NODE ==========\"      if ssh \"${SSH_OPTS[@]}\" root@\"$NODE\" true     then         echo \"SSH_STATUS=OK\"          ssh \"${SSH_OPTS[@]}\" root@\"$NODE\" \"             set -u              echo HOST=\\$(hostname -s)             echo UTC_NOW=\\$(date -u +%Y-%m-%dT%H:%M:%SZ)              echo === FAILED_UNITS ===             echo -n FAILED_UNIT_COUNT=             systemctl --failed --no-legend --plain 2>/dev/null | wc -l             systemctl --failed --no-legend --plain 2>/dev/null || true              echo === TARGET_UNITS ===             for U in \\                 homelab-backup-matrix-refresh.service \\                 homelab-backup-matrix-refresh.timer \\                 homelab-vm160-remote-backup.service \\                 homelab-vm160-remote-backup.timer \\                 homelab-cluster-admin-pull.service \\                 homelab-cluster-admin-pull.timer \\                 homelab-vm-local-dumps-2cloud.service \\                 homelab-vm-local-dumps-2cloud.timer \\                 homelab-mail-cloud-critical-upload.service \\                 homelab-mail-cloud-critical-upload.timer \\                 homelab-mail-cloud-edge-vm-upload.service \\                 homelab-mail-cloud-edge-vm-upload.timer \\                 homelab-mail-cloud-nextcloud-vm-upload.service \\                 homelab-mail-cloud-nextcloud-vm-upload.timer \\                 homelab-mail-cloud-restore-drill.service \\                 homelab-mail-cloud-restore-drill.timer \\                 homelab-verified-backup-health.service \\                 homelab-verified-backup-health.timer \\                 homelab-backup-sla-health.service \\                 homelab-backup-sla-health.timer             do                 if systemctl cat \\\"\\$U\\\" >/dev/null 2>&1                 then                     systemctl show \\\"\\$U\\\" \\                         -p Id \\                         -p LoadState \\                         -p ActiveState \\                         -p SubState \\                         -p Result \\                         -p ExecMainStatus \\                         -p UnitFileState \\                         -p ActiveEnterTimestamp \\                         -p InactiveEnterTimestamp \\                         -p ExecStart \\                         --no-pager                 else                     echo UNIT_MISSING=\\$U                 fi             done              echo === RELEVANT_TIMERS ===             systemctl list-timers --all --no-pager --no-legend 2>/dev/null |                 grep -E \\\"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\\\" ||                 true              echo === TARGET_SCRIPT_STATE ===             for F in \\                 /usr/local/sbin/homelab-backup-matrix-refresh \\                 /usr/local/sbin/homelab-vm160-remote-backup \\                 /usr/local/sbin/homelab-cluster-admin-pull \\                 /usr/local/sbin/homelab-vm-local-dumps-2cloud \\                 /usr/local/sbin/homelab-mail-cloud-critical-upload \\                 /usr/local/sbin/homelab-mail-cloud-edge-vm-upload \\                 /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload \\                 /usr/local/sbin/homelab-mail-cloud-restore-drill             do                 if test -e \\\"\\$F\\\"                 then                     stat -c \\\"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\\\" \\\"\\$F\\\"                      if /usr/bin/bash -n \\\"\\$F\\\" 2>/dev/null                     then                         echo SCRIPT_SYNTAX=\\$F:OK                     else                         echo SCRIPT_SYNTAX=\\$F:FAIL                     fi                 else                     echo FILE_MISSING=\\$F                 fi             done              echo === CURRENT_BACKUP_HEALTH ===             for H in \\                 /var/lib/homelab-health/unified-backup-matrix.txt \\                 /var/lib/homelab-health/backup-framework.txt \\                 /var/lib/homelab-health/backup-sla.txt \\                 /var/lib/homelab-health/backup-coverage-matrix.txt \\                 /var/lib/homelab-health/vm-local-dumps-cloud.txt \\                 /var/lib/homelab-health/vm-local-dumps-retention.txt \\                 /var/lib/homelab-health/vm-backup.txt \\                 /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt \\                 /var/lib/homelab-health/edge-vm-offhost-freshness.txt \\                 /var/lib/homelab-health/mail-cloud-restore-drill.txt             do                 if test -f \\\"\\$H\\\"                 then                     echo HEALTH_FILE=\\$H                     stat -c \\\"HEALTH_MTIME=%y HEALTH_SIZE=%s\\\" \\\"\\$H\\\"                     sed -n \\\"1p\\\" \\\"\\$H\\\"                 else                     echo HEALTH_FILE_MISSING=\\$H                 fi             done              echo === RCLONE_REMOTE_NAMES ===             if command -v rclone >/dev/null 2>&1             then                 rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL             else                 echo RCLONE=NOT_INSTALLED             fi              echo === ACTIVE_VZDUMP ===             pgrep -af \\\"[v]zdump\\\" || echo ACTIVE_VZDUMP=NO              echo === LOCAL_ARCHIVES ===             echo -n ARCHIVE_COUNT=             find /var/lib/vz/dump /mnt/pve \\                 -type f \\                 \\( -name \\\"vzdump-*.zst\\\" \\                    -o -name \\\"vzdump-*.lzo\\\" \\                    -o -name \\\"vzdump-*.gz\\\" \\) \\                 -printf x \\                 2>/dev/null |                 wc -c              find /var/lib/vz/dump /mnt/pve \\                 -type f \\                 \\( -name \\\"vzdump-*.zst\\\" \\                    -o -name \\\"vzdump-*.lzo\\\" \\                    -o -name \\\"vzdump-*.gz\\\" \\) \\                 -printf \\\"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\\n\\\" \\                 2>/dev/null |                 sort              echo === TARGET_UNIT_LOGS ===             journalctl \\                 -u homelab-backup-matrix-refresh.service \\                 -u homelab-vm160-remote-backup.service \\                 -u homelab-cluster-admin-pull.service \\                 -n 120 \\                 --no-pager \\                 -o short-iso \\                 2>/dev/null ||                 true         \"     else         echo \"SSH_STATUS=FAIL\"     fi done  echo \"CHANGES_MADE=NO\" echo \"SECRET_PRINTED=NO\" echo \"PRIVATE_ADDRESS_PRINTED=NO\" echo \"RESULT=OK\" echo \"BACKUP_OFFHOST_CURRENT_STATE=COMPLETE\" \n1603200 ssh -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o StrictHostKeyChecking=yes root@pve01              set -u              echo HOST=$(hostname -s)             echo UTC_NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)              echo === FAILED_UNITS ===             echo -n FAILED_UNIT_COUNT=             systemctl --failed --no-legend --plain 2>/dev/null | wc -l             systemctl --failed --no-legend --plain 2>/dev/null || true              echo === TARGET_UNITS ===             for U in                 homelab-backup-matrix-refresh.service                 homelab-backup-matrix-refresh.timer                 homelab-vm160-remote-backup.service                 homelab-vm160-remote-backup.timer                 homelab-cluster-admin-pull.service                 homelab-cluster-admin-pull.timer                 homelab-vm-local-dumps-2cloud.service                 homelab-vm-local-dumps-2cloud.timer                 homelab-mail-cloud-critical-upload.service                 homelab-mail-cloud-critical-upload.timer                 homelab-mail-cloud-edge-vm-upload.service                 homelab-mail-cloud-edge-vm-upload.timer                 homelab-mail-cloud-nextcloud-vm-upload.service                 homelab-mail-cloud-nextcloud-vm-upload.timer                 homelab-mail-cloud-restore-drill.service                 homelab-mail-cloud-restore-drill.timer                 homelab-verified-backup-health.service                 homelab-verified-backup-health.timer                 homelab-backup-sla-health.service                 homelab-backup-sla-health.timer             do                 if systemctl cat \"$U\" >/dev/null 2>&1                 then                     systemctl show \"$U\"                         -p Id                         -p LoadState                         -p ActiveState                         -p SubState                         -p Result                         -p ExecMainStatus                         -p UnitFileState                         -p ActiveEnterTimestamp                         -p InactiveEnterTimestamp                         -p ExecStart                         --no-pager                 else                     echo UNIT_MISSING=$U                 fi             done              echo === RELEVANT_TIMERS ===             systemctl list-timers --all --no-pager --no-legend 2>/dev/null |                 grep -E \"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\" ||                 true              echo === TARGET_SCRIPT_STATE ===             for F in                 /usr/local/sbin/homelab-backup-matrix-refresh                 /usr/local/sbin/homelab-vm160-remote-backup                 /usr/local/sbin/homelab-cluster-admin-pull                 /usr/local/sbin/homelab-vm-local-dumps-2cloud                 /usr/local/sbin/homelab-mail-cloud-critical-upload                 /usr/local/sbin/homelab-mail-cloud-edge-vm-upload                 /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload                 /usr/local/sbin/homelab-mail-cloud-restore-drill             do                 if test -e \"$F\"                 then                     stat -c \"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\" \"$F\"                      if /usr/bin/bash -n \"$F\" 2>/dev/null                     then                         echo SCRIPT_SYNTAX=$F:OK                     else                         echo SCRIPT_SYNTAX=$F:FAIL                     fi                 else                     echo FILE_MISSING=$F                 fi             done              echo === CURRENT_BACKUP_HEALTH ===             for H in                 /var/lib/homelab-health/unified-backup-matrix.txt                 /var/lib/homelab-health/backup-framework.txt                 /var/lib/homelab-health/backup-sla.txt                 /var/lib/homelab-health/backup-coverage-matrix.txt                 /var/lib/homelab-health/vm-local-dumps-cloud.txt                 /var/lib/homelab-health/vm-local-dumps-retention.txt                 /var/lib/homelab-health/vm-backup.txt                 /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt                 /var/lib/homelab-health/edge-vm-offhost-freshness.txt                 /var/lib/homelab-health/mail-cloud-restore-drill.txt             do                 if test -f \"$H\"                 then                     echo HEALTH_FILE=$H                     stat -c \"HEALTH_MTIME=%y HEALTH_SIZE=%s\" \"$H\"                     sed -n \"1p\" \"$H\"                 else                     echo HEALTH_FILE_MISSING=$H                 fi             done              echo === RCLONE_REMOTE_NAMES ===             if command -v rclone >/dev/null 2>&1             then                 rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL             else                 echo RCLONE=NOT_INSTALLED             fi              echo === ACTIVE_VZDUMP ===             pgrep -af \"[v]zdump\" || echo ACTIVE_VZDUMP=NO              echo === LOCAL_ARCHIVES ===             echo -n ARCHIVE_COUNT=             find /var/lib/vz/dump /mnt/pve                 -type f                 \\( -name \"vzdump-*.zst\"                    -o -name \"vzdump-*.lzo\"                    -o -name \"vzdump-*.gz\" \\)                 -printf x                 2>/dev/null |                 wc -c              find /var/lib/vz/dump /mnt/pve                 -type f                 \\( -name \"vzdump-*.zst\"                    -o -name \"vzdump-*.lzo\"                    -o -name \"vzdump-*.gz\" \\)                 -printf \"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\\n\"                 2>/dev/null |                 sort              echo === TARGET_UNIT_LOGS ===             journalctl                 -u homelab-backup-matrix-refresh.service                 -u homelab-vm160-remote-backup.service                 -u homelab-cluster-admin-pull.service                 -n 120                 --no-pager                 -o short-iso                 2>/dev/null ||                 true         \n1603210 bash -c              set -u              echo HOST=$(hostname -s)             echo UTC_NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)              echo === FAILED_UNITS ===             echo -n FAILED_UNIT_COUNT=             systemctl --failed --no-legend --plain 2>/dev/null | wc -l             systemctl --failed --no-legend --plain 2>/dev/null || true              echo === TARGET_UNITS ===             for U in                 homelab-backup-matrix-refresh.service                 homelab-backup-matrix-refresh.timer                 homelab-vm160-remote-backup.service                 homelab-vm160-remote-backup.timer                 homelab-cluster-admin-pull.service                 homelab-cluster-admin-pull.timer                 homelab-vm-local-dumps-2cloud.service                 homelab-vm-local-dumps-2cloud.timer                 homelab-mail-cloud-critical-upload.service                 homelab-mail-cloud-critical-upload.timer                 homelab-mail-cloud-edge-vm-upload.service                 homelab-mail-cloud-edge-vm-upload.timer                 homelab-mail-cloud-nextcloud-vm-upload.service                 homelab-mail-cloud-nextcloud-vm-upload.timer                 homelab-mail-cloud-restore-drill.service                 homelab-mail-cloud-restore-drill.timer                 homelab-verified-backup-health.service                 homelab-verified-backup-health.timer                 homelab-backup-sla-health.service                 homelab-backup-sla-health.timer             do                 if systemctl cat \"$U\" >/dev/null 2>&1                 then                     systemctl show \"$U\"                         -p Id                         -p LoadState                         -p ActiveState                         -p SubState                         -p Result                         -p ExecMainStatus                         -p UnitFileState                         -p ActiveEnterTimestamp                         -p InactiveEnterTimestamp                         -p ExecStart                         --no-pager                 else                     echo UNIT_MISSING=$U                 fi             done              echo === RELEVANT_TIMERS ===             systemctl list-timers --all --no-pager --no-legend 2>/dev/null |                 grep -E \"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\" ||                 true              echo === TARGET_SCRIPT_STATE ===             for F in                 /usr/local/sbin/homelab-backup-matrix-refresh                 /usr/local/sbin/homelab-vm160-remote-backup                 /usr/local/sbin/homelab-cluster-admin-pull                 /usr/local/sbin/homelab-vm-local-dumps-2cloud                 /usr/local/sbin/homelab-mail-cloud-critical-upload                 /usr/local/sbin/homelab-mail-cloud-edge-vm-upload                 /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload                 /usr/local/sbin/homelab-mail-cloud-restore-drill             do                 if test -e \"$F\"                 then                     stat -c \"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\" \"$F\"                      if /usr/bin/bash -n \"$F\" 2>/dev/null                     then                         echo SCRIPT_SYNTAX=$F:OK                     else                         echo SCRIPT_SYNTAX=$F:FAIL                     fi                 else                     echo FILE_MISSING=$F                 fi             done              echo === CURRENT_BACKUP_HEALTH ===             for H in                 /var/lib/homelab-health/unified-backup-matrix.txt                 /var/lib/homelab-health/backup-framework.txt                 /var/lib/homelab-health/backup-sla.txt                 /var/lib/homelab-health/backup-coverage-matrix.txt                 /var/lib/homelab-health/vm-local-dumps-cloud.txt                 /var/lib/homelab-health/vm-local-dumps-retention.txt                 /var/lib/homelab-health/vm-backup.txt                 /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt                 /var/lib/homelab-health/edge-vm-offhost-freshness.txt                 /var/lib/homelab-health/mail-cloud-restore-drill.txt             do                 if test -f \"$H\"                 then                     echo HEALTH_FILE=$H                     stat -c \"HEALTH_MTIME=%y HEALTH_SIZE=%s\" \"$H\"                     sed -n \"1p\" \"$H\"                 else                     echo HEALTH_FILE_MISSING=$H                 fi             done              echo === RCLONE_REMOTE_NAMES ===             if command -v rclone >/dev/null 2>&1             then                 rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL             else                 echo RCLONE=NOT_INSTALLED             fi              echo === ACTIVE_VZDUMP ===             pgrep -af \"[v]zdump\" || echo ACTIVE_VZDUMP=NO              echo === LOCAL_ARCHIVES ===             echo -n ARCHIVE_COUNT=             find /var/lib/vz/dump /mnt/pve                 -type f                 \\( -name \"vzdump-*.zst\"                    -o -name \"vzdump-*.lzo\"                    -o -name \"vzdump-*.gz\" \\)                 -printf x                 2>/dev/null |                 wc -c              find /var/lib/vz/dump /mnt/pve                 -type f                 \\( -name \"vzdump-*.zst\"                    -o -name \"vzdump-*.lzo\"                    -o -name \"vzdump-*.gz\" \\)                 -printf \"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\\n\"                 2>/dev/null |                 sort              echo === TARGET_UNIT_LOGS ===             journalctl                 -u homelab-backup-matrix-refresh.service                 -u homelab-vm160-remote-backup.service                 -u homelab-cluster-admin-pull.service                 -n 120                 --no-pager                 -o short-iso                 2>/dev/null ||                 true         \n=== LOCAL_ARCHIVES ===\nARCHIVE_COUNT=12\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst SIZE=323668231 MTIME=2026-07-19T03:45:46.2209130400\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst SIZE=326601118 MTIME=2026-07-20T03:30:08.2096782720\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst SIZE=329541843 MTIME=2026-07-21T03:30:09.7688696330\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-110-2026_07_22-03_30_01.tar.zst SIZE=332951369 MTIME=2026-07-22T03:30:08.7686076500\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst SIZE=222745965 MTIME=2026-07-19T03:45:57.2700790730\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst SIZE=222761273 MTIME=2026-07-20T03:30:13.7937629440\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst SIZE=222777732 MTIME=2026-07-21T03:30:15.9059638290\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-112-2026_07_22-03_30_09.tar.zst SIZE=222794054 MTIME=2026-07-22T03:30:14.2846933120\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst SIZE=7021297699 MTIME=2026-07-19T03:31:06.8755351020\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst SIZE=7016275481 MTIME=2026-07-20T03:31:03.2165123470\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst SIZE=7021952392 MTIME=2026-07-21T03:31:06.5717414950\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-150-2026_07_22-03_30_14.vma.zst SIZE=7052176114 MTIME=2026-07-22T03:31:02.8954482120\n=== TARGET_UNIT_LOGS ===\n2026-07-22T05:48:02+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T05:48:03+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T05:48:03+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T05:48:03+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n2026-07-22T06:11:12+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T06:11:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T06:11:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T06:11:13+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n2026-07-22T06:40:29+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T06:40:30+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T06:40:30+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T06:40:30+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n2026-07-22T07:02:40+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T07:02:41+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T07:02:41+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T07:02:41+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n2026-07-22T07:35:14+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T07:35:15+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T07:35:15+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T07:35:15+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n2026-07-22T08:10:42+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T08:10:43+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T08:10:43+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T08:10:43+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n2026-07-22T08:37:12+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T08:37:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T08:37:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T08:37:13+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n2026-07-22T08:40:15+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T08:40:16+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T08:40:16+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T08:40:16+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n2026-07-22T08:53:36+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T08:53:37+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T08:53:37+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T08:53:37+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n2026-07-22T10:11:14+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T10:11:15+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T10:11:15+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T10:11:15+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n========== NODE=pve02 ==========\nSSH_STATUS=OK\nHOST=pve02\nUTC_NOW=2026-07-22T14:53:54Z\n=== FAILED_UNITS ===\nFAILED_UNIT_COUNT=0\n=== TARGET_UNITS ===\nUNIT_MISSING=homelab-backup-matrix-refresh.service\nUNIT_MISSING=homelab-backup-matrix-refresh.timer\nUNIT_MISSING=homelab-vm160-remote-backup.service\nUNIT_MISSING=homelab-vm160-remote-backup.timer\nUNIT_MISSING=homelab-cluster-admin-pull.service\nUNIT_MISSING=homelab-cluster-admin-pull.timer\nUNIT_MISSING=homelab-vm-local-dumps-2cloud.service\nUNIT_MISSING=homelab-vm-local-dumps-2cloud.timer\nUNIT_MISSING=homelab-mail-cloud-critical-upload.service\nUNIT_MISSING=homelab-mail-cloud-critical-upload.timer\nUNIT_MISSING=homelab-mail-cloud-edge-vm-upload.service\nUNIT_MISSING=homelab-mail-cloud-edge-vm-upload.timer\nUNIT_MISSING=homelab-mail-cloud-nextcloud-vm-upload.service\nUNIT_MISSING=homelab-mail-cloud-nextcloud-vm-upload.timer\nUNIT_MISSING=homelab-mail-cloud-restore-drill.service\nUNIT_MISSING=homelab-mail-cloud-restore-drill.timer\nUNIT_MISSING=homelab-verified-backup-health.service\nUNIT_MISSING=homelab-verified-backup-health.timer\nUNIT_MISSING=homelab-backup-sla-health.service\nUNIT_MISSING=homelab-backup-sla-health.timer\n=== RELEVANT_TIMERS ===\n=== TARGET_SCRIPT_STATE ===\nFILE_MISSING=/usr/local/sbin/homelab-backup-matrix-refresh\nFILE_MISSING=/usr/local/sbin/homelab-vm160-remote-backup\nFILE_MISSING=/usr/local/sbin/homelab-cluster-admin-pull\nFILE_MISSING=/usr/local/sbin/homelab-vm-local-dumps-2cloud\nFILE_MISSING=/usr/local/sbin/homelab-mail-cloud-critical-upload\nFILE_MISSING=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload\nFILE_MISSING=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload\nFILE_MISSING=/usr/local/sbin/homelab-mail-cloud-restore-drill\n=== CURRENT_BACKUP_HEALTH ===\nHEALTH_FILE_MISSING=/var/lib/homelab-health/unified-backup-matrix.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/backup-framework.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/backup-sla.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/backup-coverage-matrix.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/vm-local-dumps-cloud.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/vm-local-dumps-retention.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/vm-backup.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/edge-vm-offhost-freshness.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/mail-cloud-restore-drill.txt\n=== RCLONE_REMOTE_NAMES ===\npve01-mail-01:\npve01-mail-01-crypt:\npve01-mail-02:\npve01-mail-02-crypt:\npve01-mail-03:\npve01-mail-03-crypt:\npve01-mail-04:\npve01-mail-04-crypt:\npve01-mail-05:\npve01-mail-05-crypt:\npve02-mail-01:\npve02-mail-01-crypt:\npve02-mail-02:\npve02-mail-02-crypt:\npve02-mail-03:\npve02-mail-03-crypt:\npve02-mail-04:\npve02-mail-04-crypt:\npve03-mail-01:\npve03-mail-01-crypt:\npve03-mail-02:\npve03-mail-02-crypt:\npve03-mail-03:\npve03-mail-03-crypt:\npve03-mail-04:\npve03-mail-04-crypt:\n=== ACTIVE_VZDUMP ===\n3930634 bash -c              set -u              echo HOST=$(hostname -s)             echo UTC_NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)              echo === FAILED_UNITS ===             echo -n FAILED_UNIT_COUNT=             systemctl --failed --no-legend --plain 2>/dev/null | wc -l             systemctl --failed --no-legend --plain 2>/dev/null || true              echo === TARGET_UNITS ===             for U in                 homelab-backup-matrix-refresh.service                 homelab-backup-matrix-refresh.timer                 homelab-vm160-remote-backup.service                 homelab-vm160-remote-backup.timer                 homelab-cluster-admin-pull.service                 homelab-cluster-admin-pull.timer                 homelab-vm-local-dumps-2cloud.service                 homelab-vm-local-dumps-2cloud.timer                 homelab-mail-cloud-critical-upload.service                 homelab-mail-cloud-critical-upload.timer                 homelab-mail-cloud-edge-vm-upload.service                 homelab-mail-cloud-edge-vm-upload.timer                 homelab-mail-cloud-nextcloud-vm-upload.service                 homelab-mail-cloud-nextcloud-vm-upload.timer                 homelab-mail-cloud-restore-drill.service                 homelab-mail-cloud-restore-drill.timer                 homelab-verified-backup-health.service                 homelab-verified-backup-health.timer                 homelab-backup-sla-health.service                 homelab-backup-sla-health.timer             do                 if systemctl cat \"$U\" >/dev/null 2>&1                 then                     systemctl show \"$U\"                         -p Id                         -p LoadState                         -p ActiveState                         -p SubState                         -p Result                         -p ExecMainStatus                         -p UnitFileState                         -p ActiveEnterTimestamp                         -p InactiveEnterTimestamp                         -p ExecStart                         --no-pager                 else                     echo UNIT_MISSING=$U                 fi             done              echo === RELEVANT_TIMERS ===             systemctl list-timers --all --no-pager --no-legend 2>/dev/null |                 grep -E \"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\" ||                 true              echo === TARGET_SCRIPT_STATE ===             for F in                 /usr/local/sbin/homelab-backup-matrix-refresh                 /usr/local/sbin/homelab-vm160-remote-backup                 /usr/local/sbin/homelab-cluster-admin-pull                 /usr/local/sbin/homelab-vm-local-dumps-2cloud                 /usr/local/sbin/homelab-mail-cloud-critical-upload                 /usr/local/sbin/homelab-mail-cloud-edge-vm-upload                 /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload                 /usr/local/sbin/homelab-mail-cloud-restore-drill             do                 if test -e \"$F\"                 then                     stat -c \"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\" \"$F\"                      if /usr/bin/bash -n \"$F\" 2>/dev/null                     then                         echo SCRIPT_SYNTAX=$F:OK                     else                         echo SCRIPT_SYNTAX=$F:FAIL                     fi                 else                     echo FILE_MISSING=$F                 fi             done              echo === CURRENT_BACKUP_HEALTH ===             for H in                 /var/lib/homelab-health/unified-backup-matrix.txt                 /var/lib/homelab-health/backup-framework.txt                 /var/lib/homelab-health/backup-sla.txt                 /var/lib/homelab-health/backup-coverage-matrix.txt                 /var/lib/homelab-health/vm-local-dumps-cloud.txt                 /var/lib/homelab-health/vm-local-dumps-retention.txt                 /var/lib/homelab-health/vm-backup.txt                 /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt                 /var/lib/homelab-health/edge-vm-offhost-freshness.txt                 /var/lib/homelab-health/mail-cloud-restore-drill.txt             do                 if test -f \"$H\"                 then                     echo HEALTH_FILE=$H                     stat -c \"HEALTH_MTIME=%y HEALTH_SIZE=%s\" \"$H\"                     sed -n \"1p\" \"$H\"                 else                     echo HEALTH_FILE_MISSING=$H                 fi             done              echo === RCLONE_REMOTE_NAMES ===             if command -v rclone >/dev/null 2>&1             then                 rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL             else                 echo RCLONE=NOT_INSTALLED             fi              echo === ACTIVE_VZDUMP ===             pgrep -af \"[v]zdump\" || echo ACTIVE_VZDUMP=NO              echo === LOCAL_ARCHIVES ===             echo -n ARCHIVE_COUNT=             find /var/lib/vz/dump /mnt/pve                 -type f                 \\( -name \"vzdump-*.zst\"                    -o -name \"vzdump-*.lzo\"                    -o -name \"vzdump-*.gz\" \\)                 -printf x                 2>/dev/null |                 wc -c              find /var/lib/vz/dump /mnt/pve                 -type f                 \\( -name \"vzdump-*.zst\"                    -o -name \"vzdump-*.lzo\"                    -o -name \"vzdump-*.gz\" \\)                 -printf \"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\\n\"                 2>/dev/null |                 sort              echo === TARGET_UNIT_LOGS ===             journalctl                 -u homelab-backup-matrix-refresh.service                 -u homelab-vm160-remote-backup.service                 -u homelab-cluster-admin-pull.service                 -n 120                 --no-pager                 -o short-iso                 2>/dev/null ||                 true         \n=== LOCAL_ARCHIVES ===\nARCHIVE_COUNT=10\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst SIZE=284443608 MTIME=2026-07-19T03:47:41.8525503200\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.tar.zst SIZE=286538526 MTIME=2026-07-20T03:30:18.4386603270\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.tar.zst SIZE=288349157 MTIME=2026-07-21T03:30:21.2480085730\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-111-2026_07_22-03_30_01.tar.zst SIZE=290059742 MTIME=2026-07-22T03:30:22.8748248510\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst SIZE=220049041 MTIME=2026-07-19T03:47:58.9256407960\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.tar.zst SIZE=220065825 MTIME=2026-07-20T03:30:30.8287264090\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.tar.zst SIZE=220080970 MTIME=2026-07-21T03:30:33.3910787180\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-113-2026_07_22-03_30_23.tar.zst SIZE=220095608 MTIME=2026-07-22T03:30:40.6809207660\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.vma.zst SIZE=868048369 MTIME=2026-06-30T03:31:08.4903256780\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.vma.zst SIZE=997503422 MTIME=2026-07-01T03:31:11.1700059310\n=== TARGET_UNIT_LOGS ===\n-- No entries --\n========== NODE=pve03 ==========\nSSH_STATUS=OK\nHOST=pve03\nUTC_NOW=2026-07-22T14:53:55Z\n=== FAILED_UNITS ===\nFAILED_UNIT_COUNT=1\nhomelab-immutable-offline-verify.service loaded failed failed Verify immutable offline backup image\n=== TARGET_UNITS ===\nUNIT_MISSING=homelab-backup-matrix-refresh.service\nUNIT_MISSING=homelab-backup-matrix-refresh.timer\nUNIT_MISSING=homelab-vm160-remote-backup.service\nUNIT_MISSING=homelab-vm160-remote-backup.timer\nUNIT_MISSING=homelab-cluster-admin-pull.service\nUNIT_MISSING=homelab-cluster-admin-pull.timer\nUNIT_MISSING=homelab-vm-local-dumps-2cloud.service\nUNIT_MISSING=homelab-vm-local-dumps-2cloud.timer\nUNIT_MISSING=homelab-mail-cloud-critical-upload.service\nUNIT_MISSING=homelab-mail-cloud-critical-upload.timer\nUNIT_MISSING=homelab-mail-cloud-edge-vm-upload.service\nUNIT_MISSING=homelab-mail-cloud-edge-vm-upload.timer\nUNIT_MISSING=homelab-mail-cloud-nextcloud-vm-upload.service\nUNIT_MISSING=homelab-mail-cloud-nextcloud-vm-upload.timer\nUNIT_MISSING=homelab-mail-cloud-restore-drill.service\nUNIT_MISSING=homelab-mail-cloud-restore-drill.timer\nUNIT_MISSING=homelab-verified-backup-health.service\nUNIT_MISSING=homelab-verified-backup-health.timer\nUNIT_MISSING=homelab-backup-sla-health.service\nUNIT_MISSING=homelab-backup-sla-health.timer\n=== RELEVANT_TIMERS ===\n=== TARGET_SCRIPT_STATE ===\nFILE_MISSING=/usr/local/sbin/homelab-backup-matrix-refresh\nFILE_MISSING=/usr/local/sbin/homelab-vm160-remote-backup\nFILE_MISSING=/usr/local/sbin/homelab-cluster-admin-pull\nFILE_MISSING=/usr/local/sbin/homelab-vm-local-dumps-2cloud\nFILE_MISSING=/usr/local/sbin/homelab-mail-cloud-critical-upload\nFILE_MISSING=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload\nFILE_MISSING=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload\nFILE_MISSING=/usr/local/sbin/homelab-mail-cloud-restore-drill\n=== CURRENT_BACKUP_HEALTH ===\nHEALTH_FILE_MISSING=/var/lib/homelab-health/unified-backup-matrix.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/backup-framework.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/backup-sla.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/backup-coverage-matrix.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/vm-local-dumps-cloud.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/vm-local-dumps-retention.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/vm-backup.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/edge-vm-offhost-freshness.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/mail-cloud-restore-drill.txt\n=== RCLONE_REMOTE_NAMES ===\npve01-mail-01:\npve01-mail-01-crypt:\npve01-mail-02:\npve01-mail-02-crypt:\npve01-mail-03:\npve01-mail-03-crypt:\npve01-mail-04:\npve01-mail-04-crypt:\npve01-mail-05:\npve01-mail-05-crypt:\npve02-mail-01:\npve02-mail-01-crypt:\npve02-mail-02:\npve02-mail-02-crypt:\npve02-mail-03:\npve02-mail-03-crypt:\npve02-mail-04:\npve02-mail-04-crypt:\npve03-mail-01:\npve03-mail-01-crypt:\npve03-mail-02:\npve03-mail-02-crypt:\npve03-mail-03:\npve03-mail-03-crypt:\npve03-mail-04:\npve03-mail-04-crypt:\n=== ACTIVE_VZDUMP ===\n2313989 bash -c              set -u              echo HOST=$(hostname -s)             echo UTC_NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)              echo === FAILED_UNITS ===             echo -n FAILED_UNIT_COUNT=             systemctl --failed --no-legend --plain 2>/dev/null | wc -l             systemctl --failed --no-legend --plain 2>/dev/null || true              echo === TARGET_UNITS ===             for U in                 homelab-backup-matrix-refresh.service                 homelab-backup-matrix-refresh.timer                 homelab-vm160-remote-backup.service                 homelab-vm160-remote-backup.timer                 homelab-cluster-admin-pull.service                 homelab-cluster-admin-pull.timer                 homelab-vm-local-dumps-2cloud.service                 homelab-vm-local-dumps-2cloud.timer                 homelab-mail-cloud-critical-upload.service                 homelab-mail-cloud-critical-upload.timer                 homelab-mail-cloud-edge-vm-upload.service                 homelab-mail-cloud-edge-vm-upload.timer                 homelab-mail-cloud-nextcloud-vm-upload.service                 homelab-mail-cloud-nextcloud-vm-upload.timer                 homelab-mail-cloud-restore-drill.service                 homelab-mail-cloud-restore-drill.timer                 homelab-verified-backup-health.service                 homelab-verified-backup-health.timer                 homelab-backup-sla-health.service                 homelab-backup-sla-health.timer             do                 if systemctl cat \"$U\" >/dev/null 2>&1                 then                     systemctl show \"$U\"                         -p Id                         -p LoadState                         -p ActiveState                         -p SubState                         -p Result                         -p ExecMainStatus                         -p UnitFileState                         -p ActiveEnterTimestamp                         -p InactiveEnterTimestamp                         -p ExecStart                         --no-pager                 else                     echo UNIT_MISSING=$U                 fi             done              echo === RELEVANT_TIMERS ===             systemctl list-timers --all --no-pager --no-legend 2>/dev/null |                 grep -E \"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\" ||                 true              echo === TARGET_SCRIPT_STATE ===             for F in                 /usr/local/sbin/homelab-backup-matrix-refresh                 /usr/local/sbin/homelab-vm160-remote-backup                 /usr/local/sbin/homelab-cluster-admin-pull                 /usr/local/sbin/homelab-vm-local-dumps-2cloud                 /usr/local/sbin/homelab-mail-cloud-critical-upload                 /usr/local/sbin/homelab-mail-cloud-edge-vm-upload                 /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload                 /usr/local/sbin/homelab-mail-cloud-restore-drill             do                 if test -e \"$F\"                 then                     stat -c \"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\" \"$F\"                      if /usr/bin/bash -n \"$F\" 2>/dev/null                     then                         echo SCRIPT_SYNTAX=$F:OK                     else                         echo SCRIPT_SYNTAX=$F:FAIL                     fi                 else                     echo FILE_MISSING=$F                 fi             done              echo === CURRENT_BACKUP_HEALTH ===             for H in                 /var/lib/homelab-health/unified-backup-matrix.txt                 /var/lib/homelab-health/backup-framework.txt                 /var/lib/homelab-health/backup-sla.txt                 /var/lib/homelab-health/backup-coverage-matrix.txt                 /var/lib/homelab-health/vm-local-dumps-cloud.txt                 /var/lib/homelab-health/vm-local-dumps-retention.txt                 /var/lib/homelab-health/vm-backup.txt                 /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt                 /var/lib/homelab-health/edge-vm-offhost-freshness.txt                 /var/lib/homelab-health/mail-cloud-restore-drill.txt             do                 if test -f \"$H\"                 then                     echo HEALTH_FILE=$H                     stat -c \"HEALTH_MTIME=%y HEALTH_SIZE=%s\" \"$H\"                     sed -n \"1p\" \"$H\"                 else                     echo HEALTH_FILE_MISSING=$H                 fi             done              echo === RCLONE_REMOTE_NAMES ===             if command -v rclone >/dev/null 2>&1             then                 rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL             else                 echo RCLONE=NOT_INSTALLED             fi              echo === ACTIVE_VZDUMP ===             pgrep -af \"[v]zdump\" || echo ACTIVE_VZDUMP=NO              echo === LOCAL_ARCHIVES ===             echo -n ARCHIVE_COUNT=             find /var/lib/vz/dump /mnt/pve                 -type f                 \\( -name \"vzdump-*.zst\"                    -o -name \"vzdump-*.lzo\"                    -o -name \"vzdump-*.gz\" \\)                 -printf x                 2>/dev/null |                 wc -c              find /var/lib/vz/dump /mnt/pve                 -type f                 \\( -name \"vzdump-*.zst\"                    -o -name \"vzdump-*.lzo\"                    -o -name \"vzdump-*.gz\" \\)                 -printf \"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\\n\"                 2>/dev/null |                 sort              echo === TARGET_UNIT_LOGS ===             journalctl                 -u homelab-backup-matrix-refresh.service                 -u homelab-vm160-remote-backup.service                 -u homelab-cluster-admin-pull.service                 -n 120                 --no-pager                 -o short-iso                 2>/dev/null ||                 true         \n=== LOCAL_ARCHIVES ===\nARCHIVE_COUNT=8\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.vma.zst SIZE=11013723404 MTIME=2026-07-19T03:33:53.3463878040\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.vma.zst SIZE=11116472459 MTIME=2026-07-20T03:33:53.2839064370\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.vma.zst SIZE=11043834894 MTIME=2026-07-21T03:33:54.3725716230\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-130-2026_07_22-03_30_02.vma.zst SIZE=11072513783 MTIME=2026-07-22T03:33:58.8961071480\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.vma.zst SIZE=838951976 MTIME=2026-07-19T03:34:33.1034859540\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.vma.zst SIZE=838817925 MTIME=2026-07-20T03:34:37.1719936670\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.vma.zst SIZE=1137072350 MTIME=2026-07-21T03:34:37.6276811360\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-190-2026_07_22-03_33_59.vma.zst SIZE=1177407443 MTIME=2026-07-22T03:34:44.3412293530\n=== TARGET_UNIT_LOGS ===\n-- No entries --\nCHANGES_MADE=NO\nSECRET_PRINTED=NO\nPRIVATE_ADDRESS_PRINTED=NO\nRESULT=OK\nBACKUP_OFFHOST_CURRENT_STATE=COMPLETE\n"
}
