From 4619971bb7f4584455055e617dc391ab4441ffe9 Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Tue, 21 Jul 2026 11:35:23 +0000 Subject: [PATCH] runtime: publish BACKUP-DUAL-CLOUD-PREFLIGHT-059 --- .../BACKUP-DUAL-CLOUD-PREFLIGHT-059.json | 30 ++ .../BACKUP-DUAL-CLOUD-PREFLIGHT-059.txt | 275 +++++++++++++++ runtime/latest.json | 18 +- runtime/latest.txt | 318 ++++++++++++++---- 4 files changed, 571 insertions(+), 70 deletions(-) create mode 100644 runtime/history/BACKUP-DUAL-CLOUD-PREFLIGHT-059.json create mode 100644 runtime/history/BACKUP-DUAL-CLOUD-PREFLIGHT-059.txt diff --git a/runtime/history/BACKUP-DUAL-CLOUD-PREFLIGHT-059.json b/runtime/history/BACKUP-DUAL-CLOUD-PREFLIGHT-059.json new file mode 100644 index 0000000..20903e4 --- /dev/null +++ b/runtime/history/BACKUP-DUAL-CLOUD-PREFLIGHT-059.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "BACKUP-DUAL-CLOUD-PREFLIGHT-059", + "status": "FAIL", + "rc": 1, + "host": "pve01", + "mode": "read-only", + "component": "backup-audit", + "started_at_utc": "2026-07-21T11:35:22Z", + "finished_at_utc": "2026-07-21T11:35:23Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1", + "command_sha256": "c6f4134d15a9f641ddaf67535760d0abede973dd08222a37af05612a6f580479", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "f375162905664fc3de53bec3ea2b2485775e05bcfbd421a3c63010def946a753", + "sanitized_output_sha256": "f375162905664fc3de53bec3ea2b2485775e05bcfbd421a3c63010def946a753", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "=== BACKUP_JOBS ===\n[{\"compress\":\"zstd\",\"enabled\":1,\"id\":\"homelab-nightly-all\",\"mailnotification\":\"failure\",\"mode\":\"snapshot\",\"next-run\":1784680200,\"prune-backups\":{\"keep-daily\":\"4\"},\"schedule\":\"03:30\",\"storage\":\"local\",\"type\":\"vzdump\",\"vmid\":\"110,111,112,113,130,150,190\"}]\n=== RCLONE_REMOTES ===\npve01-mail-01:\npve01-mail-01-crypt:\npve01-mail-02:\npve01-mail-02-crypt:\npve01-mail-03:\npve01-mail-03-crypt:\npve01-mail-04:\npve01-mail-04-crypt:\npve01-mail-05:\npve01-mail-05-crypt:\npve02-mail-01:\npve02-mail-01-crypt:\npve02-mail-02:\npve02-mail-02-crypt:\npve02-mail-03:\npve02-mail-03-crypt:\npve02-mail-04:\npve02-mail-04-crypt:\npve03-mail-01:\npve03-mail-01-crypt:\npve03-mail-02:\npve03-mail-02-crypt:\npve03-mail-03:\npve03-mail-03-crypt:\npve03-mail-04:\npve03-mail-04-crypt:\n=== BACKUP_TIMERS ===\nWed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:14 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service\nWed 2026-07-22 00:09:43 MSK 9h Tue 2026-07-21 00:09:32 MSK 14h ago homelab-backup-freshness.timer homelab-backup-freshness.service\nWed 2026-07-22 02:15:00 MSK 11h Tue 2026-07-21 02:15:05 MSK 12h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service\nWed 2026-07-22 03:10:02 MSK 12h Tue 2026-07-21 03:13:14 MSK 11h ago homelab-emergency-backup.timer homelab-emergency-backup.service\nWed 2026-07-22 03:35:00 MSK 12h Tue 2026-07-21 03:35:14 MSK 11h ago filebrowser-backup.timer filebrowser-backup.service\nWed 2026-07-22 03:55:02 MSK 13h Tue 2026-07-21 03:51:25 MSK 10h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service\nWed 2026-07-22 04:17:29 MSK 13h Tue 2026-07-21 04:19:17 MSK 10h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service\nWed 2026-07-22 04:26:54 MSK 13h Tue 2026-07-21 04:28:14 MSK 10h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service\nWed 2026-07-22 04:38:04 MSK 14h Tue 2026-07-21 04:34:48 MSK 10h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service\nWed 2026-07-22 05:01:44 MSK 14h Tue 2026-07-21 04:57:14 MSK 9h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service\nWed 2026-07-22 05:23:56 MSK 14h Tue 2026-07-21 05:17:57 MSK 9h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service\nWed 2026-07-22 05:27:13 MSK 14h Tue 2026-07-21 05:29:05 MSK 9h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service\nWed 2026-07-22 05:47:19 MSK 15h Tue 2026-07-21 05:46:14 MSK 8h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service\nWed 2026-07-22 06:01:19 MSK 15h Tue 2026-07-21 05:57:30 MSK 8h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service\nWed 2026-07-22 06:27:43 MSK 15h Tue 2026-07-21 06:32:25 MSK 8h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service\nWed 2026-07-22 06:38:51 MSK 16h Tue 2026-07-21 06:43:54 MSK 7h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service\nWed 2026-07-22 06:39:22 MSK 16h Tue 2026-07-21 06:40:25 MSK 7h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service\nWed 2026-07-22 06:44:16 MSK 16h Tue 2026-07-21 06:40:25 MSK 7h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service\nWed 2026-07-22 06:52:06 MSK 16h Tue 2026-07-21 06:47:47 MSK 7h ago homelab-beszel-backup.timer homelab-beszel-backup.service\nWed 2026-07-22 06:58:58 MSK 16h Tue 2026-07-21 06:57:30 MSK 7h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service\nWed 2026-07-22 07:02:27 MSK 16h Tue 2026-07-21 07:05:30 MSK 7h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service\nWed 2026-07-22 07:07:14 MSK 16h Tue 2026-07-21 07:18:55 MSK 7h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service\nWed 2026-07-22 07:35:06 MSK 16h Tue 2026-07-21 07:36:59 MSK 6h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service\nWed 2026-07-22 07:38:21 MSK 17h Tue 2026-07-21 07:36:14 MSK 6h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service\nWed 2026-07-22 07:41:49 MSK 17h Tue 2026-07-21 07:53:41 MSK 6h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service\nWed 2026-07-22 08:10:37 MSK 17h Tue 2026-07-21 08:07:25 MSK 6h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service\nWed 2026-07-22 08:37:02 MSK 18h Tue 2026-07-21 08:30:07 MSK 6h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service\nWed 2026-07-22 08:40:13 MSK 18h Tue 2026-07-21 08:50:14 MSK 5h 45min ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service\nWed 2026-07-22 08:53:36 MSK 18h Tue 2026-07-21 08:41:14 MSK 5h 54min ago memos-backup.timer memos-backup.service\nSun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service\nSun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service\nSun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service\nSun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service\nMon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 7h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service\n=== BACKUP_UNITS ===\ndpkg-db-backup.service static -\nfilebrowser-backup.service static -\nhomelab-alertmanager-backup.service static -\nhomelab-appbackup-dockge-stacks.service static -\nhomelab-appbackup-mariadb-batch.service static -\nhomelab-appbackup-npmplus-edge.service static -\nhomelab-appbackup-postgres-batch.service static -\nhomelab-appbackup-sqlite-batch.service static -\nhomelab-authentik-backup.service static -\nhomelab-auto-backup.service static -\nhomelab-backup-coverage-matrix.service static -\nhomelab-backup-freshness.service static -\nhomelab-backup-healthcheck.service static -\nhomelab-backup-matrix-refresh.service static -\nhomelab-backup-sla-health.service static -\nhomelab-beszel-backup.service static -\nhomelab-blackbox-exporter-backup.service static -\nhomelab-cluster-admin-incident-engine-backup.service static -\nhomelab-crypto-portfolio-backup.service static -\nhomelab-edge-vm-vzdump-backup.service static -\nhomelab-emergency-backup.service static -\nhomelab-extended-appbackup.service static -\nhomelab-gitea-backup.service static -\nhomelab-gitea-secondary-backup.service static -\nhomelab-legacy-monitoring-backups.service static -\nhomelab-mail-cloud-critical-upload.service static -\nhomelab-mail-cloud-edge-vm-upload.service static -\nhomelab-mail-cloud-nextcloud-vm-upload.service static -\nhomelab-mail-cloud-restore-drill.service static -\nhomelab-npmplus-kuma-config-backup.service static -\nhomelab-router-running-config-mail-cloud.service static -\nhomelab-vaultwarden-backup.service static -\nhomelab-verified-backup-health.service static -\nhomelab-vm-backup-health.service static -\nhomelab-vm-backup-policy.service static -\nhomelab-vm-full-backup.service static -\nhomelab-vm-local-dumps-2cloud.service static -\nhomelab-vm160-remote-backup.service static -\nimmich-media-backup.service static -\nmemos-backup.service static -\nnetbird-vps-backup.service static -\npaperless-backup.service static -\nskladchik-reports-monitor-backup-restore-check.service static -\nskladchik-reports-monitor-backup.service static -\ndpkg-db-backup.timer enabled enabled\nfilebrowser-backup.timer enabled enabled\nhomelab-alertmanager-backup.timer enabled enabled\nhomelab-appbackup-dockge-stacks.timer enabled enabled\nhomelab-appbackup-mariadb-batch.timer enabled enabled\nhomelab-appbackup-npmplus-edge.timer enabled enabled\nhomelab-appbackup-postgres-batch.timer enabled enabled\nhomelab-appbackup-sqlite-batch.timer enabled enabled\nhomelab-authentik-backup.timer disabled enabled\nhomelab-auto-backup.timer enabled enabled\nhomelab-backup-coverage-matrix.timer enabled enabled\nhomelab-backup-freshness.timer enabled enabled\nhomelab-backup-healthcheck.timer enabled enabled\nhomelab-backup-matrix-refresh.timer disabled enabled\nhomelab-backup-sla-health.timer enabled enabled\nhomelab-beszel-backup.timer enabled enabled\nhomelab-blackbox-exporter-backup.timer enabled enabled\nhomelab-cluster-admin-incident-engine-backup.timer enabled enabled\nhomelab-crypto-portfolio-backup.timer enabled enabled\nhomelab-edge-vm-vzdump-backup.timer disabled enabled\nhomelab-emergency-backup.timer enabled enabled\nhomelab-extended-appbackup.timer enabled enabled\nhomelab-gitea-backup.timer disabled enabled\nhomelab-gitea-secondary-backup.timer enabled enabled\nhomelab-legacy-monitoring-backups.timer enabled enabled\nhomelab-mail-cloud-critical-upload.timer enabled enabled\nhomelab-mail-cloud-edge-vm-upload.timer enabled enabled\nhomelab-mail-cloud-nextcloud-vm-upload.timer enabled enabled\nhomelab-mail-cloud-restore-drill.timer enabled enabled\nhomelab-npmplus-kuma-config-backup.timer disabled enabled\nhomelab-router-running-config-mail-cloud.timer enabled enabled\nhomelab-vaultwarden-backup.timer disabled enabled\nhomelab-verified-backup-health.timer enabled enabled\nhomelab-vm-backup-health.timer enabled enabled\nhomelab-vm-backup-policy.timer enabled enabled\nhomelab-vm-full-backup.timer enabled enabled\nhomelab-vm-local-dumps-2cloud.timer enabled enabled\nhomelab-vm160-remote-backup.timer disabled enabled\nimmich-media-backup.timer disabled enabled\nmemos-backup.timer enabled enabled\nnetbird-vps-backup.timer disabled enabled\npaperless-backup.timer disabled enabled\nskladchik-reports-monitor-backup-restore-check.timer enabled enabled\nskladchik-reports-monitor-backup.timer enabled enabled\n=== PIPELINE_FILES ===\n/etc/cron.d/vzdump\n/etc/systemd/system/homelab-edge-vm-offhost-freshness.timer\n/etc/systemd/system/homelab-edge-vm-vzdump-backup.service\n/etc/systemd/system/homelab-edge-vm-vzdump-backup.timer\n/etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.service\n/etc/systemd/system/homelab-vm-full-backup.service\n/etc/systemd/system/multi-user.target.wants/pve-guests.service\n/etc/systemd/system/pve-manager.service\n/etc/systemd/system/timers.target.wants/homelab-edge-vm-offhost-freshness.timer\n/usr/local/sbin/appbackup-dockge-stacks\n/usr/local/sbin/appbackup-dockge-stacks.bak-chunkall-20260710T011410Z\n/usr/local/sbin/appbackup-restore-check\n/usr/local/sbin/appbackup-restore-check.bak-chunkall-20260710T011410Z\n/usr/local/sbin/appbackup-restore-check.bak-dual-read-20260711T051924Z\n/usr/local/sbin/appbackup-restore-check.bak-flat-read-20260711T052854Z\n/usr/local/sbin/appbackup-restore-check.bak-flat-read-v2-20260711T053129Z\n/usr/local/sbin/appbackup-retention\n/usr/local/sbin/appbackup-retention.bak-dual-read-20260711T051924Z\n/usr/local/sbin/backupctl\n/usr/local/sbin/backup-restore-check\n/usr/local/sbin/backup-restore-check.bak-chunkall-20260710T011410Z\n/usr/local/sbin/backup-retention\n/usr/local/sbin/homelab-backup-edge-vm-vzdump\n/usr/local/sbin/homelab-capacity-risk\n/usr/local/sbin/homelab-change-transaction\n/usr/local/sbin/homelab-chat-run\n/usr/local/sbin/homelab-edge-vm-offhost-freshness\n/usr/local/sbin/homelab-extended-appbackup-run\n/usr/local/sbin/homelab-golden-state-extended-overlay\n/usr/local/sbin/homelab-golden-state-index\n/usr/local/sbin/homelab-health-metrics\n/usr/local/sbin/homelab-ingress-standby-refresh\n/usr/local/sbin/homelab-mail-cloud-critical-upload\n/usr/local/sbin/homelab-mail-cloud-critical-upload.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload\n/usr/local/sbin/homelab-mail-cloud-restore-drill\n/usr/local/sbin/homelab-mail-cloud-restore-drill.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-rclone-cat-chunked\n/usr/local/sbin/homelab-rclone-cat-flat-chunked\n/usr/local/sbin/homelab-rclone-copy-chunked\n/usr/local/sbin/homelab-rclone-copy-flat-chunked\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh\n/usr/local/sbin/homelab-rclone-required-check.sh\n/usr/local/sbin/homelab-router-hard-power-cycle\n/usr/local/sbin/homelab-router-running-config-mail-cloud\n/usr/local/sbin/homelab-router-running-config-mail-cloud.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-safe-exec\n/usr/local/sbin/homelab-secret-exposure-guard\n/usr/local/sbin/homelab-stage-2cloud\n/usr/local/sbin/homelab-stage-2cloud.bak-global-chunkonly-20260710T011036Z\n/usr/local/sbin/homelab-stage-2cloud-flat\n/usr/local/sbin/homelab-stage-2cloud-flat-canary\n/usr/local/sbin/homelab-stage-verify-chunked\n/usr/local/sbin/homelab-stage-verify-flat-chunked\n/usr/local/sbin/homelab-vm160-remote-backup\n/usr/local/sbin/homelab-vm-backup-health\n/usr/local/sbin/homelab-vm-backup-policy\n/usr/local/sbin/homelab-vm-full-backup\n/usr/local/sbin/homelab-vm-full-backup.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-vm-full-restore-drill\n/usr/local/sbin/homelab-vm-full-restore-drill.bak.20260710T080231Z\n/usr/local/sbin/homelab-vm-full-restore-drill.bak-docker-check-v2-20260710T050044Z\n/usr/local/sbin/homelab-vm-full-restore-drill.bak-isolated-ip-20260710T044331Z\n/usr/local/sbin/homelab-vm-full-retention\n/usr/local/sbin/homelab-vm-local-dumps-2cloud\n/usr/local/sbin/homelab-vm-local-dumps-retention\n/usr/local/sbin/skladchik-reports-monitor-backup\n/usr/local/sbin/skladchik-reports-monitor-backup.pre-restore-check-20260710T065311Z.bak\n/usr/local/sbin/skladchik-reports-monitor-backup.pre-session-keepalive-20260710T071605Z.bak\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check.pre-session-keepalive-20260710T071605Z.bak\n=== LOCAL_BACKUPS ===\n7022386770\t2026-07-18T03:31:03.6566057240\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.vma.zst\n7021952392\t2026-07-21T03:31:06.5717414950\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst\n7021297699\t2026-07-19T03:31:06.8755351020\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst\n7016275481\t2026-07-20T03:31:03.2165123470\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst\n329541843\t2026-07-21T03:30:09.7688696330\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst\n326601118\t2026-07-20T03:30:08.2096782720\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst\n323668231\t2026-07-19T03:45:46.2209130400\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst\n321121649\t2026-07-18T03:30:08.0777625900\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.tar.zst\n222777732\t2026-07-21T03:30:15.9059638290\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst\n222761273\t2026-07-20T03:30:13.7937629440\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst\n222745965\t2026-07-19T03:45:57.2700790730\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst\n222727760\t2026-07-18T03:30:13.5578457230\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.tar.zst\n3021\t2026-07-19T03:31:07.4745284120\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.log\n3020\t2026-07-21T03:31:07.3945472590\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.log\n3018\t2026-07-18T03:31:03.9197988860\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.log\n3017\t2026-07-20T03:31:04.1706719350\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.log\n1089\t2026-07-20T03:30:14.0079985330\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.log\n1086\t2026-07-20T03:30:08.4640241540\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.log\n989\t2026-07-21T03:30:16.1364976560\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.log\n989\t2026-07-18T03:30:13.7816006690\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.log\n986\t2026-07-21T03:30:10.0285327490\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.log\n986\t2026-07-18T03:30:08.3002411830\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.log\n731\t2026-07-19T03:45:57.5197000270\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.log\n729\t2026-07-19T03:45:46.4444121980\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.log\n" +} diff --git a/runtime/history/BACKUP-DUAL-CLOUD-PREFLIGHT-059.txt b/runtime/history/BACKUP-DUAL-CLOUD-PREFLIGHT-059.txt new file mode 100644 index 0000000..d1acdf4 --- /dev/null +++ b/runtime/history/BACKUP-DUAL-CLOUD-PREFLIGHT-059.txt @@ -0,0 +1,275 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=BACKUP-DUAL-CLOUD-PREFLIGHT-059 +STATUS=FAIL +RC=1 +HOST=pve01 +MODE=read-only +COMPONENT=backup-audit +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1 +COMMAND_SHA256=c6f4134d15a9f641ddaf67535760d0abede973dd08222a37af05612a6f580479 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=false +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=f375162905664fc3de53bec3ea2b2485775e05bcfbd421a3c63010def946a753 +SANITIZED_OUTPUT_SHA256=f375162905664fc3de53bec3ea2b2485775e05bcfbd421a3c63010def946a753 +OUTPUT_BEGIN +=== BACKUP_JOBS === +[{"compress":"zstd","enabled":1,"id":"homelab-nightly-all","mailnotification":"failure","mode":"snapshot","next-run":1784680200,"prune-backups":{"keep-daily":"4"},"schedule":"03:30","storage":"local","type":"vzdump","vmid":"110,111,112,113,130,150,190"}] +=== RCLONE_REMOTES === +pve01-mail-01: +pve01-mail-01-crypt: +pve01-mail-02: +pve01-mail-02-crypt: +pve01-mail-03: +pve01-mail-03-crypt: +pve01-mail-04: +pve01-mail-04-crypt: +pve01-mail-05: +pve01-mail-05-crypt: +pve02-mail-01: +pve02-mail-01-crypt: +pve02-mail-02: +pve02-mail-02-crypt: +pve02-mail-03: +pve02-mail-03-crypt: +pve02-mail-04: +pve02-mail-04-crypt: +pve03-mail-01: +pve03-mail-01-crypt: +pve03-mail-02: +pve03-mail-02-crypt: +pve03-mail-03: +pve03-mail-03-crypt: +pve03-mail-04: +pve03-mail-04-crypt: +=== BACKUP_TIMERS === +Wed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:14 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service +Wed 2026-07-22 00:09:43 MSK 9h Tue 2026-07-21 00:09:32 MSK 14h ago homelab-backup-freshness.timer homelab-backup-freshness.service +Wed 2026-07-22 02:15:00 MSK 11h Tue 2026-07-21 02:15:05 MSK 12h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service +Wed 2026-07-22 03:10:02 MSK 12h Tue 2026-07-21 03:13:14 MSK 11h ago homelab-emergency-backup.timer homelab-emergency-backup.service +Wed 2026-07-22 03:35:00 MSK 12h Tue 2026-07-21 03:35:14 MSK 11h ago filebrowser-backup.timer filebrowser-backup.service +Wed 2026-07-22 03:55:02 MSK 13h Tue 2026-07-21 03:51:25 MSK 10h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service +Wed 2026-07-22 04:17:29 MSK 13h Tue 2026-07-21 04:19:17 MSK 10h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service +Wed 2026-07-22 04:26:54 MSK 13h Tue 2026-07-21 04:28:14 MSK 10h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service +Wed 2026-07-22 04:38:04 MSK 14h Tue 2026-07-21 04:34:48 MSK 10h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service +Wed 2026-07-22 05:01:44 MSK 14h Tue 2026-07-21 04:57:14 MSK 9h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service +Wed 2026-07-22 05:23:56 MSK 14h Tue 2026-07-21 05:17:57 MSK 9h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service +Wed 2026-07-22 05:27:13 MSK 14h Tue 2026-07-21 05:29:05 MSK 9h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service +Wed 2026-07-22 05:47:19 MSK 15h Tue 2026-07-21 05:46:14 MSK 8h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service +Wed 2026-07-22 06:01:19 MSK 15h Tue 2026-07-21 05:57:30 MSK 8h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service +Wed 2026-07-22 06:27:43 MSK 15h Tue 2026-07-21 06:32:25 MSK 8h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service +Wed 2026-07-22 06:38:51 MSK 16h Tue 2026-07-21 06:43:54 MSK 7h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service +Wed 2026-07-22 06:39:22 MSK 16h Tue 2026-07-21 06:40:25 MSK 7h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service +Wed 2026-07-22 06:44:16 MSK 16h Tue 2026-07-21 06:40:25 MSK 7h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service +Wed 2026-07-22 06:52:06 MSK 16h Tue 2026-07-21 06:47:47 MSK 7h ago homelab-beszel-backup.timer homelab-beszel-backup.service +Wed 2026-07-22 06:58:58 MSK 16h Tue 2026-07-21 06:57:30 MSK 7h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service +Wed 2026-07-22 07:02:27 MSK 16h Tue 2026-07-21 07:05:30 MSK 7h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service +Wed 2026-07-22 07:07:14 MSK 16h Tue 2026-07-21 07:18:55 MSK 7h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service +Wed 2026-07-22 07:35:06 MSK 16h Tue 2026-07-21 07:36:59 MSK 6h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service +Wed 2026-07-22 07:38:21 MSK 17h Tue 2026-07-21 07:36:14 MSK 6h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service +Wed 2026-07-22 07:41:49 MSK 17h Tue 2026-07-21 07:53:41 MSK 6h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service +Wed 2026-07-22 08:10:37 MSK 17h Tue 2026-07-21 08:07:25 MSK 6h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service +Wed 2026-07-22 08:37:02 MSK 18h Tue 2026-07-21 08:30:07 MSK 6h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service +Wed 2026-07-22 08:40:13 MSK 18h Tue 2026-07-21 08:50:14 MSK 5h 45min ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service +Wed 2026-07-22 08:53:36 MSK 18h Tue 2026-07-21 08:41:14 MSK 5h 54min ago memos-backup.timer memos-backup.service +Sun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service +Sun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service +Sun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service +Sun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service +Mon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 7h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service +=== BACKUP_UNITS === +dpkg-db-backup.service static - +filebrowser-backup.service static - +homelab-alertmanager-backup.service static - +homelab-appbackup-dockge-stacks.service static - +homelab-appbackup-mariadb-batch.service static - +homelab-appbackup-npmplus-edge.service static - +homelab-appbackup-postgres-batch.service static - +homelab-appbackup-sqlite-batch.service static - +homelab-authentik-backup.service static - +homelab-auto-backup.service static - +homelab-backup-coverage-matrix.service static - +homelab-backup-freshness.service static - +homelab-backup-healthcheck.service static - +homelab-backup-matrix-refresh.service static - +homelab-backup-sla-health.service static - +homelab-beszel-backup.service static - +homelab-blackbox-exporter-backup.service static - +homelab-cluster-admin-incident-engine-backup.service static - +homelab-crypto-portfolio-backup.service static - +homelab-edge-vm-vzdump-backup.service static - +homelab-emergency-backup.service static - +homelab-extended-appbackup.service static - +homelab-gitea-backup.service static - +homelab-gitea-secondary-backup.service static - +homelab-legacy-monitoring-backups.service static - +homelab-mail-cloud-critical-upload.service static - +homelab-mail-cloud-edge-vm-upload.service static - +homelab-mail-cloud-nextcloud-vm-upload.service static - +homelab-mail-cloud-restore-drill.service static - +homelab-npmplus-kuma-config-backup.service static - +homelab-router-running-config-mail-cloud.service static - +homelab-vaultwarden-backup.service static - +homelab-verified-backup-health.service static - +homelab-vm-backup-health.service static - +homelab-vm-backup-policy.service static - +homelab-vm-full-backup.service static - +homelab-vm-local-dumps-2cloud.service static - +homelab-vm160-remote-backup.service static - +immich-media-backup.service static - +memos-backup.service static - +netbird-vps-backup.service static - +paperless-backup.service static - +skladchik-reports-monitor-backup-restore-check.service static - +skladchik-reports-monitor-backup.service static - +dpkg-db-backup.timer enabled enabled +filebrowser-backup.timer enabled enabled +homelab-alertmanager-backup.timer enabled enabled +homelab-appbackup-dockge-stacks.timer enabled enabled +homelab-appbackup-mariadb-batch.timer enabled enabled +homelab-appbackup-npmplus-edge.timer enabled enabled +homelab-appbackup-postgres-batch.timer enabled enabled +homelab-appbackup-sqlite-batch.timer enabled enabled +homelab-authentik-backup.timer disabled enabled +homelab-auto-backup.timer enabled enabled +homelab-backup-coverage-matrix.timer enabled enabled +homelab-backup-freshness.timer enabled enabled +homelab-backup-healthcheck.timer enabled enabled +homelab-backup-matrix-refresh.timer disabled enabled +homelab-backup-sla-health.timer enabled enabled +homelab-beszel-backup.timer enabled enabled +homelab-blackbox-exporter-backup.timer enabled enabled +homelab-cluster-admin-incident-engine-backup.timer enabled enabled +homelab-crypto-portfolio-backup.timer enabled enabled +homelab-edge-vm-vzdump-backup.timer disabled enabled +homelab-emergency-backup.timer enabled enabled +homelab-extended-appbackup.timer enabled enabled +homelab-gitea-backup.timer disabled enabled +homelab-gitea-secondary-backup.timer enabled enabled +homelab-legacy-monitoring-backups.timer enabled enabled +homelab-mail-cloud-critical-upload.timer enabled enabled +homelab-mail-cloud-edge-vm-upload.timer enabled enabled +homelab-mail-cloud-nextcloud-vm-upload.timer enabled enabled +homelab-mail-cloud-restore-drill.timer enabled enabled +homelab-npmplus-kuma-config-backup.timer disabled enabled +homelab-router-running-config-mail-cloud.timer enabled enabled +homelab-vaultwarden-backup.timer disabled enabled +homelab-verified-backup-health.timer enabled enabled +homelab-vm-backup-health.timer enabled enabled +homelab-vm-backup-policy.timer enabled enabled +homelab-vm-full-backup.timer enabled enabled +homelab-vm-local-dumps-2cloud.timer enabled enabled +homelab-vm160-remote-backup.timer disabled enabled +immich-media-backup.timer disabled enabled +memos-backup.timer enabled enabled +netbird-vps-backup.timer disabled enabled +paperless-backup.timer disabled enabled +skladchik-reports-monitor-backup-restore-check.timer enabled enabled +skladchik-reports-monitor-backup.timer enabled enabled +=== PIPELINE_FILES === +/etc/cron.d/vzdump +/etc/systemd/system/homelab-edge-vm-offhost-freshness.timer +/etc/systemd/system/homelab-edge-vm-vzdump-backup.service +/etc/systemd/system/homelab-edge-vm-vzdump-backup.timer +/etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.service +/etc/systemd/system/homelab-vm-full-backup.service +/etc/systemd/system/multi-user.target.wants/pve-guests.service +/etc/systemd/system/pve-manager.service +/etc/systemd/system/timers.target.wants/homelab-edge-vm-offhost-freshness.timer +/usr/local/sbin/appbackup-dockge-stacks +/usr/local/sbin/appbackup-dockge-stacks.bak-chunkall-20260710T011410Z +/usr/local/sbin/appbackup-restore-check +/usr/local/sbin/appbackup-restore-check.bak-chunkall-20260710T011410Z +/usr/local/sbin/appbackup-restore-check.bak-dual-read-20260711T051924Z +/usr/local/sbin/appbackup-restore-check.bak-flat-read-20260711T052854Z +/usr/local/sbin/appbackup-restore-check.bak-flat-read-v2-20260711T053129Z +/usr/local/sbin/appbackup-retention +/usr/local/sbin/appbackup-retention.bak-dual-read-20260711T051924Z +/usr/local/sbin/backupctl +/usr/local/sbin/backup-restore-check +/usr/local/sbin/backup-restore-check.bak-chunkall-20260710T011410Z +/usr/local/sbin/backup-retention +/usr/local/sbin/homelab-backup-edge-vm-vzdump +/usr/local/sbin/homelab-capacity-risk +/usr/local/sbin/homelab-change-transaction +/usr/local/sbin/homelab-chat-run +/usr/local/sbin/homelab-edge-vm-offhost-freshness +/usr/local/sbin/homelab-extended-appbackup-run +/usr/local/sbin/homelab-golden-state-extended-overlay +/usr/local/sbin/homelab-golden-state-index +/usr/local/sbin/homelab-health-metrics +/usr/local/sbin/homelab-ingress-standby-refresh +/usr/local/sbin/homelab-mail-cloud-critical-upload +/usr/local/sbin/homelab-mail-cloud-critical-upload.bak-chunkall-20260710T011410Z +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload +/usr/local/sbin/homelab-mail-cloud-restore-drill +/usr/local/sbin/homelab-mail-cloud-restore-drill.bak-chunkall-20260710T011410Z +/usr/local/sbin/homelab-rclone-cat-chunked +/usr/local/sbin/homelab-rclone-cat-flat-chunked +/usr/local/sbin/homelab-rclone-copy-chunked +/usr/local/sbin/homelab-rclone-copy-flat-chunked +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh +/usr/local/sbin/homelab-rclone-required-check.sh +/usr/local/sbin/homelab-router-hard-power-cycle +/usr/local/sbin/homelab-router-running-config-mail-cloud +/usr/local/sbin/homelab-router-running-config-mail-cloud.bak-chunkall-20260710T011410Z +/usr/local/sbin/homelab-safe-exec +/usr/local/sbin/homelab-secret-exposure-guard +/usr/local/sbin/homelab-stage-2cloud +/usr/local/sbin/homelab-stage-2cloud.bak-global-chunkonly-20260710T011036Z +/usr/local/sbin/homelab-stage-2cloud-flat +/usr/local/sbin/homelab-stage-2cloud-flat-canary +/usr/local/sbin/homelab-stage-verify-chunked +/usr/local/sbin/homelab-stage-verify-flat-chunked +/usr/local/sbin/homelab-vm160-remote-backup +/usr/local/sbin/homelab-vm-backup-health +/usr/local/sbin/homelab-vm-backup-policy +/usr/local/sbin/homelab-vm-full-backup +/usr/local/sbin/homelab-vm-full-backup.bak-chunkall-20260710T011410Z +/usr/local/sbin/homelab-vm-full-restore-drill +/usr/local/sbin/homelab-vm-full-restore-drill.bak.20260710T080231Z +/usr/local/sbin/homelab-vm-full-restore-drill.bak-docker-check-v2-20260710T050044Z +/usr/local/sbin/homelab-vm-full-restore-drill.bak-isolated-ip-20260710T044331Z +/usr/local/sbin/homelab-vm-full-retention +/usr/local/sbin/homelab-vm-local-dumps-2cloud +/usr/local/sbin/homelab-vm-local-dumps-retention +/usr/local/sbin/skladchik-reports-monitor-backup +/usr/local/sbin/skladchik-reports-monitor-backup.pre-restore-check-20260710T065311Z.bak +/usr/local/sbin/skladchik-reports-monitor-backup.pre-session-keepalive-20260710T071605Z.bak +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check.pre-session-keepalive-20260710T071605Z.bak +=== LOCAL_BACKUPS === +7022386770 2026-07-18T03:31:03.6566057240 /var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.vma.zst +7021952392 2026-07-21T03:31:06.5717414950 /var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst +7021297699 2026-07-19T03:31:06.8755351020 /var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst +7016275481 2026-07-20T03:31:03.2165123470 /var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst +329541843 2026-07-21T03:30:09.7688696330 /var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst +326601118 2026-07-20T03:30:08.2096782720 /var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst +323668231 2026-07-19T03:45:46.2209130400 /var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst +321121649 2026-07-18T03:30:08.0777625900 /var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.tar.zst +222777732 2026-07-21T03:30:15.9059638290 /var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst +222761273 2026-07-20T03:30:13.7937629440 /var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst +222745965 2026-07-19T03:45:57.2700790730 /var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst +222727760 2026-07-18T03:30:13.5578457230 /var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.tar.zst +3021 2026-07-19T03:31:07.4745284120 /var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.log +3020 2026-07-21T03:31:07.3945472590 /var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.log +3018 2026-07-18T03:31:03.9197988860 /var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.log +3017 2026-07-20T03:31:04.1706719350 /var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.log +1089 2026-07-20T03:30:14.0079985330 /var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.log +1086 2026-07-20T03:30:08.4640241540 /var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.log +989 2026-07-21T03:30:16.1364976560 /var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.log +989 2026-07-18T03:30:13.7816006690 /var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.log +986 2026-07-21T03:30:10.0285327490 /var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.log +986 2026-07-18T03:30:08.3002411830 /var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.log +731 2026-07-19T03:45:57.5197000270 /var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.log +729 2026-07-19T03:45:46.4444121980 /var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.log + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 89c49cb..20903e4 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "BACKUP-COVERAGE-MATRIX-058", - "status": "OK", - "rc": 0, + "command_id": "BACKUP-DUAL-CLOUD-PREFLIGHT-059", + "status": "FAIL", + "rc": 1, "host": "pve01", "mode": "read-only", "component": "backup-audit", - "started_at_utc": "2026-07-21T11:33:10Z", - "finished_at_utc": "2026-07-21T11:33:10Z", + "started_at_utc": "2026-07-21T11:35:22Z", + "finished_at_utc": "2026-07-21T11:35:23Z", "reference_register_checked": true, "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", "error_register_checked": true, "error_register_sha256": "f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1", - "command_sha256": "fe94aa788ebacd90258b858b36ebc3281bf8fac23b6891c779266673f4097044", + "command_sha256": "c6f4134d15a9f641ddaf67535760d0abede973dd08222a37af05612a6f580479", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -22,9 +22,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "deb5d7538452c026a02d069058efac10b3eadf8933bb3bc7540b3fc5982372ff", - "sanitized_output_sha256": "deb5d7538452c026a02d069058efac10b3eadf8933bb3bc7540b3fc5982372ff", + "raw_evidence_sha256": "f375162905664fc3de53bec3ea2b2485775e05bcfbd421a3c63010def946a753", + "sanitized_output_sha256": "f375162905664fc3de53bec3ea2b2485775e05bcfbd421a3c63010def946a753", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "STATUS=FAIL\nTS=2026-07-21T07:02:41Z\nTYPE=unified-backup-matrix\nMATRIX_MODE=authoritative\nJOBS_TOTAL=41\nJOBS_OK=30\nJOBS_BAD=6\nJOBS_UNMAPPED=0\nJOBS_DISABLED=5\nSTATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_JSON=/etc/homelab-backup-matrix/matrix.json\nAUTHORITY=canonical\nCOMPAT_OUTPUTS=29\nCOMPAT_MAP=/var/lib/homelab-backup-matrix/compat-map.json\nJOB=dpkg-db-backup STATUS=OK EVIDENCE=execution_only AGE=36147 MAX_AGE=172800\nJOB=filebrowser-backup STATUS=OK EVIDENCE=artifact_or_health AGE=14443 MAX_AGE=259200\nJOB=filebrowser-restore-validate STATUS=STALE EVIDENCE=artifact_or_health AGE=1016424 MAX_AGE=864000\nJOB=homelab-appbackup-dockge-stacks STATUS=OK EVIDENCE=artifact_or_health AGE=22248 MAX_AGE=172800\nJOB=homelab-appbackup-mariadb-batch STATUS=OK EVIDENCE=execution_only AGE=15387 MAX_AGE=172800\nJOB=homelab-appbackup-npmplus-edge STATUS=OK EVIDENCE=artifact_or_health AGE=20573 MAX_AGE=172800\nJOB=homelab-appbackup-postgres-batch STATUS=OK EVIDENCE=execution_only AGE=16416 MAX_AGE=172800\nJOB=homelab-appbackup-sqlite-batch STATUS=OK EVIDENCE=execution_only AGE=18327 MAX_AGE=172800\nJOB=homelab-authentik-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987790 MAX_AGE=172800\nJOB=homelab-auto-backup STATUS=OK EVIDENCE=execution_only AGE=195424 MAX_AGE=864000\nJOB=homelab-backup-coverage-matrix STATUS=OK EVIDENCE=artifact_or_health AGE=4346 MAX_AGE=172800\nJOB=homelab-backup-freshness STATUS=OK EVIDENCE=execution_only AGE=35589 MAX_AGE=172800\nJOB=homelab-backup-healthcheck STATUS=OK EVIDENCE=execution_only AGE=10631 MAX_AGE=172800\nJOB=homelab-backup-sla-health STATUS=FAIL EVIDENCE=artifact_or_health AGE=144 MAX_AGE=172800\nJOB=homelab-edge-vm-offhost-freshness STATUS=OK EVIDENCE=artifact_or_health AGE=18426 MAX_AGE=172800\nJOB=homelab-emergency-backup STATUS=OK EVIDENCE=artifact_or_health AGE=24539 MAX_AGE=172800\nJOB=homelab-extended-appbackup STATUS=OK EVIDENCE=artifact_or_health AGE=14156 MAX_AGE=172800\nJOB=homelab-gitea-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987785 MAX_AGE=172800\nJOB=homelab-golden-state-index STATUS=OK EVIDENCE=artifact_or_health AGE=4112 MAX_AGE=172800\nJOB=homelab-mail-cloud-critical-upload STATUS=OK EVIDENCE=artifact_or_health AGE=11997 MAX_AGE=172800\nJOB=homelab-mail-cloud-edge-vm-upload STATUS=OK EVIDENCE=artifact_or_health AGE=174045 MAX_AGE=864000\nJOB=homelab-mail-cloud-nextcloud-vm-upload STATUS=OK EVIDENCE=artifact_or_health AGE=7575 MAX_AGE=864000\nJOB=homelab-mail-cloud-restore-drill STATUS=STALE EVIDENCE=artifact_or_health AGE=988269 MAX_AGE=864000\nJOB=homelab-npmplus-kuma-config-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=60071 MAX_AGE=172800\nJOB=homelab-router-running-config-mail-cloud STATUS=OK EVIDENCE=artifact_or_health AGE=17076 MAX_AGE=172800\nJOB=homelab-vaultwarden-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987768 MAX_AGE=172800\nJOB=homelab-verified-backup-health STATUS=OK EVIDENCE=artifact_or_health AGE=11845 MAX_AGE=172800\nJOB=homelab-vm-backup-health STATUS=OK EVIDENCE=artifact_or_health AGE=8742 MAX_AGE=172800\nJOB=homelab-vm-backup-policy STATUS=OK EVIDENCE=artifact_or_health AGE=5548 MAX_AGE=172800\nJOB=homelab-vm-full-backup STATUS=OK EVIDENCE=artifact_or_health AGE=24778 MAX_AGE=172800\nJOB=homelab-vm-full-restore-drill STATUS=OK EVIDENCE=artifact_or_health AGE=523085 MAX_AGE=10368000\nJOB=homelab-vm-local-dumps-2cloud STATUS=OK EVIDENCE=artifact_or_health AGE=172141 MAX_AGE=864000\nJOB=homelab-vm-local-dumps-retention STATUS=OK EVIDENCE=artifact_or_health AGE=168839 MAX_AGE=864000\nJOB=immich-media-offhost-sync STATUS=OK EVIDENCE=artifact_or_health AGE=4286 MAX_AGE=259200\nJOB=memos-backup STATUS=OK EVIDENCE=artifact_or_health AGE=14547 MAX_AGE=259200\nJOB=netbird-vps-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=1747896 MAX_AGE=864000\nJOB=paperless-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=16347 MAX_AGE=259200\nJOB=paperless-restore-dry-run STATUS=STALE EVIDENCE=artifact_or_health AGE=1001986 MAX_AGE=864000\nJOB=skladchik-reports-monitor-backup STATUS=OK EVIDENCE=artifact_or_health AGE=19639 MAX_AGE=172800\nJOB=skladchik-reports-monitor-backup-restore-check STATUS=OK EVIDENCE=artifact_or_health AGE=190273 MAX_AGE=864000\nJOB=vm160-remote-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=26607 MAX_AGE=172800\n" + "output": "=== BACKUP_JOBS ===\n[{\"compress\":\"zstd\",\"enabled\":1,\"id\":\"homelab-nightly-all\",\"mailnotification\":\"failure\",\"mode\":\"snapshot\",\"next-run\":1784680200,\"prune-backups\":{\"keep-daily\":\"4\"},\"schedule\":\"03:30\",\"storage\":\"local\",\"type\":\"vzdump\",\"vmid\":\"110,111,112,113,130,150,190\"}]\n=== RCLONE_REMOTES ===\npve01-mail-01:\npve01-mail-01-crypt:\npve01-mail-02:\npve01-mail-02-crypt:\npve01-mail-03:\npve01-mail-03-crypt:\npve01-mail-04:\npve01-mail-04-crypt:\npve01-mail-05:\npve01-mail-05-crypt:\npve02-mail-01:\npve02-mail-01-crypt:\npve02-mail-02:\npve02-mail-02-crypt:\npve02-mail-03:\npve02-mail-03-crypt:\npve02-mail-04:\npve02-mail-04-crypt:\npve03-mail-01:\npve03-mail-01-crypt:\npve03-mail-02:\npve03-mail-02-crypt:\npve03-mail-03:\npve03-mail-03-crypt:\npve03-mail-04:\npve03-mail-04-crypt:\n=== BACKUP_TIMERS ===\nWed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:14 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service\nWed 2026-07-22 00:09:43 MSK 9h Tue 2026-07-21 00:09:32 MSK 14h ago homelab-backup-freshness.timer homelab-backup-freshness.service\nWed 2026-07-22 02:15:00 MSK 11h Tue 2026-07-21 02:15:05 MSK 12h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service\nWed 2026-07-22 03:10:02 MSK 12h Tue 2026-07-21 03:13:14 MSK 11h ago homelab-emergency-backup.timer homelab-emergency-backup.service\nWed 2026-07-22 03:35:00 MSK 12h Tue 2026-07-21 03:35:14 MSK 11h ago filebrowser-backup.timer filebrowser-backup.service\nWed 2026-07-22 03:55:02 MSK 13h Tue 2026-07-21 03:51:25 MSK 10h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service\nWed 2026-07-22 04:17:29 MSK 13h Tue 2026-07-21 04:19:17 MSK 10h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service\nWed 2026-07-22 04:26:54 MSK 13h Tue 2026-07-21 04:28:14 MSK 10h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service\nWed 2026-07-22 04:38:04 MSK 14h Tue 2026-07-21 04:34:48 MSK 10h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service\nWed 2026-07-22 05:01:44 MSK 14h Tue 2026-07-21 04:57:14 MSK 9h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service\nWed 2026-07-22 05:23:56 MSK 14h Tue 2026-07-21 05:17:57 MSK 9h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service\nWed 2026-07-22 05:27:13 MSK 14h Tue 2026-07-21 05:29:05 MSK 9h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service\nWed 2026-07-22 05:47:19 MSK 15h Tue 2026-07-21 05:46:14 MSK 8h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service\nWed 2026-07-22 06:01:19 MSK 15h Tue 2026-07-21 05:57:30 MSK 8h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service\nWed 2026-07-22 06:27:43 MSK 15h Tue 2026-07-21 06:32:25 MSK 8h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service\nWed 2026-07-22 06:38:51 MSK 16h Tue 2026-07-21 06:43:54 MSK 7h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service\nWed 2026-07-22 06:39:22 MSK 16h Tue 2026-07-21 06:40:25 MSK 7h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service\nWed 2026-07-22 06:44:16 MSK 16h Tue 2026-07-21 06:40:25 MSK 7h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service\nWed 2026-07-22 06:52:06 MSK 16h Tue 2026-07-21 06:47:47 MSK 7h ago homelab-beszel-backup.timer homelab-beszel-backup.service\nWed 2026-07-22 06:58:58 MSK 16h Tue 2026-07-21 06:57:30 MSK 7h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service\nWed 2026-07-22 07:02:27 MSK 16h Tue 2026-07-21 07:05:30 MSK 7h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service\nWed 2026-07-22 07:07:14 MSK 16h Tue 2026-07-21 07:18:55 MSK 7h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service\nWed 2026-07-22 07:35:06 MSK 16h Tue 2026-07-21 07:36:59 MSK 6h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service\nWed 2026-07-22 07:38:21 MSK 17h Tue 2026-07-21 07:36:14 MSK 6h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service\nWed 2026-07-22 07:41:49 MSK 17h Tue 2026-07-21 07:53:41 MSK 6h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service\nWed 2026-07-22 08:10:37 MSK 17h Tue 2026-07-21 08:07:25 MSK 6h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service\nWed 2026-07-22 08:37:02 MSK 18h Tue 2026-07-21 08:30:07 MSK 6h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service\nWed 2026-07-22 08:40:13 MSK 18h Tue 2026-07-21 08:50:14 MSK 5h 45min ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service\nWed 2026-07-22 08:53:36 MSK 18h Tue 2026-07-21 08:41:14 MSK 5h 54min ago memos-backup.timer memos-backup.service\nSun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service\nSun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service\nSun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service\nSun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service\nMon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 7h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service\n=== BACKUP_UNITS ===\ndpkg-db-backup.service static -\nfilebrowser-backup.service static -\nhomelab-alertmanager-backup.service static -\nhomelab-appbackup-dockge-stacks.service static -\nhomelab-appbackup-mariadb-batch.service static -\nhomelab-appbackup-npmplus-edge.service static -\nhomelab-appbackup-postgres-batch.service static -\nhomelab-appbackup-sqlite-batch.service static -\nhomelab-authentik-backup.service static -\nhomelab-auto-backup.service static -\nhomelab-backup-coverage-matrix.service static -\nhomelab-backup-freshness.service static -\nhomelab-backup-healthcheck.service static -\nhomelab-backup-matrix-refresh.service static -\nhomelab-backup-sla-health.service static -\nhomelab-beszel-backup.service static -\nhomelab-blackbox-exporter-backup.service static -\nhomelab-cluster-admin-incident-engine-backup.service static -\nhomelab-crypto-portfolio-backup.service static -\nhomelab-edge-vm-vzdump-backup.service static -\nhomelab-emergency-backup.service static -\nhomelab-extended-appbackup.service static -\nhomelab-gitea-backup.service static -\nhomelab-gitea-secondary-backup.service static -\nhomelab-legacy-monitoring-backups.service static -\nhomelab-mail-cloud-critical-upload.service static -\nhomelab-mail-cloud-edge-vm-upload.service static -\nhomelab-mail-cloud-nextcloud-vm-upload.service static -\nhomelab-mail-cloud-restore-drill.service static -\nhomelab-npmplus-kuma-config-backup.service static -\nhomelab-router-running-config-mail-cloud.service static -\nhomelab-vaultwarden-backup.service static -\nhomelab-verified-backup-health.service static -\nhomelab-vm-backup-health.service static -\nhomelab-vm-backup-policy.service static -\nhomelab-vm-full-backup.service static -\nhomelab-vm-local-dumps-2cloud.service static -\nhomelab-vm160-remote-backup.service static -\nimmich-media-backup.service static -\nmemos-backup.service static -\nnetbird-vps-backup.service static -\npaperless-backup.service static -\nskladchik-reports-monitor-backup-restore-check.service static -\nskladchik-reports-monitor-backup.service static -\ndpkg-db-backup.timer enabled enabled\nfilebrowser-backup.timer enabled enabled\nhomelab-alertmanager-backup.timer enabled enabled\nhomelab-appbackup-dockge-stacks.timer enabled enabled\nhomelab-appbackup-mariadb-batch.timer enabled enabled\nhomelab-appbackup-npmplus-edge.timer enabled enabled\nhomelab-appbackup-postgres-batch.timer enabled enabled\nhomelab-appbackup-sqlite-batch.timer enabled enabled\nhomelab-authentik-backup.timer disabled enabled\nhomelab-auto-backup.timer enabled enabled\nhomelab-backup-coverage-matrix.timer enabled enabled\nhomelab-backup-freshness.timer enabled enabled\nhomelab-backup-healthcheck.timer enabled enabled\nhomelab-backup-matrix-refresh.timer disabled enabled\nhomelab-backup-sla-health.timer enabled enabled\nhomelab-beszel-backup.timer enabled enabled\nhomelab-blackbox-exporter-backup.timer enabled enabled\nhomelab-cluster-admin-incident-engine-backup.timer enabled enabled\nhomelab-crypto-portfolio-backup.timer enabled enabled\nhomelab-edge-vm-vzdump-backup.timer disabled enabled\nhomelab-emergency-backup.timer enabled enabled\nhomelab-extended-appbackup.timer enabled enabled\nhomelab-gitea-backup.timer disabled enabled\nhomelab-gitea-secondary-backup.timer enabled enabled\nhomelab-legacy-monitoring-backups.timer enabled enabled\nhomelab-mail-cloud-critical-upload.timer enabled enabled\nhomelab-mail-cloud-edge-vm-upload.timer enabled enabled\nhomelab-mail-cloud-nextcloud-vm-upload.timer enabled enabled\nhomelab-mail-cloud-restore-drill.timer enabled enabled\nhomelab-npmplus-kuma-config-backup.timer disabled enabled\nhomelab-router-running-config-mail-cloud.timer enabled enabled\nhomelab-vaultwarden-backup.timer disabled enabled\nhomelab-verified-backup-health.timer enabled enabled\nhomelab-vm-backup-health.timer enabled enabled\nhomelab-vm-backup-policy.timer enabled enabled\nhomelab-vm-full-backup.timer enabled enabled\nhomelab-vm-local-dumps-2cloud.timer enabled enabled\nhomelab-vm160-remote-backup.timer disabled enabled\nimmich-media-backup.timer disabled enabled\nmemos-backup.timer enabled enabled\nnetbird-vps-backup.timer disabled enabled\npaperless-backup.timer disabled enabled\nskladchik-reports-monitor-backup-restore-check.timer enabled enabled\nskladchik-reports-monitor-backup.timer enabled enabled\n=== PIPELINE_FILES ===\n/etc/cron.d/vzdump\n/etc/systemd/system/homelab-edge-vm-offhost-freshness.timer\n/etc/systemd/system/homelab-edge-vm-vzdump-backup.service\n/etc/systemd/system/homelab-edge-vm-vzdump-backup.timer\n/etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.service\n/etc/systemd/system/homelab-vm-full-backup.service\n/etc/systemd/system/multi-user.target.wants/pve-guests.service\n/etc/systemd/system/pve-manager.service\n/etc/systemd/system/timers.target.wants/homelab-edge-vm-offhost-freshness.timer\n/usr/local/sbin/appbackup-dockge-stacks\n/usr/local/sbin/appbackup-dockge-stacks.bak-chunkall-20260710T011410Z\n/usr/local/sbin/appbackup-restore-check\n/usr/local/sbin/appbackup-restore-check.bak-chunkall-20260710T011410Z\n/usr/local/sbin/appbackup-restore-check.bak-dual-read-20260711T051924Z\n/usr/local/sbin/appbackup-restore-check.bak-flat-read-20260711T052854Z\n/usr/local/sbin/appbackup-restore-check.bak-flat-read-v2-20260711T053129Z\n/usr/local/sbin/appbackup-retention\n/usr/local/sbin/appbackup-retention.bak-dual-read-20260711T051924Z\n/usr/local/sbin/backupctl\n/usr/local/sbin/backup-restore-check\n/usr/local/sbin/backup-restore-check.bak-chunkall-20260710T011410Z\n/usr/local/sbin/backup-retention\n/usr/local/sbin/homelab-backup-edge-vm-vzdump\n/usr/local/sbin/homelab-capacity-risk\n/usr/local/sbin/homelab-change-transaction\n/usr/local/sbin/homelab-chat-run\n/usr/local/sbin/homelab-edge-vm-offhost-freshness\n/usr/local/sbin/homelab-extended-appbackup-run\n/usr/local/sbin/homelab-golden-state-extended-overlay\n/usr/local/sbin/homelab-golden-state-index\n/usr/local/sbin/homelab-health-metrics\n/usr/local/sbin/homelab-ingress-standby-refresh\n/usr/local/sbin/homelab-mail-cloud-critical-upload\n/usr/local/sbin/homelab-mail-cloud-critical-upload.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload\n/usr/local/sbin/homelab-mail-cloud-restore-drill\n/usr/local/sbin/homelab-mail-cloud-restore-drill.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-rclone-cat-chunked\n/usr/local/sbin/homelab-rclone-cat-flat-chunked\n/usr/local/sbin/homelab-rclone-copy-chunked\n/usr/local/sbin/homelab-rclone-copy-flat-chunked\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh\n/usr/local/sbin/homelab-rclone-required-check.sh\n/usr/local/sbin/homelab-router-hard-power-cycle\n/usr/local/sbin/homelab-router-running-config-mail-cloud\n/usr/local/sbin/homelab-router-running-config-mail-cloud.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-safe-exec\n/usr/local/sbin/homelab-secret-exposure-guard\n/usr/local/sbin/homelab-stage-2cloud\n/usr/local/sbin/homelab-stage-2cloud.bak-global-chunkonly-20260710T011036Z\n/usr/local/sbin/homelab-stage-2cloud-flat\n/usr/local/sbin/homelab-stage-2cloud-flat-canary\n/usr/local/sbin/homelab-stage-verify-chunked\n/usr/local/sbin/homelab-stage-verify-flat-chunked\n/usr/local/sbin/homelab-vm160-remote-backup\n/usr/local/sbin/homelab-vm-backup-health\n/usr/local/sbin/homelab-vm-backup-policy\n/usr/local/sbin/homelab-vm-full-backup\n/usr/local/sbin/homelab-vm-full-backup.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-vm-full-restore-drill\n/usr/local/sbin/homelab-vm-full-restore-drill.bak.20260710T080231Z\n/usr/local/sbin/homelab-vm-full-restore-drill.bak-docker-check-v2-20260710T050044Z\n/usr/local/sbin/homelab-vm-full-restore-drill.bak-isolated-ip-20260710T044331Z\n/usr/local/sbin/homelab-vm-full-retention\n/usr/local/sbin/homelab-vm-local-dumps-2cloud\n/usr/local/sbin/homelab-vm-local-dumps-retention\n/usr/local/sbin/skladchik-reports-monitor-backup\n/usr/local/sbin/skladchik-reports-monitor-backup.pre-restore-check-20260710T065311Z.bak\n/usr/local/sbin/skladchik-reports-monitor-backup.pre-session-keepalive-20260710T071605Z.bak\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check.pre-session-keepalive-20260710T071605Z.bak\n=== LOCAL_BACKUPS ===\n7022386770\t2026-07-18T03:31:03.6566057240\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.vma.zst\n7021952392\t2026-07-21T03:31:06.5717414950\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst\n7021297699\t2026-07-19T03:31:06.8755351020\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst\n7016275481\t2026-07-20T03:31:03.2165123470\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst\n329541843\t2026-07-21T03:30:09.7688696330\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst\n326601118\t2026-07-20T03:30:08.2096782720\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst\n323668231\t2026-07-19T03:45:46.2209130400\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst\n321121649\t2026-07-18T03:30:08.0777625900\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.tar.zst\n222777732\t2026-07-21T03:30:15.9059638290\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst\n222761273\t2026-07-20T03:30:13.7937629440\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst\n222745965\t2026-07-19T03:45:57.2700790730\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst\n222727760\t2026-07-18T03:30:13.5578457230\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.tar.zst\n3021\t2026-07-19T03:31:07.4745284120\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.log\n3020\t2026-07-21T03:31:07.3945472590\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.log\n3018\t2026-07-18T03:31:03.9197988860\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.log\n3017\t2026-07-20T03:31:04.1706719350\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.log\n1089\t2026-07-20T03:30:14.0079985330\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.log\n1086\t2026-07-20T03:30:08.4640241540\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.log\n989\t2026-07-21T03:30:16.1364976560\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.log\n989\t2026-07-18T03:30:13.7816006690\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.log\n986\t2026-07-21T03:30:10.0285327490\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.log\n986\t2026-07-18T03:30:08.3002411830\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.log\n731\t2026-07-19T03:45:57.5197000270\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.log\n729\t2026-07-19T03:45:46.4444121980\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.log\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 47f1db8..d1acdf4 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,7 +1,7 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=BACKUP-COVERAGE-MATRIX-058 -STATUS=OK -RC=0 +COMMAND_ID=BACKUP-DUAL-CLOUD-PREFLIGHT-059 +STATUS=FAIL +RC=1 HOST=pve01 MODE=read-only COMPONENT=backup-audit @@ -9,71 +9,267 @@ REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1 -COMMAND_SHA256=fe94aa788ebacd90258b858b36ebc3281bf8fac23b6891c779266673f4097044 +COMMAND_SHA256=c6f4134d15a9f641ddaf67535760d0abede973dd08222a37af05612a6f580479 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGES_MADE=false SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=deb5d7538452c026a02d069058efac10b3eadf8933bb3bc7540b3fc5982372ff -SANITIZED_OUTPUT_SHA256=deb5d7538452c026a02d069058efac10b3eadf8933bb3bc7540b3fc5982372ff +RAW_EVIDENCE_SHA256=f375162905664fc3de53bec3ea2b2485775e05bcfbd421a3c63010def946a753 +SANITIZED_OUTPUT_SHA256=f375162905664fc3de53bec3ea2b2485775e05bcfbd421a3c63010def946a753 OUTPUT_BEGIN -STATUS=FAIL -TS=2026-07-21T07:02:41Z -TYPE=unified-backup-matrix -MATRIX_MODE=authoritative -JOBS_TOTAL=41 -JOBS_OK=30 -JOBS_BAD=6 -JOBS_UNMAPPED=0 -JOBS_DISABLED=5 -STATE_JSON=/var/lib/homelab-backup-matrix/state.json -MATRIX_JSON=/etc/homelab-backup-matrix/matrix.json -AUTHORITY=canonical -COMPAT_OUTPUTS=29 -COMPAT_MAP=/var/lib/homelab-backup-matrix/compat-map.json -JOB=dpkg-db-backup STATUS=OK EVIDENCE=execution_only AGE=36147 MAX_AGE=172800 -JOB=filebrowser-backup STATUS=OK EVIDENCE=artifact_or_health AGE=14443 MAX_AGE=259200 -JOB=filebrowser-restore-validate STATUS=STALE EVIDENCE=artifact_or_health AGE=1016424 MAX_AGE=864000 -JOB=homelab-appbackup-dockge-stacks STATUS=OK EVIDENCE=artifact_or_health AGE=22248 MAX_AGE=172800 -JOB=homelab-appbackup-mariadb-batch STATUS=OK EVIDENCE=execution_only AGE=15387 MAX_AGE=172800 -JOB=homelab-appbackup-npmplus-edge STATUS=OK EVIDENCE=artifact_or_health AGE=20573 MAX_AGE=172800 -JOB=homelab-appbackup-postgres-batch STATUS=OK EVIDENCE=execution_only AGE=16416 MAX_AGE=172800 -JOB=homelab-appbackup-sqlite-batch STATUS=OK EVIDENCE=execution_only AGE=18327 MAX_AGE=172800 -JOB=homelab-authentik-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987790 MAX_AGE=172800 -JOB=homelab-auto-backup STATUS=OK EVIDENCE=execution_only AGE=195424 MAX_AGE=864000 -JOB=homelab-backup-coverage-matrix STATUS=OK EVIDENCE=artifact_or_health AGE=4346 MAX_AGE=172800 -JOB=homelab-backup-freshness STATUS=OK EVIDENCE=execution_only AGE=35589 MAX_AGE=172800 -JOB=homelab-backup-healthcheck STATUS=OK EVIDENCE=execution_only AGE=10631 MAX_AGE=172800 -JOB=homelab-backup-sla-health STATUS=FAIL EVIDENCE=artifact_or_health AGE=144 MAX_AGE=172800 -JOB=homelab-edge-vm-offhost-freshness STATUS=OK EVIDENCE=artifact_or_health AGE=18426 MAX_AGE=172800 -JOB=homelab-emergency-backup STATUS=OK EVIDENCE=artifact_or_health AGE=24539 MAX_AGE=172800 -JOB=homelab-extended-appbackup STATUS=OK EVIDENCE=artifact_or_health AGE=14156 MAX_AGE=172800 -JOB=homelab-gitea-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987785 MAX_AGE=172800 -JOB=homelab-golden-state-index STATUS=OK EVIDENCE=artifact_or_health AGE=4112 MAX_AGE=172800 -JOB=homelab-mail-cloud-critical-upload STATUS=OK EVIDENCE=artifact_or_health AGE=11997 MAX_AGE=172800 -JOB=homelab-mail-cloud-edge-vm-upload STATUS=OK EVIDENCE=artifact_or_health AGE=174045 MAX_AGE=864000 -JOB=homelab-mail-cloud-nextcloud-vm-upload STATUS=OK EVIDENCE=artifact_or_health AGE=7575 MAX_AGE=864000 -JOB=homelab-mail-cloud-restore-drill STATUS=STALE EVIDENCE=artifact_or_health AGE=988269 MAX_AGE=864000 -JOB=homelab-npmplus-kuma-config-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=60071 MAX_AGE=172800 -JOB=homelab-router-running-config-mail-cloud STATUS=OK EVIDENCE=artifact_or_health AGE=17076 MAX_AGE=172800 -JOB=homelab-vaultwarden-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987768 MAX_AGE=172800 -JOB=homelab-verified-backup-health STATUS=OK EVIDENCE=artifact_or_health AGE=11845 MAX_AGE=172800 -JOB=homelab-vm-backup-health STATUS=OK EVIDENCE=artifact_or_health AGE=8742 MAX_AGE=172800 -JOB=homelab-vm-backup-policy STATUS=OK EVIDENCE=artifact_or_health AGE=5548 MAX_AGE=172800 -JOB=homelab-vm-full-backup STATUS=OK EVIDENCE=artifact_or_health AGE=24778 MAX_AGE=172800 -JOB=homelab-vm-full-restore-drill STATUS=OK EVIDENCE=artifact_or_health AGE=523085 MAX_AGE=10368000 -JOB=homelab-vm-local-dumps-2cloud STATUS=OK EVIDENCE=artifact_or_health AGE=172141 MAX_AGE=864000 -JOB=homelab-vm-local-dumps-retention STATUS=OK EVIDENCE=artifact_or_health AGE=168839 MAX_AGE=864000 -JOB=immich-media-offhost-sync STATUS=OK EVIDENCE=artifact_or_health AGE=4286 MAX_AGE=259200 -JOB=memos-backup STATUS=OK EVIDENCE=artifact_or_health AGE=14547 MAX_AGE=259200 -JOB=netbird-vps-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=1747896 MAX_AGE=864000 -JOB=paperless-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=16347 MAX_AGE=259200 -JOB=paperless-restore-dry-run STATUS=STALE EVIDENCE=artifact_or_health AGE=1001986 MAX_AGE=864000 -JOB=skladchik-reports-monitor-backup STATUS=OK EVIDENCE=artifact_or_health AGE=19639 MAX_AGE=172800 -JOB=skladchik-reports-monitor-backup-restore-check STATUS=OK EVIDENCE=artifact_or_health AGE=190273 MAX_AGE=864000 -JOB=vm160-remote-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=26607 MAX_AGE=172800 +=== BACKUP_JOBS === +[{"compress":"zstd","enabled":1,"id":"homelab-nightly-all","mailnotification":"failure","mode":"snapshot","next-run":1784680200,"prune-backups":{"keep-daily":"4"},"schedule":"03:30","storage":"local","type":"vzdump","vmid":"110,111,112,113,130,150,190"}] +=== RCLONE_REMOTES === +pve01-mail-01: +pve01-mail-01-crypt: +pve01-mail-02: +pve01-mail-02-crypt: +pve01-mail-03: +pve01-mail-03-crypt: +pve01-mail-04: +pve01-mail-04-crypt: +pve01-mail-05: +pve01-mail-05-crypt: +pve02-mail-01: +pve02-mail-01-crypt: +pve02-mail-02: +pve02-mail-02-crypt: +pve02-mail-03: +pve02-mail-03-crypt: +pve02-mail-04: +pve02-mail-04-crypt: +pve03-mail-01: +pve03-mail-01-crypt: +pve03-mail-02: +pve03-mail-02-crypt: +pve03-mail-03: +pve03-mail-03-crypt: +pve03-mail-04: +pve03-mail-04-crypt: +=== BACKUP_TIMERS === +Wed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:14 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service +Wed 2026-07-22 00:09:43 MSK 9h Tue 2026-07-21 00:09:32 MSK 14h ago homelab-backup-freshness.timer homelab-backup-freshness.service +Wed 2026-07-22 02:15:00 MSK 11h Tue 2026-07-21 02:15:05 MSK 12h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service +Wed 2026-07-22 03:10:02 MSK 12h Tue 2026-07-21 03:13:14 MSK 11h ago homelab-emergency-backup.timer homelab-emergency-backup.service +Wed 2026-07-22 03:35:00 MSK 12h Tue 2026-07-21 03:35:14 MSK 11h ago filebrowser-backup.timer filebrowser-backup.service +Wed 2026-07-22 03:55:02 MSK 13h Tue 2026-07-21 03:51:25 MSK 10h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service +Wed 2026-07-22 04:17:29 MSK 13h Tue 2026-07-21 04:19:17 MSK 10h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service +Wed 2026-07-22 04:26:54 MSK 13h Tue 2026-07-21 04:28:14 MSK 10h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service +Wed 2026-07-22 04:38:04 MSK 14h Tue 2026-07-21 04:34:48 MSK 10h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service +Wed 2026-07-22 05:01:44 MSK 14h Tue 2026-07-21 04:57:14 MSK 9h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service +Wed 2026-07-22 05:23:56 MSK 14h Tue 2026-07-21 05:17:57 MSK 9h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service +Wed 2026-07-22 05:27:13 MSK 14h Tue 2026-07-21 05:29:05 MSK 9h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service +Wed 2026-07-22 05:47:19 MSK 15h Tue 2026-07-21 05:46:14 MSK 8h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service +Wed 2026-07-22 06:01:19 MSK 15h Tue 2026-07-21 05:57:30 MSK 8h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service +Wed 2026-07-22 06:27:43 MSK 15h Tue 2026-07-21 06:32:25 MSK 8h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service +Wed 2026-07-22 06:38:51 MSK 16h Tue 2026-07-21 06:43:54 MSK 7h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service +Wed 2026-07-22 06:39:22 MSK 16h Tue 2026-07-21 06:40:25 MSK 7h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service +Wed 2026-07-22 06:44:16 MSK 16h Tue 2026-07-21 06:40:25 MSK 7h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service +Wed 2026-07-22 06:52:06 MSK 16h Tue 2026-07-21 06:47:47 MSK 7h ago homelab-beszel-backup.timer homelab-beszel-backup.service +Wed 2026-07-22 06:58:58 MSK 16h Tue 2026-07-21 06:57:30 MSK 7h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service +Wed 2026-07-22 07:02:27 MSK 16h Tue 2026-07-21 07:05:30 MSK 7h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service +Wed 2026-07-22 07:07:14 MSK 16h Tue 2026-07-21 07:18:55 MSK 7h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service +Wed 2026-07-22 07:35:06 MSK 16h Tue 2026-07-21 07:36:59 MSK 6h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service +Wed 2026-07-22 07:38:21 MSK 17h Tue 2026-07-21 07:36:14 MSK 6h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service +Wed 2026-07-22 07:41:49 MSK 17h Tue 2026-07-21 07:53:41 MSK 6h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service +Wed 2026-07-22 08:10:37 MSK 17h Tue 2026-07-21 08:07:25 MSK 6h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service +Wed 2026-07-22 08:37:02 MSK 18h Tue 2026-07-21 08:30:07 MSK 6h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service +Wed 2026-07-22 08:40:13 MSK 18h Tue 2026-07-21 08:50:14 MSK 5h 45min ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service +Wed 2026-07-22 08:53:36 MSK 18h Tue 2026-07-21 08:41:14 MSK 5h 54min ago memos-backup.timer memos-backup.service +Sun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service +Sun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service +Sun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service +Sun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service +Mon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 7h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service +=== BACKUP_UNITS === +dpkg-db-backup.service static - +filebrowser-backup.service static - +homelab-alertmanager-backup.service static - +homelab-appbackup-dockge-stacks.service static - +homelab-appbackup-mariadb-batch.service static - +homelab-appbackup-npmplus-edge.service static - +homelab-appbackup-postgres-batch.service static - +homelab-appbackup-sqlite-batch.service static - +homelab-authentik-backup.service static - +homelab-auto-backup.service static - +homelab-backup-coverage-matrix.service static - +homelab-backup-freshness.service static - +homelab-backup-healthcheck.service static - +homelab-backup-matrix-refresh.service static - +homelab-backup-sla-health.service static - +homelab-beszel-backup.service static - +homelab-blackbox-exporter-backup.service static - +homelab-cluster-admin-incident-engine-backup.service static - +homelab-crypto-portfolio-backup.service static - +homelab-edge-vm-vzdump-backup.service static - +homelab-emergency-backup.service static - +homelab-extended-appbackup.service static - +homelab-gitea-backup.service static - +homelab-gitea-secondary-backup.service static - +homelab-legacy-monitoring-backups.service static - +homelab-mail-cloud-critical-upload.service static - +homelab-mail-cloud-edge-vm-upload.service static - +homelab-mail-cloud-nextcloud-vm-upload.service static - +homelab-mail-cloud-restore-drill.service static - +homelab-npmplus-kuma-config-backup.service static - +homelab-router-running-config-mail-cloud.service static - +homelab-vaultwarden-backup.service static - +homelab-verified-backup-health.service static - +homelab-vm-backup-health.service static - +homelab-vm-backup-policy.service static - +homelab-vm-full-backup.service static - +homelab-vm-local-dumps-2cloud.service static - +homelab-vm160-remote-backup.service static - +immich-media-backup.service static - +memos-backup.service static - +netbird-vps-backup.service static - +paperless-backup.service static - +skladchik-reports-monitor-backup-restore-check.service static - +skladchik-reports-monitor-backup.service static - +dpkg-db-backup.timer enabled enabled +filebrowser-backup.timer enabled enabled +homelab-alertmanager-backup.timer enabled enabled +homelab-appbackup-dockge-stacks.timer enabled enabled +homelab-appbackup-mariadb-batch.timer enabled enabled +homelab-appbackup-npmplus-edge.timer enabled enabled +homelab-appbackup-postgres-batch.timer enabled enabled +homelab-appbackup-sqlite-batch.timer enabled enabled +homelab-authentik-backup.timer disabled enabled +homelab-auto-backup.timer enabled enabled +homelab-backup-coverage-matrix.timer enabled enabled +homelab-backup-freshness.timer enabled enabled +homelab-backup-healthcheck.timer enabled enabled +homelab-backup-matrix-refresh.timer disabled enabled +homelab-backup-sla-health.timer enabled enabled +homelab-beszel-backup.timer enabled enabled +homelab-blackbox-exporter-backup.timer enabled enabled +homelab-cluster-admin-incident-engine-backup.timer enabled enabled +homelab-crypto-portfolio-backup.timer enabled enabled +homelab-edge-vm-vzdump-backup.timer disabled enabled +homelab-emergency-backup.timer enabled enabled +homelab-extended-appbackup.timer enabled enabled +homelab-gitea-backup.timer disabled enabled +homelab-gitea-secondary-backup.timer enabled enabled +homelab-legacy-monitoring-backups.timer enabled enabled +homelab-mail-cloud-critical-upload.timer enabled enabled +homelab-mail-cloud-edge-vm-upload.timer enabled enabled +homelab-mail-cloud-nextcloud-vm-upload.timer enabled enabled +homelab-mail-cloud-restore-drill.timer enabled enabled +homelab-npmplus-kuma-config-backup.timer disabled enabled +homelab-router-running-config-mail-cloud.timer enabled enabled +homelab-vaultwarden-backup.timer disabled enabled +homelab-verified-backup-health.timer enabled enabled +homelab-vm-backup-health.timer enabled enabled +homelab-vm-backup-policy.timer enabled enabled +homelab-vm-full-backup.timer enabled enabled +homelab-vm-local-dumps-2cloud.timer enabled enabled +homelab-vm160-remote-backup.timer disabled enabled +immich-media-backup.timer disabled enabled +memos-backup.timer enabled enabled +netbird-vps-backup.timer disabled enabled +paperless-backup.timer disabled enabled +skladchik-reports-monitor-backup-restore-check.timer enabled enabled +skladchik-reports-monitor-backup.timer enabled enabled +=== PIPELINE_FILES === +/etc/cron.d/vzdump +/etc/systemd/system/homelab-edge-vm-offhost-freshness.timer +/etc/systemd/system/homelab-edge-vm-vzdump-backup.service +/etc/systemd/system/homelab-edge-vm-vzdump-backup.timer +/etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.service +/etc/systemd/system/homelab-vm-full-backup.service +/etc/systemd/system/multi-user.target.wants/pve-guests.service +/etc/systemd/system/pve-manager.service +/etc/systemd/system/timers.target.wants/homelab-edge-vm-offhost-freshness.timer +/usr/local/sbin/appbackup-dockge-stacks +/usr/local/sbin/appbackup-dockge-stacks.bak-chunkall-20260710T011410Z +/usr/local/sbin/appbackup-restore-check +/usr/local/sbin/appbackup-restore-check.bak-chunkall-20260710T011410Z +/usr/local/sbin/appbackup-restore-check.bak-dual-read-20260711T051924Z +/usr/local/sbin/appbackup-restore-check.bak-flat-read-20260711T052854Z +/usr/local/sbin/appbackup-restore-check.bak-flat-read-v2-20260711T053129Z +/usr/local/sbin/appbackup-retention +/usr/local/sbin/appbackup-retention.bak-dual-read-20260711T051924Z +/usr/local/sbin/backupctl +/usr/local/sbin/backup-restore-check +/usr/local/sbin/backup-restore-check.bak-chunkall-20260710T011410Z +/usr/local/sbin/backup-retention +/usr/local/sbin/homelab-backup-edge-vm-vzdump +/usr/local/sbin/homelab-capacity-risk +/usr/local/sbin/homelab-change-transaction +/usr/local/sbin/homelab-chat-run +/usr/local/sbin/homelab-edge-vm-offhost-freshness +/usr/local/sbin/homelab-extended-appbackup-run +/usr/local/sbin/homelab-golden-state-extended-overlay +/usr/local/sbin/homelab-golden-state-index +/usr/local/sbin/homelab-health-metrics +/usr/local/sbin/homelab-ingress-standby-refresh +/usr/local/sbin/homelab-mail-cloud-critical-upload +/usr/local/sbin/homelab-mail-cloud-critical-upload.bak-chunkall-20260710T011410Z +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload +/usr/local/sbin/homelab-mail-cloud-restore-drill +/usr/local/sbin/homelab-mail-cloud-restore-drill.bak-chunkall-20260710T011410Z +/usr/local/sbin/homelab-rclone-cat-chunked +/usr/local/sbin/homelab-rclone-cat-flat-chunked +/usr/local/sbin/homelab-rclone-copy-chunked +/usr/local/sbin/homelab-rclone-copy-flat-chunked +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh +/usr/local/sbin/homelab-rclone-required-check.sh +/usr/local/sbin/homelab-router-hard-power-cycle +/usr/local/sbin/homelab-router-running-config-mail-cloud +/usr/local/sbin/homelab-router-running-config-mail-cloud.bak-chunkall-20260710T011410Z +/usr/local/sbin/homelab-safe-exec +/usr/local/sbin/homelab-secret-exposure-guard +/usr/local/sbin/homelab-stage-2cloud +/usr/local/sbin/homelab-stage-2cloud.bak-global-chunkonly-20260710T011036Z +/usr/local/sbin/homelab-stage-2cloud-flat +/usr/local/sbin/homelab-stage-2cloud-flat-canary +/usr/local/sbin/homelab-stage-verify-chunked +/usr/local/sbin/homelab-stage-verify-flat-chunked +/usr/local/sbin/homelab-vm160-remote-backup +/usr/local/sbin/homelab-vm-backup-health +/usr/local/sbin/homelab-vm-backup-policy +/usr/local/sbin/homelab-vm-full-backup +/usr/local/sbin/homelab-vm-full-backup.bak-chunkall-20260710T011410Z +/usr/local/sbin/homelab-vm-full-restore-drill +/usr/local/sbin/homelab-vm-full-restore-drill.bak.20260710T080231Z +/usr/local/sbin/homelab-vm-full-restore-drill.bak-docker-check-v2-20260710T050044Z +/usr/local/sbin/homelab-vm-full-restore-drill.bak-isolated-ip-20260710T044331Z +/usr/local/sbin/homelab-vm-full-retention +/usr/local/sbin/homelab-vm-local-dumps-2cloud +/usr/local/sbin/homelab-vm-local-dumps-retention +/usr/local/sbin/skladchik-reports-monitor-backup +/usr/local/sbin/skladchik-reports-monitor-backup.pre-restore-check-20260710T065311Z.bak +/usr/local/sbin/skladchik-reports-monitor-backup.pre-session-keepalive-20260710T071605Z.bak +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check.pre-session-keepalive-20260710T071605Z.bak +=== LOCAL_BACKUPS === +7022386770 2026-07-18T03:31:03.6566057240 /var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.vma.zst +7021952392 2026-07-21T03:31:06.5717414950 /var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst +7021297699 2026-07-19T03:31:06.8755351020 /var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst +7016275481 2026-07-20T03:31:03.2165123470 /var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst +329541843 2026-07-21T03:30:09.7688696330 /var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst +326601118 2026-07-20T03:30:08.2096782720 /var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst +323668231 2026-07-19T03:45:46.2209130400 /var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst +321121649 2026-07-18T03:30:08.0777625900 /var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.tar.zst +222777732 2026-07-21T03:30:15.9059638290 /var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst +222761273 2026-07-20T03:30:13.7937629440 /var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst +222745965 2026-07-19T03:45:57.2700790730 /var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst +222727760 2026-07-18T03:30:13.5578457230 /var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.tar.zst +3021 2026-07-19T03:31:07.4745284120 /var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.log +3020 2026-07-21T03:31:07.3945472590 /var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.log +3018 2026-07-18T03:31:03.9197988860 /var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.log +3017 2026-07-20T03:31:04.1706719350 /var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.log +1089 2026-07-20T03:30:14.0079985330 /var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.log +1086 2026-07-20T03:30:08.4640241540 /var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.log +989 2026-07-21T03:30:16.1364976560 /var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.log +989 2026-07-18T03:30:13.7816006690 /var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.log +986 2026-07-21T03:30:10.0285327490 /var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.log +986 2026-07-18T03:30:08.3002411830 /var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.log +731 2026-07-19T03:45:57.5197000270 /var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.log +729 2026-07-19T03:45:46.4444121980 /var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.log OUTPUT_END CHAT_OUTPUT_END