diff --git a/runtime/history/BACKUP-DUAL-CLOUD-RECONCILE-060.json b/runtime/history/BACKUP-DUAL-CLOUD-RECONCILE-060.json new file mode 100644 index 0000000..f7005f3 --- /dev/null +++ b/runtime/history/BACKUP-DUAL-CLOUD-RECONCILE-060.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "BACKUP-DUAL-CLOUD-RECONCILE-060", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "backup-audit", + "started_at_utc": "2026-07-21T11:38:23Z", + "finished_at_utc": "2026-07-21T11:38:23Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1", + "command_sha256": "be39adf6237697b760f095d503f1fdb9e25eed004c8108b2b1d338318d4794a6", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "63db4a98c9859a2dece1cd7a0f56c08584ac144e96fedc91c7d16a907335b8e9", + "sanitized_output_sha256": "63db4a98c9859a2dece1cd7a0f56c08584ac144e96fedc91c7d16a907335b8e9", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "=== EXACT_IMMUTABLE_RESULT ===\n{\n \"schema_version\": 1,\n \"channel\": \"homelab-runtime\",\n \"command_id\": \"BACKUP-DUAL-CLOUD-PREFLIGHT-059\",\n \"status\": \"FAIL\",\n \"rc\": 1,\n \"host\": \"pve01\",\n \"mode\": \"read-only\",\n \"component\": \"backup-audit\",\n \"started_at_utc\": \"2026-07-21T11:35:22Z\",\n \"finished_at_utc\": \"2026-07-21T11:35:23Z\",\n \"reference_register_checked\": true,\n \"reference_sha256\": \"5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e\",\n \"error_register_checked\": true,\n \"error_register_sha256\": \"f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1\",\n \"command_sha256\": \"c6f4134d15a9f641ddaf67535760d0abede973dd08222a37af05612a6f580479\",\n \"duplicate_failed_command_blocked\": false,\n \"block_reason\": null,\n \"execution_started\": true,\n \"changes_made\": false,\n \"sanitized\": true,\n \"secrets_included\": false,\n \"private_addresses_included\": false,\n \"raw_evidence_retained_locally\": true,\n \"raw_evidence_sha256\": \"f375162905664fc3de53bec3ea2b2485775e05bcfbd421a3c63010def946a753\",\n \"sanitized_output_sha256\": \"f375162905664fc3de53bec3ea2b2485775e05bcfbd421a3c63010def946a753\",\n \"output_truncated_in_json\": false,\n \"full_sanitized_output_url\": \"https://git.gram1.ru/.well-known/homelab-runtime/latest.txt\",\n \"output\": \"=== BACKUP_JOBS ===\\n[{\\\"compress\\\":\\\"zstd\\\",\\\"enabled\\\":1,\\\"id\\\":\\\"homelab-nightly-all\\\",\\\"mailnotification\\\":\\\"failure\\\",\\\"mode\\\":\\\"snapshot\\\",\\\"next-run\\\":1784680200,\\\"prune-backups\\\":{\\\"keep-daily\\\":\\\"4\\\"},\\\"schedule\\\":\\\"03:30\\\",\\\"storage\\\":\\\"local\\\",\\\"type\\\":\\\"vzdump\\\",\\\"vmid\\\":\\\"110,111,112,113,130,150,190\\\"}]\\n=== RCLONE_REMOTES ===\\npve01-mail-01:\\npve01-mail-01-crypt:\\npve01-mail-02:\\npve01-mail-02-crypt:\\npve01-mail-03:\\npve01-mail-03-crypt:\\npve01-mail-04:\\npve01-mail-04-crypt:\\npve01-mail-05:\\npve01-mail-05-crypt:\\npve02-mail-01:\\npve02-mail-01-crypt:\\npve02-mail-02:\\npve02-mail-02-crypt:\\npve02-mail-03:\\npve02-mail-03-crypt:\\npve02-mail-04:\\npve02-mail-04-crypt:\\npve03-mail-01:\\npve03-mail-01-crypt:\\npve03-mail-02:\\npve03-mail-02-crypt:\\npve03-mail-03:\\npve03-mail-03-crypt:\\npve03-mail-04:\\npve03-mail-04-crypt:\\n=== BACKUP_TIMERS ===\\nWed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:14 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service\\nWed 2026-07-22 00:09:43 MSK 9h Tue 2026-07-21 00:09:32 MSK 14h ago homelab-backup-freshness.timer homelab-backup-freshness.service\\nWed 2026-07-22 02:15:00 MSK 11h Tue 2026-07-21 02:15:05 MSK 12h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service\\nWed 2026-07-22 03:10:02 MSK 12h Tue 2026-07-21 03:13:14 MSK 11h ago homelab-emergency-backup.timer homelab-emergency-backup.service\\nWed 2026-07-22 03:35:00 MSK 12h Tue 2026-07-21 03:35:14 MSK 11h ago filebrowser-backup.timer filebrowser-backup.service\\nWed 2026-07-22 03:55:02 MSK 13h Tue 2026-07-21 03:51:25 MSK 10h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service\\nWed 2026-07-22 04:17:29 MSK 13h Tue 2026-07-21 04:19:17 MSK 10h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service\\nWed 2026-07-22 04:26:54 MSK 13h Tue 2026-07-21 04:28:14 MSK 10h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service\\nWed 2026-07-22 04:38:04 MSK 14h Tue 2026-07-21 04:34:48 MSK 10h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service\\nWed 2026-07-22 05:01:44 MSK 14h Tue 2026-07-21 04:57:14 MSK 9h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service\\nWed 2026-07-22 05:23:56 MSK 14h Tue 2026-07-21 05:17:57 MSK 9h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service\\nWed 2026-07-22 05:27:13 MSK 14h Tue 2026-07-21 05:29:05 MSK 9h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service\\nWed 2026-07-22 05:47:19 MSK 15h Tue 2026-07-21 05:46:14 MSK 8h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service\\nWed 2026-07-22 06:01:19 MSK 15h Tue 2026-07-21 05:57:30 MSK 8h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service\\nWed 2026-07-22 06:27:43 MSK 15h Tue 2026-07-21 06:32:25 MSK 8h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service\\nWed 2026-07-22 06:38:51 MSK 16h Tue 2026-07-21 06:43:54 MSK 7h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service\\nWed 2026-07-22 06:39:22 MSK 16h Tue 2026-07-21 06:40:25 MSK 7h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service\\nWed 2026-07-22 06:44:16 MSK 16h Tue 2026-07-21 06:40:25 MSK 7h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service\\nWed 2026-07-22 06:52:06 MSK 16h Tue 2026-07-21 06:47:47 MSK 7h ago homelab-beszel-backup.timer homelab-beszel-backup.service\\nWed 2026-07-22 06:58:58 MSK 16h Tue 2026-07-21 06:57:30 MSK 7h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service\\nWed 2026-07-22 07:02:27 MSK 16h Tue 2026-07-21 07:05:30 MSK 7h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service\\nWed 2026-07-22 07:07:14 MSK 16h Tue 2026-07-21 07:18:55 MSK 7h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service\\nWed 2026-07-22 07:35:06 MSK 16h Tue 2026-07-21 07:36:59 MSK 6h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service\\nWed 2026-07-22 07:38:21 MSK 17h Tue 2026-07-21 07:36:14 MSK 6h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service\\nWed 2026-07-22 07:41:49 MSK 17h Tue 2026-07-21 07:53:41 MSK 6h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service\\nWed 2026-07-22 08:10:37 MSK 17h Tue 2026-07-21 08:07:25 MSK 6h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service\\nWed 2026-07-22 08:37:02 MSK 18h Tue 2026-07-21 08:30:07 MSK 6h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service\\nWed 2026-07-22 08:40:13 MSK 18h Tue 2026-07-21 08:50:14 MSK 5h 45min ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service\\nWed 2026-07-22 08:53:36 MSK 18h Tue 2026-07-21 08:41:14 MSK 5h 54min ago memos-backup.timer memos-backup.service\\nSun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service\\nSun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service\\nSun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service\\nSun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service\\nMon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 7h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service\\n=== BACKUP_UNITS ===\\ndpkg-db-backup.service static -\\nfilebrowser-backup.service static -\\nhomelab-alertmanager-backup.service static -\\nhomelab-appbackup-dockge-stacks.service static -\\nhomelab-appbackup-mariadb-batch.service static -\\nhomelab-appbackup-npmplus-edge.service static -\\nhomelab-appbackup-postgres-batch.service static -\\nhomelab-appbackup-sqlite-batch.service static -\\nhomelab-authentik-backup.service static -\\nhomelab-auto-backup.service static -\\nhomelab-backup-coverage-matrix.service static -\\nhomelab-backup-freshness.service static -\\nhomelab-backup-healthcheck.service static -\\nhomelab-backup-matrix-refresh.service static -\\nhomelab-backup-sla-health.service static -\\nhomelab-beszel-backup.service static -\\nhomelab-blackbox-exporter-backup.service static -\\nhomelab-cluster-admin-incident-engine-backup.service static -\\nhomelab-crypto-portfolio-backup.service static -\\nhomelab-edge-vm-vzdump-backup.service static -\\nhomelab-emergency-backup.service static -\\nhomelab-extended-appbackup.service static -\\nhomelab-gitea-backup.service static -\\nhomelab-gitea-secondary-backup.service static -\\nhomelab-legacy-monitoring-backups.service static -\\nhomelab-mail-cloud-critical-upload.service static -\\nhomelab-mail-cloud-edge-vm-upload.service static -\\nhomelab-mail-cloud-nextcloud-vm-upload.service static -\\nhomelab-mail-cloud-restore-drill.service static -\\nhomelab-npmplus-kuma-config-backup.service static -\\nhomelab-router-running-config-mail-cloud.service static -\\nhomelab-vaultwarden-backup.service static -\\nhomelab-verified-backup-health.service static -\\nhomelab-vm-backup-health.service static -\\nhomelab-vm-backup-policy.service static -\\nhomelab-vm-full-backup.service static -\\nhomelab-vm-local-dumps-2cloud.service static -\\nhomelab-vm160-remote-backup.service static -\\nimmich-media-backup.service static -\\nmemos-backup.service static -\\nnetbird-vps-backup.service static -\\npaperless-backup.service static -\\nskladchik-reports-monitor-backup-restore-check.service static -\\nskladchik-reports-monitor-backup.service static -\\ndpkg-db-backup.timer enabled enabled\\nfilebrowser-backup.timer enabled enabled\\nhomelab-alertmanager-backup.timer enabled enabled\\nhomelab-appbackup-dockge-stacks.timer enabled enabled\\nhomelab-appbackup-mariadb-batch.timer enabled enabled\\nhomelab-appbackup-npmplus-edge.timer enabled enabled\\nhomelab-appbackup-postgres-batch.timer enabled enabled\\nhomelab-appbackup-sqlite-batch.timer enabled enabled\\nhomelab-authentik-backup.timer disabled enabled\\nhomelab-auto-backup.timer enabled enabled\\nhomelab-backup-coverage-matrix.timer enabled enabled\\nhomelab-backup-freshness.timer enabled enabled\\nhomelab-backup-healthcheck.timer enabled enabled\\nhomelab-backup-matrix-refresh.timer disabled enabled\\nhomelab-backup-sla-health.timer enabled enabled\\nhomelab-beszel-backup.timer enabled enabled\\nhomelab-blackbox-exporter-backup.timer enabled enabled\\nhomelab-cluster-admin-incident-engine-backup.timer enabled enabled\\nhomelab-crypto-portfolio-backup.timer enabled enabled\\nhomelab-edge-vm-vzdump-backup.timer disabled enabled\\nhomelab-emergency-backup.timer enabled enabled\\nhomelab-extended-appbackup.timer enabled enabled\\nhomelab-gitea-backup.timer disabled enabled\\nhomelab-gitea-secondary-backup.timer enabled enabled\\nhomelab-legacy-monitoring-backups.timer enabled enabled\\nhomelab-mail-cloud-critical-upload.timer enabled enabled\\nhomelab-mail-cloud-edge-vm-upload.timer enabled enabled\\nhomelab-mail-cloud-nextcloud-vm-upload.timer enabled enabled\\nhomelab-mail-cloud-restore-drill.timer enabled enabled\\nhomelab-npmplus-kuma-config-backup.timer disabled enabled\\nhomelab-router-running-config-mail-cloud.timer enabled enabled\\nhomelab-vaultwarden-backup.timer disabled enabled\\nhomelab-verified-backup-health.timer enabled enabled\\nhomelab-vm-backup-health.timer enabled enabled\\nhomelab-vm-backup-policy.timer enabled enabled\\nhomelab-vm-full-backup.timer enabled enabled\\nhomelab-vm-local-dumps-2cloud.timer enabled enabled\\nhomelab-vm160-remote-backup.timer disabled enabled\\nimmich-media-backup.timer disabled enabled\\nmemos-backup.timer enabled enabled\\nnetbird-vps-backup.timer disabled enabled\\npaperless-backup.timer disabled enabled\\nskladchik-reports-monitor-backup-restore-check.timer enabled enabled\\nskladchik-reports-monitor-backup.timer enabled enabled\\n=== PIPELINE_FILES ===\\n/etc/cron.d/vzdump\\n/etc/systemd/system/homelab-edge-vm-offhost-freshness.timer\\n/etc/systemd/system/homelab-edge-vm-vzdump-backup.service\\n/etc/systemd/system/homelab-edge-vm-vzdump-backup.timer\\n/etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.service\\n/etc/systemd/system/homelab-vm-full-backup.service\\n/etc/systemd/system/multi-user.target.wants/pve-guests.service\\n/etc/systemd/system/pve-manager.service\\n/etc/systemd/system/timers.target.wants/homelab-edge-vm-offhost-freshness.timer\\n/usr/local/sbin/appbackup-dockge-stacks\\n/usr/local/sbin/appbackup-dockge-stacks.bak-chunkall-20260710T011410Z\\n/usr/local/sbin/appbackup-restore-check\\n/usr/local/sbin/appbackup-restore-check.bak-chunkall-20260710T011410Z\\n/usr/local/sbin/appbackup-restore-check.bak-dual-read-20260711T051924Z\\n/usr/local/sbin/appbackup-restore-check.bak-flat-read-20260711T052854Z\\n/usr/local/sbin/appbackup-restore-check.bak-flat-read-v2-20260711T053129Z\\n/usr/local/sbin/appbackup-retention\\n/usr/local/sbin/appbackup-retention.bak-dual-read-20260711T051924Z\\n/usr/local/sbin/backupctl\\n/usr/local/sbin/backup-restore-check\\n/usr/local/sbin/backup-restore-check.bak-chunkall-20260710T011410Z\\n/usr/local/sbin/backup-retention\\n/usr/local/sbin/homelab-backup-edge-vm-vzdump\\n/usr/local/sbin/homelab-capacity-risk\\n/usr/local/sbin/homelab-change-transaction\\n/usr/local/sbin/homelab-chat-run\\n/usr/local/sbin/homelab-edge-vm-offhost-freshness\\n/usr/local/sbin/homelab-extended-appbackup-run\\n/usr/local/sbin/homelab-golden-state-extended-overlay\\n/usr/local/sbin/homelab-golden-state-index\\n/usr/local/sbin/homelab-health-metrics\\n/usr/local/sbin/homelab-ingress-standby-refresh\\n/usr/local/sbin/homelab-mail-cloud-critical-upload\\n/usr/local/sbin/homelab-mail-cloud-critical-upload.bak-chunkall-20260710T011410Z\\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload\\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload\\n/usr/local/sbin/homelab-mail-cloud-restore-drill\\n/usr/local/sbin/homelab-mail-cloud-restore-drill.bak-chunkall-20260710T011410Z\\n/usr/local/sbin/homelab-rclone-cat-chunked\\n/usr/local/sbin/homelab-rclone-cat-flat-chunked\\n/usr/local/sbin/homelab-rclone-copy-chunked\\n/usr/local/sbin/homelab-rclone-copy-flat-chunked\\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh\\n/usr/local/sbin/homelab-rclone-required-check.sh\\n/usr/local/sbin/homelab-router-hard-power-cycle\\n/usr/local/sbin/homelab-router-running-config-mail-cloud\\n/usr/local/sbin/homelab-router-running-config-mail-cloud.bak-chunkall-20260710T011410Z\\n/usr/local/sbin/homelab-safe-exec\\n/usr/local/sbin/homelab-secret-exposure-guard\\n/usr/local/sbin/homelab-stage-2cloud\\n/usr/local/sbin/homelab-stage-2cloud.bak-global-chunkonly-20260710T011036Z\\n/usr/local/sbin/homelab-stage-2cloud-flat\\n/usr/local/sbin/homelab-stage-2cloud-flat-canary\\n/usr/local/sbin/homelab-stage-verify-chunked\\n/usr/local/sbin/homelab-stage-verify-flat-chunked\\n/usr/local/sbin/homelab-vm160-remote-backup\\n/usr/local/sbin/homelab-vm-backup-health\\n/usr/local/sbin/homelab-vm-backup-policy\\n/usr/local/sbin/homelab-vm-full-backup\\n/usr/local/sbin/homelab-vm-full-backup.bak-chunkall-20260710T011410Z\\n/usr/local/sbin/homelab-vm-full-restore-drill\\n/usr/local/sbin/homelab-vm-full-restore-drill.bak.20260710T080231Z\\n/usr/local/sbin/homelab-vm-full-restore-drill.bak-docker-check-v2-20260710T050044Z\\n/usr/local/sbin/homelab-vm-full-restore-drill.bak-isolated-ip-20260710T044331Z\\n/usr/local/sbin/homelab-vm-full-retention\\n/usr/local/sbin/homelab-vm-local-dumps-2cloud\\n/usr/local/sbin/homelab-vm-local-dumps-retention\\n/usr/local/sbin/skladchik-reports-monitor-backup\\n/usr/local/sbin/skladchik-reports-monitor-backup.pre-restore-check-20260710T065311Z.bak\\n/usr/local/sbin/skladchik-reports-monitor-backup.pre-session-keepalive-20260710T071605Z.bak\\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check\\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check.pre-session-keepalive-20260710T071605Z.bak\\n=== LOCAL_BACKUPS ===\\n7022386770\\t2026-07-18T03:31:03.6566057240\\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.vma.zst\\n7021952392\\t2026-07-21T03:31:06.5717414950\\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst\\n7021297699\\t2026-07-19T03:31:06.8755351020\\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst\\n7016275481\\t2026-07-20T03:31:03.2165123470\\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst\\n329541843\\t2026-07-21T03:30:09.7688696330\\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst\\n326601118\\t2026-07-20T03:30:08.2096782720\\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst\\n323668231\\t2026-07-19T03:45:46.2209130400\\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst\\n321121649\\t2026-07-18T03:30:08.0777625900\\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.tar.zst\\n222777732\\t2026-07-21T03:30:15.9059638290\\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst\\n222761273\\t2026-07-20T03:30:13.7937629440\\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst\\n222745965\\t2026-07-19T03:45:57.2700790730\\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst\\n222727760\\t2026-07-18T03:30:13.5578457230\\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.tar.zst\\n3021\\t2026-07-19T03:31:07.4745284120\\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.log\\n3020\\t2026-07-21T03:31:07.3945472590\\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.log\\n3018\\t2026-07-18T03:31:03.9197988860\\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.log\\n3017\\t2026-07-20T03:31:04.1706719350\\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.log\\n1089\\t2026-07-20T03:30:14.0079985330\\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.log\\n1086\\t2026-07-20T03:30:08.4640241540\\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.log\\n989\\t2026-07-21T03:30:16.1364976560\\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.log\\n989\\t2026-07-18T03:30:13.7816006690\\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.log\\n986\\t2026-07-21T03:30:10.0285327490\\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.log\\n986\\t2026-07-18T03:30:08.3002411830\\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.log\\n731\\t2026-07-19T03:45:57.5197000270\\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.log\\n729\\t2026-07-19T03:45:46.4444121980\\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.log\\n\"\n}\n\nEXACT_GITEA=FOUND\n=== LOCAL_RAW_RESULT ===\nFILE=/var/lib/homelab-chat-run/raw/BACKUP-DUAL-CLOUD-PREFLIGHT-059-20260721T113522Z.log SIZE=21897 MTIME=2026-07-21 14:35:23.197206410 +0300\n=== BACKUP_JOBS ===\n[{\"compress\":\"zstd\",\"enabled\":1,\"id\":\"homelab-nightly-all\",\"mailnotification\":\"failure\",\"mode\":\"snapshot\",\"next-run\":1784680200,\"prune-backups\":{\"keep-daily\":\"4\"},\"schedule\":\"03:30\",\"storage\":\"local\",\"type\":\"vzdump\",\"vmid\":\"110,111,112,113,130,150,190\"}]\n=== RCLONE_REMOTES ===\npve01-mail-01:\npve01-mail-01-crypt:\npve01-mail-02:\npve01-mail-02-crypt:\npve01-mail-03:\npve01-mail-03-crypt:\npve01-mail-04:\npve01-mail-04-crypt:\npve01-mail-05:\npve01-mail-05-crypt:\npve02-mail-01:\npve02-mail-01-crypt:\npve02-mail-02:\npve02-mail-02-crypt:\npve02-mail-03:\npve02-mail-03-crypt:\npve02-mail-04:\npve02-mail-04-crypt:\npve03-mail-01:\npve03-mail-01-crypt:\npve03-mail-02:\npve03-mail-02-crypt:\npve03-mail-03:\npve03-mail-03-crypt:\npve03-mail-04:\npve03-mail-04-crypt:\n=== BACKUP_TIMERS ===\nWed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:14 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service\nWed 2026-07-22 00:09:43 MSK 9h Tue 2026-07-21 00:09:32 MSK 14h ago homelab-backup-freshness.timer homelab-backup-freshness.service\nWed 2026-07-22 02:15:00 MSK 11h Tue 2026-07-21 02:15:05 MSK 12h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service\nWed 2026-07-22 03:10:02 MSK 12h Tue 2026-07-21 03:13:14 MSK 11h ago homelab-emergency-backup.timer homelab-emergency-backup.service\nWed 2026-07-22 03:35:00 MSK 12h Tue 2026-07-21 03:35:14 MSK 11h ago filebrowser-backup.timer filebrowser-backup.service\nWed 2026-07-22 03:55:02 MSK 13h Tue 2026-07-21 03:51:25 MSK 10h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service\nWed 2026-07-22 04:17:29 MSK 13h Tue 2026-07-21 04:19:17 MSK 10h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service\nWed 2026-07-22 04:26:54 MSK 13h Tue 2026-07-21 04:28:14 MSK 10h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service\nWed 2026-07-22 04:38:04 MSK 14h Tue 2026-07-21 04:34:48 MSK 10h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service\nWed 2026-07-22 05:01:44 MSK 14h Tue 2026-07-21 04:57:14 MSK 9h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service\nWed 2026-07-22 05:23:56 MSK 14h Tue 2026-07-21 05:17:57 MSK 9h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service\nWed 2026-07-22 05:27:13 MSK 14h Tue 2026-07-21 05:29:05 MSK 9h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service\nWed 2026-07-22 05:47:19 MSK 15h Tue 2026-07-21 05:46:14 MSK 8h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service\nWed 2026-07-22 06:01:19 MSK 15h Tue 2026-07-21 05:57:30 MSK 8h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service\nWed 2026-07-22 06:27:43 MSK 15h Tue 2026-07-21 06:32:25 MSK 8h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service\nWed 2026-07-22 06:38:51 MSK 16h Tue 2026-07-21 06:43:54 MSK 7h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service\nWed 2026-07-22 06:39:22 MSK 16h Tue 2026-07-21 06:40:25 MSK 7h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service\nWed 2026-07-22 06:44:16 MSK 16h Tue 2026-07-21 06:40:25 MSK 7h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service\nWed 2026-07-22 06:52:06 MSK 16h Tue 2026-07-21 06:47:47 MSK 7h ago homelab-beszel-backup.timer homelab-beszel-backup.service\nWed 2026-07-22 06:58:58 MSK 16h Tue 2026-07-21 06:57:30 MSK 7h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service\nWed 2026-07-22 07:02:27 MSK 16h Tue 2026-07-21 07:05:30 MSK 7h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service\nWed 2026-07-22 07:07:14 MSK 16h Tue 2026-07-21 07:18:55 MSK 7h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service\nWed 2026-07-22 07:35:06 MSK 16h Tue 2026-07-21 07:36:59 MSK 6h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service\nWed 2026-07-22 07:38:21 MSK 17h Tue 2026-07-21 07:36:14 MSK 6h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service\nWed 2026-07-22 07:41:49 MSK 17h Tue 2026-07-21 07:53:41 MSK 6h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service\nWed 2026-07-22 08:10:37 MSK 17h Tue 2026-07-21 08:07:25 MSK 6h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service\nWed 2026-07-22 08:37:02 MSK 18h Tue 2026-07-21 08:30:07 MSK 6h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service\nWed 2026-07-22 08:40:13 MSK 18h Tue 2026-07-21 08:50:14 MSK 5h 45min ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service\nWed 2026-07-22 08:53:36 MSK 18h Tue 2026-07-21 08:41:14 MSK 5h 54min ago memos-backup.timer memos-backup.service\nSun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service\nSun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service\nSun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service\nSun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service\nMon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 7h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service\n=== BACKUP_UNITS ===\ndpkg-db-backup.service static -\nfilebrowser-backup.service static -\nhomelab-alertmanager-backup.service static -\nhomelab-appbackup-dockge-stacks.service static -\nhomelab-appbackup-mariadb-batch.service static -\nhomelab-appbackup-npmplus-edge.service static -\nhomelab-appbackup-postgres-batch.service static -\nhomelab-appbackup-sqlite-batch.service static -\nhomelab-authentik-backup.service static -\nhomelab-auto-backup.service static -\nhomelab-backup-coverage-matrix.service static -\nhomelab-backup-freshness.service static -\nhomelab-backup-healthcheck.service static -\nhomelab-backup-matrix-refresh.service static -\nhomelab-backup-sla-health.service static -\nhomelab-beszel-backup.service static -\nhomelab-blackbox-exporter-backup.service static -\nhomelab-cluster-admin-incident-engine-backup.service static -\nhomelab-crypto-portfolio-backup.service static -\nhomelab-edge-vm-vzdump-backup.service static -\nhomelab-emergency-backup.service static -\nhomelab-extended-appbackup.service static -\nhomelab-gitea-backup.service static -\nhomelab-gitea-secondary-backup.service static -\nhomelab-legacy-monitoring-backups.service static -\nhomelab-mail-cloud-critical-upload.service static -\nhomelab-mail-cloud-edge-vm-upload.service static -\nhomelab-mail-cloud-nextcloud-vm-upload.service static -\nhomelab-mail-cloud-restore-drill.service static -\nhomelab-npmplus-kuma-config-backup.service static -\nhomelab-router-running-config-mail-cloud.service static -\nhomelab-vaultwarden-backup.service static -\nhomelab-verified-backup-health.service static -\nhomelab-vm-backup-health.service static -\nhomelab-vm-backup-policy.service static -\nhomelab-vm-full-backup.service static -\nhomelab-vm-local-dumps-2cloud.service static -\nhomelab-vm160-remote-backup.service static -\nimmich-media-backup.service static -\nmemos-backup.service static -\nnetbird-vps-backup.service static -\npaperless-backup.service static -\nskladchik-reports-monitor-backup-restore-check.service static -\nskladchik-reports-monitor-backup.service static -\ndpkg-db-backup.timer enabled enabled\nfilebrowser-backup.timer enabled enabled\nhomelab-alertmanager-backup.timer enabled enabled\nhomelab-appbackup-dockge-stacks.timer enabled enabled\nhomelab-appbackup-mariadb-batch.timer enabled enabled\nhomelab-appbackup-npmplus-edge.timer enabled enabled\nhomelab-appbackup-postgres-batch.timer enabled enabled\nhomelab-appbackup-sqlite-batch.timer enabled enabled\nhomelab-authentik-backup.timer disabled enabled\nhomelab-auto-backup.timer enabled enabled\nhomelab-backup-coverage-matrix.timer enabled enabled\nhomelab-backup-freshness.timer enabled enabled\nhomelab-backup-healthcheck.timer enabled enabled\nhomelab-backup-matrix-refresh.timer disabled enabled\nhomelab-backup-sla-health.timer enabled enabled\nhomelab-beszel-backup.timer enabled enabled\nhomelab-blackbox-exporter-backup.timer enabled enabled\nhomelab-cluster-admin-incident-engine-backup.timer enabled enabled\nhomelab-crypto-portfolio-backup.timer enabled enabled\nhomelab-edge-vm-vzdump-backup.timer disabled enabled\nhomelab-emergency-backup.timer enabled enabled\nhomelab-extended-appbackup.timer enabled enabled\nhomelab-gitea-backup.timer disabled enabled\nhomelab-gitea-secondary-backup.timer enabled enabled\nhomelab-legacy-monitoring-backups.timer enabled enabled\nhomelab-mail-cloud-critical-upload.timer enabled enabled\nhomelab-mail-cloud-edge-vm-upload.timer enabled enabled\nhomelab-mail-cloud-nextcloud-vm-upload.timer enabled enabled\nhomelab-mail-cloud-restore-drill.timer enabled enabled\nhomelab-npmplus-kuma-config-backup.timer disabled enabled\nhomelab-router-running-config-mail-cloud.timer enabled enabled\nhomelab-vaultwarden-backup.timer disabled enabled\nhomelab-verified-backup-health.timer enabled enabled\nhomelab-vm-backup-health.timer enabled enabled\nhomelab-vm-backup-policy.timer enabled enabled\nhomelab-vm-full-backup.timer enabled enabled\nhomelab-vm-local-dumps-2cloud.timer enabled enabled\nhomelab-vm160-remote-backup.timer disabled enabled\nimmich-media-backup.timer disabled enabled\nmemos-backup.timer enabled enabled\nnetbird-vps-backup.timer disabled enabled\npaperless-backup.timer disabled enabled\nskladchik-reports-monitor-backup-restore-check.timer enabled enabled\nskladchik-reports-monitor-backup.timer enabled enabled\n=== PIPELINE_FILES ===\n/etc/cron.d/vzdump\n/etc/systemd/system/homelab-edge-vm-offhost-freshness.timer\n/etc/systemd/system/homelab-edge-vm-vzdump-backup.service\n/etc/systemd/system/homelab-edge-vm-vzdump-backup.timer\n/etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.service\n/etc/systemd/system/homelab-vm-full-backup.service\n/etc/systemd/system/multi-user.target.wants/pve-guests.service\n/etc/systemd/system/pve-manager.service\n/etc/systemd/system/timers.target.wants/homelab-edge-vm-offhost-freshness.timer\n/usr/local/sbin/appbackup-dockge-stacks\n/usr/local/sbin/appbackup-dockge-stacks.bak-chunkall-20260710T011410Z\n/usr/local/sbin/appbackup-restore-check\n/usr/local/sbin/appbackup-restore-check.bak-chunkall-20260710T011410Z\n/usr/local/sbin/appbackup-restore-check.bak-dual-read-20260711T051924Z\n/usr/local/sbin/appbackup-restore-check.bak-flat-read-20260711T052854Z\n/usr/local/sbin/appbackup-restore-check.bak-flat-read-v2-20260711T053129Z\n/usr/local/sbin/appbackup-retention\n/usr/local/sbin/appbackup-retention.bak-dual-read-20260711T051924Z\n/usr/local/sbin/backupctl\n/usr/local/sbin/backup-restore-check\n/usr/local/sbin/backup-restore-check.bak-chunkall-20260710T011410Z\n/usr/local/sbin/backup-retention\n/usr/local/sbin/homelab-backup-edge-vm-vzdump\n/usr/local/sbin/homelab-capacity-risk\n/usr/local/sbin/homelab-change-transaction\n/usr/local/sbin/homelab-chat-run\n/usr/local/sbin/homelab-edge-vm-offhost-freshness\n/usr/local/sbin/homelab-extended-appbackup-run\n/usr/local/sbin/homelab-golden-state-extended-overlay\n/usr/local/sbin/homelab-golden-state-index\n/usr/local/sbin/homelab-health-metrics\n/usr/local/sbin/homelab-ingress-standby-refresh\n/usr/local/sbin/homelab-mail-cloud-critical-upload\n/usr/local/sbin/homelab-mail-cloud-critical-upload.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload\n/usr/local/sbin/homelab-mail-cloud-restore-drill\n/usr/local/sbin/homelab-mail-cloud-restore-drill.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-rclone-cat-chunked\n/usr/local/sbin/homelab-rclone-cat-flat-chunked\n/usr/local/sbin/homelab-rclone-copy-chunked\n/usr/local/sbin/homelab-rclone-copy-flat-chunked\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh\n/usr/local/sbin/homelab-rclone-required-check.sh\n/usr/local/sbin/homelab-router-hard-power-cycle\n/usr/local/sbin/homelab-router-running-config-mail-cloud\n/usr/local/sbin/homelab-router-running-config-mail-cloud.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-safe-exec\n/usr/local/sbin/homelab-secret-exposure-guard\n/usr/local/sbin/homelab-stage-2cloud\n/usr/local/sbin/homelab-stage-2cloud.bak-global-chunkonly-20260710T011036Z\n/usr/local/sbin/homelab-stage-2cloud-flat\n/usr/local/sbin/homelab-stage-2cloud-flat-canary\n/usr/local/sbin/homelab-stage-verify-chunked\n/usr/local/sbin/homelab-stage-verify-flat-chunked\n/usr/local/sbin/homelab-vm160-remote-backup\n/usr/local/sbin/homelab-vm-backup-health\n/usr/local/sbin/homelab-vm-backup-policy\n/usr/local/sbin/homelab-vm-full-backup\n/usr/local/sbin/homelab-vm-full-backup.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-vm-full-restore-drill\n/usr/local/sbin/homelab-vm-full-restore-drill.bak.20260710T080231Z\n/usr/local/sbin/homelab-vm-full-restore-drill.bak-docker-check-v2-20260710T050044Z\n/usr/local/sbin/homelab-vm-full-restore-drill.bak-isolated-ip-20260710T044331Z\n/usr/local/sbin/homelab-vm-full-retention\n/usr/local/sbin/homelab-vm-local-dumps-2cloud\n/usr/local/sbin/homelab-vm-local-dumps-retention\n/usr/local/sbin/skladchik-reports-monitor-backup\n/usr/local/sbin/skladchik-reports-monitor-backup.pre-restore-check-20260710T065311Z.bak\n/usr/local/sbin/skladchik-reports-monitor-backup.pre-session-keepalive-20260710T071605Z.bak\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check.pre-session-keepalive-20260710T071605Z.bak\n=== LOCAL_BACKUPS ===\n7022386770\t2026-07-18T03:31:03.6566057240\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.vma.zst\n7021952392\t2026-07-21T03:31:06.5717414950\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst\n7021297699\t2026-07-19T03:31:06.8755351020\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst\n7016275481\t2026-07-20T03:31:03.2165123470\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst\n329541843\t2026-07-21T03:30:09.7688696330\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst\n326601118\t2026-07-20T03:30:08.2096782720\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst\n323668231\t2026-07-19T03:45:46.2209130400\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst\n321121649\t2026-07-18T03:30:08.0777625900\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.tar.zst\n222777732\t2026-07-21T03:30:15.9059638290\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst\n222761273\t2026-07-20T03:30:13.7937629440\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst\n222745965\t2026-07-19T03:45:57.2700790730\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst\n222727760\t2026-07-18T03:30:13.5578457230\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.tar.zst\n3021\t2026-07-19T03:31:07.4745284120\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.log\n3020\t2026-07-21T03:31:07.3945472590\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.log\n3018\t2026-07-18T03:31:03.9197988860\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.log\n3017\t2026-07-20T03:31:04.1706719350\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.log\n1089\t2026-07-20T03:30:14.0079985330\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.log\n1086\t2026-07-20T03:30:08.4640241540\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.log\n989\t2026-07-21T03:30:16.1364976560\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.log\n989\t2026-07-18T03:30:13.7816006690\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.log\n986\t2026-07-21T03:30:10.0285327490\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.log\n986\t2026-07-18T03:30:08.3002411830\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.log\n731\t2026-07-19T03:45:57.5197000270\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.log\n729\t2026-07-19T03:45:46.4444121980\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.log\n" +} diff --git a/runtime/history/BACKUP-DUAL-CLOUD-RECONCILE-060.txt b/runtime/history/BACKUP-DUAL-CLOUD-RECONCILE-060.txt new file mode 100644 index 0000000..b1ae9ba --- /dev/null +++ b/runtime/history/BACKUP-DUAL-CLOUD-RECONCILE-060.txt @@ -0,0 +1,310 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=BACKUP-DUAL-CLOUD-RECONCILE-060 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=backup-audit +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1 +COMMAND_SHA256=be39adf6237697b760f095d503f1fdb9e25eed004c8108b2b1d338318d4794a6 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=false +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=63db4a98c9859a2dece1cd7a0f56c08584ac144e96fedc91c7d16a907335b8e9 +SANITIZED_OUTPUT_SHA256=63db4a98c9859a2dece1cd7a0f56c08584ac144e96fedc91c7d16a907335b8e9 +OUTPUT_BEGIN +=== EXACT_IMMUTABLE_RESULT === +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "BACKUP-DUAL-CLOUD-PREFLIGHT-059", + "status": "FAIL", + "rc": 1, + "host": "pve01", + "mode": "read-only", + "component": "backup-audit", + "started_at_utc": "2026-07-21T11:35:22Z", + "finished_at_utc": "2026-07-21T11:35:23Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1", + "command_sha256": "c6f4134d15a9f641ddaf67535760d0abede973dd08222a37af05612a6f580479", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "f375162905664fc3de53bec3ea2b2485775e05bcfbd421a3c63010def946a753", + "sanitized_output_sha256": "f375162905664fc3de53bec3ea2b2485775e05bcfbd421a3c63010def946a753", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "=== BACKUP_JOBS ===\n[{\"compress\":\"zstd\",\"enabled\":1,\"id\":\"homelab-nightly-all\",\"mailnotification\":\"failure\",\"mode\":\"snapshot\",\"next-run\":1784680200,\"prune-backups\":{\"keep-daily\":\"4\"},\"schedule\":\"03:30\",\"storage\":\"local\",\"type\":\"vzdump\",\"vmid\":\"110,111,112,113,130,150,190\"}]\n=== RCLONE_REMOTES ===\npve01-mail-01:\npve01-mail-01-crypt:\npve01-mail-02:\npve01-mail-02-crypt:\npve01-mail-03:\npve01-mail-03-crypt:\npve01-mail-04:\npve01-mail-04-crypt:\npve01-mail-05:\npve01-mail-05-crypt:\npve02-mail-01:\npve02-mail-01-crypt:\npve02-mail-02:\npve02-mail-02-crypt:\npve02-mail-03:\npve02-mail-03-crypt:\npve02-mail-04:\npve02-mail-04-crypt:\npve03-mail-01:\npve03-mail-01-crypt:\npve03-mail-02:\npve03-mail-02-crypt:\npve03-mail-03:\npve03-mail-03-crypt:\npve03-mail-04:\npve03-mail-04-crypt:\n=== BACKUP_TIMERS ===\nWed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:14 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service\nWed 2026-07-22 00:09:43 MSK 9h Tue 2026-07-21 00:09:32 MSK 14h ago homelab-backup-freshness.timer homelab-backup-freshness.service\nWed 2026-07-22 02:15:00 MSK 11h Tue 2026-07-21 02:15:05 MSK 12h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service\nWed 2026-07-22 03:10:02 MSK 12h Tue 2026-07-21 03:13:14 MSK 11h ago homelab-emergency-backup.timer homelab-emergency-backup.service\nWed 2026-07-22 03:35:00 MSK 12h Tue 2026-07-21 03:35:14 MSK 11h ago filebrowser-backup.timer filebrowser-backup.service\nWed 2026-07-22 03:55:02 MSK 13h Tue 2026-07-21 03:51:25 MSK 10h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service\nWed 2026-07-22 04:17:29 MSK 13h Tue 2026-07-21 04:19:17 MSK 10h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service\nWed 2026-07-22 04:26:54 MSK 13h Tue 2026-07-21 04:28:14 MSK 10h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service\nWed 2026-07-22 04:38:04 MSK 14h Tue 2026-07-21 04:34:48 MSK 10h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service\nWed 2026-07-22 05:01:44 MSK 14h Tue 2026-07-21 04:57:14 MSK 9h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service\nWed 2026-07-22 05:23:56 MSK 14h Tue 2026-07-21 05:17:57 MSK 9h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service\nWed 2026-07-22 05:27:13 MSK 14h Tue 2026-07-21 05:29:05 MSK 9h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service\nWed 2026-07-22 05:47:19 MSK 15h Tue 2026-07-21 05:46:14 MSK 8h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service\nWed 2026-07-22 06:01:19 MSK 15h Tue 2026-07-21 05:57:30 MSK 8h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service\nWed 2026-07-22 06:27:43 MSK 15h Tue 2026-07-21 06:32:25 MSK 8h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service\nWed 2026-07-22 06:38:51 MSK 16h Tue 2026-07-21 06:43:54 MSK 7h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service\nWed 2026-07-22 06:39:22 MSK 16h Tue 2026-07-21 06:40:25 MSK 7h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service\nWed 2026-07-22 06:44:16 MSK 16h Tue 2026-07-21 06:40:25 MSK 7h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service\nWed 2026-07-22 06:52:06 MSK 16h Tue 2026-07-21 06:47:47 MSK 7h ago homelab-beszel-backup.timer homelab-beszel-backup.service\nWed 2026-07-22 06:58:58 MSK 16h Tue 2026-07-21 06:57:30 MSK 7h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service\nWed 2026-07-22 07:02:27 MSK 16h Tue 2026-07-21 07:05:30 MSK 7h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service\nWed 2026-07-22 07:07:14 MSK 16h Tue 2026-07-21 07:18:55 MSK 7h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service\nWed 2026-07-22 07:35:06 MSK 16h Tue 2026-07-21 07:36:59 MSK 6h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service\nWed 2026-07-22 07:38:21 MSK 17h Tue 2026-07-21 07:36:14 MSK 6h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service\nWed 2026-07-22 07:41:49 MSK 17h Tue 2026-07-21 07:53:41 MSK 6h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service\nWed 2026-07-22 08:10:37 MSK 17h Tue 2026-07-21 08:07:25 MSK 6h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service\nWed 2026-07-22 08:37:02 MSK 18h Tue 2026-07-21 08:30:07 MSK 6h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service\nWed 2026-07-22 08:40:13 MSK 18h Tue 2026-07-21 08:50:14 MSK 5h 45min ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service\nWed 2026-07-22 08:53:36 MSK 18h Tue 2026-07-21 08:41:14 MSK 5h 54min ago memos-backup.timer memos-backup.service\nSun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service\nSun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service\nSun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service\nSun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service\nMon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 7h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service\n=== BACKUP_UNITS ===\ndpkg-db-backup.service static -\nfilebrowser-backup.service static -\nhomelab-alertmanager-backup.service static -\nhomelab-appbackup-dockge-stacks.service static -\nhomelab-appbackup-mariadb-batch.service static -\nhomelab-appbackup-npmplus-edge.service static -\nhomelab-appbackup-postgres-batch.service static -\nhomelab-appbackup-sqlite-batch.service static -\nhomelab-authentik-backup.service static -\nhomelab-auto-backup.service static -\nhomelab-backup-coverage-matrix.service static -\nhomelab-backup-freshness.service static -\nhomelab-backup-healthcheck.service static -\nhomelab-backup-matrix-refresh.service static -\nhomelab-backup-sla-health.service static -\nhomelab-beszel-backup.service static -\nhomelab-blackbox-exporter-backup.service static -\nhomelab-cluster-admin-incident-engine-backup.service static -\nhomelab-crypto-portfolio-backup.service static -\nhomelab-edge-vm-vzdump-backup.service static -\nhomelab-emergency-backup.service static -\nhomelab-extended-appbackup.service static -\nhomelab-gitea-backup.service static -\nhomelab-gitea-secondary-backup.service static -\nhomelab-legacy-monitoring-backups.service static -\nhomelab-mail-cloud-critical-upload.service static -\nhomelab-mail-cloud-edge-vm-upload.service static -\nhomelab-mail-cloud-nextcloud-vm-upload.service static -\nhomelab-mail-cloud-restore-drill.service static -\nhomelab-npmplus-kuma-config-backup.service static -\nhomelab-router-running-config-mail-cloud.service static -\nhomelab-vaultwarden-backup.service static -\nhomelab-verified-backup-health.service static -\nhomelab-vm-backup-health.service static -\nhomelab-vm-backup-policy.service static -\nhomelab-vm-full-backup.service static -\nhomelab-vm-local-dumps-2cloud.service static -\nhomelab-vm160-remote-backup.service static -\nimmich-media-backup.service static -\nmemos-backup.service static -\nnetbird-vps-backup.service static -\npaperless-backup.service static -\nskladchik-reports-monitor-backup-restore-check.service static -\nskladchik-reports-monitor-backup.service static -\ndpkg-db-backup.timer enabled enabled\nfilebrowser-backup.timer enabled enabled\nhomelab-alertmanager-backup.timer enabled enabled\nhomelab-appbackup-dockge-stacks.timer enabled enabled\nhomelab-appbackup-mariadb-batch.timer enabled enabled\nhomelab-appbackup-npmplus-edge.timer enabled enabled\nhomelab-appbackup-postgres-batch.timer enabled enabled\nhomelab-appbackup-sqlite-batch.timer enabled enabled\nhomelab-authentik-backup.timer disabled enabled\nhomelab-auto-backup.timer enabled enabled\nhomelab-backup-coverage-matrix.timer enabled enabled\nhomelab-backup-freshness.timer enabled enabled\nhomelab-backup-healthcheck.timer enabled enabled\nhomelab-backup-matrix-refresh.timer disabled enabled\nhomelab-backup-sla-health.timer enabled enabled\nhomelab-beszel-backup.timer enabled enabled\nhomelab-blackbox-exporter-backup.timer enabled enabled\nhomelab-cluster-admin-incident-engine-backup.timer enabled enabled\nhomelab-crypto-portfolio-backup.timer enabled enabled\nhomelab-edge-vm-vzdump-backup.timer disabled enabled\nhomelab-emergency-backup.timer enabled enabled\nhomelab-extended-appbackup.timer enabled enabled\nhomelab-gitea-backup.timer disabled enabled\nhomelab-gitea-secondary-backup.timer enabled enabled\nhomelab-legacy-monitoring-backups.timer enabled enabled\nhomelab-mail-cloud-critical-upload.timer enabled enabled\nhomelab-mail-cloud-edge-vm-upload.timer enabled enabled\nhomelab-mail-cloud-nextcloud-vm-upload.timer enabled enabled\nhomelab-mail-cloud-restore-drill.timer enabled enabled\nhomelab-npmplus-kuma-config-backup.timer disabled enabled\nhomelab-router-running-config-mail-cloud.timer enabled enabled\nhomelab-vaultwarden-backup.timer disabled enabled\nhomelab-verified-backup-health.timer enabled enabled\nhomelab-vm-backup-health.timer enabled enabled\nhomelab-vm-backup-policy.timer enabled enabled\nhomelab-vm-full-backup.timer enabled enabled\nhomelab-vm-local-dumps-2cloud.timer enabled enabled\nhomelab-vm160-remote-backup.timer disabled enabled\nimmich-media-backup.timer disabled enabled\nmemos-backup.timer enabled enabled\nnetbird-vps-backup.timer disabled enabled\npaperless-backup.timer disabled enabled\nskladchik-reports-monitor-backup-restore-check.timer enabled enabled\nskladchik-reports-monitor-backup.timer enabled enabled\n=== PIPELINE_FILES ===\n/etc/cron.d/vzdump\n/etc/systemd/system/homelab-edge-vm-offhost-freshness.timer\n/etc/systemd/system/homelab-edge-vm-vzdump-backup.service\n/etc/systemd/system/homelab-edge-vm-vzdump-backup.timer\n/etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.service\n/etc/systemd/system/homelab-vm-full-backup.service\n/etc/systemd/system/multi-user.target.wants/pve-guests.service\n/etc/systemd/system/pve-manager.service\n/etc/systemd/system/timers.target.wants/homelab-edge-vm-offhost-freshness.timer\n/usr/local/sbin/appbackup-dockge-stacks\n/usr/local/sbin/appbackup-dockge-stacks.bak-chunkall-20260710T011410Z\n/usr/local/sbin/appbackup-restore-check\n/usr/local/sbin/appbackup-restore-check.bak-chunkall-20260710T011410Z\n/usr/local/sbin/appbackup-restore-check.bak-dual-read-20260711T051924Z\n/usr/local/sbin/appbackup-restore-check.bak-flat-read-20260711T052854Z\n/usr/local/sbin/appbackup-restore-check.bak-flat-read-v2-20260711T053129Z\n/usr/local/sbin/appbackup-retention\n/usr/local/sbin/appbackup-retention.bak-dual-read-20260711T051924Z\n/usr/local/sbin/backupctl\n/usr/local/sbin/backup-restore-check\n/usr/local/sbin/backup-restore-check.bak-chunkall-20260710T011410Z\n/usr/local/sbin/backup-retention\n/usr/local/sbin/homelab-backup-edge-vm-vzdump\n/usr/local/sbin/homelab-capacity-risk\n/usr/local/sbin/homelab-change-transaction\n/usr/local/sbin/homelab-chat-run\n/usr/local/sbin/homelab-edge-vm-offhost-freshness\n/usr/local/sbin/homelab-extended-appbackup-run\n/usr/local/sbin/homelab-golden-state-extended-overlay\n/usr/local/sbin/homelab-golden-state-index\n/usr/local/sbin/homelab-health-metrics\n/usr/local/sbin/homelab-ingress-standby-refresh\n/usr/local/sbin/homelab-mail-cloud-critical-upload\n/usr/local/sbin/homelab-mail-cloud-critical-upload.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload\n/usr/local/sbin/homelab-mail-cloud-restore-drill\n/usr/local/sbin/homelab-mail-cloud-restore-drill.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-rclone-cat-chunked\n/usr/local/sbin/homelab-rclone-cat-flat-chunked\n/usr/local/sbin/homelab-rclone-copy-chunked\n/usr/local/sbin/homelab-rclone-copy-flat-chunked\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh\n/usr/local/sbin/homelab-rclone-required-check.sh\n/usr/local/sbin/homelab-router-hard-power-cycle\n/usr/local/sbin/homelab-router-running-config-mail-cloud\n/usr/local/sbin/homelab-router-running-config-mail-cloud.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-safe-exec\n/usr/local/sbin/homelab-secret-exposure-guard\n/usr/local/sbin/homelab-stage-2cloud\n/usr/local/sbin/homelab-stage-2cloud.bak-global-chunkonly-20260710T011036Z\n/usr/local/sbin/homelab-stage-2cloud-flat\n/usr/local/sbin/homelab-stage-2cloud-flat-canary\n/usr/local/sbin/homelab-stage-verify-chunked\n/usr/local/sbin/homelab-stage-verify-flat-chunked\n/usr/local/sbin/homelab-vm160-remote-backup\n/usr/local/sbin/homelab-vm-backup-health\n/usr/local/sbin/homelab-vm-backup-policy\n/usr/local/sbin/homelab-vm-full-backup\n/usr/local/sbin/homelab-vm-full-backup.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-vm-full-restore-drill\n/usr/local/sbin/homelab-vm-full-restore-drill.bak.20260710T080231Z\n/usr/local/sbin/homelab-vm-full-restore-drill.bak-docker-check-v2-20260710T050044Z\n/usr/local/sbin/homelab-vm-full-restore-drill.bak-isolated-ip-20260710T044331Z\n/usr/local/sbin/homelab-vm-full-retention\n/usr/local/sbin/homelab-vm-local-dumps-2cloud\n/usr/local/sbin/homelab-vm-local-dumps-retention\n/usr/local/sbin/skladchik-reports-monitor-backup\n/usr/local/sbin/skladchik-reports-monitor-backup.pre-restore-check-20260710T065311Z.bak\n/usr/local/sbin/skladchik-reports-monitor-backup.pre-session-keepalive-20260710T071605Z.bak\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check.pre-session-keepalive-20260710T071605Z.bak\n=== LOCAL_BACKUPS ===\n7022386770\t2026-07-18T03:31:03.6566057240\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.vma.zst\n7021952392\t2026-07-21T03:31:06.5717414950\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst\n7021297699\t2026-07-19T03:31:06.8755351020\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst\n7016275481\t2026-07-20T03:31:03.2165123470\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst\n329541843\t2026-07-21T03:30:09.7688696330\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst\n326601118\t2026-07-20T03:30:08.2096782720\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst\n323668231\t2026-07-19T03:45:46.2209130400\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst\n321121649\t2026-07-18T03:30:08.0777625900\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.tar.zst\n222777732\t2026-07-21T03:30:15.9059638290\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst\n222761273\t2026-07-20T03:30:13.7937629440\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst\n222745965\t2026-07-19T03:45:57.2700790730\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst\n222727760\t2026-07-18T03:30:13.5578457230\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.tar.zst\n3021\t2026-07-19T03:31:07.4745284120\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.log\n3020\t2026-07-21T03:31:07.3945472590\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.log\n3018\t2026-07-18T03:31:03.9197988860\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.log\n3017\t2026-07-20T03:31:04.1706719350\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.log\n1089\t2026-07-20T03:30:14.0079985330\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.log\n1086\t2026-07-20T03:30:08.4640241540\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.log\n989\t2026-07-21T03:30:16.1364976560\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.log\n989\t2026-07-18T03:30:13.7816006690\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.log\n986\t2026-07-21T03:30:10.0285327490\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.log\n986\t2026-07-18T03:30:08.3002411830\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.log\n731\t2026-07-19T03:45:57.5197000270\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.log\n729\t2026-07-19T03:45:46.4444121980\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.log\n" +} + +EXACT_GITEA=FOUND +=== LOCAL_RAW_RESULT === +FILE=/var/lib/homelab-chat-run/raw/BACKUP-DUAL-CLOUD-PREFLIGHT-059-20260721T113522Z.log SIZE=21897 MTIME=2026-07-21 14:35:23.197206410 +0300 +=== BACKUP_JOBS === +[{"compress":"zstd","enabled":1,"id":"homelab-nightly-all","mailnotification":"failure","mode":"snapshot","next-run":1784680200,"prune-backups":{"keep-daily":"4"},"schedule":"03:30","storage":"local","type":"vzdump","vmid":"110,111,112,113,130,150,190"}] +=== RCLONE_REMOTES === +pve01-mail-01: +pve01-mail-01-crypt: +pve01-mail-02: +pve01-mail-02-crypt: +pve01-mail-03: +pve01-mail-03-crypt: +pve01-mail-04: +pve01-mail-04-crypt: +pve01-mail-05: +pve01-mail-05-crypt: +pve02-mail-01: +pve02-mail-01-crypt: +pve02-mail-02: +pve02-mail-02-crypt: +pve02-mail-03: +pve02-mail-03-crypt: +pve02-mail-04: +pve02-mail-04-crypt: +pve03-mail-01: +pve03-mail-01-crypt: +pve03-mail-02: +pve03-mail-02-crypt: +pve03-mail-03: +pve03-mail-03-crypt: +pve03-mail-04: +pve03-mail-04-crypt: +=== BACKUP_TIMERS === +Wed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:14 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service +Wed 2026-07-22 00:09:43 MSK 9h Tue 2026-07-21 00:09:32 MSK 14h ago homelab-backup-freshness.timer homelab-backup-freshness.service +Wed 2026-07-22 02:15:00 MSK 11h Tue 2026-07-21 02:15:05 MSK 12h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service +Wed 2026-07-22 03:10:02 MSK 12h Tue 2026-07-21 03:13:14 MSK 11h ago homelab-emergency-backup.timer homelab-emergency-backup.service +Wed 2026-07-22 03:35:00 MSK 12h Tue 2026-07-21 03:35:14 MSK 11h ago filebrowser-backup.timer filebrowser-backup.service +Wed 2026-07-22 03:55:02 MSK 13h Tue 2026-07-21 03:51:25 MSK 10h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service +Wed 2026-07-22 04:17:29 MSK 13h Tue 2026-07-21 04:19:17 MSK 10h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service +Wed 2026-07-22 04:26:54 MSK 13h Tue 2026-07-21 04:28:14 MSK 10h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service +Wed 2026-07-22 04:38:04 MSK 14h Tue 2026-07-21 04:34:48 MSK 10h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service +Wed 2026-07-22 05:01:44 MSK 14h Tue 2026-07-21 04:57:14 MSK 9h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service +Wed 2026-07-22 05:23:56 MSK 14h Tue 2026-07-21 05:17:57 MSK 9h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service +Wed 2026-07-22 05:27:13 MSK 14h Tue 2026-07-21 05:29:05 MSK 9h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service +Wed 2026-07-22 05:47:19 MSK 15h Tue 2026-07-21 05:46:14 MSK 8h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service +Wed 2026-07-22 06:01:19 MSK 15h Tue 2026-07-21 05:57:30 MSK 8h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service +Wed 2026-07-22 06:27:43 MSK 15h Tue 2026-07-21 06:32:25 MSK 8h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service +Wed 2026-07-22 06:38:51 MSK 16h Tue 2026-07-21 06:43:54 MSK 7h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service +Wed 2026-07-22 06:39:22 MSK 16h Tue 2026-07-21 06:40:25 MSK 7h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service +Wed 2026-07-22 06:44:16 MSK 16h Tue 2026-07-21 06:40:25 MSK 7h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service +Wed 2026-07-22 06:52:06 MSK 16h Tue 2026-07-21 06:47:47 MSK 7h ago homelab-beszel-backup.timer homelab-beszel-backup.service +Wed 2026-07-22 06:58:58 MSK 16h Tue 2026-07-21 06:57:30 MSK 7h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service +Wed 2026-07-22 07:02:27 MSK 16h Tue 2026-07-21 07:05:30 MSK 7h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service +Wed 2026-07-22 07:07:14 MSK 16h Tue 2026-07-21 07:18:55 MSK 7h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service +Wed 2026-07-22 07:35:06 MSK 16h Tue 2026-07-21 07:36:59 MSK 6h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service +Wed 2026-07-22 07:38:21 MSK 17h Tue 2026-07-21 07:36:14 MSK 6h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service +Wed 2026-07-22 07:41:49 MSK 17h Tue 2026-07-21 07:53:41 MSK 6h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service +Wed 2026-07-22 08:10:37 MSK 17h Tue 2026-07-21 08:07:25 MSK 6h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service +Wed 2026-07-22 08:37:02 MSK 18h Tue 2026-07-21 08:30:07 MSK 6h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service +Wed 2026-07-22 08:40:13 MSK 18h Tue 2026-07-21 08:50:14 MSK 5h 45min ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service +Wed 2026-07-22 08:53:36 MSK 18h Tue 2026-07-21 08:41:14 MSK 5h 54min ago memos-backup.timer memos-backup.service +Sun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service +Sun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service +Sun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service +Sun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service +Mon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 7h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service +=== BACKUP_UNITS === +dpkg-db-backup.service static - +filebrowser-backup.service static - +homelab-alertmanager-backup.service static - +homelab-appbackup-dockge-stacks.service static - +homelab-appbackup-mariadb-batch.service static - +homelab-appbackup-npmplus-edge.service static - +homelab-appbackup-postgres-batch.service static - +homelab-appbackup-sqlite-batch.service static - +homelab-authentik-backup.service static - +homelab-auto-backup.service static - +homelab-backup-coverage-matrix.service static - +homelab-backup-freshness.service static - +homelab-backup-healthcheck.service static - +homelab-backup-matrix-refresh.service static - +homelab-backup-sla-health.service static - +homelab-beszel-backup.service static - +homelab-blackbox-exporter-backup.service static - +homelab-cluster-admin-incident-engine-backup.service static - +homelab-crypto-portfolio-backup.service static - +homelab-edge-vm-vzdump-backup.service static - +homelab-emergency-backup.service static - +homelab-extended-appbackup.service static - +homelab-gitea-backup.service static - +homelab-gitea-secondary-backup.service static - +homelab-legacy-monitoring-backups.service static - +homelab-mail-cloud-critical-upload.service static - +homelab-mail-cloud-edge-vm-upload.service static - +homelab-mail-cloud-nextcloud-vm-upload.service static - +homelab-mail-cloud-restore-drill.service static - +homelab-npmplus-kuma-config-backup.service static - +homelab-router-running-config-mail-cloud.service static - +homelab-vaultwarden-backup.service static - +homelab-verified-backup-health.service static - +homelab-vm-backup-health.service static - +homelab-vm-backup-policy.service static - +homelab-vm-full-backup.service static - +homelab-vm-local-dumps-2cloud.service static - +homelab-vm160-remote-backup.service static - +immich-media-backup.service static - +memos-backup.service static - +netbird-vps-backup.service static - +paperless-backup.service static - +skladchik-reports-monitor-backup-restore-check.service static - +skladchik-reports-monitor-backup.service static - +dpkg-db-backup.timer enabled enabled +filebrowser-backup.timer enabled enabled +homelab-alertmanager-backup.timer enabled enabled +homelab-appbackup-dockge-stacks.timer enabled enabled +homelab-appbackup-mariadb-batch.timer enabled enabled +homelab-appbackup-npmplus-edge.timer enabled enabled +homelab-appbackup-postgres-batch.timer enabled enabled +homelab-appbackup-sqlite-batch.timer enabled enabled +homelab-authentik-backup.timer disabled enabled +homelab-auto-backup.timer enabled enabled +homelab-backup-coverage-matrix.timer enabled enabled +homelab-backup-freshness.timer enabled enabled +homelab-backup-healthcheck.timer enabled enabled +homelab-backup-matrix-refresh.timer disabled enabled +homelab-backup-sla-health.timer enabled enabled +homelab-beszel-backup.timer enabled enabled +homelab-blackbox-exporter-backup.timer enabled enabled +homelab-cluster-admin-incident-engine-backup.timer enabled enabled +homelab-crypto-portfolio-backup.timer enabled enabled +homelab-edge-vm-vzdump-backup.timer disabled enabled +homelab-emergency-backup.timer enabled enabled +homelab-extended-appbackup.timer enabled enabled +homelab-gitea-backup.timer disabled enabled +homelab-gitea-secondary-backup.timer enabled enabled +homelab-legacy-monitoring-backups.timer enabled enabled +homelab-mail-cloud-critical-upload.timer enabled enabled +homelab-mail-cloud-edge-vm-upload.timer enabled enabled +homelab-mail-cloud-nextcloud-vm-upload.timer enabled enabled +homelab-mail-cloud-restore-drill.timer enabled enabled +homelab-npmplus-kuma-config-backup.timer disabled enabled +homelab-router-running-config-mail-cloud.timer enabled enabled +homelab-vaultwarden-backup.timer disabled enabled +homelab-verified-backup-health.timer enabled enabled +homelab-vm-backup-health.timer enabled enabled +homelab-vm-backup-policy.timer enabled enabled +homelab-vm-full-backup.timer enabled enabled +homelab-vm-local-dumps-2cloud.timer enabled enabled +homelab-vm160-remote-backup.timer disabled enabled +immich-media-backup.timer disabled enabled +memos-backup.timer enabled enabled +netbird-vps-backup.timer disabled enabled +paperless-backup.timer disabled enabled +skladchik-reports-monitor-backup-restore-check.timer enabled enabled +skladchik-reports-monitor-backup.timer enabled enabled +=== PIPELINE_FILES === +/etc/cron.d/vzdump +/etc/systemd/system/homelab-edge-vm-offhost-freshness.timer +/etc/systemd/system/homelab-edge-vm-vzdump-backup.service +/etc/systemd/system/homelab-edge-vm-vzdump-backup.timer +/etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.service +/etc/systemd/system/homelab-vm-full-backup.service +/etc/systemd/system/multi-user.target.wants/pve-guests.service +/etc/systemd/system/pve-manager.service +/etc/systemd/system/timers.target.wants/homelab-edge-vm-offhost-freshness.timer +/usr/local/sbin/appbackup-dockge-stacks +/usr/local/sbin/appbackup-dockge-stacks.bak-chunkall-20260710T011410Z +/usr/local/sbin/appbackup-restore-check +/usr/local/sbin/appbackup-restore-check.bak-chunkall-20260710T011410Z +/usr/local/sbin/appbackup-restore-check.bak-dual-read-20260711T051924Z +/usr/local/sbin/appbackup-restore-check.bak-flat-read-20260711T052854Z +/usr/local/sbin/appbackup-restore-check.bak-flat-read-v2-20260711T053129Z +/usr/local/sbin/appbackup-retention +/usr/local/sbin/appbackup-retention.bak-dual-read-20260711T051924Z +/usr/local/sbin/backupctl +/usr/local/sbin/backup-restore-check +/usr/local/sbin/backup-restore-check.bak-chunkall-20260710T011410Z +/usr/local/sbin/backup-retention +/usr/local/sbin/homelab-backup-edge-vm-vzdump +/usr/local/sbin/homelab-capacity-risk +/usr/local/sbin/homelab-change-transaction +/usr/local/sbin/homelab-chat-run +/usr/local/sbin/homelab-edge-vm-offhost-freshness +/usr/local/sbin/homelab-extended-appbackup-run +/usr/local/sbin/homelab-golden-state-extended-overlay +/usr/local/sbin/homelab-golden-state-index +/usr/local/sbin/homelab-health-metrics +/usr/local/sbin/homelab-ingress-standby-refresh +/usr/local/sbin/homelab-mail-cloud-critical-upload +/usr/local/sbin/homelab-mail-cloud-critical-upload.bak-chunkall-20260710T011410Z +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload +/usr/local/sbin/homelab-mail-cloud-restore-drill +/usr/local/sbin/homelab-mail-cloud-restore-drill.bak-chunkall-20260710T011410Z +/usr/local/sbin/homelab-rclone-cat-chunked +/usr/local/sbin/homelab-rclone-cat-flat-chunked +/usr/local/sbin/homelab-rclone-copy-chunked +/usr/local/sbin/homelab-rclone-copy-flat-chunked +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh +/usr/local/sbin/homelab-rclone-required-check.sh +/usr/local/sbin/homelab-router-hard-power-cycle +/usr/local/sbin/homelab-router-running-config-mail-cloud +/usr/local/sbin/homelab-router-running-config-mail-cloud.bak-chunkall-20260710T011410Z +/usr/local/sbin/homelab-safe-exec +/usr/local/sbin/homelab-secret-exposure-guard +/usr/local/sbin/homelab-stage-2cloud +/usr/local/sbin/homelab-stage-2cloud.bak-global-chunkonly-20260710T011036Z +/usr/local/sbin/homelab-stage-2cloud-flat +/usr/local/sbin/homelab-stage-2cloud-flat-canary +/usr/local/sbin/homelab-stage-verify-chunked +/usr/local/sbin/homelab-stage-verify-flat-chunked +/usr/local/sbin/homelab-vm160-remote-backup +/usr/local/sbin/homelab-vm-backup-health +/usr/local/sbin/homelab-vm-backup-policy +/usr/local/sbin/homelab-vm-full-backup +/usr/local/sbin/homelab-vm-full-backup.bak-chunkall-20260710T011410Z +/usr/local/sbin/homelab-vm-full-restore-drill +/usr/local/sbin/homelab-vm-full-restore-drill.bak.20260710T080231Z +/usr/local/sbin/homelab-vm-full-restore-drill.bak-docker-check-v2-20260710T050044Z +/usr/local/sbin/homelab-vm-full-restore-drill.bak-isolated-ip-20260710T044331Z +/usr/local/sbin/homelab-vm-full-retention +/usr/local/sbin/homelab-vm-local-dumps-2cloud +/usr/local/sbin/homelab-vm-local-dumps-retention +/usr/local/sbin/skladchik-reports-monitor-backup +/usr/local/sbin/skladchik-reports-monitor-backup.pre-restore-check-20260710T065311Z.bak +/usr/local/sbin/skladchik-reports-monitor-backup.pre-session-keepalive-20260710T071605Z.bak +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check.pre-session-keepalive-20260710T071605Z.bak +=== LOCAL_BACKUPS === +7022386770 2026-07-18T03:31:03.6566057240 /var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.vma.zst +7021952392 2026-07-21T03:31:06.5717414950 /var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst +7021297699 2026-07-19T03:31:06.8755351020 /var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst +7016275481 2026-07-20T03:31:03.2165123470 /var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst +329541843 2026-07-21T03:30:09.7688696330 /var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst +326601118 2026-07-20T03:30:08.2096782720 /var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst +323668231 2026-07-19T03:45:46.2209130400 /var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst +321121649 2026-07-18T03:30:08.0777625900 /var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.tar.zst +222777732 2026-07-21T03:30:15.9059638290 /var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst +222761273 2026-07-20T03:30:13.7937629440 /var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst +222745965 2026-07-19T03:45:57.2700790730 /var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst +222727760 2026-07-18T03:30:13.5578457230 /var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.tar.zst +3021 2026-07-19T03:31:07.4745284120 /var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.log +3020 2026-07-21T03:31:07.3945472590 /var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.log +3018 2026-07-18T03:31:03.9197988860 /var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.log +3017 2026-07-20T03:31:04.1706719350 /var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.log +1089 2026-07-20T03:30:14.0079985330 /var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.log +1086 2026-07-20T03:30:08.4640241540 /var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.log +989 2026-07-21T03:30:16.1364976560 /var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.log +989 2026-07-18T03:30:13.7816006690 /var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.log +986 2026-07-21T03:30:10.0285327490 /var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.log +986 2026-07-18T03:30:08.3002411830 /var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.log +731 2026-07-19T03:45:57.5197000270 /var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.log +729 2026-07-19T03:45:46.4444121980 /var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.log + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 20903e4..f7005f3 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "BACKUP-DUAL-CLOUD-PREFLIGHT-059", - "status": "FAIL", - "rc": 1, + "command_id": "BACKUP-DUAL-CLOUD-RECONCILE-060", + "status": "OK", + "rc": 0, "host": "pve01", "mode": "read-only", "component": "backup-audit", - "started_at_utc": "2026-07-21T11:35:22Z", - "finished_at_utc": "2026-07-21T11:35:23Z", + "started_at_utc": "2026-07-21T11:38:23Z", + "finished_at_utc": "2026-07-21T11:38:23Z", "reference_register_checked": true, "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", "error_register_checked": true, "error_register_sha256": "f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1", - "command_sha256": "c6f4134d15a9f641ddaf67535760d0abede973dd08222a37af05612a6f580479", + "command_sha256": "be39adf6237697b760f095d503f1fdb9e25eed004c8108b2b1d338318d4794a6", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -22,9 +22,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "f375162905664fc3de53bec3ea2b2485775e05bcfbd421a3c63010def946a753", - "sanitized_output_sha256": "f375162905664fc3de53bec3ea2b2485775e05bcfbd421a3c63010def946a753", + "raw_evidence_sha256": "63db4a98c9859a2dece1cd7a0f56c08584ac144e96fedc91c7d16a907335b8e9", + "sanitized_output_sha256": "63db4a98c9859a2dece1cd7a0f56c08584ac144e96fedc91c7d16a907335b8e9", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "=== BACKUP_JOBS ===\n[{\"compress\":\"zstd\",\"enabled\":1,\"id\":\"homelab-nightly-all\",\"mailnotification\":\"failure\",\"mode\":\"snapshot\",\"next-run\":1784680200,\"prune-backups\":{\"keep-daily\":\"4\"},\"schedule\":\"03:30\",\"storage\":\"local\",\"type\":\"vzdump\",\"vmid\":\"110,111,112,113,130,150,190\"}]\n=== RCLONE_REMOTES ===\npve01-mail-01:\npve01-mail-01-crypt:\npve01-mail-02:\npve01-mail-02-crypt:\npve01-mail-03:\npve01-mail-03-crypt:\npve01-mail-04:\npve01-mail-04-crypt:\npve01-mail-05:\npve01-mail-05-crypt:\npve02-mail-01:\npve02-mail-01-crypt:\npve02-mail-02:\npve02-mail-02-crypt:\npve02-mail-03:\npve02-mail-03-crypt:\npve02-mail-04:\npve02-mail-04-crypt:\npve03-mail-01:\npve03-mail-01-crypt:\npve03-mail-02:\npve03-mail-02-crypt:\npve03-mail-03:\npve03-mail-03-crypt:\npve03-mail-04:\npve03-mail-04-crypt:\n=== BACKUP_TIMERS ===\nWed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:14 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service\nWed 2026-07-22 00:09:43 MSK 9h Tue 2026-07-21 00:09:32 MSK 14h ago homelab-backup-freshness.timer homelab-backup-freshness.service\nWed 2026-07-22 02:15:00 MSK 11h Tue 2026-07-21 02:15:05 MSK 12h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service\nWed 2026-07-22 03:10:02 MSK 12h Tue 2026-07-21 03:13:14 MSK 11h ago homelab-emergency-backup.timer homelab-emergency-backup.service\nWed 2026-07-22 03:35:00 MSK 12h Tue 2026-07-21 03:35:14 MSK 11h ago filebrowser-backup.timer filebrowser-backup.service\nWed 2026-07-22 03:55:02 MSK 13h Tue 2026-07-21 03:51:25 MSK 10h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service\nWed 2026-07-22 04:17:29 MSK 13h Tue 2026-07-21 04:19:17 MSK 10h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service\nWed 2026-07-22 04:26:54 MSK 13h Tue 2026-07-21 04:28:14 MSK 10h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service\nWed 2026-07-22 04:38:04 MSK 14h Tue 2026-07-21 04:34:48 MSK 10h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service\nWed 2026-07-22 05:01:44 MSK 14h Tue 2026-07-21 04:57:14 MSK 9h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service\nWed 2026-07-22 05:23:56 MSK 14h Tue 2026-07-21 05:17:57 MSK 9h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service\nWed 2026-07-22 05:27:13 MSK 14h Tue 2026-07-21 05:29:05 MSK 9h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service\nWed 2026-07-22 05:47:19 MSK 15h Tue 2026-07-21 05:46:14 MSK 8h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service\nWed 2026-07-22 06:01:19 MSK 15h Tue 2026-07-21 05:57:30 MSK 8h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service\nWed 2026-07-22 06:27:43 MSK 15h Tue 2026-07-21 06:32:25 MSK 8h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service\nWed 2026-07-22 06:38:51 MSK 16h Tue 2026-07-21 06:43:54 MSK 7h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service\nWed 2026-07-22 06:39:22 MSK 16h Tue 2026-07-21 06:40:25 MSK 7h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service\nWed 2026-07-22 06:44:16 MSK 16h Tue 2026-07-21 06:40:25 MSK 7h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service\nWed 2026-07-22 06:52:06 MSK 16h Tue 2026-07-21 06:47:47 MSK 7h ago homelab-beszel-backup.timer homelab-beszel-backup.service\nWed 2026-07-22 06:58:58 MSK 16h Tue 2026-07-21 06:57:30 MSK 7h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service\nWed 2026-07-22 07:02:27 MSK 16h Tue 2026-07-21 07:05:30 MSK 7h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service\nWed 2026-07-22 07:07:14 MSK 16h Tue 2026-07-21 07:18:55 MSK 7h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service\nWed 2026-07-22 07:35:06 MSK 16h Tue 2026-07-21 07:36:59 MSK 6h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service\nWed 2026-07-22 07:38:21 MSK 17h Tue 2026-07-21 07:36:14 MSK 6h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service\nWed 2026-07-22 07:41:49 MSK 17h Tue 2026-07-21 07:53:41 MSK 6h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service\nWed 2026-07-22 08:10:37 MSK 17h Tue 2026-07-21 08:07:25 MSK 6h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service\nWed 2026-07-22 08:37:02 MSK 18h Tue 2026-07-21 08:30:07 MSK 6h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service\nWed 2026-07-22 08:40:13 MSK 18h Tue 2026-07-21 08:50:14 MSK 5h 45min ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service\nWed 2026-07-22 08:53:36 MSK 18h Tue 2026-07-21 08:41:14 MSK 5h 54min ago memos-backup.timer memos-backup.service\nSun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service\nSun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service\nSun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service\nSun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service\nMon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 7h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service\n=== BACKUP_UNITS ===\ndpkg-db-backup.service static -\nfilebrowser-backup.service static -\nhomelab-alertmanager-backup.service static -\nhomelab-appbackup-dockge-stacks.service static -\nhomelab-appbackup-mariadb-batch.service static -\nhomelab-appbackup-npmplus-edge.service static -\nhomelab-appbackup-postgres-batch.service static -\nhomelab-appbackup-sqlite-batch.service static -\nhomelab-authentik-backup.service static -\nhomelab-auto-backup.service static -\nhomelab-backup-coverage-matrix.service static -\nhomelab-backup-freshness.service static -\nhomelab-backup-healthcheck.service static -\nhomelab-backup-matrix-refresh.service static -\nhomelab-backup-sla-health.service static -\nhomelab-beszel-backup.service static -\nhomelab-blackbox-exporter-backup.service static -\nhomelab-cluster-admin-incident-engine-backup.service static -\nhomelab-crypto-portfolio-backup.service static -\nhomelab-edge-vm-vzdump-backup.service static -\nhomelab-emergency-backup.service static -\nhomelab-extended-appbackup.service static -\nhomelab-gitea-backup.service static -\nhomelab-gitea-secondary-backup.service static -\nhomelab-legacy-monitoring-backups.service static -\nhomelab-mail-cloud-critical-upload.service static -\nhomelab-mail-cloud-edge-vm-upload.service static -\nhomelab-mail-cloud-nextcloud-vm-upload.service static -\nhomelab-mail-cloud-restore-drill.service static -\nhomelab-npmplus-kuma-config-backup.service static -\nhomelab-router-running-config-mail-cloud.service static -\nhomelab-vaultwarden-backup.service static -\nhomelab-verified-backup-health.service static -\nhomelab-vm-backup-health.service static -\nhomelab-vm-backup-policy.service static -\nhomelab-vm-full-backup.service static -\nhomelab-vm-local-dumps-2cloud.service static -\nhomelab-vm160-remote-backup.service static -\nimmich-media-backup.service static -\nmemos-backup.service static -\nnetbird-vps-backup.service static -\npaperless-backup.service static -\nskladchik-reports-monitor-backup-restore-check.service static -\nskladchik-reports-monitor-backup.service static -\ndpkg-db-backup.timer enabled enabled\nfilebrowser-backup.timer enabled enabled\nhomelab-alertmanager-backup.timer enabled enabled\nhomelab-appbackup-dockge-stacks.timer enabled enabled\nhomelab-appbackup-mariadb-batch.timer enabled enabled\nhomelab-appbackup-npmplus-edge.timer enabled enabled\nhomelab-appbackup-postgres-batch.timer enabled enabled\nhomelab-appbackup-sqlite-batch.timer enabled enabled\nhomelab-authentik-backup.timer disabled enabled\nhomelab-auto-backup.timer enabled enabled\nhomelab-backup-coverage-matrix.timer enabled enabled\nhomelab-backup-freshness.timer enabled enabled\nhomelab-backup-healthcheck.timer enabled enabled\nhomelab-backup-matrix-refresh.timer disabled enabled\nhomelab-backup-sla-health.timer enabled enabled\nhomelab-beszel-backup.timer enabled enabled\nhomelab-blackbox-exporter-backup.timer enabled enabled\nhomelab-cluster-admin-incident-engine-backup.timer enabled enabled\nhomelab-crypto-portfolio-backup.timer enabled enabled\nhomelab-edge-vm-vzdump-backup.timer disabled enabled\nhomelab-emergency-backup.timer enabled enabled\nhomelab-extended-appbackup.timer enabled enabled\nhomelab-gitea-backup.timer disabled enabled\nhomelab-gitea-secondary-backup.timer enabled enabled\nhomelab-legacy-monitoring-backups.timer enabled enabled\nhomelab-mail-cloud-critical-upload.timer enabled enabled\nhomelab-mail-cloud-edge-vm-upload.timer enabled enabled\nhomelab-mail-cloud-nextcloud-vm-upload.timer enabled enabled\nhomelab-mail-cloud-restore-drill.timer enabled enabled\nhomelab-npmplus-kuma-config-backup.timer disabled enabled\nhomelab-router-running-config-mail-cloud.timer enabled enabled\nhomelab-vaultwarden-backup.timer disabled enabled\nhomelab-verified-backup-health.timer enabled enabled\nhomelab-vm-backup-health.timer enabled enabled\nhomelab-vm-backup-policy.timer enabled enabled\nhomelab-vm-full-backup.timer enabled enabled\nhomelab-vm-local-dumps-2cloud.timer enabled enabled\nhomelab-vm160-remote-backup.timer disabled enabled\nimmich-media-backup.timer disabled enabled\nmemos-backup.timer enabled enabled\nnetbird-vps-backup.timer disabled enabled\npaperless-backup.timer disabled enabled\nskladchik-reports-monitor-backup-restore-check.timer enabled enabled\nskladchik-reports-monitor-backup.timer enabled enabled\n=== PIPELINE_FILES ===\n/etc/cron.d/vzdump\n/etc/systemd/system/homelab-edge-vm-offhost-freshness.timer\n/etc/systemd/system/homelab-edge-vm-vzdump-backup.service\n/etc/systemd/system/homelab-edge-vm-vzdump-backup.timer\n/etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.service\n/etc/systemd/system/homelab-vm-full-backup.service\n/etc/systemd/system/multi-user.target.wants/pve-guests.service\n/etc/systemd/system/pve-manager.service\n/etc/systemd/system/timers.target.wants/homelab-edge-vm-offhost-freshness.timer\n/usr/local/sbin/appbackup-dockge-stacks\n/usr/local/sbin/appbackup-dockge-stacks.bak-chunkall-20260710T011410Z\n/usr/local/sbin/appbackup-restore-check\n/usr/local/sbin/appbackup-restore-check.bak-chunkall-20260710T011410Z\n/usr/local/sbin/appbackup-restore-check.bak-dual-read-20260711T051924Z\n/usr/local/sbin/appbackup-restore-check.bak-flat-read-20260711T052854Z\n/usr/local/sbin/appbackup-restore-check.bak-flat-read-v2-20260711T053129Z\n/usr/local/sbin/appbackup-retention\n/usr/local/sbin/appbackup-retention.bak-dual-read-20260711T051924Z\n/usr/local/sbin/backupctl\n/usr/local/sbin/backup-restore-check\n/usr/local/sbin/backup-restore-check.bak-chunkall-20260710T011410Z\n/usr/local/sbin/backup-retention\n/usr/local/sbin/homelab-backup-edge-vm-vzdump\n/usr/local/sbin/homelab-capacity-risk\n/usr/local/sbin/homelab-change-transaction\n/usr/local/sbin/homelab-chat-run\n/usr/local/sbin/homelab-edge-vm-offhost-freshness\n/usr/local/sbin/homelab-extended-appbackup-run\n/usr/local/sbin/homelab-golden-state-extended-overlay\n/usr/local/sbin/homelab-golden-state-index\n/usr/local/sbin/homelab-health-metrics\n/usr/local/sbin/homelab-ingress-standby-refresh\n/usr/local/sbin/homelab-mail-cloud-critical-upload\n/usr/local/sbin/homelab-mail-cloud-critical-upload.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload\n/usr/local/sbin/homelab-mail-cloud-restore-drill\n/usr/local/sbin/homelab-mail-cloud-restore-drill.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-rclone-cat-chunked\n/usr/local/sbin/homelab-rclone-cat-flat-chunked\n/usr/local/sbin/homelab-rclone-copy-chunked\n/usr/local/sbin/homelab-rclone-copy-flat-chunked\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh\n/usr/local/sbin/homelab-rclone-required-check.sh\n/usr/local/sbin/homelab-router-hard-power-cycle\n/usr/local/sbin/homelab-router-running-config-mail-cloud\n/usr/local/sbin/homelab-router-running-config-mail-cloud.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-safe-exec\n/usr/local/sbin/homelab-secret-exposure-guard\n/usr/local/sbin/homelab-stage-2cloud\n/usr/local/sbin/homelab-stage-2cloud.bak-global-chunkonly-20260710T011036Z\n/usr/local/sbin/homelab-stage-2cloud-flat\n/usr/local/sbin/homelab-stage-2cloud-flat-canary\n/usr/local/sbin/homelab-stage-verify-chunked\n/usr/local/sbin/homelab-stage-verify-flat-chunked\n/usr/local/sbin/homelab-vm160-remote-backup\n/usr/local/sbin/homelab-vm-backup-health\n/usr/local/sbin/homelab-vm-backup-policy\n/usr/local/sbin/homelab-vm-full-backup\n/usr/local/sbin/homelab-vm-full-backup.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-vm-full-restore-drill\n/usr/local/sbin/homelab-vm-full-restore-drill.bak.20260710T080231Z\n/usr/local/sbin/homelab-vm-full-restore-drill.bak-docker-check-v2-20260710T050044Z\n/usr/local/sbin/homelab-vm-full-restore-drill.bak-isolated-ip-20260710T044331Z\n/usr/local/sbin/homelab-vm-full-retention\n/usr/local/sbin/homelab-vm-local-dumps-2cloud\n/usr/local/sbin/homelab-vm-local-dumps-retention\n/usr/local/sbin/skladchik-reports-monitor-backup\n/usr/local/sbin/skladchik-reports-monitor-backup.pre-restore-check-20260710T065311Z.bak\n/usr/local/sbin/skladchik-reports-monitor-backup.pre-session-keepalive-20260710T071605Z.bak\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check.pre-session-keepalive-20260710T071605Z.bak\n=== LOCAL_BACKUPS ===\n7022386770\t2026-07-18T03:31:03.6566057240\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.vma.zst\n7021952392\t2026-07-21T03:31:06.5717414950\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst\n7021297699\t2026-07-19T03:31:06.8755351020\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst\n7016275481\t2026-07-20T03:31:03.2165123470\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst\n329541843\t2026-07-21T03:30:09.7688696330\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst\n326601118\t2026-07-20T03:30:08.2096782720\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst\n323668231\t2026-07-19T03:45:46.2209130400\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst\n321121649\t2026-07-18T03:30:08.0777625900\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.tar.zst\n222777732\t2026-07-21T03:30:15.9059638290\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst\n222761273\t2026-07-20T03:30:13.7937629440\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst\n222745965\t2026-07-19T03:45:57.2700790730\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst\n222727760\t2026-07-18T03:30:13.5578457230\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.tar.zst\n3021\t2026-07-19T03:31:07.4745284120\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.log\n3020\t2026-07-21T03:31:07.3945472590\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.log\n3018\t2026-07-18T03:31:03.9197988860\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.log\n3017\t2026-07-20T03:31:04.1706719350\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.log\n1089\t2026-07-20T03:30:14.0079985330\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.log\n1086\t2026-07-20T03:30:08.4640241540\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.log\n989\t2026-07-21T03:30:16.1364976560\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.log\n989\t2026-07-18T03:30:13.7816006690\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.log\n986\t2026-07-21T03:30:10.0285327490\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.log\n986\t2026-07-18T03:30:08.3002411830\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.log\n731\t2026-07-19T03:45:57.5197000270\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.log\n729\t2026-07-19T03:45:46.4444121980\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.log\n" + "output": "=== EXACT_IMMUTABLE_RESULT ===\n{\n \"schema_version\": 1,\n \"channel\": \"homelab-runtime\",\n \"command_id\": \"BACKUP-DUAL-CLOUD-PREFLIGHT-059\",\n \"status\": \"FAIL\",\n \"rc\": 1,\n \"host\": \"pve01\",\n \"mode\": \"read-only\",\n \"component\": \"backup-audit\",\n \"started_at_utc\": \"2026-07-21T11:35:22Z\",\n \"finished_at_utc\": \"2026-07-21T11:35:23Z\",\n \"reference_register_checked\": true,\n \"reference_sha256\": \"5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e\",\n \"error_register_checked\": true,\n \"error_register_sha256\": \"f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1\",\n \"command_sha256\": \"c6f4134d15a9f641ddaf67535760d0abede973dd08222a37af05612a6f580479\",\n \"duplicate_failed_command_blocked\": false,\n \"block_reason\": null,\n \"execution_started\": true,\n \"changes_made\": false,\n \"sanitized\": true,\n \"secrets_included\": false,\n \"private_addresses_included\": false,\n \"raw_evidence_retained_locally\": true,\n \"raw_evidence_sha256\": \"f375162905664fc3de53bec3ea2b2485775e05bcfbd421a3c63010def946a753\",\n \"sanitized_output_sha256\": \"f375162905664fc3de53bec3ea2b2485775e05bcfbd421a3c63010def946a753\",\n \"output_truncated_in_json\": false,\n \"full_sanitized_output_url\": \"https://git.gram1.ru/.well-known/homelab-runtime/latest.txt\",\n \"output\": \"=== BACKUP_JOBS ===\\n[{\\\"compress\\\":\\\"zstd\\\",\\\"enabled\\\":1,\\\"id\\\":\\\"homelab-nightly-all\\\",\\\"mailnotification\\\":\\\"failure\\\",\\\"mode\\\":\\\"snapshot\\\",\\\"next-run\\\":1784680200,\\\"prune-backups\\\":{\\\"keep-daily\\\":\\\"4\\\"},\\\"schedule\\\":\\\"03:30\\\",\\\"storage\\\":\\\"local\\\",\\\"type\\\":\\\"vzdump\\\",\\\"vmid\\\":\\\"110,111,112,113,130,150,190\\\"}]\\n=== RCLONE_REMOTES ===\\npve01-mail-01:\\npve01-mail-01-crypt:\\npve01-mail-02:\\npve01-mail-02-crypt:\\npve01-mail-03:\\npve01-mail-03-crypt:\\npve01-mail-04:\\npve01-mail-04-crypt:\\npve01-mail-05:\\npve01-mail-05-crypt:\\npve02-mail-01:\\npve02-mail-01-crypt:\\npve02-mail-02:\\npve02-mail-02-crypt:\\npve02-mail-03:\\npve02-mail-03-crypt:\\npve02-mail-04:\\npve02-mail-04-crypt:\\npve03-mail-01:\\npve03-mail-01-crypt:\\npve03-mail-02:\\npve03-mail-02-crypt:\\npve03-mail-03:\\npve03-mail-03-crypt:\\npve03-mail-04:\\npve03-mail-04-crypt:\\n=== BACKUP_TIMERS ===\\nWed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:14 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service\\nWed 2026-07-22 00:09:43 MSK 9h Tue 2026-07-21 00:09:32 MSK 14h ago homelab-backup-freshness.timer homelab-backup-freshness.service\\nWed 2026-07-22 02:15:00 MSK 11h Tue 2026-07-21 02:15:05 MSK 12h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service\\nWed 2026-07-22 03:10:02 MSK 12h Tue 2026-07-21 03:13:14 MSK 11h ago homelab-emergency-backup.timer homelab-emergency-backup.service\\nWed 2026-07-22 03:35:00 MSK 12h Tue 2026-07-21 03:35:14 MSK 11h ago filebrowser-backup.timer filebrowser-backup.service\\nWed 2026-07-22 03:55:02 MSK 13h Tue 2026-07-21 03:51:25 MSK 10h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service\\nWed 2026-07-22 04:17:29 MSK 13h Tue 2026-07-21 04:19:17 MSK 10h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service\\nWed 2026-07-22 04:26:54 MSK 13h Tue 2026-07-21 04:28:14 MSK 10h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service\\nWed 2026-07-22 04:38:04 MSK 14h Tue 2026-07-21 04:34:48 MSK 10h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service\\nWed 2026-07-22 05:01:44 MSK 14h Tue 2026-07-21 04:57:14 MSK 9h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service\\nWed 2026-07-22 05:23:56 MSK 14h Tue 2026-07-21 05:17:57 MSK 9h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service\\nWed 2026-07-22 05:27:13 MSK 14h Tue 2026-07-21 05:29:05 MSK 9h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service\\nWed 2026-07-22 05:47:19 MSK 15h Tue 2026-07-21 05:46:14 MSK 8h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service\\nWed 2026-07-22 06:01:19 MSK 15h Tue 2026-07-21 05:57:30 MSK 8h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service\\nWed 2026-07-22 06:27:43 MSK 15h Tue 2026-07-21 06:32:25 MSK 8h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service\\nWed 2026-07-22 06:38:51 MSK 16h Tue 2026-07-21 06:43:54 MSK 7h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service\\nWed 2026-07-22 06:39:22 MSK 16h Tue 2026-07-21 06:40:25 MSK 7h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service\\nWed 2026-07-22 06:44:16 MSK 16h Tue 2026-07-21 06:40:25 MSK 7h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service\\nWed 2026-07-22 06:52:06 MSK 16h Tue 2026-07-21 06:47:47 MSK 7h ago homelab-beszel-backup.timer homelab-beszel-backup.service\\nWed 2026-07-22 06:58:58 MSK 16h Tue 2026-07-21 06:57:30 MSK 7h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service\\nWed 2026-07-22 07:02:27 MSK 16h Tue 2026-07-21 07:05:30 MSK 7h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service\\nWed 2026-07-22 07:07:14 MSK 16h Tue 2026-07-21 07:18:55 MSK 7h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service\\nWed 2026-07-22 07:35:06 MSK 16h Tue 2026-07-21 07:36:59 MSK 6h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service\\nWed 2026-07-22 07:38:21 MSK 17h Tue 2026-07-21 07:36:14 MSK 6h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service\\nWed 2026-07-22 07:41:49 MSK 17h Tue 2026-07-21 07:53:41 MSK 6h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service\\nWed 2026-07-22 08:10:37 MSK 17h Tue 2026-07-21 08:07:25 MSK 6h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service\\nWed 2026-07-22 08:37:02 MSK 18h Tue 2026-07-21 08:30:07 MSK 6h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service\\nWed 2026-07-22 08:40:13 MSK 18h Tue 2026-07-21 08:50:14 MSK 5h 45min ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service\\nWed 2026-07-22 08:53:36 MSK 18h Tue 2026-07-21 08:41:14 MSK 5h 54min ago memos-backup.timer memos-backup.service\\nSun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service\\nSun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service\\nSun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service\\nSun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service\\nMon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 7h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service\\n=== BACKUP_UNITS ===\\ndpkg-db-backup.service static -\\nfilebrowser-backup.service static -\\nhomelab-alertmanager-backup.service static -\\nhomelab-appbackup-dockge-stacks.service static -\\nhomelab-appbackup-mariadb-batch.service static -\\nhomelab-appbackup-npmplus-edge.service static -\\nhomelab-appbackup-postgres-batch.service static -\\nhomelab-appbackup-sqlite-batch.service static -\\nhomelab-authentik-backup.service static -\\nhomelab-auto-backup.service static -\\nhomelab-backup-coverage-matrix.service static -\\nhomelab-backup-freshness.service static -\\nhomelab-backup-healthcheck.service static -\\nhomelab-backup-matrix-refresh.service static -\\nhomelab-backup-sla-health.service static -\\nhomelab-beszel-backup.service static -\\nhomelab-blackbox-exporter-backup.service static -\\nhomelab-cluster-admin-incident-engine-backup.service static -\\nhomelab-crypto-portfolio-backup.service static -\\nhomelab-edge-vm-vzdump-backup.service static -\\nhomelab-emergency-backup.service static -\\nhomelab-extended-appbackup.service static -\\nhomelab-gitea-backup.service static -\\nhomelab-gitea-secondary-backup.service static -\\nhomelab-legacy-monitoring-backups.service static -\\nhomelab-mail-cloud-critical-upload.service static -\\nhomelab-mail-cloud-edge-vm-upload.service static -\\nhomelab-mail-cloud-nextcloud-vm-upload.service static -\\nhomelab-mail-cloud-restore-drill.service static -\\nhomelab-npmplus-kuma-config-backup.service static -\\nhomelab-router-running-config-mail-cloud.service static -\\nhomelab-vaultwarden-backup.service static -\\nhomelab-verified-backup-health.service static -\\nhomelab-vm-backup-health.service static -\\nhomelab-vm-backup-policy.service static -\\nhomelab-vm-full-backup.service static -\\nhomelab-vm-local-dumps-2cloud.service static -\\nhomelab-vm160-remote-backup.service static -\\nimmich-media-backup.service static -\\nmemos-backup.service static -\\nnetbird-vps-backup.service static -\\npaperless-backup.service static -\\nskladchik-reports-monitor-backup-restore-check.service static -\\nskladchik-reports-monitor-backup.service static -\\ndpkg-db-backup.timer enabled enabled\\nfilebrowser-backup.timer enabled enabled\\nhomelab-alertmanager-backup.timer enabled enabled\\nhomelab-appbackup-dockge-stacks.timer enabled enabled\\nhomelab-appbackup-mariadb-batch.timer enabled enabled\\nhomelab-appbackup-npmplus-edge.timer enabled enabled\\nhomelab-appbackup-postgres-batch.timer enabled enabled\\nhomelab-appbackup-sqlite-batch.timer enabled enabled\\nhomelab-authentik-backup.timer disabled enabled\\nhomelab-auto-backup.timer enabled enabled\\nhomelab-backup-coverage-matrix.timer enabled enabled\\nhomelab-backup-freshness.timer enabled enabled\\nhomelab-backup-healthcheck.timer enabled enabled\\nhomelab-backup-matrix-refresh.timer disabled enabled\\nhomelab-backup-sla-health.timer enabled enabled\\nhomelab-beszel-backup.timer enabled enabled\\nhomelab-blackbox-exporter-backup.timer enabled enabled\\nhomelab-cluster-admin-incident-engine-backup.timer enabled enabled\\nhomelab-crypto-portfolio-backup.timer enabled enabled\\nhomelab-edge-vm-vzdump-backup.timer disabled enabled\\nhomelab-emergency-backup.timer enabled enabled\\nhomelab-extended-appbackup.timer enabled enabled\\nhomelab-gitea-backup.timer disabled enabled\\nhomelab-gitea-secondary-backup.timer enabled enabled\\nhomelab-legacy-monitoring-backups.timer enabled enabled\\nhomelab-mail-cloud-critical-upload.timer enabled enabled\\nhomelab-mail-cloud-edge-vm-upload.timer enabled enabled\\nhomelab-mail-cloud-nextcloud-vm-upload.timer enabled enabled\\nhomelab-mail-cloud-restore-drill.timer enabled enabled\\nhomelab-npmplus-kuma-config-backup.timer disabled enabled\\nhomelab-router-running-config-mail-cloud.timer enabled enabled\\nhomelab-vaultwarden-backup.timer disabled enabled\\nhomelab-verified-backup-health.timer enabled enabled\\nhomelab-vm-backup-health.timer enabled enabled\\nhomelab-vm-backup-policy.timer enabled enabled\\nhomelab-vm-full-backup.timer enabled enabled\\nhomelab-vm-local-dumps-2cloud.timer enabled enabled\\nhomelab-vm160-remote-backup.timer disabled enabled\\nimmich-media-backup.timer disabled enabled\\nmemos-backup.timer enabled enabled\\nnetbird-vps-backup.timer disabled enabled\\npaperless-backup.timer disabled enabled\\nskladchik-reports-monitor-backup-restore-check.timer enabled enabled\\nskladchik-reports-monitor-backup.timer enabled enabled\\n=== PIPELINE_FILES ===\\n/etc/cron.d/vzdump\\n/etc/systemd/system/homelab-edge-vm-offhost-freshness.timer\\n/etc/systemd/system/homelab-edge-vm-vzdump-backup.service\\n/etc/systemd/system/homelab-edge-vm-vzdump-backup.timer\\n/etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.service\\n/etc/systemd/system/homelab-vm-full-backup.service\\n/etc/systemd/system/multi-user.target.wants/pve-guests.service\\n/etc/systemd/system/pve-manager.service\\n/etc/systemd/system/timers.target.wants/homelab-edge-vm-offhost-freshness.timer\\n/usr/local/sbin/appbackup-dockge-stacks\\n/usr/local/sbin/appbackup-dockge-stacks.bak-chunkall-20260710T011410Z\\n/usr/local/sbin/appbackup-restore-check\\n/usr/local/sbin/appbackup-restore-check.bak-chunkall-20260710T011410Z\\n/usr/local/sbin/appbackup-restore-check.bak-dual-read-20260711T051924Z\\n/usr/local/sbin/appbackup-restore-check.bak-flat-read-20260711T052854Z\\n/usr/local/sbin/appbackup-restore-check.bak-flat-read-v2-20260711T053129Z\\n/usr/local/sbin/appbackup-retention\\n/usr/local/sbin/appbackup-retention.bak-dual-read-20260711T051924Z\\n/usr/local/sbin/backupctl\\n/usr/local/sbin/backup-restore-check\\n/usr/local/sbin/backup-restore-check.bak-chunkall-20260710T011410Z\\n/usr/local/sbin/backup-retention\\n/usr/local/sbin/homelab-backup-edge-vm-vzdump\\n/usr/local/sbin/homelab-capacity-risk\\n/usr/local/sbin/homelab-change-transaction\\n/usr/local/sbin/homelab-chat-run\\n/usr/local/sbin/homelab-edge-vm-offhost-freshness\\n/usr/local/sbin/homelab-extended-appbackup-run\\n/usr/local/sbin/homelab-golden-state-extended-overlay\\n/usr/local/sbin/homelab-golden-state-index\\n/usr/local/sbin/homelab-health-metrics\\n/usr/local/sbin/homelab-ingress-standby-refresh\\n/usr/local/sbin/homelab-mail-cloud-critical-upload\\n/usr/local/sbin/homelab-mail-cloud-critical-upload.bak-chunkall-20260710T011410Z\\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload\\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload\\n/usr/local/sbin/homelab-mail-cloud-restore-drill\\n/usr/local/sbin/homelab-mail-cloud-restore-drill.bak-chunkall-20260710T011410Z\\n/usr/local/sbin/homelab-rclone-cat-chunked\\n/usr/local/sbin/homelab-rclone-cat-flat-chunked\\n/usr/local/sbin/homelab-rclone-copy-chunked\\n/usr/local/sbin/homelab-rclone-copy-flat-chunked\\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh\\n/usr/local/sbin/homelab-rclone-required-check.sh\\n/usr/local/sbin/homelab-router-hard-power-cycle\\n/usr/local/sbin/homelab-router-running-config-mail-cloud\\n/usr/local/sbin/homelab-router-running-config-mail-cloud.bak-chunkall-20260710T011410Z\\n/usr/local/sbin/homelab-safe-exec\\n/usr/local/sbin/homelab-secret-exposure-guard\\n/usr/local/sbin/homelab-stage-2cloud\\n/usr/local/sbin/homelab-stage-2cloud.bak-global-chunkonly-20260710T011036Z\\n/usr/local/sbin/homelab-stage-2cloud-flat\\n/usr/local/sbin/homelab-stage-2cloud-flat-canary\\n/usr/local/sbin/homelab-stage-verify-chunked\\n/usr/local/sbin/homelab-stage-verify-flat-chunked\\n/usr/local/sbin/homelab-vm160-remote-backup\\n/usr/local/sbin/homelab-vm-backup-health\\n/usr/local/sbin/homelab-vm-backup-policy\\n/usr/local/sbin/homelab-vm-full-backup\\n/usr/local/sbin/homelab-vm-full-backup.bak-chunkall-20260710T011410Z\\n/usr/local/sbin/homelab-vm-full-restore-drill\\n/usr/local/sbin/homelab-vm-full-restore-drill.bak.20260710T080231Z\\n/usr/local/sbin/homelab-vm-full-restore-drill.bak-docker-check-v2-20260710T050044Z\\n/usr/local/sbin/homelab-vm-full-restore-drill.bak-isolated-ip-20260710T044331Z\\n/usr/local/sbin/homelab-vm-full-retention\\n/usr/local/sbin/homelab-vm-local-dumps-2cloud\\n/usr/local/sbin/homelab-vm-local-dumps-retention\\n/usr/local/sbin/skladchik-reports-monitor-backup\\n/usr/local/sbin/skladchik-reports-monitor-backup.pre-restore-check-20260710T065311Z.bak\\n/usr/local/sbin/skladchik-reports-monitor-backup.pre-session-keepalive-20260710T071605Z.bak\\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check\\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check.pre-session-keepalive-20260710T071605Z.bak\\n=== LOCAL_BACKUPS ===\\n7022386770\\t2026-07-18T03:31:03.6566057240\\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.vma.zst\\n7021952392\\t2026-07-21T03:31:06.5717414950\\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst\\n7021297699\\t2026-07-19T03:31:06.8755351020\\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst\\n7016275481\\t2026-07-20T03:31:03.2165123470\\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst\\n329541843\\t2026-07-21T03:30:09.7688696330\\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst\\n326601118\\t2026-07-20T03:30:08.2096782720\\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst\\n323668231\\t2026-07-19T03:45:46.2209130400\\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst\\n321121649\\t2026-07-18T03:30:08.0777625900\\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.tar.zst\\n222777732\\t2026-07-21T03:30:15.9059638290\\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst\\n222761273\\t2026-07-20T03:30:13.7937629440\\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst\\n222745965\\t2026-07-19T03:45:57.2700790730\\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst\\n222727760\\t2026-07-18T03:30:13.5578457230\\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.tar.zst\\n3021\\t2026-07-19T03:31:07.4745284120\\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.log\\n3020\\t2026-07-21T03:31:07.3945472590\\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.log\\n3018\\t2026-07-18T03:31:03.9197988860\\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.log\\n3017\\t2026-07-20T03:31:04.1706719350\\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.log\\n1089\\t2026-07-20T03:30:14.0079985330\\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.log\\n1086\\t2026-07-20T03:30:08.4640241540\\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.log\\n989\\t2026-07-21T03:30:16.1364976560\\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.log\\n989\\t2026-07-18T03:30:13.7816006690\\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.log\\n986\\t2026-07-21T03:30:10.0285327490\\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.log\\n986\\t2026-07-18T03:30:08.3002411830\\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.log\\n731\\t2026-07-19T03:45:57.5197000270\\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.log\\n729\\t2026-07-19T03:45:46.4444121980\\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.log\\n\"\n}\n\nEXACT_GITEA=FOUND\n=== LOCAL_RAW_RESULT ===\nFILE=/var/lib/homelab-chat-run/raw/BACKUP-DUAL-CLOUD-PREFLIGHT-059-20260721T113522Z.log SIZE=21897 MTIME=2026-07-21 14:35:23.197206410 +0300\n=== BACKUP_JOBS ===\n[{\"compress\":\"zstd\",\"enabled\":1,\"id\":\"homelab-nightly-all\",\"mailnotification\":\"failure\",\"mode\":\"snapshot\",\"next-run\":1784680200,\"prune-backups\":{\"keep-daily\":\"4\"},\"schedule\":\"03:30\",\"storage\":\"local\",\"type\":\"vzdump\",\"vmid\":\"110,111,112,113,130,150,190\"}]\n=== RCLONE_REMOTES ===\npve01-mail-01:\npve01-mail-01-crypt:\npve01-mail-02:\npve01-mail-02-crypt:\npve01-mail-03:\npve01-mail-03-crypt:\npve01-mail-04:\npve01-mail-04-crypt:\npve01-mail-05:\npve01-mail-05-crypt:\npve02-mail-01:\npve02-mail-01-crypt:\npve02-mail-02:\npve02-mail-02-crypt:\npve02-mail-03:\npve02-mail-03-crypt:\npve02-mail-04:\npve02-mail-04-crypt:\npve03-mail-01:\npve03-mail-01-crypt:\npve03-mail-02:\npve03-mail-02-crypt:\npve03-mail-03:\npve03-mail-03-crypt:\npve03-mail-04:\npve03-mail-04-crypt:\n=== BACKUP_TIMERS ===\nWed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:14 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service\nWed 2026-07-22 00:09:43 MSK 9h Tue 2026-07-21 00:09:32 MSK 14h ago homelab-backup-freshness.timer homelab-backup-freshness.service\nWed 2026-07-22 02:15:00 MSK 11h Tue 2026-07-21 02:15:05 MSK 12h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service\nWed 2026-07-22 03:10:02 MSK 12h Tue 2026-07-21 03:13:14 MSK 11h ago homelab-emergency-backup.timer homelab-emergency-backup.service\nWed 2026-07-22 03:35:00 MSK 12h Tue 2026-07-21 03:35:14 MSK 11h ago filebrowser-backup.timer filebrowser-backup.service\nWed 2026-07-22 03:55:02 MSK 13h Tue 2026-07-21 03:51:25 MSK 10h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service\nWed 2026-07-22 04:17:29 MSK 13h Tue 2026-07-21 04:19:17 MSK 10h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service\nWed 2026-07-22 04:26:54 MSK 13h Tue 2026-07-21 04:28:14 MSK 10h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service\nWed 2026-07-22 04:38:04 MSK 14h Tue 2026-07-21 04:34:48 MSK 10h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service\nWed 2026-07-22 05:01:44 MSK 14h Tue 2026-07-21 04:57:14 MSK 9h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service\nWed 2026-07-22 05:23:56 MSK 14h Tue 2026-07-21 05:17:57 MSK 9h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service\nWed 2026-07-22 05:27:13 MSK 14h Tue 2026-07-21 05:29:05 MSK 9h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service\nWed 2026-07-22 05:47:19 MSK 15h Tue 2026-07-21 05:46:14 MSK 8h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service\nWed 2026-07-22 06:01:19 MSK 15h Tue 2026-07-21 05:57:30 MSK 8h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service\nWed 2026-07-22 06:27:43 MSK 15h Tue 2026-07-21 06:32:25 MSK 8h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service\nWed 2026-07-22 06:38:51 MSK 16h Tue 2026-07-21 06:43:54 MSK 7h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service\nWed 2026-07-22 06:39:22 MSK 16h Tue 2026-07-21 06:40:25 MSK 7h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service\nWed 2026-07-22 06:44:16 MSK 16h Tue 2026-07-21 06:40:25 MSK 7h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service\nWed 2026-07-22 06:52:06 MSK 16h Tue 2026-07-21 06:47:47 MSK 7h ago homelab-beszel-backup.timer homelab-beszel-backup.service\nWed 2026-07-22 06:58:58 MSK 16h Tue 2026-07-21 06:57:30 MSK 7h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service\nWed 2026-07-22 07:02:27 MSK 16h Tue 2026-07-21 07:05:30 MSK 7h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service\nWed 2026-07-22 07:07:14 MSK 16h Tue 2026-07-21 07:18:55 MSK 7h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service\nWed 2026-07-22 07:35:06 MSK 16h Tue 2026-07-21 07:36:59 MSK 6h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service\nWed 2026-07-22 07:38:21 MSK 17h Tue 2026-07-21 07:36:14 MSK 6h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service\nWed 2026-07-22 07:41:49 MSK 17h Tue 2026-07-21 07:53:41 MSK 6h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service\nWed 2026-07-22 08:10:37 MSK 17h Tue 2026-07-21 08:07:25 MSK 6h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service\nWed 2026-07-22 08:37:02 MSK 18h Tue 2026-07-21 08:30:07 MSK 6h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service\nWed 2026-07-22 08:40:13 MSK 18h Tue 2026-07-21 08:50:14 MSK 5h 45min ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service\nWed 2026-07-22 08:53:36 MSK 18h Tue 2026-07-21 08:41:14 MSK 5h 54min ago memos-backup.timer memos-backup.service\nSun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service\nSun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service\nSun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service\nSun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service\nMon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 7h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service\n=== BACKUP_UNITS ===\ndpkg-db-backup.service static -\nfilebrowser-backup.service static -\nhomelab-alertmanager-backup.service static -\nhomelab-appbackup-dockge-stacks.service static -\nhomelab-appbackup-mariadb-batch.service static -\nhomelab-appbackup-npmplus-edge.service static -\nhomelab-appbackup-postgres-batch.service static -\nhomelab-appbackup-sqlite-batch.service static -\nhomelab-authentik-backup.service static -\nhomelab-auto-backup.service static -\nhomelab-backup-coverage-matrix.service static -\nhomelab-backup-freshness.service static -\nhomelab-backup-healthcheck.service static -\nhomelab-backup-matrix-refresh.service static -\nhomelab-backup-sla-health.service static -\nhomelab-beszel-backup.service static -\nhomelab-blackbox-exporter-backup.service static -\nhomelab-cluster-admin-incident-engine-backup.service static -\nhomelab-crypto-portfolio-backup.service static -\nhomelab-edge-vm-vzdump-backup.service static -\nhomelab-emergency-backup.service static -\nhomelab-extended-appbackup.service static -\nhomelab-gitea-backup.service static -\nhomelab-gitea-secondary-backup.service static -\nhomelab-legacy-monitoring-backups.service static -\nhomelab-mail-cloud-critical-upload.service static -\nhomelab-mail-cloud-edge-vm-upload.service static -\nhomelab-mail-cloud-nextcloud-vm-upload.service static -\nhomelab-mail-cloud-restore-drill.service static -\nhomelab-npmplus-kuma-config-backup.service static -\nhomelab-router-running-config-mail-cloud.service static -\nhomelab-vaultwarden-backup.service static -\nhomelab-verified-backup-health.service static -\nhomelab-vm-backup-health.service static -\nhomelab-vm-backup-policy.service static -\nhomelab-vm-full-backup.service static -\nhomelab-vm-local-dumps-2cloud.service static -\nhomelab-vm160-remote-backup.service static -\nimmich-media-backup.service static -\nmemos-backup.service static -\nnetbird-vps-backup.service static -\npaperless-backup.service static -\nskladchik-reports-monitor-backup-restore-check.service static -\nskladchik-reports-monitor-backup.service static -\ndpkg-db-backup.timer enabled enabled\nfilebrowser-backup.timer enabled enabled\nhomelab-alertmanager-backup.timer enabled enabled\nhomelab-appbackup-dockge-stacks.timer enabled enabled\nhomelab-appbackup-mariadb-batch.timer enabled enabled\nhomelab-appbackup-npmplus-edge.timer enabled enabled\nhomelab-appbackup-postgres-batch.timer enabled enabled\nhomelab-appbackup-sqlite-batch.timer enabled enabled\nhomelab-authentik-backup.timer disabled enabled\nhomelab-auto-backup.timer enabled enabled\nhomelab-backup-coverage-matrix.timer enabled enabled\nhomelab-backup-freshness.timer enabled enabled\nhomelab-backup-healthcheck.timer enabled enabled\nhomelab-backup-matrix-refresh.timer disabled enabled\nhomelab-backup-sla-health.timer enabled enabled\nhomelab-beszel-backup.timer enabled enabled\nhomelab-blackbox-exporter-backup.timer enabled enabled\nhomelab-cluster-admin-incident-engine-backup.timer enabled enabled\nhomelab-crypto-portfolio-backup.timer enabled enabled\nhomelab-edge-vm-vzdump-backup.timer disabled enabled\nhomelab-emergency-backup.timer enabled enabled\nhomelab-extended-appbackup.timer enabled enabled\nhomelab-gitea-backup.timer disabled enabled\nhomelab-gitea-secondary-backup.timer enabled enabled\nhomelab-legacy-monitoring-backups.timer enabled enabled\nhomelab-mail-cloud-critical-upload.timer enabled enabled\nhomelab-mail-cloud-edge-vm-upload.timer enabled enabled\nhomelab-mail-cloud-nextcloud-vm-upload.timer enabled enabled\nhomelab-mail-cloud-restore-drill.timer enabled enabled\nhomelab-npmplus-kuma-config-backup.timer disabled enabled\nhomelab-router-running-config-mail-cloud.timer enabled enabled\nhomelab-vaultwarden-backup.timer disabled enabled\nhomelab-verified-backup-health.timer enabled enabled\nhomelab-vm-backup-health.timer enabled enabled\nhomelab-vm-backup-policy.timer enabled enabled\nhomelab-vm-full-backup.timer enabled enabled\nhomelab-vm-local-dumps-2cloud.timer enabled enabled\nhomelab-vm160-remote-backup.timer disabled enabled\nimmich-media-backup.timer disabled enabled\nmemos-backup.timer enabled enabled\nnetbird-vps-backup.timer disabled enabled\npaperless-backup.timer disabled enabled\nskladchik-reports-monitor-backup-restore-check.timer enabled enabled\nskladchik-reports-monitor-backup.timer enabled enabled\n=== PIPELINE_FILES ===\n/etc/cron.d/vzdump\n/etc/systemd/system/homelab-edge-vm-offhost-freshness.timer\n/etc/systemd/system/homelab-edge-vm-vzdump-backup.service\n/etc/systemd/system/homelab-edge-vm-vzdump-backup.timer\n/etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.service\n/etc/systemd/system/homelab-vm-full-backup.service\n/etc/systemd/system/multi-user.target.wants/pve-guests.service\n/etc/systemd/system/pve-manager.service\n/etc/systemd/system/timers.target.wants/homelab-edge-vm-offhost-freshness.timer\n/usr/local/sbin/appbackup-dockge-stacks\n/usr/local/sbin/appbackup-dockge-stacks.bak-chunkall-20260710T011410Z\n/usr/local/sbin/appbackup-restore-check\n/usr/local/sbin/appbackup-restore-check.bak-chunkall-20260710T011410Z\n/usr/local/sbin/appbackup-restore-check.bak-dual-read-20260711T051924Z\n/usr/local/sbin/appbackup-restore-check.bak-flat-read-20260711T052854Z\n/usr/local/sbin/appbackup-restore-check.bak-flat-read-v2-20260711T053129Z\n/usr/local/sbin/appbackup-retention\n/usr/local/sbin/appbackup-retention.bak-dual-read-20260711T051924Z\n/usr/local/sbin/backupctl\n/usr/local/sbin/backup-restore-check\n/usr/local/sbin/backup-restore-check.bak-chunkall-20260710T011410Z\n/usr/local/sbin/backup-retention\n/usr/local/sbin/homelab-backup-edge-vm-vzdump\n/usr/local/sbin/homelab-capacity-risk\n/usr/local/sbin/homelab-change-transaction\n/usr/local/sbin/homelab-chat-run\n/usr/local/sbin/homelab-edge-vm-offhost-freshness\n/usr/local/sbin/homelab-extended-appbackup-run\n/usr/local/sbin/homelab-golden-state-extended-overlay\n/usr/local/sbin/homelab-golden-state-index\n/usr/local/sbin/homelab-health-metrics\n/usr/local/sbin/homelab-ingress-standby-refresh\n/usr/local/sbin/homelab-mail-cloud-critical-upload\n/usr/local/sbin/homelab-mail-cloud-critical-upload.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload\n/usr/local/sbin/homelab-mail-cloud-restore-drill\n/usr/local/sbin/homelab-mail-cloud-restore-drill.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-rclone-cat-chunked\n/usr/local/sbin/homelab-rclone-cat-flat-chunked\n/usr/local/sbin/homelab-rclone-copy-chunked\n/usr/local/sbin/homelab-rclone-copy-flat-chunked\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh\n/usr/local/sbin/homelab-rclone-required-check.sh\n/usr/local/sbin/homelab-router-hard-power-cycle\n/usr/local/sbin/homelab-router-running-config-mail-cloud\n/usr/local/sbin/homelab-router-running-config-mail-cloud.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-safe-exec\n/usr/local/sbin/homelab-secret-exposure-guard\n/usr/local/sbin/homelab-stage-2cloud\n/usr/local/sbin/homelab-stage-2cloud.bak-global-chunkonly-20260710T011036Z\n/usr/local/sbin/homelab-stage-2cloud-flat\n/usr/local/sbin/homelab-stage-2cloud-flat-canary\n/usr/local/sbin/homelab-stage-verify-chunked\n/usr/local/sbin/homelab-stage-verify-flat-chunked\n/usr/local/sbin/homelab-vm160-remote-backup\n/usr/local/sbin/homelab-vm-backup-health\n/usr/local/sbin/homelab-vm-backup-policy\n/usr/local/sbin/homelab-vm-full-backup\n/usr/local/sbin/homelab-vm-full-backup.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-vm-full-restore-drill\n/usr/local/sbin/homelab-vm-full-restore-drill.bak.20260710T080231Z\n/usr/local/sbin/homelab-vm-full-restore-drill.bak-docker-check-v2-20260710T050044Z\n/usr/local/sbin/homelab-vm-full-restore-drill.bak-isolated-ip-20260710T044331Z\n/usr/local/sbin/homelab-vm-full-retention\n/usr/local/sbin/homelab-vm-local-dumps-2cloud\n/usr/local/sbin/homelab-vm-local-dumps-retention\n/usr/local/sbin/skladchik-reports-monitor-backup\n/usr/local/sbin/skladchik-reports-monitor-backup.pre-restore-check-20260710T065311Z.bak\n/usr/local/sbin/skladchik-reports-monitor-backup.pre-session-keepalive-20260710T071605Z.bak\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check.pre-session-keepalive-20260710T071605Z.bak\n=== LOCAL_BACKUPS ===\n7022386770\t2026-07-18T03:31:03.6566057240\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.vma.zst\n7021952392\t2026-07-21T03:31:06.5717414950\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst\n7021297699\t2026-07-19T03:31:06.8755351020\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst\n7016275481\t2026-07-20T03:31:03.2165123470\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst\n329541843\t2026-07-21T03:30:09.7688696330\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst\n326601118\t2026-07-20T03:30:08.2096782720\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst\n323668231\t2026-07-19T03:45:46.2209130400\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst\n321121649\t2026-07-18T03:30:08.0777625900\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.tar.zst\n222777732\t2026-07-21T03:30:15.9059638290\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst\n222761273\t2026-07-20T03:30:13.7937629440\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst\n222745965\t2026-07-19T03:45:57.2700790730\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst\n222727760\t2026-07-18T03:30:13.5578457230\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.tar.zst\n3021\t2026-07-19T03:31:07.4745284120\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.log\n3020\t2026-07-21T03:31:07.3945472590\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.log\n3018\t2026-07-18T03:31:03.9197988860\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.log\n3017\t2026-07-20T03:31:04.1706719350\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.log\n1089\t2026-07-20T03:30:14.0079985330\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.log\n1086\t2026-07-20T03:30:08.4640241540\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.log\n989\t2026-07-21T03:30:16.1364976560\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.log\n989\t2026-07-18T03:30:13.7816006690\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.log\n986\t2026-07-21T03:30:10.0285327490\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.log\n986\t2026-07-18T03:30:08.3002411830\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.log\n731\t2026-07-19T03:45:57.5197000270\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.log\n729\t2026-07-19T03:45:46.4444121980\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.log\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index d1acdf4..b1ae9ba 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,7 +1,7 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=BACKUP-DUAL-CLOUD-PREFLIGHT-059 -STATUS=FAIL -RC=1 +COMMAND_ID=BACKUP-DUAL-CLOUD-RECONCILE-060 +STATUS=OK +RC=0 HOST=pve01 MODE=read-only COMPONENT=backup-audit @@ -9,16 +9,51 @@ REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1 -COMMAND_SHA256=c6f4134d15a9f641ddaf67535760d0abede973dd08222a37af05612a6f580479 +COMMAND_SHA256=be39adf6237697b760f095d503f1fdb9e25eed004c8108b2b1d338318d4794a6 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGES_MADE=false SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=f375162905664fc3de53bec3ea2b2485775e05bcfbd421a3c63010def946a753 -SANITIZED_OUTPUT_SHA256=f375162905664fc3de53bec3ea2b2485775e05bcfbd421a3c63010def946a753 +RAW_EVIDENCE_SHA256=63db4a98c9859a2dece1cd7a0f56c08584ac144e96fedc91c7d16a907335b8e9 +SANITIZED_OUTPUT_SHA256=63db4a98c9859a2dece1cd7a0f56c08584ac144e96fedc91c7d16a907335b8e9 OUTPUT_BEGIN +=== EXACT_IMMUTABLE_RESULT === +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "BACKUP-DUAL-CLOUD-PREFLIGHT-059", + "status": "FAIL", + "rc": 1, + "host": "pve01", + "mode": "read-only", + "component": "backup-audit", + "started_at_utc": "2026-07-21T11:35:22Z", + "finished_at_utc": "2026-07-21T11:35:23Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1", + "command_sha256": "c6f4134d15a9f641ddaf67535760d0abede973dd08222a37af05612a6f580479", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "f375162905664fc3de53bec3ea2b2485775e05bcfbd421a3c63010def946a753", + "sanitized_output_sha256": "f375162905664fc3de53bec3ea2b2485775e05bcfbd421a3c63010def946a753", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "=== BACKUP_JOBS ===\n[{\"compress\":\"zstd\",\"enabled\":1,\"id\":\"homelab-nightly-all\",\"mailnotification\":\"failure\",\"mode\":\"snapshot\",\"next-run\":1784680200,\"prune-backups\":{\"keep-daily\":\"4\"},\"schedule\":\"03:30\",\"storage\":\"local\",\"type\":\"vzdump\",\"vmid\":\"110,111,112,113,130,150,190\"}]\n=== RCLONE_REMOTES ===\npve01-mail-01:\npve01-mail-01-crypt:\npve01-mail-02:\npve01-mail-02-crypt:\npve01-mail-03:\npve01-mail-03-crypt:\npve01-mail-04:\npve01-mail-04-crypt:\npve01-mail-05:\npve01-mail-05-crypt:\npve02-mail-01:\npve02-mail-01-crypt:\npve02-mail-02:\npve02-mail-02-crypt:\npve02-mail-03:\npve02-mail-03-crypt:\npve02-mail-04:\npve02-mail-04-crypt:\npve03-mail-01:\npve03-mail-01-crypt:\npve03-mail-02:\npve03-mail-02-crypt:\npve03-mail-03:\npve03-mail-03-crypt:\npve03-mail-04:\npve03-mail-04-crypt:\n=== BACKUP_TIMERS ===\nWed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:14 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service\nWed 2026-07-22 00:09:43 MSK 9h Tue 2026-07-21 00:09:32 MSK 14h ago homelab-backup-freshness.timer homelab-backup-freshness.service\nWed 2026-07-22 02:15:00 MSK 11h Tue 2026-07-21 02:15:05 MSK 12h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service\nWed 2026-07-22 03:10:02 MSK 12h Tue 2026-07-21 03:13:14 MSK 11h ago homelab-emergency-backup.timer homelab-emergency-backup.service\nWed 2026-07-22 03:35:00 MSK 12h Tue 2026-07-21 03:35:14 MSK 11h ago filebrowser-backup.timer filebrowser-backup.service\nWed 2026-07-22 03:55:02 MSK 13h Tue 2026-07-21 03:51:25 MSK 10h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service\nWed 2026-07-22 04:17:29 MSK 13h Tue 2026-07-21 04:19:17 MSK 10h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service\nWed 2026-07-22 04:26:54 MSK 13h Tue 2026-07-21 04:28:14 MSK 10h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service\nWed 2026-07-22 04:38:04 MSK 14h Tue 2026-07-21 04:34:48 MSK 10h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service\nWed 2026-07-22 05:01:44 MSK 14h Tue 2026-07-21 04:57:14 MSK 9h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service\nWed 2026-07-22 05:23:56 MSK 14h Tue 2026-07-21 05:17:57 MSK 9h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service\nWed 2026-07-22 05:27:13 MSK 14h Tue 2026-07-21 05:29:05 MSK 9h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service\nWed 2026-07-22 05:47:19 MSK 15h Tue 2026-07-21 05:46:14 MSK 8h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service\nWed 2026-07-22 06:01:19 MSK 15h Tue 2026-07-21 05:57:30 MSK 8h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service\nWed 2026-07-22 06:27:43 MSK 15h Tue 2026-07-21 06:32:25 MSK 8h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service\nWed 2026-07-22 06:38:51 MSK 16h Tue 2026-07-21 06:43:54 MSK 7h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service\nWed 2026-07-22 06:39:22 MSK 16h Tue 2026-07-21 06:40:25 MSK 7h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service\nWed 2026-07-22 06:44:16 MSK 16h Tue 2026-07-21 06:40:25 MSK 7h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service\nWed 2026-07-22 06:52:06 MSK 16h Tue 2026-07-21 06:47:47 MSK 7h ago homelab-beszel-backup.timer homelab-beszel-backup.service\nWed 2026-07-22 06:58:58 MSK 16h Tue 2026-07-21 06:57:30 MSK 7h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service\nWed 2026-07-22 07:02:27 MSK 16h Tue 2026-07-21 07:05:30 MSK 7h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service\nWed 2026-07-22 07:07:14 MSK 16h Tue 2026-07-21 07:18:55 MSK 7h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service\nWed 2026-07-22 07:35:06 MSK 16h Tue 2026-07-21 07:36:59 MSK 6h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service\nWed 2026-07-22 07:38:21 MSK 17h Tue 2026-07-21 07:36:14 MSK 6h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service\nWed 2026-07-22 07:41:49 MSK 17h Tue 2026-07-21 07:53:41 MSK 6h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service\nWed 2026-07-22 08:10:37 MSK 17h Tue 2026-07-21 08:07:25 MSK 6h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service\nWed 2026-07-22 08:37:02 MSK 18h Tue 2026-07-21 08:30:07 MSK 6h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service\nWed 2026-07-22 08:40:13 MSK 18h Tue 2026-07-21 08:50:14 MSK 5h 45min ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service\nWed 2026-07-22 08:53:36 MSK 18h Tue 2026-07-21 08:41:14 MSK 5h 54min ago memos-backup.timer memos-backup.service\nSun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service\nSun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service\nSun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service\nSun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service\nMon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 7h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service\n=== BACKUP_UNITS ===\ndpkg-db-backup.service static -\nfilebrowser-backup.service static -\nhomelab-alertmanager-backup.service static -\nhomelab-appbackup-dockge-stacks.service static -\nhomelab-appbackup-mariadb-batch.service static -\nhomelab-appbackup-npmplus-edge.service static -\nhomelab-appbackup-postgres-batch.service static -\nhomelab-appbackup-sqlite-batch.service static -\nhomelab-authentik-backup.service static -\nhomelab-auto-backup.service static -\nhomelab-backup-coverage-matrix.service static -\nhomelab-backup-freshness.service static -\nhomelab-backup-healthcheck.service static -\nhomelab-backup-matrix-refresh.service static -\nhomelab-backup-sla-health.service static -\nhomelab-beszel-backup.service static -\nhomelab-blackbox-exporter-backup.service static -\nhomelab-cluster-admin-incident-engine-backup.service static -\nhomelab-crypto-portfolio-backup.service static -\nhomelab-edge-vm-vzdump-backup.service static -\nhomelab-emergency-backup.service static -\nhomelab-extended-appbackup.service static -\nhomelab-gitea-backup.service static -\nhomelab-gitea-secondary-backup.service static -\nhomelab-legacy-monitoring-backups.service static -\nhomelab-mail-cloud-critical-upload.service static -\nhomelab-mail-cloud-edge-vm-upload.service static -\nhomelab-mail-cloud-nextcloud-vm-upload.service static -\nhomelab-mail-cloud-restore-drill.service static -\nhomelab-npmplus-kuma-config-backup.service static -\nhomelab-router-running-config-mail-cloud.service static -\nhomelab-vaultwarden-backup.service static -\nhomelab-verified-backup-health.service static -\nhomelab-vm-backup-health.service static -\nhomelab-vm-backup-policy.service static -\nhomelab-vm-full-backup.service static -\nhomelab-vm-local-dumps-2cloud.service static -\nhomelab-vm160-remote-backup.service static -\nimmich-media-backup.service static -\nmemos-backup.service static -\nnetbird-vps-backup.service static -\npaperless-backup.service static -\nskladchik-reports-monitor-backup-restore-check.service static -\nskladchik-reports-monitor-backup.service static -\ndpkg-db-backup.timer enabled enabled\nfilebrowser-backup.timer enabled enabled\nhomelab-alertmanager-backup.timer enabled enabled\nhomelab-appbackup-dockge-stacks.timer enabled enabled\nhomelab-appbackup-mariadb-batch.timer enabled enabled\nhomelab-appbackup-npmplus-edge.timer enabled enabled\nhomelab-appbackup-postgres-batch.timer enabled enabled\nhomelab-appbackup-sqlite-batch.timer enabled enabled\nhomelab-authentik-backup.timer disabled enabled\nhomelab-auto-backup.timer enabled enabled\nhomelab-backup-coverage-matrix.timer enabled enabled\nhomelab-backup-freshness.timer enabled enabled\nhomelab-backup-healthcheck.timer enabled enabled\nhomelab-backup-matrix-refresh.timer disabled enabled\nhomelab-backup-sla-health.timer enabled enabled\nhomelab-beszel-backup.timer enabled enabled\nhomelab-blackbox-exporter-backup.timer enabled enabled\nhomelab-cluster-admin-incident-engine-backup.timer enabled enabled\nhomelab-crypto-portfolio-backup.timer enabled enabled\nhomelab-edge-vm-vzdump-backup.timer disabled enabled\nhomelab-emergency-backup.timer enabled enabled\nhomelab-extended-appbackup.timer enabled enabled\nhomelab-gitea-backup.timer disabled enabled\nhomelab-gitea-secondary-backup.timer enabled enabled\nhomelab-legacy-monitoring-backups.timer enabled enabled\nhomelab-mail-cloud-critical-upload.timer enabled enabled\nhomelab-mail-cloud-edge-vm-upload.timer enabled enabled\nhomelab-mail-cloud-nextcloud-vm-upload.timer enabled enabled\nhomelab-mail-cloud-restore-drill.timer enabled enabled\nhomelab-npmplus-kuma-config-backup.timer disabled enabled\nhomelab-router-running-config-mail-cloud.timer enabled enabled\nhomelab-vaultwarden-backup.timer disabled enabled\nhomelab-verified-backup-health.timer enabled enabled\nhomelab-vm-backup-health.timer enabled enabled\nhomelab-vm-backup-policy.timer enabled enabled\nhomelab-vm-full-backup.timer enabled enabled\nhomelab-vm-local-dumps-2cloud.timer enabled enabled\nhomelab-vm160-remote-backup.timer disabled enabled\nimmich-media-backup.timer disabled enabled\nmemos-backup.timer enabled enabled\nnetbird-vps-backup.timer disabled enabled\npaperless-backup.timer disabled enabled\nskladchik-reports-monitor-backup-restore-check.timer enabled enabled\nskladchik-reports-monitor-backup.timer enabled enabled\n=== PIPELINE_FILES ===\n/etc/cron.d/vzdump\n/etc/systemd/system/homelab-edge-vm-offhost-freshness.timer\n/etc/systemd/system/homelab-edge-vm-vzdump-backup.service\n/etc/systemd/system/homelab-edge-vm-vzdump-backup.timer\n/etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.service\n/etc/systemd/system/homelab-vm-full-backup.service\n/etc/systemd/system/multi-user.target.wants/pve-guests.service\n/etc/systemd/system/pve-manager.service\n/etc/systemd/system/timers.target.wants/homelab-edge-vm-offhost-freshness.timer\n/usr/local/sbin/appbackup-dockge-stacks\n/usr/local/sbin/appbackup-dockge-stacks.bak-chunkall-20260710T011410Z\n/usr/local/sbin/appbackup-restore-check\n/usr/local/sbin/appbackup-restore-check.bak-chunkall-20260710T011410Z\n/usr/local/sbin/appbackup-restore-check.bak-dual-read-20260711T051924Z\n/usr/local/sbin/appbackup-restore-check.bak-flat-read-20260711T052854Z\n/usr/local/sbin/appbackup-restore-check.bak-flat-read-v2-20260711T053129Z\n/usr/local/sbin/appbackup-retention\n/usr/local/sbin/appbackup-retention.bak-dual-read-20260711T051924Z\n/usr/local/sbin/backupctl\n/usr/local/sbin/backup-restore-check\n/usr/local/sbin/backup-restore-check.bak-chunkall-20260710T011410Z\n/usr/local/sbin/backup-retention\n/usr/local/sbin/homelab-backup-edge-vm-vzdump\n/usr/local/sbin/homelab-capacity-risk\n/usr/local/sbin/homelab-change-transaction\n/usr/local/sbin/homelab-chat-run\n/usr/local/sbin/homelab-edge-vm-offhost-freshness\n/usr/local/sbin/homelab-extended-appbackup-run\n/usr/local/sbin/homelab-golden-state-extended-overlay\n/usr/local/sbin/homelab-golden-state-index\n/usr/local/sbin/homelab-health-metrics\n/usr/local/sbin/homelab-ingress-standby-refresh\n/usr/local/sbin/homelab-mail-cloud-critical-upload\n/usr/local/sbin/homelab-mail-cloud-critical-upload.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload\n/usr/local/sbin/homelab-mail-cloud-restore-drill\n/usr/local/sbin/homelab-mail-cloud-restore-drill.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-rclone-cat-chunked\n/usr/local/sbin/homelab-rclone-cat-flat-chunked\n/usr/local/sbin/homelab-rclone-copy-chunked\n/usr/local/sbin/homelab-rclone-copy-flat-chunked\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh\n/usr/local/sbin/homelab-rclone-required-check.sh\n/usr/local/sbin/homelab-router-hard-power-cycle\n/usr/local/sbin/homelab-router-running-config-mail-cloud\n/usr/local/sbin/homelab-router-running-config-mail-cloud.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-safe-exec\n/usr/local/sbin/homelab-secret-exposure-guard\n/usr/local/sbin/homelab-stage-2cloud\n/usr/local/sbin/homelab-stage-2cloud.bak-global-chunkonly-20260710T011036Z\n/usr/local/sbin/homelab-stage-2cloud-flat\n/usr/local/sbin/homelab-stage-2cloud-flat-canary\n/usr/local/sbin/homelab-stage-verify-chunked\n/usr/local/sbin/homelab-stage-verify-flat-chunked\n/usr/local/sbin/homelab-vm160-remote-backup\n/usr/local/sbin/homelab-vm-backup-health\n/usr/local/sbin/homelab-vm-backup-policy\n/usr/local/sbin/homelab-vm-full-backup\n/usr/local/sbin/homelab-vm-full-backup.bak-chunkall-20260710T011410Z\n/usr/local/sbin/homelab-vm-full-restore-drill\n/usr/local/sbin/homelab-vm-full-restore-drill.bak.20260710T080231Z\n/usr/local/sbin/homelab-vm-full-restore-drill.bak-docker-check-v2-20260710T050044Z\n/usr/local/sbin/homelab-vm-full-restore-drill.bak-isolated-ip-20260710T044331Z\n/usr/local/sbin/homelab-vm-full-retention\n/usr/local/sbin/homelab-vm-local-dumps-2cloud\n/usr/local/sbin/homelab-vm-local-dumps-retention\n/usr/local/sbin/skladchik-reports-monitor-backup\n/usr/local/sbin/skladchik-reports-monitor-backup.pre-restore-check-20260710T065311Z.bak\n/usr/local/sbin/skladchik-reports-monitor-backup.pre-session-keepalive-20260710T071605Z.bak\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check.pre-session-keepalive-20260710T071605Z.bak\n=== LOCAL_BACKUPS ===\n7022386770\t2026-07-18T03:31:03.6566057240\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.vma.zst\n7021952392\t2026-07-21T03:31:06.5717414950\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst\n7021297699\t2026-07-19T03:31:06.8755351020\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst\n7016275481\t2026-07-20T03:31:03.2165123470\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst\n329541843\t2026-07-21T03:30:09.7688696330\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst\n326601118\t2026-07-20T03:30:08.2096782720\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst\n323668231\t2026-07-19T03:45:46.2209130400\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst\n321121649\t2026-07-18T03:30:08.0777625900\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.tar.zst\n222777732\t2026-07-21T03:30:15.9059638290\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst\n222761273\t2026-07-20T03:30:13.7937629440\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst\n222745965\t2026-07-19T03:45:57.2700790730\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst\n222727760\t2026-07-18T03:30:13.5578457230\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.tar.zst\n3021\t2026-07-19T03:31:07.4745284120\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.log\n3020\t2026-07-21T03:31:07.3945472590\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.log\n3018\t2026-07-18T03:31:03.9197988860\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.log\n3017\t2026-07-20T03:31:04.1706719350\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.log\n1089\t2026-07-20T03:30:14.0079985330\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.log\n1086\t2026-07-20T03:30:08.4640241540\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.log\n989\t2026-07-21T03:30:16.1364976560\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.log\n989\t2026-07-18T03:30:13.7816006690\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.log\n986\t2026-07-21T03:30:10.0285327490\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.log\n986\t2026-07-18T03:30:08.3002411830\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.log\n731\t2026-07-19T03:45:57.5197000270\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.log\n729\t2026-07-19T03:45:46.4444121980\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.log\n" +} + +EXACT_GITEA=FOUND +=== LOCAL_RAW_RESULT === +FILE=/var/lib/homelab-chat-run/raw/BACKUP-DUAL-CLOUD-PREFLIGHT-059-20260721T113522Z.log SIZE=21897 MTIME=2026-07-21 14:35:23.197206410 +0300 === BACKUP_JOBS === [{"compress":"zstd","enabled":1,"id":"homelab-nightly-all","mailnotification":"failure","mode":"snapshot","next-run":1784680200,"prune-backups":{"keep-daily":"4"},"schedule":"03:30","storage":"local","type":"vzdump","vmid":"110,111,112,113,130,150,190"}] === RCLONE_REMOTES ===