diff --git a/runtime/history/BACKUP-CLUSTER-13CLOUD-TOPOLOGY-062.json b/runtime/history/BACKUP-CLUSTER-13CLOUD-TOPOLOGY-062.json new file mode 100644 index 0000000..0c80d5e --- /dev/null +++ b/runtime/history/BACKUP-CLUSTER-13CLOUD-TOPOLOGY-062.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "BACKUP-CLUSTER-13CLOUD-TOPOLOGY-062", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "backup-audit", + "started_at_utc": "2026-07-21T11:49:15Z", + "finished_at_utc": "2026-07-21T11:49:26Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1", + "command_sha256": "eca35846e7097b9c238bf8c082e02c81f0b2893473e27ddb0a5b0854d5d3703d", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "3ba8aaf65323f1801f1eb63016759f4767a9d5be8bb21f15d92042e408395afd", + "sanitized_output_sha256": "3ba8aaf65323f1801f1eb63016759f4767a9d5be8bb21f15d92042e408395afd", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "=== CLUSTER_BACKUP_JOBS ===\n[{\"compress\":\"zstd\",\"enabled\":1,\"id\":\"homelab-nightly-all\",\"mailnotification\":\"failure\",\"mode\":\"snapshot\",\"next-run\":1784680200,\"prune-backups\":{\"keep-daily\":\"4\"},\"schedule\":\"03:30\",\"storage\":\"local\",\"type\":\"vzdump\",\"vmid\":\"110,111,112,113,130,150,190\"}]\n=== CLUSTER_STORAGES ===\nName Type Status Total (KiB) Used (KiB) Available (KiB) %\nlocal dir active 102626232 43175760 54191208 42.07%\nlocal-lvm lvmthin active 848797696 280188119 568609576 33.01%\n========== NODE=pve01 ==========\npve01.gram1.ru\n=== RCLONE_REMOTES ===\npve01-mail-01:\npve01-mail-01-crypt:\npve01-mail-02:\npve01-mail-02-crypt:\npve01-mail-03:\npve01-mail-03-crypt:\npve01-mail-04:\npve01-mail-04-crypt:\npve01-mail-05:\npve01-mail-05-crypt:\npve02-mail-01:\npve02-mail-01-crypt:\npve02-mail-02:\npve02-mail-02-crypt:\npve02-mail-03:\npve02-mail-03-crypt:\npve02-mail-04:\npve02-mail-04-crypt:\npve03-mail-01:\npve03-mail-01-crypt:\npve03-mail-02:\npve03-mail-02-crypt:\npve03-mail-03:\npve03-mail-03-crypt:\npve03-mail-04:\npve03-mail-04-crypt:\nNODE_REMOTE_COUNT=26\nREMOTE_HEALTH node=pve01 remote=pve01-mail-01: status=OK\n=== BACKUP_TIMERS ===\nWed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:14 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service\nWed 2026-07-22 00:09:43 MSK 9h Tue 2026-07-21 00:09:32 MSK 14h ago homelab-backup-freshness.timer homelab-backup-freshness.service\nWed 2026-07-22 02:15:00 MSK 11h Tue 2026-07-21 02:15:05 MSK 12h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service\nWed 2026-07-22 03:10:02 MSK 12h Tue 2026-07-21 03:13:14 MSK 11h ago homelab-emergency-backup.timer homelab-emergency-backup.service\nWed 2026-07-22 03:35:00 MSK 12h Tue 2026-07-21 03:35:14 MSK 11h ago filebrowser-backup.timer filebrowser-backup.service\nWed 2026-07-22 03:55:02 MSK 13h Tue 2026-07-21 03:51:25 MSK 10h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service\nWed 2026-07-22 04:17:29 MSK 13h Tue 2026-07-21 04:19:17 MSK 10h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service\nWed 2026-07-22 04:26:54 MSK 13h Tue 2026-07-21 04:28:14 MSK 10h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service\nWed 2026-07-22 04:38:04 MSK 13h Tue 2026-07-21 04:34:48 MSK 10h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service\nWed 2026-07-22 05:01:44 MSK 14h Tue 2026-07-21 04:57:14 MSK 9h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service\nWed 2026-07-22 05:23:56 MSK 14h Tue 2026-07-21 05:17:57 MSK 9h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service\nWed 2026-07-22 05:27:13 MSK 14h Tue 2026-07-21 05:29:05 MSK 9h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service\nWed 2026-07-22 05:47:19 MSK 14h Tue 2026-07-21 05:46:14 MSK 9h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service\nWed 2026-07-22 06:01:19 MSK 15h Tue 2026-07-21 05:57:30 MSK 8h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service\nWed 2026-07-22 06:27:43 MSK 15h Tue 2026-07-21 06:32:25 MSK 8h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service\nWed 2026-07-22 06:38:51 MSK 15h Tue 2026-07-21 06:43:54 MSK 8h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service\nWed 2026-07-22 06:39:22 MSK 15h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service\nWed 2026-07-22 06:44:16 MSK 15h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service\nWed 2026-07-22 06:52:06 MSK 16h Tue 2026-07-21 06:47:47 MSK 8h ago homelab-beszel-backup.timer homelab-beszel-backup.service\nWed 2026-07-22 06:58:58 MSK 16h Tue 2026-07-21 06:57:30 MSK 7h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service\nWed 2026-07-22 07:02:27 MSK 16h Tue 2026-07-21 07:05:30 MSK 7h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service\nWed 2026-07-22 07:07:14 MSK 16h Tue 2026-07-21 07:18:55 MSK 7h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service\nWed 2026-07-22 07:35:06 MSK 16h Tue 2026-07-21 07:36:59 MSK 7h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service\nWed 2026-07-22 07:38:21 MSK 16h Tue 2026-07-21 07:36:14 MSK 7h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service\nWed 2026-07-22 07:41:49 MSK 16h Tue 2026-07-21 07:53:41 MSK 6h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service\nWed 2026-07-22 08:10:37 MSK 17h Tue 2026-07-21 08:07:25 MSK 6h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service\nWed 2026-07-22 08:37:02 MSK 17h Tue 2026-07-21 08:30:07 MSK 6h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service\nWed 2026-07-22 08:40:13 MSK 17h Tue 2026-07-21 08:50:14 MSK 5h 59min ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service\nWed 2026-07-22 08:53:36 MSK 18h Tue 2026-07-21 08:41:14 MSK 6h ago memos-backup.timer memos-backup.service\nSun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service\nSun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service\nSun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service\nSun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service\nMon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 7h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service\n=== BACKUP_UNITS ===\ndpkg-db-backup.service static -\nfilebrowser-backup.service static -\nhomelab-alertmanager-backup.service static -\nhomelab-appbackup-dockge-stacks.service static -\nhomelab-appbackup-mariadb-batch.service static -\nhomelab-appbackup-npmplus-edge.service static -\nhomelab-appbackup-postgres-batch.service static -\nhomelab-appbackup-sqlite-batch.service static -\nhomelab-authentik-backup.service static -\nhomelab-auto-backup.service static -\nhomelab-backup-coverage-matrix.service static -\nhomelab-backup-freshness.service static -\nhomelab-backup-healthcheck.service static -\nhomelab-backup-matrix-refresh.service static -\nhomelab-backup-sla-health.service static -\nhomelab-beszel-backup.service static -\nhomelab-blackbox-exporter-backup.service static -\nhomelab-cluster-admin-incident-engine-backup.service static -\nhomelab-crypto-portfolio-backup.service static -\nhomelab-edge-vm-vzdump-backup.service static -\nhomelab-emergency-backup.service static -\nhomelab-extended-appbackup.service static -\nhomelab-gitea-backup.service static -\nhomelab-gitea-secondary-backup.service static -\nhomelab-legacy-monitoring-backups.service static -\nhomelab-mail-cloud-critical-upload.service static -\nhomelab-mail-cloud-edge-vm-upload.service static -\nhomelab-mail-cloud-nextcloud-vm-upload.service static -\nhomelab-mail-cloud-restore-drill.service static -\nhomelab-npmplus-kuma-config-backup.service static -\nhomelab-router-running-config-mail-cloud.service static -\nhomelab-vaultwarden-backup.service static -\nhomelab-verified-backup-health.service static -\nhomelab-vm-backup-health.service static -\nhomelab-vm-backup-policy.service static -\nhomelab-vm-full-backup.service static -\nhomelab-vm-local-dumps-2cloud.service static -\nhomelab-vm160-remote-backup.service static -\nimmich-media-backup.service static -\nmemos-backup.service static -\nnetbird-vps-backup.service static -\npaperless-backup.service static -\nskladchik-reports-monitor-backup-restore-check.service static -\nskladchik-reports-monitor-backup.service static -\ndpkg-db-backup.timer enabled enabled\nfilebrowser-backup.timer enabled enabled\nhomelab-alertmanager-backup.timer enabled enabled\nhomelab-appbackup-dockge-stacks.timer enabled enabled\nhomelab-appbackup-mariadb-batch.timer enabled enabled\nhomelab-appbackup-npmplus-edge.timer enabled enabled\nhomelab-appbackup-postgres-batch.timer enabled enabled\nhomelab-appbackup-sqlite-batch.timer enabled enabled\nhomelab-authentik-backup.timer disabled enabled\nhomelab-auto-backup.timer enabled enabled\nhomelab-backup-coverage-matrix.timer enabled enabled\nhomelab-backup-freshness.timer enabled enabled\nhomelab-backup-healthcheck.timer enabled enabled\nhomelab-backup-matrix-refresh.timer disabled enabled\nhomelab-backup-sla-health.timer enabled enabled\nhomelab-beszel-backup.timer enabled enabled\nhomelab-blackbox-exporter-backup.timer enabled enabled\nhomelab-cluster-admin-incident-engine-backup.timer enabled enabled\nhomelab-crypto-portfolio-backup.timer enabled enabled\nhomelab-edge-vm-vzdump-backup.timer disabled enabled\nhomelab-emergency-backup.timer enabled enabled\nhomelab-extended-appbackup.timer enabled enabled\nhomelab-gitea-backup.timer disabled enabled\nhomelab-gitea-secondary-backup.timer enabled enabled\nhomelab-legacy-monitoring-backups.timer enabled enabled\nhomelab-mail-cloud-critical-upload.timer enabled enabled\nhomelab-mail-cloud-edge-vm-upload.timer enabled enabled\nhomelab-mail-cloud-nextcloud-vm-upload.timer enabled enabled\nhomelab-mail-cloud-restore-drill.timer enabled enabled\nhomelab-npmplus-kuma-config-backup.timer disabled enabled\nhomelab-router-running-config-mail-cloud.timer enabled enabled\nhomelab-vaultwarden-backup.timer disabled enabled\nhomelab-verified-backup-health.timer enabled enabled\nhomelab-vm-backup-health.timer enabled enabled\nhomelab-vm-backup-policy.timer enabled enabled\nhomelab-vm-full-backup.timer enabled enabled\nhomelab-vm-local-dumps-2cloud.timer enabled enabled\nhomelab-vm160-remote-backup.timer disabled enabled\nimmich-media-backup.timer disabled enabled\nmemos-backup.timer enabled enabled\nnetbird-vps-backup.timer disabled enabled\npaperless-backup.timer disabled enabled\nskladchik-reports-monitor-backup-restore-check.timer enabled enabled\nskladchik-reports-monitor-backup.timer enabled enabled\n=== BACKUP_PIPELINE_FILES ===\n=== LOCAL_BACKUP_ARCHIVES ===\n7022386770\t2026-07-18T03:31:03.6566057240\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.vma.zst\n7021952392\t2026-07-21T03:31:06.5717414950\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst\n7021297699\t2026-07-19T03:31:06.8755351020\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst\n7016275481\t2026-07-20T03:31:03.2165123470\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst\n329541843\t2026-07-21T03:30:09.7688696330\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst\n326601118\t2026-07-20T03:30:08.2096782720\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst\n323668231\t2026-07-19T03:45:46.2209130400\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst\n321121649\t2026-07-18T03:30:08.0777625900\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.tar.zst\n222777732\t2026-07-21T03:30:15.9059638290\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst\n222761273\t2026-07-20T03:30:13.7937629440\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst\n222745965\t2026-07-19T03:45:57.2700790730\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst\n222727760\t2026-07-18T03:30:13.5578457230\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.tar.zst\n3021\t2026-07-19T03:31:07.4745284120\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.log\n3020\t2026-07-21T03:31:07.3945472590\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.log\n3018\t2026-07-18T03:31:03.9197988860\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.log\n3017\t2026-07-20T03:31:04.1706719350\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.log\n1089\t2026-07-20T03:30:14.0079985330\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.log\n1086\t2026-07-20T03:30:08.4640241540\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.log\n989\t2026-07-21T03:30:16.1364976560\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.log\n989\t2026-07-18T03:30:13.7816006690\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.log\n986\t2026-07-21T03:30:10.0285327490\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.log\n986\t2026-07-18T03:30:08.3002411830\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.log\n731\t2026-07-19T03:45:57.5197000270\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.log\n729\t2026-07-19T03:45:46.4444121980\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.log\n========== NODE=pve02 ==========\npve02.gram1.ru\n=== RCLONE_REMOTES ===\npve01-mail-01:\npve01-mail-01-crypt:\npve01-mail-02:\npve01-mail-02-crypt:\npve01-mail-03:\npve01-mail-03-crypt:\npve01-mail-04:\npve01-mail-04-crypt:\npve01-mail-05:\npve01-mail-05-crypt:\npve02-mail-01:\npve02-mail-01-crypt:\npve02-mail-02:\npve02-mail-02-crypt:\npve02-mail-03:\npve02-mail-03-crypt:\npve02-mail-04:\npve02-mail-04-crypt:\npve03-mail-01:\npve03-mail-01-crypt:\npve03-mail-02:\npve03-mail-02-crypt:\npve03-mail-03:\npve03-mail-03-crypt:\npve03-mail-04:\npve03-mail-04-crypt:\nNODE_REMOTE_COUNT=26\nREMOTE_HEALTH node=pve02 remote=pve01-mail-01: status=OK\n=== BACKUP_TIMERS ===\nTue 2026-07-21 18:41:03 MSK 3h 51min Tue 2026-07-21 12:41:22 MSK 2h 7min ago forum-prod-fresh5-cloud-backup.timer forum-prod-fresh5-cloud-backup.service\nWed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:22 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service\nWed 2026-07-22 04:20:00 MSK 13h Tue 2026-07-21 04:20:12 MSK 10h ago homelab-cluster-admin-vm-mail-cloud-backup.timer homelab-cluster-admin-vm-mail-cloud-backup.service\n=== BACKUP_UNITS ===\ncodevipe-cloud-backup.service static -\ndpkg-db-backup.service static -\nforum-prod-fresh5-cloud-backup.service static -\nhomelab-cluster-admin-vm-mail-cloud-backup.service static -\ncodevipe-cloud-backup.timer disabled enabled\ndpkg-db-backup.timer enabled enabled\nforum-prod-fresh5-cloud-backup.timer enabled enabled\nhomelab-cluster-admin-vm-mail-cloud-backup.timer enabled enabled\n=== BACKUP_PIPELINE_FILES ===\n=== LOCAL_BACKUP_ARCHIVES ===\n997503422\t2026-07-01T03:31:11.1700059310\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.vma.zst\n868048369\t2026-06-30T03:31:08.4903256780\t/var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.vma.zst\n288349157\t2026-07-21T03:30:21.2480085730\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.tar.zst\n286538526\t2026-07-20T03:30:18.4386603270\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.tar.zst\n284443608\t2026-07-19T03:47:41.8525503200\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst\n283022034\t2026-07-18T03:30:19.7617214460\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_18-03_30_03.tar.zst\n220080970\t2026-07-21T03:30:33.3910787180\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.tar.zst\n220065825\t2026-07-20T03:30:30.8287264090\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.tar.zst\n220049041\t2026-07-19T03:47:58.9256407960\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst\n220033534\t2026-07-18T03:30:37.2878143100\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_18-03_30_20.tar.zst\n7547\t2026-07-18T01:09:39.2467331080\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_18-01_02_44.log\n7443\t2026-07-18T01:45:13.3149404960\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_18-01_38_20.log\n5796\t2026-07-17T21:55:21.5648458010\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_17-21_30_20.log\n2685\t2026-06-30T03:31:09.1616927060\t/var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.log\n2590\t2026-07-01T03:31:11.9500372070\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.log\n1088\t2026-07-20T03:30:31.0784689220\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.log\n1085\t2026-07-20T03:30:18.7427551960\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.log\n988\t2026-07-21T03:30:33.6364088960\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.log\n988\t2026-07-18T03:30:37.5450736960\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_18-03_30_20.log\n985\t2026-07-21T03:30:21.5465818340\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.log\n985\t2026-07-18T03:30:20.0884248500\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_18-03_30_03.log\n731\t2026-07-19T03:47:59.2238139650\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.log\n728\t2026-07-19T03:47:42.1183953260\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.log\n========== NODE=pve03 ==========\npve03.gram1.ru\n=== RCLONE_REMOTES ===\npve01-mail-01:\npve01-mail-01-crypt:\npve01-mail-02:\npve01-mail-02-crypt:\npve01-mail-03:\npve01-mail-03-crypt:\npve01-mail-04:\npve01-mail-04-crypt:\npve01-mail-05:\npve01-mail-05-crypt:\npve02-mail-01:\npve02-mail-01-crypt:\npve02-mail-02:\npve02-mail-02-crypt:\npve02-mail-03:\npve02-mail-03-crypt:\npve02-mail-04:\npve02-mail-04-crypt:\npve03-mail-01:\npve03-mail-01-crypt:\npve03-mail-02:\npve03-mail-02-crypt:\npve03-mail-03:\npve03-mail-03-crypt:\npve03-mail-04:\npve03-mail-04-crypt:\nNODE_REMOTE_COUNT=26\nREMOTE_HEALTH node=pve03 remote=pve01-mail-01: status=OK\n=== BACKUP_TIMERS ===\nWed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:10 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service\n=== BACKUP_UNITS ===\ndpkg-db-backup.service static -\ndpkg-db-backup.timer enabled enabled\n=== BACKUP_PIPELINE_FILES ===\n=== LOCAL_BACKUP_ARCHIVES ===\n11116472459\t2026-07-20T03:33:53.2839064370\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.vma.zst\n11043834894\t2026-07-21T03:33:54.3725716230\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.vma.zst\n11013723404\t2026-07-19T03:33:53.3463878040\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.vma.zst\n10994820121\t2026-07-18T03:33:47.1540038630\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_18-03_30_02.vma.zst\n1137072350\t2026-07-21T03:34:37.6276811360\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.vma.zst\n838951976\t2026-07-19T03:34:33.1034859540\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.vma.zst\n838817925\t2026-07-20T03:34:37.1719936670\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.vma.zst\n785249297\t2026-07-18T03:34:27.1531047780\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_18-03_33_48.vma.zst\n7106\t2026-07-02T03:37:53.1040459570\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_02-03_30_02.log\n7102\t2026-07-03T03:36:49.1036784480\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_03-03_30_05.log\n6564\t2026-07-18T03:33:48.0107527290\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_18-03_30_02.log\n6560\t2026-07-20T03:33:54.9283349630\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.log\n6559\t2026-07-21T03:33:55.1428157340\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.log\n6459\t2026-07-19T03:33:53.7631656400\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.log\n6246\t2026-07-04T03:33:39.2197744030\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_04-03_30_02.log\n4107\t2026-07-07T03:31:51.2516816970\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_07-03_30_01.log\n2642\t2026-07-21T03:34:38.4491706750\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.log\n2500\t2026-07-20T03:34:37.2013439550\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.log\n2404\t2026-07-19T03:34:34.1266444600\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.log\n2305\t2026-07-18T03:34:27.7179455720\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_18-03_33_48.log\n=== CLOUD_REMOTE_SUMMARY ===\nEXPECTED_TOTAL=13\nDETECTED_TOTAL=78\nREMOTE_COUNT=MISMATCH\n" +} diff --git a/runtime/history/BACKUP-CLUSTER-13CLOUD-TOPOLOGY-062.txt b/runtime/history/BACKUP-CLUSTER-13CLOUD-TOPOLOGY-062.txt new file mode 100644 index 0000000..a81d89e --- /dev/null +++ b/runtime/history/BACKUP-CLUSTER-13CLOUD-TOPOLOGY-062.txt @@ -0,0 +1,342 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=BACKUP-CLUSTER-13CLOUD-TOPOLOGY-062 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=backup-audit +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1 +COMMAND_SHA256=eca35846e7097b9c238bf8c082e02c81f0b2893473e27ddb0a5b0854d5d3703d +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=false +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=3ba8aaf65323f1801f1eb63016759f4767a9d5be8bb21f15d92042e408395afd +SANITIZED_OUTPUT_SHA256=3ba8aaf65323f1801f1eb63016759f4767a9d5be8bb21f15d92042e408395afd +OUTPUT_BEGIN +=== CLUSTER_BACKUP_JOBS === +[{"compress":"zstd","enabled":1,"id":"homelab-nightly-all","mailnotification":"failure","mode":"snapshot","next-run":1784680200,"prune-backups":{"keep-daily":"4"},"schedule":"03:30","storage":"local","type":"vzdump","vmid":"110,111,112,113,130,150,190"}] +=== CLUSTER_STORAGES === +Name Type Status Total (KiB) Used (KiB) Available (KiB) % +local dir active 102626232 43175760 54191208 42.07% +local-lvm lvmthin active 848797696 280188119 568609576 33.01% +========== NODE=pve01 ========== +pve01.gram1.ru +=== RCLONE_REMOTES === +pve01-mail-01: +pve01-mail-01-crypt: +pve01-mail-02: +pve01-mail-02-crypt: +pve01-mail-03: +pve01-mail-03-crypt: +pve01-mail-04: +pve01-mail-04-crypt: +pve01-mail-05: +pve01-mail-05-crypt: +pve02-mail-01: +pve02-mail-01-crypt: +pve02-mail-02: +pve02-mail-02-crypt: +pve02-mail-03: +pve02-mail-03-crypt: +pve02-mail-04: +pve02-mail-04-crypt: +pve03-mail-01: +pve03-mail-01-crypt: +pve03-mail-02: +pve03-mail-02-crypt: +pve03-mail-03: +pve03-mail-03-crypt: +pve03-mail-04: +pve03-mail-04-crypt: +NODE_REMOTE_COUNT=26 +REMOTE_HEALTH node=pve01 remote=pve01-mail-01: status=OK +=== BACKUP_TIMERS === +Wed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:14 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service +Wed 2026-07-22 00:09:43 MSK 9h Tue 2026-07-21 00:09:32 MSK 14h ago homelab-backup-freshness.timer homelab-backup-freshness.service +Wed 2026-07-22 02:15:00 MSK 11h Tue 2026-07-21 02:15:05 MSK 12h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service +Wed 2026-07-22 03:10:02 MSK 12h Tue 2026-07-21 03:13:14 MSK 11h ago homelab-emergency-backup.timer homelab-emergency-backup.service +Wed 2026-07-22 03:35:00 MSK 12h Tue 2026-07-21 03:35:14 MSK 11h ago filebrowser-backup.timer filebrowser-backup.service +Wed 2026-07-22 03:55:02 MSK 13h Tue 2026-07-21 03:51:25 MSK 10h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service +Wed 2026-07-22 04:17:29 MSK 13h Tue 2026-07-21 04:19:17 MSK 10h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service +Wed 2026-07-22 04:26:54 MSK 13h Tue 2026-07-21 04:28:14 MSK 10h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service +Wed 2026-07-22 04:38:04 MSK 13h Tue 2026-07-21 04:34:48 MSK 10h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service +Wed 2026-07-22 05:01:44 MSK 14h Tue 2026-07-21 04:57:14 MSK 9h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service +Wed 2026-07-22 05:23:56 MSK 14h Tue 2026-07-21 05:17:57 MSK 9h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service +Wed 2026-07-22 05:27:13 MSK 14h Tue 2026-07-21 05:29:05 MSK 9h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service +Wed 2026-07-22 05:47:19 MSK 14h Tue 2026-07-21 05:46:14 MSK 9h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service +Wed 2026-07-22 06:01:19 MSK 15h Tue 2026-07-21 05:57:30 MSK 8h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service +Wed 2026-07-22 06:27:43 MSK 15h Tue 2026-07-21 06:32:25 MSK 8h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service +Wed 2026-07-22 06:38:51 MSK 15h Tue 2026-07-21 06:43:54 MSK 8h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service +Wed 2026-07-22 06:39:22 MSK 15h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service +Wed 2026-07-22 06:44:16 MSK 15h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service +Wed 2026-07-22 06:52:06 MSK 16h Tue 2026-07-21 06:47:47 MSK 8h ago homelab-beszel-backup.timer homelab-beszel-backup.service +Wed 2026-07-22 06:58:58 MSK 16h Tue 2026-07-21 06:57:30 MSK 7h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service +Wed 2026-07-22 07:02:27 MSK 16h Tue 2026-07-21 07:05:30 MSK 7h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service +Wed 2026-07-22 07:07:14 MSK 16h Tue 2026-07-21 07:18:55 MSK 7h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service +Wed 2026-07-22 07:35:06 MSK 16h Tue 2026-07-21 07:36:59 MSK 7h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service +Wed 2026-07-22 07:38:21 MSK 16h Tue 2026-07-21 07:36:14 MSK 7h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service +Wed 2026-07-22 07:41:49 MSK 16h Tue 2026-07-21 07:53:41 MSK 6h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service +Wed 2026-07-22 08:10:37 MSK 17h Tue 2026-07-21 08:07:25 MSK 6h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service +Wed 2026-07-22 08:37:02 MSK 17h Tue 2026-07-21 08:30:07 MSK 6h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service +Wed 2026-07-22 08:40:13 MSK 17h Tue 2026-07-21 08:50:14 MSK 5h 59min ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service +Wed 2026-07-22 08:53:36 MSK 18h Tue 2026-07-21 08:41:14 MSK 6h ago memos-backup.timer memos-backup.service +Sun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service +Sun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service +Sun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service +Sun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service +Mon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 7h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service +=== BACKUP_UNITS === +dpkg-db-backup.service static - +filebrowser-backup.service static - +homelab-alertmanager-backup.service static - +homelab-appbackup-dockge-stacks.service static - +homelab-appbackup-mariadb-batch.service static - +homelab-appbackup-npmplus-edge.service static - +homelab-appbackup-postgres-batch.service static - +homelab-appbackup-sqlite-batch.service static - +homelab-authentik-backup.service static - +homelab-auto-backup.service static - +homelab-backup-coverage-matrix.service static - +homelab-backup-freshness.service static - +homelab-backup-healthcheck.service static - +homelab-backup-matrix-refresh.service static - +homelab-backup-sla-health.service static - +homelab-beszel-backup.service static - +homelab-blackbox-exporter-backup.service static - +homelab-cluster-admin-incident-engine-backup.service static - +homelab-crypto-portfolio-backup.service static - +homelab-edge-vm-vzdump-backup.service static - +homelab-emergency-backup.service static - +homelab-extended-appbackup.service static - +homelab-gitea-backup.service static - +homelab-gitea-secondary-backup.service static - +homelab-legacy-monitoring-backups.service static - +homelab-mail-cloud-critical-upload.service static - +homelab-mail-cloud-edge-vm-upload.service static - +homelab-mail-cloud-nextcloud-vm-upload.service static - +homelab-mail-cloud-restore-drill.service static - +homelab-npmplus-kuma-config-backup.service static - +homelab-router-running-config-mail-cloud.service static - +homelab-vaultwarden-backup.service static - +homelab-verified-backup-health.service static - +homelab-vm-backup-health.service static - +homelab-vm-backup-policy.service static - +homelab-vm-full-backup.service static - +homelab-vm-local-dumps-2cloud.service static - +homelab-vm160-remote-backup.service static - +immich-media-backup.service static - +memos-backup.service static - +netbird-vps-backup.service static - +paperless-backup.service static - +skladchik-reports-monitor-backup-restore-check.service static - +skladchik-reports-monitor-backup.service static - +dpkg-db-backup.timer enabled enabled +filebrowser-backup.timer enabled enabled +homelab-alertmanager-backup.timer enabled enabled +homelab-appbackup-dockge-stacks.timer enabled enabled +homelab-appbackup-mariadb-batch.timer enabled enabled +homelab-appbackup-npmplus-edge.timer enabled enabled +homelab-appbackup-postgres-batch.timer enabled enabled +homelab-appbackup-sqlite-batch.timer enabled enabled +homelab-authentik-backup.timer disabled enabled +homelab-auto-backup.timer enabled enabled +homelab-backup-coverage-matrix.timer enabled enabled +homelab-backup-freshness.timer enabled enabled +homelab-backup-healthcheck.timer enabled enabled +homelab-backup-matrix-refresh.timer disabled enabled +homelab-backup-sla-health.timer enabled enabled +homelab-beszel-backup.timer enabled enabled +homelab-blackbox-exporter-backup.timer enabled enabled +homelab-cluster-admin-incident-engine-backup.timer enabled enabled +homelab-crypto-portfolio-backup.timer enabled enabled +homelab-edge-vm-vzdump-backup.timer disabled enabled +homelab-emergency-backup.timer enabled enabled +homelab-extended-appbackup.timer enabled enabled +homelab-gitea-backup.timer disabled enabled +homelab-gitea-secondary-backup.timer enabled enabled +homelab-legacy-monitoring-backups.timer enabled enabled +homelab-mail-cloud-critical-upload.timer enabled enabled +homelab-mail-cloud-edge-vm-upload.timer enabled enabled +homelab-mail-cloud-nextcloud-vm-upload.timer enabled enabled +homelab-mail-cloud-restore-drill.timer enabled enabled +homelab-npmplus-kuma-config-backup.timer disabled enabled +homelab-router-running-config-mail-cloud.timer enabled enabled +homelab-vaultwarden-backup.timer disabled enabled +homelab-verified-backup-health.timer enabled enabled +homelab-vm-backup-health.timer enabled enabled +homelab-vm-backup-policy.timer enabled enabled +homelab-vm-full-backup.timer enabled enabled +homelab-vm-local-dumps-2cloud.timer enabled enabled +homelab-vm160-remote-backup.timer disabled enabled +immich-media-backup.timer disabled enabled +memos-backup.timer enabled enabled +netbird-vps-backup.timer disabled enabled +paperless-backup.timer disabled enabled +skladchik-reports-monitor-backup-restore-check.timer enabled enabled +skladchik-reports-monitor-backup.timer enabled enabled +=== BACKUP_PIPELINE_FILES === +=== LOCAL_BACKUP_ARCHIVES === +7022386770 2026-07-18T03:31:03.6566057240 /var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.vma.zst +7021952392 2026-07-21T03:31:06.5717414950 /var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst +7021297699 2026-07-19T03:31:06.8755351020 /var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst +7016275481 2026-07-20T03:31:03.2165123470 /var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst +329541843 2026-07-21T03:30:09.7688696330 /var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst +326601118 2026-07-20T03:30:08.2096782720 /var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst +323668231 2026-07-19T03:45:46.2209130400 /var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst +321121649 2026-07-18T03:30:08.0777625900 /var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.tar.zst +222777732 2026-07-21T03:30:15.9059638290 /var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst +222761273 2026-07-20T03:30:13.7937629440 /var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst +222745965 2026-07-19T03:45:57.2700790730 /var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst +222727760 2026-07-18T03:30:13.5578457230 /var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.tar.zst +3021 2026-07-19T03:31:07.4745284120 /var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.log +3020 2026-07-21T03:31:07.3945472590 /var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.log +3018 2026-07-18T03:31:03.9197988860 /var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.log +3017 2026-07-20T03:31:04.1706719350 /var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.log +1089 2026-07-20T03:30:14.0079985330 /var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.log +1086 2026-07-20T03:30:08.4640241540 /var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.log +989 2026-07-21T03:30:16.1364976560 /var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.log +989 2026-07-18T03:30:13.7816006690 /var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.log +986 2026-07-21T03:30:10.0285327490 /var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.log +986 2026-07-18T03:30:08.3002411830 /var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.log +731 2026-07-19T03:45:57.5197000270 /var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.log +729 2026-07-19T03:45:46.4444121980 /var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.log +========== NODE=pve02 ========== +pve02.gram1.ru +=== RCLONE_REMOTES === +pve01-mail-01: +pve01-mail-01-crypt: +pve01-mail-02: +pve01-mail-02-crypt: +pve01-mail-03: +pve01-mail-03-crypt: +pve01-mail-04: +pve01-mail-04-crypt: +pve01-mail-05: +pve01-mail-05-crypt: +pve02-mail-01: +pve02-mail-01-crypt: +pve02-mail-02: +pve02-mail-02-crypt: +pve02-mail-03: +pve02-mail-03-crypt: +pve02-mail-04: +pve02-mail-04-crypt: +pve03-mail-01: +pve03-mail-01-crypt: +pve03-mail-02: +pve03-mail-02-crypt: +pve03-mail-03: +pve03-mail-03-crypt: +pve03-mail-04: +pve03-mail-04-crypt: +NODE_REMOTE_COUNT=26 +REMOTE_HEALTH node=pve02 remote=pve01-mail-01: status=OK +=== BACKUP_TIMERS === +Tue 2026-07-21 18:41:03 MSK 3h 51min Tue 2026-07-21 12:41:22 MSK 2h 7min ago forum-prod-fresh5-cloud-backup.timer forum-prod-fresh5-cloud-backup.service +Wed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:22 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service +Wed 2026-07-22 04:20:00 MSK 13h Tue 2026-07-21 04:20:12 MSK 10h ago homelab-cluster-admin-vm-mail-cloud-backup.timer homelab-cluster-admin-vm-mail-cloud-backup.service +=== BACKUP_UNITS === +codevipe-cloud-backup.service static - +dpkg-db-backup.service static - +forum-prod-fresh5-cloud-backup.service static - +homelab-cluster-admin-vm-mail-cloud-backup.service static - +codevipe-cloud-backup.timer disabled enabled +dpkg-db-backup.timer enabled enabled +forum-prod-fresh5-cloud-backup.timer enabled enabled +homelab-cluster-admin-vm-mail-cloud-backup.timer enabled enabled +=== BACKUP_PIPELINE_FILES === +=== LOCAL_BACKUP_ARCHIVES === +997503422 2026-07-01T03:31:11.1700059310 /var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.vma.zst +868048369 2026-06-30T03:31:08.4903256780 /var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.vma.zst +288349157 2026-07-21T03:30:21.2480085730 /var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.tar.zst +286538526 2026-07-20T03:30:18.4386603270 /var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.tar.zst +284443608 2026-07-19T03:47:41.8525503200 /var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst +283022034 2026-07-18T03:30:19.7617214460 /var/lib/vz/dump/vzdump-lxc-111-2026_07_18-03_30_03.tar.zst +220080970 2026-07-21T03:30:33.3910787180 /var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.tar.zst +220065825 2026-07-20T03:30:30.8287264090 /var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.tar.zst +220049041 2026-07-19T03:47:58.9256407960 /var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst +220033534 2026-07-18T03:30:37.2878143100 /var/lib/vz/dump/vzdump-lxc-113-2026_07_18-03_30_20.tar.zst +7547 2026-07-18T01:09:39.2467331080 /var/lib/vz/dump/vzdump-qemu-160-2026_07_18-01_02_44.log +7443 2026-07-18T01:45:13.3149404960 /var/lib/vz/dump/vzdump-qemu-160-2026_07_18-01_38_20.log +5796 2026-07-17T21:55:21.5648458010 /var/lib/vz/dump/vzdump-qemu-160-2026_07_17-21_30_20.log +2685 2026-06-30T03:31:09.1616927060 /var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.log +2590 2026-07-01T03:31:11.9500372070 /var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.log +1088 2026-07-20T03:30:31.0784689220 /var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.log +1085 2026-07-20T03:30:18.7427551960 /var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.log +988 2026-07-21T03:30:33.6364088960 /var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.log +988 2026-07-18T03:30:37.5450736960 /var/lib/vz/dump/vzdump-lxc-113-2026_07_18-03_30_20.log +985 2026-07-21T03:30:21.5465818340 /var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.log +985 2026-07-18T03:30:20.0884248500 /var/lib/vz/dump/vzdump-lxc-111-2026_07_18-03_30_03.log +731 2026-07-19T03:47:59.2238139650 /var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.log +728 2026-07-19T03:47:42.1183953260 /var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.log +========== NODE=pve03 ========== +pve03.gram1.ru +=== RCLONE_REMOTES === +pve01-mail-01: +pve01-mail-01-crypt: +pve01-mail-02: +pve01-mail-02-crypt: +pve01-mail-03: +pve01-mail-03-crypt: +pve01-mail-04: +pve01-mail-04-crypt: +pve01-mail-05: +pve01-mail-05-crypt: +pve02-mail-01: +pve02-mail-01-crypt: +pve02-mail-02: +pve02-mail-02-crypt: +pve02-mail-03: +pve02-mail-03-crypt: +pve02-mail-04: +pve02-mail-04-crypt: +pve03-mail-01: +pve03-mail-01-crypt: +pve03-mail-02: +pve03-mail-02-crypt: +pve03-mail-03: +pve03-mail-03-crypt: +pve03-mail-04: +pve03-mail-04-crypt: +NODE_REMOTE_COUNT=26 +REMOTE_HEALTH node=pve03 remote=pve01-mail-01: status=OK +=== BACKUP_TIMERS === +Wed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:10 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service +=== BACKUP_UNITS === +dpkg-db-backup.service static - +dpkg-db-backup.timer enabled enabled +=== BACKUP_PIPELINE_FILES === +=== LOCAL_BACKUP_ARCHIVES === +11116472459 2026-07-20T03:33:53.2839064370 /var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.vma.zst +11043834894 2026-07-21T03:33:54.3725716230 /var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.vma.zst +11013723404 2026-07-19T03:33:53.3463878040 /var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.vma.zst +10994820121 2026-07-18T03:33:47.1540038630 /var/lib/vz/dump/vzdump-qemu-130-2026_07_18-03_30_02.vma.zst +1137072350 2026-07-21T03:34:37.6276811360 /var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.vma.zst +838951976 2026-07-19T03:34:33.1034859540 /var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.vma.zst +838817925 2026-07-20T03:34:37.1719936670 /var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.vma.zst +785249297 2026-07-18T03:34:27.1531047780 /var/lib/vz/dump/vzdump-qemu-190-2026_07_18-03_33_48.vma.zst +7106 2026-07-02T03:37:53.1040459570 /var/lib/vz/dump/vzdump-qemu-130-2026_07_02-03_30_02.log +7102 2026-07-03T03:36:49.1036784480 /var/lib/vz/dump/vzdump-qemu-130-2026_07_03-03_30_05.log +6564 2026-07-18T03:33:48.0107527290 /var/lib/vz/dump/vzdump-qemu-130-2026_07_18-03_30_02.log +6560 2026-07-20T03:33:54.9283349630 /var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.log +6559 2026-07-21T03:33:55.1428157340 /var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.log +6459 2026-07-19T03:33:53.7631656400 /var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.log +6246 2026-07-04T03:33:39.2197744030 /var/lib/vz/dump/vzdump-qemu-130-2026_07_04-03_30_02.log +4107 2026-07-07T03:31:51.2516816970 /var/lib/vz/dump/vzdump-qemu-130-2026_07_07-03_30_01.log +2642 2026-07-21T03:34:38.4491706750 /var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.log +2500 2026-07-20T03:34:37.2013439550 /var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.log +2404 2026-07-19T03:34:34.1266444600 /var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.log +2305 2026-07-18T03:34:27.7179455720 /var/lib/vz/dump/vzdump-qemu-190-2026_07_18-03_33_48.log +=== CLOUD_REMOTE_SUMMARY === +EXPECTED_TOTAL=13 +DETECTED_TOTAL=78 +REMOTE_COUNT=MISMATCH + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index b02c6a9..0c80d5e 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "BACKUP-PIPELINE-DEFINITION-AUDIT-061", + "command_id": "BACKUP-CLUSTER-13CLOUD-TOPOLOGY-062", "status": "OK", "rc": 0, "host": "pve01", "mode": "read-only", "component": "backup-audit", - "started_at_utc": "2026-07-21T11:46:07Z", - "finished_at_utc": "2026-07-21T11:46:09Z", + "started_at_utc": "2026-07-21T11:49:15Z", + "finished_at_utc": "2026-07-21T11:49:26Z", "reference_register_checked": true, "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", "error_register_checked": true, "error_register_sha256": "f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1", - "command_sha256": "08de7a8820803f855426b27b3ecfd2e646209715ff5ac4aba48b73fa45623b28", + "command_sha256": "eca35846e7097b9c238bf8c082e02c81f0b2893473e27ddb0a5b0854d5d3703d", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -22,9 +22,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "5b51699fff305570e41040468ef3f04a84e21c0c977695ed1834aa40c01946b5", - "sanitized_output_sha256": "384c3f223883553a582cbf25ff144f7ff8a0a1dc79110ffe089b7154a12aed74", + "raw_evidence_sha256": "3ba8aaf65323f1801f1eb63016759f4767a9d5be8bb21f15d92042e408395afd", + "sanitized_output_sha256": "3ba8aaf65323f1801f1eb63016759f4767a9d5be8bb21f15d92042e408395afd", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "===== TIMER dpkg-db-backup.timer =====\n# /usr/lib/systemd/system/dpkg-db-backup.timer\n[Unit]\nDescription=Daily dpkg database backup timer\nDocumentation=man:dpkg(1)\n\n[Timer]\nOnCalendar=daily\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE dpkg-db-backup.service =====\n# /usr/lib/systemd/system/dpkg-db-backup.service\n[Unit]\nDescription=Daily dpkg database backup service\nDocumentation=man:dpkg(1)\n\n[Service]\nType=oneshot\nExecStart=/usr/libexec/dpkg/dpkg-db-backup\n\n# /etc/systemd/system/dpkg-db-backup.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/libexec/dpkg/dpkg-db-backup ; argv[]=/usr/libexec/dpkg/dpkg-db-backup ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER filebrowser-backup.timer =====\n# /etc/systemd/system/filebrowser-backup.timer\n[Unit]\nDescription=Run File Browser backup\n\n[Timer]\nOnCalendar=*-*-* 03:35:00\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE filebrowser-backup.service =====\n# /etc/systemd/system/filebrowser-backup.service\n[Unit]\nDescription=File Browser backup\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/root/filebrowser-backup.sh\n\n# /etc/systemd/system/filebrowser-backup.service.d/10-superseded-noop.conf\n[Service]\nExecStart=\nExecStart=/usr/local/sbin/homelab-superseded-unit-ok filebrowser-backup.service superseded_by_current_appbackup_restore_proof\n\n# /etc/systemd/system/filebrowser-backup.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok filebrowser-backup.service superseded_by_current_appbackup_restore_proof ; ignore_errors=no ; start_time=[Tue 2026-07-21 03:35:14 MSK] ; stop_time=[Tue 2026-07-21 03:35:14 MSK] ; pid=2211390 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-alertmanager-backup.timer =====\n# /etc/systemd/system/homelab-alertmanager-backup.timer\n[Unit]\nDescription=Daily Alertmanager two-cloud backup\n\n[Timer]\nOnCalendar=*-*-* 06:35:00\nRandomizedDelaySec=10m\nPersistent=true\nUnit=homelab-alertmanager-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-alertmanager-backup.service =====\n# /etc/systemd/system/homelab-alertmanager-backup.service\n[Unit]\nDescription=Alertmanager encrypted two-cloud application backup\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-alertmanager-backup-run\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-alertmanager-backup-run ; argv[]=/usr/local/sbin/homelab-alertmanager-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 06:40:25 MSK] ; stop_time=[Tue 2026-07-21 06:40:54 MSK] ; pid=2820982 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-appbackup-dockge-stacks.timer =====\n# /etc/systemd/system/homelab-appbackup-dockge-stacks.timer\n[Unit]\nDescription=Daily homelab app-level backup timer for dockge-stacks\n\n[Timer]\nOnCalendar=*-*-* 03:40:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-appbackup-dockge-stacks.service =====\n# /etc/systemd/system/homelab-appbackup-dockge-stacks.service\n[Unit]\nDescription=Homelab app-level backup for dockge-stacks\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/appbackupctl run\nExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run dockge-stacks\n\n# /etc/systemd/system/homelab-appbackup-dockge-stacks.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/appbackupctl ; argv[]=/usr/local/sbin/appbackupctl run ; ignore_errors=no ; start_time=[Tue 2026-07-21 03:51:25 MSK] ; stop_time=[Tue 2026-07-21 03:51:53 MSK] ; pid=2261529 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-appbackup-mariadb-batch.timer =====\n# /etc/systemd/system/homelab-appbackup-mariadb-batch.timer\n[Unit]\nDescription=Daily Homelab MariaDB app batch backup timer\n\n[Timer]\nOnCalendar=*-*-* 05:35:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-appbackup-mariadb-batch.service =====\n# /etc/systemd/system/homelab-appbackup-mariadb-batch.service\n[Unit]\nDescription=Homelab MariaDB app batch backup\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/mariadb-apps-backup-run\nExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run bookstack\n\n# /etc/systemd/system/homelab-appbackup-mariadb-batch.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/mariadb-apps-backup-run ; argv[]=/usr/local/sbin/mariadb-apps-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 05:46:14 MSK] ; stop_time=[Tue 2026-07-21 05:46:44 MSK] ; pid=2623964 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-appbackup-npmplus-edge.timer =====\n# /etc/systemd/system/homelab-appbackup-npmplus-edge.timer\n[Unit]\nDescription=Daily homelab app-level backup timer for npmplus-edge\n\n[Timer]\nOnCalendar=*-*-* 04:05:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-appbackup-npmplus-edge.service =====\n# /etc/systemd/system/homelab-appbackup-npmplus-edge.service\n[Unit]\nDescription=Homelab app-level backup for npmplus-edge\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/npmplus-edge-backup-run\nExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run npmplus-edge\n\n# /etc/systemd/system/homelab-appbackup-npmplus-edge.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/npmplus-edge-backup-run ; argv[]=/usr/local/sbin/npmplus-edge-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 04:19:17 MSK] ; stop_time=[Tue 2026-07-21 04:19:48 MSK] ; pid=2360589 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-appbackup-postgres-batch.timer =====\n# /etc/systemd/system/homelab-appbackup-postgres-batch.timer\n[Unit]\nDescription=Daily Homelab Postgres app batch backup timer\n\n[Timer]\nOnCalendar=*-*-* 05:10:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-appbackup-postgres-batch.service =====\n# /etc/systemd/system/homelab-appbackup-postgres-batch.service\n[Unit]\nDescription=Homelab Postgres app batch backup\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/postgres-apps-backup-run\nExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run authentik\nExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run paperless\nExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run immich\nExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run netbox\n\n# /etc/systemd/system/homelab-appbackup-postgres-batch.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/postgres-apps-backup-run ; argv[]=/usr/local/sbin/postgres-apps-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 05:29:05 MSK] ; stop_time=[Tue 2026-07-21 05:32:29 MSK] ; pid=2568901 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-appbackup-sqlite-batch.timer =====\n# /etc/systemd/system/homelab-appbackup-sqlite-batch.timer\n[Unit]\nDescription=Daily Homelab SQLite app batch backup timer\n\n[Timer]\nOnCalendar=*-*-* 04:45:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-appbackup-sqlite-batch.service =====\n# /etc/systemd/system/homelab-appbackup-sqlite-batch.service\n[Unit]\nDescription=Homelab SQLite app batch backup\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/sqlite-apps-backup-run\nExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run vaultwarden\nExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run gitea\nExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run uptime-kuma\nExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run gotify\n\n# /etc/systemd/system/homelab-appbackup-sqlite-batch.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/sqlite-apps-backup-run ; argv[]=/usr/local/sbin/sqlite-apps-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 04:57:14 MSK] ; stop_time=[Tue 2026-07-21 05:00:36 MSK] ; pid=2476638 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-authentik-backup.timer =====\n# /etc/systemd/system/homelab-authentik-backup.timer\n[Unit]\nDescription=Run Homelab Authentik backup chain daily\n\n[Timer]\nOnCalendar=*-*-* 04:50:00\nRandomizedDelaySec=10min\nPersistent=true\nUnit=homelab-authentik-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-authentik-backup.service =====\n# /etc/systemd/system/homelab-authentik-backup.service\n[Unit]\nDescription=Homelab Authentik backup chain\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-backup-authentik\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-backup-authentik ; argv[]=/usr/local/sbin/homelab-backup-authentik ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER homelab-auto-backup.timer =====\n# /etc/systemd/system/homelab-auto-backup.timer\n[Unit]\nDescription=Run homelab automatic full backup weekly\n\n[Timer]\nOnCalendar=Sun *-*-* 03:30:00\nPersistent=true\nRandomizedDelaySec=20m\nUnit=homelab-auto-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-auto-backup.service =====\n# /etc/systemd/system/homelab-auto-backup.service\n[Unit]\nDescription=Homelab automatic full backup with retention\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/root/homelab-auto-backup-with-retention.sh\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\n\n# /etc/systemd/system/homelab-auto-backup.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/root/homelab-auto-backup-with-retention.sh ; argv[]=/root/homelab-auto-backup-with-retention.sh ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER homelab-backup-coverage-matrix.timer =====\n# /etc/systemd/system/homelab-backup-coverage-matrix.timer\n[Unit]\nDescription=Homelab backup coverage matrix timer\n\n[Timer]\nOnCalendar=*-*-* 08:35:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-backup-coverage-matrix.service =====\n# /etc/systemd/system/homelab-backup-coverage-matrix.service\n[Unit]\nDescription=Homelab backup coverage matrix\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-backup-coverage-matrix\n\n# /etc/systemd/system/homelab-backup-coverage-matrix.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-backup-coverage-matrix ; argv[]=/usr/local/sbin/homelab-backup-coverage-matrix ; ignore_errors=no ; start_time=[Tue 2026-07-21 08:50:14 MSK] ; stop_time=[Tue 2026-07-21 08:50:15 MSK] ; pid=3277860 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-backup-freshness.timer =====\n# /etc/systemd/system/homelab-backup-freshness.timer\n[Unit]\nDescription=Run homelab backup freshness collector daily\n\n[Timer]\nOnCalendar=daily\nPersistent=true\nRandomizedDelaySec=20m\nUnit=homelab-backup-freshness.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-backup-freshness.service =====\n# /etc/systemd/system/homelab-backup-freshness.service\n[Unit]\nDescription=Homelab backup freshness collector\nDocumentation=man:systemd.service(5)\nAfter=local-fs.target\n\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/default/homelab-backup-freshness\nExecStart=/usr/local/sbin/homelab-backup-freshness.sh\n\n# /etc/systemd/system/homelab-backup-freshness.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-backup-freshness.sh ; argv[]=/usr/local/sbin/homelab-backup-freshness.sh ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER homelab-backup-healthcheck.timer =====\n# /etc/systemd/system/homelab-backup-healthcheck.timer\n[Unit]\nDescription=Run homelab backup healthcheck daily\n\n[Timer]\nOnCalendar=*-*-* 07:00:00\nPersistent=true\nRandomizedDelaySec=10m\nUnit=homelab-backup-healthcheck.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-backup-healthcheck.service =====\n# /etc/systemd/system/homelab-backup-healthcheck.service\n[Unit]\nDescription=Homelab backup healthcheck\n\n[Service]\nType=oneshot\nExecStart=/root/homelab-backup-healthcheck-wrapper.sh\n\n# /etc/systemd/system/homelab-backup-healthcheck.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/root/homelab-backup-healthcheck-wrapper.sh ; argv[]=/root/homelab-backup-healthcheck-wrapper.sh ; ignore_errors=no ; start_time=[Tue 2026-07-21 07:05:30 MSK] ; stop_time=[Tue 2026-07-21 07:05:43 MSK] ; pid=2900293 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-backup-matrix-refresh.timer =====\n# /etc/systemd/system/homelab-backup-matrix-refresh.timer\n[Unit]\nDescription=Refresh unified homelab backup matrix state\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=5min\nAccuracySec=30s\nPersistent=true\nUnit=homelab-backup-matrix-refresh.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-backup-matrix-refresh.service =====\n# /etc/systemd/system/homelab-backup-matrix-refresh.service\n[Unit]\nDescription=Generate unified homelab backup matrix state\nAfter=local-fs.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-backup-matrix-refresh\nTimeoutStartSec=5min\nResult=exit-code\nExecMainStatus=1\nExecStart={ path=/usr/local/sbin/homelab-backup-matrix-refresh ; argv[]=/usr/local/sbin/homelab-backup-matrix-refresh ; ignore_errors=no ; start_time=[Tue 2026-07-21 10:02:41 MSK] ; stop_time=[Tue 2026-07-21 10:02:42 MSK] ; pid=3525445 ; code=exited ; status=1 }\nActiveState=failed\n===== TIMER homelab-backup-sla-health.timer =====\n# /etc/systemd/system/homelab-backup-sla-health.timer\n[Unit]\nDescription=Homelab backup freshness SLA health timer\n\n[Timer]\nOnCalendar=*-*-* 08:00:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-backup-sla-health.service =====\n# /etc/systemd/system/homelab-backup-sla-health.service\n[Unit]\nDescription=Homelab backup freshness SLA health\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-backup-sla-health\n\n# /etc/systemd/system/homelab-backup-sla-health.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-backup-sla-health ; argv[]=/usr/local/sbin/homelab-backup-sla-health ; ignore_errors=no ; start_time=[Tue 2026-07-21 08:07:25 MSK] ; stop_time=[Tue 2026-07-21 08:07:26 MSK] ; pid=3122093 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-beszel-backup.timer =====\n# /etc/systemd/system/homelab-beszel-backup.timer\n[Unit]\nDescription=Daily Beszel encrypted two-cloud backup\n\n[Timer]\nOnCalendar=*-*-* 06:45:00\nRandomizedDelaySec=10m\nPersistent=true\nUnit=homelab-beszel-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-beszel-backup.service =====\n# /etc/systemd/system/homelab-beszel-backup.service\n[Unit]\nDescription=Beszel encrypted two-cloud application backup\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-beszel-backup-run\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-beszel-backup-run ; argv[]=/usr/local/sbin/homelab-beszel-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 06:47:47 MSK] ; stop_time=[Tue 2026-07-21 06:48:22 MSK] ; pid=2848350 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-blackbox-exporter-backup.timer =====\n# /etc/systemd/system/homelab-blackbox-exporter-backup.timer\n[Unit]\nDescription=Daily Blackbox Exporter encrypted two-cloud backup\n\n[Timer]\nOnCalendar=*-*-* 06:55:00\nRandomizedDelaySec=10m\nPersistent=true\nUnit=homelab-blackbox-exporter-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-blackbox-exporter-backup.service =====\n# /etc/systemd/system/homelab-blackbox-exporter-backup.service\n[Unit]\nDescription=Blackbox Exporter encrypted two-cloud application backup\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-blackbox-exporter-backup-run\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-blackbox-exporter-backup-run ; argv[]=/usr/local/sbin/homelab-blackbox-exporter-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 06:57:30 MSK] ; stop_time=[Tue 2026-07-21 06:57:54 MSK] ; pid=2876084 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-cluster-admin-incident-engine-backup.timer =====\n# /etc/systemd/system/homelab-cluster-admin-incident-engine-backup.timer\n[Unit]\nDescription=Daily Cluster Admin Incident Engine two-cloud backup\n\n[Timer]\nOnCalendar=*-*-* 07:25:00\nRandomizedDelaySec=15m\nPersistent=true\nUnit=homelab-cluster-admin-incident-engine-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-cluster-admin-incident-engine-backup.service =====\n# /etc/systemd/system/homelab-cluster-admin-incident-engine-backup.service\n[Unit]\nDescription=Cluster Admin Incident Engine encrypted two-cloud backup\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run ; argv[]=/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 07:36:14 MSK] ; stop_time=[Tue 2026-07-21 07:36:40 MSK] ; pid=3010566 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-crypto-portfolio-backup.timer =====\n# /etc/systemd/system/homelab-crypto-portfolio-backup.timer\n[Unit]\nDescription=Daily Gram Crypto portfolio backup\n\n[Timer]\nOnCalendar=*-*-* 06:25:00\nRandomizedDelaySec=15m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-crypto-portfolio-backup.service =====\n# /etc/systemd/system/homelab-crypto-portfolio-backup.service\n[Unit]\nDescription=Gram Crypto portfolio two-cloud backup\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/crypto-portfolio-backup-run\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/crypto-portfolio-backup-run ; argv[]=/usr/local/sbin/crypto-portfolio-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 06:32:25 MSK] ; stop_time=[Tue 2026-07-21 06:32:50 MSK] ; pid=2794060 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-edge-vm-vzdump-backup.timer =====\n# /etc/systemd/system/homelab-edge-vm-vzdump-backup.timer\n[Unit]\nDescription=Run Homelab edge-vm VM-level backup chain weekly\n\n[Timer]\nOnCalendar=Sun *-*-* 06:20:00\nRandomizedDelaySec=20min\nPersistent=true\nUnit=homelab-edge-vm-vzdump-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-edge-vm-vzdump-backup.service =====\n# /etc/systemd/system/homelab-edge-vm-vzdump-backup.service\n[Unit]\nDescription=Homelab edge-vm VM-level vzdump backup chain\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-backup-edge-vm-vzdump\n\n# /etc/systemd/system/homelab-edge-vm-vzdump-backup.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-backup-edge-vm-vzdump ; argv[]=/usr/local/sbin/homelab-backup-edge-vm-vzdump ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER homelab-emergency-backup.timer =====\n# /etc/systemd/system/homelab-emergency-backup.timer\n[Unit]\nDescription=Daily homelab emergency/config backup timer\n\n[Timer]\nOnCalendar=*-*-* 03:10:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-emergency-backup.service =====\n# /etc/systemd/system/homelab-emergency-backup.service\n[Unit]\nDescription=Homelab emergency/config backup to two encrypted clouds\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/backupctl run\nExecStartPost=/usr/local/sbin/backup-retention 14 dry-run\n\n# /etc/systemd/system/homelab-emergency-backup.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/backupctl ; argv[]=/usr/local/sbin/backupctl run ; ignore_errors=no ; start_time=[Tue 2026-07-21 03:13:14 MSK] ; stop_time=[Tue 2026-07-21 03:13:42 MSK] ; pid=2137825 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-extended-appbackup.timer =====\n# /etc/systemd/system/homelab-extended-appbackup.timer\n[Unit]\nDescription=Homelab extended P1/P2 appbackup timer\n\n[Timer]\nOnCalendar=*-*-* 05:55:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-extended-appbackup.service =====\n# /etc/systemd/system/homelab-extended-appbackup.service\n[Unit]\nDescription=Homelab extended P1/P2 appbackup run\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-extended-appbackup-run\n\n# /etc/systemd/system/homelab-extended-appbackup.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-extended-appbackup-run ; argv[]=/usr/local/sbin/homelab-extended-appbackup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 05:57:30 MSK] ; stop_time=[Tue 2026-07-21 06:06:46 MSK] ; pid=2671623 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-gitea-backup.timer =====\n# /etc/systemd/system/homelab-gitea-backup.timer\n[Unit]\nDescription=Run Homelab Gitea backup chain daily\n\n[Timer]\nOnCalendar=*-*-* 05:10:00\nRandomizedDelaySec=10min\nPersistent=true\nUnit=homelab-gitea-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-gitea-backup.service =====\n# /etc/systemd/system/homelab-gitea-backup.service\n[Unit]\nDescription=Homelab Gitea backup chain\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-backup-gitea\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-backup-gitea ; argv[]=/usr/local/sbin/homelab-backup-gitea ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER homelab-gitea-secondary-backup.timer =====\n# /etc/systemd/system/homelab-gitea-secondary-backup.timer\n[Unit]\nDescription=Daily verified Gitea backup to pve03\n\n[Timer]\nOnCalendar=*-*-* 04:20:00\nRandomizedDelaySec=10m\nPersistent=true\nUnit=homelab-gitea-secondary-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-gitea-secondary-backup.service =====\n# /etc/systemd/system/homelab-gitea-secondary-backup.service\n[Unit]\nDescription=Create and copy a verified Gitea backup to pve03\nAfter=pve-cluster.service network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-gitea-secondary-backup\nTimeoutStartSec=2h\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\nUMask=0077\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-gitea-secondary-backup ; argv[]=/usr/local/sbin/homelab-gitea-secondary-backup ; ignore_errors=no ; start_time=[Tue 2026-07-21 04:28:14 MSK] ; stop_time=[Tue 2026-07-21 04:28:18 MSK] ; pid=2376819 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-legacy-monitoring-backups.timer =====\n# /etc/systemd/system/homelab-legacy-monitoring-backups.timer\n[Unit]\nDescription=Daily legacy monitoring applications two-cloud backup batch\n\n[Timer]\nOnCalendar=*-*-* 07:05:00\nRandomizedDelaySec=15m\nPersistent=true\nUnit=homelab-legacy-monitoring-backups.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-legacy-monitoring-backups.service =====\n# /etc/systemd/system/homelab-legacy-monitoring-backups.service\n[Unit]\nDescription=Legacy monitoring applications encrypted two-cloud backup batch\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-legacy-monitoring-backups-run\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-legacy-monitoring-backups-run ; argv[]=/usr/local/sbin/homelab-legacy-monitoring-backups-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 07:18:55 MSK] ; stop_time=[Tue 2026-07-21 07:21:45 MSK] ; pid=2950676 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-mail-cloud-critical-upload.timer =====\n# /etc/systemd/system/homelab-mail-cloud-critical-upload.timer\n[Unit]\nDescription=Daily Homelab Mail Cloud critical/small baseline upload\n\n[Timer]\nOnCalendar=*-*-* 06:40:00\nRandomizedDelaySec=15m\nPersistent=true\nUnit=homelab-mail-cloud-critical-upload.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-mail-cloud-critical-upload.service =====\n# /etc/systemd/system/homelab-mail-cloud-critical-upload.service\n[Unit]\nDescription=Homelab Mail Cloud critical/small baseline upload\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-mail-cloud-critical-upload\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-mail-cloud-critical-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-critical-upload ; ignore_errors=no ; start_time=[Tue 2026-07-21 06:40:25 MSK] ; stop_time=[Tue 2026-07-21 06:42:44 MSK] ; pid=2820985 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-mail-cloud-edge-vm-upload.timer =====\n# /etc/systemd/system/homelab-mail-cloud-edge-vm-upload.timer\n[Unit]\nDescription=Weekly Homelab Mail Cloud edge-vm large chunked upload\n\n[Timer]\nOnCalendar=Sun *-*-* 09:30:00\nRandomizedDelaySec=30m\nPersistent=true\nUnit=homelab-mail-cloud-edge-vm-upload.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-mail-cloud-edge-vm-upload.service =====\n# /etc/systemd/system/homelab-mail-cloud-edge-vm-upload.service\n[Unit]\nDescription=Homelab Mail Cloud edge-vm large chunked upload\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER homelab-mail-cloud-nextcloud-vm-upload.timer =====\n# /etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.timer\n[Unit]\nDescription=Run Homelab Mail Cloud Nextcloud VM upload daily\n\n[Timer]\nOnCalendar=*-*-* 07:30:00\nRandomizedDelaySec=30m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-mail-cloud-nextcloud-vm-upload.service =====\n# /etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.service\n[Unit]\nDescription=Homelab Mail Cloud Nextcloud VM vzdump upload\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload ; ignore_errors=no ; start_time=[Tue 2026-07-21 07:53:41 MSK] ; stop_time=[Tue 2026-07-21 07:56:26 MSK] ; pid=3068432 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-mail-cloud-restore-drill.timer =====\n# /etc/systemd/system/homelab-mail-cloud-restore-drill.timer\n[Unit]\nDescription=Weekly Homelab Mail Cloud restore drill\n\n[Timer]\nOnCalendar=Mon *-*-* 07:20:00\nRandomizedDelaySec=20m\nPersistent=true\nUnit=homelab-mail-cloud-restore-drill.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-mail-cloud-restore-drill.service =====\n# /etc/systemd/system/homelab-mail-cloud-restore-drill.service\n[Unit]\nDescription=Homelab Mail Cloud restore drill\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-mail-cloud-restore-drill\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\n\n# /etc/systemd/system/homelab-mail-cloud-restore-drill.service.d/10-superseded-noop.conf\n[Service]\nExecStart=\nExecStart=/usr/local/sbin/homelab-superseded-unit-ok homelab-mail-cloud-restore-drill.service superseded_by_safety_net_restore_check\n\n# /etc/systemd/system/homelab-mail-cloud-restore-drill.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok homelab-mail-cloud-restore-drill.service superseded_by_safety_net_restore_check ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER homelab-npmplus-kuma-config-backup.timer =====\n# /etc/systemd/system/homelab-npmplus-kuma-config-backup.timer\n[Unit]\nDescription=Run Homelab NPMplus + Kuma config backup chain daily\n\n[Timer]\nOnCalendar=*-*-* 04:20:00\nRandomizedDelaySec=10min\nPersistent=true\nUnit=homelab-npmplus-kuma-config-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-npmplus-kuma-config-backup.service =====\n# /etc/systemd/system/homelab-npmplus-kuma-config-backup.service\n[Unit]\nDescription=Homelab NPMplus + Kuma config backup chain\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-backup-npmplus-kuma-configs\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-backup-npmplus-kuma-configs ; argv[]=/usr/local/sbin/homelab-backup-npmplus-kuma-configs ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER homelab-router-running-config-mail-cloud.timer =====\n# /etc/systemd/system/homelab-router-running-config-mail-cloud.timer\n[Unit]\nDescription=Run Homelab router running-config Mail-cloud backup daily\n\n[Timer]\nOnCalendar=*-*-* 05:15:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-router-running-config-mail-cloud.service =====\n# /etc/systemd/system/homelab-router-running-config-mail-cloud.service\n[Unit]\nDescription=Homelab router running-config Mail-cloud backup\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-router-running-config-mail-cloud\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-router-running-config-mail-cloud ; argv[]=/usr/local/sbin/homelab-router-running-config-mail-cloud ; ignore_errors=no ; start_time=[Tue 2026-07-21 05:17:57 MSK] ; stop_time=[Tue 2026-07-21 05:18:05 MSK] ; pid=2540965 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-vaultwarden-backup.timer =====\n# /etc/systemd/system/homelab-vaultwarden-backup.timer\n[Unit]\nDescription=Run Homelab Vaultwarden backup chain daily\n\n[Timer]\nOnCalendar=*-*-* 05:30:00\nRandomizedDelaySec=10min\nPersistent=true\nUnit=homelab-vaultwarden-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-vaultwarden-backup.service =====\n# /etc/systemd/system/homelab-vaultwarden-backup.service\n[Unit]\nDescription=Homelab Vaultwarden backup chain\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-backup-vaultwarden\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-backup-vaultwarden ; argv[]=/usr/local/sbin/homelab-backup-vaultwarden ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER homelab-verified-backup-health.timer =====\n# /etc/systemd/system/homelab-verified-backup-health.timer\n[Unit]\nDescription=Homelab verified backup health evidence timer\n\n[Timer]\nOnCalendar=*-*-* 06:30:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-verified-backup-health.service =====\n# /etc/systemd/system/homelab-verified-backup-health.service\n[Unit]\nDescription=Homelab verified backup health evidence\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-backup-health-evidence\n\n# /etc/systemd/system/homelab-verified-backup-health.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-backup-health-evidence ; argv[]=/usr/local/sbin/homelab-backup-health-evidence ; ignore_errors=no ; start_time=[Tue 2026-07-21 06:43:54 MSK] ; stop_time=[Tue 2026-07-21 06:45:16 MSK] ; pid=2823792 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-vm-backup-health.timer =====\n# /etc/systemd/system/homelab-vm-backup-health.timer\n[Unit]\nDescription=Homelab VM backup health timer\n\n[Timer]\nOnCalendar=*-*-* 07:30:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-vm-backup-health.service =====\n# /etc/systemd/system/homelab-vm-backup-health.service\n[Unit]\nDescription=Homelab VM backup health\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-vm-backup-health\n\n# /etc/systemd/system/homelab-vm-backup-health.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-vm-backup-health ; argv[]=/usr/local/sbin/homelab-vm-backup-health ; ignore_errors=no ; start_time=[Tue 2026-07-21 07:36:59 MSK] ; stop_time=[Tue 2026-07-21 07:36:59 MSK] ; pid=3011738 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-vm-backup-policy.timer =====\n# /etc/systemd/system/homelab-vm-backup-policy.timer\n[Unit]\nDescription=Homelab VM backup policy and feasibility health timer\n\n[Timer]\nOnCalendar=*-*-* 08:18:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-vm-backup-policy.service =====\n# /etc/systemd/system/homelab-vm-backup-policy.service\n[Unit]\nDescription=Homelab VM backup policy and feasibility health\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-vm-backup-policy\n\n# /etc/systemd/system/homelab-vm-backup-policy.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-vm-backup-policy ; argv[]=/usr/local/sbin/homelab-vm-backup-policy ; ignore_errors=no ; start_time=[Tue 2026-07-21 08:30:07 MSK] ; stop_time=[Tue 2026-07-21 08:30:13 MSK] ; pid=3198902 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-vm-full-backup.timer =====\n# /etc/systemd/system/homelab-vm-full-backup.timer\n[Unit]\nDescription=Daily full backup timer for VM170 and VM171\n\n[Timer]\nOnCalendar=*-*-* 02:15:00\nPersistent=true\nAccuracySec=1min\nUnit=homelab-vm-full-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-vm-full-backup.service =====\n# /etc/systemd/system/homelab-vm-full-backup.service\n[Unit]\nDescription=Full vzdump backup of VM170 and VM171 to offhost and two clouds\nAfter=network-online.target pve-cluster.service\nWants=network-online.target\nConditionPathIsMountPoint=/mnt/staging\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-vm-full-backup\nTimeoutStartSec=infinity\nNice=10\nIOSchedulingClass=idle\nIOSchedulingPriority=7\nUMask=0077\n\n# /etc/systemd/system/homelab-vm-full-backup.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-vm-full-backup ; argv[]=/usr/local/sbin/homelab-vm-full-backup ; ignore_errors=no ; start_time=[Tue 2026-07-21 02:15:05 MSK] ; stop_time=[Tue 2026-07-21 03:09:43 MSK] ; pid=1933641 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-vm-local-dumps-2cloud.timer =====\n# /etc/systemd/system/homelab-vm-local-dumps-2cloud.timer\n[Unit]\nDescription=Homelab VM/CT existing local dumps two-cloud verifier timer\n\n[Timer]\nOnCalendar=Sun *-*-* 10:00:00\nRandomizedDelaySec=30m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-vm-local-dumps-2cloud.service =====\n# /etc/systemd/system/homelab-vm-local-dumps-2cloud.service\n[Unit]\nDescription=Homelab VM/CT existing local dumps two-cloud verifier\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-vm-local-dumps-2cloud\n\n# /etc/systemd/system/homelab-vm-local-dumps-2cloud.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; argv[]=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER homelab-vm160-remote-backup.timer =====\n# /etc/systemd/system/homelab-vm160-remote-backup.timer\n[Unit]\nDescription=Daily VM160 remote full-image backup\n\n[Timer]\nOnCalendar=*-*-* 04:10:00\nRandomizedDelaySec=10m\nUnit=homelab-vm160-remote-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-vm160-remote-backup.service =====\n# /etc/systemd/system/homelab-vm160-remote-backup.service\n[Unit]\nDescription=VM160 full-image backup streamed from pve02 to pve03\nAfter=network-online.target pve-cluster.service\nWants=network-online.target\n\n[Service]\nType=oneshot\nTimeoutStartSec=6h\nExecStart=/usr/local/sbin/homelab-vm160-remote-backup\nResult=exit-code\nExecMainStatus=1\nExecStart={ path=/usr/local/sbin/homelab-vm160-remote-backup ; argv[]=/usr/local/sbin/homelab-vm160-remote-backup ; ignore_errors=no ; start_time=[Tue 2026-07-21 02:39:12 MSK] ; stop_time=[Tue 2026-07-21 02:39:14 MSK] ; pid=2028253 ; code=exited ; status=1 }\nActiveState=failed\n===== TIMER immich-media-backup.timer =====\n# /etc/systemd/system/immich-media-backup.timer\n[Unit]\nDescription=Daily Immich media backup timer\n\n[Timer]\nOnCalendar=*-*-* 08:15:00\nPersistent=true\nRandomizedDelaySec=10m\nUnit=immich-media-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE immich-media-backup.service =====\n# /etc/systemd/system/immich-media-backup.service\n[Unit]\nDescription=Immich media backup to staging\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/root/immich-media-backup.sh\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\n\n# /etc/systemd/system/immich-media-backup.service.d/10-superseded-noop.conf\n[Service]\nExecStart=\nExecStart=/usr/local/sbin/homelab-superseded-unit-ok immich-media-backup.service superseded_by_current_appbackup_or_trust_proof\n\n# /etc/systemd/system/immich-media-backup.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok immich-media-backup.service superseded_by_current_appbackup_or_trust_proof ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER memos-backup.timer =====\n# /etc/systemd/system/memos-backup.timer\n[Unit]\nDescription=Run Memos data backup daily\n\n[Timer]\nOnCalendar=*-*-* 08:35:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE memos-backup.service =====\n# /etc/systemd/system/memos-backup.service\n[Unit]\nDescription=Memos data backup to pve01 staging\n\n[Service]\nType=oneshot\nExecStart=/root/memos-backup.sh\n\n# /etc/systemd/system/memos-backup.service.d/10-superseded-noop.conf\n[Service]\nExecStart=\nExecStart=/usr/local/sbin/homelab-superseded-unit-ok memos-backup.service superseded_by_current_appbackup_or_trust_proof\n\n# /etc/systemd/system/memos-backup.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok memos-backup.service superseded_by_current_appbackup_or_trust_proof ; ignore_errors=no ; start_time=[Tue 2026-07-21 08:41:14 MSK] ; stop_time=[Tue 2026-07-21 08:41:14 MSK] ; pid=3239605 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER netbird-vps-backup.timer =====\n# /etc/systemd/system/netbird-vps-backup.timer\n[Unit]\nDescription=Run NetBird VPS backup daily\n\n[Timer]\nOnCalendar=*-*-* 04:25:00\nPersistent=true\nRandomizedDelaySec=15m\n\n[Install]\nWantedBy=timers.target\n===== SERVICE netbird-vps-backup.service =====\n# /etc/systemd/system/netbird-vps-backup.service\n[Unit]\nDescription=NetBird VPS backup to pve01 staging\n\n[Service]\nType=oneshot\nExecStart=/root/netbird-vps-backup.sh\n\n# /etc/systemd/system/netbird-vps-backup.service.d/10-superseded-noop.conf\n[Service]\nExecStart=\nExecStart=/usr/local/sbin/homelab-superseded-unit-ok netbird-vps-backup.service superseded_by_current_appbackup_or_trust_proof\n\n# /etc/systemd/system/netbird-vps-backup.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok netbird-vps-backup.service superseded_by_current_appbackup_or_trust_proof ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER paperless-backup.timer =====\n# /etc/systemd/system/paperless-backup.timer\n[Unit]\nDescription=Run Paperless backup daily\n\n[Timer]\nOnCalendar=*-*-* 08:50:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE paperless-backup.service =====\n# /etc/systemd/system/paperless-backup.service\n[Unit]\nDescription=Paperless backup to pve01 staging\n\n[Service]\nType=oneshot\nExecStart=/root/paperless-backup.sh\n\n# /etc/systemd/system/paperless-backup.service.d/10-superseded-noop.conf\n[Service]\nExecStart=\nExecStart=/usr/local/sbin/homelab-superseded-unit-ok paperless-backup.service superseded_by_current_appbackup_or_trust_proof\n\n# /etc/systemd/system/paperless-backup.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok paperless-backup.service superseded_by_current_appbackup_or_trust_proof ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER skladchik-reports-monitor-backup-restore-check.timer =====\n# /etc/systemd/system/skladchik-reports-monitor-backup-restore-check.timer\n[Unit]\nDescription=Weekly restore-check of Skladchik reports monitor cloud backups\n\n[Timer]\nOnCalendar=Sun *-*-* 04:55:00\nRandomizedDelaySec=30min\nPersistent=true\nUnit=skladchik-reports-monitor-backup-restore-check.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE skladchik-reports-monitor-backup-restore-check.service =====\n# /etc/systemd/system/skladchik-reports-monitor-backup-restore-check.service\n[Unit]\nDescription=Restore-check Skladchik reports monitor cloud backups\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\n\n# /etc/systemd/system/skladchik-reports-monitor-backup-restore-check.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check ; argv[]=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER skladchik-reports-monitor-backup.timer =====\n# /etc/systemd/system/skladchik-reports-monitor-backup.timer\n[Unit]\nDescription=Daily Skladchik reports monitor backup to encrypted clouds\n\n[Timer]\nOnCalendar=*-*-* 04:25:00\nRandomizedDelaySec=15min\nPersistent=true\nUnit=skladchik-reports-monitor-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE skladchik-reports-monitor-backup.service =====\n# /etc/systemd/system/skladchik-reports-monitor-backup.service\n[Unit]\nDescription=Backup Skladchik reports monitor from edge-vm to encrypted clouds\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/skladchik-reports-monitor-backup\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\n\n# /etc/systemd/system/skladchik-reports-monitor-backup.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/skladchik-reports-monitor-backup ; argv[]=/usr/local/sbin/skladchik-reports-monitor-backup ; ignore_errors=no ; start_time=[Tue 2026-07-21 04:34:48 MSK] ; stop_time=[Tue 2026-07-21 04:35:22 MSK] ; pid=2399885 ; code=exited ; status=0 }\nActiveState=inactive\n===== REFERENCED_SCRIPTS =====\n----- /opt/homelab-backup/backup-emergency-pack-v0.sh -----\n41:rclone listremotes > \"$PAYLOAD/inventory/rclone_remotes.txt\" 2>&1 || true\n59:It is deleted only after both encrypted cloud copies are uploaded and fully verified.\n60:Recovery requires offline/break-glass access to rclone crypt credentials.\n65:rm -rf \"$PAYLOAD\"\n79: \"archive_size_bytes\": $ARCHIVE_SIZE_BYTES,\n83: \"local_staging_policy\": \"delete_after_both_clouds_verified\",\n90:echo \"[STEP] local checksum\"\n91:(cd \"$STAGE\" && sha256sum \"$ARCHIVE\" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null)\n94:rclone copy \"$STAGE\" \"$REMOTE_A_JOB\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n95:homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$REMOTE_A_JOB/$ARCHIVE\"\n98:rclone copy \"$STAGE\" \"$REMOTE_B_JOB\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n99:homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$REMOTE_B_JOB/$ARCHIVE\"\n102:rclone copy \"$REMOTE_A_JOB\" \"$VERIFY/cloud_a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10; homelab-rehydrate-chunked-dir \"$VERIFY/cloud_a\"\n103:(cd \"$VERIFY/cloud_a\" && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB_ID\" '.job_id == $job' manifest.json >/dev/null)\n106:rclone copy \"$REMOTE_B_JOB\" \"$VERIFY/cloud_b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10; homelab-rehydrate-chunked-dir \"$VERIFY/cloud_b\"\n107:(cd \"$VERIFY/cloud_b\" && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB_ID\" '.job_id == $job' manifest.json >/dev/null)\n110:(cd \"$STAGE\" && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null)\n113:rclone copyto \"$STAGE/latest-backup-index.json\" \"$REMOTE_A_LATEST\"\n114:rclone copyto \"$STAGE/latest-backup-index.json\" \"$REMOTE_B_LATEST\"\n115:rclone cat \"$REMOTE_A_LATEST\" | jq -e --arg job \"$JOB_ID\" '.job_id == $job' >/dev/null\n116:rclone cat \"$REMOTE_B_LATEST\" | jq -e --arg job \"$JOB_ID\" '.job_id == $job' >/dev/null\n124: \"archive_size_bytes\": $ARCHIVE_SIZE_BYTES,\n128: \"local_backup_deleted\": true\n132:echo \"[STEP] delete local staging and verification copies\"\n133:rm -rf \"$STAGE\" \"$VERIFY\"\n138:echo \"[SUCCESS] uploaded to 2 clouds, verified from both clouds, local staging deleted\"\n----- /opt/homelab-backup/backup-emergency-pack-v0.sh.bak-chunkall-20260710T011410Z -----\n41:rclone listremotes > \"$PAYLOAD/inventory/rclone_remotes.txt\" 2>&1 || true\n59:It is deleted only after both encrypted cloud copies are uploaded and fully verified.\n60:Recovery requires offline/break-glass access to rclone crypt credentials.\n65:rm -rf \"$PAYLOAD\"\n79: \"archive_size_bytes\": $ARCHIVE_SIZE_BYTES,\n83: \"local_staging_policy\": \"delete_after_both_clouds_verified\",\n90:echo \"[STEP] local checksum\"\n91:(cd \"$STAGE\" && sha256sum \"$ARCHIVE\" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null)\n94:rclone copy \"$STAGE\" \"$REMOTE_A_JOB\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 --stats=30s\n97:rclone copy \"$STAGE\" \"$REMOTE_B_JOB\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 --stats=30s\n100:rclone copy \"$REMOTE_A_JOB\" \"$VERIFY/cloud_a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10\n101:(cd \"$VERIFY/cloud_a\" && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB_ID\" '.job_id == $job' manifest.json >/dev/null)\n104:rclone copy \"$REMOTE_B_JOB\" \"$VERIFY/cloud_b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10\n105:(cd \"$VERIFY/cloud_b\" && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB_ID\" '.job_id == $job' manifest.json >/dev/null)\n108:(cd \"$STAGE\" && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null)\n111:rclone copyto \"$STAGE/latest-backup-index.json\" \"$REMOTE_A_LATEST\"\n112:rclone copyto \"$STAGE/latest-backup-index.json\" \"$REMOTE_B_LATEST\"\n113:rclone cat \"$REMOTE_A_LATEST\" | jq -e --arg job \"$JOB_ID\" '.job_id == $job' >/dev/null\n114:rclone cat \"$REMOTE_B_LATEST\" | jq -e --arg job \"$JOB_ID\" '.job_id == $job' >/dev/null\n122: \"archive_size_bytes\": $ARCHIVE_SIZE_BYTES,\n126: \"local_backup_deleted\": true\n130:echo \"[STEP] delete local staging and verification copies\"\n131:rm -rf \"$STAGE\" \"$VERIFY\"\n136:echo \"[SUCCESS] uploaded to 2 clouds, verified from both clouds, local staging deleted\"\n----- /usr/local/sbin/appbackup-dockge-stacks -----\n25:echo \"[STEP] copy app data\"\n30:rm -rf \"$PAYLOAD\"\n42: \"archive_size_bytes\": $SIZE,\n46: \"local_staging_policy\": \"delete_after_both_clouds_verified\",\n51:echo \"[STEP] local checksum\"\n52:(cd \"$STAGE\" && sha256sum \"$ARCHIVE\" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e . manifest.json >/dev/null)\n54:echo \"[STEP] upload cloud A metadata\"; rclone copy \"$STAGE\" \"$RA\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n55:echo \"[STEP] upload cloud A archive chunks\"; rclone deletefile \"$RA/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RA/$ARCHIVE\"\n56:echo \"[STEP] upload cloud B metadata\"; rclone copy \"$STAGE\" \"$RB\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n57:echo \"[STEP] upload cloud B archive chunks\"; rclone deletefile \"$RB/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RB/$ARCHIVE\"\n58:echo \"[STEP] verify cloud A chunked\"; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-stage-verify-chunked \"$RA\" \"$ARCHIVE\" \"$VERIFY/a\" \"$JOB\"\n59:echo \"[STEP] verify cloud B chunked\"; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-stage-verify-chunked \"$RB\" \"$ARCHIVE\" \"$VERIFY/b\" \"$JOB\"\n60:echo \"[STEP] upload latest index\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RAL\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RBL\"; rclone cat \"$RAL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null; rclone cat \"$RBL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n68: \"archive_size_bytes\": $SIZE,\n72: \"local_backup_deleted\": true\n75:echo \"[STEP] delete local staging and verification\"\n76:rm -rf \"$STAGE\" \"$VERIFY\"\n79:echo \"[SUCCESS] app backup uploaded to 2 clouds, verified from both clouds, local staging deleted\"\n----- /usr/local/sbin/appbackup-dockge-stacks.bak-chunkall-20260710T011410Z -----\n25:echo \"[STEP] copy app data\"\n30:rm -rf \"$PAYLOAD\"\n42: \"archive_size_bytes\": $SIZE,\n46: \"local_staging_policy\": \"delete_after_both_clouds_verified\",\n51:echo \"[STEP] local checksum\"\n52:(cd \"$STAGE\" && sha256sum \"$ARCHIVE\" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e . manifest.json >/dev/null)\n54:rclone copy \"$STAGE\" \"$RA\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 --stats=30s\n56:rclone copy \"$STAGE\" \"$RB\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 --stats=30s\n58:rclone copy \"$RA\" \"$VERIFY/a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10\n59:(cd \"$VERIFY/a\" && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n61:rclone copy \"$RB\" \"$VERIFY/b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10\n62:(cd \"$VERIFY/b\" && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n64:(cd \"$STAGE\" && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e . manifest.json >/dev/null)\n66:rclone copyto \"$STAGE/latest-backup-index.json\" \"$RAL\"\n67:rclone copyto \"$STAGE/latest-backup-index.json\" \"$RBL\"\n68:rclone cat \"$RAL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n69:rclone cat \"$RBL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n77: \"archive_size_bytes\": $SIZE,\n81: \"local_backup_deleted\": true\n84:echo \"[STEP] delete local staging and verification\"\n85:rm -rf \"$STAGE\" \"$VERIFY\"\n88:echo \"[SUCCESS] app backup uploaded to 2 clouds, verified from both clouds, local staging deleted\"\n----- /usr/local/sbin/appbackup-restore-check -----\n22:ROOT=\"/var/tmp/homelab-app-restore-check/${JOB}\"\n24:rm -rf \"$ROOT\" && mkdir -p \"$ROOT/a\" \"$ROOT/b\"\n25:trap \"echo \\\"ERROR app restore-check failed; kept: $ROOT\\\"; exit 1\" ERR\n26:echo \"restore-check app=$APP job=$JOB schema=$SCHEMA\"\n27:rclone copy \"$RA\" \"$ROOT/a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >\"${LOGBASE}-a.log\" 2>&1\n30:(cd \"$ROOT/a\" && sha256sum -c checksums.sha256 >\"${LOGBASE}-a-sha.log\" && ARCHIVE=\"$(jq -r \".archive_name\" manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n31:echo \"OK cloud A app restore-check\"\n32:rclone copy \"$RB\" \"$ROOT/b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >\"${LOGBASE}-b.log\" 2>&1\n35:(cd \"$ROOT/b\" && sha256sum -c checksums.sha256 >\"${LOGBASE}-b-sha.log\" && ARCHIVE=\"$(jq -r \".archive_name\" manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n36:echo \"OK cloud B app restore-check\"\n37:rm -rf \"$ROOT\"\n38:rmdir /var/tmp/homelab-app-restore-check 2>/dev/null || true\n39:rm -f \"${LOGBASE}-a.log\" \"${LOGBASE}-a-sha.log\" \"${LOGBASE}-b.log\" \"${LOGBASE}-b-sha.log\"\n41:echo \"OK app restore-check both clouds\"\n----- /usr/local/sbin/appbackup-restore-check.bak-chunkall-20260710T011410Z -----\n9:ROOT=\"/var/tmp/homelab-app-restore-check/${JOB}\"\n10:rm -rf \"$ROOT\" && mkdir -p \"$ROOT/a\" \"$ROOT/b\"\n11:trap \"echo \\\"ERROR app restore-check failed; kept: $ROOT\\\"; exit 1\" ERR\n12:echo \"restore-check app=$APP job=$JOB\"\n13:rclone copy \"$RA\" \"$ROOT/a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/app-restore-a.log 2>&1\n14:(cd \"$ROOT/a\" && sha256sum -c checksums.sha256 >/tmp/app-restore-a-sha.log && ARCHIVE=\"$(jq -r .archive_name manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n15:echo \"OK cloud A app restore-check\"\n16:rclone copy \"$RB\" \"$ROOT/b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/app-restore-b.log 2>&1\n17:(cd \"$ROOT/b\" && sha256sum -c checksums.sha256 >/tmp/app-restore-b-sha.log && ARCHIVE=\"$(jq -r .archive_name manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n18:echo \"OK cloud B app restore-check\"\n19:rm -rf \"$ROOT\"; rmdir /var/tmp/homelab-app-restore-check 2>/dev/null || true\n21:echo \"OK app restore-check both clouds\"\n----- /usr/local/sbin/appbackup-restore-check.bak-dual-read-20260711T051924Z -----\n9:ROOT=\"/var/tmp/homelab-app-restore-check/${JOB}\"\n10:rm -rf \"$ROOT\" && mkdir -p \"$ROOT/a\" \"$ROOT/b\"\n11:trap \"echo \\\"ERROR app restore-check failed; kept: $ROOT\\\"; exit 1\" ERR\n12:echo \"restore-check app=$APP job=$JOB\"\n13:rclone copy \"$RA\" \"$ROOT/a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/app-restore-a.log 2>&1; homelab-rehydrate-chunked-dir \"$ROOT/a\" >>/tmp/app-restore-a.log 2>&1\n14:(cd \"$ROOT/a\" && sha256sum -c checksums.sha256 >/tmp/app-restore-a-sha.log && ARCHIVE=\"$(jq -r .archive_name manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n15:echo \"OK cloud A app restore-check\"\n16:rclone copy \"$RB\" \"$ROOT/b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/app-restore-b.log 2>&1; homelab-rehydrate-chunked-dir \"$ROOT/b\" >>/tmp/app-restore-b.log 2>&1\n17:(cd \"$ROOT/b\" && sha256sum -c checksums.sha256 >/tmp/app-restore-b-sha.log && ARCHIVE=\"$(jq -r .archive_name manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n18:echo \"OK cloud B app restore-check\"\n19:rm -rf \"$ROOT\"; rmdir /var/tmp/homelab-app-restore-check 2>/dev/null || true\n21:echo \"OK app restore-check both clouds\"\n----- /usr/local/sbin/appbackup-restore-check.bak-flat-read-20260711T052854Z -----\n22:ROOT=\"/var/tmp/homelab-app-restore-check/${JOB}\"\n24:rm -rf \"$ROOT\" && mkdir -p \"$ROOT/a\" \"$ROOT/b\"\n25:trap \"echo \\\"ERROR app restore-check failed; kept: $ROOT\\\"; exit 1\" ERR\n26:echo \"restore-check app=$APP job=$JOB schema=$SCHEMA\"\n27:rclone copy \"$RA\" \"$ROOT/a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >\"${LOGBASE}-a.log\" 2>&1\n29:(cd \"$ROOT/a\" && sha256sum -c checksums.sha256 >\"${LOGBASE}-a-sha.log\" && ARCHIVE=\"$(jq -r \".archive_name\" manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n30:echo \"OK cloud A app restore-check\"\n31:rclone copy \"$RB\" \"$ROOT/b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >\"${LOGBASE}-b.log\" 2>&1\n33:(cd \"$ROOT/b\" && sha256sum -c checksums.sha256 >\"${LOGBASE}-b-sha.log\" && ARCHIVE=\"$(jq -r \".archive_name\" manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n34:echo \"OK cloud B app restore-check\"\n35:rm -rf \"$ROOT\"\n36:rmdir /var/tmp/homelab-app-restore-check 2>/dev/null || true\n37:rm -f \"${LOGBASE}-a.log\" \"${LOGBASE}-a-sha.log\" \"${LOGBASE}-b.log\" \"${LOGBASE}-b-sha.log\"\n39:echo \"OK app restore-check both clouds\"\n----- /usr/local/sbin/appbackup-restore-check.bak-flat-read-v2-20260711T053129Z -----\n22:ROOT=\"/var/tmp/homelab-app-restore-check/${JOB}\"\n24:rm -rf \"$ROOT\" && mkdir -p \"$ROOT/a\" \"$ROOT/b\"\n25:trap \"echo \\\"ERROR app restore-check failed; kept: $ROOT\\\"; exit 1\" ERR\n26:echo \"restore-check app=$APP job=$JOB schema=$SCHEMA\"\n27:rclone copy \"$RA\" \"$ROOT/a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >\"${LOGBASE}-a.log\" 2>&1\n29:(cd \"$ROOT/a\" && sha256sum -c checksums.sha256 >\"${LOGBASE}-a-sha.log\" && ARCHIVE=\"$(jq -r \".archive_name\" manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n30:echo \"OK cloud A app restore-check\"\n31:rclone copy \"$RB\" \"$ROOT/b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >\"${LOGBASE}-b.log\" 2>&1\n33:(cd \"$ROOT/b\" && sha256sum -c checksums.sha256 >\"${LOGBASE}-b-sha.log\" && ARCHIVE=\"$(jq -r \".archive_name\" manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n34:echo \"OK cloud B app restore-check\"\n35:rm -rf \"$ROOT\"\n36:rmdir /var/tmp/homelab-app-restore-check 2>/dev/null || true\n37:rm -f \"${LOGBASE}-a.log\" \"${LOGBASE}-a-sha.log\" \"${LOGBASE}-b.log\" \"${LOGBASE}-b-sha.log\"\n39:echo \"OK app restore-check both clouds\"\n----- /usr/local/sbin/appbackup-retention -----\n32:trap \"rm -rf \\\"$WORK\\\"\" EXIT\n35: rclone lsf \"${REMOTE%/}\" --dirs-only --max-depth 1 > \"$ROOT_LIST\"\n36: rclone lsf \"${REMOTE%/}/${NODE}/apps/${CANONICAL}\" --dirs-only --max-depth 1 > \"$LEGACY_LIST\"\n39: rclone lsf \"${REMOTE%/}/hb\" --dirs-only --max-depth 1 > \"$HB_LIST\"\n40: if grep -Fxq \"a/\" \"$HB_LIST\"; then rclone lsf \"${REMOTE%/}/hb/a\" --dirs-only --max-depth 1 > \"$NEW_LIST\"; fi\n73: rclone cat \"${REMOTE%/}/${LATEST_REL}/manifest.json\" | jq -e --arg job \"$LATEST\" \".job_id == \\$job\" >/dev/null || { echo \"REFUSE latest missing/bad remote_index=$INDEX\"; exit 5; }\n78: if (( TOTAL <= KEEP )); then echo \"nothing to delete\"; continue; fi\n83: [[ -n \"$LATEST_REL\" && \"$REL\" == \"$LATEST_REL\" ]] && { echo \"REFUSE delete latest ${REL}\"; exit 7; }\n85: if [[ \"$MODE\" == apply ]]; then echo \"DELETE schema=dual path=$REL\"; rclone purge \"$TARGET\"; else echo \"WOULD_DELETE schema=dual path=$REL\"; fi\n----- /usr/local/sbin/appbackup-retention.bak-dual-read-20260711T051924Z -----\n15:for REMOTE in \"$REMOTE_A\" \"$REMOTE_B\"; do BASE=\"${REMOTE%/}/${NODE}/apps/${APP}\"; echo \"--- $BASE ---\"; if [[ \"$MODE\" == \"apply\" ]]; then rclone cat \"$BASE/$LATEST/manifest.json\" | jq -e --arg job \"$LATEST\" \".job_id == \\$job\" >/dev/null || { echo \"REFUSE latest missing/bad on $BASE\"; exit 4; }; fi; mapfile -t DIRS < <(rclone lsf \"$BASE\" --dirs-only 2>/dev/null | while read -r D; do D=\"${D%/}\"; [[ \"$D\" == homelab-${NODE}-app-${APP}-* ]] && echo \"$D\"; done | sort); TOTAL=\"${#DIRS[@]}\"; echo \"total=$TOTAL keep=$KEEP mode=$MODE latest=${LATEST:-none}\"; if (( TOTAL <= KEEP )); then echo \"nothing to delete\"; continue; fi; DELETE_COUNT=$((TOTAL-KEEP)); for ((i=0; i/dev/null && echo \"OK backup framework\" ;;\n13: restore-check) backup-restore-check ;;\n14: audit) backupctl check && backup-restore-check && backupctl retention 7 dry-run && backup-report >/dev/null && echo \"OK audit finished\" ;;\n16: remotes) source /etc/homelab-backup.env && NODE=\"$(hostname -s)\" && echo \"A=$REMOTE_A\" && echo \"B=$REMOTE_B\" && rclone lsd \"${REMOTE_A%/}/${NODE}/emergency-pack\" | tail -n 5 && rclone lsd \"${REMOTE_B%/}/${NODE}/emergency-pack\" | tail -n 5 ;;\n19: *) echo \"usage: backupctl run|status|lastlog|check|restore-check|audit|report|remotes|retention|timer\"; exit 2 ;;\n----- /usr/local/sbin/backup-restore-check -----\n8:ROOT=\"/var/tmp/homelab-restore-check/${JOB}\"\n9:rm -rf \"$ROOT\" && mkdir -p \"$ROOT/a\" \"$ROOT/b\"\n10:trap 'echo \"ERROR restore-check failed; kept: \"; exit 1' ERR\n11:echo \"restore-check job=$JOB\"\n12:rclone copy \"$RA\" \"$ROOT/a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-a.log 2>&1; homelab-rehydrate-chunked-dir \"$ROOT/a\" >>/tmp/homelab-restore-a.log 2>&1\n13:(cd \"$ROOT/a\" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-a-sha.log && ARCHIVE=\"$(jq -r .archive_name manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n14:echo \"OK cloud A restore-check\"\n15:rclone copy \"$RB\" \"$ROOT/b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-b.log 2>&1; homelab-rehydrate-chunked-dir \"$ROOT/b\" >>/tmp/homelab-restore-b.log 2>&1\n16:(cd \"$ROOT/b\" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-b-sha.log && ARCHIVE=\"$(jq -r .archive_name manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n17:echo \"OK cloud B restore-check\"\n18:rm -rf \"$ROOT\"\n20:echo \"OK restore-check both clouds\"\n----- /usr/local/sbin/backup-restore-check.bak-chunkall-20260710T011410Z -----\n8:ROOT=\"/var/tmp/homelab-restore-check/${JOB}\"\n9:rm -rf \"$ROOT\" && mkdir -p \"$ROOT/a\" \"$ROOT/b\"\n10:trap 'echo \"ERROR restore-check failed; kept: \"; exit 1' ERR\n11:echo \"restore-check job=$JOB\"\n12:rclone copy \"$RA\" \"$ROOT/a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-a.log 2>&1\n13:(cd \"$ROOT/a\" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-a-sha.log && ARCHIVE=\"$(jq -r .archive_name manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n14:echo \"OK cloud A restore-check\"\n15:rclone copy \"$RB\" \"$ROOT/b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-b.log 2>&1\n16:(cd \"$ROOT/b\" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-b-sha.log && ARCHIVE=\"$(jq -r .archive_name manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n17:echo \"OK cloud B restore-check\"\n18:rm -rf \"$ROOT\"\n20:echo \"OK restore-check both clouds\"\n----- /usr/local/sbin/backup-retention -----\n18: if [[ \"$MODE\" == \"apply\" ]]; then rclone cat \"$BASE/$LATEST/manifest.json\" | jq -e --arg job \"$LATEST\" \".job_id == \\$job\" >/dev/null || { echo \"REFUSE latest missing/bad on $BASE\"; exit 4; }; fi\n19: mapfile -t DIRS < <(rclone lsf \"$BASE\" --dirs-only 2>/dev/null | while read -r D; do D=\"${D%/}\"; [[ \"$D\" == homelab-${NODE}-${JOB}-* ]] && echo \"$D\"; done | sort)\n22: if (( TOTAL <= KEEP )); then echo \"nothing to delete\"; continue; fi\n24: for ((i=0; i \\$OUT; : > \\$ERR; vzdump '$VMID' --mode snapshot --compress zstd --dumpdir '$RUN_DIR' --remove 0 >\\$OUT 2>\\$ERR; RC=\\$?; ARCHIVE=\\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\\n' | sort -n | tail -n1 | cut -d' ' -f2-); LOGFILE=\\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.log' -printf '%T@ %p\\n' | sort -n | tail -n1 | cut -d' ' -f2-); SIZE=0; SHA=none; if [ -n \\\"\\$ARCHIVE\\\" ] && [ -f \\\"\\$ARCHIVE\\\" ]; then SIZE=\\$(stat -c %s \\\"\\$ARCHIVE\\\"); SHA=\\$(sha256sum \\\"\\$ARCHIVE\\\" | awk '{print \\$1}'); fi; echo VZDUMP_RC=\\$RC; echo ARCHIVE=\\$ARCHIVE; echo LOGFILE=\\$LOGFILE; echo SIZE=\\$SIZE; echo SHA=\\$SHA; echo OUT_TAIL=\\$(tail -n 6 \\$OUT | tr '\\n' '|'); echo ERR_TAIL=\\$(tail -n 6 \\$ERR | tr '\\n' '|')\")\"\n62:dest \"mkdir -p '$OFFHOST_DIR' && rm -f '$DEST_PATH.tmp' '$DEST_PATH.sha256'\"\n74:echo \"STATUS=OK TS=$TS TYPE=edge-vm-vzdump-backup NODE=$NODE VMID=$VMID ARCHIVE=$ARCHIVE SIZE=$SIZE SHA256=$SHA SCSI1_BACKUP0=yes NOTES=timer_run\" > \"$HEALTH_DIR/edge-vm-vzdump-backup.txt\"\n75:echo \"STATUS=OK TS=$TS TYPE=edge-vm-vzdump-offhost NODE=$NODE VMID=$VMID HOST=pve02 DEST=$DEST_PATH SIZE=$SIZE SHA256=$SHA NOTES=timer_run\" > \"$HEALTH_DIR/edge-vm-vzdump-offhost.txt\"\n76:echo \"STATUS=OK TS=$TS TYPE=edge-vm-vzdump-restore-verify HOST=pve02 VMID=$VMID ARCHIVE=$DEST_PATH SIZE=$SIZE SHA256=$SHA ZSTD_RC=0 VMA_RC=0 NOTES=timer_run\" > \"$HEALTH_DIR/edge-vm-vzdump-restore.txt\"\n77:chmod 0644 \"$HEALTH_DIR/edge-vm-vzdump-backup.txt\" \"$HEALTH_DIR/edge-vm-vzdump-offhost.txt\" \"$HEALTH_DIR/edge-vm-vzdump-restore.txt\"\n79:remote \"find '$BASE_DIR' -mindepth 1 -maxdepth 1 -type d -name 'edge-vm-*' -printf '%T@ %p\\n' | sort -nr | awk 'NR>${KEEP_LOCAL}{print \\$2}' | xargs -r rm -rf\"\n80:dest \"find '$OFFHOST_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\\n' | sort -nr | awk 'NR>${KEEP_OFFHOST}{print \\$2}' | while read f; do rm -f \\\"\\$f\\\" \\\"\\$f.sha256\\\"; done\"\n----- /usr/local/sbin/homelab-capacity-risk -----\n32:LARGEST_BYTES=\"$(find /var/lib/vz/dump -maxdepth 1 -type f \\( -name \"vzdump-qemu-*.vma.zst\" -o -name \"vzdump-lxc-*.tar.zst\" \\) -printf \"%s\\n\" 2>/dev/null | sort -n | tail -n1)\"\n----- /usr/local/sbin/homelab-chat-run -----\n272: re.compile(r\"(?i)\\brclone\\.conf\\b\"),\n388: \"reference_register_checked\": True,\n390: \"error_register_checked\": True,\n489: rm -f \\\n----- /usr/local/sbin/homelab-edge-vm-offhost-freshness -----\n9:OFFHOST_DIR=/mnt/staging/offhost/edge-vm-vzdump-from-pve03\n85: /var/lib/vz/dump/vzdump-qemu-130-*.vma.zst) ;;\n96:dest_size=\"$(field DEST_SIZE <<<\"$inventory_out\")\"\n100:[[ \"$dest_size\" =~ ^[0-9]+$ && \"$count\" =~ ^[0-9]+$ && \"$available\" =~ ^[0-9]+$ ]]\n103:last_oldest_size=0\n107: [[ \"$dest_size\" == \"$SOURCE_SIZE\" && \"$dest_sha\" == \"$SOURCE_SHA\" ]]\n110: \"$dest_size\" \"$last_oldest\" \"$last_oldest_size\" \"$last_oldest_sha\" \"$count\" \"$RESERVE\")\"\n115: [[ \"$dest_size\" -ge 0 && \"$dest_size\" -lt \"$SOURCE_SIZE\" ]]\n119: last_oldest_size=\"$(field OLDEST_SIZE <<<\"$inventory_out\")\"\n122: [[ \"$last_oldest_size\" =~ ^[0-9]+$ && \"$last_oldest_size\" -gt 1000000000 ]]\n126: \"$dest_size\" \"$last_oldest\" \"$last_oldest_size\" \"$last_oldest_sha\" \"$before_count\" \"$RESERVE\")\"\n130: [[ \"$(field DEST_SIZE <<<\"$inventory_out\")\" == \"$dest_size\" ]]\n136: remaining=$((SOURCE_SIZE-dest_size))\n140: copy_out=\"$(run_pve03 copy \"$SOURCE_PATH\" \"$SOURCE_SIZE\" \"$SOURCE_SHA\" \"$DST_IP\" \"$OFFHOST_DIR\")\"\n141: grep -Fq 'SOURCE_COPY_STATUS=OK' <<<\"$copy_out\"\n146: \"$dest_size\" \"$last_oldest\" \"$last_oldest_size\" \"$last_oldest_sha\" \"$expected_final_count\" \"$RESERVE\")\"\n156:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-backup NODE=%s VMID=%s ARCHIVE=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\\n' \\\n158: >\"$HEALTH_DIR/edge-vm-vzdump-backup.txt\"\n159:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-offhost NODE=%s VMID=%s HOST=%s DEST=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\\n' \\\n161: >\"$HEALTH_DIR/edge-vm-vzdump-offhost.txt\"\n162:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-restore-verify HOST=%s VMID=%s ARCHIVE=%s SIZE=%s SHA256=%s ZSTD_RC=0 VMA_RC=0 NOTES=nightly_existing_dump_freshness\\n' \\\n164: >\"$HEALTH_DIR/edge-vm-vzdump-restore.txt\"\n167: \"$HEALTH_DIR/edge-vm-vzdump-backup.txt\" \\\n168: \"$HEALTH_DIR/edge-vm-vzdump-offhost.txt\" \\\n169: \"$HEALTH_DIR/edge-vm-vzdump-restore.txt\"\n----- /usr/local/sbin/homelab-extended-appbackup-run -----\n12:run_one(){ APP=\"$1\"; HOST=\"$2\"; PATHS=\"$3\"; NOTE=\"$4\"; CONS=\"$5\"; JOB_ID=\"homelab-${NODE}-app-${APP}-${RUN_TS}\"; export JOB_ID; SHORT_TS=\"${RUN_TS:2:6}-${RUN_TS:9:6}\"; export BACKUP_PATH_REL=\"$(homelab-backup-path job a \"$APP\" \"$SHORT_TS\")\"; STAGE=\"${STAGING_ROOT}/${JOB_ID}\"; PAYLOAD=\"$STAGE/payload\"; ARCHIVE=\"${JOB_ID}.tar.gz\"; LOG=\"/tmp/extended-appbackup-${APP}-${RUN_TS}.log\"; rm -rf \"$STAGE\"; mkdir -p \"$PAYLOAD/local\"; echo \"### app=$APP\" >> \"$REPORT\"; echo \"host=$HOST\" >> \"$REPORT\"; echo \"source_note=$NOTE\" >> \"$REPORT\"; ssh -n -o BatchMode=yes -o ConnectTimeout=30 \"$HOST\" \"sudo tar -C / --ignore-failed-read --exclude=.env --exclude=*.env --exclude=.env.* --exclude=*.bak* -czf - $PATHS\" > \"$PAYLOAD/local/${APP}-data.tar.gz\" 2>\"$LOG\"; TAR_RC=$?; if [ \"$TAR_RC\" -eq 0 ] && tar -tzf \"$PAYLOAD/local/${APP}-data.tar.gz\" >/dev/null 2>&1; then SIZE=\"$(stat -c%s \"$PAYLOAD/local/${APP}-data.tar.gz\" 2>/dev/null || echo 0)\"; echo \"archive_payload_size_bytes=$SIZE\" >> \"$REPORT\"; if [ \"$SIZE\" -le 128 ]; then echo \"backup=BAD reason=archive_too_small\" >> \"$REPORT\"; BAD=$((BAD+1)); rm -rf \"$STAGE\"; else sha256sum \"$PAYLOAD/local/${APP}-data.tar.gz\" > \"$PAYLOAD/local/${APP}-data.tar.gz.sha256\"; jq -n --arg type \"extended-appbackup\" --arg app \"$APP\" --arg host \"$HOST\" --arg source_note \"$NOTE\" --arg archive_payload_size_bytes \"$SIZE\" --arg env_files_read \"no\" --arg secret_values_printed \"no\" --arg consistency \"$CONS\" \"{type:\\$type,app_name:\\$app,host:\\$host,source_note:\\$source_note,archive_payload_size_bytes:(\\$archive_payload_size_bytes|tonumber),env_files_read:\\$env_files_read,secret_values_printed:\\$secret_values_printed,consistency:\\$consistency}\" > \"$STAGE/metadata.json\"; tar -C \"$PAYLOAD\" -czf \"$STAGE/$ARCHIVE\" .; rm -rf \"$PAYLOAD\"; JOB_ID=\"$JOB_ID\" homelab-stage-2cloud-flat \"$APP\" \"$STAGE\" \"$ARCHIVE\" > \"$LOG\" 2>&1; RC=$?; if [ \"$RC\" -eq 0 ]; then echo \"backup=OK\" >> \"$REPORT\"; grep -E \"^\\[STEP\\]|^\\[SUCCESS\\]\" \"$LOG\" >> \"$REPORT\"; appbackupctl restore-check \"$APP\" >> \"$REPORT\" 2>&1 && echo \"restore_check=OK\" >> \"$REPORT\" || echo \"restore_check=WARN_non_registry\" >> \"$REPORT\"; OK=$((OK+1)); else echo \"backup=BAD rc=$RC\" >> \"$REPORT\"; grep -E \"^\\[ERROR\\]|\\[STEP\\]|ERROR|error|failed|Failed|required|missing|denied|forbidden|checksum|rclone|copy|mkdir|jq|tar\" \"$LOG\" | tail -n 60 >> \"$REPORT\"; BAD=$((BAD+1)); fi; fi; else echo \"tar=BAD rc=$TAR_RC\" >> \"$REPORT\"; grep -E \"ERROR|error|failed|Failed|denied|No such|Cannot|tar:\" \"$LOG\" | tail -n 40 >> \"$REPORT\"; BAD=$((BAD+1)); rm -rf \"$STAGE\"; fi; echo >> \"$REPORT\"; }\n21:run_one healthchecks \"$MON\" \"var/lib/healthchecks opt/stacks/healthchecks/local_settings.py\" \"healthchecks data plus local settings\" \"emergency_data_config_archive\"\n----- /usr/local/sbin/homelab-golden-state-extended-overlay -----\n26:rclone copyto \"$JSON\" \"$RA\" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1))\n27:rclone copyto \"$JSON\" \"$RB\" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1))\n28:rclone cat \"$RA\" 2>/dev/null | jq -e --arg cp \"$CP\" \".control_plane_job == \\$cp\" >/dev/null || BAD=$((BAD+1))\n29:rclone cat \"$RB\" 2>/dev/null | jq -e --arg cp \"$CP\" \".control_plane_job == \\$cp\" >/dev/null || BAD=$((BAD+1))\n----- /usr/local/sbin/homelab-golden-state-index -----\n28:rclone copyto \"$JSON\" \"$RA\" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1))\n29:rclone copyto \"$JSON\" \"$RB\" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1))\n30:rclone cat \"$RA\" 2>/dev/null | jq -e --arg cp \"$CP\" --arg ext \"$EXT\" \".control_plane_job == \\$cp and .health.extended_appbackup == \\$ext and .extended_appbackup_health == \\$ext\" >/dev/null || BAD=$((BAD+1))\n31:rclone cat \"$RB\" 2>/dev/null | jq -e --arg cp \"$CP\" --arg ext \"$EXT\" \".control_plane_job == \\$cp and .health.extended_appbackup == \\$ext and .extended_appbackup_health == \\$ext\" >/dev/null || BAD=$((BAD+1))\n----- /usr/local/sbin/homelab-health-metrics -----\n16:category_for_check(){\n22: *backup*|*vzdump*) printf \"%s\" \"backup\" ;;\n31:tier_for_check(){\n34: npmplus-kuma-config-*|authentik-*|gitea-*|vaultwarden-*|netbox-*|mail-cloud-restore-drill|edge-vm-vzdump-*|mkdocs-runtime-publish|npmplus-certificate-expiry) printf \"%s\" \"P0\" ;;\n69: check=\"${base%.txt}\"\n70: category=\"$(category_for_check \"$check\")\"\n71: tier=\"$(tier_for_check \"$check\")\"\n75: check_esc=\"$(esc_label \"$check\")\"\n79: labels=\"host=\\\"$host_esc\\\",check=\\\"$check_esc\\\",category=\\\"$cat_esc\\\",tier=\\\"$tier_esc\\\",file=\\\"$file_esc\\\"\"\n----- /usr/local/sbin/homelab-ingress-standby-refresh -----\n16: rm -f \"$TMP_STATE\" \"$RELAY\" \"$RESOURCES\"\n74:run_node \"$SOURCE_NODE\" vzdump 130 \\\n78: --remove 0 \\\n83:\"find /var/lib/vz/dump -maxdepth 1 -type f -name 'vzdump-qemu-130-*.vma.zst' -printf '%T@ %p\\n' | sort -rn | head -n 1 | cut -d' ' -f2-\"\n106:\"chattr -i '$DEST_FILE' '$DEST_FILE.sha256' '$DEST_FILE.env' 2>/dev/null || true; rm -f '$DEST_TMP'\"\n155: -name 'vzdump-qemu-130-*.vma.zst' \\\n164: rm -f \"$OLD\" \"$OLD.sha256\" \"$OLD.env\"\n173:run_node \"$SOURCE_NODE\" rm -f \"$SOURCE_BACKUP\"\n----- /usr/local/sbin/homelab-mail-cloud-critical-upload -----\n50:EXCLUDED=edge-vm-vzdump-large-archive,rclone-config,secrets,raw-env\n80: rclone mkdir \"${R}:$REMOTE_DIR\" > \"$WORK/logs/${SAFE}_mkdir.out\" 2>\"$WORK/logs/${SAFE}_mkdir.err\"\n82: homelab-rclone-copy-chunked \"$BUNDLE\" \"${R}:$REMOTE_DIR/$(basename \"$BUNDLE\")\" > \"$WORK/logs/${SAFE}_upload_bundle.out\" 2>\"$WORK/logs/${SAFE}_upload_bundle.err\"\n84: rclone copyto \"$BUNDLE_SHA\" \"${R}:$REMOTE_DIR/$(basename \"$BUNDLE_SHA\")\" --checksum > \"$WORK/logs/${SAFE}_upload_sha.out\" 2>\"$WORK/logs/${SAFE}_upload_sha.err\"\n86: rclone copyto \"$WORK/components/MANIFEST.txt\" \"${R}:$REMOTE_DIR/MANIFEST.txt\" --checksum > \"$WORK/logs/${SAFE}_upload_manifest.out\" 2>\"$WORK/logs/${SAFE}_upload_manifest.err\"\n88: rclone copyto \"$WORK/components/SHA256SUMS.txt\" \"${R}:$REMOTE_DIR/SHA256SUMS.txt\" --checksum > \"$WORK/logs/${SAFE}_upload_sums.out\" 2>\"$WORK/logs/${SAFE}_upload_sums.err\"\n90: homelab-rclone-cat-chunked \"${R}:$REMOTE_DIR/$(basename \"$BUNDLE\")\" > \"$WORK/download/${SAFE}-$(basename \"$BUNDLE\")\" 2>\"$WORK/logs/${SAFE}_download_bundle.err\"\n108: rm -f \"$WORK/download/${SAFE}-$(basename \"$BUNDLE\")\"\n109: rclone lsf \"${R}:$TARGET_BASE\" --dirs-only 2>/dev/null | sed 's#/$##' | sort > \"$WORK/logs/${SAFE}_dirs.txt\"\n114: [ -n \"$OLD\" ] && rclone purge \"${R}:$TARGET_BASE/$OLD\" >> \"$WORK/logs/${SAFE}_retention.out\" 2>>\"$WORK/logs/${SAFE}_retention.err\"\n140: rm -rf --one-file-system \"$OLD_DIR\"\n----- /usr/local/sbin/homelab-mail-cloud-critical-upload.bak-chunkall-20260710T011410Z -----\n50:EXCLUDED=edge-vm-vzdump-large-archive,rclone-config,secrets,raw-env\n80: rclone mkdir \"${R}:$REMOTE_DIR\" > \"$WORK/logs/${SAFE}_mkdir.out\" 2>\"$WORK/logs/${SAFE}_mkdir.err\"\n82: rclone copyto \"$BUNDLE\" \"${R}:$REMOTE_DIR/$(basename \"$BUNDLE\")\" --checksum > \"$WORK/logs/${SAFE}_upload_bundle.out\" 2>\"$WORK/logs/${SAFE}_upload_bundle.err\"\n84: rclone copyto \"$BUNDLE_SHA\" \"${R}:$REMOTE_DIR/$(basename \"$BUNDLE_SHA\")\" --checksum > \"$WORK/logs/${SAFE}_upload_sha.out\" 2>\"$WORK/logs/${SAFE}_upload_sha.err\"\n86: rclone copyto \"$WORK/components/MANIFEST.txt\" \"${R}:$REMOTE_DIR/MANIFEST.txt\" --checksum > \"$WORK/logs/${SAFE}_upload_manifest.out\" 2>\"$WORK/logs/${SAFE}_upload_manifest.err\"\n88: rclone copyto \"$WORK/components/SHA256SUMS.txt\" \"${R}:$REMOTE_DIR/SHA256SUMS.txt\" --checksum > \"$WORK/logs/${SAFE}_upload_sums.out\" 2>\"$WORK/logs/${SAFE}_upload_sums.err\"\n90: rclone copyto \"${R}:$REMOTE_DIR/$(basename \"$BUNDLE\")\" \"$WORK/download/${SAFE}-$(basename \"$BUNDLE\")\" > \"$WORK/logs/${SAFE}_download_bundle.out\" 2>\"$WORK/logs/${SAFE}_download_bundle.err\"\n108: rm -f \"$WORK/download/${SAFE}-$(basename \"$BUNDLE\")\"\n109: rclone lsf \"${R}:$TARGET_BASE\" --dirs-only 2>/dev/null | sed 's#/$##' | sort > \"$WORK/logs/${SAFE}_dirs.txt\"\n114: [ -n \"$OLD\" ] && rclone purge \"${R}:$TARGET_BASE/$OLD\" >> \"$WORK/logs/${SAFE}_retention.out\" 2>>\"$WORK/logs/${SAFE}_retention.err\"\n----- /usr/local/sbin/homelab-mail-cloud-edge-vm-upload -----\n10:TARGET_BASE=full/edge-vm-vzdump\n60:case \"$SIZE\" in \"\"|*[!0-9]*) fail size_bad 2 ;; esac\n64:case \"$SOURCE\" in /mnt/staging/offhost/edge-vm-vzdump-from-pve03/vzdump-qemu-130-*.vma.zst) ;; *) fail source_path_bad 2 ;; esac\n111:rclone about \"$TARGET_REMOTE:\" > \"$WORK/logs/about.out\" 2>\"$WORK/logs/about.err\" || ABOUT_RC=$?\n114:DIRS=\"$(rclone lsf \"$TARGET_REMOTE:$TARGET_BASE\" --dirs-only 2>/dev/null | sed 's#/$##' | sort || true)\"\n116: if rclone cat \"$TARGET_REMOTE:$TARGET_BASE/$D/$BASENAME.sha256\" 2>/dev/null | grep -q \"$SHA\"; then\n172:P1 Mail Cloud edge-vm recurring chunked vzdump upload\n188:rclone mkdir \"$TARGET_REMOTE:$REMOTE_PARTS\" > \"$WORK/logs/mkdir.out\" 2>\"$WORK/logs/mkdir.err\" || MKDIR_RC=$?\n190:rclone copy \"$PARTS_DIR\" \"$TARGET_REMOTE:$REMOTE_PARTS\" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > \"$WORK/logs/upload_parts.out\" 2>\"$WORK/logs/upload_parts.err\" || UPLOAD_PARTS_RC=$?\n192:rclone copyto \"$WORK/$BASENAME.sha256\" \"$TARGET_REMOTE:$REMOTE_SHA\" --checksum > \"$WORK/logs/upload_sha.out\" 2>\"$WORK/logs/upload_sha.err\" || UPLOAD_SHA_RC=$?\n194:rclone copyto \"$WORK/MANIFEST.txt\" \"$TARGET_REMOTE:$REMOTE_MANIFEST\" --checksum > \"$WORK/logs/upload_manifest.out\" 2>\"$WORK/logs/upload_manifest.err\" || UPLOAD_MANIFEST_RC=$?\n196:rclone copyto \"$WORK/SHA256SUMS.txt\" \"$TARGET_REMOTE:$REMOTE_SUMS\" --checksum > \"$WORK/logs/upload_sums.out\" 2>\"$WORK/logs/upload_sums.err\" || UPLOAD_SUMS_RC=$?\n199:rclone size \"$TARGET_REMOTE:$TARGET_DIR\" --json > \"$WORK/logs/remote_size.json\" 2>\"$WORK/logs/remote_size.err\" || SIZE_RC=$?\n201:rclone lsf \"$TARGET_REMOTE:$REMOTE_PARTS\" --format \"ps\" > \"$WORK/logs/remote_lsf_parts.txt\" 2>\"$WORK/logs/remote_lsf_parts.err\" || LSF_PARTS_RC=$?\n203:REMOTE_BYTES=\"$(python3 -c 'import json; print(json.load(open(\"'\"$WORK\"'/logs/remote_size.json\")).get(\"bytes\",\"\"))' 2>/dev/null || echo unknown)\"\n204:REMOTE_COUNT=\"$(python3 -c 'import json; print(json.load(open(\"'\"$WORK\"'/logs/remote_size.json\")).get(\"count\",\"\"))' 2>/dev/null || echo unknown)\"\n207:rclone copy \"$TARGET_REMOTE:$REMOTE_PARTS\" \"$DL_DIR/parts\" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > \"$WORK/logs/download_parts.out\" 2>\"$WORK/logs/download_parts.err\" || DOWNLOAD_PARTS_RC=$?\n222: rm -rf \"$DL_DIR\" \"$PARTS_DIR\"\n234: rclone lsf \"$TARGET_REMOTE:$TARGET_BASE\" --dirs-only 2>/dev/null | sed \"s#/$##\" | sort > \"$DIR_LIST\" || RETENTION_RC=$?\n239: [ -n \"$OLD\" ] && rclone purge \"$TARGET_REMOTE:$TARGET_BASE/$OLD\" >> \"$WORK/logs/retention.out\" 2>>\"$WORK/logs/retention.err\"\n----- /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload -----\n8:TARGET_BASE=full/nextcloud-vm-vzdump\n11:SRC_GLOB=/var/lib/vz/dump/vzdump-qemu-150-*.vma.zst\n23: local delete_count=\"${7:-0}\"\n25: echo \"STATUS=$status TS=$TS TYPE=mail-cloud-nextcloud-vm MODE=recurring-chunked REMOTE=$TARGET_REMOTE DEST=${TARGET_DIR:-none} ARCHIVE=${BASENAME:-none} SIZE=${SIZE:-0} SHA256=${SHA:-none} PART_SIZE=$PART_SIZE PARTS=${PART_COUNT:-0} REMOTE_PARTS=$remote_parts REMOTE_BYTES=$remote_bytes UPLOAD_OK=$upload_ok DOWNLOAD_VERIFY=$download_verify RETENTION_KEEP=$KEEP DELETE_COUNT=$delete_count ALREADY_UPLOADED=$already_uploaded SECRET_PRINTED=no NOTES=$notes\" > \"$HEALTH\"\n77:P1 Mail Cloud Nextcloud VM recurring chunked vzdump upload\n93:rclone mkdir \"$TARGET_REMOTE:$REMOTE_PARTS\" > \"$WORK/logs/mkdir.out\" 2>\"$WORK/logs/mkdir.err\" || MKDIR_RC=$?\n95:rclone copy \"$PARTS_DIR\" \"$TARGET_REMOTE:$REMOTE_PARTS\" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > \"$WORK/logs/upload_parts.out\" 2>\"$WORK/logs/upload_parts.err\" || UPLOAD_PARTS_RC=$?\n97:rclone copyto \"$WORK/$BASENAME.sha256\" \"$TARGET_REMOTE:$REMOTE_SHA\" --checksum > \"$WORK/logs/upload_sha.out\" 2>\"$WORK/logs/upload_sha.err\" || UPLOAD_SHA_RC=$?\n99:rclone copyto \"$WORK/MANIFEST.txt\" \"$TARGET_REMOTE:$REMOTE_MANIFEST\" --checksum > \"$WORK/logs/upload_manifest.out\" 2>\"$WORK/logs/upload_manifest.err\" || UPLOAD_MANIFEST_RC=$?\n101:rclone copyto \"$WORK/SHA256SUMS.txt\" \"$TARGET_REMOTE:$REMOTE_SUMS\" --checksum > \"$WORK/logs/upload_sums.out\" 2>\"$WORK/logs/upload_sums.err\" || UPLOAD_SUMS_RC=$?\n104:rclone lsf \"$TARGET_REMOTE:$REMOTE_PARTS\" --format ps > \"$WORK/logs/remote_lsf_parts.txt\" 2>\"$WORK/logs/remote_lsf_parts.err\" || LSF_PARTS_RC=$?\n110:rclone copy \"$TARGET_REMOTE:$REMOTE_PARTS\" \"$DL_DIR/parts\" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > \"$WORK/logs/download_parts.out\" 2>\"$WORK/logs/download_parts.err\" || DOWNLOAD_PARTS_RC=$?\n120:rclone lsf \"$TARGET_REMOTE:$TARGET_BASE\" --dirs-only 2>/dev/null | sed 's#/$##' | sort > \"$DIR_LIST\" || RETENTION_RC=$?\n126: [ -n \"$OLD\" ] && rclone purge \"$TARGET_REMOTE:$TARGET_BASE/$OLD\" >> \"$WORK/logs/retention.out\" 2>>\"$WORK/logs/retention.err\"\n140: rm -rf \"$PARTS_DIR\" \"$DL_DIR\"\n----- /usr/local/sbin/homelab-mail-cloud-restore-drill -----\n52: homelab-rclone-cat-chunked \"${R}:$CRIT_DEST/$CRIT_BUNDLE\" > \"$OUT\" 2>\"$WORK/logs/${SAFE}_crit_download.err\"\n83: rm -f \"$OUT\"\n108: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/MANIFEST.txt\" \"$WORK/edge/MANIFEST.txt\" > \"$WORK/logs/edge_manifest_download.out\" 2>\"$WORK/logs/edge_manifest_download.err\"\n111: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/SHA256SUMS.txt\" \"$WORK/edge/SHA256SUMS.txt\" > \"$WORK/logs/edge_sums_download.out\" 2>\"$WORK/logs/edge_sums_download.err\"\n114: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/$EDGE_ARCHIVE.sha256\" \"$WORK/edge/$EDGE_ARCHIVE.sha256\" > \"$WORK/logs/edge_archive_sha_download.out\" 2>\"$WORK/logs/edge_archive_sha_download.err\"\n117: rclone lsf \"${EDGE_REMOTE}:$EDGE_DEST/parts\" --format ps > \"$WORK/logs/edge_parts_lsf.txt\" 2>\"$WORK/logs/edge_parts_lsf.err\"\n130: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/parts/$FIRST_PART\" \"$WORK/edge/$FIRST_PART\" > \"$WORK/logs/edge_first_part_download.out\" 2>\"$WORK/logs/edge_first_part_download.err\"\n137: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/parts/$LAST_PART\" \"$WORK/edge/$LAST_PART\" > \"$WORK/logs/edge_last_part_download.out\" 2>\"$WORK/logs/edge_last_part_download.err\"\n168: rm -f \"$WORK/edge/$FIRST_PART\" \"$WORK/edge/$LAST_PART\"\n----- /usr/local/sbin/homelab-mail-cloud-restore-drill.bak-chunkall-20260710T011410Z -----\n52: rclone copyto \"${R}:$CRIT_DEST/$CRIT_BUNDLE\" \"$OUT\" > \"$WORK/logs/${SAFE}_crit_download.out\" 2>\"$WORK/logs/${SAFE}_crit_download.err\"\n83: rm -f \"$OUT\"\n108: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/MANIFEST.txt\" \"$WORK/edge/MANIFEST.txt\" > \"$WORK/logs/edge_manifest_download.out\" 2>\"$WORK/logs/edge_manifest_download.err\"\n111: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/SHA256SUMS.txt\" \"$WORK/edge/SHA256SUMS.txt\" > \"$WORK/logs/edge_sums_download.out\" 2>\"$WORK/logs/edge_sums_download.err\"\n114: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/$EDGE_ARCHIVE.sha256\" \"$WORK/edge/$EDGE_ARCHIVE.sha256\" > \"$WORK/logs/edge_archive_sha_download.out\" 2>\"$WORK/logs/edge_archive_sha_download.err\"\n117: rclone lsf \"${EDGE_REMOTE}:$EDGE_DEST/parts\" --format ps > \"$WORK/logs/edge_parts_lsf.txt\" 2>\"$WORK/logs/edge_parts_lsf.err\"\n130: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/parts/$FIRST_PART\" \"$WORK/edge/$FIRST_PART\" > \"$WORK/logs/edge_first_part_download.out\" 2>\"$WORK/logs/edge_first_part_download.err\"\n137: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/parts/$LAST_PART\" \"$WORK/edge/$LAST_PART\" > \"$WORK/logs/edge_last_part_download.out\" 2>\"$WORK/logs/edge_last_part_download.err\"\n168: rm -f \"$WORK/edge/$FIRST_PART\" \"$WORK/edge/$LAST_PART\"\n----- /usr/local/sbin/homelab-rclone-cat-chunked -----\n4:TMP_BASE=${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\n7:cleanup(){ rm -rf \"$WORK\"; }\n10:rclone copyto \"${REMOTE_FILE}.parts/MANIFEST.parts.txt\" \"$MAN\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n11:while read -r PART SIZE; do case \"$PART\" in part-*) rclone cat \"${REMOTE_FILE}.parts/$PART\" ;; esac; done < \"$MAN\"\n----- /usr/local/sbin/homelab-rclone-cat-flat-chunked -----\n5:trap \"rm -f \\\"$TMP\\\"\" EXIT\n6:rclone cat \"${DEST}.parts.txt\" > \"$TMP\"\n8:while read -r PART PART_SIZE; do [[ \"$PART\" =~ ^part-[0-9]{4}$ && \"$PART_SIZE\" =~ ^[0-9]+$ ]] || { echo \"BAD FLAT PART MANIFEST\" >&2; exit 2; }; rclone cat \"${DEST}.${PART}\"; COUNT=$((COUNT+1)); done < \"$TMP\"\n----- /usr/local/sbin/homelab-rclone-copy-chunked -----\n7:TMP_BASE=${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\n13:cleanup(){ rm -rf \"$WORK\"; }\n23:printf \"name=%s\\\\nsize=%s\\\\nsha256=%s\\\\nchunk_size=%s\\\\nparts=%s\\\\n\" \"$NAME\" \"$SIZE\" \"$SHA\" \"$CHUNK_SIZE\" \"$PART_COUNT\" > \"$WORK/MANIFEST.txt\"\n24:rclone purge \"${DEST}.parts\" >/dev/null 2>&1 || true\n25:rclone mkdir \"${DEST}.parts\"\n26:rclone copy \"$PARTS\" \"${DEST}.parts\" --transfers 2 --checkers 4 --retries 5 --low-level-retries 20 --stats 30s --log-level INFO\n27:rclone copyto \"$WORK/${NAME}.sha256\" \"${DEST}.sha256\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO\n28:rclone copyto \"$WORK/MANIFEST.txt\" \"${DEST}.manifest.txt\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO\n29:rclone copyto \"$WORK/MANIFEST.parts.txt\" \"${DEST}.parts/MANIFEST.parts.txt\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO\n30:REMOTE_COUNT=$(rclone lsf \"${DEST}.parts\" --files-only | grep -c \"^part-\" || true)\n32:ACTUAL=$(while read -r PART SIZE_PART; do case \"$PART\" in part-*) rclone cat \"${DEST}.parts/$PART\" ;; esac; done < \"$WORK/MANIFEST.parts.txt\" | sha256sum | cut -d \" \" -f 1)\n----- /usr/local/sbin/homelab-rclone-copy-flat-chunked -----\n6:TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\"\n12:trap \"rm -rf \\\"$WORK\\\"\" EXIT\n21:printf \"format=homelab-flat-chunks-v1\\nname=%s\\nsize=%s\\nsha256=%s\\nchunk_size=%s\\nparts=%s\\n\" \"$NAME\" \"$SIZE\" \"$SHA\" \"$CHUNK_SIZE\" \"$PART_COUNT\" > \"$WORK/MANIFEST.txt\"\n23:while read -r PART PART_SIZE; do [[ \"$PART\" =~ ^part-[0-9]{4}$ && \"$PART_SIZE\" =~ ^[0-9]+$ ]] || { echo \"BAD PART MANIFEST\" >&2; exit 4; }; rclone copyto \"$WORK/parts/$PART\" \"${DEST}.${PART}\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR; REMOTE_BYTES=\"$(rclone size --json \"${DEST}.${PART}\" | jq -r \".bytes // -1\")\"; [[ \"$REMOTE_BYTES\" = \"$PART_SIZE\" ]] || { echo \"REMOTE PART SIZE MISMATCH $PART\" >&2; exit 5; }; UPLOADED=$((UPLOADED+1)); done < \"$WORK/MANIFEST.parts.txt\"\n25:rclone copyto \"$WORK/${NAME}.sha256\" \"${DEST}.sha256\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n26:rclone copyto \"$WORK/MANIFEST.txt\" \"${DEST}.manifest.txt\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n27:rclone copyto \"$WORK/MANIFEST.parts.txt\" \"${DEST}.parts.txt\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n28:ACTUAL=\"$(homelab-rclone-cat-flat-chunked \"$DEST\" | sha256sum | awk \"{print \\$1}\")\"\n----- /usr/local/sbin/homelab-rclone-crypt-healthcheck.sh -----\n4:CRYPT_FILE=\"/etc/homelab/rclone/required-crypt-remotes.txt\"\n18: \"checksums/\"\n21:echo \"=== homelab rclone crypt healthcheck: $(hostname) ===\"\n40: if ! rclone listremotes | sed 's/:$//' | grep -qx \"$REMOTE\"; then\n47: if ! rclone about \"${REMOTE}:\"; then\n48: echo \"FAIL: rclone about failed: $REMOTE\"\n54: ROOT_LIST=\"$(rclone lsf \"${REMOTE}:/\" || true)\"\n76: if rclone cat \"${REMOTE}:/latest-backup-index.json\" | jq -e '.schema and .remote and .status and .items' >/dev/null; then\n----- /usr/local/sbin/homelab-rclone-required-check.sh -----\n4:RAW_FILE=\"/etc/homelab/rclone/required-remotes.txt\"\n5:CRYPT_FILE=\"/etc/homelab/rclone/required-crypt-remotes.txt\"\n9:echo \"=== homelab rclone required check: $(hostname) ===\"\n12:echo \"=== installed rclone ===\"\n13:rclone version | head -5\n17:CONFIGURED=\"$(rclone listremotes 2>/dev/null | sed 's/:$//' || true)\"\n19: echo \"No rclone remotes configured yet.\"\n----- /usr/local/sbin/homelab-router-hard-power-cycle -----\n12:case \"$MODE\" in check|cycle) ;; *) echo HARD_WRAPPER_MODE_INVALID=yes; exit 64;; esac\n23:SAFE_OUT=\"$(printf '%s\\n' \"$OUT\" | sed -E '/(PASSWORD|TOKEN|SECRET|PRIVATE_KEY|ROOT_PASSWORD|ENCRYPTION_KEY|COOKIE|SESSION|AUTHORIZATION|API_KEY|LOCAL_KEY|rclone[.]conf|[.]env)/Id')\"\n----- /usr/local/sbin/homelab-router-running-config-mail-cloud -----\n62: rm -f \"$CONFIG\"\n66: rm -f \"$CONFIG\"\n73: if ( cd \"$BACKUP_ROOT\" && sha256sum -c \"$(basename \"$SHA_FILE\")\" ) >\"$WORK/sha-check.out\" 2>\"$WORK/sha-check.err\"; then\n79: if rclone mkdir \"$REMOTE:$DEST\" >\"$WORK/rclone-mkdir.out\" 2>\"$WORK/rclone-mkdir.err\" && homelab-rclone-copy-chunked \"$ARCHIVE\" \"$REMOTE:$DEST/$(basename \"$ARCHIVE\")\" >\"$WORK/rclone-upload.out\" 2>\"$WORK/rclone-upload.err\" && rclone copyto \"$SHA_FILE\" \"$REMOTE:$DEST/$(basename \"$SHA_FILE\")\" --checksum >\"$WORK/rclone-upload-sha.out\" 2>\"$WORK/rclone-upload-sha.err\"; then\n86: if homelab-rclone-cat-chunked \"$REMOTE:$DEST/$(basename \"$ARCHIVE\")\" >\"$DL\" 2>\"$WORK/rclone-download.err\"; then\n93:rclone lsf \"$REMOTE:$TARGET_BASE\" --dirs-only 2>/dev/null | sed 's#/$##' | sort > \"$DIR_LIST\" || RETENTION_RC=$?\n98: [ -n \"$OLD\" ] && rclone purge \"$REMOTE:$TARGET_BASE/$OLD\" >>\"$WORK/retention.out\" 2>>\"$WORK/retention.err\"\n103: rm -f \"$old\" \"$old.sha256\"\n116:rm -rf \"$WORK\"\n----- /usr/local/sbin/homelab-router-running-config-mail-cloud.bak-chunkall-20260710T011410Z -----\n62: rm -f \"$CONFIG\"\n66: rm -f \"$CONFIG\"\n73: if ( cd \"$BACKUP_ROOT\" && sha256sum -c \"$(basename \"$SHA_FILE\")\" ) >\"$WORK/sha-check.out\" 2>\"$WORK/sha-check.err\"; then\n79: if rclone mkdir \"$REMOTE:$DEST\" >\"$WORK/rclone-mkdir.out\" 2>\"$WORK/rclone-mkdir.err\" && rclone copyto \"$ARCHIVE\" \"$REMOTE:$DEST/$(basename \"$ARCHIVE\")\" --checksum >\"$WORK/rclone-upload.out\" 2>\"$WORK/rclone-upload.err\" && rclone copyto \"$SHA_FILE\" \"$REMOTE:$DEST/$(basename \"$SHA_FILE\")\" --checksum >\"$WORK/rclone-upload-sha.out\" 2>\"$WORK/rclone-upload-sha.err\"; then\n86: if rclone copyto \"$REMOTE:$DEST/$(basename \"$ARCHIVE\")\" \"$DL\" >\"$WORK/rclone-download.out\" 2>\"$WORK/rclone-download.err\"; then\n93:rclone lsf \"$REMOTE:$TARGET_BASE\" --dirs-only 2>/dev/null | sed 's#/$##' | sort > \"$DIR_LIST\" || RETENTION_RC=$?\n98: [ -n \"$OLD\" ] && rclone purge \"$REMOTE:$TARGET_BASE/$OLD\" >>\"$WORK/retention.out\" 2>>\"$WORK/retention.err\"\n103: rm -f \"$old\" \"$old.sha256\"\n116:rm -rf \"$WORK\"\n----- /usr/local/sbin/homelab-secret-exposure-guard -----\n18:BADFILES=\"$(find \"$REPO\" -path \"$REPO/.git\" -prune -o -type f \\( -iname \"*.key\" -o -iname \"*.pem\" -o -iname \"*.p12\" -o -iname \"*.pfx\" -o -iname \"[SENSITIVE_FILE]\" -o -iname \".env\" -o -iname \".env.*\" \\) -print 2>/dev/null)\"\n----- /usr/local/sbin/homelab-stage-2cloud -----\n20:jq -n --arg schema \"homelab-app-backup-manifest-v1\" --arg job \"$JOB\" --arg node \"$NODE\" --arg app \"$APP\" --arg created \"$(date -u +%Y%m%dT%H%M%SZ)\" --arg archive \"$ARCHIVE\" --arg sha \"$SHA\" --arg ra \"$RA\" --arg rb \"$RB\" --argjson size \"$SIZE\" --argjson meta \"$META\" '$meta + {schema:$schema,job_id:$job,node:$node,app_name:$app,created_utc:$created,archive_name:$archive,archive_size_bytes:$size,archive_sha256:$sha,remote_a_job_path:$ra,remote_b_job_path:$rb,local_staging_policy:\"delete_after_both_clouds_verified\",verification_policy:\"full_roundtrip_download_plus_sha256\"}' > \"$STAGE/manifest.json\"\n22:echo \"[STEP] local checksum\"; (cd \"$STAGE\" && sha256sum \"$ARCHIVE\" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e . manifest.json >/dev/null)\n23:echo \"[STEP] upload cloud A metadata\"; rclone copy \"$STAGE\" \"$RA\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n24:echo \"[STEP] upload cloud A archive chunks\"; rclone deletefile \"$RA/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RA/$ARCHIVE\"\n25:echo \"[STEP] upload cloud B metadata\"; rclone copy \"$STAGE\" \"$RB\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n26:echo \"[STEP] upload cloud B archive chunks\"; rclone deletefile \"$RB/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RB/$ARCHIVE\"\n27:echo \"[STEP] verify cloud A chunked\"; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-stage-verify-chunked \"$RA\" \"$ARCHIVE\" \"$VERIFY/a\" \"$JOB\"\n28:echo \"[STEP] verify cloud B chunked\"; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-stage-verify-chunked \"$RB\" \"$ARCHIVE\" \"$VERIFY/b\" \"$JOB\"\n29:echo \"[STEP] upload latest index\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RAL\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RBL\"; rclone cat \"$RAL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null; rclone cat \"$RBL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n30:jq \". + {status:\\\"success\\\",local_backup_deleted:true}\" \"$STAGE/manifest.json\" > \"$STATE_ROOT/apps/latest-${APP}.json\"\n31:echo \"[STEP] delete local staging\"; rm -rf \"$STAGE\" \"$VERIFY\"; test ! -d \"$STAGE\" && test ! -d \"$VERIFY\"\n33:echo \"[SUCCESS] uploaded to 2 clouds, verified from both clouds, local staging deleted\"\n----- /usr/local/sbin/homelab-stage-2cloud.bak-global-chunkonly-20260710T011036Z -----\n20:jq -n --arg schema \"homelab-app-backup-manifest-v1\" --arg job \"$JOB\" --arg node \"$NODE\" --arg app \"$APP\" --arg created \"$(date -u +%Y%m%dT%H%M%SZ)\" --arg archive \"$ARCHIVE\" --arg sha \"$SHA\" --arg ra \"$RA\" --arg rb \"$RB\" --argjson size \"$SIZE\" --argjson meta \"$META\" '$meta + {schema:$schema,job_id:$job,node:$node,app_name:$app,created_utc:$created,archive_name:$archive,archive_size_bytes:$size,archive_sha256:$sha,remote_a_job_path:$ra,remote_b_job_path:$rb,local_staging_policy:\"delete_after_both_clouds_verified\",verification_policy:\"full_roundtrip_download_plus_sha256\"}' > \"$STAGE/manifest.json\"\n22:echo \"[STEP] local checksum\"; (cd \"$STAGE\" && sha256sum \"$ARCHIVE\" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e . manifest.json >/dev/null)\n23:echo \"[STEP] upload cloud A\"; rclone copy \"$STAGE\" \"$RA\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 --stats=30s\n24:echo \"[STEP] upload cloud B\"; rclone copy \"$STAGE\" \"$RB\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 --stats=30s\n25:echo \"[STEP] verify cloud A\"; rclone copy \"$RA\" \"$VERIFY/a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10; (cd \"$VERIFY/a\" && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n26:echo \"[STEP] verify cloud B\"; rclone copy \"$RB\" \"$VERIFY/b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10; (cd \"$VERIFY/b\" && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n27:echo \"[STEP] upload latest index\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RAL\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RBL\"; rclone cat \"$RAL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null; rclone cat \"$RBL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n28:jq \". + {status:\\\"success\\\",local_backup_deleted:true}\" \"$STAGE/manifest.json\" > \"$STATE_ROOT/apps/latest-${APP}.json\"\n29:echo \"[STEP] delete local staging\"; rm -rf \"$STAGE\" \"$VERIFY\"; test ! -d \"$STAGE\" && test ! -d \"$VERIFY\"\n31:echo \"[SUCCESS] uploaded to 2 clouds, verified from both clouds, local staging deleted\"\n----- /usr/local/sbin/homelab-stage-2cloud-flat -----\n31:jq -n --arg schema \"homelab-app-backup-manifest-v2-flat\" --arg layout \"flat-v1\" --arg job \"$JOB\" --arg node \"$NODE\" --arg app \"$APP\" --arg created \"$CREATED\" --arg archive \"$ARCHIVE\" --arg sha \"$SHA\" --arg ra \"$RA\" --arg rb \"$RB\" --arg rel \"$REL\" --argjson size \"$SIZE\" --argjson meta \"$META\" \"\\$meta + {schema:\\$schema,layout:\\$layout,job_id:\\$job,node:\\$node,app_name:\\$app,created_utc:\\$created,archive_name:\\$archive,archive_size_bytes:\\$size,archive_sha256:\\$sha,path_rel:\\$rel,remote_a_job_path:\\$ra,remote_b_job_path:\\$rb,local_staging_policy:\\\"delete_after_both_clouds_verified\\\",verification_policy:\\\"flat_chunks_full_roundtrip_sha256\\\"}\" > \"$STAGE/manifest.json\"\n33:(cd \"$STAGE\"; FILES=(\"$ARCHIVE\" manifest.json latest-backup-index.json); [[ -f metadata.json ]] && FILES+=(metadata.json); sha256sum \"${FILES[@]}\" > checksums.sha256; sha256sum -c checksums.sha256 >/dev/null; tar -tzf \"$ARCHIVE\" >/dev/null; jq -e . manifest.json >/dev/null)\n36: if rclone lsf \"$REMOTE_DIR\" --max-depth 1 2>/dev/null | grep -q .; then echo \"REFUSE NONEMPTY CANARY TARGET $REMOTE_DIR\" >&2; exit 6; fi\n37: rclone mkdir \"$REMOTE_DIR\"\n38: for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f \"$STAGE/$F\" ]] && rclone copyto \"$STAGE/$F\" \"$REMOTE_DIR/$F\" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done\n39: CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-flat-chunked \"$STAGE/$ARCHIVE\" \"$REMOTE_DIR/$ARCHIVE\"\n41: rclone copyto \"$STAGE/latest-backup-index.json\" \"$LATEST_FILE\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n42: rclone cat \"$LATEST_FILE\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n46:jq \". + {status:\\\"success\\\",local_backup_deleted:true}\" \"$STAGE/manifest.json\" > \"$STATE_ROOT/apps/latest-${APP}.json\"\n47:rm -rf \"$STAGE\" \"$VERIFY\"\n----- /usr/local/sbin/homelab-stage-2cloud-flat-canary -----\n29:jq -n --arg schema \"homelab-app-backup-manifest-v2-flat\" --arg layout \"flat-v1\" --arg job \"$JOB\" --arg node \"$NODE\" --arg app \"$APP\" --arg created \"$CREATED\" --arg archive \"$ARCHIVE\" --arg sha \"$SHA\" --arg ra \"$RA\" --arg rb \"$RB\" --arg rel \"$REL\" --argjson size \"$SIZE\" --argjson meta \"$META\" \"\\$meta + {schema:\\$schema,layout:\\$layout,job_id:\\$job,node:\\$node,app_name:\\$app,created_utc:\\$created,archive_name:\\$archive,archive_size_bytes:\\$size,archive_sha256:\\$sha,path_rel:\\$rel,remote_a_job_path:\\$ra,remote_b_job_path:\\$rb,local_staging_policy:\\\"delete_after_both_clouds_verified\\\",verification_policy:\\\"flat_chunks_full_roundtrip_sha256\\\"}\" > \"$STAGE/manifest.json\"\n31:(cd \"$STAGE\"; FILES=(\"$ARCHIVE\" manifest.json latest-backup-index.json); [[ -f metadata.json ]] && FILES+=(metadata.json); sha256sum \"${FILES[@]}\" > checksums.sha256; sha256sum -c checksums.sha256 >/dev/null; tar -tzf \"$ARCHIVE\" >/dev/null; jq -e . manifest.json >/dev/null)\n34: if rclone lsf \"$REMOTE_DIR\" --max-depth 1 2>/dev/null | grep -q .; then echo \"REFUSE NONEMPTY CANARY TARGET $REMOTE_DIR\" >&2; exit 6; fi\n35: rclone mkdir \"$REMOTE_DIR\"\n36: for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f \"$STAGE/$F\" ]] && rclone copyto \"$STAGE/$F\" \"$REMOTE_DIR/$F\" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done\n37: CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-flat-chunked \"$STAGE/$ARCHIVE\" \"$REMOTE_DIR/$ARCHIVE\"\n39: rclone copyto \"$STAGE/latest-backup-index.json\" \"$LATEST_FILE\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n40: rclone cat \"$LATEST_FILE\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n44:jq \". + {status:\\\"success\\\",local_backup_deleted:true}\" \"$STAGE/manifest.json\" > \"$STATE_ROOT/apps/latest-${APP}.json\"\n45:rm -rf \"$STAGE\" \"$VERIFY\"\n----- /usr/local/sbin/homelab-stage-verify-chunked -----\n7:rm -rf \"$VERIFY\"\n9:rclone copy \"$REMOTE_DIR\" \"$VERIFY\" --exclude \"$ARCHIVE.parts/**\" --exclude \"$ARCHIVE.parts/\" --exclude \"$ARCHIVE.sha256\" --exclude \"$ARCHIVE.manifest.txt\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --log-level ERROR\n13:if [ -s checksums.sha256 ]; then grep -F -v \" $ARCHIVE\" checksums.sha256 > .checksums-no-archive.sha256 || true; if [ -s .checksums-no-archive.sha256 ]; then sha256sum -c .checksums-no-archive.sha256; fi; EXPECTED=$(grep -F \" $ARCHIVE\" checksums.sha256 | awk '{print $1; exit}'); else EXPECTED=$(rclone cat \"${REMOTE_DIR}/${ARCHIVE}.sha256\" | awk '{print $1; exit}'); fi\n15:ACTUAL=$(homelab-rclone-cat-chunked \"${REMOTE_DIR}/${ARCHIVE}\" | sha256sum | awk '{print $1}')\n17:homelab-rclone-cat-chunked \"${REMOTE_DIR}/${ARCHIVE}\" | tar -tzf - >/dev/null\n----- /usr/local/sbin/homelab-stage-verify-flat-chunked -----\n7:rm -rf \"$VERIFY\"\n9:rclone copy \"$REMOTE_DIR\" \"$VERIFY\" --transfers 4 --checkers 8 --retries 5 --low-level-retries 20 --log-level ERROR\n11:(cd \"$VERIFY\" && test -s manifest.json && if [[ -n \"$JOB\" ]]; then jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null; fi && sha256sum -c checksums.sha256 >/dev/null && tar -tzf \"$ARCHIVE\" >/dev/null)\n----- /usr/local/sbin/homelab-vm160-remote-backup -----\n8:PARENT=/mnt/staging/vzdump\n22:SOURCE_ACTIVE=\"$(ssh \"${SSH[@]}\" \"$SOURCE\" \"pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'\")\" || SOURCE_ACTIVE_RC=$?\n27:NAME=\"vzdump-qemu-${VMID}-${TS}.vma.zst\"\n35: ssh \"${SSH[@]}\" \"$TARGET\" \"rm -rf -- '$DIR'\" >/dev/null 2>&1 || true\n41: rm -f \"$LOG\"\n46:LATEST_SIZE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\\n' | sort -nr | head -n1\")\"\n55:ssh \"${SSH[@]}\" \"$SOURCE\" \"vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7\" 2>\"$LOG\" | ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$PART'\"\n57:ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$DIR/vzdump.log'\" < \"$LOG\"\n66:rm -f \"$LOG\"\n----- /usr/local/sbin/homelab-vm-backup-health -----\n8:check_file(){\n15:check_file /var/lib/homelab-health/backup.txt auto-vmct\n16:check_file /var/lib/homelab-health/edge-vm-vzdump-backup.txt edge-vm-backup\n17:check_file /var/lib/homelab-health/edge-vm-vzdump-offhost.txt edge-vm-offhost\n18:check_file /var/lib/homelab-health/edge-vm-vzdump-restore.txt edge-vm-restore\n----- /usr/local/sbin/homelab-vm-backup-policy -----\n29:echo \"## Latest local vzdump catalog\" >> \"$REPORT\"\n30:printf \"%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n\" \"kind\" \"vmid\" \"name\" \"size_bytes\" \"mtime_utc\" \"path\" > \"$CATALOG\"\n31:for f in $(find /var/lib/vz/dump -maxdepth 1 -type f \\( -name \"vzdump-qemu-*.vma.zst\" -o -name \"vzdump-lxc-*.tar.zst\" \\) 2>/dev/null | sort); do b=\"$(basename \"$f\")\"; size=\"$(stat -c %s \"$f\" 2>/dev/null || echo 0)\"; mt=\"$(date -u -d \"@$(stat -c %Y \"$f\" 2>/dev/null || echo 0)\" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)\"; if echo \"$b\" | grep -q \"^vzdump-qemu-\"; then kind=qemu; vmid=\"$(echo \"$b\" | sed -n \"s/^vzdump-qemu-\\([0-9]\\+\\)-.*/\\1/p\")\"; name=\"$(qm config \"$vmid\" 2>/dev/null | awk -F\": \" \"/^name:/{print \\$2; exit}\")\"; else kind=lxc; vmid=\"$(echo \"$b\" | sed -n \"s/^vzdump-lxc-\\([0-9]\\+\\)-.*/\\1/p\")\"; name=\"$(pct config \"$vmid\" 2>/dev/null | awk -F\": \" \"/^hostname:/{print \\$2; exit}\")\"; fi; printf \"%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n\" \"$kind\" \"$vmid\" \"${name:-unknown}\" \"$size\" \"$mt\" \"$f\" >> \"$CATALOG\"; done\n37:echo \"- Existing local dumps can be uploaded/verified to two clouds without creating new vzdump archives.\" >> \"$REPORT\"\n38:echo \"- New full vzdump for VM 170/171 is not started automatically because each VM has 120 GB bootdisk and /var/tmp has limited free space.\" >> \"$REPORT\"\n42:echo \"## Coverage checks\" >> \"$REPORT\"\n----- /usr/local/sbin/homelab-vm-full-backup -----\n11:# shellcheck disable=SC1090\n92:for cmd in vzdump qm zstd sha256sum rclone rsync ssh flock; do\n130: -name \"vzdump-qemu-${VMID}-*.vma.zst\" \\\n138: vzdump \"$VMID\" \\\n145: --remove 0\n151: -name \"vzdump-qemu-${VMID}-*.vma.zst\" \\\n261: CHUNK_TMP_BASE=\"$BACKUP_DIR/.rclone-chunks\" homelab-rclone-copy-chunked \"$ARCHIVE\" \"${DEST_ROOT}/${NAME}\"\n----- /usr/local/sbin/homelab-vm-full-backup.bak-chunkall-20260710T011410Z -----\n11:# shellcheck disable=SC1090\n92:for cmd in vzdump qm zstd sha256sum rclone rsync ssh flock; do\n130: -name \"vzdump-qemu-${VMID}-*.vma.zst\" \\\n138: vzdump \"$VMID\" \\\n145: --remove 0\n151: -name \"vzdump-qemu-${VMID}-*.vma.zst\" \\\n264: rclone copyto \\\n267: --check-first \\\n269: --checkers 2 \\\n275: rclone copyto \\\n278: --check-first \\\n280: --checkers 2 \\\n285: echo \"Crypt-checking $NAME on $REMOTE\"\n287: rclone cryptcheck \\\n292: --checkers 1 \\\n----- /usr/local/sbin/homelab-vm-full-restore-drill -----\n77: -name \"vzdump-qemu-${source_vmid}-*.vma.zst\" \\\n152:check_inventory() {\n184:check_container_health() {\n224:check_http() {\n349:check_container_health() {\n386:check_container_health \"$test_vmid\"\n389: check_inventory \"$source_vmid\" \"$test_vmid\"\n392: check_http \"$source_vmid\" \"$test_vmid\"\n----- /usr/local/sbin/homelab-vm-full-restore-drill.bak.20260710T080231Z -----\n77: -name \"vzdump-qemu-${source_vmid}-*.vma.zst\" \\\n152:check_inventory() {\n184:check_container_health() {\n224:check_http() {\n349:check_container_health() {\n386:check_container_health \"$test_vmid\"\n389: check_inventory \"$source_vmid\" \"$test_vmid\"\n392: check_http \"$source_vmid\" \"$test_vmid\"\n----- /usr/local/sbin/homelab-vm-full-restore-drill.bak-docker-check-v2-20260710T050044Z -----\n77: -name \"vzdump-qemu-${source_vmid}-*.vma.zst\" \\\n152:check_inventory() {\n184:check_container_health() {\n224:check_http() {\n347: check_container_health \"$test_vmid\"\n350: check_inventory \"$source_vmid\" \"$test_vmid\"\n353: check_http \"$source_vmid\" \"$test_vmid\"\n----- /usr/local/sbin/homelab-vm-full-restore-drill.bak-isolated-ip-20260710T044331Z -----\n77: -name \"vzdump-qemu-${source_vmid}-*.vma.zst\" \\\n152:check_inventory() {\n184:check_container_health() {\n224:check_http() {\n337: check_container_health \"$test_vmid\"\n340: check_inventory \"$source_vmid\" \"$test_vmid\"\n343: check_http \"$source_vmid\" \"$test_vmid\"\n----- /usr/local/sbin/homelab-vm-full-retention -----\n21: -name \"vzdump-qemu-${vmid}-*.vma.zst\" \\\n33: vzdump-qemu-\"$vmid\"-*.vma.zst)\n35: rm -f \\\n73: rm -rf -- \\\"\\$BASE/\\$setname\\\"\n91: rclone lsf \"$path\" \\\n105: rclone purge \"${path}/${dir}\"\n----- /usr/local/sbin/homelab-vm-local-dumps-2cloud -----\n30:CHKS=\"$WORK/checksums.txt\"\n39:echo \"chunk_size_bytes=$CHUNK_SIZE_BYTES\" >> \"$LOG\"\n42: F=\"$(find /var/lib/vz/dump -maxdepth 1 -type f \\( -name \"vzdump-qemu-${id}-*.vma.zst\" -o -name \"vzdump-lxc-${id}-*.tar.zst\" \\) -printf \"%T@ %p\\n\" 2>/dev/null | sort -n | tail -n 1 | cut -d\" \" -f2-)\"\n53: echo \"$BN\" | grep -q \"^vzdump-qemu-\" && KIND=\"qemu\"\n54: echo \"$BN\" | grep -q \"^vzdump-lxc-\" && KIND=\"lxc\"\n64: rm -rf \"$CDIR\"\n66: echo \"SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR\" >> \"$LOG\"\n83:job,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:]\n91: \"size_bytes\":int(p[2]),\n107: \"chunk_size_bytes\":int(chunk_size),\n117: rclone mkdir \"$DEST/files\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n118: while IFS=\"$(printf \"\\t\")\" read -r id kind size sha mt bn path mode parts; do\n121: echo \"UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts\" >> \"$LOG\"\n122: rclone mkdir \"$DEST/files/$bn.chunks\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n126: rclone copyto \"$part\" \"$DEST/files/$bn.chunks/$pn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n128: rclone copyto \"$CDIR/chunks.sha256\" \"$DEST/files/$bn.chunks/chunks.sha256\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n130: echo \"UPLOAD_DIRECT dest=$DEST file=$bn size=$size\" >> \"$LOG\"\n131: rclone copyto \"$path\" \"$DEST/files/$bn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n135: rclone copyto \"$CHKS\" \"$DEST/checksums.txt\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n136: rclone copyto \"$FILES_TSV\" \"$DEST/files.tsv\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n137: rclone copyto \"$MAN\" \"$DEST/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n144: rm -rf \"$VBASE\"\n146: rclone copyto \"$DEST/manifest.json\" \"$VBASE/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n149: while IFS=\"$(printf \"\\t\")\" read -r id kind size sha mt bn path mode parts; do\n154: rclone copy \"$DEST/files/$bn.chunks\" \"$VDIR\" --filter \"+ part-*\" --filter \"+ chunks.sha256\" --filter \"- *\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n163: rm -rf \"$VDIR\"\n166: rclone copyto \"$DEST/files/$bn\" \"$VBASE/$bn\" --transfers 1 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n168: rm -f \"$VBASE/$bn\"\n171: rm -rf \"$VBASE\"\n184:rm -rf \"$VERIFY_ROOT/$JOB_ID\"\n185:rm -rf \"$CHUNK_ROOT/$JOB_ID\"\n198: --argjson chunk_size_bytes \"$CHUNK_SIZE_BYTES\" \\\n200: \"{job_id:\\$job_id,status:\\$status,node:\\$node,remote_a:\\$remote_a,remote_b:\\$remote_b,count:\\$count,total_bytes:\\$total_bytes,chunked_count:\\$chunked_count,chunk_size_bytes:\\$chunk_size_bytes,log:\\$log,local_dumps_deleted:false}\" \\\n----- /usr/local/sbin/homelab-vm-local-dumps-retention -----\n32: mapfile -t JOBS < <(rclone lsf \"$BASE/\" --dirs-only 2>/dev/null | sed 's#/$##' | sort)\n37: echo \"nothing_to_delete\" >> \"$REPORT\"\n56: rclone purge \"$PATH_REMOTE\" >> \"$REPORT\" 2>&1 && DELETED=$((DELETED+1)) || BAD=$((BAD+1))\n----- /usr/local/sbin/skladchik-reports-monitor-backup -----\n28:command -v rclone >/dev/null\n29:[ -x /usr/local/sbin/homelab-rclone-copy-chunked ] || { echo \"missing /usr/local/sbin/homelab-rclone-copy-chunked\" >&2; exit 10; }\n36:cleanup(){ rm -rf \"$TMP\"; }\n67:size_bytes=${SIZE}\n71:log \"LOCAL_ARCHIVE_OK size=$SIZE path=$ARCHIVE\"\n75: /usr/local/sbin/homelab-rclone-copy-chunked \"$ARCHIVE\" \"${DEST_BASE}/$(basename \"$ARCHIVE\")\"\n76: /usr/local/sbin/homelab-rclone-copy-chunked \"${ARCHIVE}.sha256\" \"${DEST_BASE}/$(basename \"$ARCHIVE\").sha256\"\n77: /usr/local/sbin/homelab-rclone-copy-chunked \"${OUTDIR}/manifest.txt\" \"${DEST_BASE}/manifest.txt\"\n82: mapfile -t OLD < <(rclone lsf --dirs-only \"${REMOTE}${REMOTE_BASE}/daily/\" 2>/dev/null | sed 's#/$##' | sort | head -n -\"${KEEP_DAILY}\" || true)\n83: for D in \"${OLD[@]}\"; do [ -n \"$D\" ] && rclone purge \"${REMOTE}${REMOTE_BASE}/daily/${D}\" >/dev/null 2>&1 || true; done\n85:find \"${LOCAL_ROOT}/daily\" -mindepth 1 -maxdepth 1 -type d -printf '%f\\n' 2>/dev/null | sort | head -n -\"${KEEP_DAILY}\" | while read -r D; do [ -n \"$D\" ] && rm -rf \"${LOCAL_ROOT}/daily/${D}\"; done\n----- /usr/local/sbin/skladchik-reports-monitor-backup.pre-restore-check-20260710T065311Z.bak -----\n28:command -v rclone >/dev/null\n29:[ -x /usr/local/sbin/homelab-rclone-copy-chunked ] || { echo \"missing /usr/local/sbin/homelab-rclone-copy-chunked\" >&2; exit 10; }\n36:cleanup(){ rm -rf \"$TMP\"; }\n66:size_bytes=${SIZE}\n70:log \"LOCAL_ARCHIVE_OK size=$SIZE path=$ARCHIVE\"\n74: /usr/local/sbin/homelab-rclone-copy-chunked \"$ARCHIVE\" \"${DEST_BASE}/$(basename \"$ARCHIVE\")\"\n75: /usr/local/sbin/homelab-rclone-copy-chunked \"${ARCHIVE}.sha256\" \"${DEST_BASE}/$(basename \"$ARCHIVE\").sha256\"\n76: /usr/local/sbin/homelab-rclone-copy-chunked \"${OUTDIR}/manifest.txt\" \"${DEST_BASE}/manifest.txt\"\n81: mapfile -t OLD < <(rclone lsf --dirs-only \"${REMOTE}${REMOTE_BASE}/daily/\" 2>/dev/null | sed 's#/$##' | sort | head -n -\"${KEEP_DAILY}\" || true)\n82: for D in \"${OLD[@]}\"; do [ -n \"$D\" ] && rclone purge \"${REMOTE}${REMOTE_BASE}/daily/${D}\" >/dev/null 2>&1 || true; done\n84:find \"${LOCAL_ROOT}/daily\" -mindepth 1 -maxdepth 1 -type d -printf '%f\\n' 2>/dev/null | sort | head -n -\"${KEEP_DAILY}\" | while read -r D; do [ -n \"$D\" ] && rm -rf \"${LOCAL_ROOT}/daily/${D}\"; done\n----- /usr/local/sbin/skladchik-reports-monitor-backup.pre-session-keepalive-20260710T071605Z.bak -----\n28:command -v rclone >/dev/null\n29:[ -x /usr/local/sbin/homelab-rclone-copy-chunked ] || { echo \"missing /usr/local/sbin/homelab-rclone-copy-chunked\" >&2; exit 10; }\n36:cleanup(){ rm -rf \"$TMP\"; }\n64:size_bytes=${SIZE}\n68:log \"LOCAL_ARCHIVE_OK size=$SIZE path=$ARCHIVE\"\n72: /usr/local/sbin/homelab-rclone-copy-chunked \"$ARCHIVE\" \"${DEST_BASE}/$(basename \"$ARCHIVE\")\"\n73: /usr/local/sbin/homelab-rclone-copy-chunked \"${ARCHIVE}.sha256\" \"${DEST_BASE}/$(basename \"$ARCHIVE\").sha256\"\n74: /usr/local/sbin/homelab-rclone-copy-chunked \"${OUTDIR}/manifest.txt\" \"${DEST_BASE}/manifest.txt\"\n79: mapfile -t OLD < <(rclone lsf --dirs-only \"${REMOTE}${REMOTE_BASE}/daily/\" 2>/dev/null | sed 's#/$##' | sort | head -n -\"${KEEP_DAILY}\" || true)\n80: for D in \"${OLD[@]}\"; do [ -n \"$D\" ] && rclone purge \"${REMOTE}${REMOTE_BASE}/daily/${D}\" >/dev/null 2>&1 || true; done\n82:find \"${LOCAL_ROOT}/daily\" -mindepth 1 -maxdepth 1 -type d -printf '%f\\n' 2>/dev/null | sort | head -n -\"${KEEP_DAILY}\" | while read -r D; do [ -n \"$D\" ] && rm -rf \"${LOCAL_ROOT}/daily/${D}\"; done\n----- /usr/local/sbin/skladchik-reports-monitor-backup-restore-check -----\n4:CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf\n9:: \"${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}\"\n12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\n13:LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock\n19:log(){ printf '%s RESTORE_CHECK %s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$*\" | tee -a \"$LOGDIR/restore-check.log\"; }\n22:command -v rclone >/dev/null\n28: if rclone lsf \"${remote_path}.parts/\" > \"$plist\" 2>/dev/null && grep -q '^part-' \"$plist\"; then\n29: while IFS= read -r part; do [ -n \"$part\" ] && rclone cat \"${remote_path}.parts/${part}\" >> \"$out\"; done < <(grep '^part-' \"$plist\" | sort)\n31: rclone copyto \"$remote_path\" \"$out\" >/dev/null\n33: rm -f \"$plist\"\n36:check_one_remote(){\n39: latest=\"$(rclone lsf --dirs-only \"${rbase}/\" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)\"\n85: [ \"$(sqlite3 \"$extract/var-lib-skladchik-reports-monitor/state.sqlite\" 'PRAGMA integrity_check;' | tail -n 1)\" = \"ok\" ]\n97:check_one_remote CLOUD_1 \"$CLOUD_1\"\n98:check_one_remote CLOUD_2 \"$CLOUD_2\"\n101:find \"$WORK_ROOT\" -mindepth 1 -maxdepth 1 -type d -printf '%f\\n' 2>/dev/null | sort | head -n -\"${KEEP_LOCAL}\" | while read -r D; do [ -n \"$D\" ] && rm -rf \"${WORK_ROOT}/${D}\"; done\n----- /usr/local/sbin/skladchik-reports-monitor-backup-restore-check.pre-session-keepalive-20260710T071605Z.bak -----\n4:CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf\n9:: \"${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}\"\n12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\n13:LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock\n19:log(){ printf '%s RESTORE_CHECK %s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$*\" | tee -a \"$LOGDIR/restore-check.log\"; }\n22:command -v rclone >/dev/null\n28: if rclone lsf \"${remote_path}.parts/\" > \"$plist\" 2>/dev/null && grep -q '^part-' \"$plist\"; then\n29: while IFS= read -r part; do [ -n \"$part\" ] && rclone cat \"${remote_path}.parts/${part}\" >> \"$out\"; done < <(grep '^part-' \"$plist\" | sort)\n31: rclone copyto \"$remote_path\" \"$out\" >/dev/null\n33: rm -f \"$plist\"\n36:check_one_remote(){\n39: latest=\"$(rclone lsf --dirs-only \"${rbase}/\" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)\"\n72: [ \"$(sqlite3 \"$extract/var-lib-skladchik-reports-monitor/state.sqlite\" 'PRAGMA integrity_check;' | tail -n 1)\" = \"ok\" ]\n84:check_one_remote CLOUD_1 \"$CLOUD_1\"\n85:check_one_remote CLOUD_2 \"$CLOUD_2\"\n88:find \"$WORK_ROOT\" -mindepth 1 -maxdepth 1 -type d -printf '%f\\n' 2>/dev/null | sort | head -n -\"${KEEP_LOCAL}\" | while read -r D; do [ -n \"$D\" ] && rm -rf \"${WORK_ROOT}/${D}\"; done\n" + "output": "=== CLUSTER_BACKUP_JOBS ===\n[{\"compress\":\"zstd\",\"enabled\":1,\"id\":\"homelab-nightly-all\",\"mailnotification\":\"failure\",\"mode\":\"snapshot\",\"next-run\":1784680200,\"prune-backups\":{\"keep-daily\":\"4\"},\"schedule\":\"03:30\",\"storage\":\"local\",\"type\":\"vzdump\",\"vmid\":\"110,111,112,113,130,150,190\"}]\n=== CLUSTER_STORAGES ===\nName Type Status Total (KiB) Used (KiB) Available (KiB) %\nlocal dir active 102626232 43175760 54191208 42.07%\nlocal-lvm lvmthin active 848797696 280188119 568609576 33.01%\n========== NODE=pve01 ==========\npve01.gram1.ru\n=== RCLONE_REMOTES ===\npve01-mail-01:\npve01-mail-01-crypt:\npve01-mail-02:\npve01-mail-02-crypt:\npve01-mail-03:\npve01-mail-03-crypt:\npve01-mail-04:\npve01-mail-04-crypt:\npve01-mail-05:\npve01-mail-05-crypt:\npve02-mail-01:\npve02-mail-01-crypt:\npve02-mail-02:\npve02-mail-02-crypt:\npve02-mail-03:\npve02-mail-03-crypt:\npve02-mail-04:\npve02-mail-04-crypt:\npve03-mail-01:\npve03-mail-01-crypt:\npve03-mail-02:\npve03-mail-02-crypt:\npve03-mail-03:\npve03-mail-03-crypt:\npve03-mail-04:\npve03-mail-04-crypt:\nNODE_REMOTE_COUNT=26\nREMOTE_HEALTH node=pve01 remote=pve01-mail-01: status=OK\n=== BACKUP_TIMERS ===\nWed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:14 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service\nWed 2026-07-22 00:09:43 MSK 9h Tue 2026-07-21 00:09:32 MSK 14h ago homelab-backup-freshness.timer homelab-backup-freshness.service\nWed 2026-07-22 02:15:00 MSK 11h Tue 2026-07-21 02:15:05 MSK 12h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service\nWed 2026-07-22 03:10:02 MSK 12h Tue 2026-07-21 03:13:14 MSK 11h ago homelab-emergency-backup.timer homelab-emergency-backup.service\nWed 2026-07-22 03:35:00 MSK 12h Tue 2026-07-21 03:35:14 MSK 11h ago filebrowser-backup.timer filebrowser-backup.service\nWed 2026-07-22 03:55:02 MSK 13h Tue 2026-07-21 03:51:25 MSK 10h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service\nWed 2026-07-22 04:17:29 MSK 13h Tue 2026-07-21 04:19:17 MSK 10h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service\nWed 2026-07-22 04:26:54 MSK 13h Tue 2026-07-21 04:28:14 MSK 10h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service\nWed 2026-07-22 04:38:04 MSK 13h Tue 2026-07-21 04:34:48 MSK 10h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service\nWed 2026-07-22 05:01:44 MSK 14h Tue 2026-07-21 04:57:14 MSK 9h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service\nWed 2026-07-22 05:23:56 MSK 14h Tue 2026-07-21 05:17:57 MSK 9h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service\nWed 2026-07-22 05:27:13 MSK 14h Tue 2026-07-21 05:29:05 MSK 9h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service\nWed 2026-07-22 05:47:19 MSK 14h Tue 2026-07-21 05:46:14 MSK 9h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service\nWed 2026-07-22 06:01:19 MSK 15h Tue 2026-07-21 05:57:30 MSK 8h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service\nWed 2026-07-22 06:27:43 MSK 15h Tue 2026-07-21 06:32:25 MSK 8h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service\nWed 2026-07-22 06:38:51 MSK 15h Tue 2026-07-21 06:43:54 MSK 8h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service\nWed 2026-07-22 06:39:22 MSK 15h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service\nWed 2026-07-22 06:44:16 MSK 15h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service\nWed 2026-07-22 06:52:06 MSK 16h Tue 2026-07-21 06:47:47 MSK 8h ago homelab-beszel-backup.timer homelab-beszel-backup.service\nWed 2026-07-22 06:58:58 MSK 16h Tue 2026-07-21 06:57:30 MSK 7h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service\nWed 2026-07-22 07:02:27 MSK 16h Tue 2026-07-21 07:05:30 MSK 7h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service\nWed 2026-07-22 07:07:14 MSK 16h Tue 2026-07-21 07:18:55 MSK 7h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service\nWed 2026-07-22 07:35:06 MSK 16h Tue 2026-07-21 07:36:59 MSK 7h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service\nWed 2026-07-22 07:38:21 MSK 16h Tue 2026-07-21 07:36:14 MSK 7h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service\nWed 2026-07-22 07:41:49 MSK 16h Tue 2026-07-21 07:53:41 MSK 6h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service\nWed 2026-07-22 08:10:37 MSK 17h Tue 2026-07-21 08:07:25 MSK 6h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service\nWed 2026-07-22 08:37:02 MSK 17h Tue 2026-07-21 08:30:07 MSK 6h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service\nWed 2026-07-22 08:40:13 MSK 17h Tue 2026-07-21 08:50:14 MSK 5h 59min ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service\nWed 2026-07-22 08:53:36 MSK 18h Tue 2026-07-21 08:41:14 MSK 6h ago memos-backup.timer memos-backup.service\nSun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service\nSun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service\nSun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service\nSun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service\nMon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 7h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service\n=== BACKUP_UNITS ===\ndpkg-db-backup.service static -\nfilebrowser-backup.service static -\nhomelab-alertmanager-backup.service static -\nhomelab-appbackup-dockge-stacks.service static -\nhomelab-appbackup-mariadb-batch.service static -\nhomelab-appbackup-npmplus-edge.service static -\nhomelab-appbackup-postgres-batch.service static -\nhomelab-appbackup-sqlite-batch.service static -\nhomelab-authentik-backup.service static -\nhomelab-auto-backup.service static -\nhomelab-backup-coverage-matrix.service static -\nhomelab-backup-freshness.service static -\nhomelab-backup-healthcheck.service static -\nhomelab-backup-matrix-refresh.service static -\nhomelab-backup-sla-health.service static -\nhomelab-beszel-backup.service static -\nhomelab-blackbox-exporter-backup.service static -\nhomelab-cluster-admin-incident-engine-backup.service static -\nhomelab-crypto-portfolio-backup.service static -\nhomelab-edge-vm-vzdump-backup.service static -\nhomelab-emergency-backup.service static -\nhomelab-extended-appbackup.service static -\nhomelab-gitea-backup.service static -\nhomelab-gitea-secondary-backup.service static -\nhomelab-legacy-monitoring-backups.service static -\nhomelab-mail-cloud-critical-upload.service static -\nhomelab-mail-cloud-edge-vm-upload.service static -\nhomelab-mail-cloud-nextcloud-vm-upload.service static -\nhomelab-mail-cloud-restore-drill.service static -\nhomelab-npmplus-kuma-config-backup.service static -\nhomelab-router-running-config-mail-cloud.service static -\nhomelab-vaultwarden-backup.service static -\nhomelab-verified-backup-health.service static -\nhomelab-vm-backup-health.service static -\nhomelab-vm-backup-policy.service static -\nhomelab-vm-full-backup.service static -\nhomelab-vm-local-dumps-2cloud.service static -\nhomelab-vm160-remote-backup.service static -\nimmich-media-backup.service static -\nmemos-backup.service static -\nnetbird-vps-backup.service static -\npaperless-backup.service static -\nskladchik-reports-monitor-backup-restore-check.service static -\nskladchik-reports-monitor-backup.service static -\ndpkg-db-backup.timer enabled enabled\nfilebrowser-backup.timer enabled enabled\nhomelab-alertmanager-backup.timer enabled enabled\nhomelab-appbackup-dockge-stacks.timer enabled enabled\nhomelab-appbackup-mariadb-batch.timer enabled enabled\nhomelab-appbackup-npmplus-edge.timer enabled enabled\nhomelab-appbackup-postgres-batch.timer enabled enabled\nhomelab-appbackup-sqlite-batch.timer enabled enabled\nhomelab-authentik-backup.timer disabled enabled\nhomelab-auto-backup.timer enabled enabled\nhomelab-backup-coverage-matrix.timer enabled enabled\nhomelab-backup-freshness.timer enabled enabled\nhomelab-backup-healthcheck.timer enabled enabled\nhomelab-backup-matrix-refresh.timer disabled enabled\nhomelab-backup-sla-health.timer enabled enabled\nhomelab-beszel-backup.timer enabled enabled\nhomelab-blackbox-exporter-backup.timer enabled enabled\nhomelab-cluster-admin-incident-engine-backup.timer enabled enabled\nhomelab-crypto-portfolio-backup.timer enabled enabled\nhomelab-edge-vm-vzdump-backup.timer disabled enabled\nhomelab-emergency-backup.timer enabled enabled\nhomelab-extended-appbackup.timer enabled enabled\nhomelab-gitea-backup.timer disabled enabled\nhomelab-gitea-secondary-backup.timer enabled enabled\nhomelab-legacy-monitoring-backups.timer enabled enabled\nhomelab-mail-cloud-critical-upload.timer enabled enabled\nhomelab-mail-cloud-edge-vm-upload.timer enabled enabled\nhomelab-mail-cloud-nextcloud-vm-upload.timer enabled enabled\nhomelab-mail-cloud-restore-drill.timer enabled enabled\nhomelab-npmplus-kuma-config-backup.timer disabled enabled\nhomelab-router-running-config-mail-cloud.timer enabled enabled\nhomelab-vaultwarden-backup.timer disabled enabled\nhomelab-verified-backup-health.timer enabled enabled\nhomelab-vm-backup-health.timer enabled enabled\nhomelab-vm-backup-policy.timer enabled enabled\nhomelab-vm-full-backup.timer enabled enabled\nhomelab-vm-local-dumps-2cloud.timer enabled enabled\nhomelab-vm160-remote-backup.timer disabled enabled\nimmich-media-backup.timer disabled enabled\nmemos-backup.timer enabled enabled\nnetbird-vps-backup.timer disabled enabled\npaperless-backup.timer disabled enabled\nskladchik-reports-monitor-backup-restore-check.timer enabled enabled\nskladchik-reports-monitor-backup.timer enabled enabled\n=== BACKUP_PIPELINE_FILES ===\n=== LOCAL_BACKUP_ARCHIVES ===\n7022386770\t2026-07-18T03:31:03.6566057240\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.vma.zst\n7021952392\t2026-07-21T03:31:06.5717414950\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst\n7021297699\t2026-07-19T03:31:06.8755351020\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst\n7016275481\t2026-07-20T03:31:03.2165123470\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst\n329541843\t2026-07-21T03:30:09.7688696330\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst\n326601118\t2026-07-20T03:30:08.2096782720\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst\n323668231\t2026-07-19T03:45:46.2209130400\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst\n321121649\t2026-07-18T03:30:08.0777625900\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.tar.zst\n222777732\t2026-07-21T03:30:15.9059638290\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst\n222761273\t2026-07-20T03:30:13.7937629440\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst\n222745965\t2026-07-19T03:45:57.2700790730\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst\n222727760\t2026-07-18T03:30:13.5578457230\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.tar.zst\n3021\t2026-07-19T03:31:07.4745284120\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.log\n3020\t2026-07-21T03:31:07.3945472590\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.log\n3018\t2026-07-18T03:31:03.9197988860\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.log\n3017\t2026-07-20T03:31:04.1706719350\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.log\n1089\t2026-07-20T03:30:14.0079985330\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.log\n1086\t2026-07-20T03:30:08.4640241540\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.log\n989\t2026-07-21T03:30:16.1364976560\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.log\n989\t2026-07-18T03:30:13.7816006690\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.log\n986\t2026-07-21T03:30:10.0285327490\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.log\n986\t2026-07-18T03:30:08.3002411830\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.log\n731\t2026-07-19T03:45:57.5197000270\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.log\n729\t2026-07-19T03:45:46.4444121980\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.log\n========== NODE=pve02 ==========\npve02.gram1.ru\n=== RCLONE_REMOTES ===\npve01-mail-01:\npve01-mail-01-crypt:\npve01-mail-02:\npve01-mail-02-crypt:\npve01-mail-03:\npve01-mail-03-crypt:\npve01-mail-04:\npve01-mail-04-crypt:\npve01-mail-05:\npve01-mail-05-crypt:\npve02-mail-01:\npve02-mail-01-crypt:\npve02-mail-02:\npve02-mail-02-crypt:\npve02-mail-03:\npve02-mail-03-crypt:\npve02-mail-04:\npve02-mail-04-crypt:\npve03-mail-01:\npve03-mail-01-crypt:\npve03-mail-02:\npve03-mail-02-crypt:\npve03-mail-03:\npve03-mail-03-crypt:\npve03-mail-04:\npve03-mail-04-crypt:\nNODE_REMOTE_COUNT=26\nREMOTE_HEALTH node=pve02 remote=pve01-mail-01: status=OK\n=== BACKUP_TIMERS ===\nTue 2026-07-21 18:41:03 MSK 3h 51min Tue 2026-07-21 12:41:22 MSK 2h 7min ago forum-prod-fresh5-cloud-backup.timer forum-prod-fresh5-cloud-backup.service\nWed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:22 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service\nWed 2026-07-22 04:20:00 MSK 13h Tue 2026-07-21 04:20:12 MSK 10h ago homelab-cluster-admin-vm-mail-cloud-backup.timer homelab-cluster-admin-vm-mail-cloud-backup.service\n=== BACKUP_UNITS ===\ncodevipe-cloud-backup.service static -\ndpkg-db-backup.service static -\nforum-prod-fresh5-cloud-backup.service static -\nhomelab-cluster-admin-vm-mail-cloud-backup.service static -\ncodevipe-cloud-backup.timer disabled enabled\ndpkg-db-backup.timer enabled enabled\nforum-prod-fresh5-cloud-backup.timer enabled enabled\nhomelab-cluster-admin-vm-mail-cloud-backup.timer enabled enabled\n=== BACKUP_PIPELINE_FILES ===\n=== LOCAL_BACKUP_ARCHIVES ===\n997503422\t2026-07-01T03:31:11.1700059310\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.vma.zst\n868048369\t2026-06-30T03:31:08.4903256780\t/var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.vma.zst\n288349157\t2026-07-21T03:30:21.2480085730\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.tar.zst\n286538526\t2026-07-20T03:30:18.4386603270\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.tar.zst\n284443608\t2026-07-19T03:47:41.8525503200\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst\n283022034\t2026-07-18T03:30:19.7617214460\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_18-03_30_03.tar.zst\n220080970\t2026-07-21T03:30:33.3910787180\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.tar.zst\n220065825\t2026-07-20T03:30:30.8287264090\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.tar.zst\n220049041\t2026-07-19T03:47:58.9256407960\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst\n220033534\t2026-07-18T03:30:37.2878143100\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_18-03_30_20.tar.zst\n7547\t2026-07-18T01:09:39.2467331080\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_18-01_02_44.log\n7443\t2026-07-18T01:45:13.3149404960\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_18-01_38_20.log\n5796\t2026-07-17T21:55:21.5648458010\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_17-21_30_20.log\n2685\t2026-06-30T03:31:09.1616927060\t/var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.log\n2590\t2026-07-01T03:31:11.9500372070\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.log\n1088\t2026-07-20T03:30:31.0784689220\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.log\n1085\t2026-07-20T03:30:18.7427551960\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.log\n988\t2026-07-21T03:30:33.6364088960\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.log\n988\t2026-07-18T03:30:37.5450736960\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_18-03_30_20.log\n985\t2026-07-21T03:30:21.5465818340\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.log\n985\t2026-07-18T03:30:20.0884248500\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_18-03_30_03.log\n731\t2026-07-19T03:47:59.2238139650\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.log\n728\t2026-07-19T03:47:42.1183953260\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.log\n========== NODE=pve03 ==========\npve03.gram1.ru\n=== RCLONE_REMOTES ===\npve01-mail-01:\npve01-mail-01-crypt:\npve01-mail-02:\npve01-mail-02-crypt:\npve01-mail-03:\npve01-mail-03-crypt:\npve01-mail-04:\npve01-mail-04-crypt:\npve01-mail-05:\npve01-mail-05-crypt:\npve02-mail-01:\npve02-mail-01-crypt:\npve02-mail-02:\npve02-mail-02-crypt:\npve02-mail-03:\npve02-mail-03-crypt:\npve02-mail-04:\npve02-mail-04-crypt:\npve03-mail-01:\npve03-mail-01-crypt:\npve03-mail-02:\npve03-mail-02-crypt:\npve03-mail-03:\npve03-mail-03-crypt:\npve03-mail-04:\npve03-mail-04-crypt:\nNODE_REMOTE_COUNT=26\nREMOTE_HEALTH node=pve03 remote=pve01-mail-01: status=OK\n=== BACKUP_TIMERS ===\nWed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:10 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service\n=== BACKUP_UNITS ===\ndpkg-db-backup.service static -\ndpkg-db-backup.timer enabled enabled\n=== BACKUP_PIPELINE_FILES ===\n=== LOCAL_BACKUP_ARCHIVES ===\n11116472459\t2026-07-20T03:33:53.2839064370\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.vma.zst\n11043834894\t2026-07-21T03:33:54.3725716230\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.vma.zst\n11013723404\t2026-07-19T03:33:53.3463878040\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.vma.zst\n10994820121\t2026-07-18T03:33:47.1540038630\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_18-03_30_02.vma.zst\n1137072350\t2026-07-21T03:34:37.6276811360\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.vma.zst\n838951976\t2026-07-19T03:34:33.1034859540\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.vma.zst\n838817925\t2026-07-20T03:34:37.1719936670\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.vma.zst\n785249297\t2026-07-18T03:34:27.1531047780\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_18-03_33_48.vma.zst\n7106\t2026-07-02T03:37:53.1040459570\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_02-03_30_02.log\n7102\t2026-07-03T03:36:49.1036784480\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_03-03_30_05.log\n6564\t2026-07-18T03:33:48.0107527290\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_18-03_30_02.log\n6560\t2026-07-20T03:33:54.9283349630\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.log\n6559\t2026-07-21T03:33:55.1428157340\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.log\n6459\t2026-07-19T03:33:53.7631656400\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.log\n6246\t2026-07-04T03:33:39.2197744030\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_04-03_30_02.log\n4107\t2026-07-07T03:31:51.2516816970\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_07-03_30_01.log\n2642\t2026-07-21T03:34:38.4491706750\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.log\n2500\t2026-07-20T03:34:37.2013439550\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.log\n2404\t2026-07-19T03:34:34.1266444600\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.log\n2305\t2026-07-18T03:34:27.7179455720\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_18-03_33_48.log\n=== CLOUD_REMOTE_SUMMARY ===\nEXPECTED_TOTAL=13\nDETECTED_TOTAL=78\nREMOTE_COUNT=MISMATCH\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 43215c7..a81d89e 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,5 +1,5 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=BACKUP-PIPELINE-DEFINITION-AUDIT-061 +COMMAND_ID=BACKUP-CLUSTER-13CLOUD-TOPOLOGY-062 STATUS=OK RC=0 HOST=pve01 @@ -9,2031 +9,334 @@ REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1 -COMMAND_SHA256=08de7a8820803f855426b27b3ecfd2e646209715ff5ac4aba48b73fa45623b28 +COMMAND_SHA256=eca35846e7097b9c238bf8c082e02c81f0b2893473e27ddb0a5b0854d5d3703d DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGES_MADE=false SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=5b51699fff305570e41040468ef3f04a84e21c0c977695ed1834aa40c01946b5 -SANITIZED_OUTPUT_SHA256=384c3f223883553a582cbf25ff144f7ff8a0a1dc79110ffe089b7154a12aed74 +RAW_EVIDENCE_SHA256=3ba8aaf65323f1801f1eb63016759f4767a9d5be8bb21f15d92042e408395afd +SANITIZED_OUTPUT_SHA256=3ba8aaf65323f1801f1eb63016759f4767a9d5be8bb21f15d92042e408395afd OUTPUT_BEGIN -===== TIMER dpkg-db-backup.timer ===== -# /usr/lib/systemd/system/dpkg-db-backup.timer -[Unit] -Description=Daily dpkg database backup timer -Documentation=man:dpkg(1) - -[Timer] -OnCalendar=daily -Persistent=true - -[Install] -WantedBy=timers.target -===== SERVICE dpkg-db-backup.service ===== -# /usr/lib/systemd/system/dpkg-db-backup.service -[Unit] -Description=Daily dpkg database backup service -Documentation=man:dpkg(1) - -[Service] -Type=oneshot -ExecStart=/usr/libexec/dpkg/dpkg-db-backup - -# /etc/systemd/system/dpkg-db-backup.service.d/90-backup-matrix-refresh.conf -[Service] -ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/libexec/dpkg/dpkg-db-backup ; argv[]=/usr/libexec/dpkg/dpkg-db-backup ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } -ActiveState=inactive -===== TIMER filebrowser-backup.timer ===== -# /etc/systemd/system/filebrowser-backup.timer -[Unit] -Description=Run File Browser backup - -[Timer] -OnCalendar=*-*-* 03:35:00 -Persistent=true - -[Install] -WantedBy=timers.target -===== SERVICE filebrowser-backup.service ===== -# /etc/systemd/system/filebrowser-backup.service -[Unit] -Description=File Browser backup -After=network-online.target - -[Service] -Type=oneshot -ExecStart=/root/filebrowser-backup.sh - -# /etc/systemd/system/filebrowser-backup.service.d/10-superseded-noop.conf -[Service] -ExecStart= -ExecStart=/usr/local/sbin/homelab-superseded-unit-ok filebrowser-backup.service superseded_by_current_appbackup_restore_proof - -# /etc/systemd/system/filebrowser-backup.service.d/90-backup-matrix-refresh.conf -[Service] -ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok filebrowser-backup.service superseded_by_current_appbackup_restore_proof ; ignore_errors=no ; start_time=[Tue 2026-07-21 03:35:14 MSK] ; stop_time=[Tue 2026-07-21 03:35:14 MSK] ; pid=2211390 ; code=exited ; status=0 } -ActiveState=inactive -===== TIMER homelab-alertmanager-backup.timer ===== -# /etc/systemd/system/homelab-alertmanager-backup.timer -[Unit] -Description=Daily Alertmanager two-cloud backup - -[Timer] -OnCalendar=*-*-* 06:35:00 -RandomizedDelaySec=10m -Persistent=true -Unit=homelab-alertmanager-backup.service - -[Install] -WantedBy=timers.target -===== SERVICE homelab-alertmanager-backup.service ===== -# /etc/systemd/system/homelab-alertmanager-backup.service -[Unit] -Description=Alertmanager encrypted two-cloud application backup -After=network-online.target -Wants=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-alertmanager-backup-run -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-alertmanager-backup-run ; argv[]=/usr/local/sbin/homelab-alertmanager-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 06:40:25 MSK] ; stop_time=[Tue 2026-07-21 06:40:54 MSK] ; pid=2820982 ; code=exited ; status=0 } -ActiveState=inactive -===== TIMER homelab-appbackup-dockge-stacks.timer ===== -# /etc/systemd/system/homelab-appbackup-dockge-stacks.timer -[Unit] -Description=Daily homelab app-level backup timer for dockge-stacks - -[Timer] -OnCalendar=*-*-* 03:40:00 -RandomizedDelaySec=20m -Persistent=true - -[Install] -WantedBy=timers.target -===== SERVICE homelab-appbackup-dockge-stacks.service ===== -# /etc/systemd/system/homelab-appbackup-dockge-stacks.service -[Unit] -Description=Homelab app-level backup for dockge-stacks - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/appbackupctl run -ExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run dockge-stacks - -# /etc/systemd/system/homelab-appbackup-dockge-stacks.service.d/90-backup-matrix-refresh.conf -[Service] -ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/appbackupctl ; argv[]=/usr/local/sbin/appbackupctl run ; ignore_errors=no ; start_time=[Tue 2026-07-21 03:51:25 MSK] ; stop_time=[Tue 2026-07-21 03:51:53 MSK] ; pid=2261529 ; code=exited ; status=0 } -ActiveState=inactive -===== TIMER homelab-appbackup-mariadb-batch.timer ===== -# /etc/systemd/system/homelab-appbackup-mariadb-batch.timer -[Unit] -Description=Daily Homelab MariaDB app batch backup timer - -[Timer] -OnCalendar=*-*-* 05:35:00 -RandomizedDelaySec=20m -Persistent=true - -[Install] -WantedBy=timers.target -===== SERVICE homelab-appbackup-mariadb-batch.service ===== -# /etc/systemd/system/homelab-appbackup-mariadb-batch.service -[Unit] -Description=Homelab MariaDB app batch backup - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/mariadb-apps-backup-run -ExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run bookstack - -# /etc/systemd/system/homelab-appbackup-mariadb-batch.service.d/90-backup-matrix-refresh.conf -[Service] -ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/mariadb-apps-backup-run ; argv[]=/usr/local/sbin/mariadb-apps-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 05:46:14 MSK] ; stop_time=[Tue 2026-07-21 05:46:44 MSK] ; pid=2623964 ; code=exited ; status=0 } -ActiveState=inactive -===== TIMER homelab-appbackup-npmplus-edge.timer ===== -# /etc/systemd/system/homelab-appbackup-npmplus-edge.timer -[Unit] -Description=Daily homelab app-level backup timer for npmplus-edge - -[Timer] -OnCalendar=*-*-* 04:05:00 -RandomizedDelaySec=20m -Persistent=true - -[Install] -WantedBy=timers.target -===== SERVICE homelab-appbackup-npmplus-edge.service ===== -# /etc/systemd/system/homelab-appbackup-npmplus-edge.service -[Unit] -Description=Homelab app-level backup for npmplus-edge - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/npmplus-edge-backup-run -ExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run npmplus-edge - -# /etc/systemd/system/homelab-appbackup-npmplus-edge.service.d/90-backup-matrix-refresh.conf -[Service] -ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/npmplus-edge-backup-run ; argv[]=/usr/local/sbin/npmplus-edge-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 04:19:17 MSK] ; stop_time=[Tue 2026-07-21 04:19:48 MSK] ; pid=2360589 ; code=exited ; status=0 } -ActiveState=inactive -===== TIMER homelab-appbackup-postgres-batch.timer ===== -# /etc/systemd/system/homelab-appbackup-postgres-batch.timer -[Unit] -Description=Daily Homelab Postgres app batch backup timer - -[Timer] -OnCalendar=*-*-* 05:10:00 -RandomizedDelaySec=20m -Persistent=true - -[Install] -WantedBy=timers.target -===== SERVICE homelab-appbackup-postgres-batch.service ===== -# /etc/systemd/system/homelab-appbackup-postgres-batch.service -[Unit] -Description=Homelab Postgres app batch backup - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/postgres-apps-backup-run -ExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run authentik -ExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run paperless -ExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run immich -ExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run netbox - -# /etc/systemd/system/homelab-appbackup-postgres-batch.service.d/90-backup-matrix-refresh.conf -[Service] -ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/postgres-apps-backup-run ; argv[]=/usr/local/sbin/postgres-apps-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 05:29:05 MSK] ; stop_time=[Tue 2026-07-21 05:32:29 MSK] ; pid=2568901 ; code=exited ; status=0 } -ActiveState=inactive -===== TIMER homelab-appbackup-sqlite-batch.timer ===== -# /etc/systemd/system/homelab-appbackup-sqlite-batch.timer -[Unit] -Description=Daily Homelab SQLite app batch backup timer - -[Timer] -OnCalendar=*-*-* 04:45:00 -RandomizedDelaySec=20m -Persistent=true - -[Install] -WantedBy=timers.target -===== SERVICE homelab-appbackup-sqlite-batch.service ===== -# /etc/systemd/system/homelab-appbackup-sqlite-batch.service -[Unit] -Description=Homelab SQLite app batch backup - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/sqlite-apps-backup-run -ExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run vaultwarden -ExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run gitea -ExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run uptime-kuma -ExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run gotify - -# /etc/systemd/system/homelab-appbackup-sqlite-batch.service.d/90-backup-matrix-refresh.conf -[Service] -ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/sqlite-apps-backup-run ; argv[]=/usr/local/sbin/sqlite-apps-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 04:57:14 MSK] ; stop_time=[Tue 2026-07-21 05:00:36 MSK] ; pid=2476638 ; code=exited ; status=0 } -ActiveState=inactive -===== TIMER homelab-authentik-backup.timer ===== -# /etc/systemd/system/homelab-authentik-backup.timer -[Unit] -Description=Run Homelab Authentik backup chain daily - -[Timer] -OnCalendar=*-*-* 04:50:00 -RandomizedDelaySec=10min -Persistent=true -Unit=homelab-authentik-backup.service - -[Install] -WantedBy=timers.target -===== SERVICE homelab-authentik-backup.service ===== -# /etc/systemd/system/homelab-authentik-backup.service -[Unit] -Description=Homelab Authentik backup chain -Wants=network-online.target -After=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-backup-authentik -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-backup-authentik ; argv[]=/usr/local/sbin/homelab-backup-authentik ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } -ActiveState=inactive -===== TIMER homelab-auto-backup.timer ===== -# /etc/systemd/system/homelab-auto-backup.timer -[Unit] -Description=Run homelab automatic full backup weekly - -[Timer] -OnCalendar=Sun *-*-* 03:30:00 -Persistent=true -RandomizedDelaySec=20m -Unit=homelab-auto-backup.service - -[Install] -WantedBy=timers.target -===== SERVICE homelab-auto-backup.service ===== -# /etc/systemd/system/homelab-auto-backup.service -[Unit] -Description=Homelab automatic full backup with retention -Wants=network-online.target -After=network-online.target - -[Service] -Type=oneshot -ExecStart=/root/homelab-auto-backup-with-retention.sh -Nice=10 -IOSchedulingClass=best-effort -IOSchedulingPriority=7 - -# /etc/systemd/system/homelab-auto-backup.service.d/90-backup-matrix-refresh.conf -[Service] -ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service -Result=success -ExecMainStatus=0 -ExecStart={ path=/root/homelab-auto-backup-with-retention.sh ; argv[]=/root/homelab-auto-backup-with-retention.sh ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } -ActiveState=inactive -===== TIMER homelab-backup-coverage-matrix.timer ===== -# /etc/systemd/system/homelab-backup-coverage-matrix.timer -[Unit] -Description=Homelab backup coverage matrix timer - -[Timer] -OnCalendar=*-*-* 08:35:00 -RandomizedDelaySec=20m -Persistent=true - -[Install] -WantedBy=timers.target -===== SERVICE homelab-backup-coverage-matrix.service ===== -# /etc/systemd/system/homelab-backup-coverage-matrix.service -[Unit] -Description=Homelab backup coverage matrix - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-backup-coverage-matrix - -# /etc/systemd/system/homelab-backup-coverage-matrix.service.d/90-backup-matrix-refresh.conf -[Service] -ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-backup-coverage-matrix ; argv[]=/usr/local/sbin/homelab-backup-coverage-matrix ; ignore_errors=no ; start_time=[Tue 2026-07-21 08:50:14 MSK] ; stop_time=[Tue 2026-07-21 08:50:15 MSK] ; pid=3277860 ; code=exited ; status=0 } -ActiveState=inactive -===== TIMER homelab-backup-freshness.timer ===== -# /etc/systemd/system/homelab-backup-freshness.timer -[Unit] -Description=Run homelab backup freshness collector daily - -[Timer] -OnCalendar=daily -Persistent=true -RandomizedDelaySec=20m -Unit=homelab-backup-freshness.service - -[Install] -WantedBy=timers.target -===== SERVICE homelab-backup-freshness.service ===== -# /etc/systemd/system/homelab-backup-freshness.service -[Unit] -Description=Homelab backup freshness collector -Documentation=man:systemd.service(5) -After=local-fs.target - -[Service] -Type=oneshot -EnvironmentFile=-/etc/default/homelab-backup-freshness -ExecStart=/usr/local/sbin/homelab-backup-freshness.sh - -# /etc/systemd/system/homelab-backup-freshness.service.d/90-backup-matrix-refresh.conf -[Service] -ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-backup-freshness.sh ; argv[]=/usr/local/sbin/homelab-backup-freshness.sh ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } -ActiveState=inactive -===== TIMER homelab-backup-healthcheck.timer ===== -# /etc/systemd/system/homelab-backup-healthcheck.timer -[Unit] -Description=Run homelab backup healthcheck daily - -[Timer] -OnCalendar=*-*-* 07:00:00 -Persistent=true -RandomizedDelaySec=10m -Unit=homelab-backup-healthcheck.service - -[Install] -WantedBy=timers.target -===== SERVICE homelab-backup-healthcheck.service ===== -# /etc/systemd/system/homelab-backup-healthcheck.service -[Unit] -Description=Homelab backup healthcheck - -[Service] -Type=oneshot -ExecStart=/root/homelab-backup-healthcheck-wrapper.sh - -# /etc/systemd/system/homelab-backup-healthcheck.service.d/90-backup-matrix-refresh.conf -[Service] -ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service -Result=success -ExecMainStatus=0 -ExecStart={ path=/root/homelab-backup-healthcheck-wrapper.sh ; argv[]=/root/homelab-backup-healthcheck-wrapper.sh ; ignore_errors=no ; start_time=[Tue 2026-07-21 07:05:30 MSK] ; stop_time=[Tue 2026-07-21 07:05:43 MSK] ; pid=2900293 ; code=exited ; status=0 } -ActiveState=inactive -===== TIMER homelab-backup-matrix-refresh.timer ===== -# /etc/systemd/system/homelab-backup-matrix-refresh.timer -[Unit] -Description=Refresh unified homelab backup matrix state - -[Timer] -OnBootSec=2min -OnUnitActiveSec=5min -AccuracySec=30s -Persistent=true -Unit=homelab-backup-matrix-refresh.service - -[Install] -WantedBy=timers.target -===== SERVICE homelab-backup-matrix-refresh.service ===== -# /etc/systemd/system/homelab-backup-matrix-refresh.service -[Unit] -Description=Generate unified homelab backup matrix state -After=local-fs.target - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-backup-matrix-refresh -TimeoutStartSec=5min -Result=exit-code -ExecMainStatus=1 -ExecStart={ path=/usr/local/sbin/homelab-backup-matrix-refresh ; argv[]=/usr/local/sbin/homelab-backup-matrix-refresh ; ignore_errors=no ; start_time=[Tue 2026-07-21 10:02:41 MSK] ; stop_time=[Tue 2026-07-21 10:02:42 MSK] ; pid=3525445 ; code=exited ; status=1 } -ActiveState=failed -===== TIMER homelab-backup-sla-health.timer ===== -# /etc/systemd/system/homelab-backup-sla-health.timer -[Unit] -Description=Homelab backup freshness SLA health timer - -[Timer] -OnCalendar=*-*-* 08:00:00 -RandomizedDelaySec=20m -Persistent=true - -[Install] -WantedBy=timers.target -===== SERVICE homelab-backup-sla-health.service ===== -# /etc/systemd/system/homelab-backup-sla-health.service -[Unit] -Description=Homelab backup freshness SLA health -After=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-backup-sla-health - -# /etc/systemd/system/homelab-backup-sla-health.service.d/90-backup-matrix-refresh.conf -[Service] -ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-backup-sla-health ; argv[]=/usr/local/sbin/homelab-backup-sla-health ; ignore_errors=no ; start_time=[Tue 2026-07-21 08:07:25 MSK] ; stop_time=[Tue 2026-07-21 08:07:26 MSK] ; pid=3122093 ; code=exited ; status=0 } -ActiveState=inactive -===== TIMER homelab-beszel-backup.timer ===== -# /etc/systemd/system/homelab-beszel-backup.timer -[Unit] -Description=Daily Beszel encrypted two-cloud backup - -[Timer] -OnCalendar=*-*-* 06:45:00 -RandomizedDelaySec=10m -Persistent=true -Unit=homelab-beszel-backup.service - -[Install] -WantedBy=timers.target -===== SERVICE homelab-beszel-backup.service ===== -# /etc/systemd/system/homelab-beszel-backup.service -[Unit] -Description=Beszel encrypted two-cloud application backup -After=network-online.target -Wants=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-beszel-backup-run -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-beszel-backup-run ; argv[]=/usr/local/sbin/homelab-beszel-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 06:47:47 MSK] ; stop_time=[Tue 2026-07-21 06:48:22 MSK] ; pid=2848350 ; code=exited ; status=0 } -ActiveState=inactive -===== TIMER homelab-blackbox-exporter-backup.timer ===== -# /etc/systemd/system/homelab-blackbox-exporter-backup.timer -[Unit] -Description=Daily Blackbox Exporter encrypted two-cloud backup - -[Timer] -OnCalendar=*-*-* 06:55:00 -RandomizedDelaySec=10m -Persistent=true -Unit=homelab-blackbox-exporter-backup.service - -[Install] -WantedBy=timers.target -===== SERVICE homelab-blackbox-exporter-backup.service ===== -# /etc/systemd/system/homelab-blackbox-exporter-backup.service -[Unit] -Description=Blackbox Exporter encrypted two-cloud application backup -After=network-online.target -Wants=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-blackbox-exporter-backup-run -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-blackbox-exporter-backup-run ; argv[]=/usr/local/sbin/homelab-blackbox-exporter-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 06:57:30 MSK] ; stop_time=[Tue 2026-07-21 06:57:54 MSK] ; pid=2876084 ; code=exited ; status=0 } -ActiveState=inactive -===== TIMER homelab-cluster-admin-incident-engine-backup.timer ===== -# /etc/systemd/system/homelab-cluster-admin-incident-engine-backup.timer -[Unit] -Description=Daily Cluster Admin Incident Engine two-cloud backup - -[Timer] -OnCalendar=*-*-* 07:25:00 -RandomizedDelaySec=15m -Persistent=true -Unit=homelab-cluster-admin-incident-engine-backup.service - -[Install] -WantedBy=timers.target -===== SERVICE homelab-cluster-admin-incident-engine-backup.service ===== -# /etc/systemd/system/homelab-cluster-admin-incident-engine-backup.service -[Unit] -Description=Cluster Admin Incident Engine encrypted two-cloud backup -After=network-online.target -Wants=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run ; argv[]=/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 07:36:14 MSK] ; stop_time=[Tue 2026-07-21 07:36:40 MSK] ; pid=3010566 ; code=exited ; status=0 } -ActiveState=inactive -===== TIMER homelab-crypto-portfolio-backup.timer ===== -# /etc/systemd/system/homelab-crypto-portfolio-backup.timer -[Unit] -Description=Daily Gram Crypto portfolio backup - -[Timer] -OnCalendar=*-*-* 06:25:00 -RandomizedDelaySec=15m -Persistent=true - -[Install] -WantedBy=timers.target -===== SERVICE homelab-crypto-portfolio-backup.service ===== -# /etc/systemd/system/homelab-crypto-portfolio-backup.service -[Unit] -Description=Gram Crypto portfolio two-cloud backup -After=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/crypto-portfolio-backup-run -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/crypto-portfolio-backup-run ; argv[]=/usr/local/sbin/crypto-portfolio-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 06:32:25 MSK] ; stop_time=[Tue 2026-07-21 06:32:50 MSK] ; pid=2794060 ; code=exited ; status=0 } -ActiveState=inactive -===== TIMER homelab-edge-vm-vzdump-backup.timer ===== -# /etc/systemd/system/homelab-edge-vm-vzdump-backup.timer -[Unit] -Description=Run Homelab edge-vm VM-level backup chain weekly - -[Timer] -OnCalendar=Sun *-*-* 06:20:00 -RandomizedDelaySec=20min -Persistent=true -Unit=homelab-edge-vm-vzdump-backup.service - -[Install] -WantedBy=timers.target -===== SERVICE homelab-edge-vm-vzdump-backup.service ===== -# /etc/systemd/system/homelab-edge-vm-vzdump-backup.service -[Unit] -Description=Homelab edge-vm VM-level vzdump backup chain -Wants=network-online.target -After=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-backup-edge-vm-vzdump - -# /etc/systemd/system/homelab-edge-vm-vzdump-backup.service.d/90-backup-matrix-refresh.conf -[Service] -ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-backup-edge-vm-vzdump ; argv[]=/usr/local/sbin/homelab-backup-edge-vm-vzdump ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } -ActiveState=inactive -===== TIMER homelab-emergency-backup.timer ===== -# /etc/systemd/system/homelab-emergency-backup.timer -[Unit] -Description=Daily homelab emergency/config backup timer - -[Timer] -OnCalendar=*-*-* 03:10:00 -RandomizedDelaySec=20m -Persistent=true - -[Install] -WantedBy=timers.target -===== SERVICE homelab-emergency-backup.service ===== -# /etc/systemd/system/homelab-emergency-backup.service -[Unit] -Description=Homelab emergency/config backup to two encrypted clouds - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/backupctl run -ExecStartPost=/usr/local/sbin/backup-retention 14 dry-run - -# /etc/systemd/system/homelab-emergency-backup.service.d/90-backup-matrix-refresh.conf -[Service] -ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/backupctl ; argv[]=/usr/local/sbin/backupctl run ; ignore_errors=no ; start_time=[Tue 2026-07-21 03:13:14 MSK] ; stop_time=[Tue 2026-07-21 03:13:42 MSK] ; pid=2137825 ; code=exited ; status=0 } -ActiveState=inactive -===== TIMER homelab-extended-appbackup.timer ===== -# /etc/systemd/system/homelab-extended-appbackup.timer -[Unit] -Description=Homelab extended P1/P2 appbackup timer - -[Timer] -OnCalendar=*-*-* 05:55:00 -RandomizedDelaySec=20m -Persistent=true - -[Install] -WantedBy=timers.target -===== SERVICE homelab-extended-appbackup.service ===== -# /etc/systemd/system/homelab-extended-appbackup.service -[Unit] -Description=Homelab extended P1/P2 appbackup run -After=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-extended-appbackup-run - -# /etc/systemd/system/homelab-extended-appbackup.service.d/90-backup-matrix-refresh.conf -[Service] -ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-extended-appbackup-run ; argv[]=/usr/local/sbin/homelab-extended-appbackup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 05:57:30 MSK] ; stop_time=[Tue 2026-07-21 06:06:46 MSK] ; pid=2671623 ; code=exited ; status=0 } -ActiveState=inactive -===== TIMER homelab-gitea-backup.timer ===== -# /etc/systemd/system/homelab-gitea-backup.timer -[Unit] -Description=Run Homelab Gitea backup chain daily - -[Timer] -OnCalendar=*-*-* 05:10:00 -RandomizedDelaySec=10min -Persistent=true -Unit=homelab-gitea-backup.service - -[Install] -WantedBy=timers.target -===== SERVICE homelab-gitea-backup.service ===== -# /etc/systemd/system/homelab-gitea-backup.service -[Unit] -Description=Homelab Gitea backup chain -Wants=network-online.target -After=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-backup-gitea -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-backup-gitea ; argv[]=/usr/local/sbin/homelab-backup-gitea ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } -ActiveState=inactive -===== TIMER homelab-gitea-secondary-backup.timer ===== -# /etc/systemd/system/homelab-gitea-secondary-backup.timer -[Unit] -Description=Daily verified Gitea backup to pve03 - -[Timer] -OnCalendar=*-*-* 04:20:00 -RandomizedDelaySec=10m -Persistent=true -Unit=homelab-gitea-secondary-backup.service - -[Install] -WantedBy=timers.target -===== SERVICE homelab-gitea-secondary-backup.service ===== -# /etc/systemd/system/homelab-gitea-secondary-backup.service -[Unit] -Description=Create and copy a verified Gitea backup to pve03 -After=pve-cluster.service network-online.target -Wants=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-gitea-secondary-backup -TimeoutStartSec=2h -Nice=10 -IOSchedulingClass=best-effort -IOSchedulingPriority=7 -UMask=0077 -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-gitea-secondary-backup ; argv[]=/usr/local/sbin/homelab-gitea-secondary-backup ; ignore_errors=no ; start_time=[Tue 2026-07-21 04:28:14 MSK] ; stop_time=[Tue 2026-07-21 04:28:18 MSK] ; pid=2376819 ; code=exited ; status=0 } -ActiveState=inactive -===== TIMER homelab-legacy-monitoring-backups.timer ===== -# /etc/systemd/system/homelab-legacy-monitoring-backups.timer -[Unit] -Description=Daily legacy monitoring applications two-cloud backup batch - -[Timer] -OnCalendar=*-*-* 07:05:00 -RandomizedDelaySec=15m -Persistent=true -Unit=homelab-legacy-monitoring-backups.service - -[Install] -WantedBy=timers.target -===== SERVICE homelab-legacy-monitoring-backups.service ===== -# /etc/systemd/system/homelab-legacy-monitoring-backups.service -[Unit] -Description=Legacy monitoring applications encrypted two-cloud backup batch -After=network-online.target -Wants=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-legacy-monitoring-backups-run -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-legacy-monitoring-backups-run ; argv[]=/usr/local/sbin/homelab-legacy-monitoring-backups-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 07:18:55 MSK] ; stop_time=[Tue 2026-07-21 07:21:45 MSK] ; pid=2950676 ; code=exited ; status=0 } -ActiveState=inactive -===== TIMER homelab-mail-cloud-critical-upload.timer ===== -# /etc/systemd/system/homelab-mail-cloud-critical-upload.timer -[Unit] -Description=Daily Homelab Mail Cloud critical/small baseline upload - -[Timer] -OnCalendar=*-*-* 06:40:00 -RandomizedDelaySec=15m -Persistent=true -Unit=homelab-mail-cloud-critical-upload.service - -[Install] -WantedBy=timers.target -===== SERVICE homelab-mail-cloud-critical-upload.service ===== -# /etc/systemd/system/homelab-mail-cloud-critical-upload.service -[Unit] -Description=Homelab Mail Cloud critical/small baseline upload -After=network-online.target -Wants=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-mail-cloud-critical-upload -Nice=10 -IOSchedulingClass=best-effort -IOSchedulingPriority=7 -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-mail-cloud-critical-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-critical-upload ; ignore_errors=no ; start_time=[Tue 2026-07-21 06:40:25 MSK] ; stop_time=[Tue 2026-07-21 06:42:44 MSK] ; pid=2820985 ; code=exited ; status=0 } -ActiveState=inactive -===== TIMER homelab-mail-cloud-edge-vm-upload.timer ===== -# /etc/systemd/system/homelab-mail-cloud-edge-vm-upload.timer -[Unit] -Description=Weekly Homelab Mail Cloud edge-vm large chunked upload - -[Timer] -OnCalendar=Sun *-*-* 09:30:00 -RandomizedDelaySec=30m -Persistent=true -Unit=homelab-mail-cloud-edge-vm-upload.service - -[Install] -WantedBy=timers.target -===== SERVICE homelab-mail-cloud-edge-vm-upload.service ===== -# /etc/systemd/system/homelab-mail-cloud-edge-vm-upload.service -[Unit] -Description=Homelab Mail Cloud edge-vm large chunked upload -After=network-online.target -Wants=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload -Nice=10 -IOSchedulingClass=best-effort -IOSchedulingPriority=7 -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } -ActiveState=inactive -===== TIMER homelab-mail-cloud-nextcloud-vm-upload.timer ===== -# /etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.timer -[Unit] -Description=Run Homelab Mail Cloud Nextcloud VM upload daily - -[Timer] -OnCalendar=*-*-* 07:30:00 -RandomizedDelaySec=30m -Persistent=true - -[Install] -WantedBy=timers.target -===== SERVICE homelab-mail-cloud-nextcloud-vm-upload.service ===== -# /etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.service -[Unit] -Description=Homelab Mail Cloud Nextcloud VM vzdump upload -Wants=network-online.target -After=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload -Nice=10 -IOSchedulingClass=best-effort -IOSchedulingPriority=7 -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload ; ignore_errors=no ; start_time=[Tue 2026-07-21 07:53:41 MSK] ; stop_time=[Tue 2026-07-21 07:56:26 MSK] ; pid=3068432 ; code=exited ; status=0 } -ActiveState=inactive -===== TIMER homelab-mail-cloud-restore-drill.timer ===== -# /etc/systemd/system/homelab-mail-cloud-restore-drill.timer -[Unit] -Description=Weekly Homelab Mail Cloud restore drill - -[Timer] -OnCalendar=Mon *-*-* 07:20:00 -RandomizedDelaySec=20m -Persistent=true -Unit=homelab-mail-cloud-restore-drill.service - -[Install] -WantedBy=timers.target -===== SERVICE homelab-mail-cloud-restore-drill.service ===== -# /etc/systemd/system/homelab-mail-cloud-restore-drill.service -[Unit] -Description=Homelab Mail Cloud restore drill -After=network-online.target -Wants=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-mail-cloud-restore-drill -Nice=10 -IOSchedulingClass=best-effort -IOSchedulingPriority=7 - -# /etc/systemd/system/homelab-mail-cloud-restore-drill.service.d/10-superseded-noop.conf -[Service] -ExecStart= -ExecStart=/usr/local/sbin/homelab-superseded-unit-ok homelab-mail-cloud-restore-drill.service superseded_by_safety_net_restore_check - -# /etc/systemd/system/homelab-mail-cloud-restore-drill.service.d/90-backup-matrix-refresh.conf -[Service] -ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok homelab-mail-cloud-restore-drill.service superseded_by_safety_net_restore_check ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } -ActiveState=inactive -===== TIMER homelab-npmplus-kuma-config-backup.timer ===== -# /etc/systemd/system/homelab-npmplus-kuma-config-backup.timer -[Unit] -Description=Run Homelab NPMplus + Kuma config backup chain daily - -[Timer] -OnCalendar=*-*-* 04:20:00 -RandomizedDelaySec=10min -Persistent=true -Unit=homelab-npmplus-kuma-config-backup.service - -[Install] -WantedBy=timers.target -===== SERVICE homelab-npmplus-kuma-config-backup.service ===== -# /etc/systemd/system/homelab-npmplus-kuma-config-backup.service -[Unit] -Description=Homelab NPMplus + Kuma config backup chain -Wants=network-online.target -After=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-backup-npmplus-kuma-configs -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-backup-npmplus-kuma-configs ; argv[]=/usr/local/sbin/homelab-backup-npmplus-kuma-configs ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } -ActiveState=inactive -===== TIMER homelab-router-running-config-mail-cloud.timer ===== -# /etc/systemd/system/homelab-router-running-config-mail-cloud.timer -[Unit] -Description=Run Homelab router running-config Mail-cloud backup daily - -[Timer] -OnCalendar=*-*-* 05:15:00 -RandomizedDelaySec=20m -Persistent=true - -[Install] -WantedBy=timers.target -===== SERVICE homelab-router-running-config-mail-cloud.service ===== -# /etc/systemd/system/homelab-router-running-config-mail-cloud.service -[Unit] -Description=Homelab router running-config Mail-cloud backup -Wants=network-online.target -After=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-router-running-config-mail-cloud -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-router-running-config-mail-cloud ; argv[]=/usr/local/sbin/homelab-router-running-config-mail-cloud ; ignore_errors=no ; start_time=[Tue 2026-07-21 05:17:57 MSK] ; stop_time=[Tue 2026-07-21 05:18:05 MSK] ; pid=2540965 ; code=exited ; status=0 } -ActiveState=inactive -===== TIMER homelab-vaultwarden-backup.timer ===== -# /etc/systemd/system/homelab-vaultwarden-backup.timer -[Unit] -Description=Run Homelab Vaultwarden backup chain daily - -[Timer] -OnCalendar=*-*-* 05:30:00 -RandomizedDelaySec=10min -Persistent=true -Unit=homelab-vaultwarden-backup.service - -[Install] -WantedBy=timers.target -===== SERVICE homelab-vaultwarden-backup.service ===== -# /etc/systemd/system/homelab-vaultwarden-backup.service -[Unit] -Description=Homelab Vaultwarden backup chain -Wants=network-online.target -After=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-backup-vaultwarden -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-backup-vaultwarden ; argv[]=/usr/local/sbin/homelab-backup-vaultwarden ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } -ActiveState=inactive -===== TIMER homelab-verified-backup-health.timer ===== -# /etc/systemd/system/homelab-verified-backup-health.timer -[Unit] -Description=Homelab verified backup health evidence timer - -[Timer] -OnCalendar=*-*-* 06:30:00 -RandomizedDelaySec=20m -Persistent=true - -[Install] -WantedBy=timers.target -===== SERVICE homelab-verified-backup-health.service ===== -# /etc/systemd/system/homelab-verified-backup-health.service -[Unit] -Description=Homelab verified backup health evidence - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-backup-health-evidence - -# /etc/systemd/system/homelab-verified-backup-health.service.d/90-backup-matrix-refresh.conf -[Service] -ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-backup-health-evidence ; argv[]=/usr/local/sbin/homelab-backup-health-evidence ; ignore_errors=no ; start_time=[Tue 2026-07-21 06:43:54 MSK] ; stop_time=[Tue 2026-07-21 06:45:16 MSK] ; pid=2823792 ; code=exited ; status=0 } -ActiveState=inactive -===== TIMER homelab-vm-backup-health.timer ===== -# /etc/systemd/system/homelab-vm-backup-health.timer -[Unit] -Description=Homelab VM backup health timer - -[Timer] -OnCalendar=*-*-* 07:30:00 -RandomizedDelaySec=20m -Persistent=true - -[Install] -WantedBy=timers.target -===== SERVICE homelab-vm-backup-health.service ===== -# /etc/systemd/system/homelab-vm-backup-health.service -[Unit] -Description=Homelab VM backup health - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-vm-backup-health - -# /etc/systemd/system/homelab-vm-backup-health.service.d/90-backup-matrix-refresh.conf -[Service] -ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-vm-backup-health ; argv[]=/usr/local/sbin/homelab-vm-backup-health ; ignore_errors=no ; start_time=[Tue 2026-07-21 07:36:59 MSK] ; stop_time=[Tue 2026-07-21 07:36:59 MSK] ; pid=3011738 ; code=exited ; status=0 } -ActiveState=inactive -===== TIMER homelab-vm-backup-policy.timer ===== -# /etc/systemd/system/homelab-vm-backup-policy.timer -[Unit] -Description=Homelab VM backup policy and feasibility health timer - -[Timer] -OnCalendar=*-*-* 08:18:00 -RandomizedDelaySec=20m -Persistent=true - -[Install] -WantedBy=timers.target -===== SERVICE homelab-vm-backup-policy.service ===== -# /etc/systemd/system/homelab-vm-backup-policy.service -[Unit] -Description=Homelab VM backup policy and feasibility health -After=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-vm-backup-policy - -# /etc/systemd/system/homelab-vm-backup-policy.service.d/90-backup-matrix-refresh.conf -[Service] -ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-vm-backup-policy ; argv[]=/usr/local/sbin/homelab-vm-backup-policy ; ignore_errors=no ; start_time=[Tue 2026-07-21 08:30:07 MSK] ; stop_time=[Tue 2026-07-21 08:30:13 MSK] ; pid=3198902 ; code=exited ; status=0 } -ActiveState=inactive -===== TIMER homelab-vm-full-backup.timer ===== -# /etc/systemd/system/homelab-vm-full-backup.timer -[Unit] -Description=Daily full backup timer for VM170 and VM171 - -[Timer] -OnCalendar=*-*-* 02:15:00 -Persistent=true -AccuracySec=1min -Unit=homelab-vm-full-backup.service - -[Install] -WantedBy=timers.target -===== SERVICE homelab-vm-full-backup.service ===== -# /etc/systemd/system/homelab-vm-full-backup.service -[Unit] -Description=Full vzdump backup of VM170 and VM171 to offhost and two clouds -After=network-online.target pve-cluster.service -Wants=network-online.target -ConditionPathIsMountPoint=/mnt/staging - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-vm-full-backup -TimeoutStartSec=infinity -Nice=10 -IOSchedulingClass=idle -IOSchedulingPriority=7 -UMask=0077 - -# /etc/systemd/system/homelab-vm-full-backup.service.d/90-backup-matrix-refresh.conf -[Service] -ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-vm-full-backup ; argv[]=/usr/local/sbin/homelab-vm-full-backup ; ignore_errors=no ; start_time=[Tue 2026-07-21 02:15:05 MSK] ; stop_time=[Tue 2026-07-21 03:09:43 MSK] ; pid=1933641 ; code=exited ; status=0 } -ActiveState=inactive -===== TIMER homelab-vm-local-dumps-2cloud.timer ===== -# /etc/systemd/system/homelab-vm-local-dumps-2cloud.timer -[Unit] -Description=Homelab VM/CT existing local dumps two-cloud verifier timer - -[Timer] -OnCalendar=Sun *-*-* 10:00:00 -RandomizedDelaySec=30m -Persistent=true - -[Install] -WantedBy=timers.target -===== SERVICE homelab-vm-local-dumps-2cloud.service ===== -# /etc/systemd/system/homelab-vm-local-dumps-2cloud.service -[Unit] -Description=Homelab VM/CT existing local dumps two-cloud verifier -After=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-vm-local-dumps-2cloud - -# /etc/systemd/system/homelab-vm-local-dumps-2cloud.service.d/90-backup-matrix-refresh.conf -[Service] -ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; argv[]=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } -ActiveState=inactive -===== TIMER homelab-vm160-remote-backup.timer ===== -# /etc/systemd/system/homelab-vm160-remote-backup.timer -[Unit] -Description=Daily VM160 remote full-image backup - -[Timer] -OnCalendar=*-*-* 04:10:00 -RandomizedDelaySec=10m -Unit=homelab-vm160-remote-backup.service - -[Install] -WantedBy=timers.target -===== SERVICE homelab-vm160-remote-backup.service ===== -# /etc/systemd/system/homelab-vm160-remote-backup.service -[Unit] -Description=VM160 full-image backup streamed from pve02 to pve03 -After=network-online.target pve-cluster.service -Wants=network-online.target - -[Service] -Type=oneshot -TimeoutStartSec=6h -ExecStart=/usr/local/sbin/homelab-vm160-remote-backup -Result=exit-code -ExecMainStatus=1 -ExecStart={ path=/usr/local/sbin/homelab-vm160-remote-backup ; argv[]=/usr/local/sbin/homelab-vm160-remote-backup ; ignore_errors=no ; start_time=[Tue 2026-07-21 02:39:12 MSK] ; stop_time=[Tue 2026-07-21 02:39:14 MSK] ; pid=2028253 ; code=exited ; status=1 } -ActiveState=failed -===== TIMER immich-media-backup.timer ===== -# /etc/systemd/system/immich-media-backup.timer -[Unit] -Description=Daily Immich media backup timer - -[Timer] -OnCalendar=*-*-* 08:15:00 -Persistent=true -RandomizedDelaySec=10m -Unit=immich-media-backup.service - -[Install] -WantedBy=timers.target -===== SERVICE immich-media-backup.service ===== -# /etc/systemd/system/immich-media-backup.service -[Unit] -Description=Immich media backup to staging -Wants=network-online.target -After=network-online.target - -[Service] -Type=oneshot -ExecStart=/root/immich-media-backup.sh -Nice=10 -IOSchedulingClass=best-effort -IOSchedulingPriority=7 - -# /etc/systemd/system/immich-media-backup.service.d/10-superseded-noop.conf -[Service] -ExecStart= -ExecStart=/usr/local/sbin/homelab-superseded-unit-ok immich-media-backup.service superseded_by_current_appbackup_or_trust_proof - -# /etc/systemd/system/immich-media-backup.service.d/90-backup-matrix-refresh.conf -[Service] -ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok immich-media-backup.service superseded_by_current_appbackup_or_trust_proof ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } -ActiveState=inactive -===== TIMER memos-backup.timer ===== -# /etc/systemd/system/memos-backup.timer -[Unit] -Description=Run Memos data backup daily - -[Timer] -OnCalendar=*-*-* 08:35:00 -RandomizedDelaySec=20m -Persistent=true - -[Install] -WantedBy=timers.target -===== SERVICE memos-backup.service ===== -# /etc/systemd/system/memos-backup.service -[Unit] -Description=Memos data backup to pve01 staging - -[Service] -Type=oneshot -ExecStart=/root/memos-backup.sh - -# /etc/systemd/system/memos-backup.service.d/10-superseded-noop.conf -[Service] -ExecStart= -ExecStart=/usr/local/sbin/homelab-superseded-unit-ok memos-backup.service superseded_by_current_appbackup_or_trust_proof - -# /etc/systemd/system/memos-backup.service.d/90-backup-matrix-refresh.conf -[Service] -ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok memos-backup.service superseded_by_current_appbackup_or_trust_proof ; ignore_errors=no ; start_time=[Tue 2026-07-21 08:41:14 MSK] ; stop_time=[Tue 2026-07-21 08:41:14 MSK] ; pid=3239605 ; code=exited ; status=0 } -ActiveState=inactive -===== TIMER netbird-vps-backup.timer ===== -# /etc/systemd/system/netbird-vps-backup.timer -[Unit] -Description=Run NetBird VPS backup daily - -[Timer] -OnCalendar=*-*-* 04:25:00 -Persistent=true -RandomizedDelaySec=15m - -[Install] -WantedBy=timers.target -===== SERVICE netbird-vps-backup.service ===== -# /etc/systemd/system/netbird-vps-backup.service -[Unit] -Description=NetBird VPS backup to pve01 staging - -[Service] -Type=oneshot -ExecStart=/root/netbird-vps-backup.sh - -# /etc/systemd/system/netbird-vps-backup.service.d/10-superseded-noop.conf -[Service] -ExecStart= -ExecStart=/usr/local/sbin/homelab-superseded-unit-ok netbird-vps-backup.service superseded_by_current_appbackup_or_trust_proof - -# /etc/systemd/system/netbird-vps-backup.service.d/90-backup-matrix-refresh.conf -[Service] -ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok netbird-vps-backup.service superseded_by_current_appbackup_or_trust_proof ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } -ActiveState=inactive -===== TIMER paperless-backup.timer ===== -# /etc/systemd/system/paperless-backup.timer -[Unit] -Description=Run Paperless backup daily - -[Timer] -OnCalendar=*-*-* 08:50:00 -RandomizedDelaySec=20m -Persistent=true - -[Install] -WantedBy=timers.target -===== SERVICE paperless-backup.service ===== -# /etc/systemd/system/paperless-backup.service -[Unit] -Description=Paperless backup to pve01 staging - -[Service] -Type=oneshot -ExecStart=/root/paperless-backup.sh - -# /etc/systemd/system/paperless-backup.service.d/10-superseded-noop.conf -[Service] -ExecStart= -ExecStart=/usr/local/sbin/homelab-superseded-unit-ok paperless-backup.service superseded_by_current_appbackup_or_trust_proof - -# /etc/systemd/system/paperless-backup.service.d/90-backup-matrix-refresh.conf -[Service] -ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok paperless-backup.service superseded_by_current_appbackup_or_trust_proof ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } -ActiveState=inactive -===== TIMER skladchik-reports-monitor-backup-restore-check.timer ===== -# /etc/systemd/system/skladchik-reports-monitor-backup-restore-check.timer -[Unit] -Description=Weekly restore-check of Skladchik reports monitor cloud backups - -[Timer] -OnCalendar=Sun *-*-* 04:55:00 -RandomizedDelaySec=30min -Persistent=true -Unit=skladchik-reports-monitor-backup-restore-check.service - -[Install] -WantedBy=timers.target -===== SERVICE skladchik-reports-monitor-backup-restore-check.service ===== -# /etc/systemd/system/skladchik-reports-monitor-backup-restore-check.service -[Unit] -Description=Restore-check Skladchik reports monitor cloud backups -Wants=network-online.target -After=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check -Nice=10 -IOSchedulingClass=best-effort -IOSchedulingPriority=7 - -# /etc/systemd/system/skladchik-reports-monitor-backup-restore-check.service.d/90-backup-matrix-refresh.conf -[Service] -ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check ; argv[]=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } -ActiveState=inactive -===== TIMER skladchik-reports-monitor-backup.timer ===== -# /etc/systemd/system/skladchik-reports-monitor-backup.timer -[Unit] -Description=Daily Skladchik reports monitor backup to encrypted clouds - -[Timer] -OnCalendar=*-*-* 04:25:00 -RandomizedDelaySec=15min -Persistent=true -Unit=skladchik-reports-monitor-backup.service - -[Install] -WantedBy=timers.target -===== SERVICE skladchik-reports-monitor-backup.service ===== -# /etc/systemd/system/skladchik-reports-monitor-backup.service -[Unit] -Description=Backup Skladchik reports monitor from edge-vm to encrypted clouds -Wants=network-online.target -After=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/skladchik-reports-monitor-backup -Nice=10 -IOSchedulingClass=best-effort -IOSchedulingPriority=7 - -# /etc/systemd/system/skladchik-reports-monitor-backup.service.d/90-backup-matrix-refresh.conf -[Service] -ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/skladchik-reports-monitor-backup ; argv[]=/usr/local/sbin/skladchik-reports-monitor-backup ; ignore_errors=no ; start_time=[Tue 2026-07-21 04:34:48 MSK] ; stop_time=[Tue 2026-07-21 04:35:22 MSK] ; pid=2399885 ; code=exited ; status=0 } -ActiveState=inactive -===== REFERENCED_SCRIPTS ===== ------ /opt/homelab-backup/backup-emergency-pack-v0.sh ----- -41:rclone listremotes > "$PAYLOAD/inventory/rclone_remotes.txt" 2>&1 || true -59:It is deleted only after both encrypted cloud copies are uploaded and fully verified. -60:Recovery requires offline/break-glass access to rclone crypt credentials. -65:rm -rf "$PAYLOAD" -79: "archive_size_bytes": $ARCHIVE_SIZE_BYTES, -83: "local_staging_policy": "delete_after_both_clouds_verified", -90:echo "[STEP] local checksum" -91:(cd "$STAGE" && sha256sum "$ARCHIVE" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null) -94:rclone copy "$STAGE" "$REMOTE_A_JOB" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s -95:homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$REMOTE_A_JOB/$ARCHIVE" -98:rclone copy "$STAGE" "$REMOTE_B_JOB" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s -99:homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$REMOTE_B_JOB/$ARCHIVE" -102:rclone copy "$REMOTE_A_JOB" "$VERIFY/cloud_a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10; homelab-rehydrate-chunked-dir "$VERIFY/cloud_a" -103:(cd "$VERIFY/cloud_a" && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB_ID" '.job_id == $job' manifest.json >/dev/null) -106:rclone copy "$REMOTE_B_JOB" "$VERIFY/cloud_b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10; homelab-rehydrate-chunked-dir "$VERIFY/cloud_b" -107:(cd "$VERIFY/cloud_b" && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB_ID" '.job_id == $job' manifest.json >/dev/null) -110:(cd "$STAGE" && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null) -113:rclone copyto "$STAGE/latest-backup-index.json" "$REMOTE_A_LATEST" -114:rclone copyto "$STAGE/latest-backup-index.json" "$REMOTE_B_LATEST" -115:rclone cat "$REMOTE_A_LATEST" | jq -e --arg job "$JOB_ID" '.job_id == $job' >/dev/null -116:rclone cat "$REMOTE_B_LATEST" | jq -e --arg job "$JOB_ID" '.job_id == $job' >/dev/null -124: "archive_size_bytes": $ARCHIVE_SIZE_BYTES, -128: "local_backup_deleted": true -132:echo "[STEP] delete local staging and verification copies" -133:rm -rf "$STAGE" "$VERIFY" -138:echo "[SUCCESS] uploaded to 2 clouds, verified from both clouds, local staging deleted" ------ /opt/homelab-backup/backup-emergency-pack-v0.sh.bak-chunkall-20260710T011410Z ----- -41:rclone listremotes > "$PAYLOAD/inventory/rclone_remotes.txt" 2>&1 || true -59:It is deleted only after both encrypted cloud copies are uploaded and fully verified. -60:Recovery requires offline/break-glass access to rclone crypt credentials. -65:rm -rf "$PAYLOAD" -79: "archive_size_bytes": $ARCHIVE_SIZE_BYTES, -83: "local_staging_policy": "delete_after_both_clouds_verified", -90:echo "[STEP] local checksum" -91:(cd "$STAGE" && sha256sum "$ARCHIVE" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null) -94:rclone copy "$STAGE" "$REMOTE_A_JOB" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 --stats=30s -97:rclone copy "$STAGE" "$REMOTE_B_JOB" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 --stats=30s -100:rclone copy "$REMOTE_A_JOB" "$VERIFY/cloud_a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 -101:(cd "$VERIFY/cloud_a" && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB_ID" '.job_id == $job' manifest.json >/dev/null) -104:rclone copy "$REMOTE_B_JOB" "$VERIFY/cloud_b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 -105:(cd "$VERIFY/cloud_b" && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB_ID" '.job_id == $job' manifest.json >/dev/null) -108:(cd "$STAGE" && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null) -111:rclone copyto "$STAGE/latest-backup-index.json" "$REMOTE_A_LATEST" -112:rclone copyto "$STAGE/latest-backup-index.json" "$REMOTE_B_LATEST" -113:rclone cat "$REMOTE_A_LATEST" | jq -e --arg job "$JOB_ID" '.job_id == $job' >/dev/null -114:rclone cat "$REMOTE_B_LATEST" | jq -e --arg job "$JOB_ID" '.job_id == $job' >/dev/null -122: "archive_size_bytes": $ARCHIVE_SIZE_BYTES, -126: "local_backup_deleted": true -130:echo "[STEP] delete local staging and verification copies" -131:rm -rf "$STAGE" "$VERIFY" -136:echo "[SUCCESS] uploaded to 2 clouds, verified from both clouds, local staging deleted" ------ /usr/local/sbin/appbackup-dockge-stacks ----- -25:echo "[STEP] copy app data" -30:rm -rf "$PAYLOAD" -42: "archive_size_bytes": $SIZE, -46: "local_staging_policy": "delete_after_both_clouds_verified", -51:echo "[STEP] local checksum" -52:(cd "$STAGE" && sha256sum "$ARCHIVE" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e . manifest.json >/dev/null) -54:echo "[STEP] upload cloud A metadata"; rclone copy "$STAGE" "$RA" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s -55:echo "[STEP] upload cloud A archive chunks"; rclone deletefile "$RA/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RA/$ARCHIVE" -56:echo "[STEP] upload cloud B metadata"; rclone copy "$STAGE" "$RB" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s -57:echo "[STEP] upload cloud B archive chunks"; rclone deletefile "$RB/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RB/$ARCHIVE" -58:echo "[STEP] verify cloud A chunked"; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-stage-verify-chunked "$RA" "$ARCHIVE" "$VERIFY/a" "$JOB" -59:echo "[STEP] verify cloud B chunked"; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-stage-verify-chunked "$RB" "$ARCHIVE" "$VERIFY/b" "$JOB" -60:echo "[STEP] upload latest index"; rclone copyto "$STAGE/latest-backup-index.json" "$RAL"; rclone copyto "$STAGE/latest-backup-index.json" "$RBL"; rclone cat "$RAL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null; rclone cat "$RBL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null -68: "archive_size_bytes": $SIZE, -72: "local_backup_deleted": true -75:echo "[STEP] delete local staging and verification" -76:rm -rf "$STAGE" "$VERIFY" -79:echo "[SUCCESS] app backup uploaded to 2 clouds, verified from both clouds, local staging deleted" ------ /usr/local/sbin/appbackup-dockge-stacks.bak-chunkall-20260710T011410Z ----- -25:echo "[STEP] copy app data" -30:rm -rf "$PAYLOAD" -42: "archive_size_bytes": $SIZE, -46: "local_staging_policy": "delete_after_both_clouds_verified", -51:echo "[STEP] local checksum" -52:(cd "$STAGE" && sha256sum "$ARCHIVE" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e . manifest.json >/dev/null) -54:rclone copy "$STAGE" "$RA" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 --stats=30s -56:rclone copy "$STAGE" "$RB" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 --stats=30s -58:rclone copy "$RA" "$VERIFY/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 -59:(cd "$VERIFY/a" && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null) -61:rclone copy "$RB" "$VERIFY/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 -62:(cd "$VERIFY/b" && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null) -64:(cd "$STAGE" && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e . manifest.json >/dev/null) -66:rclone copyto "$STAGE/latest-backup-index.json" "$RAL" -67:rclone copyto "$STAGE/latest-backup-index.json" "$RBL" -68:rclone cat "$RAL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null -69:rclone cat "$RBL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null -77: "archive_size_bytes": $SIZE, -81: "local_backup_deleted": true -84:echo "[STEP] delete local staging and verification" -85:rm -rf "$STAGE" "$VERIFY" -88:echo "[SUCCESS] app backup uploaded to 2 clouds, verified from both clouds, local staging deleted" ------ /usr/local/sbin/appbackup-restore-check ----- -22:ROOT="/var/tmp/homelab-app-restore-check/${JOB}" -24:rm -rf "$ROOT" && mkdir -p "$ROOT/a" "$ROOT/b" -25:trap "echo \"ERROR app restore-check failed; kept: $ROOT\"; exit 1" ERR -26:echo "restore-check app=$APP job=$JOB schema=$SCHEMA" -27:rclone copy "$RA" "$ROOT/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >"${LOGBASE}-a.log" 2>&1 -30:(cd "$ROOT/a" && sha256sum -c checksums.sha256 >"${LOGBASE}-a-sha.log" && ARCHIVE="$(jq -r ".archive_name" manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null) -31:echo "OK cloud A app restore-check" -32:rclone copy "$RB" "$ROOT/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >"${LOGBASE}-b.log" 2>&1 -35:(cd "$ROOT/b" && sha256sum -c checksums.sha256 >"${LOGBASE}-b-sha.log" && ARCHIVE="$(jq -r ".archive_name" manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null) -36:echo "OK cloud B app restore-check" -37:rm -rf "$ROOT" -38:rmdir /var/tmp/homelab-app-restore-check 2>/dev/null || true -39:rm -f "${LOGBASE}-a.log" "${LOGBASE}-a-sha.log" "${LOGBASE}-b.log" "${LOGBASE}-b-sha.log" -41:echo "OK app restore-check both clouds" ------ /usr/local/sbin/appbackup-restore-check.bak-chunkall-20260710T011410Z ----- -9:ROOT="/var/tmp/homelab-app-restore-check/${JOB}" -10:rm -rf "$ROOT" && mkdir -p "$ROOT/a" "$ROOT/b" -11:trap "echo \"ERROR app restore-check failed; kept: $ROOT\"; exit 1" ERR -12:echo "restore-check app=$APP job=$JOB" -13:rclone copy "$RA" "$ROOT/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/app-restore-a.log 2>&1 -14:(cd "$ROOT/a" && sha256sum -c checksums.sha256 >/tmp/app-restore-a-sha.log && ARCHIVE="$(jq -r .archive_name manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null) -15:echo "OK cloud A app restore-check" -16:rclone copy "$RB" "$ROOT/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/app-restore-b.log 2>&1 -17:(cd "$ROOT/b" && sha256sum -c checksums.sha256 >/tmp/app-restore-b-sha.log && ARCHIVE="$(jq -r .archive_name manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null) -18:echo "OK cloud B app restore-check" -19:rm -rf "$ROOT"; rmdir /var/tmp/homelab-app-restore-check 2>/dev/null || true -21:echo "OK app restore-check both clouds" ------ /usr/local/sbin/appbackup-restore-check.bak-dual-read-20260711T051924Z ----- -9:ROOT="/var/tmp/homelab-app-restore-check/${JOB}" -10:rm -rf "$ROOT" && mkdir -p "$ROOT/a" "$ROOT/b" -11:trap "echo \"ERROR app restore-check failed; kept: $ROOT\"; exit 1" ERR -12:echo "restore-check app=$APP job=$JOB" -13:rclone copy "$RA" "$ROOT/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/app-restore-a.log 2>&1; homelab-rehydrate-chunked-dir "$ROOT/a" >>/tmp/app-restore-a.log 2>&1 -14:(cd "$ROOT/a" && sha256sum -c checksums.sha256 >/tmp/app-restore-a-sha.log && ARCHIVE="$(jq -r .archive_name manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null) -15:echo "OK cloud A app restore-check" -16:rclone copy "$RB" "$ROOT/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/app-restore-b.log 2>&1; homelab-rehydrate-chunked-dir "$ROOT/b" >>/tmp/app-restore-b.log 2>&1 -17:(cd "$ROOT/b" && sha256sum -c checksums.sha256 >/tmp/app-restore-b-sha.log && ARCHIVE="$(jq -r .archive_name manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null) -18:echo "OK cloud B app restore-check" -19:rm -rf "$ROOT"; rmdir /var/tmp/homelab-app-restore-check 2>/dev/null || true -21:echo "OK app restore-check both clouds" ------ /usr/local/sbin/appbackup-restore-check.bak-flat-read-20260711T052854Z ----- -22:ROOT="/var/tmp/homelab-app-restore-check/${JOB}" -24:rm -rf "$ROOT" && mkdir -p "$ROOT/a" "$ROOT/b" -25:trap "echo \"ERROR app restore-check failed; kept: $ROOT\"; exit 1" ERR -26:echo "restore-check app=$APP job=$JOB schema=$SCHEMA" -27:rclone copy "$RA" "$ROOT/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >"${LOGBASE}-a.log" 2>&1 -29:(cd "$ROOT/a" && sha256sum -c checksums.sha256 >"${LOGBASE}-a-sha.log" && ARCHIVE="$(jq -r ".archive_name" manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null) -30:echo "OK cloud A app restore-check" -31:rclone copy "$RB" "$ROOT/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >"${LOGBASE}-b.log" 2>&1 -33:(cd "$ROOT/b" && sha256sum -c checksums.sha256 >"${LOGBASE}-b-sha.log" && ARCHIVE="$(jq -r ".archive_name" manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null) -34:echo "OK cloud B app restore-check" -35:rm -rf "$ROOT" -36:rmdir /var/tmp/homelab-app-restore-check 2>/dev/null || true -37:rm -f "${LOGBASE}-a.log" "${LOGBASE}-a-sha.log" "${LOGBASE}-b.log" "${LOGBASE}-b-sha.log" -39:echo "OK app restore-check both clouds" ------ /usr/local/sbin/appbackup-restore-check.bak-flat-read-v2-20260711T053129Z ----- -22:ROOT="/var/tmp/homelab-app-restore-check/${JOB}" -24:rm -rf "$ROOT" && mkdir -p "$ROOT/a" "$ROOT/b" -25:trap "echo \"ERROR app restore-check failed; kept: $ROOT\"; exit 1" ERR -26:echo "restore-check app=$APP job=$JOB schema=$SCHEMA" -27:rclone copy "$RA" "$ROOT/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >"${LOGBASE}-a.log" 2>&1 -29:(cd "$ROOT/a" && sha256sum -c checksums.sha256 >"${LOGBASE}-a-sha.log" && ARCHIVE="$(jq -r ".archive_name" manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null) -30:echo "OK cloud A app restore-check" -31:rclone copy "$RB" "$ROOT/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >"${LOGBASE}-b.log" 2>&1 -33:(cd "$ROOT/b" && sha256sum -c checksums.sha256 >"${LOGBASE}-b-sha.log" && ARCHIVE="$(jq -r ".archive_name" manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null) -34:echo "OK cloud B app restore-check" -35:rm -rf "$ROOT" -36:rmdir /var/tmp/homelab-app-restore-check 2>/dev/null || true -37:rm -f "${LOGBASE}-a.log" "${LOGBASE}-a-sha.log" "${LOGBASE}-b.log" "${LOGBASE}-b-sha.log" -39:echo "OK app restore-check both clouds" ------ /usr/local/sbin/appbackup-retention ----- -32:trap "rm -rf \"$WORK\"" EXIT -35: rclone lsf "${REMOTE%/}" --dirs-only --max-depth 1 > "$ROOT_LIST" -36: rclone lsf "${REMOTE%/}/${NODE}/apps/${CANONICAL}" --dirs-only --max-depth 1 > "$LEGACY_LIST" -39: rclone lsf "${REMOTE%/}/hb" --dirs-only --max-depth 1 > "$HB_LIST" -40: if grep -Fxq "a/" "$HB_LIST"; then rclone lsf "${REMOTE%/}/hb/a" --dirs-only --max-depth 1 > "$NEW_LIST"; fi -73: rclone cat "${REMOTE%/}/${LATEST_REL}/manifest.json" | jq -e --arg job "$LATEST" ".job_id == \$job" >/dev/null || { echo "REFUSE latest missing/bad remote_index=$INDEX"; exit 5; } -78: if (( TOTAL <= KEEP )); then echo "nothing to delete"; continue; fi -83: [[ -n "$LATEST_REL" && "$REL" == "$LATEST_REL" ]] && { echo "REFUSE delete latest ${REL}"; exit 7; } -85: if [[ "$MODE" == apply ]]; then echo "DELETE schema=dual path=$REL"; rclone purge "$TARGET"; else echo "WOULD_DELETE schema=dual path=$REL"; fi ------ /usr/local/sbin/appbackup-retention.bak-dual-read-20260711T051924Z ----- -15:for REMOTE in "$REMOTE_A" "$REMOTE_B"; do BASE="${REMOTE%/}/${NODE}/apps/${APP}"; echo "--- $BASE ---"; if [[ "$MODE" == "apply" ]]; then rclone cat "$BASE/$LATEST/manifest.json" | jq -e --arg job "$LATEST" ".job_id == \$job" >/dev/null || { echo "REFUSE latest missing/bad on $BASE"; exit 4; }; fi; mapfile -t DIRS < <(rclone lsf "$BASE" --dirs-only 2>/dev/null | while read -r D; do D="${D%/}"; [[ "$D" == homelab-${NODE}-app-${APP}-* ]] && echo "$D"; done | sort); TOTAL="${#DIRS[@]}"; echo "total=$TOTAL keep=$KEEP mode=$MODE latest=${LATEST:-none}"; if (( TOTAL <= KEEP )); then echo "nothing to delete"; continue; fi; DELETE_COUNT=$((TOTAL-KEEP)); for ((i=0; i/dev/null && echo "OK backup framework" ;; -13: restore-check) backup-restore-check ;; -14: audit) backupctl check && backup-restore-check && backupctl retention 7 dry-run && backup-report >/dev/null && echo "OK audit finished" ;; -16: remotes) source /etc/homelab-backup.env && NODE="$(hostname -s)" && echo "A=$REMOTE_A" && echo "B=$REMOTE_B" && rclone lsd "${REMOTE_A%/}/${NODE}/emergency-pack" | tail -n 5 && rclone lsd "${REMOTE_B%/}/${NODE}/emergency-pack" | tail -n 5 ;; -19: *) echo "usage: backupctl run|status|lastlog|check|restore-check|audit|report|remotes|retention|timer"; exit 2 ;; ------ /usr/local/sbin/backup-restore-check ----- -8:ROOT="/var/tmp/homelab-restore-check/${JOB}" -9:rm -rf "$ROOT" && mkdir -p "$ROOT/a" "$ROOT/b" -10:trap 'echo "ERROR restore-check failed; kept: "; exit 1' ERR -11:echo "restore-check job=$JOB" -12:rclone copy "$RA" "$ROOT/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-a.log 2>&1; homelab-rehydrate-chunked-dir "$ROOT/a" >>/tmp/homelab-restore-a.log 2>&1 -13:(cd "$ROOT/a" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-a-sha.log && ARCHIVE="$(jq -r .archive_name manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null) -14:echo "OK cloud A restore-check" -15:rclone copy "$RB" "$ROOT/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-b.log 2>&1; homelab-rehydrate-chunked-dir "$ROOT/b" >>/tmp/homelab-restore-b.log 2>&1 -16:(cd "$ROOT/b" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-b-sha.log && ARCHIVE="$(jq -r .archive_name manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null) -17:echo "OK cloud B restore-check" -18:rm -rf "$ROOT" -20:echo "OK restore-check both clouds" ------ /usr/local/sbin/backup-restore-check.bak-chunkall-20260710T011410Z ----- -8:ROOT="/var/tmp/homelab-restore-check/${JOB}" -9:rm -rf "$ROOT" && mkdir -p "$ROOT/a" "$ROOT/b" -10:trap 'echo "ERROR restore-check failed; kept: "; exit 1' ERR -11:echo "restore-check job=$JOB" -12:rclone copy "$RA" "$ROOT/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-a.log 2>&1 -13:(cd "$ROOT/a" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-a-sha.log && ARCHIVE="$(jq -r .archive_name manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null) -14:echo "OK cloud A restore-check" -15:rclone copy "$RB" "$ROOT/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-b.log 2>&1 -16:(cd "$ROOT/b" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-b-sha.log && ARCHIVE="$(jq -r .archive_name manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null) -17:echo "OK cloud B restore-check" -18:rm -rf "$ROOT" -20:echo "OK restore-check both clouds" ------ /usr/local/sbin/backup-retention ----- -18: if [[ "$MODE" == "apply" ]]; then rclone cat "$BASE/$LATEST/manifest.json" | jq -e --arg job "$LATEST" ".job_id == \$job" >/dev/null || { echo "REFUSE latest missing/bad on $BASE"; exit 4; }; fi -19: mapfile -t DIRS < <(rclone lsf "$BASE" --dirs-only 2>/dev/null | while read -r D; do D="${D%/}"; [[ "$D" == homelab-${NODE}-${JOB}-* ]] && echo "$D"; done | sort) -22: if (( TOTAL <= KEEP )); then echo "nothing to delete"; continue; fi -24: for ((i=0; i \$OUT; : > \$ERR; vzdump '$VMID' --mode snapshot --compress zstd --dumpdir '$RUN_DIR' --remove 0 >\$OUT 2>\$ERR; RC=\$?; ARCHIVE=\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\n' | sort -n | tail -n1 | cut -d' ' -f2-); LOGFILE=\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.log' -printf '%T@ %p\n' | sort -n | tail -n1 | cut -d' ' -f2-); SIZE=0; SHA=none; if [ -n \"\$ARCHIVE\" ] && [ -f \"\$ARCHIVE\" ]; then SIZE=\$(stat -c %s \"\$ARCHIVE\"); SHA=\$(sha256sum \"\$ARCHIVE\" | awk '{print \$1}'); fi; echo VZDUMP_RC=\$RC; echo ARCHIVE=\$ARCHIVE; echo LOGFILE=\$LOGFILE; echo SIZE=\$SIZE; echo SHA=\$SHA; echo OUT_TAIL=\$(tail -n 6 \$OUT | tr '\n' '|'); echo ERR_TAIL=\$(tail -n 6 \$ERR | tr '\n' '|')")" -62:dest "mkdir -p '$OFFHOST_DIR' && rm -f '$DEST_PATH.tmp' '$DEST_PATH.sha256'" -74:echo "STATUS=OK TS=$TS TYPE=edge-vm-vzdump-backup NODE=$NODE VMID=$VMID ARCHIVE=$ARCHIVE SIZE=$SIZE SHA256=$SHA SCSI1_BACKUP0=yes NOTES=timer_run" > "$HEALTH_DIR/edge-vm-vzdump-backup.txt" -75:echo "STATUS=OK TS=$TS TYPE=edge-vm-vzdump-offhost NODE=$NODE VMID=$VMID HOST=pve02 DEST=$DEST_PATH SIZE=$SIZE SHA256=$SHA NOTES=timer_run" > "$HEALTH_DIR/edge-vm-vzdump-offhost.txt" -76:echo "STATUS=OK TS=$TS TYPE=edge-vm-vzdump-restore-verify HOST=pve02 VMID=$VMID ARCHIVE=$DEST_PATH SIZE=$SIZE SHA256=$SHA ZSTD_RC=0 VMA_RC=0 NOTES=timer_run" > "$HEALTH_DIR/edge-vm-vzdump-restore.txt" -77:chmod 0644 "$HEALTH_DIR/edge-vm-vzdump-backup.txt" "$HEALTH_DIR/edge-vm-vzdump-offhost.txt" "$HEALTH_DIR/edge-vm-vzdump-restore.txt" -79:remote "find '$BASE_DIR' -mindepth 1 -maxdepth 1 -type d -name 'edge-vm-*' -printf '%T@ %p\n' | sort -nr | awk 'NR>${KEEP_LOCAL}{print \$2}' | xargs -r rm -rf" -80:dest "find '$OFFHOST_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\n' | sort -nr | awk 'NR>${KEEP_OFFHOST}{print \$2}' | while read f; do rm -f \"\$f\" \"\$f.sha256\"; done" ------ /usr/local/sbin/homelab-capacity-risk ----- -32:LARGEST_BYTES="$(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-*.vma.zst" -o -name "vzdump-lxc-*.tar.zst" \) -printf "%s\n" 2>/dev/null | sort -n | tail -n1)" ------ /usr/local/sbin/homelab-chat-run ----- -272: re.compile(r"(?i)\brclone\.conf\b"), -388: "reference_register_checked": True, -390: "error_register_checked": True, -489: rm -f \ ------ /usr/local/sbin/homelab-edge-vm-offhost-freshness ----- -9:OFFHOST_DIR=/mnt/staging/offhost/edge-vm-vzdump-from-pve03 -85: /var/lib/vz/dump/vzdump-qemu-130-*.vma.zst) ;; -96:dest_size="$(field DEST_SIZE <<<"$inventory_out")" -100:[[ "$dest_size" =~ ^[0-9]+$ && "$count" =~ ^[0-9]+$ && "$available" =~ ^[0-9]+$ ]] -103:last_oldest_size=0 -107: [[ "$dest_size" == "$SOURCE_SIZE" && "$dest_sha" == "$SOURCE_SHA" ]] -110: "$dest_size" "$last_oldest" "$last_oldest_size" "$last_oldest_sha" "$count" "$RESERVE")" -115: [[ "$dest_size" -ge 0 && "$dest_size" -lt "$SOURCE_SIZE" ]] -119: last_oldest_size="$(field OLDEST_SIZE <<<"$inventory_out")" -122: [[ "$last_oldest_size" =~ ^[0-9]+$ && "$last_oldest_size" -gt 1000000000 ]] -126: "$dest_size" "$last_oldest" "$last_oldest_size" "$last_oldest_sha" "$before_count" "$RESERVE")" -130: [[ "$(field DEST_SIZE <<<"$inventory_out")" == "$dest_size" ]] -136: remaining=$((SOURCE_SIZE-dest_size)) -140: copy_out="$(run_pve03 copy "$SOURCE_PATH" "$SOURCE_SIZE" "$SOURCE_SHA" "$DST_IP" "$OFFHOST_DIR")" -141: grep -Fq 'SOURCE_COPY_STATUS=OK' <<<"$copy_out" -146: "$dest_size" "$last_oldest" "$last_oldest_size" "$last_oldest_sha" "$expected_final_count" "$RESERVE")" -156:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-backup NODE=%s VMID=%s ARCHIVE=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\n' \ -158: >"$HEALTH_DIR/edge-vm-vzdump-backup.txt" -159:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-offhost NODE=%s VMID=%s HOST=%s DEST=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\n' \ -161: >"$HEALTH_DIR/edge-vm-vzdump-offhost.txt" -162:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-restore-verify HOST=%s VMID=%s ARCHIVE=%s SIZE=%s SHA256=%s ZSTD_RC=0 VMA_RC=0 NOTES=nightly_existing_dump_freshness\n' \ -164: >"$HEALTH_DIR/edge-vm-vzdump-restore.txt" -167: "$HEALTH_DIR/edge-vm-vzdump-backup.txt" \ -168: "$HEALTH_DIR/edge-vm-vzdump-offhost.txt" \ -169: "$HEALTH_DIR/edge-vm-vzdump-restore.txt" ------ /usr/local/sbin/homelab-extended-appbackup-run ----- -12:run_one(){ APP="$1"; HOST="$2"; PATHS="$3"; NOTE="$4"; CONS="$5"; JOB_ID="homelab-${NODE}-app-${APP}-${RUN_TS}"; export JOB_ID; SHORT_TS="${RUN_TS:2:6}-${RUN_TS:9:6}"; export BACKUP_PATH_REL="$(homelab-backup-path job a "$APP" "$SHORT_TS")"; STAGE="${STAGING_ROOT}/${JOB_ID}"; PAYLOAD="$STAGE/payload"; ARCHIVE="${JOB_ID}.tar.gz"; LOG="/tmp/extended-appbackup-${APP}-${RUN_TS}.log"; rm -rf "$STAGE"; mkdir -p "$PAYLOAD/local"; echo "### app=$APP" >> "$REPORT"; echo "host=$HOST" >> "$REPORT"; echo "source_note=$NOTE" >> "$REPORT"; ssh -n -o BatchMode=yes -o ConnectTimeout=30 "$HOST" "sudo tar -C / --ignore-failed-read --exclude=.env --exclude=*.env --exclude=.env.* --exclude=*.bak* -czf - $PATHS" > "$PAYLOAD/local/${APP}-data.tar.gz" 2>"$LOG"; TAR_RC=$?; if [ "$TAR_RC" -eq 0 ] && tar -tzf "$PAYLOAD/local/${APP}-data.tar.gz" >/dev/null 2>&1; then SIZE="$(stat -c%s "$PAYLOAD/local/${APP}-data.tar.gz" 2>/dev/null || echo 0)"; echo "archive_payload_size_bytes=$SIZE" >> "$REPORT"; if [ "$SIZE" -le 128 ]; then echo "backup=BAD reason=archive_too_small" >> "$REPORT"; BAD=$((BAD+1)); rm -rf "$STAGE"; else sha256sum "$PAYLOAD/local/${APP}-data.tar.gz" > "$PAYLOAD/local/${APP}-data.tar.gz.sha256"; jq -n --arg type "extended-appbackup" --arg app "$APP" --arg host "$HOST" --arg source_note "$NOTE" --arg archive_payload_size_bytes "$SIZE" --arg env_files_read "no" --arg secret_values_printed "no" --arg consistency "$CONS" "{type:\$type,app_name:\$app,host:\$host,source_note:\$source_note,archive_payload_size_bytes:(\$archive_payload_size_bytes|tonumber),env_files_read:\$env_files_read,secret_values_printed:\$secret_values_printed,consistency:\$consistency}" > "$STAGE/metadata.json"; tar -C "$PAYLOAD" -czf "$STAGE/$ARCHIVE" .; rm -rf "$PAYLOAD"; JOB_ID="$JOB_ID" homelab-stage-2cloud-flat "$APP" "$STAGE" "$ARCHIVE" > "$LOG" 2>&1; RC=$?; if [ "$RC" -eq 0 ]; then echo "backup=OK" >> "$REPORT"; grep -E "^\[STEP\]|^\[SUCCESS\]" "$LOG" >> "$REPORT"; appbackupctl restore-check "$APP" >> "$REPORT" 2>&1 && echo "restore_check=OK" >> "$REPORT" || echo "restore_check=WARN_non_registry" >> "$REPORT"; OK=$((OK+1)); else echo "backup=BAD rc=$RC" >> "$REPORT"; grep -E "^\[ERROR\]|\[STEP\]|ERROR|error|failed|Failed|required|missing|denied|forbidden|checksum|rclone|copy|mkdir|jq|tar" "$LOG" | tail -n 60 >> "$REPORT"; BAD=$((BAD+1)); fi; fi; else echo "tar=BAD rc=$TAR_RC" >> "$REPORT"; grep -E "ERROR|error|failed|Failed|denied|No such|Cannot|tar:" "$LOG" | tail -n 40 >> "$REPORT"; BAD=$((BAD+1)); rm -rf "$STAGE"; fi; echo >> "$REPORT"; } -21:run_one healthchecks "$MON" "var/lib/healthchecks opt/stacks/healthchecks/local_settings.py" "healthchecks data plus local settings" "emergency_data_config_archive" ------ /usr/local/sbin/homelab-golden-state-extended-overlay ----- -26:rclone copyto "$JSON" "$RA" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1)) -27:rclone copyto "$JSON" "$RB" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1)) -28:rclone cat "$RA" 2>/dev/null | jq -e --arg cp "$CP" ".control_plane_job == \$cp" >/dev/null || BAD=$((BAD+1)) -29:rclone cat "$RB" 2>/dev/null | jq -e --arg cp "$CP" ".control_plane_job == \$cp" >/dev/null || BAD=$((BAD+1)) ------ /usr/local/sbin/homelab-golden-state-index ----- -28:rclone copyto "$JSON" "$RA" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1)) -29:rclone copyto "$JSON" "$RB" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1)) -30:rclone cat "$RA" 2>/dev/null | jq -e --arg cp "$CP" --arg ext "$EXT" ".control_plane_job == \$cp and .health.extended_appbackup == \$ext and .extended_appbackup_health == \$ext" >/dev/null || BAD=$((BAD+1)) -31:rclone cat "$RB" 2>/dev/null | jq -e --arg cp "$CP" --arg ext "$EXT" ".control_plane_job == \$cp and .health.extended_appbackup == \$ext and .extended_appbackup_health == \$ext" >/dev/null || BAD=$((BAD+1)) ------ /usr/local/sbin/homelab-health-metrics ----- -16:category_for_check(){ -22: *backup*|*vzdump*) printf "%s" "backup" ;; -31:tier_for_check(){ -34: npmplus-kuma-config-*|authentik-*|gitea-*|vaultwarden-*|netbox-*|mail-cloud-restore-drill|edge-vm-vzdump-*|mkdocs-runtime-publish|npmplus-certificate-expiry) printf "%s" "P0" ;; -69: check="${base%.txt}" -70: category="$(category_for_check "$check")" -71: tier="$(tier_for_check "$check")" -75: check_esc="$(esc_label "$check")" -79: labels="host=\"$host_esc\",check=\"$check_esc\",category=\"$cat_esc\",tier=\"$tier_esc\",file=\"$file_esc\"" ------ /usr/local/sbin/homelab-ingress-standby-refresh ----- -16: rm -f "$TMP_STATE" "$RELAY" "$RESOURCES" -74:run_node "$SOURCE_NODE" vzdump 130 \ -78: --remove 0 \ -83:"find /var/lib/vz/dump -maxdepth 1 -type f -name 'vzdump-qemu-130-*.vma.zst' -printf '%T@ %p\n' | sort -rn | head -n 1 | cut -d' ' -f2-" -106:"chattr -i '$DEST_FILE' '$DEST_FILE.sha256' '$DEST_FILE.env' 2>/dev/null || true; rm -f '$DEST_TMP'" -155: -name 'vzdump-qemu-130-*.vma.zst' \ -164: rm -f "$OLD" "$OLD.sha256" "$OLD.env" -173:run_node "$SOURCE_NODE" rm -f "$SOURCE_BACKUP" ------ /usr/local/sbin/homelab-mail-cloud-critical-upload ----- -50:EXCLUDED=edge-vm-vzdump-large-archive,rclone-config,secrets,raw-env -80: rclone mkdir "${R}:$REMOTE_DIR" > "$WORK/logs/${SAFE}_mkdir.out" 2>"$WORK/logs/${SAFE}_mkdir.err" -82: homelab-rclone-copy-chunked "$BUNDLE" "${R}:$REMOTE_DIR/$(basename "$BUNDLE")" > "$WORK/logs/${SAFE}_upload_bundle.out" 2>"$WORK/logs/${SAFE}_upload_bundle.err" -84: rclone copyto "$BUNDLE_SHA" "${R}:$REMOTE_DIR/$(basename "$BUNDLE_SHA")" --checksum > "$WORK/logs/${SAFE}_upload_sha.out" 2>"$WORK/logs/${SAFE}_upload_sha.err" -86: rclone copyto "$WORK/components/MANIFEST.txt" "${R}:$REMOTE_DIR/MANIFEST.txt" --checksum > "$WORK/logs/${SAFE}_upload_manifest.out" 2>"$WORK/logs/${SAFE}_upload_manifest.err" -88: rclone copyto "$WORK/components/SHA256SUMS.txt" "${R}:$REMOTE_DIR/SHA256SUMS.txt" --checksum > "$WORK/logs/${SAFE}_upload_sums.out" 2>"$WORK/logs/${SAFE}_upload_sums.err" -90: homelab-rclone-cat-chunked "${R}:$REMOTE_DIR/$(basename "$BUNDLE")" > "$WORK/download/${SAFE}-$(basename "$BUNDLE")" 2>"$WORK/logs/${SAFE}_download_bundle.err" -108: rm -f "$WORK/download/${SAFE}-$(basename "$BUNDLE")" -109: rclone lsf "${R}:$TARGET_BASE" --dirs-only 2>/dev/null | sed 's#/$##' | sort > "$WORK/logs/${SAFE}_dirs.txt" -114: [ -n "$OLD" ] && rclone purge "${R}:$TARGET_BASE/$OLD" >> "$WORK/logs/${SAFE}_retention.out" 2>>"$WORK/logs/${SAFE}_retention.err" -140: rm -rf --one-file-system "$OLD_DIR" ------ /usr/local/sbin/homelab-mail-cloud-critical-upload.bak-chunkall-20260710T011410Z ----- -50:EXCLUDED=edge-vm-vzdump-large-archive,rclone-config,secrets,raw-env -80: rclone mkdir "${R}:$REMOTE_DIR" > "$WORK/logs/${SAFE}_mkdir.out" 2>"$WORK/logs/${SAFE}_mkdir.err" -82: rclone copyto "$BUNDLE" "${R}:$REMOTE_DIR/$(basename "$BUNDLE")" --checksum > "$WORK/logs/${SAFE}_upload_bundle.out" 2>"$WORK/logs/${SAFE}_upload_bundle.err" -84: rclone copyto "$BUNDLE_SHA" "${R}:$REMOTE_DIR/$(basename "$BUNDLE_SHA")" --checksum > "$WORK/logs/${SAFE}_upload_sha.out" 2>"$WORK/logs/${SAFE}_upload_sha.err" -86: rclone copyto "$WORK/components/MANIFEST.txt" "${R}:$REMOTE_DIR/MANIFEST.txt" --checksum > "$WORK/logs/${SAFE}_upload_manifest.out" 2>"$WORK/logs/${SAFE}_upload_manifest.err" -88: rclone copyto "$WORK/components/SHA256SUMS.txt" "${R}:$REMOTE_DIR/SHA256SUMS.txt" --checksum > "$WORK/logs/${SAFE}_upload_sums.out" 2>"$WORK/logs/${SAFE}_upload_sums.err" -90: rclone copyto "${R}:$REMOTE_DIR/$(basename "$BUNDLE")" "$WORK/download/${SAFE}-$(basename "$BUNDLE")" > "$WORK/logs/${SAFE}_download_bundle.out" 2>"$WORK/logs/${SAFE}_download_bundle.err" -108: rm -f "$WORK/download/${SAFE}-$(basename "$BUNDLE")" -109: rclone lsf "${R}:$TARGET_BASE" --dirs-only 2>/dev/null | sed 's#/$##' | sort > "$WORK/logs/${SAFE}_dirs.txt" -114: [ -n "$OLD" ] && rclone purge "${R}:$TARGET_BASE/$OLD" >> "$WORK/logs/${SAFE}_retention.out" 2>>"$WORK/logs/${SAFE}_retention.err" ------ /usr/local/sbin/homelab-mail-cloud-edge-vm-upload ----- -10:TARGET_BASE=full/edge-vm-vzdump -60:case "$SIZE" in ""|*[!0-9]*) fail size_bad 2 ;; esac -64:case "$SOURCE" in /mnt/staging/offhost/edge-vm-vzdump-from-pve03/vzdump-qemu-130-*.vma.zst) ;; *) fail source_path_bad 2 ;; esac -111:rclone about "$TARGET_REMOTE:" > "$WORK/logs/about.out" 2>"$WORK/logs/about.err" || ABOUT_RC=$? -114:DIRS="$(rclone lsf "$TARGET_REMOTE:$TARGET_BASE" --dirs-only 2>/dev/null | sed 's#/$##' | sort || true)" -116: if rclone cat "$TARGET_REMOTE:$TARGET_BASE/$D/$BASENAME.sha256" 2>/dev/null | grep -q "$SHA"; then -172:P1 Mail Cloud edge-vm recurring chunked vzdump upload -188:rclone mkdir "$TARGET_REMOTE:$REMOTE_PARTS" > "$WORK/logs/mkdir.out" 2>"$WORK/logs/mkdir.err" || MKDIR_RC=$? -190:rclone copy "$PARTS_DIR" "$TARGET_REMOTE:$REMOTE_PARTS" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > "$WORK/logs/upload_parts.out" 2>"$WORK/logs/upload_parts.err" || UPLOAD_PARTS_RC=$? -192:rclone copyto "$WORK/$BASENAME.sha256" "$TARGET_REMOTE:$REMOTE_SHA" --checksum > "$WORK/logs/upload_sha.out" 2>"$WORK/logs/upload_sha.err" || UPLOAD_SHA_RC=$? -194:rclone copyto "$WORK/MANIFEST.txt" "$TARGET_REMOTE:$REMOTE_MANIFEST" --checksum > "$WORK/logs/upload_manifest.out" 2>"$WORK/logs/upload_manifest.err" || UPLOAD_MANIFEST_RC=$? -196:rclone copyto "$WORK/SHA256SUMS.txt" "$TARGET_REMOTE:$REMOTE_SUMS" --checksum > "$WORK/logs/upload_sums.out" 2>"$WORK/logs/upload_sums.err" || UPLOAD_SUMS_RC=$? -199:rclone size "$TARGET_REMOTE:$TARGET_DIR" --json > "$WORK/logs/remote_size.json" 2>"$WORK/logs/remote_size.err" || SIZE_RC=$? -201:rclone lsf "$TARGET_REMOTE:$REMOTE_PARTS" --format "ps" > "$WORK/logs/remote_lsf_parts.txt" 2>"$WORK/logs/remote_lsf_parts.err" || LSF_PARTS_RC=$? -203:REMOTE_BYTES="$(python3 -c 'import json; print(json.load(open("'"$WORK"'/logs/remote_size.json")).get("bytes",""))' 2>/dev/null || echo unknown)" -204:REMOTE_COUNT="$(python3 -c 'import json; print(json.load(open("'"$WORK"'/logs/remote_size.json")).get("count",""))' 2>/dev/null || echo unknown)" -207:rclone copy "$TARGET_REMOTE:$REMOTE_PARTS" "$DL_DIR/parts" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > "$WORK/logs/download_parts.out" 2>"$WORK/logs/download_parts.err" || DOWNLOAD_PARTS_RC=$? -222: rm -rf "$DL_DIR" "$PARTS_DIR" -234: rclone lsf "$TARGET_REMOTE:$TARGET_BASE" --dirs-only 2>/dev/null | sed "s#/$##" | sort > "$DIR_LIST" || RETENTION_RC=$? -239: [ -n "$OLD" ] && rclone purge "$TARGET_REMOTE:$TARGET_BASE/$OLD" >> "$WORK/logs/retention.out" 2>>"$WORK/logs/retention.err" ------ /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload ----- -8:TARGET_BASE=full/nextcloud-vm-vzdump -11:SRC_GLOB=/var/lib/vz/dump/vzdump-qemu-150-*.vma.zst -23: local delete_count="${7:-0}" -25: echo "STATUS=$status TS=$TS TYPE=mail-cloud-nextcloud-vm MODE=recurring-chunked REMOTE=$TARGET_REMOTE DEST=${TARGET_DIR:-none} ARCHIVE=${BASENAME:-none} SIZE=${SIZE:-0} SHA256=${SHA:-none} PART_SIZE=$PART_SIZE PARTS=${PART_COUNT:-0} REMOTE_PARTS=$remote_parts REMOTE_BYTES=$remote_bytes UPLOAD_OK=$upload_ok DOWNLOAD_VERIFY=$download_verify RETENTION_KEEP=$KEEP DELETE_COUNT=$delete_count ALREADY_UPLOADED=$already_uploaded SECRET_PRINTED=no NOTES=$notes" > "$HEALTH" -77:P1 Mail Cloud Nextcloud VM recurring chunked vzdump upload -93:rclone mkdir "$TARGET_REMOTE:$REMOTE_PARTS" > "$WORK/logs/mkdir.out" 2>"$WORK/logs/mkdir.err" || MKDIR_RC=$? -95:rclone copy "$PARTS_DIR" "$TARGET_REMOTE:$REMOTE_PARTS" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > "$WORK/logs/upload_parts.out" 2>"$WORK/logs/upload_parts.err" || UPLOAD_PARTS_RC=$? -97:rclone copyto "$WORK/$BASENAME.sha256" "$TARGET_REMOTE:$REMOTE_SHA" --checksum > "$WORK/logs/upload_sha.out" 2>"$WORK/logs/upload_sha.err" || UPLOAD_SHA_RC=$? -99:rclone copyto "$WORK/MANIFEST.txt" "$TARGET_REMOTE:$REMOTE_MANIFEST" --checksum > "$WORK/logs/upload_manifest.out" 2>"$WORK/logs/upload_manifest.err" || UPLOAD_MANIFEST_RC=$? -101:rclone copyto "$WORK/SHA256SUMS.txt" "$TARGET_REMOTE:$REMOTE_SUMS" --checksum > "$WORK/logs/upload_sums.out" 2>"$WORK/logs/upload_sums.err" || UPLOAD_SUMS_RC=$? -104:rclone lsf "$TARGET_REMOTE:$REMOTE_PARTS" --format ps > "$WORK/logs/remote_lsf_parts.txt" 2>"$WORK/logs/remote_lsf_parts.err" || LSF_PARTS_RC=$? -110:rclone copy "$TARGET_REMOTE:$REMOTE_PARTS" "$DL_DIR/parts" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > "$WORK/logs/download_parts.out" 2>"$WORK/logs/download_parts.err" || DOWNLOAD_PARTS_RC=$? -120:rclone lsf "$TARGET_REMOTE:$TARGET_BASE" --dirs-only 2>/dev/null | sed 's#/$##' | sort > "$DIR_LIST" || RETENTION_RC=$? -126: [ -n "$OLD" ] && rclone purge "$TARGET_REMOTE:$TARGET_BASE/$OLD" >> "$WORK/logs/retention.out" 2>>"$WORK/logs/retention.err" -140: rm -rf "$PARTS_DIR" "$DL_DIR" ------ /usr/local/sbin/homelab-mail-cloud-restore-drill ----- -52: homelab-rclone-cat-chunked "${R}:$CRIT_DEST/$CRIT_BUNDLE" > "$OUT" 2>"$WORK/logs/${SAFE}_crit_download.err" -83: rm -f "$OUT" -108: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/MANIFEST.txt" "$WORK/edge/MANIFEST.txt" > "$WORK/logs/edge_manifest_download.out" 2>"$WORK/logs/edge_manifest_download.err" -111: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/SHA256SUMS.txt" "$WORK/edge/SHA256SUMS.txt" > "$WORK/logs/edge_sums_download.out" 2>"$WORK/logs/edge_sums_download.err" -114: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/$EDGE_ARCHIVE.sha256" "$WORK/edge/$EDGE_ARCHIVE.sha256" > "$WORK/logs/edge_archive_sha_download.out" 2>"$WORK/logs/edge_archive_sha_download.err" -117: rclone lsf "${EDGE_REMOTE}:$EDGE_DEST/parts" --format ps > "$WORK/logs/edge_parts_lsf.txt" 2>"$WORK/logs/edge_parts_lsf.err" -130: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/parts/$FIRST_PART" "$WORK/edge/$FIRST_PART" > "$WORK/logs/edge_first_part_download.out" 2>"$WORK/logs/edge_first_part_download.err" -137: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/parts/$LAST_PART" "$WORK/edge/$LAST_PART" > "$WORK/logs/edge_last_part_download.out" 2>"$WORK/logs/edge_last_part_download.err" -168: rm -f "$WORK/edge/$FIRST_PART" "$WORK/edge/$LAST_PART" ------ /usr/local/sbin/homelab-mail-cloud-restore-drill.bak-chunkall-20260710T011410Z ----- -52: rclone copyto "${R}:$CRIT_DEST/$CRIT_BUNDLE" "$OUT" > "$WORK/logs/${SAFE}_crit_download.out" 2>"$WORK/logs/${SAFE}_crit_download.err" -83: rm -f "$OUT" -108: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/MANIFEST.txt" "$WORK/edge/MANIFEST.txt" > "$WORK/logs/edge_manifest_download.out" 2>"$WORK/logs/edge_manifest_download.err" -111: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/SHA256SUMS.txt" "$WORK/edge/SHA256SUMS.txt" > "$WORK/logs/edge_sums_download.out" 2>"$WORK/logs/edge_sums_download.err" -114: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/$EDGE_ARCHIVE.sha256" "$WORK/edge/$EDGE_ARCHIVE.sha256" > "$WORK/logs/edge_archive_sha_download.out" 2>"$WORK/logs/edge_archive_sha_download.err" -117: rclone lsf "${EDGE_REMOTE}:$EDGE_DEST/parts" --format ps > "$WORK/logs/edge_parts_lsf.txt" 2>"$WORK/logs/edge_parts_lsf.err" -130: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/parts/$FIRST_PART" "$WORK/edge/$FIRST_PART" > "$WORK/logs/edge_first_part_download.out" 2>"$WORK/logs/edge_first_part_download.err" -137: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/parts/$LAST_PART" "$WORK/edge/$LAST_PART" > "$WORK/logs/edge_last_part_download.out" 2>"$WORK/logs/edge_last_part_download.err" -168: rm -f "$WORK/edge/$FIRST_PART" "$WORK/edge/$LAST_PART" ------ /usr/local/sbin/homelab-rclone-cat-chunked ----- -4:TMP_BASE=${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks} -7:cleanup(){ rm -rf "$WORK"; } -10:rclone copyto "${REMOTE_FILE}.parts/MANIFEST.parts.txt" "$MAN" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR -11:while read -r PART SIZE; do case "$PART" in part-*) rclone cat "${REMOTE_FILE}.parts/$PART" ;; esac; done < "$MAN" ------ /usr/local/sbin/homelab-rclone-cat-flat-chunked ----- -5:trap "rm -f \"$TMP\"" EXIT -6:rclone cat "${DEST}.parts.txt" > "$TMP" -8:while read -r PART PART_SIZE; do [[ "$PART" =~ ^part-[0-9]{4}$ && "$PART_SIZE" =~ ^[0-9]+$ ]] || { echo "BAD FLAT PART MANIFEST" >&2; exit 2; }; rclone cat "${DEST}.${PART}"; COUNT=$((COUNT+1)); done < "$TMP" ------ /usr/local/sbin/homelab-rclone-copy-chunked ----- -7:TMP_BASE=${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks} -13:cleanup(){ rm -rf "$WORK"; } -23:printf "name=%s\\nsize=%s\\nsha256=%s\\nchunk_size=%s\\nparts=%s\\n" "$NAME" "$SIZE" "$SHA" "$CHUNK_SIZE" "$PART_COUNT" > "$WORK/MANIFEST.txt" -24:rclone purge "${DEST}.parts" >/dev/null 2>&1 || true -25:rclone mkdir "${DEST}.parts" -26:rclone copy "$PARTS" "${DEST}.parts" --transfers 2 --checkers 4 --retries 5 --low-level-retries 20 --stats 30s --log-level INFO -27:rclone copyto "$WORK/${NAME}.sha256" "${DEST}.sha256" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO -28:rclone copyto "$WORK/MANIFEST.txt" "${DEST}.manifest.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO -29:rclone copyto "$WORK/MANIFEST.parts.txt" "${DEST}.parts/MANIFEST.parts.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO -30:REMOTE_COUNT=$(rclone lsf "${DEST}.parts" --files-only | grep -c "^part-" || true) -32:ACTUAL=$(while read -r PART SIZE_PART; do case "$PART" in part-*) rclone cat "${DEST}.parts/$PART" ;; esac; done < "$WORK/MANIFEST.parts.txt" | sha256sum | cut -d " " -f 1) ------ /usr/local/sbin/homelab-rclone-copy-flat-chunked ----- -6:TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" -12:trap "rm -rf \"$WORK\"" EXIT -21:printf "format=homelab-flat-chunks-v1\nname=%s\nsize=%s\nsha256=%s\nchunk_size=%s\nparts=%s\n" "$NAME" "$SIZE" "$SHA" "$CHUNK_SIZE" "$PART_COUNT" > "$WORK/MANIFEST.txt" -23:while read -r PART PART_SIZE; do [[ "$PART" =~ ^part-[0-9]{4}$ && "$PART_SIZE" =~ ^[0-9]+$ ]] || { echo "BAD PART MANIFEST" >&2; exit 4; }; rclone copyto "$WORK/parts/$PART" "${DEST}.${PART}" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR; REMOTE_BYTES="$(rclone size --json "${DEST}.${PART}" | jq -r ".bytes // -1")"; [[ "$REMOTE_BYTES" = "$PART_SIZE" ]] || { echo "REMOTE PART SIZE MISMATCH $PART" >&2; exit 5; }; UPLOADED=$((UPLOADED+1)); done < "$WORK/MANIFEST.parts.txt" -25:rclone copyto "$WORK/${NAME}.sha256" "${DEST}.sha256" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR -26:rclone copyto "$WORK/MANIFEST.txt" "${DEST}.manifest.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR -27:rclone copyto "$WORK/MANIFEST.parts.txt" "${DEST}.parts.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR -28:ACTUAL="$(homelab-rclone-cat-flat-chunked "$DEST" | sha256sum | awk "{print \$1}")" ------ /usr/local/sbin/homelab-rclone-crypt-healthcheck.sh ----- -4:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt" -18: "checksums/" -21:echo "=== homelab rclone crypt healthcheck: $(hostname) ===" -40: if ! rclone listremotes | sed 's/:$//' | grep -qx "$REMOTE"; then -47: if ! rclone about "${REMOTE}:"; then -48: echo "FAIL: rclone about failed: $REMOTE" -54: ROOT_LIST="$(rclone lsf "${REMOTE}:/" || true)" -76: if rclone cat "${REMOTE}:/latest-backup-index.json" | jq -e '.schema and .remote and .status and .items' >/dev/null; then ------ /usr/local/sbin/homelab-rclone-required-check.sh ----- -4:RAW_FILE="/etc/homelab/rclone/required-remotes.txt" -5:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt" -9:echo "=== homelab rclone required check: $(hostname) ===" -12:echo "=== installed rclone ===" -13:rclone version | head -5 -17:CONFIGURED="$(rclone listremotes 2>/dev/null | sed 's/:$//' || true)" -19: echo "No rclone remotes configured yet." ------ /usr/local/sbin/homelab-router-hard-power-cycle ----- -12:case "$MODE" in check|cycle) ;; *) echo HARD_WRAPPER_MODE_INVALID=yes; exit 64;; esac -23:SAFE_OUT="$(printf '%s\n' "$OUT" | sed -E '/(PASSWORD|TOKEN|SECRET|PRIVATE_KEY|ROOT_PASSWORD|ENCRYPTION_KEY|COOKIE|SESSION|AUTHORIZATION|API_KEY|LOCAL_KEY|rclone[.]conf|[.]env)/Id')" ------ /usr/local/sbin/homelab-router-running-config-mail-cloud ----- -62: rm -f "$CONFIG" -66: rm -f "$CONFIG" -73: if ( cd "$BACKUP_ROOT" && sha256sum -c "$(basename "$SHA_FILE")" ) >"$WORK/sha-check.out" 2>"$WORK/sha-check.err"; then -79: if rclone mkdir "$REMOTE:$DEST" >"$WORK/rclone-mkdir.out" 2>"$WORK/rclone-mkdir.err" && homelab-rclone-copy-chunked "$ARCHIVE" "$REMOTE:$DEST/$(basename "$ARCHIVE")" >"$WORK/rclone-upload.out" 2>"$WORK/rclone-upload.err" && rclone copyto "$SHA_FILE" "$REMOTE:$DEST/$(basename "$SHA_FILE")" --checksum >"$WORK/rclone-upload-sha.out" 2>"$WORK/rclone-upload-sha.err"; then -86: if homelab-rclone-cat-chunked "$REMOTE:$DEST/$(basename "$ARCHIVE")" >"$DL" 2>"$WORK/rclone-download.err"; then -93:rclone lsf "$REMOTE:$TARGET_BASE" --dirs-only 2>/dev/null | sed 's#/$##' | sort > "$DIR_LIST" || RETENTION_RC=$? -98: [ -n "$OLD" ] && rclone purge "$REMOTE:$TARGET_BASE/$OLD" >>"$WORK/retention.out" 2>>"$WORK/retention.err" -103: rm -f "$old" "$old.sha256" -116:rm -rf "$WORK" ------ /usr/local/sbin/homelab-router-running-config-mail-cloud.bak-chunkall-20260710T011410Z ----- -62: rm -f "$CONFIG" -66: rm -f "$CONFIG" -73: if ( cd "$BACKUP_ROOT" && sha256sum -c "$(basename "$SHA_FILE")" ) >"$WORK/sha-check.out" 2>"$WORK/sha-check.err"; then -79: if rclone mkdir "$REMOTE:$DEST" >"$WORK/rclone-mkdir.out" 2>"$WORK/rclone-mkdir.err" && rclone copyto "$ARCHIVE" "$REMOTE:$DEST/$(basename "$ARCHIVE")" --checksum >"$WORK/rclone-upload.out" 2>"$WORK/rclone-upload.err" && rclone copyto "$SHA_FILE" "$REMOTE:$DEST/$(basename "$SHA_FILE")" --checksum >"$WORK/rclone-upload-sha.out" 2>"$WORK/rclone-upload-sha.err"; then -86: if rclone copyto "$REMOTE:$DEST/$(basename "$ARCHIVE")" "$DL" >"$WORK/rclone-download.out" 2>"$WORK/rclone-download.err"; then -93:rclone lsf "$REMOTE:$TARGET_BASE" --dirs-only 2>/dev/null | sed 's#/$##' | sort > "$DIR_LIST" || RETENTION_RC=$? -98: [ -n "$OLD" ] && rclone purge "$REMOTE:$TARGET_BASE/$OLD" >>"$WORK/retention.out" 2>>"$WORK/retention.err" -103: rm -f "$old" "$old.sha256" -116:rm -rf "$WORK" ------ /usr/local/sbin/homelab-secret-exposure-guard ----- -18:BADFILES="$(find "$REPO" -path "$REPO/.git" -prune -o -type f \( -iname "*.key" -o -iname "*.pem" -o -iname "*.p12" -o -iname "*.pfx" -o -iname "[SENSITIVE_FILE]" -o -iname ".env" -o -iname ".env.*" \) -print 2>/dev/null)" ------ /usr/local/sbin/homelab-stage-2cloud ----- -20:jq -n --arg schema "homelab-app-backup-manifest-v1" --arg job "$JOB" --arg node "$NODE" --arg app "$APP" --arg created "$(date -u +%Y%m%dT%H%M%SZ)" --arg archive "$ARCHIVE" --arg sha "$SHA" --arg ra "$RA" --arg rb "$RB" --argjson size "$SIZE" --argjson meta "$META" '$meta + {schema:$schema,job_id:$job,node:$node,app_name:$app,created_utc:$created,archive_name:$archive,archive_size_bytes:$size,archive_sha256:$sha,remote_a_job_path:$ra,remote_b_job_path:$rb,local_staging_policy:"delete_after_both_clouds_verified",verification_policy:"full_roundtrip_download_plus_sha256"}' > "$STAGE/manifest.json" -22:echo "[STEP] local checksum"; (cd "$STAGE" && sha256sum "$ARCHIVE" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e . manifest.json >/dev/null) -23:echo "[STEP] upload cloud A metadata"; rclone copy "$STAGE" "$RA" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s -24:echo "[STEP] upload cloud A archive chunks"; rclone deletefile "$RA/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RA/$ARCHIVE" -25:echo "[STEP] upload cloud B metadata"; rclone copy "$STAGE" "$RB" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s -26:echo "[STEP] upload cloud B archive chunks"; rclone deletefile "$RB/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RB/$ARCHIVE" -27:echo "[STEP] verify cloud A chunked"; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-stage-verify-chunked "$RA" "$ARCHIVE" "$VERIFY/a" "$JOB" -28:echo "[STEP] verify cloud B chunked"; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-stage-verify-chunked "$RB" "$ARCHIVE" "$VERIFY/b" "$JOB" -29:echo "[STEP] upload latest index"; rclone copyto "$STAGE/latest-backup-index.json" "$RAL"; rclone copyto "$STAGE/latest-backup-index.json" "$RBL"; rclone cat "$RAL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null; rclone cat "$RBL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null -30:jq ". + {status:\"success\",local_backup_deleted:true}" "$STAGE/manifest.json" > "$STATE_ROOT/apps/latest-${APP}.json" -31:echo "[STEP] delete local staging"; rm -rf "$STAGE" "$VERIFY"; test ! -d "$STAGE" && test ! -d "$VERIFY" -33:echo "[SUCCESS] uploaded to 2 clouds, verified from both clouds, local staging deleted" ------ /usr/local/sbin/homelab-stage-2cloud.bak-global-chunkonly-20260710T011036Z ----- -20:jq -n --arg schema "homelab-app-backup-manifest-v1" --arg job "$JOB" --arg node "$NODE" --arg app "$APP" --arg created "$(date -u +%Y%m%dT%H%M%SZ)" --arg archive "$ARCHIVE" --arg sha "$SHA" --arg ra "$RA" --arg rb "$RB" --argjson size "$SIZE" --argjson meta "$META" '$meta + {schema:$schema,job_id:$job,node:$node,app_name:$app,created_utc:$created,archive_name:$archive,archive_size_bytes:$size,archive_sha256:$sha,remote_a_job_path:$ra,remote_b_job_path:$rb,local_staging_policy:"delete_after_both_clouds_verified",verification_policy:"full_roundtrip_download_plus_sha256"}' > "$STAGE/manifest.json" -22:echo "[STEP] local checksum"; (cd "$STAGE" && sha256sum "$ARCHIVE" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e . manifest.json >/dev/null) -23:echo "[STEP] upload cloud A"; rclone copy "$STAGE" "$RA" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 --stats=30s -24:echo "[STEP] upload cloud B"; rclone copy "$STAGE" "$RB" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 --stats=30s -25:echo "[STEP] verify cloud A"; rclone copy "$RA" "$VERIFY/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10; (cd "$VERIFY/a" && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null) -26:echo "[STEP] verify cloud B"; rclone copy "$RB" "$VERIFY/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10; (cd "$VERIFY/b" && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null) -27:echo "[STEP] upload latest index"; rclone copyto "$STAGE/latest-backup-index.json" "$RAL"; rclone copyto "$STAGE/latest-backup-index.json" "$RBL"; rclone cat "$RAL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null; rclone cat "$RBL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null -28:jq ". + {status:\"success\",local_backup_deleted:true}" "$STAGE/manifest.json" > "$STATE_ROOT/apps/latest-${APP}.json" -29:echo "[STEP] delete local staging"; rm -rf "$STAGE" "$VERIFY"; test ! -d "$STAGE" && test ! -d "$VERIFY" -31:echo "[SUCCESS] uploaded to 2 clouds, verified from both clouds, local staging deleted" ------ /usr/local/sbin/homelab-stage-2cloud-flat ----- -31:jq -n --arg schema "homelab-app-backup-manifest-v2-flat" --arg layout "flat-v1" --arg job "$JOB" --arg node "$NODE" --arg app "$APP" --arg created "$CREATED" --arg archive "$ARCHIVE" --arg sha "$SHA" --arg ra "$RA" --arg rb "$RB" --arg rel "$REL" --argjson size "$SIZE" --argjson meta "$META" "\$meta + {schema:\$schema,layout:\$layout,job_id:\$job,node:\$node,app_name:\$app,created_utc:\$created,archive_name:\$archive,archive_size_bytes:\$size,archive_sha256:\$sha,path_rel:\$rel,remote_a_job_path:\$ra,remote_b_job_path:\$rb,local_staging_policy:\"delete_after_both_clouds_verified\",verification_policy:\"flat_chunks_full_roundtrip_sha256\"}" > "$STAGE/manifest.json" -33:(cd "$STAGE"; FILES=("$ARCHIVE" manifest.json latest-backup-index.json); [[ -f metadata.json ]] && FILES+=(metadata.json); sha256sum "${FILES[@]}" > checksums.sha256; sha256sum -c checksums.sha256 >/dev/null; tar -tzf "$ARCHIVE" >/dev/null; jq -e . manifest.json >/dev/null) -36: if rclone lsf "$REMOTE_DIR" --max-depth 1 2>/dev/null | grep -q .; then echo "REFUSE NONEMPTY CANARY TARGET $REMOTE_DIR" >&2; exit 6; fi -37: rclone mkdir "$REMOTE_DIR" -38: for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f "$STAGE/$F" ]] && rclone copyto "$STAGE/$F" "$REMOTE_DIR/$F" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done -39: CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-flat-chunked "$STAGE/$ARCHIVE" "$REMOTE_DIR/$ARCHIVE" -41: rclone copyto "$STAGE/latest-backup-index.json" "$LATEST_FILE" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR -42: rclone cat "$LATEST_FILE" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null -46:jq ". + {status:\"success\",local_backup_deleted:true}" "$STAGE/manifest.json" > "$STATE_ROOT/apps/latest-${APP}.json" -47:rm -rf "$STAGE" "$VERIFY" ------ /usr/local/sbin/homelab-stage-2cloud-flat-canary ----- -29:jq -n --arg schema "homelab-app-backup-manifest-v2-flat" --arg layout "flat-v1" --arg job "$JOB" --arg node "$NODE" --arg app "$APP" --arg created "$CREATED" --arg archive "$ARCHIVE" --arg sha "$SHA" --arg ra "$RA" --arg rb "$RB" --arg rel "$REL" --argjson size "$SIZE" --argjson meta "$META" "\$meta + {schema:\$schema,layout:\$layout,job_id:\$job,node:\$node,app_name:\$app,created_utc:\$created,archive_name:\$archive,archive_size_bytes:\$size,archive_sha256:\$sha,path_rel:\$rel,remote_a_job_path:\$ra,remote_b_job_path:\$rb,local_staging_policy:\"delete_after_both_clouds_verified\",verification_policy:\"flat_chunks_full_roundtrip_sha256\"}" > "$STAGE/manifest.json" -31:(cd "$STAGE"; FILES=("$ARCHIVE" manifest.json latest-backup-index.json); [[ -f metadata.json ]] && FILES+=(metadata.json); sha256sum "${FILES[@]}" > checksums.sha256; sha256sum -c checksums.sha256 >/dev/null; tar -tzf "$ARCHIVE" >/dev/null; jq -e . manifest.json >/dev/null) -34: if rclone lsf "$REMOTE_DIR" --max-depth 1 2>/dev/null | grep -q .; then echo "REFUSE NONEMPTY CANARY TARGET $REMOTE_DIR" >&2; exit 6; fi -35: rclone mkdir "$REMOTE_DIR" -36: for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f "$STAGE/$F" ]] && rclone copyto "$STAGE/$F" "$REMOTE_DIR/$F" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done -37: CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-flat-chunked "$STAGE/$ARCHIVE" "$REMOTE_DIR/$ARCHIVE" -39: rclone copyto "$STAGE/latest-backup-index.json" "$LATEST_FILE" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR -40: rclone cat "$LATEST_FILE" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null -44:jq ". + {status:\"success\",local_backup_deleted:true}" "$STAGE/manifest.json" > "$STATE_ROOT/apps/latest-${APP}.json" -45:rm -rf "$STAGE" "$VERIFY" ------ /usr/local/sbin/homelab-stage-verify-chunked ----- -7:rm -rf "$VERIFY" -9:rclone copy "$REMOTE_DIR" "$VERIFY" --exclude "$ARCHIVE.parts/**" --exclude "$ARCHIVE.parts/" --exclude "$ARCHIVE.sha256" --exclude "$ARCHIVE.manifest.txt" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --log-level ERROR -13:if [ -s checksums.sha256 ]; then grep -F -v " $ARCHIVE" checksums.sha256 > .checksums-no-archive.sha256 || true; if [ -s .checksums-no-archive.sha256 ]; then sha256sum -c .checksums-no-archive.sha256; fi; EXPECTED=$(grep -F " $ARCHIVE" checksums.sha256 | awk '{print $1; exit}'); else EXPECTED=$(rclone cat "${REMOTE_DIR}/${ARCHIVE}.sha256" | awk '{print $1; exit}'); fi -15:ACTUAL=$(homelab-rclone-cat-chunked "${REMOTE_DIR}/${ARCHIVE}" | sha256sum | awk '{print $1}') -17:homelab-rclone-cat-chunked "${REMOTE_DIR}/${ARCHIVE}" | tar -tzf - >/dev/null ------ /usr/local/sbin/homelab-stage-verify-flat-chunked ----- -7:rm -rf "$VERIFY" -9:rclone copy "$REMOTE_DIR" "$VERIFY" --transfers 4 --checkers 8 --retries 5 --low-level-retries 20 --log-level ERROR -11:(cd "$VERIFY" && test -s manifest.json && if [[ -n "$JOB" ]]; then jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null; fi && sha256sum -c checksums.sha256 >/dev/null && tar -tzf "$ARCHIVE" >/dev/null) ------ /usr/local/sbin/homelab-vm160-remote-backup ----- -8:PARENT=/mnt/staging/vzdump -22:SOURCE_ACTIVE="$(ssh "${SSH[@]}" "$SOURCE" "pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'")" || SOURCE_ACTIVE_RC=$? -27:NAME="vzdump-qemu-${VMID}-${TS}.vma.zst" -35: ssh "${SSH[@]}" "$TARGET" "rm -rf -- '$DIR'" >/dev/null 2>&1 || true -41: rm -f "$LOG" -46:LATEST_SIZE="$(ssh "${SSH[@]}" "$TARGET" "find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\n' | sort -nr | head -n1")" -55:ssh "${SSH[@]}" "$SOURCE" "vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7" 2>"$LOG" | ssh "${SSH[@]}" "$TARGET" "cat > '$PART'" -57:ssh "${SSH[@]}" "$TARGET" "cat > '$DIR/vzdump.log'" < "$LOG" -66:rm -f "$LOG" ------ /usr/local/sbin/homelab-vm-backup-health ----- -8:check_file(){ -15:check_file /var/lib/homelab-health/backup.txt auto-vmct -16:check_file /var/lib/homelab-health/edge-vm-vzdump-backup.txt edge-vm-backup -17:check_file /var/lib/homelab-health/edge-vm-vzdump-offhost.txt edge-vm-offhost -18:check_file /var/lib/homelab-health/edge-vm-vzdump-restore.txt edge-vm-restore ------ /usr/local/sbin/homelab-vm-backup-policy ----- -29:echo "## Latest local vzdump catalog" >> "$REPORT" -30:printf "%s\t%s\t%s\t%s\t%s\t%s\n" "kind" "vmid" "name" "size_bytes" "mtime_utc" "path" > "$CATALOG" -31:for f in $(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-*.vma.zst" -o -name "vzdump-lxc-*.tar.zst" \) 2>/dev/null | sort); do b="$(basename "$f")"; size="$(stat -c %s "$f" 2>/dev/null || echo 0)"; mt="$(date -u -d "@$(stat -c %Y "$f" 2>/dev/null || echo 0)" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)"; if echo "$b" | grep -q "^vzdump-qemu-"; then kind=qemu; vmid="$(echo "$b" | sed -n "s/^vzdump-qemu-\([0-9]\+\)-.*/\1/p")"; name="$(qm config "$vmid" 2>/dev/null | awk -F": " "/^name:/{print \$2; exit}")"; else kind=lxc; vmid="$(echo "$b" | sed -n "s/^vzdump-lxc-\([0-9]\+\)-.*/\1/p")"; name="$(pct config "$vmid" 2>/dev/null | awk -F": " "/^hostname:/{print \$2; exit}")"; fi; printf "%s\t%s\t%s\t%s\t%s\t%s\n" "$kind" "$vmid" "${name:-unknown}" "$size" "$mt" "$f" >> "$CATALOG"; done -37:echo "- Existing local dumps can be uploaded/verified to two clouds without creating new vzdump archives." >> "$REPORT" -38:echo "- New full vzdump for VM 170/171 is not started automatically because each VM has 120 GB bootdisk and /var/tmp has limited free space." >> "$REPORT" -42:echo "## Coverage checks" >> "$REPORT" ------ /usr/local/sbin/homelab-vm-full-backup ----- -11:# shellcheck disable=SC1090 -92:for cmd in vzdump qm zstd sha256sum rclone rsync ssh flock; do -130: -name "vzdump-qemu-${VMID}-*.vma.zst" \ -138: vzdump "$VMID" \ -145: --remove 0 -151: -name "vzdump-qemu-${VMID}-*.vma.zst" \ -261: CHUNK_TMP_BASE="$BACKUP_DIR/.rclone-chunks" homelab-rclone-copy-chunked "$ARCHIVE" "${DEST_ROOT}/${NAME}" ------ /usr/local/sbin/homelab-vm-full-backup.bak-chunkall-20260710T011410Z ----- -11:# shellcheck disable=SC1090 -92:for cmd in vzdump qm zstd sha256sum rclone rsync ssh flock; do -130: -name "vzdump-qemu-${VMID}-*.vma.zst" \ -138: vzdump "$VMID" \ -145: --remove 0 -151: -name "vzdump-qemu-${VMID}-*.vma.zst" \ -264: rclone copyto \ -267: --check-first \ -269: --checkers 2 \ -275: rclone copyto \ -278: --check-first \ -280: --checkers 2 \ -285: echo "Crypt-checking $NAME on $REMOTE" -287: rclone cryptcheck \ -292: --checkers 1 \ ------ /usr/local/sbin/homelab-vm-full-restore-drill ----- -77: -name "vzdump-qemu-${source_vmid}-*.vma.zst" \ -152:check_inventory() { -184:check_container_health() { -224:check_http() { -349:check_container_health() { -386:check_container_health "$test_vmid" -389: check_inventory "$source_vmid" "$test_vmid" -392: check_http "$source_vmid" "$test_vmid" ------ /usr/local/sbin/homelab-vm-full-restore-drill.bak.20260710T080231Z ----- -77: -name "vzdump-qemu-${source_vmid}-*.vma.zst" \ -152:check_inventory() { -184:check_container_health() { -224:check_http() { -349:check_container_health() { -386:check_container_health "$test_vmid" -389: check_inventory "$source_vmid" "$test_vmid" -392: check_http "$source_vmid" "$test_vmid" ------ /usr/local/sbin/homelab-vm-full-restore-drill.bak-docker-check-v2-20260710T050044Z ----- -77: -name "vzdump-qemu-${source_vmid}-*.vma.zst" \ -152:check_inventory() { -184:check_container_health() { -224:check_http() { -347: check_container_health "$test_vmid" -350: check_inventory "$source_vmid" "$test_vmid" -353: check_http "$source_vmid" "$test_vmid" ------ /usr/local/sbin/homelab-vm-full-restore-drill.bak-isolated-ip-20260710T044331Z ----- -77: -name "vzdump-qemu-${source_vmid}-*.vma.zst" \ -152:check_inventory() { -184:check_container_health() { -224:check_http() { -337: check_container_health "$test_vmid" -340: check_inventory "$source_vmid" "$test_vmid" -343: check_http "$source_vmid" "$test_vmid" ------ /usr/local/sbin/homelab-vm-full-retention ----- -21: -name "vzdump-qemu-${vmid}-*.vma.zst" \ -33: vzdump-qemu-"$vmid"-*.vma.zst) -35: rm -f \ -73: rm -rf -- \"\$BASE/\$setname\" -91: rclone lsf "$path" \ -105: rclone purge "${path}/${dir}" ------ /usr/local/sbin/homelab-vm-local-dumps-2cloud ----- -30:CHKS="$WORK/checksums.txt" -39:echo "chunk_size_bytes=$CHUNK_SIZE_BYTES" >> "$LOG" -42: F="$(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-${id}-*.vma.zst" -o -name "vzdump-lxc-${id}-*.tar.zst" \) -printf "%T@ %p\n" 2>/dev/null | sort -n | tail -n 1 | cut -d" " -f2-)" -53: echo "$BN" | grep -q "^vzdump-qemu-" && KIND="qemu" -54: echo "$BN" | grep -q "^vzdump-lxc-" && KIND="lxc" -64: rm -rf "$CDIR" -66: echo "SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR" >> "$LOG" -83:job,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:] -91: "size_bytes":int(p[2]), -107: "chunk_size_bytes":int(chunk_size), -117: rclone mkdir "$DEST/files" >> "$LOG" 2>&1 || BAD=$((BAD+1)) -118: while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do -121: echo "UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts" >> "$LOG" -122: rclone mkdir "$DEST/files/$bn.chunks" >> "$LOG" 2>&1 || BAD=$((BAD+1)) -126: rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) -128: rclone copyto "$CDIR/chunks.sha256" "$DEST/files/$bn.chunks/chunks.sha256" >> "$LOG" 2>&1 || BAD=$((BAD+1)) -130: echo "UPLOAD_DIRECT dest=$DEST file=$bn size=$size" >> "$LOG" -131: rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) -135: rclone copyto "$CHKS" "$DEST/checksums.txt" >> "$LOG" 2>&1 || BAD=$((BAD+1)) -136: rclone copyto "$FILES_TSV" "$DEST/files.tsv" >> "$LOG" 2>&1 || BAD=$((BAD+1)) -137: rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1)) -144: rm -rf "$VBASE" -146: rclone copyto "$DEST/manifest.json" "$VBASE/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1)) -149: while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do -154: rclone copy "$DEST/files/$bn.chunks" "$VDIR" --filter "+ part-*" --filter "+ chunks.sha256" --filter "- *" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) -163: rm -rf "$VDIR" -166: rclone copyto "$DEST/files/$bn" "$VBASE/$bn" --transfers 1 >> "$LOG" 2>&1 || BAD=$((BAD+1)) -168: rm -f "$VBASE/$bn" -171: rm -rf "$VBASE" -184:rm -rf "$VERIFY_ROOT/$JOB_ID" -185:rm -rf "$CHUNK_ROOT/$JOB_ID" -198: --argjson chunk_size_bytes "$CHUNK_SIZE_BYTES" \ -200: "{job_id:\$job_id,status:\$status,node:\$node,remote_a:\$remote_a,remote_b:\$remote_b,count:\$count,total_bytes:\$total_bytes,chunked_count:\$chunked_count,chunk_size_bytes:\$chunk_size_bytes,log:\$log,local_dumps_deleted:false}" \ ------ /usr/local/sbin/homelab-vm-local-dumps-retention ----- -32: mapfile -t JOBS < <(rclone lsf "$BASE/" --dirs-only 2>/dev/null | sed 's#/$##' | sort) -37: echo "nothing_to_delete" >> "$REPORT" -56: rclone purge "$PATH_REMOTE" >> "$REPORT" 2>&1 && DELETED=$((DELETED+1)) || BAD=$((BAD+1)) ------ /usr/local/sbin/skladchik-reports-monitor-backup ----- -28:command -v rclone >/dev/null -29:[ -x /usr/local/sbin/homelab-rclone-copy-chunked ] || { echo "missing /usr/local/sbin/homelab-rclone-copy-chunked" >&2; exit 10; } -36:cleanup(){ rm -rf "$TMP"; } -67:size_bytes=${SIZE} -71:log "LOCAL_ARCHIVE_OK size=$SIZE path=$ARCHIVE" -75: /usr/local/sbin/homelab-rclone-copy-chunked "$ARCHIVE" "${DEST_BASE}/$(basename "$ARCHIVE")" -76: /usr/local/sbin/homelab-rclone-copy-chunked "${ARCHIVE}.sha256" "${DEST_BASE}/$(basename "$ARCHIVE").sha256" -77: /usr/local/sbin/homelab-rclone-copy-chunked "${OUTDIR}/manifest.txt" "${DEST_BASE}/manifest.txt" -82: mapfile -t OLD < <(rclone lsf --dirs-only "${REMOTE}${REMOTE_BASE}/daily/" 2>/dev/null | sed 's#/$##' | sort | head -n -"${KEEP_DAILY}" || true) -83: for D in "${OLD[@]}"; do [ -n "$D" ] && rclone purge "${REMOTE}${REMOTE_BASE}/daily/${D}" >/dev/null 2>&1 || true; done -85:find "${LOCAL_ROOT}/daily" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_DAILY}" | while read -r D; do [ -n "$D" ] && rm -rf "${LOCAL_ROOT}/daily/${D}"; done ------ /usr/local/sbin/skladchik-reports-monitor-backup.pre-restore-check-20260710T065311Z.bak ----- -28:command -v rclone >/dev/null -29:[ -x /usr/local/sbin/homelab-rclone-copy-chunked ] || { echo "missing /usr/local/sbin/homelab-rclone-copy-chunked" >&2; exit 10; } -36:cleanup(){ rm -rf "$TMP"; } -66:size_bytes=${SIZE} -70:log "LOCAL_ARCHIVE_OK size=$SIZE path=$ARCHIVE" -74: /usr/local/sbin/homelab-rclone-copy-chunked "$ARCHIVE" "${DEST_BASE}/$(basename "$ARCHIVE")" -75: /usr/local/sbin/homelab-rclone-copy-chunked "${ARCHIVE}.sha256" "${DEST_BASE}/$(basename "$ARCHIVE").sha256" -76: /usr/local/sbin/homelab-rclone-copy-chunked "${OUTDIR}/manifest.txt" "${DEST_BASE}/manifest.txt" -81: mapfile -t OLD < <(rclone lsf --dirs-only "${REMOTE}${REMOTE_BASE}/daily/" 2>/dev/null | sed 's#/$##' | sort | head -n -"${KEEP_DAILY}" || true) -82: for D in "${OLD[@]}"; do [ -n "$D" ] && rclone purge "${REMOTE}${REMOTE_BASE}/daily/${D}" >/dev/null 2>&1 || true; done -84:find "${LOCAL_ROOT}/daily" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_DAILY}" | while read -r D; do [ -n "$D" ] && rm -rf "${LOCAL_ROOT}/daily/${D}"; done ------ /usr/local/sbin/skladchik-reports-monitor-backup.pre-session-keepalive-20260710T071605Z.bak ----- -28:command -v rclone >/dev/null -29:[ -x /usr/local/sbin/homelab-rclone-copy-chunked ] || { echo "missing /usr/local/sbin/homelab-rclone-copy-chunked" >&2; exit 10; } -36:cleanup(){ rm -rf "$TMP"; } -64:size_bytes=${SIZE} -68:log "LOCAL_ARCHIVE_OK size=$SIZE path=$ARCHIVE" -72: /usr/local/sbin/homelab-rclone-copy-chunked "$ARCHIVE" "${DEST_BASE}/$(basename "$ARCHIVE")" -73: /usr/local/sbin/homelab-rclone-copy-chunked "${ARCHIVE}.sha256" "${DEST_BASE}/$(basename "$ARCHIVE").sha256" -74: /usr/local/sbin/homelab-rclone-copy-chunked "${OUTDIR}/manifest.txt" "${DEST_BASE}/manifest.txt" -79: mapfile -t OLD < <(rclone lsf --dirs-only "${REMOTE}${REMOTE_BASE}/daily/" 2>/dev/null | sed 's#/$##' | sort | head -n -"${KEEP_DAILY}" || true) -80: for D in "${OLD[@]}"; do [ -n "$D" ] && rclone purge "${REMOTE}${REMOTE_BASE}/daily/${D}" >/dev/null 2>&1 || true; done -82:find "${LOCAL_ROOT}/daily" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_DAILY}" | while read -r D; do [ -n "$D" ] && rm -rf "${LOCAL_ROOT}/daily/${D}"; done ------ /usr/local/sbin/skladchik-reports-monitor-backup-restore-check ----- -4:CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf -9:: "${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}" -12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt -13:LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock -19:log(){ printf '%s RESTORE_CHECK %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" | tee -a "$LOGDIR/restore-check.log"; } -22:command -v rclone >/dev/null -28: if rclone lsf "${remote_path}.parts/" > "$plist" 2>/dev/null && grep -q '^part-' "$plist"; then -29: while IFS= read -r part; do [ -n "$part" ] && rclone cat "${remote_path}.parts/${part}" >> "$out"; done < <(grep '^part-' "$plist" | sort) -31: rclone copyto "$remote_path" "$out" >/dev/null -33: rm -f "$plist" -36:check_one_remote(){ -39: latest="$(rclone lsf --dirs-only "${rbase}/" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)" -85: [ "$(sqlite3 "$extract/var-lib-skladchik-reports-monitor/state.sqlite" 'PRAGMA integrity_check;' | tail -n 1)" = "ok" ] -97:check_one_remote CLOUD_1 "$CLOUD_1" -98:check_one_remote CLOUD_2 "$CLOUD_2" -101:find "$WORK_ROOT" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_LOCAL}" | while read -r D; do [ -n "$D" ] && rm -rf "${WORK_ROOT}/${D}"; done ------ /usr/local/sbin/skladchik-reports-monitor-backup-restore-check.pre-session-keepalive-20260710T071605Z.bak ----- -4:CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf -9:: "${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}" -12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt -13:LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock -19:log(){ printf '%s RESTORE_CHECK %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" | tee -a "$LOGDIR/restore-check.log"; } -22:command -v rclone >/dev/null -28: if rclone lsf "${remote_path}.parts/" > "$plist" 2>/dev/null && grep -q '^part-' "$plist"; then -29: while IFS= read -r part; do [ -n "$part" ] && rclone cat "${remote_path}.parts/${part}" >> "$out"; done < <(grep '^part-' "$plist" | sort) -31: rclone copyto "$remote_path" "$out" >/dev/null -33: rm -f "$plist" -36:check_one_remote(){ -39: latest="$(rclone lsf --dirs-only "${rbase}/" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)" -72: [ "$(sqlite3 "$extract/var-lib-skladchik-reports-monitor/state.sqlite" 'PRAGMA integrity_check;' | tail -n 1)" = "ok" ] -84:check_one_remote CLOUD_1 "$CLOUD_1" -85:check_one_remote CLOUD_2 "$CLOUD_2" -88:find "$WORK_ROOT" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_LOCAL}" | while read -r D; do [ -n "$D" ] && rm -rf "${WORK_ROOT}/${D}"; done +=== CLUSTER_BACKUP_JOBS === +[{"compress":"zstd","enabled":1,"id":"homelab-nightly-all","mailnotification":"failure","mode":"snapshot","next-run":1784680200,"prune-backups":{"keep-daily":"4"},"schedule":"03:30","storage":"local","type":"vzdump","vmid":"110,111,112,113,130,150,190"}] +=== CLUSTER_STORAGES === +Name Type Status Total (KiB) Used (KiB) Available (KiB) % +local dir active 102626232 43175760 54191208 42.07% +local-lvm lvmthin active 848797696 280188119 568609576 33.01% +========== NODE=pve01 ========== +pve01.gram1.ru +=== RCLONE_REMOTES === +pve01-mail-01: +pve01-mail-01-crypt: +pve01-mail-02: +pve01-mail-02-crypt: +pve01-mail-03: +pve01-mail-03-crypt: +pve01-mail-04: +pve01-mail-04-crypt: +pve01-mail-05: +pve01-mail-05-crypt: +pve02-mail-01: +pve02-mail-01-crypt: +pve02-mail-02: +pve02-mail-02-crypt: +pve02-mail-03: +pve02-mail-03-crypt: +pve02-mail-04: +pve02-mail-04-crypt: +pve03-mail-01: +pve03-mail-01-crypt: +pve03-mail-02: +pve03-mail-02-crypt: +pve03-mail-03: +pve03-mail-03-crypt: +pve03-mail-04: +pve03-mail-04-crypt: +NODE_REMOTE_COUNT=26 +REMOTE_HEALTH node=pve01 remote=pve01-mail-01: status=OK +=== BACKUP_TIMERS === +Wed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:14 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service +Wed 2026-07-22 00:09:43 MSK 9h Tue 2026-07-21 00:09:32 MSK 14h ago homelab-backup-freshness.timer homelab-backup-freshness.service +Wed 2026-07-22 02:15:00 MSK 11h Tue 2026-07-21 02:15:05 MSK 12h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service +Wed 2026-07-22 03:10:02 MSK 12h Tue 2026-07-21 03:13:14 MSK 11h ago homelab-emergency-backup.timer homelab-emergency-backup.service +Wed 2026-07-22 03:35:00 MSK 12h Tue 2026-07-21 03:35:14 MSK 11h ago filebrowser-backup.timer filebrowser-backup.service +Wed 2026-07-22 03:55:02 MSK 13h Tue 2026-07-21 03:51:25 MSK 10h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service +Wed 2026-07-22 04:17:29 MSK 13h Tue 2026-07-21 04:19:17 MSK 10h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service +Wed 2026-07-22 04:26:54 MSK 13h Tue 2026-07-21 04:28:14 MSK 10h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service +Wed 2026-07-22 04:38:04 MSK 13h Tue 2026-07-21 04:34:48 MSK 10h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service +Wed 2026-07-22 05:01:44 MSK 14h Tue 2026-07-21 04:57:14 MSK 9h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service +Wed 2026-07-22 05:23:56 MSK 14h Tue 2026-07-21 05:17:57 MSK 9h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service +Wed 2026-07-22 05:27:13 MSK 14h Tue 2026-07-21 05:29:05 MSK 9h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service +Wed 2026-07-22 05:47:19 MSK 14h Tue 2026-07-21 05:46:14 MSK 9h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service +Wed 2026-07-22 06:01:19 MSK 15h Tue 2026-07-21 05:57:30 MSK 8h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service +Wed 2026-07-22 06:27:43 MSK 15h Tue 2026-07-21 06:32:25 MSK 8h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service +Wed 2026-07-22 06:38:51 MSK 15h Tue 2026-07-21 06:43:54 MSK 8h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service +Wed 2026-07-22 06:39:22 MSK 15h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service +Wed 2026-07-22 06:44:16 MSK 15h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service +Wed 2026-07-22 06:52:06 MSK 16h Tue 2026-07-21 06:47:47 MSK 8h ago homelab-beszel-backup.timer homelab-beszel-backup.service +Wed 2026-07-22 06:58:58 MSK 16h Tue 2026-07-21 06:57:30 MSK 7h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service +Wed 2026-07-22 07:02:27 MSK 16h Tue 2026-07-21 07:05:30 MSK 7h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service +Wed 2026-07-22 07:07:14 MSK 16h Tue 2026-07-21 07:18:55 MSK 7h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service +Wed 2026-07-22 07:35:06 MSK 16h Tue 2026-07-21 07:36:59 MSK 7h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service +Wed 2026-07-22 07:38:21 MSK 16h Tue 2026-07-21 07:36:14 MSK 7h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service +Wed 2026-07-22 07:41:49 MSK 16h Tue 2026-07-21 07:53:41 MSK 6h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service +Wed 2026-07-22 08:10:37 MSK 17h Tue 2026-07-21 08:07:25 MSK 6h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service +Wed 2026-07-22 08:37:02 MSK 17h Tue 2026-07-21 08:30:07 MSK 6h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service +Wed 2026-07-22 08:40:13 MSK 17h Tue 2026-07-21 08:50:14 MSK 5h 59min ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service +Wed 2026-07-22 08:53:36 MSK 18h Tue 2026-07-21 08:41:14 MSK 6h ago memos-backup.timer memos-backup.service +Sun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service +Sun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service +Sun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service +Sun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service +Mon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 7h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service +=== BACKUP_UNITS === +dpkg-db-backup.service static - +filebrowser-backup.service static - +homelab-alertmanager-backup.service static - +homelab-appbackup-dockge-stacks.service static - +homelab-appbackup-mariadb-batch.service static - +homelab-appbackup-npmplus-edge.service static - +homelab-appbackup-postgres-batch.service static - +homelab-appbackup-sqlite-batch.service static - +homelab-authentik-backup.service static - +homelab-auto-backup.service static - +homelab-backup-coverage-matrix.service static - +homelab-backup-freshness.service static - +homelab-backup-healthcheck.service static - +homelab-backup-matrix-refresh.service static - +homelab-backup-sla-health.service static - +homelab-beszel-backup.service static - +homelab-blackbox-exporter-backup.service static - +homelab-cluster-admin-incident-engine-backup.service static - +homelab-crypto-portfolio-backup.service static - +homelab-edge-vm-vzdump-backup.service static - +homelab-emergency-backup.service static - +homelab-extended-appbackup.service static - +homelab-gitea-backup.service static - +homelab-gitea-secondary-backup.service static - +homelab-legacy-monitoring-backups.service static - +homelab-mail-cloud-critical-upload.service static - +homelab-mail-cloud-edge-vm-upload.service static - +homelab-mail-cloud-nextcloud-vm-upload.service static - +homelab-mail-cloud-restore-drill.service static - +homelab-npmplus-kuma-config-backup.service static - +homelab-router-running-config-mail-cloud.service static - +homelab-vaultwarden-backup.service static - +homelab-verified-backup-health.service static - +homelab-vm-backup-health.service static - +homelab-vm-backup-policy.service static - +homelab-vm-full-backup.service static - +homelab-vm-local-dumps-2cloud.service static - +homelab-vm160-remote-backup.service static - +immich-media-backup.service static - +memos-backup.service static - +netbird-vps-backup.service static - +paperless-backup.service static - +skladchik-reports-monitor-backup-restore-check.service static - +skladchik-reports-monitor-backup.service static - +dpkg-db-backup.timer enabled enabled +filebrowser-backup.timer enabled enabled +homelab-alertmanager-backup.timer enabled enabled +homelab-appbackup-dockge-stacks.timer enabled enabled +homelab-appbackup-mariadb-batch.timer enabled enabled +homelab-appbackup-npmplus-edge.timer enabled enabled +homelab-appbackup-postgres-batch.timer enabled enabled +homelab-appbackup-sqlite-batch.timer enabled enabled +homelab-authentik-backup.timer disabled enabled +homelab-auto-backup.timer enabled enabled +homelab-backup-coverage-matrix.timer enabled enabled +homelab-backup-freshness.timer enabled enabled +homelab-backup-healthcheck.timer enabled enabled +homelab-backup-matrix-refresh.timer disabled enabled +homelab-backup-sla-health.timer enabled enabled +homelab-beszel-backup.timer enabled enabled +homelab-blackbox-exporter-backup.timer enabled enabled +homelab-cluster-admin-incident-engine-backup.timer enabled enabled +homelab-crypto-portfolio-backup.timer enabled enabled +homelab-edge-vm-vzdump-backup.timer disabled enabled +homelab-emergency-backup.timer enabled enabled +homelab-extended-appbackup.timer enabled enabled +homelab-gitea-backup.timer disabled enabled +homelab-gitea-secondary-backup.timer enabled enabled +homelab-legacy-monitoring-backups.timer enabled enabled +homelab-mail-cloud-critical-upload.timer enabled enabled +homelab-mail-cloud-edge-vm-upload.timer enabled enabled +homelab-mail-cloud-nextcloud-vm-upload.timer enabled enabled +homelab-mail-cloud-restore-drill.timer enabled enabled +homelab-npmplus-kuma-config-backup.timer disabled enabled +homelab-router-running-config-mail-cloud.timer enabled enabled +homelab-vaultwarden-backup.timer disabled enabled +homelab-verified-backup-health.timer enabled enabled +homelab-vm-backup-health.timer enabled enabled +homelab-vm-backup-policy.timer enabled enabled +homelab-vm-full-backup.timer enabled enabled +homelab-vm-local-dumps-2cloud.timer enabled enabled +homelab-vm160-remote-backup.timer disabled enabled +immich-media-backup.timer disabled enabled +memos-backup.timer enabled enabled +netbird-vps-backup.timer disabled enabled +paperless-backup.timer disabled enabled +skladchik-reports-monitor-backup-restore-check.timer enabled enabled +skladchik-reports-monitor-backup.timer enabled enabled +=== BACKUP_PIPELINE_FILES === +=== LOCAL_BACKUP_ARCHIVES === +7022386770 2026-07-18T03:31:03.6566057240 /var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.vma.zst +7021952392 2026-07-21T03:31:06.5717414950 /var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst +7021297699 2026-07-19T03:31:06.8755351020 /var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst +7016275481 2026-07-20T03:31:03.2165123470 /var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst +329541843 2026-07-21T03:30:09.7688696330 /var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst +326601118 2026-07-20T03:30:08.2096782720 /var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst +323668231 2026-07-19T03:45:46.2209130400 /var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst +321121649 2026-07-18T03:30:08.0777625900 /var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.tar.zst +222777732 2026-07-21T03:30:15.9059638290 /var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst +222761273 2026-07-20T03:30:13.7937629440 /var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst +222745965 2026-07-19T03:45:57.2700790730 /var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst +222727760 2026-07-18T03:30:13.5578457230 /var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.tar.zst +3021 2026-07-19T03:31:07.4745284120 /var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.log +3020 2026-07-21T03:31:07.3945472590 /var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.log +3018 2026-07-18T03:31:03.9197988860 /var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.log +3017 2026-07-20T03:31:04.1706719350 /var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.log +1089 2026-07-20T03:30:14.0079985330 /var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.log +1086 2026-07-20T03:30:08.4640241540 /var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.log +989 2026-07-21T03:30:16.1364976560 /var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.log +989 2026-07-18T03:30:13.7816006690 /var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.log +986 2026-07-21T03:30:10.0285327490 /var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.log +986 2026-07-18T03:30:08.3002411830 /var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.log +731 2026-07-19T03:45:57.5197000270 /var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.log +729 2026-07-19T03:45:46.4444121980 /var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.log +========== NODE=pve02 ========== +pve02.gram1.ru +=== RCLONE_REMOTES === +pve01-mail-01: +pve01-mail-01-crypt: +pve01-mail-02: +pve01-mail-02-crypt: +pve01-mail-03: +pve01-mail-03-crypt: +pve01-mail-04: +pve01-mail-04-crypt: +pve01-mail-05: +pve01-mail-05-crypt: +pve02-mail-01: +pve02-mail-01-crypt: +pve02-mail-02: +pve02-mail-02-crypt: +pve02-mail-03: +pve02-mail-03-crypt: +pve02-mail-04: +pve02-mail-04-crypt: +pve03-mail-01: +pve03-mail-01-crypt: +pve03-mail-02: +pve03-mail-02-crypt: +pve03-mail-03: +pve03-mail-03-crypt: +pve03-mail-04: +pve03-mail-04-crypt: +NODE_REMOTE_COUNT=26 +REMOTE_HEALTH node=pve02 remote=pve01-mail-01: status=OK +=== BACKUP_TIMERS === +Tue 2026-07-21 18:41:03 MSK 3h 51min Tue 2026-07-21 12:41:22 MSK 2h 7min ago forum-prod-fresh5-cloud-backup.timer forum-prod-fresh5-cloud-backup.service +Wed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:22 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service +Wed 2026-07-22 04:20:00 MSK 13h Tue 2026-07-21 04:20:12 MSK 10h ago homelab-cluster-admin-vm-mail-cloud-backup.timer homelab-cluster-admin-vm-mail-cloud-backup.service +=== BACKUP_UNITS === +codevipe-cloud-backup.service static - +dpkg-db-backup.service static - +forum-prod-fresh5-cloud-backup.service static - +homelab-cluster-admin-vm-mail-cloud-backup.service static - +codevipe-cloud-backup.timer disabled enabled +dpkg-db-backup.timer enabled enabled +forum-prod-fresh5-cloud-backup.timer enabled enabled +homelab-cluster-admin-vm-mail-cloud-backup.timer enabled enabled +=== BACKUP_PIPELINE_FILES === +=== LOCAL_BACKUP_ARCHIVES === +997503422 2026-07-01T03:31:11.1700059310 /var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.vma.zst +868048369 2026-06-30T03:31:08.4903256780 /var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.vma.zst +288349157 2026-07-21T03:30:21.2480085730 /var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.tar.zst +286538526 2026-07-20T03:30:18.4386603270 /var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.tar.zst +284443608 2026-07-19T03:47:41.8525503200 /var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst +283022034 2026-07-18T03:30:19.7617214460 /var/lib/vz/dump/vzdump-lxc-111-2026_07_18-03_30_03.tar.zst +220080970 2026-07-21T03:30:33.3910787180 /var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.tar.zst +220065825 2026-07-20T03:30:30.8287264090 /var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.tar.zst +220049041 2026-07-19T03:47:58.9256407960 /var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst +220033534 2026-07-18T03:30:37.2878143100 /var/lib/vz/dump/vzdump-lxc-113-2026_07_18-03_30_20.tar.zst +7547 2026-07-18T01:09:39.2467331080 /var/lib/vz/dump/vzdump-qemu-160-2026_07_18-01_02_44.log +7443 2026-07-18T01:45:13.3149404960 /var/lib/vz/dump/vzdump-qemu-160-2026_07_18-01_38_20.log +5796 2026-07-17T21:55:21.5648458010 /var/lib/vz/dump/vzdump-qemu-160-2026_07_17-21_30_20.log +2685 2026-06-30T03:31:09.1616927060 /var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.log +2590 2026-07-01T03:31:11.9500372070 /var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.log +1088 2026-07-20T03:30:31.0784689220 /var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.log +1085 2026-07-20T03:30:18.7427551960 /var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.log +988 2026-07-21T03:30:33.6364088960 /var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.log +988 2026-07-18T03:30:37.5450736960 /var/lib/vz/dump/vzdump-lxc-113-2026_07_18-03_30_20.log +985 2026-07-21T03:30:21.5465818340 /var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.log +985 2026-07-18T03:30:20.0884248500 /var/lib/vz/dump/vzdump-lxc-111-2026_07_18-03_30_03.log +731 2026-07-19T03:47:59.2238139650 /var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.log +728 2026-07-19T03:47:42.1183953260 /var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.log +========== NODE=pve03 ========== +pve03.gram1.ru +=== RCLONE_REMOTES === +pve01-mail-01: +pve01-mail-01-crypt: +pve01-mail-02: +pve01-mail-02-crypt: +pve01-mail-03: +pve01-mail-03-crypt: +pve01-mail-04: +pve01-mail-04-crypt: +pve01-mail-05: +pve01-mail-05-crypt: +pve02-mail-01: +pve02-mail-01-crypt: +pve02-mail-02: +pve02-mail-02-crypt: +pve02-mail-03: +pve02-mail-03-crypt: +pve02-mail-04: +pve02-mail-04-crypt: +pve03-mail-01: +pve03-mail-01-crypt: +pve03-mail-02: +pve03-mail-02-crypt: +pve03-mail-03: +pve03-mail-03-crypt: +pve03-mail-04: +pve03-mail-04-crypt: +NODE_REMOTE_COUNT=26 +REMOTE_HEALTH node=pve03 remote=pve01-mail-01: status=OK +=== BACKUP_TIMERS === +Wed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:10 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service +=== BACKUP_UNITS === +dpkg-db-backup.service static - +dpkg-db-backup.timer enabled enabled +=== BACKUP_PIPELINE_FILES === +=== LOCAL_BACKUP_ARCHIVES === +11116472459 2026-07-20T03:33:53.2839064370 /var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.vma.zst +11043834894 2026-07-21T03:33:54.3725716230 /var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.vma.zst +11013723404 2026-07-19T03:33:53.3463878040 /var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.vma.zst +10994820121 2026-07-18T03:33:47.1540038630 /var/lib/vz/dump/vzdump-qemu-130-2026_07_18-03_30_02.vma.zst +1137072350 2026-07-21T03:34:37.6276811360 /var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.vma.zst +838951976 2026-07-19T03:34:33.1034859540 /var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.vma.zst +838817925 2026-07-20T03:34:37.1719936670 /var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.vma.zst +785249297 2026-07-18T03:34:27.1531047780 /var/lib/vz/dump/vzdump-qemu-190-2026_07_18-03_33_48.vma.zst +7106 2026-07-02T03:37:53.1040459570 /var/lib/vz/dump/vzdump-qemu-130-2026_07_02-03_30_02.log +7102 2026-07-03T03:36:49.1036784480 /var/lib/vz/dump/vzdump-qemu-130-2026_07_03-03_30_05.log +6564 2026-07-18T03:33:48.0107527290 /var/lib/vz/dump/vzdump-qemu-130-2026_07_18-03_30_02.log +6560 2026-07-20T03:33:54.9283349630 /var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.log +6559 2026-07-21T03:33:55.1428157340 /var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.log +6459 2026-07-19T03:33:53.7631656400 /var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.log +6246 2026-07-04T03:33:39.2197744030 /var/lib/vz/dump/vzdump-qemu-130-2026_07_04-03_30_02.log +4107 2026-07-07T03:31:51.2516816970 /var/lib/vz/dump/vzdump-qemu-130-2026_07_07-03_30_01.log +2642 2026-07-21T03:34:38.4491706750 /var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.log +2500 2026-07-20T03:34:37.2013439550 /var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.log +2404 2026-07-19T03:34:34.1266444600 /var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.log +2305 2026-07-18T03:34:27.7179455720 /var/lib/vz/dump/vzdump-qemu-190-2026_07_18-03_33_48.log +=== CLOUD_REMOTE_SUMMARY === +EXPECTED_TOTAL=13 +DETECTED_TOTAL=78 +REMOTE_COUNT=MISMATCH OUTPUT_END CHAT_OUTPUT_END