diff --git a/runtime/history/BACKUP-CLOUD-PIPELINE-MAP-063.json b/runtime/history/BACKUP-CLOUD-PIPELINE-MAP-063.json new file mode 100644 index 0000000..d072dfb --- /dev/null +++ b/runtime/history/BACKUP-CLOUD-PIPELINE-MAP-063.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "BACKUP-CLOUD-PIPELINE-MAP-063", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "backup-audit", + "started_at_utc": "2026-07-21T11:51:46Z", + "finished_at_utc": "2026-07-21T11:51:50Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1", + "command_sha256": "6354dd51dd01e331f845ee918c861a5da7d433a7a7851f62a8a1b779b72e6921", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "b0525c6bddcc86a5d8b0f181e65bb0979a7f71eb1258d738d7f4cea89c3aedbe", + "sanitized_output_sha256": "b0525c6bddcc86a5d8b0f181e65bb0979a7f71eb1258d738d7f4cea89c3aedbe", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "===== NODE=pve01 =====\nREMOTE_COUNT=26\nREMOTE node=pve01 name=pve01-mail-01: health=OK references=UNREFERENCED\n===== NODE=pve02 =====\nREMOTE_COUNT=26\nREMOTE node=pve02 name=pve01-mail-01: health=OK references=UNREFERENCED\n===== NODE=pve03 =====\nREMOTE_COUNT=26\nREMOTE node=pve03 name=pve01-mail-01: health=OK references=UNREFERENCED\n===== SUMMARY =====\nEXPECTED_CLUSTER_REMOTES=13\nDETECTED_CLUSTER_REMOTES=78\nREMOTE_COUNT_STATUS=MISMATCH\nCLOUD_PIPELINE_MAP_COMPLETE=YES\n" +} diff --git a/runtime/history/BACKUP-CLOUD-PIPELINE-MAP-063.txt b/runtime/history/BACKUP-CLOUD-PIPELINE-MAP-063.txt new file mode 100644 index 0000000..b981bd8 --- /dev/null +++ b/runtime/history/BACKUP-CLOUD-PIPELINE-MAP-063.txt @@ -0,0 +1,38 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=BACKUP-CLOUD-PIPELINE-MAP-063 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=backup-audit +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1 +COMMAND_SHA256=6354dd51dd01e331f845ee918c861a5da7d433a7a7851f62a8a1b779b72e6921 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=false +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=b0525c6bddcc86a5d8b0f181e65bb0979a7f71eb1258d738d7f4cea89c3aedbe +SANITIZED_OUTPUT_SHA256=b0525c6bddcc86a5d8b0f181e65bb0979a7f71eb1258d738d7f4cea89c3aedbe +OUTPUT_BEGIN +===== NODE=pve01 ===== +REMOTE_COUNT=26 +REMOTE node=pve01 name=pve01-mail-01: health=OK references=UNREFERENCED +===== NODE=pve02 ===== +REMOTE_COUNT=26 +REMOTE node=pve02 name=pve01-mail-01: health=OK references=UNREFERENCED +===== NODE=pve03 ===== +REMOTE_COUNT=26 +REMOTE node=pve03 name=pve01-mail-01: health=OK references=UNREFERENCED +===== SUMMARY ===== +EXPECTED_CLUSTER_REMOTES=13 +DETECTED_CLUSTER_REMOTES=78 +REMOTE_COUNT_STATUS=MISMATCH +CLOUD_PIPELINE_MAP_COMPLETE=YES + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 0c80d5e..d072dfb 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "BACKUP-CLUSTER-13CLOUD-TOPOLOGY-062", + "command_id": "BACKUP-CLOUD-PIPELINE-MAP-063", "status": "OK", "rc": 0, "host": "pve01", "mode": "read-only", "component": "backup-audit", - "started_at_utc": "2026-07-21T11:49:15Z", - "finished_at_utc": "2026-07-21T11:49:26Z", + "started_at_utc": "2026-07-21T11:51:46Z", + "finished_at_utc": "2026-07-21T11:51:50Z", "reference_register_checked": true, "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", "error_register_checked": true, "error_register_sha256": "f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1", - "command_sha256": "eca35846e7097b9c238bf8c082e02c81f0b2893473e27ddb0a5b0854d5d3703d", + "command_sha256": "6354dd51dd01e331f845ee918c861a5da7d433a7a7851f62a8a1b779b72e6921", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -22,9 +22,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "3ba8aaf65323f1801f1eb63016759f4767a9d5be8bb21f15d92042e408395afd", - "sanitized_output_sha256": "3ba8aaf65323f1801f1eb63016759f4767a9d5be8bb21f15d92042e408395afd", + "raw_evidence_sha256": "b0525c6bddcc86a5d8b0f181e65bb0979a7f71eb1258d738d7f4cea89c3aedbe", + "sanitized_output_sha256": "b0525c6bddcc86a5d8b0f181e65bb0979a7f71eb1258d738d7f4cea89c3aedbe", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "=== CLUSTER_BACKUP_JOBS ===\n[{\"compress\":\"zstd\",\"enabled\":1,\"id\":\"homelab-nightly-all\",\"mailnotification\":\"failure\",\"mode\":\"snapshot\",\"next-run\":1784680200,\"prune-backups\":{\"keep-daily\":\"4\"},\"schedule\":\"03:30\",\"storage\":\"local\",\"type\":\"vzdump\",\"vmid\":\"110,111,112,113,130,150,190\"}]\n=== CLUSTER_STORAGES ===\nName Type Status Total (KiB) Used (KiB) Available (KiB) %\nlocal dir active 102626232 43175760 54191208 42.07%\nlocal-lvm lvmthin active 848797696 280188119 568609576 33.01%\n========== NODE=pve01 ==========\npve01.gram1.ru\n=== RCLONE_REMOTES ===\npve01-mail-01:\npve01-mail-01-crypt:\npve01-mail-02:\npve01-mail-02-crypt:\npve01-mail-03:\npve01-mail-03-crypt:\npve01-mail-04:\npve01-mail-04-crypt:\npve01-mail-05:\npve01-mail-05-crypt:\npve02-mail-01:\npve02-mail-01-crypt:\npve02-mail-02:\npve02-mail-02-crypt:\npve02-mail-03:\npve02-mail-03-crypt:\npve02-mail-04:\npve02-mail-04-crypt:\npve03-mail-01:\npve03-mail-01-crypt:\npve03-mail-02:\npve03-mail-02-crypt:\npve03-mail-03:\npve03-mail-03-crypt:\npve03-mail-04:\npve03-mail-04-crypt:\nNODE_REMOTE_COUNT=26\nREMOTE_HEALTH node=pve01 remote=pve01-mail-01: status=OK\n=== BACKUP_TIMERS ===\nWed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:14 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service\nWed 2026-07-22 00:09:43 MSK 9h Tue 2026-07-21 00:09:32 MSK 14h ago homelab-backup-freshness.timer homelab-backup-freshness.service\nWed 2026-07-22 02:15:00 MSK 11h Tue 2026-07-21 02:15:05 MSK 12h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service\nWed 2026-07-22 03:10:02 MSK 12h Tue 2026-07-21 03:13:14 MSK 11h ago homelab-emergency-backup.timer homelab-emergency-backup.service\nWed 2026-07-22 03:35:00 MSK 12h Tue 2026-07-21 03:35:14 MSK 11h ago filebrowser-backup.timer filebrowser-backup.service\nWed 2026-07-22 03:55:02 MSK 13h Tue 2026-07-21 03:51:25 MSK 10h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service\nWed 2026-07-22 04:17:29 MSK 13h Tue 2026-07-21 04:19:17 MSK 10h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service\nWed 2026-07-22 04:26:54 MSK 13h Tue 2026-07-21 04:28:14 MSK 10h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service\nWed 2026-07-22 04:38:04 MSK 13h Tue 2026-07-21 04:34:48 MSK 10h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service\nWed 2026-07-22 05:01:44 MSK 14h Tue 2026-07-21 04:57:14 MSK 9h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service\nWed 2026-07-22 05:23:56 MSK 14h Tue 2026-07-21 05:17:57 MSK 9h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service\nWed 2026-07-22 05:27:13 MSK 14h Tue 2026-07-21 05:29:05 MSK 9h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service\nWed 2026-07-22 05:47:19 MSK 14h Tue 2026-07-21 05:46:14 MSK 9h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service\nWed 2026-07-22 06:01:19 MSK 15h Tue 2026-07-21 05:57:30 MSK 8h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service\nWed 2026-07-22 06:27:43 MSK 15h Tue 2026-07-21 06:32:25 MSK 8h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service\nWed 2026-07-22 06:38:51 MSK 15h Tue 2026-07-21 06:43:54 MSK 8h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service\nWed 2026-07-22 06:39:22 MSK 15h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service\nWed 2026-07-22 06:44:16 MSK 15h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service\nWed 2026-07-22 06:52:06 MSK 16h Tue 2026-07-21 06:47:47 MSK 8h ago homelab-beszel-backup.timer homelab-beszel-backup.service\nWed 2026-07-22 06:58:58 MSK 16h Tue 2026-07-21 06:57:30 MSK 7h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service\nWed 2026-07-22 07:02:27 MSK 16h Tue 2026-07-21 07:05:30 MSK 7h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service\nWed 2026-07-22 07:07:14 MSK 16h Tue 2026-07-21 07:18:55 MSK 7h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service\nWed 2026-07-22 07:35:06 MSK 16h Tue 2026-07-21 07:36:59 MSK 7h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service\nWed 2026-07-22 07:38:21 MSK 16h Tue 2026-07-21 07:36:14 MSK 7h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service\nWed 2026-07-22 07:41:49 MSK 16h Tue 2026-07-21 07:53:41 MSK 6h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service\nWed 2026-07-22 08:10:37 MSK 17h Tue 2026-07-21 08:07:25 MSK 6h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service\nWed 2026-07-22 08:37:02 MSK 17h Tue 2026-07-21 08:30:07 MSK 6h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service\nWed 2026-07-22 08:40:13 MSK 17h Tue 2026-07-21 08:50:14 MSK 5h 59min ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service\nWed 2026-07-22 08:53:36 MSK 18h Tue 2026-07-21 08:41:14 MSK 6h ago memos-backup.timer memos-backup.service\nSun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service\nSun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service\nSun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service\nSun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service\nMon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 7h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service\n=== BACKUP_UNITS ===\ndpkg-db-backup.service static -\nfilebrowser-backup.service static -\nhomelab-alertmanager-backup.service static -\nhomelab-appbackup-dockge-stacks.service static -\nhomelab-appbackup-mariadb-batch.service static -\nhomelab-appbackup-npmplus-edge.service static -\nhomelab-appbackup-postgres-batch.service static -\nhomelab-appbackup-sqlite-batch.service static -\nhomelab-authentik-backup.service static -\nhomelab-auto-backup.service static -\nhomelab-backup-coverage-matrix.service static -\nhomelab-backup-freshness.service static -\nhomelab-backup-healthcheck.service static -\nhomelab-backup-matrix-refresh.service static -\nhomelab-backup-sla-health.service static -\nhomelab-beszel-backup.service static -\nhomelab-blackbox-exporter-backup.service static -\nhomelab-cluster-admin-incident-engine-backup.service static -\nhomelab-crypto-portfolio-backup.service static -\nhomelab-edge-vm-vzdump-backup.service static -\nhomelab-emergency-backup.service static -\nhomelab-extended-appbackup.service static -\nhomelab-gitea-backup.service static -\nhomelab-gitea-secondary-backup.service static -\nhomelab-legacy-monitoring-backups.service static -\nhomelab-mail-cloud-critical-upload.service static -\nhomelab-mail-cloud-edge-vm-upload.service static -\nhomelab-mail-cloud-nextcloud-vm-upload.service static -\nhomelab-mail-cloud-restore-drill.service static -\nhomelab-npmplus-kuma-config-backup.service static -\nhomelab-router-running-config-mail-cloud.service static -\nhomelab-vaultwarden-backup.service static -\nhomelab-verified-backup-health.service static -\nhomelab-vm-backup-health.service static -\nhomelab-vm-backup-policy.service static -\nhomelab-vm-full-backup.service static -\nhomelab-vm-local-dumps-2cloud.service static -\nhomelab-vm160-remote-backup.service static -\nimmich-media-backup.service static -\nmemos-backup.service static -\nnetbird-vps-backup.service static -\npaperless-backup.service static -\nskladchik-reports-monitor-backup-restore-check.service static -\nskladchik-reports-monitor-backup.service static -\ndpkg-db-backup.timer enabled enabled\nfilebrowser-backup.timer enabled enabled\nhomelab-alertmanager-backup.timer enabled enabled\nhomelab-appbackup-dockge-stacks.timer enabled enabled\nhomelab-appbackup-mariadb-batch.timer enabled enabled\nhomelab-appbackup-npmplus-edge.timer enabled enabled\nhomelab-appbackup-postgres-batch.timer enabled enabled\nhomelab-appbackup-sqlite-batch.timer enabled enabled\nhomelab-authentik-backup.timer disabled enabled\nhomelab-auto-backup.timer enabled enabled\nhomelab-backup-coverage-matrix.timer enabled enabled\nhomelab-backup-freshness.timer enabled enabled\nhomelab-backup-healthcheck.timer enabled enabled\nhomelab-backup-matrix-refresh.timer disabled enabled\nhomelab-backup-sla-health.timer enabled enabled\nhomelab-beszel-backup.timer enabled enabled\nhomelab-blackbox-exporter-backup.timer enabled enabled\nhomelab-cluster-admin-incident-engine-backup.timer enabled enabled\nhomelab-crypto-portfolio-backup.timer enabled enabled\nhomelab-edge-vm-vzdump-backup.timer disabled enabled\nhomelab-emergency-backup.timer enabled enabled\nhomelab-extended-appbackup.timer enabled enabled\nhomelab-gitea-backup.timer disabled enabled\nhomelab-gitea-secondary-backup.timer enabled enabled\nhomelab-legacy-monitoring-backups.timer enabled enabled\nhomelab-mail-cloud-critical-upload.timer enabled enabled\nhomelab-mail-cloud-edge-vm-upload.timer enabled enabled\nhomelab-mail-cloud-nextcloud-vm-upload.timer enabled enabled\nhomelab-mail-cloud-restore-drill.timer enabled enabled\nhomelab-npmplus-kuma-config-backup.timer disabled enabled\nhomelab-router-running-config-mail-cloud.timer enabled enabled\nhomelab-vaultwarden-backup.timer disabled enabled\nhomelab-verified-backup-health.timer enabled enabled\nhomelab-vm-backup-health.timer enabled enabled\nhomelab-vm-backup-policy.timer enabled enabled\nhomelab-vm-full-backup.timer enabled enabled\nhomelab-vm-local-dumps-2cloud.timer enabled enabled\nhomelab-vm160-remote-backup.timer disabled enabled\nimmich-media-backup.timer disabled enabled\nmemos-backup.timer enabled enabled\nnetbird-vps-backup.timer disabled enabled\npaperless-backup.timer disabled enabled\nskladchik-reports-monitor-backup-restore-check.timer enabled enabled\nskladchik-reports-monitor-backup.timer enabled enabled\n=== BACKUP_PIPELINE_FILES ===\n=== LOCAL_BACKUP_ARCHIVES ===\n7022386770\t2026-07-18T03:31:03.6566057240\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.vma.zst\n7021952392\t2026-07-21T03:31:06.5717414950\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst\n7021297699\t2026-07-19T03:31:06.8755351020\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst\n7016275481\t2026-07-20T03:31:03.2165123470\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst\n329541843\t2026-07-21T03:30:09.7688696330\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst\n326601118\t2026-07-20T03:30:08.2096782720\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst\n323668231\t2026-07-19T03:45:46.2209130400\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst\n321121649\t2026-07-18T03:30:08.0777625900\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.tar.zst\n222777732\t2026-07-21T03:30:15.9059638290\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst\n222761273\t2026-07-20T03:30:13.7937629440\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst\n222745965\t2026-07-19T03:45:57.2700790730\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst\n222727760\t2026-07-18T03:30:13.5578457230\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.tar.zst\n3021\t2026-07-19T03:31:07.4745284120\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.log\n3020\t2026-07-21T03:31:07.3945472590\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.log\n3018\t2026-07-18T03:31:03.9197988860\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.log\n3017\t2026-07-20T03:31:04.1706719350\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.log\n1089\t2026-07-20T03:30:14.0079985330\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.log\n1086\t2026-07-20T03:30:08.4640241540\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.log\n989\t2026-07-21T03:30:16.1364976560\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.log\n989\t2026-07-18T03:30:13.7816006690\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.log\n986\t2026-07-21T03:30:10.0285327490\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.log\n986\t2026-07-18T03:30:08.3002411830\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.log\n731\t2026-07-19T03:45:57.5197000270\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.log\n729\t2026-07-19T03:45:46.4444121980\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.log\n========== NODE=pve02 ==========\npve02.gram1.ru\n=== RCLONE_REMOTES ===\npve01-mail-01:\npve01-mail-01-crypt:\npve01-mail-02:\npve01-mail-02-crypt:\npve01-mail-03:\npve01-mail-03-crypt:\npve01-mail-04:\npve01-mail-04-crypt:\npve01-mail-05:\npve01-mail-05-crypt:\npve02-mail-01:\npve02-mail-01-crypt:\npve02-mail-02:\npve02-mail-02-crypt:\npve02-mail-03:\npve02-mail-03-crypt:\npve02-mail-04:\npve02-mail-04-crypt:\npve03-mail-01:\npve03-mail-01-crypt:\npve03-mail-02:\npve03-mail-02-crypt:\npve03-mail-03:\npve03-mail-03-crypt:\npve03-mail-04:\npve03-mail-04-crypt:\nNODE_REMOTE_COUNT=26\nREMOTE_HEALTH node=pve02 remote=pve01-mail-01: status=OK\n=== BACKUP_TIMERS ===\nTue 2026-07-21 18:41:03 MSK 3h 51min Tue 2026-07-21 12:41:22 MSK 2h 7min ago forum-prod-fresh5-cloud-backup.timer forum-prod-fresh5-cloud-backup.service\nWed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:22 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service\nWed 2026-07-22 04:20:00 MSK 13h Tue 2026-07-21 04:20:12 MSK 10h ago homelab-cluster-admin-vm-mail-cloud-backup.timer homelab-cluster-admin-vm-mail-cloud-backup.service\n=== BACKUP_UNITS ===\ncodevipe-cloud-backup.service static -\ndpkg-db-backup.service static -\nforum-prod-fresh5-cloud-backup.service static -\nhomelab-cluster-admin-vm-mail-cloud-backup.service static -\ncodevipe-cloud-backup.timer disabled enabled\ndpkg-db-backup.timer enabled enabled\nforum-prod-fresh5-cloud-backup.timer enabled enabled\nhomelab-cluster-admin-vm-mail-cloud-backup.timer enabled enabled\n=== BACKUP_PIPELINE_FILES ===\n=== LOCAL_BACKUP_ARCHIVES ===\n997503422\t2026-07-01T03:31:11.1700059310\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.vma.zst\n868048369\t2026-06-30T03:31:08.4903256780\t/var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.vma.zst\n288349157\t2026-07-21T03:30:21.2480085730\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.tar.zst\n286538526\t2026-07-20T03:30:18.4386603270\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.tar.zst\n284443608\t2026-07-19T03:47:41.8525503200\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst\n283022034\t2026-07-18T03:30:19.7617214460\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_18-03_30_03.tar.zst\n220080970\t2026-07-21T03:30:33.3910787180\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.tar.zst\n220065825\t2026-07-20T03:30:30.8287264090\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.tar.zst\n220049041\t2026-07-19T03:47:58.9256407960\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst\n220033534\t2026-07-18T03:30:37.2878143100\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_18-03_30_20.tar.zst\n7547\t2026-07-18T01:09:39.2467331080\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_18-01_02_44.log\n7443\t2026-07-18T01:45:13.3149404960\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_18-01_38_20.log\n5796\t2026-07-17T21:55:21.5648458010\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_17-21_30_20.log\n2685\t2026-06-30T03:31:09.1616927060\t/var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.log\n2590\t2026-07-01T03:31:11.9500372070\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.log\n1088\t2026-07-20T03:30:31.0784689220\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.log\n1085\t2026-07-20T03:30:18.7427551960\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.log\n988\t2026-07-21T03:30:33.6364088960\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.log\n988\t2026-07-18T03:30:37.5450736960\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_18-03_30_20.log\n985\t2026-07-21T03:30:21.5465818340\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.log\n985\t2026-07-18T03:30:20.0884248500\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_18-03_30_03.log\n731\t2026-07-19T03:47:59.2238139650\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.log\n728\t2026-07-19T03:47:42.1183953260\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.log\n========== NODE=pve03 ==========\npve03.gram1.ru\n=== RCLONE_REMOTES ===\npve01-mail-01:\npve01-mail-01-crypt:\npve01-mail-02:\npve01-mail-02-crypt:\npve01-mail-03:\npve01-mail-03-crypt:\npve01-mail-04:\npve01-mail-04-crypt:\npve01-mail-05:\npve01-mail-05-crypt:\npve02-mail-01:\npve02-mail-01-crypt:\npve02-mail-02:\npve02-mail-02-crypt:\npve02-mail-03:\npve02-mail-03-crypt:\npve02-mail-04:\npve02-mail-04-crypt:\npve03-mail-01:\npve03-mail-01-crypt:\npve03-mail-02:\npve03-mail-02-crypt:\npve03-mail-03:\npve03-mail-03-crypt:\npve03-mail-04:\npve03-mail-04-crypt:\nNODE_REMOTE_COUNT=26\nREMOTE_HEALTH node=pve03 remote=pve01-mail-01: status=OK\n=== BACKUP_TIMERS ===\nWed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:10 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service\n=== BACKUP_UNITS ===\ndpkg-db-backup.service static -\ndpkg-db-backup.timer enabled enabled\n=== BACKUP_PIPELINE_FILES ===\n=== LOCAL_BACKUP_ARCHIVES ===\n11116472459\t2026-07-20T03:33:53.2839064370\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.vma.zst\n11043834894\t2026-07-21T03:33:54.3725716230\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.vma.zst\n11013723404\t2026-07-19T03:33:53.3463878040\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.vma.zst\n10994820121\t2026-07-18T03:33:47.1540038630\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_18-03_30_02.vma.zst\n1137072350\t2026-07-21T03:34:37.6276811360\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.vma.zst\n838951976\t2026-07-19T03:34:33.1034859540\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.vma.zst\n838817925\t2026-07-20T03:34:37.1719936670\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.vma.zst\n785249297\t2026-07-18T03:34:27.1531047780\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_18-03_33_48.vma.zst\n7106\t2026-07-02T03:37:53.1040459570\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_02-03_30_02.log\n7102\t2026-07-03T03:36:49.1036784480\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_03-03_30_05.log\n6564\t2026-07-18T03:33:48.0107527290\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_18-03_30_02.log\n6560\t2026-07-20T03:33:54.9283349630\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.log\n6559\t2026-07-21T03:33:55.1428157340\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.log\n6459\t2026-07-19T03:33:53.7631656400\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.log\n6246\t2026-07-04T03:33:39.2197744030\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_04-03_30_02.log\n4107\t2026-07-07T03:31:51.2516816970\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_07-03_30_01.log\n2642\t2026-07-21T03:34:38.4491706750\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.log\n2500\t2026-07-20T03:34:37.2013439550\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.log\n2404\t2026-07-19T03:34:34.1266444600\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.log\n2305\t2026-07-18T03:34:27.7179455720\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_18-03_33_48.log\n=== CLOUD_REMOTE_SUMMARY ===\nEXPECTED_TOTAL=13\nDETECTED_TOTAL=78\nREMOTE_COUNT=MISMATCH\n" + "output": "===== NODE=pve01 =====\nREMOTE_COUNT=26\nREMOTE node=pve01 name=pve01-mail-01: health=OK references=UNREFERENCED\n===== NODE=pve02 =====\nREMOTE_COUNT=26\nREMOTE node=pve02 name=pve01-mail-01: health=OK references=UNREFERENCED\n===== NODE=pve03 =====\nREMOTE_COUNT=26\nREMOTE node=pve03 name=pve01-mail-01: health=OK references=UNREFERENCED\n===== SUMMARY =====\nEXPECTED_CLUSTER_REMOTES=13\nDETECTED_CLUSTER_REMOTES=78\nREMOTE_COUNT_STATUS=MISMATCH\nCLOUD_PIPELINE_MAP_COMPLETE=YES\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index a81d89e..b981bd8 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,5 +1,5 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=BACKUP-CLUSTER-13CLOUD-TOPOLOGY-062 +COMMAND_ID=BACKUP-CLOUD-PIPELINE-MAP-063 STATUS=OK RC=0 HOST=pve01 @@ -9,334 +9,30 @@ REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1 -COMMAND_SHA256=eca35846e7097b9c238bf8c082e02c81f0b2893473e27ddb0a5b0854d5d3703d +COMMAND_SHA256=6354dd51dd01e331f845ee918c861a5da7d433a7a7851f62a8a1b779b72e6921 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGES_MADE=false SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=3ba8aaf65323f1801f1eb63016759f4767a9d5be8bb21f15d92042e408395afd -SANITIZED_OUTPUT_SHA256=3ba8aaf65323f1801f1eb63016759f4767a9d5be8bb21f15d92042e408395afd +RAW_EVIDENCE_SHA256=b0525c6bddcc86a5d8b0f181e65bb0979a7f71eb1258d738d7f4cea89c3aedbe +SANITIZED_OUTPUT_SHA256=b0525c6bddcc86a5d8b0f181e65bb0979a7f71eb1258d738d7f4cea89c3aedbe OUTPUT_BEGIN -=== CLUSTER_BACKUP_JOBS === -[{"compress":"zstd","enabled":1,"id":"homelab-nightly-all","mailnotification":"failure","mode":"snapshot","next-run":1784680200,"prune-backups":{"keep-daily":"4"},"schedule":"03:30","storage":"local","type":"vzdump","vmid":"110,111,112,113,130,150,190"}] -=== CLUSTER_STORAGES === -Name Type Status Total (KiB) Used (KiB) Available (KiB) % -local dir active 102626232 43175760 54191208 42.07% -local-lvm lvmthin active 848797696 280188119 568609576 33.01% -========== NODE=pve01 ========== -pve01.gram1.ru -=== RCLONE_REMOTES === -pve01-mail-01: -pve01-mail-01-crypt: -pve01-mail-02: -pve01-mail-02-crypt: -pve01-mail-03: -pve01-mail-03-crypt: -pve01-mail-04: -pve01-mail-04-crypt: -pve01-mail-05: -pve01-mail-05-crypt: -pve02-mail-01: -pve02-mail-01-crypt: -pve02-mail-02: -pve02-mail-02-crypt: -pve02-mail-03: -pve02-mail-03-crypt: -pve02-mail-04: -pve02-mail-04-crypt: -pve03-mail-01: -pve03-mail-01-crypt: -pve03-mail-02: -pve03-mail-02-crypt: -pve03-mail-03: -pve03-mail-03-crypt: -pve03-mail-04: -pve03-mail-04-crypt: -NODE_REMOTE_COUNT=26 -REMOTE_HEALTH node=pve01 remote=pve01-mail-01: status=OK -=== BACKUP_TIMERS === -Wed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:14 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service -Wed 2026-07-22 00:09:43 MSK 9h Tue 2026-07-21 00:09:32 MSK 14h ago homelab-backup-freshness.timer homelab-backup-freshness.service -Wed 2026-07-22 02:15:00 MSK 11h Tue 2026-07-21 02:15:05 MSK 12h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service -Wed 2026-07-22 03:10:02 MSK 12h Tue 2026-07-21 03:13:14 MSK 11h ago homelab-emergency-backup.timer homelab-emergency-backup.service -Wed 2026-07-22 03:35:00 MSK 12h Tue 2026-07-21 03:35:14 MSK 11h ago filebrowser-backup.timer filebrowser-backup.service -Wed 2026-07-22 03:55:02 MSK 13h Tue 2026-07-21 03:51:25 MSK 10h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service -Wed 2026-07-22 04:17:29 MSK 13h Tue 2026-07-21 04:19:17 MSK 10h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service -Wed 2026-07-22 04:26:54 MSK 13h Tue 2026-07-21 04:28:14 MSK 10h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service -Wed 2026-07-22 04:38:04 MSK 13h Tue 2026-07-21 04:34:48 MSK 10h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service -Wed 2026-07-22 05:01:44 MSK 14h Tue 2026-07-21 04:57:14 MSK 9h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service -Wed 2026-07-22 05:23:56 MSK 14h Tue 2026-07-21 05:17:57 MSK 9h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service -Wed 2026-07-22 05:27:13 MSK 14h Tue 2026-07-21 05:29:05 MSK 9h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service -Wed 2026-07-22 05:47:19 MSK 14h Tue 2026-07-21 05:46:14 MSK 9h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service -Wed 2026-07-22 06:01:19 MSK 15h Tue 2026-07-21 05:57:30 MSK 8h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service -Wed 2026-07-22 06:27:43 MSK 15h Tue 2026-07-21 06:32:25 MSK 8h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service -Wed 2026-07-22 06:38:51 MSK 15h Tue 2026-07-21 06:43:54 MSK 8h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service -Wed 2026-07-22 06:39:22 MSK 15h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service -Wed 2026-07-22 06:44:16 MSK 15h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service -Wed 2026-07-22 06:52:06 MSK 16h Tue 2026-07-21 06:47:47 MSK 8h ago homelab-beszel-backup.timer homelab-beszel-backup.service -Wed 2026-07-22 06:58:58 MSK 16h Tue 2026-07-21 06:57:30 MSK 7h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service -Wed 2026-07-22 07:02:27 MSK 16h Tue 2026-07-21 07:05:30 MSK 7h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service -Wed 2026-07-22 07:07:14 MSK 16h Tue 2026-07-21 07:18:55 MSK 7h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service -Wed 2026-07-22 07:35:06 MSK 16h Tue 2026-07-21 07:36:59 MSK 7h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service -Wed 2026-07-22 07:38:21 MSK 16h Tue 2026-07-21 07:36:14 MSK 7h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service -Wed 2026-07-22 07:41:49 MSK 16h Tue 2026-07-21 07:53:41 MSK 6h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service -Wed 2026-07-22 08:10:37 MSK 17h Tue 2026-07-21 08:07:25 MSK 6h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service -Wed 2026-07-22 08:37:02 MSK 17h Tue 2026-07-21 08:30:07 MSK 6h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service -Wed 2026-07-22 08:40:13 MSK 17h Tue 2026-07-21 08:50:14 MSK 5h 59min ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service -Wed 2026-07-22 08:53:36 MSK 18h Tue 2026-07-21 08:41:14 MSK 6h ago memos-backup.timer memos-backup.service -Sun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service -Sun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service -Sun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service -Sun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service -Mon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 7h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service -=== BACKUP_UNITS === -dpkg-db-backup.service static - -filebrowser-backup.service static - -homelab-alertmanager-backup.service static - -homelab-appbackup-dockge-stacks.service static - -homelab-appbackup-mariadb-batch.service static - -homelab-appbackup-npmplus-edge.service static - -homelab-appbackup-postgres-batch.service static - -homelab-appbackup-sqlite-batch.service static - -homelab-authentik-backup.service static - -homelab-auto-backup.service static - -homelab-backup-coverage-matrix.service static - -homelab-backup-freshness.service static - -homelab-backup-healthcheck.service static - -homelab-backup-matrix-refresh.service static - -homelab-backup-sla-health.service static - -homelab-beszel-backup.service static - -homelab-blackbox-exporter-backup.service static - -homelab-cluster-admin-incident-engine-backup.service static - -homelab-crypto-portfolio-backup.service static - -homelab-edge-vm-vzdump-backup.service static - -homelab-emergency-backup.service static - -homelab-extended-appbackup.service static - -homelab-gitea-backup.service static - -homelab-gitea-secondary-backup.service static - -homelab-legacy-monitoring-backups.service static - -homelab-mail-cloud-critical-upload.service static - -homelab-mail-cloud-edge-vm-upload.service static - -homelab-mail-cloud-nextcloud-vm-upload.service static - -homelab-mail-cloud-restore-drill.service static - -homelab-npmplus-kuma-config-backup.service static - -homelab-router-running-config-mail-cloud.service static - -homelab-vaultwarden-backup.service static - -homelab-verified-backup-health.service static - -homelab-vm-backup-health.service static - -homelab-vm-backup-policy.service static - -homelab-vm-full-backup.service static - -homelab-vm-local-dumps-2cloud.service static - -homelab-vm160-remote-backup.service static - -immich-media-backup.service static - -memos-backup.service static - -netbird-vps-backup.service static - -paperless-backup.service static - -skladchik-reports-monitor-backup-restore-check.service static - -skladchik-reports-monitor-backup.service static - -dpkg-db-backup.timer enabled enabled -filebrowser-backup.timer enabled enabled -homelab-alertmanager-backup.timer enabled enabled -homelab-appbackup-dockge-stacks.timer enabled enabled -homelab-appbackup-mariadb-batch.timer enabled enabled -homelab-appbackup-npmplus-edge.timer enabled enabled -homelab-appbackup-postgres-batch.timer enabled enabled -homelab-appbackup-sqlite-batch.timer enabled enabled -homelab-authentik-backup.timer disabled enabled -homelab-auto-backup.timer enabled enabled -homelab-backup-coverage-matrix.timer enabled enabled -homelab-backup-freshness.timer enabled enabled -homelab-backup-healthcheck.timer enabled enabled -homelab-backup-matrix-refresh.timer disabled enabled -homelab-backup-sla-health.timer enabled enabled -homelab-beszel-backup.timer enabled enabled -homelab-blackbox-exporter-backup.timer enabled enabled -homelab-cluster-admin-incident-engine-backup.timer enabled enabled -homelab-crypto-portfolio-backup.timer enabled enabled -homelab-edge-vm-vzdump-backup.timer disabled enabled -homelab-emergency-backup.timer enabled enabled -homelab-extended-appbackup.timer enabled enabled -homelab-gitea-backup.timer disabled enabled -homelab-gitea-secondary-backup.timer enabled enabled -homelab-legacy-monitoring-backups.timer enabled enabled -homelab-mail-cloud-critical-upload.timer enabled enabled -homelab-mail-cloud-edge-vm-upload.timer enabled enabled -homelab-mail-cloud-nextcloud-vm-upload.timer enabled enabled -homelab-mail-cloud-restore-drill.timer enabled enabled -homelab-npmplus-kuma-config-backup.timer disabled enabled -homelab-router-running-config-mail-cloud.timer enabled enabled -homelab-vaultwarden-backup.timer disabled enabled -homelab-verified-backup-health.timer enabled enabled -homelab-vm-backup-health.timer enabled enabled -homelab-vm-backup-policy.timer enabled enabled -homelab-vm-full-backup.timer enabled enabled -homelab-vm-local-dumps-2cloud.timer enabled enabled -homelab-vm160-remote-backup.timer disabled enabled -immich-media-backup.timer disabled enabled -memos-backup.timer enabled enabled -netbird-vps-backup.timer disabled enabled -paperless-backup.timer disabled enabled -skladchik-reports-monitor-backup-restore-check.timer enabled enabled -skladchik-reports-monitor-backup.timer enabled enabled -=== BACKUP_PIPELINE_FILES === -=== LOCAL_BACKUP_ARCHIVES === -7022386770 2026-07-18T03:31:03.6566057240 /var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.vma.zst -7021952392 2026-07-21T03:31:06.5717414950 /var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst -7021297699 2026-07-19T03:31:06.8755351020 /var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst -7016275481 2026-07-20T03:31:03.2165123470 /var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst -329541843 2026-07-21T03:30:09.7688696330 /var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst -326601118 2026-07-20T03:30:08.2096782720 /var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst -323668231 2026-07-19T03:45:46.2209130400 /var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst -321121649 2026-07-18T03:30:08.0777625900 /var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.tar.zst -222777732 2026-07-21T03:30:15.9059638290 /var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst -222761273 2026-07-20T03:30:13.7937629440 /var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst -222745965 2026-07-19T03:45:57.2700790730 /var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst -222727760 2026-07-18T03:30:13.5578457230 /var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.tar.zst -3021 2026-07-19T03:31:07.4745284120 /var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.log -3020 2026-07-21T03:31:07.3945472590 /var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.log -3018 2026-07-18T03:31:03.9197988860 /var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.log -3017 2026-07-20T03:31:04.1706719350 /var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.log -1089 2026-07-20T03:30:14.0079985330 /var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.log -1086 2026-07-20T03:30:08.4640241540 /var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.log -989 2026-07-21T03:30:16.1364976560 /var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.log -989 2026-07-18T03:30:13.7816006690 /var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.log -986 2026-07-21T03:30:10.0285327490 /var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.log -986 2026-07-18T03:30:08.3002411830 /var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.log -731 2026-07-19T03:45:57.5197000270 /var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.log -729 2026-07-19T03:45:46.4444121980 /var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.log -========== NODE=pve02 ========== -pve02.gram1.ru -=== RCLONE_REMOTES === -pve01-mail-01: -pve01-mail-01-crypt: -pve01-mail-02: -pve01-mail-02-crypt: -pve01-mail-03: -pve01-mail-03-crypt: -pve01-mail-04: -pve01-mail-04-crypt: -pve01-mail-05: -pve01-mail-05-crypt: -pve02-mail-01: -pve02-mail-01-crypt: -pve02-mail-02: -pve02-mail-02-crypt: -pve02-mail-03: -pve02-mail-03-crypt: -pve02-mail-04: -pve02-mail-04-crypt: -pve03-mail-01: -pve03-mail-01-crypt: -pve03-mail-02: -pve03-mail-02-crypt: -pve03-mail-03: -pve03-mail-03-crypt: -pve03-mail-04: -pve03-mail-04-crypt: -NODE_REMOTE_COUNT=26 -REMOTE_HEALTH node=pve02 remote=pve01-mail-01: status=OK -=== BACKUP_TIMERS === -Tue 2026-07-21 18:41:03 MSK 3h 51min Tue 2026-07-21 12:41:22 MSK 2h 7min ago forum-prod-fresh5-cloud-backup.timer forum-prod-fresh5-cloud-backup.service -Wed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:22 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service -Wed 2026-07-22 04:20:00 MSK 13h Tue 2026-07-21 04:20:12 MSK 10h ago homelab-cluster-admin-vm-mail-cloud-backup.timer homelab-cluster-admin-vm-mail-cloud-backup.service -=== BACKUP_UNITS === -codevipe-cloud-backup.service static - -dpkg-db-backup.service static - -forum-prod-fresh5-cloud-backup.service static - -homelab-cluster-admin-vm-mail-cloud-backup.service static - -codevipe-cloud-backup.timer disabled enabled -dpkg-db-backup.timer enabled enabled -forum-prod-fresh5-cloud-backup.timer enabled enabled -homelab-cluster-admin-vm-mail-cloud-backup.timer enabled enabled -=== BACKUP_PIPELINE_FILES === -=== LOCAL_BACKUP_ARCHIVES === -997503422 2026-07-01T03:31:11.1700059310 /var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.vma.zst -868048369 2026-06-30T03:31:08.4903256780 /var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.vma.zst -288349157 2026-07-21T03:30:21.2480085730 /var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.tar.zst -286538526 2026-07-20T03:30:18.4386603270 /var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.tar.zst -284443608 2026-07-19T03:47:41.8525503200 /var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst -283022034 2026-07-18T03:30:19.7617214460 /var/lib/vz/dump/vzdump-lxc-111-2026_07_18-03_30_03.tar.zst -220080970 2026-07-21T03:30:33.3910787180 /var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.tar.zst -220065825 2026-07-20T03:30:30.8287264090 /var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.tar.zst -220049041 2026-07-19T03:47:58.9256407960 /var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst -220033534 2026-07-18T03:30:37.2878143100 /var/lib/vz/dump/vzdump-lxc-113-2026_07_18-03_30_20.tar.zst -7547 2026-07-18T01:09:39.2467331080 /var/lib/vz/dump/vzdump-qemu-160-2026_07_18-01_02_44.log -7443 2026-07-18T01:45:13.3149404960 /var/lib/vz/dump/vzdump-qemu-160-2026_07_18-01_38_20.log -5796 2026-07-17T21:55:21.5648458010 /var/lib/vz/dump/vzdump-qemu-160-2026_07_17-21_30_20.log -2685 2026-06-30T03:31:09.1616927060 /var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.log -2590 2026-07-01T03:31:11.9500372070 /var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.log -1088 2026-07-20T03:30:31.0784689220 /var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.log -1085 2026-07-20T03:30:18.7427551960 /var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.log -988 2026-07-21T03:30:33.6364088960 /var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.log -988 2026-07-18T03:30:37.5450736960 /var/lib/vz/dump/vzdump-lxc-113-2026_07_18-03_30_20.log -985 2026-07-21T03:30:21.5465818340 /var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.log -985 2026-07-18T03:30:20.0884248500 /var/lib/vz/dump/vzdump-lxc-111-2026_07_18-03_30_03.log -731 2026-07-19T03:47:59.2238139650 /var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.log -728 2026-07-19T03:47:42.1183953260 /var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.log -========== NODE=pve03 ========== -pve03.gram1.ru -=== RCLONE_REMOTES === -pve01-mail-01: -pve01-mail-01-crypt: -pve01-mail-02: -pve01-mail-02-crypt: -pve01-mail-03: -pve01-mail-03-crypt: -pve01-mail-04: -pve01-mail-04-crypt: -pve01-mail-05: -pve01-mail-05-crypt: -pve02-mail-01: -pve02-mail-01-crypt: -pve02-mail-02: -pve02-mail-02-crypt: -pve02-mail-03: -pve02-mail-03-crypt: -pve02-mail-04: -pve02-mail-04-crypt: -pve03-mail-01: -pve03-mail-01-crypt: -pve03-mail-02: -pve03-mail-02-crypt: -pve03-mail-03: -pve03-mail-03-crypt: -pve03-mail-04: -pve03-mail-04-crypt: -NODE_REMOTE_COUNT=26 -REMOTE_HEALTH node=pve03 remote=pve01-mail-01: status=OK -=== BACKUP_TIMERS === -Wed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:10 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service -=== BACKUP_UNITS === -dpkg-db-backup.service static - -dpkg-db-backup.timer enabled enabled -=== BACKUP_PIPELINE_FILES === -=== LOCAL_BACKUP_ARCHIVES === -11116472459 2026-07-20T03:33:53.2839064370 /var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.vma.zst -11043834894 2026-07-21T03:33:54.3725716230 /var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.vma.zst -11013723404 2026-07-19T03:33:53.3463878040 /var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.vma.zst -10994820121 2026-07-18T03:33:47.1540038630 /var/lib/vz/dump/vzdump-qemu-130-2026_07_18-03_30_02.vma.zst -1137072350 2026-07-21T03:34:37.6276811360 /var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.vma.zst -838951976 2026-07-19T03:34:33.1034859540 /var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.vma.zst -838817925 2026-07-20T03:34:37.1719936670 /var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.vma.zst -785249297 2026-07-18T03:34:27.1531047780 /var/lib/vz/dump/vzdump-qemu-190-2026_07_18-03_33_48.vma.zst -7106 2026-07-02T03:37:53.1040459570 /var/lib/vz/dump/vzdump-qemu-130-2026_07_02-03_30_02.log -7102 2026-07-03T03:36:49.1036784480 /var/lib/vz/dump/vzdump-qemu-130-2026_07_03-03_30_05.log -6564 2026-07-18T03:33:48.0107527290 /var/lib/vz/dump/vzdump-qemu-130-2026_07_18-03_30_02.log -6560 2026-07-20T03:33:54.9283349630 /var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.log -6559 2026-07-21T03:33:55.1428157340 /var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.log -6459 2026-07-19T03:33:53.7631656400 /var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.log -6246 2026-07-04T03:33:39.2197744030 /var/lib/vz/dump/vzdump-qemu-130-2026_07_04-03_30_02.log -4107 2026-07-07T03:31:51.2516816970 /var/lib/vz/dump/vzdump-qemu-130-2026_07_07-03_30_01.log -2642 2026-07-21T03:34:38.4491706750 /var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.log -2500 2026-07-20T03:34:37.2013439550 /var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.log -2404 2026-07-19T03:34:34.1266444600 /var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.log -2305 2026-07-18T03:34:27.7179455720 /var/lib/vz/dump/vzdump-qemu-190-2026_07_18-03_33_48.log -=== CLOUD_REMOTE_SUMMARY === -EXPECTED_TOTAL=13 -DETECTED_TOTAL=78 -REMOTE_COUNT=MISMATCH +===== NODE=pve01 ===== +REMOTE_COUNT=26 +REMOTE node=pve01 name=pve01-mail-01: health=OK references=UNREFERENCED +===== NODE=pve02 ===== +REMOTE_COUNT=26 +REMOTE node=pve02 name=pve01-mail-01: health=OK references=UNREFERENCED +===== NODE=pve03 ===== +REMOTE_COUNT=26 +REMOTE node=pve03 name=pve01-mail-01: health=OK references=UNREFERENCED +===== SUMMARY ===== +EXPECTED_CLUSTER_REMOTES=13 +DETECTED_CLUSTER_REMOTES=78 +REMOTE_COUNT_STATUS=MISMATCH +CLOUD_PIPELINE_MAP_COMPLETE=YES OUTPUT_END CHAT_OUTPUT_END