From b1db5c6dcad043c4c1e62602be27c4fc1e6f8af2 Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Tue, 21 Jul 2026 11:41:35 +0000 Subject: [PATCH] runtime: publish BACKUP-PIPELINE-LOGIC-061 --- .../history/BACKUP-PIPELINE-LOGIC-061.json | 30 ++ runtime/history/BACKUP-PIPELINE-LOGIC-061.txt | 354 ++++++++++++++ runtime/latest.json | 14 +- runtime/latest.txt | 444 ++++++++++-------- 4 files changed, 635 insertions(+), 207 deletions(-) create mode 100644 runtime/history/BACKUP-PIPELINE-LOGIC-061.json create mode 100644 runtime/history/BACKUP-PIPELINE-LOGIC-061.txt diff --git a/runtime/history/BACKUP-PIPELINE-LOGIC-061.json b/runtime/history/BACKUP-PIPELINE-LOGIC-061.json new file mode 100644 index 0000000..102bba3 --- /dev/null +++ b/runtime/history/BACKUP-PIPELINE-LOGIC-061.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "BACKUP-PIPELINE-LOGIC-061", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "backup-audit", + "started_at_utc": "2026-07-21T11:41:33Z", + "finished_at_utc": "2026-07-21T11:41:34Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1", + "command_sha256": "8cd01154fdfa3d93f159413d18952f742f79d917e9d1ae084a44f20c5a7e900c", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "6d688d5c3fe35e135dc3153564cf179967785172356e2ee3f880a2a4eed6ad65", + "sanitized_output_sha256": "f48b9d843450b5a97445697cc0fe774d88376f4ee974f0a517ae55d4581ae206", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "=== BACKUP_JOBS ===\n[{\"compress\":\"zstd\",\"enabled\":1,\"id\":\"homelab-nightly-all\",\"mailnotification\":\"failure\",\"mode\":\"snapshot\",\"next-run\":1784680200,\"prune-backups\":{\"keep-daily\":\"4\"},\"schedule\":\"03:30\",\"storage\":\"local\",\"type\":\"vzdump\",\"vmid\":\"110,111,112,113,130,150,190\"}]\n=== BACKUP_UNITS_AND_TIMERS ===\ndpkg-db-backup.service static -\nfilebrowser-backup.service static -\nhomelab-alertmanager-backup.service static -\nhomelab-appbackup-dockge-stacks.service static -\nhomelab-appbackup-mariadb-batch.service static -\nhomelab-appbackup-npmplus-edge.service static -\nhomelab-appbackup-postgres-batch.service static -\nhomelab-appbackup-sqlite-batch.service static -\nhomelab-authentik-backup.service static -\nhomelab-auto-backup.service static -\nhomelab-backup-coverage-matrix.service static -\nhomelab-backup-freshness.service static -\nhomelab-backup-healthcheck.service static -\nhomelab-backup-matrix-refresh.service static -\nhomelab-backup-sla-health.service static -\nhomelab-beszel-backup.service static -\nhomelab-blackbox-exporter-backup.service static -\nhomelab-cluster-admin-incident-engine-backup.service static -\nhomelab-crypto-portfolio-backup.service static -\nhomelab-edge-vm-vzdump-backup.service static -\nhomelab-emergency-backup.service static -\nhomelab-extended-appbackup.service static -\nhomelab-gitea-backup.service static -\nhomelab-gitea-secondary-backup.service static -\nhomelab-legacy-monitoring-backups.service static -\nhomelab-mail-cloud-critical-upload.service static -\nhomelab-mail-cloud-edge-vm-upload.service static -\nhomelab-mail-cloud-nextcloud-vm-upload.service static -\nhomelab-mail-cloud-restore-drill.service static -\nhomelab-npmplus-kuma-config-backup.service static -\nhomelab-router-running-config-mail-cloud.service static -\nhomelab-vaultwarden-backup.service static -\nhomelab-verified-backup-health.service static -\nhomelab-vm-backup-health.service static -\nhomelab-vm-backup-policy.service static -\nhomelab-vm-full-backup.service static -\nhomelab-vm-local-dumps-2cloud.service static -\nhomelab-vm160-remote-backup.service static -\nimmich-media-backup.service static -\nmemos-backup.service static -\nnetbird-vps-backup.service static -\npaperless-backup.service static -\nskladchik-reports-monitor-backup-restore-check.service static -\nskladchik-reports-monitor-backup.service static -\ndpkg-db-backup.timer enabled enabled\nfilebrowser-backup.timer enabled enabled\nhomelab-alertmanager-backup.timer enabled enabled\nhomelab-appbackup-dockge-stacks.timer enabled enabled\nhomelab-appbackup-mariadb-batch.timer enabled enabled\nhomelab-appbackup-npmplus-edge.timer enabled enabled\nhomelab-appbackup-postgres-batch.timer enabled enabled\nhomelab-appbackup-sqlite-batch.timer enabled enabled\nhomelab-authentik-backup.timer disabled enabled\nhomelab-auto-backup.timer enabled enabled\nhomelab-backup-coverage-matrix.timer enabled enabled\nhomelab-backup-freshness.timer enabled enabled\nhomelab-backup-healthcheck.timer enabled enabled\nhomelab-backup-matrix-refresh.timer disabled enabled\nhomelab-backup-sla-health.timer enabled enabled\nhomelab-beszel-backup.timer enabled enabled\nhomelab-blackbox-exporter-backup.timer enabled enabled\nhomelab-cluster-admin-incident-engine-backup.timer enabled enabled\nhomelab-crypto-portfolio-backup.timer enabled enabled\nhomelab-edge-vm-vzdump-backup.timer disabled enabled\nhomelab-emergency-backup.timer enabled enabled\nhomelab-extended-appbackup.timer enabled enabled\nhomelab-gitea-backup.timer disabled enabled\nhomelab-gitea-secondary-backup.timer enabled enabled\nhomelab-legacy-monitoring-backups.timer enabled enabled\nhomelab-mail-cloud-critical-upload.timer enabled enabled\nhomelab-mail-cloud-edge-vm-upload.timer enabled enabled\nhomelab-mail-cloud-nextcloud-vm-upload.timer enabled enabled\nhomelab-mail-cloud-restore-drill.timer enabled enabled\nhomelab-npmplus-kuma-config-backup.timer disabled enabled\nhomelab-router-running-config-mail-cloud.timer enabled enabled\nhomelab-vaultwarden-backup.timer disabled enabled\nhomelab-verified-backup-health.timer enabled enabled\nhomelab-vm-backup-health.timer enabled enabled\nhomelab-vm-backup-policy.timer enabled enabled\nhomelab-vm-full-backup.timer enabled enabled\nhomelab-vm-local-dumps-2cloud.timer enabled enabled\nhomelab-vm160-remote-backup.timer disabled enabled\nimmich-media-backup.timer disabled enabled\nmemos-backup.timer enabled enabled\nnetbird-vps-backup.timer disabled enabled\npaperless-backup.timer disabled enabled\nskladchik-reports-monitor-backup-restore-check.timer enabled enabled\nskladchik-reports-monitor-backup.timer enabled enabled\nWed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:14 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service\nWed 2026-07-22 00:09:43 MSK 9h Tue 2026-07-21 00:09:32 MSK 14h ago homelab-backup-freshness.timer homelab-backup-freshness.service\nWed 2026-07-22 02:15:00 MSK 11h Tue 2026-07-21 02:15:05 MSK 12h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service\nWed 2026-07-22 03:10:02 MSK 12h Tue 2026-07-21 03:13:14 MSK 11h ago homelab-emergency-backup.timer homelab-emergency-backup.service\nWed 2026-07-22 03:35:00 MSK 12h Tue 2026-07-21 03:35:14 MSK 11h ago filebrowser-backup.timer filebrowser-backup.service\nWed 2026-07-22 03:55:02 MSK 13h Tue 2026-07-21 03:51:25 MSK 10h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service\nWed 2026-07-22 04:17:29 MSK 13h Tue 2026-07-21 04:19:17 MSK 10h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service\nWed 2026-07-22 04:26:54 MSK 13h Tue 2026-07-21 04:28:14 MSK 10h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service\nWed 2026-07-22 04:38:04 MSK 13h Tue 2026-07-21 04:34:48 MSK 10h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service\nWed 2026-07-22 05:01:44 MSK 14h Tue 2026-07-21 04:57:14 MSK 9h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service\nWed 2026-07-22 05:23:56 MSK 14h Tue 2026-07-21 05:17:57 MSK 9h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service\nWed 2026-07-22 05:27:13 MSK 14h Tue 2026-07-21 05:29:05 MSK 9h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service\nWed 2026-07-22 05:47:19 MSK 15h Tue 2026-07-21 05:46:14 MSK 8h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service\nWed 2026-07-22 06:01:19 MSK 15h Tue 2026-07-21 05:57:30 MSK 8h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service\nWed 2026-07-22 06:27:43 MSK 15h Tue 2026-07-21 06:32:25 MSK 8h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service\nWed 2026-07-22 06:38:51 MSK 15h Tue 2026-07-21 06:43:54 MSK 7h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service\nWed 2026-07-22 06:39:22 MSK 15h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service\nWed 2026-07-22 06:44:16 MSK 16h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service\nWed 2026-07-22 06:52:06 MSK 16h Tue 2026-07-21 06:47:47 MSK 7h ago homelab-beszel-backup.timer homelab-beszel-backup.service\nWed 2026-07-22 06:58:58 MSK 16h Tue 2026-07-21 06:57:30 MSK 7h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service\nWed 2026-07-22 07:02:27 MSK 16h Tue 2026-07-21 07:05:30 MSK 7h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service\nWed 2026-07-22 07:07:14 MSK 16h Tue 2026-07-21 07:18:55 MSK 7h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service\nWed 2026-07-22 07:35:06 MSK 16h Tue 2026-07-21 07:36:59 MSK 7h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service\nWed 2026-07-22 07:38:21 MSK 16h Tue 2026-07-21 07:36:14 MSK 7h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service\nWed 2026-07-22 07:41:49 MSK 17h Tue 2026-07-21 07:53:41 MSK 6h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service\nWed 2026-07-22 08:10:37 MSK 17h Tue 2026-07-21 08:07:25 MSK 6h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service\nWed 2026-07-22 08:37:02 MSK 17h Tue 2026-07-21 08:30:07 MSK 6h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service\nWed 2026-07-22 08:40:13 MSK 17h Tue 2026-07-21 08:50:14 MSK 5h 51min ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service\nWed 2026-07-22 08:53:36 MSK 18h Tue 2026-07-21 08:41:14 MSK 6h ago memos-backup.timer memos-backup.service\nSun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service\nSun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service\nSun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service\nSun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service\nMon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 7h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service\n=== PIPELINE_LOGIC ===\n/usr/local/sbin/homelab-vm160-remote-backup:8:PARENT=/mnt/staging/vzdump\n/usr/local/sbin/homelab-vm160-remote-backup:22:SOURCE_ACTIVE=\"$(ssh \"${SSH[@]}\" \"$SOURCE\" \"pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'\")\" || SOURCE_ACTIVE_RC=$?\n/usr/local/sbin/homelab-vm160-remote-backup:27:NAME=\"vzdump-qemu-${VMID}-${TS}.vma.zst\"\n/usr/local/sbin/homelab-vm160-remote-backup:41: rm -f \"$LOG\"\n/usr/local/sbin/homelab-vm160-remote-backup:46:LATEST_SIZE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\\n' | sort -nr | head -n1\")\"\n/usr/local/sbin/homelab-vm160-remote-backup:55:ssh \"${SSH[@]}\" \"$SOURCE\" \"vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7\" 2>\"$LOG\" | ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$PART'\"\n/usr/local/sbin/homelab-vm160-remote-backup:57:ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$DIR/vzdump.log'\" < \"$LOG\"\n/usr/local/sbin/homelab-vm160-remote-backup:66:rm -f \"$LOG\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:83: rm -f \"$OUT\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:108: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/MANIFEST.txt\" \"$WORK/edge/MANIFEST.txt\" > \"$WORK/logs/edge_manifest_download.out\" 2>\"$WORK/logs/edge_manifest_download.err\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:111: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/SHA256SUMS.txt\" \"$WORK/edge/SHA256SUMS.txt\" > \"$WORK/logs/edge_sums_download.out\" 2>\"$WORK/logs/edge_sums_download.err\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:114: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/$EDGE_ARCHIVE.sha256\" \"$WORK/edge/$EDGE_ARCHIVE.sha256\" > \"$WORK/logs/edge_archive_sha_download.out\" 2>\"$WORK/logs/edge_archive_sha_download.err\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:130: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/parts/$FIRST_PART\" \"$WORK/edge/$FIRST_PART\" > \"$WORK/logs/edge_first_part_download.out\" 2>\"$WORK/logs/edge_first_part_download.err\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:137: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/parts/$LAST_PART\" \"$WORK/edge/$LAST_PART\" > \"$WORK/logs/edge_last_part_download.out\" 2>\"$WORK/logs/edge_last_part_download.err\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:168: rm -f \"$WORK/edge/$FIRST_PART\" \"$WORK/edge/$LAST_PART\"\n/usr/local/sbin/homelab-golden-state-extended-overlay:26:rclone copyto \"$JSON\" \"$RA\" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-golden-state-extended-overlay:27:rclone copyto \"$JSON\" \"$RB\" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-mail-cloud-critical-upload:84: rclone copyto \"$BUNDLE_SHA\" \"${R}:$REMOTE_DIR/$(basename \"$BUNDLE_SHA\")\" --checksum > \"$WORK/logs/${SAFE}_upload_sha.out\" 2>\"$WORK/logs/${SAFE}_upload_sha.err\"\n/usr/local/sbin/homelab-mail-cloud-critical-upload:86: rclone copyto \"$WORK/components/MANIFEST.txt\" \"${R}:$REMOTE_DIR/MANIFEST.txt\" --checksum > \"$WORK/logs/${SAFE}_upload_manifest.out\" 2>\"$WORK/logs/${SAFE}_upload_manifest.err\"\n/usr/local/sbin/homelab-mail-cloud-critical-upload:88: rclone copyto \"$WORK/components/SHA256SUMS.txt\" \"${R}:$REMOTE_DIR/SHA256SUMS.txt\" --checksum > \"$WORK/logs/${SAFE}_upload_sums.out\" 2>\"$WORK/logs/${SAFE}_upload_sums.err\"\n/usr/local/sbin/homelab-mail-cloud-critical-upload:108: rm -f \"$WORK/download/${SAFE}-$(basename \"$BUNDLE\")\"\n/usr/local/sbin/homelab-mail-cloud-critical-upload:114: [ -n \"$OLD\" ] && rclone purge \"${R}:$TARGET_BASE/$OLD\" >> \"$WORK/logs/${SAFE}_retention.out\" 2>>\"$WORK/logs/${SAFE}_retention.err\"\n/usr/local/sbin/homelab-alertmanager-backup-run:18:rm -f \"$STAGE/remote.tgz\"\n/usr/local/sbin/postgres-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@\"$H\" \"sudo rm -f /tmp/postgres-app-${APP}-${JOB_ID}.tgz\"\n/usr/local/sbin/homelab-stage-verify-flat-chunked:9:rclone copy \"$REMOTE_DIR\" \"$VERIFY\" --transfers 4 --checkers 8 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-rclone-cat-chunked:10:rclone copyto \"${REMOTE_FILE}.parts/MANIFEST.parts.txt\" \"$MAN\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-stage-verify-chunked:9:rclone copy \"$REMOTE_DIR\" \"$VERIFY\" --exclude \"$ARCHIVE.parts/**\" --exclude \"$ARCHIVE.parts/\" --exclude \"$ARCHIVE.sha256\" --exclude \"$ARCHIVE.manifest.txt\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --log-level ERROR\n/usr/local/sbin/crypto-portfolio-backup-run:17:rm -f \"$STAGE/remote.tgz\"\n/usr/local/sbin/npmplus-edge-backup-run:20:ssh -o BatchMode=yes -o ConnectTimeout=8 \"$EDGE\" \"sudo rm -f '/tmp/npmplus-edge-${JOB_ID}.tgz'\"\n/usr/local/sbin/mariadb-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@\"$H\" \"sudo rm -f /tmp/mariadb-app-${APP}-${JOB_ID}.tgz\"\n/usr/local/sbin/homelab-blackbox-exporter-backup-run:17:rm -f \"$STAGE/remote.tgz\"\n/usr/local/sbin/homelab-ingress-standby-refresh:16: rm -f \"$TMP_STATE\" \"$RELAY\" \"$RESOURCES\"\n/usr/local/sbin/homelab-ingress-standby-refresh:74:run_node \"$SOURCE_NODE\" vzdump 130 \\\n/usr/local/sbin/homelab-ingress-standby-refresh:83:\"find /var/lib/vz/dump -maxdepth 1 -type f -name 'vzdump-qemu-130-*.vma.zst' -printf '%T@ %p\\n' | sort -rn | head -n 1 | cut -d' ' -f2-\"\n/usr/local/sbin/homelab-ingress-standby-refresh:106:\"chattr -i '$DEST_FILE' '$DEST_FILE.sha256' '$DEST_FILE.env' 2>/dev/null || true; rm -f '$DEST_TMP'\"\n/usr/local/sbin/homelab-ingress-standby-refresh:155: -name 'vzdump-qemu-130-*.vma.zst' \\\n/usr/local/sbin/homelab-ingress-standby-refresh:164: rm -f \"$OLD\" \"$OLD.sha256\" \"$OLD.env\"\n/usr/local/sbin/homelab-ingress-standby-refresh:173:run_node \"$SOURCE_NODE\" rm -f \"$SOURCE_BACKUP\"\n/usr/local/sbin/homelab-stage-2cloud-flat:38: for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f \"$STAGE/$F\" ]] && rclone copyto \"$STAGE/$F\" \"$REMOTE_DIR/$F\" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done\n/usr/local/sbin/homelab-stage-2cloud-flat:41: rclone copyto \"$STAGE/latest-backup-index.json\" \"$LATEST_FILE\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-backup-authentik:27:ssh debian@[PRIVATE_IP] \"sudo docker rm -f authentik-restore-test >/dev/null 2>&1 || true\"\n/usr/local/sbin/homelab-backup-authentik:51:ssh debian@[PRIVATE_IP] \"sudo docker rm -f authentik-restore-test >/dev/null 2>&1 || true; rm -f /tmp/authentik-restore-from-archive.pg_dump\"\n/usr/local/sbin/homelab-safe-exec:621: latest_tmp.unlink(missing_ok=True)\n/usr/local/sbin/homelab-golden-state-index:28:rclone copyto \"$JSON\" \"$RA\" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-golden-state-index:29:rclone copyto \"$JSON\" \"$RB\" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-full-retention:21: -name \"vzdump-qemu-${vmid}-*.vma.zst\" \\\n/usr/local/sbin/homelab-vm-full-retention:33: vzdump-qemu-\"$vmid\"-*.vma.zst)\n/usr/local/sbin/homelab-vm-full-retention:35: rm -f \\\n/usr/local/sbin/homelab-vm-full-retention:105: rclone purge \"${path}/${dir}\"\n/usr/local/sbin/homelab-stage-2cloud:23:echo \"[STEP] upload cloud A metadata\"; rclone copy \"$STAGE\" \"$RA\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n/usr/local/sbin/homelab-stage-2cloud:24:echo \"[STEP] upload cloud A archive chunks\"; rclone deletefile \"$RA/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RA/$ARCHIVE\"\n/usr/local/sbin/homelab-stage-2cloud:25:echo \"[STEP] upload cloud B metadata\"; rclone copy \"$STAGE\" \"$RB\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n/usr/local/sbin/homelab-stage-2cloud:26:echo \"[STEP] upload cloud B archive chunks\"; rclone deletefile \"$RB/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RB/$ARCHIVE\"\n/usr/local/sbin/homelab-stage-2cloud:29:echo \"[STEP] upload latest index\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RAL\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RBL\"; rclone cat \"$RAL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null; rclone cat \"$RBL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n/usr/local/sbin/homelab-backup-npmplus-kuma-configs:90: ssh -o BatchMode=yes -o ConnectTimeout=8 debian@[PRIVATE_IP] \"rm -f '$EDGE_TGZ'\" 2>>\"$ERR\" || true\n/usr/local/sbin/homelab-stage-2cloud-flat-canary:36: for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f \"$STAGE/$F\" ]] && rclone copyto \"$STAGE/$F\" \"$REMOTE_DIR/$F\" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done\n/usr/local/sbin/homelab-stage-2cloud-flat-canary:39: rclone copyto \"$STAGE/latest-backup-index.json\" \"$LATEST_FILE\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-router-running-config-mail-cloud:62: rm -f \"$CONFIG\"\n/usr/local/sbin/homelab-router-running-config-mail-cloud:66: rm -f \"$CONFIG\"\n/usr/local/sbin/homelab-router-running-config-mail-cloud:79: if rclone mkdir \"$REMOTE:$DEST\" >\"$WORK/rclone-mkdir.out\" 2>\"$WORK/rclone-mkdir.err\" && homelab-rclone-copy-chunked \"$ARCHIVE\" \"$REMOTE:$DEST/$(basename \"$ARCHIVE\")\" >\"$WORK/rclone-upload.out\" 2>\"$WORK/rclone-upload.err\" && rclone copyto \"$SHA_FILE\" \"$REMOTE:$DEST/$(basename \"$SHA_FILE\")\" --checksum >\"$WORK/rclone-upload-sha.out\" 2>\"$WORK/rclone-upload-sha.err\"; then\n/usr/local/sbin/homelab-router-running-config-mail-cloud:98: [ -n \"$OLD\" ] && rclone purge \"$REMOTE:$TARGET_BASE/$OLD\" >>\"$WORK/retention.out\" 2>>\"$WORK/retention.err\"\n/usr/local/sbin/homelab-router-running-config-mail-cloud:103: rm -f \"$old\" \"$old.sha256\"\n/usr/local/sbin/appbackup-restore-check:27:rclone copy \"$RA\" \"$ROOT/a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >\"${LOGBASE}-a.log\" 2>&1\n/usr/local/sbin/appbackup-restore-check:32:rclone copy \"$RB\" \"$ROOT/b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >\"${LOGBASE}-b.log\" 2>&1\n/usr/local/sbin/appbackup-restore-check:39:rm -f \"${LOGBASE}-a.log\" \"${LOGBASE}-a-sha.log\" \"${LOGBASE}-b.log\" \"${LOGBASE}-b-sha.log\"\n/usr/local/sbin/homelab-capacity-risk:32:LARGEST_BYTES=\"$(find /var/lib/vz/dump -maxdepth 1 -type f \\( -name \"vzdump-qemu-*.vma.zst\" -o -name \"vzdump-lxc-*.tar.zst\" \\) -printf \"%s\\n\" 2>/dev/null | sort -n | tail -n1)\"\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:9:OFFHOST_DIR=/mnt/staging/offhost/edge-vm-vzdump-from-pve03\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:85: /var/lib/vz/dump/vzdump-qemu-130-*.vma.zst) ;;\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:156:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-backup NODE=%s VMID=%s ARCHIVE=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\\n' \\\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:158: >\"$HEALTH_DIR/edge-vm-vzdump-backup.txt\"\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:159:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-offhost NODE=%s VMID=%s HOST=%s DEST=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\\n' \\\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:161: >\"$HEALTH_DIR/edge-vm-vzdump-offhost.txt\"\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:162:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-restore-verify HOST=%s VMID=%s ARCHIVE=%s SIZE=%s SHA256=%s ZSTD_RC=0 VMA_RC=0 NOTES=nightly_existing_dump_freshness\\n' \\\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:164: >\"$HEALTH_DIR/edge-vm-vzdump-restore.txt\"\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:167: \"$HEALTH_DIR/edge-vm-vzdump-backup.txt\" \\\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:168: \"$HEALTH_DIR/edge-vm-vzdump-offhost.txt\" \\\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:169: \"$HEALTH_DIR/edge-vm-vzdump-restore.txt\"\n/usr/local/sbin/homelab-external-probe-vps:20:rm -f \"$TMP\"\n/usr/local/sbin/sqlite-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@\"$H\" \"sudo rm -f /tmp/sqlite-app-${APP}-${JOB_ID}.tgz\"\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:31: rclone copyto \"$remote_path\" \"$out\" >/dev/null\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:33: rm -f \"$plist\"\n/usr/local/sbin/skladchik-reports-monitor-backup:83: for D in \"${OLD[@]}\"; do [ -n \"$D\" ] && rclone purge \"${REMOTE}${REMOTE_BASE}/daily/${D}\" >/dev/null 2>&1 || true; done\n/usr/local/sbin/homelab-vm-local-dumps-retention:56: rclone purge \"$PATH_REMOTE\" >> \"$REPORT\" 2>&1 && DELETED=$((DELETED+1)) || BAD=$((BAD+1))\n/usr/local/sbin/homelab-mkdocs-auto-refresh:53: rm -f /tmp/homelab-mkdocs-auto-refresh-git-fetch.out /tmp/homelab-mkdocs-auto-refresh-git-fetch.err\n/usr/local/sbin/homelab-mkdocs-auto-refresh:57:rm -f /tmp/homelab-mkdocs-auto-refresh-git-fetch.out /tmp/homelab-mkdocs-auto-refresh-git-fetch.err\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:42: F=\"$(find /var/lib/vz/dump -maxdepth 1 -type f \\( -name \"vzdump-qemu-${id}-*.vma.zst\" -o -name \"vzdump-lxc-${id}-*.tar.zst\" \\) -printf \"%T@ %p\\n\" 2>/dev/null | sort -n | tail -n 1 | cut -d\" \" -f2-)\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:53: echo \"$BN\" | grep -q \"^vzdump-qemu-\" && KIND=\"qemu\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:54: echo \"$BN\" | grep -q \"^vzdump-lxc-\" && KIND=\"lxc\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:126: rclone copyto \"$part\" \"$DEST/files/$bn.chunks/$pn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:128: rclone copyto \"$CDIR/chunks.sha256\" \"$DEST/files/$bn.chunks/chunks.sha256\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:131: rclone copyto \"$path\" \"$DEST/files/$bn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:135: rclone copyto \"$CHKS\" \"$DEST/checksums.txt\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:136: rclone copyto \"$FILES_TSV\" \"$DEST/files.tsv\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:137: rclone copyto \"$MAN\" \"$DEST/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:146: rclone copyto \"$DEST/manifest.json\" \"$VBASE/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:154: rclone copy \"$DEST/files/$bn.chunks\" \"$VDIR\" --filter \"+ part-*\" --filter \"+ chunks.sha256\" --filter \"- *\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:166: rclone copyto \"$DEST/files/$bn\" \"$VBASE/$bn\" --transfers 1 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:168: rm -f \"$VBASE/$bn\"\n/usr/local/sbin/homelab-gitea-secondary-backup:18: rm -f \"$TMP\"\n/usr/local/sbin/homelab-gitea-secondary-backup:31:docker exec -u git -w /tmp gitea sh -lc \\\"rm -f /tmp/\\$NAME && gitea dump -c /data/gitea/conf/app.ini -f /tmp/\\$NAME\\\"\n/usr/local/sbin/homelab-gitea-secondary-backup:33:docker exec gitea rm -f \\\"/tmp/\\$NAME\\\"\n/usr/local/sbin/homelab-gitea-secondary-backup:100: rm -f -- \"$FILE\" \"$FILE.sha256\"\n/usr/local/sbin/homelab-gitea-secondary-backup:123: rm -f -- \"$FILE\" \"$FILE.sha256\"\n/usr/local/sbin/homelab-gitea-secondary-backup:136: rm -f -- \"$FILE\" \"$FILE.sha256\"\n/usr/local/sbin/homelab-rclone-copy-chunked:24:rclone purge \"${DEST}.parts\" >/dev/null 2>&1 || true\n/usr/local/sbin/homelab-rclone-copy-chunked:26:rclone copy \"$PARTS\" \"${DEST}.parts\" --transfers 2 --checkers 4 --retries 5 --low-level-retries 20 --stats 30s --log-level INFO\n/usr/local/sbin/homelab-rclone-copy-chunked:27:rclone copyto \"$WORK/${NAME}.sha256\" \"${DEST}.sha256\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO\n/usr/local/sbin/homelab-rclone-copy-chunked:28:rclone copyto \"$WORK/MANIFEST.txt\" \"${DEST}.manifest.txt\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO\n/usr/local/sbin/homelab-rclone-copy-chunked:29:rclone copyto \"$WORK/MANIFEST.parts.txt\" \"${DEST}.parts/MANIFEST.parts.txt\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO\n/usr/local/sbin/homelab-health-metrics-write:10:rm -f \"$tmp\"\n/usr/local/sbin/homelab-rclone-cat-flat-chunked:5:trap \"rm -f \\\"$TMP\\\"\" EXIT\n/usr/local/sbin/homelab-vm-backup-policy:29:echo \"## Latest local vzdump catalog\" >> \"$REPORT\"\n/usr/local/sbin/homelab-vm-backup-policy:31:for f in $(find /var/lib/vz/dump -maxdepth 1 -type f \\( -name \"vzdump-qemu-*.vma.zst\" -o -name \"vzdump-lxc-*.tar.zst\" \\) 2>/dev/null | sort); do b=\"$(basename \"$f\")\"; size=\"$(stat -c %s \"$f\" 2>/dev/null || echo 0)\"; mt=\"$(date -u -d \"@$(stat -c %Y \"$f\" 2>/dev/null || echo 0)\" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)\"; if echo \"$b\" | grep -q \"^vzdump-qemu-\"; then kind=qemu; vmid=\"$(echo \"$b\" | sed -n \"s/^vzdump-qemu-\\([0-9]\\+\\)-.*/\\1/p\")\"; name=\"$(qm config \"$vmid\" 2>/dev/null | awk -F\": \" \"/^name:/{print \\$2; exit}\")\"; else kind=lxc; vmid=\"$(echo \"$b\" | sed -n \"s/^vzdump-lxc-\\([0-9]\\+\\)-.*/\\1/p\")\"; name=\"$(pct config \"$vmid\" 2>/dev/null | awk -F\": \" \"/^hostname:/{print \\$2; exit}\")\"; fi; printf \"%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n\" \"$kind\" \"$vmid\" \"${name:-unknown}\" \"$size\" \"$mt\" \"$f\" >> \"$CATALOG\"; done\n/usr/local/sbin/homelab-vm-backup-policy:37:echo \"- Existing local dumps can be uploaded/verified to two clouds without creating new vzdump archives.\" >> \"$REPORT\"\n/usr/local/sbin/homelab-vm-backup-policy:38:echo \"- New full vzdump for VM 170/171 is not started automatically because each VM has 120 GB bootdisk and /var/tmp has limited free space.\" >> \"$REPORT\"\n/usr/local/sbin/homelab-context-refresh-direct:34:ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] debian@[PRIVATE_IP] \"sudo -n /usr/local/sbin/homelab-runtime-receive $id /tmp/$id.json /tmp/$id.txt && rm -f /tmp/$id.json /tmp/$id.txt\"\n/usr/local/sbin/homelab-evidence-seal:16: rm -f \"$TMP\"\n/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:15:DUMP_SIZE=\"$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 \"$REMOTE\" \"sudo -n rm -f \\\"$REMOTE_DUMP\\\"; sudo -n -u clusteradmin pg_dump --format=custom --no-owner --no-privileges --file=\\\"$REMOTE_DUMP\\\" clusteradmin; sudo -n test -s \\\"$REMOTE_DUMP\\\"; sudo -n -u clusteradmin pg_restore --list \\\"$REMOTE_DUMP\\\" >/dev/null; sudo -n chown debian:debian \\\"$REMOTE_DUMP\\\"; sudo -n chmod 600 \\\"$REMOTE_DUMP\\\"; stat -c %s \\\"$REMOTE_DUMP\\\"\")\"\n/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:18:ssh -n -o BatchMode=yes -o ConnectTimeout=8 \"$REMOTE\" \"rm -f \\\"$REMOTE_DUMP\\\"\"\n/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:23:rm -f \"$STAGE/runtime.tgz\"\n/usr/local/sbin/appbackup-retention:85: if [[ \"$MODE\" == apply ]]; then echo \"DELETE schema=dual path=$REL\"; rclone purge \"$TARGET\"; else echo \"WOULD_DELETE schema=dual path=$REL\"; fi\n/usr/local/sbin/homelab-change-transaction:195: latest_tmp.unlink(missing_ok=True)\n/usr/local/sbin/homelab-check-private-vpn-hosts:15:trap 'rm -f \"$tmp\"' EXIT\n/usr/local/sbin/homelab-health-metrics:22: *backup*|*vzdump*) printf \"%s\" \"backup\" ;;\n/usr/local/sbin/homelab-health-metrics:34: npmplus-kuma-config-*|authentik-*|gitea-*|vaultwarden-*|netbox-*|mail-cloud-restore-drill|edge-vm-vzdump-*|mkdocs-runtime-publish|npmplus-certificate-expiry) printf \"%s\" \"P0\" ;;\n/usr/local/sbin/crypto-portfolio-chart-health-check:5:trap 'rm -f \"$TMP\"' EXIT\n/usr/local/sbin/homelab-rclone-copy-flat-chunked:23:while read -r PART PART_SIZE; do [[ \"$PART\" =~ ^part-[0-9]{4}$ && \"$PART_SIZE\" =~ ^[0-9]+$ ]] || { echo \"BAD PART MANIFEST\" >&2; exit 4; }; rclone copyto \"$WORK/parts/$PART\" \"${DEST}.${PART}\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR; REMOTE_BYTES=\"$(rclone size --json \"${DEST}.${PART}\" | jq -r \".bytes // -1\")\"; [[ \"$REMOTE_BYTES\" = \"$PART_SIZE\" ]] || { echo \"REMOTE PART SIZE MISMATCH $PART\" >&2; exit 5; }; UPLOADED=$((UPLOADED+1)); done < \"$WORK/MANIFEST.parts.txt\"\n/usr/local/sbin/homelab-rclone-copy-flat-chunked:25:rclone copyto \"$WORK/${NAME}.sha256\" \"${DEST}.sha256\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-rclone-copy-flat-chunked:26:rclone copyto \"$WORK/MANIFEST.txt\" \"${DEST}.manifest.txt\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-rclone-copy-flat-chunked:27:rclone copyto \"$WORK/MANIFEST.parts.txt\" \"${DEST}.parts.txt\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:8:TARGET_BASE=full/nextcloud-vm-vzdump\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:11:SRC_GLOB=/var/lib/vz/dump/vzdump-qemu-150-*.vma.zst\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:77:P1 Mail Cloud Nextcloud VM recurring chunked vzdump upload\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:95:rclone copy \"$PARTS_DIR\" \"$TARGET_REMOTE:$REMOTE_PARTS\" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > \"$WORK/logs/upload_parts.out\" 2>\"$WORK/logs/upload_parts.err\" || UPLOAD_PARTS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:97:rclone copyto \"$WORK/$BASENAME.sha256\" \"$TARGET_REMOTE:$REMOTE_SHA\" --checksum > \"$WORK/logs/upload_sha.out\" 2>\"$WORK/logs/upload_sha.err\" || UPLOAD_SHA_RC=$?\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:99:rclone copyto \"$WORK/MANIFEST.txt\" \"$TARGET_REMOTE:$REMOTE_MANIFEST\" --checksum > \"$WORK/logs/upload_manifest.out\" 2>\"$WORK/logs/upload_manifest.err\" || UPLOAD_MANIFEST_RC=$?\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:101:rclone copyto \"$WORK/SHA256SUMS.txt\" \"$TARGET_REMOTE:$REMOTE_SUMS\" --checksum > \"$WORK/logs/upload_sums.out\" 2>\"$WORK/logs/upload_sums.err\" || UPLOAD_SUMS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:110:rclone copy \"$TARGET_REMOTE:$REMOTE_PARTS\" \"$DL_DIR/parts\" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > \"$WORK/logs/download_parts.out\" 2>\"$WORK/logs/download_parts.err\" || DOWNLOAD_PARTS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:126: [ -n \"$OLD\" ] && rclone purge \"$TARGET_REMOTE:$TARGET_BASE/$OLD\" >> \"$WORK/logs/retention.out\" 2>>\"$WORK/logs/retention.err\"\n/usr/local/sbin/homelab-rehydrate-flat-chunked-dir:10: rm -f \"$TMP_TARGET\"\n/usr/local/sbin/homelab-legacy-monitoring-backups-run:24: rm -f \"$STAGE/remote.tgz\"\n/usr/local/sbin/homelab-legacy-monitoring-backups-run:43: rm -f \"$STAGE/pve01.tgz\"\n/usr/local/sbin/homelab-legacy-monitoring-backups-run:48: rm -f \"$STAGE/$LABEL.tgz\"\n/usr/local/sbin/homelab-legacy-monitoring-backups-run:52: rm -f \"$STAGE/monitoring.tgz\"\n/usr/local/sbin/homelab-legacy-monitoring-backups-run:71: rm -f \"$STAGE/config.tgz\"\n/usr/local/sbin/homelab-cluster-admin-inventory-sync:19:ssh -n -o BatchMode=yes -o ConnectTimeout=8 \"$REMOTE\" \"rm -f $REMOTE_TMP\"\n/usr/local/sbin/homelab-chat-run:489: rm -f \\\n/usr/local/sbin/homelab-beszel-backup-run:33:rm -f \"$STAGE/remote.tgz\"\n/usr/local/sbin/backup-restore-check:12:rclone copy \"$RA\" \"$ROOT/a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-a.log 2>&1; homelab-rehydrate-chunked-dir \"$ROOT/a\" >>/tmp/homelab-restore-a.log 2>&1\n/usr/local/sbin/backup-restore-check:15:rclone copy \"$RB\" \"$ROOT/b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-b.log 2>&1; homelab-rehydrate-chunked-dir \"$ROOT/b\" >>/tmp/homelab-restore-b.log 2>&1\n/usr/local/sbin/homelab-vm-full-restore-drill:77: -name \"vzdump-qemu-${source_vmid}-*.vma.zst\" \\\n/usr/local/sbin/homelab-vm-backup-health:16:check_file /var/lib/homelab-health/edge-vm-vzdump-backup.txt edge-vm-backup\n/usr/local/sbin/homelab-vm-backup-health:17:check_file /var/lib/homelab-health/edge-vm-vzdump-offhost.txt edge-vm-offhost\n/usr/local/sbin/homelab-vm-backup-health:18:check_file /var/lib/homelab-health/edge-vm-vzdump-restore.txt edge-vm-restore\n/usr/local/sbin/homelab-ingress-failover:37: rm -f \"$RESOURCES\"\n/usr/local/sbin/homelab-ingress-failover:252: rm -f \"$tmp\"\n/usr/local/sbin/homelab-vm-full-backup:92:for cmd in vzdump qm zstd sha256sum rclone rsync ssh flock; do\n/usr/local/sbin/homelab-vm-full-backup:130: -name \"vzdump-qemu-${VMID}-*.vma.zst\" \\\n/usr/local/sbin/homelab-vm-full-backup:138: vzdump \"$VMID\" \\\n/usr/local/sbin/homelab-vm-full-backup:151: -name \"vzdump-qemu-${VMID}-*.vma.zst\" \\\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:10:TARGET_BASE=full/edge-vm-vzdump\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:64:case \"$SOURCE\" in /mnt/staging/offhost/edge-vm-vzdump-from-pve03/vzdump-qemu-130-*.vma.zst) ;; *) fail source_path_bad 2 ;; esac\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:172:P1 Mail Cloud edge-vm recurring chunked vzdump upload\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:190:rclone copy \"$PARTS_DIR\" \"$TARGET_REMOTE:$REMOTE_PARTS\" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > \"$WORK/logs/upload_parts.out\" 2>\"$WORK/logs/upload_parts.err\" || UPLOAD_PARTS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:192:rclone copyto \"$WORK/$BASENAME.sha256\" \"$TARGET_REMOTE:$REMOTE_SHA\" --checksum > \"$WORK/logs/upload_sha.out\" 2>\"$WORK/logs/upload_sha.err\" || UPLOAD_SHA_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:194:rclone copyto \"$WORK/MANIFEST.txt\" \"$TARGET_REMOTE:$REMOTE_MANIFEST\" --checksum > \"$WORK/logs/upload_manifest.out\" 2>\"$WORK/logs/upload_manifest.err\" || UPLOAD_MANIFEST_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:196:rclone copyto \"$WORK/SHA256SUMS.txt\" \"$TARGET_REMOTE:$REMOTE_SUMS\" --checksum > \"$WORK/logs/upload_sums.out\" 2>\"$WORK/logs/upload_sums.err\" || UPLOAD_SUMS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:199:rclone size \"$TARGET_REMOTE:$TARGET_DIR\" --json > \"$WORK/logs/remote_size.json\" 2>\"$WORK/logs/remote_size.err\" || SIZE_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:207:rclone copy \"$TARGET_REMOTE:$REMOTE_PARTS\" \"$DL_DIR/parts\" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > \"$WORK/logs/download_parts.out\" 2>\"$WORK/logs/download_parts.err\" || DOWNLOAD_PARTS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:239: [ -n \"$OLD\" ] && rclone purge \"$TARGET_REMOTE:$TARGET_BASE/$OLD\" >> \"$WORK/logs/retention.out\" 2>>\"$WORK/logs/retention.err\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:7:BASE_DIR=\"/mnt/staging/vzdump/edge-vm-vzdump\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:8:OFFHOST_DIR=\"/mnt/staging/offhost/edge-vm-vzdump-from-pve03\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:40:REMOTE_OUT=\"$(remote \"OUT=/tmp/edge_vm_vzdump_${VMID}_${TS}.out; ERR=/tmp/edge_vm_vzdump_${VMID}_${TS}.err; : > \\$OUT; : > \\$ERR; vzdump '$VMID' --mode snapshot --compress zstd --dumpdir '$RUN_DIR' --remove 0 >\\$OUT 2>\\$ERR; RC=\\$?; ARCHIVE=\\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\\n' | sort -n | tail -n1 | cut -d' ' -f2-); LOGFILE=\\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.log' -printf '%T@ %p\\n' | sort -n | tail -n1 | cut -d' ' -f2-); SIZE=0; SHA=none; if [ -n \\\"\\$ARCHIVE\\\" ] && [ -f \\\"\\$ARCHIVE\\\" ]; then SIZE=\\$(stat -c %s \\\"\\$ARCHIVE\\\"); SHA=\\$(sha256sum \\\"\\$ARCHIVE\\\" | awk '{print \\$1}'); fi; echo VZDUMP_RC=\\$RC; echo ARCHIVE=\\$ARCHIVE; echo LOGFILE=\\$LOGFILE; echo SIZE=\\$SIZE; echo SHA=\\$SHA; echo OUT_TAIL=\\$(tail -n 6 \\$OUT | tr '\\n' '|'); echo ERR_TAIL=\\$(tail -n 6 \\$ERR | tr '\\n' '|')\")\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:62:dest \"mkdir -p '$OFFHOST_DIR' && rm -f '$DEST_PATH.tmp' '$DEST_PATH.sha256'\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:74:echo \"STATUS=OK TS=$TS TYPE=edge-vm-vzdump-backup NODE=$NODE VMID=$VMID ARCHIVE=$ARCHIVE SIZE=$SIZE SHA256=$SHA SCSI1_BACKUP0=yes NOTES=timer_run\" > \"$HEALTH_DIR/edge-vm-vzdump-backup.txt\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:75:echo \"STATUS=OK TS=$TS TYPE=edge-vm-vzdump-offhost NODE=$NODE VMID=$VMID HOST=pve02 DEST=$DEST_PATH SIZE=$SIZE SHA256=$SHA NOTES=timer_run\" > \"$HEALTH_DIR/edge-vm-vzdump-offhost.txt\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:76:echo \"STATUS=OK TS=$TS TYPE=edge-vm-vzdump-restore-verify HOST=pve02 VMID=$VMID ARCHIVE=$DEST_PATH SIZE=$SIZE SHA256=$SHA ZSTD_RC=0 VMA_RC=0 NOTES=timer_run\" > \"$HEALTH_DIR/edge-vm-vzdump-restore.txt\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:77:chmod 0644 \"$HEALTH_DIR/edge-vm-vzdump-backup.txt\" \"$HEALTH_DIR/edge-vm-vzdump-offhost.txt\" \"$HEALTH_DIR/edge-vm-vzdump-restore.txt\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:80:dest \"find '$OFFHOST_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\\n' | sort -nr | awk 'NR>${KEEP_OFFHOST}{print \\$2}' | while read f; do rm -f \\\"\\$f\\\" \\\"\\$f.sha256\\\"; done\"\n/usr/local/sbin/appbackup-dockge-stacks:54:echo \"[STEP] upload cloud A metadata\"; rclone copy \"$STAGE\" \"$RA\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n/usr/local/sbin/appbackup-dockge-stacks:55:echo \"[STEP] upload cloud A archive chunks\"; rclone deletefile \"$RA/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RA/$ARCHIVE\"\n/usr/local/sbin/appbackup-dockge-stacks:56:echo \"[STEP] upload cloud B metadata\"; rclone copy \"$STAGE\" \"$RB\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n/usr/local/sbin/appbackup-dockge-stacks:57:echo \"[STEP] upload cloud B archive chunks\"; rclone deletefile \"$RB/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RB/$ARCHIVE\"\n/usr/local/sbin/appbackup-dockge-stacks:60:echo \"[STEP] upload latest index\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RAL\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RBL\"; rclone cat \"$RAL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null; rclone cat \"$RBL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n/usr/local/sbin/backup-retention:24: for ((i=0; i"$LOG" | ssh "${SSH[@]}" "$TARGET" "cat > '$PART'" +/usr/local/sbin/homelab-vm160-remote-backup:57:ssh "${SSH[@]}" "$TARGET" "cat > '$DIR/vzdump.log'" < "$LOG" +/usr/local/sbin/homelab-vm160-remote-backup:66:rm -f "$LOG" +/usr/local/sbin/homelab-mail-cloud-restore-drill:83: rm -f "$OUT" +/usr/local/sbin/homelab-mail-cloud-restore-drill:108: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/MANIFEST.txt" "$WORK/edge/MANIFEST.txt" > "$WORK/logs/edge_manifest_download.out" 2>"$WORK/logs/edge_manifest_download.err" +/usr/local/sbin/homelab-mail-cloud-restore-drill:111: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/SHA256SUMS.txt" "$WORK/edge/SHA256SUMS.txt" > "$WORK/logs/edge_sums_download.out" 2>"$WORK/logs/edge_sums_download.err" +/usr/local/sbin/homelab-mail-cloud-restore-drill:114: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/$EDGE_ARCHIVE.sha256" "$WORK/edge/$EDGE_ARCHIVE.sha256" > "$WORK/logs/edge_archive_sha_download.out" 2>"$WORK/logs/edge_archive_sha_download.err" +/usr/local/sbin/homelab-mail-cloud-restore-drill:130: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/parts/$FIRST_PART" "$WORK/edge/$FIRST_PART" > "$WORK/logs/edge_first_part_download.out" 2>"$WORK/logs/edge_first_part_download.err" +/usr/local/sbin/homelab-mail-cloud-restore-drill:137: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/parts/$LAST_PART" "$WORK/edge/$LAST_PART" > "$WORK/logs/edge_last_part_download.out" 2>"$WORK/logs/edge_last_part_download.err" +/usr/local/sbin/homelab-mail-cloud-restore-drill:168: rm -f "$WORK/edge/$FIRST_PART" "$WORK/edge/$LAST_PART" +/usr/local/sbin/homelab-golden-state-extended-overlay:26:rclone copyto "$JSON" "$RA" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-golden-state-extended-overlay:27:rclone copyto "$JSON" "$RB" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-mail-cloud-critical-upload:84: rclone copyto "$BUNDLE_SHA" "${R}:$REMOTE_DIR/$(basename "$BUNDLE_SHA")" --checksum > "$WORK/logs/${SAFE}_upload_sha.out" 2>"$WORK/logs/${SAFE}_upload_sha.err" +/usr/local/sbin/homelab-mail-cloud-critical-upload:86: rclone copyto "$WORK/components/MANIFEST.txt" "${R}:$REMOTE_DIR/MANIFEST.txt" --checksum > "$WORK/logs/${SAFE}_upload_manifest.out" 2>"$WORK/logs/${SAFE}_upload_manifest.err" +/usr/local/sbin/homelab-mail-cloud-critical-upload:88: rclone copyto "$WORK/components/SHA256SUMS.txt" "${R}:$REMOTE_DIR/SHA256SUMS.txt" --checksum > "$WORK/logs/${SAFE}_upload_sums.out" 2>"$WORK/logs/${SAFE}_upload_sums.err" +/usr/local/sbin/homelab-mail-cloud-critical-upload:108: rm -f "$WORK/download/${SAFE}-$(basename "$BUNDLE")" +/usr/local/sbin/homelab-mail-cloud-critical-upload:114: [ -n "$OLD" ] && rclone purge "${R}:$TARGET_BASE/$OLD" >> "$WORK/logs/${SAFE}_retention.out" 2>>"$WORK/logs/${SAFE}_retention.err" +/usr/local/sbin/homelab-alertmanager-backup-run:18:rm -f "$STAGE/remote.tgz" +/usr/local/sbin/postgres-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@"$H" "sudo rm -f /tmp/postgres-app-${APP}-${JOB_ID}.tgz" +/usr/local/sbin/homelab-stage-verify-flat-chunked:9:rclone copy "$REMOTE_DIR" "$VERIFY" --transfers 4 --checkers 8 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-rclone-cat-chunked:10:rclone copyto "${REMOTE_FILE}.parts/MANIFEST.parts.txt" "$MAN" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-stage-verify-chunked:9:rclone copy "$REMOTE_DIR" "$VERIFY" --exclude "$ARCHIVE.parts/**" --exclude "$ARCHIVE.parts/" --exclude "$ARCHIVE.sha256" --exclude "$ARCHIVE.manifest.txt" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --log-level ERROR +/usr/local/sbin/crypto-portfolio-backup-run:17:rm -f "$STAGE/remote.tgz" +/usr/local/sbin/npmplus-edge-backup-run:20:ssh -o BatchMode=yes -o ConnectTimeout=8 "$EDGE" "sudo rm -f '/tmp/npmplus-edge-${JOB_ID}.tgz'" +/usr/local/sbin/mariadb-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@"$H" "sudo rm -f /tmp/mariadb-app-${APP}-${JOB_ID}.tgz" +/usr/local/sbin/homelab-blackbox-exporter-backup-run:17:rm -f "$STAGE/remote.tgz" +/usr/local/sbin/homelab-ingress-standby-refresh:16: rm -f "$TMP_STATE" "$RELAY" "$RESOURCES" +/usr/local/sbin/homelab-ingress-standby-refresh:74:run_node "$SOURCE_NODE" vzdump 130 \ +/usr/local/sbin/homelab-ingress-standby-refresh:83:"find /var/lib/vz/dump -maxdepth 1 -type f -name 'vzdump-qemu-130-*.vma.zst' -printf '%T@ %p\n' | sort -rn | head -n 1 | cut -d' ' -f2-" +/usr/local/sbin/homelab-ingress-standby-refresh:106:"chattr -i '$DEST_FILE' '$DEST_FILE.sha256' '$DEST_FILE.env' 2>/dev/null || true; rm -f '$DEST_TMP'" +/usr/local/sbin/homelab-ingress-standby-refresh:155: -name 'vzdump-qemu-130-*.vma.zst' \ +/usr/local/sbin/homelab-ingress-standby-refresh:164: rm -f "$OLD" "$OLD.sha256" "$OLD.env" +/usr/local/sbin/homelab-ingress-standby-refresh:173:run_node "$SOURCE_NODE" rm -f "$SOURCE_BACKUP" +/usr/local/sbin/homelab-stage-2cloud-flat:38: for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f "$STAGE/$F" ]] && rclone copyto "$STAGE/$F" "$REMOTE_DIR/$F" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done +/usr/local/sbin/homelab-stage-2cloud-flat:41: rclone copyto "$STAGE/latest-backup-index.json" "$LATEST_FILE" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-backup-authentik:27:ssh debian@[PRIVATE_IP] "sudo docker rm -f authentik-restore-test >/dev/null 2>&1 || true" +/usr/local/sbin/homelab-backup-authentik:51:ssh debian@[PRIVATE_IP] "sudo docker rm -f authentik-restore-test >/dev/null 2>&1 || true; rm -f /tmp/authentik-restore-from-archive.pg_dump" +/usr/local/sbin/homelab-safe-exec:621: latest_tmp.unlink(missing_ok=True) +/usr/local/sbin/homelab-golden-state-index:28:rclone copyto "$JSON" "$RA" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-golden-state-index:29:rclone copyto "$JSON" "$RB" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-full-retention:21: -name "vzdump-qemu-${vmid}-*.vma.zst" \ +/usr/local/sbin/homelab-vm-full-retention:33: vzdump-qemu-"$vmid"-*.vma.zst) +/usr/local/sbin/homelab-vm-full-retention:35: rm -f \ +/usr/local/sbin/homelab-vm-full-retention:105: rclone purge "${path}/${dir}" +/usr/local/sbin/homelab-stage-2cloud:23:echo "[STEP] upload cloud A metadata"; rclone copy "$STAGE" "$RA" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s +/usr/local/sbin/homelab-stage-2cloud:24:echo "[STEP] upload cloud A archive chunks"; rclone deletefile "$RA/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RA/$ARCHIVE" +/usr/local/sbin/homelab-stage-2cloud:25:echo "[STEP] upload cloud B metadata"; rclone copy "$STAGE" "$RB" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s +/usr/local/sbin/homelab-stage-2cloud:26:echo "[STEP] upload cloud B archive chunks"; rclone deletefile "$RB/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RB/$ARCHIVE" +/usr/local/sbin/homelab-stage-2cloud:29:echo "[STEP] upload latest index"; rclone copyto "$STAGE/latest-backup-index.json" "$RAL"; rclone copyto "$STAGE/latest-backup-index.json" "$RBL"; rclone cat "$RAL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null; rclone cat "$RBL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null +/usr/local/sbin/homelab-backup-npmplus-kuma-configs:90: ssh -o BatchMode=yes -o ConnectTimeout=8 debian@[PRIVATE_IP] "rm -f '$EDGE_TGZ'" 2>>"$ERR" || true +/usr/local/sbin/homelab-stage-2cloud-flat-canary:36: for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f "$STAGE/$F" ]] && rclone copyto "$STAGE/$F" "$REMOTE_DIR/$F" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done +/usr/local/sbin/homelab-stage-2cloud-flat-canary:39: rclone copyto "$STAGE/latest-backup-index.json" "$LATEST_FILE" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-router-running-config-mail-cloud:62: rm -f "$CONFIG" +/usr/local/sbin/homelab-router-running-config-mail-cloud:66: rm -f "$CONFIG" +/usr/local/sbin/homelab-router-running-config-mail-cloud:79: if rclone mkdir "$REMOTE:$DEST" >"$WORK/rclone-mkdir.out" 2>"$WORK/rclone-mkdir.err" && homelab-rclone-copy-chunked "$ARCHIVE" "$REMOTE:$DEST/$(basename "$ARCHIVE")" >"$WORK/rclone-upload.out" 2>"$WORK/rclone-upload.err" && rclone copyto "$SHA_FILE" "$REMOTE:$DEST/$(basename "$SHA_FILE")" --checksum >"$WORK/rclone-upload-sha.out" 2>"$WORK/rclone-upload-sha.err"; then +/usr/local/sbin/homelab-router-running-config-mail-cloud:98: [ -n "$OLD" ] && rclone purge "$REMOTE:$TARGET_BASE/$OLD" >>"$WORK/retention.out" 2>>"$WORK/retention.err" +/usr/local/sbin/homelab-router-running-config-mail-cloud:103: rm -f "$old" "$old.sha256" +/usr/local/sbin/appbackup-restore-check:27:rclone copy "$RA" "$ROOT/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >"${LOGBASE}-a.log" 2>&1 +/usr/local/sbin/appbackup-restore-check:32:rclone copy "$RB" "$ROOT/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >"${LOGBASE}-b.log" 2>&1 +/usr/local/sbin/appbackup-restore-check:39:rm -f "${LOGBASE}-a.log" "${LOGBASE}-a-sha.log" "${LOGBASE}-b.log" "${LOGBASE}-b-sha.log" +/usr/local/sbin/homelab-capacity-risk:32:LARGEST_BYTES="$(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-*.vma.zst" -o -name "vzdump-lxc-*.tar.zst" \) -printf "%s\n" 2>/dev/null | sort -n | tail -n1)" +/usr/local/sbin/homelab-edge-vm-offhost-freshness:9:OFFHOST_DIR=/mnt/staging/offhost/edge-vm-vzdump-from-pve03 +/usr/local/sbin/homelab-edge-vm-offhost-freshness:85: /var/lib/vz/dump/vzdump-qemu-130-*.vma.zst) ;; +/usr/local/sbin/homelab-edge-vm-offhost-freshness:156:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-backup NODE=%s VMID=%s ARCHIVE=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\n' \ +/usr/local/sbin/homelab-edge-vm-offhost-freshness:158: >"$HEALTH_DIR/edge-vm-vzdump-backup.txt" +/usr/local/sbin/homelab-edge-vm-offhost-freshness:159:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-offhost NODE=%s VMID=%s HOST=%s DEST=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\n' \ +/usr/local/sbin/homelab-edge-vm-offhost-freshness:161: >"$HEALTH_DIR/edge-vm-vzdump-offhost.txt" +/usr/local/sbin/homelab-edge-vm-offhost-freshness:162:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-restore-verify HOST=%s VMID=%s ARCHIVE=%s SIZE=%s SHA256=%s ZSTD_RC=0 VMA_RC=0 NOTES=nightly_existing_dump_freshness\n' \ +/usr/local/sbin/homelab-edge-vm-offhost-freshness:164: >"$HEALTH_DIR/edge-vm-vzdump-restore.txt" +/usr/local/sbin/homelab-edge-vm-offhost-freshness:167: "$HEALTH_DIR/edge-vm-vzdump-backup.txt" \ +/usr/local/sbin/homelab-edge-vm-offhost-freshness:168: "$HEALTH_DIR/edge-vm-vzdump-offhost.txt" \ +/usr/local/sbin/homelab-edge-vm-offhost-freshness:169: "$HEALTH_DIR/edge-vm-vzdump-restore.txt" +/usr/local/sbin/homelab-external-probe-vps:20:rm -f "$TMP" +/usr/local/sbin/sqlite-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@"$H" "sudo rm -f /tmp/sqlite-app-${APP}-${JOB_ID}.tgz" +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:31: rclone copyto "$remote_path" "$out" >/dev/null +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:33: rm -f "$plist" +/usr/local/sbin/skladchik-reports-monitor-backup:83: for D in "${OLD[@]}"; do [ -n "$D" ] && rclone purge "${REMOTE}${REMOTE_BASE}/daily/${D}" >/dev/null 2>&1 || true; done +/usr/local/sbin/homelab-vm-local-dumps-retention:56: rclone purge "$PATH_REMOTE" >> "$REPORT" 2>&1 && DELETED=$((DELETED+1)) || BAD=$((BAD+1)) +/usr/local/sbin/homelab-mkdocs-auto-refresh:53: rm -f /tmp/homelab-mkdocs-auto-refresh-git-fetch.out /tmp/homelab-mkdocs-auto-refresh-git-fetch.err +/usr/local/sbin/homelab-mkdocs-auto-refresh:57:rm -f /tmp/homelab-mkdocs-auto-refresh-git-fetch.out /tmp/homelab-mkdocs-auto-refresh-git-fetch.err +/usr/local/sbin/homelab-vm-local-dumps-2cloud:42: F="$(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-${id}-*.vma.zst" -o -name "vzdump-lxc-${id}-*.tar.zst" \) -printf "%T@ %p\n" 2>/dev/null | sort -n | tail -n 1 | cut -d" " -f2-)" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:53: echo "$BN" | grep -q "^vzdump-qemu-" && KIND="qemu" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:54: echo "$BN" | grep -q "^vzdump-lxc-" && KIND="lxc" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:126: rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:128: rclone copyto "$CDIR/chunks.sha256" "$DEST/files/$bn.chunks/chunks.sha256" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:131: rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:135: rclone copyto "$CHKS" "$DEST/checksums.txt" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:136: rclone copyto "$FILES_TSV" "$DEST/files.tsv" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:137: rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:146: rclone copyto "$DEST/manifest.json" "$VBASE/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:154: rclone copy "$DEST/files/$bn.chunks" "$VDIR" --filter "+ part-*" --filter "+ chunks.sha256" --filter "- *" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:166: rclone copyto "$DEST/files/$bn" "$VBASE/$bn" --transfers 1 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:168: rm -f "$VBASE/$bn" +/usr/local/sbin/homelab-gitea-secondary-backup:18: rm -f "$TMP" +/usr/local/sbin/homelab-gitea-secondary-backup:31:docker exec -u git -w /tmp gitea sh -lc \"rm -f /tmp/\$NAME && gitea dump -c /data/gitea/conf/app.ini -f /tmp/\$NAME\" +/usr/local/sbin/homelab-gitea-secondary-backup:33:docker exec gitea rm -f \"/tmp/\$NAME\" +/usr/local/sbin/homelab-gitea-secondary-backup:100: rm -f -- "$FILE" "$FILE.sha256" +/usr/local/sbin/homelab-gitea-secondary-backup:123: rm -f -- "$FILE" "$FILE.sha256" +/usr/local/sbin/homelab-gitea-secondary-backup:136: rm -f -- "$FILE" "$FILE.sha256" +/usr/local/sbin/homelab-rclone-copy-chunked:24:rclone purge "${DEST}.parts" >/dev/null 2>&1 || true +/usr/local/sbin/homelab-rclone-copy-chunked:26:rclone copy "$PARTS" "${DEST}.parts" --transfers 2 --checkers 4 --retries 5 --low-level-retries 20 --stats 30s --log-level INFO +/usr/local/sbin/homelab-rclone-copy-chunked:27:rclone copyto "$WORK/${NAME}.sha256" "${DEST}.sha256" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO +/usr/local/sbin/homelab-rclone-copy-chunked:28:rclone copyto "$WORK/MANIFEST.txt" "${DEST}.manifest.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO +/usr/local/sbin/homelab-rclone-copy-chunked:29:rclone copyto "$WORK/MANIFEST.parts.txt" "${DEST}.parts/MANIFEST.parts.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO +/usr/local/sbin/homelab-health-metrics-write:10:rm -f "$tmp" +/usr/local/sbin/homelab-rclone-cat-flat-chunked:5:trap "rm -f \"$TMP\"" EXIT +/usr/local/sbin/homelab-vm-backup-policy:29:echo "## Latest local vzdump catalog" >> "$REPORT" +/usr/local/sbin/homelab-vm-backup-policy:31:for f in $(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-*.vma.zst" -o -name "vzdump-lxc-*.tar.zst" \) 2>/dev/null | sort); do b="$(basename "$f")"; size="$(stat -c %s "$f" 2>/dev/null || echo 0)"; mt="$(date -u -d "@$(stat -c %Y "$f" 2>/dev/null || echo 0)" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)"; if echo "$b" | grep -q "^vzdump-qemu-"; then kind=qemu; vmid="$(echo "$b" | sed -n "s/^vzdump-qemu-\([0-9]\+\)-.*/\1/p")"; name="$(qm config "$vmid" 2>/dev/null | awk -F": " "/^name:/{print \$2; exit}")"; else kind=lxc; vmid="$(echo "$b" | sed -n "s/^vzdump-lxc-\([0-9]\+\)-.*/\1/p")"; name="$(pct config "$vmid" 2>/dev/null | awk -F": " "/^hostname:/{print \$2; exit}")"; fi; printf "%s\t%s\t%s\t%s\t%s\t%s\n" "$kind" "$vmid" "${name:-unknown}" "$size" "$mt" "$f" >> "$CATALOG"; done +/usr/local/sbin/homelab-vm-backup-policy:37:echo "- Existing local dumps can be uploaded/verified to two clouds without creating new vzdump archives." >> "$REPORT" +/usr/local/sbin/homelab-vm-backup-policy:38:echo "- New full vzdump for VM 170/171 is not started automatically because each VM has 120 GB bootdisk and /var/tmp has limited free space." >> "$REPORT" +/usr/local/sbin/homelab-context-refresh-direct:34:ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] debian@[PRIVATE_IP] "sudo -n /usr/local/sbin/homelab-runtime-receive $id /tmp/$id.json /tmp/$id.txt && rm -f /tmp/$id.json /tmp/$id.txt" +/usr/local/sbin/homelab-evidence-seal:16: rm -f "$TMP" +/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:15:DUMP_SIZE="$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$REMOTE" "sudo -n rm -f \"$REMOTE_DUMP\"; sudo -n -u clusteradmin pg_dump --format=custom --no-owner --no-privileges --file=\"$REMOTE_DUMP\" clusteradmin; sudo -n test -s \"$REMOTE_DUMP\"; sudo -n -u clusteradmin pg_restore --list \"$REMOTE_DUMP\" >/dev/null; sudo -n chown debian:debian \"$REMOTE_DUMP\"; sudo -n chmod 600 \"$REMOTE_DUMP\"; stat -c %s \"$REMOTE_DUMP\"")" +/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:18:ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$REMOTE" "rm -f \"$REMOTE_DUMP\"" +/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:23:rm -f "$STAGE/runtime.tgz" +/usr/local/sbin/appbackup-retention:85: if [[ "$MODE" == apply ]]; then echo "DELETE schema=dual path=$REL"; rclone purge "$TARGET"; else echo "WOULD_DELETE schema=dual path=$REL"; fi +/usr/local/sbin/homelab-change-transaction:195: latest_tmp.unlink(missing_ok=True) +/usr/local/sbin/homelab-check-private-vpn-hosts:15:trap 'rm -f "$tmp"' EXIT +/usr/local/sbin/homelab-health-metrics:22: *backup*|*vzdump*) printf "%s" "backup" ;; +/usr/local/sbin/homelab-health-metrics:34: npmplus-kuma-config-*|authentik-*|gitea-*|vaultwarden-*|netbox-*|mail-cloud-restore-drill|edge-vm-vzdump-*|mkdocs-runtime-publish|npmplus-certificate-expiry) printf "%s" "P0" ;; +/usr/local/sbin/crypto-portfolio-chart-health-check:5:trap 'rm -f "$TMP"' EXIT +/usr/local/sbin/homelab-rclone-copy-flat-chunked:23:while read -r PART PART_SIZE; do [[ "$PART" =~ ^part-[0-9]{4}$ && "$PART_SIZE" =~ ^[0-9]+$ ]] || { echo "BAD PART MANIFEST" >&2; exit 4; }; rclone copyto "$WORK/parts/$PART" "${DEST}.${PART}" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR; REMOTE_BYTES="$(rclone size --json "${DEST}.${PART}" | jq -r ".bytes // -1")"; [[ "$REMOTE_BYTES" = "$PART_SIZE" ]] || { echo "REMOTE PART SIZE MISMATCH $PART" >&2; exit 5; }; UPLOADED=$((UPLOADED+1)); done < "$WORK/MANIFEST.parts.txt" +/usr/local/sbin/homelab-rclone-copy-flat-chunked:25:rclone copyto "$WORK/${NAME}.sha256" "${DEST}.sha256" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-rclone-copy-flat-chunked:26:rclone copyto "$WORK/MANIFEST.txt" "${DEST}.manifest.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-rclone-copy-flat-chunked:27:rclone copyto "$WORK/MANIFEST.parts.txt" "${DEST}.parts.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:8:TARGET_BASE=full/nextcloud-vm-vzdump +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:11:SRC_GLOB=/var/lib/vz/dump/vzdump-qemu-150-*.vma.zst +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:77:P1 Mail Cloud Nextcloud VM recurring chunked vzdump upload +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:95:rclone copy "$PARTS_DIR" "$TARGET_REMOTE:$REMOTE_PARTS" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > "$WORK/logs/upload_parts.out" 2>"$WORK/logs/upload_parts.err" || UPLOAD_PARTS_RC=$? +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:97:rclone copyto "$WORK/$BASENAME.sha256" "$TARGET_REMOTE:$REMOTE_SHA" --checksum > "$WORK/logs/upload_sha.out" 2>"$WORK/logs/upload_sha.err" || UPLOAD_SHA_RC=$? +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:99:rclone copyto "$WORK/MANIFEST.txt" "$TARGET_REMOTE:$REMOTE_MANIFEST" --checksum > "$WORK/logs/upload_manifest.out" 2>"$WORK/logs/upload_manifest.err" || UPLOAD_MANIFEST_RC=$? +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:101:rclone copyto "$WORK/SHA256SUMS.txt" "$TARGET_REMOTE:$REMOTE_SUMS" --checksum > "$WORK/logs/upload_sums.out" 2>"$WORK/logs/upload_sums.err" || UPLOAD_SUMS_RC=$? +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:110:rclone copy "$TARGET_REMOTE:$REMOTE_PARTS" "$DL_DIR/parts" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > "$WORK/logs/download_parts.out" 2>"$WORK/logs/download_parts.err" || DOWNLOAD_PARTS_RC=$? +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:126: [ -n "$OLD" ] && rclone purge "$TARGET_REMOTE:$TARGET_BASE/$OLD" >> "$WORK/logs/retention.out" 2>>"$WORK/logs/retention.err" +/usr/local/sbin/homelab-rehydrate-flat-chunked-dir:10: rm -f "$TMP_TARGET" +/usr/local/sbin/homelab-legacy-monitoring-backups-run:24: rm -f "$STAGE/remote.tgz" +/usr/local/sbin/homelab-legacy-monitoring-backups-run:43: rm -f "$STAGE/pve01.tgz" +/usr/local/sbin/homelab-legacy-monitoring-backups-run:48: rm -f "$STAGE/$LABEL.tgz" +/usr/local/sbin/homelab-legacy-monitoring-backups-run:52: rm -f "$STAGE/monitoring.tgz" +/usr/local/sbin/homelab-legacy-monitoring-backups-run:71: rm -f "$STAGE/config.tgz" +/usr/local/sbin/homelab-cluster-admin-inventory-sync:19:ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$REMOTE" "rm -f $REMOTE_TMP" +/usr/local/sbin/homelab-chat-run:489: rm -f \ +/usr/local/sbin/homelab-beszel-backup-run:33:rm -f "$STAGE/remote.tgz" +/usr/local/sbin/backup-restore-check:12:rclone copy "$RA" "$ROOT/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-a.log 2>&1; homelab-rehydrate-chunked-dir "$ROOT/a" >>/tmp/homelab-restore-a.log 2>&1 +/usr/local/sbin/backup-restore-check:15:rclone copy "$RB" "$ROOT/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-b.log 2>&1; homelab-rehydrate-chunked-dir "$ROOT/b" >>/tmp/homelab-restore-b.log 2>&1 +/usr/local/sbin/homelab-vm-full-restore-drill:77: -name "vzdump-qemu-${source_vmid}-*.vma.zst" \ +/usr/local/sbin/homelab-vm-backup-health:16:check_file /var/lib/homelab-health/edge-vm-vzdump-backup.txt edge-vm-backup +/usr/local/sbin/homelab-vm-backup-health:17:check_file /var/lib/homelab-health/edge-vm-vzdump-offhost.txt edge-vm-offhost +/usr/local/sbin/homelab-vm-backup-health:18:check_file /var/lib/homelab-health/edge-vm-vzdump-restore.txt edge-vm-restore +/usr/local/sbin/homelab-ingress-failover:37: rm -f "$RESOURCES" +/usr/local/sbin/homelab-ingress-failover:252: rm -f "$tmp" +/usr/local/sbin/homelab-vm-full-backup:92:for cmd in vzdump qm zstd sha256sum rclone rsync ssh flock; do +/usr/local/sbin/homelab-vm-full-backup:130: -name "vzdump-qemu-${VMID}-*.vma.zst" \ +/usr/local/sbin/homelab-vm-full-backup:138: vzdump "$VMID" \ +/usr/local/sbin/homelab-vm-full-backup:151: -name "vzdump-qemu-${VMID}-*.vma.zst" \ +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:10:TARGET_BASE=full/edge-vm-vzdump +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:64:case "$SOURCE" in /mnt/staging/offhost/edge-vm-vzdump-from-pve03/vzdump-qemu-130-*.vma.zst) ;; *) fail source_path_bad 2 ;; esac +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:172:P1 Mail Cloud edge-vm recurring chunked vzdump upload +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:190:rclone copy "$PARTS_DIR" "$TARGET_REMOTE:$REMOTE_PARTS" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > "$WORK/logs/upload_parts.out" 2>"$WORK/logs/upload_parts.err" || UPLOAD_PARTS_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:192:rclone copyto "$WORK/$BASENAME.sha256" "$TARGET_REMOTE:$REMOTE_SHA" --checksum > "$WORK/logs/upload_sha.out" 2>"$WORK/logs/upload_sha.err" || UPLOAD_SHA_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:194:rclone copyto "$WORK/MANIFEST.txt" "$TARGET_REMOTE:$REMOTE_MANIFEST" --checksum > "$WORK/logs/upload_manifest.out" 2>"$WORK/logs/upload_manifest.err" || UPLOAD_MANIFEST_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:196:rclone copyto "$WORK/SHA256SUMS.txt" "$TARGET_REMOTE:$REMOTE_SUMS" --checksum > "$WORK/logs/upload_sums.out" 2>"$WORK/logs/upload_sums.err" || UPLOAD_SUMS_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:199:rclone size "$TARGET_REMOTE:$TARGET_DIR" --json > "$WORK/logs/remote_size.json" 2>"$WORK/logs/remote_size.err" || SIZE_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:207:rclone copy "$TARGET_REMOTE:$REMOTE_PARTS" "$DL_DIR/parts" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > "$WORK/logs/download_parts.out" 2>"$WORK/logs/download_parts.err" || DOWNLOAD_PARTS_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:239: [ -n "$OLD" ] && rclone purge "$TARGET_REMOTE:$TARGET_BASE/$OLD" >> "$WORK/logs/retention.out" 2>>"$WORK/logs/retention.err" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:7:BASE_DIR="/mnt/staging/vzdump/edge-vm-vzdump" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:8:OFFHOST_DIR="/mnt/staging/offhost/edge-vm-vzdump-from-pve03" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:40:REMOTE_OUT="$(remote "OUT=/tmp/edge_vm_vzdump_${VMID}_${TS}.out; ERR=/tmp/edge_vm_vzdump_${VMID}_${TS}.err; : > \$OUT; : > \$ERR; vzdump '$VMID' --mode snapshot --compress zstd --dumpdir '$RUN_DIR' --remove 0 >\$OUT 2>\$ERR; RC=\$?; ARCHIVE=\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\n' | sort -n | tail -n1 | cut -d' ' -f2-); LOGFILE=\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.log' -printf '%T@ %p\n' | sort -n | tail -n1 | cut -d' ' -f2-); SIZE=0; SHA=none; if [ -n \"\$ARCHIVE\" ] && [ -f \"\$ARCHIVE\" ]; then SIZE=\$(stat -c %s \"\$ARCHIVE\"); SHA=\$(sha256sum \"\$ARCHIVE\" | awk '{print \$1}'); fi; echo VZDUMP_RC=\$RC; echo ARCHIVE=\$ARCHIVE; echo LOGFILE=\$LOGFILE; echo SIZE=\$SIZE; echo SHA=\$SHA; echo OUT_TAIL=\$(tail -n 6 \$OUT | tr '\n' '|'); echo ERR_TAIL=\$(tail -n 6 \$ERR | tr '\n' '|')")" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:62:dest "mkdir -p '$OFFHOST_DIR' && rm -f '$DEST_PATH.tmp' '$DEST_PATH.sha256'" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:74:echo "STATUS=OK TS=$TS TYPE=edge-vm-vzdump-backup NODE=$NODE VMID=$VMID ARCHIVE=$ARCHIVE SIZE=$SIZE SHA256=$SHA SCSI1_BACKUP0=yes NOTES=timer_run" > "$HEALTH_DIR/edge-vm-vzdump-backup.txt" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:75:echo "STATUS=OK TS=$TS TYPE=edge-vm-vzdump-offhost NODE=$NODE VMID=$VMID HOST=pve02 DEST=$DEST_PATH SIZE=$SIZE SHA256=$SHA NOTES=timer_run" > "$HEALTH_DIR/edge-vm-vzdump-offhost.txt" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:76:echo "STATUS=OK TS=$TS TYPE=edge-vm-vzdump-restore-verify HOST=pve02 VMID=$VMID ARCHIVE=$DEST_PATH SIZE=$SIZE SHA256=$SHA ZSTD_RC=0 VMA_RC=0 NOTES=timer_run" > "$HEALTH_DIR/edge-vm-vzdump-restore.txt" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:77:chmod 0644 "$HEALTH_DIR/edge-vm-vzdump-backup.txt" "$HEALTH_DIR/edge-vm-vzdump-offhost.txt" "$HEALTH_DIR/edge-vm-vzdump-restore.txt" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:80:dest "find '$OFFHOST_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\n' | sort -nr | awk 'NR>${KEEP_OFFHOST}{print \$2}' | while read f; do rm -f \"\$f\" \"\$f.sha256\"; done" +/usr/local/sbin/appbackup-dockge-stacks:54:echo "[STEP] upload cloud A metadata"; rclone copy "$STAGE" "$RA" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s +/usr/local/sbin/appbackup-dockge-stacks:55:echo "[STEP] upload cloud A archive chunks"; rclone deletefile "$RA/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RA/$ARCHIVE" +/usr/local/sbin/appbackup-dockge-stacks:56:echo "[STEP] upload cloud B metadata"; rclone copy "$STAGE" "$RB" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s +/usr/local/sbin/appbackup-dockge-stacks:57:echo "[STEP] upload cloud B archive chunks"; rclone deletefile "$RB/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RB/$ARCHIVE" +/usr/local/sbin/appbackup-dockge-stacks:60:echo "[STEP] upload latest index"; rclone copyto "$STAGE/latest-backup-index.json" "$RAL"; rclone copyto "$STAGE/latest-backup-index.json" "$RBL"; rclone cat "$RAL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null; rclone cat "$RBL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null +/usr/local/sbin/backup-retention:24: for ((i=0; i\"$LOG\" | ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$PART'\"\n/usr/local/sbin/homelab-vm160-remote-backup:57:ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$DIR/vzdump.log'\" < \"$LOG\"\n/usr/local/sbin/homelab-vm160-remote-backup:66:rm -f \"$LOG\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:83: rm -f \"$OUT\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:108: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/MANIFEST.txt\" \"$WORK/edge/MANIFEST.txt\" > \"$WORK/logs/edge_manifest_download.out\" 2>\"$WORK/logs/edge_manifest_download.err\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:111: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/SHA256SUMS.txt\" \"$WORK/edge/SHA256SUMS.txt\" > \"$WORK/logs/edge_sums_download.out\" 2>\"$WORK/logs/edge_sums_download.err\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:114: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/$EDGE_ARCHIVE.sha256\" \"$WORK/edge/$EDGE_ARCHIVE.sha256\" > \"$WORK/logs/edge_archive_sha_download.out\" 2>\"$WORK/logs/edge_archive_sha_download.err\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:130: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/parts/$FIRST_PART\" \"$WORK/edge/$FIRST_PART\" > \"$WORK/logs/edge_first_part_download.out\" 2>\"$WORK/logs/edge_first_part_download.err\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:137: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/parts/$LAST_PART\" \"$WORK/edge/$LAST_PART\" > \"$WORK/logs/edge_last_part_download.out\" 2>\"$WORK/logs/edge_last_part_download.err\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:168: rm -f \"$WORK/edge/$FIRST_PART\" \"$WORK/edge/$LAST_PART\"\n/usr/local/sbin/homelab-golden-state-extended-overlay:26:rclone copyto \"$JSON\" \"$RA\" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-golden-state-extended-overlay:27:rclone copyto \"$JSON\" \"$RB\" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-mail-cloud-critical-upload:84: rclone copyto \"$BUNDLE_SHA\" \"${R}:$REMOTE_DIR/$(basename \"$BUNDLE_SHA\")\" --checksum > \"$WORK/logs/${SAFE}_upload_sha.out\" 2>\"$WORK/logs/${SAFE}_upload_sha.err\"\n/usr/local/sbin/homelab-mail-cloud-critical-upload:86: rclone copyto \"$WORK/components/MANIFEST.txt\" \"${R}:$REMOTE_DIR/MANIFEST.txt\" --checksum > \"$WORK/logs/${SAFE}_upload_manifest.out\" 2>\"$WORK/logs/${SAFE}_upload_manifest.err\"\n/usr/local/sbin/homelab-mail-cloud-critical-upload:88: rclone copyto \"$WORK/components/SHA256SUMS.txt\" \"${R}:$REMOTE_DIR/SHA256SUMS.txt\" --checksum > \"$WORK/logs/${SAFE}_upload_sums.out\" 2>\"$WORK/logs/${SAFE}_upload_sums.err\"\n/usr/local/sbin/homelab-mail-cloud-critical-upload:108: rm -f \"$WORK/download/${SAFE}-$(basename \"$BUNDLE\")\"\n/usr/local/sbin/homelab-mail-cloud-critical-upload:114: [ -n \"$OLD\" ] && rclone purge \"${R}:$TARGET_BASE/$OLD\" >> \"$WORK/logs/${SAFE}_retention.out\" 2>>\"$WORK/logs/${SAFE}_retention.err\"\n/usr/local/sbin/homelab-alertmanager-backup-run:18:rm -f \"$STAGE/remote.tgz\"\n/usr/local/sbin/postgres-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@\"$H\" \"sudo rm -f /tmp/postgres-app-${APP}-${JOB_ID}.tgz\"\n/usr/local/sbin/homelab-stage-verify-flat-chunked:9:rclone copy \"$REMOTE_DIR\" \"$VERIFY\" --transfers 4 --checkers 8 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-rclone-cat-chunked:10:rclone copyto \"${REMOTE_FILE}.parts/MANIFEST.parts.txt\" \"$MAN\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-stage-verify-chunked:9:rclone copy \"$REMOTE_DIR\" \"$VERIFY\" --exclude \"$ARCHIVE.parts/**\" --exclude \"$ARCHIVE.parts/\" --exclude \"$ARCHIVE.sha256\" --exclude \"$ARCHIVE.manifest.txt\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --log-level ERROR\n/usr/local/sbin/crypto-portfolio-backup-run:17:rm -f \"$STAGE/remote.tgz\"\n/usr/local/sbin/npmplus-edge-backup-run:20:ssh -o BatchMode=yes -o ConnectTimeout=8 \"$EDGE\" \"sudo rm -f '/tmp/npmplus-edge-${JOB_ID}.tgz'\"\n/usr/local/sbin/mariadb-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@\"$H\" \"sudo rm -f /tmp/mariadb-app-${APP}-${JOB_ID}.tgz\"\n/usr/local/sbin/homelab-blackbox-exporter-backup-run:17:rm -f \"$STAGE/remote.tgz\"\n/usr/local/sbin/homelab-ingress-standby-refresh:16: rm -f \"$TMP_STATE\" \"$RELAY\" \"$RESOURCES\"\n/usr/local/sbin/homelab-ingress-standby-refresh:74:run_node \"$SOURCE_NODE\" vzdump 130 \\\n/usr/local/sbin/homelab-ingress-standby-refresh:83:\"find /var/lib/vz/dump -maxdepth 1 -type f -name 'vzdump-qemu-130-*.vma.zst' -printf '%T@ %p\\n' | sort -rn | head -n 1 | cut -d' ' -f2-\"\n/usr/local/sbin/homelab-ingress-standby-refresh:106:\"chattr -i '$DEST_FILE' '$DEST_FILE.sha256' '$DEST_FILE.env' 2>/dev/null || true; rm -f '$DEST_TMP'\"\n/usr/local/sbin/homelab-ingress-standby-refresh:155: -name 'vzdump-qemu-130-*.vma.zst' \\\n/usr/local/sbin/homelab-ingress-standby-refresh:164: rm -f \"$OLD\" \"$OLD.sha256\" \"$OLD.env\"\n/usr/local/sbin/homelab-ingress-standby-refresh:173:run_node \"$SOURCE_NODE\" rm -f \"$SOURCE_BACKUP\"\n/usr/local/sbin/homelab-stage-2cloud-flat:38: for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f \"$STAGE/$F\" ]] && rclone copyto \"$STAGE/$F\" \"$REMOTE_DIR/$F\" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done\n/usr/local/sbin/homelab-stage-2cloud-flat:41: rclone copyto \"$STAGE/latest-backup-index.json\" \"$LATEST_FILE\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-backup-authentik:27:ssh debian@[PRIVATE_IP] \"sudo docker rm -f authentik-restore-test >/dev/null 2>&1 || true\"\n/usr/local/sbin/homelab-backup-authentik:51:ssh debian@[PRIVATE_IP] \"sudo docker rm -f authentik-restore-test >/dev/null 2>&1 || true; rm -f /tmp/authentik-restore-from-archive.pg_dump\"\n/usr/local/sbin/homelab-safe-exec:621: latest_tmp.unlink(missing_ok=True)\n/usr/local/sbin/homelab-golden-state-index:28:rclone copyto \"$JSON\" \"$RA\" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-golden-state-index:29:rclone copyto \"$JSON\" \"$RB\" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-full-retention:21: -name \"vzdump-qemu-${vmid}-*.vma.zst\" \\\n/usr/local/sbin/homelab-vm-full-retention:33: vzdump-qemu-\"$vmid\"-*.vma.zst)\n/usr/local/sbin/homelab-vm-full-retention:35: rm -f \\\n/usr/local/sbin/homelab-vm-full-retention:105: rclone purge \"${path}/${dir}\"\n/usr/local/sbin/homelab-stage-2cloud:23:echo \"[STEP] upload cloud A metadata\"; rclone copy \"$STAGE\" \"$RA\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n/usr/local/sbin/homelab-stage-2cloud:24:echo \"[STEP] upload cloud A archive chunks\"; rclone deletefile \"$RA/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RA/$ARCHIVE\"\n/usr/local/sbin/homelab-stage-2cloud:25:echo \"[STEP] upload cloud B metadata\"; rclone copy \"$STAGE\" \"$RB\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n/usr/local/sbin/homelab-stage-2cloud:26:echo \"[STEP] upload cloud B archive chunks\"; rclone deletefile \"$RB/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RB/$ARCHIVE\"\n/usr/local/sbin/homelab-stage-2cloud:29:echo \"[STEP] upload latest index\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RAL\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RBL\"; rclone cat \"$RAL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null; rclone cat \"$RBL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n/usr/local/sbin/homelab-backup-npmplus-kuma-configs:90: ssh -o BatchMode=yes -o ConnectTimeout=8 debian@[PRIVATE_IP] \"rm -f '$EDGE_TGZ'\" 2>>\"$ERR\" || true\n/usr/local/sbin/homelab-stage-2cloud-flat-canary:36: for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f \"$STAGE/$F\" ]] && rclone copyto \"$STAGE/$F\" \"$REMOTE_DIR/$F\" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done\n/usr/local/sbin/homelab-stage-2cloud-flat-canary:39: rclone copyto \"$STAGE/latest-backup-index.json\" \"$LATEST_FILE\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-router-running-config-mail-cloud:62: rm -f \"$CONFIG\"\n/usr/local/sbin/homelab-router-running-config-mail-cloud:66: rm -f \"$CONFIG\"\n/usr/local/sbin/homelab-router-running-config-mail-cloud:79: if rclone mkdir \"$REMOTE:$DEST\" >\"$WORK/rclone-mkdir.out\" 2>\"$WORK/rclone-mkdir.err\" && homelab-rclone-copy-chunked \"$ARCHIVE\" \"$REMOTE:$DEST/$(basename \"$ARCHIVE\")\" >\"$WORK/rclone-upload.out\" 2>\"$WORK/rclone-upload.err\" && rclone copyto \"$SHA_FILE\" \"$REMOTE:$DEST/$(basename \"$SHA_FILE\")\" --checksum >\"$WORK/rclone-upload-sha.out\" 2>\"$WORK/rclone-upload-sha.err\"; then\n/usr/local/sbin/homelab-router-running-config-mail-cloud:98: [ -n \"$OLD\" ] && rclone purge \"$REMOTE:$TARGET_BASE/$OLD\" >>\"$WORK/retention.out\" 2>>\"$WORK/retention.err\"\n/usr/local/sbin/homelab-router-running-config-mail-cloud:103: rm -f \"$old\" \"$old.sha256\"\n/usr/local/sbin/appbackup-restore-check:27:rclone copy \"$RA\" \"$ROOT/a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >\"${LOGBASE}-a.log\" 2>&1\n/usr/local/sbin/appbackup-restore-check:32:rclone copy \"$RB\" \"$ROOT/b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >\"${LOGBASE}-b.log\" 2>&1\n/usr/local/sbin/appbackup-restore-check:39:rm -f \"${LOGBASE}-a.log\" \"${LOGBASE}-a-sha.log\" \"${LOGBASE}-b.log\" \"${LOGBASE}-b-sha.log\"\n/usr/local/sbin/homelab-capacity-risk:32:LARGEST_BYTES=\"$(find /var/lib/vz/dump -maxdepth 1 -type f \\( -name \"vzdump-qemu-*.vma.zst\" -o -name \"vzdump-lxc-*.tar.zst\" \\) -printf \"%s\\n\" 2>/dev/null | sort -n | tail -n1)\"\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:9:OFFHOST_DIR=/mnt/staging/offhost/edge-vm-vzdump-from-pve03\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:85: /var/lib/vz/dump/vzdump-qemu-130-*.vma.zst) ;;\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:156:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-backup NODE=%s VMID=%s ARCHIVE=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\\n' \\\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:158: >\"$HEALTH_DIR/edge-vm-vzdump-backup.txt\"\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:159:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-offhost NODE=%s VMID=%s HOST=%s DEST=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\\n' \\\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:161: >\"$HEALTH_DIR/edge-vm-vzdump-offhost.txt\"\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:162:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-restore-verify HOST=%s VMID=%s ARCHIVE=%s SIZE=%s SHA256=%s ZSTD_RC=0 VMA_RC=0 NOTES=nightly_existing_dump_freshness\\n' \\\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:164: >\"$HEALTH_DIR/edge-vm-vzdump-restore.txt\"\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:167: \"$HEALTH_DIR/edge-vm-vzdump-backup.txt\" \\\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:168: \"$HEALTH_DIR/edge-vm-vzdump-offhost.txt\" \\\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:169: \"$HEALTH_DIR/edge-vm-vzdump-restore.txt\"\n/usr/local/sbin/homelab-external-probe-vps:20:rm -f \"$TMP\"\n/usr/local/sbin/sqlite-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@\"$H\" \"sudo rm -f /tmp/sqlite-app-${APP}-${JOB_ID}.tgz\"\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:31: rclone copyto \"$remote_path\" \"$out\" >/dev/null\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:33: rm -f \"$plist\"\n/usr/local/sbin/skladchik-reports-monitor-backup:83: for D in \"${OLD[@]}\"; do [ -n \"$D\" ] && rclone purge \"${REMOTE}${REMOTE_BASE}/daily/${D}\" >/dev/null 2>&1 || true; done\n/usr/local/sbin/homelab-vm-local-dumps-retention:56: rclone purge \"$PATH_REMOTE\" >> \"$REPORT\" 2>&1 && DELETED=$((DELETED+1)) || BAD=$((BAD+1))\n/usr/local/sbin/homelab-mkdocs-auto-refresh:53: rm -f /tmp/homelab-mkdocs-auto-refresh-git-fetch.out /tmp/homelab-mkdocs-auto-refresh-git-fetch.err\n/usr/local/sbin/homelab-mkdocs-auto-refresh:57:rm -f /tmp/homelab-mkdocs-auto-refresh-git-fetch.out /tmp/homelab-mkdocs-auto-refresh-git-fetch.err\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:42: F=\"$(find /var/lib/vz/dump -maxdepth 1 -type f \\( -name \"vzdump-qemu-${id}-*.vma.zst\" -o -name \"vzdump-lxc-${id}-*.tar.zst\" \\) -printf \"%T@ %p\\n\" 2>/dev/null | sort -n | tail -n 1 | cut -d\" \" -f2-)\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:53: echo \"$BN\" | grep -q \"^vzdump-qemu-\" && KIND=\"qemu\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:54: echo \"$BN\" | grep -q \"^vzdump-lxc-\" && KIND=\"lxc\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:126: rclone copyto \"$part\" \"$DEST/files/$bn.chunks/$pn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:128: rclone copyto \"$CDIR/chunks.sha256\" \"$DEST/files/$bn.chunks/chunks.sha256\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:131: rclone copyto \"$path\" \"$DEST/files/$bn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:135: rclone copyto \"$CHKS\" \"$DEST/checksums.txt\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:136: rclone copyto \"$FILES_TSV\" \"$DEST/files.tsv\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:137: rclone copyto \"$MAN\" \"$DEST/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:146: rclone copyto \"$DEST/manifest.json\" \"$VBASE/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:154: rclone copy \"$DEST/files/$bn.chunks\" \"$VDIR\" --filter \"+ part-*\" --filter \"+ chunks.sha256\" --filter \"- *\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:166: rclone copyto \"$DEST/files/$bn\" \"$VBASE/$bn\" --transfers 1 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:168: rm -f \"$VBASE/$bn\"\n/usr/local/sbin/homelab-gitea-secondary-backup:18: rm -f \"$TMP\"\n/usr/local/sbin/homelab-gitea-secondary-backup:31:docker exec -u git -w /tmp gitea sh -lc \\\"rm -f /tmp/\\$NAME && gitea dump -c /data/gitea/conf/app.ini -f /tmp/\\$NAME\\\"\n/usr/local/sbin/homelab-gitea-secondary-backup:33:docker exec gitea rm -f \\\"/tmp/\\$NAME\\\"\n/usr/local/sbin/homelab-gitea-secondary-backup:100: rm -f -- \"$FILE\" \"$FILE.sha256\"\n/usr/local/sbin/homelab-gitea-secondary-backup:123: rm -f -- \"$FILE\" \"$FILE.sha256\"\n/usr/local/sbin/homelab-gitea-secondary-backup:136: rm -f -- \"$FILE\" \"$FILE.sha256\"\n/usr/local/sbin/homelab-rclone-copy-chunked:24:rclone purge \"${DEST}.parts\" >/dev/null 2>&1 || true\n/usr/local/sbin/homelab-rclone-copy-chunked:26:rclone copy \"$PARTS\" \"${DEST}.parts\" --transfers 2 --checkers 4 --retries 5 --low-level-retries 20 --stats 30s --log-level INFO\n/usr/local/sbin/homelab-rclone-copy-chunked:27:rclone copyto \"$WORK/${NAME}.sha256\" \"${DEST}.sha256\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO\n/usr/local/sbin/homelab-rclone-copy-chunked:28:rclone copyto \"$WORK/MANIFEST.txt\" \"${DEST}.manifest.txt\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO\n/usr/local/sbin/homelab-rclone-copy-chunked:29:rclone copyto \"$WORK/MANIFEST.parts.txt\" \"${DEST}.parts/MANIFEST.parts.txt\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO\n/usr/local/sbin/homelab-health-metrics-write:10:rm -f \"$tmp\"\n/usr/local/sbin/homelab-rclone-cat-flat-chunked:5:trap \"rm -f \\\"$TMP\\\"\" EXIT\n/usr/local/sbin/homelab-vm-backup-policy:29:echo \"## Latest local vzdump catalog\" >> \"$REPORT\"\n/usr/local/sbin/homelab-vm-backup-policy:31:for f in $(find /var/lib/vz/dump -maxdepth 1 -type f \\( -name \"vzdump-qemu-*.vma.zst\" -o -name \"vzdump-lxc-*.tar.zst\" \\) 2>/dev/null | sort); do b=\"$(basename \"$f\")\"; size=\"$(stat -c %s \"$f\" 2>/dev/null || echo 0)\"; mt=\"$(date -u -d \"@$(stat -c %Y \"$f\" 2>/dev/null || echo 0)\" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)\"; if echo \"$b\" | grep -q \"^vzdump-qemu-\"; then kind=qemu; vmid=\"$(echo \"$b\" | sed -n \"s/^vzdump-qemu-\\([0-9]\\+\\)-.*/\\1/p\")\"; name=\"$(qm config \"$vmid\" 2>/dev/null | awk -F\": \" \"/^name:/{print \\$2; exit}\")\"; else kind=lxc; vmid=\"$(echo \"$b\" | sed -n \"s/^vzdump-lxc-\\([0-9]\\+\\)-.*/\\1/p\")\"; name=\"$(pct config \"$vmid\" 2>/dev/null | awk -F\": \" \"/^hostname:/{print \\$2; exit}\")\"; fi; printf \"%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n\" \"$kind\" \"$vmid\" \"${name:-unknown}\" \"$size\" \"$mt\" \"$f\" >> \"$CATALOG\"; done\n/usr/local/sbin/homelab-vm-backup-policy:37:echo \"- Existing local dumps can be uploaded/verified to two clouds without creating new vzdump archives.\" >> \"$REPORT\"\n/usr/local/sbin/homelab-vm-backup-policy:38:echo \"- New full vzdump for VM 170/171 is not started automatically because each VM has 120 GB bootdisk and /var/tmp has limited free space.\" >> \"$REPORT\"\n/usr/local/sbin/homelab-context-refresh-direct:34:ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] debian@[PRIVATE_IP] \"sudo -n /usr/local/sbin/homelab-runtime-receive $id /tmp/$id.json /tmp/$id.txt && rm -f /tmp/$id.json /tmp/$id.txt\"\n/usr/local/sbin/homelab-evidence-seal:16: rm -f \"$TMP\"\n/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:15:DUMP_SIZE=\"$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 \"$REMOTE\" \"sudo -n rm -f \\\"$REMOTE_DUMP\\\"; sudo -n -u clusteradmin pg_dump --format=custom --no-owner --no-privileges --file=\\\"$REMOTE_DUMP\\\" clusteradmin; sudo -n test -s \\\"$REMOTE_DUMP\\\"; sudo -n -u clusteradmin pg_restore --list \\\"$REMOTE_DUMP\\\" >/dev/null; sudo -n chown debian:debian \\\"$REMOTE_DUMP\\\"; sudo -n chmod 600 \\\"$REMOTE_DUMP\\\"; stat -c %s \\\"$REMOTE_DUMP\\\"\")\"\n/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:18:ssh -n -o BatchMode=yes -o ConnectTimeout=8 \"$REMOTE\" \"rm -f \\\"$REMOTE_DUMP\\\"\"\n/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:23:rm -f \"$STAGE/runtime.tgz\"\n/usr/local/sbin/appbackup-retention:85: if [[ \"$MODE\" == apply ]]; then echo \"DELETE schema=dual path=$REL\"; rclone purge \"$TARGET\"; else echo \"WOULD_DELETE schema=dual path=$REL\"; fi\n/usr/local/sbin/homelab-change-transaction:195: latest_tmp.unlink(missing_ok=True)\n/usr/local/sbin/homelab-check-private-vpn-hosts:15:trap 'rm -f \"$tmp\"' EXIT\n/usr/local/sbin/homelab-health-metrics:22: *backup*|*vzdump*) printf \"%s\" \"backup\" ;;\n/usr/local/sbin/homelab-health-metrics:34: npmplus-kuma-config-*|authentik-*|gitea-*|vaultwarden-*|netbox-*|mail-cloud-restore-drill|edge-vm-vzdump-*|mkdocs-runtime-publish|npmplus-certificate-expiry) printf \"%s\" \"P0\" ;;\n/usr/local/sbin/crypto-portfolio-chart-health-check:5:trap 'rm -f \"$TMP\"' EXIT\n/usr/local/sbin/homelab-rclone-copy-flat-chunked:23:while read -r PART PART_SIZE; do [[ \"$PART\" =~ ^part-[0-9]{4}$ && \"$PART_SIZE\" =~ ^[0-9]+$ ]] || { echo \"BAD PART MANIFEST\" >&2; exit 4; }; rclone copyto \"$WORK/parts/$PART\" \"${DEST}.${PART}\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR; REMOTE_BYTES=\"$(rclone size --json \"${DEST}.${PART}\" | jq -r \".bytes // -1\")\"; [[ \"$REMOTE_BYTES\" = \"$PART_SIZE\" ]] || { echo \"REMOTE PART SIZE MISMATCH $PART\" >&2; exit 5; }; UPLOADED=$((UPLOADED+1)); done < \"$WORK/MANIFEST.parts.txt\"\n/usr/local/sbin/homelab-rclone-copy-flat-chunked:25:rclone copyto \"$WORK/${NAME}.sha256\" \"${DEST}.sha256\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-rclone-copy-flat-chunked:26:rclone copyto \"$WORK/MANIFEST.txt\" \"${DEST}.manifest.txt\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-rclone-copy-flat-chunked:27:rclone copyto \"$WORK/MANIFEST.parts.txt\" \"${DEST}.parts.txt\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:8:TARGET_BASE=full/nextcloud-vm-vzdump\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:11:SRC_GLOB=/var/lib/vz/dump/vzdump-qemu-150-*.vma.zst\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:77:P1 Mail Cloud Nextcloud VM recurring chunked vzdump upload\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:95:rclone copy \"$PARTS_DIR\" \"$TARGET_REMOTE:$REMOTE_PARTS\" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > \"$WORK/logs/upload_parts.out\" 2>\"$WORK/logs/upload_parts.err\" || UPLOAD_PARTS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:97:rclone copyto \"$WORK/$BASENAME.sha256\" \"$TARGET_REMOTE:$REMOTE_SHA\" --checksum > \"$WORK/logs/upload_sha.out\" 2>\"$WORK/logs/upload_sha.err\" || UPLOAD_SHA_RC=$?\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:99:rclone copyto \"$WORK/MANIFEST.txt\" \"$TARGET_REMOTE:$REMOTE_MANIFEST\" --checksum > \"$WORK/logs/upload_manifest.out\" 2>\"$WORK/logs/upload_manifest.err\" || UPLOAD_MANIFEST_RC=$?\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:101:rclone copyto \"$WORK/SHA256SUMS.txt\" \"$TARGET_REMOTE:$REMOTE_SUMS\" --checksum > \"$WORK/logs/upload_sums.out\" 2>\"$WORK/logs/upload_sums.err\" || UPLOAD_SUMS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:110:rclone copy \"$TARGET_REMOTE:$REMOTE_PARTS\" \"$DL_DIR/parts\" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > \"$WORK/logs/download_parts.out\" 2>\"$WORK/logs/download_parts.err\" || DOWNLOAD_PARTS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:126: [ -n \"$OLD\" ] && rclone purge \"$TARGET_REMOTE:$TARGET_BASE/$OLD\" >> \"$WORK/logs/retention.out\" 2>>\"$WORK/logs/retention.err\"\n/usr/local/sbin/homelab-rehydrate-flat-chunked-dir:10: rm -f \"$TMP_TARGET\"\n/usr/local/sbin/homelab-legacy-monitoring-backups-run:24: rm -f \"$STAGE/remote.tgz\"\n/usr/local/sbin/homelab-legacy-monitoring-backups-run:43: rm -f \"$STAGE/pve01.tgz\"\n/usr/local/sbin/homelab-legacy-monitoring-backups-run:48: rm -f \"$STAGE/$LABEL.tgz\"\n/usr/local/sbin/homelab-legacy-monitoring-backups-run:52: rm -f \"$STAGE/monitoring.tgz\"\n/usr/local/sbin/homelab-legacy-monitoring-backups-run:71: rm -f \"$STAGE/config.tgz\"\n/usr/local/sbin/homelab-cluster-admin-inventory-sync:19:ssh -n -o BatchMode=yes -o ConnectTimeout=8 \"$REMOTE\" \"rm -f $REMOTE_TMP\"\n/usr/local/sbin/homelab-chat-run:489: rm -f \\\n/usr/local/sbin/homelab-beszel-backup-run:33:rm -f \"$STAGE/remote.tgz\"\n/usr/local/sbin/backup-restore-check:12:rclone copy \"$RA\" \"$ROOT/a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-a.log 2>&1; homelab-rehydrate-chunked-dir \"$ROOT/a\" >>/tmp/homelab-restore-a.log 2>&1\n/usr/local/sbin/backup-restore-check:15:rclone copy \"$RB\" \"$ROOT/b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-b.log 2>&1; homelab-rehydrate-chunked-dir \"$ROOT/b\" >>/tmp/homelab-restore-b.log 2>&1\n/usr/local/sbin/homelab-vm-full-restore-drill:77: -name \"vzdump-qemu-${source_vmid}-*.vma.zst\" \\\n/usr/local/sbin/homelab-vm-backup-health:16:check_file /var/lib/homelab-health/edge-vm-vzdump-backup.txt edge-vm-backup\n/usr/local/sbin/homelab-vm-backup-health:17:check_file /var/lib/homelab-health/edge-vm-vzdump-offhost.txt edge-vm-offhost\n/usr/local/sbin/homelab-vm-backup-health:18:check_file /var/lib/homelab-health/edge-vm-vzdump-restore.txt edge-vm-restore\n/usr/local/sbin/homelab-ingress-failover:37: rm -f \"$RESOURCES\"\n/usr/local/sbin/homelab-ingress-failover:252: rm -f \"$tmp\"\n/usr/local/sbin/homelab-vm-full-backup:92:for cmd in vzdump qm zstd sha256sum rclone rsync ssh flock; do\n/usr/local/sbin/homelab-vm-full-backup:130: -name \"vzdump-qemu-${VMID}-*.vma.zst\" \\\n/usr/local/sbin/homelab-vm-full-backup:138: vzdump \"$VMID\" \\\n/usr/local/sbin/homelab-vm-full-backup:151: -name \"vzdump-qemu-${VMID}-*.vma.zst\" \\\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:10:TARGET_BASE=full/edge-vm-vzdump\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:64:case \"$SOURCE\" in /mnt/staging/offhost/edge-vm-vzdump-from-pve03/vzdump-qemu-130-*.vma.zst) ;; *) fail source_path_bad 2 ;; esac\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:172:P1 Mail Cloud edge-vm recurring chunked vzdump upload\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:190:rclone copy \"$PARTS_DIR\" \"$TARGET_REMOTE:$REMOTE_PARTS\" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > \"$WORK/logs/upload_parts.out\" 2>\"$WORK/logs/upload_parts.err\" || UPLOAD_PARTS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:192:rclone copyto \"$WORK/$BASENAME.sha256\" \"$TARGET_REMOTE:$REMOTE_SHA\" --checksum > \"$WORK/logs/upload_sha.out\" 2>\"$WORK/logs/upload_sha.err\" || UPLOAD_SHA_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:194:rclone copyto \"$WORK/MANIFEST.txt\" \"$TARGET_REMOTE:$REMOTE_MANIFEST\" --checksum > \"$WORK/logs/upload_manifest.out\" 2>\"$WORK/logs/upload_manifest.err\" || UPLOAD_MANIFEST_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:196:rclone copyto \"$WORK/SHA256SUMS.txt\" \"$TARGET_REMOTE:$REMOTE_SUMS\" --checksum > \"$WORK/logs/upload_sums.out\" 2>\"$WORK/logs/upload_sums.err\" || UPLOAD_SUMS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:199:rclone size \"$TARGET_REMOTE:$TARGET_DIR\" --json > \"$WORK/logs/remote_size.json\" 2>\"$WORK/logs/remote_size.err\" || SIZE_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:207:rclone copy \"$TARGET_REMOTE:$REMOTE_PARTS\" \"$DL_DIR/parts\" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > \"$WORK/logs/download_parts.out\" 2>\"$WORK/logs/download_parts.err\" || DOWNLOAD_PARTS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:239: [ -n \"$OLD\" ] && rclone purge \"$TARGET_REMOTE:$TARGET_BASE/$OLD\" >> \"$WORK/logs/retention.out\" 2>>\"$WORK/logs/retention.err\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:7:BASE_DIR=\"/mnt/staging/vzdump/edge-vm-vzdump\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:8:OFFHOST_DIR=\"/mnt/staging/offhost/edge-vm-vzdump-from-pve03\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:40:REMOTE_OUT=\"$(remote \"OUT=/tmp/edge_vm_vzdump_${VMID}_${TS}.out; ERR=/tmp/edge_vm_vzdump_${VMID}_${TS}.err; : > \\$OUT; : > \\$ERR; vzdump '$VMID' --mode snapshot --compress zstd --dumpdir '$RUN_DIR' --remove 0 >\\$OUT 2>\\$ERR; RC=\\$?; ARCHIVE=\\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\\n' | sort -n | tail -n1 | cut -d' ' -f2-); LOGFILE=\\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.log' -printf '%T@ %p\\n' | sort -n | tail -n1 | cut -d' ' -f2-); SIZE=0; SHA=none; if [ -n \\\"\\$ARCHIVE\\\" ] && [ -f \\\"\\$ARCHIVE\\\" ]; then SIZE=\\$(stat -c %s \\\"\\$ARCHIVE\\\"); SHA=\\$(sha256sum \\\"\\$ARCHIVE\\\" | awk '{print \\$1}'); fi; echo VZDUMP_RC=\\$RC; echo ARCHIVE=\\$ARCHIVE; echo LOGFILE=\\$LOGFILE; echo SIZE=\\$SIZE; echo SHA=\\$SHA; echo OUT_TAIL=\\$(tail -n 6 \\$OUT | tr '\\n' '|'); echo ERR_TAIL=\\$(tail -n 6 \\$ERR | tr '\\n' '|')\")\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:62:dest \"mkdir -p '$OFFHOST_DIR' && rm -f '$DEST_PATH.tmp' '$DEST_PATH.sha256'\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:74:echo \"STATUS=OK TS=$TS TYPE=edge-vm-vzdump-backup NODE=$NODE VMID=$VMID ARCHIVE=$ARCHIVE SIZE=$SIZE SHA256=$SHA SCSI1_BACKUP0=yes NOTES=timer_run\" > \"$HEALTH_DIR/edge-vm-vzdump-backup.txt\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:75:echo \"STATUS=OK TS=$TS TYPE=edge-vm-vzdump-offhost NODE=$NODE VMID=$VMID HOST=pve02 DEST=$DEST_PATH SIZE=$SIZE SHA256=$SHA NOTES=timer_run\" > \"$HEALTH_DIR/edge-vm-vzdump-offhost.txt\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:76:echo \"STATUS=OK TS=$TS TYPE=edge-vm-vzdump-restore-verify HOST=pve02 VMID=$VMID ARCHIVE=$DEST_PATH SIZE=$SIZE SHA256=$SHA ZSTD_RC=0 VMA_RC=0 NOTES=timer_run\" > \"$HEALTH_DIR/edge-vm-vzdump-restore.txt\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:77:chmod 0644 \"$HEALTH_DIR/edge-vm-vzdump-backup.txt\" \"$HEALTH_DIR/edge-vm-vzdump-offhost.txt\" \"$HEALTH_DIR/edge-vm-vzdump-restore.txt\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:80:dest \"find '$OFFHOST_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\\n' | sort -nr | awk 'NR>${KEEP_OFFHOST}{print \\$2}' | while read f; do rm -f \\\"\\$f\\\" \\\"\\$f.sha256\\\"; done\"\n/usr/local/sbin/appbackup-dockge-stacks:54:echo \"[STEP] upload cloud A metadata\"; rclone copy \"$STAGE\" \"$RA\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n/usr/local/sbin/appbackup-dockge-stacks:55:echo \"[STEP] upload cloud A archive chunks\"; rclone deletefile \"$RA/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RA/$ARCHIVE\"\n/usr/local/sbin/appbackup-dockge-stacks:56:echo \"[STEP] upload cloud B metadata\"; rclone copy \"$STAGE\" \"$RB\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n/usr/local/sbin/appbackup-dockge-stacks:57:echo \"[STEP] upload cloud B archive chunks\"; rclone deletefile \"$RB/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RB/$ARCHIVE\"\n/usr/local/sbin/appbackup-dockge-stacks:60:echo \"[STEP] upload latest index\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RAL\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RBL\"; rclone cat \"$RAL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null; rclone cat \"$RBL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n/usr/local/sbin/backup-retention:24: for ((i=0; i"$LOG" | ssh "${SSH[@]}" "$TARGET" "cat > '$PART'" +/usr/local/sbin/homelab-vm160-remote-backup:57:ssh "${SSH[@]}" "$TARGET" "cat > '$DIR/vzdump.log'" < "$LOG" +/usr/local/sbin/homelab-vm160-remote-backup:66:rm -f "$LOG" +/usr/local/sbin/homelab-mail-cloud-restore-drill:83: rm -f "$OUT" +/usr/local/sbin/homelab-mail-cloud-restore-drill:108: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/MANIFEST.txt" "$WORK/edge/MANIFEST.txt" > "$WORK/logs/edge_manifest_download.out" 2>"$WORK/logs/edge_manifest_download.err" +/usr/local/sbin/homelab-mail-cloud-restore-drill:111: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/SHA256SUMS.txt" "$WORK/edge/SHA256SUMS.txt" > "$WORK/logs/edge_sums_download.out" 2>"$WORK/logs/edge_sums_download.err" +/usr/local/sbin/homelab-mail-cloud-restore-drill:114: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/$EDGE_ARCHIVE.sha256" "$WORK/edge/$EDGE_ARCHIVE.sha256" > "$WORK/logs/edge_archive_sha_download.out" 2>"$WORK/logs/edge_archive_sha_download.err" +/usr/local/sbin/homelab-mail-cloud-restore-drill:130: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/parts/$FIRST_PART" "$WORK/edge/$FIRST_PART" > "$WORK/logs/edge_first_part_download.out" 2>"$WORK/logs/edge_first_part_download.err" +/usr/local/sbin/homelab-mail-cloud-restore-drill:137: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/parts/$LAST_PART" "$WORK/edge/$LAST_PART" > "$WORK/logs/edge_last_part_download.out" 2>"$WORK/logs/edge_last_part_download.err" +/usr/local/sbin/homelab-mail-cloud-restore-drill:168: rm -f "$WORK/edge/$FIRST_PART" "$WORK/edge/$LAST_PART" +/usr/local/sbin/homelab-golden-state-extended-overlay:26:rclone copyto "$JSON" "$RA" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-golden-state-extended-overlay:27:rclone copyto "$JSON" "$RB" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-mail-cloud-critical-upload:84: rclone copyto "$BUNDLE_SHA" "${R}:$REMOTE_DIR/$(basename "$BUNDLE_SHA")" --checksum > "$WORK/logs/${SAFE}_upload_sha.out" 2>"$WORK/logs/${SAFE}_upload_sha.err" +/usr/local/sbin/homelab-mail-cloud-critical-upload:86: rclone copyto "$WORK/components/MANIFEST.txt" "${R}:$REMOTE_DIR/MANIFEST.txt" --checksum > "$WORK/logs/${SAFE}_upload_manifest.out" 2>"$WORK/logs/${SAFE}_upload_manifest.err" +/usr/local/sbin/homelab-mail-cloud-critical-upload:88: rclone copyto "$WORK/components/SHA256SUMS.txt" "${R}:$REMOTE_DIR/SHA256SUMS.txt" --checksum > "$WORK/logs/${SAFE}_upload_sums.out" 2>"$WORK/logs/${SAFE}_upload_sums.err" +/usr/local/sbin/homelab-mail-cloud-critical-upload:108: rm -f "$WORK/download/${SAFE}-$(basename "$BUNDLE")" +/usr/local/sbin/homelab-mail-cloud-critical-upload:114: [ -n "$OLD" ] && rclone purge "${R}:$TARGET_BASE/$OLD" >> "$WORK/logs/${SAFE}_retention.out" 2>>"$WORK/logs/${SAFE}_retention.err" +/usr/local/sbin/homelab-alertmanager-backup-run:18:rm -f "$STAGE/remote.tgz" +/usr/local/sbin/postgres-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@"$H" "sudo rm -f /tmp/postgres-app-${APP}-${JOB_ID}.tgz" +/usr/local/sbin/homelab-stage-verify-flat-chunked:9:rclone copy "$REMOTE_DIR" "$VERIFY" --transfers 4 --checkers 8 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-rclone-cat-chunked:10:rclone copyto "${REMOTE_FILE}.parts/MANIFEST.parts.txt" "$MAN" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-stage-verify-chunked:9:rclone copy "$REMOTE_DIR" "$VERIFY" --exclude "$ARCHIVE.parts/**" --exclude "$ARCHIVE.parts/" --exclude "$ARCHIVE.sha256" --exclude "$ARCHIVE.manifest.txt" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --log-level ERROR +/usr/local/sbin/crypto-portfolio-backup-run:17:rm -f "$STAGE/remote.tgz" +/usr/local/sbin/npmplus-edge-backup-run:20:ssh -o BatchMode=yes -o ConnectTimeout=8 "$EDGE" "sudo rm -f '/tmp/npmplus-edge-${JOB_ID}.tgz'" +/usr/local/sbin/mariadb-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@"$H" "sudo rm -f /tmp/mariadb-app-${APP}-${JOB_ID}.tgz" +/usr/local/sbin/homelab-blackbox-exporter-backup-run:17:rm -f "$STAGE/remote.tgz" +/usr/local/sbin/homelab-ingress-standby-refresh:16: rm -f "$TMP_STATE" "$RELAY" "$RESOURCES" +/usr/local/sbin/homelab-ingress-standby-refresh:74:run_node "$SOURCE_NODE" vzdump 130 \ +/usr/local/sbin/homelab-ingress-standby-refresh:83:"find /var/lib/vz/dump -maxdepth 1 -type f -name 'vzdump-qemu-130-*.vma.zst' -printf '%T@ %p\n' | sort -rn | head -n 1 | cut -d' ' -f2-" +/usr/local/sbin/homelab-ingress-standby-refresh:106:"chattr -i '$DEST_FILE' '$DEST_FILE.sha256' '$DEST_FILE.env' 2>/dev/null || true; rm -f '$DEST_TMP'" +/usr/local/sbin/homelab-ingress-standby-refresh:155: -name 'vzdump-qemu-130-*.vma.zst' \ +/usr/local/sbin/homelab-ingress-standby-refresh:164: rm -f "$OLD" "$OLD.sha256" "$OLD.env" +/usr/local/sbin/homelab-ingress-standby-refresh:173:run_node "$SOURCE_NODE" rm -f "$SOURCE_BACKUP" +/usr/local/sbin/homelab-stage-2cloud-flat:38: for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f "$STAGE/$F" ]] && rclone copyto "$STAGE/$F" "$REMOTE_DIR/$F" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done +/usr/local/sbin/homelab-stage-2cloud-flat:41: rclone copyto "$STAGE/latest-backup-index.json" "$LATEST_FILE" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-backup-authentik:27:ssh debian@[PRIVATE_IP] "sudo docker rm -f authentik-restore-test >/dev/null 2>&1 || true" +/usr/local/sbin/homelab-backup-authentik:51:ssh debian@[PRIVATE_IP] "sudo docker rm -f authentik-restore-test >/dev/null 2>&1 || true; rm -f /tmp/authentik-restore-from-archive.pg_dump" +/usr/local/sbin/homelab-safe-exec:621: latest_tmp.unlink(missing_ok=True) +/usr/local/sbin/homelab-golden-state-index:28:rclone copyto "$JSON" "$RA" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-golden-state-index:29:rclone copyto "$JSON" "$RB" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-full-retention:21: -name "vzdump-qemu-${vmid}-*.vma.zst" \ +/usr/local/sbin/homelab-vm-full-retention:33: vzdump-qemu-"$vmid"-*.vma.zst) +/usr/local/sbin/homelab-vm-full-retention:35: rm -f \ +/usr/local/sbin/homelab-vm-full-retention:105: rclone purge "${path}/${dir}" +/usr/local/sbin/homelab-stage-2cloud:23:echo "[STEP] upload cloud A metadata"; rclone copy "$STAGE" "$RA" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s +/usr/local/sbin/homelab-stage-2cloud:24:echo "[STEP] upload cloud A archive chunks"; rclone deletefile "$RA/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RA/$ARCHIVE" +/usr/local/sbin/homelab-stage-2cloud:25:echo "[STEP] upload cloud B metadata"; rclone copy "$STAGE" "$RB" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s +/usr/local/sbin/homelab-stage-2cloud:26:echo "[STEP] upload cloud B archive chunks"; rclone deletefile "$RB/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RB/$ARCHIVE" +/usr/local/sbin/homelab-stage-2cloud:29:echo "[STEP] upload latest index"; rclone copyto "$STAGE/latest-backup-index.json" "$RAL"; rclone copyto "$STAGE/latest-backup-index.json" "$RBL"; rclone cat "$RAL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null; rclone cat "$RBL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null +/usr/local/sbin/homelab-backup-npmplus-kuma-configs:90: ssh -o BatchMode=yes -o ConnectTimeout=8 debian@[PRIVATE_IP] "rm -f '$EDGE_TGZ'" 2>>"$ERR" || true +/usr/local/sbin/homelab-stage-2cloud-flat-canary:36: for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f "$STAGE/$F" ]] && rclone copyto "$STAGE/$F" "$REMOTE_DIR/$F" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done +/usr/local/sbin/homelab-stage-2cloud-flat-canary:39: rclone copyto "$STAGE/latest-backup-index.json" "$LATEST_FILE" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-router-running-config-mail-cloud:62: rm -f "$CONFIG" +/usr/local/sbin/homelab-router-running-config-mail-cloud:66: rm -f "$CONFIG" +/usr/local/sbin/homelab-router-running-config-mail-cloud:79: if rclone mkdir "$REMOTE:$DEST" >"$WORK/rclone-mkdir.out" 2>"$WORK/rclone-mkdir.err" && homelab-rclone-copy-chunked "$ARCHIVE" "$REMOTE:$DEST/$(basename "$ARCHIVE")" >"$WORK/rclone-upload.out" 2>"$WORK/rclone-upload.err" && rclone copyto "$SHA_FILE" "$REMOTE:$DEST/$(basename "$SHA_FILE")" --checksum >"$WORK/rclone-upload-sha.out" 2>"$WORK/rclone-upload-sha.err"; then +/usr/local/sbin/homelab-router-running-config-mail-cloud:98: [ -n "$OLD" ] && rclone purge "$REMOTE:$TARGET_BASE/$OLD" >>"$WORK/retention.out" 2>>"$WORK/retention.err" +/usr/local/sbin/homelab-router-running-config-mail-cloud:103: rm -f "$old" "$old.sha256" +/usr/local/sbin/appbackup-restore-check:27:rclone copy "$RA" "$ROOT/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >"${LOGBASE}-a.log" 2>&1 +/usr/local/sbin/appbackup-restore-check:32:rclone copy "$RB" "$ROOT/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >"${LOGBASE}-b.log" 2>&1 +/usr/local/sbin/appbackup-restore-check:39:rm -f "${LOGBASE}-a.log" "${LOGBASE}-a-sha.log" "${LOGBASE}-b.log" "${LOGBASE}-b-sha.log" +/usr/local/sbin/homelab-capacity-risk:32:LARGEST_BYTES="$(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-*.vma.zst" -o -name "vzdump-lxc-*.tar.zst" \) -printf "%s\n" 2>/dev/null | sort -n | tail -n1)" +/usr/local/sbin/homelab-edge-vm-offhost-freshness:9:OFFHOST_DIR=/mnt/staging/offhost/edge-vm-vzdump-from-pve03 +/usr/local/sbin/homelab-edge-vm-offhost-freshness:85: /var/lib/vz/dump/vzdump-qemu-130-*.vma.zst) ;; +/usr/local/sbin/homelab-edge-vm-offhost-freshness:156:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-backup NODE=%s VMID=%s ARCHIVE=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\n' \ +/usr/local/sbin/homelab-edge-vm-offhost-freshness:158: >"$HEALTH_DIR/edge-vm-vzdump-backup.txt" +/usr/local/sbin/homelab-edge-vm-offhost-freshness:159:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-offhost NODE=%s VMID=%s HOST=%s DEST=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\n' \ +/usr/local/sbin/homelab-edge-vm-offhost-freshness:161: >"$HEALTH_DIR/edge-vm-vzdump-offhost.txt" +/usr/local/sbin/homelab-edge-vm-offhost-freshness:162:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-restore-verify HOST=%s VMID=%s ARCHIVE=%s SIZE=%s SHA256=%s ZSTD_RC=0 VMA_RC=0 NOTES=nightly_existing_dump_freshness\n' \ +/usr/local/sbin/homelab-edge-vm-offhost-freshness:164: >"$HEALTH_DIR/edge-vm-vzdump-restore.txt" +/usr/local/sbin/homelab-edge-vm-offhost-freshness:167: "$HEALTH_DIR/edge-vm-vzdump-backup.txt" \ +/usr/local/sbin/homelab-edge-vm-offhost-freshness:168: "$HEALTH_DIR/edge-vm-vzdump-offhost.txt" \ +/usr/local/sbin/homelab-edge-vm-offhost-freshness:169: "$HEALTH_DIR/edge-vm-vzdump-restore.txt" +/usr/local/sbin/homelab-external-probe-vps:20:rm -f "$TMP" +/usr/local/sbin/sqlite-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@"$H" "sudo rm -f /tmp/sqlite-app-${APP}-${JOB_ID}.tgz" +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:31: rclone copyto "$remote_path" "$out" >/dev/null +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:33: rm -f "$plist" +/usr/local/sbin/skladchik-reports-monitor-backup:83: for D in "${OLD[@]}"; do [ -n "$D" ] && rclone purge "${REMOTE}${REMOTE_BASE}/daily/${D}" >/dev/null 2>&1 || true; done +/usr/local/sbin/homelab-vm-local-dumps-retention:56: rclone purge "$PATH_REMOTE" >> "$REPORT" 2>&1 && DELETED=$((DELETED+1)) || BAD=$((BAD+1)) +/usr/local/sbin/homelab-mkdocs-auto-refresh:53: rm -f /tmp/homelab-mkdocs-auto-refresh-git-fetch.out /tmp/homelab-mkdocs-auto-refresh-git-fetch.err +/usr/local/sbin/homelab-mkdocs-auto-refresh:57:rm -f /tmp/homelab-mkdocs-auto-refresh-git-fetch.out /tmp/homelab-mkdocs-auto-refresh-git-fetch.err +/usr/local/sbin/homelab-vm-local-dumps-2cloud:42: F="$(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-${id}-*.vma.zst" -o -name "vzdump-lxc-${id}-*.tar.zst" \) -printf "%T@ %p\n" 2>/dev/null | sort -n | tail -n 1 | cut -d" " -f2-)" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:53: echo "$BN" | grep -q "^vzdump-qemu-" && KIND="qemu" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:54: echo "$BN" | grep -q "^vzdump-lxc-" && KIND="lxc" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:126: rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:128: rclone copyto "$CDIR/chunks.sha256" "$DEST/files/$bn.chunks/chunks.sha256" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:131: rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:135: rclone copyto "$CHKS" "$DEST/checksums.txt" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:136: rclone copyto "$FILES_TSV" "$DEST/files.tsv" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:137: rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:146: rclone copyto "$DEST/manifest.json" "$VBASE/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:154: rclone copy "$DEST/files/$bn.chunks" "$VDIR" --filter "+ part-*" --filter "+ chunks.sha256" --filter "- *" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:166: rclone copyto "$DEST/files/$bn" "$VBASE/$bn" --transfers 1 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:168: rm -f "$VBASE/$bn" +/usr/local/sbin/homelab-gitea-secondary-backup:18: rm -f "$TMP" +/usr/local/sbin/homelab-gitea-secondary-backup:31:docker exec -u git -w /tmp gitea sh -lc \"rm -f /tmp/\$NAME && gitea dump -c /data/gitea/conf/app.ini -f /tmp/\$NAME\" +/usr/local/sbin/homelab-gitea-secondary-backup:33:docker exec gitea rm -f \"/tmp/\$NAME\" +/usr/local/sbin/homelab-gitea-secondary-backup:100: rm -f -- "$FILE" "$FILE.sha256" +/usr/local/sbin/homelab-gitea-secondary-backup:123: rm -f -- "$FILE" "$FILE.sha256" +/usr/local/sbin/homelab-gitea-secondary-backup:136: rm -f -- "$FILE" "$FILE.sha256" +/usr/local/sbin/homelab-rclone-copy-chunked:24:rclone purge "${DEST}.parts" >/dev/null 2>&1 || true +/usr/local/sbin/homelab-rclone-copy-chunked:26:rclone copy "$PARTS" "${DEST}.parts" --transfers 2 --checkers 4 --retries 5 --low-level-retries 20 --stats 30s --log-level INFO +/usr/local/sbin/homelab-rclone-copy-chunked:27:rclone copyto "$WORK/${NAME}.sha256" "${DEST}.sha256" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO +/usr/local/sbin/homelab-rclone-copy-chunked:28:rclone copyto "$WORK/MANIFEST.txt" "${DEST}.manifest.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO +/usr/local/sbin/homelab-rclone-copy-chunked:29:rclone copyto "$WORK/MANIFEST.parts.txt" "${DEST}.parts/MANIFEST.parts.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO +/usr/local/sbin/homelab-health-metrics-write:10:rm -f "$tmp" +/usr/local/sbin/homelab-rclone-cat-flat-chunked:5:trap "rm -f \"$TMP\"" EXIT +/usr/local/sbin/homelab-vm-backup-policy:29:echo "## Latest local vzdump catalog" >> "$REPORT" +/usr/local/sbin/homelab-vm-backup-policy:31:for f in $(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-*.vma.zst" -o -name "vzdump-lxc-*.tar.zst" \) 2>/dev/null | sort); do b="$(basename "$f")"; size="$(stat -c %s "$f" 2>/dev/null || echo 0)"; mt="$(date -u -d "@$(stat -c %Y "$f" 2>/dev/null || echo 0)" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)"; if echo "$b" | grep -q "^vzdump-qemu-"; then kind=qemu; vmid="$(echo "$b" | sed -n "s/^vzdump-qemu-\([0-9]\+\)-.*/\1/p")"; name="$(qm config "$vmid" 2>/dev/null | awk -F": " "/^name:/{print \$2; exit}")"; else kind=lxc; vmid="$(echo "$b" | sed -n "s/^vzdump-lxc-\([0-9]\+\)-.*/\1/p")"; name="$(pct config "$vmid" 2>/dev/null | awk -F": " "/^hostname:/{print \$2; exit}")"; fi; printf "%s\t%s\t%s\t%s\t%s\t%s\n" "$kind" "$vmid" "${name:-unknown}" "$size" "$mt" "$f" >> "$CATALOG"; done +/usr/local/sbin/homelab-vm-backup-policy:37:echo "- Existing local dumps can be uploaded/verified to two clouds without creating new vzdump archives." >> "$REPORT" +/usr/local/sbin/homelab-vm-backup-policy:38:echo "- New full vzdump for VM 170/171 is not started automatically because each VM has 120 GB bootdisk and /var/tmp has limited free space." >> "$REPORT" +/usr/local/sbin/homelab-context-refresh-direct:34:ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] debian@[PRIVATE_IP] "sudo -n /usr/local/sbin/homelab-runtime-receive $id /tmp/$id.json /tmp/$id.txt && rm -f /tmp/$id.json /tmp/$id.txt" +/usr/local/sbin/homelab-evidence-seal:16: rm -f "$TMP" +/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:15:DUMP_SIZE="$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$REMOTE" "sudo -n rm -f \"$REMOTE_DUMP\"; sudo -n -u clusteradmin pg_dump --format=custom --no-owner --no-privileges --file=\"$REMOTE_DUMP\" clusteradmin; sudo -n test -s \"$REMOTE_DUMP\"; sudo -n -u clusteradmin pg_restore --list \"$REMOTE_DUMP\" >/dev/null; sudo -n chown debian:debian \"$REMOTE_DUMP\"; sudo -n chmod 600 \"$REMOTE_DUMP\"; stat -c %s \"$REMOTE_DUMP\"")" +/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:18:ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$REMOTE" "rm -f \"$REMOTE_DUMP\"" +/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:23:rm -f "$STAGE/runtime.tgz" +/usr/local/sbin/appbackup-retention:85: if [[ "$MODE" == apply ]]; then echo "DELETE schema=dual path=$REL"; rclone purge "$TARGET"; else echo "WOULD_DELETE schema=dual path=$REL"; fi +/usr/local/sbin/homelab-change-transaction:195: latest_tmp.unlink(missing_ok=True) +/usr/local/sbin/homelab-check-private-vpn-hosts:15:trap 'rm -f "$tmp"' EXIT +/usr/local/sbin/homelab-health-metrics:22: *backup*|*vzdump*) printf "%s" "backup" ;; +/usr/local/sbin/homelab-health-metrics:34: npmplus-kuma-config-*|authentik-*|gitea-*|vaultwarden-*|netbox-*|mail-cloud-restore-drill|edge-vm-vzdump-*|mkdocs-runtime-publish|npmplus-certificate-expiry) printf "%s" "P0" ;; +/usr/local/sbin/crypto-portfolio-chart-health-check:5:trap 'rm -f "$TMP"' EXIT +/usr/local/sbin/homelab-rclone-copy-flat-chunked:23:while read -r PART PART_SIZE; do [[ "$PART" =~ ^part-[0-9]{4}$ && "$PART_SIZE" =~ ^[0-9]+$ ]] || { echo "BAD PART MANIFEST" >&2; exit 4; }; rclone copyto "$WORK/parts/$PART" "${DEST}.${PART}" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR; REMOTE_BYTES="$(rclone size --json "${DEST}.${PART}" | jq -r ".bytes // -1")"; [[ "$REMOTE_BYTES" = "$PART_SIZE" ]] || { echo "REMOTE PART SIZE MISMATCH $PART" >&2; exit 5; }; UPLOADED=$((UPLOADED+1)); done < "$WORK/MANIFEST.parts.txt" +/usr/local/sbin/homelab-rclone-copy-flat-chunked:25:rclone copyto "$WORK/${NAME}.sha256" "${DEST}.sha256" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-rclone-copy-flat-chunked:26:rclone copyto "$WORK/MANIFEST.txt" "${DEST}.manifest.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-rclone-copy-flat-chunked:27:rclone copyto "$WORK/MANIFEST.parts.txt" "${DEST}.parts.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:8:TARGET_BASE=full/nextcloud-vm-vzdump +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:11:SRC_GLOB=/var/lib/vz/dump/vzdump-qemu-150-*.vma.zst +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:77:P1 Mail Cloud Nextcloud VM recurring chunked vzdump upload +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:95:rclone copy "$PARTS_DIR" "$TARGET_REMOTE:$REMOTE_PARTS" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > "$WORK/logs/upload_parts.out" 2>"$WORK/logs/upload_parts.err" || UPLOAD_PARTS_RC=$? +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:97:rclone copyto "$WORK/$BASENAME.sha256" "$TARGET_REMOTE:$REMOTE_SHA" --checksum > "$WORK/logs/upload_sha.out" 2>"$WORK/logs/upload_sha.err" || UPLOAD_SHA_RC=$? +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:99:rclone copyto "$WORK/MANIFEST.txt" "$TARGET_REMOTE:$REMOTE_MANIFEST" --checksum > "$WORK/logs/upload_manifest.out" 2>"$WORK/logs/upload_manifest.err" || UPLOAD_MANIFEST_RC=$? +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:101:rclone copyto "$WORK/SHA256SUMS.txt" "$TARGET_REMOTE:$REMOTE_SUMS" --checksum > "$WORK/logs/upload_sums.out" 2>"$WORK/logs/upload_sums.err" || UPLOAD_SUMS_RC=$? +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:110:rclone copy "$TARGET_REMOTE:$REMOTE_PARTS" "$DL_DIR/parts" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > "$WORK/logs/download_parts.out" 2>"$WORK/logs/download_parts.err" || DOWNLOAD_PARTS_RC=$? +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:126: [ -n "$OLD" ] && rclone purge "$TARGET_REMOTE:$TARGET_BASE/$OLD" >> "$WORK/logs/retention.out" 2>>"$WORK/logs/retention.err" +/usr/local/sbin/homelab-rehydrate-flat-chunked-dir:10: rm -f "$TMP_TARGET" +/usr/local/sbin/homelab-legacy-monitoring-backups-run:24: rm -f "$STAGE/remote.tgz" +/usr/local/sbin/homelab-legacy-monitoring-backups-run:43: rm -f "$STAGE/pve01.tgz" +/usr/local/sbin/homelab-legacy-monitoring-backups-run:48: rm -f "$STAGE/$LABEL.tgz" +/usr/local/sbin/homelab-legacy-monitoring-backups-run:52: rm -f "$STAGE/monitoring.tgz" +/usr/local/sbin/homelab-legacy-monitoring-backups-run:71: rm -f "$STAGE/config.tgz" +/usr/local/sbin/homelab-cluster-admin-inventory-sync:19:ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$REMOTE" "rm -f $REMOTE_TMP" +/usr/local/sbin/homelab-chat-run:489: rm -f \ +/usr/local/sbin/homelab-beszel-backup-run:33:rm -f "$STAGE/remote.tgz" +/usr/local/sbin/backup-restore-check:12:rclone copy "$RA" "$ROOT/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-a.log 2>&1; homelab-rehydrate-chunked-dir "$ROOT/a" >>/tmp/homelab-restore-a.log 2>&1 +/usr/local/sbin/backup-restore-check:15:rclone copy "$RB" "$ROOT/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-b.log 2>&1; homelab-rehydrate-chunked-dir "$ROOT/b" >>/tmp/homelab-restore-b.log 2>&1 +/usr/local/sbin/homelab-vm-full-restore-drill:77: -name "vzdump-qemu-${source_vmid}-*.vma.zst" \ +/usr/local/sbin/homelab-vm-backup-health:16:check_file /var/lib/homelab-health/edge-vm-vzdump-backup.txt edge-vm-backup +/usr/local/sbin/homelab-vm-backup-health:17:check_file /var/lib/homelab-health/edge-vm-vzdump-offhost.txt edge-vm-offhost +/usr/local/sbin/homelab-vm-backup-health:18:check_file /var/lib/homelab-health/edge-vm-vzdump-restore.txt edge-vm-restore +/usr/local/sbin/homelab-ingress-failover:37: rm -f "$RESOURCES" +/usr/local/sbin/homelab-ingress-failover:252: rm -f "$tmp" +/usr/local/sbin/homelab-vm-full-backup:92:for cmd in vzdump qm zstd sha256sum rclone rsync ssh flock; do +/usr/local/sbin/homelab-vm-full-backup:130: -name "vzdump-qemu-${VMID}-*.vma.zst" \ +/usr/local/sbin/homelab-vm-full-backup:138: vzdump "$VMID" \ +/usr/local/sbin/homelab-vm-full-backup:151: -name "vzdump-qemu-${VMID}-*.vma.zst" \ +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:10:TARGET_BASE=full/edge-vm-vzdump +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:64:case "$SOURCE" in /mnt/staging/offhost/edge-vm-vzdump-from-pve03/vzdump-qemu-130-*.vma.zst) ;; *) fail source_path_bad 2 ;; esac +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:172:P1 Mail Cloud edge-vm recurring chunked vzdump upload +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:190:rclone copy "$PARTS_DIR" "$TARGET_REMOTE:$REMOTE_PARTS" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > "$WORK/logs/upload_parts.out" 2>"$WORK/logs/upload_parts.err" || UPLOAD_PARTS_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:192:rclone copyto "$WORK/$BASENAME.sha256" "$TARGET_REMOTE:$REMOTE_SHA" --checksum > "$WORK/logs/upload_sha.out" 2>"$WORK/logs/upload_sha.err" || UPLOAD_SHA_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:194:rclone copyto "$WORK/MANIFEST.txt" "$TARGET_REMOTE:$REMOTE_MANIFEST" --checksum > "$WORK/logs/upload_manifest.out" 2>"$WORK/logs/upload_manifest.err" || UPLOAD_MANIFEST_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:196:rclone copyto "$WORK/SHA256SUMS.txt" "$TARGET_REMOTE:$REMOTE_SUMS" --checksum > "$WORK/logs/upload_sums.out" 2>"$WORK/logs/upload_sums.err" || UPLOAD_SUMS_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:199:rclone size "$TARGET_REMOTE:$TARGET_DIR" --json > "$WORK/logs/remote_size.json" 2>"$WORK/logs/remote_size.err" || SIZE_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:207:rclone copy "$TARGET_REMOTE:$REMOTE_PARTS" "$DL_DIR/parts" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > "$WORK/logs/download_parts.out" 2>"$WORK/logs/download_parts.err" || DOWNLOAD_PARTS_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:239: [ -n "$OLD" ] && rclone purge "$TARGET_REMOTE:$TARGET_BASE/$OLD" >> "$WORK/logs/retention.out" 2>>"$WORK/logs/retention.err" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:7:BASE_DIR="/mnt/staging/vzdump/edge-vm-vzdump" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:8:OFFHOST_DIR="/mnt/staging/offhost/edge-vm-vzdump-from-pve03" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:40:REMOTE_OUT="$(remote "OUT=/tmp/edge_vm_vzdump_${VMID}_${TS}.out; ERR=/tmp/edge_vm_vzdump_${VMID}_${TS}.err; : > \$OUT; : > \$ERR; vzdump '$VMID' --mode snapshot --compress zstd --dumpdir '$RUN_DIR' --remove 0 >\$OUT 2>\$ERR; RC=\$?; ARCHIVE=\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\n' | sort -n | tail -n1 | cut -d' ' -f2-); LOGFILE=\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.log' -printf '%T@ %p\n' | sort -n | tail -n1 | cut -d' ' -f2-); SIZE=0; SHA=none; if [ -n \"\$ARCHIVE\" ] && [ -f \"\$ARCHIVE\" ]; then SIZE=\$(stat -c %s \"\$ARCHIVE\"); SHA=\$(sha256sum \"\$ARCHIVE\" | awk '{print \$1}'); fi; echo VZDUMP_RC=\$RC; echo ARCHIVE=\$ARCHIVE; echo LOGFILE=\$LOGFILE; echo SIZE=\$SIZE; echo SHA=\$SHA; echo OUT_TAIL=\$(tail -n 6 \$OUT | tr '\n' '|'); echo ERR_TAIL=\$(tail -n 6 \$ERR | tr '\n' '|')")" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:62:dest "mkdir -p '$OFFHOST_DIR' && rm -f '$DEST_PATH.tmp' '$DEST_PATH.sha256'" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:74:echo "STATUS=OK TS=$TS TYPE=edge-vm-vzdump-backup NODE=$NODE VMID=$VMID ARCHIVE=$ARCHIVE SIZE=$SIZE SHA256=$SHA SCSI1_BACKUP0=yes NOTES=timer_run" > "$HEALTH_DIR/edge-vm-vzdump-backup.txt" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:75:echo "STATUS=OK TS=$TS TYPE=edge-vm-vzdump-offhost NODE=$NODE VMID=$VMID HOST=pve02 DEST=$DEST_PATH SIZE=$SIZE SHA256=$SHA NOTES=timer_run" > "$HEALTH_DIR/edge-vm-vzdump-offhost.txt" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:76:echo "STATUS=OK TS=$TS TYPE=edge-vm-vzdump-restore-verify HOST=pve02 VMID=$VMID ARCHIVE=$DEST_PATH SIZE=$SIZE SHA256=$SHA ZSTD_RC=0 VMA_RC=0 NOTES=timer_run" > "$HEALTH_DIR/edge-vm-vzdump-restore.txt" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:77:chmod 0644 "$HEALTH_DIR/edge-vm-vzdump-backup.txt" "$HEALTH_DIR/edge-vm-vzdump-offhost.txt" "$HEALTH_DIR/edge-vm-vzdump-restore.txt" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:80:dest "find '$OFFHOST_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\n' | sort -nr | awk 'NR>${KEEP_OFFHOST}{print \$2}' | while read f; do rm -f \"\$f\" \"\$f.sha256\"; done" +/usr/local/sbin/appbackup-dockge-stacks:54:echo "[STEP] upload cloud A metadata"; rclone copy "$STAGE" "$RA" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s +/usr/local/sbin/appbackup-dockge-stacks:55:echo "[STEP] upload cloud A archive chunks"; rclone deletefile "$RA/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RA/$ARCHIVE" +/usr/local/sbin/appbackup-dockge-stacks:56:echo "[STEP] upload cloud B metadata"; rclone copy "$STAGE" "$RB" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s +/usr/local/sbin/appbackup-dockge-stacks:57:echo "[STEP] upload cloud B archive chunks"; rclone deletefile "$RB/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RB/$ARCHIVE" +/usr/local/sbin/appbackup-dockge-stacks:60:echo "[STEP] upload latest index"; rclone copyto "$STAGE/latest-backup-index.json" "$RAL"; rclone copyto "$STAGE/latest-backup-index.json" "$RBL"; rclone cat "$RAL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null; rclone cat "$RBL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null +/usr/local/sbin/backup-retention:24: for ((i=0; i