From fade65d7198d7daab423b51bcce0d5594d6ec0c5 Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Tue, 21 Jul 2026 12:01:04 +0000 Subject: [PATCH] runtime: publish BACKUP-PAIR-DELETE-GUARD-066 --- .../history/BACKUP-PAIR-DELETE-GUARD-066.json | 30 + .../history/BACKUP-PAIR-DELETE-GUARD-066.txt | 1017 +++++++++++++++++ runtime/latest.json | 18 +- runtime/latest.txt | 1006 +++++++++++++++- 4 files changed, 2055 insertions(+), 16 deletions(-) create mode 100644 runtime/history/BACKUP-PAIR-DELETE-GUARD-066.json create mode 100644 runtime/history/BACKUP-PAIR-DELETE-GUARD-066.txt diff --git a/runtime/history/BACKUP-PAIR-DELETE-GUARD-066.json b/runtime/history/BACKUP-PAIR-DELETE-GUARD-066.json new file mode 100644 index 0000000..4d3dfee --- /dev/null +++ b/runtime/history/BACKUP-PAIR-DELETE-GUARD-066.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "BACKUP-PAIR-DELETE-GUARD-066", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "backup-audit", + "started_at_utc": "2026-07-21T12:01:03Z", + "finished_at_utc": "2026-07-21T12:01:03Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1", + "command_sha256": "e64555881c22fffab137df86ee957819b9922f83078f71124f8ff61e8580590d", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "d13bc63fd488f7e13f81bef392b271beacdec9c637576dc61e7dad33042b27f2", + "sanitized_output_sha256": "6c697f245387cba271db9658a3e6e5ce9d5a9ac0178945c559417c7064daa692", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "=== PIPELINES_AND_CLOUD_PAIRS ===\n========== NODE=pve01 ==========\n=== REMOTES ===\n=== TIMERS ===\nWed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:14 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service\nWed 2026-07-22 00:09:43 MSK 9h Tue 2026-07-21 00:09:32 MSK 14h ago homelab-backup-freshness.timer homelab-backup-freshness.service\nWed 2026-07-22 02:15:00 MSK 11h Tue 2026-07-21 02:15:05 MSK 12h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service\nWed 2026-07-22 03:10:02 MSK 12h Tue 2026-07-21 03:13:14 MSK 11h ago homelab-emergency-backup.timer homelab-emergency-backup.service\nWed 2026-07-22 03:35:00 MSK 12h Tue 2026-07-21 03:35:14 MSK 11h ago filebrowser-backup.timer filebrowser-backup.service\nWed 2026-07-22 03:55:02 MSK 13h Tue 2026-07-21 03:51:25 MSK 11h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service\nWed 2026-07-22 04:17:29 MSK 13h Tue 2026-07-21 04:19:17 MSK 10h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service\nWed 2026-07-22 04:26:54 MSK 13h Tue 2026-07-21 04:28:14 MSK 10h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service\nWed 2026-07-22 04:38:04 MSK 13h Tue 2026-07-21 04:34:48 MSK 10h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service\nWed 2026-07-22 05:01:44 MSK 14h Tue 2026-07-21 04:57:14 MSK 9h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service\nWed 2026-07-22 05:23:56 MSK 14h Tue 2026-07-21 05:17:57 MSK 9h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service\nWed 2026-07-22 05:27:13 MSK 14h Tue 2026-07-21 05:29:05 MSK 9h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service\nWed 2026-07-22 05:47:19 MSK 14h Tue 2026-07-21 05:46:14 MSK 9h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service\nWed 2026-07-22 06:01:19 MSK 15h Tue 2026-07-21 05:57:30 MSK 8h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service\nWed 2026-07-22 06:27:43 MSK 15h Tue 2026-07-21 06:32:25 MSK 8h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service\nWed 2026-07-22 06:38:51 MSK 15h Tue 2026-07-21 06:43:54 MSK 8h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service\nWed 2026-07-22 06:39:22 MSK 15h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service\nWed 2026-07-22 06:44:16 MSK 15h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service\nWed 2026-07-22 06:52:06 MSK 15h Tue 2026-07-21 06:47:47 MSK 8h ago homelab-beszel-backup.timer homelab-beszel-backup.service\nWed 2026-07-22 06:58:58 MSK 16h Tue 2026-07-21 06:57:30 MSK 7h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service\nWed 2026-07-22 07:02:27 MSK 16h Tue 2026-07-21 07:05:30 MSK 7h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service\nWed 2026-07-22 07:07:14 MSK 16h Tue 2026-07-21 07:18:55 MSK 7h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service\nWed 2026-07-22 07:35:06 MSK 16h Tue 2026-07-21 07:36:59 MSK 7h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service\nWed 2026-07-22 07:38:21 MSK 16h Tue 2026-07-21 07:36:14 MSK 7h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service\nWed 2026-07-22 07:41:49 MSK 16h Tue 2026-07-21 07:53:41 MSK 7h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service\nWed 2026-07-22 08:10:37 MSK 17h Tue 2026-07-21 08:07:25 MSK 6h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service\nWed 2026-07-22 08:37:02 MSK 17h Tue 2026-07-21 08:30:07 MSK 6h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service\nWed 2026-07-22 08:40:13 MSK 17h Tue 2026-07-21 08:50:14 MSK 6h ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service\nWed 2026-07-22 08:53:36 MSK 17h Tue 2026-07-21 08:41:14 MSK 6h ago memos-backup.timer memos-backup.service\nSun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service\nSun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service\nSun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service\nSun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service\nMon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 7h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service\n=== SERVICES ===\ndpkg-db-backup.service static -\nfilebrowser-backup.service static -\nhomelab-alertmanager-backup.service static -\nhomelab-appbackup-dockge-stacks.service static -\nhomelab-appbackup-mariadb-batch.service static -\nhomelab-appbackup-npmplus-edge.service static -\nhomelab-appbackup-postgres-batch.service static -\nhomelab-appbackup-sqlite-batch.service static -\nhomelab-authentik-backup.service static -\nhomelab-auto-backup.service static -\nhomelab-backup-coverage-matrix.service static -\nhomelab-backup-freshness.service static -\nhomelab-backup-healthcheck.service static -\nhomelab-backup-matrix-refresh.service static -\nhomelab-backup-sla-health.service static -\nhomelab-beszel-backup.service static -\nhomelab-blackbox-exporter-backup.service static -\nhomelab-cluster-admin-incident-engine-backup.service static -\nhomelab-crypto-portfolio-backup.service static -\nhomelab-edge-vm-vzdump-backup.service static -\nhomelab-emergency-backup.service static -\nhomelab-extended-appbackup.service static -\nhomelab-gitea-backup.service static -\nhomelab-gitea-secondary-backup.service static -\nhomelab-legacy-monitoring-backups.service static -\nhomelab-mail-cloud-critical-upload.service static -\nhomelab-mail-cloud-edge-vm-upload.service static -\nhomelab-mail-cloud-nextcloud-vm-upload.service static -\nhomelab-mail-cloud-restore-drill.service static -\nhomelab-npmplus-kuma-config-backup.service static -\nhomelab-router-running-config-mail-cloud.service static -\nhomelab-vaultwarden-backup.service static -\nhomelab-verified-backup-health.service static -\nhomelab-vm-backup-health.service static -\nhomelab-vm-backup-policy.service static -\nhomelab-vm-full-backup.service static -\nhomelab-vm-local-dumps-2cloud.service static -\nhomelab-vm160-remote-backup.service static -\nimmich-media-backup.service static -\nmemos-backup.service static -\nnetbird-vps-backup.service static -\npaperless-backup.service static -\nskladchik-reports-monitor-backup-restore-check.service static -\nskladchik-reports-monitor-backup.service static -\n/usr/local/sbin/homelab-vm160-remote-backup:8:PARENT=/mnt/staging/vzdump\n/usr/local/sbin/homelab-vm160-remote-backup:22:SOURCE_ACTIVE=\"$(ssh \"${SSH[@]}\" \"$SOURCE\" \"pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'\")\" || SOURCE_ACTIVE_RC=$?\n/usr/local/sbin/homelab-vm160-remote-backup:27:NAME=\"vzdump-qemu-${VMID}-${TS}.vma.zst\"\n/usr/local/sbin/homelab-vm160-remote-backup:35: ssh \"${SSH[@]}\" \"$TARGET\" \"rm -rf -- '$DIR'\" >/dev/null 2>&1 || true\n/usr/local/sbin/homelab-vm160-remote-backup:41: rm -f \"$LOG\"\n/usr/local/sbin/homelab-vm160-remote-backup:46:LATEST_SIZE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\\n' | sort -nr | head -n1\")\"\n/usr/local/sbin/homelab-vm160-remote-backup:55:ssh \"${SSH[@]}\" \"$SOURCE\" \"vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7\" 2>\"$LOG\" | ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$PART'\"\n/usr/local/sbin/homelab-vm160-remote-backup:57:ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$DIR/vzdump.log'\" < \"$LOG\"\n/usr/local/sbin/homelab-vm160-remote-backup:66:rm -f \"$LOG\"\n/usr/local/sbin/skladchik-reports-monitor-status:11:check_health() {\n/usr/local/sbin/skladchik-reports-monitor-status:24:check_timer() {\n/usr/local/sbin/skladchik-reports-monitor-status:38:check_timer skladchik-reports-monitor-backup.timer\n/usr/local/sbin/skladchik-reports-monitor-status:39:check_timer skladchik-reports-monitor-backup-restore-check.timer\n/usr/local/sbin/skladchik-reports-monitor-status:40:check_timer skladchik-reports-monitor-supervisor.timer\n/usr/local/sbin/skladchik-reports-monitor-status:41:check_timer skladchik-reports-monitor-monthly-selftest-watch.timer\n/usr/local/sbin/skladchik-reports-monitor-status:43:check_health \"PVE01: ежедневный бэкап\" \"/var/lib/skladchik-reports-monitor-backup/health.txt\"\n/usr/local/sbin/skladchik-reports-monitor-status:44:check_health \"PVE01: проверка восстановления\" \"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\"\n/usr/local/sbin/skladchik-reports-monitor-status:45:check_health \"PVE01: общий supervisor\" \"/var/lib/skladchik-reports-monitor-supervisor/health.txt\"\n/usr/local/sbin/skladchik-reports-monitor-status:46:check_health \"PVE01: контроль ежемесячного самотеста\" \"/var/lib/skladchik-reports-monitor-selftest-watch/health.txt\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:52: homelab-rclone-cat-chunked \"${R}:$CRIT_DEST/$CRIT_BUNDLE\" > \"$OUT\" 2>\"$WORK/logs/${SAFE}_crit_download.err\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:83: rm -f \"$OUT\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:108: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/MANIFEST.txt\" \"$WORK/edge/MANIFEST.txt\" > \"$WORK/logs/edge_manifest_download.out\" 2>\"$WORK/logs/edge_manifest_download.err\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:111: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/SHA256SUMS.txt\" \"$WORK/edge/SHA256SUMS.txt\" > \"$WORK/logs/edge_sums_download.out\" 2>\"$WORK/logs/edge_sums_download.err\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:114: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/$EDGE_ARCHIVE.sha256\" \"$WORK/edge/$EDGE_ARCHIVE.sha256\" > \"$WORK/logs/edge_archive_sha_download.out\" 2>\"$WORK/logs/edge_archive_sha_download.err\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:117: rclone lsf \"${EDGE_REMOTE}:$EDGE_DEST/parts\" --format ps > \"$WORK/logs/edge_parts_lsf.txt\" 2>\"$WORK/logs/edge_parts_lsf.err\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:130: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/parts/$FIRST_PART\" \"$WORK/edge/$FIRST_PART\" > \"$WORK/logs/edge_first_part_download.out\" 2>\"$WORK/logs/edge_first_part_download.err\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:137: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/parts/$LAST_PART\" \"$WORK/edge/$LAST_PART\" > \"$WORK/logs/edge_last_part_download.out\" 2>\"$WORK/logs/edge_last_part_download.err\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:168: rm -f \"$WORK/edge/$FIRST_PART\" \"$WORK/edge/$LAST_PART\"\n/usr/local/sbin/homelab-golden-state-extended-overlay:26:rclone copyto \"$JSON\" \"$RA\" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-golden-state-extended-overlay:27:rclone copyto \"$JSON\" \"$RB\" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-golden-state-extended-overlay:28:rclone cat \"$RA\" 2>/dev/null | jq -e --arg cp \"$CP\" \".control_plane_job == \\$cp\" >/dev/null || BAD=$((BAD+1))\n/usr/local/sbin/homelab-golden-state-extended-overlay:29:rclone cat \"$RB\" 2>/dev/null | jq -e --arg cp \"$CP\" \".control_plane_job == \\$cp\" >/dev/null || BAD=$((BAD+1))\n/usr/local/sbin/homelab-mail-cloud-critical-upload:50:EXCLUDED=edge-vm-vzdump-large-archive,rclone-config,secrets,raw-env\n/usr/local/sbin/homelab-mail-cloud-critical-upload:80: rclone mkdir \"${R}:$REMOTE_DIR\" > \"$WORK/logs/${SAFE}_mkdir.out\" 2>\"$WORK/logs/${SAFE}_mkdir.err\"\n/usr/local/sbin/homelab-mail-cloud-critical-upload:82: homelab-rclone-copy-chunked \"$BUNDLE\" \"${R}:$REMOTE_DIR/$(basename \"$BUNDLE\")\" > \"$WORK/logs/${SAFE}_upload_bundle.out\" 2>\"$WORK/logs/${SAFE}_upload_bundle.err\"\n/usr/local/sbin/homelab-mail-cloud-critical-upload:84: rclone copyto \"$BUNDLE_SHA\" \"${R}:$REMOTE_DIR/$(basename \"$BUNDLE_SHA\")\" --checksum > \"$WORK/logs/${SAFE}_upload_sha.out\" 2>\"$WORK/logs/${SAFE}_upload_sha.err\"\n/usr/local/sbin/homelab-mail-cloud-critical-upload:86: rclone copyto \"$WORK/components/MANIFEST.txt\" \"${R}:$REMOTE_DIR/MANIFEST.txt\" --checksum > \"$WORK/logs/${SAFE}_upload_manifest.out\" 2>\"$WORK/logs/${SAFE}_upload_manifest.err\"\n/usr/local/sbin/homelab-mail-cloud-critical-upload:88: rclone copyto \"$WORK/components/SHA256SUMS.txt\" \"${R}:$REMOTE_DIR/SHA256SUMS.txt\" --checksum > \"$WORK/logs/${SAFE}_upload_sums.out\" 2>\"$WORK/logs/${SAFE}_upload_sums.err\"\n/usr/local/sbin/homelab-mail-cloud-critical-upload:90: homelab-rclone-cat-chunked \"${R}:$REMOTE_DIR/$(basename \"$BUNDLE\")\" > \"$WORK/download/${SAFE}-$(basename \"$BUNDLE\")\" 2>\"$WORK/logs/${SAFE}_download_bundle.err\"\n/usr/local/sbin/homelab-mail-cloud-critical-upload:108: rm -f \"$WORK/download/${SAFE}-$(basename \"$BUNDLE\")\"\n/usr/local/sbin/homelab-mail-cloud-critical-upload:109: rclone lsf \"${R}:$TARGET_BASE\" --dirs-only 2>/dev/null | sed 's#/$##' | sort > \"$WORK/logs/${SAFE}_dirs.txt\"\n/usr/local/sbin/homelab-mail-cloud-critical-upload:114: [ -n \"$OLD\" ] && rclone purge \"${R}:$TARGET_BASE/$OLD\" >> \"$WORK/logs/${SAFE}_retention.out\" 2>>\"$WORK/logs/${SAFE}_retention.err\"\n/usr/local/sbin/homelab-mail-cloud-critical-upload:140: rm -rf --one-file-system \"$OLD_DIR\"\n/usr/local/sbin/homelab-alertmanager-backup-run:16:ssh -n -o BatchMode=yes -o ConnectTimeout=8 \"$MON\" \"sudo -n docker exec alertmanager amtool check-config /etc/alertmanager/alertmanager.yml >/dev/null && sudo -n tar -C / -czf - opt/stacks/observability-lite/compose.yaml opt/stacks/observability-lite/compose.override.yaml opt/stacks/observability-lite/alertmanager/alertmanager.yml opt/stacks/observability-lite/alertmanager/data\" > \"$STAGE/remote.tgz\"\n/usr/local/sbin/homelab-alertmanager-backup-run:18:rm -f \"$STAGE/remote.tgz\"\n/usr/local/sbin/homelab-alertmanager-backup-run:21:jq -n --arg app \"$APP\" --arg host \"monitoring\" --arg engine \"files+alertmanager-config-check\" --arg config \"ok\" --argjson files \"$FILES\" \"{app:\\$app,engine:\\$engine,source_host:\\$host,source_paths:[\\\"monitoring:/opt/stacks/observability-lite/alertmanager\\\",\\\"monitoring:/opt/stacks/observability-lite/compose.yaml\\\",\\\"monitoring:/opt/stacks/observability-lite/compose.override.yaml\\\"],config_check:\\$config,file_count:\\$files,note:\\\"Alertmanager configuration and runtime state in encrypted two-cloud backup\\\"}\" > \"$STAGE/metadata.json\"\n/usr/local/sbin/appbackupctl:13: check) bash -n /usr/local/sbin/appbackup-dockge-stacks && bash -n /usr/local/sbin/appbackup-restore-check && bash -n /usr/local/sbin/appbackup-retention && bash -n /usr/local/sbin/appbackup-report && echo \"OK appbackup framework\" ;;\n/usr/local/sbin/appbackupctl:14: restore-check) appbackup-restore-check \"${2:-dockge-stacks}\" ;;\n/usr/local/sbin/appbackupctl:17: audit) appbackupctl check && appbackupctl restore-check dockge-stacks && appbackupctl retention 7 dry-run dockge-stacks && appbackupctl report dockge-stacks >/dev/null && echo \"OK app audit finished\" ;;\n/usr/local/sbin/appbackupctl:21: *) echo \"usage: appbackupctl run [dockge-stacks|cluster-admin-incident-engine]|check|restore-check|retention|report|audit|status|lastlog|timer\"; exit 2 ;;\n/usr/local/sbin/postgres-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@\"$H\" \"sudo rm -f /tmp/postgres-app-${APP}-${JOB_ID}.tgz\"\n/usr/local/sbin/postgres-apps-backup-run:21: rm -rf \"$STAGE/payload\"\n/usr/local/sbin/homelab-backup-gitea:27:rm -rf \"$RESTORE\"\n/usr/local/sbin/homelab-backup-gitea:35:INTEGRITY=\"$(sqlite3 \"$DB\" 'PRAGMA integrity_check;')\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:4:CRYPT_FILE=\"/etc/homelab/rclone/required-crypt-remotes.txt\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:18: \"checksums/\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:21:echo \"=== homelab rclone crypt healthcheck: $(hostname) ===\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:40: if ! rclone listremotes | sed 's/:$//' | grep -qx \"$REMOTE\"; then\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:47: if ! rclone about \"${REMOTE}:\"; then\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:48: echo \"FAIL: rclone about failed: $REMOTE\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:54: ROOT_LIST=\"$(rclone lsf \"${REMOTE}:/\" || true)\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:76: if rclone cat \"${REMOTE}:/latest-backup-index.json\" | jq -e '.schema and .remote and .status and .items' >/dev/null; then\n/usr/local/sbin/homelab-check-private-vpn-hosts.realnet-20260624T140509Z:4:REAL=\"/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z\"\n/usr/local/sbin/homelab-stage-verify-flat-chunked:7:rm -rf \"$VERIFY\"\n/usr/local/sbin/homelab-stage-verify-flat-chunked:9:rclone copy \"$REMOTE_DIR\" \"$VERIFY\" --transfers 4 --checkers 8 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-stage-verify-flat-chunked:11:(cd \"$VERIFY\" && test -s manifest.json && if [[ -n \"$JOB\" ]]; then jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null; fi && sha256sum -c checksums.sha256 >/dev/null && tar -tzf \"$ARCHIVE\" >/dev/null)\n/usr/local/sbin/homelab-rclone-cat-chunked:4:TMP_BASE=${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\n/usr/local/sbin/homelab-rclone-cat-chunked:7:cleanup(){ rm -rf \"$WORK\"; }\n/usr/local/sbin/homelab-rclone-cat-chunked:10:rclone copyto \"${REMOTE_FILE}.parts/MANIFEST.parts.txt\" \"$MAN\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-rclone-cat-chunked:11:while read -r PART SIZE; do case \"$PART\" in part-*) rclone cat \"${REMOTE_FILE}.parts/$PART\" ;; esac; done < \"$MAN\"\n/usr/local/sbin/homelab-stage-verify-chunked:7:rm -rf \"$VERIFY\"\n/usr/local/sbin/homelab-stage-verify-chunked:9:rclone copy \"$REMOTE_DIR\" \"$VERIFY\" --exclude \"$ARCHIVE.parts/**\" --exclude \"$ARCHIVE.parts/\" --exclude \"$ARCHIVE.sha256\" --exclude \"$ARCHIVE.manifest.txt\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --log-level ERROR\n/usr/local/sbin/homelab-stage-verify-chunked:13:if [ -s checksums.sha256 ]; then grep -F -v \" $ARCHIVE\" checksums.sha256 > .checksums-no-archive.sha256 || true; if [ -s .checksums-no-archive.sha256 ]; then sha256sum -c .checksums-no-archive.sha256; fi; EXPECTED=$(grep -F \" $ARCHIVE\" checksums.sha256 | awk '{print $1; exit}'); else EXPECTED=$(rclone cat \"${REMOTE_DIR}/${ARCHIVE}.sha256\" | awk '{print $1; exit}'); fi\n/usr/local/sbin/homelab-stage-verify-chunked:15:ACTUAL=$(homelab-rclone-cat-chunked \"${REMOTE_DIR}/${ARCHIVE}\" | sha256sum | awk '{print $1}')\n/usr/local/sbin/homelab-stage-verify-chunked:17:homelab-rclone-cat-chunked \"${REMOTE_DIR}/${ARCHIVE}\" | tar -tzf - >/dev/null\n/usr/local/sbin/crypto-portfolio-backup-run:15:ssh -n -o BatchMode=yes -o ConnectTimeout=10 \"$CORE\" 'H=\"$(hostname -f 2>/dev/null || hostname)\"; { [ \"$H\" = \"core-apps.gram1.ru\" ] || [ \"$H\" = \"core-apps\" ]; } || exit 3; D=/opt/stacks/crypto-portfolio; T=\"$(sudo -n mktemp -d /tmp/crypto-portfolio-backup.XXXXXX)\"; trap \"sudo -n rm -rf \\\"$T\\\"\" EXIT; sudo -n install -d -m 700 \"$T/stack\" \"$T/stack/data\"; for F in compose.yaml Dockerfile requirements.txt app.py worker.py .dockerignore; do sudo -n test -s \"$D/$F\" && sudo -n cp -a \"$D/$F\" \"$T/stack/\"; done; sudo -n cp -a \"$D/templates\" \"$D/secrets\" \"$T/stack/\"; sudo -n python3 -c \"import sqlite3,sys;s=sqlite3.connect(sys.argv[1]);d=sqlite3.connect(sys.argv[2]);s.backup(d);d.close();s.close()\" \"$D/data/portfolio.db\" \"$T/stack/data/portfolio.db\"; for F in market.json worker-status.json; do sudo -n test -s \"$D/data/$F\" && sudo -n cp -a \"$D/data/$F\" \"$T/stack/data/\"; done; sudo -n tar -C \"$T\" -czf - stack' > \"$STAGE/remote.tgz\"\n/usr/local/sbin/crypto-portfolio-backup-run:17:rm -f \"$STAGE/remote.tgz\"\n/usr/local/sbin/crypto-portfolio-backup-run:18:DBI=\"$(python3 -c 'import sqlite3,sys;print(sqlite3.connect(sys.argv[1]).execute(\"PRAGMA integrity_check\").fetchone()[0])' \"$STAGE/payload/stack/data/portfolio.db\")\"\n/usr/local/sbin/npmplus-edge-backup-run:20:ssh -o BatchMode=yes -o ConnectTimeout=8 \"$EDGE\" \"sudo rm -f '/tmp/npmplus-edge-${JOB_ID}.tgz'\"\n/usr/local/sbin/npmplus-edge-backup-run:28:rm -rf \"$STAGE/payload\"\n/usr/local/sbin/mariadb-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@\"$H\" \"sudo rm -f /tmp/mariadb-app-${APP}-${JOB_ID}.tgz\"\n/usr/local/sbin/mariadb-apps-backup-run:20: rm -rf \"$STAGE/payload\"\n/usr/local/sbin/homelab-blackbox-exporter-backup-run:17:rm -f \"$STAGE/remote.tgz\"\n/usr/local/sbin/homelab-blackbox-exporter-backup-run:21:jq -n --arg app \"$APP\" --arg host \"monitoring\" --arg engine \"files+runtime-health-check\" --argjson files \"$FILES\" '{app:$app,engine:$engine,source_host:$host,source_paths:[\"monitoring:/opt/stacks/blackbox-exporter\"],runtime_health:\"ok\",file_count:$files,note:\"Blackbox Exporter compose and probe configuration in encrypted two-cloud backup\"}' > \"$STAGE/metadata.json\"\n/usr/local/sbin/homelab-ingress-standby-refresh:16: rm -f \"$TMP_STATE\" \"$RELAY\" \"$RESOURCES\"\n/usr/local/sbin/homelab-ingress-standby-refresh:74:run_node \"$SOURCE_NODE\" vzdump 130 \\\n/usr/local/sbin/homelab-ingress-standby-refresh:78: --remove 0 \\\n/usr/local/sbin/homelab-ingress-standby-refresh:83:\"find /var/lib/vz/dump -maxdepth 1 -type f -name 'vzdump-qemu-130-*.vma.zst' -printf '%T@ %p\\n' | sort -rn | head -n 1 | cut -d' ' -f2-\"\n/usr/local/sbin/homelab-ingress-standby-refresh:106:\"chattr -i '$DEST_FILE' '$DEST_FILE.sha256' '$DEST_FILE.env' 2>/dev/null || true; rm -f '$DEST_TMP'\"\n/usr/local/sbin/homelab-ingress-standby-refresh:155: -name 'vzdump-qemu-130-*.vma.zst' \\\n/usr/local/sbin/homelab-ingress-standby-refresh:164: rm -f \"$OLD\" \"$OLD.sha256\" \"$OLD.env\"\n/usr/local/sbin/homelab-ingress-standby-refresh:173:run_node \"$SOURCE_NODE\" rm -f \"$SOURCE_BACKUP\"\n/usr/local/sbin/homelab-stage-2cloud-flat:33:(cd \"$STAGE\"; FILES=(\"$ARCHIVE\" manifest.json latest-backup-index.json); [[ -f metadata.json ]] && FILES+=(metadata.json); sha256sum \"${FILES[@]}\" > checksums.sha256; sha256sum -c checksums.sha256 >/dev/null; tar -tzf \"$ARCHIVE\" >/dev/null; jq -e . manifest.json >/dev/null)\n/usr/local/sbin/homelab-stage-2cloud-flat:36: if rclone lsf \"$REMOTE_DIR\" --max-depth 1 2>/dev/null | grep -q .; then echo \"REFUSE NONEMPTY CANARY TARGET $REMOTE_DIR\" >&2; exit 6; fi\n/usr/local/sbin/homelab-stage-2cloud-flat:37: rclone mkdir \"$REMOTE_DIR\"\n/usr/local/sbin/homelab-stage-2cloud-flat:38: for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f \"$STAGE/$F\" ]] && rclone copyto \"$STAGE/$F\" \"$REMOTE_DIR/$F\" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done\n/usr/local/sbin/homelab-stage-2cloud-flat:39: CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-flat-chunked \"$STAGE/$ARCHIVE\" \"$REMOTE_DIR/$ARCHIVE\"\n/usr/local/sbin/homelab-stage-2cloud-flat:41: rclone copyto \"$STAGE/latest-backup-index.json\" \"$LATEST_FILE\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-stage-2cloud-flat:42: rclone cat \"$LATEST_FILE\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n/usr/local/sbin/homelab-stage-2cloud-flat:47:rm -rf \"$STAGE\" \"$VERIFY\"\n/usr/local/sbin/homelab-cluster-admin-inventory-export:12: return subprocess.check_output(args, text=True, stderr=subprocess.DEVNULL, timeout=30)\n/usr/local/sbin/homelab-cluster-admin-webpanel-sync:7:copy_health() {\n/usr/local/sbin/homelab-cluster-admin-webpanel-sync:17:copy_health /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt cluster-admin-vm-mail-cloud-backup.txt\n/usr/local/sbin/homelab-cluster-admin-webpanel-sync:18:copy_health /var/lib/homelab-health/overall.txt overall.txt\n/usr/local/sbin/homelab-cluster-admin-webpanel-sync:19:copy_health /var/lib/homelab-health/final-readiness.txt final-readiness.txt\n/usr/local/sbin/homelab-cluster-admin-webpanel-sync:20:copy_health /var/lib/homelab-health/cluster-admin-strict-final-seal.txt cluster-admin-strict-final-seal.txt\n/usr/local/sbin/homelab-backup-authentik:14:ssh debian@[PRIVATE_IP] \"rm -rf /tmp/auth-small-snap; mkdir -p /tmp/auth-small-snap\"\n/usr/local/sbin/homelab-backup-authentik:27:ssh debian@[PRIVATE_IP] \"sudo docker rm -f authentik-restore-test >/dev/null 2>&1 || true\"\n/usr/local/sbin/homelab-backup-authentik:51:ssh debian@[PRIVATE_IP] \"sudo docker rm -f authentik-restore-test >/dev/null 2>&1 || true; rm -f /tmp/authentik-restore-from-archive.pg_dump\"\n/usr/local/sbin/homelab-safe-exec:77: check=False,\n/usr/local/sbin/homelab-safe-exec:621: latest_tmp.unlink(missing_ok=True)\n/usr/local/sbin/homelab-golden-state-index:28:rclone copyto \"$JSON\" \"$RA\" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-golden-state-index:29:rclone copyto \"$JSON\" \"$RB\" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-golden-state-index:30:rclone cat \"$RA\" 2>/dev/null | jq -e --arg cp \"$CP\" --arg ext \"$EXT\" \".control_plane_job == \\$cp and .health.extended_appbackup == \\$ext and .extended_appbackup_health == \\$ext\" >/dev/null || BAD=$((BAD+1))\n/usr/local/sbin/homelab-golden-state-index:31:rclone cat \"$RB\" 2>/dev/null | jq -e --arg cp \"$CP\" --arg ext \"$EXT\" \".control_plane_job == \\$cp and .health.extended_appbackup == \\$ext and .extended_appbackup_health == \\$ext\" >/dev/null || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-full-retention:21: -name \"vzdump-qemu-${vmid}-*.vma.zst\" \\\n/usr/local/sbin/homelab-vm-full-retention:33: vzdump-qemu-\"$vmid\"-*.vma.zst)\n/usr/local/sbin/homelab-vm-full-retention:35: rm -f \\\n/usr/local/sbin/homelab-vm-full-retention:73: rm -rf -- \\\"\\$BASE/\\$setname\\\"\n/usr/local/sbin/homelab-vm-full-retention:91: rclone lsf \"$path\" \\\n/usr/local/sbin/homelab-vm-full-retention:105: rclone purge \"${path}/${dir}\"\n/usr/local/sbin/homelab-stage-2cloud:22:echo \"[STEP] local checksum\"; (cd \"$STAGE\" && sha256sum \"$ARCHIVE\" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e . manifest.json >/dev/null)\n/usr/local/sbin/homelab-stage-2cloud:23:echo \"[STEP] upload cloud A metadata\"; rclone copy \"$STAGE\" \"$RA\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n/usr/local/sbin/homelab-stage-2cloud:24:echo \"[STEP] upload cloud A archive chunks\"; rclone deletefile \"$RA/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RA/$ARCHIVE\"\n/usr/local/sbin/homelab-stage-2cloud:25:echo \"[STEP] upload cloud B metadata\"; rclone copy \"$STAGE\" \"$RB\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n/usr/local/sbin/homelab-stage-2cloud:26:echo \"[STEP] upload cloud B archive chunks\"; rclone deletefile \"$RB/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RB/$ARCHIVE\"\n/usr/local/sbin/homelab-stage-2cloud:27:echo \"[STEP] verify cloud A chunked\"; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-stage-verify-chunked \"$RA\" \"$ARCHIVE\" \"$VERIFY/a\" \"$JOB\"\n/usr/local/sbin/homelab-stage-2cloud:28:echo \"[STEP] verify cloud B chunked\"; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-stage-verify-chunked \"$RB\" \"$ARCHIVE\" \"$VERIFY/b\" \"$JOB\"\n/usr/local/sbin/homelab-stage-2cloud:29:echo \"[STEP] upload latest index\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RAL\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RBL\"; rclone cat \"$RAL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null; rclone cat \"$RBL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n/usr/local/sbin/homelab-stage-2cloud:31:echo \"[STEP] delete local staging\"; rm -rf \"$STAGE\" \"$VERIFY\"; test ! -d \"$STAGE\" && test ! -d \"$VERIFY\"\n/usr/local/sbin/homelab-safe-autoheal:17:SAFE_TIMERS=\"homelab-emergency-backup.timer homelab-appbackup-dockge-stacks.timer homelab-appbackup-npmplus-edge.timer homelab-appbackup-sqlite-batch.timer homelab-appbackup-postgres-batch.timer homelab-appbackup-mariadb-batch.timer homelab-verified-backup-health.timer homelab-drift-check.timer homelab-service-registry-check.timer homelab-dependency-map-check.timer homelab-vm-backup-health.timer homelab-alerting-health.timer homelab-runbook-generate.timer homelab-desired-state-sync.timer homelab-duty-admin-report.timer homelab-incident-journal.timer homelab-backup-sla-health.timer homelab-kuma-monitor-policy.timer homelab-safe-autoheal.timer homelab-overall-health.timer\"\n/usr/local/sbin/homelab-safe-autoheal:144:for cmd in homelab-alerting-health homelab-kuma-monitor-policy homelab-backup-sla-health homelab-service-registry-check homelab-dependency-map-check; do\n/usr/local/sbin/homelab-backup-npmplus-kuma-configs:26: install -D -m 0755 /usr/local/sbin/homelab-check-private-vpn-hosts \"$SNAP/pve01/usr/local/sbin/homelab-check-private-vpn-hosts\" 2>>\"$ERR\" || fail \"copy_private_vpn_script_failed\"\n/usr/local/sbin/homelab-backup-npmplus-kuma-configs:27: install -D -m 0644 /etc/systemd/system/homelab-private-vpn-hosts-health.service \"$SNAP/pve01/etc/systemd/system/homelab-private-vpn-hosts-health.service\" 2>>\"$ERR\" || fail \"copy_private_vpn_service_failed\"\n/usr/local/sbin/homelab-backup-npmplus-kuma-configs:28: install -D -m 0644 /etc/systemd/system/homelab-private-vpn-hosts-health.timer \"$SNAP/pve01/etc/systemd/system/homelab-private-vpn-hosts-health.timer\" 2>>\"$ERR\" || fail \"copy_private_vpn_timer_failed\"\n/usr/local/sbin/homelab-backup-npmplus-kuma-configs:29: install -D -m 0644 /var/lib/homelab-health/private-vpn-hosts.txt \"$SNAP/pve01/var/lib/homelab-health/private-vpn-hosts.txt\" 2>>\"$ERR\" || fail \"copy_private_vpn_health_failed\"\n/usr/local/sbin/homelab-backup-npmplus-kuma-configs:42:sudo rm -rf \"$TMP\" \"$EDGE_TGZ\"\n/usr/local/sbin/homelab-backup-npmplus-kuma-configs:82:sudo rm -rf \"$TMP\"\n/usr/local/sbin/homelab-backup-npmplus-kuma-configs:90: ssh -o BatchMode=yes -o ConnectTimeout=8 debian@[PRIVATE_IP] \"rm -f '$EDGE_TGZ'\" 2>>\"$ERR\" || true\n/usr/local/sbin/homelab-backup-npmplus-kuma-configs:99: NPM_INTEGRITY=\"$(sqlite3 \"$SNAP/edge-vm/npmplus/db/database.sqlite\" \"PRAGMA integrity_check;\" 2>>\"$ERR\" | tr '\\n' ';' || echo error)\"\n/usr/local/sbin/homelab-backup-npmplus-kuma-configs:100: KUMA_INTEGRITY=\"$(sqlite3 \"$SNAP/edge-vm/kuma/db/kuma.db\" \"PRAGMA integrity_check;\" 2>>\"$ERR\" | tr '\\n' ';' || echo error)\"\n/usr/local/sbin/homelab-backup-npmplus-kuma-configs:134: rm -rf \"$RESTORE\"\n/usr/local/sbin/homelab-backup-npmplus-kuma-configs:138: RESTORE_NPM_INTEGRITY=\"$(sqlite3 \"$RESTORE/edge-vm/npmplus/db/database.sqlite\" \"PRAGMA integrity_check;\" 2>>\"$ERR\" | tr '\\n' ';' || echo error)\"\n/usr/local/sbin/homelab-backup-npmplus-kuma-configs:139: RESTORE_KUMA_INTEGRITY=\"$(sqlite3 \"$RESTORE/edge-vm/kuma/db/kuma.db\" \"PRAGMA integrity_check;\" 2>>\"$ERR\" | tr '\\n' ';' || echo error)\"\n/usr/local/sbin/homelab-stage-2cloud-flat-canary:31:(cd \"$STAGE\"; FILES=(\"$ARCHIVE\" manifest.json latest-backup-index.json); [[ -f metadata.json ]] && FILES+=(metadata.json); sha256sum \"${FILES[@]}\" > checksums.sha256; sha256sum -c checksums.sha256 >/dev/null; tar -tzf \"$ARCHIVE\" >/dev/null; jq -e . manifest.json >/dev/null)\n/usr/local/sbin/homelab-stage-2cloud-flat-canary:34: if rclone lsf \"$REMOTE_DIR\" --max-depth 1 2>/dev/null | grep -q .; then echo \"REFUSE NONEMPTY CANARY TARGET $REMOTE_DIR\" >&2; exit 6; fi\n/usr/local/sbin/homelab-stage-2cloud-flat-canary:35: rclone mkdir \"$REMOTE_DIR\"\n/usr/local/sbin/homelab-stage-2cloud-flat-canary:36: for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f \"$STAGE/$F\" ]] && rclone copyto \"$STAGE/$F\" \"$REMOTE_DIR/$F\" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done\n/usr/local/sbin/homelab-stage-2cloud-flat-canary:37: CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-flat-chunked \"$STAGE/$ARCHIVE\" \"$REMOTE_DIR/$ARCHIVE\"\n/usr/local/sbin/homelab-stage-2cloud-flat-canary:39: rclone copyto \"$STAGE/latest-backup-index.json\" \"$LATEST_FILE\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-stage-2cloud-flat-canary:40: rclone cat \"$LATEST_FILE\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n/usr/local/sbin/homelab-stage-2cloud-flat-canary:45:rm -rf \"$STAGE\" \"$VERIFY\"\n/usr/local/sbin/homelab-router-running-config-mail-cloud:62: rm -f \"$CONFIG\"\n/usr/local/sbin/homelab-router-running-config-mail-cloud:66: rm -f \"$CONFIG\"\n/usr/local/sbin/homelab-router-running-config-mail-cloud:73: if ( cd \"$BACKUP_ROOT\" && sha256sum -c \"$(basename \"$SHA_FILE\")\" ) >\"$WORK/sha-check.out\" 2>\"$WORK/sha-check.err\"; then\n/usr/local/sbin/homelab-router-running-config-mail-cloud:79: if rclone mkdir \"$REMOTE:$DEST\" >\"$WORK/rclone-mkdir.out\" 2>\"$WORK/rclone-mkdir.err\" && homelab-rclone-copy-chunked \"$ARCHIVE\" \"$REMOTE:$DEST/$(basename \"$ARCHIVE\")\" >\"$WORK/rclone-upload.out\" 2>\"$WORK/rclone-upload.err\" && rclone copyto \"$SHA_FILE\" \"$REMOTE:$DEST/$(basename \"$SHA_FILE\")\" --checksum >\"$WORK/rclone-upload-sha.out\" 2>\"$WORK/rclone-upload-sha.err\"; then\n/usr/local/sbin/homelab-router-running-config-mail-cloud:86: if homelab-rclone-cat-chunked \"$REMOTE:$DEST/$(basename \"$ARCHIVE\")\" >\"$DL\" 2>\"$WORK/rclone-download.err\"; then\n/usr/local/sbin/homelab-router-running-config-mail-cloud:93:rclone lsf \"$REMOTE:$TARGET_BASE\" --dirs-only 2>/dev/null | sed 's#/$##' | sort > \"$DIR_LIST\" || RETENTION_RC=$?\n/usr/local/sbin/homelab-router-running-config-mail-cloud:98: [ -n \"$OLD\" ] && rclone purge \"$REMOTE:$TARGET_BASE/$OLD\" >>\"$WORK/retention.out\" 2>>\"$WORK/retention.err\"\n/usr/local/sbin/homelab-router-running-config-mail-cloud:103: rm -f \"$old\" \"$old.sha256\"\n/usr/local/sbin/homelab-router-running-config-mail-cloud:116:rm -rf \"$WORK\"\n/usr/local/sbin/appbackup-restore-check:22:ROOT=\"/var/tmp/homelab-app-restore-check/${JOB}\"\n/usr/local/sbin/appbackup-restore-check:24:rm -rf \"$ROOT\" && mkdir -p \"$ROOT/a\" \"$ROOT/b\"\n/usr/local/sbin/appbackup-restore-check:25:trap \"echo \\\"ERROR app restore-check failed; kept: $ROOT\\\"; exit 1\" ERR\n/usr/local/sbin/appbackup-restore-check:26:echo \"restore-check app=$APP job=$JOB schema=$SCHEMA\"\n/usr/local/sbin/appbackup-restore-check:27:rclone copy \"$RA\" \"$ROOT/a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >\"${LOGBASE}-a.log\" 2>&1\n/usr/local/sbin/appbackup-restore-check:30:(cd \"$ROOT/a\" && sha256sum -c checksums.sha256 >\"${LOGBASE}-a-sha.log\" && ARCHIVE=\"$(jq -r \".archive_name\" manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n/usr/local/sbin/appbackup-restore-check:31:echo \"OK cloud A app restore-check\"\n/usr/local/sbin/appbackup-restore-check:32:rclone copy \"$RB\" \"$ROOT/b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >\"${LOGBASE}-b.log\" 2>&1\n/usr/local/sbin/appbackup-restore-check:35:(cd \"$ROOT/b\" && sha256sum -c checksums.sha256 >\"${LOGBASE}-b-sha.log\" && ARCHIVE=\"$(jq -r \".archive_name\" manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n/usr/local/sbin/appbackup-restore-check:36:echo \"OK cloud B app restore-check\"\n/usr/local/sbin/appbackup-restore-check:37:rm -rf \"$ROOT\"\n/usr/local/sbin/appbackup-restore-check:38:rmdir /var/tmp/homelab-app-restore-check 2>/dev/null || true\n/usr/local/sbin/appbackup-restore-check:39:rm -f \"${LOGBASE}-a.log\" \"${LOGBASE}-a-sha.log\" \"${LOGBASE}-b.log\" \"${LOGBASE}-b-sha.log\"\n/usr/local/sbin/appbackup-restore-check:41:echo \"OK app restore-check both clouds\"\n/usr/local/sbin/backupctl:12: check) bash -n \"$SCRIPT\" && [[ -f \"$STATE\" ]] && jq -e \".status == \\\"success\\\" and .local_backup_deleted == true\" \"$STATE\" >/dev/null && echo \"OK backup framework\" ;;\n/usr/local/sbin/backupctl:13: restore-check) backup-restore-check ;;\n/usr/local/sbin/backupctl:14: audit) backupctl check && backup-restore-check && backupctl retention 7 dry-run && backup-report >/dev/null && echo \"OK audit finished\" ;;\n/usr/local/sbin/backupctl:16: remotes) source /etc/homelab-backup.env && NODE=\"$(hostname -s)\" && echo \"A=$REMOTE_A\" && echo \"B=$REMOTE_B\" && rclone lsd \"${REMOTE_A%/}/${NODE}/emergency-pack\" | tail -n 5 && rclone lsd \"${REMOTE_B%/}/${NODE}/emergency-pack\" | tail -n 5 ;;\n/usr/local/sbin/backupctl:19: *) echo \"usage: backupctl run|status|lastlog|check|restore-check|audit|report|remotes|retention|timer\"; exit 2 ;;\n/usr/local/sbin/homelab-capacity-risk:32:LARGEST_BYTES=\"$(find /var/lib/vz/dump -maxdepth 1 -type f \\( -name \"vzdump-qemu-*.vma.zst\" -o -name \"vzdump-lxc-*.tar.zst\" \\) -printf \"%s\\n\" 2>/dev/null | sort -n | tail -n1)\"\n/usr/local/sbin/homelab-backupctl:6: audit) echo \"=== EMERGENCY AUDIT ===\"; backupctl audit; echo \"=== APP FRAMEWORK ===\"; appbackupctl check; echo \"=== APP AUDITS ===\"; for app in $(app_list); do echo \"### app=$app\"; appbackup-restore-check \"$app\"; appbackup-retention 7 dry-run \"$app\"; done; echo \"OK full backup audit\" ;;\n/usr/local/sbin/homelab-staging-check.sh:11:echo \"=== homelab staging check: ${HOST} ===\"\n/usr/local/sbin/homelab-secret-exposure-guard:18:BADFILES=\"$(find \"$REPO\" -path \"$REPO/.git\" -prune -o -type f \\( -iname \"*.key\" -o -iname \"*.pem\" -o -iname \"*.p12\" -o -iname \"*.pfx\" -o -iname \"[SENSITIVE_FILE]\" -o -iname \".env\" -o -iname \".env.*\" \\) -print 2>/dev/null)\"\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:9:OFFHOST_DIR=/mnt/staging/offhost/edge-vm-vzdump-from-pve03\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:85: /var/lib/vz/dump/vzdump-qemu-130-*.vma.zst) ;;\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:140: copy_out=\"$(run_pve03 copy \"$SOURCE_PATH\" \"$SOURCE_SIZE\" \"$SOURCE_SHA\" \"$DST_IP\" \"$OFFHOST_DIR\")\"\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:141: grep -Fq 'SOURCE_COPY_STATUS=OK' <<<\"$copy_out\"\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:156:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-backup NODE=%s VMID=%s ARCHIVE=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\\n' \\\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:158: >\"$HEALTH_DIR/edge-vm-vzdump-backup.txt\"\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:159:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-offhost NODE=%s VMID=%s HOST=%s DEST=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\\n' \\\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:161: >\"$HEALTH_DIR/edge-vm-vzdump-offhost.txt\"\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:162:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-restore-verify HOST=%s VMID=%s ARCHIVE=%s SIZE=%s SHA256=%s ZSTD_RC=0 VMA_RC=0 NOTES=nightly_existing_dump_freshness\\n' \\\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:164: >\"$HEALTH_DIR/edge-vm-vzdump-restore.txt\"\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:167: \"$HEALTH_DIR/edge-vm-vzdump-backup.txt\" \\\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:168: \"$HEALTH_DIR/edge-vm-vzdump-offhost.txt\" \\\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:169: \"$HEALTH_DIR/edge-vm-vzdump-restore.txt\"\n/usr/local/sbin/homelab-external-probe-vps:20:rm -f \"$TMP\"\n/usr/local/sbin/sqlite-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@\"$H\" \"sudo rm -f /tmp/sqlite-app-${APP}-${JOB_ID}.tgz\"\n/usr/local/sbin/sqlite-apps-backup-run:21: rm -rf \"$STAGE/payload\"\n/usr/local/sbin/homelab-desired-state-sync:10:find \"$REPO\" -mindepth 1 -maxdepth 1 ! -name .git -exec rm -rf {} + 2>/dev/null\n/usr/local/sbin/homelab-desired-state-sync:24:[ -f /root/HOMELAB_SAFETY_NET_CHECKPOINT.md ] && cp -a /root/HOMELAB_SAFETY_NET_CHECKPOINT.md \"$REPO/docs/safety-net-checkpoint.md\" 2>/dev/null || true\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:4:CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:9:: \"${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}\"\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:13:LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:19:log(){ printf '%s RESTORE_CHECK %s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$*\" | tee -a \"$LOGDIR/restore-check.log\"; }\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:22:command -v rclone >/dev/null\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:28: if rclone lsf \"${remote_path}.parts/\" > \"$plist\" 2>/dev/null && grep -q '^part-' \"$plist\"; then\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:29: while IFS= read -r part; do [ -n \"$part\" ] && rclone cat \"${remote_path}.parts/${part}\" >> \"$out\"; done < <(grep '^part-' \"$plist\" | sort)\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:31: rclone copyto \"$remote_path\" \"$out\" >/dev/null\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:33: rm -f \"$plist\"\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:36:check_one_remote(){\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:39: latest=\"$(rclone lsf --dirs-only \"${rbase}/\" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)\"\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:85: [ \"$(sqlite3 \"$extract/var-lib-skladchik-reports-monitor/state.sqlite\" 'PRAGMA integrity_check;' | tail -n 1)\" = \"ok\" ]\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:97:check_one_remote CLOUD_1 \"$CLOUD_1\"\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:98:check_one_remote CLOUD_2 \"$CLOUD_2\"\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:101:find \"$WORK_ROOT\" -mindepth 1 -maxdepth 1 -type d -printf '%f\\n' 2>/dev/null | sort | head -n -\"${KEEP_LOCAL}\" | while read -r D; do [ -n \"$D\" ] && rm -rf \"${WORK_ROOT}/${D}\"; done\n/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch:77: check=False,\n/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch:101:def check() -> int:\n/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch:126: check=False,\n/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch:223: return check()\n/usr/local/sbin/skladchik-reports-monitor-backup:28:command -v rclone >/dev/null\n/usr/local/sbin/skladchik-reports-monitor-backup:29:[ -x /usr/local/sbin/homelab-rclone-copy-chunked ] || { echo \"missing /usr/local/sbin/homelab-rclone-copy-chunked\" >&2; exit 10; }\n/usr/local/sbin/skladchik-reports-monitor-backup:36:cleanup(){ rm -rf \"$TMP\"; }\n/usr/local/sbin/skladchik-reports-monitor-backup:75: /usr/local/sbin/homelab-rclone-copy-chunked \"$ARCHIVE\" \"${DEST_BASE}/$(basename \"$ARCHIVE\")\"\n/usr/local/sbin/skladchik-reports-monitor-backup:76: /usr/local/sbin/homelab-rclone-copy-chunked \"${ARCHIVE}.sha256\" \"${DEST_BASE}/$(basename \"$ARCHIVE\").sha256\"\n/usr/local/sbin/skladchik-reports-monitor-backup:77: /usr/local/sbin/homelab-rclone-copy-chunked \"${OUTDIR}/manifest.txt\" \"${DEST_BASE}/manifest.txt\"\n/usr/local/sbin/skladchik-reports-monitor-backup:82: mapfile -t OLD < <(rclone lsf --dirs-only \"${REMOTE}${REMOTE_BASE}/daily/\" 2>/dev/null | sed 's#/$##' | sort | head -n -\"${KEEP_DAILY}\" || true)\n/usr/local/sbin/skladchik-reports-monitor-backup:83: for D in \"${OLD[@]}\"; do [ -n \"$D\" ] && rclone purge \"${REMOTE}${REMOTE_BASE}/daily/${D}\" >/dev/null 2>&1 || true; done\n/usr/local/sbin/skladchik-reports-monitor-backup:85:find \"${LOCAL_ROOT}/daily\" -mindepth 1 -maxdepth 1 -type d -printf '%f\\n' 2>/dev/null | sort | head -n -\"${KEEP_DAILY}\" | while read -r D; do [ -n \"$D\" ] && rm -rf \"${LOCAL_ROOT}/daily/${D}\"; done\n/usr/local/sbin/homelab-vm-local-dumps-retention:32: mapfile -t JOBS < <(rclone lsf \"$BASE/\" --dirs-only 2>/dev/null | sed 's#/$##' | sort)\n/usr/local/sbin/homelab-vm-local-dumps-retention:56: rclone purge \"$PATH_REMOTE\" >> \"$REPORT\" 2>&1 && DELETED=$((DELETED+1)) || BAD=$((BAD+1))\n/usr/local/sbin/homelab-extended-appbackup-run:12:run_one(){ APP=\"$1\"; HOST=\"$2\"; PATHS=\"$3\"; NOTE=\"$4\"; CONS=\"$5\"; JOB_ID=\"homelab-${NODE}-app-${APP}-${RUN_TS}\"; export JOB_ID; SHORT_TS=\"${RUN_TS:2:6}-${RUN_TS:9:6}\"; export BACKUP_PATH_REL=\"$(homelab-backup-path job a \"$APP\" \"$SHORT_TS\")\"; STAGE=\"${STAGING_ROOT}/${JOB_ID}\"; PAYLOAD=\"$STAGE/payload\"; ARCHIVE=\"${JOB_ID}.tar.gz\"; LOG=\"/tmp/extended-appbackup-${APP}-${RUN_TS}.log\"; rm -rf \"$STAGE\"; mkdir -p \"$PAYLOAD/local\"; echo \"### app=$APP\" >> \"$REPORT\"; echo \"host=$HOST\" >> \"$REPORT\"; echo \"source_note=$NOTE\" >> \"$REPORT\"; ssh -n -o BatchMode=yes -o ConnectTimeout=30 \"$HOST\" \"sudo tar -C / --ignore-failed-read --exclude=.env --exclude=*.env --exclude=.env.* --exclude=*.bak* -czf - $PATHS\" > \"$PAYLOAD/local/${APP}-data.tar.gz\" 2>\"$LOG\"; TAR_RC=$?; if [ \"$TAR_RC\" -eq 0 ] && tar -tzf \"$PAYLOAD/local/${APP}-data.tar.gz\" >/dev/null 2>&1; then SIZE=\"$(stat -c%s \"$PAYLOAD/local/${APP}-data.tar.gz\" 2>/dev/null || echo 0)\"; echo \"archive_payload_size_bytes=$SIZE\" >> \"$REPORT\"; if [ \"$SIZE\" -le 128 ]; then echo \"backup=BAD reason=archive_too_small\" >> \"$REPORT\"; BAD=$((BAD+1)); rm -rf \"$STAGE\"; else sha256sum \"$PAYLOAD/local/${APP}-data.tar.gz\" > \"$PAYLOAD/local/${APP}-data.tar.gz.sha256\"; jq -n --arg type \"extended-appbackup\" --arg app \"$APP\" --arg host \"$HOST\" --arg source_note \"$NOTE\" --arg archive_payload_size_bytes \"$SIZE\" --arg env_files_read \"no\" --arg secret_values_printed \"no\" --arg consistency \"$CONS\" \"{type:\\$type,app_name:\\$app,host:\\$host,source_note:\\$source_note,archive_payload_size_bytes:(\\$archive_payload_size_bytes|tonumber),env_files_read:\\$env_files_read,secret_values_printed:\\$secret_values_printed,consistency:\\$consistency}\" > \"$STAGE/metadata.json\"; tar -C \"$PAYLOAD\" -czf \"$STAGE/$ARCHIVE\" .; rm -rf \"$PAYLOAD\"; JOB_ID=\"$JOB_ID\" homelab-stage-2cloud-flat \"$APP\" \"$STAGE\" \"$ARCHIVE\" > \"$LOG\" 2>&1; RC=$?; if [ \"$RC\" -eq 0 ]; then echo \"backup=OK\" >> \"$REPORT\"; grep -E \"^\\[STEP\\]|^\\[SUCCESS\\]\" \"$LOG\" >> \"$REPORT\"; appbackupctl restore-check \"$APP\" >> \"$REPORT\" 2>&1 && echo \"restore_check=OK\" >> \"$REPORT\" || echo \"restore_check=WARN_non_registry\" >> \"$REPORT\"; OK=$((OK+1)); else echo \"backup=BAD rc=$RC\" >> \"$REPORT\"; grep -E \"^\\[ERROR\\]|\\[STEP\\]|ERROR|error|failed|Failed|required|missing|denied|forbidden|checksum|rclone|copy|mkdir|jq|tar\" \"$LOG\" | tail -n 60 >> \"$REPORT\"; BAD=$((BAD+1)); fi; fi; else echo \"tar=BAD rc=$TAR_RC\" >> \"$REPORT\"; grep -E \"ERROR|error|failed|Failed|denied|No such|Cannot|tar:\" \"$LOG\" | tail -n 40 >> \"$REPORT\"; BAD=$((BAD+1)); rm -rf \"$STAGE\"; fi; echo >> \"$REPORT\"; }\n/usr/local/sbin/homelab-extended-appbackup-run:21:run_one healthchecks \"$MON\" \"var/lib/healthchecks opt/stacks/healthchecks/local_settings.py\" \"healthchecks data plus local settings\" \"emergency_data_config_archive\"\n/usr/local/sbin/homelab-mkdocs-auto-refresh:53: rm -f /tmp/homelab-mkdocs-auto-refresh-git-fetch.out /tmp/homelab-mkdocs-auto-refresh-git-fetch.err\n/usr/local/sbin/homelab-mkdocs-auto-refresh:57:rm -f /tmp/homelab-mkdocs-auto-refresh-git-fetch.out /tmp/homelab-mkdocs-auto-refresh-git-fetch.err\n/usr/local/sbin/homelab-mkdocs-auto-refresh:77:ssh \"${EDGE_SSH_OPTS[@]}\" \"$EDGE_HOST\" \"rm -rf '$STAGE' && mkdir -p '$STAGE'\"\n/usr/local/sbin/homelab-mkdocs-auto-refresh:85:rm -rf \"$BASE/site.new\" \"$BASE/site.old\"\n/usr/local/sbin/homelab-mkdocs-auto-refresh:93:rm -rf \"$BASE/site.old\" \"$STAGE\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:30:CHKS=\"$WORK/checksums.txt\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:42: F=\"$(find /var/lib/vz/dump -maxdepth 1 -type f \\( -name \"vzdump-qemu-${id}-*.vma.zst\" -o -name \"vzdump-lxc-${id}-*.tar.zst\" \\) -printf \"%T@ %p\\n\" 2>/dev/null | sort -n | tail -n 1 | cut -d\" \" -f2-)\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:53: echo \"$BN\" | grep -q \"^vzdump-qemu-\" && KIND=\"qemu\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:54: echo \"$BN\" | grep -q \"^vzdump-lxc-\" && KIND=\"lxc\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:64: rm -rf \"$CDIR\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:117: rclone mkdir \"$DEST/files\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:122: rclone mkdir \"$DEST/files/$bn.chunks\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:126: rclone copyto \"$part\" \"$DEST/files/$bn.chunks/$pn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:128: rclone copyto \"$CDIR/chunks.sha256\" \"$DEST/files/$bn.chunks/chunks.sha256\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:131: rclone copyto \"$path\" \"$DEST/files/$bn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:135: rclone copyto \"$CHKS\" \"$DEST/checksums.txt\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:136: rclone copyto \"$FILES_TSV\" \"$DEST/files.tsv\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:137: rclone copyto \"$MAN\" \"$DEST/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:144: rm -rf \"$VBASE\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:146: rclone copyto \"$DEST/manifest.json\" \"$VBASE/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:154: rclone copy \"$DEST/files/$bn.chunks\" \"$VDIR\" --filter \"+ part-*\" --filter \"+ chunks.sha256\" --filter \"- *\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:163: rm -rf \"$VDIR\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:166: rclone copyto \"$DEST/files/$bn\" \"$VBASE/$bn\" --transfers 1 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:168: rm -f \"$VBASE/$bn\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:171: rm -rf \"$VBASE\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:184:rm -rf \"$VERIFY_ROOT/$JOB_ID\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:185:rm -rf \"$CHUNK_ROOT/$JOB_ID\"\n/usr/local/sbin/homelab-gitea-secondary-backup:18: rm -f \"$TMP\"\n/usr/local/sbin/homelab-gitea-secondary-backup:31:docker exec -u git -w /tmp gitea sh -lc \\\"rm -f /tmp/\\$NAME && gitea dump -c /data/gitea/conf/app.ini -f /tmp/\\$NAME\\\"\n/usr/local/sbin/homelab-gitea-secondary-backup:33:docker exec gitea rm -f \\\"/tmp/\\$NAME\\\"\n/usr/local/sbin/homelab-gitea-secondary-backup:100: rm -f -- \"$FILE\" \"$FILE.sha256\"\n/usr/local/sbin/homelab-gitea-secondary-backup:123: rm -f -- \"$FILE\" \"$FILE.sha256\"\n/usr/local/sbin/homelab-gitea-secondary-backup:136: rm -f -- \"$FILE\" \"$FILE.sha256\"\n/usr/local/sbin/homelab-rclone-copy-chunked:7:TMP_BASE=${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\n/usr/local/sbin/homelab-rclone-copy-chunked:13:cleanup(){ rm -rf \"$WORK\"; }\n/usr/local/sbin/homelab-rclone-copy-chunked:24:rclone purge \"${DEST}.parts\" >/dev/null 2>&1 || true\n/usr/local/sbin/homelab-rclone-copy-chunked:25:rclone mkdir \"${DEST}.parts\"\n/usr/local/sbin/homelab-rclone-copy-chunked:26:rclone copy \"$PARTS\" \"${DEST}.parts\" --transfers 2 --checkers 4 --retries 5 --low-level-retries 20 --stats 30s --log-level INFO\n/usr/local/sbin/homelab-rclone-copy-chunked:27:rclone copyto \"$WORK/${NAME}.sha256\" \"${DEST}.sha256\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO\n/usr/local/sbin/homelab-rclone-copy-chunked:28:rclone copyto \"$WORK/MANIFEST.txt\" \"${DEST}.manifest.txt\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO\n/usr/local/sbin/homelab-rclone-copy-chunked:29:rclone copyto \"$WORK/MANIFEST.parts.txt\" \"${DEST}.parts/MANIFEST.parts.txt\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO\n/usr/local/sbin/homelab-rclone-copy-chunked:30:REMOTE_COUNT=$(rclone lsf \"${DEST}.parts\" --files-only | grep -c \"^part-\" || true)\n/usr/local/sbin/homelab-rclone-copy-chunked:32:ACTUAL=$(while read -r PART SIZE_PART; do case \"$PART\" in part-*) rclone cat \"${DEST}.parts/$PART\" ;; esac; done < \"$WORK/MANIFEST.parts.txt\" | sha256sum | cut -d \" \" -f 1)\n/usr/local/sbin/homelab-router-hard-power-cycle:12:case \"$MODE\" in check|cycle) ;; *) echo HARD_WRAPPER_MODE_INVALID=yes; exit 64;; esac\n/usr/local/sbin/homelab-router-hard-power-cycle:23:SAFE_OUT=\"$(printf '%s\\n' \"$OUT\" | sed -E '/(PASSWORD|TOKEN|SECRET|PRIVATE_KEY|ROOT_PASSWORD|ENCRYPTION_KEY|COOKIE|SESSION|AUTHORIZATION|API_KEY|LOCAL_KEY|rclone[.]conf|[.]env)/Id')\"\n/usr/local/sbin/homelab-health-metrics-write:10:rm -f \"$tmp\"\n/usr/local/sbin/homelab-rclone-cat-flat-chunked:5:trap \"rm -f \\\"$TMP\\\"\" EXIT\n/usr/local/sbin/homelab-rclone-cat-flat-chunked:6:rclone cat \"${DEST}.parts.txt\" > \"$TMP\"\n/usr/local/sbin/homelab-rclone-cat-flat-chunked:8:while read -r PART PART_SIZE; do [[ \"$PART\" =~ ^part-[0-9]{4}$ && \"$PART_SIZE\" =~ ^[0-9]+$ ]] || { echo \"BAD FLAT PART MANIFEST\" >&2; exit 2; }; rclone cat \"${DEST}.${PART}\"; COUNT=$((COUNT+1)); done < \"$TMP\"\n/usr/local/sbin/homelab-vm-backup-policy:29:echo \"## Latest local vzdump catalog\" >> \"$REPORT\"\n/usr/local/sbin/homelab-vm-backup-policy:31:for f in $(find /var/lib/vz/dump -maxdepth 1 -type f \\( -name \"vzdump-qemu-*.vma.zst\" -o -name \"vzdump-lxc-*.tar.zst\" \\) 2>/dev/null | sort); do b=\"$(basename \"$f\")\"; size=\"$(stat -c %s \"$f\" 2>/dev/null || echo 0)\"; mt=\"$(date -u -d \"@$(stat -c %Y \"$f\" 2>/dev/null || echo 0)\" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)\"; if echo \"$b\" | grep -q \"^vzdump-qemu-\"; then kind=qemu; vmid=\"$(echo \"$b\" | sed -n \"s/^vzdump-qemu-\\([0-9]\\+\\)-.*/\\1/p\")\"; name=\"$(qm config \"$vmid\" 2>/dev/null | awk -F\": \" \"/^name:/{print \\$2; exit}\")\"; else kind=lxc; vmid=\"$(echo \"$b\" | sed -n \"s/^vzdump-lxc-\\([0-9]\\+\\)-.*/\\1/p\")\"; name=\"$(pct config \"$vmid\" 2>/dev/null | awk -F\": \" \"/^hostname:/{print \\$2; exit}\")\"; fi; printf \"%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n\" \"$kind\" \"$vmid\" \"${name:-unknown}\" \"$size\" \"$mt\" \"$f\" >> \"$CATALOG\"; done\n/usr/local/sbin/homelab-vm-backup-policy:37:echo \"- Existing local dumps can be uploaded/verified to two clouds without creating new vzdump archives.\" >> \"$REPORT\"\n/usr/local/sbin/homelab-vm-backup-policy:38:echo \"- New full vzdump for VM 170/171 is not started automatically because each VM has 120 GB bootdisk and /var/tmp has limited free space.\" >> \"$REPORT\"\n/usr/local/sbin/homelab-vm-backup-policy:42:echo \"## Coverage checks\" >> \"$REPORT\"\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:4:cat /var/lib/skladchik-reports-monitor-backup/restore-check-health.txt 2>/dev/null || true\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:7:systemctl is-enabled skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/enabled=/' || true\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:8:systemctl is-active skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/active=/' || true\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:9:systemctl list-timers --all | grep skladchik-reports-monitor-backup-restore-check || true\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:12:tail -n 80 /var/log/skladchik-reports-monitor-backup/restore-check.log 2>/dev/null || true\n/usr/local/sbin/homelab-context-refresh-direct:5:trap \"rm -rf \\\"$d\\\"\" EXIT\n/usr/local/sbin/homelab-context-refresh-direct:32:printf \"{\\\"schema_version\\\":1,\\\"channel\\\":\\\"homelab-runtime\\\",\\\"command_id\\\":\\\"%s\\\",\\\"status\\\":\\\"OK\\\",\\\"rc\\\":0,\\\"host\\\":\\\"pve01\\\",\\\"mode\\\":\\\"read-only\\\",\\\"component\\\":\\\"cluster-context\\\",\\\"started_at_utc\\\":\\\"%s\\\",\\\"finished_at_utc\\\":\\\"%s\\\",\\\"reference_register_checked\\\":true,\\\"reference_sha256\\\":\\\"%s\\\",\\\"error_register_checked\\\":true,\\\"error_register_sha256\\\":\\\"%s\\\",\\\"changes_made\\\":false,\\\"sanitized\\\":true,\\\"secrets_included\\\":false,\\\"private_addresses_included\\\":false,\\\"output\\\":\\\"AUTOMATIC_CLUSTER_CONTEXT_REFRESH=OK\\\"}\\n\" \"$id\" \"$started\" \"$finished\" \"$ref\" \"$err\" >\"$d/$id.json\"\n/usr/local/sbin/homelab-context-refresh-direct:34:ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] debian@[PRIVATE_IP] \"sudo -n /usr/local/sbin/homelab-runtime-receive $id /tmp/$id.json /tmp/$id.txt && rm -f /tmp/$id.json /tmp/$id.txt\"\n/usr/local/sbin/homelab-evidence-seal:16: rm -f \"$TMP\"\n/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:15:DUMP_SIZE=\"$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 \"$REMOTE\" \"sudo -n rm -f \\\"$REMOTE_DUMP\\\"; sudo -n -u clusteradmin pg_dump --format=custom --no-owner --no-privileges --file=\\\"$REMOTE_DUMP\\\" clusteradmin; sudo -n test -s \\\"$REMOTE_DUMP\\\"; sudo -n -u clusteradmin pg_restore --list \\\"$REMOTE_DUMP\\\" >/dev/null; sudo -n chown debian:debian \\\"$REMOTE_DUMP\\\"; sudo -n chmod 600 \\\"$REMOTE_DUMP\\\"; stat -c %s \\\"$REMOTE_DUMP\\\"\")\"\n/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:18:ssh -n -o BatchMode=yes -o ConnectTimeout=8 \"$REMOTE\" \"rm -f \\\"$REMOTE_DUMP\\\"\"\n/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:23:rm -f \"$STAGE/runtime.tgz\"\n/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:25:jq -n --arg app \"$APP\" --arg host \"cluster-admin\" --arg engine \"postgresql-custom-dump+files\" --argjson files \"$FILES\" --argjson dump_size \"$DUMP_SIZE\" \"{app:\\$app,engine:\\$engine,source_host:\\$host,source_paths:[\\\"cluster-admin:/opt/cluster-admin-incident-engine\\\",\\\"cluster-admin:/etc/cluster-admin-incident-engine\\\",\\\"cluster-admin:/var/lib/cluster-admin-incident-engine\\\",\\\"cluster-admin:PostgreSQL clusteradmin\\\"],file_count:\\$files,database_dump_size_bytes:\\$dump_size,database_dump_format:\\\"custom\\\",database_restore_list_check:\\\"ok\\\",includes_runtime_credentials:true,note:\\\"Incident engine application database state and runtime configuration\\\"}\" > \"$STAGE/metadata.json\"\n/usr/local/sbin/appbackup-retention:32:trap \"rm -rf \\\"$WORK\\\"\" EXIT\n/usr/local/sbin/appbackup-retention:35: rclone lsf \"${REMOTE%/}\" --dirs-only --max-depth 1 > \"$ROOT_LIST\"\n/usr/local/sbin/appbackup-retention:36: rclone lsf \"${REMOTE%/}/${NODE}/apps/${CANONICAL}\" --dirs-only --max-depth 1 > \"$LEGACY_LIST\"\n/usr/local/sbin/appbackup-retention:39: rclone lsf \"${REMOTE%/}/hb\" --dirs-only --max-depth 1 > \"$HB_LIST\"\n/usr/local/sbin/appbackup-retention:40: if grep -Fxq \"a/\" \"$HB_LIST\"; then rclone lsf \"${REMOTE%/}/hb/a\" --dirs-only --max-depth 1 > \"$NEW_LIST\"; fi\n/usr/local/sbin/appbackup-retention:73: rclone cat \"${REMOTE%/}/${LATEST_REL}/manifest.json\" | jq -e --arg job \"$LATEST\" \".job_id == \\$job\" >/dev/null || { echo \"REFUSE latest missing/bad remote_index=$INDEX\"; exit 5; }\n/usr/local/sbin/appbackup-retention:85: if [[ \"$MODE\" == apply ]]; then echo \"DELETE schema=dual path=$REL\"; rclone purge \"$TARGET\"; else echo \"WOULD_DELETE schema=dual path=$REL\"; fi\n/usr/local/sbin/homelab-change-transaction:160: check=False,\n/usr/local/sbin/homelab-change-transaction:195: latest_tmp.unlink(missing_ok=True)\n/usr/local/sbin/homelab-change-transaction:421: \" homelab-change-transaction run PLAN --confirm PLAN\"\n/usr/local/sbin/homelab-check-private-vpn-hosts:15:trap 'rm -f \"$tmp\"' EXIT\n/usr/local/sbin/homelab-backup-vaultwarden:26:rm -rf \"$RESTORE\"\n/usr/local/sbin/homelab-backup-vaultwarden:32:INTEGRITY=\"$(sqlite3 \"$DB\" 'PRAGMA integrity_check;')\"\n/usr/local/sbin/homelab-sops-edge-secret-coverage:97: env = os.environ.copy()\n/usr/local/sbin/homelab-sops-edge-secret-coverage:113: env=os.environ.copy(),\n/usr/local/sbin/homelab-sops-edge-secret-coverage:176: checked_at = subprocess.check_output([\"date\", \"-u\", \"+%Y-%m-%dT%H:%M:%SZ\"], text=True).strip()\n/usr/local/sbin/homelab-sops-edge-secret-coverage:242: f\"CHECKED_AT_UTC={checked_at}\",\n/usr/local/sbin/homelab-health-metrics:16:category_for_check(){\n/usr/local/sbin/homelab-health-metrics:22: *backup*|*vzdump*) printf \"%s\" \"backup\" ;;\n/usr/local/sbin/homelab-health-metrics:31:tier_for_check(){\n/usr/local/sbin/homelab-health-metrics:34: npmplus-kuma-config-*|authentik-*|gitea-*|vaultwarden-*|netbox-*|mail-cloud-restore-drill|edge-vm-vzdump-*|mkdocs-runtime-publish|npmplus-certificate-expiry) printf \"%s\" \"P0\" ;;\n/usr/local/sbin/homelab-health-metrics:69: check=\"${base%.txt}\"\n/usr/local/sbin/homelab-health-metrics:70: category=\"$(category_for_check \"$check\")\"\n/usr/local/sbin/homelab-health-metrics:71: tier=\"$(tier_for_check \"$check\")\"\n/usr/local/sbin/homelab-health-metrics:75: check_esc=\"$(esc_label \"$check\")\"\n/usr/local/sbin/homelab-health-metrics:79: labels=\"host=\\\"$host_esc\\\",check=\\\"$check_esc\\\",category=\\\"$cat_esc\\\",tier=\\\"$tier_esc\\\",file=\\\"$file_esc\\\"\"\n/usr/local/sbin/homelab-backup-health-evidence:8:backupctl check >/tmp/backup-health-emergency.log 2>&1 && EMERGENCY_OK=1 || true\n/usr/local/sbin/homelab-backup-health-evidence:9:appbackupctl check >/tmp/backup-health-appframework.log 2>&1 && APPS_OK=1 || true\n/usr/local/sbin/homelab-drift-check:7:HEALTH=/var/lib/homelab-health/drift-check.txt\n/usr/local/sbin/homelab-drift-check:24:echo \"STATUS=$STATUS TS=$(date -u +%Y-%m-%dT%H:%M:%SZ) TYPE=drift-check NOTE=$NOTE DIFF=$DIFF BASE=$BASE CURRENT=$CUR\" > \"$HEALTH\"\n/usr/local/sbin/crypto-portfolio-chart-health-check:5:trap 'rm -f \"$TMP\"' EXIT\n/usr/local/sbin/homelab-rclone-copy-flat-chunked:6:TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\"\n/usr/local/sbin/homelab-rclone-copy-flat-chunked:12:trap \"rm -rf \\\"$WORK\\\"\" EXIT\n/usr/local/sbin/homelab-rclone-copy-flat-chunked:23:while read -r PART PART_SIZE; do [[ \"$PART\" =~ ^part-[0-9]{4}$ && \"$PART_SIZE\" =~ ^[0-9]+$ ]] || { echo \"BAD PART MANIFEST\" >&2; exit 4; }; rclone copyto \"$WORK/parts/$PART\" \"${DEST}.${PART}\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR; REMOTE_BYTES=\"$(rclone size --json \"${DEST}.${PART}\" | jq -r \".bytes // -1\")\"; [[ \"$REMOTE_BYTES\" = \"$PART_SIZE\" ]] || { echo \"REMOTE PART SIZE MISMATCH $PART\" >&2; exit 5; }; UPLOADED=$((UPLOADED+1)); done < \"$WORK/MANIFEST.parts.txt\"\n/usr/local/sbin/homelab-rclone-copy-flat-chunked:25:rclone copyto \"$WORK/${NAME}.sha256\" \"${DEST}.sha256\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-rclone-copy-flat-chunked:26:rclone copyto \"$WORK/MANIFEST.txt\" \"${DEST}.manifest.txt\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-rclone-copy-flat-chunked:27:rclone copyto \"$WORK/MANIFEST.parts.txt\" \"${DEST}.parts.txt\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-rclone-copy-flat-chunked:28:ACTUAL=\"$(homelab-rclone-cat-flat-chunked \"$DEST\" | sha256sum | awk \"{print \\$1}\")\"\n/usr/local/sbin/homelab-final-readiness-gate:10:check(){ name=\"$1\"; shift; echo \"### $name\" >> \"$REPORT\"; timeout \"${CHECK_TIMEOUT_SECONDS:-300}\" \"$@\" >> \"$REPORT\" 2>&1; rc=$?; if [ \"$rc\" -eq 0 ]; then echo \"PASS $name\" >> \"$REPORT\"; PASS=$((PASS+1)); else echo \"FAIL $name rc=$rc\" >> \"$REPORT\"; FAIL=$((FAIL+1)); BAD=$((BAD+1)); fi; echo \"\" >> \"$REPORT\"; }\n/usr/local/sbin/homelab-final-readiness-gate:16:check \"private overall endpoint\" bash -c \"curl -fsS http://[PRIVATE_IP]:9101/overall.txt | grep -q STATUS=\"\n/usr/local/sbin/homelab-final-readiness-gate:17:check \"public overall endpoint\" bash -c \"curl -k -fsS https://backup.gram1.ru/overall.txt | grep -q STATUS=\"\n/usr/local/sbin/homelab-final-readiness-gate:18:check \"backup framework evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/backup-framework.txt && grep -q SOURCE_STATUS=OK /var/lib/homelab-health/backup-framework.txt && grep -q FRESHNESS=FRESH /var/lib/homelab-health/backup-framework.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:19:check \"control-plane restore-check\" appbackupctl restore-check safety-net-control-plane\n/usr/local/sbin/homelab-final-readiness-gate:20:check \"uptime-kuma restore-check\" appbackupctl restore-check uptime-kuma\n/usr/local/sbin/homelab-final-readiness-gate:21:check \"backup SLA evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/backup-sla.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:22:check \"backup coverage matrix evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/backup-coverage-matrix.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:23:check \"extended appbackup evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/extended-appbackup.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:24:check \"residual review\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/residual-review.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:25:check \"forum snuffleupagus\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/forum-snuffleupagus.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:26:check \"vm170 vm171 full strategy\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/vm170-vm171-full-strategy.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:27:check \"external probe vps\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/external-probe-vps.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:28:check \"netbird vps trust\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/netbird-vps-trust.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:29:check \"netbird vps trust closure\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/netbird-vps-trust-closure.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:30:check \"external probe runner decision\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/external-probe-runner-decision.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:31:check \"netbird vps trust clean seal\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:32:check \"live tail audit\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/live-tail-audit.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:34:check \"duty admin v2\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/duty-admin-v2.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:35:check \"cluster admin observer\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-observer.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:36:check \"cluster admin restricted probes\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-restricted-probes.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:37:check \"cluster admin full observer\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-full-observer.txt && grep -q BAD=0 /var/lib/homelab-health/cluster-admin-full-observer.txt && grep -q WARN=0 /var/lib/homelab-health/cluster-admin-full-observer.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:38:check \"cluster admin VM Mail Cloud backup\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt && grep -q BAD=0 /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt && grep -q MAIL_CLOUD_2COPY_VERIFY_OK /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:39:check \"cluster admin webpanel sync\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-webpanel-sync.txt && grep -q BAD=0 /var/lib/homelab-health/cluster-admin-webpanel-sync.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:40:check \"cluster admin webpanel\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-webpanel.txt && grep -q BAD=0 /var/lib/homelab-health/cluster-admin-webpanel.txt && curl -fsS --max-time 5 http://[PRIVATE_IP]:8080/status.txt | grep -q STATUS=OK\"\n/usr/local/sbin/homelab-final-readiness-gate:41:check \"node loss readiness\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/node-loss-readiness.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:42:check \"node loss runbooks\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/node-loss-runbooks.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:43:check \"power loss readiness\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/power-loss-readiness.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:44:check \"power loss runbook\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/power-loss-runbook.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:45:check \"ungated health backlog\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/ungated-health-backlog.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:46:check \"pve03 root cleanup\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/pve03-root-cleanup.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:47:check \"pve03 staging retention\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/pve03-staging-retention.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:48:check \"pve03 staging retention proof\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/pve03-staging-retention-proof.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:49:check \"pve03 staging retention cleanup\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/pve03-staging-retention-cleanup.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:50:check \"pve03 failed unit cleanup\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/pve03-failed-unit-cleanup.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:51:check \"remote git sops age readiness\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/remote-git-sops-age-readiness.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:52:check \"remote git sops age policy\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/remote-git-sops-age-policy.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:53:check \"desired state remote target trace\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/desired-state-remote-target-trace.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:54:check \"desired state remote target\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/desired-state-remote-target.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:55:check \"Kuma monitor policy evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/kuma-monitor-policy.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:56:check \"safe autoheal evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/safe-autoheal.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:57:check \"desired-state clean evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/desired-state.txt && test -z \"$(git -C /srv/homelab-desired-state status --short)\"\"\n/usr/local/sbin/homelab-final-readiness-gate:58:check \"runbooks evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/runbooks.txt && test -s /var/lib/homelab-runbooks/INDEX.md\"\n/usr/local/sbin/homelab-final-readiness-gate:59:check \"incident journal evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/incident-journal.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:60:check \"alerting evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/alerting.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:61:check \"VM backup health evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/vm-backup.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:62:check \"VM backup policy evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/vm-backup-policy.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:63:check \"VM local dumps cloud\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/vm-local-dumps-cloud.txt && [ $(jq -r .count /var/lib/homelab-vm-backup/latest-vm-local-dumps-cloud.json) -ge 3 ] && [ $(jq -r .chunked_count /var/lib/homelab-vm-backup/latest-vm-local-dumps-cloud.json) -ge 1 ]\"\n/usr/local/sbin/homelab-final-readiness-gate:64:check \"VM local dumps retention\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/vm-local-dumps-retention.txt && grep -q ACTIVE_LEFT=0 /var/lib/homelab-health/kuma-monitor-policy.txt || grep -q STATUS=OK /var/lib/homelab-health/vm-local-dumps-retention.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:65:check \"capacity risk evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/capacity-risk.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:66:check \"secret exposure evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/secret-exposure.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:67:check \"golden state evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/golden-state.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:68:check \"service registry evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/service-registry.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:69:check \"dependency map evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/dependency-map.txt\"\n/usr/local/sbin/homelab-docker-health.sh:44: echo \"containers_without_healthcheck=$NOHEALTH\"\n/usr/local/sbin/homelab-docker-health.sh:63: echo \"# HELP homelab_docker_health_container_without_healthcheck Docker containers without native healthcheck.\"\n/usr/local/sbin/homelab-docker-health.sh:64: echo \"# TYPE homelab_docker_health_container_without_healthcheck gauge\"\n/usr/local/sbin/homelab-docker-health.sh:65: echo \"homelab_docker_health_container_without_healthcheck $NOHEALTH\"\n/usr/local/sbin/homelab-backup-coverage-matrix:17:P1_SAFE_APPS=\"actual-budget homebox linkding mealie memos node-red syncthing filebrowser healthchecks ntfy vikunja homeassistant karakeep minio n8n\"\n/usr/local/sbin/homelab-backup-coverage-matrix:38: while read -r norm raw; do\n/usr/local/sbin/homelab-backup-coverage-matrix:50: while read -r norm raw; do\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:8:TARGET_BASE=full/nextcloud-vm-vzdump\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:11:SRC_GLOB=/var/lib/vz/dump/vzdump-qemu-150-*.vma.zst\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:77:P1 Mail Cloud Nextcloud VM recurring chunked vzdump upload\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:93:rclone mkdir \"$TARGET_REMOTE:$REMOTE_PARTS\" > \"$WORK/logs/mkdir.out\" 2>\"$WORK/logs/mkdir.err\" || MKDIR_RC=$?\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:95:rclone copy \"$PARTS_DIR\" \"$TARGET_REMOTE:$REMOTE_PARTS\" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > \"$WORK/logs/upload_parts.out\" 2>\"$WORK/logs/upload_parts.err\" || UPLOAD_PARTS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:97:rclone copyto \"$WORK/$BASENAME.sha256\" \"$TARGET_REMOTE:$REMOTE_SHA\" --checksum > \"$WORK/logs/upload_sha.out\" 2>\"$WORK/logs/upload_sha.err\" || UPLOAD_SHA_RC=$?\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:99:rclone copyto \"$WORK/MANIFEST.txt\" \"$TARGET_REMOTE:$REMOTE_MANIFEST\" --checksum > \"$WORK/logs/upload_manifest.out\" 2>\"$WORK/logs/upload_manifest.err\" || UPLOAD_MANIFEST_RC=$?\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:101:rclone copyto \"$WORK/SHA256SUMS.txt\" \"$TARGET_REMOTE:$REMOTE_SUMS\" --checksum > \"$WORK/logs/upload_sums.out\" 2>\"$WORK/logs/upload_sums.err\" || UPLOAD_SUMS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:104:rclone lsf \"$TARGET_REMOTE:$REMOTE_PARTS\" --format ps > \"$WORK/logs/remote_lsf_parts.txt\" 2>\"$WORK/logs/remote_lsf_parts.err\" || LSF_PARTS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:110:rclone copy \"$TARGET_REMOTE:$REMOTE_PARTS\" \"$DL_DIR/parts\" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > \"$WORK/logs/download_parts.out\" 2>\"$WORK/logs/download_parts.err\" || DOWNLOAD_PARTS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:120:rclone lsf \"$TARGET_REMOTE:$TARGET_BASE\" --dirs-only 2>/dev/null | sed 's#/$##' | sort > \"$DIR_LIST\" || RETENTION_RC=$?\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:126: [ -n \"$OLD\" ] && rclone purge \"$TARGET_REMOTE:$TARGET_BASE/$OLD\" >> \"$WORK/logs/retention.out\" 2>>\"$WORK/logs/retention.err\"\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:140: rm -rf \"$PARTS_DIR\" \"$DL_DIR\"\n/usr/local/sbin/homelab-rehydrate-flat-chunked-dir:10: rm -f \"$TMP_TARGET\"\n/usr/local/sbin/homelab-disk-space-health.py:10:checks = [\n/usr/local/sbin/homelab-disk-space-health.py:21:for label, cmd in checks:\n/usr/local/sbin/homelab-rclone-required-check.sh:4:RAW_FILE=\"/etc/homelab/rclone/required-remotes.txt\"\n/usr/local/sbin/homelab-rclone-required-check.sh:5:CRYPT_FILE=\"/etc/homelab/rclone/required-crypt-remotes.txt\"\n/usr/local/sbin/homelab-rclone-required-check.sh:9:echo \"=== homelab rclone required check: $(hostname) ===\"\n/usr/local/sbin/homelab-rclone-required-check.sh:12:echo \"=== installed rclone ===\"\n/usr/local/sbin/homelab-rclone-required-check.sh:13:rclone version | head -5\n/usr/local/sbin/homelab-rclone-required-check.sh:17:CONFIGURED=\"$(rclone listremotes 2>/dev/null | sed 's/:$//' || true)\"\n/usr/local/sbin/homelab-rclone-required-check.sh:19: echo \"No rclone remotes configured yet.\"\n/usr/local/sbin/homelab-legacy-monitoring-backups-run:24: rm -f \"$STAGE/remote.tgz\"\n/usr/local/sbin/homelab-legacy-monitoring-backups-run:43: rm -f \"$STAGE/pve01.tgz\"\n/usr/local/sbin/homelab-legacy-monitoring-backups-run:48: rm -f \"$STAGE/$LABEL.tgz\"\n/usr/local/sbin/homelab-legacy-monitoring-backups-run:52: rm -f \"$STAGE/monitoring.tgz\"\n/usr/local/sbin/homelab-legacy-monitoring-backups-run:71: rm -f \"$STAGE/config.tgz\"\n/usr/local/sbin/homelab-cluster-admin-inventory-sync:19:ssh -n -o BatchMode=yes -o ConnectTimeout=8 \"$REMOTE\" \"rm -f $REMOTE_TMP\"\n/usr/local/sbin/homelab-chat-run:272: re.compile(r\"(?i)\\brclone\\.conf\\b\"),\n/usr/local/sbin/homelab-chat-run:388: \"reference_register_checked\": True,\n/usr/local/sbin/homelab-chat-run:390: \"error_register_checked\": True,\n/usr/local/sbin/homelab-chat-run:489: rm -f \\\n/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:10: rm -rf \"$TMPD\" 2>/dev/null || true\n/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:19:check_host() {\n/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:39:check_host \"kuma\" \"kuma.vpn.gram1.ru\" \"^(200|302)$\"\n/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:40:check_host \"homepage\" \"homepage.vpn.gram1.ru\" \"^200$\"\n/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:41:check_host \"gitea\" \"gitea.vpn.gram1.ru\" \"^200$\"\n/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:42:check_host \"filebrowser\" \"filebrowser.vpn.gram1.ru\" \"^200$\"\n/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:43:check_host \"auth\" \"auth.vpn.gram1.ru\" \"^(200|302)$\"\n/usr/local/sbin/homelab-beszel-backup-run:33:rm -f \"$STAGE/remote.tgz\"\n/usr/local/sbin/homelab-beszel-backup-run:37:while IFS= read -r DB; do DB_COUNT=$((DB_COUNT+1)); RESULT=\"$(python3 -c \"import sqlite3,sys;print(sqlite3.connect(sys.argv[1]).execute(\\\"PRAGMA integrity_check\\\").fetchone()[0])\" \"$DB\" 2>/dev/null || echo error)\"; [ \"$RESULT\" = ok ] || exit 4; DB_OK=$((DB_OK+1)); done < <(find \"$STAGE/payload\" -type f \\( -name \"*.db\" -o -name \"*.sqlite\" -o -name \"*.sqlite3\" \\) 2>/dev/null)\n/usr/local/sbin/homelab-runbook-generate:40: f\"## First checks\\n\"\n/usr/local/sbin/homelab-runbook-generate:44: f\"appbackupctl restore-check {app}\\n\"\n/usr/local/sbin/homelab-runbook-generate:47: f\"Do not restore over a live service blindly. First confirm both encrypted cloud copies pass restore-check, target VM/container is snapshotted or intentionally stopped, and dependency map is OK.\\n\\n\"\n/usr/local/sbin/homelab-runbook-generate:51: f\"homelab-service-registry-check\\n\"\n/usr/local/sbin/homelab-runbook-generate:52: f\"homelab-dependency-map-check\\n\"\n/usr/local/sbin/homelab-runbook-generate:85: \"backupctl restore-check\\n\"\n/usr/local/sbin/homelab-runbook-generate:86: \"appbackupctl restore-check safety-net-control-plane\\n\"\n/usr/local/sbin/skladchik-reports-monitor-supervisor:25:PVE_RESTORE_HEALTH = Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\n/usr/local/sbin/skladchik-reports-monitor-supervisor:254: \"skladchik-reports-monitor-backup-restore-check.timer\",\n/usr/local/sbin/skladchik-reports-monitor-supervisor:301: \"Restore-check:\",\n/usr/local/sbin/skladchik-reports-monitor-supervisor:339: mode = sys.argv[1] if len(sys.argv) > 1 else \"check\"\n/usr/local/sbin/skladchik-reports-monitor-supervisor:340: if mode not in (\"check\", \"test\"):\n/usr/local/sbin/skladchik-reports-monitor-supervisor:341: print(\"usage: skladchik-reports-monitor-supervisor [check|test]\", file=sys.stderr)\n/usr/local/sbin/backup-restore-check:8:ROOT=\"/var/tmp/homelab-restore-check/${JOB}\"\n/usr/local/sbin/backup-restore-check:9:rm -rf \"$ROOT\" && mkdir -p \"$ROOT/a\" \"$ROOT/b\"\n/usr/local/sbin/backup-restore-check:10:trap 'echo \"ERROR restore-check failed; kept: \"; exit 1' ERR\n/usr/local/sbin/backup-restore-check:11:echo \"restore-check job=$JOB\"\n/usr/local/sbin/backup-restore-check:12:rclone copy \"$RA\" \"$ROOT/a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-a.log 2>&1; homelab-rehydrate-chunked-dir \"$ROOT/a\" >>/tmp/homelab-restore-a.log 2>&1\n/usr/local/sbin/backup-restore-check:13:(cd \"$ROOT/a\" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-a-sha.log && ARCHIVE=\"$(jq -r .archive_name manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n/usr/local/sbin/backup-restore-check:14:echo \"OK cloud A restore-check\"\n/usr/local/sbin/backup-restore-check:15:rclone copy \"$RB\" \"$ROOT/b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-b.log 2>&1; homelab-rehydrate-chunked-dir \"$ROOT/b\" >>/tmp/homelab-restore-b.log 2>&1\n/usr/local/sbin/backup-restore-check:16:(cd \"$ROOT/b\" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-b-sha.log && ARCHIVE=\"$(jq -r .archive_name manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n/usr/local/sbin/backup-restore-check:17:echo \"OK cloud B restore-check\"\n/usr/local/sbin/backup-restore-check:18:rm -rf \"$ROOT\"\n/usr/local/sbin/backup-restore-check:20:echo \"OK restore-check both clouds\"\n/usr/local/sbin/homelab-vps-identity-audit:8:trap 'rm -rf \"$TMP\"' EXIT\n/usr/local/sbin/homelab-vm-full-restore-drill:77: -name \"vzdump-qemu-${source_vmid}-*.vma.zst\" \\\n/usr/local/sbin/homelab-vm-full-restore-drill:152:check_inventory() {\n/usr/local/sbin/homelab-vm-full-restore-drill:184:check_container_health() {\n/usr/local/sbin/homelab-vm-full-restore-drill:224:check_http() {\n/usr/local/sbin/homelab-vm-full-restore-drill:349:check_container_health() {\n/usr/local/sbin/homelab-vm-full-restore-drill:386:check_container_health \"$test_vmid\"\n/usr/local/sbin/homelab-vm-full-restore-drill:389: check_inventory \"$source_vmid\" \"$test_vmid\"\n/usr/local/sbin/homelab-vm-full-restore-drill:392: check_http \"$source_vmid\" \"$test_vmid\"\n/usr/local/sbin/homelab-vm-backup-health:8:check_file(){\n/usr/local/sbin/homelab-vm-backup-health:15:check_file /var/lib/homelab-health/backup.txt auto-vmct\n/usr/local/sbin/homelab-vm-backup-health:16:check_file /var/lib/homelab-health/edge-vm-vzdump-backup.txt edge-vm-backup\n/usr/local/sbin/homelab-vm-backup-health:17:check_file /var/lib/homelab-health/edge-vm-vzdump-offhost.txt edge-vm-offhost\n/usr/local/sbin/homelab-vm-backup-health:18:check_file /var/lib/homelab-health/edge-vm-vzdump-restore.txt edge-vm-restore\n/usr/local/sbin/homelab-ingress-failover:37: rm -f \"$RESOURCES\"\n/usr/local/sbin/homelab-ingress-failover:252: rm -f \"$tmp\"\n/usr/local/sbin/homelab-overall-health:8: /var/lib/homelab-health/drift-check.txt\n/usr/local/sbin/homelab-backup-matrix-refresh:10: p=subprocess.run(args,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True,check=False)\n/usr/local/sbin/homelab-backup-matrix-refresh:65: shutil.copy2(source_path,temporary)\n/usr/local/sbin/homelab-vm-full-backup:11:# shellcheck disable=SC1090\n/usr/local/sbin/homelab-vm-full-backup:92:for cmd in vzdump qm zstd sha256sum rclone rsync ssh flock; do\n/usr/local/sbin/homelab-vm-full-backup:130: -name \"vzdump-qemu-${VMID}-*.vma.zst\" \\\n/usr/local/sbin/homelab-vm-full-backup:138: vzdump \"$VMID\" \\\n/usr/local/sbin/homelab-vm-full-backup:145: --remove 0\n/usr/local/sbin/homelab-vm-full-backup:151: -name \"vzdump-qemu-${VMID}-*.vma.zst\" \\\n/usr/local/sbin/homelab-vm-full-backup:261: CHUNK_TMP_BASE=\"$BACKUP_DIR/.rclone-chunks\" homelab-rclone-copy-chunked \"$ARCHIVE\" \"${DEST_ROOT}/${NAME}\"\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:10:TARGET_BASE=full/edge-vm-vzdump\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:64:case \"$SOURCE\" in /mnt/staging/offhost/edge-vm-vzdump-from-pve03/vzdump-qemu-130-*.vma.zst) ;; *) fail source_path_bad 2 ;; esac\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:111:rclone about \"$TARGET_REMOTE:\" > \"$WORK/logs/about.out\" 2>\"$WORK/logs/about.err\" || ABOUT_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:114:DIRS=\"$(rclone lsf \"$TARGET_REMOTE:$TARGET_BASE\" --dirs-only 2>/dev/null | sed 's#/$##' | sort || true)\"\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:116: if rclone cat \"$TARGET_REMOTE:$TARGET_BASE/$D/$BASENAME.sha256\" 2>/dev/null | grep -q \"$SHA\"; then\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:172:P1 Mail Cloud edge-vm recurring chunked vzdump upload\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:188:rclone mkdir \"$TARGET_REMOTE:$REMOTE_PARTS\" > \"$WORK/logs/mkdir.out\" 2>\"$WORK/logs/mkdir.err\" || MKDIR_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:190:rclone copy \"$PARTS_DIR\" \"$TARGET_REMOTE:$REMOTE_PARTS\" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > \"$WORK/logs/upload_parts.out\" 2>\"$WORK/logs/upload_parts.err\" || UPLOAD_PARTS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:192:rclone copyto \"$WORK/$BASENAME.sha256\" \"$TARGET_REMOTE:$REMOTE_SHA\" --checksum > \"$WORK/logs/upload_sha.out\" 2>\"$WORK/logs/upload_sha.err\" || UPLOAD_SHA_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:194:rclone copyto \"$WORK/MANIFEST.txt\" \"$TARGET_REMOTE:$REMOTE_MANIFEST\" --checksum > \"$WORK/logs/upload_manifest.out\" 2>\"$WORK/logs/upload_manifest.err\" || UPLOAD_MANIFEST_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:196:rclone copyto \"$WORK/SHA256SUMS.txt\" \"$TARGET_REMOTE:$REMOTE_SUMS\" --checksum > \"$WORK/logs/upload_sums.out\" 2>\"$WORK/logs/upload_sums.err\" || UPLOAD_SUMS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:199:rclone size \"$TARGET_REMOTE:$TARGET_DIR\" --json > \"$WORK/logs/remote_size.json\" 2>\"$WORK/logs/remote_size.err\" || SIZE_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:201:rclone lsf \"$TARGET_REMOTE:$REMOTE_PARTS\" --format \"ps\" > \"$WORK/logs/remote_lsf_parts.txt\" 2>\"$WORK/logs/remote_lsf_parts.err\" || LSF_PARTS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:207:rclone copy \"$TARGET_REMOTE:$REMOTE_PARTS\" \"$DL_DIR/parts\" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > \"$WORK/logs/download_parts.out\" 2>\"$WORK/logs/download_parts.err\" || DOWNLOAD_PARTS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:222: rm -rf \"$DL_DIR\" \"$PARTS_DIR\"\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:234: rclone lsf \"$TARGET_REMOTE:$TARGET_BASE\" --dirs-only 2>/dev/null | sed \"s#/$##\" | sort > \"$DIR_LIST\" || RETENTION_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:239: [ -n \"$OLD\" ] && rclone purge \"$TARGET_REMOTE:$TARGET_BASE/$OLD\" >> \"$WORK/logs/retention.out\" 2>>\"$WORK/logs/retention.err\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:7:BASE_DIR=\"/mnt/staging/vzdump/edge-vm-vzdump\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:8:OFFHOST_DIR=\"/mnt/staging/offhost/edge-vm-vzdump-from-pve03\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:40:REMOTE_OUT=\"$(remote \"OUT=/tmp/edge_vm_vzdump_${VMID}_${TS}.out; ERR=/tmp/edge_vm_vzdump_${VMID}_${TS}.err; : > \\$OUT; : > \\$ERR; vzdump '$VMID' --mode snapshot --compress zstd --dumpdir '$RUN_DIR' --remove 0 >\\$OUT 2>\\$ERR; RC=\\$?; ARCHIVE=\\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\\n' | sort -n | tail -n1 | cut -d' ' -f2-); LOGFILE=\\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.log' -printf '%T@ %p\\n' | sort -n | tail -n1 | cut -d' ' -f2-); SIZE=0; SHA=none; if [ -n \\\"\\$ARCHIVE\\\" ] && [ -f \\\"\\$ARCHIVE\\\" ]; then SIZE=\\$(stat -c %s \\\"\\$ARCHIVE\\\"); SHA=\\$(sha256sum \\\"\\$ARCHIVE\\\" | awk '{print \\$1}'); fi; echo VZDUMP_RC=\\$RC; echo ARCHIVE=\\$ARCHIVE; echo LOGFILE=\\$LOGFILE; echo SIZE=\\$SIZE; echo SHA=\\$SHA; echo OUT_TAIL=\\$(tail -n 6 \\$OUT | tr '\\n' '|'); echo ERR_TAIL=\\$(tail -n 6 \\$ERR | tr '\\n' '|')\")\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:62:dest \"mkdir -p '$OFFHOST_DIR' && rm -f '$DEST_PATH.tmp' '$DEST_PATH.sha256'\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:74:echo \"STATUS=OK TS=$TS TYPE=edge-vm-vzdump-backup NODE=$NODE VMID=$VMID ARCHIVE=$ARCHIVE SIZE=$SIZE SHA256=$SHA SCSI1_BACKUP0=yes NOTES=timer_run\" > \"$HEALTH_DIR/edge-vm-vzdump-backup.txt\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:75:echo \"STATUS=OK TS=$TS TYPE=edge-vm-vzdump-offhost NODE=$NODE VMID=$VMID HOST=pve02 DEST=$DEST_PATH SIZE=$SIZE SHA256=$SHA NOTES=timer_run\" > \"$HEALTH_DIR/edge-vm-vzdump-offhost.txt\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:76:echo \"STATUS=OK TS=$TS TYPE=edge-vm-vzdump-restore-verify HOST=pve02 VMID=$VMID ARCHIVE=$DEST_PATH SIZE=$SIZE SHA256=$SHA ZSTD_RC=0 VMA_RC=0 NOTES=timer_run\" > \"$HEALTH_DIR/edge-vm-vzdump-restore.txt\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:77:chmod 0644 \"$HEALTH_DIR/edge-vm-vzdump-backup.txt\" \"$HEALTH_DIR/edge-vm-vzdump-offhost.txt\" \"$HEALTH_DIR/edge-vm-vzdump-restore.txt\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:79:remote \"find '$BASE_DIR' -mindepth 1 -maxdepth 1 -type d -name 'edge-vm-*' -printf '%T@ %p\\n' | sort -nr | awk 'NR>${KEEP_LOCAL}{print \\$2}' | xargs -r rm -rf\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:80:dest \"find '$OFFHOST_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\\n' | sort -nr | awk 'NR>${KEEP_OFFHOST}{print \\$2}' | while read f; do rm -f \\\"\\$f\\\" \\\"\\$f.sha256\\\"; done\"\n/usr/local/sbin/appbackup-dockge-stacks:25:echo \"[STEP] copy app data\"\n/usr/local/sbin/appbackup-dockge-stacks:30:rm -rf \"$PAYLOAD\"\n/usr/local/sbin/appbackup-dockge-stacks:51:echo \"[STEP] local checksum\"\n/usr/local/sbin/appbackup-dockge-stacks:52:(cd \"$STAGE\" && sha256sum \"$ARCHIVE\" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e . manifest.json >/dev/null)\n/usr/local/sbin/appbackup-dockge-stacks:54:echo \"[STEP] upload cloud A metadata\"; rclone copy \"$STAGE\" \"$RA\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n/usr/local/sbin/appbackup-dockge-stacks:55:echo \"[STEP] upload cloud A archive chunks\"; rclone deletefile \"$RA/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RA/$ARCHIVE\"\n/usr/local/sbin/appbackup-dockge-stacks:56:echo \"[STEP] upload cloud B metadata\"; rclone copy \"$STAGE\" \"$RB\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n/usr/local/sbin/appbackup-dockge-stacks:57:echo \"[STEP] upload cloud B archive chunks\"; rclone deletefile \"$RB/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RB/$ARCHIVE\"\n/usr/local/sbin/appbackup-dockge-stacks:58:echo \"[STEP] verify cloud A chunked\"; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-stage-verify-chunked \"$RA\" \"$ARCHIVE\" \"$VERIFY/a\" \"$JOB\"\n/usr/local/sbin/appbackup-dockge-stacks:59:echo \"[STEP] verify cloud B chunked\"; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-stage-verify-chunked \"$RB\" \"$ARCHIVE\" \"$VERIFY/b\" \"$JOB\"\n/usr/local/sbin/appbackup-dockge-stacks:60:echo \"[STEP] upload latest index\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RAL\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RBL\"; rclone cat \"$RAL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null; rclone cat \"$RBL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n/usr/local/sbin/appbackup-dockge-stacks:76:rm -rf \"$STAGE\" \"$VERIFY\"\n/usr/local/sbin/homelab-cluster-passport:23:line \"- safety checkpoint: /root/HOMELAB_SAFETY_NET_CHECKPOINT.md\"\n/usr/local/sbin/homelab-cluster-passport:65:need_ok /var/lib/homelab-health/drift-check.txt drift-check\n/usr/local/sbin/homelab-cluster-passport:77:line \"appbackupctl restore-check safety-net-control-plane\"\n/usr/local/sbin/homelab-cluster-passport:84:line \"4. For control-plane loss: appbackupctl restore-check safety-net-control-plane, then restore the verified archive from either encrypted cloud.\"\n/usr/local/sbin/backup-retention:18: if [[ \"$MODE\" == \"apply\" ]]; then rclone cat \"$BASE/$LATEST/manifest.json\" | jq -e --arg job \"$LATEST\" \".job_id == \\$job\" >/dev/null || { echo \"REFUSE latest missing/bad on $BASE\"; exit 4; }; fi\n/usr/local/sbin/backup-retention:19: mapfile -t DIRS < <(rclone lsf \"$BASE\" --dirs-only 2>/dev/null | while read -r D; do D=\"${D%/}\"; [[ \"$D\" == homelab-${NODE}-${JOB}-* ]] && echo \"$D\"; done | sort)\n/usr/local/sbin/backup-retention:24: for ((i=0; i /var/lib/homelab-health/private-vpn-hosts.txt.tmp; rc=$?; mv /var/lib/homelab-health/private-vpn-hosts.txt.tmp /var/lib/homelab-health/private-vpn-hosts.txt; chmod 0644 /var/lib/homelab-health/private-vpn-hosts.txt; exit $rc'\n/etc/systemd/system/netbird-peers-health.service:2:Description=NetBird peers health check\n/etc/systemd/system/homelab-pve03-staging-capacity-health.timer:2:Description=Run Homelab pve03 staging capacity health check\n/etc/systemd/system/timers.target.wants/homelab-private-vpn-hosts-health.timer:2:Description=Run Homelab private VPN hosts health check every minute\n/etc/systemd/system/timers.target.wants/homelab-crypto-portfolio-chart-health.timer:2:Description=Recurring Crypto portfolio chart health check\n/etc/systemd/system/timers.target.wants/homelab-edge-vm-offhost-freshness.timer:2:Description=Run Homelab edge-vm VM130 offhost freshness sync daily after nightly vzdump\n/etc/systemd/system/timers.target.wants/homelab-dependency-map-check.timer:2:Description=Homelab dependency map check timer\n/etc/systemd/system/timers.target.wants/homelab-pve03-staging-capacity-health.timer:2:Description=Run Homelab pve03 staging capacity health check\n/etc/systemd/system/timers.target.wants/homelab-drift-check.timer:2:Description=Homelab drift check timer\n/etc/systemd/system/timers.target.wants/homelab-service-registry-check.timer:2:Description=Homelab service registry health check timer\n/etc/systemd/system/timers.target.wants/homelab-backup-healthcheck.timer:2:Description=Run homelab backup healthcheck daily\n/etc/systemd/system/timers.target.wants/homelab-backup-healthcheck.timer:8:Unit=homelab-backup-healthcheck.service\n/etc/systemd/system/timers.target.wants/netbird-peers-health.timer:2:Description=Run NetBird peers health check\n/etc/systemd/system/timers.target.wants/homelab-alerting-health.timer:2:Description=Homelab alerting health check timer\n/etc/systemd/system/timers.target.wants/skladchik-reports-monitor-backup-restore-check.timer:2:Description=Weekly restore-check of Skladchik reports monitor cloud backups\n/etc/systemd/system/timers.target.wants/skladchik-reports-monitor-backup-restore-check.timer:8:Unit=skladchik-reports-monitor-backup-restore-check.service\n/etc/systemd/system/homelab-edge-vm-vzdump-backup.timer:8:Unit=homelab-edge-vm-vzdump-backup.service\n/etc/systemd/system/homelab-drift-check.timer:2:Description=Homelab drift check timer\n/etc/systemd/system/homelab-service-registry-check.timer:2:Description=Homelab service registry health check timer\n/etc/systemd/system/homelab-pve03-staging-capacity-health.service:2:Description=Homelab pve03 staging capacity health check\n/etc/systemd/system/homelab-backup-healthcheck.timer:2:Description=Run homelab backup healthcheck daily\n/etc/systemd/system/homelab-backup-healthcheck.timer:8:Unit=homelab-backup-healthcheck.service\n/etc/systemd/system/homelab-safe-exec-health.service:2:Description=Run allowlisted homelab health checks\n/etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.service:2:Description=Homelab Mail Cloud Nextcloud VM vzdump upload\n/etc/systemd/system/hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'\n/etc/systemd/system/netbird-peers-health.timer:2:Description=Run NetBird peers health check\n/etc/systemd/system/homelab-alerting-health.timer:2:Description=Homelab alerting health check timer\n/etc/systemd/system/skladchik-reports-monitor-backup-restore-check.timer:2:Description=Weekly restore-check of Skladchik reports monitor cloud backups\n/etc/systemd/system/skladchik-reports-monitor-backup-restore-check.timer:8:Unit=skladchik-reports-monitor-backup-restore-check.service\n/etc/systemd/system/suspend-then-hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'\n/etc/cron.d/vzdump:1:# cluster wide vzdump cron schedule\n7022386770\t2026-07-18T03:31:03.6566057240\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.vma.zst\n7021952392\t2026-07-21T03:31:06.5717414950\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst\n7021297699\t2026-07-19T03:31:06.8755351020\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst\n7016275481\t2026-07-20T03:31:03.2165123470\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst\n329541843\t2026-07-21T03:30:09.7688696330\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst\n326601118\t2026-07-20T03:30:08.2096782720\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst\n323668231\t2026-07-19T03:45:46.2209130400\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst\n321121649\t2026-07-18T03:30:08.0777625900\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.tar.zst\n222777732\t2026-07-21T03:30:15.9059638290\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst\n222761273\t2026-07-20T03:30:13.7937629440\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst\n222745965\t2026-07-19T03:45:57.2700790730\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst\n222727760\t2026-07-18T03:30:13.5578457230\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.tar.zst\n3021\t2026-07-19T03:31:07.4745284120\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.log\n3020\t2026-07-21T03:31:07.3945472590\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.log\n3018\t2026-07-18T03:31:03.9197988860\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.log\n3017\t2026-07-20T03:31:04.1706719350\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.log\n1089\t2026-07-20T03:30:14.0079985330\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.log\n1086\t2026-07-20T03:30:08.4640241540\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.log\n989\t2026-07-21T03:30:16.1364976560\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.log\n989\t2026-07-18T03:30:13.7816006690\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.log\n986\t2026-07-21T03:30:10.0285327490\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.log\n986\t2026-07-18T03:30:08.3002411830\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.log\n731\t2026-07-19T03:45:57.5197000270\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.log\n729\t2026-07-19T03:45:46.4444121980\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.log\n========== NODE=pve02 ==========\n=== REMOTES ===\n=== TIMERS ===\nTue 2026-07-21 18:41:03 MSK 3h 46min Tue 2026-07-21 12:41:22 MSK 2h 12min ago forum-prod-fresh5-cloud-backup.timer forum-prod-fresh5-cloud-backup.service\nWed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:22 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service\nWed 2026-07-22 04:20:00 MSK 13h Tue 2026-07-21 04:20:12 MSK 10h ago homelab-cluster-admin-vm-mail-cloud-backup.timer homelab-cluster-admin-vm-mail-cloud-backup.service\n=== SERVICES ===\ncodevipe-cloud-backup.service static -\ndpkg-db-backup.service static -\nforum-prod-fresh5-cloud-backup.service static -\nhomelab-cluster-admin-vm-mail-cloud-backup.service static -\n/usr/local/sbin/homelab-rclone-required-check.sh:4:RAW_FILE=\"/etc/homelab/rclone/required-remotes.txt\"\n/usr/local/sbin/homelab-rclone-required-check.sh:5:CRYPT_FILE=\"/etc/homelab/rclone/required-crypt-remotes.txt\"\n/usr/local/sbin/homelab-rclone-required-check.sh:9:echo \"=== homelab rclone required check: $(hostname) ===\"\n/usr/local/sbin/homelab-rclone-required-check.sh:12:echo \"=== installed rclone ===\"\n/usr/local/sbin/homelab-rclone-required-check.sh:13:rclone version | head -5\n/usr/local/sbin/homelab-rclone-required-check.sh:17:CONFIGURED=\"$(rclone listremotes 2>/dev/null | sed 's/:$//' || true)\"\n/usr/local/sbin/homelab-rclone-required-check.sh:19: echo \"No rclone remotes configured yet.\"\n/usr/local/sbin/homelab-staging-check.sh:11:echo \"=== homelab staging check: ${HOST} ===\"\n/usr/local/sbin/routerguard-watchdog-power-entrypoint:12:case \"$MODE\" in check|cycle) ;; *) echo HARD_ENTRYPOINT_MODE_INVALID=yes; exit 64;; esac\n/usr/local/sbin/routerguard-watchdog-power-entrypoint:15:if [ \"$MODE\" = check ]; then\n/usr/local/sbin/routerguard-watchdog-power-entrypoint:16: exec \"$HELPER\" check\n/usr/local/sbin/routerguard-watchdog-power-entrypoint:21:PRE=\"$(mktemp /run/routerguard-watchdog-precheck.XXXXXX)\"\n/usr/local/sbin/routerguard-watchdog-power-entrypoint:22:cleanup(){ rm -f \"$PRE\" \"$ARM\"; }\n/usr/local/sbin/routerguard-watchdog-power-entrypoint:25:\"$HELPER\" check >\"$PRE\" 2>&1\n/usr/local/sbin/routerguard-watchdog-power-entrypoint:38:rm -f \"$ARM\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:4:CRYPT_FILE=\"/etc/homelab/rclone/required-crypt-remotes.txt\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:18: \"checksums/\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:21:echo \"=== homelab rclone crypt healthcheck: $(hostname) ===\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:40: if ! rclone listremotes | sed 's/:$//' | grep -qx \"$REMOTE\"; then\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:47: if ! rclone about \"${REMOTE}:\"; then\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:48: echo \"FAIL: rclone about failed: $REMOTE\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:54: ROOT_LIST=\"$(rclone lsf \"${REMOTE}:/\" || true)\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:76: if rclone cat \"${REMOTE}:/latest-backup-index.json\" | jq -e '.schema and .remote and .status and .items' >/dev/null; then\n/usr/local/sbin/codevipe-cloud-backup.sh:28: rclone copy \"$CACHE_ROOT/$BASE\" \"$R:$REMOTE_PATH/$BASE\" --stats 30s\n/usr/local/sbin/codevipe-cloud-backup.sh:31:find \"$CACHE_ROOT\" -mindepth 1 -maxdepth 1 -type d -mtime +14 -print -exec rm -rf {} \\;\n/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:26:rm -f \"$STAGE/$base\"\n/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:34:format=tar.gz split_parts_1500M rclone_crypt_remote\n/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:44: rclone mkdir \"$dest\" >/tmp/forum-cloud-mkdir-${r}-$TS.log 2>&1\n/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:46: rclone copy \"$STAGE\" \"$dest\" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --stats-one-line >/tmp/forum-cloud-copy-${r}-$TS.log 2>&1\n/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:49: remote_count=$(rclone lsf \"$dest\" --files-only 2>/tmp/forum-cloud-lsf-${r}-$TS.err | wc -l)\n/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:55:find /root/backups/forum-prod-fresh5 -mindepth 1 -maxdepth 1 -type d -mtime +14 -exec rm -rf {} +\n/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:73: retention_remote_count=$(rclone lsf \"$r:$REMOTE_BASE/$retention_job\" --files-only 2>/dev/null | wc -l)\n/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:78: rm -rf --one-file-system \"$retention_dir\"\n/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:83: rm -f \"$retention_keep_tmp\"\n/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:24:rclone listremotes 2>/tmp/cluster-admin-rclone-listremotes.err | grep -E 'mail.*crypt|crypt.*mail' | sed 's/:$//' | sort > /tmp/cluster-admin-mail-remotes.txt\n/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:41: rm -rf \"$STAGE/$OLD_JOB\"\n/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:43: rclone purge \"$R:homelab-backups/pve02/cluster-admin-vm180/$OLD_JOB\" >/tmp/cluster-admin-rclone-purge-$R.log 2>&1\n/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:51: ls \"$d\"/vzdump-qemu-${VMID}-*.vma.zst >/dev/null 2>&1 || { echo \"DELETE_EMPTY_LOCAL_JOB=$b\"; rm -rf \"$d\"; }\n/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:84:vzdump \"$VMID\" --mode snapshot --compress zstd --dumpdir \"$JOBDIR\" --quiet 0\n/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:89:LATEST=\"$(ls -1t \"$JOBDIR\"/vzdump-qemu-${VMID}-*.vma.zst 2>/dev/null | head -n1)\"\n/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:115: rm -f \"$TMPVMA\"\n/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:124: rm -f \"$TMPVMA\"\n/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:156: rclone mkdir \"$DEST\" >/tmp/cluster-admin-rclone-mkdir-$R.log 2>&1\n/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:158: rclone copy \"$JOBDIR\" \"$DEST\" --checksum --transfers 2 --checkers 4 --retries 3 --low-level-retries 5 >/tmp/cluster-admin-rclone-copy-$R.log 2>&1\n/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:161: rclone copyto \"$JOBDIR/MANIFEST.txt\" \"$R:homelab-backups/pve02/cluster-admin-vm180/latest/MANIFEST.txt\" --retries 3 >/tmp/cluster-admin-rclone-latest-$R.log 2>&1\n/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:164: REMOTE_SHA=\"$(rclone cat \"$DEST/$BASE.sha256\" 2>/tmp/cluster-admin-rclone-catsha-$R.err | awk '{print $1}' | head -n1)\"\n/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:177:find \"$STAGE\" -mindepth 1 -maxdepth 1 -type d -printf '%f\\n' 2>/dev/null | sort -r | tail -n +4 | while read old; do rm -rf \"$STAGE/$old\"; echo \"LOCAL_STAGE_DELETED=$old\"; done\n/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:20: rm -rf -- \"$WORK\"\n/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:29: local checked_at\n/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:31: checked_at=\"$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || printf UNKNOWN)\"\n/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:37:CHECKED_AT_UTC=$checked_at\n/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:117:for CMD in awk cat chmod date grep head mkdir mktemp rclone rm sed sha256sum sort sqlite3 tail tar wc; do\n/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:131:rclone lsf \"$REMOTE\" --recursive >\"$REMOTE_LIST\" 2>\"$ERROR_FILE\" \\\n/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:149:rclone copyto \"$REMOTE$CRITICAL_REMOTE_PATH\" \"$CRITICAL_ARCHIVE\" 2>\"$ERROR_FILE\" \\\n/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:152:rclone copyto \"$REMOTE$CRITICAL_REMOTE_SHA_PATH\" \"$CRITICAL_SHA_FILE\" 2>\"$ERROR_FILE\" \\\n/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:251:INTEGRITY_RESULT=\"$(sqlite3 \"$DB_PATH\" 'PRAGMA integrity_check;' 2>\"$ERROR_FILE\")\" \\\n/usr/local/sbin/routerguard-hard-power-cycle:3:MODE=\"${1:-check}\"\n/usr/local/sbin/routerguard-hard-power-cycle:4:case \"$MODE\" in check|cycle) ;; *) echo USAGE=routerguard-hard-power-cycle_check_or_cycle; exit 64;; esac\n/usr/local/sbin/routerguard-network:48:remove_rules(){\n/usr/local/sbin/routerguard-network:91: rm -f \"$RUN/active\"\n/usr/local/sbin/routerguard-network:92: remove_rules\n/usr/local/sbin/routerguard-network:100: rm -rf \"$RUN\"\n/etc/systemd/system/suspend.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'\n/etc/systemd/system/suspend-then-hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'\n/etc/systemd/system/hybrid-sleep.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'\n/etc/systemd/system/multi-user.target.wants/corosync.service:20:# as well; before editing this line, you may want to check DefaultStandardError\n/etc/systemd/system/multi-user.target.wants/postfix.service:19:# perform 2-stage startup\n/etc/systemd/system/multi-user.target.wants/postfix.service:20:ExecStartPre=+postfix check\n/etc/systemd/system/multi-user.target.wants/postfix.service:30:# Individual Postfix daemons are started as root, optionally perform chroot\n/etc/systemd/system/multi-user.target.wants/postfix.service:52:# CAP_CHOWN can be removed.\n/etc/systemd/system/multi-user.target.wants/pve-guests.service:22:ExecStop=-/usr/bin/vzdump -stop\n/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:22:ExecStart=/usr/sbin/pve-ha-lrm start\n/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:23:ExecStop=/usr/sbin/pve-ha-lrm stop\n/etc/systemd/system/multi-user.target.wants/rrdcached.service:2:# To customize, copy this file to /etc/systemd/system/\n/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:14:ExecStart=/usr/sbin/pve-ha-crm start\n/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:15:ExecStop=/usr/sbin/pve-ha-crm stop\n/etc/systemd/system/multi-user.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'\n/etc/systemd/system/multi-user.target.wants/e2scrub_reap.service:2:Description=Remove Stale Online ext4 Metadata Check Snapshots\n/etc/systemd/system/default.target.wants/nvmefc-boot-connections.service:19:RemoveIPC=yes\n/etc/systemd/system/default.target.wants/nvmf-autoconnect.service:21:RemoveIPC=yes\n/etc/systemd/system/sysinit.target.wants/lvm2-lvmpolld.socket:10:RemoveOnStop=true\n/etc/systemd/system/sysinit.target.wants/pvenetcommit.service:8:ExecStartPre=-/bin/rm -f /etc/openvswitch/conf.db\n/etc/systemd/system/sysinit.target.wants/systemd-pstore.service:11:Description=Platform Persistent Storage Archival\n/etc/systemd/system/pve-manager.service:22:ExecStop=-/usr/bin/vzdump -stop\n/etc/systemd/system/hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'\n/etc/systemd/system/sockets.target.wants/dm-event.socket:10:RemoveOnStop=true\n/etc/systemd/system/sockets.target.wants/rrdcached.socket:2:# To customize, copy this file to /etc/systemd/system/\n/etc/cron.d/vzdump:1:# cluster wide vzdump cron schedule\n997503422\t2026-07-01T03:31:11.1700059310\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.vma.zst\n868048369\t2026-06-30T03:31:08.4903256780\t/var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.vma.zst\n288349157\t2026-07-21T03:30:21.2480085730\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.tar.zst\n286538526\t2026-07-20T03:30:18.4386603270\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.tar.zst\n284443608\t2026-07-19T03:47:41.8525503200\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst\n283022034\t2026-07-18T03:30:19.7617214460\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_18-03_30_03.tar.zst\n220080970\t2026-07-21T03:30:33.3910787180\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.tar.zst\n220065825\t2026-07-20T03:30:30.8287264090\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.tar.zst\n220049041\t2026-07-19T03:47:58.9256407960\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst\n220033534\t2026-07-18T03:30:37.2878143100\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_18-03_30_20.tar.zst\n7547\t2026-07-18T01:09:39.2467331080\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_18-01_02_44.log\n7443\t2026-07-18T01:45:13.3149404960\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_18-01_38_20.log\n5796\t2026-07-17T21:55:21.5648458010\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_17-21_30_20.log\n2685\t2026-06-30T03:31:09.1616927060\t/var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.log\n2590\t2026-07-01T03:31:11.9500372070\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.log\n1088\t2026-07-20T03:30:31.0784689220\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.log\n1085\t2026-07-20T03:30:18.7427551960\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.log\n988\t2026-07-21T03:30:33.6364088960\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.log\n988\t2026-07-18T03:30:37.5450736960\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_18-03_30_20.log\n985\t2026-07-21T03:30:21.5465818340\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.log\n985\t2026-07-18T03:30:20.0884248500\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_18-03_30_03.log\n731\t2026-07-19T03:47:59.2238139650\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.log\n728\t2026-07-19T03:47:42.1183953260\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.log\n========== NODE=pve03 ==========\n=== REMOTES ===\n=== TIMERS ===\nWed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:10 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service\n=== SERVICES ===\ndpkg-db-backup.service static -\n/usr/local/sbin/homelab-staging-check.sh:11:echo \"=== homelab staging check: ${HOST} ===\"\n/usr/local/sbin/homelab-immutable-offline-verify:11: rm -f \"$TMP\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:4:CRYPT_FILE=\"/etc/homelab/rclone/required-crypt-remotes.txt\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:18: \"checksums/\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:21:echo \"=== homelab rclone crypt healthcheck: $(hostname) ===\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:40: if ! rclone listremotes | sed 's/:$//' | grep -qx \"$REMOTE\"; then\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:47: if ! rclone about \"${REMOTE}:\"; then\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:48: echo \"FAIL: rclone about failed: $REMOTE\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:54: ROOT_LIST=\"$(rclone lsf \"${REMOTE}:/\" || true)\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:76: if rclone cat \"${REMOTE}:/latest-backup-index.json\" | jq -e '.schema and .remote and .status and .items' >/dev/null; then\n/usr/local/sbin/homelab-rclone-required-check.sh:4:RAW_FILE=\"/etc/homelab/rclone/required-remotes.txt\"\n/usr/local/sbin/homelab-rclone-required-check.sh:5:CRYPT_FILE=\"/etc/homelab/rclone/required-crypt-remotes.txt\"\n/usr/local/sbin/homelab-rclone-required-check.sh:9:echo \"=== homelab rclone required check: $(hostname) ===\"\n/usr/local/sbin/homelab-rclone-required-check.sh:12:echo \"=== installed rclone ===\"\n/usr/local/sbin/homelab-rclone-required-check.sh:13:rclone version | head -5\n/usr/local/sbin/homelab-rclone-required-check.sh:17:CONFIGURED=\"$(rclone listremotes 2>/dev/null | sed 's/:$//' || true)\"\n/usr/local/sbin/homelab-rclone-required-check.sh:19: echo \"No rclone remotes configured yet.\"\n/etc/systemd/system/hybrid-sleep.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'\n/etc/systemd/system/multi-user.target.wants/pve-guests.service:22:ExecStop=-/usr/bin/vzdump -stop\n/etc/systemd/system/multi-user.target.wants/corosync.service:20:# as well; before editing this line, you may want to check DefaultStandardError\n/etc/systemd/system/multi-user.target.wants/rrdcached.service:2:# To customize, copy this file to /etc/systemd/system/\n/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:14:ExecStart=/usr/sbin/pve-ha-crm start\n/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:15:ExecStop=/usr/sbin/pve-ha-crm stop\n/etc/systemd/system/multi-user.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'\n/etc/systemd/system/multi-user.target.wants/postfix.service:19:# perform 2-stage startup\n/etc/systemd/system/multi-user.target.wants/postfix.service:20:ExecStartPre=+postfix check\n/etc/systemd/system/multi-user.target.wants/postfix.service:30:# Individual Postfix daemons are started as root, optionally perform chroot\n/etc/systemd/system/multi-user.target.wants/postfix.service:52:# CAP_CHOWN can be removed.\n/etc/systemd/system/multi-user.target.wants/e2scrub_reap.service:2:Description=Remove Stale Online ext4 Metadata Check Snapshots\n/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:22:ExecStart=/usr/sbin/pve-ha-lrm start\n/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:23:ExecStop=/usr/sbin/pve-ha-lrm stop\n/etc/systemd/system/sysinit.target.wants/pvenetcommit.service:8:ExecStartPre=-/bin/rm -f /etc/openvswitch/conf.db\n/etc/systemd/system/sysinit.target.wants/systemd-pstore.service:11:Description=Platform Persistent Storage Archival\n/etc/systemd/system/sysinit.target.wants/lvm2-lvmpolld.socket:10:RemoveOnStop=true\n/etc/systemd/system/hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'\n/etc/systemd/system/sockets.target.wants/rrdcached.socket:2:# To customize, copy this file to /etc/systemd/system/\n/etc/systemd/system/sockets.target.wants/dm-event.socket:10:RemoveOnStop=true\n/etc/systemd/system/suspend-then-hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'\n/etc/systemd/system/default.target.wants/nvmf-autoconnect.service:21:RemoveIPC=yes\n/etc/systemd/system/default.target.wants/nvmefc-boot-connections.service:19:RemoveIPC=yes\n/etc/systemd/system/suspend.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'\n/etc/systemd/system/pve-manager.service:22:ExecStop=-/usr/bin/vzdump -stop\n/etc/cron.d/vzdump:1:# cluster wide vzdump cron schedule\n11116472459\t2026-07-20T03:33:53.2839064370\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.vma.zst\n11043834894\t2026-07-21T03:33:54.3725716230\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.vma.zst\n11013723404\t2026-07-19T03:33:53.3463878040\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.vma.zst\n10994820121\t2026-07-18T03:33:47.1540038630\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_18-03_30_02.vma.zst\n1137072350\t2026-07-21T03:34:37.6276811360\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.vma.zst\n838951976\t2026-07-19T03:34:33.1034859540\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.vma.zst\n838817925\t2026-07-20T03:34:37.1719936670\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.vma.zst\n785249297\t2026-07-18T03:34:27.1531047780\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_18-03_33_48.vma.zst\n7106\t2026-07-02T03:37:53.1040459570\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_02-03_30_02.log\n7102\t2026-07-03T03:36:49.1036784480\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_03-03_30_05.log\n6564\t2026-07-18T03:33:48.0107527290\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_18-03_30_02.log\n6560\t2026-07-20T03:33:54.9283349630\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.log\n6559\t2026-07-21T03:33:55.1428157340\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.log\n6459\t2026-07-19T03:33:53.7631656400\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.log\n6246\t2026-07-04T03:33:39.2197744030\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_04-03_30_02.log\n4107\t2026-07-07T03:31:51.2516816970\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_07-03_30_01.log\n2642\t2026-07-21T03:34:38.4491706750\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.log\n2500\t2026-07-20T03:34:37.2013439550\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.log\n2404\t2026-07-19T03:34:34.1266444600\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.log\n2305\t2026-07-18T03:34:27.7179455720\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_18-03_33_48.log\n=== CLOUD_HEALTH_AND_OBJECTS ===\n=== CURRENT_BACKUP_MATRIX ===\nSTATUS=FAIL\nTS=2026-07-21T07:02:41Z\nTYPE=unified-backup-matrix\nMATRIX_MODE=authoritative\nJOBS_TOTAL=41\nJOBS_OK=30\nJOBS_BAD=6\nJOBS_UNMAPPED=0\nJOBS_DISABLED=5\nSTATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_JSON=/etc/homelab-backup-matrix/matrix.json\nAUTHORITY=canonical\nCOMPAT_OUTPUTS=29\nCOMPAT_MAP=/var/lib/homelab-backup-matrix/compat-map.json\nJOB=dpkg-db-backup STATUS=OK EVIDENCE=execution_only AGE=36147 MAX_AGE=172800\nJOB=filebrowser-backup STATUS=OK EVIDENCE=artifact_or_health AGE=14443 MAX_AGE=259200\nJOB=filebrowser-restore-validate STATUS=STALE EVIDENCE=artifact_or_health AGE=1016424 MAX_AGE=864000\nJOB=homelab-appbackup-dockge-stacks STATUS=OK EVIDENCE=artifact_or_health AGE=22248 MAX_AGE=172800\nJOB=homelab-appbackup-mariadb-batch STATUS=OK EVIDENCE=execution_only AGE=15387 MAX_AGE=172800\nJOB=homelab-appbackup-npmplus-edge STATUS=OK EVIDENCE=artifact_or_health AGE=20573 MAX_AGE=172800\nJOB=homelab-appbackup-postgres-batch STATUS=OK EVIDENCE=execution_only AGE=16416 MAX_AGE=172800\nJOB=homelab-appbackup-sqlite-batch STATUS=OK EVIDENCE=execution_only AGE=18327 MAX_AGE=172800\nJOB=homelab-authentik-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987790 MAX_AGE=172800\nJOB=homelab-auto-backup STATUS=OK EVIDENCE=execution_only AGE=195424 MAX_AGE=864000\nJOB=homelab-backup-coverage-matrix STATUS=OK EVIDENCE=artifact_or_health AGE=4346 MAX_AGE=172800\nJOB=homelab-backup-freshness STATUS=OK EVIDENCE=execution_only AGE=35589 MAX_AGE=172800\nJOB=homelab-backup-healthcheck STATUS=OK EVIDENCE=execution_only AGE=10631 MAX_AGE=172800\nJOB=homelab-backup-sla-health STATUS=FAIL EVIDENCE=artifact_or_health AGE=144 MAX_AGE=172800\nJOB=homelab-edge-vm-offhost-freshness STATUS=OK EVIDENCE=artifact_or_health AGE=18426 MAX_AGE=172800\nJOB=homelab-emergency-backup STATUS=OK EVIDENCE=artifact_or_health AGE=24539 MAX_AGE=172800\nJOB=homelab-extended-appbackup STATUS=OK EVIDENCE=artifact_or_health AGE=14156 MAX_AGE=172800\nJOB=homelab-gitea-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987785 MAX_AGE=172800\nJOB=homelab-golden-state-index STATUS=OK EVIDENCE=artifact_or_health AGE=4112 MAX_AGE=172800\nJOB=homelab-mail-cloud-critical-upload STATUS=OK EVIDENCE=artifact_or_health AGE=11997 MAX_AGE=172800\nJOB=homelab-mail-cloud-edge-vm-upload STATUS=OK EVIDENCE=artifact_or_health AGE=174045 MAX_AGE=864000\nJOB=homelab-mail-cloud-nextcloud-vm-upload STATUS=OK EVIDENCE=artifact_or_health AGE=7575 MAX_AGE=864000\nJOB=homelab-mail-cloud-restore-drill STATUS=STALE EVIDENCE=artifact_or_health AGE=988269 MAX_AGE=864000\nJOB=homelab-npmplus-kuma-config-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=60071 MAX_AGE=172800\nJOB=homelab-router-running-config-mail-cloud STATUS=OK EVIDENCE=artifact_or_health AGE=17076 MAX_AGE=172800\nJOB=homelab-vaultwarden-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987768 MAX_AGE=172800\nJOB=homelab-verified-backup-health STATUS=OK EVIDENCE=artifact_or_health AGE=11845 MAX_AGE=172800\nJOB=homelab-vm-backup-health STATUS=OK EVIDENCE=artifact_or_health AGE=8742 MAX_AGE=172800\nJOB=homelab-vm-backup-policy STATUS=OK EVIDENCE=artifact_or_health AGE=5548 MAX_AGE=172800\nJOB=homelab-vm-full-backup STATUS=OK EVIDENCE=artifact_or_health AGE=24778 MAX_AGE=172800\nJOB=homelab-vm-full-restore-drill STATUS=OK EVIDENCE=artifact_or_health AGE=523085 MAX_AGE=10368000\nJOB=homelab-vm-local-dumps-2cloud STATUS=OK EVIDENCE=artifact_or_health AGE=172141 MAX_AGE=864000\nJOB=homelab-vm-local-dumps-retention STATUS=OK EVIDENCE=artifact_or_health AGE=168839 MAX_AGE=864000\nJOB=immich-media-offhost-sync STATUS=OK EVIDENCE=artifact_or_health AGE=4286 MAX_AGE=259200\nJOB=memos-backup STATUS=OK EVIDENCE=artifact_or_health AGE=14547 MAX_AGE=259200\nJOB=netbird-vps-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=1747896 MAX_AGE=864000\nJOB=paperless-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=16347 MAX_AGE=259200\nJOB=paperless-restore-dry-run STATUS=STALE EVIDENCE=artifact_or_health AGE=1001986 MAX_AGE=864000\nJOB=skladchik-reports-monitor-backup STATUS=OK EVIDENCE=artifact_or_health AGE=19639 MAX_AGE=172800\nJOB=skladchik-reports-monitor-backup-restore-check STATUS=OK EVIDENCE=artifact_or_health AGE=190273 MAX_AGE=864000\nJOB=vm160-remote-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=26607 MAX_AGE=172800\nPAIR_DELETE_GUARD_AUDIT=COMPLETE\n" +} diff --git a/runtime/history/BACKUP-PAIR-DELETE-GUARD-066.txt b/runtime/history/BACKUP-PAIR-DELETE-GUARD-066.txt new file mode 100644 index 0000000..c28c90f --- /dev/null +++ b/runtime/history/BACKUP-PAIR-DELETE-GUARD-066.txt @@ -0,0 +1,1017 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=BACKUP-PAIR-DELETE-GUARD-066 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=backup-audit +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1 +COMMAND_SHA256=e64555881c22fffab137df86ee957819b9922f83078f71124f8ff61e8580590d +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=false +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=d13bc63fd488f7e13f81bef392b271beacdec9c637576dc61e7dad33042b27f2 +SANITIZED_OUTPUT_SHA256=6c697f245387cba271db9658a3e6e5ce9d5a9ac0178945c559417c7064daa692 +OUTPUT_BEGIN +=== PIPELINES_AND_CLOUD_PAIRS === +========== NODE=pve01 ========== +=== REMOTES === +=== TIMERS === +Wed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:14 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service +Wed 2026-07-22 00:09:43 MSK 9h Tue 2026-07-21 00:09:32 MSK 14h ago homelab-backup-freshness.timer homelab-backup-freshness.service +Wed 2026-07-22 02:15:00 MSK 11h Tue 2026-07-21 02:15:05 MSK 12h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service +Wed 2026-07-22 03:10:02 MSK 12h Tue 2026-07-21 03:13:14 MSK 11h ago homelab-emergency-backup.timer homelab-emergency-backup.service +Wed 2026-07-22 03:35:00 MSK 12h Tue 2026-07-21 03:35:14 MSK 11h ago filebrowser-backup.timer filebrowser-backup.service +Wed 2026-07-22 03:55:02 MSK 13h Tue 2026-07-21 03:51:25 MSK 11h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service +Wed 2026-07-22 04:17:29 MSK 13h Tue 2026-07-21 04:19:17 MSK 10h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service +Wed 2026-07-22 04:26:54 MSK 13h Tue 2026-07-21 04:28:14 MSK 10h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service +Wed 2026-07-22 04:38:04 MSK 13h Tue 2026-07-21 04:34:48 MSK 10h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service +Wed 2026-07-22 05:01:44 MSK 14h Tue 2026-07-21 04:57:14 MSK 9h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service +Wed 2026-07-22 05:23:56 MSK 14h Tue 2026-07-21 05:17:57 MSK 9h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service +Wed 2026-07-22 05:27:13 MSK 14h Tue 2026-07-21 05:29:05 MSK 9h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service +Wed 2026-07-22 05:47:19 MSK 14h Tue 2026-07-21 05:46:14 MSK 9h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service +Wed 2026-07-22 06:01:19 MSK 15h Tue 2026-07-21 05:57:30 MSK 8h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service +Wed 2026-07-22 06:27:43 MSK 15h Tue 2026-07-21 06:32:25 MSK 8h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service +Wed 2026-07-22 06:38:51 MSK 15h Tue 2026-07-21 06:43:54 MSK 8h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service +Wed 2026-07-22 06:39:22 MSK 15h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service +Wed 2026-07-22 06:44:16 MSK 15h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service +Wed 2026-07-22 06:52:06 MSK 15h Tue 2026-07-21 06:47:47 MSK 8h ago homelab-beszel-backup.timer homelab-beszel-backup.service +Wed 2026-07-22 06:58:58 MSK 16h Tue 2026-07-21 06:57:30 MSK 7h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service +Wed 2026-07-22 07:02:27 MSK 16h Tue 2026-07-21 07:05:30 MSK 7h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service +Wed 2026-07-22 07:07:14 MSK 16h Tue 2026-07-21 07:18:55 MSK 7h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service +Wed 2026-07-22 07:35:06 MSK 16h Tue 2026-07-21 07:36:59 MSK 7h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service +Wed 2026-07-22 07:38:21 MSK 16h Tue 2026-07-21 07:36:14 MSK 7h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service +Wed 2026-07-22 07:41:49 MSK 16h Tue 2026-07-21 07:53:41 MSK 7h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service +Wed 2026-07-22 08:10:37 MSK 17h Tue 2026-07-21 08:07:25 MSK 6h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service +Wed 2026-07-22 08:37:02 MSK 17h Tue 2026-07-21 08:30:07 MSK 6h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service +Wed 2026-07-22 08:40:13 MSK 17h Tue 2026-07-21 08:50:14 MSK 6h ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service +Wed 2026-07-22 08:53:36 MSK 17h Tue 2026-07-21 08:41:14 MSK 6h ago memos-backup.timer memos-backup.service +Sun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service +Sun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service +Sun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service +Sun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service +Mon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 7h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service +=== SERVICES === +dpkg-db-backup.service static - +filebrowser-backup.service static - +homelab-alertmanager-backup.service static - +homelab-appbackup-dockge-stacks.service static - +homelab-appbackup-mariadb-batch.service static - +homelab-appbackup-npmplus-edge.service static - +homelab-appbackup-postgres-batch.service static - +homelab-appbackup-sqlite-batch.service static - +homelab-authentik-backup.service static - +homelab-auto-backup.service static - +homelab-backup-coverage-matrix.service static - +homelab-backup-freshness.service static - +homelab-backup-healthcheck.service static - +homelab-backup-matrix-refresh.service static - +homelab-backup-sla-health.service static - +homelab-beszel-backup.service static - +homelab-blackbox-exporter-backup.service static - +homelab-cluster-admin-incident-engine-backup.service static - +homelab-crypto-portfolio-backup.service static - +homelab-edge-vm-vzdump-backup.service static - +homelab-emergency-backup.service static - +homelab-extended-appbackup.service static - +homelab-gitea-backup.service static - +homelab-gitea-secondary-backup.service static - +homelab-legacy-monitoring-backups.service static - +homelab-mail-cloud-critical-upload.service static - +homelab-mail-cloud-edge-vm-upload.service static - +homelab-mail-cloud-nextcloud-vm-upload.service static - +homelab-mail-cloud-restore-drill.service static - +homelab-npmplus-kuma-config-backup.service static - +homelab-router-running-config-mail-cloud.service static - +homelab-vaultwarden-backup.service static - +homelab-verified-backup-health.service static - +homelab-vm-backup-health.service static - +homelab-vm-backup-policy.service static - +homelab-vm-full-backup.service static - +homelab-vm-local-dumps-2cloud.service static - +homelab-vm160-remote-backup.service static - +immich-media-backup.service static - +memos-backup.service static - +netbird-vps-backup.service static - +paperless-backup.service static - +skladchik-reports-monitor-backup-restore-check.service static - +skladchik-reports-monitor-backup.service static - +/usr/local/sbin/homelab-vm160-remote-backup:8:PARENT=/mnt/staging/vzdump +/usr/local/sbin/homelab-vm160-remote-backup:22:SOURCE_ACTIVE="$(ssh "${SSH[@]}" "$SOURCE" "pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'")" || SOURCE_ACTIVE_RC=$? +/usr/local/sbin/homelab-vm160-remote-backup:27:NAME="vzdump-qemu-${VMID}-${TS}.vma.zst" +/usr/local/sbin/homelab-vm160-remote-backup:35: ssh "${SSH[@]}" "$TARGET" "rm -rf -- '$DIR'" >/dev/null 2>&1 || true +/usr/local/sbin/homelab-vm160-remote-backup:41: rm -f "$LOG" +/usr/local/sbin/homelab-vm160-remote-backup:46:LATEST_SIZE="$(ssh "${SSH[@]}" "$TARGET" "find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\n' | sort -nr | head -n1")" +/usr/local/sbin/homelab-vm160-remote-backup:55:ssh "${SSH[@]}" "$SOURCE" "vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7" 2>"$LOG" | ssh "${SSH[@]}" "$TARGET" "cat > '$PART'" +/usr/local/sbin/homelab-vm160-remote-backup:57:ssh "${SSH[@]}" "$TARGET" "cat > '$DIR/vzdump.log'" < "$LOG" +/usr/local/sbin/homelab-vm160-remote-backup:66:rm -f "$LOG" +/usr/local/sbin/skladchik-reports-monitor-status:11:check_health() { +/usr/local/sbin/skladchik-reports-monitor-status:24:check_timer() { +/usr/local/sbin/skladchik-reports-monitor-status:38:check_timer skladchik-reports-monitor-backup.timer +/usr/local/sbin/skladchik-reports-monitor-status:39:check_timer skladchik-reports-monitor-backup-restore-check.timer +/usr/local/sbin/skladchik-reports-monitor-status:40:check_timer skladchik-reports-monitor-supervisor.timer +/usr/local/sbin/skladchik-reports-monitor-status:41:check_timer skladchik-reports-monitor-monthly-selftest-watch.timer +/usr/local/sbin/skladchik-reports-monitor-status:43:check_health "PVE01: ежедневный бэкап" "/var/lib/skladchik-reports-monitor-backup/health.txt" +/usr/local/sbin/skladchik-reports-monitor-status:44:check_health "PVE01: проверка восстановления" "/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt" +/usr/local/sbin/skladchik-reports-monitor-status:45:check_health "PVE01: общий supervisor" "/var/lib/skladchik-reports-monitor-supervisor/health.txt" +/usr/local/sbin/skladchik-reports-monitor-status:46:check_health "PVE01: контроль ежемесячного самотеста" "/var/lib/skladchik-reports-monitor-selftest-watch/health.txt" +/usr/local/sbin/homelab-mail-cloud-restore-drill:52: homelab-rclone-cat-chunked "${R}:$CRIT_DEST/$CRIT_BUNDLE" > "$OUT" 2>"$WORK/logs/${SAFE}_crit_download.err" +/usr/local/sbin/homelab-mail-cloud-restore-drill:83: rm -f "$OUT" +/usr/local/sbin/homelab-mail-cloud-restore-drill:108: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/MANIFEST.txt" "$WORK/edge/MANIFEST.txt" > "$WORK/logs/edge_manifest_download.out" 2>"$WORK/logs/edge_manifest_download.err" +/usr/local/sbin/homelab-mail-cloud-restore-drill:111: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/SHA256SUMS.txt" "$WORK/edge/SHA256SUMS.txt" > "$WORK/logs/edge_sums_download.out" 2>"$WORK/logs/edge_sums_download.err" +/usr/local/sbin/homelab-mail-cloud-restore-drill:114: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/$EDGE_ARCHIVE.sha256" "$WORK/edge/$EDGE_ARCHIVE.sha256" > "$WORK/logs/edge_archive_sha_download.out" 2>"$WORK/logs/edge_archive_sha_download.err" +/usr/local/sbin/homelab-mail-cloud-restore-drill:117: rclone lsf "${EDGE_REMOTE}:$EDGE_DEST/parts" --format ps > "$WORK/logs/edge_parts_lsf.txt" 2>"$WORK/logs/edge_parts_lsf.err" +/usr/local/sbin/homelab-mail-cloud-restore-drill:130: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/parts/$FIRST_PART" "$WORK/edge/$FIRST_PART" > "$WORK/logs/edge_first_part_download.out" 2>"$WORK/logs/edge_first_part_download.err" +/usr/local/sbin/homelab-mail-cloud-restore-drill:137: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/parts/$LAST_PART" "$WORK/edge/$LAST_PART" > "$WORK/logs/edge_last_part_download.out" 2>"$WORK/logs/edge_last_part_download.err" +/usr/local/sbin/homelab-mail-cloud-restore-drill:168: rm -f "$WORK/edge/$FIRST_PART" "$WORK/edge/$LAST_PART" +/usr/local/sbin/homelab-golden-state-extended-overlay:26:rclone copyto "$JSON" "$RA" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-golden-state-extended-overlay:27:rclone copyto "$JSON" "$RB" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-golden-state-extended-overlay:28:rclone cat "$RA" 2>/dev/null | jq -e --arg cp "$CP" ".control_plane_job == \$cp" >/dev/null || BAD=$((BAD+1)) +/usr/local/sbin/homelab-golden-state-extended-overlay:29:rclone cat "$RB" 2>/dev/null | jq -e --arg cp "$CP" ".control_plane_job == \$cp" >/dev/null || BAD=$((BAD+1)) +/usr/local/sbin/homelab-mail-cloud-critical-upload:50:EXCLUDED=edge-vm-vzdump-large-archive,rclone-config,secrets,raw-env +/usr/local/sbin/homelab-mail-cloud-critical-upload:80: rclone mkdir "${R}:$REMOTE_DIR" > "$WORK/logs/${SAFE}_mkdir.out" 2>"$WORK/logs/${SAFE}_mkdir.err" +/usr/local/sbin/homelab-mail-cloud-critical-upload:82: homelab-rclone-copy-chunked "$BUNDLE" "${R}:$REMOTE_DIR/$(basename "$BUNDLE")" > "$WORK/logs/${SAFE}_upload_bundle.out" 2>"$WORK/logs/${SAFE}_upload_bundle.err" +/usr/local/sbin/homelab-mail-cloud-critical-upload:84: rclone copyto "$BUNDLE_SHA" "${R}:$REMOTE_DIR/$(basename "$BUNDLE_SHA")" --checksum > "$WORK/logs/${SAFE}_upload_sha.out" 2>"$WORK/logs/${SAFE}_upload_sha.err" +/usr/local/sbin/homelab-mail-cloud-critical-upload:86: rclone copyto "$WORK/components/MANIFEST.txt" "${R}:$REMOTE_DIR/MANIFEST.txt" --checksum > "$WORK/logs/${SAFE}_upload_manifest.out" 2>"$WORK/logs/${SAFE}_upload_manifest.err" +/usr/local/sbin/homelab-mail-cloud-critical-upload:88: rclone copyto "$WORK/components/SHA256SUMS.txt" "${R}:$REMOTE_DIR/SHA256SUMS.txt" --checksum > "$WORK/logs/${SAFE}_upload_sums.out" 2>"$WORK/logs/${SAFE}_upload_sums.err" +/usr/local/sbin/homelab-mail-cloud-critical-upload:90: homelab-rclone-cat-chunked "${R}:$REMOTE_DIR/$(basename "$BUNDLE")" > "$WORK/download/${SAFE}-$(basename "$BUNDLE")" 2>"$WORK/logs/${SAFE}_download_bundle.err" +/usr/local/sbin/homelab-mail-cloud-critical-upload:108: rm -f "$WORK/download/${SAFE}-$(basename "$BUNDLE")" +/usr/local/sbin/homelab-mail-cloud-critical-upload:109: rclone lsf "${R}:$TARGET_BASE" --dirs-only 2>/dev/null | sed 's#/$##' | sort > "$WORK/logs/${SAFE}_dirs.txt" +/usr/local/sbin/homelab-mail-cloud-critical-upload:114: [ -n "$OLD" ] && rclone purge "${R}:$TARGET_BASE/$OLD" >> "$WORK/logs/${SAFE}_retention.out" 2>>"$WORK/logs/${SAFE}_retention.err" +/usr/local/sbin/homelab-mail-cloud-critical-upload:140: rm -rf --one-file-system "$OLD_DIR" +/usr/local/sbin/homelab-alertmanager-backup-run:16:ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$MON" "sudo -n docker exec alertmanager amtool check-config /etc/alertmanager/alertmanager.yml >/dev/null && sudo -n tar -C / -czf - opt/stacks/observability-lite/compose.yaml opt/stacks/observability-lite/compose.override.yaml opt/stacks/observability-lite/alertmanager/alertmanager.yml opt/stacks/observability-lite/alertmanager/data" > "$STAGE/remote.tgz" +/usr/local/sbin/homelab-alertmanager-backup-run:18:rm -f "$STAGE/remote.tgz" +/usr/local/sbin/homelab-alertmanager-backup-run:21:jq -n --arg app "$APP" --arg host "monitoring" --arg engine "files+alertmanager-config-check" --arg config "ok" --argjson files "$FILES" "{app:\$app,engine:\$engine,source_host:\$host,source_paths:[\"monitoring:/opt/stacks/observability-lite/alertmanager\",\"monitoring:/opt/stacks/observability-lite/compose.yaml\",\"monitoring:/opt/stacks/observability-lite/compose.override.yaml\"],config_check:\$config,file_count:\$files,note:\"Alertmanager configuration and runtime state in encrypted two-cloud backup\"}" > "$STAGE/metadata.json" +/usr/local/sbin/appbackupctl:13: check) bash -n /usr/local/sbin/appbackup-dockge-stacks && bash -n /usr/local/sbin/appbackup-restore-check && bash -n /usr/local/sbin/appbackup-retention && bash -n /usr/local/sbin/appbackup-report && echo "OK appbackup framework" ;; +/usr/local/sbin/appbackupctl:14: restore-check) appbackup-restore-check "${2:-dockge-stacks}" ;; +/usr/local/sbin/appbackupctl:17: audit) appbackupctl check && appbackupctl restore-check dockge-stacks && appbackupctl retention 7 dry-run dockge-stacks && appbackupctl report dockge-stacks >/dev/null && echo "OK app audit finished" ;; +/usr/local/sbin/appbackupctl:21: *) echo "usage: appbackupctl run [dockge-stacks|cluster-admin-incident-engine]|check|restore-check|retention|report|audit|status|lastlog|timer"; exit 2 ;; +/usr/local/sbin/postgres-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@"$H" "sudo rm -f /tmp/postgres-app-${APP}-${JOB_ID}.tgz" +/usr/local/sbin/postgres-apps-backup-run:21: rm -rf "$STAGE/payload" +/usr/local/sbin/homelab-backup-gitea:27:rm -rf "$RESTORE" +/usr/local/sbin/homelab-backup-gitea:35:INTEGRITY="$(sqlite3 "$DB" 'PRAGMA integrity_check;')" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:4:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:18: "checksums/" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:21:echo "=== homelab rclone crypt healthcheck: $(hostname) ===" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:40: if ! rclone listremotes | sed 's/:$//' | grep -qx "$REMOTE"; then +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:47: if ! rclone about "${REMOTE}:"; then +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:48: echo "FAIL: rclone about failed: $REMOTE" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:54: ROOT_LIST="$(rclone lsf "${REMOTE}:/" || true)" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:76: if rclone cat "${REMOTE}:/latest-backup-index.json" | jq -e '.schema and .remote and .status and .items' >/dev/null; then +/usr/local/sbin/homelab-check-private-vpn-hosts.realnet-20260624T140509Z:4:REAL="/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z" +/usr/local/sbin/homelab-stage-verify-flat-chunked:7:rm -rf "$VERIFY" +/usr/local/sbin/homelab-stage-verify-flat-chunked:9:rclone copy "$REMOTE_DIR" "$VERIFY" --transfers 4 --checkers 8 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-stage-verify-flat-chunked:11:(cd "$VERIFY" && test -s manifest.json && if [[ -n "$JOB" ]]; then jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null; fi && sha256sum -c checksums.sha256 >/dev/null && tar -tzf "$ARCHIVE" >/dev/null) +/usr/local/sbin/homelab-rclone-cat-chunked:4:TMP_BASE=${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks} +/usr/local/sbin/homelab-rclone-cat-chunked:7:cleanup(){ rm -rf "$WORK"; } +/usr/local/sbin/homelab-rclone-cat-chunked:10:rclone copyto "${REMOTE_FILE}.parts/MANIFEST.parts.txt" "$MAN" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-rclone-cat-chunked:11:while read -r PART SIZE; do case "$PART" in part-*) rclone cat "${REMOTE_FILE}.parts/$PART" ;; esac; done < "$MAN" +/usr/local/sbin/homelab-stage-verify-chunked:7:rm -rf "$VERIFY" +/usr/local/sbin/homelab-stage-verify-chunked:9:rclone copy "$REMOTE_DIR" "$VERIFY" --exclude "$ARCHIVE.parts/**" --exclude "$ARCHIVE.parts/" --exclude "$ARCHIVE.sha256" --exclude "$ARCHIVE.manifest.txt" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --log-level ERROR +/usr/local/sbin/homelab-stage-verify-chunked:13:if [ -s checksums.sha256 ]; then grep -F -v " $ARCHIVE" checksums.sha256 > .checksums-no-archive.sha256 || true; if [ -s .checksums-no-archive.sha256 ]; then sha256sum -c .checksums-no-archive.sha256; fi; EXPECTED=$(grep -F " $ARCHIVE" checksums.sha256 | awk '{print $1; exit}'); else EXPECTED=$(rclone cat "${REMOTE_DIR}/${ARCHIVE}.sha256" | awk '{print $1; exit}'); fi +/usr/local/sbin/homelab-stage-verify-chunked:15:ACTUAL=$(homelab-rclone-cat-chunked "${REMOTE_DIR}/${ARCHIVE}" | sha256sum | awk '{print $1}') +/usr/local/sbin/homelab-stage-verify-chunked:17:homelab-rclone-cat-chunked "${REMOTE_DIR}/${ARCHIVE}" | tar -tzf - >/dev/null +/usr/local/sbin/crypto-portfolio-backup-run:15:ssh -n -o BatchMode=yes -o ConnectTimeout=10 "$CORE" 'H="$(hostname -f 2>/dev/null || hostname)"; { [ "$H" = "core-apps.gram1.ru" ] || [ "$H" = "core-apps" ]; } || exit 3; D=/opt/stacks/crypto-portfolio; T="$(sudo -n mktemp -d /tmp/crypto-portfolio-backup.XXXXXX)"; trap "sudo -n rm -rf \"$T\"" EXIT; sudo -n install -d -m 700 "$T/stack" "$T/stack/data"; for F in compose.yaml Dockerfile requirements.txt app.py worker.py .dockerignore; do sudo -n test -s "$D/$F" && sudo -n cp -a "$D/$F" "$T/stack/"; done; sudo -n cp -a "$D/templates" "$D/secrets" "$T/stack/"; sudo -n python3 -c "import sqlite3,sys;s=sqlite3.connect(sys.argv[1]);d=sqlite3.connect(sys.argv[2]);s.backup(d);d.close();s.close()" "$D/data/portfolio.db" "$T/stack/data/portfolio.db"; for F in market.json worker-status.json; do sudo -n test -s "$D/data/$F" && sudo -n cp -a "$D/data/$F" "$T/stack/data/"; done; sudo -n tar -C "$T" -czf - stack' > "$STAGE/remote.tgz" +/usr/local/sbin/crypto-portfolio-backup-run:17:rm -f "$STAGE/remote.tgz" +/usr/local/sbin/crypto-portfolio-backup-run:18:DBI="$(python3 -c 'import sqlite3,sys;print(sqlite3.connect(sys.argv[1]).execute("PRAGMA integrity_check").fetchone()[0])' "$STAGE/payload/stack/data/portfolio.db")" +/usr/local/sbin/npmplus-edge-backup-run:20:ssh -o BatchMode=yes -o ConnectTimeout=8 "$EDGE" "sudo rm -f '/tmp/npmplus-edge-${JOB_ID}.tgz'" +/usr/local/sbin/npmplus-edge-backup-run:28:rm -rf "$STAGE/payload" +/usr/local/sbin/mariadb-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@"$H" "sudo rm -f /tmp/mariadb-app-${APP}-${JOB_ID}.tgz" +/usr/local/sbin/mariadb-apps-backup-run:20: rm -rf "$STAGE/payload" +/usr/local/sbin/homelab-blackbox-exporter-backup-run:17:rm -f "$STAGE/remote.tgz" +/usr/local/sbin/homelab-blackbox-exporter-backup-run:21:jq -n --arg app "$APP" --arg host "monitoring" --arg engine "files+runtime-health-check" --argjson files "$FILES" '{app:$app,engine:$engine,source_host:$host,source_paths:["monitoring:/opt/stacks/blackbox-exporter"],runtime_health:"ok",file_count:$files,note:"Blackbox Exporter compose and probe configuration in encrypted two-cloud backup"}' > "$STAGE/metadata.json" +/usr/local/sbin/homelab-ingress-standby-refresh:16: rm -f "$TMP_STATE" "$RELAY" "$RESOURCES" +/usr/local/sbin/homelab-ingress-standby-refresh:74:run_node "$SOURCE_NODE" vzdump 130 \ +/usr/local/sbin/homelab-ingress-standby-refresh:78: --remove 0 \ +/usr/local/sbin/homelab-ingress-standby-refresh:83:"find /var/lib/vz/dump -maxdepth 1 -type f -name 'vzdump-qemu-130-*.vma.zst' -printf '%T@ %p\n' | sort -rn | head -n 1 | cut -d' ' -f2-" +/usr/local/sbin/homelab-ingress-standby-refresh:106:"chattr -i '$DEST_FILE' '$DEST_FILE.sha256' '$DEST_FILE.env' 2>/dev/null || true; rm -f '$DEST_TMP'" +/usr/local/sbin/homelab-ingress-standby-refresh:155: -name 'vzdump-qemu-130-*.vma.zst' \ +/usr/local/sbin/homelab-ingress-standby-refresh:164: rm -f "$OLD" "$OLD.sha256" "$OLD.env" +/usr/local/sbin/homelab-ingress-standby-refresh:173:run_node "$SOURCE_NODE" rm -f "$SOURCE_BACKUP" +/usr/local/sbin/homelab-stage-2cloud-flat:33:(cd "$STAGE"; FILES=("$ARCHIVE" manifest.json latest-backup-index.json); [[ -f metadata.json ]] && FILES+=(metadata.json); sha256sum "${FILES[@]}" > checksums.sha256; sha256sum -c checksums.sha256 >/dev/null; tar -tzf "$ARCHIVE" >/dev/null; jq -e . manifest.json >/dev/null) +/usr/local/sbin/homelab-stage-2cloud-flat:36: if rclone lsf "$REMOTE_DIR" --max-depth 1 2>/dev/null | grep -q .; then echo "REFUSE NONEMPTY CANARY TARGET $REMOTE_DIR" >&2; exit 6; fi +/usr/local/sbin/homelab-stage-2cloud-flat:37: rclone mkdir "$REMOTE_DIR" +/usr/local/sbin/homelab-stage-2cloud-flat:38: for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f "$STAGE/$F" ]] && rclone copyto "$STAGE/$F" "$REMOTE_DIR/$F" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done +/usr/local/sbin/homelab-stage-2cloud-flat:39: CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-flat-chunked "$STAGE/$ARCHIVE" "$REMOTE_DIR/$ARCHIVE" +/usr/local/sbin/homelab-stage-2cloud-flat:41: rclone copyto "$STAGE/latest-backup-index.json" "$LATEST_FILE" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-stage-2cloud-flat:42: rclone cat "$LATEST_FILE" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null +/usr/local/sbin/homelab-stage-2cloud-flat:47:rm -rf "$STAGE" "$VERIFY" +/usr/local/sbin/homelab-cluster-admin-inventory-export:12: return subprocess.check_output(args, text=True, stderr=subprocess.DEVNULL, timeout=30) +/usr/local/sbin/homelab-cluster-admin-webpanel-sync:7:copy_health() { +/usr/local/sbin/homelab-cluster-admin-webpanel-sync:17:copy_health /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt cluster-admin-vm-mail-cloud-backup.txt +/usr/local/sbin/homelab-cluster-admin-webpanel-sync:18:copy_health /var/lib/homelab-health/overall.txt overall.txt +/usr/local/sbin/homelab-cluster-admin-webpanel-sync:19:copy_health /var/lib/homelab-health/final-readiness.txt final-readiness.txt +/usr/local/sbin/homelab-cluster-admin-webpanel-sync:20:copy_health /var/lib/homelab-health/cluster-admin-strict-final-seal.txt cluster-admin-strict-final-seal.txt +/usr/local/sbin/homelab-backup-authentik:14:ssh debian@[PRIVATE_IP] "rm -rf /tmp/auth-small-snap; mkdir -p /tmp/auth-small-snap" +/usr/local/sbin/homelab-backup-authentik:27:ssh debian@[PRIVATE_IP] "sudo docker rm -f authentik-restore-test >/dev/null 2>&1 || true" +/usr/local/sbin/homelab-backup-authentik:51:ssh debian@[PRIVATE_IP] "sudo docker rm -f authentik-restore-test >/dev/null 2>&1 || true; rm -f /tmp/authentik-restore-from-archive.pg_dump" +/usr/local/sbin/homelab-safe-exec:77: check=False, +/usr/local/sbin/homelab-safe-exec:621: latest_tmp.unlink(missing_ok=True) +/usr/local/sbin/homelab-golden-state-index:28:rclone copyto "$JSON" "$RA" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-golden-state-index:29:rclone copyto "$JSON" "$RB" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-golden-state-index:30:rclone cat "$RA" 2>/dev/null | jq -e --arg cp "$CP" --arg ext "$EXT" ".control_plane_job == \$cp and .health.extended_appbackup == \$ext and .extended_appbackup_health == \$ext" >/dev/null || BAD=$((BAD+1)) +/usr/local/sbin/homelab-golden-state-index:31:rclone cat "$RB" 2>/dev/null | jq -e --arg cp "$CP" --arg ext "$EXT" ".control_plane_job == \$cp and .health.extended_appbackup == \$ext and .extended_appbackup_health == \$ext" >/dev/null || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-full-retention:21: -name "vzdump-qemu-${vmid}-*.vma.zst" \ +/usr/local/sbin/homelab-vm-full-retention:33: vzdump-qemu-"$vmid"-*.vma.zst) +/usr/local/sbin/homelab-vm-full-retention:35: rm -f \ +/usr/local/sbin/homelab-vm-full-retention:73: rm -rf -- \"\$BASE/\$setname\" +/usr/local/sbin/homelab-vm-full-retention:91: rclone lsf "$path" \ +/usr/local/sbin/homelab-vm-full-retention:105: rclone purge "${path}/${dir}" +/usr/local/sbin/homelab-stage-2cloud:22:echo "[STEP] local checksum"; (cd "$STAGE" && sha256sum "$ARCHIVE" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e . manifest.json >/dev/null) +/usr/local/sbin/homelab-stage-2cloud:23:echo "[STEP] upload cloud A metadata"; rclone copy "$STAGE" "$RA" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s +/usr/local/sbin/homelab-stage-2cloud:24:echo "[STEP] upload cloud A archive chunks"; rclone deletefile "$RA/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RA/$ARCHIVE" +/usr/local/sbin/homelab-stage-2cloud:25:echo "[STEP] upload cloud B metadata"; rclone copy "$STAGE" "$RB" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s +/usr/local/sbin/homelab-stage-2cloud:26:echo "[STEP] upload cloud B archive chunks"; rclone deletefile "$RB/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RB/$ARCHIVE" +/usr/local/sbin/homelab-stage-2cloud:27:echo "[STEP] verify cloud A chunked"; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-stage-verify-chunked "$RA" "$ARCHIVE" "$VERIFY/a" "$JOB" +/usr/local/sbin/homelab-stage-2cloud:28:echo "[STEP] verify cloud B chunked"; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-stage-verify-chunked "$RB" "$ARCHIVE" "$VERIFY/b" "$JOB" +/usr/local/sbin/homelab-stage-2cloud:29:echo "[STEP] upload latest index"; rclone copyto "$STAGE/latest-backup-index.json" "$RAL"; rclone copyto "$STAGE/latest-backup-index.json" "$RBL"; rclone cat "$RAL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null; rclone cat "$RBL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null +/usr/local/sbin/homelab-stage-2cloud:31:echo "[STEP] delete local staging"; rm -rf "$STAGE" "$VERIFY"; test ! -d "$STAGE" && test ! -d "$VERIFY" +/usr/local/sbin/homelab-safe-autoheal:17:SAFE_TIMERS="homelab-emergency-backup.timer homelab-appbackup-dockge-stacks.timer homelab-appbackup-npmplus-edge.timer homelab-appbackup-sqlite-batch.timer homelab-appbackup-postgres-batch.timer homelab-appbackup-mariadb-batch.timer homelab-verified-backup-health.timer homelab-drift-check.timer homelab-service-registry-check.timer homelab-dependency-map-check.timer homelab-vm-backup-health.timer homelab-alerting-health.timer homelab-runbook-generate.timer homelab-desired-state-sync.timer homelab-duty-admin-report.timer homelab-incident-journal.timer homelab-backup-sla-health.timer homelab-kuma-monitor-policy.timer homelab-safe-autoheal.timer homelab-overall-health.timer" +/usr/local/sbin/homelab-safe-autoheal:144:for cmd in homelab-alerting-health homelab-kuma-monitor-policy homelab-backup-sla-health homelab-service-registry-check homelab-dependency-map-check; do +/usr/local/sbin/homelab-backup-npmplus-kuma-configs:26: install -D -m 0755 /usr/local/sbin/homelab-check-private-vpn-hosts "$SNAP/pve01/usr/local/sbin/homelab-check-private-vpn-hosts" 2>>"$ERR" || fail "copy_private_vpn_script_failed" +/usr/local/sbin/homelab-backup-npmplus-kuma-configs:27: install -D -m 0644 /etc/systemd/system/homelab-private-vpn-hosts-health.service "$SNAP/pve01/etc/systemd/system/homelab-private-vpn-hosts-health.service" 2>>"$ERR" || fail "copy_private_vpn_service_failed" +/usr/local/sbin/homelab-backup-npmplus-kuma-configs:28: install -D -m 0644 /etc/systemd/system/homelab-private-vpn-hosts-health.timer "$SNAP/pve01/etc/systemd/system/homelab-private-vpn-hosts-health.timer" 2>>"$ERR" || fail "copy_private_vpn_timer_failed" +/usr/local/sbin/homelab-backup-npmplus-kuma-configs:29: install -D -m 0644 /var/lib/homelab-health/private-vpn-hosts.txt "$SNAP/pve01/var/lib/homelab-health/private-vpn-hosts.txt" 2>>"$ERR" || fail "copy_private_vpn_health_failed" +/usr/local/sbin/homelab-backup-npmplus-kuma-configs:42:sudo rm -rf "$TMP" "$EDGE_TGZ" +/usr/local/sbin/homelab-backup-npmplus-kuma-configs:82:sudo rm -rf "$TMP" +/usr/local/sbin/homelab-backup-npmplus-kuma-configs:90: ssh -o BatchMode=yes -o ConnectTimeout=8 debian@[PRIVATE_IP] "rm -f '$EDGE_TGZ'" 2>>"$ERR" || true +/usr/local/sbin/homelab-backup-npmplus-kuma-configs:99: NPM_INTEGRITY="$(sqlite3 "$SNAP/edge-vm/npmplus/db/database.sqlite" "PRAGMA integrity_check;" 2>>"$ERR" | tr '\n' ';' || echo error)" +/usr/local/sbin/homelab-backup-npmplus-kuma-configs:100: KUMA_INTEGRITY="$(sqlite3 "$SNAP/edge-vm/kuma/db/kuma.db" "PRAGMA integrity_check;" 2>>"$ERR" | tr '\n' ';' || echo error)" +/usr/local/sbin/homelab-backup-npmplus-kuma-configs:134: rm -rf "$RESTORE" +/usr/local/sbin/homelab-backup-npmplus-kuma-configs:138: RESTORE_NPM_INTEGRITY="$(sqlite3 "$RESTORE/edge-vm/npmplus/db/database.sqlite" "PRAGMA integrity_check;" 2>>"$ERR" | tr '\n' ';' || echo error)" +/usr/local/sbin/homelab-backup-npmplus-kuma-configs:139: RESTORE_KUMA_INTEGRITY="$(sqlite3 "$RESTORE/edge-vm/kuma/db/kuma.db" "PRAGMA integrity_check;" 2>>"$ERR" | tr '\n' ';' || echo error)" +/usr/local/sbin/homelab-stage-2cloud-flat-canary:31:(cd "$STAGE"; FILES=("$ARCHIVE" manifest.json latest-backup-index.json); [[ -f metadata.json ]] && FILES+=(metadata.json); sha256sum "${FILES[@]}" > checksums.sha256; sha256sum -c checksums.sha256 >/dev/null; tar -tzf "$ARCHIVE" >/dev/null; jq -e . manifest.json >/dev/null) +/usr/local/sbin/homelab-stage-2cloud-flat-canary:34: if rclone lsf "$REMOTE_DIR" --max-depth 1 2>/dev/null | grep -q .; then echo "REFUSE NONEMPTY CANARY TARGET $REMOTE_DIR" >&2; exit 6; fi +/usr/local/sbin/homelab-stage-2cloud-flat-canary:35: rclone mkdir "$REMOTE_DIR" +/usr/local/sbin/homelab-stage-2cloud-flat-canary:36: for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f "$STAGE/$F" ]] && rclone copyto "$STAGE/$F" "$REMOTE_DIR/$F" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done +/usr/local/sbin/homelab-stage-2cloud-flat-canary:37: CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-flat-chunked "$STAGE/$ARCHIVE" "$REMOTE_DIR/$ARCHIVE" +/usr/local/sbin/homelab-stage-2cloud-flat-canary:39: rclone copyto "$STAGE/latest-backup-index.json" "$LATEST_FILE" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-stage-2cloud-flat-canary:40: rclone cat "$LATEST_FILE" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null +/usr/local/sbin/homelab-stage-2cloud-flat-canary:45:rm -rf "$STAGE" "$VERIFY" +/usr/local/sbin/homelab-router-running-config-mail-cloud:62: rm -f "$CONFIG" +/usr/local/sbin/homelab-router-running-config-mail-cloud:66: rm -f "$CONFIG" +/usr/local/sbin/homelab-router-running-config-mail-cloud:73: if ( cd "$BACKUP_ROOT" && sha256sum -c "$(basename "$SHA_FILE")" ) >"$WORK/sha-check.out" 2>"$WORK/sha-check.err"; then +/usr/local/sbin/homelab-router-running-config-mail-cloud:79: if rclone mkdir "$REMOTE:$DEST" >"$WORK/rclone-mkdir.out" 2>"$WORK/rclone-mkdir.err" && homelab-rclone-copy-chunked "$ARCHIVE" "$REMOTE:$DEST/$(basename "$ARCHIVE")" >"$WORK/rclone-upload.out" 2>"$WORK/rclone-upload.err" && rclone copyto "$SHA_FILE" "$REMOTE:$DEST/$(basename "$SHA_FILE")" --checksum >"$WORK/rclone-upload-sha.out" 2>"$WORK/rclone-upload-sha.err"; then +/usr/local/sbin/homelab-router-running-config-mail-cloud:86: if homelab-rclone-cat-chunked "$REMOTE:$DEST/$(basename "$ARCHIVE")" >"$DL" 2>"$WORK/rclone-download.err"; then +/usr/local/sbin/homelab-router-running-config-mail-cloud:93:rclone lsf "$REMOTE:$TARGET_BASE" --dirs-only 2>/dev/null | sed 's#/$##' | sort > "$DIR_LIST" || RETENTION_RC=$? +/usr/local/sbin/homelab-router-running-config-mail-cloud:98: [ -n "$OLD" ] && rclone purge "$REMOTE:$TARGET_BASE/$OLD" >>"$WORK/retention.out" 2>>"$WORK/retention.err" +/usr/local/sbin/homelab-router-running-config-mail-cloud:103: rm -f "$old" "$old.sha256" +/usr/local/sbin/homelab-router-running-config-mail-cloud:116:rm -rf "$WORK" +/usr/local/sbin/appbackup-restore-check:22:ROOT="/var/tmp/homelab-app-restore-check/${JOB}" +/usr/local/sbin/appbackup-restore-check:24:rm -rf "$ROOT" && mkdir -p "$ROOT/a" "$ROOT/b" +/usr/local/sbin/appbackup-restore-check:25:trap "echo \"ERROR app restore-check failed; kept: $ROOT\"; exit 1" ERR +/usr/local/sbin/appbackup-restore-check:26:echo "restore-check app=$APP job=$JOB schema=$SCHEMA" +/usr/local/sbin/appbackup-restore-check:27:rclone copy "$RA" "$ROOT/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >"${LOGBASE}-a.log" 2>&1 +/usr/local/sbin/appbackup-restore-check:30:(cd "$ROOT/a" && sha256sum -c checksums.sha256 >"${LOGBASE}-a-sha.log" && ARCHIVE="$(jq -r ".archive_name" manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null) +/usr/local/sbin/appbackup-restore-check:31:echo "OK cloud A app restore-check" +/usr/local/sbin/appbackup-restore-check:32:rclone copy "$RB" "$ROOT/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >"${LOGBASE}-b.log" 2>&1 +/usr/local/sbin/appbackup-restore-check:35:(cd "$ROOT/b" && sha256sum -c checksums.sha256 >"${LOGBASE}-b-sha.log" && ARCHIVE="$(jq -r ".archive_name" manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null) +/usr/local/sbin/appbackup-restore-check:36:echo "OK cloud B app restore-check" +/usr/local/sbin/appbackup-restore-check:37:rm -rf "$ROOT" +/usr/local/sbin/appbackup-restore-check:38:rmdir /var/tmp/homelab-app-restore-check 2>/dev/null || true +/usr/local/sbin/appbackup-restore-check:39:rm -f "${LOGBASE}-a.log" "${LOGBASE}-a-sha.log" "${LOGBASE}-b.log" "${LOGBASE}-b-sha.log" +/usr/local/sbin/appbackup-restore-check:41:echo "OK app restore-check both clouds" +/usr/local/sbin/backupctl:12: check) bash -n "$SCRIPT" && [[ -f "$STATE" ]] && jq -e ".status == \"success\" and .local_backup_deleted == true" "$STATE" >/dev/null && echo "OK backup framework" ;; +/usr/local/sbin/backupctl:13: restore-check) backup-restore-check ;; +/usr/local/sbin/backupctl:14: audit) backupctl check && backup-restore-check && backupctl retention 7 dry-run && backup-report >/dev/null && echo "OK audit finished" ;; +/usr/local/sbin/backupctl:16: remotes) source /etc/homelab-backup.env && NODE="$(hostname -s)" && echo "A=$REMOTE_A" && echo "B=$REMOTE_B" && rclone lsd "${REMOTE_A%/}/${NODE}/emergency-pack" | tail -n 5 && rclone lsd "${REMOTE_B%/}/${NODE}/emergency-pack" | tail -n 5 ;; +/usr/local/sbin/backupctl:19: *) echo "usage: backupctl run|status|lastlog|check|restore-check|audit|report|remotes|retention|timer"; exit 2 ;; +/usr/local/sbin/homelab-capacity-risk:32:LARGEST_BYTES="$(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-*.vma.zst" -o -name "vzdump-lxc-*.tar.zst" \) -printf "%s\n" 2>/dev/null | sort -n | tail -n1)" +/usr/local/sbin/homelab-backupctl:6: audit) echo "=== EMERGENCY AUDIT ==="; backupctl audit; echo "=== APP FRAMEWORK ==="; appbackupctl check; echo "=== APP AUDITS ==="; for app in $(app_list); do echo "### app=$app"; appbackup-restore-check "$app"; appbackup-retention 7 dry-run "$app"; done; echo "OK full backup audit" ;; +/usr/local/sbin/homelab-staging-check.sh:11:echo "=== homelab staging check: ${HOST} ===" +/usr/local/sbin/homelab-secret-exposure-guard:18:BADFILES="$(find "$REPO" -path "$REPO/.git" -prune -o -type f \( -iname "*.key" -o -iname "*.pem" -o -iname "*.p12" -o -iname "*.pfx" -o -iname "[SENSITIVE_FILE]" -o -iname ".env" -o -iname ".env.*" \) -print 2>/dev/null)" +/usr/local/sbin/homelab-edge-vm-offhost-freshness:9:OFFHOST_DIR=/mnt/staging/offhost/edge-vm-vzdump-from-pve03 +/usr/local/sbin/homelab-edge-vm-offhost-freshness:85: /var/lib/vz/dump/vzdump-qemu-130-*.vma.zst) ;; +/usr/local/sbin/homelab-edge-vm-offhost-freshness:140: copy_out="$(run_pve03 copy "$SOURCE_PATH" "$SOURCE_SIZE" "$SOURCE_SHA" "$DST_IP" "$OFFHOST_DIR")" +/usr/local/sbin/homelab-edge-vm-offhost-freshness:141: grep -Fq 'SOURCE_COPY_STATUS=OK' <<<"$copy_out" +/usr/local/sbin/homelab-edge-vm-offhost-freshness:156:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-backup NODE=%s VMID=%s ARCHIVE=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\n' \ +/usr/local/sbin/homelab-edge-vm-offhost-freshness:158: >"$HEALTH_DIR/edge-vm-vzdump-backup.txt" +/usr/local/sbin/homelab-edge-vm-offhost-freshness:159:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-offhost NODE=%s VMID=%s HOST=%s DEST=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\n' \ +/usr/local/sbin/homelab-edge-vm-offhost-freshness:161: >"$HEALTH_DIR/edge-vm-vzdump-offhost.txt" +/usr/local/sbin/homelab-edge-vm-offhost-freshness:162:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-restore-verify HOST=%s VMID=%s ARCHIVE=%s SIZE=%s SHA256=%s ZSTD_RC=0 VMA_RC=0 NOTES=nightly_existing_dump_freshness\n' \ +/usr/local/sbin/homelab-edge-vm-offhost-freshness:164: >"$HEALTH_DIR/edge-vm-vzdump-restore.txt" +/usr/local/sbin/homelab-edge-vm-offhost-freshness:167: "$HEALTH_DIR/edge-vm-vzdump-backup.txt" \ +/usr/local/sbin/homelab-edge-vm-offhost-freshness:168: "$HEALTH_DIR/edge-vm-vzdump-offhost.txt" \ +/usr/local/sbin/homelab-edge-vm-offhost-freshness:169: "$HEALTH_DIR/edge-vm-vzdump-restore.txt" +/usr/local/sbin/homelab-external-probe-vps:20:rm -f "$TMP" +/usr/local/sbin/sqlite-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@"$H" "sudo rm -f /tmp/sqlite-app-${APP}-${JOB_ID}.tgz" +/usr/local/sbin/sqlite-apps-backup-run:21: rm -rf "$STAGE/payload" +/usr/local/sbin/homelab-desired-state-sync:10:find "$REPO" -mindepth 1 -maxdepth 1 ! -name .git -exec rm -rf {} + 2>/dev/null +/usr/local/sbin/homelab-desired-state-sync:24:[ -f /root/HOMELAB_SAFETY_NET_CHECKPOINT.md ] && cp -a /root/HOMELAB_SAFETY_NET_CHECKPOINT.md "$REPO/docs/safety-net-checkpoint.md" 2>/dev/null || true +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:4:CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:9:: "${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}" +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:13:LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:19:log(){ printf '%s RESTORE_CHECK %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" | tee -a "$LOGDIR/restore-check.log"; } +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:22:command -v rclone >/dev/null +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:28: if rclone lsf "${remote_path}.parts/" > "$plist" 2>/dev/null && grep -q '^part-' "$plist"; then +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:29: while IFS= read -r part; do [ -n "$part" ] && rclone cat "${remote_path}.parts/${part}" >> "$out"; done < <(grep '^part-' "$plist" | sort) +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:31: rclone copyto "$remote_path" "$out" >/dev/null +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:33: rm -f "$plist" +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:36:check_one_remote(){ +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:39: latest="$(rclone lsf --dirs-only "${rbase}/" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)" +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:85: [ "$(sqlite3 "$extract/var-lib-skladchik-reports-monitor/state.sqlite" 'PRAGMA integrity_check;' | tail -n 1)" = "ok" ] +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:97:check_one_remote CLOUD_1 "$CLOUD_1" +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:98:check_one_remote CLOUD_2 "$CLOUD_2" +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:101:find "$WORK_ROOT" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_LOCAL}" | while read -r D; do [ -n "$D" ] && rm -rf "${WORK_ROOT}/${D}"; done +/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch:77: check=False, +/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch:101:def check() -> int: +/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch:126: check=False, +/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch:223: return check() +/usr/local/sbin/skladchik-reports-monitor-backup:28:command -v rclone >/dev/null +/usr/local/sbin/skladchik-reports-monitor-backup:29:[ -x /usr/local/sbin/homelab-rclone-copy-chunked ] || { echo "missing /usr/local/sbin/homelab-rclone-copy-chunked" >&2; exit 10; } +/usr/local/sbin/skladchik-reports-monitor-backup:36:cleanup(){ rm -rf "$TMP"; } +/usr/local/sbin/skladchik-reports-monitor-backup:75: /usr/local/sbin/homelab-rclone-copy-chunked "$ARCHIVE" "${DEST_BASE}/$(basename "$ARCHIVE")" +/usr/local/sbin/skladchik-reports-monitor-backup:76: /usr/local/sbin/homelab-rclone-copy-chunked "${ARCHIVE}.sha256" "${DEST_BASE}/$(basename "$ARCHIVE").sha256" +/usr/local/sbin/skladchik-reports-monitor-backup:77: /usr/local/sbin/homelab-rclone-copy-chunked "${OUTDIR}/manifest.txt" "${DEST_BASE}/manifest.txt" +/usr/local/sbin/skladchik-reports-monitor-backup:82: mapfile -t OLD < <(rclone lsf --dirs-only "${REMOTE}${REMOTE_BASE}/daily/" 2>/dev/null | sed 's#/$##' | sort | head -n -"${KEEP_DAILY}" || true) +/usr/local/sbin/skladchik-reports-monitor-backup:83: for D in "${OLD[@]}"; do [ -n "$D" ] && rclone purge "${REMOTE}${REMOTE_BASE}/daily/${D}" >/dev/null 2>&1 || true; done +/usr/local/sbin/skladchik-reports-monitor-backup:85:find "${LOCAL_ROOT}/daily" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_DAILY}" | while read -r D; do [ -n "$D" ] && rm -rf "${LOCAL_ROOT}/daily/${D}"; done +/usr/local/sbin/homelab-vm-local-dumps-retention:32: mapfile -t JOBS < <(rclone lsf "$BASE/" --dirs-only 2>/dev/null | sed 's#/$##' | sort) +/usr/local/sbin/homelab-vm-local-dumps-retention:56: rclone purge "$PATH_REMOTE" >> "$REPORT" 2>&1 && DELETED=$((DELETED+1)) || BAD=$((BAD+1)) +/usr/local/sbin/homelab-extended-appbackup-run:12:run_one(){ APP="$1"; HOST="$2"; PATHS="$3"; NOTE="$4"; CONS="$5"; JOB_ID="homelab-${NODE}-app-${APP}-${RUN_TS}"; export JOB_ID; SHORT_TS="${RUN_TS:2:6}-${RUN_TS:9:6}"; export BACKUP_PATH_REL="$(homelab-backup-path job a "$APP" "$SHORT_TS")"; STAGE="${STAGING_ROOT}/${JOB_ID}"; PAYLOAD="$STAGE/payload"; ARCHIVE="${JOB_ID}.tar.gz"; LOG="/tmp/extended-appbackup-${APP}-${RUN_TS}.log"; rm -rf "$STAGE"; mkdir -p "$PAYLOAD/local"; echo "### app=$APP" >> "$REPORT"; echo "host=$HOST" >> "$REPORT"; echo "source_note=$NOTE" >> "$REPORT"; ssh -n -o BatchMode=yes -o ConnectTimeout=30 "$HOST" "sudo tar -C / --ignore-failed-read --exclude=.env --exclude=*.env --exclude=.env.* --exclude=*.bak* -czf - $PATHS" > "$PAYLOAD/local/${APP}-data.tar.gz" 2>"$LOG"; TAR_RC=$?; if [ "$TAR_RC" -eq 0 ] && tar -tzf "$PAYLOAD/local/${APP}-data.tar.gz" >/dev/null 2>&1; then SIZE="$(stat -c%s "$PAYLOAD/local/${APP}-data.tar.gz" 2>/dev/null || echo 0)"; echo "archive_payload_size_bytes=$SIZE" >> "$REPORT"; if [ "$SIZE" -le 128 ]; then echo "backup=BAD reason=archive_too_small" >> "$REPORT"; BAD=$((BAD+1)); rm -rf "$STAGE"; else sha256sum "$PAYLOAD/local/${APP}-data.tar.gz" > "$PAYLOAD/local/${APP}-data.tar.gz.sha256"; jq -n --arg type "extended-appbackup" --arg app "$APP" --arg host "$HOST" --arg source_note "$NOTE" --arg archive_payload_size_bytes "$SIZE" --arg env_files_read "no" --arg secret_values_printed "no" --arg consistency "$CONS" "{type:\$type,app_name:\$app,host:\$host,source_note:\$source_note,archive_payload_size_bytes:(\$archive_payload_size_bytes|tonumber),env_files_read:\$env_files_read,secret_values_printed:\$secret_values_printed,consistency:\$consistency}" > "$STAGE/metadata.json"; tar -C "$PAYLOAD" -czf "$STAGE/$ARCHIVE" .; rm -rf "$PAYLOAD"; JOB_ID="$JOB_ID" homelab-stage-2cloud-flat "$APP" "$STAGE" "$ARCHIVE" > "$LOG" 2>&1; RC=$?; if [ "$RC" -eq 0 ]; then echo "backup=OK" >> "$REPORT"; grep -E "^\[STEP\]|^\[SUCCESS\]" "$LOG" >> "$REPORT"; appbackupctl restore-check "$APP" >> "$REPORT" 2>&1 && echo "restore_check=OK" >> "$REPORT" || echo "restore_check=WARN_non_registry" >> "$REPORT"; OK=$((OK+1)); else echo "backup=BAD rc=$RC" >> "$REPORT"; grep -E "^\[ERROR\]|\[STEP\]|ERROR|error|failed|Failed|required|missing|denied|forbidden|checksum|rclone|copy|mkdir|jq|tar" "$LOG" | tail -n 60 >> "$REPORT"; BAD=$((BAD+1)); fi; fi; else echo "tar=BAD rc=$TAR_RC" >> "$REPORT"; grep -E "ERROR|error|failed|Failed|denied|No such|Cannot|tar:" "$LOG" | tail -n 40 >> "$REPORT"; BAD=$((BAD+1)); rm -rf "$STAGE"; fi; echo >> "$REPORT"; } +/usr/local/sbin/homelab-extended-appbackup-run:21:run_one healthchecks "$MON" "var/lib/healthchecks opt/stacks/healthchecks/local_settings.py" "healthchecks data plus local settings" "emergency_data_config_archive" +/usr/local/sbin/homelab-mkdocs-auto-refresh:53: rm -f /tmp/homelab-mkdocs-auto-refresh-git-fetch.out /tmp/homelab-mkdocs-auto-refresh-git-fetch.err +/usr/local/sbin/homelab-mkdocs-auto-refresh:57:rm -f /tmp/homelab-mkdocs-auto-refresh-git-fetch.out /tmp/homelab-mkdocs-auto-refresh-git-fetch.err +/usr/local/sbin/homelab-mkdocs-auto-refresh:77:ssh "${EDGE_SSH_OPTS[@]}" "$EDGE_HOST" "rm -rf '$STAGE' && mkdir -p '$STAGE'" +/usr/local/sbin/homelab-mkdocs-auto-refresh:85:rm -rf "$BASE/site.new" "$BASE/site.old" +/usr/local/sbin/homelab-mkdocs-auto-refresh:93:rm -rf "$BASE/site.old" "$STAGE" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:30:CHKS="$WORK/checksums.txt" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:42: F="$(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-${id}-*.vma.zst" -o -name "vzdump-lxc-${id}-*.tar.zst" \) -printf "%T@ %p\n" 2>/dev/null | sort -n | tail -n 1 | cut -d" " -f2-)" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:53: echo "$BN" | grep -q "^vzdump-qemu-" && KIND="qemu" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:54: echo "$BN" | grep -q "^vzdump-lxc-" && KIND="lxc" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:64: rm -rf "$CDIR" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:117: rclone mkdir "$DEST/files" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:122: rclone mkdir "$DEST/files/$bn.chunks" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:126: rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:128: rclone copyto "$CDIR/chunks.sha256" "$DEST/files/$bn.chunks/chunks.sha256" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:131: rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:135: rclone copyto "$CHKS" "$DEST/checksums.txt" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:136: rclone copyto "$FILES_TSV" "$DEST/files.tsv" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:137: rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:144: rm -rf "$VBASE" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:146: rclone copyto "$DEST/manifest.json" "$VBASE/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:154: rclone copy "$DEST/files/$bn.chunks" "$VDIR" --filter "+ part-*" --filter "+ chunks.sha256" --filter "- *" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:163: rm -rf "$VDIR" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:166: rclone copyto "$DEST/files/$bn" "$VBASE/$bn" --transfers 1 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:168: rm -f "$VBASE/$bn" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:171: rm -rf "$VBASE" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:184:rm -rf "$VERIFY_ROOT/$JOB_ID" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:185:rm -rf "$CHUNK_ROOT/$JOB_ID" +/usr/local/sbin/homelab-gitea-secondary-backup:18: rm -f "$TMP" +/usr/local/sbin/homelab-gitea-secondary-backup:31:docker exec -u git -w /tmp gitea sh -lc \"rm -f /tmp/\$NAME && gitea dump -c /data/gitea/conf/app.ini -f /tmp/\$NAME\" +/usr/local/sbin/homelab-gitea-secondary-backup:33:docker exec gitea rm -f \"/tmp/\$NAME\" +/usr/local/sbin/homelab-gitea-secondary-backup:100: rm -f -- "$FILE" "$FILE.sha256" +/usr/local/sbin/homelab-gitea-secondary-backup:123: rm -f -- "$FILE" "$FILE.sha256" +/usr/local/sbin/homelab-gitea-secondary-backup:136: rm -f -- "$FILE" "$FILE.sha256" +/usr/local/sbin/homelab-rclone-copy-chunked:7:TMP_BASE=${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks} +/usr/local/sbin/homelab-rclone-copy-chunked:13:cleanup(){ rm -rf "$WORK"; } +/usr/local/sbin/homelab-rclone-copy-chunked:24:rclone purge "${DEST}.parts" >/dev/null 2>&1 || true +/usr/local/sbin/homelab-rclone-copy-chunked:25:rclone mkdir "${DEST}.parts" +/usr/local/sbin/homelab-rclone-copy-chunked:26:rclone copy "$PARTS" "${DEST}.parts" --transfers 2 --checkers 4 --retries 5 --low-level-retries 20 --stats 30s --log-level INFO +/usr/local/sbin/homelab-rclone-copy-chunked:27:rclone copyto "$WORK/${NAME}.sha256" "${DEST}.sha256" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO +/usr/local/sbin/homelab-rclone-copy-chunked:28:rclone copyto "$WORK/MANIFEST.txt" "${DEST}.manifest.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO +/usr/local/sbin/homelab-rclone-copy-chunked:29:rclone copyto "$WORK/MANIFEST.parts.txt" "${DEST}.parts/MANIFEST.parts.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO +/usr/local/sbin/homelab-rclone-copy-chunked:30:REMOTE_COUNT=$(rclone lsf "${DEST}.parts" --files-only | grep -c "^part-" || true) +/usr/local/sbin/homelab-rclone-copy-chunked:32:ACTUAL=$(while read -r PART SIZE_PART; do case "$PART" in part-*) rclone cat "${DEST}.parts/$PART" ;; esac; done < "$WORK/MANIFEST.parts.txt" | sha256sum | cut -d " " -f 1) +/usr/local/sbin/homelab-router-hard-power-cycle:12:case "$MODE" in check|cycle) ;; *) echo HARD_WRAPPER_MODE_INVALID=yes; exit 64;; esac +/usr/local/sbin/homelab-router-hard-power-cycle:23:SAFE_OUT="$(printf '%s\n' "$OUT" | sed -E '/(PASSWORD|TOKEN|SECRET|PRIVATE_KEY|ROOT_PASSWORD|ENCRYPTION_KEY|COOKIE|SESSION|AUTHORIZATION|API_KEY|LOCAL_KEY|rclone[.]conf|[.]env)/Id')" +/usr/local/sbin/homelab-health-metrics-write:10:rm -f "$tmp" +/usr/local/sbin/homelab-rclone-cat-flat-chunked:5:trap "rm -f \"$TMP\"" EXIT +/usr/local/sbin/homelab-rclone-cat-flat-chunked:6:rclone cat "${DEST}.parts.txt" > "$TMP" +/usr/local/sbin/homelab-rclone-cat-flat-chunked:8:while read -r PART PART_SIZE; do [[ "$PART" =~ ^part-[0-9]{4}$ && "$PART_SIZE" =~ ^[0-9]+$ ]] || { echo "BAD FLAT PART MANIFEST" >&2; exit 2; }; rclone cat "${DEST}.${PART}"; COUNT=$((COUNT+1)); done < "$TMP" +/usr/local/sbin/homelab-vm-backup-policy:29:echo "## Latest local vzdump catalog" >> "$REPORT" +/usr/local/sbin/homelab-vm-backup-policy:31:for f in $(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-*.vma.zst" -o -name "vzdump-lxc-*.tar.zst" \) 2>/dev/null | sort); do b="$(basename "$f")"; size="$(stat -c %s "$f" 2>/dev/null || echo 0)"; mt="$(date -u -d "@$(stat -c %Y "$f" 2>/dev/null || echo 0)" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)"; if echo "$b" | grep -q "^vzdump-qemu-"; then kind=qemu; vmid="$(echo "$b" | sed -n "s/^vzdump-qemu-\([0-9]\+\)-.*/\1/p")"; name="$(qm config "$vmid" 2>/dev/null | awk -F": " "/^name:/{print \$2; exit}")"; else kind=lxc; vmid="$(echo "$b" | sed -n "s/^vzdump-lxc-\([0-9]\+\)-.*/\1/p")"; name="$(pct config "$vmid" 2>/dev/null | awk -F": " "/^hostname:/{print \$2; exit}")"; fi; printf "%s\t%s\t%s\t%s\t%s\t%s\n" "$kind" "$vmid" "${name:-unknown}" "$size" "$mt" "$f" >> "$CATALOG"; done +/usr/local/sbin/homelab-vm-backup-policy:37:echo "- Existing local dumps can be uploaded/verified to two clouds without creating new vzdump archives." >> "$REPORT" +/usr/local/sbin/homelab-vm-backup-policy:38:echo "- New full vzdump for VM 170/171 is not started automatically because each VM has 120 GB bootdisk and /var/tmp has limited free space." >> "$REPORT" +/usr/local/sbin/homelab-vm-backup-policy:42:echo "## Coverage checks" >> "$REPORT" +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:4:cat /var/lib/skladchik-reports-monitor-backup/restore-check-health.txt 2>/dev/null || true +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:7:systemctl is-enabled skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/enabled=/' || true +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:8:systemctl is-active skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/active=/' || true +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:9:systemctl list-timers --all | grep skladchik-reports-monitor-backup-restore-check || true +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:12:tail -n 80 /var/log/skladchik-reports-monitor-backup/restore-check.log 2>/dev/null || true +/usr/local/sbin/homelab-context-refresh-direct:5:trap "rm -rf \"$d\"" EXIT +/usr/local/sbin/homelab-context-refresh-direct:32:printf "{\"schema_version\":1,\"channel\":\"homelab-runtime\",\"command_id\":\"%s\",\"status\":\"OK\",\"rc\":0,\"host\":\"pve01\",\"mode\":\"read-only\",\"component\":\"cluster-context\",\"started_at_utc\":\"%s\",\"finished_at_utc\":\"%s\",\"reference_register_checked\":true,\"reference_sha256\":\"%s\",\"error_register_checked\":true,\"error_register_sha256\":\"%s\",\"changes_made\":false,\"sanitized\":true,\"secrets_included\":false,\"private_addresses_included\":false,\"output\":\"AUTOMATIC_CLUSTER_CONTEXT_REFRESH=OK\"}\n" "$id" "$started" "$finished" "$ref" "$err" >"$d/$id.json" +/usr/local/sbin/homelab-context-refresh-direct:34:ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] debian@[PRIVATE_IP] "sudo -n /usr/local/sbin/homelab-runtime-receive $id /tmp/$id.json /tmp/$id.txt && rm -f /tmp/$id.json /tmp/$id.txt" +/usr/local/sbin/homelab-evidence-seal:16: rm -f "$TMP" +/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:15:DUMP_SIZE="$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$REMOTE" "sudo -n rm -f \"$REMOTE_DUMP\"; sudo -n -u clusteradmin pg_dump --format=custom --no-owner --no-privileges --file=\"$REMOTE_DUMP\" clusteradmin; sudo -n test -s \"$REMOTE_DUMP\"; sudo -n -u clusteradmin pg_restore --list \"$REMOTE_DUMP\" >/dev/null; sudo -n chown debian:debian \"$REMOTE_DUMP\"; sudo -n chmod 600 \"$REMOTE_DUMP\"; stat -c %s \"$REMOTE_DUMP\"")" +/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:18:ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$REMOTE" "rm -f \"$REMOTE_DUMP\"" +/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:23:rm -f "$STAGE/runtime.tgz" +/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:25:jq -n --arg app "$APP" --arg host "cluster-admin" --arg engine "postgresql-custom-dump+files" --argjson files "$FILES" --argjson dump_size "$DUMP_SIZE" "{app:\$app,engine:\$engine,source_host:\$host,source_paths:[\"cluster-admin:/opt/cluster-admin-incident-engine\",\"cluster-admin:/etc/cluster-admin-incident-engine\",\"cluster-admin:/var/lib/cluster-admin-incident-engine\",\"cluster-admin:PostgreSQL clusteradmin\"],file_count:\$files,database_dump_size_bytes:\$dump_size,database_dump_format:\"custom\",database_restore_list_check:\"ok\",includes_runtime_credentials:true,note:\"Incident engine application database state and runtime configuration\"}" > "$STAGE/metadata.json" +/usr/local/sbin/appbackup-retention:32:trap "rm -rf \"$WORK\"" EXIT +/usr/local/sbin/appbackup-retention:35: rclone lsf "${REMOTE%/}" --dirs-only --max-depth 1 > "$ROOT_LIST" +/usr/local/sbin/appbackup-retention:36: rclone lsf "${REMOTE%/}/${NODE}/apps/${CANONICAL}" --dirs-only --max-depth 1 > "$LEGACY_LIST" +/usr/local/sbin/appbackup-retention:39: rclone lsf "${REMOTE%/}/hb" --dirs-only --max-depth 1 > "$HB_LIST" +/usr/local/sbin/appbackup-retention:40: if grep -Fxq "a/" "$HB_LIST"; then rclone lsf "${REMOTE%/}/hb/a" --dirs-only --max-depth 1 > "$NEW_LIST"; fi +/usr/local/sbin/appbackup-retention:73: rclone cat "${REMOTE%/}/${LATEST_REL}/manifest.json" | jq -e --arg job "$LATEST" ".job_id == \$job" >/dev/null || { echo "REFUSE latest missing/bad remote_index=$INDEX"; exit 5; } +/usr/local/sbin/appbackup-retention:85: if [[ "$MODE" == apply ]]; then echo "DELETE schema=dual path=$REL"; rclone purge "$TARGET"; else echo "WOULD_DELETE schema=dual path=$REL"; fi +/usr/local/sbin/homelab-change-transaction:160: check=False, +/usr/local/sbin/homelab-change-transaction:195: latest_tmp.unlink(missing_ok=True) +/usr/local/sbin/homelab-change-transaction:421: " homelab-change-transaction run PLAN --confirm PLAN" +/usr/local/sbin/homelab-check-private-vpn-hosts:15:trap 'rm -f "$tmp"' EXIT +/usr/local/sbin/homelab-backup-vaultwarden:26:rm -rf "$RESTORE" +/usr/local/sbin/homelab-backup-vaultwarden:32:INTEGRITY="$(sqlite3 "$DB" 'PRAGMA integrity_check;')" +/usr/local/sbin/homelab-sops-edge-secret-coverage:97: env = os.environ.copy() +/usr/local/sbin/homelab-sops-edge-secret-coverage:113: env=os.environ.copy(), +/usr/local/sbin/homelab-sops-edge-secret-coverage:176: checked_at = subprocess.check_output(["date", "-u", "+%Y-%m-%dT%H:%M:%SZ"], text=True).strip() +/usr/local/sbin/homelab-sops-edge-secret-coverage:242: f"CHECKED_AT_UTC={checked_at}", +/usr/local/sbin/homelab-health-metrics:16:category_for_check(){ +/usr/local/sbin/homelab-health-metrics:22: *backup*|*vzdump*) printf "%s" "backup" ;; +/usr/local/sbin/homelab-health-metrics:31:tier_for_check(){ +/usr/local/sbin/homelab-health-metrics:34: npmplus-kuma-config-*|authentik-*|gitea-*|vaultwarden-*|netbox-*|mail-cloud-restore-drill|edge-vm-vzdump-*|mkdocs-runtime-publish|npmplus-certificate-expiry) printf "%s" "P0" ;; +/usr/local/sbin/homelab-health-metrics:69: check="${base%.txt}" +/usr/local/sbin/homelab-health-metrics:70: category="$(category_for_check "$check")" +/usr/local/sbin/homelab-health-metrics:71: tier="$(tier_for_check "$check")" +/usr/local/sbin/homelab-health-metrics:75: check_esc="$(esc_label "$check")" +/usr/local/sbin/homelab-health-metrics:79: labels="host=\"$host_esc\",check=\"$check_esc\",category=\"$cat_esc\",tier=\"$tier_esc\",file=\"$file_esc\"" +/usr/local/sbin/homelab-backup-health-evidence:8:backupctl check >/tmp/backup-health-emergency.log 2>&1 && EMERGENCY_OK=1 || true +/usr/local/sbin/homelab-backup-health-evidence:9:appbackupctl check >/tmp/backup-health-appframework.log 2>&1 && APPS_OK=1 || true +/usr/local/sbin/homelab-drift-check:7:HEALTH=/var/lib/homelab-health/drift-check.txt +/usr/local/sbin/homelab-drift-check:24:echo "STATUS=$STATUS TS=$(date -u +%Y-%m-%dT%H:%M:%SZ) TYPE=drift-check NOTE=$NOTE DIFF=$DIFF BASE=$BASE CURRENT=$CUR" > "$HEALTH" +/usr/local/sbin/crypto-portfolio-chart-health-check:5:trap 'rm -f "$TMP"' EXIT +/usr/local/sbin/homelab-rclone-copy-flat-chunked:6:TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" +/usr/local/sbin/homelab-rclone-copy-flat-chunked:12:trap "rm -rf \"$WORK\"" EXIT +/usr/local/sbin/homelab-rclone-copy-flat-chunked:23:while read -r PART PART_SIZE; do [[ "$PART" =~ ^part-[0-9]{4}$ && "$PART_SIZE" =~ ^[0-9]+$ ]] || { echo "BAD PART MANIFEST" >&2; exit 4; }; rclone copyto "$WORK/parts/$PART" "${DEST}.${PART}" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR; REMOTE_BYTES="$(rclone size --json "${DEST}.${PART}" | jq -r ".bytes // -1")"; [[ "$REMOTE_BYTES" = "$PART_SIZE" ]] || { echo "REMOTE PART SIZE MISMATCH $PART" >&2; exit 5; }; UPLOADED=$((UPLOADED+1)); done < "$WORK/MANIFEST.parts.txt" +/usr/local/sbin/homelab-rclone-copy-flat-chunked:25:rclone copyto "$WORK/${NAME}.sha256" "${DEST}.sha256" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-rclone-copy-flat-chunked:26:rclone copyto "$WORK/MANIFEST.txt" "${DEST}.manifest.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-rclone-copy-flat-chunked:27:rclone copyto "$WORK/MANIFEST.parts.txt" "${DEST}.parts.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-rclone-copy-flat-chunked:28:ACTUAL="$(homelab-rclone-cat-flat-chunked "$DEST" | sha256sum | awk "{print \$1}")" +/usr/local/sbin/homelab-final-readiness-gate:10:check(){ name="$1"; shift; echo "### $name" >> "$REPORT"; timeout "${CHECK_TIMEOUT_SECONDS:-300}" "$@" >> "$REPORT" 2>&1; rc=$?; if [ "$rc" -eq 0 ]; then echo "PASS $name" >> "$REPORT"; PASS=$((PASS+1)); else echo "FAIL $name rc=$rc" >> "$REPORT"; FAIL=$((FAIL+1)); BAD=$((BAD+1)); fi; echo "" >> "$REPORT"; } +/usr/local/sbin/homelab-final-readiness-gate:16:check "private overall endpoint" bash -c "curl -fsS http://[PRIVATE_IP]:9101/overall.txt | grep -q STATUS=" +/usr/local/sbin/homelab-final-readiness-gate:17:check "public overall endpoint" bash -c "curl -k -fsS https://backup.gram1.ru/overall.txt | grep -q STATUS=" +/usr/local/sbin/homelab-final-readiness-gate:18:check "backup framework evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/backup-framework.txt && grep -q SOURCE_STATUS=OK /var/lib/homelab-health/backup-framework.txt && grep -q FRESHNESS=FRESH /var/lib/homelab-health/backup-framework.txt" +/usr/local/sbin/homelab-final-readiness-gate:19:check "control-plane restore-check" appbackupctl restore-check safety-net-control-plane +/usr/local/sbin/homelab-final-readiness-gate:20:check "uptime-kuma restore-check" appbackupctl restore-check uptime-kuma +/usr/local/sbin/homelab-final-readiness-gate:21:check "backup SLA evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/backup-sla.txt" +/usr/local/sbin/homelab-final-readiness-gate:22:check "backup coverage matrix evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/backup-coverage-matrix.txt" +/usr/local/sbin/homelab-final-readiness-gate:23:check "extended appbackup evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/extended-appbackup.txt" +/usr/local/sbin/homelab-final-readiness-gate:24:check "residual review" bash -c "grep -q STATUS=OK /var/lib/homelab-health/residual-review.txt" +/usr/local/sbin/homelab-final-readiness-gate:25:check "forum snuffleupagus" bash -c "grep -q STATUS=OK /var/lib/homelab-health/forum-snuffleupagus.txt" +/usr/local/sbin/homelab-final-readiness-gate:26:check "vm170 vm171 full strategy" bash -c "grep -q STATUS=OK /var/lib/homelab-health/vm170-vm171-full-strategy.txt" +/usr/local/sbin/homelab-final-readiness-gate:27:check "external probe vps" bash -c "grep -q STATUS=OK /var/lib/homelab-health/external-probe-vps.txt" +/usr/local/sbin/homelab-final-readiness-gate:28:check "netbird vps trust" bash -c "grep -q STATUS=OK /var/lib/homelab-health/netbird-vps-trust.txt" +/usr/local/sbin/homelab-final-readiness-gate:29:check "netbird vps trust closure" bash -c "grep -q STATUS=OK /var/lib/homelab-health/netbird-vps-trust-closure.txt" +/usr/local/sbin/homelab-final-readiness-gate:30:check "external probe runner decision" bash -c "grep -q STATUS=OK /var/lib/homelab-health/external-probe-runner-decision.txt" +/usr/local/sbin/homelab-final-readiness-gate:31:check "netbird vps trust clean seal" bash -c "grep -q STATUS=OK /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt" +/usr/local/sbin/homelab-final-readiness-gate:32:check "live tail audit" bash -c "grep -q STATUS=OK /var/lib/homelab-health/live-tail-audit.txt" +/usr/local/sbin/homelab-final-readiness-gate:34:check "duty admin v2" bash -c "grep -q STATUS=OK /var/lib/homelab-health/duty-admin-v2.txt" +/usr/local/sbin/homelab-final-readiness-gate:35:check "cluster admin observer" bash -c "grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-observer.txt" +/usr/local/sbin/homelab-final-readiness-gate:36:check "cluster admin restricted probes" bash -c "grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-restricted-probes.txt" +/usr/local/sbin/homelab-final-readiness-gate:37:check "cluster admin full observer" bash -c "grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-full-observer.txt && grep -q BAD=0 /var/lib/homelab-health/cluster-admin-full-observer.txt && grep -q WARN=0 /var/lib/homelab-health/cluster-admin-full-observer.txt" +/usr/local/sbin/homelab-final-readiness-gate:38:check "cluster admin VM Mail Cloud backup" bash -c "grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt && grep -q BAD=0 /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt && grep -q MAIL_CLOUD_2COPY_VERIFY_OK /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt" +/usr/local/sbin/homelab-final-readiness-gate:39:check "cluster admin webpanel sync" bash -c "grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-webpanel-sync.txt && grep -q BAD=0 /var/lib/homelab-health/cluster-admin-webpanel-sync.txt" +/usr/local/sbin/homelab-final-readiness-gate:40:check "cluster admin webpanel" bash -c "grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-webpanel.txt && grep -q BAD=0 /var/lib/homelab-health/cluster-admin-webpanel.txt && curl -fsS --max-time 5 http://[PRIVATE_IP]:8080/status.txt | grep -q STATUS=OK" +/usr/local/sbin/homelab-final-readiness-gate:41:check "node loss readiness" bash -c "grep -q STATUS=OK /var/lib/homelab-health/node-loss-readiness.txt" +/usr/local/sbin/homelab-final-readiness-gate:42:check "node loss runbooks" bash -c "grep -q STATUS=OK /var/lib/homelab-health/node-loss-runbooks.txt" +/usr/local/sbin/homelab-final-readiness-gate:43:check "power loss readiness" bash -c "grep -q STATUS=OK /var/lib/homelab-health/power-loss-readiness.txt" +/usr/local/sbin/homelab-final-readiness-gate:44:check "power loss runbook" bash -c "grep -q STATUS=OK /var/lib/homelab-health/power-loss-runbook.txt" +/usr/local/sbin/homelab-final-readiness-gate:45:check "ungated health backlog" bash -c "grep -q STATUS=OK /var/lib/homelab-health/ungated-health-backlog.txt" +/usr/local/sbin/homelab-final-readiness-gate:46:check "pve03 root cleanup" bash -c "grep -q STATUS=OK /var/lib/homelab-health/pve03-root-cleanup.txt" +/usr/local/sbin/homelab-final-readiness-gate:47:check "pve03 staging retention" bash -c "grep -q STATUS=OK /var/lib/homelab-health/pve03-staging-retention.txt" +/usr/local/sbin/homelab-final-readiness-gate:48:check "pve03 staging retention proof" bash -c "grep -q STATUS=OK /var/lib/homelab-health/pve03-staging-retention-proof.txt" +/usr/local/sbin/homelab-final-readiness-gate:49:check "pve03 staging retention cleanup" bash -c "grep -q STATUS=OK /var/lib/homelab-health/pve03-staging-retention-cleanup.txt" +/usr/local/sbin/homelab-final-readiness-gate:50:check "pve03 failed unit cleanup" bash -c "grep -q STATUS=OK /var/lib/homelab-health/pve03-failed-unit-cleanup.txt" +/usr/local/sbin/homelab-final-readiness-gate:51:check "remote git sops age readiness" bash -c "grep -q STATUS=OK /var/lib/homelab-health/remote-git-sops-age-readiness.txt" +/usr/local/sbin/homelab-final-readiness-gate:52:check "remote git sops age policy" bash -c "grep -q STATUS=OK /var/lib/homelab-health/remote-git-sops-age-policy.txt" +/usr/local/sbin/homelab-final-readiness-gate:53:check "desired state remote target trace" bash -c "grep -q STATUS=OK /var/lib/homelab-health/desired-state-remote-target-trace.txt" +/usr/local/sbin/homelab-final-readiness-gate:54:check "desired state remote target" bash -c "grep -q STATUS=OK /var/lib/homelab-health/desired-state-remote-target.txt" +/usr/local/sbin/homelab-final-readiness-gate:55:check "Kuma monitor policy evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/kuma-monitor-policy.txt" +/usr/local/sbin/homelab-final-readiness-gate:56:check "safe autoheal evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/safe-autoheal.txt" +/usr/local/sbin/homelab-final-readiness-gate:57:check "desired-state clean evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/desired-state.txt && test -z "$(git -C /srv/homelab-desired-state status --short)"" +/usr/local/sbin/homelab-final-readiness-gate:58:check "runbooks evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/runbooks.txt && test -s /var/lib/homelab-runbooks/INDEX.md" +/usr/local/sbin/homelab-final-readiness-gate:59:check "incident journal evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/incident-journal.txt" +/usr/local/sbin/homelab-final-readiness-gate:60:check "alerting evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/alerting.txt" +/usr/local/sbin/homelab-final-readiness-gate:61:check "VM backup health evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/vm-backup.txt" +/usr/local/sbin/homelab-final-readiness-gate:62:check "VM backup policy evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/vm-backup-policy.txt" +/usr/local/sbin/homelab-final-readiness-gate:63:check "VM local dumps cloud" bash -c "grep -q STATUS=OK /var/lib/homelab-health/vm-local-dumps-cloud.txt && [ $(jq -r .count /var/lib/homelab-vm-backup/latest-vm-local-dumps-cloud.json) -ge 3 ] && [ $(jq -r .chunked_count /var/lib/homelab-vm-backup/latest-vm-local-dumps-cloud.json) -ge 1 ]" +/usr/local/sbin/homelab-final-readiness-gate:64:check "VM local dumps retention" bash -c "grep -q STATUS=OK /var/lib/homelab-health/vm-local-dumps-retention.txt && grep -q ACTIVE_LEFT=0 /var/lib/homelab-health/kuma-monitor-policy.txt || grep -q STATUS=OK /var/lib/homelab-health/vm-local-dumps-retention.txt" +/usr/local/sbin/homelab-final-readiness-gate:65:check "capacity risk evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/capacity-risk.txt" +/usr/local/sbin/homelab-final-readiness-gate:66:check "secret exposure evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/secret-exposure.txt" +/usr/local/sbin/homelab-final-readiness-gate:67:check "golden state evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/golden-state.txt" +/usr/local/sbin/homelab-final-readiness-gate:68:check "service registry evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/service-registry.txt" +/usr/local/sbin/homelab-final-readiness-gate:69:check "dependency map evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/dependency-map.txt" +/usr/local/sbin/homelab-docker-health.sh:44: echo "containers_without_healthcheck=$NOHEALTH" +/usr/local/sbin/homelab-docker-health.sh:63: echo "# HELP homelab_docker_health_container_without_healthcheck Docker containers without native healthcheck." +/usr/local/sbin/homelab-docker-health.sh:64: echo "# TYPE homelab_docker_health_container_without_healthcheck gauge" +/usr/local/sbin/homelab-docker-health.sh:65: echo "homelab_docker_health_container_without_healthcheck $NOHEALTH" +/usr/local/sbin/homelab-backup-coverage-matrix:17:P1_SAFE_APPS="actual-budget homebox linkding mealie memos node-red syncthing filebrowser healthchecks ntfy vikunja homeassistant karakeep minio n8n" +/usr/local/sbin/homelab-backup-coverage-matrix:38: while read -r norm raw; do +/usr/local/sbin/homelab-backup-coverage-matrix:50: while read -r norm raw; do +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:8:TARGET_BASE=full/nextcloud-vm-vzdump +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:11:SRC_GLOB=/var/lib/vz/dump/vzdump-qemu-150-*.vma.zst +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:77:P1 Mail Cloud Nextcloud VM recurring chunked vzdump upload +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:93:rclone mkdir "$TARGET_REMOTE:$REMOTE_PARTS" > "$WORK/logs/mkdir.out" 2>"$WORK/logs/mkdir.err" || MKDIR_RC=$? +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:95:rclone copy "$PARTS_DIR" "$TARGET_REMOTE:$REMOTE_PARTS" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > "$WORK/logs/upload_parts.out" 2>"$WORK/logs/upload_parts.err" || UPLOAD_PARTS_RC=$? +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:97:rclone copyto "$WORK/$BASENAME.sha256" "$TARGET_REMOTE:$REMOTE_SHA" --checksum > "$WORK/logs/upload_sha.out" 2>"$WORK/logs/upload_sha.err" || UPLOAD_SHA_RC=$? +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:99:rclone copyto "$WORK/MANIFEST.txt" "$TARGET_REMOTE:$REMOTE_MANIFEST" --checksum > "$WORK/logs/upload_manifest.out" 2>"$WORK/logs/upload_manifest.err" || UPLOAD_MANIFEST_RC=$? +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:101:rclone copyto "$WORK/SHA256SUMS.txt" "$TARGET_REMOTE:$REMOTE_SUMS" --checksum > "$WORK/logs/upload_sums.out" 2>"$WORK/logs/upload_sums.err" || UPLOAD_SUMS_RC=$? +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:104:rclone lsf "$TARGET_REMOTE:$REMOTE_PARTS" --format ps > "$WORK/logs/remote_lsf_parts.txt" 2>"$WORK/logs/remote_lsf_parts.err" || LSF_PARTS_RC=$? +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:110:rclone copy "$TARGET_REMOTE:$REMOTE_PARTS" "$DL_DIR/parts" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > "$WORK/logs/download_parts.out" 2>"$WORK/logs/download_parts.err" || DOWNLOAD_PARTS_RC=$? +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:120:rclone lsf "$TARGET_REMOTE:$TARGET_BASE" --dirs-only 2>/dev/null | sed 's#/$##' | sort > "$DIR_LIST" || RETENTION_RC=$? +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:126: [ -n "$OLD" ] && rclone purge "$TARGET_REMOTE:$TARGET_BASE/$OLD" >> "$WORK/logs/retention.out" 2>>"$WORK/logs/retention.err" +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:140: rm -rf "$PARTS_DIR" "$DL_DIR" +/usr/local/sbin/homelab-rehydrate-flat-chunked-dir:10: rm -f "$TMP_TARGET" +/usr/local/sbin/homelab-disk-space-health.py:10:checks = [ +/usr/local/sbin/homelab-disk-space-health.py:21:for label, cmd in checks: +/usr/local/sbin/homelab-rclone-required-check.sh:4:RAW_FILE="/etc/homelab/rclone/required-remotes.txt" +/usr/local/sbin/homelab-rclone-required-check.sh:5:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt" +/usr/local/sbin/homelab-rclone-required-check.sh:9:echo "=== homelab rclone required check: $(hostname) ===" +/usr/local/sbin/homelab-rclone-required-check.sh:12:echo "=== installed rclone ===" +/usr/local/sbin/homelab-rclone-required-check.sh:13:rclone version | head -5 +/usr/local/sbin/homelab-rclone-required-check.sh:17:CONFIGURED="$(rclone listremotes 2>/dev/null | sed 's/:$//' || true)" +/usr/local/sbin/homelab-rclone-required-check.sh:19: echo "No rclone remotes configured yet." +/usr/local/sbin/homelab-legacy-monitoring-backups-run:24: rm -f "$STAGE/remote.tgz" +/usr/local/sbin/homelab-legacy-monitoring-backups-run:43: rm -f "$STAGE/pve01.tgz" +/usr/local/sbin/homelab-legacy-monitoring-backups-run:48: rm -f "$STAGE/$LABEL.tgz" +/usr/local/sbin/homelab-legacy-monitoring-backups-run:52: rm -f "$STAGE/monitoring.tgz" +/usr/local/sbin/homelab-legacy-monitoring-backups-run:71: rm -f "$STAGE/config.tgz" +/usr/local/sbin/homelab-cluster-admin-inventory-sync:19:ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$REMOTE" "rm -f $REMOTE_TMP" +/usr/local/sbin/homelab-chat-run:272: re.compile(r"(?i)\brclone\.conf\b"), +/usr/local/sbin/homelab-chat-run:388: "reference_register_checked": True, +/usr/local/sbin/homelab-chat-run:390: "error_register_checked": True, +/usr/local/sbin/homelab-chat-run:489: rm -f \ +/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:10: rm -rf "$TMPD" 2>/dev/null || true +/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:19:check_host() { +/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:39:check_host "kuma" "kuma.vpn.gram1.ru" "^(200|302)$" +/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:40:check_host "homepage" "homepage.vpn.gram1.ru" "^200$" +/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:41:check_host "gitea" "gitea.vpn.gram1.ru" "^200$" +/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:42:check_host "filebrowser" "filebrowser.vpn.gram1.ru" "^200$" +/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:43:check_host "auth" "auth.vpn.gram1.ru" "^(200|302)$" +/usr/local/sbin/homelab-beszel-backup-run:33:rm -f "$STAGE/remote.tgz" +/usr/local/sbin/homelab-beszel-backup-run:37:while IFS= read -r DB; do DB_COUNT=$((DB_COUNT+1)); RESULT="$(python3 -c "import sqlite3,sys;print(sqlite3.connect(sys.argv[1]).execute(\"PRAGMA integrity_check\").fetchone()[0])" "$DB" 2>/dev/null || echo error)"; [ "$RESULT" = ok ] || exit 4; DB_OK=$((DB_OK+1)); done < <(find "$STAGE/payload" -type f \( -name "*.db" -o -name "*.sqlite" -o -name "*.sqlite3" \) 2>/dev/null) +/usr/local/sbin/homelab-runbook-generate:40: f"## First checks\n" +/usr/local/sbin/homelab-runbook-generate:44: f"appbackupctl restore-check {app}\n" +/usr/local/sbin/homelab-runbook-generate:47: f"Do not restore over a live service blindly. First confirm both encrypted cloud copies pass restore-check, target VM/container is snapshotted or intentionally stopped, and dependency map is OK.\n\n" +/usr/local/sbin/homelab-runbook-generate:51: f"homelab-service-registry-check\n" +/usr/local/sbin/homelab-runbook-generate:52: f"homelab-dependency-map-check\n" +/usr/local/sbin/homelab-runbook-generate:85: "backupctl restore-check\n" +/usr/local/sbin/homelab-runbook-generate:86: "appbackupctl restore-check safety-net-control-plane\n" +/usr/local/sbin/skladchik-reports-monitor-supervisor:25:PVE_RESTORE_HEALTH = Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt") +/usr/local/sbin/skladchik-reports-monitor-supervisor:254: "skladchik-reports-monitor-backup-restore-check.timer", +/usr/local/sbin/skladchik-reports-monitor-supervisor:301: "Restore-check:", +/usr/local/sbin/skladchik-reports-monitor-supervisor:339: mode = sys.argv[1] if len(sys.argv) > 1 else "check" +/usr/local/sbin/skladchik-reports-monitor-supervisor:340: if mode not in ("check", "test"): +/usr/local/sbin/skladchik-reports-monitor-supervisor:341: print("usage: skladchik-reports-monitor-supervisor [check|test]", file=sys.stderr) +/usr/local/sbin/backup-restore-check:8:ROOT="/var/tmp/homelab-restore-check/${JOB}" +/usr/local/sbin/backup-restore-check:9:rm -rf "$ROOT" && mkdir -p "$ROOT/a" "$ROOT/b" +/usr/local/sbin/backup-restore-check:10:trap 'echo "ERROR restore-check failed; kept: "; exit 1' ERR +/usr/local/sbin/backup-restore-check:11:echo "restore-check job=$JOB" +/usr/local/sbin/backup-restore-check:12:rclone copy "$RA" "$ROOT/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-a.log 2>&1; homelab-rehydrate-chunked-dir "$ROOT/a" >>/tmp/homelab-restore-a.log 2>&1 +/usr/local/sbin/backup-restore-check:13:(cd "$ROOT/a" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-a-sha.log && ARCHIVE="$(jq -r .archive_name manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null) +/usr/local/sbin/backup-restore-check:14:echo "OK cloud A restore-check" +/usr/local/sbin/backup-restore-check:15:rclone copy "$RB" "$ROOT/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-b.log 2>&1; homelab-rehydrate-chunked-dir "$ROOT/b" >>/tmp/homelab-restore-b.log 2>&1 +/usr/local/sbin/backup-restore-check:16:(cd "$ROOT/b" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-b-sha.log && ARCHIVE="$(jq -r .archive_name manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null) +/usr/local/sbin/backup-restore-check:17:echo "OK cloud B restore-check" +/usr/local/sbin/backup-restore-check:18:rm -rf "$ROOT" +/usr/local/sbin/backup-restore-check:20:echo "OK restore-check both clouds" +/usr/local/sbin/homelab-vps-identity-audit:8:trap 'rm -rf "$TMP"' EXIT +/usr/local/sbin/homelab-vm-full-restore-drill:77: -name "vzdump-qemu-${source_vmid}-*.vma.zst" \ +/usr/local/sbin/homelab-vm-full-restore-drill:152:check_inventory() { +/usr/local/sbin/homelab-vm-full-restore-drill:184:check_container_health() { +/usr/local/sbin/homelab-vm-full-restore-drill:224:check_http() { +/usr/local/sbin/homelab-vm-full-restore-drill:349:check_container_health() { +/usr/local/sbin/homelab-vm-full-restore-drill:386:check_container_health "$test_vmid" +/usr/local/sbin/homelab-vm-full-restore-drill:389: check_inventory "$source_vmid" "$test_vmid" +/usr/local/sbin/homelab-vm-full-restore-drill:392: check_http "$source_vmid" "$test_vmid" +/usr/local/sbin/homelab-vm-backup-health:8:check_file(){ +/usr/local/sbin/homelab-vm-backup-health:15:check_file /var/lib/homelab-health/backup.txt auto-vmct +/usr/local/sbin/homelab-vm-backup-health:16:check_file /var/lib/homelab-health/edge-vm-vzdump-backup.txt edge-vm-backup +/usr/local/sbin/homelab-vm-backup-health:17:check_file /var/lib/homelab-health/edge-vm-vzdump-offhost.txt edge-vm-offhost +/usr/local/sbin/homelab-vm-backup-health:18:check_file /var/lib/homelab-health/edge-vm-vzdump-restore.txt edge-vm-restore +/usr/local/sbin/homelab-ingress-failover:37: rm -f "$RESOURCES" +/usr/local/sbin/homelab-ingress-failover:252: rm -f "$tmp" +/usr/local/sbin/homelab-overall-health:8: /var/lib/homelab-health/drift-check.txt +/usr/local/sbin/homelab-backup-matrix-refresh:10: p=subprocess.run(args,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True,check=False) +/usr/local/sbin/homelab-backup-matrix-refresh:65: shutil.copy2(source_path,temporary) +/usr/local/sbin/homelab-vm-full-backup:11:# shellcheck disable=SC1090 +/usr/local/sbin/homelab-vm-full-backup:92:for cmd in vzdump qm zstd sha256sum rclone rsync ssh flock; do +/usr/local/sbin/homelab-vm-full-backup:130: -name "vzdump-qemu-${VMID}-*.vma.zst" \ +/usr/local/sbin/homelab-vm-full-backup:138: vzdump "$VMID" \ +/usr/local/sbin/homelab-vm-full-backup:145: --remove 0 +/usr/local/sbin/homelab-vm-full-backup:151: -name "vzdump-qemu-${VMID}-*.vma.zst" \ +/usr/local/sbin/homelab-vm-full-backup:261: CHUNK_TMP_BASE="$BACKUP_DIR/.rclone-chunks" homelab-rclone-copy-chunked "$ARCHIVE" "${DEST_ROOT}/${NAME}" +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:10:TARGET_BASE=full/edge-vm-vzdump +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:64:case "$SOURCE" in /mnt/staging/offhost/edge-vm-vzdump-from-pve03/vzdump-qemu-130-*.vma.zst) ;; *) fail source_path_bad 2 ;; esac +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:111:rclone about "$TARGET_REMOTE:" > "$WORK/logs/about.out" 2>"$WORK/logs/about.err" || ABOUT_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:114:DIRS="$(rclone lsf "$TARGET_REMOTE:$TARGET_BASE" --dirs-only 2>/dev/null | sed 's#/$##' | sort || true)" +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:116: if rclone cat "$TARGET_REMOTE:$TARGET_BASE/$D/$BASENAME.sha256" 2>/dev/null | grep -q "$SHA"; then +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:172:P1 Mail Cloud edge-vm recurring chunked vzdump upload +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:188:rclone mkdir "$TARGET_REMOTE:$REMOTE_PARTS" > "$WORK/logs/mkdir.out" 2>"$WORK/logs/mkdir.err" || MKDIR_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:190:rclone copy "$PARTS_DIR" "$TARGET_REMOTE:$REMOTE_PARTS" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > "$WORK/logs/upload_parts.out" 2>"$WORK/logs/upload_parts.err" || UPLOAD_PARTS_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:192:rclone copyto "$WORK/$BASENAME.sha256" "$TARGET_REMOTE:$REMOTE_SHA" --checksum > "$WORK/logs/upload_sha.out" 2>"$WORK/logs/upload_sha.err" || UPLOAD_SHA_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:194:rclone copyto "$WORK/MANIFEST.txt" "$TARGET_REMOTE:$REMOTE_MANIFEST" --checksum > "$WORK/logs/upload_manifest.out" 2>"$WORK/logs/upload_manifest.err" || UPLOAD_MANIFEST_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:196:rclone copyto "$WORK/SHA256SUMS.txt" "$TARGET_REMOTE:$REMOTE_SUMS" --checksum > "$WORK/logs/upload_sums.out" 2>"$WORK/logs/upload_sums.err" || UPLOAD_SUMS_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:199:rclone size "$TARGET_REMOTE:$TARGET_DIR" --json > "$WORK/logs/remote_size.json" 2>"$WORK/logs/remote_size.err" || SIZE_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:201:rclone lsf "$TARGET_REMOTE:$REMOTE_PARTS" --format "ps" > "$WORK/logs/remote_lsf_parts.txt" 2>"$WORK/logs/remote_lsf_parts.err" || LSF_PARTS_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:207:rclone copy "$TARGET_REMOTE:$REMOTE_PARTS" "$DL_DIR/parts" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > "$WORK/logs/download_parts.out" 2>"$WORK/logs/download_parts.err" || DOWNLOAD_PARTS_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:222: rm -rf "$DL_DIR" "$PARTS_DIR" +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:234: rclone lsf "$TARGET_REMOTE:$TARGET_BASE" --dirs-only 2>/dev/null | sed "s#/$##" | sort > "$DIR_LIST" || RETENTION_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:239: [ -n "$OLD" ] && rclone purge "$TARGET_REMOTE:$TARGET_BASE/$OLD" >> "$WORK/logs/retention.out" 2>>"$WORK/logs/retention.err" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:7:BASE_DIR="/mnt/staging/vzdump/edge-vm-vzdump" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:8:OFFHOST_DIR="/mnt/staging/offhost/edge-vm-vzdump-from-pve03" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:40:REMOTE_OUT="$(remote "OUT=/tmp/edge_vm_vzdump_${VMID}_${TS}.out; ERR=/tmp/edge_vm_vzdump_${VMID}_${TS}.err; : > \$OUT; : > \$ERR; vzdump '$VMID' --mode snapshot --compress zstd --dumpdir '$RUN_DIR' --remove 0 >\$OUT 2>\$ERR; RC=\$?; ARCHIVE=\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\n' | sort -n | tail -n1 | cut -d' ' -f2-); LOGFILE=\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.log' -printf '%T@ %p\n' | sort -n | tail -n1 | cut -d' ' -f2-); SIZE=0; SHA=none; if [ -n \"\$ARCHIVE\" ] && [ -f \"\$ARCHIVE\" ]; then SIZE=\$(stat -c %s \"\$ARCHIVE\"); SHA=\$(sha256sum \"\$ARCHIVE\" | awk '{print \$1}'); fi; echo VZDUMP_RC=\$RC; echo ARCHIVE=\$ARCHIVE; echo LOGFILE=\$LOGFILE; echo SIZE=\$SIZE; echo SHA=\$SHA; echo OUT_TAIL=\$(tail -n 6 \$OUT | tr '\n' '|'); echo ERR_TAIL=\$(tail -n 6 \$ERR | tr '\n' '|')")" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:62:dest "mkdir -p '$OFFHOST_DIR' && rm -f '$DEST_PATH.tmp' '$DEST_PATH.sha256'" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:74:echo "STATUS=OK TS=$TS TYPE=edge-vm-vzdump-backup NODE=$NODE VMID=$VMID ARCHIVE=$ARCHIVE SIZE=$SIZE SHA256=$SHA SCSI1_BACKUP0=yes NOTES=timer_run" > "$HEALTH_DIR/edge-vm-vzdump-backup.txt" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:75:echo "STATUS=OK TS=$TS TYPE=edge-vm-vzdump-offhost NODE=$NODE VMID=$VMID HOST=pve02 DEST=$DEST_PATH SIZE=$SIZE SHA256=$SHA NOTES=timer_run" > "$HEALTH_DIR/edge-vm-vzdump-offhost.txt" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:76:echo "STATUS=OK TS=$TS TYPE=edge-vm-vzdump-restore-verify HOST=pve02 VMID=$VMID ARCHIVE=$DEST_PATH SIZE=$SIZE SHA256=$SHA ZSTD_RC=0 VMA_RC=0 NOTES=timer_run" > "$HEALTH_DIR/edge-vm-vzdump-restore.txt" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:77:chmod 0644 "$HEALTH_DIR/edge-vm-vzdump-backup.txt" "$HEALTH_DIR/edge-vm-vzdump-offhost.txt" "$HEALTH_DIR/edge-vm-vzdump-restore.txt" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:79:remote "find '$BASE_DIR' -mindepth 1 -maxdepth 1 -type d -name 'edge-vm-*' -printf '%T@ %p\n' | sort -nr | awk 'NR>${KEEP_LOCAL}{print \$2}' | xargs -r rm -rf" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:80:dest "find '$OFFHOST_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\n' | sort -nr | awk 'NR>${KEEP_OFFHOST}{print \$2}' | while read f; do rm -f \"\$f\" \"\$f.sha256\"; done" +/usr/local/sbin/appbackup-dockge-stacks:25:echo "[STEP] copy app data" +/usr/local/sbin/appbackup-dockge-stacks:30:rm -rf "$PAYLOAD" +/usr/local/sbin/appbackup-dockge-stacks:51:echo "[STEP] local checksum" +/usr/local/sbin/appbackup-dockge-stacks:52:(cd "$STAGE" && sha256sum "$ARCHIVE" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e . manifest.json >/dev/null) +/usr/local/sbin/appbackup-dockge-stacks:54:echo "[STEP] upload cloud A metadata"; rclone copy "$STAGE" "$RA" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s +/usr/local/sbin/appbackup-dockge-stacks:55:echo "[STEP] upload cloud A archive chunks"; rclone deletefile "$RA/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RA/$ARCHIVE" +/usr/local/sbin/appbackup-dockge-stacks:56:echo "[STEP] upload cloud B metadata"; rclone copy "$STAGE" "$RB" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s +/usr/local/sbin/appbackup-dockge-stacks:57:echo "[STEP] upload cloud B archive chunks"; rclone deletefile "$RB/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RB/$ARCHIVE" +/usr/local/sbin/appbackup-dockge-stacks:58:echo "[STEP] verify cloud A chunked"; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-stage-verify-chunked "$RA" "$ARCHIVE" "$VERIFY/a" "$JOB" +/usr/local/sbin/appbackup-dockge-stacks:59:echo "[STEP] verify cloud B chunked"; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-stage-verify-chunked "$RB" "$ARCHIVE" "$VERIFY/b" "$JOB" +/usr/local/sbin/appbackup-dockge-stacks:60:echo "[STEP] upload latest index"; rclone copyto "$STAGE/latest-backup-index.json" "$RAL"; rclone copyto "$STAGE/latest-backup-index.json" "$RBL"; rclone cat "$RAL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null; rclone cat "$RBL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null +/usr/local/sbin/appbackup-dockge-stacks:76:rm -rf "$STAGE" "$VERIFY" +/usr/local/sbin/homelab-cluster-passport:23:line "- safety checkpoint: /root/HOMELAB_SAFETY_NET_CHECKPOINT.md" +/usr/local/sbin/homelab-cluster-passport:65:need_ok /var/lib/homelab-health/drift-check.txt drift-check +/usr/local/sbin/homelab-cluster-passport:77:line "appbackupctl restore-check safety-net-control-plane" +/usr/local/sbin/homelab-cluster-passport:84:line "4. For control-plane loss: appbackupctl restore-check safety-net-control-plane, then restore the verified archive from either encrypted cloud." +/usr/local/sbin/backup-retention:18: if [[ "$MODE" == "apply" ]]; then rclone cat "$BASE/$LATEST/manifest.json" | jq -e --arg job "$LATEST" ".job_id == \$job" >/dev/null || { echo "REFUSE latest missing/bad on $BASE"; exit 4; }; fi +/usr/local/sbin/backup-retention:19: mapfile -t DIRS < <(rclone lsf "$BASE" --dirs-only 2>/dev/null | while read -r D; do D="${D%/}"; [[ "$D" == homelab-${NODE}-${JOB}-* ]] && echo "$D"; done | sort) +/usr/local/sbin/backup-retention:24: for ((i=0; i /var/lib/homelab-health/private-vpn-hosts.txt.tmp; rc=$?; mv /var/lib/homelab-health/private-vpn-hosts.txt.tmp /var/lib/homelab-health/private-vpn-hosts.txt; chmod 0644 /var/lib/homelab-health/private-vpn-hosts.txt; exit $rc' +/etc/systemd/system/netbird-peers-health.service:2:Description=NetBird peers health check +/etc/systemd/system/homelab-pve03-staging-capacity-health.timer:2:Description=Run Homelab pve03 staging capacity health check +/etc/systemd/system/timers.target.wants/homelab-private-vpn-hosts-health.timer:2:Description=Run Homelab private VPN hosts health check every minute +/etc/systemd/system/timers.target.wants/homelab-crypto-portfolio-chart-health.timer:2:Description=Recurring Crypto portfolio chart health check +/etc/systemd/system/timers.target.wants/homelab-edge-vm-offhost-freshness.timer:2:Description=Run Homelab edge-vm VM130 offhost freshness sync daily after nightly vzdump +/etc/systemd/system/timers.target.wants/homelab-dependency-map-check.timer:2:Description=Homelab dependency map check timer +/etc/systemd/system/timers.target.wants/homelab-pve03-staging-capacity-health.timer:2:Description=Run Homelab pve03 staging capacity health check +/etc/systemd/system/timers.target.wants/homelab-drift-check.timer:2:Description=Homelab drift check timer +/etc/systemd/system/timers.target.wants/homelab-service-registry-check.timer:2:Description=Homelab service registry health check timer +/etc/systemd/system/timers.target.wants/homelab-backup-healthcheck.timer:2:Description=Run homelab backup healthcheck daily +/etc/systemd/system/timers.target.wants/homelab-backup-healthcheck.timer:8:Unit=homelab-backup-healthcheck.service +/etc/systemd/system/timers.target.wants/netbird-peers-health.timer:2:Description=Run NetBird peers health check +/etc/systemd/system/timers.target.wants/homelab-alerting-health.timer:2:Description=Homelab alerting health check timer +/etc/systemd/system/timers.target.wants/skladchik-reports-monitor-backup-restore-check.timer:2:Description=Weekly restore-check of Skladchik reports monitor cloud backups +/etc/systemd/system/timers.target.wants/skladchik-reports-monitor-backup-restore-check.timer:8:Unit=skladchik-reports-monitor-backup-restore-check.service +/etc/systemd/system/homelab-edge-vm-vzdump-backup.timer:8:Unit=homelab-edge-vm-vzdump-backup.service +/etc/systemd/system/homelab-drift-check.timer:2:Description=Homelab drift check timer +/etc/systemd/system/homelab-service-registry-check.timer:2:Description=Homelab service registry health check timer +/etc/systemd/system/homelab-pve03-staging-capacity-health.service:2:Description=Homelab pve03 staging capacity health check +/etc/systemd/system/homelab-backup-healthcheck.timer:2:Description=Run homelab backup healthcheck daily +/etc/systemd/system/homelab-backup-healthcheck.timer:8:Unit=homelab-backup-healthcheck.service +/etc/systemd/system/homelab-safe-exec-health.service:2:Description=Run allowlisted homelab health checks +/etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.service:2:Description=Homelab Mail Cloud Nextcloud VM vzdump upload +/etc/systemd/system/hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub' +/etc/systemd/system/netbird-peers-health.timer:2:Description=Run NetBird peers health check +/etc/systemd/system/homelab-alerting-health.timer:2:Description=Homelab alerting health check timer +/etc/systemd/system/skladchik-reports-monitor-backup-restore-check.timer:2:Description=Weekly restore-check of Skladchik reports monitor cloud backups +/etc/systemd/system/skladchik-reports-monitor-backup-restore-check.timer:8:Unit=skladchik-reports-monitor-backup-restore-check.service +/etc/systemd/system/suspend-then-hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub' +/etc/cron.d/vzdump:1:# cluster wide vzdump cron schedule +7022386770 2026-07-18T03:31:03.6566057240 /var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.vma.zst +7021952392 2026-07-21T03:31:06.5717414950 /var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst +7021297699 2026-07-19T03:31:06.8755351020 /var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst +7016275481 2026-07-20T03:31:03.2165123470 /var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst +329541843 2026-07-21T03:30:09.7688696330 /var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst +326601118 2026-07-20T03:30:08.2096782720 /var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst +323668231 2026-07-19T03:45:46.2209130400 /var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst +321121649 2026-07-18T03:30:08.0777625900 /var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.tar.zst +222777732 2026-07-21T03:30:15.9059638290 /var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst +222761273 2026-07-20T03:30:13.7937629440 /var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst +222745965 2026-07-19T03:45:57.2700790730 /var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst +222727760 2026-07-18T03:30:13.5578457230 /var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.tar.zst +3021 2026-07-19T03:31:07.4745284120 /var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.log +3020 2026-07-21T03:31:07.3945472590 /var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.log +3018 2026-07-18T03:31:03.9197988860 /var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.log +3017 2026-07-20T03:31:04.1706719350 /var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.log +1089 2026-07-20T03:30:14.0079985330 /var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.log +1086 2026-07-20T03:30:08.4640241540 /var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.log +989 2026-07-21T03:30:16.1364976560 /var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.log +989 2026-07-18T03:30:13.7816006690 /var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.log +986 2026-07-21T03:30:10.0285327490 /var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.log +986 2026-07-18T03:30:08.3002411830 /var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.log +731 2026-07-19T03:45:57.5197000270 /var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.log +729 2026-07-19T03:45:46.4444121980 /var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.log +========== NODE=pve02 ========== +=== REMOTES === +=== TIMERS === +Tue 2026-07-21 18:41:03 MSK 3h 46min Tue 2026-07-21 12:41:22 MSK 2h 12min ago forum-prod-fresh5-cloud-backup.timer forum-prod-fresh5-cloud-backup.service +Wed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:22 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service +Wed 2026-07-22 04:20:00 MSK 13h Tue 2026-07-21 04:20:12 MSK 10h ago homelab-cluster-admin-vm-mail-cloud-backup.timer homelab-cluster-admin-vm-mail-cloud-backup.service +=== SERVICES === +codevipe-cloud-backup.service static - +dpkg-db-backup.service static - +forum-prod-fresh5-cloud-backup.service static - +homelab-cluster-admin-vm-mail-cloud-backup.service static - +/usr/local/sbin/homelab-rclone-required-check.sh:4:RAW_FILE="/etc/homelab/rclone/required-remotes.txt" +/usr/local/sbin/homelab-rclone-required-check.sh:5:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt" +/usr/local/sbin/homelab-rclone-required-check.sh:9:echo "=== homelab rclone required check: $(hostname) ===" +/usr/local/sbin/homelab-rclone-required-check.sh:12:echo "=== installed rclone ===" +/usr/local/sbin/homelab-rclone-required-check.sh:13:rclone version | head -5 +/usr/local/sbin/homelab-rclone-required-check.sh:17:CONFIGURED="$(rclone listremotes 2>/dev/null | sed 's/:$//' || true)" +/usr/local/sbin/homelab-rclone-required-check.sh:19: echo "No rclone remotes configured yet." +/usr/local/sbin/homelab-staging-check.sh:11:echo "=== homelab staging check: ${HOST} ===" +/usr/local/sbin/routerguard-watchdog-power-entrypoint:12:case "$MODE" in check|cycle) ;; *) echo HARD_ENTRYPOINT_MODE_INVALID=yes; exit 64;; esac +/usr/local/sbin/routerguard-watchdog-power-entrypoint:15:if [ "$MODE" = check ]; then +/usr/local/sbin/routerguard-watchdog-power-entrypoint:16: exec "$HELPER" check +/usr/local/sbin/routerguard-watchdog-power-entrypoint:21:PRE="$(mktemp /run/routerguard-watchdog-precheck.XXXXXX)" +/usr/local/sbin/routerguard-watchdog-power-entrypoint:22:cleanup(){ rm -f "$PRE" "$ARM"; } +/usr/local/sbin/routerguard-watchdog-power-entrypoint:25:"$HELPER" check >"$PRE" 2>&1 +/usr/local/sbin/routerguard-watchdog-power-entrypoint:38:rm -f "$ARM" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:4:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:18: "checksums/" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:21:echo "=== homelab rclone crypt healthcheck: $(hostname) ===" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:40: if ! rclone listremotes | sed 's/:$//' | grep -qx "$REMOTE"; then +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:47: if ! rclone about "${REMOTE}:"; then +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:48: echo "FAIL: rclone about failed: $REMOTE" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:54: ROOT_LIST="$(rclone lsf "${REMOTE}:/" || true)" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:76: if rclone cat "${REMOTE}:/latest-backup-index.json" | jq -e '.schema and .remote and .status and .items' >/dev/null; then +/usr/local/sbin/codevipe-cloud-backup.sh:28: rclone copy "$CACHE_ROOT/$BASE" "$R:$REMOTE_PATH/$BASE" --stats 30s +/usr/local/sbin/codevipe-cloud-backup.sh:31:find "$CACHE_ROOT" -mindepth 1 -maxdepth 1 -type d -mtime +14 -print -exec rm -rf {} \; +/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:26:rm -f "$STAGE/$base" +/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:34:format=tar.gz split_parts_1500M rclone_crypt_remote +/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:44: rclone mkdir "$dest" >/tmp/forum-cloud-mkdir-${r}-$TS.log 2>&1 +/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:46: rclone copy "$STAGE" "$dest" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --stats-one-line >/tmp/forum-cloud-copy-${r}-$TS.log 2>&1 +/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:49: remote_count=$(rclone lsf "$dest" --files-only 2>/tmp/forum-cloud-lsf-${r}-$TS.err | wc -l) +/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:55:find /root/backups/forum-prod-fresh5 -mindepth 1 -maxdepth 1 -type d -mtime +14 -exec rm -rf {} + +/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:73: retention_remote_count=$(rclone lsf "$r:$REMOTE_BASE/$retention_job" --files-only 2>/dev/null | wc -l) +/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:78: rm -rf --one-file-system "$retention_dir" +/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:83: rm -f "$retention_keep_tmp" +/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:24:rclone listremotes 2>/tmp/cluster-admin-rclone-listremotes.err | grep -E 'mail.*crypt|crypt.*mail' | sed 's/:$//' | sort > /tmp/cluster-admin-mail-remotes.txt +/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:41: rm -rf "$STAGE/$OLD_JOB" +/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:43: rclone purge "$R:homelab-backups/pve02/cluster-admin-vm180/$OLD_JOB" >/tmp/cluster-admin-rclone-purge-$R.log 2>&1 +/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:51: ls "$d"/vzdump-qemu-${VMID}-*.vma.zst >/dev/null 2>&1 || { echo "DELETE_EMPTY_LOCAL_JOB=$b"; rm -rf "$d"; } +/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:84:vzdump "$VMID" --mode snapshot --compress zstd --dumpdir "$JOBDIR" --quiet 0 +/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:89:LATEST="$(ls -1t "$JOBDIR"/vzdump-qemu-${VMID}-*.vma.zst 2>/dev/null | head -n1)" +/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:115: rm -f "$TMPVMA" +/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:124: rm -f "$TMPVMA" +/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:156: rclone mkdir "$DEST" >/tmp/cluster-admin-rclone-mkdir-$R.log 2>&1 +/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:158: rclone copy "$JOBDIR" "$DEST" --checksum --transfers 2 --checkers 4 --retries 3 --low-level-retries 5 >/tmp/cluster-admin-rclone-copy-$R.log 2>&1 +/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:161: rclone copyto "$JOBDIR/MANIFEST.txt" "$R:homelab-backups/pve02/cluster-admin-vm180/latest/MANIFEST.txt" --retries 3 >/tmp/cluster-admin-rclone-latest-$R.log 2>&1 +/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:164: REMOTE_SHA="$(rclone cat "$DEST/$BASE.sha256" 2>/tmp/cluster-admin-rclone-catsha-$R.err | awk '{print $1}' | head -n1)" +/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:177:find "$STAGE" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort -r | tail -n +4 | while read old; do rm -rf "$STAGE/$old"; echo "LOCAL_STAGE_DELETED=$old"; done +/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:20: rm -rf -- "$WORK" +/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:29: local checked_at +/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:31: checked_at="$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || printf UNKNOWN)" +/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:37:CHECKED_AT_UTC=$checked_at +/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:117:for CMD in awk cat chmod date grep head mkdir mktemp rclone rm sed sha256sum sort sqlite3 tail tar wc; do +/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:131:rclone lsf "$REMOTE" --recursive >"$REMOTE_LIST" 2>"$ERROR_FILE" \ +/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:149:rclone copyto "$REMOTE$CRITICAL_REMOTE_PATH" "$CRITICAL_ARCHIVE" 2>"$ERROR_FILE" \ +/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:152:rclone copyto "$REMOTE$CRITICAL_REMOTE_SHA_PATH" "$CRITICAL_SHA_FILE" 2>"$ERROR_FILE" \ +/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:251:INTEGRITY_RESULT="$(sqlite3 "$DB_PATH" 'PRAGMA integrity_check;' 2>"$ERROR_FILE")" \ +/usr/local/sbin/routerguard-hard-power-cycle:3:MODE="${1:-check}" +/usr/local/sbin/routerguard-hard-power-cycle:4:case "$MODE" in check|cycle) ;; *) echo USAGE=routerguard-hard-power-cycle_check_or_cycle; exit 64;; esac +/usr/local/sbin/routerguard-network:48:remove_rules(){ +/usr/local/sbin/routerguard-network:91: rm -f "$RUN/active" +/usr/local/sbin/routerguard-network:92: remove_rules +/usr/local/sbin/routerguard-network:100: rm -rf "$RUN" +/etc/systemd/system/suspend.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub' +/etc/systemd/system/suspend-then-hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub' +/etc/systemd/system/hybrid-sleep.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub' +/etc/systemd/system/multi-user.target.wants/corosync.service:20:# as well; before editing this line, you may want to check DefaultStandardError +/etc/systemd/system/multi-user.target.wants/postfix.service:19:# perform 2-stage startup +/etc/systemd/system/multi-user.target.wants/postfix.service:20:ExecStartPre=+postfix check +/etc/systemd/system/multi-user.target.wants/postfix.service:30:# Individual Postfix daemons are started as root, optionally perform chroot +/etc/systemd/system/multi-user.target.wants/postfix.service:52:# CAP_CHOWN can be removed. +/etc/systemd/system/multi-user.target.wants/pve-guests.service:22:ExecStop=-/usr/bin/vzdump -stop +/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:22:ExecStart=/usr/sbin/pve-ha-lrm start +/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:23:ExecStop=/usr/sbin/pve-ha-lrm stop +/etc/systemd/system/multi-user.target.wants/rrdcached.service:2:# To customize, copy this file to /etc/systemd/system/ +/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:14:ExecStart=/usr/sbin/pve-ha-crm start +/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:15:ExecStop=/usr/sbin/pve-ha-crm stop +/etc/systemd/system/multi-user.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub' +/etc/systemd/system/multi-user.target.wants/e2scrub_reap.service:2:Description=Remove Stale Online ext4 Metadata Check Snapshots +/etc/systemd/system/default.target.wants/nvmefc-boot-connections.service:19:RemoveIPC=yes +/etc/systemd/system/default.target.wants/nvmf-autoconnect.service:21:RemoveIPC=yes +/etc/systemd/system/sysinit.target.wants/lvm2-lvmpolld.socket:10:RemoveOnStop=true +/etc/systemd/system/sysinit.target.wants/pvenetcommit.service:8:ExecStartPre=-/bin/rm -f /etc/openvswitch/conf.db +/etc/systemd/system/sysinit.target.wants/systemd-pstore.service:11:Description=Platform Persistent Storage Archival +/etc/systemd/system/pve-manager.service:22:ExecStop=-/usr/bin/vzdump -stop +/etc/systemd/system/hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub' +/etc/systemd/system/sockets.target.wants/dm-event.socket:10:RemoveOnStop=true +/etc/systemd/system/sockets.target.wants/rrdcached.socket:2:# To customize, copy this file to /etc/systemd/system/ +/etc/cron.d/vzdump:1:# cluster wide vzdump cron schedule +997503422 2026-07-01T03:31:11.1700059310 /var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.vma.zst +868048369 2026-06-30T03:31:08.4903256780 /var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.vma.zst +288349157 2026-07-21T03:30:21.2480085730 /var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.tar.zst +286538526 2026-07-20T03:30:18.4386603270 /var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.tar.zst +284443608 2026-07-19T03:47:41.8525503200 /var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst +283022034 2026-07-18T03:30:19.7617214460 /var/lib/vz/dump/vzdump-lxc-111-2026_07_18-03_30_03.tar.zst +220080970 2026-07-21T03:30:33.3910787180 /var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.tar.zst +220065825 2026-07-20T03:30:30.8287264090 /var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.tar.zst +220049041 2026-07-19T03:47:58.9256407960 /var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst +220033534 2026-07-18T03:30:37.2878143100 /var/lib/vz/dump/vzdump-lxc-113-2026_07_18-03_30_20.tar.zst +7547 2026-07-18T01:09:39.2467331080 /var/lib/vz/dump/vzdump-qemu-160-2026_07_18-01_02_44.log +7443 2026-07-18T01:45:13.3149404960 /var/lib/vz/dump/vzdump-qemu-160-2026_07_18-01_38_20.log +5796 2026-07-17T21:55:21.5648458010 /var/lib/vz/dump/vzdump-qemu-160-2026_07_17-21_30_20.log +2685 2026-06-30T03:31:09.1616927060 /var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.log +2590 2026-07-01T03:31:11.9500372070 /var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.log +1088 2026-07-20T03:30:31.0784689220 /var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.log +1085 2026-07-20T03:30:18.7427551960 /var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.log +988 2026-07-21T03:30:33.6364088960 /var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.log +988 2026-07-18T03:30:37.5450736960 /var/lib/vz/dump/vzdump-lxc-113-2026_07_18-03_30_20.log +985 2026-07-21T03:30:21.5465818340 /var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.log +985 2026-07-18T03:30:20.0884248500 /var/lib/vz/dump/vzdump-lxc-111-2026_07_18-03_30_03.log +731 2026-07-19T03:47:59.2238139650 /var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.log +728 2026-07-19T03:47:42.1183953260 /var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.log +========== NODE=pve03 ========== +=== REMOTES === +=== TIMERS === +Wed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:10 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service +=== SERVICES === +dpkg-db-backup.service static - +/usr/local/sbin/homelab-staging-check.sh:11:echo "=== homelab staging check: ${HOST} ===" +/usr/local/sbin/homelab-immutable-offline-verify:11: rm -f "$TMP" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:4:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:18: "checksums/" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:21:echo "=== homelab rclone crypt healthcheck: $(hostname) ===" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:40: if ! rclone listremotes | sed 's/:$//' | grep -qx "$REMOTE"; then +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:47: if ! rclone about "${REMOTE}:"; then +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:48: echo "FAIL: rclone about failed: $REMOTE" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:54: ROOT_LIST="$(rclone lsf "${REMOTE}:/" || true)" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:76: if rclone cat "${REMOTE}:/latest-backup-index.json" | jq -e '.schema and .remote and .status and .items' >/dev/null; then +/usr/local/sbin/homelab-rclone-required-check.sh:4:RAW_FILE="/etc/homelab/rclone/required-remotes.txt" +/usr/local/sbin/homelab-rclone-required-check.sh:5:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt" +/usr/local/sbin/homelab-rclone-required-check.sh:9:echo "=== homelab rclone required check: $(hostname) ===" +/usr/local/sbin/homelab-rclone-required-check.sh:12:echo "=== installed rclone ===" +/usr/local/sbin/homelab-rclone-required-check.sh:13:rclone version | head -5 +/usr/local/sbin/homelab-rclone-required-check.sh:17:CONFIGURED="$(rclone listremotes 2>/dev/null | sed 's/:$//' || true)" +/usr/local/sbin/homelab-rclone-required-check.sh:19: echo "No rclone remotes configured yet." +/etc/systemd/system/hybrid-sleep.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub' +/etc/systemd/system/multi-user.target.wants/pve-guests.service:22:ExecStop=-/usr/bin/vzdump -stop +/etc/systemd/system/multi-user.target.wants/corosync.service:20:# as well; before editing this line, you may want to check DefaultStandardError +/etc/systemd/system/multi-user.target.wants/rrdcached.service:2:# To customize, copy this file to /etc/systemd/system/ +/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:14:ExecStart=/usr/sbin/pve-ha-crm start +/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:15:ExecStop=/usr/sbin/pve-ha-crm stop +/etc/systemd/system/multi-user.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub' +/etc/systemd/system/multi-user.target.wants/postfix.service:19:# perform 2-stage startup +/etc/systemd/system/multi-user.target.wants/postfix.service:20:ExecStartPre=+postfix check +/etc/systemd/system/multi-user.target.wants/postfix.service:30:# Individual Postfix daemons are started as root, optionally perform chroot +/etc/systemd/system/multi-user.target.wants/postfix.service:52:# CAP_CHOWN can be removed. +/etc/systemd/system/multi-user.target.wants/e2scrub_reap.service:2:Description=Remove Stale Online ext4 Metadata Check Snapshots +/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:22:ExecStart=/usr/sbin/pve-ha-lrm start +/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:23:ExecStop=/usr/sbin/pve-ha-lrm stop +/etc/systemd/system/sysinit.target.wants/pvenetcommit.service:8:ExecStartPre=-/bin/rm -f /etc/openvswitch/conf.db +/etc/systemd/system/sysinit.target.wants/systemd-pstore.service:11:Description=Platform Persistent Storage Archival +/etc/systemd/system/sysinit.target.wants/lvm2-lvmpolld.socket:10:RemoveOnStop=true +/etc/systemd/system/hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub' +/etc/systemd/system/sockets.target.wants/rrdcached.socket:2:# To customize, copy this file to /etc/systemd/system/ +/etc/systemd/system/sockets.target.wants/dm-event.socket:10:RemoveOnStop=true +/etc/systemd/system/suspend-then-hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub' +/etc/systemd/system/default.target.wants/nvmf-autoconnect.service:21:RemoveIPC=yes +/etc/systemd/system/default.target.wants/nvmefc-boot-connections.service:19:RemoveIPC=yes +/etc/systemd/system/suspend.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub' +/etc/systemd/system/pve-manager.service:22:ExecStop=-/usr/bin/vzdump -stop +/etc/cron.d/vzdump:1:# cluster wide vzdump cron schedule +11116472459 2026-07-20T03:33:53.2839064370 /var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.vma.zst +11043834894 2026-07-21T03:33:54.3725716230 /var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.vma.zst +11013723404 2026-07-19T03:33:53.3463878040 /var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.vma.zst +10994820121 2026-07-18T03:33:47.1540038630 /var/lib/vz/dump/vzdump-qemu-130-2026_07_18-03_30_02.vma.zst +1137072350 2026-07-21T03:34:37.6276811360 /var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.vma.zst +838951976 2026-07-19T03:34:33.1034859540 /var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.vma.zst +838817925 2026-07-20T03:34:37.1719936670 /var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.vma.zst +785249297 2026-07-18T03:34:27.1531047780 /var/lib/vz/dump/vzdump-qemu-190-2026_07_18-03_33_48.vma.zst +7106 2026-07-02T03:37:53.1040459570 /var/lib/vz/dump/vzdump-qemu-130-2026_07_02-03_30_02.log +7102 2026-07-03T03:36:49.1036784480 /var/lib/vz/dump/vzdump-qemu-130-2026_07_03-03_30_05.log +6564 2026-07-18T03:33:48.0107527290 /var/lib/vz/dump/vzdump-qemu-130-2026_07_18-03_30_02.log +6560 2026-07-20T03:33:54.9283349630 /var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.log +6559 2026-07-21T03:33:55.1428157340 /var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.log +6459 2026-07-19T03:33:53.7631656400 /var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.log +6246 2026-07-04T03:33:39.2197744030 /var/lib/vz/dump/vzdump-qemu-130-2026_07_04-03_30_02.log +4107 2026-07-07T03:31:51.2516816970 /var/lib/vz/dump/vzdump-qemu-130-2026_07_07-03_30_01.log +2642 2026-07-21T03:34:38.4491706750 /var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.log +2500 2026-07-20T03:34:37.2013439550 /var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.log +2404 2026-07-19T03:34:34.1266444600 /var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.log +2305 2026-07-18T03:34:27.7179455720 /var/lib/vz/dump/vzdump-qemu-190-2026_07_18-03_33_48.log +=== CLOUD_HEALTH_AND_OBJECTS === +=== CURRENT_BACKUP_MATRIX === +STATUS=FAIL +TS=2026-07-21T07:02:41Z +TYPE=unified-backup-matrix +MATRIX_MODE=authoritative +JOBS_TOTAL=41 +JOBS_OK=30 +JOBS_BAD=6 +JOBS_UNMAPPED=0 +JOBS_DISABLED=5 +STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_JSON=/etc/homelab-backup-matrix/matrix.json +AUTHORITY=canonical +COMPAT_OUTPUTS=29 +COMPAT_MAP=/var/lib/homelab-backup-matrix/compat-map.json +JOB=dpkg-db-backup STATUS=OK EVIDENCE=execution_only AGE=36147 MAX_AGE=172800 +JOB=filebrowser-backup STATUS=OK EVIDENCE=artifact_or_health AGE=14443 MAX_AGE=259200 +JOB=filebrowser-restore-validate STATUS=STALE EVIDENCE=artifact_or_health AGE=1016424 MAX_AGE=864000 +JOB=homelab-appbackup-dockge-stacks STATUS=OK EVIDENCE=artifact_or_health AGE=22248 MAX_AGE=172800 +JOB=homelab-appbackup-mariadb-batch STATUS=OK EVIDENCE=execution_only AGE=15387 MAX_AGE=172800 +JOB=homelab-appbackup-npmplus-edge STATUS=OK EVIDENCE=artifact_or_health AGE=20573 MAX_AGE=172800 +JOB=homelab-appbackup-postgres-batch STATUS=OK EVIDENCE=execution_only AGE=16416 MAX_AGE=172800 +JOB=homelab-appbackup-sqlite-batch STATUS=OK EVIDENCE=execution_only AGE=18327 MAX_AGE=172800 +JOB=homelab-authentik-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987790 MAX_AGE=172800 +JOB=homelab-auto-backup STATUS=OK EVIDENCE=execution_only AGE=195424 MAX_AGE=864000 +JOB=homelab-backup-coverage-matrix STATUS=OK EVIDENCE=artifact_or_health AGE=4346 MAX_AGE=172800 +JOB=homelab-backup-freshness STATUS=OK EVIDENCE=execution_only AGE=35589 MAX_AGE=172800 +JOB=homelab-backup-healthcheck STATUS=OK EVIDENCE=execution_only AGE=10631 MAX_AGE=172800 +JOB=homelab-backup-sla-health STATUS=FAIL EVIDENCE=artifact_or_health AGE=144 MAX_AGE=172800 +JOB=homelab-edge-vm-offhost-freshness STATUS=OK EVIDENCE=artifact_or_health AGE=18426 MAX_AGE=172800 +JOB=homelab-emergency-backup STATUS=OK EVIDENCE=artifact_or_health AGE=24539 MAX_AGE=172800 +JOB=homelab-extended-appbackup STATUS=OK EVIDENCE=artifact_or_health AGE=14156 MAX_AGE=172800 +JOB=homelab-gitea-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987785 MAX_AGE=172800 +JOB=homelab-golden-state-index STATUS=OK EVIDENCE=artifact_or_health AGE=4112 MAX_AGE=172800 +JOB=homelab-mail-cloud-critical-upload STATUS=OK EVIDENCE=artifact_or_health AGE=11997 MAX_AGE=172800 +JOB=homelab-mail-cloud-edge-vm-upload STATUS=OK EVIDENCE=artifact_or_health AGE=174045 MAX_AGE=864000 +JOB=homelab-mail-cloud-nextcloud-vm-upload STATUS=OK EVIDENCE=artifact_or_health AGE=7575 MAX_AGE=864000 +JOB=homelab-mail-cloud-restore-drill STATUS=STALE EVIDENCE=artifact_or_health AGE=988269 MAX_AGE=864000 +JOB=homelab-npmplus-kuma-config-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=60071 MAX_AGE=172800 +JOB=homelab-router-running-config-mail-cloud STATUS=OK EVIDENCE=artifact_or_health AGE=17076 MAX_AGE=172800 +JOB=homelab-vaultwarden-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987768 MAX_AGE=172800 +JOB=homelab-verified-backup-health STATUS=OK EVIDENCE=artifact_or_health AGE=11845 MAX_AGE=172800 +JOB=homelab-vm-backup-health STATUS=OK EVIDENCE=artifact_or_health AGE=8742 MAX_AGE=172800 +JOB=homelab-vm-backup-policy STATUS=OK EVIDENCE=artifact_or_health AGE=5548 MAX_AGE=172800 +JOB=homelab-vm-full-backup STATUS=OK EVIDENCE=artifact_or_health AGE=24778 MAX_AGE=172800 +JOB=homelab-vm-full-restore-drill STATUS=OK EVIDENCE=artifact_or_health AGE=523085 MAX_AGE=10368000 +JOB=homelab-vm-local-dumps-2cloud STATUS=OK EVIDENCE=artifact_or_health AGE=172141 MAX_AGE=864000 +JOB=homelab-vm-local-dumps-retention STATUS=OK EVIDENCE=artifact_or_health AGE=168839 MAX_AGE=864000 +JOB=immich-media-offhost-sync STATUS=OK EVIDENCE=artifact_or_health AGE=4286 MAX_AGE=259200 +JOB=memos-backup STATUS=OK EVIDENCE=artifact_or_health AGE=14547 MAX_AGE=259200 +JOB=netbird-vps-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=1747896 MAX_AGE=864000 +JOB=paperless-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=16347 MAX_AGE=259200 +JOB=paperless-restore-dry-run STATUS=STALE EVIDENCE=artifact_or_health AGE=1001986 MAX_AGE=864000 +JOB=skladchik-reports-monitor-backup STATUS=OK EVIDENCE=artifact_or_health AGE=19639 MAX_AGE=172800 +JOB=skladchik-reports-monitor-backup-restore-check STATUS=OK EVIDENCE=artifact_or_health AGE=190273 MAX_AGE=864000 +JOB=vm160-remote-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=26607 MAX_AGE=172800 +PAIR_DELETE_GUARD_AUDIT=COMPLETE + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 36a7ed8..4d3dfee 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "BACKUP-CLOUD-CONTENTS-065", - "status": "FAIL", - "rc": 2, + "command_id": "BACKUP-PAIR-DELETE-GUARD-066", + "status": "OK", + "rc": 0, "host": "pve01", "mode": "read-only", "component": "backup-audit", - "started_at_utc": "2026-07-21T11:58:57Z", - "finished_at_utc": "2026-07-21T11:58:57Z", + "started_at_utc": "2026-07-21T12:01:03Z", + "finished_at_utc": "2026-07-21T12:01:03Z", "reference_register_checked": true, "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", "error_register_checked": true, "error_register_sha256": "f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1", - "command_sha256": "452b162f3a4d66e4471b9cccf75bfeb2631e0d2552eb0a8e1325b757d2e6f1df", + "command_sha256": "e64555881c22fffab137df86ee957819b9922f83078f71124f8ff61e8580590d", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -22,9 +22,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "2bc4df84f54e8e6175eeb80e00fc91fcb32008a2e5abbaeab989bcec88af8173", - "sanitized_output_sha256": "2bc4df84f54e8e6175eeb80e00fc91fcb32008a2e5abbaeab989bcec88af8173", + "raw_evidence_sha256": "d13bc63fd488f7e13f81bef392b271beacdec9c637576dc61e7dad33042b27f2", + "sanitized_output_sha256": "6c697f245387cba271db9658a3e6e5ce9d5a9ac0178945c559417c7064daa692", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "/usr/bin/bash: -c: line 1: unexpected EOF while looking for matching `\"'\n" + "output": "=== PIPELINES_AND_CLOUD_PAIRS ===\n========== NODE=pve01 ==========\n=== REMOTES ===\n=== TIMERS ===\nWed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:14 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service\nWed 2026-07-22 00:09:43 MSK 9h Tue 2026-07-21 00:09:32 MSK 14h ago homelab-backup-freshness.timer homelab-backup-freshness.service\nWed 2026-07-22 02:15:00 MSK 11h Tue 2026-07-21 02:15:05 MSK 12h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service\nWed 2026-07-22 03:10:02 MSK 12h Tue 2026-07-21 03:13:14 MSK 11h ago homelab-emergency-backup.timer homelab-emergency-backup.service\nWed 2026-07-22 03:35:00 MSK 12h Tue 2026-07-21 03:35:14 MSK 11h ago filebrowser-backup.timer filebrowser-backup.service\nWed 2026-07-22 03:55:02 MSK 13h Tue 2026-07-21 03:51:25 MSK 11h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service\nWed 2026-07-22 04:17:29 MSK 13h Tue 2026-07-21 04:19:17 MSK 10h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service\nWed 2026-07-22 04:26:54 MSK 13h Tue 2026-07-21 04:28:14 MSK 10h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service\nWed 2026-07-22 04:38:04 MSK 13h Tue 2026-07-21 04:34:48 MSK 10h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service\nWed 2026-07-22 05:01:44 MSK 14h Tue 2026-07-21 04:57:14 MSK 9h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service\nWed 2026-07-22 05:23:56 MSK 14h Tue 2026-07-21 05:17:57 MSK 9h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service\nWed 2026-07-22 05:27:13 MSK 14h Tue 2026-07-21 05:29:05 MSK 9h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service\nWed 2026-07-22 05:47:19 MSK 14h Tue 2026-07-21 05:46:14 MSK 9h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service\nWed 2026-07-22 06:01:19 MSK 15h Tue 2026-07-21 05:57:30 MSK 8h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service\nWed 2026-07-22 06:27:43 MSK 15h Tue 2026-07-21 06:32:25 MSK 8h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service\nWed 2026-07-22 06:38:51 MSK 15h Tue 2026-07-21 06:43:54 MSK 8h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service\nWed 2026-07-22 06:39:22 MSK 15h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service\nWed 2026-07-22 06:44:16 MSK 15h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service\nWed 2026-07-22 06:52:06 MSK 15h Tue 2026-07-21 06:47:47 MSK 8h ago homelab-beszel-backup.timer homelab-beszel-backup.service\nWed 2026-07-22 06:58:58 MSK 16h Tue 2026-07-21 06:57:30 MSK 7h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service\nWed 2026-07-22 07:02:27 MSK 16h Tue 2026-07-21 07:05:30 MSK 7h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service\nWed 2026-07-22 07:07:14 MSK 16h Tue 2026-07-21 07:18:55 MSK 7h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service\nWed 2026-07-22 07:35:06 MSK 16h Tue 2026-07-21 07:36:59 MSK 7h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service\nWed 2026-07-22 07:38:21 MSK 16h Tue 2026-07-21 07:36:14 MSK 7h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service\nWed 2026-07-22 07:41:49 MSK 16h Tue 2026-07-21 07:53:41 MSK 7h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service\nWed 2026-07-22 08:10:37 MSK 17h Tue 2026-07-21 08:07:25 MSK 6h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service\nWed 2026-07-22 08:37:02 MSK 17h Tue 2026-07-21 08:30:07 MSK 6h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service\nWed 2026-07-22 08:40:13 MSK 17h Tue 2026-07-21 08:50:14 MSK 6h ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service\nWed 2026-07-22 08:53:36 MSK 17h Tue 2026-07-21 08:41:14 MSK 6h ago memos-backup.timer memos-backup.service\nSun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service\nSun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service\nSun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service\nSun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service\nMon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 7h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service\n=== SERVICES ===\ndpkg-db-backup.service static -\nfilebrowser-backup.service static -\nhomelab-alertmanager-backup.service static -\nhomelab-appbackup-dockge-stacks.service static -\nhomelab-appbackup-mariadb-batch.service static -\nhomelab-appbackup-npmplus-edge.service static -\nhomelab-appbackup-postgres-batch.service static -\nhomelab-appbackup-sqlite-batch.service static -\nhomelab-authentik-backup.service static -\nhomelab-auto-backup.service static -\nhomelab-backup-coverage-matrix.service static -\nhomelab-backup-freshness.service static -\nhomelab-backup-healthcheck.service static -\nhomelab-backup-matrix-refresh.service static -\nhomelab-backup-sla-health.service static -\nhomelab-beszel-backup.service static -\nhomelab-blackbox-exporter-backup.service static -\nhomelab-cluster-admin-incident-engine-backup.service static -\nhomelab-crypto-portfolio-backup.service static -\nhomelab-edge-vm-vzdump-backup.service static -\nhomelab-emergency-backup.service static -\nhomelab-extended-appbackup.service static -\nhomelab-gitea-backup.service static -\nhomelab-gitea-secondary-backup.service static -\nhomelab-legacy-monitoring-backups.service static -\nhomelab-mail-cloud-critical-upload.service static -\nhomelab-mail-cloud-edge-vm-upload.service static -\nhomelab-mail-cloud-nextcloud-vm-upload.service static -\nhomelab-mail-cloud-restore-drill.service static -\nhomelab-npmplus-kuma-config-backup.service static -\nhomelab-router-running-config-mail-cloud.service static -\nhomelab-vaultwarden-backup.service static -\nhomelab-verified-backup-health.service static -\nhomelab-vm-backup-health.service static -\nhomelab-vm-backup-policy.service static -\nhomelab-vm-full-backup.service static -\nhomelab-vm-local-dumps-2cloud.service static -\nhomelab-vm160-remote-backup.service static -\nimmich-media-backup.service static -\nmemos-backup.service static -\nnetbird-vps-backup.service static -\npaperless-backup.service static -\nskladchik-reports-monitor-backup-restore-check.service static -\nskladchik-reports-monitor-backup.service static -\n/usr/local/sbin/homelab-vm160-remote-backup:8:PARENT=/mnt/staging/vzdump\n/usr/local/sbin/homelab-vm160-remote-backup:22:SOURCE_ACTIVE=\"$(ssh \"${SSH[@]}\" \"$SOURCE\" \"pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'\")\" || SOURCE_ACTIVE_RC=$?\n/usr/local/sbin/homelab-vm160-remote-backup:27:NAME=\"vzdump-qemu-${VMID}-${TS}.vma.zst\"\n/usr/local/sbin/homelab-vm160-remote-backup:35: ssh \"${SSH[@]}\" \"$TARGET\" \"rm -rf -- '$DIR'\" >/dev/null 2>&1 || true\n/usr/local/sbin/homelab-vm160-remote-backup:41: rm -f \"$LOG\"\n/usr/local/sbin/homelab-vm160-remote-backup:46:LATEST_SIZE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\\n' | sort -nr | head -n1\")\"\n/usr/local/sbin/homelab-vm160-remote-backup:55:ssh \"${SSH[@]}\" \"$SOURCE\" \"vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7\" 2>\"$LOG\" | ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$PART'\"\n/usr/local/sbin/homelab-vm160-remote-backup:57:ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$DIR/vzdump.log'\" < \"$LOG\"\n/usr/local/sbin/homelab-vm160-remote-backup:66:rm -f \"$LOG\"\n/usr/local/sbin/skladchik-reports-monitor-status:11:check_health() {\n/usr/local/sbin/skladchik-reports-monitor-status:24:check_timer() {\n/usr/local/sbin/skladchik-reports-monitor-status:38:check_timer skladchik-reports-monitor-backup.timer\n/usr/local/sbin/skladchik-reports-monitor-status:39:check_timer skladchik-reports-monitor-backup-restore-check.timer\n/usr/local/sbin/skladchik-reports-monitor-status:40:check_timer skladchik-reports-monitor-supervisor.timer\n/usr/local/sbin/skladchik-reports-monitor-status:41:check_timer skladchik-reports-monitor-monthly-selftest-watch.timer\n/usr/local/sbin/skladchik-reports-monitor-status:43:check_health \"PVE01: ежедневный бэкап\" \"/var/lib/skladchik-reports-monitor-backup/health.txt\"\n/usr/local/sbin/skladchik-reports-monitor-status:44:check_health \"PVE01: проверка восстановления\" \"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\"\n/usr/local/sbin/skladchik-reports-monitor-status:45:check_health \"PVE01: общий supervisor\" \"/var/lib/skladchik-reports-monitor-supervisor/health.txt\"\n/usr/local/sbin/skladchik-reports-monitor-status:46:check_health \"PVE01: контроль ежемесячного самотеста\" \"/var/lib/skladchik-reports-monitor-selftest-watch/health.txt\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:52: homelab-rclone-cat-chunked \"${R}:$CRIT_DEST/$CRIT_BUNDLE\" > \"$OUT\" 2>\"$WORK/logs/${SAFE}_crit_download.err\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:83: rm -f \"$OUT\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:108: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/MANIFEST.txt\" \"$WORK/edge/MANIFEST.txt\" > \"$WORK/logs/edge_manifest_download.out\" 2>\"$WORK/logs/edge_manifest_download.err\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:111: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/SHA256SUMS.txt\" \"$WORK/edge/SHA256SUMS.txt\" > \"$WORK/logs/edge_sums_download.out\" 2>\"$WORK/logs/edge_sums_download.err\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:114: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/$EDGE_ARCHIVE.sha256\" \"$WORK/edge/$EDGE_ARCHIVE.sha256\" > \"$WORK/logs/edge_archive_sha_download.out\" 2>\"$WORK/logs/edge_archive_sha_download.err\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:117: rclone lsf \"${EDGE_REMOTE}:$EDGE_DEST/parts\" --format ps > \"$WORK/logs/edge_parts_lsf.txt\" 2>\"$WORK/logs/edge_parts_lsf.err\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:130: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/parts/$FIRST_PART\" \"$WORK/edge/$FIRST_PART\" > \"$WORK/logs/edge_first_part_download.out\" 2>\"$WORK/logs/edge_first_part_download.err\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:137: rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/parts/$LAST_PART\" \"$WORK/edge/$LAST_PART\" > \"$WORK/logs/edge_last_part_download.out\" 2>\"$WORK/logs/edge_last_part_download.err\"\n/usr/local/sbin/homelab-mail-cloud-restore-drill:168: rm -f \"$WORK/edge/$FIRST_PART\" \"$WORK/edge/$LAST_PART\"\n/usr/local/sbin/homelab-golden-state-extended-overlay:26:rclone copyto \"$JSON\" \"$RA\" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-golden-state-extended-overlay:27:rclone copyto \"$JSON\" \"$RB\" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-golden-state-extended-overlay:28:rclone cat \"$RA\" 2>/dev/null | jq -e --arg cp \"$CP\" \".control_plane_job == \\$cp\" >/dev/null || BAD=$((BAD+1))\n/usr/local/sbin/homelab-golden-state-extended-overlay:29:rclone cat \"$RB\" 2>/dev/null | jq -e --arg cp \"$CP\" \".control_plane_job == \\$cp\" >/dev/null || BAD=$((BAD+1))\n/usr/local/sbin/homelab-mail-cloud-critical-upload:50:EXCLUDED=edge-vm-vzdump-large-archive,rclone-config,secrets,raw-env\n/usr/local/sbin/homelab-mail-cloud-critical-upload:80: rclone mkdir \"${R}:$REMOTE_DIR\" > \"$WORK/logs/${SAFE}_mkdir.out\" 2>\"$WORK/logs/${SAFE}_mkdir.err\"\n/usr/local/sbin/homelab-mail-cloud-critical-upload:82: homelab-rclone-copy-chunked \"$BUNDLE\" \"${R}:$REMOTE_DIR/$(basename \"$BUNDLE\")\" > \"$WORK/logs/${SAFE}_upload_bundle.out\" 2>\"$WORK/logs/${SAFE}_upload_bundle.err\"\n/usr/local/sbin/homelab-mail-cloud-critical-upload:84: rclone copyto \"$BUNDLE_SHA\" \"${R}:$REMOTE_DIR/$(basename \"$BUNDLE_SHA\")\" --checksum > \"$WORK/logs/${SAFE}_upload_sha.out\" 2>\"$WORK/logs/${SAFE}_upload_sha.err\"\n/usr/local/sbin/homelab-mail-cloud-critical-upload:86: rclone copyto \"$WORK/components/MANIFEST.txt\" \"${R}:$REMOTE_DIR/MANIFEST.txt\" --checksum > \"$WORK/logs/${SAFE}_upload_manifest.out\" 2>\"$WORK/logs/${SAFE}_upload_manifest.err\"\n/usr/local/sbin/homelab-mail-cloud-critical-upload:88: rclone copyto \"$WORK/components/SHA256SUMS.txt\" \"${R}:$REMOTE_DIR/SHA256SUMS.txt\" --checksum > \"$WORK/logs/${SAFE}_upload_sums.out\" 2>\"$WORK/logs/${SAFE}_upload_sums.err\"\n/usr/local/sbin/homelab-mail-cloud-critical-upload:90: homelab-rclone-cat-chunked \"${R}:$REMOTE_DIR/$(basename \"$BUNDLE\")\" > \"$WORK/download/${SAFE}-$(basename \"$BUNDLE\")\" 2>\"$WORK/logs/${SAFE}_download_bundle.err\"\n/usr/local/sbin/homelab-mail-cloud-critical-upload:108: rm -f \"$WORK/download/${SAFE}-$(basename \"$BUNDLE\")\"\n/usr/local/sbin/homelab-mail-cloud-critical-upload:109: rclone lsf \"${R}:$TARGET_BASE\" --dirs-only 2>/dev/null | sed 's#/$##' | sort > \"$WORK/logs/${SAFE}_dirs.txt\"\n/usr/local/sbin/homelab-mail-cloud-critical-upload:114: [ -n \"$OLD\" ] && rclone purge \"${R}:$TARGET_BASE/$OLD\" >> \"$WORK/logs/${SAFE}_retention.out\" 2>>\"$WORK/logs/${SAFE}_retention.err\"\n/usr/local/sbin/homelab-mail-cloud-critical-upload:140: rm -rf --one-file-system \"$OLD_DIR\"\n/usr/local/sbin/homelab-alertmanager-backup-run:16:ssh -n -o BatchMode=yes -o ConnectTimeout=8 \"$MON\" \"sudo -n docker exec alertmanager amtool check-config /etc/alertmanager/alertmanager.yml >/dev/null && sudo -n tar -C / -czf - opt/stacks/observability-lite/compose.yaml opt/stacks/observability-lite/compose.override.yaml opt/stacks/observability-lite/alertmanager/alertmanager.yml opt/stacks/observability-lite/alertmanager/data\" > \"$STAGE/remote.tgz\"\n/usr/local/sbin/homelab-alertmanager-backup-run:18:rm -f \"$STAGE/remote.tgz\"\n/usr/local/sbin/homelab-alertmanager-backup-run:21:jq -n --arg app \"$APP\" --arg host \"monitoring\" --arg engine \"files+alertmanager-config-check\" --arg config \"ok\" --argjson files \"$FILES\" \"{app:\\$app,engine:\\$engine,source_host:\\$host,source_paths:[\\\"monitoring:/opt/stacks/observability-lite/alertmanager\\\",\\\"monitoring:/opt/stacks/observability-lite/compose.yaml\\\",\\\"monitoring:/opt/stacks/observability-lite/compose.override.yaml\\\"],config_check:\\$config,file_count:\\$files,note:\\\"Alertmanager configuration and runtime state in encrypted two-cloud backup\\\"}\" > \"$STAGE/metadata.json\"\n/usr/local/sbin/appbackupctl:13: check) bash -n /usr/local/sbin/appbackup-dockge-stacks && bash -n /usr/local/sbin/appbackup-restore-check && bash -n /usr/local/sbin/appbackup-retention && bash -n /usr/local/sbin/appbackup-report && echo \"OK appbackup framework\" ;;\n/usr/local/sbin/appbackupctl:14: restore-check) appbackup-restore-check \"${2:-dockge-stacks}\" ;;\n/usr/local/sbin/appbackupctl:17: audit) appbackupctl check && appbackupctl restore-check dockge-stacks && appbackupctl retention 7 dry-run dockge-stacks && appbackupctl report dockge-stacks >/dev/null && echo \"OK app audit finished\" ;;\n/usr/local/sbin/appbackupctl:21: *) echo \"usage: appbackupctl run [dockge-stacks|cluster-admin-incident-engine]|check|restore-check|retention|report|audit|status|lastlog|timer\"; exit 2 ;;\n/usr/local/sbin/postgres-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@\"$H\" \"sudo rm -f /tmp/postgres-app-${APP}-${JOB_ID}.tgz\"\n/usr/local/sbin/postgres-apps-backup-run:21: rm -rf \"$STAGE/payload\"\n/usr/local/sbin/homelab-backup-gitea:27:rm -rf \"$RESTORE\"\n/usr/local/sbin/homelab-backup-gitea:35:INTEGRITY=\"$(sqlite3 \"$DB\" 'PRAGMA integrity_check;')\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:4:CRYPT_FILE=\"/etc/homelab/rclone/required-crypt-remotes.txt\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:18: \"checksums/\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:21:echo \"=== homelab rclone crypt healthcheck: $(hostname) ===\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:40: if ! rclone listremotes | sed 's/:$//' | grep -qx \"$REMOTE\"; then\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:47: if ! rclone about \"${REMOTE}:\"; then\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:48: echo \"FAIL: rclone about failed: $REMOTE\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:54: ROOT_LIST=\"$(rclone lsf \"${REMOTE}:/\" || true)\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:76: if rclone cat \"${REMOTE}:/latest-backup-index.json\" | jq -e '.schema and .remote and .status and .items' >/dev/null; then\n/usr/local/sbin/homelab-check-private-vpn-hosts.realnet-20260624T140509Z:4:REAL=\"/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z\"\n/usr/local/sbin/homelab-stage-verify-flat-chunked:7:rm -rf \"$VERIFY\"\n/usr/local/sbin/homelab-stage-verify-flat-chunked:9:rclone copy \"$REMOTE_DIR\" \"$VERIFY\" --transfers 4 --checkers 8 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-stage-verify-flat-chunked:11:(cd \"$VERIFY\" && test -s manifest.json && if [[ -n \"$JOB\" ]]; then jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null; fi && sha256sum -c checksums.sha256 >/dev/null && tar -tzf \"$ARCHIVE\" >/dev/null)\n/usr/local/sbin/homelab-rclone-cat-chunked:4:TMP_BASE=${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\n/usr/local/sbin/homelab-rclone-cat-chunked:7:cleanup(){ rm -rf \"$WORK\"; }\n/usr/local/sbin/homelab-rclone-cat-chunked:10:rclone copyto \"${REMOTE_FILE}.parts/MANIFEST.parts.txt\" \"$MAN\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-rclone-cat-chunked:11:while read -r PART SIZE; do case \"$PART\" in part-*) rclone cat \"${REMOTE_FILE}.parts/$PART\" ;; esac; done < \"$MAN\"\n/usr/local/sbin/homelab-stage-verify-chunked:7:rm -rf \"$VERIFY\"\n/usr/local/sbin/homelab-stage-verify-chunked:9:rclone copy \"$REMOTE_DIR\" \"$VERIFY\" --exclude \"$ARCHIVE.parts/**\" --exclude \"$ARCHIVE.parts/\" --exclude \"$ARCHIVE.sha256\" --exclude \"$ARCHIVE.manifest.txt\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --log-level ERROR\n/usr/local/sbin/homelab-stage-verify-chunked:13:if [ -s checksums.sha256 ]; then grep -F -v \" $ARCHIVE\" checksums.sha256 > .checksums-no-archive.sha256 || true; if [ -s .checksums-no-archive.sha256 ]; then sha256sum -c .checksums-no-archive.sha256; fi; EXPECTED=$(grep -F \" $ARCHIVE\" checksums.sha256 | awk '{print $1; exit}'); else EXPECTED=$(rclone cat \"${REMOTE_DIR}/${ARCHIVE}.sha256\" | awk '{print $1; exit}'); fi\n/usr/local/sbin/homelab-stage-verify-chunked:15:ACTUAL=$(homelab-rclone-cat-chunked \"${REMOTE_DIR}/${ARCHIVE}\" | sha256sum | awk '{print $1}')\n/usr/local/sbin/homelab-stage-verify-chunked:17:homelab-rclone-cat-chunked \"${REMOTE_DIR}/${ARCHIVE}\" | tar -tzf - >/dev/null\n/usr/local/sbin/crypto-portfolio-backup-run:15:ssh -n -o BatchMode=yes -o ConnectTimeout=10 \"$CORE\" 'H=\"$(hostname -f 2>/dev/null || hostname)\"; { [ \"$H\" = \"core-apps.gram1.ru\" ] || [ \"$H\" = \"core-apps\" ]; } || exit 3; D=/opt/stacks/crypto-portfolio; T=\"$(sudo -n mktemp -d /tmp/crypto-portfolio-backup.XXXXXX)\"; trap \"sudo -n rm -rf \\\"$T\\\"\" EXIT; sudo -n install -d -m 700 \"$T/stack\" \"$T/stack/data\"; for F in compose.yaml Dockerfile requirements.txt app.py worker.py .dockerignore; do sudo -n test -s \"$D/$F\" && sudo -n cp -a \"$D/$F\" \"$T/stack/\"; done; sudo -n cp -a \"$D/templates\" \"$D/secrets\" \"$T/stack/\"; sudo -n python3 -c \"import sqlite3,sys;s=sqlite3.connect(sys.argv[1]);d=sqlite3.connect(sys.argv[2]);s.backup(d);d.close();s.close()\" \"$D/data/portfolio.db\" \"$T/stack/data/portfolio.db\"; for F in market.json worker-status.json; do sudo -n test -s \"$D/data/$F\" && sudo -n cp -a \"$D/data/$F\" \"$T/stack/data/\"; done; sudo -n tar -C \"$T\" -czf - stack' > \"$STAGE/remote.tgz\"\n/usr/local/sbin/crypto-portfolio-backup-run:17:rm -f \"$STAGE/remote.tgz\"\n/usr/local/sbin/crypto-portfolio-backup-run:18:DBI=\"$(python3 -c 'import sqlite3,sys;print(sqlite3.connect(sys.argv[1]).execute(\"PRAGMA integrity_check\").fetchone()[0])' \"$STAGE/payload/stack/data/portfolio.db\")\"\n/usr/local/sbin/npmplus-edge-backup-run:20:ssh -o BatchMode=yes -o ConnectTimeout=8 \"$EDGE\" \"sudo rm -f '/tmp/npmplus-edge-${JOB_ID}.tgz'\"\n/usr/local/sbin/npmplus-edge-backup-run:28:rm -rf \"$STAGE/payload\"\n/usr/local/sbin/mariadb-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@\"$H\" \"sudo rm -f /tmp/mariadb-app-${APP}-${JOB_ID}.tgz\"\n/usr/local/sbin/mariadb-apps-backup-run:20: rm -rf \"$STAGE/payload\"\n/usr/local/sbin/homelab-blackbox-exporter-backup-run:17:rm -f \"$STAGE/remote.tgz\"\n/usr/local/sbin/homelab-blackbox-exporter-backup-run:21:jq -n --arg app \"$APP\" --arg host \"monitoring\" --arg engine \"files+runtime-health-check\" --argjson files \"$FILES\" '{app:$app,engine:$engine,source_host:$host,source_paths:[\"monitoring:/opt/stacks/blackbox-exporter\"],runtime_health:\"ok\",file_count:$files,note:\"Blackbox Exporter compose and probe configuration in encrypted two-cloud backup\"}' > \"$STAGE/metadata.json\"\n/usr/local/sbin/homelab-ingress-standby-refresh:16: rm -f \"$TMP_STATE\" \"$RELAY\" \"$RESOURCES\"\n/usr/local/sbin/homelab-ingress-standby-refresh:74:run_node \"$SOURCE_NODE\" vzdump 130 \\\n/usr/local/sbin/homelab-ingress-standby-refresh:78: --remove 0 \\\n/usr/local/sbin/homelab-ingress-standby-refresh:83:\"find /var/lib/vz/dump -maxdepth 1 -type f -name 'vzdump-qemu-130-*.vma.zst' -printf '%T@ %p\\n' | sort -rn | head -n 1 | cut -d' ' -f2-\"\n/usr/local/sbin/homelab-ingress-standby-refresh:106:\"chattr -i '$DEST_FILE' '$DEST_FILE.sha256' '$DEST_FILE.env' 2>/dev/null || true; rm -f '$DEST_TMP'\"\n/usr/local/sbin/homelab-ingress-standby-refresh:155: -name 'vzdump-qemu-130-*.vma.zst' \\\n/usr/local/sbin/homelab-ingress-standby-refresh:164: rm -f \"$OLD\" \"$OLD.sha256\" \"$OLD.env\"\n/usr/local/sbin/homelab-ingress-standby-refresh:173:run_node \"$SOURCE_NODE\" rm -f \"$SOURCE_BACKUP\"\n/usr/local/sbin/homelab-stage-2cloud-flat:33:(cd \"$STAGE\"; FILES=(\"$ARCHIVE\" manifest.json latest-backup-index.json); [[ -f metadata.json ]] && FILES+=(metadata.json); sha256sum \"${FILES[@]}\" > checksums.sha256; sha256sum -c checksums.sha256 >/dev/null; tar -tzf \"$ARCHIVE\" >/dev/null; jq -e . manifest.json >/dev/null)\n/usr/local/sbin/homelab-stage-2cloud-flat:36: if rclone lsf \"$REMOTE_DIR\" --max-depth 1 2>/dev/null | grep -q .; then echo \"REFUSE NONEMPTY CANARY TARGET $REMOTE_DIR\" >&2; exit 6; fi\n/usr/local/sbin/homelab-stage-2cloud-flat:37: rclone mkdir \"$REMOTE_DIR\"\n/usr/local/sbin/homelab-stage-2cloud-flat:38: for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f \"$STAGE/$F\" ]] && rclone copyto \"$STAGE/$F\" \"$REMOTE_DIR/$F\" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done\n/usr/local/sbin/homelab-stage-2cloud-flat:39: CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-flat-chunked \"$STAGE/$ARCHIVE\" \"$REMOTE_DIR/$ARCHIVE\"\n/usr/local/sbin/homelab-stage-2cloud-flat:41: rclone copyto \"$STAGE/latest-backup-index.json\" \"$LATEST_FILE\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-stage-2cloud-flat:42: rclone cat \"$LATEST_FILE\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n/usr/local/sbin/homelab-stage-2cloud-flat:47:rm -rf \"$STAGE\" \"$VERIFY\"\n/usr/local/sbin/homelab-cluster-admin-inventory-export:12: return subprocess.check_output(args, text=True, stderr=subprocess.DEVNULL, timeout=30)\n/usr/local/sbin/homelab-cluster-admin-webpanel-sync:7:copy_health() {\n/usr/local/sbin/homelab-cluster-admin-webpanel-sync:17:copy_health /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt cluster-admin-vm-mail-cloud-backup.txt\n/usr/local/sbin/homelab-cluster-admin-webpanel-sync:18:copy_health /var/lib/homelab-health/overall.txt overall.txt\n/usr/local/sbin/homelab-cluster-admin-webpanel-sync:19:copy_health /var/lib/homelab-health/final-readiness.txt final-readiness.txt\n/usr/local/sbin/homelab-cluster-admin-webpanel-sync:20:copy_health /var/lib/homelab-health/cluster-admin-strict-final-seal.txt cluster-admin-strict-final-seal.txt\n/usr/local/sbin/homelab-backup-authentik:14:ssh debian@[PRIVATE_IP] \"rm -rf /tmp/auth-small-snap; mkdir -p /tmp/auth-small-snap\"\n/usr/local/sbin/homelab-backup-authentik:27:ssh debian@[PRIVATE_IP] \"sudo docker rm -f authentik-restore-test >/dev/null 2>&1 || true\"\n/usr/local/sbin/homelab-backup-authentik:51:ssh debian@[PRIVATE_IP] \"sudo docker rm -f authentik-restore-test >/dev/null 2>&1 || true; rm -f /tmp/authentik-restore-from-archive.pg_dump\"\n/usr/local/sbin/homelab-safe-exec:77: check=False,\n/usr/local/sbin/homelab-safe-exec:621: latest_tmp.unlink(missing_ok=True)\n/usr/local/sbin/homelab-golden-state-index:28:rclone copyto \"$JSON\" \"$RA\" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-golden-state-index:29:rclone copyto \"$JSON\" \"$RB\" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-golden-state-index:30:rclone cat \"$RA\" 2>/dev/null | jq -e --arg cp \"$CP\" --arg ext \"$EXT\" \".control_plane_job == \\$cp and .health.extended_appbackup == \\$ext and .extended_appbackup_health == \\$ext\" >/dev/null || BAD=$((BAD+1))\n/usr/local/sbin/homelab-golden-state-index:31:rclone cat \"$RB\" 2>/dev/null | jq -e --arg cp \"$CP\" --arg ext \"$EXT\" \".control_plane_job == \\$cp and .health.extended_appbackup == \\$ext and .extended_appbackup_health == \\$ext\" >/dev/null || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-full-retention:21: -name \"vzdump-qemu-${vmid}-*.vma.zst\" \\\n/usr/local/sbin/homelab-vm-full-retention:33: vzdump-qemu-\"$vmid\"-*.vma.zst)\n/usr/local/sbin/homelab-vm-full-retention:35: rm -f \\\n/usr/local/sbin/homelab-vm-full-retention:73: rm -rf -- \\\"\\$BASE/\\$setname\\\"\n/usr/local/sbin/homelab-vm-full-retention:91: rclone lsf \"$path\" \\\n/usr/local/sbin/homelab-vm-full-retention:105: rclone purge \"${path}/${dir}\"\n/usr/local/sbin/homelab-stage-2cloud:22:echo \"[STEP] local checksum\"; (cd \"$STAGE\" && sha256sum \"$ARCHIVE\" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e . manifest.json >/dev/null)\n/usr/local/sbin/homelab-stage-2cloud:23:echo \"[STEP] upload cloud A metadata\"; rclone copy \"$STAGE\" \"$RA\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n/usr/local/sbin/homelab-stage-2cloud:24:echo \"[STEP] upload cloud A archive chunks\"; rclone deletefile \"$RA/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RA/$ARCHIVE\"\n/usr/local/sbin/homelab-stage-2cloud:25:echo \"[STEP] upload cloud B metadata\"; rclone copy \"$STAGE\" \"$RB\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n/usr/local/sbin/homelab-stage-2cloud:26:echo \"[STEP] upload cloud B archive chunks\"; rclone deletefile \"$RB/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RB/$ARCHIVE\"\n/usr/local/sbin/homelab-stage-2cloud:27:echo \"[STEP] verify cloud A chunked\"; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-stage-verify-chunked \"$RA\" \"$ARCHIVE\" \"$VERIFY/a\" \"$JOB\"\n/usr/local/sbin/homelab-stage-2cloud:28:echo \"[STEP] verify cloud B chunked\"; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-stage-verify-chunked \"$RB\" \"$ARCHIVE\" \"$VERIFY/b\" \"$JOB\"\n/usr/local/sbin/homelab-stage-2cloud:29:echo \"[STEP] upload latest index\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RAL\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RBL\"; rclone cat \"$RAL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null; rclone cat \"$RBL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n/usr/local/sbin/homelab-stage-2cloud:31:echo \"[STEP] delete local staging\"; rm -rf \"$STAGE\" \"$VERIFY\"; test ! -d \"$STAGE\" && test ! -d \"$VERIFY\"\n/usr/local/sbin/homelab-safe-autoheal:17:SAFE_TIMERS=\"homelab-emergency-backup.timer homelab-appbackup-dockge-stacks.timer homelab-appbackup-npmplus-edge.timer homelab-appbackup-sqlite-batch.timer homelab-appbackup-postgres-batch.timer homelab-appbackup-mariadb-batch.timer homelab-verified-backup-health.timer homelab-drift-check.timer homelab-service-registry-check.timer homelab-dependency-map-check.timer homelab-vm-backup-health.timer homelab-alerting-health.timer homelab-runbook-generate.timer homelab-desired-state-sync.timer homelab-duty-admin-report.timer homelab-incident-journal.timer homelab-backup-sla-health.timer homelab-kuma-monitor-policy.timer homelab-safe-autoheal.timer homelab-overall-health.timer\"\n/usr/local/sbin/homelab-safe-autoheal:144:for cmd in homelab-alerting-health homelab-kuma-monitor-policy homelab-backup-sla-health homelab-service-registry-check homelab-dependency-map-check; do\n/usr/local/sbin/homelab-backup-npmplus-kuma-configs:26: install -D -m 0755 /usr/local/sbin/homelab-check-private-vpn-hosts \"$SNAP/pve01/usr/local/sbin/homelab-check-private-vpn-hosts\" 2>>\"$ERR\" || fail \"copy_private_vpn_script_failed\"\n/usr/local/sbin/homelab-backup-npmplus-kuma-configs:27: install -D -m 0644 /etc/systemd/system/homelab-private-vpn-hosts-health.service \"$SNAP/pve01/etc/systemd/system/homelab-private-vpn-hosts-health.service\" 2>>\"$ERR\" || fail \"copy_private_vpn_service_failed\"\n/usr/local/sbin/homelab-backup-npmplus-kuma-configs:28: install -D -m 0644 /etc/systemd/system/homelab-private-vpn-hosts-health.timer \"$SNAP/pve01/etc/systemd/system/homelab-private-vpn-hosts-health.timer\" 2>>\"$ERR\" || fail \"copy_private_vpn_timer_failed\"\n/usr/local/sbin/homelab-backup-npmplus-kuma-configs:29: install -D -m 0644 /var/lib/homelab-health/private-vpn-hosts.txt \"$SNAP/pve01/var/lib/homelab-health/private-vpn-hosts.txt\" 2>>\"$ERR\" || fail \"copy_private_vpn_health_failed\"\n/usr/local/sbin/homelab-backup-npmplus-kuma-configs:42:sudo rm -rf \"$TMP\" \"$EDGE_TGZ\"\n/usr/local/sbin/homelab-backup-npmplus-kuma-configs:82:sudo rm -rf \"$TMP\"\n/usr/local/sbin/homelab-backup-npmplus-kuma-configs:90: ssh -o BatchMode=yes -o ConnectTimeout=8 debian@[PRIVATE_IP] \"rm -f '$EDGE_TGZ'\" 2>>\"$ERR\" || true\n/usr/local/sbin/homelab-backup-npmplus-kuma-configs:99: NPM_INTEGRITY=\"$(sqlite3 \"$SNAP/edge-vm/npmplus/db/database.sqlite\" \"PRAGMA integrity_check;\" 2>>\"$ERR\" | tr '\\n' ';' || echo error)\"\n/usr/local/sbin/homelab-backup-npmplus-kuma-configs:100: KUMA_INTEGRITY=\"$(sqlite3 \"$SNAP/edge-vm/kuma/db/kuma.db\" \"PRAGMA integrity_check;\" 2>>\"$ERR\" | tr '\\n' ';' || echo error)\"\n/usr/local/sbin/homelab-backup-npmplus-kuma-configs:134: rm -rf \"$RESTORE\"\n/usr/local/sbin/homelab-backup-npmplus-kuma-configs:138: RESTORE_NPM_INTEGRITY=\"$(sqlite3 \"$RESTORE/edge-vm/npmplus/db/database.sqlite\" \"PRAGMA integrity_check;\" 2>>\"$ERR\" | tr '\\n' ';' || echo error)\"\n/usr/local/sbin/homelab-backup-npmplus-kuma-configs:139: RESTORE_KUMA_INTEGRITY=\"$(sqlite3 \"$RESTORE/edge-vm/kuma/db/kuma.db\" \"PRAGMA integrity_check;\" 2>>\"$ERR\" | tr '\\n' ';' || echo error)\"\n/usr/local/sbin/homelab-stage-2cloud-flat-canary:31:(cd \"$STAGE\"; FILES=(\"$ARCHIVE\" manifest.json latest-backup-index.json); [[ -f metadata.json ]] && FILES+=(metadata.json); sha256sum \"${FILES[@]}\" > checksums.sha256; sha256sum -c checksums.sha256 >/dev/null; tar -tzf \"$ARCHIVE\" >/dev/null; jq -e . manifest.json >/dev/null)\n/usr/local/sbin/homelab-stage-2cloud-flat-canary:34: if rclone lsf \"$REMOTE_DIR\" --max-depth 1 2>/dev/null | grep -q .; then echo \"REFUSE NONEMPTY CANARY TARGET $REMOTE_DIR\" >&2; exit 6; fi\n/usr/local/sbin/homelab-stage-2cloud-flat-canary:35: rclone mkdir \"$REMOTE_DIR\"\n/usr/local/sbin/homelab-stage-2cloud-flat-canary:36: for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f \"$STAGE/$F\" ]] && rclone copyto \"$STAGE/$F\" \"$REMOTE_DIR/$F\" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done\n/usr/local/sbin/homelab-stage-2cloud-flat-canary:37: CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-flat-chunked \"$STAGE/$ARCHIVE\" \"$REMOTE_DIR/$ARCHIVE\"\n/usr/local/sbin/homelab-stage-2cloud-flat-canary:39: rclone copyto \"$STAGE/latest-backup-index.json\" \"$LATEST_FILE\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-stage-2cloud-flat-canary:40: rclone cat \"$LATEST_FILE\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n/usr/local/sbin/homelab-stage-2cloud-flat-canary:45:rm -rf \"$STAGE\" \"$VERIFY\"\n/usr/local/sbin/homelab-router-running-config-mail-cloud:62: rm -f \"$CONFIG\"\n/usr/local/sbin/homelab-router-running-config-mail-cloud:66: rm -f \"$CONFIG\"\n/usr/local/sbin/homelab-router-running-config-mail-cloud:73: if ( cd \"$BACKUP_ROOT\" && sha256sum -c \"$(basename \"$SHA_FILE\")\" ) >\"$WORK/sha-check.out\" 2>\"$WORK/sha-check.err\"; then\n/usr/local/sbin/homelab-router-running-config-mail-cloud:79: if rclone mkdir \"$REMOTE:$DEST\" >\"$WORK/rclone-mkdir.out\" 2>\"$WORK/rclone-mkdir.err\" && homelab-rclone-copy-chunked \"$ARCHIVE\" \"$REMOTE:$DEST/$(basename \"$ARCHIVE\")\" >\"$WORK/rclone-upload.out\" 2>\"$WORK/rclone-upload.err\" && rclone copyto \"$SHA_FILE\" \"$REMOTE:$DEST/$(basename \"$SHA_FILE\")\" --checksum >\"$WORK/rclone-upload-sha.out\" 2>\"$WORK/rclone-upload-sha.err\"; then\n/usr/local/sbin/homelab-router-running-config-mail-cloud:86: if homelab-rclone-cat-chunked \"$REMOTE:$DEST/$(basename \"$ARCHIVE\")\" >\"$DL\" 2>\"$WORK/rclone-download.err\"; then\n/usr/local/sbin/homelab-router-running-config-mail-cloud:93:rclone lsf \"$REMOTE:$TARGET_BASE\" --dirs-only 2>/dev/null | sed 's#/$##' | sort > \"$DIR_LIST\" || RETENTION_RC=$?\n/usr/local/sbin/homelab-router-running-config-mail-cloud:98: [ -n \"$OLD\" ] && rclone purge \"$REMOTE:$TARGET_BASE/$OLD\" >>\"$WORK/retention.out\" 2>>\"$WORK/retention.err\"\n/usr/local/sbin/homelab-router-running-config-mail-cloud:103: rm -f \"$old\" \"$old.sha256\"\n/usr/local/sbin/homelab-router-running-config-mail-cloud:116:rm -rf \"$WORK\"\n/usr/local/sbin/appbackup-restore-check:22:ROOT=\"/var/tmp/homelab-app-restore-check/${JOB}\"\n/usr/local/sbin/appbackup-restore-check:24:rm -rf \"$ROOT\" && mkdir -p \"$ROOT/a\" \"$ROOT/b\"\n/usr/local/sbin/appbackup-restore-check:25:trap \"echo \\\"ERROR app restore-check failed; kept: $ROOT\\\"; exit 1\" ERR\n/usr/local/sbin/appbackup-restore-check:26:echo \"restore-check app=$APP job=$JOB schema=$SCHEMA\"\n/usr/local/sbin/appbackup-restore-check:27:rclone copy \"$RA\" \"$ROOT/a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >\"${LOGBASE}-a.log\" 2>&1\n/usr/local/sbin/appbackup-restore-check:30:(cd \"$ROOT/a\" && sha256sum -c checksums.sha256 >\"${LOGBASE}-a-sha.log\" && ARCHIVE=\"$(jq -r \".archive_name\" manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n/usr/local/sbin/appbackup-restore-check:31:echo \"OK cloud A app restore-check\"\n/usr/local/sbin/appbackup-restore-check:32:rclone copy \"$RB\" \"$ROOT/b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >\"${LOGBASE}-b.log\" 2>&1\n/usr/local/sbin/appbackup-restore-check:35:(cd \"$ROOT/b\" && sha256sum -c checksums.sha256 >\"${LOGBASE}-b-sha.log\" && ARCHIVE=\"$(jq -r \".archive_name\" manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n/usr/local/sbin/appbackup-restore-check:36:echo \"OK cloud B app restore-check\"\n/usr/local/sbin/appbackup-restore-check:37:rm -rf \"$ROOT\"\n/usr/local/sbin/appbackup-restore-check:38:rmdir /var/tmp/homelab-app-restore-check 2>/dev/null || true\n/usr/local/sbin/appbackup-restore-check:39:rm -f \"${LOGBASE}-a.log\" \"${LOGBASE}-a-sha.log\" \"${LOGBASE}-b.log\" \"${LOGBASE}-b-sha.log\"\n/usr/local/sbin/appbackup-restore-check:41:echo \"OK app restore-check both clouds\"\n/usr/local/sbin/backupctl:12: check) bash -n \"$SCRIPT\" && [[ -f \"$STATE\" ]] && jq -e \".status == \\\"success\\\" and .local_backup_deleted == true\" \"$STATE\" >/dev/null && echo \"OK backup framework\" ;;\n/usr/local/sbin/backupctl:13: restore-check) backup-restore-check ;;\n/usr/local/sbin/backupctl:14: audit) backupctl check && backup-restore-check && backupctl retention 7 dry-run && backup-report >/dev/null && echo \"OK audit finished\" ;;\n/usr/local/sbin/backupctl:16: remotes) source /etc/homelab-backup.env && NODE=\"$(hostname -s)\" && echo \"A=$REMOTE_A\" && echo \"B=$REMOTE_B\" && rclone lsd \"${REMOTE_A%/}/${NODE}/emergency-pack\" | tail -n 5 && rclone lsd \"${REMOTE_B%/}/${NODE}/emergency-pack\" | tail -n 5 ;;\n/usr/local/sbin/backupctl:19: *) echo \"usage: backupctl run|status|lastlog|check|restore-check|audit|report|remotes|retention|timer\"; exit 2 ;;\n/usr/local/sbin/homelab-capacity-risk:32:LARGEST_BYTES=\"$(find /var/lib/vz/dump -maxdepth 1 -type f \\( -name \"vzdump-qemu-*.vma.zst\" -o -name \"vzdump-lxc-*.tar.zst\" \\) -printf \"%s\\n\" 2>/dev/null | sort -n | tail -n1)\"\n/usr/local/sbin/homelab-backupctl:6: audit) echo \"=== EMERGENCY AUDIT ===\"; backupctl audit; echo \"=== APP FRAMEWORK ===\"; appbackupctl check; echo \"=== APP AUDITS ===\"; for app in $(app_list); do echo \"### app=$app\"; appbackup-restore-check \"$app\"; appbackup-retention 7 dry-run \"$app\"; done; echo \"OK full backup audit\" ;;\n/usr/local/sbin/homelab-staging-check.sh:11:echo \"=== homelab staging check: ${HOST} ===\"\n/usr/local/sbin/homelab-secret-exposure-guard:18:BADFILES=\"$(find \"$REPO\" -path \"$REPO/.git\" -prune -o -type f \\( -iname \"*.key\" -o -iname \"*.pem\" -o -iname \"*.p12\" -o -iname \"*.pfx\" -o -iname \"[SENSITIVE_FILE]\" -o -iname \".env\" -o -iname \".env.*\" \\) -print 2>/dev/null)\"\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:9:OFFHOST_DIR=/mnt/staging/offhost/edge-vm-vzdump-from-pve03\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:85: /var/lib/vz/dump/vzdump-qemu-130-*.vma.zst) ;;\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:140: copy_out=\"$(run_pve03 copy \"$SOURCE_PATH\" \"$SOURCE_SIZE\" \"$SOURCE_SHA\" \"$DST_IP\" \"$OFFHOST_DIR\")\"\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:141: grep -Fq 'SOURCE_COPY_STATUS=OK' <<<\"$copy_out\"\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:156:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-backup NODE=%s VMID=%s ARCHIVE=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\\n' \\\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:158: >\"$HEALTH_DIR/edge-vm-vzdump-backup.txt\"\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:159:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-offhost NODE=%s VMID=%s HOST=%s DEST=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\\n' \\\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:161: >\"$HEALTH_DIR/edge-vm-vzdump-offhost.txt\"\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:162:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-restore-verify HOST=%s VMID=%s ARCHIVE=%s SIZE=%s SHA256=%s ZSTD_RC=0 VMA_RC=0 NOTES=nightly_existing_dump_freshness\\n' \\\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:164: >\"$HEALTH_DIR/edge-vm-vzdump-restore.txt\"\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:167: \"$HEALTH_DIR/edge-vm-vzdump-backup.txt\" \\\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:168: \"$HEALTH_DIR/edge-vm-vzdump-offhost.txt\" \\\n/usr/local/sbin/homelab-edge-vm-offhost-freshness:169: \"$HEALTH_DIR/edge-vm-vzdump-restore.txt\"\n/usr/local/sbin/homelab-external-probe-vps:20:rm -f \"$TMP\"\n/usr/local/sbin/sqlite-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@\"$H\" \"sudo rm -f /tmp/sqlite-app-${APP}-${JOB_ID}.tgz\"\n/usr/local/sbin/sqlite-apps-backup-run:21: rm -rf \"$STAGE/payload\"\n/usr/local/sbin/homelab-desired-state-sync:10:find \"$REPO\" -mindepth 1 -maxdepth 1 ! -name .git -exec rm -rf {} + 2>/dev/null\n/usr/local/sbin/homelab-desired-state-sync:24:[ -f /root/HOMELAB_SAFETY_NET_CHECKPOINT.md ] && cp -a /root/HOMELAB_SAFETY_NET_CHECKPOINT.md \"$REPO/docs/safety-net-checkpoint.md\" 2>/dev/null || true\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:4:CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:9:: \"${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}\"\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:13:LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:19:log(){ printf '%s RESTORE_CHECK %s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$*\" | tee -a \"$LOGDIR/restore-check.log\"; }\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:22:command -v rclone >/dev/null\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:28: if rclone lsf \"${remote_path}.parts/\" > \"$plist\" 2>/dev/null && grep -q '^part-' \"$plist\"; then\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:29: while IFS= read -r part; do [ -n \"$part\" ] && rclone cat \"${remote_path}.parts/${part}\" >> \"$out\"; done < <(grep '^part-' \"$plist\" | sort)\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:31: rclone copyto \"$remote_path\" \"$out\" >/dev/null\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:33: rm -f \"$plist\"\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:36:check_one_remote(){\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:39: latest=\"$(rclone lsf --dirs-only \"${rbase}/\" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)\"\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:85: [ \"$(sqlite3 \"$extract/var-lib-skladchik-reports-monitor/state.sqlite\" 'PRAGMA integrity_check;' | tail -n 1)\" = \"ok\" ]\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:97:check_one_remote CLOUD_1 \"$CLOUD_1\"\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:98:check_one_remote CLOUD_2 \"$CLOUD_2\"\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:101:find \"$WORK_ROOT\" -mindepth 1 -maxdepth 1 -type d -printf '%f\\n' 2>/dev/null | sort | head -n -\"${KEEP_LOCAL}\" | while read -r D; do [ -n \"$D\" ] && rm -rf \"${WORK_ROOT}/${D}\"; done\n/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch:77: check=False,\n/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch:101:def check() -> int:\n/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch:126: check=False,\n/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch:223: return check()\n/usr/local/sbin/skladchik-reports-monitor-backup:28:command -v rclone >/dev/null\n/usr/local/sbin/skladchik-reports-monitor-backup:29:[ -x /usr/local/sbin/homelab-rclone-copy-chunked ] || { echo \"missing /usr/local/sbin/homelab-rclone-copy-chunked\" >&2; exit 10; }\n/usr/local/sbin/skladchik-reports-monitor-backup:36:cleanup(){ rm -rf \"$TMP\"; }\n/usr/local/sbin/skladchik-reports-monitor-backup:75: /usr/local/sbin/homelab-rclone-copy-chunked \"$ARCHIVE\" \"${DEST_BASE}/$(basename \"$ARCHIVE\")\"\n/usr/local/sbin/skladchik-reports-monitor-backup:76: /usr/local/sbin/homelab-rclone-copy-chunked \"${ARCHIVE}.sha256\" \"${DEST_BASE}/$(basename \"$ARCHIVE\").sha256\"\n/usr/local/sbin/skladchik-reports-monitor-backup:77: /usr/local/sbin/homelab-rclone-copy-chunked \"${OUTDIR}/manifest.txt\" \"${DEST_BASE}/manifest.txt\"\n/usr/local/sbin/skladchik-reports-monitor-backup:82: mapfile -t OLD < <(rclone lsf --dirs-only \"${REMOTE}${REMOTE_BASE}/daily/\" 2>/dev/null | sed 's#/$##' | sort | head -n -\"${KEEP_DAILY}\" || true)\n/usr/local/sbin/skladchik-reports-monitor-backup:83: for D in \"${OLD[@]}\"; do [ -n \"$D\" ] && rclone purge \"${REMOTE}${REMOTE_BASE}/daily/${D}\" >/dev/null 2>&1 || true; done\n/usr/local/sbin/skladchik-reports-monitor-backup:85:find \"${LOCAL_ROOT}/daily\" -mindepth 1 -maxdepth 1 -type d -printf '%f\\n' 2>/dev/null | sort | head -n -\"${KEEP_DAILY}\" | while read -r D; do [ -n \"$D\" ] && rm -rf \"${LOCAL_ROOT}/daily/${D}\"; done\n/usr/local/sbin/homelab-vm-local-dumps-retention:32: mapfile -t JOBS < <(rclone lsf \"$BASE/\" --dirs-only 2>/dev/null | sed 's#/$##' | sort)\n/usr/local/sbin/homelab-vm-local-dumps-retention:56: rclone purge \"$PATH_REMOTE\" >> \"$REPORT\" 2>&1 && DELETED=$((DELETED+1)) || BAD=$((BAD+1))\n/usr/local/sbin/homelab-extended-appbackup-run:12:run_one(){ APP=\"$1\"; HOST=\"$2\"; PATHS=\"$3\"; NOTE=\"$4\"; CONS=\"$5\"; JOB_ID=\"homelab-${NODE}-app-${APP}-${RUN_TS}\"; export JOB_ID; SHORT_TS=\"${RUN_TS:2:6}-${RUN_TS:9:6}\"; export BACKUP_PATH_REL=\"$(homelab-backup-path job a \"$APP\" \"$SHORT_TS\")\"; STAGE=\"${STAGING_ROOT}/${JOB_ID}\"; PAYLOAD=\"$STAGE/payload\"; ARCHIVE=\"${JOB_ID}.tar.gz\"; LOG=\"/tmp/extended-appbackup-${APP}-${RUN_TS}.log\"; rm -rf \"$STAGE\"; mkdir -p \"$PAYLOAD/local\"; echo \"### app=$APP\" >> \"$REPORT\"; echo \"host=$HOST\" >> \"$REPORT\"; echo \"source_note=$NOTE\" >> \"$REPORT\"; ssh -n -o BatchMode=yes -o ConnectTimeout=30 \"$HOST\" \"sudo tar -C / --ignore-failed-read --exclude=.env --exclude=*.env --exclude=.env.* --exclude=*.bak* -czf - $PATHS\" > \"$PAYLOAD/local/${APP}-data.tar.gz\" 2>\"$LOG\"; TAR_RC=$?; if [ \"$TAR_RC\" -eq 0 ] && tar -tzf \"$PAYLOAD/local/${APP}-data.tar.gz\" >/dev/null 2>&1; then SIZE=\"$(stat -c%s \"$PAYLOAD/local/${APP}-data.tar.gz\" 2>/dev/null || echo 0)\"; echo \"archive_payload_size_bytes=$SIZE\" >> \"$REPORT\"; if [ \"$SIZE\" -le 128 ]; then echo \"backup=BAD reason=archive_too_small\" >> \"$REPORT\"; BAD=$((BAD+1)); rm -rf \"$STAGE\"; else sha256sum \"$PAYLOAD/local/${APP}-data.tar.gz\" > \"$PAYLOAD/local/${APP}-data.tar.gz.sha256\"; jq -n --arg type \"extended-appbackup\" --arg app \"$APP\" --arg host \"$HOST\" --arg source_note \"$NOTE\" --arg archive_payload_size_bytes \"$SIZE\" --arg env_files_read \"no\" --arg secret_values_printed \"no\" --arg consistency \"$CONS\" \"{type:\\$type,app_name:\\$app,host:\\$host,source_note:\\$source_note,archive_payload_size_bytes:(\\$archive_payload_size_bytes|tonumber),env_files_read:\\$env_files_read,secret_values_printed:\\$secret_values_printed,consistency:\\$consistency}\" > \"$STAGE/metadata.json\"; tar -C \"$PAYLOAD\" -czf \"$STAGE/$ARCHIVE\" .; rm -rf \"$PAYLOAD\"; JOB_ID=\"$JOB_ID\" homelab-stage-2cloud-flat \"$APP\" \"$STAGE\" \"$ARCHIVE\" > \"$LOG\" 2>&1; RC=$?; if [ \"$RC\" -eq 0 ]; then echo \"backup=OK\" >> \"$REPORT\"; grep -E \"^\\[STEP\\]|^\\[SUCCESS\\]\" \"$LOG\" >> \"$REPORT\"; appbackupctl restore-check \"$APP\" >> \"$REPORT\" 2>&1 && echo \"restore_check=OK\" >> \"$REPORT\" || echo \"restore_check=WARN_non_registry\" >> \"$REPORT\"; OK=$((OK+1)); else echo \"backup=BAD rc=$RC\" >> \"$REPORT\"; grep -E \"^\\[ERROR\\]|\\[STEP\\]|ERROR|error|failed|Failed|required|missing|denied|forbidden|checksum|rclone|copy|mkdir|jq|tar\" \"$LOG\" | tail -n 60 >> \"$REPORT\"; BAD=$((BAD+1)); fi; fi; else echo \"tar=BAD rc=$TAR_RC\" >> \"$REPORT\"; grep -E \"ERROR|error|failed|Failed|denied|No such|Cannot|tar:\" \"$LOG\" | tail -n 40 >> \"$REPORT\"; BAD=$((BAD+1)); rm -rf \"$STAGE\"; fi; echo >> \"$REPORT\"; }\n/usr/local/sbin/homelab-extended-appbackup-run:21:run_one healthchecks \"$MON\" \"var/lib/healthchecks opt/stacks/healthchecks/local_settings.py\" \"healthchecks data plus local settings\" \"emergency_data_config_archive\"\n/usr/local/sbin/homelab-mkdocs-auto-refresh:53: rm -f /tmp/homelab-mkdocs-auto-refresh-git-fetch.out /tmp/homelab-mkdocs-auto-refresh-git-fetch.err\n/usr/local/sbin/homelab-mkdocs-auto-refresh:57:rm -f /tmp/homelab-mkdocs-auto-refresh-git-fetch.out /tmp/homelab-mkdocs-auto-refresh-git-fetch.err\n/usr/local/sbin/homelab-mkdocs-auto-refresh:77:ssh \"${EDGE_SSH_OPTS[@]}\" \"$EDGE_HOST\" \"rm -rf '$STAGE' && mkdir -p '$STAGE'\"\n/usr/local/sbin/homelab-mkdocs-auto-refresh:85:rm -rf \"$BASE/site.new\" \"$BASE/site.old\"\n/usr/local/sbin/homelab-mkdocs-auto-refresh:93:rm -rf \"$BASE/site.old\" \"$STAGE\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:30:CHKS=\"$WORK/checksums.txt\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:42: F=\"$(find /var/lib/vz/dump -maxdepth 1 -type f \\( -name \"vzdump-qemu-${id}-*.vma.zst\" -o -name \"vzdump-lxc-${id}-*.tar.zst\" \\) -printf \"%T@ %p\\n\" 2>/dev/null | sort -n | tail -n 1 | cut -d\" \" -f2-)\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:53: echo \"$BN\" | grep -q \"^vzdump-qemu-\" && KIND=\"qemu\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:54: echo \"$BN\" | grep -q \"^vzdump-lxc-\" && KIND=\"lxc\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:64: rm -rf \"$CDIR\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:117: rclone mkdir \"$DEST/files\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:122: rclone mkdir \"$DEST/files/$bn.chunks\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:126: rclone copyto \"$part\" \"$DEST/files/$bn.chunks/$pn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:128: rclone copyto \"$CDIR/chunks.sha256\" \"$DEST/files/$bn.chunks/chunks.sha256\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:131: rclone copyto \"$path\" \"$DEST/files/$bn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:135: rclone copyto \"$CHKS\" \"$DEST/checksums.txt\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:136: rclone copyto \"$FILES_TSV\" \"$DEST/files.tsv\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:137: rclone copyto \"$MAN\" \"$DEST/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:144: rm -rf \"$VBASE\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:146: rclone copyto \"$DEST/manifest.json\" \"$VBASE/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:154: rclone copy \"$DEST/files/$bn.chunks\" \"$VDIR\" --filter \"+ part-*\" --filter \"+ chunks.sha256\" --filter \"- *\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:163: rm -rf \"$VDIR\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:166: rclone copyto \"$DEST/files/$bn\" \"$VBASE/$bn\" --transfers 1 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:168: rm -f \"$VBASE/$bn\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:171: rm -rf \"$VBASE\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:184:rm -rf \"$VERIFY_ROOT/$JOB_ID\"\n/usr/local/sbin/homelab-vm-local-dumps-2cloud:185:rm -rf \"$CHUNK_ROOT/$JOB_ID\"\n/usr/local/sbin/homelab-gitea-secondary-backup:18: rm -f \"$TMP\"\n/usr/local/sbin/homelab-gitea-secondary-backup:31:docker exec -u git -w /tmp gitea sh -lc \\\"rm -f /tmp/\\$NAME && gitea dump -c /data/gitea/conf/app.ini -f /tmp/\\$NAME\\\"\n/usr/local/sbin/homelab-gitea-secondary-backup:33:docker exec gitea rm -f \\\"/tmp/\\$NAME\\\"\n/usr/local/sbin/homelab-gitea-secondary-backup:100: rm -f -- \"$FILE\" \"$FILE.sha256\"\n/usr/local/sbin/homelab-gitea-secondary-backup:123: rm -f -- \"$FILE\" \"$FILE.sha256\"\n/usr/local/sbin/homelab-gitea-secondary-backup:136: rm -f -- \"$FILE\" \"$FILE.sha256\"\n/usr/local/sbin/homelab-rclone-copy-chunked:7:TMP_BASE=${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\n/usr/local/sbin/homelab-rclone-copy-chunked:13:cleanup(){ rm -rf \"$WORK\"; }\n/usr/local/sbin/homelab-rclone-copy-chunked:24:rclone purge \"${DEST}.parts\" >/dev/null 2>&1 || true\n/usr/local/sbin/homelab-rclone-copy-chunked:25:rclone mkdir \"${DEST}.parts\"\n/usr/local/sbin/homelab-rclone-copy-chunked:26:rclone copy \"$PARTS\" \"${DEST}.parts\" --transfers 2 --checkers 4 --retries 5 --low-level-retries 20 --stats 30s --log-level INFO\n/usr/local/sbin/homelab-rclone-copy-chunked:27:rclone copyto \"$WORK/${NAME}.sha256\" \"${DEST}.sha256\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO\n/usr/local/sbin/homelab-rclone-copy-chunked:28:rclone copyto \"$WORK/MANIFEST.txt\" \"${DEST}.manifest.txt\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO\n/usr/local/sbin/homelab-rclone-copy-chunked:29:rclone copyto \"$WORK/MANIFEST.parts.txt\" \"${DEST}.parts/MANIFEST.parts.txt\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO\n/usr/local/sbin/homelab-rclone-copy-chunked:30:REMOTE_COUNT=$(rclone lsf \"${DEST}.parts\" --files-only | grep -c \"^part-\" || true)\n/usr/local/sbin/homelab-rclone-copy-chunked:32:ACTUAL=$(while read -r PART SIZE_PART; do case \"$PART\" in part-*) rclone cat \"${DEST}.parts/$PART\" ;; esac; done < \"$WORK/MANIFEST.parts.txt\" | sha256sum | cut -d \" \" -f 1)\n/usr/local/sbin/homelab-router-hard-power-cycle:12:case \"$MODE\" in check|cycle) ;; *) echo HARD_WRAPPER_MODE_INVALID=yes; exit 64;; esac\n/usr/local/sbin/homelab-router-hard-power-cycle:23:SAFE_OUT=\"$(printf '%s\\n' \"$OUT\" | sed -E '/(PASSWORD|TOKEN|SECRET|PRIVATE_KEY|ROOT_PASSWORD|ENCRYPTION_KEY|COOKIE|SESSION|AUTHORIZATION|API_KEY|LOCAL_KEY|rclone[.]conf|[.]env)/Id')\"\n/usr/local/sbin/homelab-health-metrics-write:10:rm -f \"$tmp\"\n/usr/local/sbin/homelab-rclone-cat-flat-chunked:5:trap \"rm -f \\\"$TMP\\\"\" EXIT\n/usr/local/sbin/homelab-rclone-cat-flat-chunked:6:rclone cat \"${DEST}.parts.txt\" > \"$TMP\"\n/usr/local/sbin/homelab-rclone-cat-flat-chunked:8:while read -r PART PART_SIZE; do [[ \"$PART\" =~ ^part-[0-9]{4}$ && \"$PART_SIZE\" =~ ^[0-9]+$ ]] || { echo \"BAD FLAT PART MANIFEST\" >&2; exit 2; }; rclone cat \"${DEST}.${PART}\"; COUNT=$((COUNT+1)); done < \"$TMP\"\n/usr/local/sbin/homelab-vm-backup-policy:29:echo \"## Latest local vzdump catalog\" >> \"$REPORT\"\n/usr/local/sbin/homelab-vm-backup-policy:31:for f in $(find /var/lib/vz/dump -maxdepth 1 -type f \\( -name \"vzdump-qemu-*.vma.zst\" -o -name \"vzdump-lxc-*.tar.zst\" \\) 2>/dev/null | sort); do b=\"$(basename \"$f\")\"; size=\"$(stat -c %s \"$f\" 2>/dev/null || echo 0)\"; mt=\"$(date -u -d \"@$(stat -c %Y \"$f\" 2>/dev/null || echo 0)\" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)\"; if echo \"$b\" | grep -q \"^vzdump-qemu-\"; then kind=qemu; vmid=\"$(echo \"$b\" | sed -n \"s/^vzdump-qemu-\\([0-9]\\+\\)-.*/\\1/p\")\"; name=\"$(qm config \"$vmid\" 2>/dev/null | awk -F\": \" \"/^name:/{print \\$2; exit}\")\"; else kind=lxc; vmid=\"$(echo \"$b\" | sed -n \"s/^vzdump-lxc-\\([0-9]\\+\\)-.*/\\1/p\")\"; name=\"$(pct config \"$vmid\" 2>/dev/null | awk -F\": \" \"/^hostname:/{print \\$2; exit}\")\"; fi; printf \"%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n\" \"$kind\" \"$vmid\" \"${name:-unknown}\" \"$size\" \"$mt\" \"$f\" >> \"$CATALOG\"; done\n/usr/local/sbin/homelab-vm-backup-policy:37:echo \"- Existing local dumps can be uploaded/verified to two clouds without creating new vzdump archives.\" >> \"$REPORT\"\n/usr/local/sbin/homelab-vm-backup-policy:38:echo \"- New full vzdump for VM 170/171 is not started automatically because each VM has 120 GB bootdisk and /var/tmp has limited free space.\" >> \"$REPORT\"\n/usr/local/sbin/homelab-vm-backup-policy:42:echo \"## Coverage checks\" >> \"$REPORT\"\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:4:cat /var/lib/skladchik-reports-monitor-backup/restore-check-health.txt 2>/dev/null || true\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:7:systemctl is-enabled skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/enabled=/' || true\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:8:systemctl is-active skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/active=/' || true\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:9:systemctl list-timers --all | grep skladchik-reports-monitor-backup-restore-check || true\n/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:12:tail -n 80 /var/log/skladchik-reports-monitor-backup/restore-check.log 2>/dev/null || true\n/usr/local/sbin/homelab-context-refresh-direct:5:trap \"rm -rf \\\"$d\\\"\" EXIT\n/usr/local/sbin/homelab-context-refresh-direct:32:printf \"{\\\"schema_version\\\":1,\\\"channel\\\":\\\"homelab-runtime\\\",\\\"command_id\\\":\\\"%s\\\",\\\"status\\\":\\\"OK\\\",\\\"rc\\\":0,\\\"host\\\":\\\"pve01\\\",\\\"mode\\\":\\\"read-only\\\",\\\"component\\\":\\\"cluster-context\\\",\\\"started_at_utc\\\":\\\"%s\\\",\\\"finished_at_utc\\\":\\\"%s\\\",\\\"reference_register_checked\\\":true,\\\"reference_sha256\\\":\\\"%s\\\",\\\"error_register_checked\\\":true,\\\"error_register_sha256\\\":\\\"%s\\\",\\\"changes_made\\\":false,\\\"sanitized\\\":true,\\\"secrets_included\\\":false,\\\"private_addresses_included\\\":false,\\\"output\\\":\\\"AUTOMATIC_CLUSTER_CONTEXT_REFRESH=OK\\\"}\\n\" \"$id\" \"$started\" \"$finished\" \"$ref\" \"$err\" >\"$d/$id.json\"\n/usr/local/sbin/homelab-context-refresh-direct:34:ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] debian@[PRIVATE_IP] \"sudo -n /usr/local/sbin/homelab-runtime-receive $id /tmp/$id.json /tmp/$id.txt && rm -f /tmp/$id.json /tmp/$id.txt\"\n/usr/local/sbin/homelab-evidence-seal:16: rm -f \"$TMP\"\n/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:15:DUMP_SIZE=\"$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 \"$REMOTE\" \"sudo -n rm -f \\\"$REMOTE_DUMP\\\"; sudo -n -u clusteradmin pg_dump --format=custom --no-owner --no-privileges --file=\\\"$REMOTE_DUMP\\\" clusteradmin; sudo -n test -s \\\"$REMOTE_DUMP\\\"; sudo -n -u clusteradmin pg_restore --list \\\"$REMOTE_DUMP\\\" >/dev/null; sudo -n chown debian:debian \\\"$REMOTE_DUMP\\\"; sudo -n chmod 600 \\\"$REMOTE_DUMP\\\"; stat -c %s \\\"$REMOTE_DUMP\\\"\")\"\n/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:18:ssh -n -o BatchMode=yes -o ConnectTimeout=8 \"$REMOTE\" \"rm -f \\\"$REMOTE_DUMP\\\"\"\n/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:23:rm -f \"$STAGE/runtime.tgz\"\n/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:25:jq -n --arg app \"$APP\" --arg host \"cluster-admin\" --arg engine \"postgresql-custom-dump+files\" --argjson files \"$FILES\" --argjson dump_size \"$DUMP_SIZE\" \"{app:\\$app,engine:\\$engine,source_host:\\$host,source_paths:[\\\"cluster-admin:/opt/cluster-admin-incident-engine\\\",\\\"cluster-admin:/etc/cluster-admin-incident-engine\\\",\\\"cluster-admin:/var/lib/cluster-admin-incident-engine\\\",\\\"cluster-admin:PostgreSQL clusteradmin\\\"],file_count:\\$files,database_dump_size_bytes:\\$dump_size,database_dump_format:\\\"custom\\\",database_restore_list_check:\\\"ok\\\",includes_runtime_credentials:true,note:\\\"Incident engine application database state and runtime configuration\\\"}\" > \"$STAGE/metadata.json\"\n/usr/local/sbin/appbackup-retention:32:trap \"rm -rf \\\"$WORK\\\"\" EXIT\n/usr/local/sbin/appbackup-retention:35: rclone lsf \"${REMOTE%/}\" --dirs-only --max-depth 1 > \"$ROOT_LIST\"\n/usr/local/sbin/appbackup-retention:36: rclone lsf \"${REMOTE%/}/${NODE}/apps/${CANONICAL}\" --dirs-only --max-depth 1 > \"$LEGACY_LIST\"\n/usr/local/sbin/appbackup-retention:39: rclone lsf \"${REMOTE%/}/hb\" --dirs-only --max-depth 1 > \"$HB_LIST\"\n/usr/local/sbin/appbackup-retention:40: if grep -Fxq \"a/\" \"$HB_LIST\"; then rclone lsf \"${REMOTE%/}/hb/a\" --dirs-only --max-depth 1 > \"$NEW_LIST\"; fi\n/usr/local/sbin/appbackup-retention:73: rclone cat \"${REMOTE%/}/${LATEST_REL}/manifest.json\" | jq -e --arg job \"$LATEST\" \".job_id == \\$job\" >/dev/null || { echo \"REFUSE latest missing/bad remote_index=$INDEX\"; exit 5; }\n/usr/local/sbin/appbackup-retention:85: if [[ \"$MODE\" == apply ]]; then echo \"DELETE schema=dual path=$REL\"; rclone purge \"$TARGET\"; else echo \"WOULD_DELETE schema=dual path=$REL\"; fi\n/usr/local/sbin/homelab-change-transaction:160: check=False,\n/usr/local/sbin/homelab-change-transaction:195: latest_tmp.unlink(missing_ok=True)\n/usr/local/sbin/homelab-change-transaction:421: \" homelab-change-transaction run PLAN --confirm PLAN\"\n/usr/local/sbin/homelab-check-private-vpn-hosts:15:trap 'rm -f \"$tmp\"' EXIT\n/usr/local/sbin/homelab-backup-vaultwarden:26:rm -rf \"$RESTORE\"\n/usr/local/sbin/homelab-backup-vaultwarden:32:INTEGRITY=\"$(sqlite3 \"$DB\" 'PRAGMA integrity_check;')\"\n/usr/local/sbin/homelab-sops-edge-secret-coverage:97: env = os.environ.copy()\n/usr/local/sbin/homelab-sops-edge-secret-coverage:113: env=os.environ.copy(),\n/usr/local/sbin/homelab-sops-edge-secret-coverage:176: checked_at = subprocess.check_output([\"date\", \"-u\", \"+%Y-%m-%dT%H:%M:%SZ\"], text=True).strip()\n/usr/local/sbin/homelab-sops-edge-secret-coverage:242: f\"CHECKED_AT_UTC={checked_at}\",\n/usr/local/sbin/homelab-health-metrics:16:category_for_check(){\n/usr/local/sbin/homelab-health-metrics:22: *backup*|*vzdump*) printf \"%s\" \"backup\" ;;\n/usr/local/sbin/homelab-health-metrics:31:tier_for_check(){\n/usr/local/sbin/homelab-health-metrics:34: npmplus-kuma-config-*|authentik-*|gitea-*|vaultwarden-*|netbox-*|mail-cloud-restore-drill|edge-vm-vzdump-*|mkdocs-runtime-publish|npmplus-certificate-expiry) printf \"%s\" \"P0\" ;;\n/usr/local/sbin/homelab-health-metrics:69: check=\"${base%.txt}\"\n/usr/local/sbin/homelab-health-metrics:70: category=\"$(category_for_check \"$check\")\"\n/usr/local/sbin/homelab-health-metrics:71: tier=\"$(tier_for_check \"$check\")\"\n/usr/local/sbin/homelab-health-metrics:75: check_esc=\"$(esc_label \"$check\")\"\n/usr/local/sbin/homelab-health-metrics:79: labels=\"host=\\\"$host_esc\\\",check=\\\"$check_esc\\\",category=\\\"$cat_esc\\\",tier=\\\"$tier_esc\\\",file=\\\"$file_esc\\\"\"\n/usr/local/sbin/homelab-backup-health-evidence:8:backupctl check >/tmp/backup-health-emergency.log 2>&1 && EMERGENCY_OK=1 || true\n/usr/local/sbin/homelab-backup-health-evidence:9:appbackupctl check >/tmp/backup-health-appframework.log 2>&1 && APPS_OK=1 || true\n/usr/local/sbin/homelab-drift-check:7:HEALTH=/var/lib/homelab-health/drift-check.txt\n/usr/local/sbin/homelab-drift-check:24:echo \"STATUS=$STATUS TS=$(date -u +%Y-%m-%dT%H:%M:%SZ) TYPE=drift-check NOTE=$NOTE DIFF=$DIFF BASE=$BASE CURRENT=$CUR\" > \"$HEALTH\"\n/usr/local/sbin/crypto-portfolio-chart-health-check:5:trap 'rm -f \"$TMP\"' EXIT\n/usr/local/sbin/homelab-rclone-copy-flat-chunked:6:TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\"\n/usr/local/sbin/homelab-rclone-copy-flat-chunked:12:trap \"rm -rf \\\"$WORK\\\"\" EXIT\n/usr/local/sbin/homelab-rclone-copy-flat-chunked:23:while read -r PART PART_SIZE; do [[ \"$PART\" =~ ^part-[0-9]{4}$ && \"$PART_SIZE\" =~ ^[0-9]+$ ]] || { echo \"BAD PART MANIFEST\" >&2; exit 4; }; rclone copyto \"$WORK/parts/$PART\" \"${DEST}.${PART}\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR; REMOTE_BYTES=\"$(rclone size --json \"${DEST}.${PART}\" | jq -r \".bytes // -1\")\"; [[ \"$REMOTE_BYTES\" = \"$PART_SIZE\" ]] || { echo \"REMOTE PART SIZE MISMATCH $PART\" >&2; exit 5; }; UPLOADED=$((UPLOADED+1)); done < \"$WORK/MANIFEST.parts.txt\"\n/usr/local/sbin/homelab-rclone-copy-flat-chunked:25:rclone copyto \"$WORK/${NAME}.sha256\" \"${DEST}.sha256\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-rclone-copy-flat-chunked:26:rclone copyto \"$WORK/MANIFEST.txt\" \"${DEST}.manifest.txt\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-rclone-copy-flat-chunked:27:rclone copyto \"$WORK/MANIFEST.parts.txt\" \"${DEST}.parts.txt\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n/usr/local/sbin/homelab-rclone-copy-flat-chunked:28:ACTUAL=\"$(homelab-rclone-cat-flat-chunked \"$DEST\" | sha256sum | awk \"{print \\$1}\")\"\n/usr/local/sbin/homelab-final-readiness-gate:10:check(){ name=\"$1\"; shift; echo \"### $name\" >> \"$REPORT\"; timeout \"${CHECK_TIMEOUT_SECONDS:-300}\" \"$@\" >> \"$REPORT\" 2>&1; rc=$?; if [ \"$rc\" -eq 0 ]; then echo \"PASS $name\" >> \"$REPORT\"; PASS=$((PASS+1)); else echo \"FAIL $name rc=$rc\" >> \"$REPORT\"; FAIL=$((FAIL+1)); BAD=$((BAD+1)); fi; echo \"\" >> \"$REPORT\"; }\n/usr/local/sbin/homelab-final-readiness-gate:16:check \"private overall endpoint\" bash -c \"curl -fsS http://[PRIVATE_IP]:9101/overall.txt | grep -q STATUS=\"\n/usr/local/sbin/homelab-final-readiness-gate:17:check \"public overall endpoint\" bash -c \"curl -k -fsS https://backup.gram1.ru/overall.txt | grep -q STATUS=\"\n/usr/local/sbin/homelab-final-readiness-gate:18:check \"backup framework evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/backup-framework.txt && grep -q SOURCE_STATUS=OK /var/lib/homelab-health/backup-framework.txt && grep -q FRESHNESS=FRESH /var/lib/homelab-health/backup-framework.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:19:check \"control-plane restore-check\" appbackupctl restore-check safety-net-control-plane\n/usr/local/sbin/homelab-final-readiness-gate:20:check \"uptime-kuma restore-check\" appbackupctl restore-check uptime-kuma\n/usr/local/sbin/homelab-final-readiness-gate:21:check \"backup SLA evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/backup-sla.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:22:check \"backup coverage matrix evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/backup-coverage-matrix.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:23:check \"extended appbackup evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/extended-appbackup.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:24:check \"residual review\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/residual-review.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:25:check \"forum snuffleupagus\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/forum-snuffleupagus.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:26:check \"vm170 vm171 full strategy\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/vm170-vm171-full-strategy.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:27:check \"external probe vps\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/external-probe-vps.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:28:check \"netbird vps trust\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/netbird-vps-trust.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:29:check \"netbird vps trust closure\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/netbird-vps-trust-closure.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:30:check \"external probe runner decision\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/external-probe-runner-decision.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:31:check \"netbird vps trust clean seal\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:32:check \"live tail audit\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/live-tail-audit.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:34:check \"duty admin v2\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/duty-admin-v2.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:35:check \"cluster admin observer\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-observer.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:36:check \"cluster admin restricted probes\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-restricted-probes.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:37:check \"cluster admin full observer\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-full-observer.txt && grep -q BAD=0 /var/lib/homelab-health/cluster-admin-full-observer.txt && grep -q WARN=0 /var/lib/homelab-health/cluster-admin-full-observer.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:38:check \"cluster admin VM Mail Cloud backup\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt && grep -q BAD=0 /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt && grep -q MAIL_CLOUD_2COPY_VERIFY_OK /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:39:check \"cluster admin webpanel sync\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-webpanel-sync.txt && grep -q BAD=0 /var/lib/homelab-health/cluster-admin-webpanel-sync.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:40:check \"cluster admin webpanel\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-webpanel.txt && grep -q BAD=0 /var/lib/homelab-health/cluster-admin-webpanel.txt && curl -fsS --max-time 5 http://[PRIVATE_IP]:8080/status.txt | grep -q STATUS=OK\"\n/usr/local/sbin/homelab-final-readiness-gate:41:check \"node loss readiness\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/node-loss-readiness.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:42:check \"node loss runbooks\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/node-loss-runbooks.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:43:check \"power loss readiness\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/power-loss-readiness.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:44:check \"power loss runbook\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/power-loss-runbook.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:45:check \"ungated health backlog\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/ungated-health-backlog.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:46:check \"pve03 root cleanup\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/pve03-root-cleanup.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:47:check \"pve03 staging retention\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/pve03-staging-retention.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:48:check \"pve03 staging retention proof\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/pve03-staging-retention-proof.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:49:check \"pve03 staging retention cleanup\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/pve03-staging-retention-cleanup.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:50:check \"pve03 failed unit cleanup\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/pve03-failed-unit-cleanup.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:51:check \"remote git sops age readiness\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/remote-git-sops-age-readiness.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:52:check \"remote git sops age policy\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/remote-git-sops-age-policy.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:53:check \"desired state remote target trace\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/desired-state-remote-target-trace.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:54:check \"desired state remote target\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/desired-state-remote-target.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:55:check \"Kuma monitor policy evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/kuma-monitor-policy.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:56:check \"safe autoheal evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/safe-autoheal.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:57:check \"desired-state clean evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/desired-state.txt && test -z \"$(git -C /srv/homelab-desired-state status --short)\"\"\n/usr/local/sbin/homelab-final-readiness-gate:58:check \"runbooks evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/runbooks.txt && test -s /var/lib/homelab-runbooks/INDEX.md\"\n/usr/local/sbin/homelab-final-readiness-gate:59:check \"incident journal evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/incident-journal.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:60:check \"alerting evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/alerting.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:61:check \"VM backup health evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/vm-backup.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:62:check \"VM backup policy evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/vm-backup-policy.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:63:check \"VM local dumps cloud\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/vm-local-dumps-cloud.txt && [ $(jq -r .count /var/lib/homelab-vm-backup/latest-vm-local-dumps-cloud.json) -ge 3 ] && [ $(jq -r .chunked_count /var/lib/homelab-vm-backup/latest-vm-local-dumps-cloud.json) -ge 1 ]\"\n/usr/local/sbin/homelab-final-readiness-gate:64:check \"VM local dumps retention\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/vm-local-dumps-retention.txt && grep -q ACTIVE_LEFT=0 /var/lib/homelab-health/kuma-monitor-policy.txt || grep -q STATUS=OK /var/lib/homelab-health/vm-local-dumps-retention.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:65:check \"capacity risk evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/capacity-risk.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:66:check \"secret exposure evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/secret-exposure.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:67:check \"golden state evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/golden-state.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:68:check \"service registry evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/service-registry.txt\"\n/usr/local/sbin/homelab-final-readiness-gate:69:check \"dependency map evidence\" bash -c \"grep -q STATUS=OK /var/lib/homelab-health/dependency-map.txt\"\n/usr/local/sbin/homelab-docker-health.sh:44: echo \"containers_without_healthcheck=$NOHEALTH\"\n/usr/local/sbin/homelab-docker-health.sh:63: echo \"# HELP homelab_docker_health_container_without_healthcheck Docker containers without native healthcheck.\"\n/usr/local/sbin/homelab-docker-health.sh:64: echo \"# TYPE homelab_docker_health_container_without_healthcheck gauge\"\n/usr/local/sbin/homelab-docker-health.sh:65: echo \"homelab_docker_health_container_without_healthcheck $NOHEALTH\"\n/usr/local/sbin/homelab-backup-coverage-matrix:17:P1_SAFE_APPS=\"actual-budget homebox linkding mealie memos node-red syncthing filebrowser healthchecks ntfy vikunja homeassistant karakeep minio n8n\"\n/usr/local/sbin/homelab-backup-coverage-matrix:38: while read -r norm raw; do\n/usr/local/sbin/homelab-backup-coverage-matrix:50: while read -r norm raw; do\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:8:TARGET_BASE=full/nextcloud-vm-vzdump\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:11:SRC_GLOB=/var/lib/vz/dump/vzdump-qemu-150-*.vma.zst\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:77:P1 Mail Cloud Nextcloud VM recurring chunked vzdump upload\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:93:rclone mkdir \"$TARGET_REMOTE:$REMOTE_PARTS\" > \"$WORK/logs/mkdir.out\" 2>\"$WORK/logs/mkdir.err\" || MKDIR_RC=$?\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:95:rclone copy \"$PARTS_DIR\" \"$TARGET_REMOTE:$REMOTE_PARTS\" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > \"$WORK/logs/upload_parts.out\" 2>\"$WORK/logs/upload_parts.err\" || UPLOAD_PARTS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:97:rclone copyto \"$WORK/$BASENAME.sha256\" \"$TARGET_REMOTE:$REMOTE_SHA\" --checksum > \"$WORK/logs/upload_sha.out\" 2>\"$WORK/logs/upload_sha.err\" || UPLOAD_SHA_RC=$?\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:99:rclone copyto \"$WORK/MANIFEST.txt\" \"$TARGET_REMOTE:$REMOTE_MANIFEST\" --checksum > \"$WORK/logs/upload_manifest.out\" 2>\"$WORK/logs/upload_manifest.err\" || UPLOAD_MANIFEST_RC=$?\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:101:rclone copyto \"$WORK/SHA256SUMS.txt\" \"$TARGET_REMOTE:$REMOTE_SUMS\" --checksum > \"$WORK/logs/upload_sums.out\" 2>\"$WORK/logs/upload_sums.err\" || UPLOAD_SUMS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:104:rclone lsf \"$TARGET_REMOTE:$REMOTE_PARTS\" --format ps > \"$WORK/logs/remote_lsf_parts.txt\" 2>\"$WORK/logs/remote_lsf_parts.err\" || LSF_PARTS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:110:rclone copy \"$TARGET_REMOTE:$REMOTE_PARTS\" \"$DL_DIR/parts\" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > \"$WORK/logs/download_parts.out\" 2>\"$WORK/logs/download_parts.err\" || DOWNLOAD_PARTS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:120:rclone lsf \"$TARGET_REMOTE:$TARGET_BASE\" --dirs-only 2>/dev/null | sed 's#/$##' | sort > \"$DIR_LIST\" || RETENTION_RC=$?\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:126: [ -n \"$OLD\" ] && rclone purge \"$TARGET_REMOTE:$TARGET_BASE/$OLD\" >> \"$WORK/logs/retention.out\" 2>>\"$WORK/logs/retention.err\"\n/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:140: rm -rf \"$PARTS_DIR\" \"$DL_DIR\"\n/usr/local/sbin/homelab-rehydrate-flat-chunked-dir:10: rm -f \"$TMP_TARGET\"\n/usr/local/sbin/homelab-disk-space-health.py:10:checks = [\n/usr/local/sbin/homelab-disk-space-health.py:21:for label, cmd in checks:\n/usr/local/sbin/homelab-rclone-required-check.sh:4:RAW_FILE=\"/etc/homelab/rclone/required-remotes.txt\"\n/usr/local/sbin/homelab-rclone-required-check.sh:5:CRYPT_FILE=\"/etc/homelab/rclone/required-crypt-remotes.txt\"\n/usr/local/sbin/homelab-rclone-required-check.sh:9:echo \"=== homelab rclone required check: $(hostname) ===\"\n/usr/local/sbin/homelab-rclone-required-check.sh:12:echo \"=== installed rclone ===\"\n/usr/local/sbin/homelab-rclone-required-check.sh:13:rclone version | head -5\n/usr/local/sbin/homelab-rclone-required-check.sh:17:CONFIGURED=\"$(rclone listremotes 2>/dev/null | sed 's/:$//' || true)\"\n/usr/local/sbin/homelab-rclone-required-check.sh:19: echo \"No rclone remotes configured yet.\"\n/usr/local/sbin/homelab-legacy-monitoring-backups-run:24: rm -f \"$STAGE/remote.tgz\"\n/usr/local/sbin/homelab-legacy-monitoring-backups-run:43: rm -f \"$STAGE/pve01.tgz\"\n/usr/local/sbin/homelab-legacy-monitoring-backups-run:48: rm -f \"$STAGE/$LABEL.tgz\"\n/usr/local/sbin/homelab-legacy-monitoring-backups-run:52: rm -f \"$STAGE/monitoring.tgz\"\n/usr/local/sbin/homelab-legacy-monitoring-backups-run:71: rm -f \"$STAGE/config.tgz\"\n/usr/local/sbin/homelab-cluster-admin-inventory-sync:19:ssh -n -o BatchMode=yes -o ConnectTimeout=8 \"$REMOTE\" \"rm -f $REMOTE_TMP\"\n/usr/local/sbin/homelab-chat-run:272: re.compile(r\"(?i)\\brclone\\.conf\\b\"),\n/usr/local/sbin/homelab-chat-run:388: \"reference_register_checked\": True,\n/usr/local/sbin/homelab-chat-run:390: \"error_register_checked\": True,\n/usr/local/sbin/homelab-chat-run:489: rm -f \\\n/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:10: rm -rf \"$TMPD\" 2>/dev/null || true\n/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:19:check_host() {\n/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:39:check_host \"kuma\" \"kuma.vpn.gram1.ru\" \"^(200|302)$\"\n/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:40:check_host \"homepage\" \"homepage.vpn.gram1.ru\" \"^200$\"\n/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:41:check_host \"gitea\" \"gitea.vpn.gram1.ru\" \"^200$\"\n/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:42:check_host \"filebrowser\" \"filebrowser.vpn.gram1.ru\" \"^200$\"\n/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:43:check_host \"auth\" \"auth.vpn.gram1.ru\" \"^(200|302)$\"\n/usr/local/sbin/homelab-beszel-backup-run:33:rm -f \"$STAGE/remote.tgz\"\n/usr/local/sbin/homelab-beszel-backup-run:37:while IFS= read -r DB; do DB_COUNT=$((DB_COUNT+1)); RESULT=\"$(python3 -c \"import sqlite3,sys;print(sqlite3.connect(sys.argv[1]).execute(\\\"PRAGMA integrity_check\\\").fetchone()[0])\" \"$DB\" 2>/dev/null || echo error)\"; [ \"$RESULT\" = ok ] || exit 4; DB_OK=$((DB_OK+1)); done < <(find \"$STAGE/payload\" -type f \\( -name \"*.db\" -o -name \"*.sqlite\" -o -name \"*.sqlite3\" \\) 2>/dev/null)\n/usr/local/sbin/homelab-runbook-generate:40: f\"## First checks\\n\"\n/usr/local/sbin/homelab-runbook-generate:44: f\"appbackupctl restore-check {app}\\n\"\n/usr/local/sbin/homelab-runbook-generate:47: f\"Do not restore over a live service blindly. First confirm both encrypted cloud copies pass restore-check, target VM/container is snapshotted or intentionally stopped, and dependency map is OK.\\n\\n\"\n/usr/local/sbin/homelab-runbook-generate:51: f\"homelab-service-registry-check\\n\"\n/usr/local/sbin/homelab-runbook-generate:52: f\"homelab-dependency-map-check\\n\"\n/usr/local/sbin/homelab-runbook-generate:85: \"backupctl restore-check\\n\"\n/usr/local/sbin/homelab-runbook-generate:86: \"appbackupctl restore-check safety-net-control-plane\\n\"\n/usr/local/sbin/skladchik-reports-monitor-supervisor:25:PVE_RESTORE_HEALTH = Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\n/usr/local/sbin/skladchik-reports-monitor-supervisor:254: \"skladchik-reports-monitor-backup-restore-check.timer\",\n/usr/local/sbin/skladchik-reports-monitor-supervisor:301: \"Restore-check:\",\n/usr/local/sbin/skladchik-reports-monitor-supervisor:339: mode = sys.argv[1] if len(sys.argv) > 1 else \"check\"\n/usr/local/sbin/skladchik-reports-monitor-supervisor:340: if mode not in (\"check\", \"test\"):\n/usr/local/sbin/skladchik-reports-monitor-supervisor:341: print(\"usage: skladchik-reports-monitor-supervisor [check|test]\", file=sys.stderr)\n/usr/local/sbin/backup-restore-check:8:ROOT=\"/var/tmp/homelab-restore-check/${JOB}\"\n/usr/local/sbin/backup-restore-check:9:rm -rf \"$ROOT\" && mkdir -p \"$ROOT/a\" \"$ROOT/b\"\n/usr/local/sbin/backup-restore-check:10:trap 'echo \"ERROR restore-check failed; kept: \"; exit 1' ERR\n/usr/local/sbin/backup-restore-check:11:echo \"restore-check job=$JOB\"\n/usr/local/sbin/backup-restore-check:12:rclone copy \"$RA\" \"$ROOT/a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-a.log 2>&1; homelab-rehydrate-chunked-dir \"$ROOT/a\" >>/tmp/homelab-restore-a.log 2>&1\n/usr/local/sbin/backup-restore-check:13:(cd \"$ROOT/a\" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-a-sha.log && ARCHIVE=\"$(jq -r .archive_name manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n/usr/local/sbin/backup-restore-check:14:echo \"OK cloud A restore-check\"\n/usr/local/sbin/backup-restore-check:15:rclone copy \"$RB\" \"$ROOT/b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-b.log 2>&1; homelab-rehydrate-chunked-dir \"$ROOT/b\" >>/tmp/homelab-restore-b.log 2>&1\n/usr/local/sbin/backup-restore-check:16:(cd \"$ROOT/b\" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-b-sha.log && ARCHIVE=\"$(jq -r .archive_name manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n/usr/local/sbin/backup-restore-check:17:echo \"OK cloud B restore-check\"\n/usr/local/sbin/backup-restore-check:18:rm -rf \"$ROOT\"\n/usr/local/sbin/backup-restore-check:20:echo \"OK restore-check both clouds\"\n/usr/local/sbin/homelab-vps-identity-audit:8:trap 'rm -rf \"$TMP\"' EXIT\n/usr/local/sbin/homelab-vm-full-restore-drill:77: -name \"vzdump-qemu-${source_vmid}-*.vma.zst\" \\\n/usr/local/sbin/homelab-vm-full-restore-drill:152:check_inventory() {\n/usr/local/sbin/homelab-vm-full-restore-drill:184:check_container_health() {\n/usr/local/sbin/homelab-vm-full-restore-drill:224:check_http() {\n/usr/local/sbin/homelab-vm-full-restore-drill:349:check_container_health() {\n/usr/local/sbin/homelab-vm-full-restore-drill:386:check_container_health \"$test_vmid\"\n/usr/local/sbin/homelab-vm-full-restore-drill:389: check_inventory \"$source_vmid\" \"$test_vmid\"\n/usr/local/sbin/homelab-vm-full-restore-drill:392: check_http \"$source_vmid\" \"$test_vmid\"\n/usr/local/sbin/homelab-vm-backup-health:8:check_file(){\n/usr/local/sbin/homelab-vm-backup-health:15:check_file /var/lib/homelab-health/backup.txt auto-vmct\n/usr/local/sbin/homelab-vm-backup-health:16:check_file /var/lib/homelab-health/edge-vm-vzdump-backup.txt edge-vm-backup\n/usr/local/sbin/homelab-vm-backup-health:17:check_file /var/lib/homelab-health/edge-vm-vzdump-offhost.txt edge-vm-offhost\n/usr/local/sbin/homelab-vm-backup-health:18:check_file /var/lib/homelab-health/edge-vm-vzdump-restore.txt edge-vm-restore\n/usr/local/sbin/homelab-ingress-failover:37: rm -f \"$RESOURCES\"\n/usr/local/sbin/homelab-ingress-failover:252: rm -f \"$tmp\"\n/usr/local/sbin/homelab-overall-health:8: /var/lib/homelab-health/drift-check.txt\n/usr/local/sbin/homelab-backup-matrix-refresh:10: p=subprocess.run(args,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True,check=False)\n/usr/local/sbin/homelab-backup-matrix-refresh:65: shutil.copy2(source_path,temporary)\n/usr/local/sbin/homelab-vm-full-backup:11:# shellcheck disable=SC1090\n/usr/local/sbin/homelab-vm-full-backup:92:for cmd in vzdump qm zstd sha256sum rclone rsync ssh flock; do\n/usr/local/sbin/homelab-vm-full-backup:130: -name \"vzdump-qemu-${VMID}-*.vma.zst\" \\\n/usr/local/sbin/homelab-vm-full-backup:138: vzdump \"$VMID\" \\\n/usr/local/sbin/homelab-vm-full-backup:145: --remove 0\n/usr/local/sbin/homelab-vm-full-backup:151: -name \"vzdump-qemu-${VMID}-*.vma.zst\" \\\n/usr/local/sbin/homelab-vm-full-backup:261: CHUNK_TMP_BASE=\"$BACKUP_DIR/.rclone-chunks\" homelab-rclone-copy-chunked \"$ARCHIVE\" \"${DEST_ROOT}/${NAME}\"\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:10:TARGET_BASE=full/edge-vm-vzdump\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:64:case \"$SOURCE\" in /mnt/staging/offhost/edge-vm-vzdump-from-pve03/vzdump-qemu-130-*.vma.zst) ;; *) fail source_path_bad 2 ;; esac\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:111:rclone about \"$TARGET_REMOTE:\" > \"$WORK/logs/about.out\" 2>\"$WORK/logs/about.err\" || ABOUT_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:114:DIRS=\"$(rclone lsf \"$TARGET_REMOTE:$TARGET_BASE\" --dirs-only 2>/dev/null | sed 's#/$##' | sort || true)\"\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:116: if rclone cat \"$TARGET_REMOTE:$TARGET_BASE/$D/$BASENAME.sha256\" 2>/dev/null | grep -q \"$SHA\"; then\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:172:P1 Mail Cloud edge-vm recurring chunked vzdump upload\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:188:rclone mkdir \"$TARGET_REMOTE:$REMOTE_PARTS\" > \"$WORK/logs/mkdir.out\" 2>\"$WORK/logs/mkdir.err\" || MKDIR_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:190:rclone copy \"$PARTS_DIR\" \"$TARGET_REMOTE:$REMOTE_PARTS\" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > \"$WORK/logs/upload_parts.out\" 2>\"$WORK/logs/upload_parts.err\" || UPLOAD_PARTS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:192:rclone copyto \"$WORK/$BASENAME.sha256\" \"$TARGET_REMOTE:$REMOTE_SHA\" --checksum > \"$WORK/logs/upload_sha.out\" 2>\"$WORK/logs/upload_sha.err\" || UPLOAD_SHA_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:194:rclone copyto \"$WORK/MANIFEST.txt\" \"$TARGET_REMOTE:$REMOTE_MANIFEST\" --checksum > \"$WORK/logs/upload_manifest.out\" 2>\"$WORK/logs/upload_manifest.err\" || UPLOAD_MANIFEST_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:196:rclone copyto \"$WORK/SHA256SUMS.txt\" \"$TARGET_REMOTE:$REMOTE_SUMS\" --checksum > \"$WORK/logs/upload_sums.out\" 2>\"$WORK/logs/upload_sums.err\" || UPLOAD_SUMS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:199:rclone size \"$TARGET_REMOTE:$TARGET_DIR\" --json > \"$WORK/logs/remote_size.json\" 2>\"$WORK/logs/remote_size.err\" || SIZE_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:201:rclone lsf \"$TARGET_REMOTE:$REMOTE_PARTS\" --format \"ps\" > \"$WORK/logs/remote_lsf_parts.txt\" 2>\"$WORK/logs/remote_lsf_parts.err\" || LSF_PARTS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:207:rclone copy \"$TARGET_REMOTE:$REMOTE_PARTS\" \"$DL_DIR/parts\" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > \"$WORK/logs/download_parts.out\" 2>\"$WORK/logs/download_parts.err\" || DOWNLOAD_PARTS_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:222: rm -rf \"$DL_DIR\" \"$PARTS_DIR\"\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:234: rclone lsf \"$TARGET_REMOTE:$TARGET_BASE\" --dirs-only 2>/dev/null | sed \"s#/$##\" | sort > \"$DIR_LIST\" || RETENTION_RC=$?\n/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:239: [ -n \"$OLD\" ] && rclone purge \"$TARGET_REMOTE:$TARGET_BASE/$OLD\" >> \"$WORK/logs/retention.out\" 2>>\"$WORK/logs/retention.err\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:7:BASE_DIR=\"/mnt/staging/vzdump/edge-vm-vzdump\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:8:OFFHOST_DIR=\"/mnt/staging/offhost/edge-vm-vzdump-from-pve03\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:40:REMOTE_OUT=\"$(remote \"OUT=/tmp/edge_vm_vzdump_${VMID}_${TS}.out; ERR=/tmp/edge_vm_vzdump_${VMID}_${TS}.err; : > \\$OUT; : > \\$ERR; vzdump '$VMID' --mode snapshot --compress zstd --dumpdir '$RUN_DIR' --remove 0 >\\$OUT 2>\\$ERR; RC=\\$?; ARCHIVE=\\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\\n' | sort -n | tail -n1 | cut -d' ' -f2-); LOGFILE=\\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.log' -printf '%T@ %p\\n' | sort -n | tail -n1 | cut -d' ' -f2-); SIZE=0; SHA=none; if [ -n \\\"\\$ARCHIVE\\\" ] && [ -f \\\"\\$ARCHIVE\\\" ]; then SIZE=\\$(stat -c %s \\\"\\$ARCHIVE\\\"); SHA=\\$(sha256sum \\\"\\$ARCHIVE\\\" | awk '{print \\$1}'); fi; echo VZDUMP_RC=\\$RC; echo ARCHIVE=\\$ARCHIVE; echo LOGFILE=\\$LOGFILE; echo SIZE=\\$SIZE; echo SHA=\\$SHA; echo OUT_TAIL=\\$(tail -n 6 \\$OUT | tr '\\n' '|'); echo ERR_TAIL=\\$(tail -n 6 \\$ERR | tr '\\n' '|')\")\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:62:dest \"mkdir -p '$OFFHOST_DIR' && rm -f '$DEST_PATH.tmp' '$DEST_PATH.sha256'\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:74:echo \"STATUS=OK TS=$TS TYPE=edge-vm-vzdump-backup NODE=$NODE VMID=$VMID ARCHIVE=$ARCHIVE SIZE=$SIZE SHA256=$SHA SCSI1_BACKUP0=yes NOTES=timer_run\" > \"$HEALTH_DIR/edge-vm-vzdump-backup.txt\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:75:echo \"STATUS=OK TS=$TS TYPE=edge-vm-vzdump-offhost NODE=$NODE VMID=$VMID HOST=pve02 DEST=$DEST_PATH SIZE=$SIZE SHA256=$SHA NOTES=timer_run\" > \"$HEALTH_DIR/edge-vm-vzdump-offhost.txt\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:76:echo \"STATUS=OK TS=$TS TYPE=edge-vm-vzdump-restore-verify HOST=pve02 VMID=$VMID ARCHIVE=$DEST_PATH SIZE=$SIZE SHA256=$SHA ZSTD_RC=0 VMA_RC=0 NOTES=timer_run\" > \"$HEALTH_DIR/edge-vm-vzdump-restore.txt\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:77:chmod 0644 \"$HEALTH_DIR/edge-vm-vzdump-backup.txt\" \"$HEALTH_DIR/edge-vm-vzdump-offhost.txt\" \"$HEALTH_DIR/edge-vm-vzdump-restore.txt\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:79:remote \"find '$BASE_DIR' -mindepth 1 -maxdepth 1 -type d -name 'edge-vm-*' -printf '%T@ %p\\n' | sort -nr | awk 'NR>${KEEP_LOCAL}{print \\$2}' | xargs -r rm -rf\"\n/usr/local/sbin/homelab-backup-edge-vm-vzdump:80:dest \"find '$OFFHOST_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\\n' | sort -nr | awk 'NR>${KEEP_OFFHOST}{print \\$2}' | while read f; do rm -f \\\"\\$f\\\" \\\"\\$f.sha256\\\"; done\"\n/usr/local/sbin/appbackup-dockge-stacks:25:echo \"[STEP] copy app data\"\n/usr/local/sbin/appbackup-dockge-stacks:30:rm -rf \"$PAYLOAD\"\n/usr/local/sbin/appbackup-dockge-stacks:51:echo \"[STEP] local checksum\"\n/usr/local/sbin/appbackup-dockge-stacks:52:(cd \"$STAGE\" && sha256sum \"$ARCHIVE\" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e . manifest.json >/dev/null)\n/usr/local/sbin/appbackup-dockge-stacks:54:echo \"[STEP] upload cloud A metadata\"; rclone copy \"$STAGE\" \"$RA\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n/usr/local/sbin/appbackup-dockge-stacks:55:echo \"[STEP] upload cloud A archive chunks\"; rclone deletefile \"$RA/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RA/$ARCHIVE\"\n/usr/local/sbin/appbackup-dockge-stacks:56:echo \"[STEP] upload cloud B metadata\"; rclone copy \"$STAGE\" \"$RB\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n/usr/local/sbin/appbackup-dockge-stacks:57:echo \"[STEP] upload cloud B archive chunks\"; rclone deletefile \"$RB/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RB/$ARCHIVE\"\n/usr/local/sbin/appbackup-dockge-stacks:58:echo \"[STEP] verify cloud A chunked\"; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-stage-verify-chunked \"$RA\" \"$ARCHIVE\" \"$VERIFY/a\" \"$JOB\"\n/usr/local/sbin/appbackup-dockge-stacks:59:echo \"[STEP] verify cloud B chunked\"; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-stage-verify-chunked \"$RB\" \"$ARCHIVE\" \"$VERIFY/b\" \"$JOB\"\n/usr/local/sbin/appbackup-dockge-stacks:60:echo \"[STEP] upload latest index\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RAL\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RBL\"; rclone cat \"$RAL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null; rclone cat \"$RBL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n/usr/local/sbin/appbackup-dockge-stacks:76:rm -rf \"$STAGE\" \"$VERIFY\"\n/usr/local/sbin/homelab-cluster-passport:23:line \"- safety checkpoint: /root/HOMELAB_SAFETY_NET_CHECKPOINT.md\"\n/usr/local/sbin/homelab-cluster-passport:65:need_ok /var/lib/homelab-health/drift-check.txt drift-check\n/usr/local/sbin/homelab-cluster-passport:77:line \"appbackupctl restore-check safety-net-control-plane\"\n/usr/local/sbin/homelab-cluster-passport:84:line \"4. For control-plane loss: appbackupctl restore-check safety-net-control-plane, then restore the verified archive from either encrypted cloud.\"\n/usr/local/sbin/backup-retention:18: if [[ \"$MODE\" == \"apply\" ]]; then rclone cat \"$BASE/$LATEST/manifest.json\" | jq -e --arg job \"$LATEST\" \".job_id == \\$job\" >/dev/null || { echo \"REFUSE latest missing/bad on $BASE\"; exit 4; }; fi\n/usr/local/sbin/backup-retention:19: mapfile -t DIRS < <(rclone lsf \"$BASE\" --dirs-only 2>/dev/null | while read -r D; do D=\"${D%/}\"; [[ \"$D\" == homelab-${NODE}-${JOB}-* ]] && echo \"$D\"; done | sort)\n/usr/local/sbin/backup-retention:24: for ((i=0; i /var/lib/homelab-health/private-vpn-hosts.txt.tmp; rc=$?; mv /var/lib/homelab-health/private-vpn-hosts.txt.tmp /var/lib/homelab-health/private-vpn-hosts.txt; chmod 0644 /var/lib/homelab-health/private-vpn-hosts.txt; exit $rc'\n/etc/systemd/system/netbird-peers-health.service:2:Description=NetBird peers health check\n/etc/systemd/system/homelab-pve03-staging-capacity-health.timer:2:Description=Run Homelab pve03 staging capacity health check\n/etc/systemd/system/timers.target.wants/homelab-private-vpn-hosts-health.timer:2:Description=Run Homelab private VPN hosts health check every minute\n/etc/systemd/system/timers.target.wants/homelab-crypto-portfolio-chart-health.timer:2:Description=Recurring Crypto portfolio chart health check\n/etc/systemd/system/timers.target.wants/homelab-edge-vm-offhost-freshness.timer:2:Description=Run Homelab edge-vm VM130 offhost freshness sync daily after nightly vzdump\n/etc/systemd/system/timers.target.wants/homelab-dependency-map-check.timer:2:Description=Homelab dependency map check timer\n/etc/systemd/system/timers.target.wants/homelab-pve03-staging-capacity-health.timer:2:Description=Run Homelab pve03 staging capacity health check\n/etc/systemd/system/timers.target.wants/homelab-drift-check.timer:2:Description=Homelab drift check timer\n/etc/systemd/system/timers.target.wants/homelab-service-registry-check.timer:2:Description=Homelab service registry health check timer\n/etc/systemd/system/timers.target.wants/homelab-backup-healthcheck.timer:2:Description=Run homelab backup healthcheck daily\n/etc/systemd/system/timers.target.wants/homelab-backup-healthcheck.timer:8:Unit=homelab-backup-healthcheck.service\n/etc/systemd/system/timers.target.wants/netbird-peers-health.timer:2:Description=Run NetBird peers health check\n/etc/systemd/system/timers.target.wants/homelab-alerting-health.timer:2:Description=Homelab alerting health check timer\n/etc/systemd/system/timers.target.wants/skladchik-reports-monitor-backup-restore-check.timer:2:Description=Weekly restore-check of Skladchik reports monitor cloud backups\n/etc/systemd/system/timers.target.wants/skladchik-reports-monitor-backup-restore-check.timer:8:Unit=skladchik-reports-monitor-backup-restore-check.service\n/etc/systemd/system/homelab-edge-vm-vzdump-backup.timer:8:Unit=homelab-edge-vm-vzdump-backup.service\n/etc/systemd/system/homelab-drift-check.timer:2:Description=Homelab drift check timer\n/etc/systemd/system/homelab-service-registry-check.timer:2:Description=Homelab service registry health check timer\n/etc/systemd/system/homelab-pve03-staging-capacity-health.service:2:Description=Homelab pve03 staging capacity health check\n/etc/systemd/system/homelab-backup-healthcheck.timer:2:Description=Run homelab backup healthcheck daily\n/etc/systemd/system/homelab-backup-healthcheck.timer:8:Unit=homelab-backup-healthcheck.service\n/etc/systemd/system/homelab-safe-exec-health.service:2:Description=Run allowlisted homelab health checks\n/etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.service:2:Description=Homelab Mail Cloud Nextcloud VM vzdump upload\n/etc/systemd/system/hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'\n/etc/systemd/system/netbird-peers-health.timer:2:Description=Run NetBird peers health check\n/etc/systemd/system/homelab-alerting-health.timer:2:Description=Homelab alerting health check timer\n/etc/systemd/system/skladchik-reports-monitor-backup-restore-check.timer:2:Description=Weekly restore-check of Skladchik reports monitor cloud backups\n/etc/systemd/system/skladchik-reports-monitor-backup-restore-check.timer:8:Unit=skladchik-reports-monitor-backup-restore-check.service\n/etc/systemd/system/suspend-then-hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'\n/etc/cron.d/vzdump:1:# cluster wide vzdump cron schedule\n7022386770\t2026-07-18T03:31:03.6566057240\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.vma.zst\n7021952392\t2026-07-21T03:31:06.5717414950\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst\n7021297699\t2026-07-19T03:31:06.8755351020\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst\n7016275481\t2026-07-20T03:31:03.2165123470\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst\n329541843\t2026-07-21T03:30:09.7688696330\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst\n326601118\t2026-07-20T03:30:08.2096782720\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst\n323668231\t2026-07-19T03:45:46.2209130400\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst\n321121649\t2026-07-18T03:30:08.0777625900\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.tar.zst\n222777732\t2026-07-21T03:30:15.9059638290\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst\n222761273\t2026-07-20T03:30:13.7937629440\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst\n222745965\t2026-07-19T03:45:57.2700790730\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst\n222727760\t2026-07-18T03:30:13.5578457230\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.tar.zst\n3021\t2026-07-19T03:31:07.4745284120\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.log\n3020\t2026-07-21T03:31:07.3945472590\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.log\n3018\t2026-07-18T03:31:03.9197988860\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.log\n3017\t2026-07-20T03:31:04.1706719350\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.log\n1089\t2026-07-20T03:30:14.0079985330\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.log\n1086\t2026-07-20T03:30:08.4640241540\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.log\n989\t2026-07-21T03:30:16.1364976560\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.log\n989\t2026-07-18T03:30:13.7816006690\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.log\n986\t2026-07-21T03:30:10.0285327490\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.log\n986\t2026-07-18T03:30:08.3002411830\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.log\n731\t2026-07-19T03:45:57.5197000270\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.log\n729\t2026-07-19T03:45:46.4444121980\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.log\n========== NODE=pve02 ==========\n=== REMOTES ===\n=== TIMERS ===\nTue 2026-07-21 18:41:03 MSK 3h 46min Tue 2026-07-21 12:41:22 MSK 2h 12min ago forum-prod-fresh5-cloud-backup.timer forum-prod-fresh5-cloud-backup.service\nWed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:22 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service\nWed 2026-07-22 04:20:00 MSK 13h Tue 2026-07-21 04:20:12 MSK 10h ago homelab-cluster-admin-vm-mail-cloud-backup.timer homelab-cluster-admin-vm-mail-cloud-backup.service\n=== SERVICES ===\ncodevipe-cloud-backup.service static -\ndpkg-db-backup.service static -\nforum-prod-fresh5-cloud-backup.service static -\nhomelab-cluster-admin-vm-mail-cloud-backup.service static -\n/usr/local/sbin/homelab-rclone-required-check.sh:4:RAW_FILE=\"/etc/homelab/rclone/required-remotes.txt\"\n/usr/local/sbin/homelab-rclone-required-check.sh:5:CRYPT_FILE=\"/etc/homelab/rclone/required-crypt-remotes.txt\"\n/usr/local/sbin/homelab-rclone-required-check.sh:9:echo \"=== homelab rclone required check: $(hostname) ===\"\n/usr/local/sbin/homelab-rclone-required-check.sh:12:echo \"=== installed rclone ===\"\n/usr/local/sbin/homelab-rclone-required-check.sh:13:rclone version | head -5\n/usr/local/sbin/homelab-rclone-required-check.sh:17:CONFIGURED=\"$(rclone listremotes 2>/dev/null | sed 's/:$//' || true)\"\n/usr/local/sbin/homelab-rclone-required-check.sh:19: echo \"No rclone remotes configured yet.\"\n/usr/local/sbin/homelab-staging-check.sh:11:echo \"=== homelab staging check: ${HOST} ===\"\n/usr/local/sbin/routerguard-watchdog-power-entrypoint:12:case \"$MODE\" in check|cycle) ;; *) echo HARD_ENTRYPOINT_MODE_INVALID=yes; exit 64;; esac\n/usr/local/sbin/routerguard-watchdog-power-entrypoint:15:if [ \"$MODE\" = check ]; then\n/usr/local/sbin/routerguard-watchdog-power-entrypoint:16: exec \"$HELPER\" check\n/usr/local/sbin/routerguard-watchdog-power-entrypoint:21:PRE=\"$(mktemp /run/routerguard-watchdog-precheck.XXXXXX)\"\n/usr/local/sbin/routerguard-watchdog-power-entrypoint:22:cleanup(){ rm -f \"$PRE\" \"$ARM\"; }\n/usr/local/sbin/routerguard-watchdog-power-entrypoint:25:\"$HELPER\" check >\"$PRE\" 2>&1\n/usr/local/sbin/routerguard-watchdog-power-entrypoint:38:rm -f \"$ARM\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:4:CRYPT_FILE=\"/etc/homelab/rclone/required-crypt-remotes.txt\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:18: \"checksums/\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:21:echo \"=== homelab rclone crypt healthcheck: $(hostname) ===\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:40: if ! rclone listremotes | sed 's/:$//' | grep -qx \"$REMOTE\"; then\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:47: if ! rclone about \"${REMOTE}:\"; then\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:48: echo \"FAIL: rclone about failed: $REMOTE\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:54: ROOT_LIST=\"$(rclone lsf \"${REMOTE}:/\" || true)\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:76: if rclone cat \"${REMOTE}:/latest-backup-index.json\" | jq -e '.schema and .remote and .status and .items' >/dev/null; then\n/usr/local/sbin/codevipe-cloud-backup.sh:28: rclone copy \"$CACHE_ROOT/$BASE\" \"$R:$REMOTE_PATH/$BASE\" --stats 30s\n/usr/local/sbin/codevipe-cloud-backup.sh:31:find \"$CACHE_ROOT\" -mindepth 1 -maxdepth 1 -type d -mtime +14 -print -exec rm -rf {} \\;\n/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:26:rm -f \"$STAGE/$base\"\n/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:34:format=tar.gz split_parts_1500M rclone_crypt_remote\n/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:44: rclone mkdir \"$dest\" >/tmp/forum-cloud-mkdir-${r}-$TS.log 2>&1\n/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:46: rclone copy \"$STAGE\" \"$dest\" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --stats-one-line >/tmp/forum-cloud-copy-${r}-$TS.log 2>&1\n/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:49: remote_count=$(rclone lsf \"$dest\" --files-only 2>/tmp/forum-cloud-lsf-${r}-$TS.err | wc -l)\n/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:55:find /root/backups/forum-prod-fresh5 -mindepth 1 -maxdepth 1 -type d -mtime +14 -exec rm -rf {} +\n/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:73: retention_remote_count=$(rclone lsf \"$r:$REMOTE_BASE/$retention_job\" --files-only 2>/dev/null | wc -l)\n/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:78: rm -rf --one-file-system \"$retention_dir\"\n/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:83: rm -f \"$retention_keep_tmp\"\n/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:24:rclone listremotes 2>/tmp/cluster-admin-rclone-listremotes.err | grep -E 'mail.*crypt|crypt.*mail' | sed 's/:$//' | sort > /tmp/cluster-admin-mail-remotes.txt\n/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:41: rm -rf \"$STAGE/$OLD_JOB\"\n/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:43: rclone purge \"$R:homelab-backups/pve02/cluster-admin-vm180/$OLD_JOB\" >/tmp/cluster-admin-rclone-purge-$R.log 2>&1\n/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:51: ls \"$d\"/vzdump-qemu-${VMID}-*.vma.zst >/dev/null 2>&1 || { echo \"DELETE_EMPTY_LOCAL_JOB=$b\"; rm -rf \"$d\"; }\n/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:84:vzdump \"$VMID\" --mode snapshot --compress zstd --dumpdir \"$JOBDIR\" --quiet 0\n/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:89:LATEST=\"$(ls -1t \"$JOBDIR\"/vzdump-qemu-${VMID}-*.vma.zst 2>/dev/null | head -n1)\"\n/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:115: rm -f \"$TMPVMA\"\n/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:124: rm -f \"$TMPVMA\"\n/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:156: rclone mkdir \"$DEST\" >/tmp/cluster-admin-rclone-mkdir-$R.log 2>&1\n/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:158: rclone copy \"$JOBDIR\" \"$DEST\" --checksum --transfers 2 --checkers 4 --retries 3 --low-level-retries 5 >/tmp/cluster-admin-rclone-copy-$R.log 2>&1\n/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:161: rclone copyto \"$JOBDIR/MANIFEST.txt\" \"$R:homelab-backups/pve02/cluster-admin-vm180/latest/MANIFEST.txt\" --retries 3 >/tmp/cluster-admin-rclone-latest-$R.log 2>&1\n/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:164: REMOTE_SHA=\"$(rclone cat \"$DEST/$BASE.sha256\" 2>/tmp/cluster-admin-rclone-catsha-$R.err | awk '{print $1}' | head -n1)\"\n/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:177:find \"$STAGE\" -mindepth 1 -maxdepth 1 -type d -printf '%f\\n' 2>/dev/null | sort -r | tail -n +4 | while read old; do rm -rf \"$STAGE/$old\"; echo \"LOCAL_STAGE_DELETED=$old\"; done\n/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:20: rm -rf -- \"$WORK\"\n/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:29: local checked_at\n/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:31: checked_at=\"$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || printf UNKNOWN)\"\n/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:37:CHECKED_AT_UTC=$checked_at\n/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:117:for CMD in awk cat chmod date grep head mkdir mktemp rclone rm sed sha256sum sort sqlite3 tail tar wc; do\n/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:131:rclone lsf \"$REMOTE\" --recursive >\"$REMOTE_LIST\" 2>\"$ERROR_FILE\" \\\n/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:149:rclone copyto \"$REMOTE$CRITICAL_REMOTE_PATH\" \"$CRITICAL_ARCHIVE\" 2>\"$ERROR_FILE\" \\\n/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:152:rclone copyto \"$REMOTE$CRITICAL_REMOTE_SHA_PATH\" \"$CRITICAL_SHA_FILE\" 2>\"$ERROR_FILE\" \\\n/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:251:INTEGRITY_RESULT=\"$(sqlite3 \"$DB_PATH\" 'PRAGMA integrity_check;' 2>\"$ERROR_FILE\")\" \\\n/usr/local/sbin/routerguard-hard-power-cycle:3:MODE=\"${1:-check}\"\n/usr/local/sbin/routerguard-hard-power-cycle:4:case \"$MODE\" in check|cycle) ;; *) echo USAGE=routerguard-hard-power-cycle_check_or_cycle; exit 64;; esac\n/usr/local/sbin/routerguard-network:48:remove_rules(){\n/usr/local/sbin/routerguard-network:91: rm -f \"$RUN/active\"\n/usr/local/sbin/routerguard-network:92: remove_rules\n/usr/local/sbin/routerguard-network:100: rm -rf \"$RUN\"\n/etc/systemd/system/suspend.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'\n/etc/systemd/system/suspend-then-hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'\n/etc/systemd/system/hybrid-sleep.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'\n/etc/systemd/system/multi-user.target.wants/corosync.service:20:# as well; before editing this line, you may want to check DefaultStandardError\n/etc/systemd/system/multi-user.target.wants/postfix.service:19:# perform 2-stage startup\n/etc/systemd/system/multi-user.target.wants/postfix.service:20:ExecStartPre=+postfix check\n/etc/systemd/system/multi-user.target.wants/postfix.service:30:# Individual Postfix daemons are started as root, optionally perform chroot\n/etc/systemd/system/multi-user.target.wants/postfix.service:52:# CAP_CHOWN can be removed.\n/etc/systemd/system/multi-user.target.wants/pve-guests.service:22:ExecStop=-/usr/bin/vzdump -stop\n/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:22:ExecStart=/usr/sbin/pve-ha-lrm start\n/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:23:ExecStop=/usr/sbin/pve-ha-lrm stop\n/etc/systemd/system/multi-user.target.wants/rrdcached.service:2:# To customize, copy this file to /etc/systemd/system/\n/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:14:ExecStart=/usr/sbin/pve-ha-crm start\n/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:15:ExecStop=/usr/sbin/pve-ha-crm stop\n/etc/systemd/system/multi-user.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'\n/etc/systemd/system/multi-user.target.wants/e2scrub_reap.service:2:Description=Remove Stale Online ext4 Metadata Check Snapshots\n/etc/systemd/system/default.target.wants/nvmefc-boot-connections.service:19:RemoveIPC=yes\n/etc/systemd/system/default.target.wants/nvmf-autoconnect.service:21:RemoveIPC=yes\n/etc/systemd/system/sysinit.target.wants/lvm2-lvmpolld.socket:10:RemoveOnStop=true\n/etc/systemd/system/sysinit.target.wants/pvenetcommit.service:8:ExecStartPre=-/bin/rm -f /etc/openvswitch/conf.db\n/etc/systemd/system/sysinit.target.wants/systemd-pstore.service:11:Description=Platform Persistent Storage Archival\n/etc/systemd/system/pve-manager.service:22:ExecStop=-/usr/bin/vzdump -stop\n/etc/systemd/system/hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'\n/etc/systemd/system/sockets.target.wants/dm-event.socket:10:RemoveOnStop=true\n/etc/systemd/system/sockets.target.wants/rrdcached.socket:2:# To customize, copy this file to /etc/systemd/system/\n/etc/cron.d/vzdump:1:# cluster wide vzdump cron schedule\n997503422\t2026-07-01T03:31:11.1700059310\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.vma.zst\n868048369\t2026-06-30T03:31:08.4903256780\t/var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.vma.zst\n288349157\t2026-07-21T03:30:21.2480085730\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.tar.zst\n286538526\t2026-07-20T03:30:18.4386603270\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.tar.zst\n284443608\t2026-07-19T03:47:41.8525503200\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst\n283022034\t2026-07-18T03:30:19.7617214460\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_18-03_30_03.tar.zst\n220080970\t2026-07-21T03:30:33.3910787180\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.tar.zst\n220065825\t2026-07-20T03:30:30.8287264090\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.tar.zst\n220049041\t2026-07-19T03:47:58.9256407960\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst\n220033534\t2026-07-18T03:30:37.2878143100\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_18-03_30_20.tar.zst\n7547\t2026-07-18T01:09:39.2467331080\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_18-01_02_44.log\n7443\t2026-07-18T01:45:13.3149404960\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_18-01_38_20.log\n5796\t2026-07-17T21:55:21.5648458010\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_17-21_30_20.log\n2685\t2026-06-30T03:31:09.1616927060\t/var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.log\n2590\t2026-07-01T03:31:11.9500372070\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.log\n1088\t2026-07-20T03:30:31.0784689220\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.log\n1085\t2026-07-20T03:30:18.7427551960\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.log\n988\t2026-07-21T03:30:33.6364088960\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.log\n988\t2026-07-18T03:30:37.5450736960\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_18-03_30_20.log\n985\t2026-07-21T03:30:21.5465818340\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.log\n985\t2026-07-18T03:30:20.0884248500\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_18-03_30_03.log\n731\t2026-07-19T03:47:59.2238139650\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.log\n728\t2026-07-19T03:47:42.1183953260\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.log\n========== NODE=pve03 ==========\n=== REMOTES ===\n=== TIMERS ===\nWed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:10 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service\n=== SERVICES ===\ndpkg-db-backup.service static -\n/usr/local/sbin/homelab-staging-check.sh:11:echo \"=== homelab staging check: ${HOST} ===\"\n/usr/local/sbin/homelab-immutable-offline-verify:11: rm -f \"$TMP\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:4:CRYPT_FILE=\"/etc/homelab/rclone/required-crypt-remotes.txt\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:18: \"checksums/\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:21:echo \"=== homelab rclone crypt healthcheck: $(hostname) ===\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:40: if ! rclone listremotes | sed 's/:$//' | grep -qx \"$REMOTE\"; then\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:47: if ! rclone about \"${REMOTE}:\"; then\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:48: echo \"FAIL: rclone about failed: $REMOTE\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:54: ROOT_LIST=\"$(rclone lsf \"${REMOTE}:/\" || true)\"\n/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:76: if rclone cat \"${REMOTE}:/latest-backup-index.json\" | jq -e '.schema and .remote and .status and .items' >/dev/null; then\n/usr/local/sbin/homelab-rclone-required-check.sh:4:RAW_FILE=\"/etc/homelab/rclone/required-remotes.txt\"\n/usr/local/sbin/homelab-rclone-required-check.sh:5:CRYPT_FILE=\"/etc/homelab/rclone/required-crypt-remotes.txt\"\n/usr/local/sbin/homelab-rclone-required-check.sh:9:echo \"=== homelab rclone required check: $(hostname) ===\"\n/usr/local/sbin/homelab-rclone-required-check.sh:12:echo \"=== installed rclone ===\"\n/usr/local/sbin/homelab-rclone-required-check.sh:13:rclone version | head -5\n/usr/local/sbin/homelab-rclone-required-check.sh:17:CONFIGURED=\"$(rclone listremotes 2>/dev/null | sed 's/:$//' || true)\"\n/usr/local/sbin/homelab-rclone-required-check.sh:19: echo \"No rclone remotes configured yet.\"\n/etc/systemd/system/hybrid-sleep.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'\n/etc/systemd/system/multi-user.target.wants/pve-guests.service:22:ExecStop=-/usr/bin/vzdump -stop\n/etc/systemd/system/multi-user.target.wants/corosync.service:20:# as well; before editing this line, you may want to check DefaultStandardError\n/etc/systemd/system/multi-user.target.wants/rrdcached.service:2:# To customize, copy this file to /etc/systemd/system/\n/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:14:ExecStart=/usr/sbin/pve-ha-crm start\n/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:15:ExecStop=/usr/sbin/pve-ha-crm stop\n/etc/systemd/system/multi-user.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'\n/etc/systemd/system/multi-user.target.wants/postfix.service:19:# perform 2-stage startup\n/etc/systemd/system/multi-user.target.wants/postfix.service:20:ExecStartPre=+postfix check\n/etc/systemd/system/multi-user.target.wants/postfix.service:30:# Individual Postfix daemons are started as root, optionally perform chroot\n/etc/systemd/system/multi-user.target.wants/postfix.service:52:# CAP_CHOWN can be removed.\n/etc/systemd/system/multi-user.target.wants/e2scrub_reap.service:2:Description=Remove Stale Online ext4 Metadata Check Snapshots\n/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:22:ExecStart=/usr/sbin/pve-ha-lrm start\n/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:23:ExecStop=/usr/sbin/pve-ha-lrm stop\n/etc/systemd/system/sysinit.target.wants/pvenetcommit.service:8:ExecStartPre=-/bin/rm -f /etc/openvswitch/conf.db\n/etc/systemd/system/sysinit.target.wants/systemd-pstore.service:11:Description=Platform Persistent Storage Archival\n/etc/systemd/system/sysinit.target.wants/lvm2-lvmpolld.socket:10:RemoveOnStop=true\n/etc/systemd/system/hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'\n/etc/systemd/system/sockets.target.wants/rrdcached.socket:2:# To customize, copy this file to /etc/systemd/system/\n/etc/systemd/system/sockets.target.wants/dm-event.socket:10:RemoveOnStop=true\n/etc/systemd/system/suspend-then-hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'\n/etc/systemd/system/default.target.wants/nvmf-autoconnect.service:21:RemoveIPC=yes\n/etc/systemd/system/default.target.wants/nvmefc-boot-connections.service:19:RemoveIPC=yes\n/etc/systemd/system/suspend.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'\n/etc/systemd/system/pve-manager.service:22:ExecStop=-/usr/bin/vzdump -stop\n/etc/cron.d/vzdump:1:# cluster wide vzdump cron schedule\n11116472459\t2026-07-20T03:33:53.2839064370\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.vma.zst\n11043834894\t2026-07-21T03:33:54.3725716230\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.vma.zst\n11013723404\t2026-07-19T03:33:53.3463878040\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.vma.zst\n10994820121\t2026-07-18T03:33:47.1540038630\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_18-03_30_02.vma.zst\n1137072350\t2026-07-21T03:34:37.6276811360\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.vma.zst\n838951976\t2026-07-19T03:34:33.1034859540\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.vma.zst\n838817925\t2026-07-20T03:34:37.1719936670\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.vma.zst\n785249297\t2026-07-18T03:34:27.1531047780\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_18-03_33_48.vma.zst\n7106\t2026-07-02T03:37:53.1040459570\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_02-03_30_02.log\n7102\t2026-07-03T03:36:49.1036784480\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_03-03_30_05.log\n6564\t2026-07-18T03:33:48.0107527290\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_18-03_30_02.log\n6560\t2026-07-20T03:33:54.9283349630\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.log\n6559\t2026-07-21T03:33:55.1428157340\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.log\n6459\t2026-07-19T03:33:53.7631656400\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.log\n6246\t2026-07-04T03:33:39.2197744030\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_04-03_30_02.log\n4107\t2026-07-07T03:31:51.2516816970\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_07-03_30_01.log\n2642\t2026-07-21T03:34:38.4491706750\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.log\n2500\t2026-07-20T03:34:37.2013439550\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.log\n2404\t2026-07-19T03:34:34.1266444600\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.log\n2305\t2026-07-18T03:34:27.7179455720\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_18-03_33_48.log\n=== CLOUD_HEALTH_AND_OBJECTS ===\n=== CURRENT_BACKUP_MATRIX ===\nSTATUS=FAIL\nTS=2026-07-21T07:02:41Z\nTYPE=unified-backup-matrix\nMATRIX_MODE=authoritative\nJOBS_TOTAL=41\nJOBS_OK=30\nJOBS_BAD=6\nJOBS_UNMAPPED=0\nJOBS_DISABLED=5\nSTATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_JSON=/etc/homelab-backup-matrix/matrix.json\nAUTHORITY=canonical\nCOMPAT_OUTPUTS=29\nCOMPAT_MAP=/var/lib/homelab-backup-matrix/compat-map.json\nJOB=dpkg-db-backup STATUS=OK EVIDENCE=execution_only AGE=36147 MAX_AGE=172800\nJOB=filebrowser-backup STATUS=OK EVIDENCE=artifact_or_health AGE=14443 MAX_AGE=259200\nJOB=filebrowser-restore-validate STATUS=STALE EVIDENCE=artifact_or_health AGE=1016424 MAX_AGE=864000\nJOB=homelab-appbackup-dockge-stacks STATUS=OK EVIDENCE=artifact_or_health AGE=22248 MAX_AGE=172800\nJOB=homelab-appbackup-mariadb-batch STATUS=OK EVIDENCE=execution_only AGE=15387 MAX_AGE=172800\nJOB=homelab-appbackup-npmplus-edge STATUS=OK EVIDENCE=artifact_or_health AGE=20573 MAX_AGE=172800\nJOB=homelab-appbackup-postgres-batch STATUS=OK EVIDENCE=execution_only AGE=16416 MAX_AGE=172800\nJOB=homelab-appbackup-sqlite-batch STATUS=OK EVIDENCE=execution_only AGE=18327 MAX_AGE=172800\nJOB=homelab-authentik-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987790 MAX_AGE=172800\nJOB=homelab-auto-backup STATUS=OK EVIDENCE=execution_only AGE=195424 MAX_AGE=864000\nJOB=homelab-backup-coverage-matrix STATUS=OK EVIDENCE=artifact_or_health AGE=4346 MAX_AGE=172800\nJOB=homelab-backup-freshness STATUS=OK EVIDENCE=execution_only AGE=35589 MAX_AGE=172800\nJOB=homelab-backup-healthcheck STATUS=OK EVIDENCE=execution_only AGE=10631 MAX_AGE=172800\nJOB=homelab-backup-sla-health STATUS=FAIL EVIDENCE=artifact_or_health AGE=144 MAX_AGE=172800\nJOB=homelab-edge-vm-offhost-freshness STATUS=OK EVIDENCE=artifact_or_health AGE=18426 MAX_AGE=172800\nJOB=homelab-emergency-backup STATUS=OK EVIDENCE=artifact_or_health AGE=24539 MAX_AGE=172800\nJOB=homelab-extended-appbackup STATUS=OK EVIDENCE=artifact_or_health AGE=14156 MAX_AGE=172800\nJOB=homelab-gitea-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987785 MAX_AGE=172800\nJOB=homelab-golden-state-index STATUS=OK EVIDENCE=artifact_or_health AGE=4112 MAX_AGE=172800\nJOB=homelab-mail-cloud-critical-upload STATUS=OK EVIDENCE=artifact_or_health AGE=11997 MAX_AGE=172800\nJOB=homelab-mail-cloud-edge-vm-upload STATUS=OK EVIDENCE=artifact_or_health AGE=174045 MAX_AGE=864000\nJOB=homelab-mail-cloud-nextcloud-vm-upload STATUS=OK EVIDENCE=artifact_or_health AGE=7575 MAX_AGE=864000\nJOB=homelab-mail-cloud-restore-drill STATUS=STALE EVIDENCE=artifact_or_health AGE=988269 MAX_AGE=864000\nJOB=homelab-npmplus-kuma-config-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=60071 MAX_AGE=172800\nJOB=homelab-router-running-config-mail-cloud STATUS=OK EVIDENCE=artifact_or_health AGE=17076 MAX_AGE=172800\nJOB=homelab-vaultwarden-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987768 MAX_AGE=172800\nJOB=homelab-verified-backup-health STATUS=OK EVIDENCE=artifact_or_health AGE=11845 MAX_AGE=172800\nJOB=homelab-vm-backup-health STATUS=OK EVIDENCE=artifact_or_health AGE=8742 MAX_AGE=172800\nJOB=homelab-vm-backup-policy STATUS=OK EVIDENCE=artifact_or_health AGE=5548 MAX_AGE=172800\nJOB=homelab-vm-full-backup STATUS=OK EVIDENCE=artifact_or_health AGE=24778 MAX_AGE=172800\nJOB=homelab-vm-full-restore-drill STATUS=OK EVIDENCE=artifact_or_health AGE=523085 MAX_AGE=10368000\nJOB=homelab-vm-local-dumps-2cloud STATUS=OK EVIDENCE=artifact_or_health AGE=172141 MAX_AGE=864000\nJOB=homelab-vm-local-dumps-retention STATUS=OK EVIDENCE=artifact_or_health AGE=168839 MAX_AGE=864000\nJOB=immich-media-offhost-sync STATUS=OK EVIDENCE=artifact_or_health AGE=4286 MAX_AGE=259200\nJOB=memos-backup STATUS=OK EVIDENCE=artifact_or_health AGE=14547 MAX_AGE=259200\nJOB=netbird-vps-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=1747896 MAX_AGE=864000\nJOB=paperless-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=16347 MAX_AGE=259200\nJOB=paperless-restore-dry-run STATUS=STALE EVIDENCE=artifact_or_health AGE=1001986 MAX_AGE=864000\nJOB=skladchik-reports-monitor-backup STATUS=OK EVIDENCE=artifact_or_health AGE=19639 MAX_AGE=172800\nJOB=skladchik-reports-monitor-backup-restore-check STATUS=OK EVIDENCE=artifact_or_health AGE=190273 MAX_AGE=864000\nJOB=vm160-remote-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=26607 MAX_AGE=172800\nPAIR_DELETE_GUARD_AUDIT=COMPLETE\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index ded3dd6..c28c90f 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,7 +1,7 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=BACKUP-CLOUD-CONTENTS-065 -STATUS=FAIL -RC=2 +COMMAND_ID=BACKUP-PAIR-DELETE-GUARD-066 +STATUS=OK +RC=0 HOST=pve01 MODE=read-only COMPONENT=backup-audit @@ -9,17 +9,1009 @@ REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1 -COMMAND_SHA256=452b162f3a4d66e4471b9cccf75bfeb2631e0d2552eb0a8e1325b757d2e6f1df +COMMAND_SHA256=e64555881c22fffab137df86ee957819b9922f83078f71124f8ff61e8580590d DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGES_MADE=false SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=2bc4df84f54e8e6175eeb80e00fc91fcb32008a2e5abbaeab989bcec88af8173 -SANITIZED_OUTPUT_SHA256=2bc4df84f54e8e6175eeb80e00fc91fcb32008a2e5abbaeab989bcec88af8173 +RAW_EVIDENCE_SHA256=d13bc63fd488f7e13f81bef392b271beacdec9c637576dc61e7dad33042b27f2 +SANITIZED_OUTPUT_SHA256=6c697f245387cba271db9658a3e6e5ce9d5a9ac0178945c559417c7064daa692 OUTPUT_BEGIN -/usr/bin/bash: -c: line 1: unexpected EOF while looking for matching `"' +=== PIPELINES_AND_CLOUD_PAIRS === +========== NODE=pve01 ========== +=== REMOTES === +=== TIMERS === +Wed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:14 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service +Wed 2026-07-22 00:09:43 MSK 9h Tue 2026-07-21 00:09:32 MSK 14h ago homelab-backup-freshness.timer homelab-backup-freshness.service +Wed 2026-07-22 02:15:00 MSK 11h Tue 2026-07-21 02:15:05 MSK 12h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service +Wed 2026-07-22 03:10:02 MSK 12h Tue 2026-07-21 03:13:14 MSK 11h ago homelab-emergency-backup.timer homelab-emergency-backup.service +Wed 2026-07-22 03:35:00 MSK 12h Tue 2026-07-21 03:35:14 MSK 11h ago filebrowser-backup.timer filebrowser-backup.service +Wed 2026-07-22 03:55:02 MSK 13h Tue 2026-07-21 03:51:25 MSK 11h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service +Wed 2026-07-22 04:17:29 MSK 13h Tue 2026-07-21 04:19:17 MSK 10h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service +Wed 2026-07-22 04:26:54 MSK 13h Tue 2026-07-21 04:28:14 MSK 10h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service +Wed 2026-07-22 04:38:04 MSK 13h Tue 2026-07-21 04:34:48 MSK 10h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service +Wed 2026-07-22 05:01:44 MSK 14h Tue 2026-07-21 04:57:14 MSK 9h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service +Wed 2026-07-22 05:23:56 MSK 14h Tue 2026-07-21 05:17:57 MSK 9h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service +Wed 2026-07-22 05:27:13 MSK 14h Tue 2026-07-21 05:29:05 MSK 9h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service +Wed 2026-07-22 05:47:19 MSK 14h Tue 2026-07-21 05:46:14 MSK 9h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service +Wed 2026-07-22 06:01:19 MSK 15h Tue 2026-07-21 05:57:30 MSK 8h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service +Wed 2026-07-22 06:27:43 MSK 15h Tue 2026-07-21 06:32:25 MSK 8h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service +Wed 2026-07-22 06:38:51 MSK 15h Tue 2026-07-21 06:43:54 MSK 8h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service +Wed 2026-07-22 06:39:22 MSK 15h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service +Wed 2026-07-22 06:44:16 MSK 15h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service +Wed 2026-07-22 06:52:06 MSK 15h Tue 2026-07-21 06:47:47 MSK 8h ago homelab-beszel-backup.timer homelab-beszel-backup.service +Wed 2026-07-22 06:58:58 MSK 16h Tue 2026-07-21 06:57:30 MSK 7h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service +Wed 2026-07-22 07:02:27 MSK 16h Tue 2026-07-21 07:05:30 MSK 7h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service +Wed 2026-07-22 07:07:14 MSK 16h Tue 2026-07-21 07:18:55 MSK 7h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service +Wed 2026-07-22 07:35:06 MSK 16h Tue 2026-07-21 07:36:59 MSK 7h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service +Wed 2026-07-22 07:38:21 MSK 16h Tue 2026-07-21 07:36:14 MSK 7h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service +Wed 2026-07-22 07:41:49 MSK 16h Tue 2026-07-21 07:53:41 MSK 7h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service +Wed 2026-07-22 08:10:37 MSK 17h Tue 2026-07-21 08:07:25 MSK 6h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service +Wed 2026-07-22 08:37:02 MSK 17h Tue 2026-07-21 08:30:07 MSK 6h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service +Wed 2026-07-22 08:40:13 MSK 17h Tue 2026-07-21 08:50:14 MSK 6h ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service +Wed 2026-07-22 08:53:36 MSK 17h Tue 2026-07-21 08:41:14 MSK 6h ago memos-backup.timer memos-backup.service +Sun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service +Sun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service +Sun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service +Sun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service +Mon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 7h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service +=== SERVICES === +dpkg-db-backup.service static - +filebrowser-backup.service static - +homelab-alertmanager-backup.service static - +homelab-appbackup-dockge-stacks.service static - +homelab-appbackup-mariadb-batch.service static - +homelab-appbackup-npmplus-edge.service static - +homelab-appbackup-postgres-batch.service static - +homelab-appbackup-sqlite-batch.service static - +homelab-authentik-backup.service static - +homelab-auto-backup.service static - +homelab-backup-coverage-matrix.service static - +homelab-backup-freshness.service static - +homelab-backup-healthcheck.service static - +homelab-backup-matrix-refresh.service static - +homelab-backup-sla-health.service static - +homelab-beszel-backup.service static - +homelab-blackbox-exporter-backup.service static - +homelab-cluster-admin-incident-engine-backup.service static - +homelab-crypto-portfolio-backup.service static - +homelab-edge-vm-vzdump-backup.service static - +homelab-emergency-backup.service static - +homelab-extended-appbackup.service static - +homelab-gitea-backup.service static - +homelab-gitea-secondary-backup.service static - +homelab-legacy-monitoring-backups.service static - +homelab-mail-cloud-critical-upload.service static - +homelab-mail-cloud-edge-vm-upload.service static - +homelab-mail-cloud-nextcloud-vm-upload.service static - +homelab-mail-cloud-restore-drill.service static - +homelab-npmplus-kuma-config-backup.service static - +homelab-router-running-config-mail-cloud.service static - +homelab-vaultwarden-backup.service static - +homelab-verified-backup-health.service static - +homelab-vm-backup-health.service static - +homelab-vm-backup-policy.service static - +homelab-vm-full-backup.service static - +homelab-vm-local-dumps-2cloud.service static - +homelab-vm160-remote-backup.service static - +immich-media-backup.service static - +memos-backup.service static - +netbird-vps-backup.service static - +paperless-backup.service static - +skladchik-reports-monitor-backup-restore-check.service static - +skladchik-reports-monitor-backup.service static - +/usr/local/sbin/homelab-vm160-remote-backup:8:PARENT=/mnt/staging/vzdump +/usr/local/sbin/homelab-vm160-remote-backup:22:SOURCE_ACTIVE="$(ssh "${SSH[@]}" "$SOURCE" "pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'")" || SOURCE_ACTIVE_RC=$? +/usr/local/sbin/homelab-vm160-remote-backup:27:NAME="vzdump-qemu-${VMID}-${TS}.vma.zst" +/usr/local/sbin/homelab-vm160-remote-backup:35: ssh "${SSH[@]}" "$TARGET" "rm -rf -- '$DIR'" >/dev/null 2>&1 || true +/usr/local/sbin/homelab-vm160-remote-backup:41: rm -f "$LOG" +/usr/local/sbin/homelab-vm160-remote-backup:46:LATEST_SIZE="$(ssh "${SSH[@]}" "$TARGET" "find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\n' | sort -nr | head -n1")" +/usr/local/sbin/homelab-vm160-remote-backup:55:ssh "${SSH[@]}" "$SOURCE" "vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7" 2>"$LOG" | ssh "${SSH[@]}" "$TARGET" "cat > '$PART'" +/usr/local/sbin/homelab-vm160-remote-backup:57:ssh "${SSH[@]}" "$TARGET" "cat > '$DIR/vzdump.log'" < "$LOG" +/usr/local/sbin/homelab-vm160-remote-backup:66:rm -f "$LOG" +/usr/local/sbin/skladchik-reports-monitor-status:11:check_health() { +/usr/local/sbin/skladchik-reports-monitor-status:24:check_timer() { +/usr/local/sbin/skladchik-reports-monitor-status:38:check_timer skladchik-reports-monitor-backup.timer +/usr/local/sbin/skladchik-reports-monitor-status:39:check_timer skladchik-reports-monitor-backup-restore-check.timer +/usr/local/sbin/skladchik-reports-monitor-status:40:check_timer skladchik-reports-monitor-supervisor.timer +/usr/local/sbin/skladchik-reports-monitor-status:41:check_timer skladchik-reports-monitor-monthly-selftest-watch.timer +/usr/local/sbin/skladchik-reports-monitor-status:43:check_health "PVE01: ежедневный бэкап" "/var/lib/skladchik-reports-monitor-backup/health.txt" +/usr/local/sbin/skladchik-reports-monitor-status:44:check_health "PVE01: проверка восстановления" "/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt" +/usr/local/sbin/skladchik-reports-monitor-status:45:check_health "PVE01: общий supervisor" "/var/lib/skladchik-reports-monitor-supervisor/health.txt" +/usr/local/sbin/skladchik-reports-monitor-status:46:check_health "PVE01: контроль ежемесячного самотеста" "/var/lib/skladchik-reports-monitor-selftest-watch/health.txt" +/usr/local/sbin/homelab-mail-cloud-restore-drill:52: homelab-rclone-cat-chunked "${R}:$CRIT_DEST/$CRIT_BUNDLE" > "$OUT" 2>"$WORK/logs/${SAFE}_crit_download.err" +/usr/local/sbin/homelab-mail-cloud-restore-drill:83: rm -f "$OUT" +/usr/local/sbin/homelab-mail-cloud-restore-drill:108: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/MANIFEST.txt" "$WORK/edge/MANIFEST.txt" > "$WORK/logs/edge_manifest_download.out" 2>"$WORK/logs/edge_manifest_download.err" +/usr/local/sbin/homelab-mail-cloud-restore-drill:111: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/SHA256SUMS.txt" "$WORK/edge/SHA256SUMS.txt" > "$WORK/logs/edge_sums_download.out" 2>"$WORK/logs/edge_sums_download.err" +/usr/local/sbin/homelab-mail-cloud-restore-drill:114: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/$EDGE_ARCHIVE.sha256" "$WORK/edge/$EDGE_ARCHIVE.sha256" > "$WORK/logs/edge_archive_sha_download.out" 2>"$WORK/logs/edge_archive_sha_download.err" +/usr/local/sbin/homelab-mail-cloud-restore-drill:117: rclone lsf "${EDGE_REMOTE}:$EDGE_DEST/parts" --format ps > "$WORK/logs/edge_parts_lsf.txt" 2>"$WORK/logs/edge_parts_lsf.err" +/usr/local/sbin/homelab-mail-cloud-restore-drill:130: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/parts/$FIRST_PART" "$WORK/edge/$FIRST_PART" > "$WORK/logs/edge_first_part_download.out" 2>"$WORK/logs/edge_first_part_download.err" +/usr/local/sbin/homelab-mail-cloud-restore-drill:137: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/parts/$LAST_PART" "$WORK/edge/$LAST_PART" > "$WORK/logs/edge_last_part_download.out" 2>"$WORK/logs/edge_last_part_download.err" +/usr/local/sbin/homelab-mail-cloud-restore-drill:168: rm -f "$WORK/edge/$FIRST_PART" "$WORK/edge/$LAST_PART" +/usr/local/sbin/homelab-golden-state-extended-overlay:26:rclone copyto "$JSON" "$RA" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-golden-state-extended-overlay:27:rclone copyto "$JSON" "$RB" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-golden-state-extended-overlay:28:rclone cat "$RA" 2>/dev/null | jq -e --arg cp "$CP" ".control_plane_job == \$cp" >/dev/null || BAD=$((BAD+1)) +/usr/local/sbin/homelab-golden-state-extended-overlay:29:rclone cat "$RB" 2>/dev/null | jq -e --arg cp "$CP" ".control_plane_job == \$cp" >/dev/null || BAD=$((BAD+1)) +/usr/local/sbin/homelab-mail-cloud-critical-upload:50:EXCLUDED=edge-vm-vzdump-large-archive,rclone-config,secrets,raw-env +/usr/local/sbin/homelab-mail-cloud-critical-upload:80: rclone mkdir "${R}:$REMOTE_DIR" > "$WORK/logs/${SAFE}_mkdir.out" 2>"$WORK/logs/${SAFE}_mkdir.err" +/usr/local/sbin/homelab-mail-cloud-critical-upload:82: homelab-rclone-copy-chunked "$BUNDLE" "${R}:$REMOTE_DIR/$(basename "$BUNDLE")" > "$WORK/logs/${SAFE}_upload_bundle.out" 2>"$WORK/logs/${SAFE}_upload_bundle.err" +/usr/local/sbin/homelab-mail-cloud-critical-upload:84: rclone copyto "$BUNDLE_SHA" "${R}:$REMOTE_DIR/$(basename "$BUNDLE_SHA")" --checksum > "$WORK/logs/${SAFE}_upload_sha.out" 2>"$WORK/logs/${SAFE}_upload_sha.err" +/usr/local/sbin/homelab-mail-cloud-critical-upload:86: rclone copyto "$WORK/components/MANIFEST.txt" "${R}:$REMOTE_DIR/MANIFEST.txt" --checksum > "$WORK/logs/${SAFE}_upload_manifest.out" 2>"$WORK/logs/${SAFE}_upload_manifest.err" +/usr/local/sbin/homelab-mail-cloud-critical-upload:88: rclone copyto "$WORK/components/SHA256SUMS.txt" "${R}:$REMOTE_DIR/SHA256SUMS.txt" --checksum > "$WORK/logs/${SAFE}_upload_sums.out" 2>"$WORK/logs/${SAFE}_upload_sums.err" +/usr/local/sbin/homelab-mail-cloud-critical-upload:90: homelab-rclone-cat-chunked "${R}:$REMOTE_DIR/$(basename "$BUNDLE")" > "$WORK/download/${SAFE}-$(basename "$BUNDLE")" 2>"$WORK/logs/${SAFE}_download_bundle.err" +/usr/local/sbin/homelab-mail-cloud-critical-upload:108: rm -f "$WORK/download/${SAFE}-$(basename "$BUNDLE")" +/usr/local/sbin/homelab-mail-cloud-critical-upload:109: rclone lsf "${R}:$TARGET_BASE" --dirs-only 2>/dev/null | sed 's#/$##' | sort > "$WORK/logs/${SAFE}_dirs.txt" +/usr/local/sbin/homelab-mail-cloud-critical-upload:114: [ -n "$OLD" ] && rclone purge "${R}:$TARGET_BASE/$OLD" >> "$WORK/logs/${SAFE}_retention.out" 2>>"$WORK/logs/${SAFE}_retention.err" +/usr/local/sbin/homelab-mail-cloud-critical-upload:140: rm -rf --one-file-system "$OLD_DIR" +/usr/local/sbin/homelab-alertmanager-backup-run:16:ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$MON" "sudo -n docker exec alertmanager amtool check-config /etc/alertmanager/alertmanager.yml >/dev/null && sudo -n tar -C / -czf - opt/stacks/observability-lite/compose.yaml opt/stacks/observability-lite/compose.override.yaml opt/stacks/observability-lite/alertmanager/alertmanager.yml opt/stacks/observability-lite/alertmanager/data" > "$STAGE/remote.tgz" +/usr/local/sbin/homelab-alertmanager-backup-run:18:rm -f "$STAGE/remote.tgz" +/usr/local/sbin/homelab-alertmanager-backup-run:21:jq -n --arg app "$APP" --arg host "monitoring" --arg engine "files+alertmanager-config-check" --arg config "ok" --argjson files "$FILES" "{app:\$app,engine:\$engine,source_host:\$host,source_paths:[\"monitoring:/opt/stacks/observability-lite/alertmanager\",\"monitoring:/opt/stacks/observability-lite/compose.yaml\",\"monitoring:/opt/stacks/observability-lite/compose.override.yaml\"],config_check:\$config,file_count:\$files,note:\"Alertmanager configuration and runtime state in encrypted two-cloud backup\"}" > "$STAGE/metadata.json" +/usr/local/sbin/appbackupctl:13: check) bash -n /usr/local/sbin/appbackup-dockge-stacks && bash -n /usr/local/sbin/appbackup-restore-check && bash -n /usr/local/sbin/appbackup-retention && bash -n /usr/local/sbin/appbackup-report && echo "OK appbackup framework" ;; +/usr/local/sbin/appbackupctl:14: restore-check) appbackup-restore-check "${2:-dockge-stacks}" ;; +/usr/local/sbin/appbackupctl:17: audit) appbackupctl check && appbackupctl restore-check dockge-stacks && appbackupctl retention 7 dry-run dockge-stacks && appbackupctl report dockge-stacks >/dev/null && echo "OK app audit finished" ;; +/usr/local/sbin/appbackupctl:21: *) echo "usage: appbackupctl run [dockge-stacks|cluster-admin-incident-engine]|check|restore-check|retention|report|audit|status|lastlog|timer"; exit 2 ;; +/usr/local/sbin/postgres-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@"$H" "sudo rm -f /tmp/postgres-app-${APP}-${JOB_ID}.tgz" +/usr/local/sbin/postgres-apps-backup-run:21: rm -rf "$STAGE/payload" +/usr/local/sbin/homelab-backup-gitea:27:rm -rf "$RESTORE" +/usr/local/sbin/homelab-backup-gitea:35:INTEGRITY="$(sqlite3 "$DB" 'PRAGMA integrity_check;')" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:4:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:18: "checksums/" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:21:echo "=== homelab rclone crypt healthcheck: $(hostname) ===" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:40: if ! rclone listremotes | sed 's/:$//' | grep -qx "$REMOTE"; then +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:47: if ! rclone about "${REMOTE}:"; then +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:48: echo "FAIL: rclone about failed: $REMOTE" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:54: ROOT_LIST="$(rclone lsf "${REMOTE}:/" || true)" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:76: if rclone cat "${REMOTE}:/latest-backup-index.json" | jq -e '.schema and .remote and .status and .items' >/dev/null; then +/usr/local/sbin/homelab-check-private-vpn-hosts.realnet-20260624T140509Z:4:REAL="/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z" +/usr/local/sbin/homelab-stage-verify-flat-chunked:7:rm -rf "$VERIFY" +/usr/local/sbin/homelab-stage-verify-flat-chunked:9:rclone copy "$REMOTE_DIR" "$VERIFY" --transfers 4 --checkers 8 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-stage-verify-flat-chunked:11:(cd "$VERIFY" && test -s manifest.json && if [[ -n "$JOB" ]]; then jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null; fi && sha256sum -c checksums.sha256 >/dev/null && tar -tzf "$ARCHIVE" >/dev/null) +/usr/local/sbin/homelab-rclone-cat-chunked:4:TMP_BASE=${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks} +/usr/local/sbin/homelab-rclone-cat-chunked:7:cleanup(){ rm -rf "$WORK"; } +/usr/local/sbin/homelab-rclone-cat-chunked:10:rclone copyto "${REMOTE_FILE}.parts/MANIFEST.parts.txt" "$MAN" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-rclone-cat-chunked:11:while read -r PART SIZE; do case "$PART" in part-*) rclone cat "${REMOTE_FILE}.parts/$PART" ;; esac; done < "$MAN" +/usr/local/sbin/homelab-stage-verify-chunked:7:rm -rf "$VERIFY" +/usr/local/sbin/homelab-stage-verify-chunked:9:rclone copy "$REMOTE_DIR" "$VERIFY" --exclude "$ARCHIVE.parts/**" --exclude "$ARCHIVE.parts/" --exclude "$ARCHIVE.sha256" --exclude "$ARCHIVE.manifest.txt" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --log-level ERROR +/usr/local/sbin/homelab-stage-verify-chunked:13:if [ -s checksums.sha256 ]; then grep -F -v " $ARCHIVE" checksums.sha256 > .checksums-no-archive.sha256 || true; if [ -s .checksums-no-archive.sha256 ]; then sha256sum -c .checksums-no-archive.sha256; fi; EXPECTED=$(grep -F " $ARCHIVE" checksums.sha256 | awk '{print $1; exit}'); else EXPECTED=$(rclone cat "${REMOTE_DIR}/${ARCHIVE}.sha256" | awk '{print $1; exit}'); fi +/usr/local/sbin/homelab-stage-verify-chunked:15:ACTUAL=$(homelab-rclone-cat-chunked "${REMOTE_DIR}/${ARCHIVE}" | sha256sum | awk '{print $1}') +/usr/local/sbin/homelab-stage-verify-chunked:17:homelab-rclone-cat-chunked "${REMOTE_DIR}/${ARCHIVE}" | tar -tzf - >/dev/null +/usr/local/sbin/crypto-portfolio-backup-run:15:ssh -n -o BatchMode=yes -o ConnectTimeout=10 "$CORE" 'H="$(hostname -f 2>/dev/null || hostname)"; { [ "$H" = "core-apps.gram1.ru" ] || [ "$H" = "core-apps" ]; } || exit 3; D=/opt/stacks/crypto-portfolio; T="$(sudo -n mktemp -d /tmp/crypto-portfolio-backup.XXXXXX)"; trap "sudo -n rm -rf \"$T\"" EXIT; sudo -n install -d -m 700 "$T/stack" "$T/stack/data"; for F in compose.yaml Dockerfile requirements.txt app.py worker.py .dockerignore; do sudo -n test -s "$D/$F" && sudo -n cp -a "$D/$F" "$T/stack/"; done; sudo -n cp -a "$D/templates" "$D/secrets" "$T/stack/"; sudo -n python3 -c "import sqlite3,sys;s=sqlite3.connect(sys.argv[1]);d=sqlite3.connect(sys.argv[2]);s.backup(d);d.close();s.close()" "$D/data/portfolio.db" "$T/stack/data/portfolio.db"; for F in market.json worker-status.json; do sudo -n test -s "$D/data/$F" && sudo -n cp -a "$D/data/$F" "$T/stack/data/"; done; sudo -n tar -C "$T" -czf - stack' > "$STAGE/remote.tgz" +/usr/local/sbin/crypto-portfolio-backup-run:17:rm -f "$STAGE/remote.tgz" +/usr/local/sbin/crypto-portfolio-backup-run:18:DBI="$(python3 -c 'import sqlite3,sys;print(sqlite3.connect(sys.argv[1]).execute("PRAGMA integrity_check").fetchone()[0])' "$STAGE/payload/stack/data/portfolio.db")" +/usr/local/sbin/npmplus-edge-backup-run:20:ssh -o BatchMode=yes -o ConnectTimeout=8 "$EDGE" "sudo rm -f '/tmp/npmplus-edge-${JOB_ID}.tgz'" +/usr/local/sbin/npmplus-edge-backup-run:28:rm -rf "$STAGE/payload" +/usr/local/sbin/mariadb-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@"$H" "sudo rm -f /tmp/mariadb-app-${APP}-${JOB_ID}.tgz" +/usr/local/sbin/mariadb-apps-backup-run:20: rm -rf "$STAGE/payload" +/usr/local/sbin/homelab-blackbox-exporter-backup-run:17:rm -f "$STAGE/remote.tgz" +/usr/local/sbin/homelab-blackbox-exporter-backup-run:21:jq -n --arg app "$APP" --arg host "monitoring" --arg engine "files+runtime-health-check" --argjson files "$FILES" '{app:$app,engine:$engine,source_host:$host,source_paths:["monitoring:/opt/stacks/blackbox-exporter"],runtime_health:"ok",file_count:$files,note:"Blackbox Exporter compose and probe configuration in encrypted two-cloud backup"}' > "$STAGE/metadata.json" +/usr/local/sbin/homelab-ingress-standby-refresh:16: rm -f "$TMP_STATE" "$RELAY" "$RESOURCES" +/usr/local/sbin/homelab-ingress-standby-refresh:74:run_node "$SOURCE_NODE" vzdump 130 \ +/usr/local/sbin/homelab-ingress-standby-refresh:78: --remove 0 \ +/usr/local/sbin/homelab-ingress-standby-refresh:83:"find /var/lib/vz/dump -maxdepth 1 -type f -name 'vzdump-qemu-130-*.vma.zst' -printf '%T@ %p\n' | sort -rn | head -n 1 | cut -d' ' -f2-" +/usr/local/sbin/homelab-ingress-standby-refresh:106:"chattr -i '$DEST_FILE' '$DEST_FILE.sha256' '$DEST_FILE.env' 2>/dev/null || true; rm -f '$DEST_TMP'" +/usr/local/sbin/homelab-ingress-standby-refresh:155: -name 'vzdump-qemu-130-*.vma.zst' \ +/usr/local/sbin/homelab-ingress-standby-refresh:164: rm -f "$OLD" "$OLD.sha256" "$OLD.env" +/usr/local/sbin/homelab-ingress-standby-refresh:173:run_node "$SOURCE_NODE" rm -f "$SOURCE_BACKUP" +/usr/local/sbin/homelab-stage-2cloud-flat:33:(cd "$STAGE"; FILES=("$ARCHIVE" manifest.json latest-backup-index.json); [[ -f metadata.json ]] && FILES+=(metadata.json); sha256sum "${FILES[@]}" > checksums.sha256; sha256sum -c checksums.sha256 >/dev/null; tar -tzf "$ARCHIVE" >/dev/null; jq -e . manifest.json >/dev/null) +/usr/local/sbin/homelab-stage-2cloud-flat:36: if rclone lsf "$REMOTE_DIR" --max-depth 1 2>/dev/null | grep -q .; then echo "REFUSE NONEMPTY CANARY TARGET $REMOTE_DIR" >&2; exit 6; fi +/usr/local/sbin/homelab-stage-2cloud-flat:37: rclone mkdir "$REMOTE_DIR" +/usr/local/sbin/homelab-stage-2cloud-flat:38: for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f "$STAGE/$F" ]] && rclone copyto "$STAGE/$F" "$REMOTE_DIR/$F" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done +/usr/local/sbin/homelab-stage-2cloud-flat:39: CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-flat-chunked "$STAGE/$ARCHIVE" "$REMOTE_DIR/$ARCHIVE" +/usr/local/sbin/homelab-stage-2cloud-flat:41: rclone copyto "$STAGE/latest-backup-index.json" "$LATEST_FILE" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-stage-2cloud-flat:42: rclone cat "$LATEST_FILE" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null +/usr/local/sbin/homelab-stage-2cloud-flat:47:rm -rf "$STAGE" "$VERIFY" +/usr/local/sbin/homelab-cluster-admin-inventory-export:12: return subprocess.check_output(args, text=True, stderr=subprocess.DEVNULL, timeout=30) +/usr/local/sbin/homelab-cluster-admin-webpanel-sync:7:copy_health() { +/usr/local/sbin/homelab-cluster-admin-webpanel-sync:17:copy_health /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt cluster-admin-vm-mail-cloud-backup.txt +/usr/local/sbin/homelab-cluster-admin-webpanel-sync:18:copy_health /var/lib/homelab-health/overall.txt overall.txt +/usr/local/sbin/homelab-cluster-admin-webpanel-sync:19:copy_health /var/lib/homelab-health/final-readiness.txt final-readiness.txt +/usr/local/sbin/homelab-cluster-admin-webpanel-sync:20:copy_health /var/lib/homelab-health/cluster-admin-strict-final-seal.txt cluster-admin-strict-final-seal.txt +/usr/local/sbin/homelab-backup-authentik:14:ssh debian@[PRIVATE_IP] "rm -rf /tmp/auth-small-snap; mkdir -p /tmp/auth-small-snap" +/usr/local/sbin/homelab-backup-authentik:27:ssh debian@[PRIVATE_IP] "sudo docker rm -f authentik-restore-test >/dev/null 2>&1 || true" +/usr/local/sbin/homelab-backup-authentik:51:ssh debian@[PRIVATE_IP] "sudo docker rm -f authentik-restore-test >/dev/null 2>&1 || true; rm -f /tmp/authentik-restore-from-archive.pg_dump" +/usr/local/sbin/homelab-safe-exec:77: check=False, +/usr/local/sbin/homelab-safe-exec:621: latest_tmp.unlink(missing_ok=True) +/usr/local/sbin/homelab-golden-state-index:28:rclone copyto "$JSON" "$RA" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-golden-state-index:29:rclone copyto "$JSON" "$RB" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-golden-state-index:30:rclone cat "$RA" 2>/dev/null | jq -e --arg cp "$CP" --arg ext "$EXT" ".control_plane_job == \$cp and .health.extended_appbackup == \$ext and .extended_appbackup_health == \$ext" >/dev/null || BAD=$((BAD+1)) +/usr/local/sbin/homelab-golden-state-index:31:rclone cat "$RB" 2>/dev/null | jq -e --arg cp "$CP" --arg ext "$EXT" ".control_plane_job == \$cp and .health.extended_appbackup == \$ext and .extended_appbackup_health == \$ext" >/dev/null || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-full-retention:21: -name "vzdump-qemu-${vmid}-*.vma.zst" \ +/usr/local/sbin/homelab-vm-full-retention:33: vzdump-qemu-"$vmid"-*.vma.zst) +/usr/local/sbin/homelab-vm-full-retention:35: rm -f \ +/usr/local/sbin/homelab-vm-full-retention:73: rm -rf -- \"\$BASE/\$setname\" +/usr/local/sbin/homelab-vm-full-retention:91: rclone lsf "$path" \ +/usr/local/sbin/homelab-vm-full-retention:105: rclone purge "${path}/${dir}" +/usr/local/sbin/homelab-stage-2cloud:22:echo "[STEP] local checksum"; (cd "$STAGE" && sha256sum "$ARCHIVE" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e . manifest.json >/dev/null) +/usr/local/sbin/homelab-stage-2cloud:23:echo "[STEP] upload cloud A metadata"; rclone copy "$STAGE" "$RA" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s +/usr/local/sbin/homelab-stage-2cloud:24:echo "[STEP] upload cloud A archive chunks"; rclone deletefile "$RA/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RA/$ARCHIVE" +/usr/local/sbin/homelab-stage-2cloud:25:echo "[STEP] upload cloud B metadata"; rclone copy "$STAGE" "$RB" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s +/usr/local/sbin/homelab-stage-2cloud:26:echo "[STEP] upload cloud B archive chunks"; rclone deletefile "$RB/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RB/$ARCHIVE" +/usr/local/sbin/homelab-stage-2cloud:27:echo "[STEP] verify cloud A chunked"; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-stage-verify-chunked "$RA" "$ARCHIVE" "$VERIFY/a" "$JOB" +/usr/local/sbin/homelab-stage-2cloud:28:echo "[STEP] verify cloud B chunked"; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-stage-verify-chunked "$RB" "$ARCHIVE" "$VERIFY/b" "$JOB" +/usr/local/sbin/homelab-stage-2cloud:29:echo "[STEP] upload latest index"; rclone copyto "$STAGE/latest-backup-index.json" "$RAL"; rclone copyto "$STAGE/latest-backup-index.json" "$RBL"; rclone cat "$RAL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null; rclone cat "$RBL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null +/usr/local/sbin/homelab-stage-2cloud:31:echo "[STEP] delete local staging"; rm -rf "$STAGE" "$VERIFY"; test ! -d "$STAGE" && test ! -d "$VERIFY" +/usr/local/sbin/homelab-safe-autoheal:17:SAFE_TIMERS="homelab-emergency-backup.timer homelab-appbackup-dockge-stacks.timer homelab-appbackup-npmplus-edge.timer homelab-appbackup-sqlite-batch.timer homelab-appbackup-postgres-batch.timer homelab-appbackup-mariadb-batch.timer homelab-verified-backup-health.timer homelab-drift-check.timer homelab-service-registry-check.timer homelab-dependency-map-check.timer homelab-vm-backup-health.timer homelab-alerting-health.timer homelab-runbook-generate.timer homelab-desired-state-sync.timer homelab-duty-admin-report.timer homelab-incident-journal.timer homelab-backup-sla-health.timer homelab-kuma-monitor-policy.timer homelab-safe-autoheal.timer homelab-overall-health.timer" +/usr/local/sbin/homelab-safe-autoheal:144:for cmd in homelab-alerting-health homelab-kuma-monitor-policy homelab-backup-sla-health homelab-service-registry-check homelab-dependency-map-check; do +/usr/local/sbin/homelab-backup-npmplus-kuma-configs:26: install -D -m 0755 /usr/local/sbin/homelab-check-private-vpn-hosts "$SNAP/pve01/usr/local/sbin/homelab-check-private-vpn-hosts" 2>>"$ERR" || fail "copy_private_vpn_script_failed" +/usr/local/sbin/homelab-backup-npmplus-kuma-configs:27: install -D -m 0644 /etc/systemd/system/homelab-private-vpn-hosts-health.service "$SNAP/pve01/etc/systemd/system/homelab-private-vpn-hosts-health.service" 2>>"$ERR" || fail "copy_private_vpn_service_failed" +/usr/local/sbin/homelab-backup-npmplus-kuma-configs:28: install -D -m 0644 /etc/systemd/system/homelab-private-vpn-hosts-health.timer "$SNAP/pve01/etc/systemd/system/homelab-private-vpn-hosts-health.timer" 2>>"$ERR" || fail "copy_private_vpn_timer_failed" +/usr/local/sbin/homelab-backup-npmplus-kuma-configs:29: install -D -m 0644 /var/lib/homelab-health/private-vpn-hosts.txt "$SNAP/pve01/var/lib/homelab-health/private-vpn-hosts.txt" 2>>"$ERR" || fail "copy_private_vpn_health_failed" +/usr/local/sbin/homelab-backup-npmplus-kuma-configs:42:sudo rm -rf "$TMP" "$EDGE_TGZ" +/usr/local/sbin/homelab-backup-npmplus-kuma-configs:82:sudo rm -rf "$TMP" +/usr/local/sbin/homelab-backup-npmplus-kuma-configs:90: ssh -o BatchMode=yes -o ConnectTimeout=8 debian@[PRIVATE_IP] "rm -f '$EDGE_TGZ'" 2>>"$ERR" || true +/usr/local/sbin/homelab-backup-npmplus-kuma-configs:99: NPM_INTEGRITY="$(sqlite3 "$SNAP/edge-vm/npmplus/db/database.sqlite" "PRAGMA integrity_check;" 2>>"$ERR" | tr '\n' ';' || echo error)" +/usr/local/sbin/homelab-backup-npmplus-kuma-configs:100: KUMA_INTEGRITY="$(sqlite3 "$SNAP/edge-vm/kuma/db/kuma.db" "PRAGMA integrity_check;" 2>>"$ERR" | tr '\n' ';' || echo error)" +/usr/local/sbin/homelab-backup-npmplus-kuma-configs:134: rm -rf "$RESTORE" +/usr/local/sbin/homelab-backup-npmplus-kuma-configs:138: RESTORE_NPM_INTEGRITY="$(sqlite3 "$RESTORE/edge-vm/npmplus/db/database.sqlite" "PRAGMA integrity_check;" 2>>"$ERR" | tr '\n' ';' || echo error)" +/usr/local/sbin/homelab-backup-npmplus-kuma-configs:139: RESTORE_KUMA_INTEGRITY="$(sqlite3 "$RESTORE/edge-vm/kuma/db/kuma.db" "PRAGMA integrity_check;" 2>>"$ERR" | tr '\n' ';' || echo error)" +/usr/local/sbin/homelab-stage-2cloud-flat-canary:31:(cd "$STAGE"; FILES=("$ARCHIVE" manifest.json latest-backup-index.json); [[ -f metadata.json ]] && FILES+=(metadata.json); sha256sum "${FILES[@]}" > checksums.sha256; sha256sum -c checksums.sha256 >/dev/null; tar -tzf "$ARCHIVE" >/dev/null; jq -e . manifest.json >/dev/null) +/usr/local/sbin/homelab-stage-2cloud-flat-canary:34: if rclone lsf "$REMOTE_DIR" --max-depth 1 2>/dev/null | grep -q .; then echo "REFUSE NONEMPTY CANARY TARGET $REMOTE_DIR" >&2; exit 6; fi +/usr/local/sbin/homelab-stage-2cloud-flat-canary:35: rclone mkdir "$REMOTE_DIR" +/usr/local/sbin/homelab-stage-2cloud-flat-canary:36: for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f "$STAGE/$F" ]] && rclone copyto "$STAGE/$F" "$REMOTE_DIR/$F" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done +/usr/local/sbin/homelab-stage-2cloud-flat-canary:37: CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-flat-chunked "$STAGE/$ARCHIVE" "$REMOTE_DIR/$ARCHIVE" +/usr/local/sbin/homelab-stage-2cloud-flat-canary:39: rclone copyto "$STAGE/latest-backup-index.json" "$LATEST_FILE" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-stage-2cloud-flat-canary:40: rclone cat "$LATEST_FILE" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null +/usr/local/sbin/homelab-stage-2cloud-flat-canary:45:rm -rf "$STAGE" "$VERIFY" +/usr/local/sbin/homelab-router-running-config-mail-cloud:62: rm -f "$CONFIG" +/usr/local/sbin/homelab-router-running-config-mail-cloud:66: rm -f "$CONFIG" +/usr/local/sbin/homelab-router-running-config-mail-cloud:73: if ( cd "$BACKUP_ROOT" && sha256sum -c "$(basename "$SHA_FILE")" ) >"$WORK/sha-check.out" 2>"$WORK/sha-check.err"; then +/usr/local/sbin/homelab-router-running-config-mail-cloud:79: if rclone mkdir "$REMOTE:$DEST" >"$WORK/rclone-mkdir.out" 2>"$WORK/rclone-mkdir.err" && homelab-rclone-copy-chunked "$ARCHIVE" "$REMOTE:$DEST/$(basename "$ARCHIVE")" >"$WORK/rclone-upload.out" 2>"$WORK/rclone-upload.err" && rclone copyto "$SHA_FILE" "$REMOTE:$DEST/$(basename "$SHA_FILE")" --checksum >"$WORK/rclone-upload-sha.out" 2>"$WORK/rclone-upload-sha.err"; then +/usr/local/sbin/homelab-router-running-config-mail-cloud:86: if homelab-rclone-cat-chunked "$REMOTE:$DEST/$(basename "$ARCHIVE")" >"$DL" 2>"$WORK/rclone-download.err"; then +/usr/local/sbin/homelab-router-running-config-mail-cloud:93:rclone lsf "$REMOTE:$TARGET_BASE" --dirs-only 2>/dev/null | sed 's#/$##' | sort > "$DIR_LIST" || RETENTION_RC=$? +/usr/local/sbin/homelab-router-running-config-mail-cloud:98: [ -n "$OLD" ] && rclone purge "$REMOTE:$TARGET_BASE/$OLD" >>"$WORK/retention.out" 2>>"$WORK/retention.err" +/usr/local/sbin/homelab-router-running-config-mail-cloud:103: rm -f "$old" "$old.sha256" +/usr/local/sbin/homelab-router-running-config-mail-cloud:116:rm -rf "$WORK" +/usr/local/sbin/appbackup-restore-check:22:ROOT="/var/tmp/homelab-app-restore-check/${JOB}" +/usr/local/sbin/appbackup-restore-check:24:rm -rf "$ROOT" && mkdir -p "$ROOT/a" "$ROOT/b" +/usr/local/sbin/appbackup-restore-check:25:trap "echo \"ERROR app restore-check failed; kept: $ROOT\"; exit 1" ERR +/usr/local/sbin/appbackup-restore-check:26:echo "restore-check app=$APP job=$JOB schema=$SCHEMA" +/usr/local/sbin/appbackup-restore-check:27:rclone copy "$RA" "$ROOT/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >"${LOGBASE}-a.log" 2>&1 +/usr/local/sbin/appbackup-restore-check:30:(cd "$ROOT/a" && sha256sum -c checksums.sha256 >"${LOGBASE}-a-sha.log" && ARCHIVE="$(jq -r ".archive_name" manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null) +/usr/local/sbin/appbackup-restore-check:31:echo "OK cloud A app restore-check" +/usr/local/sbin/appbackup-restore-check:32:rclone copy "$RB" "$ROOT/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >"${LOGBASE}-b.log" 2>&1 +/usr/local/sbin/appbackup-restore-check:35:(cd "$ROOT/b" && sha256sum -c checksums.sha256 >"${LOGBASE}-b-sha.log" && ARCHIVE="$(jq -r ".archive_name" manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null) +/usr/local/sbin/appbackup-restore-check:36:echo "OK cloud B app restore-check" +/usr/local/sbin/appbackup-restore-check:37:rm -rf "$ROOT" +/usr/local/sbin/appbackup-restore-check:38:rmdir /var/tmp/homelab-app-restore-check 2>/dev/null || true +/usr/local/sbin/appbackup-restore-check:39:rm -f "${LOGBASE}-a.log" "${LOGBASE}-a-sha.log" "${LOGBASE}-b.log" "${LOGBASE}-b-sha.log" +/usr/local/sbin/appbackup-restore-check:41:echo "OK app restore-check both clouds" +/usr/local/sbin/backupctl:12: check) bash -n "$SCRIPT" && [[ -f "$STATE" ]] && jq -e ".status == \"success\" and .local_backup_deleted == true" "$STATE" >/dev/null && echo "OK backup framework" ;; +/usr/local/sbin/backupctl:13: restore-check) backup-restore-check ;; +/usr/local/sbin/backupctl:14: audit) backupctl check && backup-restore-check && backupctl retention 7 dry-run && backup-report >/dev/null && echo "OK audit finished" ;; +/usr/local/sbin/backupctl:16: remotes) source /etc/homelab-backup.env && NODE="$(hostname -s)" && echo "A=$REMOTE_A" && echo "B=$REMOTE_B" && rclone lsd "${REMOTE_A%/}/${NODE}/emergency-pack" | tail -n 5 && rclone lsd "${REMOTE_B%/}/${NODE}/emergency-pack" | tail -n 5 ;; +/usr/local/sbin/backupctl:19: *) echo "usage: backupctl run|status|lastlog|check|restore-check|audit|report|remotes|retention|timer"; exit 2 ;; +/usr/local/sbin/homelab-capacity-risk:32:LARGEST_BYTES="$(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-*.vma.zst" -o -name "vzdump-lxc-*.tar.zst" \) -printf "%s\n" 2>/dev/null | sort -n | tail -n1)" +/usr/local/sbin/homelab-backupctl:6: audit) echo "=== EMERGENCY AUDIT ==="; backupctl audit; echo "=== APP FRAMEWORK ==="; appbackupctl check; echo "=== APP AUDITS ==="; for app in $(app_list); do echo "### app=$app"; appbackup-restore-check "$app"; appbackup-retention 7 dry-run "$app"; done; echo "OK full backup audit" ;; +/usr/local/sbin/homelab-staging-check.sh:11:echo "=== homelab staging check: ${HOST} ===" +/usr/local/sbin/homelab-secret-exposure-guard:18:BADFILES="$(find "$REPO" -path "$REPO/.git" -prune -o -type f \( -iname "*.key" -o -iname "*.pem" -o -iname "*.p12" -o -iname "*.pfx" -o -iname "[SENSITIVE_FILE]" -o -iname ".env" -o -iname ".env.*" \) -print 2>/dev/null)" +/usr/local/sbin/homelab-edge-vm-offhost-freshness:9:OFFHOST_DIR=/mnt/staging/offhost/edge-vm-vzdump-from-pve03 +/usr/local/sbin/homelab-edge-vm-offhost-freshness:85: /var/lib/vz/dump/vzdump-qemu-130-*.vma.zst) ;; +/usr/local/sbin/homelab-edge-vm-offhost-freshness:140: copy_out="$(run_pve03 copy "$SOURCE_PATH" "$SOURCE_SIZE" "$SOURCE_SHA" "$DST_IP" "$OFFHOST_DIR")" +/usr/local/sbin/homelab-edge-vm-offhost-freshness:141: grep -Fq 'SOURCE_COPY_STATUS=OK' <<<"$copy_out" +/usr/local/sbin/homelab-edge-vm-offhost-freshness:156:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-backup NODE=%s VMID=%s ARCHIVE=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\n' \ +/usr/local/sbin/homelab-edge-vm-offhost-freshness:158: >"$HEALTH_DIR/edge-vm-vzdump-backup.txt" +/usr/local/sbin/homelab-edge-vm-offhost-freshness:159:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-offhost NODE=%s VMID=%s HOST=%s DEST=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\n' \ +/usr/local/sbin/homelab-edge-vm-offhost-freshness:161: >"$HEALTH_DIR/edge-vm-vzdump-offhost.txt" +/usr/local/sbin/homelab-edge-vm-offhost-freshness:162:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-restore-verify HOST=%s VMID=%s ARCHIVE=%s SIZE=%s SHA256=%s ZSTD_RC=0 VMA_RC=0 NOTES=nightly_existing_dump_freshness\n' \ +/usr/local/sbin/homelab-edge-vm-offhost-freshness:164: >"$HEALTH_DIR/edge-vm-vzdump-restore.txt" +/usr/local/sbin/homelab-edge-vm-offhost-freshness:167: "$HEALTH_DIR/edge-vm-vzdump-backup.txt" \ +/usr/local/sbin/homelab-edge-vm-offhost-freshness:168: "$HEALTH_DIR/edge-vm-vzdump-offhost.txt" \ +/usr/local/sbin/homelab-edge-vm-offhost-freshness:169: "$HEALTH_DIR/edge-vm-vzdump-restore.txt" +/usr/local/sbin/homelab-external-probe-vps:20:rm -f "$TMP" +/usr/local/sbin/sqlite-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@"$H" "sudo rm -f /tmp/sqlite-app-${APP}-${JOB_ID}.tgz" +/usr/local/sbin/sqlite-apps-backup-run:21: rm -rf "$STAGE/payload" +/usr/local/sbin/homelab-desired-state-sync:10:find "$REPO" -mindepth 1 -maxdepth 1 ! -name .git -exec rm -rf {} + 2>/dev/null +/usr/local/sbin/homelab-desired-state-sync:24:[ -f /root/HOMELAB_SAFETY_NET_CHECKPOINT.md ] && cp -a /root/HOMELAB_SAFETY_NET_CHECKPOINT.md "$REPO/docs/safety-net-checkpoint.md" 2>/dev/null || true +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:4:CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:9:: "${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}" +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:13:LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:19:log(){ printf '%s RESTORE_CHECK %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" | tee -a "$LOGDIR/restore-check.log"; } +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:22:command -v rclone >/dev/null +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:28: if rclone lsf "${remote_path}.parts/" > "$plist" 2>/dev/null && grep -q '^part-' "$plist"; then +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:29: while IFS= read -r part; do [ -n "$part" ] && rclone cat "${remote_path}.parts/${part}" >> "$out"; done < <(grep '^part-' "$plist" | sort) +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:31: rclone copyto "$remote_path" "$out" >/dev/null +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:33: rm -f "$plist" +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:36:check_one_remote(){ +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:39: latest="$(rclone lsf --dirs-only "${rbase}/" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)" +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:85: [ "$(sqlite3 "$extract/var-lib-skladchik-reports-monitor/state.sqlite" 'PRAGMA integrity_check;' | tail -n 1)" = "ok" ] +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:97:check_one_remote CLOUD_1 "$CLOUD_1" +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:98:check_one_remote CLOUD_2 "$CLOUD_2" +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:101:find "$WORK_ROOT" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_LOCAL}" | while read -r D; do [ -n "$D" ] && rm -rf "${WORK_ROOT}/${D}"; done +/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch:77: check=False, +/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch:101:def check() -> int: +/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch:126: check=False, +/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch:223: return check() +/usr/local/sbin/skladchik-reports-monitor-backup:28:command -v rclone >/dev/null +/usr/local/sbin/skladchik-reports-monitor-backup:29:[ -x /usr/local/sbin/homelab-rclone-copy-chunked ] || { echo "missing /usr/local/sbin/homelab-rclone-copy-chunked" >&2; exit 10; } +/usr/local/sbin/skladchik-reports-monitor-backup:36:cleanup(){ rm -rf "$TMP"; } +/usr/local/sbin/skladchik-reports-monitor-backup:75: /usr/local/sbin/homelab-rclone-copy-chunked "$ARCHIVE" "${DEST_BASE}/$(basename "$ARCHIVE")" +/usr/local/sbin/skladchik-reports-monitor-backup:76: /usr/local/sbin/homelab-rclone-copy-chunked "${ARCHIVE}.sha256" "${DEST_BASE}/$(basename "$ARCHIVE").sha256" +/usr/local/sbin/skladchik-reports-monitor-backup:77: /usr/local/sbin/homelab-rclone-copy-chunked "${OUTDIR}/manifest.txt" "${DEST_BASE}/manifest.txt" +/usr/local/sbin/skladchik-reports-monitor-backup:82: mapfile -t OLD < <(rclone lsf --dirs-only "${REMOTE}${REMOTE_BASE}/daily/" 2>/dev/null | sed 's#/$##' | sort | head -n -"${KEEP_DAILY}" || true) +/usr/local/sbin/skladchik-reports-monitor-backup:83: for D in "${OLD[@]}"; do [ -n "$D" ] && rclone purge "${REMOTE}${REMOTE_BASE}/daily/${D}" >/dev/null 2>&1 || true; done +/usr/local/sbin/skladchik-reports-monitor-backup:85:find "${LOCAL_ROOT}/daily" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_DAILY}" | while read -r D; do [ -n "$D" ] && rm -rf "${LOCAL_ROOT}/daily/${D}"; done +/usr/local/sbin/homelab-vm-local-dumps-retention:32: mapfile -t JOBS < <(rclone lsf "$BASE/" --dirs-only 2>/dev/null | sed 's#/$##' | sort) +/usr/local/sbin/homelab-vm-local-dumps-retention:56: rclone purge "$PATH_REMOTE" >> "$REPORT" 2>&1 && DELETED=$((DELETED+1)) || BAD=$((BAD+1)) +/usr/local/sbin/homelab-extended-appbackup-run:12:run_one(){ APP="$1"; HOST="$2"; PATHS="$3"; NOTE="$4"; CONS="$5"; JOB_ID="homelab-${NODE}-app-${APP}-${RUN_TS}"; export JOB_ID; SHORT_TS="${RUN_TS:2:6}-${RUN_TS:9:6}"; export BACKUP_PATH_REL="$(homelab-backup-path job a "$APP" "$SHORT_TS")"; STAGE="${STAGING_ROOT}/${JOB_ID}"; PAYLOAD="$STAGE/payload"; ARCHIVE="${JOB_ID}.tar.gz"; LOG="/tmp/extended-appbackup-${APP}-${RUN_TS}.log"; rm -rf "$STAGE"; mkdir -p "$PAYLOAD/local"; echo "### app=$APP" >> "$REPORT"; echo "host=$HOST" >> "$REPORT"; echo "source_note=$NOTE" >> "$REPORT"; ssh -n -o BatchMode=yes -o ConnectTimeout=30 "$HOST" "sudo tar -C / --ignore-failed-read --exclude=.env --exclude=*.env --exclude=.env.* --exclude=*.bak* -czf - $PATHS" > "$PAYLOAD/local/${APP}-data.tar.gz" 2>"$LOG"; TAR_RC=$?; if [ "$TAR_RC" -eq 0 ] && tar -tzf "$PAYLOAD/local/${APP}-data.tar.gz" >/dev/null 2>&1; then SIZE="$(stat -c%s "$PAYLOAD/local/${APP}-data.tar.gz" 2>/dev/null || echo 0)"; echo "archive_payload_size_bytes=$SIZE" >> "$REPORT"; if [ "$SIZE" -le 128 ]; then echo "backup=BAD reason=archive_too_small" >> "$REPORT"; BAD=$((BAD+1)); rm -rf "$STAGE"; else sha256sum "$PAYLOAD/local/${APP}-data.tar.gz" > "$PAYLOAD/local/${APP}-data.tar.gz.sha256"; jq -n --arg type "extended-appbackup" --arg app "$APP" --arg host "$HOST" --arg source_note "$NOTE" --arg archive_payload_size_bytes "$SIZE" --arg env_files_read "no" --arg secret_values_printed "no" --arg consistency "$CONS" "{type:\$type,app_name:\$app,host:\$host,source_note:\$source_note,archive_payload_size_bytes:(\$archive_payload_size_bytes|tonumber),env_files_read:\$env_files_read,secret_values_printed:\$secret_values_printed,consistency:\$consistency}" > "$STAGE/metadata.json"; tar -C "$PAYLOAD" -czf "$STAGE/$ARCHIVE" .; rm -rf "$PAYLOAD"; JOB_ID="$JOB_ID" homelab-stage-2cloud-flat "$APP" "$STAGE" "$ARCHIVE" > "$LOG" 2>&1; RC=$?; if [ "$RC" -eq 0 ]; then echo "backup=OK" >> "$REPORT"; grep -E "^\[STEP\]|^\[SUCCESS\]" "$LOG" >> "$REPORT"; appbackupctl restore-check "$APP" >> "$REPORT" 2>&1 && echo "restore_check=OK" >> "$REPORT" || echo "restore_check=WARN_non_registry" >> "$REPORT"; OK=$((OK+1)); else echo "backup=BAD rc=$RC" >> "$REPORT"; grep -E "^\[ERROR\]|\[STEP\]|ERROR|error|failed|Failed|required|missing|denied|forbidden|checksum|rclone|copy|mkdir|jq|tar" "$LOG" | tail -n 60 >> "$REPORT"; BAD=$((BAD+1)); fi; fi; else echo "tar=BAD rc=$TAR_RC" >> "$REPORT"; grep -E "ERROR|error|failed|Failed|denied|No such|Cannot|tar:" "$LOG" | tail -n 40 >> "$REPORT"; BAD=$((BAD+1)); rm -rf "$STAGE"; fi; echo >> "$REPORT"; } +/usr/local/sbin/homelab-extended-appbackup-run:21:run_one healthchecks "$MON" "var/lib/healthchecks opt/stacks/healthchecks/local_settings.py" "healthchecks data plus local settings" "emergency_data_config_archive" +/usr/local/sbin/homelab-mkdocs-auto-refresh:53: rm -f /tmp/homelab-mkdocs-auto-refresh-git-fetch.out /tmp/homelab-mkdocs-auto-refresh-git-fetch.err +/usr/local/sbin/homelab-mkdocs-auto-refresh:57:rm -f /tmp/homelab-mkdocs-auto-refresh-git-fetch.out /tmp/homelab-mkdocs-auto-refresh-git-fetch.err +/usr/local/sbin/homelab-mkdocs-auto-refresh:77:ssh "${EDGE_SSH_OPTS[@]}" "$EDGE_HOST" "rm -rf '$STAGE' && mkdir -p '$STAGE'" +/usr/local/sbin/homelab-mkdocs-auto-refresh:85:rm -rf "$BASE/site.new" "$BASE/site.old" +/usr/local/sbin/homelab-mkdocs-auto-refresh:93:rm -rf "$BASE/site.old" "$STAGE" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:30:CHKS="$WORK/checksums.txt" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:42: F="$(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-${id}-*.vma.zst" -o -name "vzdump-lxc-${id}-*.tar.zst" \) -printf "%T@ %p\n" 2>/dev/null | sort -n | tail -n 1 | cut -d" " -f2-)" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:53: echo "$BN" | grep -q "^vzdump-qemu-" && KIND="qemu" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:54: echo "$BN" | grep -q "^vzdump-lxc-" && KIND="lxc" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:64: rm -rf "$CDIR" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:117: rclone mkdir "$DEST/files" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:122: rclone mkdir "$DEST/files/$bn.chunks" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:126: rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:128: rclone copyto "$CDIR/chunks.sha256" "$DEST/files/$bn.chunks/chunks.sha256" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:131: rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:135: rclone copyto "$CHKS" "$DEST/checksums.txt" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:136: rclone copyto "$FILES_TSV" "$DEST/files.tsv" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:137: rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:144: rm -rf "$VBASE" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:146: rclone copyto "$DEST/manifest.json" "$VBASE/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:154: rclone copy "$DEST/files/$bn.chunks" "$VDIR" --filter "+ part-*" --filter "+ chunks.sha256" --filter "- *" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:163: rm -rf "$VDIR" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:166: rclone copyto "$DEST/files/$bn" "$VBASE/$bn" --transfers 1 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +/usr/local/sbin/homelab-vm-local-dumps-2cloud:168: rm -f "$VBASE/$bn" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:171: rm -rf "$VBASE" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:184:rm -rf "$VERIFY_ROOT/$JOB_ID" +/usr/local/sbin/homelab-vm-local-dumps-2cloud:185:rm -rf "$CHUNK_ROOT/$JOB_ID" +/usr/local/sbin/homelab-gitea-secondary-backup:18: rm -f "$TMP" +/usr/local/sbin/homelab-gitea-secondary-backup:31:docker exec -u git -w /tmp gitea sh -lc \"rm -f /tmp/\$NAME && gitea dump -c /data/gitea/conf/app.ini -f /tmp/\$NAME\" +/usr/local/sbin/homelab-gitea-secondary-backup:33:docker exec gitea rm -f \"/tmp/\$NAME\" +/usr/local/sbin/homelab-gitea-secondary-backup:100: rm -f -- "$FILE" "$FILE.sha256" +/usr/local/sbin/homelab-gitea-secondary-backup:123: rm -f -- "$FILE" "$FILE.sha256" +/usr/local/sbin/homelab-gitea-secondary-backup:136: rm -f -- "$FILE" "$FILE.sha256" +/usr/local/sbin/homelab-rclone-copy-chunked:7:TMP_BASE=${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks} +/usr/local/sbin/homelab-rclone-copy-chunked:13:cleanup(){ rm -rf "$WORK"; } +/usr/local/sbin/homelab-rclone-copy-chunked:24:rclone purge "${DEST}.parts" >/dev/null 2>&1 || true +/usr/local/sbin/homelab-rclone-copy-chunked:25:rclone mkdir "${DEST}.parts" +/usr/local/sbin/homelab-rclone-copy-chunked:26:rclone copy "$PARTS" "${DEST}.parts" --transfers 2 --checkers 4 --retries 5 --low-level-retries 20 --stats 30s --log-level INFO +/usr/local/sbin/homelab-rclone-copy-chunked:27:rclone copyto "$WORK/${NAME}.sha256" "${DEST}.sha256" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO +/usr/local/sbin/homelab-rclone-copy-chunked:28:rclone copyto "$WORK/MANIFEST.txt" "${DEST}.manifest.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO +/usr/local/sbin/homelab-rclone-copy-chunked:29:rclone copyto "$WORK/MANIFEST.parts.txt" "${DEST}.parts/MANIFEST.parts.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO +/usr/local/sbin/homelab-rclone-copy-chunked:30:REMOTE_COUNT=$(rclone lsf "${DEST}.parts" --files-only | grep -c "^part-" || true) +/usr/local/sbin/homelab-rclone-copy-chunked:32:ACTUAL=$(while read -r PART SIZE_PART; do case "$PART" in part-*) rclone cat "${DEST}.parts/$PART" ;; esac; done < "$WORK/MANIFEST.parts.txt" | sha256sum | cut -d " " -f 1) +/usr/local/sbin/homelab-router-hard-power-cycle:12:case "$MODE" in check|cycle) ;; *) echo HARD_WRAPPER_MODE_INVALID=yes; exit 64;; esac +/usr/local/sbin/homelab-router-hard-power-cycle:23:SAFE_OUT="$(printf '%s\n' "$OUT" | sed -E '/(PASSWORD|TOKEN|SECRET|PRIVATE_KEY|ROOT_PASSWORD|ENCRYPTION_KEY|COOKIE|SESSION|AUTHORIZATION|API_KEY|LOCAL_KEY|rclone[.]conf|[.]env)/Id')" +/usr/local/sbin/homelab-health-metrics-write:10:rm -f "$tmp" +/usr/local/sbin/homelab-rclone-cat-flat-chunked:5:trap "rm -f \"$TMP\"" EXIT +/usr/local/sbin/homelab-rclone-cat-flat-chunked:6:rclone cat "${DEST}.parts.txt" > "$TMP" +/usr/local/sbin/homelab-rclone-cat-flat-chunked:8:while read -r PART PART_SIZE; do [[ "$PART" =~ ^part-[0-9]{4}$ && "$PART_SIZE" =~ ^[0-9]+$ ]] || { echo "BAD FLAT PART MANIFEST" >&2; exit 2; }; rclone cat "${DEST}.${PART}"; COUNT=$((COUNT+1)); done < "$TMP" +/usr/local/sbin/homelab-vm-backup-policy:29:echo "## Latest local vzdump catalog" >> "$REPORT" +/usr/local/sbin/homelab-vm-backup-policy:31:for f in $(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-*.vma.zst" -o -name "vzdump-lxc-*.tar.zst" \) 2>/dev/null | sort); do b="$(basename "$f")"; size="$(stat -c %s "$f" 2>/dev/null || echo 0)"; mt="$(date -u -d "@$(stat -c %Y "$f" 2>/dev/null || echo 0)" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)"; if echo "$b" | grep -q "^vzdump-qemu-"; then kind=qemu; vmid="$(echo "$b" | sed -n "s/^vzdump-qemu-\([0-9]\+\)-.*/\1/p")"; name="$(qm config "$vmid" 2>/dev/null | awk -F": " "/^name:/{print \$2; exit}")"; else kind=lxc; vmid="$(echo "$b" | sed -n "s/^vzdump-lxc-\([0-9]\+\)-.*/\1/p")"; name="$(pct config "$vmid" 2>/dev/null | awk -F": " "/^hostname:/{print \$2; exit}")"; fi; printf "%s\t%s\t%s\t%s\t%s\t%s\n" "$kind" "$vmid" "${name:-unknown}" "$size" "$mt" "$f" >> "$CATALOG"; done +/usr/local/sbin/homelab-vm-backup-policy:37:echo "- Existing local dumps can be uploaded/verified to two clouds without creating new vzdump archives." >> "$REPORT" +/usr/local/sbin/homelab-vm-backup-policy:38:echo "- New full vzdump for VM 170/171 is not started automatically because each VM has 120 GB bootdisk and /var/tmp has limited free space." >> "$REPORT" +/usr/local/sbin/homelab-vm-backup-policy:42:echo "## Coverage checks" >> "$REPORT" +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:4:cat /var/lib/skladchik-reports-monitor-backup/restore-check-health.txt 2>/dev/null || true +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:7:systemctl is-enabled skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/enabled=/' || true +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:8:systemctl is-active skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/active=/' || true +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:9:systemctl list-timers --all | grep skladchik-reports-monitor-backup-restore-check || true +/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:12:tail -n 80 /var/log/skladchik-reports-monitor-backup/restore-check.log 2>/dev/null || true +/usr/local/sbin/homelab-context-refresh-direct:5:trap "rm -rf \"$d\"" EXIT +/usr/local/sbin/homelab-context-refresh-direct:32:printf "{\"schema_version\":1,\"channel\":\"homelab-runtime\",\"command_id\":\"%s\",\"status\":\"OK\",\"rc\":0,\"host\":\"pve01\",\"mode\":\"read-only\",\"component\":\"cluster-context\",\"started_at_utc\":\"%s\",\"finished_at_utc\":\"%s\",\"reference_register_checked\":true,\"reference_sha256\":\"%s\",\"error_register_checked\":true,\"error_register_sha256\":\"%s\",\"changes_made\":false,\"sanitized\":true,\"secrets_included\":false,\"private_addresses_included\":false,\"output\":\"AUTOMATIC_CLUSTER_CONTEXT_REFRESH=OK\"}\n" "$id" "$started" "$finished" "$ref" "$err" >"$d/$id.json" +/usr/local/sbin/homelab-context-refresh-direct:34:ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] debian@[PRIVATE_IP] "sudo -n /usr/local/sbin/homelab-runtime-receive $id /tmp/$id.json /tmp/$id.txt && rm -f /tmp/$id.json /tmp/$id.txt" +/usr/local/sbin/homelab-evidence-seal:16: rm -f "$TMP" +/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:15:DUMP_SIZE="$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$REMOTE" "sudo -n rm -f \"$REMOTE_DUMP\"; sudo -n -u clusteradmin pg_dump --format=custom --no-owner --no-privileges --file=\"$REMOTE_DUMP\" clusteradmin; sudo -n test -s \"$REMOTE_DUMP\"; sudo -n -u clusteradmin pg_restore --list \"$REMOTE_DUMP\" >/dev/null; sudo -n chown debian:debian \"$REMOTE_DUMP\"; sudo -n chmod 600 \"$REMOTE_DUMP\"; stat -c %s \"$REMOTE_DUMP\"")" +/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:18:ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$REMOTE" "rm -f \"$REMOTE_DUMP\"" +/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:23:rm -f "$STAGE/runtime.tgz" +/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:25:jq -n --arg app "$APP" --arg host "cluster-admin" --arg engine "postgresql-custom-dump+files" --argjson files "$FILES" --argjson dump_size "$DUMP_SIZE" "{app:\$app,engine:\$engine,source_host:\$host,source_paths:[\"cluster-admin:/opt/cluster-admin-incident-engine\",\"cluster-admin:/etc/cluster-admin-incident-engine\",\"cluster-admin:/var/lib/cluster-admin-incident-engine\",\"cluster-admin:PostgreSQL clusteradmin\"],file_count:\$files,database_dump_size_bytes:\$dump_size,database_dump_format:\"custom\",database_restore_list_check:\"ok\",includes_runtime_credentials:true,note:\"Incident engine application database state and runtime configuration\"}" > "$STAGE/metadata.json" +/usr/local/sbin/appbackup-retention:32:trap "rm -rf \"$WORK\"" EXIT +/usr/local/sbin/appbackup-retention:35: rclone lsf "${REMOTE%/}" --dirs-only --max-depth 1 > "$ROOT_LIST" +/usr/local/sbin/appbackup-retention:36: rclone lsf "${REMOTE%/}/${NODE}/apps/${CANONICAL}" --dirs-only --max-depth 1 > "$LEGACY_LIST" +/usr/local/sbin/appbackup-retention:39: rclone lsf "${REMOTE%/}/hb" --dirs-only --max-depth 1 > "$HB_LIST" +/usr/local/sbin/appbackup-retention:40: if grep -Fxq "a/" "$HB_LIST"; then rclone lsf "${REMOTE%/}/hb/a" --dirs-only --max-depth 1 > "$NEW_LIST"; fi +/usr/local/sbin/appbackup-retention:73: rclone cat "${REMOTE%/}/${LATEST_REL}/manifest.json" | jq -e --arg job "$LATEST" ".job_id == \$job" >/dev/null || { echo "REFUSE latest missing/bad remote_index=$INDEX"; exit 5; } +/usr/local/sbin/appbackup-retention:85: if [[ "$MODE" == apply ]]; then echo "DELETE schema=dual path=$REL"; rclone purge "$TARGET"; else echo "WOULD_DELETE schema=dual path=$REL"; fi +/usr/local/sbin/homelab-change-transaction:160: check=False, +/usr/local/sbin/homelab-change-transaction:195: latest_tmp.unlink(missing_ok=True) +/usr/local/sbin/homelab-change-transaction:421: " homelab-change-transaction run PLAN --confirm PLAN" +/usr/local/sbin/homelab-check-private-vpn-hosts:15:trap 'rm -f "$tmp"' EXIT +/usr/local/sbin/homelab-backup-vaultwarden:26:rm -rf "$RESTORE" +/usr/local/sbin/homelab-backup-vaultwarden:32:INTEGRITY="$(sqlite3 "$DB" 'PRAGMA integrity_check;')" +/usr/local/sbin/homelab-sops-edge-secret-coverage:97: env = os.environ.copy() +/usr/local/sbin/homelab-sops-edge-secret-coverage:113: env=os.environ.copy(), +/usr/local/sbin/homelab-sops-edge-secret-coverage:176: checked_at = subprocess.check_output(["date", "-u", "+%Y-%m-%dT%H:%M:%SZ"], text=True).strip() +/usr/local/sbin/homelab-sops-edge-secret-coverage:242: f"CHECKED_AT_UTC={checked_at}", +/usr/local/sbin/homelab-health-metrics:16:category_for_check(){ +/usr/local/sbin/homelab-health-metrics:22: *backup*|*vzdump*) printf "%s" "backup" ;; +/usr/local/sbin/homelab-health-metrics:31:tier_for_check(){ +/usr/local/sbin/homelab-health-metrics:34: npmplus-kuma-config-*|authentik-*|gitea-*|vaultwarden-*|netbox-*|mail-cloud-restore-drill|edge-vm-vzdump-*|mkdocs-runtime-publish|npmplus-certificate-expiry) printf "%s" "P0" ;; +/usr/local/sbin/homelab-health-metrics:69: check="${base%.txt}" +/usr/local/sbin/homelab-health-metrics:70: category="$(category_for_check "$check")" +/usr/local/sbin/homelab-health-metrics:71: tier="$(tier_for_check "$check")" +/usr/local/sbin/homelab-health-metrics:75: check_esc="$(esc_label "$check")" +/usr/local/sbin/homelab-health-metrics:79: labels="host=\"$host_esc\",check=\"$check_esc\",category=\"$cat_esc\",tier=\"$tier_esc\",file=\"$file_esc\"" +/usr/local/sbin/homelab-backup-health-evidence:8:backupctl check >/tmp/backup-health-emergency.log 2>&1 && EMERGENCY_OK=1 || true +/usr/local/sbin/homelab-backup-health-evidence:9:appbackupctl check >/tmp/backup-health-appframework.log 2>&1 && APPS_OK=1 || true +/usr/local/sbin/homelab-drift-check:7:HEALTH=/var/lib/homelab-health/drift-check.txt +/usr/local/sbin/homelab-drift-check:24:echo "STATUS=$STATUS TS=$(date -u +%Y-%m-%dT%H:%M:%SZ) TYPE=drift-check NOTE=$NOTE DIFF=$DIFF BASE=$BASE CURRENT=$CUR" > "$HEALTH" +/usr/local/sbin/crypto-portfolio-chart-health-check:5:trap 'rm -f "$TMP"' EXIT +/usr/local/sbin/homelab-rclone-copy-flat-chunked:6:TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" +/usr/local/sbin/homelab-rclone-copy-flat-chunked:12:trap "rm -rf \"$WORK\"" EXIT +/usr/local/sbin/homelab-rclone-copy-flat-chunked:23:while read -r PART PART_SIZE; do [[ "$PART" =~ ^part-[0-9]{4}$ && "$PART_SIZE" =~ ^[0-9]+$ ]] || { echo "BAD PART MANIFEST" >&2; exit 4; }; rclone copyto "$WORK/parts/$PART" "${DEST}.${PART}" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR; REMOTE_BYTES="$(rclone size --json "${DEST}.${PART}" | jq -r ".bytes // -1")"; [[ "$REMOTE_BYTES" = "$PART_SIZE" ]] || { echo "REMOTE PART SIZE MISMATCH $PART" >&2; exit 5; }; UPLOADED=$((UPLOADED+1)); done < "$WORK/MANIFEST.parts.txt" +/usr/local/sbin/homelab-rclone-copy-flat-chunked:25:rclone copyto "$WORK/${NAME}.sha256" "${DEST}.sha256" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-rclone-copy-flat-chunked:26:rclone copyto "$WORK/MANIFEST.txt" "${DEST}.manifest.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-rclone-copy-flat-chunked:27:rclone copyto "$WORK/MANIFEST.parts.txt" "${DEST}.parts.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR +/usr/local/sbin/homelab-rclone-copy-flat-chunked:28:ACTUAL="$(homelab-rclone-cat-flat-chunked "$DEST" | sha256sum | awk "{print \$1}")" +/usr/local/sbin/homelab-final-readiness-gate:10:check(){ name="$1"; shift; echo "### $name" >> "$REPORT"; timeout "${CHECK_TIMEOUT_SECONDS:-300}" "$@" >> "$REPORT" 2>&1; rc=$?; if [ "$rc" -eq 0 ]; then echo "PASS $name" >> "$REPORT"; PASS=$((PASS+1)); else echo "FAIL $name rc=$rc" >> "$REPORT"; FAIL=$((FAIL+1)); BAD=$((BAD+1)); fi; echo "" >> "$REPORT"; } +/usr/local/sbin/homelab-final-readiness-gate:16:check "private overall endpoint" bash -c "curl -fsS http://[PRIVATE_IP]:9101/overall.txt | grep -q STATUS=" +/usr/local/sbin/homelab-final-readiness-gate:17:check "public overall endpoint" bash -c "curl -k -fsS https://backup.gram1.ru/overall.txt | grep -q STATUS=" +/usr/local/sbin/homelab-final-readiness-gate:18:check "backup framework evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/backup-framework.txt && grep -q SOURCE_STATUS=OK /var/lib/homelab-health/backup-framework.txt && grep -q FRESHNESS=FRESH /var/lib/homelab-health/backup-framework.txt" +/usr/local/sbin/homelab-final-readiness-gate:19:check "control-plane restore-check" appbackupctl restore-check safety-net-control-plane +/usr/local/sbin/homelab-final-readiness-gate:20:check "uptime-kuma restore-check" appbackupctl restore-check uptime-kuma +/usr/local/sbin/homelab-final-readiness-gate:21:check "backup SLA evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/backup-sla.txt" +/usr/local/sbin/homelab-final-readiness-gate:22:check "backup coverage matrix evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/backup-coverage-matrix.txt" +/usr/local/sbin/homelab-final-readiness-gate:23:check "extended appbackup evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/extended-appbackup.txt" +/usr/local/sbin/homelab-final-readiness-gate:24:check "residual review" bash -c "grep -q STATUS=OK /var/lib/homelab-health/residual-review.txt" +/usr/local/sbin/homelab-final-readiness-gate:25:check "forum snuffleupagus" bash -c "grep -q STATUS=OK /var/lib/homelab-health/forum-snuffleupagus.txt" +/usr/local/sbin/homelab-final-readiness-gate:26:check "vm170 vm171 full strategy" bash -c "grep -q STATUS=OK /var/lib/homelab-health/vm170-vm171-full-strategy.txt" +/usr/local/sbin/homelab-final-readiness-gate:27:check "external probe vps" bash -c "grep -q STATUS=OK /var/lib/homelab-health/external-probe-vps.txt" +/usr/local/sbin/homelab-final-readiness-gate:28:check "netbird vps trust" bash -c "grep -q STATUS=OK /var/lib/homelab-health/netbird-vps-trust.txt" +/usr/local/sbin/homelab-final-readiness-gate:29:check "netbird vps trust closure" bash -c "grep -q STATUS=OK /var/lib/homelab-health/netbird-vps-trust-closure.txt" +/usr/local/sbin/homelab-final-readiness-gate:30:check "external probe runner decision" bash -c "grep -q STATUS=OK /var/lib/homelab-health/external-probe-runner-decision.txt" +/usr/local/sbin/homelab-final-readiness-gate:31:check "netbird vps trust clean seal" bash -c "grep -q STATUS=OK /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt" +/usr/local/sbin/homelab-final-readiness-gate:32:check "live tail audit" bash -c "grep -q STATUS=OK /var/lib/homelab-health/live-tail-audit.txt" +/usr/local/sbin/homelab-final-readiness-gate:34:check "duty admin v2" bash -c "grep -q STATUS=OK /var/lib/homelab-health/duty-admin-v2.txt" +/usr/local/sbin/homelab-final-readiness-gate:35:check "cluster admin observer" bash -c "grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-observer.txt" +/usr/local/sbin/homelab-final-readiness-gate:36:check "cluster admin restricted probes" bash -c "grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-restricted-probes.txt" +/usr/local/sbin/homelab-final-readiness-gate:37:check "cluster admin full observer" bash -c "grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-full-observer.txt && grep -q BAD=0 /var/lib/homelab-health/cluster-admin-full-observer.txt && grep -q WARN=0 /var/lib/homelab-health/cluster-admin-full-observer.txt" +/usr/local/sbin/homelab-final-readiness-gate:38:check "cluster admin VM Mail Cloud backup" bash -c "grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt && grep -q BAD=0 /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt && grep -q MAIL_CLOUD_2COPY_VERIFY_OK /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt" +/usr/local/sbin/homelab-final-readiness-gate:39:check "cluster admin webpanel sync" bash -c "grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-webpanel-sync.txt && grep -q BAD=0 /var/lib/homelab-health/cluster-admin-webpanel-sync.txt" +/usr/local/sbin/homelab-final-readiness-gate:40:check "cluster admin webpanel" bash -c "grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-webpanel.txt && grep -q BAD=0 /var/lib/homelab-health/cluster-admin-webpanel.txt && curl -fsS --max-time 5 http://[PRIVATE_IP]:8080/status.txt | grep -q STATUS=OK" +/usr/local/sbin/homelab-final-readiness-gate:41:check "node loss readiness" bash -c "grep -q STATUS=OK /var/lib/homelab-health/node-loss-readiness.txt" +/usr/local/sbin/homelab-final-readiness-gate:42:check "node loss runbooks" bash -c "grep -q STATUS=OK /var/lib/homelab-health/node-loss-runbooks.txt" +/usr/local/sbin/homelab-final-readiness-gate:43:check "power loss readiness" bash -c "grep -q STATUS=OK /var/lib/homelab-health/power-loss-readiness.txt" +/usr/local/sbin/homelab-final-readiness-gate:44:check "power loss runbook" bash -c "grep -q STATUS=OK /var/lib/homelab-health/power-loss-runbook.txt" +/usr/local/sbin/homelab-final-readiness-gate:45:check "ungated health backlog" bash -c "grep -q STATUS=OK /var/lib/homelab-health/ungated-health-backlog.txt" +/usr/local/sbin/homelab-final-readiness-gate:46:check "pve03 root cleanup" bash -c "grep -q STATUS=OK /var/lib/homelab-health/pve03-root-cleanup.txt" +/usr/local/sbin/homelab-final-readiness-gate:47:check "pve03 staging retention" bash -c "grep -q STATUS=OK /var/lib/homelab-health/pve03-staging-retention.txt" +/usr/local/sbin/homelab-final-readiness-gate:48:check "pve03 staging retention proof" bash -c "grep -q STATUS=OK /var/lib/homelab-health/pve03-staging-retention-proof.txt" +/usr/local/sbin/homelab-final-readiness-gate:49:check "pve03 staging retention cleanup" bash -c "grep -q STATUS=OK /var/lib/homelab-health/pve03-staging-retention-cleanup.txt" +/usr/local/sbin/homelab-final-readiness-gate:50:check "pve03 failed unit cleanup" bash -c "grep -q STATUS=OK /var/lib/homelab-health/pve03-failed-unit-cleanup.txt" +/usr/local/sbin/homelab-final-readiness-gate:51:check "remote git sops age readiness" bash -c "grep -q STATUS=OK /var/lib/homelab-health/remote-git-sops-age-readiness.txt" +/usr/local/sbin/homelab-final-readiness-gate:52:check "remote git sops age policy" bash -c "grep -q STATUS=OK /var/lib/homelab-health/remote-git-sops-age-policy.txt" +/usr/local/sbin/homelab-final-readiness-gate:53:check "desired state remote target trace" bash -c "grep -q STATUS=OK /var/lib/homelab-health/desired-state-remote-target-trace.txt" +/usr/local/sbin/homelab-final-readiness-gate:54:check "desired state remote target" bash -c "grep -q STATUS=OK /var/lib/homelab-health/desired-state-remote-target.txt" +/usr/local/sbin/homelab-final-readiness-gate:55:check "Kuma monitor policy evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/kuma-monitor-policy.txt" +/usr/local/sbin/homelab-final-readiness-gate:56:check "safe autoheal evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/safe-autoheal.txt" +/usr/local/sbin/homelab-final-readiness-gate:57:check "desired-state clean evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/desired-state.txt && test -z "$(git -C /srv/homelab-desired-state status --short)"" +/usr/local/sbin/homelab-final-readiness-gate:58:check "runbooks evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/runbooks.txt && test -s /var/lib/homelab-runbooks/INDEX.md" +/usr/local/sbin/homelab-final-readiness-gate:59:check "incident journal evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/incident-journal.txt" +/usr/local/sbin/homelab-final-readiness-gate:60:check "alerting evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/alerting.txt" +/usr/local/sbin/homelab-final-readiness-gate:61:check "VM backup health evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/vm-backup.txt" +/usr/local/sbin/homelab-final-readiness-gate:62:check "VM backup policy evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/vm-backup-policy.txt" +/usr/local/sbin/homelab-final-readiness-gate:63:check "VM local dumps cloud" bash -c "grep -q STATUS=OK /var/lib/homelab-health/vm-local-dumps-cloud.txt && [ $(jq -r .count /var/lib/homelab-vm-backup/latest-vm-local-dumps-cloud.json) -ge 3 ] && [ $(jq -r .chunked_count /var/lib/homelab-vm-backup/latest-vm-local-dumps-cloud.json) -ge 1 ]" +/usr/local/sbin/homelab-final-readiness-gate:64:check "VM local dumps retention" bash -c "grep -q STATUS=OK /var/lib/homelab-health/vm-local-dumps-retention.txt && grep -q ACTIVE_LEFT=0 /var/lib/homelab-health/kuma-monitor-policy.txt || grep -q STATUS=OK /var/lib/homelab-health/vm-local-dumps-retention.txt" +/usr/local/sbin/homelab-final-readiness-gate:65:check "capacity risk evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/capacity-risk.txt" +/usr/local/sbin/homelab-final-readiness-gate:66:check "secret exposure evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/secret-exposure.txt" +/usr/local/sbin/homelab-final-readiness-gate:67:check "golden state evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/golden-state.txt" +/usr/local/sbin/homelab-final-readiness-gate:68:check "service registry evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/service-registry.txt" +/usr/local/sbin/homelab-final-readiness-gate:69:check "dependency map evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/dependency-map.txt" +/usr/local/sbin/homelab-docker-health.sh:44: echo "containers_without_healthcheck=$NOHEALTH" +/usr/local/sbin/homelab-docker-health.sh:63: echo "# HELP homelab_docker_health_container_without_healthcheck Docker containers without native healthcheck." +/usr/local/sbin/homelab-docker-health.sh:64: echo "# TYPE homelab_docker_health_container_without_healthcheck gauge" +/usr/local/sbin/homelab-docker-health.sh:65: echo "homelab_docker_health_container_without_healthcheck $NOHEALTH" +/usr/local/sbin/homelab-backup-coverage-matrix:17:P1_SAFE_APPS="actual-budget homebox linkding mealie memos node-red syncthing filebrowser healthchecks ntfy vikunja homeassistant karakeep minio n8n" +/usr/local/sbin/homelab-backup-coverage-matrix:38: while read -r norm raw; do +/usr/local/sbin/homelab-backup-coverage-matrix:50: while read -r norm raw; do +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:8:TARGET_BASE=full/nextcloud-vm-vzdump +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:11:SRC_GLOB=/var/lib/vz/dump/vzdump-qemu-150-*.vma.zst +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:77:P1 Mail Cloud Nextcloud VM recurring chunked vzdump upload +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:93:rclone mkdir "$TARGET_REMOTE:$REMOTE_PARTS" > "$WORK/logs/mkdir.out" 2>"$WORK/logs/mkdir.err" || MKDIR_RC=$? +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:95:rclone copy "$PARTS_DIR" "$TARGET_REMOTE:$REMOTE_PARTS" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > "$WORK/logs/upload_parts.out" 2>"$WORK/logs/upload_parts.err" || UPLOAD_PARTS_RC=$? +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:97:rclone copyto "$WORK/$BASENAME.sha256" "$TARGET_REMOTE:$REMOTE_SHA" --checksum > "$WORK/logs/upload_sha.out" 2>"$WORK/logs/upload_sha.err" || UPLOAD_SHA_RC=$? +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:99:rclone copyto "$WORK/MANIFEST.txt" "$TARGET_REMOTE:$REMOTE_MANIFEST" --checksum > "$WORK/logs/upload_manifest.out" 2>"$WORK/logs/upload_manifest.err" || UPLOAD_MANIFEST_RC=$? +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:101:rclone copyto "$WORK/SHA256SUMS.txt" "$TARGET_REMOTE:$REMOTE_SUMS" --checksum > "$WORK/logs/upload_sums.out" 2>"$WORK/logs/upload_sums.err" || UPLOAD_SUMS_RC=$? +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:104:rclone lsf "$TARGET_REMOTE:$REMOTE_PARTS" --format ps > "$WORK/logs/remote_lsf_parts.txt" 2>"$WORK/logs/remote_lsf_parts.err" || LSF_PARTS_RC=$? +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:110:rclone copy "$TARGET_REMOTE:$REMOTE_PARTS" "$DL_DIR/parts" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > "$WORK/logs/download_parts.out" 2>"$WORK/logs/download_parts.err" || DOWNLOAD_PARTS_RC=$? +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:120:rclone lsf "$TARGET_REMOTE:$TARGET_BASE" --dirs-only 2>/dev/null | sed 's#/$##' | sort > "$DIR_LIST" || RETENTION_RC=$? +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:126: [ -n "$OLD" ] && rclone purge "$TARGET_REMOTE:$TARGET_BASE/$OLD" >> "$WORK/logs/retention.out" 2>>"$WORK/logs/retention.err" +/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:140: rm -rf "$PARTS_DIR" "$DL_DIR" +/usr/local/sbin/homelab-rehydrate-flat-chunked-dir:10: rm -f "$TMP_TARGET" +/usr/local/sbin/homelab-disk-space-health.py:10:checks = [ +/usr/local/sbin/homelab-disk-space-health.py:21:for label, cmd in checks: +/usr/local/sbin/homelab-rclone-required-check.sh:4:RAW_FILE="/etc/homelab/rclone/required-remotes.txt" +/usr/local/sbin/homelab-rclone-required-check.sh:5:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt" +/usr/local/sbin/homelab-rclone-required-check.sh:9:echo "=== homelab rclone required check: $(hostname) ===" +/usr/local/sbin/homelab-rclone-required-check.sh:12:echo "=== installed rclone ===" +/usr/local/sbin/homelab-rclone-required-check.sh:13:rclone version | head -5 +/usr/local/sbin/homelab-rclone-required-check.sh:17:CONFIGURED="$(rclone listremotes 2>/dev/null | sed 's/:$//' || true)" +/usr/local/sbin/homelab-rclone-required-check.sh:19: echo "No rclone remotes configured yet." +/usr/local/sbin/homelab-legacy-monitoring-backups-run:24: rm -f "$STAGE/remote.tgz" +/usr/local/sbin/homelab-legacy-monitoring-backups-run:43: rm -f "$STAGE/pve01.tgz" +/usr/local/sbin/homelab-legacy-monitoring-backups-run:48: rm -f "$STAGE/$LABEL.tgz" +/usr/local/sbin/homelab-legacy-monitoring-backups-run:52: rm -f "$STAGE/monitoring.tgz" +/usr/local/sbin/homelab-legacy-monitoring-backups-run:71: rm -f "$STAGE/config.tgz" +/usr/local/sbin/homelab-cluster-admin-inventory-sync:19:ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$REMOTE" "rm -f $REMOTE_TMP" +/usr/local/sbin/homelab-chat-run:272: re.compile(r"(?i)\brclone\.conf\b"), +/usr/local/sbin/homelab-chat-run:388: "reference_register_checked": True, +/usr/local/sbin/homelab-chat-run:390: "error_register_checked": True, +/usr/local/sbin/homelab-chat-run:489: rm -f \ +/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:10: rm -rf "$TMPD" 2>/dev/null || true +/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:19:check_host() { +/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:39:check_host "kuma" "kuma.vpn.gram1.ru" "^(200|302)$" +/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:40:check_host "homepage" "homepage.vpn.gram1.ru" "^200$" +/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:41:check_host "gitea" "gitea.vpn.gram1.ru" "^200$" +/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:42:check_host "filebrowser" "filebrowser.vpn.gram1.ru" "^200$" +/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:43:check_host "auth" "auth.vpn.gram1.ru" "^(200|302)$" +/usr/local/sbin/homelab-beszel-backup-run:33:rm -f "$STAGE/remote.tgz" +/usr/local/sbin/homelab-beszel-backup-run:37:while IFS= read -r DB; do DB_COUNT=$((DB_COUNT+1)); RESULT="$(python3 -c "import sqlite3,sys;print(sqlite3.connect(sys.argv[1]).execute(\"PRAGMA integrity_check\").fetchone()[0])" "$DB" 2>/dev/null || echo error)"; [ "$RESULT" = ok ] || exit 4; DB_OK=$((DB_OK+1)); done < <(find "$STAGE/payload" -type f \( -name "*.db" -o -name "*.sqlite" -o -name "*.sqlite3" \) 2>/dev/null) +/usr/local/sbin/homelab-runbook-generate:40: f"## First checks\n" +/usr/local/sbin/homelab-runbook-generate:44: f"appbackupctl restore-check {app}\n" +/usr/local/sbin/homelab-runbook-generate:47: f"Do not restore over a live service blindly. First confirm both encrypted cloud copies pass restore-check, target VM/container is snapshotted or intentionally stopped, and dependency map is OK.\n\n" +/usr/local/sbin/homelab-runbook-generate:51: f"homelab-service-registry-check\n" +/usr/local/sbin/homelab-runbook-generate:52: f"homelab-dependency-map-check\n" +/usr/local/sbin/homelab-runbook-generate:85: "backupctl restore-check\n" +/usr/local/sbin/homelab-runbook-generate:86: "appbackupctl restore-check safety-net-control-plane\n" +/usr/local/sbin/skladchik-reports-monitor-supervisor:25:PVE_RESTORE_HEALTH = Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt") +/usr/local/sbin/skladchik-reports-monitor-supervisor:254: "skladchik-reports-monitor-backup-restore-check.timer", +/usr/local/sbin/skladchik-reports-monitor-supervisor:301: "Restore-check:", +/usr/local/sbin/skladchik-reports-monitor-supervisor:339: mode = sys.argv[1] if len(sys.argv) > 1 else "check" +/usr/local/sbin/skladchik-reports-monitor-supervisor:340: if mode not in ("check", "test"): +/usr/local/sbin/skladchik-reports-monitor-supervisor:341: print("usage: skladchik-reports-monitor-supervisor [check|test]", file=sys.stderr) +/usr/local/sbin/backup-restore-check:8:ROOT="/var/tmp/homelab-restore-check/${JOB}" +/usr/local/sbin/backup-restore-check:9:rm -rf "$ROOT" && mkdir -p "$ROOT/a" "$ROOT/b" +/usr/local/sbin/backup-restore-check:10:trap 'echo "ERROR restore-check failed; kept: "; exit 1' ERR +/usr/local/sbin/backup-restore-check:11:echo "restore-check job=$JOB" +/usr/local/sbin/backup-restore-check:12:rclone copy "$RA" "$ROOT/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-a.log 2>&1; homelab-rehydrate-chunked-dir "$ROOT/a" >>/tmp/homelab-restore-a.log 2>&1 +/usr/local/sbin/backup-restore-check:13:(cd "$ROOT/a" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-a-sha.log && ARCHIVE="$(jq -r .archive_name manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null) +/usr/local/sbin/backup-restore-check:14:echo "OK cloud A restore-check" +/usr/local/sbin/backup-restore-check:15:rclone copy "$RB" "$ROOT/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-b.log 2>&1; homelab-rehydrate-chunked-dir "$ROOT/b" >>/tmp/homelab-restore-b.log 2>&1 +/usr/local/sbin/backup-restore-check:16:(cd "$ROOT/b" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-b-sha.log && ARCHIVE="$(jq -r .archive_name manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null) +/usr/local/sbin/backup-restore-check:17:echo "OK cloud B restore-check" +/usr/local/sbin/backup-restore-check:18:rm -rf "$ROOT" +/usr/local/sbin/backup-restore-check:20:echo "OK restore-check both clouds" +/usr/local/sbin/homelab-vps-identity-audit:8:trap 'rm -rf "$TMP"' EXIT +/usr/local/sbin/homelab-vm-full-restore-drill:77: -name "vzdump-qemu-${source_vmid}-*.vma.zst" \ +/usr/local/sbin/homelab-vm-full-restore-drill:152:check_inventory() { +/usr/local/sbin/homelab-vm-full-restore-drill:184:check_container_health() { +/usr/local/sbin/homelab-vm-full-restore-drill:224:check_http() { +/usr/local/sbin/homelab-vm-full-restore-drill:349:check_container_health() { +/usr/local/sbin/homelab-vm-full-restore-drill:386:check_container_health "$test_vmid" +/usr/local/sbin/homelab-vm-full-restore-drill:389: check_inventory "$source_vmid" "$test_vmid" +/usr/local/sbin/homelab-vm-full-restore-drill:392: check_http "$source_vmid" "$test_vmid" +/usr/local/sbin/homelab-vm-backup-health:8:check_file(){ +/usr/local/sbin/homelab-vm-backup-health:15:check_file /var/lib/homelab-health/backup.txt auto-vmct +/usr/local/sbin/homelab-vm-backup-health:16:check_file /var/lib/homelab-health/edge-vm-vzdump-backup.txt edge-vm-backup +/usr/local/sbin/homelab-vm-backup-health:17:check_file /var/lib/homelab-health/edge-vm-vzdump-offhost.txt edge-vm-offhost +/usr/local/sbin/homelab-vm-backup-health:18:check_file /var/lib/homelab-health/edge-vm-vzdump-restore.txt edge-vm-restore +/usr/local/sbin/homelab-ingress-failover:37: rm -f "$RESOURCES" +/usr/local/sbin/homelab-ingress-failover:252: rm -f "$tmp" +/usr/local/sbin/homelab-overall-health:8: /var/lib/homelab-health/drift-check.txt +/usr/local/sbin/homelab-backup-matrix-refresh:10: p=subprocess.run(args,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True,check=False) +/usr/local/sbin/homelab-backup-matrix-refresh:65: shutil.copy2(source_path,temporary) +/usr/local/sbin/homelab-vm-full-backup:11:# shellcheck disable=SC1090 +/usr/local/sbin/homelab-vm-full-backup:92:for cmd in vzdump qm zstd sha256sum rclone rsync ssh flock; do +/usr/local/sbin/homelab-vm-full-backup:130: -name "vzdump-qemu-${VMID}-*.vma.zst" \ +/usr/local/sbin/homelab-vm-full-backup:138: vzdump "$VMID" \ +/usr/local/sbin/homelab-vm-full-backup:145: --remove 0 +/usr/local/sbin/homelab-vm-full-backup:151: -name "vzdump-qemu-${VMID}-*.vma.zst" \ +/usr/local/sbin/homelab-vm-full-backup:261: CHUNK_TMP_BASE="$BACKUP_DIR/.rclone-chunks" homelab-rclone-copy-chunked "$ARCHIVE" "${DEST_ROOT}/${NAME}" +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:10:TARGET_BASE=full/edge-vm-vzdump +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:64:case "$SOURCE" in /mnt/staging/offhost/edge-vm-vzdump-from-pve03/vzdump-qemu-130-*.vma.zst) ;; *) fail source_path_bad 2 ;; esac +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:111:rclone about "$TARGET_REMOTE:" > "$WORK/logs/about.out" 2>"$WORK/logs/about.err" || ABOUT_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:114:DIRS="$(rclone lsf "$TARGET_REMOTE:$TARGET_BASE" --dirs-only 2>/dev/null | sed 's#/$##' | sort || true)" +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:116: if rclone cat "$TARGET_REMOTE:$TARGET_BASE/$D/$BASENAME.sha256" 2>/dev/null | grep -q "$SHA"; then +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:172:P1 Mail Cloud edge-vm recurring chunked vzdump upload +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:188:rclone mkdir "$TARGET_REMOTE:$REMOTE_PARTS" > "$WORK/logs/mkdir.out" 2>"$WORK/logs/mkdir.err" || MKDIR_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:190:rclone copy "$PARTS_DIR" "$TARGET_REMOTE:$REMOTE_PARTS" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > "$WORK/logs/upload_parts.out" 2>"$WORK/logs/upload_parts.err" || UPLOAD_PARTS_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:192:rclone copyto "$WORK/$BASENAME.sha256" "$TARGET_REMOTE:$REMOTE_SHA" --checksum > "$WORK/logs/upload_sha.out" 2>"$WORK/logs/upload_sha.err" || UPLOAD_SHA_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:194:rclone copyto "$WORK/MANIFEST.txt" "$TARGET_REMOTE:$REMOTE_MANIFEST" --checksum > "$WORK/logs/upload_manifest.out" 2>"$WORK/logs/upload_manifest.err" || UPLOAD_MANIFEST_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:196:rclone copyto "$WORK/SHA256SUMS.txt" "$TARGET_REMOTE:$REMOTE_SUMS" --checksum > "$WORK/logs/upload_sums.out" 2>"$WORK/logs/upload_sums.err" || UPLOAD_SUMS_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:199:rclone size "$TARGET_REMOTE:$TARGET_DIR" --json > "$WORK/logs/remote_size.json" 2>"$WORK/logs/remote_size.err" || SIZE_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:201:rclone lsf "$TARGET_REMOTE:$REMOTE_PARTS" --format "ps" > "$WORK/logs/remote_lsf_parts.txt" 2>"$WORK/logs/remote_lsf_parts.err" || LSF_PARTS_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:207:rclone copy "$TARGET_REMOTE:$REMOTE_PARTS" "$DL_DIR/parts" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > "$WORK/logs/download_parts.out" 2>"$WORK/logs/download_parts.err" || DOWNLOAD_PARTS_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:222: rm -rf "$DL_DIR" "$PARTS_DIR" +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:234: rclone lsf "$TARGET_REMOTE:$TARGET_BASE" --dirs-only 2>/dev/null | sed "s#/$##" | sort > "$DIR_LIST" || RETENTION_RC=$? +/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:239: [ -n "$OLD" ] && rclone purge "$TARGET_REMOTE:$TARGET_BASE/$OLD" >> "$WORK/logs/retention.out" 2>>"$WORK/logs/retention.err" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:7:BASE_DIR="/mnt/staging/vzdump/edge-vm-vzdump" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:8:OFFHOST_DIR="/mnt/staging/offhost/edge-vm-vzdump-from-pve03" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:40:REMOTE_OUT="$(remote "OUT=/tmp/edge_vm_vzdump_${VMID}_${TS}.out; ERR=/tmp/edge_vm_vzdump_${VMID}_${TS}.err; : > \$OUT; : > \$ERR; vzdump '$VMID' --mode snapshot --compress zstd --dumpdir '$RUN_DIR' --remove 0 >\$OUT 2>\$ERR; RC=\$?; ARCHIVE=\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\n' | sort -n | tail -n1 | cut -d' ' -f2-); LOGFILE=\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.log' -printf '%T@ %p\n' | sort -n | tail -n1 | cut -d' ' -f2-); SIZE=0; SHA=none; if [ -n \"\$ARCHIVE\" ] && [ -f \"\$ARCHIVE\" ]; then SIZE=\$(stat -c %s \"\$ARCHIVE\"); SHA=\$(sha256sum \"\$ARCHIVE\" | awk '{print \$1}'); fi; echo VZDUMP_RC=\$RC; echo ARCHIVE=\$ARCHIVE; echo LOGFILE=\$LOGFILE; echo SIZE=\$SIZE; echo SHA=\$SHA; echo OUT_TAIL=\$(tail -n 6 \$OUT | tr '\n' '|'); echo ERR_TAIL=\$(tail -n 6 \$ERR | tr '\n' '|')")" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:62:dest "mkdir -p '$OFFHOST_DIR' && rm -f '$DEST_PATH.tmp' '$DEST_PATH.sha256'" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:74:echo "STATUS=OK TS=$TS TYPE=edge-vm-vzdump-backup NODE=$NODE VMID=$VMID ARCHIVE=$ARCHIVE SIZE=$SIZE SHA256=$SHA SCSI1_BACKUP0=yes NOTES=timer_run" > "$HEALTH_DIR/edge-vm-vzdump-backup.txt" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:75:echo "STATUS=OK TS=$TS TYPE=edge-vm-vzdump-offhost NODE=$NODE VMID=$VMID HOST=pve02 DEST=$DEST_PATH SIZE=$SIZE SHA256=$SHA NOTES=timer_run" > "$HEALTH_DIR/edge-vm-vzdump-offhost.txt" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:76:echo "STATUS=OK TS=$TS TYPE=edge-vm-vzdump-restore-verify HOST=pve02 VMID=$VMID ARCHIVE=$DEST_PATH SIZE=$SIZE SHA256=$SHA ZSTD_RC=0 VMA_RC=0 NOTES=timer_run" > "$HEALTH_DIR/edge-vm-vzdump-restore.txt" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:77:chmod 0644 "$HEALTH_DIR/edge-vm-vzdump-backup.txt" "$HEALTH_DIR/edge-vm-vzdump-offhost.txt" "$HEALTH_DIR/edge-vm-vzdump-restore.txt" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:79:remote "find '$BASE_DIR' -mindepth 1 -maxdepth 1 -type d -name 'edge-vm-*' -printf '%T@ %p\n' | sort -nr | awk 'NR>${KEEP_LOCAL}{print \$2}' | xargs -r rm -rf" +/usr/local/sbin/homelab-backup-edge-vm-vzdump:80:dest "find '$OFFHOST_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\n' | sort -nr | awk 'NR>${KEEP_OFFHOST}{print \$2}' | while read f; do rm -f \"\$f\" \"\$f.sha256\"; done" +/usr/local/sbin/appbackup-dockge-stacks:25:echo "[STEP] copy app data" +/usr/local/sbin/appbackup-dockge-stacks:30:rm -rf "$PAYLOAD" +/usr/local/sbin/appbackup-dockge-stacks:51:echo "[STEP] local checksum" +/usr/local/sbin/appbackup-dockge-stacks:52:(cd "$STAGE" && sha256sum "$ARCHIVE" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e . manifest.json >/dev/null) +/usr/local/sbin/appbackup-dockge-stacks:54:echo "[STEP] upload cloud A metadata"; rclone copy "$STAGE" "$RA" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s +/usr/local/sbin/appbackup-dockge-stacks:55:echo "[STEP] upload cloud A archive chunks"; rclone deletefile "$RA/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RA/$ARCHIVE" +/usr/local/sbin/appbackup-dockge-stacks:56:echo "[STEP] upload cloud B metadata"; rclone copy "$STAGE" "$RB" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s +/usr/local/sbin/appbackup-dockge-stacks:57:echo "[STEP] upload cloud B archive chunks"; rclone deletefile "$RB/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RB/$ARCHIVE" +/usr/local/sbin/appbackup-dockge-stacks:58:echo "[STEP] verify cloud A chunked"; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-stage-verify-chunked "$RA" "$ARCHIVE" "$VERIFY/a" "$JOB" +/usr/local/sbin/appbackup-dockge-stacks:59:echo "[STEP] verify cloud B chunked"; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-stage-verify-chunked "$RB" "$ARCHIVE" "$VERIFY/b" "$JOB" +/usr/local/sbin/appbackup-dockge-stacks:60:echo "[STEP] upload latest index"; rclone copyto "$STAGE/latest-backup-index.json" "$RAL"; rclone copyto "$STAGE/latest-backup-index.json" "$RBL"; rclone cat "$RAL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null; rclone cat "$RBL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null +/usr/local/sbin/appbackup-dockge-stacks:76:rm -rf "$STAGE" "$VERIFY" +/usr/local/sbin/homelab-cluster-passport:23:line "- safety checkpoint: /root/HOMELAB_SAFETY_NET_CHECKPOINT.md" +/usr/local/sbin/homelab-cluster-passport:65:need_ok /var/lib/homelab-health/drift-check.txt drift-check +/usr/local/sbin/homelab-cluster-passport:77:line "appbackupctl restore-check safety-net-control-plane" +/usr/local/sbin/homelab-cluster-passport:84:line "4. For control-plane loss: appbackupctl restore-check safety-net-control-plane, then restore the verified archive from either encrypted cloud." +/usr/local/sbin/backup-retention:18: if [[ "$MODE" == "apply" ]]; then rclone cat "$BASE/$LATEST/manifest.json" | jq -e --arg job "$LATEST" ".job_id == \$job" >/dev/null || { echo "REFUSE latest missing/bad on $BASE"; exit 4; }; fi +/usr/local/sbin/backup-retention:19: mapfile -t DIRS < <(rclone lsf "$BASE" --dirs-only 2>/dev/null | while read -r D; do D="${D%/}"; [[ "$D" == homelab-${NODE}-${JOB}-* ]] && echo "$D"; done | sort) +/usr/local/sbin/backup-retention:24: for ((i=0; i /var/lib/homelab-health/private-vpn-hosts.txt.tmp; rc=$?; mv /var/lib/homelab-health/private-vpn-hosts.txt.tmp /var/lib/homelab-health/private-vpn-hosts.txt; chmod 0644 /var/lib/homelab-health/private-vpn-hosts.txt; exit $rc' +/etc/systemd/system/netbird-peers-health.service:2:Description=NetBird peers health check +/etc/systemd/system/homelab-pve03-staging-capacity-health.timer:2:Description=Run Homelab pve03 staging capacity health check +/etc/systemd/system/timers.target.wants/homelab-private-vpn-hosts-health.timer:2:Description=Run Homelab private VPN hosts health check every minute +/etc/systemd/system/timers.target.wants/homelab-crypto-portfolio-chart-health.timer:2:Description=Recurring Crypto portfolio chart health check +/etc/systemd/system/timers.target.wants/homelab-edge-vm-offhost-freshness.timer:2:Description=Run Homelab edge-vm VM130 offhost freshness sync daily after nightly vzdump +/etc/systemd/system/timers.target.wants/homelab-dependency-map-check.timer:2:Description=Homelab dependency map check timer +/etc/systemd/system/timers.target.wants/homelab-pve03-staging-capacity-health.timer:2:Description=Run Homelab pve03 staging capacity health check +/etc/systemd/system/timers.target.wants/homelab-drift-check.timer:2:Description=Homelab drift check timer +/etc/systemd/system/timers.target.wants/homelab-service-registry-check.timer:2:Description=Homelab service registry health check timer +/etc/systemd/system/timers.target.wants/homelab-backup-healthcheck.timer:2:Description=Run homelab backup healthcheck daily +/etc/systemd/system/timers.target.wants/homelab-backup-healthcheck.timer:8:Unit=homelab-backup-healthcheck.service +/etc/systemd/system/timers.target.wants/netbird-peers-health.timer:2:Description=Run NetBird peers health check +/etc/systemd/system/timers.target.wants/homelab-alerting-health.timer:2:Description=Homelab alerting health check timer +/etc/systemd/system/timers.target.wants/skladchik-reports-monitor-backup-restore-check.timer:2:Description=Weekly restore-check of Skladchik reports monitor cloud backups +/etc/systemd/system/timers.target.wants/skladchik-reports-monitor-backup-restore-check.timer:8:Unit=skladchik-reports-monitor-backup-restore-check.service +/etc/systemd/system/homelab-edge-vm-vzdump-backup.timer:8:Unit=homelab-edge-vm-vzdump-backup.service +/etc/systemd/system/homelab-drift-check.timer:2:Description=Homelab drift check timer +/etc/systemd/system/homelab-service-registry-check.timer:2:Description=Homelab service registry health check timer +/etc/systemd/system/homelab-pve03-staging-capacity-health.service:2:Description=Homelab pve03 staging capacity health check +/etc/systemd/system/homelab-backup-healthcheck.timer:2:Description=Run homelab backup healthcheck daily +/etc/systemd/system/homelab-backup-healthcheck.timer:8:Unit=homelab-backup-healthcheck.service +/etc/systemd/system/homelab-safe-exec-health.service:2:Description=Run allowlisted homelab health checks +/etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.service:2:Description=Homelab Mail Cloud Nextcloud VM vzdump upload +/etc/systemd/system/hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub' +/etc/systemd/system/netbird-peers-health.timer:2:Description=Run NetBird peers health check +/etc/systemd/system/homelab-alerting-health.timer:2:Description=Homelab alerting health check timer +/etc/systemd/system/skladchik-reports-monitor-backup-restore-check.timer:2:Description=Weekly restore-check of Skladchik reports monitor cloud backups +/etc/systemd/system/skladchik-reports-monitor-backup-restore-check.timer:8:Unit=skladchik-reports-monitor-backup-restore-check.service +/etc/systemd/system/suspend-then-hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub' +/etc/cron.d/vzdump:1:# cluster wide vzdump cron schedule +7022386770 2026-07-18T03:31:03.6566057240 /var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.vma.zst +7021952392 2026-07-21T03:31:06.5717414950 /var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst +7021297699 2026-07-19T03:31:06.8755351020 /var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst +7016275481 2026-07-20T03:31:03.2165123470 /var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst +329541843 2026-07-21T03:30:09.7688696330 /var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst +326601118 2026-07-20T03:30:08.2096782720 /var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst +323668231 2026-07-19T03:45:46.2209130400 /var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst +321121649 2026-07-18T03:30:08.0777625900 /var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.tar.zst +222777732 2026-07-21T03:30:15.9059638290 /var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst +222761273 2026-07-20T03:30:13.7937629440 /var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst +222745965 2026-07-19T03:45:57.2700790730 /var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst +222727760 2026-07-18T03:30:13.5578457230 /var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.tar.zst +3021 2026-07-19T03:31:07.4745284120 /var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.log +3020 2026-07-21T03:31:07.3945472590 /var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.log +3018 2026-07-18T03:31:03.9197988860 /var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.log +3017 2026-07-20T03:31:04.1706719350 /var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.log +1089 2026-07-20T03:30:14.0079985330 /var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.log +1086 2026-07-20T03:30:08.4640241540 /var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.log +989 2026-07-21T03:30:16.1364976560 /var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.log +989 2026-07-18T03:30:13.7816006690 /var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.log +986 2026-07-21T03:30:10.0285327490 /var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.log +986 2026-07-18T03:30:08.3002411830 /var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.log +731 2026-07-19T03:45:57.5197000270 /var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.log +729 2026-07-19T03:45:46.4444121980 /var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.log +========== NODE=pve02 ========== +=== REMOTES === +=== TIMERS === +Tue 2026-07-21 18:41:03 MSK 3h 46min Tue 2026-07-21 12:41:22 MSK 2h 12min ago forum-prod-fresh5-cloud-backup.timer forum-prod-fresh5-cloud-backup.service +Wed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:22 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service +Wed 2026-07-22 04:20:00 MSK 13h Tue 2026-07-21 04:20:12 MSK 10h ago homelab-cluster-admin-vm-mail-cloud-backup.timer homelab-cluster-admin-vm-mail-cloud-backup.service +=== SERVICES === +codevipe-cloud-backup.service static - +dpkg-db-backup.service static - +forum-prod-fresh5-cloud-backup.service static - +homelab-cluster-admin-vm-mail-cloud-backup.service static - +/usr/local/sbin/homelab-rclone-required-check.sh:4:RAW_FILE="/etc/homelab/rclone/required-remotes.txt" +/usr/local/sbin/homelab-rclone-required-check.sh:5:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt" +/usr/local/sbin/homelab-rclone-required-check.sh:9:echo "=== homelab rclone required check: $(hostname) ===" +/usr/local/sbin/homelab-rclone-required-check.sh:12:echo "=== installed rclone ===" +/usr/local/sbin/homelab-rclone-required-check.sh:13:rclone version | head -5 +/usr/local/sbin/homelab-rclone-required-check.sh:17:CONFIGURED="$(rclone listremotes 2>/dev/null | sed 's/:$//' || true)" +/usr/local/sbin/homelab-rclone-required-check.sh:19: echo "No rclone remotes configured yet." +/usr/local/sbin/homelab-staging-check.sh:11:echo "=== homelab staging check: ${HOST} ===" +/usr/local/sbin/routerguard-watchdog-power-entrypoint:12:case "$MODE" in check|cycle) ;; *) echo HARD_ENTRYPOINT_MODE_INVALID=yes; exit 64;; esac +/usr/local/sbin/routerguard-watchdog-power-entrypoint:15:if [ "$MODE" = check ]; then +/usr/local/sbin/routerguard-watchdog-power-entrypoint:16: exec "$HELPER" check +/usr/local/sbin/routerguard-watchdog-power-entrypoint:21:PRE="$(mktemp /run/routerguard-watchdog-precheck.XXXXXX)" +/usr/local/sbin/routerguard-watchdog-power-entrypoint:22:cleanup(){ rm -f "$PRE" "$ARM"; } +/usr/local/sbin/routerguard-watchdog-power-entrypoint:25:"$HELPER" check >"$PRE" 2>&1 +/usr/local/sbin/routerguard-watchdog-power-entrypoint:38:rm -f "$ARM" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:4:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:18: "checksums/" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:21:echo "=== homelab rclone crypt healthcheck: $(hostname) ===" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:40: if ! rclone listremotes | sed 's/:$//' | grep -qx "$REMOTE"; then +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:47: if ! rclone about "${REMOTE}:"; then +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:48: echo "FAIL: rclone about failed: $REMOTE" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:54: ROOT_LIST="$(rclone lsf "${REMOTE}:/" || true)" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:76: if rclone cat "${REMOTE}:/latest-backup-index.json" | jq -e '.schema and .remote and .status and .items' >/dev/null; then +/usr/local/sbin/codevipe-cloud-backup.sh:28: rclone copy "$CACHE_ROOT/$BASE" "$R:$REMOTE_PATH/$BASE" --stats 30s +/usr/local/sbin/codevipe-cloud-backup.sh:31:find "$CACHE_ROOT" -mindepth 1 -maxdepth 1 -type d -mtime +14 -print -exec rm -rf {} \; +/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:26:rm -f "$STAGE/$base" +/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:34:format=tar.gz split_parts_1500M rclone_crypt_remote +/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:44: rclone mkdir "$dest" >/tmp/forum-cloud-mkdir-${r}-$TS.log 2>&1 +/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:46: rclone copy "$STAGE" "$dest" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --stats-one-line >/tmp/forum-cloud-copy-${r}-$TS.log 2>&1 +/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:49: remote_count=$(rclone lsf "$dest" --files-only 2>/tmp/forum-cloud-lsf-${r}-$TS.err | wc -l) +/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:55:find /root/backups/forum-prod-fresh5 -mindepth 1 -maxdepth 1 -type d -mtime +14 -exec rm -rf {} + +/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:73: retention_remote_count=$(rclone lsf "$r:$REMOTE_BASE/$retention_job" --files-only 2>/dev/null | wc -l) +/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:78: rm -rf --one-file-system "$retention_dir" +/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:83: rm -f "$retention_keep_tmp" +/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:24:rclone listremotes 2>/tmp/cluster-admin-rclone-listremotes.err | grep -E 'mail.*crypt|crypt.*mail' | sed 's/:$//' | sort > /tmp/cluster-admin-mail-remotes.txt +/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:41: rm -rf "$STAGE/$OLD_JOB" +/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:43: rclone purge "$R:homelab-backups/pve02/cluster-admin-vm180/$OLD_JOB" >/tmp/cluster-admin-rclone-purge-$R.log 2>&1 +/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:51: ls "$d"/vzdump-qemu-${VMID}-*.vma.zst >/dev/null 2>&1 || { echo "DELETE_EMPTY_LOCAL_JOB=$b"; rm -rf "$d"; } +/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:84:vzdump "$VMID" --mode snapshot --compress zstd --dumpdir "$JOBDIR" --quiet 0 +/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:89:LATEST="$(ls -1t "$JOBDIR"/vzdump-qemu-${VMID}-*.vma.zst 2>/dev/null | head -n1)" +/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:115: rm -f "$TMPVMA" +/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:124: rm -f "$TMPVMA" +/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:156: rclone mkdir "$DEST" >/tmp/cluster-admin-rclone-mkdir-$R.log 2>&1 +/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:158: rclone copy "$JOBDIR" "$DEST" --checksum --transfers 2 --checkers 4 --retries 3 --low-level-retries 5 >/tmp/cluster-admin-rclone-copy-$R.log 2>&1 +/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:161: rclone copyto "$JOBDIR/MANIFEST.txt" "$R:homelab-backups/pve02/cluster-admin-vm180/latest/MANIFEST.txt" --retries 3 >/tmp/cluster-admin-rclone-latest-$R.log 2>&1 +/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:164: REMOTE_SHA="$(rclone cat "$DEST/$BASE.sha256" 2>/tmp/cluster-admin-rclone-catsha-$R.err | awk '{print $1}' | head -n1)" +/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:177:find "$STAGE" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort -r | tail -n +4 | while read old; do rm -rf "$STAGE/$old"; echo "LOCAL_STAGE_DELETED=$old"; done +/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:20: rm -rf -- "$WORK" +/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:29: local checked_at +/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:31: checked_at="$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || printf UNKNOWN)" +/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:37:CHECKED_AT_UTC=$checked_at +/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:117:for CMD in awk cat chmod date grep head mkdir mktemp rclone rm sed sha256sum sort sqlite3 tail tar wc; do +/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:131:rclone lsf "$REMOTE" --recursive >"$REMOTE_LIST" 2>"$ERROR_FILE" \ +/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:149:rclone copyto "$REMOTE$CRITICAL_REMOTE_PATH" "$CRITICAL_ARCHIVE" 2>"$ERROR_FILE" \ +/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:152:rclone copyto "$REMOTE$CRITICAL_REMOTE_SHA_PATH" "$CRITICAL_SHA_FILE" 2>"$ERROR_FILE" \ +/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:251:INTEGRITY_RESULT="$(sqlite3 "$DB_PATH" 'PRAGMA integrity_check;' 2>"$ERROR_FILE")" \ +/usr/local/sbin/routerguard-hard-power-cycle:3:MODE="${1:-check}" +/usr/local/sbin/routerguard-hard-power-cycle:4:case "$MODE" in check|cycle) ;; *) echo USAGE=routerguard-hard-power-cycle_check_or_cycle; exit 64;; esac +/usr/local/sbin/routerguard-network:48:remove_rules(){ +/usr/local/sbin/routerguard-network:91: rm -f "$RUN/active" +/usr/local/sbin/routerguard-network:92: remove_rules +/usr/local/sbin/routerguard-network:100: rm -rf "$RUN" +/etc/systemd/system/suspend.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub' +/etc/systemd/system/suspend-then-hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub' +/etc/systemd/system/hybrid-sleep.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub' +/etc/systemd/system/multi-user.target.wants/corosync.service:20:# as well; before editing this line, you may want to check DefaultStandardError +/etc/systemd/system/multi-user.target.wants/postfix.service:19:# perform 2-stage startup +/etc/systemd/system/multi-user.target.wants/postfix.service:20:ExecStartPre=+postfix check +/etc/systemd/system/multi-user.target.wants/postfix.service:30:# Individual Postfix daemons are started as root, optionally perform chroot +/etc/systemd/system/multi-user.target.wants/postfix.service:52:# CAP_CHOWN can be removed. +/etc/systemd/system/multi-user.target.wants/pve-guests.service:22:ExecStop=-/usr/bin/vzdump -stop +/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:22:ExecStart=/usr/sbin/pve-ha-lrm start +/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:23:ExecStop=/usr/sbin/pve-ha-lrm stop +/etc/systemd/system/multi-user.target.wants/rrdcached.service:2:# To customize, copy this file to /etc/systemd/system/ +/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:14:ExecStart=/usr/sbin/pve-ha-crm start +/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:15:ExecStop=/usr/sbin/pve-ha-crm stop +/etc/systemd/system/multi-user.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub' +/etc/systemd/system/multi-user.target.wants/e2scrub_reap.service:2:Description=Remove Stale Online ext4 Metadata Check Snapshots +/etc/systemd/system/default.target.wants/nvmefc-boot-connections.service:19:RemoveIPC=yes +/etc/systemd/system/default.target.wants/nvmf-autoconnect.service:21:RemoveIPC=yes +/etc/systemd/system/sysinit.target.wants/lvm2-lvmpolld.socket:10:RemoveOnStop=true +/etc/systemd/system/sysinit.target.wants/pvenetcommit.service:8:ExecStartPre=-/bin/rm -f /etc/openvswitch/conf.db +/etc/systemd/system/sysinit.target.wants/systemd-pstore.service:11:Description=Platform Persistent Storage Archival +/etc/systemd/system/pve-manager.service:22:ExecStop=-/usr/bin/vzdump -stop +/etc/systemd/system/hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub' +/etc/systemd/system/sockets.target.wants/dm-event.socket:10:RemoveOnStop=true +/etc/systemd/system/sockets.target.wants/rrdcached.socket:2:# To customize, copy this file to /etc/systemd/system/ +/etc/cron.d/vzdump:1:# cluster wide vzdump cron schedule +997503422 2026-07-01T03:31:11.1700059310 /var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.vma.zst +868048369 2026-06-30T03:31:08.4903256780 /var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.vma.zst +288349157 2026-07-21T03:30:21.2480085730 /var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.tar.zst +286538526 2026-07-20T03:30:18.4386603270 /var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.tar.zst +284443608 2026-07-19T03:47:41.8525503200 /var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst +283022034 2026-07-18T03:30:19.7617214460 /var/lib/vz/dump/vzdump-lxc-111-2026_07_18-03_30_03.tar.zst +220080970 2026-07-21T03:30:33.3910787180 /var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.tar.zst +220065825 2026-07-20T03:30:30.8287264090 /var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.tar.zst +220049041 2026-07-19T03:47:58.9256407960 /var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst +220033534 2026-07-18T03:30:37.2878143100 /var/lib/vz/dump/vzdump-lxc-113-2026_07_18-03_30_20.tar.zst +7547 2026-07-18T01:09:39.2467331080 /var/lib/vz/dump/vzdump-qemu-160-2026_07_18-01_02_44.log +7443 2026-07-18T01:45:13.3149404960 /var/lib/vz/dump/vzdump-qemu-160-2026_07_18-01_38_20.log +5796 2026-07-17T21:55:21.5648458010 /var/lib/vz/dump/vzdump-qemu-160-2026_07_17-21_30_20.log +2685 2026-06-30T03:31:09.1616927060 /var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.log +2590 2026-07-01T03:31:11.9500372070 /var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.log +1088 2026-07-20T03:30:31.0784689220 /var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.log +1085 2026-07-20T03:30:18.7427551960 /var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.log +988 2026-07-21T03:30:33.6364088960 /var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.log +988 2026-07-18T03:30:37.5450736960 /var/lib/vz/dump/vzdump-lxc-113-2026_07_18-03_30_20.log +985 2026-07-21T03:30:21.5465818340 /var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.log +985 2026-07-18T03:30:20.0884248500 /var/lib/vz/dump/vzdump-lxc-111-2026_07_18-03_30_03.log +731 2026-07-19T03:47:59.2238139650 /var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.log +728 2026-07-19T03:47:42.1183953260 /var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.log +========== NODE=pve03 ========== +=== REMOTES === +=== TIMERS === +Wed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:10 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service +=== SERVICES === +dpkg-db-backup.service static - +/usr/local/sbin/homelab-staging-check.sh:11:echo "=== homelab staging check: ${HOST} ===" +/usr/local/sbin/homelab-immutable-offline-verify:11: rm -f "$TMP" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:4:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:18: "checksums/" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:21:echo "=== homelab rclone crypt healthcheck: $(hostname) ===" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:40: if ! rclone listremotes | sed 's/:$//' | grep -qx "$REMOTE"; then +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:47: if ! rclone about "${REMOTE}:"; then +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:48: echo "FAIL: rclone about failed: $REMOTE" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:54: ROOT_LIST="$(rclone lsf "${REMOTE}:/" || true)" +/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:76: if rclone cat "${REMOTE}:/latest-backup-index.json" | jq -e '.schema and .remote and .status and .items' >/dev/null; then +/usr/local/sbin/homelab-rclone-required-check.sh:4:RAW_FILE="/etc/homelab/rclone/required-remotes.txt" +/usr/local/sbin/homelab-rclone-required-check.sh:5:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt" +/usr/local/sbin/homelab-rclone-required-check.sh:9:echo "=== homelab rclone required check: $(hostname) ===" +/usr/local/sbin/homelab-rclone-required-check.sh:12:echo "=== installed rclone ===" +/usr/local/sbin/homelab-rclone-required-check.sh:13:rclone version | head -5 +/usr/local/sbin/homelab-rclone-required-check.sh:17:CONFIGURED="$(rclone listremotes 2>/dev/null | sed 's/:$//' || true)" +/usr/local/sbin/homelab-rclone-required-check.sh:19: echo "No rclone remotes configured yet." +/etc/systemd/system/hybrid-sleep.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub' +/etc/systemd/system/multi-user.target.wants/pve-guests.service:22:ExecStop=-/usr/bin/vzdump -stop +/etc/systemd/system/multi-user.target.wants/corosync.service:20:# as well; before editing this line, you may want to check DefaultStandardError +/etc/systemd/system/multi-user.target.wants/rrdcached.service:2:# To customize, copy this file to /etc/systemd/system/ +/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:14:ExecStart=/usr/sbin/pve-ha-crm start +/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:15:ExecStop=/usr/sbin/pve-ha-crm stop +/etc/systemd/system/multi-user.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub' +/etc/systemd/system/multi-user.target.wants/postfix.service:19:# perform 2-stage startup +/etc/systemd/system/multi-user.target.wants/postfix.service:20:ExecStartPre=+postfix check +/etc/systemd/system/multi-user.target.wants/postfix.service:30:# Individual Postfix daemons are started as root, optionally perform chroot +/etc/systemd/system/multi-user.target.wants/postfix.service:52:# CAP_CHOWN can be removed. +/etc/systemd/system/multi-user.target.wants/e2scrub_reap.service:2:Description=Remove Stale Online ext4 Metadata Check Snapshots +/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:22:ExecStart=/usr/sbin/pve-ha-lrm start +/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:23:ExecStop=/usr/sbin/pve-ha-lrm stop +/etc/systemd/system/sysinit.target.wants/pvenetcommit.service:8:ExecStartPre=-/bin/rm -f /etc/openvswitch/conf.db +/etc/systemd/system/sysinit.target.wants/systemd-pstore.service:11:Description=Platform Persistent Storage Archival +/etc/systemd/system/sysinit.target.wants/lvm2-lvmpolld.socket:10:RemoveOnStop=true +/etc/systemd/system/hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub' +/etc/systemd/system/sockets.target.wants/rrdcached.socket:2:# To customize, copy this file to /etc/systemd/system/ +/etc/systemd/system/sockets.target.wants/dm-event.socket:10:RemoveOnStop=true +/etc/systemd/system/suspend-then-hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub' +/etc/systemd/system/default.target.wants/nvmf-autoconnect.service:21:RemoveIPC=yes +/etc/systemd/system/default.target.wants/nvmefc-boot-connections.service:19:RemoveIPC=yes +/etc/systemd/system/suspend.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub' +/etc/systemd/system/pve-manager.service:22:ExecStop=-/usr/bin/vzdump -stop +/etc/cron.d/vzdump:1:# cluster wide vzdump cron schedule +11116472459 2026-07-20T03:33:53.2839064370 /var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.vma.zst +11043834894 2026-07-21T03:33:54.3725716230 /var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.vma.zst +11013723404 2026-07-19T03:33:53.3463878040 /var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.vma.zst +10994820121 2026-07-18T03:33:47.1540038630 /var/lib/vz/dump/vzdump-qemu-130-2026_07_18-03_30_02.vma.zst +1137072350 2026-07-21T03:34:37.6276811360 /var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.vma.zst +838951976 2026-07-19T03:34:33.1034859540 /var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.vma.zst +838817925 2026-07-20T03:34:37.1719936670 /var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.vma.zst +785249297 2026-07-18T03:34:27.1531047780 /var/lib/vz/dump/vzdump-qemu-190-2026_07_18-03_33_48.vma.zst +7106 2026-07-02T03:37:53.1040459570 /var/lib/vz/dump/vzdump-qemu-130-2026_07_02-03_30_02.log +7102 2026-07-03T03:36:49.1036784480 /var/lib/vz/dump/vzdump-qemu-130-2026_07_03-03_30_05.log +6564 2026-07-18T03:33:48.0107527290 /var/lib/vz/dump/vzdump-qemu-130-2026_07_18-03_30_02.log +6560 2026-07-20T03:33:54.9283349630 /var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.log +6559 2026-07-21T03:33:55.1428157340 /var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.log +6459 2026-07-19T03:33:53.7631656400 /var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.log +6246 2026-07-04T03:33:39.2197744030 /var/lib/vz/dump/vzdump-qemu-130-2026_07_04-03_30_02.log +4107 2026-07-07T03:31:51.2516816970 /var/lib/vz/dump/vzdump-qemu-130-2026_07_07-03_30_01.log +2642 2026-07-21T03:34:38.4491706750 /var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.log +2500 2026-07-20T03:34:37.2013439550 /var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.log +2404 2026-07-19T03:34:34.1266444600 /var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.log +2305 2026-07-18T03:34:27.7179455720 /var/lib/vz/dump/vzdump-qemu-190-2026_07_18-03_33_48.log +=== CLOUD_HEALTH_AND_OBJECTS === +=== CURRENT_BACKUP_MATRIX === +STATUS=FAIL +TS=2026-07-21T07:02:41Z +TYPE=unified-backup-matrix +MATRIX_MODE=authoritative +JOBS_TOTAL=41 +JOBS_OK=30 +JOBS_BAD=6 +JOBS_UNMAPPED=0 +JOBS_DISABLED=5 +STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_JSON=/etc/homelab-backup-matrix/matrix.json +AUTHORITY=canonical +COMPAT_OUTPUTS=29 +COMPAT_MAP=/var/lib/homelab-backup-matrix/compat-map.json +JOB=dpkg-db-backup STATUS=OK EVIDENCE=execution_only AGE=36147 MAX_AGE=172800 +JOB=filebrowser-backup STATUS=OK EVIDENCE=artifact_or_health AGE=14443 MAX_AGE=259200 +JOB=filebrowser-restore-validate STATUS=STALE EVIDENCE=artifact_or_health AGE=1016424 MAX_AGE=864000 +JOB=homelab-appbackup-dockge-stacks STATUS=OK EVIDENCE=artifact_or_health AGE=22248 MAX_AGE=172800 +JOB=homelab-appbackup-mariadb-batch STATUS=OK EVIDENCE=execution_only AGE=15387 MAX_AGE=172800 +JOB=homelab-appbackup-npmplus-edge STATUS=OK EVIDENCE=artifact_or_health AGE=20573 MAX_AGE=172800 +JOB=homelab-appbackup-postgres-batch STATUS=OK EVIDENCE=execution_only AGE=16416 MAX_AGE=172800 +JOB=homelab-appbackup-sqlite-batch STATUS=OK EVIDENCE=execution_only AGE=18327 MAX_AGE=172800 +JOB=homelab-authentik-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987790 MAX_AGE=172800 +JOB=homelab-auto-backup STATUS=OK EVIDENCE=execution_only AGE=195424 MAX_AGE=864000 +JOB=homelab-backup-coverage-matrix STATUS=OK EVIDENCE=artifact_or_health AGE=4346 MAX_AGE=172800 +JOB=homelab-backup-freshness STATUS=OK EVIDENCE=execution_only AGE=35589 MAX_AGE=172800 +JOB=homelab-backup-healthcheck STATUS=OK EVIDENCE=execution_only AGE=10631 MAX_AGE=172800 +JOB=homelab-backup-sla-health STATUS=FAIL EVIDENCE=artifact_or_health AGE=144 MAX_AGE=172800 +JOB=homelab-edge-vm-offhost-freshness STATUS=OK EVIDENCE=artifact_or_health AGE=18426 MAX_AGE=172800 +JOB=homelab-emergency-backup STATUS=OK EVIDENCE=artifact_or_health AGE=24539 MAX_AGE=172800 +JOB=homelab-extended-appbackup STATUS=OK EVIDENCE=artifact_or_health AGE=14156 MAX_AGE=172800 +JOB=homelab-gitea-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987785 MAX_AGE=172800 +JOB=homelab-golden-state-index STATUS=OK EVIDENCE=artifact_or_health AGE=4112 MAX_AGE=172800 +JOB=homelab-mail-cloud-critical-upload STATUS=OK EVIDENCE=artifact_or_health AGE=11997 MAX_AGE=172800 +JOB=homelab-mail-cloud-edge-vm-upload STATUS=OK EVIDENCE=artifact_or_health AGE=174045 MAX_AGE=864000 +JOB=homelab-mail-cloud-nextcloud-vm-upload STATUS=OK EVIDENCE=artifact_or_health AGE=7575 MAX_AGE=864000 +JOB=homelab-mail-cloud-restore-drill STATUS=STALE EVIDENCE=artifact_or_health AGE=988269 MAX_AGE=864000 +JOB=homelab-npmplus-kuma-config-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=60071 MAX_AGE=172800 +JOB=homelab-router-running-config-mail-cloud STATUS=OK EVIDENCE=artifact_or_health AGE=17076 MAX_AGE=172800 +JOB=homelab-vaultwarden-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987768 MAX_AGE=172800 +JOB=homelab-verified-backup-health STATUS=OK EVIDENCE=artifact_or_health AGE=11845 MAX_AGE=172800 +JOB=homelab-vm-backup-health STATUS=OK EVIDENCE=artifact_or_health AGE=8742 MAX_AGE=172800 +JOB=homelab-vm-backup-policy STATUS=OK EVIDENCE=artifact_or_health AGE=5548 MAX_AGE=172800 +JOB=homelab-vm-full-backup STATUS=OK EVIDENCE=artifact_or_health AGE=24778 MAX_AGE=172800 +JOB=homelab-vm-full-restore-drill STATUS=OK EVIDENCE=artifact_or_health AGE=523085 MAX_AGE=10368000 +JOB=homelab-vm-local-dumps-2cloud STATUS=OK EVIDENCE=artifact_or_health AGE=172141 MAX_AGE=864000 +JOB=homelab-vm-local-dumps-retention STATUS=OK EVIDENCE=artifact_or_health AGE=168839 MAX_AGE=864000 +JOB=immich-media-offhost-sync STATUS=OK EVIDENCE=artifact_or_health AGE=4286 MAX_AGE=259200 +JOB=memos-backup STATUS=OK EVIDENCE=artifact_or_health AGE=14547 MAX_AGE=259200 +JOB=netbird-vps-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=1747896 MAX_AGE=864000 +JOB=paperless-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=16347 MAX_AGE=259200 +JOB=paperless-restore-dry-run STATUS=STALE EVIDENCE=artifact_or_health AGE=1001986 MAX_AGE=864000 +JOB=skladchik-reports-monitor-backup STATUS=OK EVIDENCE=artifact_or_health AGE=19639 MAX_AGE=172800 +JOB=skladchik-reports-monitor-backup-restore-check STATUS=OK EVIDENCE=artifact_or_health AGE=190273 MAX_AGE=864000 +JOB=vm160-remote-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=26607 MAX_AGE=172800 +PAIR_DELETE_GUARD_AUDIT=COMPLETE OUTPUT_END CHAT_OUTPUT_END