{
  "schema_version": 1,
  "channel": "homelab-runtime",
  "command_id": "BACKUP-PIPELINE-DEFINITION-AUDIT-061",
  "status": "OK",
  "rc": 0,
  "host": "pve01",
  "mode": "read-only",
  "component": "backup-audit",
  "started_at_utc": "2026-07-21T11:46:07Z",
  "finished_at_utc": "2026-07-21T11:46:09Z",
  "reference_register_checked": true,
  "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
  "error_register_checked": true,
  "error_register_sha256": "f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1",
  "command_sha256": "08de7a8820803f855426b27b3ecfd2e646209715ff5ac4aba48b73fa45623b28",
  "duplicate_failed_command_blocked": false,
  "block_reason": null,
  "execution_started": true,
  "changes_made": false,
  "sanitized": true,
  "secrets_included": false,
  "private_addresses_included": false,
  "raw_evidence_retained_locally": true,
  "raw_evidence_sha256": "5b51699fff305570e41040468ef3f04a84e21c0c977695ed1834aa40c01946b5",
  "sanitized_output_sha256": "384c3f223883553a582cbf25ff144f7ff8a0a1dc79110ffe089b7154a12aed74",
  "output_truncated_in_json": false,
  "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
  "output": "===== TIMER dpkg-db-backup.timer =====\n# /usr/lib/systemd/system/dpkg-db-backup.timer\n[Unit]\nDescription=Daily dpkg database backup timer\nDocumentation=man:dpkg(1)\n\n[Timer]\nOnCalendar=daily\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE dpkg-db-backup.service =====\n# /usr/lib/systemd/system/dpkg-db-backup.service\n[Unit]\nDescription=Daily dpkg database backup service\nDocumentation=man:dpkg(1)\n\n[Service]\nType=oneshot\nExecStart=/usr/libexec/dpkg/dpkg-db-backup\n\n# /etc/systemd/system/dpkg-db-backup.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/libexec/dpkg/dpkg-db-backup ; argv[]=/usr/libexec/dpkg/dpkg-db-backup ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER filebrowser-backup.timer =====\n# /etc/systemd/system/filebrowser-backup.timer\n[Unit]\nDescription=Run File Browser backup\n\n[Timer]\nOnCalendar=*-*-* 03:35:00\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE filebrowser-backup.service =====\n# /etc/systemd/system/filebrowser-backup.service\n[Unit]\nDescription=File Browser backup\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/root/filebrowser-backup.sh\n\n# /etc/systemd/system/filebrowser-backup.service.d/10-superseded-noop.conf\n[Service]\nExecStart=\nExecStart=/usr/local/sbin/homelab-superseded-unit-ok filebrowser-backup.service superseded_by_current_appbackup_restore_proof\n\n# /etc/systemd/system/filebrowser-backup.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok filebrowser-backup.service superseded_by_current_appbackup_restore_proof ; ignore_errors=no ; start_time=[Tue 2026-07-21 03:35:14 MSK] ; stop_time=[Tue 2026-07-21 03:35:14 MSK] ; pid=2211390 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-alertmanager-backup.timer =====\n# /etc/systemd/system/homelab-alertmanager-backup.timer\n[Unit]\nDescription=Daily Alertmanager two-cloud backup\n\n[Timer]\nOnCalendar=*-*-* 06:35:00\nRandomizedDelaySec=10m\nPersistent=true\nUnit=homelab-alertmanager-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-alertmanager-backup.service =====\n# /etc/systemd/system/homelab-alertmanager-backup.service\n[Unit]\nDescription=Alertmanager encrypted two-cloud application backup\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-alertmanager-backup-run\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-alertmanager-backup-run ; argv[]=/usr/local/sbin/homelab-alertmanager-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 06:40:25 MSK] ; stop_time=[Tue 2026-07-21 06:40:54 MSK] ; pid=2820982 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-appbackup-dockge-stacks.timer =====\n# /etc/systemd/system/homelab-appbackup-dockge-stacks.timer\n[Unit]\nDescription=Daily homelab app-level backup timer for dockge-stacks\n\n[Timer]\nOnCalendar=*-*-* 03:40:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-appbackup-dockge-stacks.service =====\n# /etc/systemd/system/homelab-appbackup-dockge-stacks.service\n[Unit]\nDescription=Homelab app-level backup for dockge-stacks\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/appbackupctl run\nExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run dockge-stacks\n\n# /etc/systemd/system/homelab-appbackup-dockge-stacks.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/appbackupctl ; argv[]=/usr/local/sbin/appbackupctl run ; ignore_errors=no ; start_time=[Tue 2026-07-21 03:51:25 MSK] ; stop_time=[Tue 2026-07-21 03:51:53 MSK] ; pid=2261529 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-appbackup-mariadb-batch.timer =====\n# /etc/systemd/system/homelab-appbackup-mariadb-batch.timer\n[Unit]\nDescription=Daily Homelab MariaDB app batch backup timer\n\n[Timer]\nOnCalendar=*-*-* 05:35:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-appbackup-mariadb-batch.service =====\n# /etc/systemd/system/homelab-appbackup-mariadb-batch.service\n[Unit]\nDescription=Homelab MariaDB app batch backup\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/mariadb-apps-backup-run\nExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run bookstack\n\n# /etc/systemd/system/homelab-appbackup-mariadb-batch.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/mariadb-apps-backup-run ; argv[]=/usr/local/sbin/mariadb-apps-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 05:46:14 MSK] ; stop_time=[Tue 2026-07-21 05:46:44 MSK] ; pid=2623964 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-appbackup-npmplus-edge.timer =====\n# /etc/systemd/system/homelab-appbackup-npmplus-edge.timer\n[Unit]\nDescription=Daily homelab app-level backup timer for npmplus-edge\n\n[Timer]\nOnCalendar=*-*-* 04:05:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-appbackup-npmplus-edge.service =====\n# /etc/systemd/system/homelab-appbackup-npmplus-edge.service\n[Unit]\nDescription=Homelab app-level backup for npmplus-edge\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/npmplus-edge-backup-run\nExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run npmplus-edge\n\n# /etc/systemd/system/homelab-appbackup-npmplus-edge.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/npmplus-edge-backup-run ; argv[]=/usr/local/sbin/npmplus-edge-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 04:19:17 MSK] ; stop_time=[Tue 2026-07-21 04:19:48 MSK] ; pid=2360589 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-appbackup-postgres-batch.timer =====\n# /etc/systemd/system/homelab-appbackup-postgres-batch.timer\n[Unit]\nDescription=Daily Homelab Postgres app batch backup timer\n\n[Timer]\nOnCalendar=*-*-* 05:10:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-appbackup-postgres-batch.service =====\n# /etc/systemd/system/homelab-appbackup-postgres-batch.service\n[Unit]\nDescription=Homelab Postgres app batch backup\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/postgres-apps-backup-run\nExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run authentik\nExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run paperless\nExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run immich\nExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run netbox\n\n# /etc/systemd/system/homelab-appbackup-postgres-batch.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/postgres-apps-backup-run ; argv[]=/usr/local/sbin/postgres-apps-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 05:29:05 MSK] ; stop_time=[Tue 2026-07-21 05:32:29 MSK] ; pid=2568901 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-appbackup-sqlite-batch.timer =====\n# /etc/systemd/system/homelab-appbackup-sqlite-batch.timer\n[Unit]\nDescription=Daily Homelab SQLite app batch backup timer\n\n[Timer]\nOnCalendar=*-*-* 04:45:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-appbackup-sqlite-batch.service =====\n# /etc/systemd/system/homelab-appbackup-sqlite-batch.service\n[Unit]\nDescription=Homelab SQLite app batch backup\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/sqlite-apps-backup-run\nExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run vaultwarden\nExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run gitea\nExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run uptime-kuma\nExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run gotify\n\n# /etc/systemd/system/homelab-appbackup-sqlite-batch.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/sqlite-apps-backup-run ; argv[]=/usr/local/sbin/sqlite-apps-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 04:57:14 MSK] ; stop_time=[Tue 2026-07-21 05:00:36 MSK] ; pid=2476638 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-authentik-backup.timer =====\n# /etc/systemd/system/homelab-authentik-backup.timer\n[Unit]\nDescription=Run Homelab Authentik backup chain daily\n\n[Timer]\nOnCalendar=*-*-* 04:50:00\nRandomizedDelaySec=10min\nPersistent=true\nUnit=homelab-authentik-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-authentik-backup.service =====\n# /etc/systemd/system/homelab-authentik-backup.service\n[Unit]\nDescription=Homelab Authentik backup chain\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-backup-authentik\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-backup-authentik ; argv[]=/usr/local/sbin/homelab-backup-authentik ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER homelab-auto-backup.timer =====\n# /etc/systemd/system/homelab-auto-backup.timer\n[Unit]\nDescription=Run homelab automatic full backup weekly\n\n[Timer]\nOnCalendar=Sun *-*-* 03:30:00\nPersistent=true\nRandomizedDelaySec=20m\nUnit=homelab-auto-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-auto-backup.service =====\n# /etc/systemd/system/homelab-auto-backup.service\n[Unit]\nDescription=Homelab automatic full backup with retention\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/root/homelab-auto-backup-with-retention.sh\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\n\n# /etc/systemd/system/homelab-auto-backup.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/root/homelab-auto-backup-with-retention.sh ; argv[]=/root/homelab-auto-backup-with-retention.sh ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER homelab-backup-coverage-matrix.timer =====\n# /etc/systemd/system/homelab-backup-coverage-matrix.timer\n[Unit]\nDescription=Homelab backup coverage matrix timer\n\n[Timer]\nOnCalendar=*-*-* 08:35:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-backup-coverage-matrix.service =====\n# /etc/systemd/system/homelab-backup-coverage-matrix.service\n[Unit]\nDescription=Homelab backup coverage matrix\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-backup-coverage-matrix\n\n# /etc/systemd/system/homelab-backup-coverage-matrix.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-backup-coverage-matrix ; argv[]=/usr/local/sbin/homelab-backup-coverage-matrix ; ignore_errors=no ; start_time=[Tue 2026-07-21 08:50:14 MSK] ; stop_time=[Tue 2026-07-21 08:50:15 MSK] ; pid=3277860 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-backup-freshness.timer =====\n# /etc/systemd/system/homelab-backup-freshness.timer\n[Unit]\nDescription=Run homelab backup freshness collector daily\n\n[Timer]\nOnCalendar=daily\nPersistent=true\nRandomizedDelaySec=20m\nUnit=homelab-backup-freshness.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-backup-freshness.service =====\n# /etc/systemd/system/homelab-backup-freshness.service\n[Unit]\nDescription=Homelab backup freshness collector\nDocumentation=man:systemd.service(5)\nAfter=local-fs.target\n\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/default/homelab-backup-freshness\nExecStart=/usr/local/sbin/homelab-backup-freshness.sh\n\n# /etc/systemd/system/homelab-backup-freshness.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-backup-freshness.sh ; argv[]=/usr/local/sbin/homelab-backup-freshness.sh ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER homelab-backup-healthcheck.timer =====\n# /etc/systemd/system/homelab-backup-healthcheck.timer\n[Unit]\nDescription=Run homelab backup healthcheck daily\n\n[Timer]\nOnCalendar=*-*-* 07:00:00\nPersistent=true\nRandomizedDelaySec=10m\nUnit=homelab-backup-healthcheck.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-backup-healthcheck.service =====\n# /etc/systemd/system/homelab-backup-healthcheck.service\n[Unit]\nDescription=Homelab backup healthcheck\n\n[Service]\nType=oneshot\nExecStart=/root/homelab-backup-healthcheck-wrapper.sh\n\n# /etc/systemd/system/homelab-backup-healthcheck.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/root/homelab-backup-healthcheck-wrapper.sh ; argv[]=/root/homelab-backup-healthcheck-wrapper.sh ; ignore_errors=no ; start_time=[Tue 2026-07-21 07:05:30 MSK] ; stop_time=[Tue 2026-07-21 07:05:43 MSK] ; pid=2900293 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-backup-matrix-refresh.timer =====\n# /etc/systemd/system/homelab-backup-matrix-refresh.timer\n[Unit]\nDescription=Refresh unified homelab backup matrix state\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=5min\nAccuracySec=30s\nPersistent=true\nUnit=homelab-backup-matrix-refresh.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-backup-matrix-refresh.service =====\n# /etc/systemd/system/homelab-backup-matrix-refresh.service\n[Unit]\nDescription=Generate unified homelab backup matrix state\nAfter=local-fs.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-backup-matrix-refresh\nTimeoutStartSec=5min\nResult=exit-code\nExecMainStatus=1\nExecStart={ path=/usr/local/sbin/homelab-backup-matrix-refresh ; argv[]=/usr/local/sbin/homelab-backup-matrix-refresh ; ignore_errors=no ; start_time=[Tue 2026-07-21 10:02:41 MSK] ; stop_time=[Tue 2026-07-21 10:02:42 MSK] ; pid=3525445 ; code=exited ; status=1 }\nActiveState=failed\n===== TIMER homelab-backup-sla-health.timer =====\n# /etc/systemd/system/homelab-backup-sla-health.timer\n[Unit]\nDescription=Homelab backup freshness SLA health timer\n\n[Timer]\nOnCalendar=*-*-* 08:00:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-backup-sla-health.service =====\n# /etc/systemd/system/homelab-backup-sla-health.service\n[Unit]\nDescription=Homelab backup freshness SLA health\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-backup-sla-health\n\n# /etc/systemd/system/homelab-backup-sla-health.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-backup-sla-health ; argv[]=/usr/local/sbin/homelab-backup-sla-health ; ignore_errors=no ; start_time=[Tue 2026-07-21 08:07:25 MSK] ; stop_time=[Tue 2026-07-21 08:07:26 MSK] ; pid=3122093 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-beszel-backup.timer =====\n# /etc/systemd/system/homelab-beszel-backup.timer\n[Unit]\nDescription=Daily Beszel encrypted two-cloud backup\n\n[Timer]\nOnCalendar=*-*-* 06:45:00\nRandomizedDelaySec=10m\nPersistent=true\nUnit=homelab-beszel-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-beszel-backup.service =====\n# /etc/systemd/system/homelab-beszel-backup.service\n[Unit]\nDescription=Beszel encrypted two-cloud application backup\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-beszel-backup-run\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-beszel-backup-run ; argv[]=/usr/local/sbin/homelab-beszel-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 06:47:47 MSK] ; stop_time=[Tue 2026-07-21 06:48:22 MSK] ; pid=2848350 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-blackbox-exporter-backup.timer =====\n# /etc/systemd/system/homelab-blackbox-exporter-backup.timer\n[Unit]\nDescription=Daily Blackbox Exporter encrypted two-cloud backup\n\n[Timer]\nOnCalendar=*-*-* 06:55:00\nRandomizedDelaySec=10m\nPersistent=true\nUnit=homelab-blackbox-exporter-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-blackbox-exporter-backup.service =====\n# /etc/systemd/system/homelab-blackbox-exporter-backup.service\n[Unit]\nDescription=Blackbox Exporter encrypted two-cloud application backup\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-blackbox-exporter-backup-run\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-blackbox-exporter-backup-run ; argv[]=/usr/local/sbin/homelab-blackbox-exporter-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 06:57:30 MSK] ; stop_time=[Tue 2026-07-21 06:57:54 MSK] ; pid=2876084 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-cluster-admin-incident-engine-backup.timer =====\n# /etc/systemd/system/homelab-cluster-admin-incident-engine-backup.timer\n[Unit]\nDescription=Daily Cluster Admin Incident Engine two-cloud backup\n\n[Timer]\nOnCalendar=*-*-* 07:25:00\nRandomizedDelaySec=15m\nPersistent=true\nUnit=homelab-cluster-admin-incident-engine-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-cluster-admin-incident-engine-backup.service =====\n# /etc/systemd/system/homelab-cluster-admin-incident-engine-backup.service\n[Unit]\nDescription=Cluster Admin Incident Engine encrypted two-cloud backup\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run ; argv[]=/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 07:36:14 MSK] ; stop_time=[Tue 2026-07-21 07:36:40 MSK] ; pid=3010566 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-crypto-portfolio-backup.timer =====\n# /etc/systemd/system/homelab-crypto-portfolio-backup.timer\n[Unit]\nDescription=Daily Gram Crypto portfolio backup\n\n[Timer]\nOnCalendar=*-*-* 06:25:00\nRandomizedDelaySec=15m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-crypto-portfolio-backup.service =====\n# /etc/systemd/system/homelab-crypto-portfolio-backup.service\n[Unit]\nDescription=Gram Crypto portfolio two-cloud backup\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/crypto-portfolio-backup-run\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/crypto-portfolio-backup-run ; argv[]=/usr/local/sbin/crypto-portfolio-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 06:32:25 MSK] ; stop_time=[Tue 2026-07-21 06:32:50 MSK] ; pid=2794060 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-edge-vm-vzdump-backup.timer =====\n# /etc/systemd/system/homelab-edge-vm-vzdump-backup.timer\n[Unit]\nDescription=Run Homelab edge-vm VM-level backup chain weekly\n\n[Timer]\nOnCalendar=Sun *-*-* 06:20:00\nRandomizedDelaySec=20min\nPersistent=true\nUnit=homelab-edge-vm-vzdump-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-edge-vm-vzdump-backup.service =====\n# /etc/systemd/system/homelab-edge-vm-vzdump-backup.service\n[Unit]\nDescription=Homelab edge-vm VM-level vzdump backup chain\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-backup-edge-vm-vzdump\n\n# /etc/systemd/system/homelab-edge-vm-vzdump-backup.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-backup-edge-vm-vzdump ; argv[]=/usr/local/sbin/homelab-backup-edge-vm-vzdump ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER homelab-emergency-backup.timer =====\n# /etc/systemd/system/homelab-emergency-backup.timer\n[Unit]\nDescription=Daily homelab emergency/config backup timer\n\n[Timer]\nOnCalendar=*-*-* 03:10:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-emergency-backup.service =====\n# /etc/systemd/system/homelab-emergency-backup.service\n[Unit]\nDescription=Homelab emergency/config backup to two encrypted clouds\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/backupctl run\nExecStartPost=/usr/local/sbin/backup-retention 14 dry-run\n\n# /etc/systemd/system/homelab-emergency-backup.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/backupctl ; argv[]=/usr/local/sbin/backupctl run ; ignore_errors=no ; start_time=[Tue 2026-07-21 03:13:14 MSK] ; stop_time=[Tue 2026-07-21 03:13:42 MSK] ; pid=2137825 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-extended-appbackup.timer =====\n# /etc/systemd/system/homelab-extended-appbackup.timer\n[Unit]\nDescription=Homelab extended P1/P2 appbackup timer\n\n[Timer]\nOnCalendar=*-*-* 05:55:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-extended-appbackup.service =====\n# /etc/systemd/system/homelab-extended-appbackup.service\n[Unit]\nDescription=Homelab extended P1/P2 appbackup run\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-extended-appbackup-run\n\n# /etc/systemd/system/homelab-extended-appbackup.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-extended-appbackup-run ; argv[]=/usr/local/sbin/homelab-extended-appbackup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 05:57:30 MSK] ; stop_time=[Tue 2026-07-21 06:06:46 MSK] ; pid=2671623 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-gitea-backup.timer =====\n# /etc/systemd/system/homelab-gitea-backup.timer\n[Unit]\nDescription=Run Homelab Gitea backup chain daily\n\n[Timer]\nOnCalendar=*-*-* 05:10:00\nRandomizedDelaySec=10min\nPersistent=true\nUnit=homelab-gitea-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-gitea-backup.service =====\n# /etc/systemd/system/homelab-gitea-backup.service\n[Unit]\nDescription=Homelab Gitea backup chain\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-backup-gitea\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-backup-gitea ; argv[]=/usr/local/sbin/homelab-backup-gitea ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER homelab-gitea-secondary-backup.timer =====\n# /etc/systemd/system/homelab-gitea-secondary-backup.timer\n[Unit]\nDescription=Daily verified Gitea backup to pve03\n\n[Timer]\nOnCalendar=*-*-* 04:20:00\nRandomizedDelaySec=10m\nPersistent=true\nUnit=homelab-gitea-secondary-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-gitea-secondary-backup.service =====\n# /etc/systemd/system/homelab-gitea-secondary-backup.service\n[Unit]\nDescription=Create and copy a verified Gitea backup to pve03\nAfter=pve-cluster.service network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-gitea-secondary-backup\nTimeoutStartSec=2h\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\nUMask=0077\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-gitea-secondary-backup ; argv[]=/usr/local/sbin/homelab-gitea-secondary-backup ; ignore_errors=no ; start_time=[Tue 2026-07-21 04:28:14 MSK] ; stop_time=[Tue 2026-07-21 04:28:18 MSK] ; pid=2376819 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-legacy-monitoring-backups.timer =====\n# /etc/systemd/system/homelab-legacy-monitoring-backups.timer\n[Unit]\nDescription=Daily legacy monitoring applications two-cloud backup batch\n\n[Timer]\nOnCalendar=*-*-* 07:05:00\nRandomizedDelaySec=15m\nPersistent=true\nUnit=homelab-legacy-monitoring-backups.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-legacy-monitoring-backups.service =====\n# /etc/systemd/system/homelab-legacy-monitoring-backups.service\n[Unit]\nDescription=Legacy monitoring applications encrypted two-cloud backup batch\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-legacy-monitoring-backups-run\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-legacy-monitoring-backups-run ; argv[]=/usr/local/sbin/homelab-legacy-monitoring-backups-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 07:18:55 MSK] ; stop_time=[Tue 2026-07-21 07:21:45 MSK] ; pid=2950676 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-mail-cloud-critical-upload.timer =====\n# /etc/systemd/system/homelab-mail-cloud-critical-upload.timer\n[Unit]\nDescription=Daily Homelab Mail Cloud critical/small baseline upload\n\n[Timer]\nOnCalendar=*-*-* 06:40:00\nRandomizedDelaySec=15m\nPersistent=true\nUnit=homelab-mail-cloud-critical-upload.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-mail-cloud-critical-upload.service =====\n# /etc/systemd/system/homelab-mail-cloud-critical-upload.service\n[Unit]\nDescription=Homelab Mail Cloud critical/small baseline upload\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-mail-cloud-critical-upload\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-mail-cloud-critical-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-critical-upload ; ignore_errors=no ; start_time=[Tue 2026-07-21 06:40:25 MSK] ; stop_time=[Tue 2026-07-21 06:42:44 MSK] ; pid=2820985 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-mail-cloud-edge-vm-upload.timer =====\n# /etc/systemd/system/homelab-mail-cloud-edge-vm-upload.timer\n[Unit]\nDescription=Weekly Homelab Mail Cloud edge-vm large chunked upload\n\n[Timer]\nOnCalendar=Sun *-*-* 09:30:00\nRandomizedDelaySec=30m\nPersistent=true\nUnit=homelab-mail-cloud-edge-vm-upload.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-mail-cloud-edge-vm-upload.service =====\n# /etc/systemd/system/homelab-mail-cloud-edge-vm-upload.service\n[Unit]\nDescription=Homelab Mail Cloud edge-vm large chunked upload\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER homelab-mail-cloud-nextcloud-vm-upload.timer =====\n# /etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.timer\n[Unit]\nDescription=Run Homelab Mail Cloud Nextcloud VM upload daily\n\n[Timer]\nOnCalendar=*-*-* 07:30:00\nRandomizedDelaySec=30m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-mail-cloud-nextcloud-vm-upload.service =====\n# /etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.service\n[Unit]\nDescription=Homelab Mail Cloud Nextcloud VM vzdump upload\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload ; ignore_errors=no ; start_time=[Tue 2026-07-21 07:53:41 MSK] ; stop_time=[Tue 2026-07-21 07:56:26 MSK] ; pid=3068432 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-mail-cloud-restore-drill.timer =====\n# /etc/systemd/system/homelab-mail-cloud-restore-drill.timer\n[Unit]\nDescription=Weekly Homelab Mail Cloud restore drill\n\n[Timer]\nOnCalendar=Mon *-*-* 07:20:00\nRandomizedDelaySec=20m\nPersistent=true\nUnit=homelab-mail-cloud-restore-drill.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-mail-cloud-restore-drill.service =====\n# /etc/systemd/system/homelab-mail-cloud-restore-drill.service\n[Unit]\nDescription=Homelab Mail Cloud restore drill\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-mail-cloud-restore-drill\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\n\n# /etc/systemd/system/homelab-mail-cloud-restore-drill.service.d/10-superseded-noop.conf\n[Service]\nExecStart=\nExecStart=/usr/local/sbin/homelab-superseded-unit-ok homelab-mail-cloud-restore-drill.service superseded_by_safety_net_restore_check\n\n# /etc/systemd/system/homelab-mail-cloud-restore-drill.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok homelab-mail-cloud-restore-drill.service superseded_by_safety_net_restore_check ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER homelab-npmplus-kuma-config-backup.timer =====\n# /etc/systemd/system/homelab-npmplus-kuma-config-backup.timer\n[Unit]\nDescription=Run Homelab NPMplus + Kuma config backup chain daily\n\n[Timer]\nOnCalendar=*-*-* 04:20:00\nRandomizedDelaySec=10min\nPersistent=true\nUnit=homelab-npmplus-kuma-config-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-npmplus-kuma-config-backup.service =====\n# /etc/systemd/system/homelab-npmplus-kuma-config-backup.service\n[Unit]\nDescription=Homelab NPMplus + Kuma config backup chain\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-backup-npmplus-kuma-configs\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-backup-npmplus-kuma-configs ; argv[]=/usr/local/sbin/homelab-backup-npmplus-kuma-configs ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER homelab-router-running-config-mail-cloud.timer =====\n# /etc/systemd/system/homelab-router-running-config-mail-cloud.timer\n[Unit]\nDescription=Run Homelab router running-config Mail-cloud backup daily\n\n[Timer]\nOnCalendar=*-*-* 05:15:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-router-running-config-mail-cloud.service =====\n# /etc/systemd/system/homelab-router-running-config-mail-cloud.service\n[Unit]\nDescription=Homelab router running-config Mail-cloud backup\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-router-running-config-mail-cloud\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-router-running-config-mail-cloud ; argv[]=/usr/local/sbin/homelab-router-running-config-mail-cloud ; ignore_errors=no ; start_time=[Tue 2026-07-21 05:17:57 MSK] ; stop_time=[Tue 2026-07-21 05:18:05 MSK] ; pid=2540965 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-vaultwarden-backup.timer =====\n# /etc/systemd/system/homelab-vaultwarden-backup.timer\n[Unit]\nDescription=Run Homelab Vaultwarden backup chain daily\n\n[Timer]\nOnCalendar=*-*-* 05:30:00\nRandomizedDelaySec=10min\nPersistent=true\nUnit=homelab-vaultwarden-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-vaultwarden-backup.service =====\n# /etc/systemd/system/homelab-vaultwarden-backup.service\n[Unit]\nDescription=Homelab Vaultwarden backup chain\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-backup-vaultwarden\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-backup-vaultwarden ; argv[]=/usr/local/sbin/homelab-backup-vaultwarden ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER homelab-verified-backup-health.timer =====\n# /etc/systemd/system/homelab-verified-backup-health.timer\n[Unit]\nDescription=Homelab verified backup health evidence timer\n\n[Timer]\nOnCalendar=*-*-* 06:30:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-verified-backup-health.service =====\n# /etc/systemd/system/homelab-verified-backup-health.service\n[Unit]\nDescription=Homelab verified backup health evidence\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-backup-health-evidence\n\n# /etc/systemd/system/homelab-verified-backup-health.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-backup-health-evidence ; argv[]=/usr/local/sbin/homelab-backup-health-evidence ; ignore_errors=no ; start_time=[Tue 2026-07-21 06:43:54 MSK] ; stop_time=[Tue 2026-07-21 06:45:16 MSK] ; pid=2823792 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-vm-backup-health.timer =====\n# /etc/systemd/system/homelab-vm-backup-health.timer\n[Unit]\nDescription=Homelab VM backup health timer\n\n[Timer]\nOnCalendar=*-*-* 07:30:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-vm-backup-health.service =====\n# /etc/systemd/system/homelab-vm-backup-health.service\n[Unit]\nDescription=Homelab VM backup health\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-vm-backup-health\n\n# /etc/systemd/system/homelab-vm-backup-health.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-vm-backup-health ; argv[]=/usr/local/sbin/homelab-vm-backup-health ; ignore_errors=no ; start_time=[Tue 2026-07-21 07:36:59 MSK] ; stop_time=[Tue 2026-07-21 07:36:59 MSK] ; pid=3011738 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-vm-backup-policy.timer =====\n# /etc/systemd/system/homelab-vm-backup-policy.timer\n[Unit]\nDescription=Homelab VM backup policy and feasibility health timer\n\n[Timer]\nOnCalendar=*-*-* 08:18:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-vm-backup-policy.service =====\n# /etc/systemd/system/homelab-vm-backup-policy.service\n[Unit]\nDescription=Homelab VM backup policy and feasibility health\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-vm-backup-policy\n\n# /etc/systemd/system/homelab-vm-backup-policy.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-vm-backup-policy ; argv[]=/usr/local/sbin/homelab-vm-backup-policy ; ignore_errors=no ; start_time=[Tue 2026-07-21 08:30:07 MSK] ; stop_time=[Tue 2026-07-21 08:30:13 MSK] ; pid=3198902 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-vm-full-backup.timer =====\n# /etc/systemd/system/homelab-vm-full-backup.timer\n[Unit]\nDescription=Daily full backup timer for VM170 and VM171\n\n[Timer]\nOnCalendar=*-*-* 02:15:00\nPersistent=true\nAccuracySec=1min\nUnit=homelab-vm-full-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-vm-full-backup.service =====\n# /etc/systemd/system/homelab-vm-full-backup.service\n[Unit]\nDescription=Full vzdump backup of VM170 and VM171 to offhost and two clouds\nAfter=network-online.target pve-cluster.service\nWants=network-online.target\nConditionPathIsMountPoint=/mnt/staging\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-vm-full-backup\nTimeoutStartSec=infinity\nNice=10\nIOSchedulingClass=idle\nIOSchedulingPriority=7\nUMask=0077\n\n# /etc/systemd/system/homelab-vm-full-backup.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-vm-full-backup ; argv[]=/usr/local/sbin/homelab-vm-full-backup ; ignore_errors=no ; start_time=[Tue 2026-07-21 02:15:05 MSK] ; stop_time=[Tue 2026-07-21 03:09:43 MSK] ; pid=1933641 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER homelab-vm-local-dumps-2cloud.timer =====\n# /etc/systemd/system/homelab-vm-local-dumps-2cloud.timer\n[Unit]\nDescription=Homelab VM/CT existing local dumps two-cloud verifier timer\n\n[Timer]\nOnCalendar=Sun *-*-* 10:00:00\nRandomizedDelaySec=30m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-vm-local-dumps-2cloud.service =====\n# /etc/systemd/system/homelab-vm-local-dumps-2cloud.service\n[Unit]\nDescription=Homelab VM/CT existing local dumps two-cloud verifier\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-vm-local-dumps-2cloud\n\n# /etc/systemd/system/homelab-vm-local-dumps-2cloud.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; argv[]=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER homelab-vm160-remote-backup.timer =====\n# /etc/systemd/system/homelab-vm160-remote-backup.timer\n[Unit]\nDescription=Daily VM160 remote full-image backup\n\n[Timer]\nOnCalendar=*-*-* 04:10:00\nRandomizedDelaySec=10m\nUnit=homelab-vm160-remote-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE homelab-vm160-remote-backup.service =====\n# /etc/systemd/system/homelab-vm160-remote-backup.service\n[Unit]\nDescription=VM160 full-image backup streamed from pve02 to pve03\nAfter=network-online.target pve-cluster.service\nWants=network-online.target\n\n[Service]\nType=oneshot\nTimeoutStartSec=6h\nExecStart=/usr/local/sbin/homelab-vm160-remote-backup\nResult=exit-code\nExecMainStatus=1\nExecStart={ path=/usr/local/sbin/homelab-vm160-remote-backup ; argv[]=/usr/local/sbin/homelab-vm160-remote-backup ; ignore_errors=no ; start_time=[Tue 2026-07-21 02:39:12 MSK] ; stop_time=[Tue 2026-07-21 02:39:14 MSK] ; pid=2028253 ; code=exited ; status=1 }\nActiveState=failed\n===== TIMER immich-media-backup.timer =====\n# /etc/systemd/system/immich-media-backup.timer\n[Unit]\nDescription=Daily Immich media backup timer\n\n[Timer]\nOnCalendar=*-*-* 08:15:00\nPersistent=true\nRandomizedDelaySec=10m\nUnit=immich-media-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE immich-media-backup.service =====\n# /etc/systemd/system/immich-media-backup.service\n[Unit]\nDescription=Immich media backup to staging\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/root/immich-media-backup.sh\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\n\n# /etc/systemd/system/immich-media-backup.service.d/10-superseded-noop.conf\n[Service]\nExecStart=\nExecStart=/usr/local/sbin/homelab-superseded-unit-ok immich-media-backup.service superseded_by_current_appbackup_or_trust_proof\n\n# /etc/systemd/system/immich-media-backup.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok immich-media-backup.service superseded_by_current_appbackup_or_trust_proof ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER memos-backup.timer =====\n# /etc/systemd/system/memos-backup.timer\n[Unit]\nDescription=Run Memos data backup daily\n\n[Timer]\nOnCalendar=*-*-* 08:35:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE memos-backup.service =====\n# /etc/systemd/system/memos-backup.service\n[Unit]\nDescription=Memos data backup to pve01 staging\n\n[Service]\nType=oneshot\nExecStart=/root/memos-backup.sh\n\n# /etc/systemd/system/memos-backup.service.d/10-superseded-noop.conf\n[Service]\nExecStart=\nExecStart=/usr/local/sbin/homelab-superseded-unit-ok memos-backup.service superseded_by_current_appbackup_or_trust_proof\n\n# /etc/systemd/system/memos-backup.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok memos-backup.service superseded_by_current_appbackup_or_trust_proof ; ignore_errors=no ; start_time=[Tue 2026-07-21 08:41:14 MSK] ; stop_time=[Tue 2026-07-21 08:41:14 MSK] ; pid=3239605 ; code=exited ; status=0 }\nActiveState=inactive\n===== TIMER netbird-vps-backup.timer =====\n# /etc/systemd/system/netbird-vps-backup.timer\n[Unit]\nDescription=Run NetBird VPS backup daily\n\n[Timer]\nOnCalendar=*-*-* 04:25:00\nPersistent=true\nRandomizedDelaySec=15m\n\n[Install]\nWantedBy=timers.target\n===== SERVICE netbird-vps-backup.service =====\n# /etc/systemd/system/netbird-vps-backup.service\n[Unit]\nDescription=NetBird VPS backup to pve01 staging\n\n[Service]\nType=oneshot\nExecStart=/root/netbird-vps-backup.sh\n\n# /etc/systemd/system/netbird-vps-backup.service.d/10-superseded-noop.conf\n[Service]\nExecStart=\nExecStart=/usr/local/sbin/homelab-superseded-unit-ok netbird-vps-backup.service superseded_by_current_appbackup_or_trust_proof\n\n# /etc/systemd/system/netbird-vps-backup.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok netbird-vps-backup.service superseded_by_current_appbackup_or_trust_proof ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER paperless-backup.timer =====\n# /etc/systemd/system/paperless-backup.timer\n[Unit]\nDescription=Run Paperless backup daily\n\n[Timer]\nOnCalendar=*-*-* 08:50:00\nRandomizedDelaySec=20m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n===== SERVICE paperless-backup.service =====\n# /etc/systemd/system/paperless-backup.service\n[Unit]\nDescription=Paperless backup to pve01 staging\n\n[Service]\nType=oneshot\nExecStart=/root/paperless-backup.sh\n\n# /etc/systemd/system/paperless-backup.service.d/10-superseded-noop.conf\n[Service]\nExecStart=\nExecStart=/usr/local/sbin/homelab-superseded-unit-ok paperless-backup.service superseded_by_current_appbackup_or_trust_proof\n\n# /etc/systemd/system/paperless-backup.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok paperless-backup.service superseded_by_current_appbackup_or_trust_proof ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER skladchik-reports-monitor-backup-restore-check.timer =====\n# /etc/systemd/system/skladchik-reports-monitor-backup-restore-check.timer\n[Unit]\nDescription=Weekly restore-check of Skladchik reports monitor cloud backups\n\n[Timer]\nOnCalendar=Sun *-*-* 04:55:00\nRandomizedDelaySec=30min\nPersistent=true\nUnit=skladchik-reports-monitor-backup-restore-check.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE skladchik-reports-monitor-backup-restore-check.service =====\n# /etc/systemd/system/skladchik-reports-monitor-backup-restore-check.service\n[Unit]\nDescription=Restore-check Skladchik reports monitor cloud backups\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\n\n# /etc/systemd/system/skladchik-reports-monitor-backup-restore-check.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check ; argv[]=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\n===== TIMER skladchik-reports-monitor-backup.timer =====\n# /etc/systemd/system/skladchik-reports-monitor-backup.timer\n[Unit]\nDescription=Daily Skladchik reports monitor backup to encrypted clouds\n\n[Timer]\nOnCalendar=*-*-* 04:25:00\nRandomizedDelaySec=15min\nPersistent=true\nUnit=skladchik-reports-monitor-backup.service\n\n[Install]\nWantedBy=timers.target\n===== SERVICE skladchik-reports-monitor-backup.service =====\n# /etc/systemd/system/skladchik-reports-monitor-backup.service\n[Unit]\nDescription=Backup Skladchik reports monitor from edge-vm to encrypted clouds\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/skladchik-reports-monitor-backup\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\n\n# /etc/systemd/system/skladchik-reports-monitor-backup.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/skladchik-reports-monitor-backup ; argv[]=/usr/local/sbin/skladchik-reports-monitor-backup ; ignore_errors=no ; start_time=[Tue 2026-07-21 04:34:48 MSK] ; stop_time=[Tue 2026-07-21 04:35:22 MSK] ; pid=2399885 ; code=exited ; status=0 }\nActiveState=inactive\n===== REFERENCED_SCRIPTS =====\n----- /opt/homelab-backup/backup-emergency-pack-v0.sh -----\n41:rclone listremotes > \"$PAYLOAD/inventory/rclone_remotes.txt\" 2>&1 || true\n59:It is deleted only after both encrypted cloud copies are uploaded and fully verified.\n60:Recovery requires offline/break-glass access to rclone crypt credentials.\n65:rm -rf \"$PAYLOAD\"\n79:  \"archive_size_bytes\": $ARCHIVE_SIZE_BYTES,\n83:  \"local_staging_policy\": \"delete_after_both_clouds_verified\",\n90:echo \"[STEP] local checksum\"\n91:(cd \"$STAGE\" && sha256sum \"$ARCHIVE\" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null)\n94:rclone copy \"$STAGE\" \"$REMOTE_A_JOB\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n95:homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$REMOTE_A_JOB/$ARCHIVE\"\n98:rclone copy \"$STAGE\" \"$REMOTE_B_JOB\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n99:homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$REMOTE_B_JOB/$ARCHIVE\"\n102:rclone copy \"$REMOTE_A_JOB\" \"$VERIFY/cloud_a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10; homelab-rehydrate-chunked-dir \"$VERIFY/cloud_a\"\n103:(cd \"$VERIFY/cloud_a\" && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB_ID\" '.job_id == $job' manifest.json >/dev/null)\n106:rclone copy \"$REMOTE_B_JOB\" \"$VERIFY/cloud_b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10; homelab-rehydrate-chunked-dir \"$VERIFY/cloud_b\"\n107:(cd \"$VERIFY/cloud_b\" && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB_ID\" '.job_id == $job' manifest.json >/dev/null)\n110:(cd \"$STAGE\" && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null)\n113:rclone copyto \"$STAGE/latest-backup-index.json\" \"$REMOTE_A_LATEST\"\n114:rclone copyto \"$STAGE/latest-backup-index.json\" \"$REMOTE_B_LATEST\"\n115:rclone cat \"$REMOTE_A_LATEST\" | jq -e --arg job \"$JOB_ID\" '.job_id == $job' >/dev/null\n116:rclone cat \"$REMOTE_B_LATEST\" | jq -e --arg job \"$JOB_ID\" '.job_id == $job' >/dev/null\n124:  \"archive_size_bytes\": $ARCHIVE_SIZE_BYTES,\n128:  \"local_backup_deleted\": true\n132:echo \"[STEP] delete local staging and verification copies\"\n133:rm -rf \"$STAGE\" \"$VERIFY\"\n138:echo \"[SUCCESS] uploaded to 2 clouds, verified from both clouds, local staging deleted\"\n----- /opt/homelab-backup/backup-emergency-pack-v0.sh.bak-chunkall-20260710T011410Z -----\n41:rclone listremotes > \"$PAYLOAD/inventory/rclone_remotes.txt\" 2>&1 || true\n59:It is deleted only after both encrypted cloud copies are uploaded and fully verified.\n60:Recovery requires offline/break-glass access to rclone crypt credentials.\n65:rm -rf \"$PAYLOAD\"\n79:  \"archive_size_bytes\": $ARCHIVE_SIZE_BYTES,\n83:  \"local_staging_policy\": \"delete_after_both_clouds_verified\",\n90:echo \"[STEP] local checksum\"\n91:(cd \"$STAGE\" && sha256sum \"$ARCHIVE\" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null)\n94:rclone copy \"$STAGE\" \"$REMOTE_A_JOB\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 --stats=30s\n97:rclone copy \"$STAGE\" \"$REMOTE_B_JOB\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 --stats=30s\n100:rclone copy \"$REMOTE_A_JOB\" \"$VERIFY/cloud_a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10\n101:(cd \"$VERIFY/cloud_a\" && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB_ID\" '.job_id == $job' manifest.json >/dev/null)\n104:rclone copy \"$REMOTE_B_JOB\" \"$VERIFY/cloud_b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10\n105:(cd \"$VERIFY/cloud_b\" && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB_ID\" '.job_id == $job' manifest.json >/dev/null)\n108:(cd \"$STAGE\" && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null)\n111:rclone copyto \"$STAGE/latest-backup-index.json\" \"$REMOTE_A_LATEST\"\n112:rclone copyto \"$STAGE/latest-backup-index.json\" \"$REMOTE_B_LATEST\"\n113:rclone cat \"$REMOTE_A_LATEST\" | jq -e --arg job \"$JOB_ID\" '.job_id == $job' >/dev/null\n114:rclone cat \"$REMOTE_B_LATEST\" | jq -e --arg job \"$JOB_ID\" '.job_id == $job' >/dev/null\n122:  \"archive_size_bytes\": $ARCHIVE_SIZE_BYTES,\n126:  \"local_backup_deleted\": true\n130:echo \"[STEP] delete local staging and verification copies\"\n131:rm -rf \"$STAGE\" \"$VERIFY\"\n136:echo \"[SUCCESS] uploaded to 2 clouds, verified from both clouds, local staging deleted\"\n----- /usr/local/sbin/appbackup-dockge-stacks -----\n25:echo \"[STEP] copy app data\"\n30:rm -rf \"$PAYLOAD\"\n42:  \"archive_size_bytes\": $SIZE,\n46:  \"local_staging_policy\": \"delete_after_both_clouds_verified\",\n51:echo \"[STEP] local checksum\"\n52:(cd \"$STAGE\" && sha256sum \"$ARCHIVE\" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e . manifest.json >/dev/null)\n54:echo \"[STEP] upload cloud A metadata\"; rclone copy \"$STAGE\" \"$RA\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n55:echo \"[STEP] upload cloud A archive chunks\"; rclone deletefile \"$RA/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RA/$ARCHIVE\"\n56:echo \"[STEP] upload cloud B metadata\"; rclone copy \"$STAGE\" \"$RB\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n57:echo \"[STEP] upload cloud B archive chunks\"; rclone deletefile \"$RB/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RB/$ARCHIVE\"\n58:echo \"[STEP] verify cloud A chunked\"; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-stage-verify-chunked \"$RA\" \"$ARCHIVE\" \"$VERIFY/a\" \"$JOB\"\n59:echo \"[STEP] verify cloud B chunked\"; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-stage-verify-chunked \"$RB\" \"$ARCHIVE\" \"$VERIFY/b\" \"$JOB\"\n60:echo \"[STEP] upload latest index\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RAL\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RBL\"; rclone cat \"$RAL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null; rclone cat \"$RBL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n68:  \"archive_size_bytes\": $SIZE,\n72:  \"local_backup_deleted\": true\n75:echo \"[STEP] delete local staging and verification\"\n76:rm -rf \"$STAGE\" \"$VERIFY\"\n79:echo \"[SUCCESS] app backup uploaded to 2 clouds, verified from both clouds, local staging deleted\"\n----- /usr/local/sbin/appbackup-dockge-stacks.bak-chunkall-20260710T011410Z -----\n25:echo \"[STEP] copy app data\"\n30:rm -rf \"$PAYLOAD\"\n42:  \"archive_size_bytes\": $SIZE,\n46:  \"local_staging_policy\": \"delete_after_both_clouds_verified\",\n51:echo \"[STEP] local checksum\"\n52:(cd \"$STAGE\" && sha256sum \"$ARCHIVE\" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e . manifest.json >/dev/null)\n54:rclone copy \"$STAGE\" \"$RA\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 --stats=30s\n56:rclone copy \"$STAGE\" \"$RB\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 --stats=30s\n58:rclone copy \"$RA\" \"$VERIFY/a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10\n59:(cd \"$VERIFY/a\" && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n61:rclone copy \"$RB\" \"$VERIFY/b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10\n62:(cd \"$VERIFY/b\" && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n64:(cd \"$STAGE\" && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e . manifest.json >/dev/null)\n66:rclone copyto \"$STAGE/latest-backup-index.json\" \"$RAL\"\n67:rclone copyto \"$STAGE/latest-backup-index.json\" \"$RBL\"\n68:rclone cat \"$RAL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n69:rclone cat \"$RBL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n77:  \"archive_size_bytes\": $SIZE,\n81:  \"local_backup_deleted\": true\n84:echo \"[STEP] delete local staging and verification\"\n85:rm -rf \"$STAGE\" \"$VERIFY\"\n88:echo \"[SUCCESS] app backup uploaded to 2 clouds, verified from both clouds, local staging deleted\"\n----- /usr/local/sbin/appbackup-restore-check -----\n22:ROOT=\"/var/tmp/homelab-app-restore-check/${JOB}\"\n24:rm -rf \"$ROOT\" && mkdir -p \"$ROOT/a\" \"$ROOT/b\"\n25:trap \"echo \\\"ERROR app restore-check failed; kept: $ROOT\\\"; exit 1\" ERR\n26:echo \"restore-check app=$APP job=$JOB schema=$SCHEMA\"\n27:rclone copy \"$RA\" \"$ROOT/a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >\"${LOGBASE}-a.log\" 2>&1\n30:(cd \"$ROOT/a\" && sha256sum -c checksums.sha256 >\"${LOGBASE}-a-sha.log\" && ARCHIVE=\"$(jq -r \".archive_name\" manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n31:echo \"OK cloud A app restore-check\"\n32:rclone copy \"$RB\" \"$ROOT/b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >\"${LOGBASE}-b.log\" 2>&1\n35:(cd \"$ROOT/b\" && sha256sum -c checksums.sha256 >\"${LOGBASE}-b-sha.log\" && ARCHIVE=\"$(jq -r \".archive_name\" manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n36:echo \"OK cloud B app restore-check\"\n37:rm -rf \"$ROOT\"\n38:rmdir /var/tmp/homelab-app-restore-check 2>/dev/null || true\n39:rm -f \"${LOGBASE}-a.log\" \"${LOGBASE}-a-sha.log\" \"${LOGBASE}-b.log\" \"${LOGBASE}-b-sha.log\"\n41:echo \"OK app restore-check both clouds\"\n----- /usr/local/sbin/appbackup-restore-check.bak-chunkall-20260710T011410Z -----\n9:ROOT=\"/var/tmp/homelab-app-restore-check/${JOB}\"\n10:rm -rf \"$ROOT\" && mkdir -p \"$ROOT/a\" \"$ROOT/b\"\n11:trap \"echo \\\"ERROR app restore-check failed; kept: $ROOT\\\"; exit 1\" ERR\n12:echo \"restore-check app=$APP job=$JOB\"\n13:rclone copy \"$RA\" \"$ROOT/a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/app-restore-a.log 2>&1\n14:(cd \"$ROOT/a\" && sha256sum -c checksums.sha256 >/tmp/app-restore-a-sha.log && ARCHIVE=\"$(jq -r .archive_name manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n15:echo \"OK cloud A app restore-check\"\n16:rclone copy \"$RB\" \"$ROOT/b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/app-restore-b.log 2>&1\n17:(cd \"$ROOT/b\" && sha256sum -c checksums.sha256 >/tmp/app-restore-b-sha.log && ARCHIVE=\"$(jq -r .archive_name manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n18:echo \"OK cloud B app restore-check\"\n19:rm -rf \"$ROOT\"; rmdir /var/tmp/homelab-app-restore-check 2>/dev/null || true\n21:echo \"OK app restore-check both clouds\"\n----- /usr/local/sbin/appbackup-restore-check.bak-dual-read-20260711T051924Z -----\n9:ROOT=\"/var/tmp/homelab-app-restore-check/${JOB}\"\n10:rm -rf \"$ROOT\" && mkdir -p \"$ROOT/a\" \"$ROOT/b\"\n11:trap \"echo \\\"ERROR app restore-check failed; kept: $ROOT\\\"; exit 1\" ERR\n12:echo \"restore-check app=$APP job=$JOB\"\n13:rclone copy \"$RA\" \"$ROOT/a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/app-restore-a.log 2>&1; homelab-rehydrate-chunked-dir \"$ROOT/a\" >>/tmp/app-restore-a.log 2>&1\n14:(cd \"$ROOT/a\" && sha256sum -c checksums.sha256 >/tmp/app-restore-a-sha.log && ARCHIVE=\"$(jq -r .archive_name manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n15:echo \"OK cloud A app restore-check\"\n16:rclone copy \"$RB\" \"$ROOT/b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/app-restore-b.log 2>&1; homelab-rehydrate-chunked-dir \"$ROOT/b\" >>/tmp/app-restore-b.log 2>&1\n17:(cd \"$ROOT/b\" && sha256sum -c checksums.sha256 >/tmp/app-restore-b-sha.log && ARCHIVE=\"$(jq -r .archive_name manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n18:echo \"OK cloud B app restore-check\"\n19:rm -rf \"$ROOT\"; rmdir /var/tmp/homelab-app-restore-check 2>/dev/null || true\n21:echo \"OK app restore-check both clouds\"\n----- /usr/local/sbin/appbackup-restore-check.bak-flat-read-20260711T052854Z -----\n22:ROOT=\"/var/tmp/homelab-app-restore-check/${JOB}\"\n24:rm -rf \"$ROOT\" && mkdir -p \"$ROOT/a\" \"$ROOT/b\"\n25:trap \"echo \\\"ERROR app restore-check failed; kept: $ROOT\\\"; exit 1\" ERR\n26:echo \"restore-check app=$APP job=$JOB schema=$SCHEMA\"\n27:rclone copy \"$RA\" \"$ROOT/a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >\"${LOGBASE}-a.log\" 2>&1\n29:(cd \"$ROOT/a\" && sha256sum -c checksums.sha256 >\"${LOGBASE}-a-sha.log\" && ARCHIVE=\"$(jq -r \".archive_name\" manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n30:echo \"OK cloud A app restore-check\"\n31:rclone copy \"$RB\" \"$ROOT/b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >\"${LOGBASE}-b.log\" 2>&1\n33:(cd \"$ROOT/b\" && sha256sum -c checksums.sha256 >\"${LOGBASE}-b-sha.log\" && ARCHIVE=\"$(jq -r \".archive_name\" manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n34:echo \"OK cloud B app restore-check\"\n35:rm -rf \"$ROOT\"\n36:rmdir /var/tmp/homelab-app-restore-check 2>/dev/null || true\n37:rm -f \"${LOGBASE}-a.log\" \"${LOGBASE}-a-sha.log\" \"${LOGBASE}-b.log\" \"${LOGBASE}-b-sha.log\"\n39:echo \"OK app restore-check both clouds\"\n----- /usr/local/sbin/appbackup-restore-check.bak-flat-read-v2-20260711T053129Z -----\n22:ROOT=\"/var/tmp/homelab-app-restore-check/${JOB}\"\n24:rm -rf \"$ROOT\" && mkdir -p \"$ROOT/a\" \"$ROOT/b\"\n25:trap \"echo \\\"ERROR app restore-check failed; kept: $ROOT\\\"; exit 1\" ERR\n26:echo \"restore-check app=$APP job=$JOB schema=$SCHEMA\"\n27:rclone copy \"$RA\" \"$ROOT/a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >\"${LOGBASE}-a.log\" 2>&1\n29:(cd \"$ROOT/a\" && sha256sum -c checksums.sha256 >\"${LOGBASE}-a-sha.log\" && ARCHIVE=\"$(jq -r \".archive_name\" manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n30:echo \"OK cloud A app restore-check\"\n31:rclone copy \"$RB\" \"$ROOT/b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >\"${LOGBASE}-b.log\" 2>&1\n33:(cd \"$ROOT/b\" && sha256sum -c checksums.sha256 >\"${LOGBASE}-b-sha.log\" && ARCHIVE=\"$(jq -r \".archive_name\" manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n34:echo \"OK cloud B app restore-check\"\n35:rm -rf \"$ROOT\"\n36:rmdir /var/tmp/homelab-app-restore-check 2>/dev/null || true\n37:rm -f \"${LOGBASE}-a.log\" \"${LOGBASE}-a-sha.log\" \"${LOGBASE}-b.log\" \"${LOGBASE}-b-sha.log\"\n39:echo \"OK app restore-check both clouds\"\n----- /usr/local/sbin/appbackup-retention -----\n32:trap \"rm -rf \\\"$WORK\\\"\" EXIT\n35:  rclone lsf \"${REMOTE%/}\" --dirs-only --max-depth 1 > \"$ROOT_LIST\"\n36:  rclone lsf \"${REMOTE%/}/${NODE}/apps/${CANONICAL}\" --dirs-only --max-depth 1 > \"$LEGACY_LIST\"\n39:    rclone lsf \"${REMOTE%/}/hb\" --dirs-only --max-depth 1 > \"$HB_LIST\"\n40:    if grep -Fxq \"a/\" \"$HB_LIST\"; then rclone lsf \"${REMOTE%/}/hb/a\" --dirs-only --max-depth 1 > \"$NEW_LIST\"; fi\n73:    rclone cat \"${REMOTE%/}/${LATEST_REL}/manifest.json\" | jq -e --arg job \"$LATEST\" \".job_id == \\$job\" >/dev/null || { echo \"REFUSE latest missing/bad remote_index=$INDEX\"; exit 5; }\n78:  if (( TOTAL <= KEEP )); then echo \"nothing to delete\"; continue; fi\n83:    [[ -n \"$LATEST_REL\" && \"$REL\" == \"$LATEST_REL\" ]] && { echo \"REFUSE delete latest ${REL}\"; exit 7; }\n85:    if [[ \"$MODE\" == apply ]]; then echo \"DELETE schema=dual path=$REL\"; rclone purge \"$TARGET\"; else echo \"WOULD_DELETE schema=dual path=$REL\"; fi\n----- /usr/local/sbin/appbackup-retention.bak-dual-read-20260711T051924Z -----\n15:for REMOTE in \"$REMOTE_A\" \"$REMOTE_B\"; do BASE=\"${REMOTE%/}/${NODE}/apps/${APP}\"; echo \"--- $BASE ---\"; if [[ \"$MODE\" == \"apply\" ]]; then rclone cat \"$BASE/$LATEST/manifest.json\" | jq -e --arg job \"$LATEST\" \".job_id == \\$job\" >/dev/null || { echo \"REFUSE latest missing/bad on $BASE\"; exit 4; }; fi; mapfile -t DIRS < <(rclone lsf \"$BASE\" --dirs-only 2>/dev/null | while read -r D; do D=\"${D%/}\"; [[ \"$D\" == homelab-${NODE}-app-${APP}-* ]] && echo \"$D\"; done | sort); TOTAL=\"${#DIRS[@]}\"; echo \"total=$TOTAL keep=$KEEP mode=$MODE latest=${LATEST:-none}\"; if (( TOTAL <= KEEP )); then echo \"nothing to delete\"; continue; fi; DELETE_COUNT=$((TOTAL-KEEP)); for ((i=0; i<DELETE_COUNT; i++)); do TARGET_NAME=\"${DIRS[$i]}\"; TARGET=\"$BASE/$TARGET_NAME\"; [[ -n \"$LATEST\" && \"$TARGET_NAME\" == \"$LATEST\" ]] && { echo \"REFUSE delete latest $TARGET\"; exit 5; }; if [[ \"$MODE\" == \"apply\" ]]; then echo \"DELETE $TARGET\"; rclone purge \"$TARGET\"; else echo \"WOULD_DELETE $TARGET\"; fi; done; done\n----- /usr/local/sbin/backupctl -----\n12:  check) bash -n \"$SCRIPT\" && [[ -f \"$STATE\" ]] && jq -e \".status == \\\"success\\\" and .local_backup_deleted == true\" \"$STATE\" >/dev/null && echo \"OK backup framework\" ;;\n13:  restore-check) backup-restore-check ;;\n14:  audit) backupctl check && backup-restore-check && backupctl retention 7 dry-run && backup-report >/dev/null && echo \"OK audit finished\" ;;\n16:  remotes) source /etc/homelab-backup.env && NODE=\"$(hostname -s)\" && echo \"A=$REMOTE_A\" && echo \"B=$REMOTE_B\" && rclone lsd \"${REMOTE_A%/}/${NODE}/emergency-pack\" | tail -n 5 && rclone lsd \"${REMOTE_B%/}/${NODE}/emergency-pack\" | tail -n 5 ;;\n19:  *) echo \"usage: backupctl run|status|lastlog|check|restore-check|audit|report|remotes|retention|timer\"; exit 2 ;;\n----- /usr/local/sbin/backup-restore-check -----\n8:ROOT=\"/var/tmp/homelab-restore-check/${JOB}\"\n9:rm -rf \"$ROOT\" && mkdir -p \"$ROOT/a\" \"$ROOT/b\"\n10:trap 'echo \"ERROR restore-check failed; kept: \"; exit 1' ERR\n11:echo \"restore-check job=$JOB\"\n12:rclone copy \"$RA\" \"$ROOT/a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-a.log 2>&1; homelab-rehydrate-chunked-dir \"$ROOT/a\" >>/tmp/homelab-restore-a.log 2>&1\n13:(cd \"$ROOT/a\" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-a-sha.log && ARCHIVE=\"$(jq -r .archive_name manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n14:echo \"OK cloud A restore-check\"\n15:rclone copy \"$RB\" \"$ROOT/b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-b.log 2>&1; homelab-rehydrate-chunked-dir \"$ROOT/b\" >>/tmp/homelab-restore-b.log 2>&1\n16:(cd \"$ROOT/b\" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-b-sha.log && ARCHIVE=\"$(jq -r .archive_name manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n17:echo \"OK cloud B restore-check\"\n18:rm -rf \"$ROOT\"\n20:echo \"OK restore-check both clouds\"\n----- /usr/local/sbin/backup-restore-check.bak-chunkall-20260710T011410Z -----\n8:ROOT=\"/var/tmp/homelab-restore-check/${JOB}\"\n9:rm -rf \"$ROOT\" && mkdir -p \"$ROOT/a\" \"$ROOT/b\"\n10:trap 'echo \"ERROR restore-check failed; kept: \"; exit 1' ERR\n11:echo \"restore-check job=$JOB\"\n12:rclone copy \"$RA\" \"$ROOT/a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-a.log 2>&1\n13:(cd \"$ROOT/a\" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-a-sha.log && ARCHIVE=\"$(jq -r .archive_name manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n14:echo \"OK cloud A restore-check\"\n15:rclone copy \"$RB\" \"$ROOT/b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-b.log 2>&1\n16:(cd \"$ROOT/b\" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-b-sha.log && ARCHIVE=\"$(jq -r .archive_name manifest.json)\" && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n17:echo \"OK cloud B restore-check\"\n18:rm -rf \"$ROOT\"\n20:echo \"OK restore-check both clouds\"\n----- /usr/local/sbin/backup-retention -----\n18:  if [[ \"$MODE\" == \"apply\" ]]; then rclone cat \"$BASE/$LATEST/manifest.json\" | jq -e --arg job \"$LATEST\" \".job_id == \\$job\" >/dev/null || { echo \"REFUSE latest missing/bad on $BASE\"; exit 4; }; fi\n19:  mapfile -t DIRS < <(rclone lsf \"$BASE\" --dirs-only 2>/dev/null | while read -r D; do D=\"${D%/}\"; [[ \"$D\" == homelab-${NODE}-${JOB}-* ]] && echo \"$D\"; done | sort)\n22:  if (( TOTAL <= KEEP )); then echo \"nothing to delete\"; continue; fi\n24:  for ((i=0; i<DELETE_COUNT; i++)); do TARGET_NAME=\"${DIRS[$i]}\"; TARGET=\"$BASE/$TARGET_NAME\"; [[ -n \"$LATEST\" && \"$TARGET_NAME\" == \"$LATEST\" ]] && { echo \"REFUSE delete latest $TARGET\"; exit 5; }; if [[ \"$MODE\" == \"apply\" ]]; then echo \"DELETE $TARGET\"; rclone purge \"$TARGET\"; else echo \"WOULD_DELETE $TARGET\"; fi; done\n----- /usr/local/sbin/homelab-backup-edge-vm-vzdump -----\n7:BASE_DIR=\"/mnt/staging/vzdump/edge-vm-vzdump\"\n8:OFFHOST_DIR=\"/mnt/staging/offhost/edge-vm-vzdump-from-pve03\"\n40:REMOTE_OUT=\"$(remote \"OUT=/tmp/edge_vm_vzdump_${VMID}_${TS}.out; ERR=/tmp/edge_vm_vzdump_${VMID}_${TS}.err; : > \\$OUT; : > \\$ERR; vzdump '$VMID' --mode snapshot --compress zstd --dumpdir '$RUN_DIR' --remove 0 >\\$OUT 2>\\$ERR; RC=\\$?; ARCHIVE=\\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\\n' | sort -n | tail -n1 | cut -d' ' -f2-); LOGFILE=\\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.log' -printf '%T@ %p\\n' | sort -n | tail -n1 | cut -d' ' -f2-); SIZE=0; SHA=none; if [ -n \\\"\\$ARCHIVE\\\" ] && [ -f \\\"\\$ARCHIVE\\\" ]; then SIZE=\\$(stat -c %s \\\"\\$ARCHIVE\\\"); SHA=\\$(sha256sum \\\"\\$ARCHIVE\\\" | awk '{print \\$1}'); fi; echo VZDUMP_RC=\\$RC; echo ARCHIVE=\\$ARCHIVE; echo LOGFILE=\\$LOGFILE; echo SIZE=\\$SIZE; echo SHA=\\$SHA; echo OUT_TAIL=\\$(tail -n 6 \\$OUT | tr '\\n' '|'); echo ERR_TAIL=\\$(tail -n 6 \\$ERR | tr '\\n' '|')\")\"\n62:dest \"mkdir -p '$OFFHOST_DIR' && rm -f '$DEST_PATH.tmp' '$DEST_PATH.sha256'\"\n74:echo \"STATUS=OK TS=$TS TYPE=edge-vm-vzdump-backup NODE=$NODE VMID=$VMID ARCHIVE=$ARCHIVE SIZE=$SIZE SHA256=$SHA SCSI1_BACKUP0=yes NOTES=timer_run\" > \"$HEALTH_DIR/edge-vm-vzdump-backup.txt\"\n75:echo \"STATUS=OK TS=$TS TYPE=edge-vm-vzdump-offhost NODE=$NODE VMID=$VMID HOST=pve02 DEST=$DEST_PATH SIZE=$SIZE SHA256=$SHA NOTES=timer_run\" > \"$HEALTH_DIR/edge-vm-vzdump-offhost.txt\"\n76:echo \"STATUS=OK TS=$TS TYPE=edge-vm-vzdump-restore-verify HOST=pve02 VMID=$VMID ARCHIVE=$DEST_PATH SIZE=$SIZE SHA256=$SHA ZSTD_RC=0 VMA_RC=0 NOTES=timer_run\" > \"$HEALTH_DIR/edge-vm-vzdump-restore.txt\"\n77:chmod 0644 \"$HEALTH_DIR/edge-vm-vzdump-backup.txt\" \"$HEALTH_DIR/edge-vm-vzdump-offhost.txt\" \"$HEALTH_DIR/edge-vm-vzdump-restore.txt\"\n79:remote \"find '$BASE_DIR' -mindepth 1 -maxdepth 1 -type d -name 'edge-vm-*' -printf '%T@ %p\\n' | sort -nr | awk 'NR>${KEEP_LOCAL}{print \\$2}' | xargs -r rm -rf\"\n80:dest \"find '$OFFHOST_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\\n' | sort -nr | awk 'NR>${KEEP_OFFHOST}{print \\$2}' | while read f; do rm -f \\\"\\$f\\\" \\\"\\$f.sha256\\\"; done\"\n----- /usr/local/sbin/homelab-capacity-risk -----\n32:LARGEST_BYTES=\"$(find /var/lib/vz/dump -maxdepth 1 -type f \\( -name \"vzdump-qemu-*.vma.zst\" -o -name \"vzdump-lxc-*.tar.zst\" \\) -printf \"%s\\n\" 2>/dev/null | sort -n | tail -n1)\"\n----- /usr/local/sbin/homelab-chat-run -----\n272:        re.compile(r\"(?i)\\brclone\\.conf\\b\"),\n388:    \"reference_register_checked\": True,\n390:    \"error_register_checked\": True,\n489:    rm -f \\\n----- /usr/local/sbin/homelab-edge-vm-offhost-freshness -----\n9:OFFHOST_DIR=/mnt/staging/offhost/edge-vm-vzdump-from-pve03\n85:  /var/lib/vz/dump/vzdump-qemu-130-*.vma.zst) ;;\n96:dest_size=\"$(field DEST_SIZE <<<\"$inventory_out\")\"\n100:[[ \"$dest_size\" =~ ^[0-9]+$ && \"$count\" =~ ^[0-9]+$ && \"$available\" =~ ^[0-9]+$ ]]\n103:last_oldest_size=0\n107:  [[ \"$dest_size\" == \"$SOURCE_SIZE\" && \"$dest_sha\" == \"$SOURCE_SHA\" ]]\n110:    \"$dest_size\" \"$last_oldest\" \"$last_oldest_size\" \"$last_oldest_sha\" \"$count\" \"$RESERVE\")\"\n115:  [[ \"$dest_size\" -ge 0 && \"$dest_size\" -lt \"$SOURCE_SIZE\" ]]\n119:    last_oldest_size=\"$(field OLDEST_SIZE <<<\"$inventory_out\")\"\n122:    [[ \"$last_oldest_size\" =~ ^[0-9]+$ && \"$last_oldest_size\" -gt 1000000000 ]]\n126:      \"$dest_size\" \"$last_oldest\" \"$last_oldest_size\" \"$last_oldest_sha\" \"$before_count\" \"$RESERVE\")\"\n130:    [[ \"$(field DEST_SIZE <<<\"$inventory_out\")\" == \"$dest_size\" ]]\n136:  remaining=$((SOURCE_SIZE-dest_size))\n140:  copy_out=\"$(run_pve03 copy \"$SOURCE_PATH\" \"$SOURCE_SIZE\" \"$SOURCE_SHA\" \"$DST_IP\" \"$OFFHOST_DIR\")\"\n141:  grep -Fq 'SOURCE_COPY_STATUS=OK' <<<\"$copy_out\"\n146:    \"$dest_size\" \"$last_oldest\" \"$last_oldest_size\" \"$last_oldest_sha\" \"$expected_final_count\" \"$RESERVE\")\"\n156:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-backup NODE=%s VMID=%s ARCHIVE=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\\n' \\\n158:  >\"$HEALTH_DIR/edge-vm-vzdump-backup.txt\"\n159:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-offhost NODE=%s VMID=%s HOST=%s DEST=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\\n' \\\n161:  >\"$HEALTH_DIR/edge-vm-vzdump-offhost.txt\"\n162:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-restore-verify HOST=%s VMID=%s ARCHIVE=%s SIZE=%s SHA256=%s ZSTD_RC=0 VMA_RC=0 NOTES=nightly_existing_dump_freshness\\n' \\\n164:  >\"$HEALTH_DIR/edge-vm-vzdump-restore.txt\"\n167:  \"$HEALTH_DIR/edge-vm-vzdump-backup.txt\" \\\n168:  \"$HEALTH_DIR/edge-vm-vzdump-offhost.txt\" \\\n169:  \"$HEALTH_DIR/edge-vm-vzdump-restore.txt\"\n----- /usr/local/sbin/homelab-extended-appbackup-run -----\n12:run_one(){ APP=\"$1\"; HOST=\"$2\"; PATHS=\"$3\"; NOTE=\"$4\"; CONS=\"$5\"; JOB_ID=\"homelab-${NODE}-app-${APP}-${RUN_TS}\"; export JOB_ID; SHORT_TS=\"${RUN_TS:2:6}-${RUN_TS:9:6}\"; export BACKUP_PATH_REL=\"$(homelab-backup-path job a \"$APP\" \"$SHORT_TS\")\"; STAGE=\"${STAGING_ROOT}/${JOB_ID}\"; PAYLOAD=\"$STAGE/payload\"; ARCHIVE=\"${JOB_ID}.tar.gz\"; LOG=\"/tmp/extended-appbackup-${APP}-${RUN_TS}.log\"; rm -rf \"$STAGE\"; mkdir -p \"$PAYLOAD/local\"; echo \"### app=$APP\" >> \"$REPORT\"; echo \"host=$HOST\" >> \"$REPORT\"; echo \"source_note=$NOTE\" >> \"$REPORT\"; ssh -n -o BatchMode=yes -o ConnectTimeout=30 \"$HOST\" \"sudo tar -C / --ignore-failed-read --exclude=.env --exclude=*.env --exclude=.env.* --exclude=*.bak* -czf - $PATHS\" > \"$PAYLOAD/local/${APP}-data.tar.gz\" 2>\"$LOG\"; TAR_RC=$?; if [ \"$TAR_RC\" -eq 0 ] && tar -tzf \"$PAYLOAD/local/${APP}-data.tar.gz\" >/dev/null 2>&1; then SIZE=\"$(stat -c%s \"$PAYLOAD/local/${APP}-data.tar.gz\" 2>/dev/null || echo 0)\"; echo \"archive_payload_size_bytes=$SIZE\" >> \"$REPORT\"; if [ \"$SIZE\" -le 128 ]; then echo \"backup=BAD reason=archive_too_small\" >> \"$REPORT\"; BAD=$((BAD+1)); rm -rf \"$STAGE\"; else sha256sum \"$PAYLOAD/local/${APP}-data.tar.gz\" > \"$PAYLOAD/local/${APP}-data.tar.gz.sha256\"; jq -n --arg type \"extended-appbackup\" --arg app \"$APP\" --arg host \"$HOST\" --arg source_note \"$NOTE\" --arg archive_payload_size_bytes \"$SIZE\" --arg env_files_read \"no\" --arg secret_values_printed \"no\" --arg consistency \"$CONS\" \"{type:\\$type,app_name:\\$app,host:\\$host,source_note:\\$source_note,archive_payload_size_bytes:(\\$archive_payload_size_bytes|tonumber),env_files_read:\\$env_files_read,secret_values_printed:\\$secret_values_printed,consistency:\\$consistency}\" > \"$STAGE/metadata.json\"; tar -C \"$PAYLOAD\" -czf \"$STAGE/$ARCHIVE\" .; rm -rf \"$PAYLOAD\"; JOB_ID=\"$JOB_ID\" homelab-stage-2cloud-flat \"$APP\" \"$STAGE\" \"$ARCHIVE\" > \"$LOG\" 2>&1; RC=$?; if [ \"$RC\" -eq 0 ]; then echo \"backup=OK\" >> \"$REPORT\"; grep -E \"^\\[STEP\\]|^\\[SUCCESS\\]\" \"$LOG\" >> \"$REPORT\"; appbackupctl restore-check \"$APP\" >> \"$REPORT\" 2>&1 && echo \"restore_check=OK\" >> \"$REPORT\" || echo \"restore_check=WARN_non_registry\" >> \"$REPORT\"; OK=$((OK+1)); else echo \"backup=BAD rc=$RC\" >> \"$REPORT\"; grep -E \"^\\[ERROR\\]|\\[STEP\\]|ERROR|error|failed|Failed|required|missing|denied|forbidden|checksum|rclone|copy|mkdir|jq|tar\" \"$LOG\" | tail -n 60 >> \"$REPORT\"; BAD=$((BAD+1)); fi; fi; else echo \"tar=BAD rc=$TAR_RC\" >> \"$REPORT\"; grep -E \"ERROR|error|failed|Failed|denied|No such|Cannot|tar:\" \"$LOG\" | tail -n 40 >> \"$REPORT\"; BAD=$((BAD+1)); rm -rf \"$STAGE\"; fi; echo >> \"$REPORT\"; }\n21:run_one healthchecks \"$MON\" \"var/lib/healthchecks opt/stacks/healthchecks/local_settings.py\" \"healthchecks data plus local settings\" \"emergency_data_config_archive\"\n----- /usr/local/sbin/homelab-golden-state-extended-overlay -----\n26:rclone copyto \"$JSON\" \"$RA\" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1))\n27:rclone copyto \"$JSON\" \"$RB\" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1))\n28:rclone cat \"$RA\" 2>/dev/null | jq -e --arg cp \"$CP\" \".control_plane_job == \\$cp\" >/dev/null || BAD=$((BAD+1))\n29:rclone cat \"$RB\" 2>/dev/null | jq -e --arg cp \"$CP\" \".control_plane_job == \\$cp\" >/dev/null || BAD=$((BAD+1))\n----- /usr/local/sbin/homelab-golden-state-index -----\n28:rclone copyto \"$JSON\" \"$RA\" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1))\n29:rclone copyto \"$JSON\" \"$RB\" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1))\n30:rclone cat \"$RA\" 2>/dev/null | jq -e --arg cp \"$CP\" --arg ext \"$EXT\" \".control_plane_job == \\$cp and .health.extended_appbackup == \\$ext and .extended_appbackup_health == \\$ext\" >/dev/null || BAD=$((BAD+1))\n31:rclone cat \"$RB\" 2>/dev/null | jq -e --arg cp \"$CP\" --arg ext \"$EXT\" \".control_plane_job == \\$cp and .health.extended_appbackup == \\$ext and .extended_appbackup_health == \\$ext\" >/dev/null || BAD=$((BAD+1))\n----- /usr/local/sbin/homelab-health-metrics -----\n16:category_for_check(){\n22:    *backup*|*vzdump*) printf \"%s\" \"backup\" ;;\n31:tier_for_check(){\n34:    npmplus-kuma-config-*|authentik-*|gitea-*|vaultwarden-*|netbox-*|mail-cloud-restore-drill|edge-vm-vzdump-*|mkdocs-runtime-publish|npmplus-certificate-expiry) printf \"%s\" \"P0\" ;;\n69:    check=\"${base%.txt}\"\n70:    category=\"$(category_for_check \"$check\")\"\n71:    tier=\"$(tier_for_check \"$check\")\"\n75:    check_esc=\"$(esc_label \"$check\")\"\n79:    labels=\"host=\\\"$host_esc\\\",check=\\\"$check_esc\\\",category=\\\"$cat_esc\\\",tier=\\\"$tier_esc\\\",file=\\\"$file_esc\\\"\"\n----- /usr/local/sbin/homelab-ingress-standby-refresh -----\n16:    rm -f \"$TMP_STATE\" \"$RELAY\" \"$RESOURCES\"\n74:run_node \"$SOURCE_NODE\" vzdump 130 \\\n78:    --remove 0 \\\n83:\"find /var/lib/vz/dump -maxdepth 1 -type f -name 'vzdump-qemu-130-*.vma.zst' -printf '%T@ %p\\n' | sort -rn | head -n 1 | cut -d' ' -f2-\"\n106:\"chattr -i '$DEST_FILE' '$DEST_FILE.sha256' '$DEST_FILE.env' 2>/dev/null || true; rm -f '$DEST_TMP'\"\n155:        -name 'vzdump-qemu-130-*.vma.zst' \\\n164:        rm -f \"$OLD\" \"$OLD.sha256\" \"$OLD.env\"\n173:run_node \"$SOURCE_NODE\" rm -f \"$SOURCE_BACKUP\"\n----- /usr/local/sbin/homelab-mail-cloud-critical-upload -----\n50:EXCLUDED=edge-vm-vzdump-large-archive,rclone-config,secrets,raw-env\n80:    rclone mkdir \"${R}:$REMOTE_DIR\" > \"$WORK/logs/${SAFE}_mkdir.out\" 2>\"$WORK/logs/${SAFE}_mkdir.err\"\n82:    homelab-rclone-copy-chunked \"$BUNDLE\" \"${R}:$REMOTE_DIR/$(basename \"$BUNDLE\")\" > \"$WORK/logs/${SAFE}_upload_bundle.out\" 2>\"$WORK/logs/${SAFE}_upload_bundle.err\"\n84:    rclone copyto \"$BUNDLE_SHA\" \"${R}:$REMOTE_DIR/$(basename \"$BUNDLE_SHA\")\" --checksum > \"$WORK/logs/${SAFE}_upload_sha.out\" 2>\"$WORK/logs/${SAFE}_upload_sha.err\"\n86:    rclone copyto \"$WORK/components/MANIFEST.txt\" \"${R}:$REMOTE_DIR/MANIFEST.txt\" --checksum > \"$WORK/logs/${SAFE}_upload_manifest.out\" 2>\"$WORK/logs/${SAFE}_upload_manifest.err\"\n88:    rclone copyto \"$WORK/components/SHA256SUMS.txt\" \"${R}:$REMOTE_DIR/SHA256SUMS.txt\" --checksum > \"$WORK/logs/${SAFE}_upload_sums.out\" 2>\"$WORK/logs/${SAFE}_upload_sums.err\"\n90:    homelab-rclone-cat-chunked \"${R}:$REMOTE_DIR/$(basename \"$BUNDLE\")\" > \"$WORK/download/${SAFE}-$(basename \"$BUNDLE\")\" 2>\"$WORK/logs/${SAFE}_download_bundle.err\"\n108:      rm -f \"$WORK/download/${SAFE}-$(basename \"$BUNDLE\")\"\n109:      rclone lsf \"${R}:$TARGET_BASE\" --dirs-only 2>/dev/null | sed 's#/$##' | sort > \"$WORK/logs/${SAFE}_dirs.txt\"\n114:          [ -n \"$OLD\" ] && rclone purge \"${R}:$TARGET_BASE/$OLD\" >> \"$WORK/logs/${SAFE}_retention.out\" 2>>\"$WORK/logs/${SAFE}_retention.err\"\n140:      rm -rf --one-file-system \"$OLD_DIR\"\n----- /usr/local/sbin/homelab-mail-cloud-critical-upload.bak-chunkall-20260710T011410Z -----\n50:EXCLUDED=edge-vm-vzdump-large-archive,rclone-config,secrets,raw-env\n80:    rclone mkdir \"${R}:$REMOTE_DIR\" > \"$WORK/logs/${SAFE}_mkdir.out\" 2>\"$WORK/logs/${SAFE}_mkdir.err\"\n82:    rclone copyto \"$BUNDLE\" \"${R}:$REMOTE_DIR/$(basename \"$BUNDLE\")\" --checksum > \"$WORK/logs/${SAFE}_upload_bundle.out\" 2>\"$WORK/logs/${SAFE}_upload_bundle.err\"\n84:    rclone copyto \"$BUNDLE_SHA\" \"${R}:$REMOTE_DIR/$(basename \"$BUNDLE_SHA\")\" --checksum > \"$WORK/logs/${SAFE}_upload_sha.out\" 2>\"$WORK/logs/${SAFE}_upload_sha.err\"\n86:    rclone copyto \"$WORK/components/MANIFEST.txt\" \"${R}:$REMOTE_DIR/MANIFEST.txt\" --checksum > \"$WORK/logs/${SAFE}_upload_manifest.out\" 2>\"$WORK/logs/${SAFE}_upload_manifest.err\"\n88:    rclone copyto \"$WORK/components/SHA256SUMS.txt\" \"${R}:$REMOTE_DIR/SHA256SUMS.txt\" --checksum > \"$WORK/logs/${SAFE}_upload_sums.out\" 2>\"$WORK/logs/${SAFE}_upload_sums.err\"\n90:    rclone copyto \"${R}:$REMOTE_DIR/$(basename \"$BUNDLE\")\" \"$WORK/download/${SAFE}-$(basename \"$BUNDLE\")\" > \"$WORK/logs/${SAFE}_download_bundle.out\" 2>\"$WORK/logs/${SAFE}_download_bundle.err\"\n108:      rm -f \"$WORK/download/${SAFE}-$(basename \"$BUNDLE\")\"\n109:      rclone lsf \"${R}:$TARGET_BASE\" --dirs-only 2>/dev/null | sed 's#/$##' | sort > \"$WORK/logs/${SAFE}_dirs.txt\"\n114:          [ -n \"$OLD\" ] && rclone purge \"${R}:$TARGET_BASE/$OLD\" >> \"$WORK/logs/${SAFE}_retention.out\" 2>>\"$WORK/logs/${SAFE}_retention.err\"\n----- /usr/local/sbin/homelab-mail-cloud-edge-vm-upload -----\n10:TARGET_BASE=full/edge-vm-vzdump\n60:case \"$SIZE\" in \"\"|*[!0-9]*) fail size_bad 2 ;; esac\n64:case \"$SOURCE\" in /mnt/staging/offhost/edge-vm-vzdump-from-pve03/vzdump-qemu-130-*.vma.zst) ;; *) fail source_path_bad 2 ;; esac\n111:rclone about \"$TARGET_REMOTE:\" > \"$WORK/logs/about.out\" 2>\"$WORK/logs/about.err\" || ABOUT_RC=$?\n114:DIRS=\"$(rclone lsf \"$TARGET_REMOTE:$TARGET_BASE\" --dirs-only 2>/dev/null | sed 's#/$##' | sort || true)\"\n116:  if rclone cat \"$TARGET_REMOTE:$TARGET_BASE/$D/$BASENAME.sha256\" 2>/dev/null | grep -q \"$SHA\"; then\n172:P1 Mail Cloud edge-vm recurring chunked vzdump upload\n188:rclone mkdir \"$TARGET_REMOTE:$REMOTE_PARTS\" > \"$WORK/logs/mkdir.out\" 2>\"$WORK/logs/mkdir.err\" || MKDIR_RC=$?\n190:rclone copy \"$PARTS_DIR\" \"$TARGET_REMOTE:$REMOTE_PARTS\" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > \"$WORK/logs/upload_parts.out\" 2>\"$WORK/logs/upload_parts.err\" || UPLOAD_PARTS_RC=$?\n192:rclone copyto \"$WORK/$BASENAME.sha256\" \"$TARGET_REMOTE:$REMOTE_SHA\" --checksum > \"$WORK/logs/upload_sha.out\" 2>\"$WORK/logs/upload_sha.err\" || UPLOAD_SHA_RC=$?\n194:rclone copyto \"$WORK/MANIFEST.txt\" \"$TARGET_REMOTE:$REMOTE_MANIFEST\" --checksum > \"$WORK/logs/upload_manifest.out\" 2>\"$WORK/logs/upload_manifest.err\" || UPLOAD_MANIFEST_RC=$?\n196:rclone copyto \"$WORK/SHA256SUMS.txt\" \"$TARGET_REMOTE:$REMOTE_SUMS\" --checksum > \"$WORK/logs/upload_sums.out\" 2>\"$WORK/logs/upload_sums.err\" || UPLOAD_SUMS_RC=$?\n199:rclone size \"$TARGET_REMOTE:$TARGET_DIR\" --json > \"$WORK/logs/remote_size.json\" 2>\"$WORK/logs/remote_size.err\" || SIZE_RC=$?\n201:rclone lsf \"$TARGET_REMOTE:$REMOTE_PARTS\" --format \"ps\" > \"$WORK/logs/remote_lsf_parts.txt\" 2>\"$WORK/logs/remote_lsf_parts.err\" || LSF_PARTS_RC=$?\n203:REMOTE_BYTES=\"$(python3 -c 'import json; print(json.load(open(\"'\"$WORK\"'/logs/remote_size.json\")).get(\"bytes\",\"\"))' 2>/dev/null || echo unknown)\"\n204:REMOTE_COUNT=\"$(python3 -c 'import json; print(json.load(open(\"'\"$WORK\"'/logs/remote_size.json\")).get(\"count\",\"\"))' 2>/dev/null || echo unknown)\"\n207:rclone copy \"$TARGET_REMOTE:$REMOTE_PARTS\" \"$DL_DIR/parts\" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > \"$WORK/logs/download_parts.out\" 2>\"$WORK/logs/download_parts.err\" || DOWNLOAD_PARTS_RC=$?\n222:  rm -rf \"$DL_DIR\" \"$PARTS_DIR\"\n234:  rclone lsf \"$TARGET_REMOTE:$TARGET_BASE\" --dirs-only 2>/dev/null | sed \"s#/$##\" | sort > \"$DIR_LIST\" || RETENTION_RC=$?\n239:      [ -n \"$OLD\" ] && rclone purge \"$TARGET_REMOTE:$TARGET_BASE/$OLD\" >> \"$WORK/logs/retention.out\" 2>>\"$WORK/logs/retention.err\"\n----- /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload -----\n8:TARGET_BASE=full/nextcloud-vm-vzdump\n11:SRC_GLOB=/var/lib/vz/dump/vzdump-qemu-150-*.vma.zst\n23:  local delete_count=\"${7:-0}\"\n25:  echo \"STATUS=$status TS=$TS TYPE=mail-cloud-nextcloud-vm MODE=recurring-chunked REMOTE=$TARGET_REMOTE DEST=${TARGET_DIR:-none} ARCHIVE=${BASENAME:-none} SIZE=${SIZE:-0} SHA256=${SHA:-none} PART_SIZE=$PART_SIZE PARTS=${PART_COUNT:-0} REMOTE_PARTS=$remote_parts REMOTE_BYTES=$remote_bytes UPLOAD_OK=$upload_ok DOWNLOAD_VERIFY=$download_verify RETENTION_KEEP=$KEEP DELETE_COUNT=$delete_count ALREADY_UPLOADED=$already_uploaded SECRET_PRINTED=no NOTES=$notes\" > \"$HEALTH\"\n77:P1 Mail Cloud Nextcloud VM recurring chunked vzdump upload\n93:rclone mkdir \"$TARGET_REMOTE:$REMOTE_PARTS\" > \"$WORK/logs/mkdir.out\" 2>\"$WORK/logs/mkdir.err\" || MKDIR_RC=$?\n95:rclone copy \"$PARTS_DIR\" \"$TARGET_REMOTE:$REMOTE_PARTS\" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > \"$WORK/logs/upload_parts.out\" 2>\"$WORK/logs/upload_parts.err\" || UPLOAD_PARTS_RC=$?\n97:rclone copyto \"$WORK/$BASENAME.sha256\" \"$TARGET_REMOTE:$REMOTE_SHA\" --checksum > \"$WORK/logs/upload_sha.out\" 2>\"$WORK/logs/upload_sha.err\" || UPLOAD_SHA_RC=$?\n99:rclone copyto \"$WORK/MANIFEST.txt\" \"$TARGET_REMOTE:$REMOTE_MANIFEST\" --checksum > \"$WORK/logs/upload_manifest.out\" 2>\"$WORK/logs/upload_manifest.err\" || UPLOAD_MANIFEST_RC=$?\n101:rclone copyto \"$WORK/SHA256SUMS.txt\" \"$TARGET_REMOTE:$REMOTE_SUMS\" --checksum > \"$WORK/logs/upload_sums.out\" 2>\"$WORK/logs/upload_sums.err\" || UPLOAD_SUMS_RC=$?\n104:rclone lsf \"$TARGET_REMOTE:$REMOTE_PARTS\" --format ps > \"$WORK/logs/remote_lsf_parts.txt\" 2>\"$WORK/logs/remote_lsf_parts.err\" || LSF_PARTS_RC=$?\n110:rclone copy \"$TARGET_REMOTE:$REMOTE_PARTS\" \"$DL_DIR/parts\" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > \"$WORK/logs/download_parts.out\" 2>\"$WORK/logs/download_parts.err\" || DOWNLOAD_PARTS_RC=$?\n120:rclone lsf \"$TARGET_REMOTE:$TARGET_BASE\" --dirs-only 2>/dev/null | sed 's#/$##' | sort > \"$DIR_LIST\" || RETENTION_RC=$?\n126:    [ -n \"$OLD\" ] && rclone purge \"$TARGET_REMOTE:$TARGET_BASE/$OLD\" >> \"$WORK/logs/retention.out\" 2>>\"$WORK/logs/retention.err\"\n140:  rm -rf \"$PARTS_DIR\" \"$DL_DIR\"\n----- /usr/local/sbin/homelab-mail-cloud-restore-drill -----\n52:    homelab-rclone-cat-chunked \"${R}:$CRIT_DEST/$CRIT_BUNDLE\" > \"$OUT\" 2>\"$WORK/logs/${SAFE}_crit_download.err\"\n83:      rm -f \"$OUT\"\n108:  rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/MANIFEST.txt\" \"$WORK/edge/MANIFEST.txt\" > \"$WORK/logs/edge_manifest_download.out\" 2>\"$WORK/logs/edge_manifest_download.err\"\n111:  rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/SHA256SUMS.txt\" \"$WORK/edge/SHA256SUMS.txt\" > \"$WORK/logs/edge_sums_download.out\" 2>\"$WORK/logs/edge_sums_download.err\"\n114:  rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/$EDGE_ARCHIVE.sha256\" \"$WORK/edge/$EDGE_ARCHIVE.sha256\" > \"$WORK/logs/edge_archive_sha_download.out\" 2>\"$WORK/logs/edge_archive_sha_download.err\"\n117:  rclone lsf \"${EDGE_REMOTE}:$EDGE_DEST/parts\" --format ps > \"$WORK/logs/edge_parts_lsf.txt\" 2>\"$WORK/logs/edge_parts_lsf.err\"\n130:  rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/parts/$FIRST_PART\" \"$WORK/edge/$FIRST_PART\" > \"$WORK/logs/edge_first_part_download.out\" 2>\"$WORK/logs/edge_first_part_download.err\"\n137:    rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/parts/$LAST_PART\" \"$WORK/edge/$LAST_PART\" > \"$WORK/logs/edge_last_part_download.out\" 2>\"$WORK/logs/edge_last_part_download.err\"\n168:    rm -f \"$WORK/edge/$FIRST_PART\" \"$WORK/edge/$LAST_PART\"\n----- /usr/local/sbin/homelab-mail-cloud-restore-drill.bak-chunkall-20260710T011410Z -----\n52:    rclone copyto \"${R}:$CRIT_DEST/$CRIT_BUNDLE\" \"$OUT\" > \"$WORK/logs/${SAFE}_crit_download.out\" 2>\"$WORK/logs/${SAFE}_crit_download.err\"\n83:      rm -f \"$OUT\"\n108:  rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/MANIFEST.txt\" \"$WORK/edge/MANIFEST.txt\" > \"$WORK/logs/edge_manifest_download.out\" 2>\"$WORK/logs/edge_manifest_download.err\"\n111:  rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/SHA256SUMS.txt\" \"$WORK/edge/SHA256SUMS.txt\" > \"$WORK/logs/edge_sums_download.out\" 2>\"$WORK/logs/edge_sums_download.err\"\n114:  rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/$EDGE_ARCHIVE.sha256\" \"$WORK/edge/$EDGE_ARCHIVE.sha256\" > \"$WORK/logs/edge_archive_sha_download.out\" 2>\"$WORK/logs/edge_archive_sha_download.err\"\n117:  rclone lsf \"${EDGE_REMOTE}:$EDGE_DEST/parts\" --format ps > \"$WORK/logs/edge_parts_lsf.txt\" 2>\"$WORK/logs/edge_parts_lsf.err\"\n130:  rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/parts/$FIRST_PART\" \"$WORK/edge/$FIRST_PART\" > \"$WORK/logs/edge_first_part_download.out\" 2>\"$WORK/logs/edge_first_part_download.err\"\n137:    rclone copyto \"${EDGE_REMOTE}:$EDGE_DEST/parts/$LAST_PART\" \"$WORK/edge/$LAST_PART\" > \"$WORK/logs/edge_last_part_download.out\" 2>\"$WORK/logs/edge_last_part_download.err\"\n168:    rm -f \"$WORK/edge/$FIRST_PART\" \"$WORK/edge/$LAST_PART\"\n----- /usr/local/sbin/homelab-rclone-cat-chunked -----\n4:TMP_BASE=${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\n7:cleanup(){ rm -rf \"$WORK\"; }\n10:rclone copyto \"${REMOTE_FILE}.parts/MANIFEST.parts.txt\" \"$MAN\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n11:while read -r PART SIZE; do case \"$PART\" in part-*) rclone cat \"${REMOTE_FILE}.parts/$PART\" ;; esac; done < \"$MAN\"\n----- /usr/local/sbin/homelab-rclone-cat-flat-chunked -----\n5:trap \"rm -f \\\"$TMP\\\"\" EXIT\n6:rclone cat \"${DEST}.parts.txt\" > \"$TMP\"\n8:while read -r PART PART_SIZE; do [[ \"$PART\" =~ ^part-[0-9]{4}$ && \"$PART_SIZE\" =~ ^[0-9]+$ ]] || { echo \"BAD FLAT PART MANIFEST\" >&2; exit 2; }; rclone cat \"${DEST}.${PART}\"; COUNT=$((COUNT+1)); done < \"$TMP\"\n----- /usr/local/sbin/homelab-rclone-copy-chunked -----\n7:TMP_BASE=${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\n13:cleanup(){ rm -rf \"$WORK\"; }\n23:printf \"name=%s\\\\nsize=%s\\\\nsha256=%s\\\\nchunk_size=%s\\\\nparts=%s\\\\n\" \"$NAME\" \"$SIZE\" \"$SHA\" \"$CHUNK_SIZE\" \"$PART_COUNT\" > \"$WORK/MANIFEST.txt\"\n24:rclone purge \"${DEST}.parts\" >/dev/null 2>&1 || true\n25:rclone mkdir \"${DEST}.parts\"\n26:rclone copy \"$PARTS\" \"${DEST}.parts\" --transfers 2 --checkers 4 --retries 5 --low-level-retries 20 --stats 30s --log-level INFO\n27:rclone copyto \"$WORK/${NAME}.sha256\" \"${DEST}.sha256\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO\n28:rclone copyto \"$WORK/MANIFEST.txt\" \"${DEST}.manifest.txt\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO\n29:rclone copyto \"$WORK/MANIFEST.parts.txt\" \"${DEST}.parts/MANIFEST.parts.txt\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO\n30:REMOTE_COUNT=$(rclone lsf \"${DEST}.parts\" --files-only | grep -c \"^part-\" || true)\n32:ACTUAL=$(while read -r PART SIZE_PART; do case \"$PART\" in part-*) rclone cat \"${DEST}.parts/$PART\" ;; esac; done < \"$WORK/MANIFEST.parts.txt\" | sha256sum | cut -d \" \" -f 1)\n----- /usr/local/sbin/homelab-rclone-copy-flat-chunked -----\n6:TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\"\n12:trap \"rm -rf \\\"$WORK\\\"\" EXIT\n21:printf \"format=homelab-flat-chunks-v1\\nname=%s\\nsize=%s\\nsha256=%s\\nchunk_size=%s\\nparts=%s\\n\" \"$NAME\" \"$SIZE\" \"$SHA\" \"$CHUNK_SIZE\" \"$PART_COUNT\" > \"$WORK/MANIFEST.txt\"\n23:while read -r PART PART_SIZE; do [[ \"$PART\" =~ ^part-[0-9]{4}$ && \"$PART_SIZE\" =~ ^[0-9]+$ ]] || { echo \"BAD PART MANIFEST\" >&2; exit 4; }; rclone copyto \"$WORK/parts/$PART\" \"${DEST}.${PART}\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR; REMOTE_BYTES=\"$(rclone size --json \"${DEST}.${PART}\" | jq -r \".bytes // -1\")\"; [[ \"$REMOTE_BYTES\" = \"$PART_SIZE\" ]] || { echo \"REMOTE PART SIZE MISMATCH $PART\" >&2; exit 5; }; UPLOADED=$((UPLOADED+1)); done < \"$WORK/MANIFEST.parts.txt\"\n25:rclone copyto \"$WORK/${NAME}.sha256\" \"${DEST}.sha256\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n26:rclone copyto \"$WORK/MANIFEST.txt\" \"${DEST}.manifest.txt\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n27:rclone copyto \"$WORK/MANIFEST.parts.txt\" \"${DEST}.parts.txt\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n28:ACTUAL=\"$(homelab-rclone-cat-flat-chunked \"$DEST\" | sha256sum | awk \"{print \\$1}\")\"\n----- /usr/local/sbin/homelab-rclone-crypt-healthcheck.sh -----\n4:CRYPT_FILE=\"/etc/homelab/rclone/required-crypt-remotes.txt\"\n18:  \"checksums/\"\n21:echo \"=== homelab rclone crypt healthcheck: $(hostname) ===\"\n40:  if ! rclone listremotes | sed 's/:$//' | grep -qx \"$REMOTE\"; then\n47:  if ! rclone about \"${REMOTE}:\"; then\n48:    echo \"FAIL: rclone about failed: $REMOTE\"\n54:  ROOT_LIST=\"$(rclone lsf \"${REMOTE}:/\" || true)\"\n76:  if rclone cat \"${REMOTE}:/latest-backup-index.json\" | jq -e '.schema and .remote and .status and .items' >/dev/null; then\n----- /usr/local/sbin/homelab-rclone-required-check.sh -----\n4:RAW_FILE=\"/etc/homelab/rclone/required-remotes.txt\"\n5:CRYPT_FILE=\"/etc/homelab/rclone/required-crypt-remotes.txt\"\n9:echo \"=== homelab rclone required check: $(hostname) ===\"\n12:echo \"=== installed rclone ===\"\n13:rclone version | head -5\n17:CONFIGURED=\"$(rclone listremotes 2>/dev/null | sed 's/:$//' || true)\"\n19:  echo \"No rclone remotes configured yet.\"\n----- /usr/local/sbin/homelab-router-hard-power-cycle -----\n12:case \"$MODE\" in check|cycle) ;; *) echo HARD_WRAPPER_MODE_INVALID=yes; exit 64;; esac\n23:SAFE_OUT=\"$(printf '%s\\n' \"$OUT\" | sed -E '/(PASSWORD|TOKEN|SECRET|PRIVATE_KEY|ROOT_PASSWORD|ENCRYPTION_KEY|COOKIE|SESSION|AUTHORIZATION|API_KEY|LOCAL_KEY|rclone[.]conf|[.]env)/Id')\"\n----- /usr/local/sbin/homelab-router-running-config-mail-cloud -----\n62:  rm -f \"$CONFIG\"\n66:  rm -f \"$CONFIG\"\n73:  if ( cd \"$BACKUP_ROOT\" && sha256sum -c \"$(basename \"$SHA_FILE\")\" ) >\"$WORK/sha-check.out\" 2>\"$WORK/sha-check.err\"; then\n79:  if rclone mkdir \"$REMOTE:$DEST\" >\"$WORK/rclone-mkdir.out\" 2>\"$WORK/rclone-mkdir.err\" && homelab-rclone-copy-chunked \"$ARCHIVE\" \"$REMOTE:$DEST/$(basename \"$ARCHIVE\")\" >\"$WORK/rclone-upload.out\" 2>\"$WORK/rclone-upload.err\" && rclone copyto \"$SHA_FILE\" \"$REMOTE:$DEST/$(basename \"$SHA_FILE\")\" --checksum >\"$WORK/rclone-upload-sha.out\" 2>\"$WORK/rclone-upload-sha.err\"; then\n86:  if homelab-rclone-cat-chunked \"$REMOTE:$DEST/$(basename \"$ARCHIVE\")\" >\"$DL\" 2>\"$WORK/rclone-download.err\"; then\n93:rclone lsf \"$REMOTE:$TARGET_BASE\" --dirs-only 2>/dev/null | sed 's#/$##' | sort > \"$DIR_LIST\" || RETENTION_RC=$?\n98:    [ -n \"$OLD\" ] && rclone purge \"$REMOTE:$TARGET_BASE/$OLD\" >>\"$WORK/retention.out\" 2>>\"$WORK/retention.err\"\n103:  rm -f \"$old\" \"$old.sha256\"\n116:rm -rf \"$WORK\"\n----- /usr/local/sbin/homelab-router-running-config-mail-cloud.bak-chunkall-20260710T011410Z -----\n62:  rm -f \"$CONFIG\"\n66:  rm -f \"$CONFIG\"\n73:  if ( cd \"$BACKUP_ROOT\" && sha256sum -c \"$(basename \"$SHA_FILE\")\" ) >\"$WORK/sha-check.out\" 2>\"$WORK/sha-check.err\"; then\n79:  if rclone mkdir \"$REMOTE:$DEST\" >\"$WORK/rclone-mkdir.out\" 2>\"$WORK/rclone-mkdir.err\" && rclone copyto \"$ARCHIVE\" \"$REMOTE:$DEST/$(basename \"$ARCHIVE\")\" --checksum >\"$WORK/rclone-upload.out\" 2>\"$WORK/rclone-upload.err\" && rclone copyto \"$SHA_FILE\" \"$REMOTE:$DEST/$(basename \"$SHA_FILE\")\" --checksum >\"$WORK/rclone-upload-sha.out\" 2>\"$WORK/rclone-upload-sha.err\"; then\n86:  if rclone copyto \"$REMOTE:$DEST/$(basename \"$ARCHIVE\")\" \"$DL\" >\"$WORK/rclone-download.out\" 2>\"$WORK/rclone-download.err\"; then\n93:rclone lsf \"$REMOTE:$TARGET_BASE\" --dirs-only 2>/dev/null | sed 's#/$##' | sort > \"$DIR_LIST\" || RETENTION_RC=$?\n98:    [ -n \"$OLD\" ] && rclone purge \"$REMOTE:$TARGET_BASE/$OLD\" >>\"$WORK/retention.out\" 2>>\"$WORK/retention.err\"\n103:  rm -f \"$old\" \"$old.sha256\"\n116:rm -rf \"$WORK\"\n----- /usr/local/sbin/homelab-secret-exposure-guard -----\n18:BADFILES=\"$(find \"$REPO\" -path \"$REPO/.git\" -prune -o -type f \\( -iname \"*.key\" -o -iname \"*.pem\" -o -iname \"*.p12\" -o -iname \"*.pfx\" -o -iname \"[SENSITIVE_FILE]\" -o -iname \".env\" -o -iname \".env.*\" \\) -print 2>/dev/null)\"\n----- /usr/local/sbin/homelab-stage-2cloud -----\n20:jq -n --arg schema \"homelab-app-backup-manifest-v1\" --arg job \"$JOB\" --arg node \"$NODE\" --arg app \"$APP\" --arg created \"$(date -u +%Y%m%dT%H%M%SZ)\" --arg archive \"$ARCHIVE\" --arg sha \"$SHA\" --arg ra \"$RA\" --arg rb \"$RB\" --argjson size \"$SIZE\" --argjson meta \"$META\" '$meta + {schema:$schema,job_id:$job,node:$node,app_name:$app,created_utc:$created,archive_name:$archive,archive_size_bytes:$size,archive_sha256:$sha,remote_a_job_path:$ra,remote_b_job_path:$rb,local_staging_policy:\"delete_after_both_clouds_verified\",verification_policy:\"full_roundtrip_download_plus_sha256\"}' > \"$STAGE/manifest.json\"\n22:echo \"[STEP] local checksum\"; (cd \"$STAGE\" && sha256sum \"$ARCHIVE\" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e . manifest.json >/dev/null)\n23:echo \"[STEP] upload cloud A metadata\"; rclone copy \"$STAGE\" \"$RA\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n24:echo \"[STEP] upload cloud A archive chunks\"; rclone deletefile \"$RA/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RA/$ARCHIVE\"\n25:echo \"[STEP] upload cloud B metadata\"; rclone copy \"$STAGE\" \"$RB\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n26:echo \"[STEP] upload cloud B archive chunks\"; rclone deletefile \"$RB/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RB/$ARCHIVE\"\n27:echo \"[STEP] verify cloud A chunked\"; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-stage-verify-chunked \"$RA\" \"$ARCHIVE\" \"$VERIFY/a\" \"$JOB\"\n28:echo \"[STEP] verify cloud B chunked\"; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-stage-verify-chunked \"$RB\" \"$ARCHIVE\" \"$VERIFY/b\" \"$JOB\"\n29:echo \"[STEP] upload latest index\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RAL\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RBL\"; rclone cat \"$RAL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null; rclone cat \"$RBL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n30:jq \". + {status:\\\"success\\\",local_backup_deleted:true}\" \"$STAGE/manifest.json\" > \"$STATE_ROOT/apps/latest-${APP}.json\"\n31:echo \"[STEP] delete local staging\"; rm -rf \"$STAGE\" \"$VERIFY\"; test ! -d \"$STAGE\" && test ! -d \"$VERIFY\"\n33:echo \"[SUCCESS] uploaded to 2 clouds, verified from both clouds, local staging deleted\"\n----- /usr/local/sbin/homelab-stage-2cloud.bak-global-chunkonly-20260710T011036Z -----\n20:jq -n --arg schema \"homelab-app-backup-manifest-v1\" --arg job \"$JOB\" --arg node \"$NODE\" --arg app \"$APP\" --arg created \"$(date -u +%Y%m%dT%H%M%SZ)\" --arg archive \"$ARCHIVE\" --arg sha \"$SHA\" --arg ra \"$RA\" --arg rb \"$RB\" --argjson size \"$SIZE\" --argjson meta \"$META\" '$meta + {schema:$schema,job_id:$job,node:$node,app_name:$app,created_utc:$created,archive_name:$archive,archive_size_bytes:$size,archive_sha256:$sha,remote_a_job_path:$ra,remote_b_job_path:$rb,local_staging_policy:\"delete_after_both_clouds_verified\",verification_policy:\"full_roundtrip_download_plus_sha256\"}' > \"$STAGE/manifest.json\"\n22:echo \"[STEP] local checksum\"; (cd \"$STAGE\" && sha256sum \"$ARCHIVE\" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e . manifest.json >/dev/null)\n23:echo \"[STEP] upload cloud A\"; rclone copy \"$STAGE\" \"$RA\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 --stats=30s\n24:echo \"[STEP] upload cloud B\"; rclone copy \"$STAGE\" \"$RB\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 --stats=30s\n25:echo \"[STEP] verify cloud A\"; rclone copy \"$RA\" \"$VERIFY/a\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10; (cd \"$VERIFY/a\" && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n26:echo \"[STEP] verify cloud B\"; rclone copy \"$RB\" \"$VERIFY/b\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10; (cd \"$VERIFY/b\" && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null)\n27:echo \"[STEP] upload latest index\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RAL\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RBL\"; rclone cat \"$RAL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null; rclone cat \"$RBL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n28:jq \". + {status:\\\"success\\\",local_backup_deleted:true}\" \"$STAGE/manifest.json\" > \"$STATE_ROOT/apps/latest-${APP}.json\"\n29:echo \"[STEP] delete local staging\"; rm -rf \"$STAGE\" \"$VERIFY\"; test ! -d \"$STAGE\" && test ! -d \"$VERIFY\"\n31:echo \"[SUCCESS] uploaded to 2 clouds, verified from both clouds, local staging deleted\"\n----- /usr/local/sbin/homelab-stage-2cloud-flat -----\n31:jq -n --arg schema \"homelab-app-backup-manifest-v2-flat\" --arg layout \"flat-v1\" --arg job \"$JOB\" --arg node \"$NODE\" --arg app \"$APP\" --arg created \"$CREATED\" --arg archive \"$ARCHIVE\" --arg sha \"$SHA\" --arg ra \"$RA\" --arg rb \"$RB\" --arg rel \"$REL\" --argjson size \"$SIZE\" --argjson meta \"$META\" \"\\$meta + {schema:\\$schema,layout:\\$layout,job_id:\\$job,node:\\$node,app_name:\\$app,created_utc:\\$created,archive_name:\\$archive,archive_size_bytes:\\$size,archive_sha256:\\$sha,path_rel:\\$rel,remote_a_job_path:\\$ra,remote_b_job_path:\\$rb,local_staging_policy:\\\"delete_after_both_clouds_verified\\\",verification_policy:\\\"flat_chunks_full_roundtrip_sha256\\\"}\" > \"$STAGE/manifest.json\"\n33:(cd \"$STAGE\"; FILES=(\"$ARCHIVE\" manifest.json latest-backup-index.json); [[ -f metadata.json ]] && FILES+=(metadata.json); sha256sum \"${FILES[@]}\" > checksums.sha256; sha256sum -c checksums.sha256 >/dev/null; tar -tzf \"$ARCHIVE\" >/dev/null; jq -e . manifest.json >/dev/null)\n36:  if rclone lsf \"$REMOTE_DIR\" --max-depth 1 2>/dev/null | grep -q .; then echo \"REFUSE NONEMPTY CANARY TARGET $REMOTE_DIR\" >&2; exit 6; fi\n37:  rclone mkdir \"$REMOTE_DIR\"\n38:  for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f \"$STAGE/$F\" ]] && rclone copyto \"$STAGE/$F\" \"$REMOTE_DIR/$F\" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done\n39:  CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-flat-chunked \"$STAGE/$ARCHIVE\" \"$REMOTE_DIR/$ARCHIVE\"\n41:  rclone copyto \"$STAGE/latest-backup-index.json\" \"$LATEST_FILE\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n42:  rclone cat \"$LATEST_FILE\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n46:jq \". + {status:\\\"success\\\",local_backup_deleted:true}\" \"$STAGE/manifest.json\" > \"$STATE_ROOT/apps/latest-${APP}.json\"\n47:rm -rf \"$STAGE\" \"$VERIFY\"\n----- /usr/local/sbin/homelab-stage-2cloud-flat-canary -----\n29:jq -n --arg schema \"homelab-app-backup-manifest-v2-flat\" --arg layout \"flat-v1\" --arg job \"$JOB\" --arg node \"$NODE\" --arg app \"$APP\" --arg created \"$CREATED\" --arg archive \"$ARCHIVE\" --arg sha \"$SHA\" --arg ra \"$RA\" --arg rb \"$RB\" --arg rel \"$REL\" --argjson size \"$SIZE\" --argjson meta \"$META\" \"\\$meta + {schema:\\$schema,layout:\\$layout,job_id:\\$job,node:\\$node,app_name:\\$app,created_utc:\\$created,archive_name:\\$archive,archive_size_bytes:\\$size,archive_sha256:\\$sha,path_rel:\\$rel,remote_a_job_path:\\$ra,remote_b_job_path:\\$rb,local_staging_policy:\\\"delete_after_both_clouds_verified\\\",verification_policy:\\\"flat_chunks_full_roundtrip_sha256\\\"}\" > \"$STAGE/manifest.json\"\n31:(cd \"$STAGE\"; FILES=(\"$ARCHIVE\" manifest.json latest-backup-index.json); [[ -f metadata.json ]] && FILES+=(metadata.json); sha256sum \"${FILES[@]}\" > checksums.sha256; sha256sum -c checksums.sha256 >/dev/null; tar -tzf \"$ARCHIVE\" >/dev/null; jq -e . manifest.json >/dev/null)\n34:  if rclone lsf \"$REMOTE_DIR\" --max-depth 1 2>/dev/null | grep -q .; then echo \"REFUSE NONEMPTY CANARY TARGET $REMOTE_DIR\" >&2; exit 6; fi\n35:  rclone mkdir \"$REMOTE_DIR\"\n36:  for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f \"$STAGE/$F\" ]] && rclone copyto \"$STAGE/$F\" \"$REMOTE_DIR/$F\" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done\n37:  CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-flat-chunked \"$STAGE/$ARCHIVE\" \"$REMOTE_DIR/$ARCHIVE\"\n39:  rclone copyto \"$STAGE/latest-backup-index.json\" \"$LATEST_FILE\" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR\n40:  rclone cat \"$LATEST_FILE\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n44:jq \". + {status:\\\"success\\\",local_backup_deleted:true}\" \"$STAGE/manifest.json\" > \"$STATE_ROOT/apps/latest-${APP}.json\"\n45:rm -rf \"$STAGE\" \"$VERIFY\"\n----- /usr/local/sbin/homelab-stage-verify-chunked -----\n7:rm -rf \"$VERIFY\"\n9:rclone copy \"$REMOTE_DIR\" \"$VERIFY\" --exclude \"$ARCHIVE.parts/**\" --exclude \"$ARCHIVE.parts/\" --exclude \"$ARCHIVE.sha256\" --exclude \"$ARCHIVE.manifest.txt\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --log-level ERROR\n13:if [ -s checksums.sha256 ]; then grep -F -v \"  $ARCHIVE\" checksums.sha256 > .checksums-no-archive.sha256 || true; if [ -s .checksums-no-archive.sha256 ]; then sha256sum -c .checksums-no-archive.sha256; fi; EXPECTED=$(grep -F \"  $ARCHIVE\" checksums.sha256 | awk '{print $1; exit}'); else EXPECTED=$(rclone cat \"${REMOTE_DIR}/${ARCHIVE}.sha256\" | awk '{print $1; exit}'); fi\n15:ACTUAL=$(homelab-rclone-cat-chunked \"${REMOTE_DIR}/${ARCHIVE}\" | sha256sum | awk '{print $1}')\n17:homelab-rclone-cat-chunked \"${REMOTE_DIR}/${ARCHIVE}\" | tar -tzf - >/dev/null\n----- /usr/local/sbin/homelab-stage-verify-flat-chunked -----\n7:rm -rf \"$VERIFY\"\n9:rclone copy \"$REMOTE_DIR\" \"$VERIFY\" --transfers 4 --checkers 8 --retries 5 --low-level-retries 20 --log-level ERROR\n11:(cd \"$VERIFY\" && test -s manifest.json && if [[ -n \"$JOB\" ]]; then jq -e --arg job \"$JOB\" \".job_id == \\$job\" manifest.json >/dev/null; fi && sha256sum -c checksums.sha256 >/dev/null && tar -tzf \"$ARCHIVE\" >/dev/null)\n----- /usr/local/sbin/homelab-vm160-remote-backup -----\n8:PARENT=/mnt/staging/vzdump\n22:SOURCE_ACTIVE=\"$(ssh \"${SSH[@]}\" \"$SOURCE\" \"pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'\")\" || SOURCE_ACTIVE_RC=$?\n27:NAME=\"vzdump-qemu-${VMID}-${TS}.vma.zst\"\n35:    ssh \"${SSH[@]}\" \"$TARGET\" \"rm -rf -- '$DIR'\" >/dev/null 2>&1 || true\n41:  rm -f \"$LOG\"\n46:LATEST_SIZE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\\n' | sort -nr | head -n1\")\"\n55:ssh \"${SSH[@]}\" \"$SOURCE\" \"vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7\" 2>\"$LOG\" | ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$PART'\"\n57:ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$DIR/vzdump.log'\" < \"$LOG\"\n66:rm -f \"$LOG\"\n----- /usr/local/sbin/homelab-vm-backup-health -----\n8:check_file(){\n15:check_file /var/lib/homelab-health/backup.txt auto-vmct\n16:check_file /var/lib/homelab-health/edge-vm-vzdump-backup.txt edge-vm-backup\n17:check_file /var/lib/homelab-health/edge-vm-vzdump-offhost.txt edge-vm-offhost\n18:check_file /var/lib/homelab-health/edge-vm-vzdump-restore.txt edge-vm-restore\n----- /usr/local/sbin/homelab-vm-backup-policy -----\n29:echo \"## Latest local vzdump catalog\" >> \"$REPORT\"\n30:printf \"%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n\" \"kind\" \"vmid\" \"name\" \"size_bytes\" \"mtime_utc\" \"path\" > \"$CATALOG\"\n31:for f in $(find /var/lib/vz/dump -maxdepth 1 -type f \\( -name \"vzdump-qemu-*.vma.zst\" -o -name \"vzdump-lxc-*.tar.zst\" \\) 2>/dev/null | sort); do b=\"$(basename \"$f\")\"; size=\"$(stat -c %s \"$f\" 2>/dev/null || echo 0)\"; mt=\"$(date -u -d \"@$(stat -c %Y \"$f\" 2>/dev/null || echo 0)\" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)\"; if echo \"$b\" | grep -q \"^vzdump-qemu-\"; then kind=qemu; vmid=\"$(echo \"$b\" | sed -n \"s/^vzdump-qemu-\\([0-9]\\+\\)-.*/\\1/p\")\"; name=\"$(qm config \"$vmid\" 2>/dev/null | awk -F\": \" \"/^name:/{print \\$2; exit}\")\"; else kind=lxc; vmid=\"$(echo \"$b\" | sed -n \"s/^vzdump-lxc-\\([0-9]\\+\\)-.*/\\1/p\")\"; name=\"$(pct config \"$vmid\" 2>/dev/null | awk -F\": \" \"/^hostname:/{print \\$2; exit}\")\"; fi; printf \"%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n\" \"$kind\" \"$vmid\" \"${name:-unknown}\" \"$size\" \"$mt\" \"$f\" >> \"$CATALOG\"; done\n37:echo \"- Existing local dumps can be uploaded/verified to two clouds without creating new vzdump archives.\" >> \"$REPORT\"\n38:echo \"- New full vzdump for VM 170/171 is not started automatically because each VM has 120 GB bootdisk and /var/tmp has limited free space.\" >> \"$REPORT\"\n42:echo \"## Coverage checks\" >> \"$REPORT\"\n----- /usr/local/sbin/homelab-vm-full-backup -----\n11:# shellcheck disable=SC1090\n92:for cmd in vzdump qm zstd sha256sum rclone rsync ssh flock; do\n130:            -name \"vzdump-qemu-${VMID}-*.vma.zst\" \\\n138:    vzdump \"$VMID\" \\\n145:        --remove 0\n151:            -name \"vzdump-qemu-${VMID}-*.vma.zst\" \\\n261:        CHUNK_TMP_BASE=\"$BACKUP_DIR/.rclone-chunks\" homelab-rclone-copy-chunked \"$ARCHIVE\" \"${DEST_ROOT}/${NAME}\"\n----- /usr/local/sbin/homelab-vm-full-backup.bak-chunkall-20260710T011410Z -----\n11:# shellcheck disable=SC1090\n92:for cmd in vzdump qm zstd sha256sum rclone rsync ssh flock; do\n130:            -name \"vzdump-qemu-${VMID}-*.vma.zst\" \\\n138:    vzdump \"$VMID\" \\\n145:        --remove 0\n151:            -name \"vzdump-qemu-${VMID}-*.vma.zst\" \\\n264:        rclone copyto \\\n267:            --check-first \\\n269:            --checkers 2 \\\n275:        rclone copyto \\\n278:            --check-first \\\n280:            --checkers 2 \\\n285:        echo \"Crypt-checking $NAME on $REMOTE\"\n287:        rclone cryptcheck \\\n292:            --checkers 1 \\\n----- /usr/local/sbin/homelab-vm-full-restore-drill -----\n77:        -name \"vzdump-qemu-${source_vmid}-*.vma.zst\" \\\n152:check_inventory() {\n184:check_container_health() {\n224:check_http() {\n349:check_container_health() {\n386:check_container_health \"$test_vmid\"\n389:    check_inventory \"$source_vmid\" \"$test_vmid\"\n392:    check_http \"$source_vmid\" \"$test_vmid\"\n----- /usr/local/sbin/homelab-vm-full-restore-drill.bak.20260710T080231Z -----\n77:        -name \"vzdump-qemu-${source_vmid}-*.vma.zst\" \\\n152:check_inventory() {\n184:check_container_health() {\n224:check_http() {\n349:check_container_health() {\n386:check_container_health \"$test_vmid\"\n389:    check_inventory \"$source_vmid\" \"$test_vmid\"\n392:    check_http \"$source_vmid\" \"$test_vmid\"\n----- /usr/local/sbin/homelab-vm-full-restore-drill.bak-docker-check-v2-20260710T050044Z -----\n77:        -name \"vzdump-qemu-${source_vmid}-*.vma.zst\" \\\n152:check_inventory() {\n184:check_container_health() {\n224:check_http() {\n347:    check_container_health \"$test_vmid\"\n350:    check_inventory \"$source_vmid\" \"$test_vmid\"\n353:    check_http \"$source_vmid\" \"$test_vmid\"\n----- /usr/local/sbin/homelab-vm-full-restore-drill.bak-isolated-ip-20260710T044331Z -----\n77:        -name \"vzdump-qemu-${source_vmid}-*.vma.zst\" \\\n152:check_inventory() {\n184:check_container_health() {\n224:check_http() {\n337:    check_container_health \"$test_vmid\"\n340:    check_inventory \"$source_vmid\" \"$test_vmid\"\n343:    check_http \"$source_vmid\" \"$test_vmid\"\n----- /usr/local/sbin/homelab-vm-full-retention -----\n21:                -name \"vzdump-qemu-${vmid}-*.vma.zst\" \\\n33:                vzdump-qemu-\"$vmid\"-*.vma.zst)\n35:                    rm -f \\\n73:                         rm -rf -- \\\"\\$BASE/\\$setname\\\"\n91:        rclone lsf \"$path\" \\\n105:        rclone purge \"${path}/${dir}\"\n----- /usr/local/sbin/homelab-vm-local-dumps-2cloud -----\n30:CHKS=\"$WORK/checksums.txt\"\n39:echo \"chunk_size_bytes=$CHUNK_SIZE_BYTES\" >> \"$LOG\"\n42:  F=\"$(find /var/lib/vz/dump -maxdepth 1 -type f \\( -name \"vzdump-qemu-${id}-*.vma.zst\" -o -name \"vzdump-lxc-${id}-*.tar.zst\" \\) -printf \"%T@ %p\\n\" 2>/dev/null | sort -n | tail -n 1 | cut -d\" \" -f2-)\"\n53:  echo \"$BN\" | grep -q \"^vzdump-qemu-\" && KIND=\"qemu\"\n54:  echo \"$BN\" | grep -q \"^vzdump-lxc-\" && KIND=\"lxc\"\n64:    rm -rf \"$CDIR\"\n66:    echo \"SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR\" >> \"$LOG\"\n83:job,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:]\n91:            \"size_bytes\":int(p[2]),\n107:    \"chunk_size_bytes\":int(chunk_size),\n117:  rclone mkdir \"$DEST/files\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n118:  while IFS=\"$(printf \"\\t\")\" read -r id kind size sha mt bn path mode parts; do\n121:      echo \"UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts\" >> \"$LOG\"\n122:      rclone mkdir \"$DEST/files/$bn.chunks\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n126:        rclone copyto \"$part\" \"$DEST/files/$bn.chunks/$pn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n128:      rclone copyto \"$CDIR/chunks.sha256\" \"$DEST/files/$bn.chunks/chunks.sha256\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n130:      echo \"UPLOAD_DIRECT dest=$DEST file=$bn size=$size\" >> \"$LOG\"\n131:      rclone copyto \"$path\" \"$DEST/files/$bn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n135:  rclone copyto \"$CHKS\" \"$DEST/checksums.txt\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n136:  rclone copyto \"$FILES_TSV\" \"$DEST/files.tsv\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n137:  rclone copyto \"$MAN\" \"$DEST/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n144:  rm -rf \"$VBASE\"\n146:  rclone copyto \"$DEST/manifest.json\" \"$VBASE/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n149:  while IFS=\"$(printf \"\\t\")\" read -r id kind size sha mt bn path mode parts; do\n154:      rclone copy \"$DEST/files/$bn.chunks\" \"$VDIR\" --filter \"+ part-*\" --filter \"+ chunks.sha256\" --filter \"- *\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n163:      rm -rf \"$VDIR\"\n166:      rclone copyto \"$DEST/files/$bn\" \"$VBASE/$bn\" --transfers 1 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n168:      rm -f \"$VBASE/$bn\"\n171:  rm -rf \"$VBASE\"\n184:rm -rf \"$VERIFY_ROOT/$JOB_ID\"\n185:rm -rf \"$CHUNK_ROOT/$JOB_ID\"\n198:  --argjson chunk_size_bytes \"$CHUNK_SIZE_BYTES\" \\\n200:  \"{job_id:\\$job_id,status:\\$status,node:\\$node,remote_a:\\$remote_a,remote_b:\\$remote_b,count:\\$count,total_bytes:\\$total_bytes,chunked_count:\\$chunked_count,chunk_size_bytes:\\$chunk_size_bytes,log:\\$log,local_dumps_deleted:false}\" \\\n----- /usr/local/sbin/homelab-vm-local-dumps-retention -----\n32:  mapfile -t JOBS < <(rclone lsf \"$BASE/\" --dirs-only 2>/dev/null | sed 's#/$##' | sort)\n37:    echo \"nothing_to_delete\" >> \"$REPORT\"\n56:        rclone purge \"$PATH_REMOTE\" >> \"$REPORT\" 2>&1 && DELETED=$((DELETED+1)) || BAD=$((BAD+1))\n----- /usr/local/sbin/skladchik-reports-monitor-backup -----\n28:command -v rclone >/dev/null\n29:[ -x /usr/local/sbin/homelab-rclone-copy-chunked ] || { echo \"missing /usr/local/sbin/homelab-rclone-copy-chunked\" >&2; exit 10; }\n36:cleanup(){ rm -rf \"$TMP\"; }\n67:size_bytes=${SIZE}\n71:log \"LOCAL_ARCHIVE_OK size=$SIZE path=$ARCHIVE\"\n75:  /usr/local/sbin/homelab-rclone-copy-chunked \"$ARCHIVE\" \"${DEST_BASE}/$(basename \"$ARCHIVE\")\"\n76:  /usr/local/sbin/homelab-rclone-copy-chunked \"${ARCHIVE}.sha256\" \"${DEST_BASE}/$(basename \"$ARCHIVE\").sha256\"\n77:  /usr/local/sbin/homelab-rclone-copy-chunked \"${OUTDIR}/manifest.txt\" \"${DEST_BASE}/manifest.txt\"\n82:  mapfile -t OLD < <(rclone lsf --dirs-only \"${REMOTE}${REMOTE_BASE}/daily/\" 2>/dev/null | sed 's#/$##' | sort | head -n -\"${KEEP_DAILY}\" || true)\n83:  for D in \"${OLD[@]}\"; do [ -n \"$D\" ] && rclone purge \"${REMOTE}${REMOTE_BASE}/daily/${D}\" >/dev/null 2>&1 || true; done\n85:find \"${LOCAL_ROOT}/daily\" -mindepth 1 -maxdepth 1 -type d -printf '%f\\n' 2>/dev/null | sort | head -n -\"${KEEP_DAILY}\" | while read -r D; do [ -n \"$D\" ] && rm -rf \"${LOCAL_ROOT}/daily/${D}\"; done\n----- /usr/local/sbin/skladchik-reports-monitor-backup.pre-restore-check-20260710T065311Z.bak -----\n28:command -v rclone >/dev/null\n29:[ -x /usr/local/sbin/homelab-rclone-copy-chunked ] || { echo \"missing /usr/local/sbin/homelab-rclone-copy-chunked\" >&2; exit 10; }\n36:cleanup(){ rm -rf \"$TMP\"; }\n66:size_bytes=${SIZE}\n70:log \"LOCAL_ARCHIVE_OK size=$SIZE path=$ARCHIVE\"\n74:  /usr/local/sbin/homelab-rclone-copy-chunked \"$ARCHIVE\" \"${DEST_BASE}/$(basename \"$ARCHIVE\")\"\n75:  /usr/local/sbin/homelab-rclone-copy-chunked \"${ARCHIVE}.sha256\" \"${DEST_BASE}/$(basename \"$ARCHIVE\").sha256\"\n76:  /usr/local/sbin/homelab-rclone-copy-chunked \"${OUTDIR}/manifest.txt\" \"${DEST_BASE}/manifest.txt\"\n81:  mapfile -t OLD < <(rclone lsf --dirs-only \"${REMOTE}${REMOTE_BASE}/daily/\" 2>/dev/null | sed 's#/$##' | sort | head -n -\"${KEEP_DAILY}\" || true)\n82:  for D in \"${OLD[@]}\"; do [ -n \"$D\" ] && rclone purge \"${REMOTE}${REMOTE_BASE}/daily/${D}\" >/dev/null 2>&1 || true; done\n84:find \"${LOCAL_ROOT}/daily\" -mindepth 1 -maxdepth 1 -type d -printf '%f\\n' 2>/dev/null | sort | head -n -\"${KEEP_DAILY}\" | while read -r D; do [ -n \"$D\" ] && rm -rf \"${LOCAL_ROOT}/daily/${D}\"; done\n----- /usr/local/sbin/skladchik-reports-monitor-backup.pre-session-keepalive-20260710T071605Z.bak -----\n28:command -v rclone >/dev/null\n29:[ -x /usr/local/sbin/homelab-rclone-copy-chunked ] || { echo \"missing /usr/local/sbin/homelab-rclone-copy-chunked\" >&2; exit 10; }\n36:cleanup(){ rm -rf \"$TMP\"; }\n64:size_bytes=${SIZE}\n68:log \"LOCAL_ARCHIVE_OK size=$SIZE path=$ARCHIVE\"\n72:  /usr/local/sbin/homelab-rclone-copy-chunked \"$ARCHIVE\" \"${DEST_BASE}/$(basename \"$ARCHIVE\")\"\n73:  /usr/local/sbin/homelab-rclone-copy-chunked \"${ARCHIVE}.sha256\" \"${DEST_BASE}/$(basename \"$ARCHIVE\").sha256\"\n74:  /usr/local/sbin/homelab-rclone-copy-chunked \"${OUTDIR}/manifest.txt\" \"${DEST_BASE}/manifest.txt\"\n79:  mapfile -t OLD < <(rclone lsf --dirs-only \"${REMOTE}${REMOTE_BASE}/daily/\" 2>/dev/null | sed 's#/$##' | sort | head -n -\"${KEEP_DAILY}\" || true)\n80:  for D in \"${OLD[@]}\"; do [ -n \"$D\" ] && rclone purge \"${REMOTE}${REMOTE_BASE}/daily/${D}\" >/dev/null 2>&1 || true; done\n82:find \"${LOCAL_ROOT}/daily\" -mindepth 1 -maxdepth 1 -type d -printf '%f\\n' 2>/dev/null | sort | head -n -\"${KEEP_DAILY}\" | while read -r D; do [ -n \"$D\" ] && rm -rf \"${LOCAL_ROOT}/daily/${D}\"; done\n----- /usr/local/sbin/skladchik-reports-monitor-backup-restore-check -----\n4:CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf\n9:: \"${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}\"\n12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\n13:LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock\n19:log(){ printf '%s RESTORE_CHECK %s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$*\" | tee -a \"$LOGDIR/restore-check.log\"; }\n22:command -v rclone >/dev/null\n28:  if rclone lsf \"${remote_path}.parts/\" > \"$plist\" 2>/dev/null && grep -q '^part-' \"$plist\"; then\n29:    while IFS= read -r part; do [ -n \"$part\" ] && rclone cat \"${remote_path}.parts/${part}\" >> \"$out\"; done < <(grep '^part-' \"$plist\" | sort)\n31:    rclone copyto \"$remote_path\" \"$out\" >/dev/null\n33:  rm -f \"$plist\"\n36:check_one_remote(){\n39:  latest=\"$(rclone lsf --dirs-only \"${rbase}/\" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)\"\n85:    [ \"$(sqlite3 \"$extract/var-lib-skladchik-reports-monitor/state.sqlite\" 'PRAGMA integrity_check;' | tail -n 1)\" = \"ok\" ]\n97:check_one_remote CLOUD_1 \"$CLOUD_1\"\n98:check_one_remote CLOUD_2 \"$CLOUD_2\"\n101:find \"$WORK_ROOT\" -mindepth 1 -maxdepth 1 -type d -printf '%f\\n' 2>/dev/null | sort | head -n -\"${KEEP_LOCAL}\" | while read -r D; do [ -n \"$D\" ] && rm -rf \"${WORK_ROOT}/${D}\"; done\n----- /usr/local/sbin/skladchik-reports-monitor-backup-restore-check.pre-session-keepalive-20260710T071605Z.bak -----\n4:CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf\n9:: \"${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}\"\n12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\n13:LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock\n19:log(){ printf '%s RESTORE_CHECK %s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \"$*\" | tee -a \"$LOGDIR/restore-check.log\"; }\n22:command -v rclone >/dev/null\n28:  if rclone lsf \"${remote_path}.parts/\" > \"$plist\" 2>/dev/null && grep -q '^part-' \"$plist\"; then\n29:    while IFS= read -r part; do [ -n \"$part\" ] && rclone cat \"${remote_path}.parts/${part}\" >> \"$out\"; done < <(grep '^part-' \"$plist\" | sort)\n31:    rclone copyto \"$remote_path\" \"$out\" >/dev/null\n33:  rm -f \"$plist\"\n36:check_one_remote(){\n39:  latest=\"$(rclone lsf --dirs-only \"${rbase}/\" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)\"\n72:    [ \"$(sqlite3 \"$extract/var-lib-skladchik-reports-monitor/state.sqlite\" 'PRAGMA integrity_check;' | tail -n 1)\" = \"ok\" ]\n84:check_one_remote CLOUD_1 \"$CLOUD_1\"\n85:check_one_remote CLOUD_2 \"$CLOUD_2\"\n88:find \"$WORK_ROOT\" -mindepth 1 -maxdepth 1 -type d -printf '%f\\n' 2>/dev/null | sort | head -n -\"${KEEP_LOCAL}\" | while read -r D; do [ -n \"$D\" ] && rm -rf \"${WORK_ROOT}/${D}\"; done\n"
}
