CHAT_OUTPUT_BEGIN
COMMAND_ID=BACKUP-PIPELINE-DEFINITION-AUDIT-061
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=backup-audit
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1
COMMAND_SHA256=08de7a8820803f855426b27b3ecfd2e646209715ff5ac4aba48b73fa45623b28
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGES_MADE=false
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=5b51699fff305570e41040468ef3f04a84e21c0c977695ed1834aa40c01946b5
SANITIZED_OUTPUT_SHA256=384c3f223883553a582cbf25ff144f7ff8a0a1dc79110ffe089b7154a12aed74
OUTPUT_BEGIN
===== TIMER dpkg-db-backup.timer =====
# /usr/lib/systemd/system/dpkg-db-backup.timer
[Unit]
Description=Daily dpkg database backup timer
Documentation=man:dpkg(1)

[Timer]
OnCalendar=daily
Persistent=true

[Install]
WantedBy=timers.target
===== SERVICE dpkg-db-backup.service =====
# /usr/lib/systemd/system/dpkg-db-backup.service
[Unit]
Description=Daily dpkg database backup service
Documentation=man:dpkg(1)

[Service]
Type=oneshot
ExecStart=/usr/libexec/dpkg/dpkg-db-backup

# /etc/systemd/system/dpkg-db-backup.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/libexec/dpkg/dpkg-db-backup ; argv[]=/usr/libexec/dpkg/dpkg-db-backup ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
ActiveState=inactive
===== TIMER filebrowser-backup.timer =====
# /etc/systemd/system/filebrowser-backup.timer
[Unit]
Description=Run File Browser backup

[Timer]
OnCalendar=*-*-* 03:35:00
Persistent=true

[Install]
WantedBy=timers.target
===== SERVICE filebrowser-backup.service =====
# /etc/systemd/system/filebrowser-backup.service
[Unit]
Description=File Browser backup
After=network-online.target

[Service]
Type=oneshot
ExecStart=/root/filebrowser-backup.sh

# /etc/systemd/system/filebrowser-backup.service.d/10-superseded-noop.conf
[Service]
ExecStart=
ExecStart=/usr/local/sbin/homelab-superseded-unit-ok filebrowser-backup.service superseded_by_current_appbackup_restore_proof

# /etc/systemd/system/filebrowser-backup.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok filebrowser-backup.service superseded_by_current_appbackup_restore_proof ; ignore_errors=no ; start_time=[Tue 2026-07-21 03:35:14 MSK] ; stop_time=[Tue 2026-07-21 03:35:14 MSK] ; pid=2211390 ; code=exited ; status=0 }
ActiveState=inactive
===== TIMER homelab-alertmanager-backup.timer =====
# /etc/systemd/system/homelab-alertmanager-backup.timer
[Unit]
Description=Daily Alertmanager two-cloud backup

[Timer]
OnCalendar=*-*-* 06:35:00
RandomizedDelaySec=10m
Persistent=true
Unit=homelab-alertmanager-backup.service

[Install]
WantedBy=timers.target
===== SERVICE homelab-alertmanager-backup.service =====
# /etc/systemd/system/homelab-alertmanager-backup.service
[Unit]
Description=Alertmanager encrypted two-cloud application backup
After=network-online.target
Wants=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-alertmanager-backup-run
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-alertmanager-backup-run ; argv[]=/usr/local/sbin/homelab-alertmanager-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 06:40:25 MSK] ; stop_time=[Tue 2026-07-21 06:40:54 MSK] ; pid=2820982 ; code=exited ; status=0 }
ActiveState=inactive
===== TIMER homelab-appbackup-dockge-stacks.timer =====
# /etc/systemd/system/homelab-appbackup-dockge-stacks.timer
[Unit]
Description=Daily homelab app-level backup timer for dockge-stacks

[Timer]
OnCalendar=*-*-* 03:40:00
RandomizedDelaySec=20m
Persistent=true

[Install]
WantedBy=timers.target
===== SERVICE homelab-appbackup-dockge-stacks.service =====
# /etc/systemd/system/homelab-appbackup-dockge-stacks.service
[Unit]
Description=Homelab app-level backup for dockge-stacks

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/appbackupctl run
ExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run dockge-stacks

# /etc/systemd/system/homelab-appbackup-dockge-stacks.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/appbackupctl ; argv[]=/usr/local/sbin/appbackupctl run ; ignore_errors=no ; start_time=[Tue 2026-07-21 03:51:25 MSK] ; stop_time=[Tue 2026-07-21 03:51:53 MSK] ; pid=2261529 ; code=exited ; status=0 }
ActiveState=inactive
===== TIMER homelab-appbackup-mariadb-batch.timer =====
# /etc/systemd/system/homelab-appbackup-mariadb-batch.timer
[Unit]
Description=Daily Homelab MariaDB app batch backup timer

[Timer]
OnCalendar=*-*-* 05:35:00
RandomizedDelaySec=20m
Persistent=true

[Install]
WantedBy=timers.target
===== SERVICE homelab-appbackup-mariadb-batch.service =====
# /etc/systemd/system/homelab-appbackup-mariadb-batch.service
[Unit]
Description=Homelab MariaDB app batch backup

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/mariadb-apps-backup-run
ExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run bookstack

# /etc/systemd/system/homelab-appbackup-mariadb-batch.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/mariadb-apps-backup-run ; argv[]=/usr/local/sbin/mariadb-apps-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 05:46:14 MSK] ; stop_time=[Tue 2026-07-21 05:46:44 MSK] ; pid=2623964 ; code=exited ; status=0 }
ActiveState=inactive
===== TIMER homelab-appbackup-npmplus-edge.timer =====
# /etc/systemd/system/homelab-appbackup-npmplus-edge.timer
[Unit]
Description=Daily homelab app-level backup timer for npmplus-edge

[Timer]
OnCalendar=*-*-* 04:05:00
RandomizedDelaySec=20m
Persistent=true

[Install]
WantedBy=timers.target
===== SERVICE homelab-appbackup-npmplus-edge.service =====
# /etc/systemd/system/homelab-appbackup-npmplus-edge.service
[Unit]
Description=Homelab app-level backup for npmplus-edge

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/npmplus-edge-backup-run
ExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run npmplus-edge

# /etc/systemd/system/homelab-appbackup-npmplus-edge.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/npmplus-edge-backup-run ; argv[]=/usr/local/sbin/npmplus-edge-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 04:19:17 MSK] ; stop_time=[Tue 2026-07-21 04:19:48 MSK] ; pid=2360589 ; code=exited ; status=0 }
ActiveState=inactive
===== TIMER homelab-appbackup-postgres-batch.timer =====
# /etc/systemd/system/homelab-appbackup-postgres-batch.timer
[Unit]
Description=Daily Homelab Postgres app batch backup timer

[Timer]
OnCalendar=*-*-* 05:10:00
RandomizedDelaySec=20m
Persistent=true

[Install]
WantedBy=timers.target
===== SERVICE homelab-appbackup-postgres-batch.service =====
# /etc/systemd/system/homelab-appbackup-postgres-batch.service
[Unit]
Description=Homelab Postgres app batch backup

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/postgres-apps-backup-run
ExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run authentik
ExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run paperless
ExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run immich
ExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run netbox

# /etc/systemd/system/homelab-appbackup-postgres-batch.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/postgres-apps-backup-run ; argv[]=/usr/local/sbin/postgres-apps-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 05:29:05 MSK] ; stop_time=[Tue 2026-07-21 05:32:29 MSK] ; pid=2568901 ; code=exited ; status=0 }
ActiveState=inactive
===== TIMER homelab-appbackup-sqlite-batch.timer =====
# /etc/systemd/system/homelab-appbackup-sqlite-batch.timer
[Unit]
Description=Daily Homelab SQLite app batch backup timer

[Timer]
OnCalendar=*-*-* 04:45:00
RandomizedDelaySec=20m
Persistent=true

[Install]
WantedBy=timers.target
===== SERVICE homelab-appbackup-sqlite-batch.service =====
# /etc/systemd/system/homelab-appbackup-sqlite-batch.service
[Unit]
Description=Homelab SQLite app batch backup

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/sqlite-apps-backup-run
ExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run vaultwarden
ExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run gitea
ExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run uptime-kuma
ExecStartPost=/usr/local/sbin/appbackupctl retention 14 dry-run gotify

# /etc/systemd/system/homelab-appbackup-sqlite-batch.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/sqlite-apps-backup-run ; argv[]=/usr/local/sbin/sqlite-apps-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 04:57:14 MSK] ; stop_time=[Tue 2026-07-21 05:00:36 MSK] ; pid=2476638 ; code=exited ; status=0 }
ActiveState=inactive
===== TIMER homelab-authentik-backup.timer =====
# /etc/systemd/system/homelab-authentik-backup.timer
[Unit]
Description=Run Homelab Authentik backup chain daily

[Timer]
OnCalendar=*-*-* 04:50:00
RandomizedDelaySec=10min
Persistent=true
Unit=homelab-authentik-backup.service

[Install]
WantedBy=timers.target
===== SERVICE homelab-authentik-backup.service =====
# /etc/systemd/system/homelab-authentik-backup.service
[Unit]
Description=Homelab Authentik backup chain
Wants=network-online.target
After=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-backup-authentik
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-backup-authentik ; argv[]=/usr/local/sbin/homelab-backup-authentik ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
ActiveState=inactive
===== TIMER homelab-auto-backup.timer =====
# /etc/systemd/system/homelab-auto-backup.timer
[Unit]
Description=Run homelab automatic full backup weekly

[Timer]
OnCalendar=Sun *-*-* 03:30:00
Persistent=true
RandomizedDelaySec=20m
Unit=homelab-auto-backup.service

[Install]
WantedBy=timers.target
===== SERVICE homelab-auto-backup.service =====
# /etc/systemd/system/homelab-auto-backup.service
[Unit]
Description=Homelab automatic full backup with retention
Wants=network-online.target
After=network-online.target

[Service]
Type=oneshot
ExecStart=/root/homelab-auto-backup-with-retention.sh
Nice=10
IOSchedulingClass=best-effort
IOSchedulingPriority=7

# /etc/systemd/system/homelab-auto-backup.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/root/homelab-auto-backup-with-retention.sh ; argv[]=/root/homelab-auto-backup-with-retention.sh ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
ActiveState=inactive
===== TIMER homelab-backup-coverage-matrix.timer =====
# /etc/systemd/system/homelab-backup-coverage-matrix.timer
[Unit]
Description=Homelab backup coverage matrix timer

[Timer]
OnCalendar=*-*-* 08:35:00
RandomizedDelaySec=20m
Persistent=true

[Install]
WantedBy=timers.target
===== SERVICE homelab-backup-coverage-matrix.service =====
# /etc/systemd/system/homelab-backup-coverage-matrix.service
[Unit]
Description=Homelab backup coverage matrix

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-backup-coverage-matrix

# /etc/systemd/system/homelab-backup-coverage-matrix.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-backup-coverage-matrix ; argv[]=/usr/local/sbin/homelab-backup-coverage-matrix ; ignore_errors=no ; start_time=[Tue 2026-07-21 08:50:14 MSK] ; stop_time=[Tue 2026-07-21 08:50:15 MSK] ; pid=3277860 ; code=exited ; status=0 }
ActiveState=inactive
===== TIMER homelab-backup-freshness.timer =====
# /etc/systemd/system/homelab-backup-freshness.timer
[Unit]
Description=Run homelab backup freshness collector daily

[Timer]
OnCalendar=daily
Persistent=true
RandomizedDelaySec=20m
Unit=homelab-backup-freshness.service

[Install]
WantedBy=timers.target
===== SERVICE homelab-backup-freshness.service =====
# /etc/systemd/system/homelab-backup-freshness.service
[Unit]
Description=Homelab backup freshness collector
Documentation=man:systemd.service(5)
After=local-fs.target

[Service]
Type=oneshot
EnvironmentFile=-/etc/default/homelab-backup-freshness
ExecStart=/usr/local/sbin/homelab-backup-freshness.sh

# /etc/systemd/system/homelab-backup-freshness.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-backup-freshness.sh ; argv[]=/usr/local/sbin/homelab-backup-freshness.sh ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
ActiveState=inactive
===== TIMER homelab-backup-healthcheck.timer =====
# /etc/systemd/system/homelab-backup-healthcheck.timer
[Unit]
Description=Run homelab backup healthcheck daily

[Timer]
OnCalendar=*-*-* 07:00:00
Persistent=true
RandomizedDelaySec=10m
Unit=homelab-backup-healthcheck.service

[Install]
WantedBy=timers.target
===== SERVICE homelab-backup-healthcheck.service =====
# /etc/systemd/system/homelab-backup-healthcheck.service
[Unit]
Description=Homelab backup healthcheck

[Service]
Type=oneshot
ExecStart=/root/homelab-backup-healthcheck-wrapper.sh

# /etc/systemd/system/homelab-backup-healthcheck.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/root/homelab-backup-healthcheck-wrapper.sh ; argv[]=/root/homelab-backup-healthcheck-wrapper.sh ; ignore_errors=no ; start_time=[Tue 2026-07-21 07:05:30 MSK] ; stop_time=[Tue 2026-07-21 07:05:43 MSK] ; pid=2900293 ; code=exited ; status=0 }
ActiveState=inactive
===== TIMER homelab-backup-matrix-refresh.timer =====
# /etc/systemd/system/homelab-backup-matrix-refresh.timer
[Unit]
Description=Refresh unified homelab backup matrix state

[Timer]
OnBootSec=2min
OnUnitActiveSec=5min
AccuracySec=30s
Persistent=true
Unit=homelab-backup-matrix-refresh.service

[Install]
WantedBy=timers.target
===== SERVICE homelab-backup-matrix-refresh.service =====
# /etc/systemd/system/homelab-backup-matrix-refresh.service
[Unit]
Description=Generate unified homelab backup matrix state
After=local-fs.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-backup-matrix-refresh
TimeoutStartSec=5min
Result=exit-code
ExecMainStatus=1
ExecStart={ path=/usr/local/sbin/homelab-backup-matrix-refresh ; argv[]=/usr/local/sbin/homelab-backup-matrix-refresh ; ignore_errors=no ; start_time=[Tue 2026-07-21 10:02:41 MSK] ; stop_time=[Tue 2026-07-21 10:02:42 MSK] ; pid=3525445 ; code=exited ; status=1 }
ActiveState=failed
===== TIMER homelab-backup-sla-health.timer =====
# /etc/systemd/system/homelab-backup-sla-health.timer
[Unit]
Description=Homelab backup freshness SLA health timer

[Timer]
OnCalendar=*-*-* 08:00:00
RandomizedDelaySec=20m
Persistent=true

[Install]
WantedBy=timers.target
===== SERVICE homelab-backup-sla-health.service =====
# /etc/systemd/system/homelab-backup-sla-health.service
[Unit]
Description=Homelab backup freshness SLA health
After=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-backup-sla-health

# /etc/systemd/system/homelab-backup-sla-health.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-backup-sla-health ; argv[]=/usr/local/sbin/homelab-backup-sla-health ; ignore_errors=no ; start_time=[Tue 2026-07-21 08:07:25 MSK] ; stop_time=[Tue 2026-07-21 08:07:26 MSK] ; pid=3122093 ; code=exited ; status=0 }
ActiveState=inactive
===== TIMER homelab-beszel-backup.timer =====
# /etc/systemd/system/homelab-beszel-backup.timer
[Unit]
Description=Daily Beszel encrypted two-cloud backup

[Timer]
OnCalendar=*-*-* 06:45:00
RandomizedDelaySec=10m
Persistent=true
Unit=homelab-beszel-backup.service

[Install]
WantedBy=timers.target
===== SERVICE homelab-beszel-backup.service =====
# /etc/systemd/system/homelab-beszel-backup.service
[Unit]
Description=Beszel encrypted two-cloud application backup
After=network-online.target
Wants=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-beszel-backup-run
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-beszel-backup-run ; argv[]=/usr/local/sbin/homelab-beszel-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 06:47:47 MSK] ; stop_time=[Tue 2026-07-21 06:48:22 MSK] ; pid=2848350 ; code=exited ; status=0 }
ActiveState=inactive
===== TIMER homelab-blackbox-exporter-backup.timer =====
# /etc/systemd/system/homelab-blackbox-exporter-backup.timer
[Unit]
Description=Daily Blackbox Exporter encrypted two-cloud backup

[Timer]
OnCalendar=*-*-* 06:55:00
RandomizedDelaySec=10m
Persistent=true
Unit=homelab-blackbox-exporter-backup.service

[Install]
WantedBy=timers.target
===== SERVICE homelab-blackbox-exporter-backup.service =====
# /etc/systemd/system/homelab-blackbox-exporter-backup.service
[Unit]
Description=Blackbox Exporter encrypted two-cloud application backup
After=network-online.target
Wants=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-blackbox-exporter-backup-run
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-blackbox-exporter-backup-run ; argv[]=/usr/local/sbin/homelab-blackbox-exporter-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 06:57:30 MSK] ; stop_time=[Tue 2026-07-21 06:57:54 MSK] ; pid=2876084 ; code=exited ; status=0 }
ActiveState=inactive
===== TIMER homelab-cluster-admin-incident-engine-backup.timer =====
# /etc/systemd/system/homelab-cluster-admin-incident-engine-backup.timer
[Unit]
Description=Daily Cluster Admin Incident Engine two-cloud backup

[Timer]
OnCalendar=*-*-* 07:25:00
RandomizedDelaySec=15m
Persistent=true
Unit=homelab-cluster-admin-incident-engine-backup.service

[Install]
WantedBy=timers.target
===== SERVICE homelab-cluster-admin-incident-engine-backup.service =====
# /etc/systemd/system/homelab-cluster-admin-incident-engine-backup.service
[Unit]
Description=Cluster Admin Incident Engine encrypted two-cloud backup
After=network-online.target
Wants=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run ; argv[]=/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 07:36:14 MSK] ; stop_time=[Tue 2026-07-21 07:36:40 MSK] ; pid=3010566 ; code=exited ; status=0 }
ActiveState=inactive
===== TIMER homelab-crypto-portfolio-backup.timer =====
# /etc/systemd/system/homelab-crypto-portfolio-backup.timer
[Unit]
Description=Daily Gram Crypto portfolio backup

[Timer]
OnCalendar=*-*-* 06:25:00
RandomizedDelaySec=15m
Persistent=true

[Install]
WantedBy=timers.target
===== SERVICE homelab-crypto-portfolio-backup.service =====
# /etc/systemd/system/homelab-crypto-portfolio-backup.service
[Unit]
Description=Gram Crypto portfolio two-cloud backup
After=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/crypto-portfolio-backup-run
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/crypto-portfolio-backup-run ; argv[]=/usr/local/sbin/crypto-portfolio-backup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 06:32:25 MSK] ; stop_time=[Tue 2026-07-21 06:32:50 MSK] ; pid=2794060 ; code=exited ; status=0 }
ActiveState=inactive
===== TIMER homelab-edge-vm-vzdump-backup.timer =====
# /etc/systemd/system/homelab-edge-vm-vzdump-backup.timer
[Unit]
Description=Run Homelab edge-vm VM-level backup chain weekly

[Timer]
OnCalendar=Sun *-*-* 06:20:00
RandomizedDelaySec=20min
Persistent=true
Unit=homelab-edge-vm-vzdump-backup.service

[Install]
WantedBy=timers.target
===== SERVICE homelab-edge-vm-vzdump-backup.service =====
# /etc/systemd/system/homelab-edge-vm-vzdump-backup.service
[Unit]
Description=Homelab edge-vm VM-level vzdump backup chain
Wants=network-online.target
After=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-backup-edge-vm-vzdump

# /etc/systemd/system/homelab-edge-vm-vzdump-backup.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-backup-edge-vm-vzdump ; argv[]=/usr/local/sbin/homelab-backup-edge-vm-vzdump ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
ActiveState=inactive
===== TIMER homelab-emergency-backup.timer =====
# /etc/systemd/system/homelab-emergency-backup.timer
[Unit]
Description=Daily homelab emergency/config backup timer

[Timer]
OnCalendar=*-*-* 03:10:00
RandomizedDelaySec=20m
Persistent=true

[Install]
WantedBy=timers.target
===== SERVICE homelab-emergency-backup.service =====
# /etc/systemd/system/homelab-emergency-backup.service
[Unit]
Description=Homelab emergency/config backup to two encrypted clouds

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/backupctl run
ExecStartPost=/usr/local/sbin/backup-retention 14 dry-run

# /etc/systemd/system/homelab-emergency-backup.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/backupctl ; argv[]=/usr/local/sbin/backupctl run ; ignore_errors=no ; start_time=[Tue 2026-07-21 03:13:14 MSK] ; stop_time=[Tue 2026-07-21 03:13:42 MSK] ; pid=2137825 ; code=exited ; status=0 }
ActiveState=inactive
===== TIMER homelab-extended-appbackup.timer =====
# /etc/systemd/system/homelab-extended-appbackup.timer
[Unit]
Description=Homelab extended P1/P2 appbackup timer

[Timer]
OnCalendar=*-*-* 05:55:00
RandomizedDelaySec=20m
Persistent=true

[Install]
WantedBy=timers.target
===== SERVICE homelab-extended-appbackup.service =====
# /etc/systemd/system/homelab-extended-appbackup.service
[Unit]
Description=Homelab extended P1/P2 appbackup run
After=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-extended-appbackup-run

# /etc/systemd/system/homelab-extended-appbackup.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-extended-appbackup-run ; argv[]=/usr/local/sbin/homelab-extended-appbackup-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 05:57:30 MSK] ; stop_time=[Tue 2026-07-21 06:06:46 MSK] ; pid=2671623 ; code=exited ; status=0 }
ActiveState=inactive
===== TIMER homelab-gitea-backup.timer =====
# /etc/systemd/system/homelab-gitea-backup.timer
[Unit]
Description=Run Homelab Gitea backup chain daily

[Timer]
OnCalendar=*-*-* 05:10:00
RandomizedDelaySec=10min
Persistent=true
Unit=homelab-gitea-backup.service

[Install]
WantedBy=timers.target
===== SERVICE homelab-gitea-backup.service =====
# /etc/systemd/system/homelab-gitea-backup.service
[Unit]
Description=Homelab Gitea backup chain
Wants=network-online.target
After=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-backup-gitea
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-backup-gitea ; argv[]=/usr/local/sbin/homelab-backup-gitea ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
ActiveState=inactive
===== TIMER homelab-gitea-secondary-backup.timer =====
# /etc/systemd/system/homelab-gitea-secondary-backup.timer
[Unit]
Description=Daily verified Gitea backup to pve03

[Timer]
OnCalendar=*-*-* 04:20:00
RandomizedDelaySec=10m
Persistent=true
Unit=homelab-gitea-secondary-backup.service

[Install]
WantedBy=timers.target
===== SERVICE homelab-gitea-secondary-backup.service =====
# /etc/systemd/system/homelab-gitea-secondary-backup.service
[Unit]
Description=Create and copy a verified Gitea backup to pve03
After=pve-cluster.service network-online.target
Wants=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-gitea-secondary-backup
TimeoutStartSec=2h
Nice=10
IOSchedulingClass=best-effort
IOSchedulingPriority=7
UMask=0077
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-gitea-secondary-backup ; argv[]=/usr/local/sbin/homelab-gitea-secondary-backup ; ignore_errors=no ; start_time=[Tue 2026-07-21 04:28:14 MSK] ; stop_time=[Tue 2026-07-21 04:28:18 MSK] ; pid=2376819 ; code=exited ; status=0 }
ActiveState=inactive
===== TIMER homelab-legacy-monitoring-backups.timer =====
# /etc/systemd/system/homelab-legacy-monitoring-backups.timer
[Unit]
Description=Daily legacy monitoring applications two-cloud backup batch

[Timer]
OnCalendar=*-*-* 07:05:00
RandomizedDelaySec=15m
Persistent=true
Unit=homelab-legacy-monitoring-backups.service

[Install]
WantedBy=timers.target
===== SERVICE homelab-legacy-monitoring-backups.service =====
# /etc/systemd/system/homelab-legacy-monitoring-backups.service
[Unit]
Description=Legacy monitoring applications encrypted two-cloud backup batch
After=network-online.target
Wants=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-legacy-monitoring-backups-run
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-legacy-monitoring-backups-run ; argv[]=/usr/local/sbin/homelab-legacy-monitoring-backups-run ; ignore_errors=no ; start_time=[Tue 2026-07-21 07:18:55 MSK] ; stop_time=[Tue 2026-07-21 07:21:45 MSK] ; pid=2950676 ; code=exited ; status=0 }
ActiveState=inactive
===== TIMER homelab-mail-cloud-critical-upload.timer =====
# /etc/systemd/system/homelab-mail-cloud-critical-upload.timer
[Unit]
Description=Daily Homelab Mail Cloud critical/small baseline upload

[Timer]
OnCalendar=*-*-* 06:40:00
RandomizedDelaySec=15m
Persistent=true
Unit=homelab-mail-cloud-critical-upload.service

[Install]
WantedBy=timers.target
===== SERVICE homelab-mail-cloud-critical-upload.service =====
# /etc/systemd/system/homelab-mail-cloud-critical-upload.service
[Unit]
Description=Homelab Mail Cloud critical/small baseline upload
After=network-online.target
Wants=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-mail-cloud-critical-upload
Nice=10
IOSchedulingClass=best-effort
IOSchedulingPriority=7
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-mail-cloud-critical-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-critical-upload ; ignore_errors=no ; start_time=[Tue 2026-07-21 06:40:25 MSK] ; stop_time=[Tue 2026-07-21 06:42:44 MSK] ; pid=2820985 ; code=exited ; status=0 }
ActiveState=inactive
===== TIMER homelab-mail-cloud-edge-vm-upload.timer =====
# /etc/systemd/system/homelab-mail-cloud-edge-vm-upload.timer
[Unit]
Description=Weekly Homelab Mail Cloud edge-vm large chunked upload

[Timer]
OnCalendar=Sun *-*-* 09:30:00
RandomizedDelaySec=30m
Persistent=true
Unit=homelab-mail-cloud-edge-vm-upload.service

[Install]
WantedBy=timers.target
===== SERVICE homelab-mail-cloud-edge-vm-upload.service =====
# /etc/systemd/system/homelab-mail-cloud-edge-vm-upload.service
[Unit]
Description=Homelab Mail Cloud edge-vm large chunked upload
After=network-online.target
Wants=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload
Nice=10
IOSchedulingClass=best-effort
IOSchedulingPriority=7
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
ActiveState=inactive
===== TIMER homelab-mail-cloud-nextcloud-vm-upload.timer =====
# /etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.timer
[Unit]
Description=Run Homelab Mail Cloud Nextcloud VM upload daily

[Timer]
OnCalendar=*-*-* 07:30:00
RandomizedDelaySec=30m
Persistent=true

[Install]
WantedBy=timers.target
===== SERVICE homelab-mail-cloud-nextcloud-vm-upload.service =====
# /etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.service
[Unit]
Description=Homelab Mail Cloud Nextcloud VM vzdump upload
Wants=network-online.target
After=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload
Nice=10
IOSchedulingClass=best-effort
IOSchedulingPriority=7
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload ; ignore_errors=no ; start_time=[Tue 2026-07-21 07:53:41 MSK] ; stop_time=[Tue 2026-07-21 07:56:26 MSK] ; pid=3068432 ; code=exited ; status=0 }
ActiveState=inactive
===== TIMER homelab-mail-cloud-restore-drill.timer =====
# /etc/systemd/system/homelab-mail-cloud-restore-drill.timer
[Unit]
Description=Weekly Homelab Mail Cloud restore drill

[Timer]
OnCalendar=Mon *-*-* 07:20:00
RandomizedDelaySec=20m
Persistent=true
Unit=homelab-mail-cloud-restore-drill.service

[Install]
WantedBy=timers.target
===== SERVICE homelab-mail-cloud-restore-drill.service =====
# /etc/systemd/system/homelab-mail-cloud-restore-drill.service
[Unit]
Description=Homelab Mail Cloud restore drill
After=network-online.target
Wants=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-mail-cloud-restore-drill
Nice=10
IOSchedulingClass=best-effort
IOSchedulingPriority=7

# /etc/systemd/system/homelab-mail-cloud-restore-drill.service.d/10-superseded-noop.conf
[Service]
ExecStart=
ExecStart=/usr/local/sbin/homelab-superseded-unit-ok homelab-mail-cloud-restore-drill.service superseded_by_safety_net_restore_check

# /etc/systemd/system/homelab-mail-cloud-restore-drill.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok homelab-mail-cloud-restore-drill.service superseded_by_safety_net_restore_check ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
ActiveState=inactive
===== TIMER homelab-npmplus-kuma-config-backup.timer =====
# /etc/systemd/system/homelab-npmplus-kuma-config-backup.timer
[Unit]
Description=Run Homelab NPMplus + Kuma config backup chain daily

[Timer]
OnCalendar=*-*-* 04:20:00
RandomizedDelaySec=10min
Persistent=true
Unit=homelab-npmplus-kuma-config-backup.service

[Install]
WantedBy=timers.target
===== SERVICE homelab-npmplus-kuma-config-backup.service =====
# /etc/systemd/system/homelab-npmplus-kuma-config-backup.service
[Unit]
Description=Homelab NPMplus + Kuma config backup chain
Wants=network-online.target
After=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-backup-npmplus-kuma-configs
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-backup-npmplus-kuma-configs ; argv[]=/usr/local/sbin/homelab-backup-npmplus-kuma-configs ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
ActiveState=inactive
===== TIMER homelab-router-running-config-mail-cloud.timer =====
# /etc/systemd/system/homelab-router-running-config-mail-cloud.timer
[Unit]
Description=Run Homelab router running-config Mail-cloud backup daily

[Timer]
OnCalendar=*-*-* 05:15:00
RandomizedDelaySec=20m
Persistent=true

[Install]
WantedBy=timers.target
===== SERVICE homelab-router-running-config-mail-cloud.service =====
# /etc/systemd/system/homelab-router-running-config-mail-cloud.service
[Unit]
Description=Homelab router running-config Mail-cloud backup
Wants=network-online.target
After=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-router-running-config-mail-cloud
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-router-running-config-mail-cloud ; argv[]=/usr/local/sbin/homelab-router-running-config-mail-cloud ; ignore_errors=no ; start_time=[Tue 2026-07-21 05:17:57 MSK] ; stop_time=[Tue 2026-07-21 05:18:05 MSK] ; pid=2540965 ; code=exited ; status=0 }
ActiveState=inactive
===== TIMER homelab-vaultwarden-backup.timer =====
# /etc/systemd/system/homelab-vaultwarden-backup.timer
[Unit]
Description=Run Homelab Vaultwarden backup chain daily

[Timer]
OnCalendar=*-*-* 05:30:00
RandomizedDelaySec=10min
Persistent=true
Unit=homelab-vaultwarden-backup.service

[Install]
WantedBy=timers.target
===== SERVICE homelab-vaultwarden-backup.service =====
# /etc/systemd/system/homelab-vaultwarden-backup.service
[Unit]
Description=Homelab Vaultwarden backup chain
Wants=network-online.target
After=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-backup-vaultwarden
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-backup-vaultwarden ; argv[]=/usr/local/sbin/homelab-backup-vaultwarden ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
ActiveState=inactive
===== TIMER homelab-verified-backup-health.timer =====
# /etc/systemd/system/homelab-verified-backup-health.timer
[Unit]
Description=Homelab verified backup health evidence timer

[Timer]
OnCalendar=*-*-* 06:30:00
RandomizedDelaySec=20m
Persistent=true

[Install]
WantedBy=timers.target
===== SERVICE homelab-verified-backup-health.service =====
# /etc/systemd/system/homelab-verified-backup-health.service
[Unit]
Description=Homelab verified backup health evidence

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-backup-health-evidence

# /etc/systemd/system/homelab-verified-backup-health.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-backup-health-evidence ; argv[]=/usr/local/sbin/homelab-backup-health-evidence ; ignore_errors=no ; start_time=[Tue 2026-07-21 06:43:54 MSK] ; stop_time=[Tue 2026-07-21 06:45:16 MSK] ; pid=2823792 ; code=exited ; status=0 }
ActiveState=inactive
===== TIMER homelab-vm-backup-health.timer =====
# /etc/systemd/system/homelab-vm-backup-health.timer
[Unit]
Description=Homelab VM backup health timer

[Timer]
OnCalendar=*-*-* 07:30:00
RandomizedDelaySec=20m
Persistent=true

[Install]
WantedBy=timers.target
===== SERVICE homelab-vm-backup-health.service =====
# /etc/systemd/system/homelab-vm-backup-health.service
[Unit]
Description=Homelab VM backup health

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-vm-backup-health

# /etc/systemd/system/homelab-vm-backup-health.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-vm-backup-health ; argv[]=/usr/local/sbin/homelab-vm-backup-health ; ignore_errors=no ; start_time=[Tue 2026-07-21 07:36:59 MSK] ; stop_time=[Tue 2026-07-21 07:36:59 MSK] ; pid=3011738 ; code=exited ; status=0 }
ActiveState=inactive
===== TIMER homelab-vm-backup-policy.timer =====
# /etc/systemd/system/homelab-vm-backup-policy.timer
[Unit]
Description=Homelab VM backup policy and feasibility health timer

[Timer]
OnCalendar=*-*-* 08:18:00
RandomizedDelaySec=20m
Persistent=true

[Install]
WantedBy=timers.target
===== SERVICE homelab-vm-backup-policy.service =====
# /etc/systemd/system/homelab-vm-backup-policy.service
[Unit]
Description=Homelab VM backup policy and feasibility health
After=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-vm-backup-policy

# /etc/systemd/system/homelab-vm-backup-policy.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-vm-backup-policy ; argv[]=/usr/local/sbin/homelab-vm-backup-policy ; ignore_errors=no ; start_time=[Tue 2026-07-21 08:30:07 MSK] ; stop_time=[Tue 2026-07-21 08:30:13 MSK] ; pid=3198902 ; code=exited ; status=0 }
ActiveState=inactive
===== TIMER homelab-vm-full-backup.timer =====
# /etc/systemd/system/homelab-vm-full-backup.timer
[Unit]
Description=Daily full backup timer for VM170 and VM171

[Timer]
OnCalendar=*-*-* 02:15:00
Persistent=true
AccuracySec=1min
Unit=homelab-vm-full-backup.service

[Install]
WantedBy=timers.target
===== SERVICE homelab-vm-full-backup.service =====
# /etc/systemd/system/homelab-vm-full-backup.service
[Unit]
Description=Full vzdump backup of VM170 and VM171 to offhost and two clouds
After=network-online.target pve-cluster.service
Wants=network-online.target
ConditionPathIsMountPoint=/mnt/staging

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-vm-full-backup
TimeoutStartSec=infinity
Nice=10
IOSchedulingClass=idle
IOSchedulingPriority=7
UMask=0077

# /etc/systemd/system/homelab-vm-full-backup.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-vm-full-backup ; argv[]=/usr/local/sbin/homelab-vm-full-backup ; ignore_errors=no ; start_time=[Tue 2026-07-21 02:15:05 MSK] ; stop_time=[Tue 2026-07-21 03:09:43 MSK] ; pid=1933641 ; code=exited ; status=0 }
ActiveState=inactive
===== TIMER homelab-vm-local-dumps-2cloud.timer =====
# /etc/systemd/system/homelab-vm-local-dumps-2cloud.timer
[Unit]
Description=Homelab VM/CT existing local dumps two-cloud verifier timer

[Timer]
OnCalendar=Sun *-*-* 10:00:00
RandomizedDelaySec=30m
Persistent=true

[Install]
WantedBy=timers.target
===== SERVICE homelab-vm-local-dumps-2cloud.service =====
# /etc/systemd/system/homelab-vm-local-dumps-2cloud.service
[Unit]
Description=Homelab VM/CT existing local dumps two-cloud verifier
After=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-vm-local-dumps-2cloud

# /etc/systemd/system/homelab-vm-local-dumps-2cloud.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; argv[]=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
ActiveState=inactive
===== TIMER homelab-vm160-remote-backup.timer =====
# /etc/systemd/system/homelab-vm160-remote-backup.timer
[Unit]
Description=Daily VM160 remote full-image backup

[Timer]
OnCalendar=*-*-* 04:10:00
RandomizedDelaySec=10m
Unit=homelab-vm160-remote-backup.service

[Install]
WantedBy=timers.target
===== SERVICE homelab-vm160-remote-backup.service =====
# /etc/systemd/system/homelab-vm160-remote-backup.service
[Unit]
Description=VM160 full-image backup streamed from pve02 to pve03
After=network-online.target pve-cluster.service
Wants=network-online.target

[Service]
Type=oneshot
TimeoutStartSec=6h
ExecStart=/usr/local/sbin/homelab-vm160-remote-backup
Result=exit-code
ExecMainStatus=1
ExecStart={ path=/usr/local/sbin/homelab-vm160-remote-backup ; argv[]=/usr/local/sbin/homelab-vm160-remote-backup ; ignore_errors=no ; start_time=[Tue 2026-07-21 02:39:12 MSK] ; stop_time=[Tue 2026-07-21 02:39:14 MSK] ; pid=2028253 ; code=exited ; status=1 }
ActiveState=failed
===== TIMER immich-media-backup.timer =====
# /etc/systemd/system/immich-media-backup.timer
[Unit]
Description=Daily Immich media backup timer

[Timer]
OnCalendar=*-*-* 08:15:00
Persistent=true
RandomizedDelaySec=10m
Unit=immich-media-backup.service

[Install]
WantedBy=timers.target
===== SERVICE immich-media-backup.service =====
# /etc/systemd/system/immich-media-backup.service
[Unit]
Description=Immich media backup to staging
Wants=network-online.target
After=network-online.target

[Service]
Type=oneshot
ExecStart=/root/immich-media-backup.sh
Nice=10
IOSchedulingClass=best-effort
IOSchedulingPriority=7

# /etc/systemd/system/immich-media-backup.service.d/10-superseded-noop.conf
[Service]
ExecStart=
ExecStart=/usr/local/sbin/homelab-superseded-unit-ok immich-media-backup.service superseded_by_current_appbackup_or_trust_proof

# /etc/systemd/system/immich-media-backup.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok immich-media-backup.service superseded_by_current_appbackup_or_trust_proof ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
ActiveState=inactive
===== TIMER memos-backup.timer =====
# /etc/systemd/system/memos-backup.timer
[Unit]
Description=Run Memos data backup daily

[Timer]
OnCalendar=*-*-* 08:35:00
RandomizedDelaySec=20m
Persistent=true

[Install]
WantedBy=timers.target
===== SERVICE memos-backup.service =====
# /etc/systemd/system/memos-backup.service
[Unit]
Description=Memos data backup to pve01 staging

[Service]
Type=oneshot
ExecStart=/root/memos-backup.sh

# /etc/systemd/system/memos-backup.service.d/10-superseded-noop.conf
[Service]
ExecStart=
ExecStart=/usr/local/sbin/homelab-superseded-unit-ok memos-backup.service superseded_by_current_appbackup_or_trust_proof

# /etc/systemd/system/memos-backup.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok memos-backup.service superseded_by_current_appbackup_or_trust_proof ; ignore_errors=no ; start_time=[Tue 2026-07-21 08:41:14 MSK] ; stop_time=[Tue 2026-07-21 08:41:14 MSK] ; pid=3239605 ; code=exited ; status=0 }
ActiveState=inactive
===== TIMER netbird-vps-backup.timer =====
# /etc/systemd/system/netbird-vps-backup.timer
[Unit]
Description=Run NetBird VPS backup daily

[Timer]
OnCalendar=*-*-* 04:25:00
Persistent=true
RandomizedDelaySec=15m

[Install]
WantedBy=timers.target
===== SERVICE netbird-vps-backup.service =====
# /etc/systemd/system/netbird-vps-backup.service
[Unit]
Description=NetBird VPS backup to pve01 staging

[Service]
Type=oneshot
ExecStart=/root/netbird-vps-backup.sh

# /etc/systemd/system/netbird-vps-backup.service.d/10-superseded-noop.conf
[Service]
ExecStart=
ExecStart=/usr/local/sbin/homelab-superseded-unit-ok netbird-vps-backup.service superseded_by_current_appbackup_or_trust_proof

# /etc/systemd/system/netbird-vps-backup.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok netbird-vps-backup.service superseded_by_current_appbackup_or_trust_proof ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
ActiveState=inactive
===== TIMER paperless-backup.timer =====
# /etc/systemd/system/paperless-backup.timer
[Unit]
Description=Run Paperless backup daily

[Timer]
OnCalendar=*-*-* 08:50:00
RandomizedDelaySec=20m
Persistent=true

[Install]
WantedBy=timers.target
===== SERVICE paperless-backup.service =====
# /etc/systemd/system/paperless-backup.service
[Unit]
Description=Paperless backup to pve01 staging

[Service]
Type=oneshot
ExecStart=/root/paperless-backup.sh

# /etc/systemd/system/paperless-backup.service.d/10-superseded-noop.conf
[Service]
ExecStart=
ExecStart=/usr/local/sbin/homelab-superseded-unit-ok paperless-backup.service superseded_by_current_appbackup_or_trust_proof

# /etc/systemd/system/paperless-backup.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok paperless-backup.service superseded_by_current_appbackup_or_trust_proof ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
ActiveState=inactive
===== TIMER skladchik-reports-monitor-backup-restore-check.timer =====
# /etc/systemd/system/skladchik-reports-monitor-backup-restore-check.timer
[Unit]
Description=Weekly restore-check of Skladchik reports monitor cloud backups

[Timer]
OnCalendar=Sun *-*-* 04:55:00
RandomizedDelaySec=30min
Persistent=true
Unit=skladchik-reports-monitor-backup-restore-check.service

[Install]
WantedBy=timers.target
===== SERVICE skladchik-reports-monitor-backup-restore-check.service =====
# /etc/systemd/system/skladchik-reports-monitor-backup-restore-check.service
[Unit]
Description=Restore-check Skladchik reports monitor cloud backups
Wants=network-online.target
After=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check
Nice=10
IOSchedulingClass=best-effort
IOSchedulingPriority=7

# /etc/systemd/system/skladchik-reports-monitor-backup-restore-check.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check ; argv[]=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
ActiveState=inactive
===== TIMER skladchik-reports-monitor-backup.timer =====
# /etc/systemd/system/skladchik-reports-monitor-backup.timer
[Unit]
Description=Daily Skladchik reports monitor backup to encrypted clouds

[Timer]
OnCalendar=*-*-* 04:25:00
RandomizedDelaySec=15min
Persistent=true
Unit=skladchik-reports-monitor-backup.service

[Install]
WantedBy=timers.target
===== SERVICE skladchik-reports-monitor-backup.service =====
# /etc/systemd/system/skladchik-reports-monitor-backup.service
[Unit]
Description=Backup Skladchik reports monitor from edge-vm to encrypted clouds
Wants=network-online.target
After=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/skladchik-reports-monitor-backup
Nice=10
IOSchedulingClass=best-effort
IOSchedulingPriority=7

# /etc/systemd/system/skladchik-reports-monitor-backup.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/skladchik-reports-monitor-backup ; argv[]=/usr/local/sbin/skladchik-reports-monitor-backup ; ignore_errors=no ; start_time=[Tue 2026-07-21 04:34:48 MSK] ; stop_time=[Tue 2026-07-21 04:35:22 MSK] ; pid=2399885 ; code=exited ; status=0 }
ActiveState=inactive
===== REFERENCED_SCRIPTS =====
----- /opt/homelab-backup/backup-emergency-pack-v0.sh -----
41:rclone listremotes > "$PAYLOAD/inventory/rclone_remotes.txt" 2>&1 || true
59:It is deleted only after both encrypted cloud copies are uploaded and fully verified.
60:Recovery requires offline/break-glass access to rclone crypt credentials.
65:rm -rf "$PAYLOAD"
79:  "archive_size_bytes": $ARCHIVE_SIZE_BYTES,
83:  "local_staging_policy": "delete_after_both_clouds_verified",
90:echo "[STEP] local checksum"
91:(cd "$STAGE" && sha256sum "$ARCHIVE" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null)
94:rclone copy "$STAGE" "$REMOTE_A_JOB" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s
95:homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$REMOTE_A_JOB/$ARCHIVE"
98:rclone copy "$STAGE" "$REMOTE_B_JOB" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s
99:homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$REMOTE_B_JOB/$ARCHIVE"
102:rclone copy "$REMOTE_A_JOB" "$VERIFY/cloud_a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10; homelab-rehydrate-chunked-dir "$VERIFY/cloud_a"
103:(cd "$VERIFY/cloud_a" && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB_ID" '.job_id == $job' manifest.json >/dev/null)
106:rclone copy "$REMOTE_B_JOB" "$VERIFY/cloud_b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10; homelab-rehydrate-chunked-dir "$VERIFY/cloud_b"
107:(cd "$VERIFY/cloud_b" && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB_ID" '.job_id == $job' manifest.json >/dev/null)
110:(cd "$STAGE" && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null)
113:rclone copyto "$STAGE/latest-backup-index.json" "$REMOTE_A_LATEST"
114:rclone copyto "$STAGE/latest-backup-index.json" "$REMOTE_B_LATEST"
115:rclone cat "$REMOTE_A_LATEST" | jq -e --arg job "$JOB_ID" '.job_id == $job' >/dev/null
116:rclone cat "$REMOTE_B_LATEST" | jq -e --arg job "$JOB_ID" '.job_id == $job' >/dev/null
124:  "archive_size_bytes": $ARCHIVE_SIZE_BYTES,
128:  "local_backup_deleted": true
132:echo "[STEP] delete local staging and verification copies"
133:rm -rf "$STAGE" "$VERIFY"
138:echo "[SUCCESS] uploaded to 2 clouds, verified from both clouds, local staging deleted"
----- /opt/homelab-backup/backup-emergency-pack-v0.sh.bak-chunkall-20260710T011410Z -----
41:rclone listremotes > "$PAYLOAD/inventory/rclone_remotes.txt" 2>&1 || true
59:It is deleted only after both encrypted cloud copies are uploaded and fully verified.
60:Recovery requires offline/break-glass access to rclone crypt credentials.
65:rm -rf "$PAYLOAD"
79:  "archive_size_bytes": $ARCHIVE_SIZE_BYTES,
83:  "local_staging_policy": "delete_after_both_clouds_verified",
90:echo "[STEP] local checksum"
91:(cd "$STAGE" && sha256sum "$ARCHIVE" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null)
94:rclone copy "$STAGE" "$REMOTE_A_JOB" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 --stats=30s
97:rclone copy "$STAGE" "$REMOTE_B_JOB" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 --stats=30s
100:rclone copy "$REMOTE_A_JOB" "$VERIFY/cloud_a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10
101:(cd "$VERIFY/cloud_a" && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB_ID" '.job_id == $job' manifest.json >/dev/null)
104:rclone copy "$REMOTE_B_JOB" "$VERIFY/cloud_b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10
105:(cd "$VERIFY/cloud_b" && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB_ID" '.job_id == $job' manifest.json >/dev/null)
108:(cd "$STAGE" && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null)
111:rclone copyto "$STAGE/latest-backup-index.json" "$REMOTE_A_LATEST"
112:rclone copyto "$STAGE/latest-backup-index.json" "$REMOTE_B_LATEST"
113:rclone cat "$REMOTE_A_LATEST" | jq -e --arg job "$JOB_ID" '.job_id == $job' >/dev/null
114:rclone cat "$REMOTE_B_LATEST" | jq -e --arg job "$JOB_ID" '.job_id == $job' >/dev/null
122:  "archive_size_bytes": $ARCHIVE_SIZE_BYTES,
126:  "local_backup_deleted": true
130:echo "[STEP] delete local staging and verification copies"
131:rm -rf "$STAGE" "$VERIFY"
136:echo "[SUCCESS] uploaded to 2 clouds, verified from both clouds, local staging deleted"
----- /usr/local/sbin/appbackup-dockge-stacks -----
25:echo "[STEP] copy app data"
30:rm -rf "$PAYLOAD"
42:  "archive_size_bytes": $SIZE,
46:  "local_staging_policy": "delete_after_both_clouds_verified",
51:echo "[STEP] local checksum"
52:(cd "$STAGE" && sha256sum "$ARCHIVE" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e . manifest.json >/dev/null)
54:echo "[STEP] upload cloud A metadata"; rclone copy "$STAGE" "$RA" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s
55:echo "[STEP] upload cloud A archive chunks"; rclone deletefile "$RA/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RA/$ARCHIVE"
56:echo "[STEP] upload cloud B metadata"; rclone copy "$STAGE" "$RB" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s
57:echo "[STEP] upload cloud B archive chunks"; rclone deletefile "$RB/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RB/$ARCHIVE"
58:echo "[STEP] verify cloud A chunked"; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-stage-verify-chunked "$RA" "$ARCHIVE" "$VERIFY/a" "$JOB"
59:echo "[STEP] verify cloud B chunked"; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-stage-verify-chunked "$RB" "$ARCHIVE" "$VERIFY/b" "$JOB"
60:echo "[STEP] upload latest index"; rclone copyto "$STAGE/latest-backup-index.json" "$RAL"; rclone copyto "$STAGE/latest-backup-index.json" "$RBL"; rclone cat "$RAL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null; rclone cat "$RBL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null
68:  "archive_size_bytes": $SIZE,
72:  "local_backup_deleted": true
75:echo "[STEP] delete local staging and verification"
76:rm -rf "$STAGE" "$VERIFY"
79:echo "[SUCCESS] app backup uploaded to 2 clouds, verified from both clouds, local staging deleted"
----- /usr/local/sbin/appbackup-dockge-stacks.bak-chunkall-20260710T011410Z -----
25:echo "[STEP] copy app data"
30:rm -rf "$PAYLOAD"
42:  "archive_size_bytes": $SIZE,
46:  "local_staging_policy": "delete_after_both_clouds_verified",
51:echo "[STEP] local checksum"
52:(cd "$STAGE" && sha256sum "$ARCHIVE" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e . manifest.json >/dev/null)
54:rclone copy "$STAGE" "$RA" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 --stats=30s
56:rclone copy "$STAGE" "$RB" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 --stats=30s
58:rclone copy "$RA" "$VERIFY/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10
59:(cd "$VERIFY/a" && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null)
61:rclone copy "$RB" "$VERIFY/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10
62:(cd "$VERIFY/b" && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null)
64:(cd "$STAGE" && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e . manifest.json >/dev/null)
66:rclone copyto "$STAGE/latest-backup-index.json" "$RAL"
67:rclone copyto "$STAGE/latest-backup-index.json" "$RBL"
68:rclone cat "$RAL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null
69:rclone cat "$RBL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null
77:  "archive_size_bytes": $SIZE,
81:  "local_backup_deleted": true
84:echo "[STEP] delete local staging and verification"
85:rm -rf "$STAGE" "$VERIFY"
88:echo "[SUCCESS] app backup uploaded to 2 clouds, verified from both clouds, local staging deleted"
----- /usr/local/sbin/appbackup-restore-check -----
22:ROOT="/var/tmp/homelab-app-restore-check/${JOB}"
24:rm -rf "$ROOT" && mkdir -p "$ROOT/a" "$ROOT/b"
25:trap "echo \"ERROR app restore-check failed; kept: $ROOT\"; exit 1" ERR
26:echo "restore-check app=$APP job=$JOB schema=$SCHEMA"
27:rclone copy "$RA" "$ROOT/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >"${LOGBASE}-a.log" 2>&1
30:(cd "$ROOT/a" && sha256sum -c checksums.sha256 >"${LOGBASE}-a-sha.log" && ARCHIVE="$(jq -r ".archive_name" manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null)
31:echo "OK cloud A app restore-check"
32:rclone copy "$RB" "$ROOT/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >"${LOGBASE}-b.log" 2>&1
35:(cd "$ROOT/b" && sha256sum -c checksums.sha256 >"${LOGBASE}-b-sha.log" && ARCHIVE="$(jq -r ".archive_name" manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null)
36:echo "OK cloud B app restore-check"
37:rm -rf "$ROOT"
38:rmdir /var/tmp/homelab-app-restore-check 2>/dev/null || true
39:rm -f "${LOGBASE}-a.log" "${LOGBASE}-a-sha.log" "${LOGBASE}-b.log" "${LOGBASE}-b-sha.log"
41:echo "OK app restore-check both clouds"
----- /usr/local/sbin/appbackup-restore-check.bak-chunkall-20260710T011410Z -----
9:ROOT="/var/tmp/homelab-app-restore-check/${JOB}"
10:rm -rf "$ROOT" && mkdir -p "$ROOT/a" "$ROOT/b"
11:trap "echo \"ERROR app restore-check failed; kept: $ROOT\"; exit 1" ERR
12:echo "restore-check app=$APP job=$JOB"
13:rclone copy "$RA" "$ROOT/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/app-restore-a.log 2>&1
14:(cd "$ROOT/a" && sha256sum -c checksums.sha256 >/tmp/app-restore-a-sha.log && ARCHIVE="$(jq -r .archive_name manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null)
15:echo "OK cloud A app restore-check"
16:rclone copy "$RB" "$ROOT/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/app-restore-b.log 2>&1
17:(cd "$ROOT/b" && sha256sum -c checksums.sha256 >/tmp/app-restore-b-sha.log && ARCHIVE="$(jq -r .archive_name manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null)
18:echo "OK cloud B app restore-check"
19:rm -rf "$ROOT"; rmdir /var/tmp/homelab-app-restore-check 2>/dev/null || true
21:echo "OK app restore-check both clouds"
----- /usr/local/sbin/appbackup-restore-check.bak-dual-read-20260711T051924Z -----
9:ROOT="/var/tmp/homelab-app-restore-check/${JOB}"
10:rm -rf "$ROOT" && mkdir -p "$ROOT/a" "$ROOT/b"
11:trap "echo \"ERROR app restore-check failed; kept: $ROOT\"; exit 1" ERR
12:echo "restore-check app=$APP job=$JOB"
13:rclone copy "$RA" "$ROOT/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/app-restore-a.log 2>&1; homelab-rehydrate-chunked-dir "$ROOT/a" >>/tmp/app-restore-a.log 2>&1
14:(cd "$ROOT/a" && sha256sum -c checksums.sha256 >/tmp/app-restore-a-sha.log && ARCHIVE="$(jq -r .archive_name manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null)
15:echo "OK cloud A app restore-check"
16:rclone copy "$RB" "$ROOT/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/app-restore-b.log 2>&1; homelab-rehydrate-chunked-dir "$ROOT/b" >>/tmp/app-restore-b.log 2>&1
17:(cd "$ROOT/b" && sha256sum -c checksums.sha256 >/tmp/app-restore-b-sha.log && ARCHIVE="$(jq -r .archive_name manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null)
18:echo "OK cloud B app restore-check"
19:rm -rf "$ROOT"; rmdir /var/tmp/homelab-app-restore-check 2>/dev/null || true
21:echo "OK app restore-check both clouds"
----- /usr/local/sbin/appbackup-restore-check.bak-flat-read-20260711T052854Z -----
22:ROOT="/var/tmp/homelab-app-restore-check/${JOB}"
24:rm -rf "$ROOT" && mkdir -p "$ROOT/a" "$ROOT/b"
25:trap "echo \"ERROR app restore-check failed; kept: $ROOT\"; exit 1" ERR
26:echo "restore-check app=$APP job=$JOB schema=$SCHEMA"
27:rclone copy "$RA" "$ROOT/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >"${LOGBASE}-a.log" 2>&1
29:(cd "$ROOT/a" && sha256sum -c checksums.sha256 >"${LOGBASE}-a-sha.log" && ARCHIVE="$(jq -r ".archive_name" manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null)
30:echo "OK cloud A app restore-check"
31:rclone copy "$RB" "$ROOT/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >"${LOGBASE}-b.log" 2>&1
33:(cd "$ROOT/b" && sha256sum -c checksums.sha256 >"${LOGBASE}-b-sha.log" && ARCHIVE="$(jq -r ".archive_name" manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null)
34:echo "OK cloud B app restore-check"
35:rm -rf "$ROOT"
36:rmdir /var/tmp/homelab-app-restore-check 2>/dev/null || true
37:rm -f "${LOGBASE}-a.log" "${LOGBASE}-a-sha.log" "${LOGBASE}-b.log" "${LOGBASE}-b-sha.log"
39:echo "OK app restore-check both clouds"
----- /usr/local/sbin/appbackup-restore-check.bak-flat-read-v2-20260711T053129Z -----
22:ROOT="/var/tmp/homelab-app-restore-check/${JOB}"
24:rm -rf "$ROOT" && mkdir -p "$ROOT/a" "$ROOT/b"
25:trap "echo \"ERROR app restore-check failed; kept: $ROOT\"; exit 1" ERR
26:echo "restore-check app=$APP job=$JOB schema=$SCHEMA"
27:rclone copy "$RA" "$ROOT/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >"${LOGBASE}-a.log" 2>&1
29:(cd "$ROOT/a" && sha256sum -c checksums.sha256 >"${LOGBASE}-a-sha.log" && ARCHIVE="$(jq -r ".archive_name" manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null)
30:echo "OK cloud A app restore-check"
31:rclone copy "$RB" "$ROOT/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >"${LOGBASE}-b.log" 2>&1
33:(cd "$ROOT/b" && sha256sum -c checksums.sha256 >"${LOGBASE}-b-sha.log" && ARCHIVE="$(jq -r ".archive_name" manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null)
34:echo "OK cloud B app restore-check"
35:rm -rf "$ROOT"
36:rmdir /var/tmp/homelab-app-restore-check 2>/dev/null || true
37:rm -f "${LOGBASE}-a.log" "${LOGBASE}-a-sha.log" "${LOGBASE}-b.log" "${LOGBASE}-b-sha.log"
39:echo "OK app restore-check both clouds"
----- /usr/local/sbin/appbackup-retention -----
32:trap "rm -rf \"$WORK\"" EXIT
35:  rclone lsf "${REMOTE%/}" --dirs-only --max-depth 1 > "$ROOT_LIST"
36:  rclone lsf "${REMOTE%/}/${NODE}/apps/${CANONICAL}" --dirs-only --max-depth 1 > "$LEGACY_LIST"
39:    rclone lsf "${REMOTE%/}/hb" --dirs-only --max-depth 1 > "$HB_LIST"
40:    if grep -Fxq "a/" "$HB_LIST"; then rclone lsf "${REMOTE%/}/hb/a" --dirs-only --max-depth 1 > "$NEW_LIST"; fi
73:    rclone cat "${REMOTE%/}/${LATEST_REL}/manifest.json" | jq -e --arg job "$LATEST" ".job_id == \$job" >/dev/null || { echo "REFUSE latest missing/bad remote_index=$INDEX"; exit 5; }
78:  if (( TOTAL <= KEEP )); then echo "nothing to delete"; continue; fi
83:    [[ -n "$LATEST_REL" && "$REL" == "$LATEST_REL" ]] && { echo "REFUSE delete latest ${REL}"; exit 7; }
85:    if [[ "$MODE" == apply ]]; then echo "DELETE schema=dual path=$REL"; rclone purge "$TARGET"; else echo "WOULD_DELETE schema=dual path=$REL"; fi
----- /usr/local/sbin/appbackup-retention.bak-dual-read-20260711T051924Z -----
15:for REMOTE in "$REMOTE_A" "$REMOTE_B"; do BASE="${REMOTE%/}/${NODE}/apps/${APP}"; echo "--- $BASE ---"; if [[ "$MODE" == "apply" ]]; then rclone cat "$BASE/$LATEST/manifest.json" | jq -e --arg job "$LATEST" ".job_id == \$job" >/dev/null || { echo "REFUSE latest missing/bad on $BASE"; exit 4; }; fi; mapfile -t DIRS < <(rclone lsf "$BASE" --dirs-only 2>/dev/null | while read -r D; do D="${D%/}"; [[ "$D" == homelab-${NODE}-app-${APP}-* ]] && echo "$D"; done | sort); TOTAL="${#DIRS[@]}"; echo "total=$TOTAL keep=$KEEP mode=$MODE latest=${LATEST:-none}"; if (( TOTAL <= KEEP )); then echo "nothing to delete"; continue; fi; DELETE_COUNT=$((TOTAL-KEEP)); for ((i=0; i<DELETE_COUNT; i++)); do TARGET_NAME="${DIRS[$i]}"; TARGET="$BASE/$TARGET_NAME"; [[ -n "$LATEST" && "$TARGET_NAME" == "$LATEST" ]] && { echo "REFUSE delete latest $TARGET"; exit 5; }; if [[ "$MODE" == "apply" ]]; then echo "DELETE $TARGET"; rclone purge "$TARGET"; else echo "WOULD_DELETE $TARGET"; fi; done; done
----- /usr/local/sbin/backupctl -----
12:  check) bash -n "$SCRIPT" && [[ -f "$STATE" ]] && jq -e ".status == \"success\" and .local_backup_deleted == true" "$STATE" >/dev/null && echo "OK backup framework" ;;
13:  restore-check) backup-restore-check ;;
14:  audit) backupctl check && backup-restore-check && backupctl retention 7 dry-run && backup-report >/dev/null && echo "OK audit finished" ;;
16:  remotes) source /etc/homelab-backup.env && NODE="$(hostname -s)" && echo "A=$REMOTE_A" && echo "B=$REMOTE_B" && rclone lsd "${REMOTE_A%/}/${NODE}/emergency-pack" | tail -n 5 && rclone lsd "${REMOTE_B%/}/${NODE}/emergency-pack" | tail -n 5 ;;
19:  *) echo "usage: backupctl run|status|lastlog|check|restore-check|audit|report|remotes|retention|timer"; exit 2 ;;
----- /usr/local/sbin/backup-restore-check -----
8:ROOT="/var/tmp/homelab-restore-check/${JOB}"
9:rm -rf "$ROOT" && mkdir -p "$ROOT/a" "$ROOT/b"
10:trap 'echo "ERROR restore-check failed; kept: "; exit 1' ERR
11:echo "restore-check job=$JOB"
12:rclone copy "$RA" "$ROOT/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-a.log 2>&1; homelab-rehydrate-chunked-dir "$ROOT/a" >>/tmp/homelab-restore-a.log 2>&1
13:(cd "$ROOT/a" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-a-sha.log && ARCHIVE="$(jq -r .archive_name manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null)
14:echo "OK cloud A restore-check"
15:rclone copy "$RB" "$ROOT/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-b.log 2>&1; homelab-rehydrate-chunked-dir "$ROOT/b" >>/tmp/homelab-restore-b.log 2>&1
16:(cd "$ROOT/b" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-b-sha.log && ARCHIVE="$(jq -r .archive_name manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null)
17:echo "OK cloud B restore-check"
18:rm -rf "$ROOT"
20:echo "OK restore-check both clouds"
----- /usr/local/sbin/backup-restore-check.bak-chunkall-20260710T011410Z -----
8:ROOT="/var/tmp/homelab-restore-check/${JOB}"
9:rm -rf "$ROOT" && mkdir -p "$ROOT/a" "$ROOT/b"
10:trap 'echo "ERROR restore-check failed; kept: "; exit 1' ERR
11:echo "restore-check job=$JOB"
12:rclone copy "$RA" "$ROOT/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-a.log 2>&1
13:(cd "$ROOT/a" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-a-sha.log && ARCHIVE="$(jq -r .archive_name manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null)
14:echo "OK cloud A restore-check"
15:rclone copy "$RB" "$ROOT/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-b.log 2>&1
16:(cd "$ROOT/b" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-b-sha.log && ARCHIVE="$(jq -r .archive_name manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null)
17:echo "OK cloud B restore-check"
18:rm -rf "$ROOT"
20:echo "OK restore-check both clouds"
----- /usr/local/sbin/backup-retention -----
18:  if [[ "$MODE" == "apply" ]]; then rclone cat "$BASE/$LATEST/manifest.json" | jq -e --arg job "$LATEST" ".job_id == \$job" >/dev/null || { echo "REFUSE latest missing/bad on $BASE"; exit 4; }; fi
19:  mapfile -t DIRS < <(rclone lsf "$BASE" --dirs-only 2>/dev/null | while read -r D; do D="${D%/}"; [[ "$D" == homelab-${NODE}-${JOB}-* ]] && echo "$D"; done | sort)
22:  if (( TOTAL <= KEEP )); then echo "nothing to delete"; continue; fi
24:  for ((i=0; i<DELETE_COUNT; i++)); do TARGET_NAME="${DIRS[$i]}"; TARGET="$BASE/$TARGET_NAME"; [[ -n "$LATEST" && "$TARGET_NAME" == "$LATEST" ]] && { echo "REFUSE delete latest $TARGET"; exit 5; }; if [[ "$MODE" == "apply" ]]; then echo "DELETE $TARGET"; rclone purge "$TARGET"; else echo "WOULD_DELETE $TARGET"; fi; done
----- /usr/local/sbin/homelab-backup-edge-vm-vzdump -----
7:BASE_DIR="/mnt/staging/vzdump/edge-vm-vzdump"
8:OFFHOST_DIR="/mnt/staging/offhost/edge-vm-vzdump-from-pve03"
40:REMOTE_OUT="$(remote "OUT=/tmp/edge_vm_vzdump_${VMID}_${TS}.out; ERR=/tmp/edge_vm_vzdump_${VMID}_${TS}.err; : > \$OUT; : > \$ERR; vzdump '$VMID' --mode snapshot --compress zstd --dumpdir '$RUN_DIR' --remove 0 >\$OUT 2>\$ERR; RC=\$?; ARCHIVE=\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\n' | sort -n | tail -n1 | cut -d' ' -f2-); LOGFILE=\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.log' -printf '%T@ %p\n' | sort -n | tail -n1 | cut -d' ' -f2-); SIZE=0; SHA=none; if [ -n \"\$ARCHIVE\" ] && [ -f \"\$ARCHIVE\" ]; then SIZE=\$(stat -c %s \"\$ARCHIVE\"); SHA=\$(sha256sum \"\$ARCHIVE\" | awk '{print \$1}'); fi; echo VZDUMP_RC=\$RC; echo ARCHIVE=\$ARCHIVE; echo LOGFILE=\$LOGFILE; echo SIZE=\$SIZE; echo SHA=\$SHA; echo OUT_TAIL=\$(tail -n 6 \$OUT | tr '\n' '|'); echo ERR_TAIL=\$(tail -n 6 \$ERR | tr '\n' '|')")"
62:dest "mkdir -p '$OFFHOST_DIR' && rm -f '$DEST_PATH.tmp' '$DEST_PATH.sha256'"
74:echo "STATUS=OK TS=$TS TYPE=edge-vm-vzdump-backup NODE=$NODE VMID=$VMID ARCHIVE=$ARCHIVE SIZE=$SIZE SHA256=$SHA SCSI1_BACKUP0=yes NOTES=timer_run" > "$HEALTH_DIR/edge-vm-vzdump-backup.txt"
75:echo "STATUS=OK TS=$TS TYPE=edge-vm-vzdump-offhost NODE=$NODE VMID=$VMID HOST=pve02 DEST=$DEST_PATH SIZE=$SIZE SHA256=$SHA NOTES=timer_run" > "$HEALTH_DIR/edge-vm-vzdump-offhost.txt"
76:echo "STATUS=OK TS=$TS TYPE=edge-vm-vzdump-restore-verify HOST=pve02 VMID=$VMID ARCHIVE=$DEST_PATH SIZE=$SIZE SHA256=$SHA ZSTD_RC=0 VMA_RC=0 NOTES=timer_run" > "$HEALTH_DIR/edge-vm-vzdump-restore.txt"
77:chmod 0644 "$HEALTH_DIR/edge-vm-vzdump-backup.txt" "$HEALTH_DIR/edge-vm-vzdump-offhost.txt" "$HEALTH_DIR/edge-vm-vzdump-restore.txt"
79:remote "find '$BASE_DIR' -mindepth 1 -maxdepth 1 -type d -name 'edge-vm-*' -printf '%T@ %p\n' | sort -nr | awk 'NR>${KEEP_LOCAL}{print \$2}' | xargs -r rm -rf"
80:dest "find '$OFFHOST_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\n' | sort -nr | awk 'NR>${KEEP_OFFHOST}{print \$2}' | while read f; do rm -f \"\$f\" \"\$f.sha256\"; done"
----- /usr/local/sbin/homelab-capacity-risk -----
32:LARGEST_BYTES="$(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-*.vma.zst" -o -name "vzdump-lxc-*.tar.zst" \) -printf "%s\n" 2>/dev/null | sort -n | tail -n1)"
----- /usr/local/sbin/homelab-chat-run -----
272:        re.compile(r"(?i)\brclone\.conf\b"),
388:    "reference_register_checked": True,
390:    "error_register_checked": True,
489:    rm -f \
----- /usr/local/sbin/homelab-edge-vm-offhost-freshness -----
9:OFFHOST_DIR=/mnt/staging/offhost/edge-vm-vzdump-from-pve03
85:  /var/lib/vz/dump/vzdump-qemu-130-*.vma.zst) ;;
96:dest_size="$(field DEST_SIZE <<<"$inventory_out")"
100:[[ "$dest_size" =~ ^[0-9]+$ && "$count" =~ ^[0-9]+$ && "$available" =~ ^[0-9]+$ ]]
103:last_oldest_size=0
107:  [[ "$dest_size" == "$SOURCE_SIZE" && "$dest_sha" == "$SOURCE_SHA" ]]
110:    "$dest_size" "$last_oldest" "$last_oldest_size" "$last_oldest_sha" "$count" "$RESERVE")"
115:  [[ "$dest_size" -ge 0 && "$dest_size" -lt "$SOURCE_SIZE" ]]
119:    last_oldest_size="$(field OLDEST_SIZE <<<"$inventory_out")"
122:    [[ "$last_oldest_size" =~ ^[0-9]+$ && "$last_oldest_size" -gt 1000000000 ]]
126:      "$dest_size" "$last_oldest" "$last_oldest_size" "$last_oldest_sha" "$before_count" "$RESERVE")"
130:    [[ "$(field DEST_SIZE <<<"$inventory_out")" == "$dest_size" ]]
136:  remaining=$((SOURCE_SIZE-dest_size))
140:  copy_out="$(run_pve03 copy "$SOURCE_PATH" "$SOURCE_SIZE" "$SOURCE_SHA" "$DST_IP" "$OFFHOST_DIR")"
141:  grep -Fq 'SOURCE_COPY_STATUS=OK' <<<"$copy_out"
146:    "$dest_size" "$last_oldest" "$last_oldest_size" "$last_oldest_sha" "$expected_final_count" "$RESERVE")"
156:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-backup NODE=%s VMID=%s ARCHIVE=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\n' \
158:  >"$HEALTH_DIR/edge-vm-vzdump-backup.txt"
159:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-offhost NODE=%s VMID=%s HOST=%s DEST=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\n' \
161:  >"$HEALTH_DIR/edge-vm-vzdump-offhost.txt"
162:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-restore-verify HOST=%s VMID=%s ARCHIVE=%s SIZE=%s SHA256=%s ZSTD_RC=0 VMA_RC=0 NOTES=nightly_existing_dump_freshness\n' \
164:  >"$HEALTH_DIR/edge-vm-vzdump-restore.txt"
167:  "$HEALTH_DIR/edge-vm-vzdump-backup.txt" \
168:  "$HEALTH_DIR/edge-vm-vzdump-offhost.txt" \
169:  "$HEALTH_DIR/edge-vm-vzdump-restore.txt"
----- /usr/local/sbin/homelab-extended-appbackup-run -----
12:run_one(){ APP="$1"; HOST="$2"; PATHS="$3"; NOTE="$4"; CONS="$5"; JOB_ID="homelab-${NODE}-app-${APP}-${RUN_TS}"; export JOB_ID; SHORT_TS="${RUN_TS:2:6}-${RUN_TS:9:6}"; export BACKUP_PATH_REL="$(homelab-backup-path job a "$APP" "$SHORT_TS")"; STAGE="${STAGING_ROOT}/${JOB_ID}"; PAYLOAD="$STAGE/payload"; ARCHIVE="${JOB_ID}.tar.gz"; LOG="/tmp/extended-appbackup-${APP}-${RUN_TS}.log"; rm -rf "$STAGE"; mkdir -p "$PAYLOAD/local"; echo "### app=$APP" >> "$REPORT"; echo "host=$HOST" >> "$REPORT"; echo "source_note=$NOTE" >> "$REPORT"; ssh -n -o BatchMode=yes -o ConnectTimeout=30 "$HOST" "sudo tar -C / --ignore-failed-read --exclude=.env --exclude=*.env --exclude=.env.* --exclude=*.bak* -czf - $PATHS" > "$PAYLOAD/local/${APP}-data.tar.gz" 2>"$LOG"; TAR_RC=$?; if [ "$TAR_RC" -eq 0 ] && tar -tzf "$PAYLOAD/local/${APP}-data.tar.gz" >/dev/null 2>&1; then SIZE="$(stat -c%s "$PAYLOAD/local/${APP}-data.tar.gz" 2>/dev/null || echo 0)"; echo "archive_payload_size_bytes=$SIZE" >> "$REPORT"; if [ "$SIZE" -le 128 ]; then echo "backup=BAD reason=archive_too_small" >> "$REPORT"; BAD=$((BAD+1)); rm -rf "$STAGE"; else sha256sum "$PAYLOAD/local/${APP}-data.tar.gz" > "$PAYLOAD/local/${APP}-data.tar.gz.sha256"; jq -n --arg type "extended-appbackup" --arg app "$APP" --arg host "$HOST" --arg source_note "$NOTE" --arg archive_payload_size_bytes "$SIZE" --arg env_files_read "no" --arg secret_values_printed "no" --arg consistency "$CONS" "{type:\$type,app_name:\$app,host:\$host,source_note:\$source_note,archive_payload_size_bytes:(\$archive_payload_size_bytes|tonumber),env_files_read:\$env_files_read,secret_values_printed:\$secret_values_printed,consistency:\$consistency}" > "$STAGE/metadata.json"; tar -C "$PAYLOAD" -czf "$STAGE/$ARCHIVE" .; rm -rf "$PAYLOAD"; JOB_ID="$JOB_ID" homelab-stage-2cloud-flat "$APP" "$STAGE" "$ARCHIVE" > "$LOG" 2>&1; RC=$?; if [ "$RC" -eq 0 ]; then echo "backup=OK" >> "$REPORT"; grep -E "^\[STEP\]|^\[SUCCESS\]" "$LOG" >> "$REPORT"; appbackupctl restore-check "$APP" >> "$REPORT" 2>&1 && echo "restore_check=OK" >> "$REPORT" || echo "restore_check=WARN_non_registry" >> "$REPORT"; OK=$((OK+1)); else echo "backup=BAD rc=$RC" >> "$REPORT"; grep -E "^\[ERROR\]|\[STEP\]|ERROR|error|failed|Failed|required|missing|denied|forbidden|checksum|rclone|copy|mkdir|jq|tar" "$LOG" | tail -n 60 >> "$REPORT"; BAD=$((BAD+1)); fi; fi; else echo "tar=BAD rc=$TAR_RC" >> "$REPORT"; grep -E "ERROR|error|failed|Failed|denied|No such|Cannot|tar:" "$LOG" | tail -n 40 >> "$REPORT"; BAD=$((BAD+1)); rm -rf "$STAGE"; fi; echo >> "$REPORT"; }
21:run_one healthchecks "$MON" "var/lib/healthchecks opt/stacks/healthchecks/local_settings.py" "healthchecks data plus local settings" "emergency_data_config_archive"
----- /usr/local/sbin/homelab-golden-state-extended-overlay -----
26:rclone copyto "$JSON" "$RA" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1))
27:rclone copyto "$JSON" "$RB" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1))
28:rclone cat "$RA" 2>/dev/null | jq -e --arg cp "$CP" ".control_plane_job == \$cp" >/dev/null || BAD=$((BAD+1))
29:rclone cat "$RB" 2>/dev/null | jq -e --arg cp "$CP" ".control_plane_job == \$cp" >/dev/null || BAD=$((BAD+1))
----- /usr/local/sbin/homelab-golden-state-index -----
28:rclone copyto "$JSON" "$RA" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1))
29:rclone copyto "$JSON" "$RB" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1))
30:rclone cat "$RA" 2>/dev/null | jq -e --arg cp "$CP" --arg ext "$EXT" ".control_plane_job == \$cp and .health.extended_appbackup == \$ext and .extended_appbackup_health == \$ext" >/dev/null || BAD=$((BAD+1))
31:rclone cat "$RB" 2>/dev/null | jq -e --arg cp "$CP" --arg ext "$EXT" ".control_plane_job == \$cp and .health.extended_appbackup == \$ext and .extended_appbackup_health == \$ext" >/dev/null || BAD=$((BAD+1))
----- /usr/local/sbin/homelab-health-metrics -----
16:category_for_check(){
22:    *backup*|*vzdump*) printf "%s" "backup" ;;
31:tier_for_check(){
34:    npmplus-kuma-config-*|authentik-*|gitea-*|vaultwarden-*|netbox-*|mail-cloud-restore-drill|edge-vm-vzdump-*|mkdocs-runtime-publish|npmplus-certificate-expiry) printf "%s" "P0" ;;
69:    check="${base%.txt}"
70:    category="$(category_for_check "$check")"
71:    tier="$(tier_for_check "$check")"
75:    check_esc="$(esc_label "$check")"
79:    labels="host=\"$host_esc\",check=\"$check_esc\",category=\"$cat_esc\",tier=\"$tier_esc\",file=\"$file_esc\""
----- /usr/local/sbin/homelab-ingress-standby-refresh -----
16:    rm -f "$TMP_STATE" "$RELAY" "$RESOURCES"
74:run_node "$SOURCE_NODE" vzdump 130 \
78:    --remove 0 \
83:"find /var/lib/vz/dump -maxdepth 1 -type f -name 'vzdump-qemu-130-*.vma.zst' -printf '%T@ %p\n' | sort -rn | head -n 1 | cut -d' ' -f2-"
106:"chattr -i '$DEST_FILE' '$DEST_FILE.sha256' '$DEST_FILE.env' 2>/dev/null || true; rm -f '$DEST_TMP'"
155:        -name 'vzdump-qemu-130-*.vma.zst' \
164:        rm -f "$OLD" "$OLD.sha256" "$OLD.env"
173:run_node "$SOURCE_NODE" rm -f "$SOURCE_BACKUP"
----- /usr/local/sbin/homelab-mail-cloud-critical-upload -----
50:EXCLUDED=edge-vm-vzdump-large-archive,rclone-config,secrets,raw-env
80:    rclone mkdir "${R}:$REMOTE_DIR" > "$WORK/logs/${SAFE}_mkdir.out" 2>"$WORK/logs/${SAFE}_mkdir.err"
82:    homelab-rclone-copy-chunked "$BUNDLE" "${R}:$REMOTE_DIR/$(basename "$BUNDLE")" > "$WORK/logs/${SAFE}_upload_bundle.out" 2>"$WORK/logs/${SAFE}_upload_bundle.err"
84:    rclone copyto "$BUNDLE_SHA" "${R}:$REMOTE_DIR/$(basename "$BUNDLE_SHA")" --checksum > "$WORK/logs/${SAFE}_upload_sha.out" 2>"$WORK/logs/${SAFE}_upload_sha.err"
86:    rclone copyto "$WORK/components/MANIFEST.txt" "${R}:$REMOTE_DIR/MANIFEST.txt" --checksum > "$WORK/logs/${SAFE}_upload_manifest.out" 2>"$WORK/logs/${SAFE}_upload_manifest.err"
88:    rclone copyto "$WORK/components/SHA256SUMS.txt" "${R}:$REMOTE_DIR/SHA256SUMS.txt" --checksum > "$WORK/logs/${SAFE}_upload_sums.out" 2>"$WORK/logs/${SAFE}_upload_sums.err"
90:    homelab-rclone-cat-chunked "${R}:$REMOTE_DIR/$(basename "$BUNDLE")" > "$WORK/download/${SAFE}-$(basename "$BUNDLE")" 2>"$WORK/logs/${SAFE}_download_bundle.err"
108:      rm -f "$WORK/download/${SAFE}-$(basename "$BUNDLE")"
109:      rclone lsf "${R}:$TARGET_BASE" --dirs-only 2>/dev/null | sed 's#/$##' | sort > "$WORK/logs/${SAFE}_dirs.txt"
114:          [ -n "$OLD" ] && rclone purge "${R}:$TARGET_BASE/$OLD" >> "$WORK/logs/${SAFE}_retention.out" 2>>"$WORK/logs/${SAFE}_retention.err"
140:      rm -rf --one-file-system "$OLD_DIR"
----- /usr/local/sbin/homelab-mail-cloud-critical-upload.bak-chunkall-20260710T011410Z -----
50:EXCLUDED=edge-vm-vzdump-large-archive,rclone-config,secrets,raw-env
80:    rclone mkdir "${R}:$REMOTE_DIR" > "$WORK/logs/${SAFE}_mkdir.out" 2>"$WORK/logs/${SAFE}_mkdir.err"
82:    rclone copyto "$BUNDLE" "${R}:$REMOTE_DIR/$(basename "$BUNDLE")" --checksum > "$WORK/logs/${SAFE}_upload_bundle.out" 2>"$WORK/logs/${SAFE}_upload_bundle.err"
84:    rclone copyto "$BUNDLE_SHA" "${R}:$REMOTE_DIR/$(basename "$BUNDLE_SHA")" --checksum > "$WORK/logs/${SAFE}_upload_sha.out" 2>"$WORK/logs/${SAFE}_upload_sha.err"
86:    rclone copyto "$WORK/components/MANIFEST.txt" "${R}:$REMOTE_DIR/MANIFEST.txt" --checksum > "$WORK/logs/${SAFE}_upload_manifest.out" 2>"$WORK/logs/${SAFE}_upload_manifest.err"
88:    rclone copyto "$WORK/components/SHA256SUMS.txt" "${R}:$REMOTE_DIR/SHA256SUMS.txt" --checksum > "$WORK/logs/${SAFE}_upload_sums.out" 2>"$WORK/logs/${SAFE}_upload_sums.err"
90:    rclone copyto "${R}:$REMOTE_DIR/$(basename "$BUNDLE")" "$WORK/download/${SAFE}-$(basename "$BUNDLE")" > "$WORK/logs/${SAFE}_download_bundle.out" 2>"$WORK/logs/${SAFE}_download_bundle.err"
108:      rm -f "$WORK/download/${SAFE}-$(basename "$BUNDLE")"
109:      rclone lsf "${R}:$TARGET_BASE" --dirs-only 2>/dev/null | sed 's#/$##' | sort > "$WORK/logs/${SAFE}_dirs.txt"
114:          [ -n "$OLD" ] && rclone purge "${R}:$TARGET_BASE/$OLD" >> "$WORK/logs/${SAFE}_retention.out" 2>>"$WORK/logs/${SAFE}_retention.err"
----- /usr/local/sbin/homelab-mail-cloud-edge-vm-upload -----
10:TARGET_BASE=full/edge-vm-vzdump
60:case "$SIZE" in ""|*[!0-9]*) fail size_bad 2 ;; esac
64:case "$SOURCE" in /mnt/staging/offhost/edge-vm-vzdump-from-pve03/vzdump-qemu-130-*.vma.zst) ;; *) fail source_path_bad 2 ;; esac
111:rclone about "$TARGET_REMOTE:" > "$WORK/logs/about.out" 2>"$WORK/logs/about.err" || ABOUT_RC=$?
114:DIRS="$(rclone lsf "$TARGET_REMOTE:$TARGET_BASE" --dirs-only 2>/dev/null | sed 's#/$##' | sort || true)"
116:  if rclone cat "$TARGET_REMOTE:$TARGET_BASE/$D/$BASENAME.sha256" 2>/dev/null | grep -q "$SHA"; then
172:P1 Mail Cloud edge-vm recurring chunked vzdump upload
188:rclone mkdir "$TARGET_REMOTE:$REMOTE_PARTS" > "$WORK/logs/mkdir.out" 2>"$WORK/logs/mkdir.err" || MKDIR_RC=$?
190:rclone copy "$PARTS_DIR" "$TARGET_REMOTE:$REMOTE_PARTS" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > "$WORK/logs/upload_parts.out" 2>"$WORK/logs/upload_parts.err" || UPLOAD_PARTS_RC=$?
192:rclone copyto "$WORK/$BASENAME.sha256" "$TARGET_REMOTE:$REMOTE_SHA" --checksum > "$WORK/logs/upload_sha.out" 2>"$WORK/logs/upload_sha.err" || UPLOAD_SHA_RC=$?
194:rclone copyto "$WORK/MANIFEST.txt" "$TARGET_REMOTE:$REMOTE_MANIFEST" --checksum > "$WORK/logs/upload_manifest.out" 2>"$WORK/logs/upload_manifest.err" || UPLOAD_MANIFEST_RC=$?
196:rclone copyto "$WORK/SHA256SUMS.txt" "$TARGET_REMOTE:$REMOTE_SUMS" --checksum > "$WORK/logs/upload_sums.out" 2>"$WORK/logs/upload_sums.err" || UPLOAD_SUMS_RC=$?
199:rclone size "$TARGET_REMOTE:$TARGET_DIR" --json > "$WORK/logs/remote_size.json" 2>"$WORK/logs/remote_size.err" || SIZE_RC=$?
201:rclone lsf "$TARGET_REMOTE:$REMOTE_PARTS" --format "ps" > "$WORK/logs/remote_lsf_parts.txt" 2>"$WORK/logs/remote_lsf_parts.err" || LSF_PARTS_RC=$?
203:REMOTE_BYTES="$(python3 -c 'import json; print(json.load(open("'"$WORK"'/logs/remote_size.json")).get("bytes",""))' 2>/dev/null || echo unknown)"
204:REMOTE_COUNT="$(python3 -c 'import json; print(json.load(open("'"$WORK"'/logs/remote_size.json")).get("count",""))' 2>/dev/null || echo unknown)"
207:rclone copy "$TARGET_REMOTE:$REMOTE_PARTS" "$DL_DIR/parts" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > "$WORK/logs/download_parts.out" 2>"$WORK/logs/download_parts.err" || DOWNLOAD_PARTS_RC=$?
222:  rm -rf "$DL_DIR" "$PARTS_DIR"
234:  rclone lsf "$TARGET_REMOTE:$TARGET_BASE" --dirs-only 2>/dev/null | sed "s#/$##" | sort > "$DIR_LIST" || RETENTION_RC=$?
239:      [ -n "$OLD" ] && rclone purge "$TARGET_REMOTE:$TARGET_BASE/$OLD" >> "$WORK/logs/retention.out" 2>>"$WORK/logs/retention.err"
----- /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload -----
8:TARGET_BASE=full/nextcloud-vm-vzdump
11:SRC_GLOB=/var/lib/vz/dump/vzdump-qemu-150-*.vma.zst
23:  local delete_count="${7:-0}"
25:  echo "STATUS=$status TS=$TS TYPE=mail-cloud-nextcloud-vm MODE=recurring-chunked REMOTE=$TARGET_REMOTE DEST=${TARGET_DIR:-none} ARCHIVE=${BASENAME:-none} SIZE=${SIZE:-0} SHA256=${SHA:-none} PART_SIZE=$PART_SIZE PARTS=${PART_COUNT:-0} REMOTE_PARTS=$remote_parts REMOTE_BYTES=$remote_bytes UPLOAD_OK=$upload_ok DOWNLOAD_VERIFY=$download_verify RETENTION_KEEP=$KEEP DELETE_COUNT=$delete_count ALREADY_UPLOADED=$already_uploaded SECRET_PRINTED=no NOTES=$notes" > "$HEALTH"
77:P1 Mail Cloud Nextcloud VM recurring chunked vzdump upload
93:rclone mkdir "$TARGET_REMOTE:$REMOTE_PARTS" > "$WORK/logs/mkdir.out" 2>"$WORK/logs/mkdir.err" || MKDIR_RC=$?
95:rclone copy "$PARTS_DIR" "$TARGET_REMOTE:$REMOTE_PARTS" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > "$WORK/logs/upload_parts.out" 2>"$WORK/logs/upload_parts.err" || UPLOAD_PARTS_RC=$?
97:rclone copyto "$WORK/$BASENAME.sha256" "$TARGET_REMOTE:$REMOTE_SHA" --checksum > "$WORK/logs/upload_sha.out" 2>"$WORK/logs/upload_sha.err" || UPLOAD_SHA_RC=$?
99:rclone copyto "$WORK/MANIFEST.txt" "$TARGET_REMOTE:$REMOTE_MANIFEST" --checksum > "$WORK/logs/upload_manifest.out" 2>"$WORK/logs/upload_manifest.err" || UPLOAD_MANIFEST_RC=$?
101:rclone copyto "$WORK/SHA256SUMS.txt" "$TARGET_REMOTE:$REMOTE_SUMS" --checksum > "$WORK/logs/upload_sums.out" 2>"$WORK/logs/upload_sums.err" || UPLOAD_SUMS_RC=$?
104:rclone lsf "$TARGET_REMOTE:$REMOTE_PARTS" --format ps > "$WORK/logs/remote_lsf_parts.txt" 2>"$WORK/logs/remote_lsf_parts.err" || LSF_PARTS_RC=$?
110:rclone copy "$TARGET_REMOTE:$REMOTE_PARTS" "$DL_DIR/parts" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > "$WORK/logs/download_parts.out" 2>"$WORK/logs/download_parts.err" || DOWNLOAD_PARTS_RC=$?
120:rclone lsf "$TARGET_REMOTE:$TARGET_BASE" --dirs-only 2>/dev/null | sed 's#/$##' | sort > "$DIR_LIST" || RETENTION_RC=$?
126:    [ -n "$OLD" ] && rclone purge "$TARGET_REMOTE:$TARGET_BASE/$OLD" >> "$WORK/logs/retention.out" 2>>"$WORK/logs/retention.err"
140:  rm -rf "$PARTS_DIR" "$DL_DIR"
----- /usr/local/sbin/homelab-mail-cloud-restore-drill -----
52:    homelab-rclone-cat-chunked "${R}:$CRIT_DEST/$CRIT_BUNDLE" > "$OUT" 2>"$WORK/logs/${SAFE}_crit_download.err"
83:      rm -f "$OUT"
108:  rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/MANIFEST.txt" "$WORK/edge/MANIFEST.txt" > "$WORK/logs/edge_manifest_download.out" 2>"$WORK/logs/edge_manifest_download.err"
111:  rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/SHA256SUMS.txt" "$WORK/edge/SHA256SUMS.txt" > "$WORK/logs/edge_sums_download.out" 2>"$WORK/logs/edge_sums_download.err"
114:  rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/$EDGE_ARCHIVE.sha256" "$WORK/edge/$EDGE_ARCHIVE.sha256" > "$WORK/logs/edge_archive_sha_download.out" 2>"$WORK/logs/edge_archive_sha_download.err"
117:  rclone lsf "${EDGE_REMOTE}:$EDGE_DEST/parts" --format ps > "$WORK/logs/edge_parts_lsf.txt" 2>"$WORK/logs/edge_parts_lsf.err"
130:  rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/parts/$FIRST_PART" "$WORK/edge/$FIRST_PART" > "$WORK/logs/edge_first_part_download.out" 2>"$WORK/logs/edge_first_part_download.err"
137:    rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/parts/$LAST_PART" "$WORK/edge/$LAST_PART" > "$WORK/logs/edge_last_part_download.out" 2>"$WORK/logs/edge_last_part_download.err"
168:    rm -f "$WORK/edge/$FIRST_PART" "$WORK/edge/$LAST_PART"
----- /usr/local/sbin/homelab-mail-cloud-restore-drill.bak-chunkall-20260710T011410Z -----
52:    rclone copyto "${R}:$CRIT_DEST/$CRIT_BUNDLE" "$OUT" > "$WORK/logs/${SAFE}_crit_download.out" 2>"$WORK/logs/${SAFE}_crit_download.err"
83:      rm -f "$OUT"
108:  rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/MANIFEST.txt" "$WORK/edge/MANIFEST.txt" > "$WORK/logs/edge_manifest_download.out" 2>"$WORK/logs/edge_manifest_download.err"
111:  rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/SHA256SUMS.txt" "$WORK/edge/SHA256SUMS.txt" > "$WORK/logs/edge_sums_download.out" 2>"$WORK/logs/edge_sums_download.err"
114:  rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/$EDGE_ARCHIVE.sha256" "$WORK/edge/$EDGE_ARCHIVE.sha256" > "$WORK/logs/edge_archive_sha_download.out" 2>"$WORK/logs/edge_archive_sha_download.err"
117:  rclone lsf "${EDGE_REMOTE}:$EDGE_DEST/parts" --format ps > "$WORK/logs/edge_parts_lsf.txt" 2>"$WORK/logs/edge_parts_lsf.err"
130:  rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/parts/$FIRST_PART" "$WORK/edge/$FIRST_PART" > "$WORK/logs/edge_first_part_download.out" 2>"$WORK/logs/edge_first_part_download.err"
137:    rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/parts/$LAST_PART" "$WORK/edge/$LAST_PART" > "$WORK/logs/edge_last_part_download.out" 2>"$WORK/logs/edge_last_part_download.err"
168:    rm -f "$WORK/edge/$FIRST_PART" "$WORK/edge/$LAST_PART"
----- /usr/local/sbin/homelab-rclone-cat-chunked -----
4:TMP_BASE=${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}
7:cleanup(){ rm -rf "$WORK"; }
10:rclone copyto "${REMOTE_FILE}.parts/MANIFEST.parts.txt" "$MAN" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR
11:while read -r PART SIZE; do case "$PART" in part-*) rclone cat "${REMOTE_FILE}.parts/$PART" ;; esac; done < "$MAN"
----- /usr/local/sbin/homelab-rclone-cat-flat-chunked -----
5:trap "rm -f \"$TMP\"" EXIT
6:rclone cat "${DEST}.parts.txt" > "$TMP"
8:while read -r PART PART_SIZE; do [[ "$PART" =~ ^part-[0-9]{4}$ && "$PART_SIZE" =~ ^[0-9]+$ ]] || { echo "BAD FLAT PART MANIFEST" >&2; exit 2; }; rclone cat "${DEST}.${PART}"; COUNT=$((COUNT+1)); done < "$TMP"
----- /usr/local/sbin/homelab-rclone-copy-chunked -----
7:TMP_BASE=${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}
13:cleanup(){ rm -rf "$WORK"; }
23:printf "name=%s\\nsize=%s\\nsha256=%s\\nchunk_size=%s\\nparts=%s\\n" "$NAME" "$SIZE" "$SHA" "$CHUNK_SIZE" "$PART_COUNT" > "$WORK/MANIFEST.txt"
24:rclone purge "${DEST}.parts" >/dev/null 2>&1 || true
25:rclone mkdir "${DEST}.parts"
26:rclone copy "$PARTS" "${DEST}.parts" --transfers 2 --checkers 4 --retries 5 --low-level-retries 20 --stats 30s --log-level INFO
27:rclone copyto "$WORK/${NAME}.sha256" "${DEST}.sha256" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO
28:rclone copyto "$WORK/MANIFEST.txt" "${DEST}.manifest.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO
29:rclone copyto "$WORK/MANIFEST.parts.txt" "${DEST}.parts/MANIFEST.parts.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO
30:REMOTE_COUNT=$(rclone lsf "${DEST}.parts" --files-only | grep -c "^part-" || true)
32:ACTUAL=$(while read -r PART SIZE_PART; do case "$PART" in part-*) rclone cat "${DEST}.parts/$PART" ;; esac; done < "$WORK/MANIFEST.parts.txt" | sha256sum | cut -d " " -f 1)
----- /usr/local/sbin/homelab-rclone-copy-flat-chunked -----
6:TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}"
12:trap "rm -rf \"$WORK\"" EXIT
21:printf "format=homelab-flat-chunks-v1\nname=%s\nsize=%s\nsha256=%s\nchunk_size=%s\nparts=%s\n" "$NAME" "$SIZE" "$SHA" "$CHUNK_SIZE" "$PART_COUNT" > "$WORK/MANIFEST.txt"
23:while read -r PART PART_SIZE; do [[ "$PART" =~ ^part-[0-9]{4}$ && "$PART_SIZE" =~ ^[0-9]+$ ]] || { echo "BAD PART MANIFEST" >&2; exit 4; }; rclone copyto "$WORK/parts/$PART" "${DEST}.${PART}" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR; REMOTE_BYTES="$(rclone size --json "${DEST}.${PART}" | jq -r ".bytes // -1")"; [[ "$REMOTE_BYTES" = "$PART_SIZE" ]] || { echo "REMOTE PART SIZE MISMATCH $PART" >&2; exit 5; }; UPLOADED=$((UPLOADED+1)); done < "$WORK/MANIFEST.parts.txt"
25:rclone copyto "$WORK/${NAME}.sha256" "${DEST}.sha256" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR
26:rclone copyto "$WORK/MANIFEST.txt" "${DEST}.manifest.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR
27:rclone copyto "$WORK/MANIFEST.parts.txt" "${DEST}.parts.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR
28:ACTUAL="$(homelab-rclone-cat-flat-chunked "$DEST" | sha256sum | awk "{print \$1}")"
----- /usr/local/sbin/homelab-rclone-crypt-healthcheck.sh -----
4:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt"
18:  "checksums/"
21:echo "=== homelab rclone crypt healthcheck: $(hostname) ==="
40:  if ! rclone listremotes | sed 's/:$//' | grep -qx "$REMOTE"; then
47:  if ! rclone about "${REMOTE}:"; then
48:    echo "FAIL: rclone about failed: $REMOTE"
54:  ROOT_LIST="$(rclone lsf "${REMOTE}:/" || true)"
76:  if rclone cat "${REMOTE}:/latest-backup-index.json" | jq -e '.schema and .remote and .status and .items' >/dev/null; then
----- /usr/local/sbin/homelab-rclone-required-check.sh -----
4:RAW_FILE="/etc/homelab/rclone/required-remotes.txt"
5:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt"
9:echo "=== homelab rclone required check: $(hostname) ==="
12:echo "=== installed rclone ==="
13:rclone version | head -5
17:CONFIGURED="$(rclone listremotes 2>/dev/null | sed 's/:$//' || true)"
19:  echo "No rclone remotes configured yet."
----- /usr/local/sbin/homelab-router-hard-power-cycle -----
12:case "$MODE" in check|cycle) ;; *) echo HARD_WRAPPER_MODE_INVALID=yes; exit 64;; esac
23:SAFE_OUT="$(printf '%s\n' "$OUT" | sed -E '/(PASSWORD|TOKEN|SECRET|PRIVATE_KEY|ROOT_PASSWORD|ENCRYPTION_KEY|COOKIE|SESSION|AUTHORIZATION|API_KEY|LOCAL_KEY|rclone[.]conf|[.]env)/Id')"
----- /usr/local/sbin/homelab-router-running-config-mail-cloud -----
62:  rm -f "$CONFIG"
66:  rm -f "$CONFIG"
73:  if ( cd "$BACKUP_ROOT" && sha256sum -c "$(basename "$SHA_FILE")" ) >"$WORK/sha-check.out" 2>"$WORK/sha-check.err"; then
79:  if rclone mkdir "$REMOTE:$DEST" >"$WORK/rclone-mkdir.out" 2>"$WORK/rclone-mkdir.err" && homelab-rclone-copy-chunked "$ARCHIVE" "$REMOTE:$DEST/$(basename "$ARCHIVE")" >"$WORK/rclone-upload.out" 2>"$WORK/rclone-upload.err" && rclone copyto "$SHA_FILE" "$REMOTE:$DEST/$(basename "$SHA_FILE")" --checksum >"$WORK/rclone-upload-sha.out" 2>"$WORK/rclone-upload-sha.err"; then
86:  if homelab-rclone-cat-chunked "$REMOTE:$DEST/$(basename "$ARCHIVE")" >"$DL" 2>"$WORK/rclone-download.err"; then
93:rclone lsf "$REMOTE:$TARGET_BASE" --dirs-only 2>/dev/null | sed 's#/$##' | sort > "$DIR_LIST" || RETENTION_RC=$?
98:    [ -n "$OLD" ] && rclone purge "$REMOTE:$TARGET_BASE/$OLD" >>"$WORK/retention.out" 2>>"$WORK/retention.err"
103:  rm -f "$old" "$old.sha256"
116:rm -rf "$WORK"
----- /usr/local/sbin/homelab-router-running-config-mail-cloud.bak-chunkall-20260710T011410Z -----
62:  rm -f "$CONFIG"
66:  rm -f "$CONFIG"
73:  if ( cd "$BACKUP_ROOT" && sha256sum -c "$(basename "$SHA_FILE")" ) >"$WORK/sha-check.out" 2>"$WORK/sha-check.err"; then
79:  if rclone mkdir "$REMOTE:$DEST" >"$WORK/rclone-mkdir.out" 2>"$WORK/rclone-mkdir.err" && rclone copyto "$ARCHIVE" "$REMOTE:$DEST/$(basename "$ARCHIVE")" --checksum >"$WORK/rclone-upload.out" 2>"$WORK/rclone-upload.err" && rclone copyto "$SHA_FILE" "$REMOTE:$DEST/$(basename "$SHA_FILE")" --checksum >"$WORK/rclone-upload-sha.out" 2>"$WORK/rclone-upload-sha.err"; then
86:  if rclone copyto "$REMOTE:$DEST/$(basename "$ARCHIVE")" "$DL" >"$WORK/rclone-download.out" 2>"$WORK/rclone-download.err"; then
93:rclone lsf "$REMOTE:$TARGET_BASE" --dirs-only 2>/dev/null | sed 's#/$##' | sort > "$DIR_LIST" || RETENTION_RC=$?
98:    [ -n "$OLD" ] && rclone purge "$REMOTE:$TARGET_BASE/$OLD" >>"$WORK/retention.out" 2>>"$WORK/retention.err"
103:  rm -f "$old" "$old.sha256"
116:rm -rf "$WORK"
----- /usr/local/sbin/homelab-secret-exposure-guard -----
18:BADFILES="$(find "$REPO" -path "$REPO/.git" -prune -o -type f \( -iname "*.key" -o -iname "*.pem" -o -iname "*.p12" -o -iname "*.pfx" -o -iname "[SENSITIVE_FILE]" -o -iname ".env" -o -iname ".env.*" \) -print 2>/dev/null)"
----- /usr/local/sbin/homelab-stage-2cloud -----
20:jq -n --arg schema "homelab-app-backup-manifest-v1" --arg job "$JOB" --arg node "$NODE" --arg app "$APP" --arg created "$(date -u +%Y%m%dT%H%M%SZ)" --arg archive "$ARCHIVE" --arg sha "$SHA" --arg ra "$RA" --arg rb "$RB" --argjson size "$SIZE" --argjson meta "$META" '$meta + {schema:$schema,job_id:$job,node:$node,app_name:$app,created_utc:$created,archive_name:$archive,archive_size_bytes:$size,archive_sha256:$sha,remote_a_job_path:$ra,remote_b_job_path:$rb,local_staging_policy:"delete_after_both_clouds_verified",verification_policy:"full_roundtrip_download_plus_sha256"}' > "$STAGE/manifest.json"
22:echo "[STEP] local checksum"; (cd "$STAGE" && sha256sum "$ARCHIVE" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e . manifest.json >/dev/null)
23:echo "[STEP] upload cloud A metadata"; rclone copy "$STAGE" "$RA" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s
24:echo "[STEP] upload cloud A archive chunks"; rclone deletefile "$RA/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RA/$ARCHIVE"
25:echo "[STEP] upload cloud B metadata"; rclone copy "$STAGE" "$RB" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s
26:echo "[STEP] upload cloud B archive chunks"; rclone deletefile "$RB/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RB/$ARCHIVE"
27:echo "[STEP] verify cloud A chunked"; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-stage-verify-chunked "$RA" "$ARCHIVE" "$VERIFY/a" "$JOB"
28:echo "[STEP] verify cloud B chunked"; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-stage-verify-chunked "$RB" "$ARCHIVE" "$VERIFY/b" "$JOB"
29:echo "[STEP] upload latest index"; rclone copyto "$STAGE/latest-backup-index.json" "$RAL"; rclone copyto "$STAGE/latest-backup-index.json" "$RBL"; rclone cat "$RAL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null; rclone cat "$RBL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null
30:jq ". + {status:\"success\",local_backup_deleted:true}" "$STAGE/manifest.json" > "$STATE_ROOT/apps/latest-${APP}.json"
31:echo "[STEP] delete local staging"; rm -rf "$STAGE" "$VERIFY"; test ! -d "$STAGE" && test ! -d "$VERIFY"
33:echo "[SUCCESS] uploaded to 2 clouds, verified from both clouds, local staging deleted"
----- /usr/local/sbin/homelab-stage-2cloud.bak-global-chunkonly-20260710T011036Z -----
20:jq -n --arg schema "homelab-app-backup-manifest-v1" --arg job "$JOB" --arg node "$NODE" --arg app "$APP" --arg created "$(date -u +%Y%m%dT%H%M%SZ)" --arg archive "$ARCHIVE" --arg sha "$SHA" --arg ra "$RA" --arg rb "$RB" --argjson size "$SIZE" --argjson meta "$META" '$meta + {schema:$schema,job_id:$job,node:$node,app_name:$app,created_utc:$created,archive_name:$archive,archive_size_bytes:$size,archive_sha256:$sha,remote_a_job_path:$ra,remote_b_job_path:$rb,local_staging_policy:"delete_after_both_clouds_verified",verification_policy:"full_roundtrip_download_plus_sha256"}' > "$STAGE/manifest.json"
22:echo "[STEP] local checksum"; (cd "$STAGE" && sha256sum "$ARCHIVE" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e . manifest.json >/dev/null)
23:echo "[STEP] upload cloud A"; rclone copy "$STAGE" "$RA" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 --stats=30s
24:echo "[STEP] upload cloud B"; rclone copy "$STAGE" "$RB" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 --stats=30s
25:echo "[STEP] verify cloud A"; rclone copy "$RA" "$VERIFY/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10; (cd "$VERIFY/a" && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null)
26:echo "[STEP] verify cloud B"; rclone copy "$RB" "$VERIFY/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10; (cd "$VERIFY/b" && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null)
27:echo "[STEP] upload latest index"; rclone copyto "$STAGE/latest-backup-index.json" "$RAL"; rclone copyto "$STAGE/latest-backup-index.json" "$RBL"; rclone cat "$RAL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null; rclone cat "$RBL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null
28:jq ". + {status:\"success\",local_backup_deleted:true}" "$STAGE/manifest.json" > "$STATE_ROOT/apps/latest-${APP}.json"
29:echo "[STEP] delete local staging"; rm -rf "$STAGE" "$VERIFY"; test ! -d "$STAGE" && test ! -d "$VERIFY"
31:echo "[SUCCESS] uploaded to 2 clouds, verified from both clouds, local staging deleted"
----- /usr/local/sbin/homelab-stage-2cloud-flat -----
31:jq -n --arg schema "homelab-app-backup-manifest-v2-flat" --arg layout "flat-v1" --arg job "$JOB" --arg node "$NODE" --arg app "$APP" --arg created "$CREATED" --arg archive "$ARCHIVE" --arg sha "$SHA" --arg ra "$RA" --arg rb "$RB" --arg rel "$REL" --argjson size "$SIZE" --argjson meta "$META" "\$meta + {schema:\$schema,layout:\$layout,job_id:\$job,node:\$node,app_name:\$app,created_utc:\$created,archive_name:\$archive,archive_size_bytes:\$size,archive_sha256:\$sha,path_rel:\$rel,remote_a_job_path:\$ra,remote_b_job_path:\$rb,local_staging_policy:\"delete_after_both_clouds_verified\",verification_policy:\"flat_chunks_full_roundtrip_sha256\"}" > "$STAGE/manifest.json"
33:(cd "$STAGE"; FILES=("$ARCHIVE" manifest.json latest-backup-index.json); [[ -f metadata.json ]] && FILES+=(metadata.json); sha256sum "${FILES[@]}" > checksums.sha256; sha256sum -c checksums.sha256 >/dev/null; tar -tzf "$ARCHIVE" >/dev/null; jq -e . manifest.json >/dev/null)
36:  if rclone lsf "$REMOTE_DIR" --max-depth 1 2>/dev/null | grep -q .; then echo "REFUSE NONEMPTY CANARY TARGET $REMOTE_DIR" >&2; exit 6; fi
37:  rclone mkdir "$REMOTE_DIR"
38:  for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f "$STAGE/$F" ]] && rclone copyto "$STAGE/$F" "$REMOTE_DIR/$F" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done
39:  CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-flat-chunked "$STAGE/$ARCHIVE" "$REMOTE_DIR/$ARCHIVE"
41:  rclone copyto "$STAGE/latest-backup-index.json" "$LATEST_FILE" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR
42:  rclone cat "$LATEST_FILE" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null
46:jq ". + {status:\"success\",local_backup_deleted:true}" "$STAGE/manifest.json" > "$STATE_ROOT/apps/latest-${APP}.json"
47:rm -rf "$STAGE" "$VERIFY"
----- /usr/local/sbin/homelab-stage-2cloud-flat-canary -----
29:jq -n --arg schema "homelab-app-backup-manifest-v2-flat" --arg layout "flat-v1" --arg job "$JOB" --arg node "$NODE" --arg app "$APP" --arg created "$CREATED" --arg archive "$ARCHIVE" --arg sha "$SHA" --arg ra "$RA" --arg rb "$RB" --arg rel "$REL" --argjson size "$SIZE" --argjson meta "$META" "\$meta + {schema:\$schema,layout:\$layout,job_id:\$job,node:\$node,app_name:\$app,created_utc:\$created,archive_name:\$archive,archive_size_bytes:\$size,archive_sha256:\$sha,path_rel:\$rel,remote_a_job_path:\$ra,remote_b_job_path:\$rb,local_staging_policy:\"delete_after_both_clouds_verified\",verification_policy:\"flat_chunks_full_roundtrip_sha256\"}" > "$STAGE/manifest.json"
31:(cd "$STAGE"; FILES=("$ARCHIVE" manifest.json latest-backup-index.json); [[ -f metadata.json ]] && FILES+=(metadata.json); sha256sum "${FILES[@]}" > checksums.sha256; sha256sum -c checksums.sha256 >/dev/null; tar -tzf "$ARCHIVE" >/dev/null; jq -e . manifest.json >/dev/null)
34:  if rclone lsf "$REMOTE_DIR" --max-depth 1 2>/dev/null | grep -q .; then echo "REFUSE NONEMPTY CANARY TARGET $REMOTE_DIR" >&2; exit 6; fi
35:  rclone mkdir "$REMOTE_DIR"
36:  for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f "$STAGE/$F" ]] && rclone copyto "$STAGE/$F" "$REMOTE_DIR/$F" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done
37:  CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-flat-chunked "$STAGE/$ARCHIVE" "$REMOTE_DIR/$ARCHIVE"
39:  rclone copyto "$STAGE/latest-backup-index.json" "$LATEST_FILE" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR
40:  rclone cat "$LATEST_FILE" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null
44:jq ". + {status:\"success\",local_backup_deleted:true}" "$STAGE/manifest.json" > "$STATE_ROOT/apps/latest-${APP}.json"
45:rm -rf "$STAGE" "$VERIFY"
----- /usr/local/sbin/homelab-stage-verify-chunked -----
7:rm -rf "$VERIFY"
9:rclone copy "$REMOTE_DIR" "$VERIFY" --exclude "$ARCHIVE.parts/**" --exclude "$ARCHIVE.parts/" --exclude "$ARCHIVE.sha256" --exclude "$ARCHIVE.manifest.txt" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --log-level ERROR
13:if [ -s checksums.sha256 ]; then grep -F -v "  $ARCHIVE" checksums.sha256 > .checksums-no-archive.sha256 || true; if [ -s .checksums-no-archive.sha256 ]; then sha256sum -c .checksums-no-archive.sha256; fi; EXPECTED=$(grep -F "  $ARCHIVE" checksums.sha256 | awk '{print $1; exit}'); else EXPECTED=$(rclone cat "${REMOTE_DIR}/${ARCHIVE}.sha256" | awk '{print $1; exit}'); fi
15:ACTUAL=$(homelab-rclone-cat-chunked "${REMOTE_DIR}/${ARCHIVE}" | sha256sum | awk '{print $1}')
17:homelab-rclone-cat-chunked "${REMOTE_DIR}/${ARCHIVE}" | tar -tzf - >/dev/null
----- /usr/local/sbin/homelab-stage-verify-flat-chunked -----
7:rm -rf "$VERIFY"
9:rclone copy "$REMOTE_DIR" "$VERIFY" --transfers 4 --checkers 8 --retries 5 --low-level-retries 20 --log-level ERROR
11:(cd "$VERIFY" && test -s manifest.json && if [[ -n "$JOB" ]]; then jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null; fi && sha256sum -c checksums.sha256 >/dev/null && tar -tzf "$ARCHIVE" >/dev/null)
----- /usr/local/sbin/homelab-vm160-remote-backup -----
8:PARENT=/mnt/staging/vzdump
22:SOURCE_ACTIVE="$(ssh "${SSH[@]}" "$SOURCE" "pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'")" || SOURCE_ACTIVE_RC=$?
27:NAME="vzdump-qemu-${VMID}-${TS}.vma.zst"
35:    ssh "${SSH[@]}" "$TARGET" "rm -rf -- '$DIR'" >/dev/null 2>&1 || true
41:  rm -f "$LOG"
46:LATEST_SIZE="$(ssh "${SSH[@]}" "$TARGET" "find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\n' | sort -nr | head -n1")"
55:ssh "${SSH[@]}" "$SOURCE" "vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7" 2>"$LOG" | ssh "${SSH[@]}" "$TARGET" "cat > '$PART'"
57:ssh "${SSH[@]}" "$TARGET" "cat > '$DIR/vzdump.log'" < "$LOG"
66:rm -f "$LOG"
----- /usr/local/sbin/homelab-vm-backup-health -----
8:check_file(){
15:check_file /var/lib/homelab-health/backup.txt auto-vmct
16:check_file /var/lib/homelab-health/edge-vm-vzdump-backup.txt edge-vm-backup
17:check_file /var/lib/homelab-health/edge-vm-vzdump-offhost.txt edge-vm-offhost
18:check_file /var/lib/homelab-health/edge-vm-vzdump-restore.txt edge-vm-restore
----- /usr/local/sbin/homelab-vm-backup-policy -----
29:echo "## Latest local vzdump catalog" >> "$REPORT"
30:printf "%s\t%s\t%s\t%s\t%s\t%s\n" "kind" "vmid" "name" "size_bytes" "mtime_utc" "path" > "$CATALOG"
31:for f in $(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-*.vma.zst" -o -name "vzdump-lxc-*.tar.zst" \) 2>/dev/null | sort); do b="$(basename "$f")"; size="$(stat -c %s "$f" 2>/dev/null || echo 0)"; mt="$(date -u -d "@$(stat -c %Y "$f" 2>/dev/null || echo 0)" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)"; if echo "$b" | grep -q "^vzdump-qemu-"; then kind=qemu; vmid="$(echo "$b" | sed -n "s/^vzdump-qemu-\([0-9]\+\)-.*/\1/p")"; name="$(qm config "$vmid" 2>/dev/null | awk -F": " "/^name:/{print \$2; exit}")"; else kind=lxc; vmid="$(echo "$b" | sed -n "s/^vzdump-lxc-\([0-9]\+\)-.*/\1/p")"; name="$(pct config "$vmid" 2>/dev/null | awk -F": " "/^hostname:/{print \$2; exit}")"; fi; printf "%s\t%s\t%s\t%s\t%s\t%s\n" "$kind" "$vmid" "${name:-unknown}" "$size" "$mt" "$f" >> "$CATALOG"; done
37:echo "- Existing local dumps can be uploaded/verified to two clouds without creating new vzdump archives." >> "$REPORT"
38:echo "- New full vzdump for VM 170/171 is not started automatically because each VM has 120 GB bootdisk and /var/tmp has limited free space." >> "$REPORT"
42:echo "## Coverage checks" >> "$REPORT"
----- /usr/local/sbin/homelab-vm-full-backup -----
11:# shellcheck disable=SC1090
92:for cmd in vzdump qm zstd sha256sum rclone rsync ssh flock; do
130:            -name "vzdump-qemu-${VMID}-*.vma.zst" \
138:    vzdump "$VMID" \
145:        --remove 0
151:            -name "vzdump-qemu-${VMID}-*.vma.zst" \
261:        CHUNK_TMP_BASE="$BACKUP_DIR/.rclone-chunks" homelab-rclone-copy-chunked "$ARCHIVE" "${DEST_ROOT}/${NAME}"
----- /usr/local/sbin/homelab-vm-full-backup.bak-chunkall-20260710T011410Z -----
11:# shellcheck disable=SC1090
92:for cmd in vzdump qm zstd sha256sum rclone rsync ssh flock; do
130:            -name "vzdump-qemu-${VMID}-*.vma.zst" \
138:    vzdump "$VMID" \
145:        --remove 0
151:            -name "vzdump-qemu-${VMID}-*.vma.zst" \
264:        rclone copyto \
267:            --check-first \
269:            --checkers 2 \
275:        rclone copyto \
278:            --check-first \
280:            --checkers 2 \
285:        echo "Crypt-checking $NAME on $REMOTE"
287:        rclone cryptcheck \
292:            --checkers 1 \
----- /usr/local/sbin/homelab-vm-full-restore-drill -----
77:        -name "vzdump-qemu-${source_vmid}-*.vma.zst" \
152:check_inventory() {
184:check_container_health() {
224:check_http() {
349:check_container_health() {
386:check_container_health "$test_vmid"
389:    check_inventory "$source_vmid" "$test_vmid"
392:    check_http "$source_vmid" "$test_vmid"
----- /usr/local/sbin/homelab-vm-full-restore-drill.bak.20260710T080231Z -----
77:        -name "vzdump-qemu-${source_vmid}-*.vma.zst" \
152:check_inventory() {
184:check_container_health() {
224:check_http() {
349:check_container_health() {
386:check_container_health "$test_vmid"
389:    check_inventory "$source_vmid" "$test_vmid"
392:    check_http "$source_vmid" "$test_vmid"
----- /usr/local/sbin/homelab-vm-full-restore-drill.bak-docker-check-v2-20260710T050044Z -----
77:        -name "vzdump-qemu-${source_vmid}-*.vma.zst" \
152:check_inventory() {
184:check_container_health() {
224:check_http() {
347:    check_container_health "$test_vmid"
350:    check_inventory "$source_vmid" "$test_vmid"
353:    check_http "$source_vmid" "$test_vmid"
----- /usr/local/sbin/homelab-vm-full-restore-drill.bak-isolated-ip-20260710T044331Z -----
77:        -name "vzdump-qemu-${source_vmid}-*.vma.zst" \
152:check_inventory() {
184:check_container_health() {
224:check_http() {
337:    check_container_health "$test_vmid"
340:    check_inventory "$source_vmid" "$test_vmid"
343:    check_http "$source_vmid" "$test_vmid"
----- /usr/local/sbin/homelab-vm-full-retention -----
21:                -name "vzdump-qemu-${vmid}-*.vma.zst" \
33:                vzdump-qemu-"$vmid"-*.vma.zst)
35:                    rm -f \
73:                         rm -rf -- \"\$BASE/\$setname\"
91:        rclone lsf "$path" \
105:        rclone purge "${path}/${dir}"
----- /usr/local/sbin/homelab-vm-local-dumps-2cloud -----
30:CHKS="$WORK/checksums.txt"
39:echo "chunk_size_bytes=$CHUNK_SIZE_BYTES" >> "$LOG"
42:  F="$(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-${id}-*.vma.zst" -o -name "vzdump-lxc-${id}-*.tar.zst" \) -printf "%T@ %p\n" 2>/dev/null | sort -n | tail -n 1 | cut -d" " -f2-)"
53:  echo "$BN" | grep -q "^vzdump-qemu-" && KIND="qemu"
54:  echo "$BN" | grep -q "^vzdump-lxc-" && KIND="lxc"
64:    rm -rf "$CDIR"
66:    echo "SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR" >> "$LOG"
83:job,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:]
91:            "size_bytes":int(p[2]),
107:    "chunk_size_bytes":int(chunk_size),
117:  rclone mkdir "$DEST/files" >> "$LOG" 2>&1 || BAD=$((BAD+1))
118:  while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do
121:      echo "UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts" >> "$LOG"
122:      rclone mkdir "$DEST/files/$bn.chunks" >> "$LOG" 2>&1 || BAD=$((BAD+1))
126:        rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
128:      rclone copyto "$CDIR/chunks.sha256" "$DEST/files/$bn.chunks/chunks.sha256" >> "$LOG" 2>&1 || BAD=$((BAD+1))
130:      echo "UPLOAD_DIRECT dest=$DEST file=$bn size=$size" >> "$LOG"
131:      rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
135:  rclone copyto "$CHKS" "$DEST/checksums.txt" >> "$LOG" 2>&1 || BAD=$((BAD+1))
136:  rclone copyto "$FILES_TSV" "$DEST/files.tsv" >> "$LOG" 2>&1 || BAD=$((BAD+1))
137:  rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
144:  rm -rf "$VBASE"
146:  rclone copyto "$DEST/manifest.json" "$VBASE/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
149:  while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do
154:      rclone copy "$DEST/files/$bn.chunks" "$VDIR" --filter "+ part-*" --filter "+ chunks.sha256" --filter "- *" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
163:      rm -rf "$VDIR"
166:      rclone copyto "$DEST/files/$bn" "$VBASE/$bn" --transfers 1 >> "$LOG" 2>&1 || BAD=$((BAD+1))
168:      rm -f "$VBASE/$bn"
171:  rm -rf "$VBASE"
184:rm -rf "$VERIFY_ROOT/$JOB_ID"
185:rm -rf "$CHUNK_ROOT/$JOB_ID"
198:  --argjson chunk_size_bytes "$CHUNK_SIZE_BYTES" \
200:  "{job_id:\$job_id,status:\$status,node:\$node,remote_a:\$remote_a,remote_b:\$remote_b,count:\$count,total_bytes:\$total_bytes,chunked_count:\$chunked_count,chunk_size_bytes:\$chunk_size_bytes,log:\$log,local_dumps_deleted:false}" \
----- /usr/local/sbin/homelab-vm-local-dumps-retention -----
32:  mapfile -t JOBS < <(rclone lsf "$BASE/" --dirs-only 2>/dev/null | sed 's#/$##' | sort)
37:    echo "nothing_to_delete" >> "$REPORT"
56:        rclone purge "$PATH_REMOTE" >> "$REPORT" 2>&1 && DELETED=$((DELETED+1)) || BAD=$((BAD+1))
----- /usr/local/sbin/skladchik-reports-monitor-backup -----
28:command -v rclone >/dev/null
29:[ -x /usr/local/sbin/homelab-rclone-copy-chunked ] || { echo "missing /usr/local/sbin/homelab-rclone-copy-chunked" >&2; exit 10; }
36:cleanup(){ rm -rf "$TMP"; }
67:size_bytes=${SIZE}
71:log "LOCAL_ARCHIVE_OK size=$SIZE path=$ARCHIVE"
75:  /usr/local/sbin/homelab-rclone-copy-chunked "$ARCHIVE" "${DEST_BASE}/$(basename "$ARCHIVE")"
76:  /usr/local/sbin/homelab-rclone-copy-chunked "${ARCHIVE}.sha256" "${DEST_BASE}/$(basename "$ARCHIVE").sha256"
77:  /usr/local/sbin/homelab-rclone-copy-chunked "${OUTDIR}/manifest.txt" "${DEST_BASE}/manifest.txt"
82:  mapfile -t OLD < <(rclone lsf --dirs-only "${REMOTE}${REMOTE_BASE}/daily/" 2>/dev/null | sed 's#/$##' | sort | head -n -"${KEEP_DAILY}" || true)
83:  for D in "${OLD[@]}"; do [ -n "$D" ] && rclone purge "${REMOTE}${REMOTE_BASE}/daily/${D}" >/dev/null 2>&1 || true; done
85:find "${LOCAL_ROOT}/daily" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_DAILY}" | while read -r D; do [ -n "$D" ] && rm -rf "${LOCAL_ROOT}/daily/${D}"; done
----- /usr/local/sbin/skladchik-reports-monitor-backup.pre-restore-check-20260710T065311Z.bak -----
28:command -v rclone >/dev/null
29:[ -x /usr/local/sbin/homelab-rclone-copy-chunked ] || { echo "missing /usr/local/sbin/homelab-rclone-copy-chunked" >&2; exit 10; }
36:cleanup(){ rm -rf "$TMP"; }
66:size_bytes=${SIZE}
70:log "LOCAL_ARCHIVE_OK size=$SIZE path=$ARCHIVE"
74:  /usr/local/sbin/homelab-rclone-copy-chunked "$ARCHIVE" "${DEST_BASE}/$(basename "$ARCHIVE")"
75:  /usr/local/sbin/homelab-rclone-copy-chunked "${ARCHIVE}.sha256" "${DEST_BASE}/$(basename "$ARCHIVE").sha256"
76:  /usr/local/sbin/homelab-rclone-copy-chunked "${OUTDIR}/manifest.txt" "${DEST_BASE}/manifest.txt"
81:  mapfile -t OLD < <(rclone lsf --dirs-only "${REMOTE}${REMOTE_BASE}/daily/" 2>/dev/null | sed 's#/$##' | sort | head -n -"${KEEP_DAILY}" || true)
82:  for D in "${OLD[@]}"; do [ -n "$D" ] && rclone purge "${REMOTE}${REMOTE_BASE}/daily/${D}" >/dev/null 2>&1 || true; done
84:find "${LOCAL_ROOT}/daily" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_DAILY}" | while read -r D; do [ -n "$D" ] && rm -rf "${LOCAL_ROOT}/daily/${D}"; done
----- /usr/local/sbin/skladchik-reports-monitor-backup.pre-session-keepalive-20260710T071605Z.bak -----
28:command -v rclone >/dev/null
29:[ -x /usr/local/sbin/homelab-rclone-copy-chunked ] || { echo "missing /usr/local/sbin/homelab-rclone-copy-chunked" >&2; exit 10; }
36:cleanup(){ rm -rf "$TMP"; }
64:size_bytes=${SIZE}
68:log "LOCAL_ARCHIVE_OK size=$SIZE path=$ARCHIVE"
72:  /usr/local/sbin/homelab-rclone-copy-chunked "$ARCHIVE" "${DEST_BASE}/$(basename "$ARCHIVE")"
73:  /usr/local/sbin/homelab-rclone-copy-chunked "${ARCHIVE}.sha256" "${DEST_BASE}/$(basename "$ARCHIVE").sha256"
74:  /usr/local/sbin/homelab-rclone-copy-chunked "${OUTDIR}/manifest.txt" "${DEST_BASE}/manifest.txt"
79:  mapfile -t OLD < <(rclone lsf --dirs-only "${REMOTE}${REMOTE_BASE}/daily/" 2>/dev/null | sed 's#/$##' | sort | head -n -"${KEEP_DAILY}" || true)
80:  for D in "${OLD[@]}"; do [ -n "$D" ] && rclone purge "${REMOTE}${REMOTE_BASE}/daily/${D}" >/dev/null 2>&1 || true; done
82:find "${LOCAL_ROOT}/daily" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_DAILY}" | while read -r D; do [ -n "$D" ] && rm -rf "${LOCAL_ROOT}/daily/${D}"; done
----- /usr/local/sbin/skladchik-reports-monitor-backup-restore-check -----
4:CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf
9:: "${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}"
12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt
13:LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock
19:log(){ printf '%s RESTORE_CHECK %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" | tee -a "$LOGDIR/restore-check.log"; }
22:command -v rclone >/dev/null
28:  if rclone lsf "${remote_path}.parts/" > "$plist" 2>/dev/null && grep -q '^part-' "$plist"; then
29:    while IFS= read -r part; do [ -n "$part" ] && rclone cat "${remote_path}.parts/${part}" >> "$out"; done < <(grep '^part-' "$plist" | sort)
31:    rclone copyto "$remote_path" "$out" >/dev/null
33:  rm -f "$plist"
36:check_one_remote(){
39:  latest="$(rclone lsf --dirs-only "${rbase}/" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)"
85:    [ "$(sqlite3 "$extract/var-lib-skladchik-reports-monitor/state.sqlite" 'PRAGMA integrity_check;' | tail -n 1)" = "ok" ]
97:check_one_remote CLOUD_1 "$CLOUD_1"
98:check_one_remote CLOUD_2 "$CLOUD_2"
101:find "$WORK_ROOT" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_LOCAL}" | while read -r D; do [ -n "$D" ] && rm -rf "${WORK_ROOT}/${D}"; done
----- /usr/local/sbin/skladchik-reports-monitor-backup-restore-check.pre-session-keepalive-20260710T071605Z.bak -----
4:CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf
9:: "${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}"
12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt
13:LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock
19:log(){ printf '%s RESTORE_CHECK %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" | tee -a "$LOGDIR/restore-check.log"; }
22:command -v rclone >/dev/null
28:  if rclone lsf "${remote_path}.parts/" > "$plist" 2>/dev/null && grep -q '^part-' "$plist"; then
29:    while IFS= read -r part; do [ -n "$part" ] && rclone cat "${remote_path}.parts/${part}" >> "$out"; done < <(grep '^part-' "$plist" | sort)
31:    rclone copyto "$remote_path" "$out" >/dev/null
33:  rm -f "$plist"
36:check_one_remote(){
39:  latest="$(rclone lsf --dirs-only "${rbase}/" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)"
72:    [ "$(sqlite3 "$extract/var-lib-skladchik-reports-monitor/state.sqlite" 'PRAGMA integrity_check;' | tail -n 1)" = "ok" ]
84:check_one_remote CLOUD_1 "$CLOUD_1"
85:check_one_remote CLOUD_2 "$CLOUD_2"
88:find "$WORK_ROOT" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_LOCAL}" | while read -r D; do [ -n "$D" ] && rm -rf "${WORK_ROOT}/${D}"; done

OUTPUT_END
CHAT_OUTPUT_END