homelab-runtime-publisher
2329 lines
349 KiB
Plaintext
2329 lines
349 KiB
Plaintext
CHAT_OUTPUT_BEGIN
|
|
COMMAND_ID=BACKUP-REMEDIATION-PLAN-068
|
|
STATUS=OK
|
|
RC=0
|
|
HOST=pve01
|
|
MODE=read-only
|
|
COMPONENT=backup-audit
|
|
REFERENCE_REGISTER_CHECK=OK
|
|
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
|
|
ERROR_REGISTER_CHECK=OK
|
|
ERROR_REGISTER_SHA256=f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1
|
|
COMMAND_SHA256=9d10d1dc1a1c86fff398eeee0cce63e8603d904cfeae85ab30889b3ce4b3d365
|
|
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
|
EXECUTION_STARTED=true
|
|
CHANGES_MADE=false
|
|
SANITIZED=yes
|
|
SECRETS_INCLUDED=no
|
|
PRIVATE_ADDRESSES_INCLUDED=no
|
|
RAW_EVIDENCE_SHA256=b9c60641fc3a6b9d066c7842ca5efffe88376fd789a670b7962390be23670f6b
|
|
SANITIZED_OUTPUT_SHA256=e8e42de7ad82ca585b60bce951ddc7f139fc66209ab685181f46be6d3d1d7f81
|
|
OUTPUT_BEGIN
|
|
========== BACKUP-CLUSTER-13CLOUD-TOPOLOGY-062 ==========
|
|
[{"compress":"zstd","enabled":1,"id":"homelab-nightly-all","mailnotification":"failure","mode":"snapshot","next-run":1784680200,"prune-backups":{"keep-daily":"4"},"schedule":"03:30","storage":"local","type":"vzdump","vmid":"110,111,112,113,130,150,190"}]
|
|
========== NODE=pve01 ==========
|
|
=== RCLONE_REMOTES ===
|
|
NODE_REMOTE_COUNT=26
|
|
REMOTE_HEALTH node=pve01 remote=pve01-mail-01: status=OK
|
|
=== BACKUP_TIMERS ===
|
|
Wed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:14 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service
|
|
Wed 2026-07-22 00:09:43 MSK 9h Tue 2026-07-21 00:09:32 MSK 14h ago homelab-backup-freshness.timer homelab-backup-freshness.service
|
|
Wed 2026-07-22 02:15:00 MSK 11h Tue 2026-07-21 02:15:05 MSK 12h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service
|
|
Wed 2026-07-22 03:10:02 MSK 12h Tue 2026-07-21 03:13:14 MSK 11h ago homelab-emergency-backup.timer homelab-emergency-backup.service
|
|
Wed 2026-07-22 03:35:00 MSK 12h Tue 2026-07-21 03:35:14 MSK 11h ago filebrowser-backup.timer filebrowser-backup.service
|
|
Wed 2026-07-22 03:55:02 MSK 13h Tue 2026-07-21 03:51:25 MSK 10h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service
|
|
Wed 2026-07-22 04:17:29 MSK 13h Tue 2026-07-21 04:19:17 MSK 10h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service
|
|
Wed 2026-07-22 04:26:54 MSK 13h Tue 2026-07-21 04:28:14 MSK 10h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service
|
|
Wed 2026-07-22 04:38:04 MSK 13h Tue 2026-07-21 04:34:48 MSK 10h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service
|
|
Wed 2026-07-22 05:01:44 MSK 14h Tue 2026-07-21 04:57:14 MSK 9h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service
|
|
Wed 2026-07-22 05:23:56 MSK 14h Tue 2026-07-21 05:17:57 MSK 9h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service
|
|
Wed 2026-07-22 05:27:13 MSK 14h Tue 2026-07-21 05:29:05 MSK 9h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service
|
|
Wed 2026-07-22 05:47:19 MSK 14h Tue 2026-07-21 05:46:14 MSK 9h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service
|
|
Wed 2026-07-22 06:01:19 MSK 15h Tue 2026-07-21 05:57:30 MSK 8h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service
|
|
Wed 2026-07-22 06:27:43 MSK 15h Tue 2026-07-21 06:32:25 MSK 8h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service
|
|
Wed 2026-07-22 06:38:51 MSK 15h Tue 2026-07-21 06:43:54 MSK 8h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service
|
|
Wed 2026-07-22 06:39:22 MSK 15h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service
|
|
Wed 2026-07-22 06:44:16 MSK 15h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service
|
|
Wed 2026-07-22 06:52:06 MSK 16h Tue 2026-07-21 06:47:47 MSK 8h ago homelab-beszel-backup.timer homelab-beszel-backup.service
|
|
Wed 2026-07-22 06:58:58 MSK 16h Tue 2026-07-21 06:57:30 MSK 7h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service
|
|
Wed 2026-07-22 07:02:27 MSK 16h Tue 2026-07-21 07:05:30 MSK 7h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service
|
|
Wed 2026-07-22 07:07:14 MSK 16h Tue 2026-07-21 07:18:55 MSK 7h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service
|
|
Wed 2026-07-22 07:35:06 MSK 16h Tue 2026-07-21 07:36:59 MSK 7h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service
|
|
Wed 2026-07-22 07:38:21 MSK 16h Tue 2026-07-21 07:36:14 MSK 7h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service
|
|
Wed 2026-07-22 07:41:49 MSK 16h Tue 2026-07-21 07:53:41 MSK 6h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service
|
|
Wed 2026-07-22 08:10:37 MSK 17h Tue 2026-07-21 08:07:25 MSK 6h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service
|
|
Wed 2026-07-22 08:37:02 MSK 17h Tue 2026-07-21 08:30:07 MSK 6h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service
|
|
Wed 2026-07-22 08:40:13 MSK 17h Tue 2026-07-21 08:50:14 MSK 5h 59min ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service
|
|
Wed 2026-07-22 08:53:36 MSK 18h Tue 2026-07-21 08:41:14 MSK 6h ago memos-backup.timer memos-backup.service
|
|
Sun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service
|
|
Sun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service
|
|
Sun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service
|
|
Sun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service
|
|
Mon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 7h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service
|
|
dpkg-db-backup.service static -
|
|
filebrowser-backup.service static -
|
|
homelab-alertmanager-backup.service static -
|
|
homelab-appbackup-dockge-stacks.service static -
|
|
homelab-appbackup-mariadb-batch.service static -
|
|
homelab-appbackup-npmplus-edge.service static -
|
|
homelab-appbackup-postgres-batch.service static -
|
|
homelab-appbackup-sqlite-batch.service static -
|
|
homelab-authentik-backup.service static -
|
|
homelab-auto-backup.service static -
|
|
homelab-backup-coverage-matrix.service static -
|
|
homelab-backup-freshness.service static -
|
|
homelab-backup-healthcheck.service static -
|
|
homelab-backup-matrix-refresh.service static -
|
|
homelab-backup-sla-health.service static -
|
|
homelab-beszel-backup.service static -
|
|
homelab-blackbox-exporter-backup.service static -
|
|
homelab-cluster-admin-incident-engine-backup.service static -
|
|
homelab-crypto-portfolio-backup.service static -
|
|
homelab-edge-vm-vzdump-backup.service static -
|
|
homelab-emergency-backup.service static -
|
|
homelab-extended-appbackup.service static -
|
|
homelab-gitea-backup.service static -
|
|
homelab-gitea-secondary-backup.service static -
|
|
homelab-legacy-monitoring-backups.service static -
|
|
homelab-mail-cloud-critical-upload.service static -
|
|
homelab-mail-cloud-edge-vm-upload.service static -
|
|
homelab-mail-cloud-nextcloud-vm-upload.service static -
|
|
homelab-mail-cloud-restore-drill.service static -
|
|
homelab-npmplus-kuma-config-backup.service static -
|
|
homelab-router-running-config-mail-cloud.service static -
|
|
homelab-vaultwarden-backup.service static -
|
|
homelab-verified-backup-health.service static -
|
|
homelab-vm-backup-health.service static -
|
|
homelab-vm-backup-policy.service static -
|
|
homelab-vm-full-backup.service static -
|
|
homelab-vm-local-dumps-2cloud.service static -
|
|
homelab-vm160-remote-backup.service static -
|
|
immich-media-backup.service static -
|
|
memos-backup.service static -
|
|
netbird-vps-backup.service static -
|
|
paperless-backup.service static -
|
|
skladchik-reports-monitor-backup-restore-check.service static -
|
|
skladchik-reports-monitor-backup.service static -
|
|
dpkg-db-backup.timer enabled enabled
|
|
filebrowser-backup.timer enabled enabled
|
|
homelab-alertmanager-backup.timer enabled enabled
|
|
homelab-appbackup-dockge-stacks.timer enabled enabled
|
|
homelab-appbackup-mariadb-batch.timer enabled enabled
|
|
homelab-appbackup-npmplus-edge.timer enabled enabled
|
|
homelab-appbackup-postgres-batch.timer enabled enabled
|
|
homelab-appbackup-sqlite-batch.timer enabled enabled
|
|
homelab-authentik-backup.timer disabled enabled
|
|
homelab-auto-backup.timer enabled enabled
|
|
homelab-backup-coverage-matrix.timer enabled enabled
|
|
homelab-backup-freshness.timer enabled enabled
|
|
homelab-backup-healthcheck.timer enabled enabled
|
|
homelab-backup-matrix-refresh.timer disabled enabled
|
|
homelab-backup-sla-health.timer enabled enabled
|
|
homelab-beszel-backup.timer enabled enabled
|
|
homelab-blackbox-exporter-backup.timer enabled enabled
|
|
homelab-cluster-admin-incident-engine-backup.timer enabled enabled
|
|
homelab-crypto-portfolio-backup.timer enabled enabled
|
|
homelab-edge-vm-vzdump-backup.timer disabled enabled
|
|
homelab-emergency-backup.timer enabled enabled
|
|
homelab-extended-appbackup.timer enabled enabled
|
|
homelab-gitea-backup.timer disabled enabled
|
|
homelab-gitea-secondary-backup.timer enabled enabled
|
|
homelab-legacy-monitoring-backups.timer enabled enabled
|
|
homelab-mail-cloud-critical-upload.timer enabled enabled
|
|
homelab-mail-cloud-edge-vm-upload.timer enabled enabled
|
|
homelab-mail-cloud-nextcloud-vm-upload.timer enabled enabled
|
|
homelab-mail-cloud-restore-drill.timer enabled enabled
|
|
homelab-npmplus-kuma-config-backup.timer disabled enabled
|
|
homelab-router-running-config-mail-cloud.timer enabled enabled
|
|
homelab-vaultwarden-backup.timer disabled enabled
|
|
homelab-verified-backup-health.timer enabled enabled
|
|
homelab-vm-backup-health.timer enabled enabled
|
|
homelab-vm-backup-policy.timer enabled enabled
|
|
homelab-vm-full-backup.timer enabled enabled
|
|
homelab-vm-local-dumps-2cloud.timer enabled enabled
|
|
homelab-vm160-remote-backup.timer disabled enabled
|
|
immich-media-backup.timer disabled enabled
|
|
memos-backup.timer enabled enabled
|
|
netbird-vps-backup.timer disabled enabled
|
|
paperless-backup.timer disabled enabled
|
|
skladchik-reports-monitor-backup-restore-check.timer enabled enabled
|
|
skladchik-reports-monitor-backup.timer enabled enabled
|
|
=== LOCAL_BACKUP_ARCHIVES ===
|
|
========== NODE=pve02 ==========
|
|
=== RCLONE_REMOTES ===
|
|
NODE_REMOTE_COUNT=26
|
|
REMOTE_HEALTH node=pve02 remote=pve01-mail-01: status=OK
|
|
=== BACKUP_TIMERS ===
|
|
Tue 2026-07-21 18:41:03 MSK 3h 51min Tue 2026-07-21 12:41:22 MSK 2h 7min ago forum-prod-fresh5-cloud-backup.timer forum-prod-fresh5-cloud-backup.service
|
|
Wed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:22 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service
|
|
Wed 2026-07-22 04:20:00 MSK 13h Tue 2026-07-21 04:20:12 MSK 10h ago homelab-cluster-admin-vm-mail-cloud-backup.timer homelab-cluster-admin-vm-mail-cloud-backup.service
|
|
codevipe-cloud-backup.service static -
|
|
dpkg-db-backup.service static -
|
|
forum-prod-fresh5-cloud-backup.service static -
|
|
homelab-cluster-admin-vm-mail-cloud-backup.service static -
|
|
codevipe-cloud-backup.timer disabled enabled
|
|
dpkg-db-backup.timer enabled enabled
|
|
forum-prod-fresh5-cloud-backup.timer enabled enabled
|
|
homelab-cluster-admin-vm-mail-cloud-backup.timer enabled enabled
|
|
=== LOCAL_BACKUP_ARCHIVES ===
|
|
========== NODE=pve03 ==========
|
|
=== RCLONE_REMOTES ===
|
|
NODE_REMOTE_COUNT=26
|
|
REMOTE_HEALTH node=pve03 remote=pve01-mail-01: status=OK
|
|
=== BACKUP_TIMERS ===
|
|
Wed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:10 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service
|
|
dpkg-db-backup.service static -
|
|
dpkg-db-backup.timer enabled enabled
|
|
=== LOCAL_BACKUP_ARCHIVES ===
|
|
=== CLOUD_REMOTE_SUMMARY ===
|
|
REMOTE_COUNT=MISMATCH
|
|
========== BACKUP-CLOUD-PIPELINE-MAP-063 ==========
|
|
===== NODE=pve01 =====
|
|
REMOTE_COUNT=26
|
|
REMOTE node=pve01 name=pve01-mail-01: health=OK references=UNREFERENCED
|
|
===== NODE=pve02 =====
|
|
REMOTE_COUNT=26
|
|
REMOTE node=pve02 name=pve01-mail-01: health=OK references=UNREFERENCED
|
|
===== NODE=pve03 =====
|
|
REMOTE_COUNT=26
|
|
REMOTE node=pve03 name=pve01-mail-01: health=OK references=UNREFERENCED
|
|
EXPECTED_CLUSTER_REMOTES=13
|
|
DETECTED_CLUSTER_REMOTES=78
|
|
REMOTE_COUNT_STATUS=MISMATCH
|
|
========== BACKUP-JOB-CLOUD-MATRIX-064 ==========
|
|
========== NODE=pve01 ==========
|
|
=== REMOTES ===
|
|
=== TIMERS ===
|
|
Wed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:14 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service
|
|
Wed 2026-07-22 00:09:43 MSK 9h Tue 2026-07-21 00:09:32 MSK 14h ago homelab-backup-freshness.timer homelab-backup-freshness.service
|
|
Wed 2026-07-22 02:15:00 MSK 11h Tue 2026-07-21 02:15:05 MSK 12h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service
|
|
Wed 2026-07-22 03:10:02 MSK 12h Tue 2026-07-21 03:13:14 MSK 11h ago homelab-emergency-backup.timer homelab-emergency-backup.service
|
|
Wed 2026-07-22 03:35:00 MSK 12h Tue 2026-07-21 03:35:14 MSK 11h ago filebrowser-backup.timer filebrowser-backup.service
|
|
Wed 2026-07-22 03:55:02 MSK 13h Tue 2026-07-21 03:51:25 MSK 11h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service
|
|
Wed 2026-07-22 04:17:29 MSK 13h Tue 2026-07-21 04:19:17 MSK 10h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service
|
|
Wed 2026-07-22 04:26:54 MSK 13h Tue 2026-07-21 04:28:14 MSK 10h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service
|
|
Wed 2026-07-22 04:38:04 MSK 13h Tue 2026-07-21 04:34:48 MSK 10h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service
|
|
Wed 2026-07-22 05:01:44 MSK 14h Tue 2026-07-21 04:57:14 MSK 9h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service
|
|
Wed 2026-07-22 05:23:56 MSK 14h Tue 2026-07-21 05:17:57 MSK 9h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service
|
|
Wed 2026-07-22 05:27:13 MSK 14h Tue 2026-07-21 05:29:05 MSK 9h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service
|
|
Wed 2026-07-22 05:47:19 MSK 14h Tue 2026-07-21 05:46:14 MSK 9h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service
|
|
Wed 2026-07-22 06:01:19 MSK 15h Tue 2026-07-21 05:57:30 MSK 8h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service
|
|
Wed 2026-07-22 06:27:43 MSK 15h Tue 2026-07-21 06:32:25 MSK 8h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service
|
|
Wed 2026-07-22 06:38:51 MSK 15h Tue 2026-07-21 06:43:54 MSK 8h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service
|
|
Wed 2026-07-22 06:39:22 MSK 15h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service
|
|
Wed 2026-07-22 06:44:16 MSK 15h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service
|
|
Wed 2026-07-22 06:52:06 MSK 15h Tue 2026-07-21 06:47:47 MSK 8h ago homelab-beszel-backup.timer homelab-beszel-backup.service
|
|
Wed 2026-07-22 06:58:58 MSK 16h Tue 2026-07-21 06:57:30 MSK 7h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service
|
|
Wed 2026-07-22 07:02:27 MSK 16h Tue 2026-07-21 07:05:30 MSK 7h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service
|
|
Wed 2026-07-22 07:07:14 MSK 16h Tue 2026-07-21 07:18:55 MSK 7h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service
|
|
Wed 2026-07-22 07:35:06 MSK 16h Tue 2026-07-21 07:36:59 MSK 7h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service
|
|
Wed 2026-07-22 07:38:21 MSK 16h Tue 2026-07-21 07:36:14 MSK 7h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service
|
|
Wed 2026-07-22 07:41:49 MSK 16h Tue 2026-07-21 07:53:41 MSK 7h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service
|
|
Wed 2026-07-22 08:10:37 MSK 17h Tue 2026-07-21 08:07:25 MSK 6h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service
|
|
Wed 2026-07-22 08:37:02 MSK 17h Tue 2026-07-21 08:30:07 MSK 6h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service
|
|
Wed 2026-07-22 08:40:13 MSK 17h Tue 2026-07-21 08:50:14 MSK 6h ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service
|
|
Wed 2026-07-22 08:53:36 MSK 17h Tue 2026-07-21 08:41:14 MSK 6h ago memos-backup.timer memos-backup.service
|
|
Sun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service
|
|
Sun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service
|
|
Sun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service
|
|
Sun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service
|
|
Mon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 7h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service
|
|
=== SERVICES ===
|
|
dpkg-db-backup.service static -
|
|
filebrowser-backup.service static -
|
|
homelab-alertmanager-backup.service static -
|
|
homelab-appbackup-dockge-stacks.service static -
|
|
homelab-appbackup-mariadb-batch.service static -
|
|
homelab-appbackup-npmplus-edge.service static -
|
|
homelab-appbackup-postgres-batch.service static -
|
|
homelab-appbackup-sqlite-batch.service static -
|
|
homelab-authentik-backup.service static -
|
|
homelab-auto-backup.service static -
|
|
homelab-backup-coverage-matrix.service static -
|
|
homelab-backup-freshness.service static -
|
|
homelab-backup-healthcheck.service static -
|
|
homelab-backup-matrix-refresh.service static -
|
|
homelab-backup-sla-health.service static -
|
|
homelab-beszel-backup.service static -
|
|
homelab-blackbox-exporter-backup.service static -
|
|
homelab-cluster-admin-incident-engine-backup.service static -
|
|
homelab-crypto-portfolio-backup.service static -
|
|
homelab-edge-vm-vzdump-backup.service static -
|
|
homelab-emergency-backup.service static -
|
|
homelab-extended-appbackup.service static -
|
|
homelab-gitea-backup.service static -
|
|
homelab-gitea-secondary-backup.service static -
|
|
homelab-legacy-monitoring-backups.service static -
|
|
homelab-mail-cloud-critical-upload.service static -
|
|
homelab-mail-cloud-edge-vm-upload.service static -
|
|
homelab-mail-cloud-nextcloud-vm-upload.service static -
|
|
homelab-mail-cloud-restore-drill.service static -
|
|
homelab-npmplus-kuma-config-backup.service static -
|
|
homelab-router-running-config-mail-cloud.service static -
|
|
homelab-vaultwarden-backup.service static -
|
|
homelab-verified-backup-health.service static -
|
|
homelab-vm-backup-health.service static -
|
|
homelab-vm-backup-policy.service static -
|
|
homelab-vm-full-backup.service static -
|
|
homelab-vm-local-dumps-2cloud.service static -
|
|
homelab-vm160-remote-backup.service static -
|
|
immich-media-backup.service static -
|
|
memos-backup.service static -
|
|
netbird-vps-backup.service static -
|
|
paperless-backup.service static -
|
|
skladchik-reports-monitor-backup-restore-check.service static -
|
|
skladchik-reports-monitor-backup.service static -
|
|
/usr/local/sbin/homelab-vm160-remote-backup:8:PARENT=/mnt/staging/vzdump
|
|
/usr/local/sbin/homelab-vm160-remote-backup:22:SOURCE_ACTIVE="$(ssh "${SSH[@]}" "$SOURCE" "pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'")" || SOURCE_ACTIVE_RC=$?
|
|
/usr/local/sbin/homelab-vm160-remote-backup:27:NAME="vzdump-qemu-${VMID}-${TS}.vma.zst"
|
|
/usr/local/sbin/homelab-vm160-remote-backup:35: ssh "${SSH[@]}" "$TARGET" "rm -rf -- '$DIR'" >/dev/null 2>&1 || true
|
|
/usr/local/sbin/homelab-vm160-remote-backup:41: rm -f "$LOG"
|
|
/usr/local/sbin/homelab-vm160-remote-backup:46:LATEST_SIZE="$(ssh "${SSH[@]}" "$TARGET" "find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\n' | sort -nr | head -n1")"
|
|
/usr/local/sbin/homelab-vm160-remote-backup:55:ssh "${SSH[@]}" "$SOURCE" "vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7" 2>"$LOG" | ssh "${SSH[@]}" "$TARGET" "cat > '$PART'"
|
|
/usr/local/sbin/homelab-vm160-remote-backup:57:ssh "${SSH[@]}" "$TARGET" "cat > '$DIR/vzdump.log'" < "$LOG"
|
|
/usr/local/sbin/homelab-vm160-remote-backup:66:rm -f "$LOG"
|
|
/usr/local/sbin/skladchik-reports-monitor-status:11:check_health() {
|
|
/usr/local/sbin/skladchik-reports-monitor-status:24:check_timer() {
|
|
/usr/local/sbin/skladchik-reports-monitor-status:38:check_timer skladchik-reports-monitor-backup.timer
|
|
/usr/local/sbin/skladchik-reports-monitor-status:39:check_timer skladchik-reports-monitor-backup-restore-check.timer
|
|
/usr/local/sbin/skladchik-reports-monitor-status:40:check_timer skladchik-reports-monitor-supervisor.timer
|
|
/usr/local/sbin/skladchik-reports-monitor-status:41:check_timer skladchik-reports-monitor-monthly-selftest-watch.timer
|
|
/usr/local/sbin/skladchik-reports-monitor-status:43:check_health "PVE01: ежедневный бэкап" "/var/lib/skladchik-reports-monitor-backup/health.txt"
|
|
/usr/local/sbin/skladchik-reports-monitor-status:44:check_health "PVE01: проверка восстановления" "/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt"
|
|
/usr/local/sbin/skladchik-reports-monitor-status:45:check_health "PVE01: общий supervisor" "/var/lib/skladchik-reports-monitor-supervisor/health.txt"
|
|
/usr/local/sbin/skladchik-reports-monitor-status:46:check_health "PVE01: контроль ежемесячного самотеста" "/var/lib/skladchik-reports-monitor-selftest-watch/health.txt"
|
|
/usr/local/sbin/homelab-mail-cloud-restore-drill:83: rm -f "$OUT"
|
|
/usr/local/sbin/homelab-mail-cloud-restore-drill:108: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/MANIFEST.txt" "$WORK/edge/MANIFEST.txt" > "$WORK/logs/edge_manifest_download.out" 2>"$WORK/logs/edge_manifest_download.err"
|
|
/usr/local/sbin/homelab-mail-cloud-restore-drill:111: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/SHA256SUMS.txt" "$WORK/edge/SHA256SUMS.txt" > "$WORK/logs/edge_sums_download.out" 2>"$WORK/logs/edge_sums_download.err"
|
|
/usr/local/sbin/homelab-mail-cloud-restore-drill:114: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/$EDGE_ARCHIVE.sha256" "$WORK/edge/$EDGE_ARCHIVE.sha256" > "$WORK/logs/edge_archive_sha_download.out" 2>"$WORK/logs/edge_archive_sha_download.err"
|
|
/usr/local/sbin/homelab-mail-cloud-restore-drill:117: rclone lsf "${EDGE_REMOTE}:$EDGE_DEST/parts" --format ps > "$WORK/logs/edge_parts_lsf.txt" 2>"$WORK/logs/edge_parts_lsf.err"
|
|
/usr/local/sbin/homelab-mail-cloud-restore-drill:130: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/parts/$FIRST_PART" "$WORK/edge/$FIRST_PART" > "$WORK/logs/edge_first_part_download.out" 2>"$WORK/logs/edge_first_part_download.err"
|
|
/usr/local/sbin/homelab-mail-cloud-restore-drill:137: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/parts/$LAST_PART" "$WORK/edge/$LAST_PART" > "$WORK/logs/edge_last_part_download.out" 2>"$WORK/logs/edge_last_part_download.err"
|
|
/usr/local/sbin/homelab-mail-cloud-restore-drill:168: rm -f "$WORK/edge/$FIRST_PART" "$WORK/edge/$LAST_PART"
|
|
/usr/local/sbin/homelab-golden-state-extended-overlay:26:rclone copyto "$JSON" "$RA" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-golden-state-extended-overlay:27:rclone copyto "$JSON" "$RB" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-mail-cloud-critical-upload:80: rclone mkdir "${R}:$REMOTE_DIR" > "$WORK/logs/${SAFE}_mkdir.out" 2>"$WORK/logs/${SAFE}_mkdir.err"
|
|
/usr/local/sbin/homelab-mail-cloud-critical-upload:82: homelab-rclone-copy-chunked "$BUNDLE" "${R}:$REMOTE_DIR/$(basename "$BUNDLE")" > "$WORK/logs/${SAFE}_upload_bundle.out" 2>"$WORK/logs/${SAFE}_upload_bundle.err"
|
|
/usr/local/sbin/homelab-mail-cloud-critical-upload:84: rclone copyto "$BUNDLE_SHA" "${R}:$REMOTE_DIR/$(basename "$BUNDLE_SHA")" --checksum > "$WORK/logs/${SAFE}_upload_sha.out" 2>"$WORK/logs/${SAFE}_upload_sha.err"
|
|
/usr/local/sbin/homelab-mail-cloud-critical-upload:86: rclone copyto "$WORK/components/MANIFEST.txt" "${R}:$REMOTE_DIR/MANIFEST.txt" --checksum > "$WORK/logs/${SAFE}_upload_manifest.out" 2>"$WORK/logs/${SAFE}_upload_manifest.err"
|
|
/usr/local/sbin/homelab-mail-cloud-critical-upload:88: rclone copyto "$WORK/components/SHA256SUMS.txt" "${R}:$REMOTE_DIR/SHA256SUMS.txt" --checksum > "$WORK/logs/${SAFE}_upload_sums.out" 2>"$WORK/logs/${SAFE}_upload_sums.err"
|
|
/usr/local/sbin/homelab-mail-cloud-critical-upload:90: homelab-rclone-cat-chunked "${R}:$REMOTE_DIR/$(basename "$BUNDLE")" > "$WORK/download/${SAFE}-$(basename "$BUNDLE")" 2>"$WORK/logs/${SAFE}_download_bundle.err"
|
|
/usr/local/sbin/homelab-mail-cloud-critical-upload:108: rm -f "$WORK/download/${SAFE}-$(basename "$BUNDLE")"
|
|
/usr/local/sbin/homelab-mail-cloud-critical-upload:140: rm -rf --one-file-system "$OLD_DIR"
|
|
/usr/local/sbin/homelab-alertmanager-backup-run:16:ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$MON" "sudo -n docker exec alertmanager amtool check-config /etc/alertmanager/alertmanager.yml >/dev/null && sudo -n tar -C / -czf - opt/stacks/observability-lite/compose.yaml opt/stacks/observability-lite/compose.override.yaml opt/stacks/observability-lite/alertmanager/alertmanager.yml opt/stacks/observability-lite/alertmanager/data" > "$STAGE/remote.tgz"
|
|
/usr/local/sbin/homelab-alertmanager-backup-run:18:rm -f "$STAGE/remote.tgz"
|
|
/usr/local/sbin/homelab-alertmanager-backup-run:21:jq -n --arg app "$APP" --arg host "monitoring" --arg engine "files+alertmanager-config-check" --arg config "ok" --argjson files "$FILES" "{app:\$app,engine:\$engine,source_host:\$host,source_paths:[\"monitoring:/opt/stacks/observability-lite/alertmanager\",\"monitoring:/opt/stacks/observability-lite/compose.yaml\",\"monitoring:/opt/stacks/observability-lite/compose.override.yaml\"],config_check:\$config,file_count:\$files,note:\"Alertmanager configuration and runtime state in encrypted two-cloud backup\"}" > "$STAGE/metadata.json"
|
|
/usr/local/sbin/appbackupctl:21: *) echo "usage: appbackupctl run [dockge-stacks|cluster-admin-incident-engine]|check|restore-check|retention|report|audit|status|lastlog|timer"; exit 2 ;;
|
|
/usr/local/sbin/postgres-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@"$H" "sudo rm -f /tmp/postgres-app-${APP}-${JOB_ID}.tgz"
|
|
/usr/local/sbin/postgres-apps-backup-run:21: rm -rf "$STAGE/payload"
|
|
/usr/local/sbin/homelab-backup-gitea:27:rm -rf "$RESTORE"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:4:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:18: "checksums/"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:21:echo "=== homelab rclone crypt healthcheck: $(hostname) ==="
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:40: if ! rclone listremotes | sed 's/:$//' | grep -qx "$REMOTE"; then
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:47: if ! rclone about "${REMOTE}:"; then
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:48: echo "FAIL: rclone about failed: $REMOTE"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:54: ROOT_LIST="$(rclone lsf "${REMOTE}:/" || true)"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:76: if rclone cat "${REMOTE}:/latest-backup-index.json" | jq -e '.schema and .remote and .status and .items' >/dev/null; then
|
|
/usr/local/sbin/homelab-stage-verify-flat-chunked:7:rm -rf "$VERIFY"
|
|
/usr/local/sbin/homelab-stage-verify-flat-chunked:9:rclone copy "$REMOTE_DIR" "$VERIFY" --transfers 4 --checkers 8 --retries 5 --low-level-retries 20 --log-level ERROR
|
|
/usr/local/sbin/homelab-stage-verify-flat-chunked:11:(cd "$VERIFY" && test -s manifest.json && if [[ -n "$JOB" ]]; then jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null; fi && sha256sum -c checksums.sha256 >/dev/null && tar -tzf "$ARCHIVE" >/dev/null)
|
|
/usr/local/sbin/homelab-rclone-cat-chunked:7:cleanup(){ rm -rf "$WORK"; }
|
|
/usr/local/sbin/homelab-rclone-cat-chunked:10:rclone copyto "${REMOTE_FILE}.parts/MANIFEST.parts.txt" "$MAN" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR
|
|
/usr/local/sbin/homelab-rclone-cat-chunked:11:while read -r PART SIZE; do case "$PART" in part-*) rclone cat "${REMOTE_FILE}.parts/$PART" ;; esac; done < "$MAN"
|
|
/usr/local/sbin/homelab-stage-verify-chunked:7:rm -rf "$VERIFY"
|
|
/usr/local/sbin/homelab-stage-verify-chunked:9:rclone copy "$REMOTE_DIR" "$VERIFY" --exclude "$ARCHIVE.parts/**" --exclude "$ARCHIVE.parts/" --exclude "$ARCHIVE.sha256" --exclude "$ARCHIVE.manifest.txt" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --log-level ERROR
|
|
/usr/local/sbin/homelab-stage-verify-chunked:13:if [ -s checksums.sha256 ]; then grep -F -v " $ARCHIVE" checksums.sha256 > .checksums-no-archive.sha256 || true; if [ -s .checksums-no-archive.sha256 ]; then sha256sum -c .checksums-no-archive.sha256; fi; EXPECTED=$(grep -F " $ARCHIVE" checksums.sha256 | awk '{print $1; exit}'); else EXPECTED=$(rclone cat "${REMOTE_DIR}/${ARCHIVE}.sha256" | awk '{print $1; exit}'); fi
|
|
/usr/local/sbin/homelab-stage-verify-chunked:15:ACTUAL=$(homelab-rclone-cat-chunked "${REMOTE_DIR}/${ARCHIVE}" | sha256sum | awk '{print $1}')
|
|
/usr/local/sbin/homelab-stage-verify-chunked:17:homelab-rclone-cat-chunked "${REMOTE_DIR}/${ARCHIVE}" | tar -tzf - >/dev/null
|
|
/usr/local/sbin/crypto-portfolio-backup-run:15:ssh -n -o BatchMode=yes -o ConnectTimeout=10 "$CORE" 'H="$(hostname -f 2>/dev/null || hostname)"; { [ "$H" = "core-apps.gram1.ru" ] || [ "$H" = "core-apps" ]; } || exit 3; D=/opt/stacks/crypto-portfolio; T="$(sudo -n mktemp -d /tmp/crypto-portfolio-backup.XXXXXX)"; trap "sudo -n rm -rf \"$T\"" EXIT; sudo -n install -d -m 700 "$T/stack" "$T/stack/data"; for F in compose.yaml Dockerfile requirements.txt app.py worker.py .dockerignore; do sudo -n test -s "$D/$F" && sudo -n cp -a "$D/$F" "$T/stack/"; done; sudo -n cp -a "$D/templates" "$D/secrets" "$T/stack/"; sudo -n python3 -c "import sqlite3,sys;s=sqlite3.connect(sys.argv[1]);d=sqlite3.connect(sys.argv[2]);s.backup(d);d.close();s.close()" "$D/data/portfolio.db" "$T/stack/data/portfolio.db"; for F in market.json worker-status.json; do sudo -n test -s "$D/data/$F" && sudo -n cp -a "$D/data/$F" "$T/stack/data/"; done; sudo -n tar -C "$T" -czf - stack' > "$STAGE/remote.tgz"
|
|
/usr/local/sbin/crypto-portfolio-backup-run:17:rm -f "$STAGE/remote.tgz"
|
|
/usr/local/sbin/npmplus-edge-backup-run:20:ssh -o BatchMode=yes -o ConnectTimeout=8 "$EDGE" "sudo rm -f '/tmp/npmplus-edge-${JOB_ID}.tgz'"
|
|
/usr/local/sbin/npmplus-edge-backup-run:28:rm -rf "$STAGE/payload"
|
|
/usr/local/sbin/mariadb-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@"$H" "sudo rm -f /tmp/mariadb-app-${APP}-${JOB_ID}.tgz"
|
|
/usr/local/sbin/mariadb-apps-backup-run:20: rm -rf "$STAGE/payload"
|
|
/usr/local/sbin/homelab-blackbox-exporter-backup-run:17:rm -f "$STAGE/remote.tgz"
|
|
/usr/local/sbin/homelab-blackbox-exporter-backup-run:21:jq -n --arg app "$APP" --arg host "monitoring" --arg engine "files+runtime-health-check" --argjson files "$FILES" '{app:$app,engine:$engine,source_host:$host,source_paths:["monitoring:/opt/stacks/blackbox-exporter"],runtime_health:"ok",file_count:$files,note:"Blackbox Exporter compose and probe configuration in encrypted two-cloud backup"}' > "$STAGE/metadata.json"
|
|
/usr/local/sbin/homelab-ingress-standby-refresh:16: rm -f "$TMP_STATE" "$RELAY" "$RESOURCES"
|
|
/usr/local/sbin/homelab-ingress-standby-refresh:106:"chattr -i '$DEST_FILE' '$DEST_FILE.sha256' '$DEST_FILE.env' 2>/dev/null || true; rm -f '$DEST_TMP'"
|
|
/usr/local/sbin/homelab-ingress-standby-refresh:164: rm -f "$OLD" "$OLD.sha256" "$OLD.env"
|
|
/usr/local/sbin/homelab-ingress-standby-refresh:173:run_node "$SOURCE_NODE" rm -f "$SOURCE_BACKUP"
|
|
/usr/local/sbin/homelab-stage-2cloud-flat:33:(cd "$STAGE"; FILES=("$ARCHIVE" manifest.json latest-backup-index.json); [[ -f metadata.json ]] && FILES+=(metadata.json); sha256sum "${FILES[@]}" > checksums.sha256; sha256sum -c checksums.sha256 >/dev/null; tar -tzf "$ARCHIVE" >/dev/null; jq -e . manifest.json >/dev/null)
|
|
/usr/local/sbin/homelab-stage-2cloud-flat:36: if rclone lsf "$REMOTE_DIR" --max-depth 1 2>/dev/null | grep -q .; then echo "REFUSE NONEMPTY CANARY TARGET $REMOTE_DIR" >&2; exit 6; fi
|
|
/usr/local/sbin/homelab-stage-2cloud-flat:37: rclone mkdir "$REMOTE_DIR"
|
|
/usr/local/sbin/homelab-stage-2cloud-flat:38: for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f "$STAGE/$F" ]] && rclone copyto "$STAGE/$F" "$REMOTE_DIR/$F" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done
|
|
/usr/local/sbin/homelab-stage-2cloud-flat:39: CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-flat-chunked "$STAGE/$ARCHIVE" "$REMOTE_DIR/$ARCHIVE"
|
|
/usr/local/sbin/homelab-stage-2cloud-flat:41: rclone copyto "$STAGE/latest-backup-index.json" "$LATEST_FILE" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR
|
|
/usr/local/sbin/homelab-stage-2cloud-flat:47:rm -rf "$STAGE" "$VERIFY"
|
|
/usr/local/sbin/homelab-cluster-admin-webpanel-sync:7:copy_health() {
|
|
/usr/local/sbin/homelab-cluster-admin-webpanel-sync:17:copy_health /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt cluster-admin-vm-mail-cloud-backup.txt
|
|
/usr/local/sbin/homelab-cluster-admin-webpanel-sync:18:copy_health /var/lib/homelab-health/overall.txt overall.txt
|
|
/usr/local/sbin/homelab-cluster-admin-webpanel-sync:19:copy_health /var/lib/homelab-health/final-readiness.txt final-readiness.txt
|
|
/usr/local/sbin/homelab-cluster-admin-webpanel-sync:20:copy_health /var/lib/homelab-health/cluster-admin-strict-final-seal.txt cluster-admin-strict-final-seal.txt
|
|
/usr/local/sbin/homelab-backup-authentik:14:ssh debian@[PRIVATE_IP] "rm -rf /tmp/auth-small-snap; mkdir -p /tmp/auth-small-snap"
|
|
/usr/local/sbin/homelab-backup-authentik:27:ssh debian@[PRIVATE_IP] "sudo docker rm -f authentik-restore-test >/dev/null 2>&1 || true"
|
|
/usr/local/sbin/homelab-backup-authentik:51:ssh debian@[PRIVATE_IP] "sudo docker rm -f authentik-restore-test >/dev/null 2>&1 || true; rm -f /tmp/authentik-restore-from-archive.pg_dump"
|
|
/usr/local/sbin/homelab-safe-exec:621: latest_tmp.unlink(missing_ok=True)
|
|
/usr/local/sbin/homelab-golden-state-index:28:rclone copyto "$JSON" "$RA" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-golden-state-index:29:rclone copyto "$JSON" "$RB" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-golden-state-index:30:rclone cat "$RA" 2>/dev/null | jq -e --arg cp "$CP" --arg ext "$EXT" ".control_plane_job == \$cp and .health.extended_appbackup == \$ext and .extended_appbackup_health == \$ext" >/dev/null || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-golden-state-index:31:rclone cat "$RB" 2>/dev/null | jq -e --arg cp "$CP" --arg ext "$EXT" ".control_plane_job == \$cp and .health.extended_appbackup == \$ext and .extended_appbackup_health == \$ext" >/dev/null || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-vm-full-retention:35: rm -f \
|
|
/usr/local/sbin/homelab-vm-full-retention:73: rm -rf -- \"\$BASE/\$setname\"
|
|
/usr/local/sbin/homelab-stage-2cloud:22:echo "[STEP] local checksum"; (cd "$STAGE" && sha256sum "$ARCHIVE" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e . manifest.json >/dev/null)
|
|
/usr/local/sbin/homelab-stage-2cloud:23:echo "[STEP] upload cloud A metadata"; rclone copy "$STAGE" "$RA" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s
|
|
/usr/local/sbin/homelab-stage-2cloud:25:echo "[STEP] upload cloud B metadata"; rclone copy "$STAGE" "$RB" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s
|
|
/usr/local/sbin/homelab-stage-2cloud:29:echo "[STEP] upload latest index"; rclone copyto "$STAGE/latest-backup-index.json" "$RAL"; rclone copyto "$STAGE/latest-backup-index.json" "$RBL"; rclone cat "$RAL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null; rclone cat "$RBL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null
|
|
/usr/local/sbin/homelab-stage-2cloud:31:echo "[STEP] delete local staging"; rm -rf "$STAGE" "$VERIFY"; test ! -d "$STAGE" && test ! -d "$VERIFY"
|
|
/usr/local/sbin/homelab-safe-autoheal:17:SAFE_TIMERS="homelab-emergency-backup.timer homelab-appbackup-dockge-stacks.timer homelab-appbackup-npmplus-edge.timer homelab-appbackup-sqlite-batch.timer homelab-appbackup-postgres-batch.timer homelab-appbackup-mariadb-batch.timer homelab-verified-backup-health.timer homelab-drift-check.timer homelab-service-registry-check.timer homelab-dependency-map-check.timer homelab-vm-backup-health.timer homelab-alerting-health.timer homelab-runbook-generate.timer homelab-desired-state-sync.timer homelab-duty-admin-report.timer homelab-incident-journal.timer homelab-backup-sla-health.timer homelab-kuma-monitor-policy.timer homelab-safe-autoheal.timer homelab-overall-health.timer"
|
|
/usr/local/sbin/homelab-safe-autoheal:144:for cmd in homelab-alerting-health homelab-kuma-monitor-policy homelab-backup-sla-health homelab-service-registry-check homelab-dependency-map-check; do
|
|
/usr/local/sbin/homelab-backup-npmplus-kuma-configs:26: install -D -m 0755 /usr/local/sbin/homelab-check-private-vpn-hosts "$SNAP/pve01/usr/local/sbin/homelab-check-private-vpn-hosts" 2>>"$ERR" || fail "copy_private_vpn_script_failed"
|
|
/usr/local/sbin/homelab-backup-npmplus-kuma-configs:27: install -D -m 0644 /etc/systemd/system/homelab-private-vpn-hosts-health.service "$SNAP/pve01/etc/systemd/system/homelab-private-vpn-hosts-health.service" 2>>"$ERR" || fail "copy_private_vpn_service_failed"
|
|
/usr/local/sbin/homelab-backup-npmplus-kuma-configs:28: install -D -m 0644 /etc/systemd/system/homelab-private-vpn-hosts-health.timer "$SNAP/pve01/etc/systemd/system/homelab-private-vpn-hosts-health.timer" 2>>"$ERR" || fail "copy_private_vpn_timer_failed"
|
|
/usr/local/sbin/homelab-backup-npmplus-kuma-configs:29: install -D -m 0644 /var/lib/homelab-health/private-vpn-hosts.txt "$SNAP/pve01/var/lib/homelab-health/private-vpn-hosts.txt" 2>>"$ERR" || fail "copy_private_vpn_health_failed"
|
|
/usr/local/sbin/homelab-backup-npmplus-kuma-configs:42:sudo rm -rf "$TMP" "$EDGE_TGZ"
|
|
/usr/local/sbin/homelab-backup-npmplus-kuma-configs:82:sudo rm -rf "$TMP"
|
|
/usr/local/sbin/homelab-backup-npmplus-kuma-configs:90: ssh -o BatchMode=yes -o ConnectTimeout=8 debian@[PRIVATE_IP] "rm -f '$EDGE_TGZ'" 2>>"$ERR" || true
|
|
/usr/local/sbin/homelab-backup-npmplus-kuma-configs:99: NPM_INTEGRITY="$(sqlite3 "$SNAP/edge-vm/npmplus/db/database.sqlite" "PRAGMA integrity_check;" 2>>"$ERR" | tr '\n' ';' || echo error)"
|
|
/usr/local/sbin/homelab-backup-npmplus-kuma-configs:100: KUMA_INTEGRITY="$(sqlite3 "$SNAP/edge-vm/kuma/db/kuma.db" "PRAGMA integrity_check;" 2>>"$ERR" | tr '\n' ';' || echo error)"
|
|
/usr/local/sbin/homelab-backup-npmplus-kuma-configs:134: rm -rf "$RESTORE"
|
|
/usr/local/sbin/homelab-backup-npmplus-kuma-configs:138: RESTORE_NPM_INTEGRITY="$(sqlite3 "$RESTORE/edge-vm/npmplus/db/database.sqlite" "PRAGMA integrity_check;" 2>>"$ERR" | tr '\n' ';' || echo error)"
|
|
/usr/local/sbin/homelab-backup-npmplus-kuma-configs:139: RESTORE_KUMA_INTEGRITY="$(sqlite3 "$RESTORE/edge-vm/kuma/db/kuma.db" "PRAGMA integrity_check;" 2>>"$ERR" | tr '\n' ';' || echo error)"
|
|
/usr/local/sbin/homelab-stage-2cloud-flat-canary:31:(cd "$STAGE"; FILES=("$ARCHIVE" manifest.json latest-backup-index.json); [[ -f metadata.json ]] && FILES+=(metadata.json); sha256sum "${FILES[@]}" > checksums.sha256; sha256sum -c checksums.sha256 >/dev/null; tar -tzf "$ARCHIVE" >/dev/null; jq -e . manifest.json >/dev/null)
|
|
/usr/local/sbin/homelab-stage-2cloud-flat-canary:34: if rclone lsf "$REMOTE_DIR" --max-depth 1 2>/dev/null | grep -q .; then echo "REFUSE NONEMPTY CANARY TARGET $REMOTE_DIR" >&2; exit 6; fi
|
|
/usr/local/sbin/homelab-stage-2cloud-flat-canary:35: rclone mkdir "$REMOTE_DIR"
|
|
/usr/local/sbin/homelab-stage-2cloud-flat-canary:36: for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f "$STAGE/$F" ]] && rclone copyto "$STAGE/$F" "$REMOTE_DIR/$F" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done
|
|
/usr/local/sbin/homelab-stage-2cloud-flat-canary:37: CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-flat-chunked "$STAGE/$ARCHIVE" "$REMOTE_DIR/$ARCHIVE"
|
|
/usr/local/sbin/homelab-stage-2cloud-flat-canary:39: rclone copyto "$STAGE/latest-backup-index.json" "$LATEST_FILE" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR
|
|
/usr/local/sbin/homelab-stage-2cloud-flat-canary:45:rm -rf "$STAGE" "$VERIFY"
|
|
/usr/local/sbin/homelab-router-running-config-mail-cloud:62: rm -f "$CONFIG"
|
|
/usr/local/sbin/homelab-router-running-config-mail-cloud:66: rm -f "$CONFIG"
|
|
/usr/local/sbin/homelab-router-running-config-mail-cloud:79: if rclone mkdir "$REMOTE:$DEST" >"$WORK/rclone-mkdir.out" 2>"$WORK/rclone-mkdir.err" && homelab-rclone-copy-chunked "$ARCHIVE" "$REMOTE:$DEST/$(basename "$ARCHIVE")" >"$WORK/rclone-upload.out" 2>"$WORK/rclone-upload.err" && rclone copyto "$SHA_FILE" "$REMOTE:$DEST/$(basename "$SHA_FILE")" --checksum >"$WORK/rclone-upload-sha.out" 2>"$WORK/rclone-upload-sha.err"; then
|
|
/usr/local/sbin/homelab-router-running-config-mail-cloud:86: if homelab-rclone-cat-chunked "$REMOTE:$DEST/$(basename "$ARCHIVE")" >"$DL" 2>"$WORK/rclone-download.err"; then
|
|
/usr/local/sbin/homelab-router-running-config-mail-cloud:93:rclone lsf "$REMOTE:$TARGET_BASE" --dirs-only 2>/dev/null | sed 's#/$##' | sort > "$DIR_LIST" || RETENTION_RC=$?
|
|
/usr/local/sbin/homelab-router-running-config-mail-cloud:98: [ -n "$OLD" ] && rclone purge "$REMOTE:$TARGET_BASE/$OLD" >>"$WORK/retention.out" 2>>"$WORK/retention.err"
|
|
/usr/local/sbin/homelab-router-running-config-mail-cloud:103: rm -f "$old" "$old.sha256"
|
|
/usr/local/sbin/homelab-router-running-config-mail-cloud:116:rm -rf "$WORK"
|
|
/usr/local/sbin/appbackup-restore-check:24:rm -rf "$ROOT" && mkdir -p "$ROOT/a" "$ROOT/b"
|
|
/usr/local/sbin/appbackup-restore-check:25:trap "echo \"ERROR app restore-check failed; kept: $ROOT\"; exit 1" ERR
|
|
/usr/local/sbin/appbackup-restore-check:27:rclone copy "$RA" "$ROOT/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >"${LOGBASE}-a.log" 2>&1
|
|
/usr/local/sbin/appbackup-restore-check:30:(cd "$ROOT/a" && sha256sum -c checksums.sha256 >"${LOGBASE}-a-sha.log" && ARCHIVE="$(jq -r ".archive_name" manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null)
|
|
/usr/local/sbin/appbackup-restore-check:32:rclone copy "$RB" "$ROOT/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >"${LOGBASE}-b.log" 2>&1
|
|
/usr/local/sbin/appbackup-restore-check:35:(cd "$ROOT/b" && sha256sum -c checksums.sha256 >"${LOGBASE}-b-sha.log" && ARCHIVE="$(jq -r ".archive_name" manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null)
|
|
/usr/local/sbin/appbackup-restore-check:37:rm -rf "$ROOT"
|
|
/usr/local/sbin/appbackup-restore-check:39:rm -f "${LOGBASE}-a.log" "${LOGBASE}-a-sha.log" "${LOGBASE}-b.log" "${LOGBASE}-b-sha.log"
|
|
/usr/local/sbin/backupctl:12: check) bash -n "$SCRIPT" && [[ -f "$STATE" ]] && jq -e ".status == \"success\" and .local_backup_deleted == true" "$STATE" >/dev/null && echo "OK backup framework" ;;
|
|
/usr/local/sbin/backupctl:16: remotes) source /etc/homelab-backup.env && NODE="$(hostname -s)" && echo "A=$REMOTE_A" && echo "B=$REMOTE_B" && rclone lsd "${REMOTE_A%/}/${NODE}/emergency-pack" | tail -n 5 && rclone lsd "${REMOTE_B%/}/${NODE}/emergency-pack" | tail -n 5 ;;
|
|
/usr/local/sbin/backupctl:19: *) echo "usage: backupctl run|status|lastlog|check|restore-check|audit|report|remotes|retention|timer"; exit 2 ;;
|
|
/usr/local/sbin/homelab-edge-vm-offhost-freshness:156:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-backup NODE=%s VMID=%s ARCHIVE=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\n' \
|
|
/usr/local/sbin/homelab-edge-vm-offhost-freshness:158: >"$HEALTH_DIR/edge-vm-vzdump-backup.txt"
|
|
/usr/local/sbin/homelab-edge-vm-offhost-freshness:159:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-offhost NODE=%s VMID=%s HOST=%s DEST=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\n' \
|
|
/usr/local/sbin/homelab-edge-vm-offhost-freshness:161: >"$HEALTH_DIR/edge-vm-vzdump-offhost.txt"
|
|
/usr/local/sbin/homelab-edge-vm-offhost-freshness:164: >"$HEALTH_DIR/edge-vm-vzdump-restore.txt"
|
|
/usr/local/sbin/homelab-edge-vm-offhost-freshness:167: "$HEALTH_DIR/edge-vm-vzdump-backup.txt" \
|
|
/usr/local/sbin/homelab-edge-vm-offhost-freshness:168: "$HEALTH_DIR/edge-vm-vzdump-offhost.txt" \
|
|
/usr/local/sbin/homelab-edge-vm-offhost-freshness:169: "$HEALTH_DIR/edge-vm-vzdump-restore.txt"
|
|
/usr/local/sbin/homelab-external-probe-vps:20:rm -f "$TMP"
|
|
/usr/local/sbin/sqlite-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@"$H" "sudo rm -f /tmp/sqlite-app-${APP}-${JOB_ID}.tgz"
|
|
/usr/local/sbin/sqlite-apps-backup-run:21: rm -rf "$STAGE/payload"
|
|
/usr/local/sbin/homelab-desired-state-sync:10:find "$REPO" -mindepth 1 -maxdepth 1 ! -name .git -exec rm -rf {} + 2>/dev/null
|
|
/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt
|
|
/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:28: if rclone lsf "${remote_path}.parts/" > "$plist" 2>/dev/null && grep -q '^part-' "$plist"; then
|
|
/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:29: while IFS= read -r part; do [ -n "$part" ] && rclone cat "${remote_path}.parts/${part}" >> "$out"; done < <(grep '^part-' "$plist" | sort)
|
|
/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:31: rclone copyto "$remote_path" "$out" >/dev/null
|
|
/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:33: rm -f "$plist"
|
|
/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:36:check_one_remote(){
|
|
/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:97:check_one_remote CLOUD_1 "$CLOUD_1"
|
|
/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:98:check_one_remote CLOUD_2 "$CLOUD_2"
|
|
/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:101:find "$WORK_ROOT" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_LOCAL}" | while read -r D; do [ -n "$D" ] && rm -rf "${WORK_ROOT}/${D}"; done
|
|
/usr/local/sbin/skladchik-reports-monitor-backup:29:[ -x /usr/local/sbin/homelab-rclone-copy-chunked ] || { echo "missing /usr/local/sbin/homelab-rclone-copy-chunked" >&2; exit 10; }
|
|
/usr/local/sbin/skladchik-reports-monitor-backup:36:cleanup(){ rm -rf "$TMP"; }
|
|
/usr/local/sbin/skladchik-reports-monitor-backup:82: mapfile -t OLD < <(rclone lsf --dirs-only "${REMOTE}${REMOTE_BASE}/daily/" 2>/dev/null | sed 's#/$##' | sort | head -n -"${KEEP_DAILY}" || true)
|
|
/usr/local/sbin/skladchik-reports-monitor-backup:83: for D in "${OLD[@]}"; do [ -n "$D" ] && rclone purge "${REMOTE}${REMOTE_BASE}/daily/${D}" >/dev/null 2>&1 || true; done
|
|
/usr/local/sbin/skladchik-reports-monitor-backup:85:find "${LOCAL_ROOT}/daily" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_DAILY}" | while read -r D; do [ -n "$D" ] && rm -rf "${LOCAL_ROOT}/daily/${D}"; done
|
|
/usr/local/sbin/homelab-vm-local-dumps-retention:56: rclone purge "$PATH_REMOTE" >> "$REPORT" 2>&1 && DELETED=$((DELETED+1)) || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-extended-appbackup-run:12:run_one(){ APP="$1"; HOST="$2"; PATHS="$3"; NOTE="$4"; CONS="$5"; JOB_ID="homelab-${NODE}-app-${APP}-${RUN_TS}"; export JOB_ID; SHORT_TS="${RUN_TS:2:6}-${RUN_TS:9:6}"; export BACKUP_PATH_REL="$(homelab-backup-path job a "$APP" "$SHORT_TS")"; STAGE="${STAGING_ROOT}/${JOB_ID}"; PAYLOAD="$STAGE/payload"; ARCHIVE="${JOB_ID}.tar.gz"; LOG="/tmp/extended-appbackup-${APP}-${RUN_TS}.log"; rm -rf "$STAGE"; mkdir -p "$PAYLOAD/local"; echo "### app=$APP" >> "$REPORT"; echo "host=$HOST" >> "$REPORT"; echo "source_note=$NOTE" >> "$REPORT"; ssh -n -o BatchMode=yes -o ConnectTimeout=30 "$HOST" "sudo tar -C / --ignore-failed-read --exclude=.env --exclude=*.env --exclude=.env.* --exclude=*.bak* -czf - $PATHS" > "$PAYLOAD/local/${APP}-data.tar.gz" 2>"$LOG"; TAR_RC=$?; if [ "$TAR_RC" -eq 0 ] && tar -tzf "$PAYLOAD/local/${APP}-data.tar.gz" >/dev/null 2>&1; then SIZE="$(stat -c%s "$PAYLOAD/local/${APP}-data.tar.gz" 2>/dev/null || echo 0)"; echo "archive_payload_size_bytes=$SIZE" >> "$REPORT"; if [ "$SIZE" -le 128 ]; then echo "backup=BAD reason=archive_too_small" >> "$REPORT"; BAD=$((BAD+1)); rm -rf "$STAGE"; else sha256sum "$PAYLOAD/local/${APP}-data.tar.gz" > "$PAYLOAD/local/${APP}-data.tar.gz.sha256"; jq -n --arg type "extended-appbackup" --arg app "$APP" --arg host "$HOST" --arg source_note "$NOTE" --arg archive_payload_size_bytes "$SIZE" --arg env_files_read "no" --arg secret_values_printed "no" --arg consistency "$CONS" "{type:\$type,app_name:\$app,host:\$host,source_note:\$source_note,archive_payload_size_bytes:(\$archive_payload_size_bytes|tonumber),env_files_read:\$env_files_read,secret_values_printed:\$secret_values_printed,consistency:\$consistency}" > "$STAGE/metadata.json"; tar -C "$PAYLOAD" -czf "$STAGE/$ARCHIVE" .; rm -rf "$PAYLOAD"; JOB_ID="$JOB_ID" homelab-stage-2cloud-flat "$APP" "$STAGE" "$ARCHIVE" > "$LOG" 2>&1; RC=$?; if [ "$RC" -eq 0 ]; then echo "backup=OK" >> "$REPORT"; grep -E "^\[STEP\]|^\[SUCCESS\]" "$LOG" >> "$REPORT"; appbackupctl restore-check "$APP" >> "$REPORT" 2>&1 && echo "restore_check=OK" >> "$REPORT" || echo "restore_check=WARN_non_registry" >> "$REPORT"; OK=$((OK+1)); else echo "backup=BAD rc=$RC" >> "$REPORT"; grep -E "^\[ERROR\]|\[STEP\]|ERROR|error|failed|Failed|required|missing|denied|forbidden|checksum|rclone|copy|mkdir|jq|tar" "$LOG" | tail -n 60 >> "$REPORT"; BAD=$((BAD+1)); fi; fi; else echo "tar=BAD rc=$TAR_RC" >> "$REPORT"; grep -E "ERROR|error|failed|Failed|denied|No such|Cannot|tar:" "$LOG" | tail -n 40 >> "$REPORT"; BAD=$((BAD+1)); rm -rf "$STAGE"; fi; echo >> "$REPORT"; }
|
|
/usr/local/sbin/homelab-extended-appbackup-run:21:run_one healthchecks "$MON" "var/lib/healthchecks opt/stacks/healthchecks/local_settings.py" "healthchecks data plus local settings" "emergency_data_config_archive"
|
|
/usr/local/sbin/homelab-mkdocs-auto-refresh:53: rm -f /tmp/homelab-mkdocs-auto-refresh-git-fetch.out /tmp/homelab-mkdocs-auto-refresh-git-fetch.err
|
|
/usr/local/sbin/homelab-mkdocs-auto-refresh:57:rm -f /tmp/homelab-mkdocs-auto-refresh-git-fetch.out /tmp/homelab-mkdocs-auto-refresh-git-fetch.err
|
|
/usr/local/sbin/homelab-mkdocs-auto-refresh:77:ssh "${EDGE_SSH_OPTS[@]}" "$EDGE_HOST" "rm -rf '$STAGE' && mkdir -p '$STAGE'"
|
|
/usr/local/sbin/homelab-mkdocs-auto-refresh:85:rm -rf "$BASE/site.new" "$BASE/site.old"
|
|
/usr/local/sbin/homelab-mkdocs-auto-refresh:93:rm -rf "$BASE/site.old" "$STAGE"
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:30:CHKS="$WORK/checksums.txt"
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:64: rm -rf "$CDIR"
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:126: rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:128: rclone copyto "$CDIR/chunks.sha256" "$DEST/files/$bn.chunks/chunks.sha256" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:131: rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:135: rclone copyto "$CHKS" "$DEST/checksums.txt" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:136: rclone copyto "$FILES_TSV" "$DEST/files.tsv" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:137: rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:144: rm -rf "$VBASE"
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:146: rclone copyto "$DEST/manifest.json" "$VBASE/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:154: rclone copy "$DEST/files/$bn.chunks" "$VDIR" --filter "+ part-*" --filter "+ chunks.sha256" --filter "- *" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:163: rm -rf "$VDIR"
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:166: rclone copyto "$DEST/files/$bn" "$VBASE/$bn" --transfers 1 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:168: rm -f "$VBASE/$bn"
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:171: rm -rf "$VBASE"
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:184:rm -rf "$VERIFY_ROOT/$JOB_ID"
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:185:rm -rf "$CHUNK_ROOT/$JOB_ID"
|
|
/usr/local/sbin/homelab-gitea-secondary-backup:18: rm -f "$TMP"
|
|
/usr/local/sbin/homelab-gitea-secondary-backup:31:docker exec -u git -w /tmp gitea sh -lc \"rm -f /tmp/\$NAME && gitea dump -c /data/gitea/conf/app.ini -f /tmp/\$NAME\"
|
|
/usr/local/sbin/homelab-gitea-secondary-backup:33:docker exec gitea rm -f \"/tmp/\$NAME\"
|
|
/usr/local/sbin/homelab-gitea-secondary-backup:100: rm -f -- "$FILE" "$FILE.sha256"
|
|
/usr/local/sbin/homelab-gitea-secondary-backup:123: rm -f -- "$FILE" "$FILE.sha256"
|
|
/usr/local/sbin/homelab-gitea-secondary-backup:136: rm -f -- "$FILE" "$FILE.sha256"
|
|
/usr/local/sbin/homelab-rclone-copy-chunked:13:cleanup(){ rm -rf "$WORK"; }
|
|
/usr/local/sbin/homelab-rclone-copy-chunked:26:rclone copy "$PARTS" "${DEST}.parts" --transfers 2 --checkers 4 --retries 5 --low-level-retries 20 --stats 30s --log-level INFO
|
|
/usr/local/sbin/homelab-rclone-copy-chunked:27:rclone copyto "$WORK/${NAME}.sha256" "${DEST}.sha256" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO
|
|
/usr/local/sbin/homelab-rclone-copy-chunked:28:rclone copyto "$WORK/MANIFEST.txt" "${DEST}.manifest.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO
|
|
/usr/local/sbin/homelab-rclone-copy-chunked:29:rclone copyto "$WORK/MANIFEST.parts.txt" "${DEST}.parts/MANIFEST.parts.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO
|
|
/usr/local/sbin/homelab-rclone-copy-chunked:30:REMOTE_COUNT=$(rclone lsf "${DEST}.parts" --files-only | grep -c "^part-" || true)
|
|
/usr/local/sbin/homelab-router-hard-power-cycle:23:SAFE_OUT="$(printf '%s\n' "$OUT" | sed -E '/(PASSWORD|TOKEN|SECRET|PRIVATE_KEY|ROOT_PASSWORD|ENCRYPTION_KEY|COOKIE|SESSION|AUTHORIZATION|API_KEY|LOCAL_KEY|rclone[.]conf|[.]env)/Id')"
|
|
/usr/local/sbin/homelab-health-metrics-write:10:rm -f "$tmp"
|
|
/usr/local/sbin/homelab-rclone-cat-flat-chunked:5:trap "rm -f \"$TMP\"" EXIT
|
|
/usr/local/sbin/homelab-rclone-cat-flat-chunked:8:while read -r PART PART_SIZE; do [[ "$PART" =~ ^part-[0-9]{4}$ && "$PART_SIZE" =~ ^[0-9]+$ ]] || { echo "BAD FLAT PART MANIFEST" >&2; exit 2; }; rclone cat "${DEST}.${PART}"; COUNT=$((COUNT+1)); done < "$TMP"
|
|
/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:4:cat /var/lib/skladchik-reports-monitor-backup/restore-check-health.txt 2>/dev/null || true
|
|
/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:7:systemctl is-enabled skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/enabled=/' || true
|
|
/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:8:systemctl is-active skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/active=/' || true
|
|
/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:9:systemctl list-timers --all | grep skladchik-reports-monitor-backup-restore-check || true
|
|
/usr/local/sbin/homelab-context-refresh-direct:5:trap "rm -rf \"$d\"" EXIT
|
|
/usr/local/sbin/homelab-context-refresh-direct:32:printf "{\"schema_version\":1,\"channel\":\"homelab-runtime\",\"command_id\":\"%s\",\"status\":\"OK\",\"rc\":0,\"host\":\"pve01\",\"mode\":\"read-only\",\"component\":\"cluster-context\",\"started_at_utc\":\"%s\",\"finished_at_utc\":\"%s\",\"reference_register_checked\":true,\"reference_sha256\":\"%s\",\"error_register_checked\":true,\"error_register_sha256\":\"%s\",\"changes_made\":false,\"sanitized\":true,\"secrets_included\":false,\"private_addresses_included\":false,\"output\":\"AUTOMATIC_CLUSTER_CONTEXT_REFRESH=OK\"}\n" "$id" "$started" "$finished" "$ref" "$err" >"$d/$id.json"
|
|
/usr/local/sbin/homelab-context-refresh-direct:34:ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] debian@[PRIVATE_IP] "sudo -n /usr/local/sbin/homelab-runtime-receive $id /tmp/$id.json /tmp/$id.txt && rm -f /tmp/$id.json /tmp/$id.txt"
|
|
/usr/local/sbin/homelab-evidence-seal:16: rm -f "$TMP"
|
|
/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:15:DUMP_SIZE="$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$REMOTE" "sudo -n rm -f \"$REMOTE_DUMP\"; sudo -n -u clusteradmin pg_dump --format=custom --no-owner --no-privileges --file=\"$REMOTE_DUMP\" clusteradmin; sudo -n test -s \"$REMOTE_DUMP\"; sudo -n -u clusteradmin pg_restore --list \"$REMOTE_DUMP\" >/dev/null; sudo -n chown debian:debian \"$REMOTE_DUMP\"; sudo -n chmod 600 \"$REMOTE_DUMP\"; stat -c %s \"$REMOTE_DUMP\"")"
|
|
/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:18:ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$REMOTE" "rm -f \"$REMOTE_DUMP\""
|
|
/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:23:rm -f "$STAGE/runtime.tgz"
|
|
/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:25:jq -n --arg app "$APP" --arg host "cluster-admin" --arg engine "postgresql-custom-dump+files" --argjson files "$FILES" --argjson dump_size "$DUMP_SIZE" "{app:\$app,engine:\$engine,source_host:\$host,source_paths:[\"cluster-admin:/opt/cluster-admin-incident-engine\",\"cluster-admin:/etc/cluster-admin-incident-engine\",\"cluster-admin:/var/lib/cluster-admin-incident-engine\",\"cluster-admin:PostgreSQL clusteradmin\"],file_count:\$files,database_dump_size_bytes:\$dump_size,database_dump_format:\"custom\",database_restore_list_check:\"ok\",includes_runtime_credentials:true,note:\"Incident engine application database state and runtime configuration\"}" > "$STAGE/metadata.json"
|
|
/usr/local/sbin/appbackup-retention:32:trap "rm -rf \"$WORK\"" EXIT
|
|
/usr/local/sbin/appbackup-retention:35: rclone lsf "${REMOTE%/}" --dirs-only --max-depth 1 > "$ROOT_LIST"
|
|
/usr/local/sbin/appbackup-retention:36: rclone lsf "${REMOTE%/}/${NODE}/apps/${CANONICAL}" --dirs-only --max-depth 1 > "$LEGACY_LIST"
|
|
/usr/local/sbin/appbackup-retention:39: rclone lsf "${REMOTE%/}/hb" --dirs-only --max-depth 1 > "$HB_LIST"
|
|
/usr/local/sbin/appbackup-retention:40: if grep -Fxq "a/" "$HB_LIST"; then rclone lsf "${REMOTE%/}/hb/a" --dirs-only --max-depth 1 > "$NEW_LIST"; fi
|
|
/usr/local/sbin/appbackup-retention:73: rclone cat "${REMOTE%/}/${LATEST_REL}/manifest.json" | jq -e --arg job "$LATEST" ".job_id == \$job" >/dev/null || { echo "REFUSE latest missing/bad remote_index=$INDEX"; exit 5; }
|
|
/usr/local/sbin/homelab-change-transaction:195: latest_tmp.unlink(missing_ok=True)
|
|
/usr/local/sbin/homelab-change-transaction:421: " homelab-change-transaction run PLAN --confirm PLAN"
|
|
/usr/local/sbin/homelab-check-private-vpn-hosts:15:trap 'rm -f "$tmp"' EXIT
|
|
/usr/local/sbin/homelab-backup-vaultwarden:26:rm -rf "$RESTORE"
|
|
/usr/local/sbin/homelab-health-metrics:16:category_for_check(){
|
|
/usr/local/sbin/homelab-health-metrics:22: *backup*|*vzdump*) printf "%s" "backup" ;;
|
|
/usr/local/sbin/homelab-health-metrics:31:tier_for_check(){
|
|
/usr/local/sbin/homelab-health-metrics:34: npmplus-kuma-config-*|authentik-*|gitea-*|vaultwarden-*|netbox-*|mail-cloud-restore-drill|edge-vm-vzdump-*|mkdocs-runtime-publish|npmplus-certificate-expiry) printf "%s" "P0" ;;
|
|
/usr/local/sbin/homelab-health-metrics:69: check="${base%.txt}"
|
|
/usr/local/sbin/homelab-health-metrics:70: category="$(category_for_check "$check")"
|
|
/usr/local/sbin/homelab-health-metrics:71: tier="$(tier_for_check "$check")"
|
|
/usr/local/sbin/homelab-health-metrics:75: check_esc="$(esc_label "$check")"
|
|
/usr/local/sbin/homelab-health-metrics:79: labels="host=\"$host_esc\",check=\"$check_esc\",category=\"$cat_esc\",tier=\"$tier_esc\",file=\"$file_esc\""
|
|
/usr/local/sbin/homelab-backup-health-evidence:8:backupctl check >/tmp/backup-health-emergency.log 2>&1 && EMERGENCY_OK=1 || true
|
|
/usr/local/sbin/homelab-backup-health-evidence:9:appbackupctl check >/tmp/backup-health-appframework.log 2>&1 && APPS_OK=1 || true
|
|
/usr/local/sbin/homelab-drift-check:7:HEALTH=/var/lib/homelab-health/drift-check.txt
|
|
/usr/local/sbin/homelab-drift-check:24:echo "STATUS=$STATUS TS=$(date -u +%Y-%m-%dT%H:%M:%SZ) TYPE=drift-check NOTE=$NOTE DIFF=$DIFF BASE=$BASE CURRENT=$CUR" > "$HEALTH"
|
|
/usr/local/sbin/crypto-portfolio-chart-health-check:5:trap 'rm -f "$TMP"' EXIT
|
|
/usr/local/sbin/homelab-rclone-copy-flat-chunked:12:trap "rm -rf \"$WORK\"" EXIT
|
|
/usr/local/sbin/homelab-rclone-copy-flat-chunked:23:while read -r PART PART_SIZE; do [[ "$PART" =~ ^part-[0-9]{4}$ && "$PART_SIZE" =~ ^[0-9]+$ ]] || { echo "BAD PART MANIFEST" >&2; exit 4; }; rclone copyto "$WORK/parts/$PART" "${DEST}.${PART}" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR; REMOTE_BYTES="$(rclone size --json "${DEST}.${PART}" | jq -r ".bytes // -1")"; [[ "$REMOTE_BYTES" = "$PART_SIZE" ]] || { echo "REMOTE PART SIZE MISMATCH $PART" >&2; exit 5; }; UPLOADED=$((UPLOADED+1)); done < "$WORK/MANIFEST.parts.txt"
|
|
/usr/local/sbin/homelab-rclone-copy-flat-chunked:25:rclone copyto "$WORK/${NAME}.sha256" "${DEST}.sha256" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR
|
|
/usr/local/sbin/homelab-rclone-copy-flat-chunked:26:rclone copyto "$WORK/MANIFEST.txt" "${DEST}.manifest.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR
|
|
/usr/local/sbin/homelab-rclone-copy-flat-chunked:27:rclone copyto "$WORK/MANIFEST.parts.txt" "${DEST}.parts.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR
|
|
/usr/local/sbin/homelab-final-readiness-gate:10:check(){ name="$1"; shift; echo "### $name" >> "$REPORT"; timeout "${CHECK_TIMEOUT_SECONDS:-300}" "$@" >> "$REPORT" 2>&1; rc=$?; if [ "$rc" -eq 0 ]; then echo "PASS $name" >> "$REPORT"; PASS=$((PASS+1)); else echo "FAIL $name rc=$rc" >> "$REPORT"; FAIL=$((FAIL+1)); BAD=$((BAD+1)); fi; echo "" >> "$REPORT"; }
|
|
/usr/local/sbin/homelab-final-readiness-gate:18:check "backup framework evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/backup-framework.txt && grep -q SOURCE_STATUS=OK /var/lib/homelab-health/backup-framework.txt && grep -q FRESHNESS=FRESH /var/lib/homelab-health/backup-framework.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:21:check "backup SLA evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/backup-sla.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:22:check "backup coverage matrix evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/backup-coverage-matrix.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:23:check "extended appbackup evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/extended-appbackup.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:24:check "residual review" bash -c "grep -q STATUS=OK /var/lib/homelab-health/residual-review.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:25:check "forum snuffleupagus" bash -c "grep -q STATUS=OK /var/lib/homelab-health/forum-snuffleupagus.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:26:check "vm170 vm171 full strategy" bash -c "grep -q STATUS=OK /var/lib/homelab-health/vm170-vm171-full-strategy.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:27:check "external probe vps" bash -c "grep -q STATUS=OK /var/lib/homelab-health/external-probe-vps.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:28:check "netbird vps trust" bash -c "grep -q STATUS=OK /var/lib/homelab-health/netbird-vps-trust.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:29:check "netbird vps trust closure" bash -c "grep -q STATUS=OK /var/lib/homelab-health/netbird-vps-trust-closure.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:30:check "external probe runner decision" bash -c "grep -q STATUS=OK /var/lib/homelab-health/external-probe-runner-decision.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:31:check "netbird vps trust clean seal" bash -c "grep -q STATUS=OK /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:32:check "live tail audit" bash -c "grep -q STATUS=OK /var/lib/homelab-health/live-tail-audit.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:34:check "duty admin v2" bash -c "grep -q STATUS=OK /var/lib/homelab-health/duty-admin-v2.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:35:check "cluster admin observer" bash -c "grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-observer.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:36:check "cluster admin restricted probes" bash -c "grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-restricted-probes.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:37:check "cluster admin full observer" bash -c "grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-full-observer.txt && grep -q BAD=0 /var/lib/homelab-health/cluster-admin-full-observer.txt && grep -q WARN=0 /var/lib/homelab-health/cluster-admin-full-observer.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:38:check "cluster admin VM Mail Cloud backup" bash -c "grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt && grep -q BAD=0 /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt && grep -q MAIL_CLOUD_2COPY_VERIFY_OK /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:39:check "cluster admin webpanel sync" bash -c "grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-webpanel-sync.txt && grep -q BAD=0 /var/lib/homelab-health/cluster-admin-webpanel-sync.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:40:check "cluster admin webpanel" bash -c "grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-webpanel.txt && grep -q BAD=0 /var/lib/homelab-health/cluster-admin-webpanel.txt && curl -fsS --max-time 5 http://[PRIVATE_IP]:8080/status.txt | grep -q STATUS=OK"
|
|
/usr/local/sbin/homelab-final-readiness-gate:41:check "node loss readiness" bash -c "grep -q STATUS=OK /var/lib/homelab-health/node-loss-readiness.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:42:check "node loss runbooks" bash -c "grep -q STATUS=OK /var/lib/homelab-health/node-loss-runbooks.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:43:check "power loss readiness" bash -c "grep -q STATUS=OK /var/lib/homelab-health/power-loss-readiness.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:44:check "power loss runbook" bash -c "grep -q STATUS=OK /var/lib/homelab-health/power-loss-runbook.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:45:check "ungated health backlog" bash -c "grep -q STATUS=OK /var/lib/homelab-health/ungated-health-backlog.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:46:check "pve03 root cleanup" bash -c "grep -q STATUS=OK /var/lib/homelab-health/pve03-root-cleanup.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:47:check "pve03 staging retention" bash -c "grep -q STATUS=OK /var/lib/homelab-health/pve03-staging-retention.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:48:check "pve03 staging retention proof" bash -c "grep -q STATUS=OK /var/lib/homelab-health/pve03-staging-retention-proof.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:49:check "pve03 staging retention cleanup" bash -c "grep -q STATUS=OK /var/lib/homelab-health/pve03-staging-retention-cleanup.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:50:check "pve03 failed unit cleanup" bash -c "grep -q STATUS=OK /var/lib/homelab-health/pve03-failed-unit-cleanup.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:51:check "remote git sops age readiness" bash -c "grep -q STATUS=OK /var/lib/homelab-health/remote-git-sops-age-readiness.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:52:check "remote git sops age policy" bash -c "grep -q STATUS=OK /var/lib/homelab-health/remote-git-sops-age-policy.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:53:check "desired state remote target trace" bash -c "grep -q STATUS=OK /var/lib/homelab-health/desired-state-remote-target-trace.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:54:check "desired state remote target" bash -c "grep -q STATUS=OK /var/lib/homelab-health/desired-state-remote-target.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:55:check "Kuma monitor policy evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/kuma-monitor-policy.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:56:check "safe autoheal evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/safe-autoheal.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:57:check "desired-state clean evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/desired-state.txt && test -z "$(git -C /srv/homelab-desired-state status --short)""
|
|
/usr/local/sbin/homelab-final-readiness-gate:58:check "runbooks evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/runbooks.txt && test -s /var/lib/homelab-runbooks/INDEX.md"
|
|
/usr/local/sbin/homelab-final-readiness-gate:59:check "incident journal evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/incident-journal.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:60:check "alerting evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/alerting.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:61:check "VM backup health evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/vm-backup.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:62:check "VM backup policy evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/vm-backup-policy.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:63:check "VM local dumps cloud" bash -c "grep -q STATUS=OK /var/lib/homelab-health/vm-local-dumps-cloud.txt && [ $(jq -r .count /var/lib/homelab-vm-backup/latest-vm-local-dumps-cloud.json) -ge 3 ] && [ $(jq -r .chunked_count /var/lib/homelab-vm-backup/latest-vm-local-dumps-cloud.json) -ge 1 ]"
|
|
/usr/local/sbin/homelab-final-readiness-gate:64:check "VM local dumps retention" bash -c "grep -q STATUS=OK /var/lib/homelab-health/vm-local-dumps-retention.txt && grep -q ACTIVE_LEFT=0 /var/lib/homelab-health/kuma-monitor-policy.txt || grep -q STATUS=OK /var/lib/homelab-health/vm-local-dumps-retention.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:65:check "capacity risk evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/capacity-risk.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:66:check "secret exposure evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/secret-exposure.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:67:check "golden state evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/golden-state.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:68:check "service registry evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/service-registry.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:69:check "dependency map evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/dependency-map.txt"
|
|
/usr/local/sbin/homelab-docker-health.sh:44: echo "containers_without_healthcheck=$NOHEALTH"
|
|
/usr/local/sbin/homelab-docker-health.sh:63: echo "# HELP homelab_docker_health_container_without_healthcheck Docker containers without native healthcheck."
|
|
/usr/local/sbin/homelab-docker-health.sh:64: echo "# TYPE homelab_docker_health_container_without_healthcheck gauge"
|
|
/usr/local/sbin/homelab-docker-health.sh:65: echo "homelab_docker_health_container_without_healthcheck $NOHEALTH"
|
|
/usr/local/sbin/homelab-backup-coverage-matrix:17:P1_SAFE_APPS="actual-budget homebox linkding mealie memos node-red syncthing filebrowser healthchecks ntfy vikunja homeassistant karakeep minio n8n"
|
|
/usr/local/sbin/homelab-backup-coverage-matrix:38: while read -r norm raw; do
|
|
/usr/local/sbin/homelab-backup-coverage-matrix:50: while read -r norm raw; do
|
|
/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:93:rclone mkdir "$TARGET_REMOTE:$REMOTE_PARTS" > "$WORK/logs/mkdir.out" 2>"$WORK/logs/mkdir.err" || MKDIR_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:95:rclone copy "$PARTS_DIR" "$TARGET_REMOTE:$REMOTE_PARTS" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > "$WORK/logs/upload_parts.out" 2>"$WORK/logs/upload_parts.err" || UPLOAD_PARTS_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:97:rclone copyto "$WORK/$BASENAME.sha256" "$TARGET_REMOTE:$REMOTE_SHA" --checksum > "$WORK/logs/upload_sha.out" 2>"$WORK/logs/upload_sha.err" || UPLOAD_SHA_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:99:rclone copyto "$WORK/MANIFEST.txt" "$TARGET_REMOTE:$REMOTE_MANIFEST" --checksum > "$WORK/logs/upload_manifest.out" 2>"$WORK/logs/upload_manifest.err" || UPLOAD_MANIFEST_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:101:rclone copyto "$WORK/SHA256SUMS.txt" "$TARGET_REMOTE:$REMOTE_SUMS" --checksum > "$WORK/logs/upload_sums.out" 2>"$WORK/logs/upload_sums.err" || UPLOAD_SUMS_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:104:rclone lsf "$TARGET_REMOTE:$REMOTE_PARTS" --format ps > "$WORK/logs/remote_lsf_parts.txt" 2>"$WORK/logs/remote_lsf_parts.err" || LSF_PARTS_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:110:rclone copy "$TARGET_REMOTE:$REMOTE_PARTS" "$DL_DIR/parts" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > "$WORK/logs/download_parts.out" 2>"$WORK/logs/download_parts.err" || DOWNLOAD_PARTS_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:120:rclone lsf "$TARGET_REMOTE:$TARGET_BASE" --dirs-only 2>/dev/null | sed 's#/$##' | sort > "$DIR_LIST" || RETENTION_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:126: [ -n "$OLD" ] && rclone purge "$TARGET_REMOTE:$TARGET_BASE/$OLD" >> "$WORK/logs/retention.out" 2>>"$WORK/logs/retention.err"
|
|
/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:140: rm -rf "$PARTS_DIR" "$DL_DIR"
|
|
/usr/local/sbin/homelab-rehydrate-flat-chunked-dir:10: rm -f "$TMP_TARGET"
|
|
/usr/local/sbin/homelab-disk-space-health.py:10:checks = [
|
|
/usr/local/sbin/homelab-disk-space-health.py:21:for label, cmd in checks:
|
|
/usr/local/sbin/homelab-rclone-required-check.sh:4:RAW_FILE="/etc/homelab/rclone/required-remotes.txt"
|
|
/usr/local/sbin/homelab-rclone-required-check.sh:5:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt"
|
|
/usr/local/sbin/homelab-rclone-required-check.sh:17:CONFIGURED="$(rclone listremotes 2>/dev/null | sed 's/:$//' || true)"
|
|
/usr/local/sbin/homelab-rclone-required-check.sh:19: echo "No rclone remotes configured yet."
|
|
/usr/local/sbin/homelab-legacy-monitoring-backups-run:24: rm -f "$STAGE/remote.tgz"
|
|
/usr/local/sbin/homelab-legacy-monitoring-backups-run:43: rm -f "$STAGE/pve01.tgz"
|
|
/usr/local/sbin/homelab-legacy-monitoring-backups-run:48: rm -f "$STAGE/$LABEL.tgz"
|
|
/usr/local/sbin/homelab-legacy-monitoring-backups-run:52: rm -f "$STAGE/monitoring.tgz"
|
|
/usr/local/sbin/homelab-legacy-monitoring-backups-run:71: rm -f "$STAGE/config.tgz"
|
|
/usr/local/sbin/homelab-cluster-admin-inventory-sync:19:ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$REMOTE" "rm -f $REMOTE_TMP"
|
|
/usr/local/sbin/homelab-chat-run:390: "error_register_checked": True,
|
|
/usr/local/sbin/homelab-chat-run:489: rm -f \
|
|
/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:10: rm -rf "$TMPD" 2>/dev/null || true
|
|
/usr/local/sbin/homelab-beszel-backup-run:33:rm -f "$STAGE/remote.tgz"
|
|
/usr/local/sbin/homelab-beszel-backup-run:37:while IFS= read -r DB; do DB_COUNT=$((DB_COUNT+1)); RESULT="$(python3 -c "import sqlite3,sys;print(sqlite3.connect(sys.argv[1]).execute(\"PRAGMA integrity_check\").fetchone()[0])" "$DB" 2>/dev/null || echo error)"; [ "$RESULT" = ok ] || exit 4; DB_OK=$((DB_OK+1)); done < <(find "$STAGE/payload" -type f \( -name "*.db" -o -name "*.sqlite" -o -name "*.sqlite3" \) 2>/dev/null)
|
|
/usr/local/sbin/homelab-runbook-generate:47: f"Do not restore over a live service blindly. First confirm both encrypted cloud copies pass restore-check, target VM/container is snapshotted or intentionally stopped, and dependency map is OK.\n\n"
|
|
/usr/local/sbin/homelab-runbook-generate:51: f"homelab-service-registry-check\n"
|
|
/usr/local/sbin/skladchik-reports-monitor-supervisor:25:PVE_RESTORE_HEALTH = Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt")
|
|
/usr/local/sbin/skladchik-reports-monitor-supervisor:254: "skladchik-reports-monitor-backup-restore-check.timer",
|
|
/usr/local/sbin/backup-restore-check:9:rm -rf "$ROOT" && mkdir -p "$ROOT/a" "$ROOT/b"
|
|
/usr/local/sbin/backup-restore-check:10:trap 'echo "ERROR restore-check failed; kept: "; exit 1' ERR
|
|
/usr/local/sbin/backup-restore-check:12:rclone copy "$RA" "$ROOT/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-a.log 2>&1; homelab-rehydrate-chunked-dir "$ROOT/a" >>/tmp/homelab-restore-a.log 2>&1
|
|
/usr/local/sbin/backup-restore-check:13:(cd "$ROOT/a" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-a-sha.log && ARCHIVE="$(jq -r .archive_name manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null)
|
|
/usr/local/sbin/backup-restore-check:15:rclone copy "$RB" "$ROOT/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-b.log 2>&1; homelab-rehydrate-chunked-dir "$ROOT/b" >>/tmp/homelab-restore-b.log 2>&1
|
|
/usr/local/sbin/backup-restore-check:16:(cd "$ROOT/b" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-b-sha.log && ARCHIVE="$(jq -r .archive_name manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null)
|
|
/usr/local/sbin/backup-restore-check:18:rm -rf "$ROOT"
|
|
/usr/local/sbin/homelab-vps-identity-audit:8:trap 'rm -rf "$TMP"' EXIT
|
|
/usr/local/sbin/homelab-vm-full-restore-drill:184:check_container_health() {
|
|
/usr/local/sbin/homelab-vm-full-restore-drill:349:check_container_health() {
|
|
/usr/local/sbin/homelab-vm-full-restore-drill:386:check_container_health "$test_vmid"
|
|
/usr/local/sbin/homelab-vm-backup-health:8:check_file(){
|
|
/usr/local/sbin/homelab-vm-backup-health:15:check_file /var/lib/homelab-health/backup.txt auto-vmct
|
|
/usr/local/sbin/homelab-vm-backup-health:16:check_file /var/lib/homelab-health/edge-vm-vzdump-backup.txt edge-vm-backup
|
|
/usr/local/sbin/homelab-vm-backup-health:17:check_file /var/lib/homelab-health/edge-vm-vzdump-offhost.txt edge-vm-offhost
|
|
/usr/local/sbin/homelab-vm-backup-health:18:check_file /var/lib/homelab-health/edge-vm-vzdump-restore.txt edge-vm-restore
|
|
/usr/local/sbin/homelab-ingress-failover:37: rm -f "$RESOURCES"
|
|
/usr/local/sbin/homelab-ingress-failover:252: rm -f "$tmp"
|
|
/usr/local/sbin/homelab-overall-health:8: /var/lib/homelab-health/drift-check.txt
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:64:case "$SOURCE" in /mnt/staging/offhost/edge-vm-vzdump-from-pve03/vzdump-qemu-130-*.vma.zst) ;; *) fail source_path_bad 2 ;; esac
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:111:rclone about "$TARGET_REMOTE:" > "$WORK/logs/about.out" 2>"$WORK/logs/about.err" || ABOUT_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:114:DIRS="$(rclone lsf "$TARGET_REMOTE:$TARGET_BASE" --dirs-only 2>/dev/null | sed 's#/$##' | sort || true)"
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:116: if rclone cat "$TARGET_REMOTE:$TARGET_BASE/$D/$BASENAME.sha256" 2>/dev/null | grep -q "$SHA"; then
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:188:rclone mkdir "$TARGET_REMOTE:$REMOTE_PARTS" > "$WORK/logs/mkdir.out" 2>"$WORK/logs/mkdir.err" || MKDIR_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:190:rclone copy "$PARTS_DIR" "$TARGET_REMOTE:$REMOTE_PARTS" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > "$WORK/logs/upload_parts.out" 2>"$WORK/logs/upload_parts.err" || UPLOAD_PARTS_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:192:rclone copyto "$WORK/$BASENAME.sha256" "$TARGET_REMOTE:$REMOTE_SHA" --checksum > "$WORK/logs/upload_sha.out" 2>"$WORK/logs/upload_sha.err" || UPLOAD_SHA_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:194:rclone copyto "$WORK/MANIFEST.txt" "$TARGET_REMOTE:$REMOTE_MANIFEST" --checksum > "$WORK/logs/upload_manifest.out" 2>"$WORK/logs/upload_manifest.err" || UPLOAD_MANIFEST_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:196:rclone copyto "$WORK/SHA256SUMS.txt" "$TARGET_REMOTE:$REMOTE_SUMS" --checksum > "$WORK/logs/upload_sums.out" 2>"$WORK/logs/upload_sums.err" || UPLOAD_SUMS_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:199:rclone size "$TARGET_REMOTE:$TARGET_DIR" --json > "$WORK/logs/remote_size.json" 2>"$WORK/logs/remote_size.err" || SIZE_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:201:rclone lsf "$TARGET_REMOTE:$REMOTE_PARTS" --format "ps" > "$WORK/logs/remote_lsf_parts.txt" 2>"$WORK/logs/remote_lsf_parts.err" || LSF_PARTS_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:207:rclone copy "$TARGET_REMOTE:$REMOTE_PARTS" "$DL_DIR/parts" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > "$WORK/logs/download_parts.out" 2>"$WORK/logs/download_parts.err" || DOWNLOAD_PARTS_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:222: rm -rf "$DL_DIR" "$PARTS_DIR"
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:234: rclone lsf "$TARGET_REMOTE:$TARGET_BASE" --dirs-only 2>/dev/null | sed "s#/$##" | sort > "$DIR_LIST" || RETENTION_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:239: [ -n "$OLD" ] && rclone purge "$TARGET_REMOTE:$TARGET_BASE/$OLD" >> "$WORK/logs/retention.out" 2>>"$WORK/logs/retention.err"
|
|
/usr/local/sbin/homelab-backup-edge-vm-vzdump:40:REMOTE_OUT="$(remote "OUT=/tmp/edge_vm_vzdump_${VMID}_${TS}.out; ERR=/tmp/edge_vm_vzdump_${VMID}_${TS}.err; : > \$OUT; : > \$ERR; vzdump '$VMID' --mode snapshot --compress zstd --dumpdir '$RUN_DIR' --remove 0 >\$OUT 2>\$ERR; RC=\$?; ARCHIVE=\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\n' | sort -n | tail -n1 | cut -d' ' -f2-); LOGFILE=\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.log' -printf '%T@ %p\n' | sort -n | tail -n1 | cut -d' ' -f2-); SIZE=0; SHA=none; if [ -n \"\$ARCHIVE\" ] && [ -f \"\$ARCHIVE\" ]; then SIZE=\$(stat -c %s \"\$ARCHIVE\"); SHA=\$(sha256sum \"\$ARCHIVE\" | awk '{print \$1}'); fi; echo VZDUMP_RC=\$RC; echo ARCHIVE=\$ARCHIVE; echo LOGFILE=\$LOGFILE; echo SIZE=\$SIZE; echo SHA=\$SHA; echo OUT_TAIL=\$(tail -n 6 \$OUT | tr '\n' '|'); echo ERR_TAIL=\$(tail -n 6 \$ERR | tr '\n' '|')")"
|
|
/usr/local/sbin/homelab-backup-edge-vm-vzdump:62:dest "mkdir -p '$OFFHOST_DIR' && rm -f '$DEST_PATH.tmp' '$DEST_PATH.sha256'"
|
|
/usr/local/sbin/homelab-backup-edge-vm-vzdump:74:echo "STATUS=OK TS=$TS TYPE=edge-vm-vzdump-backup NODE=$NODE VMID=$VMID ARCHIVE=$ARCHIVE SIZE=$SIZE SHA256=$SHA SCSI1_BACKUP0=yes NOTES=timer_run" > "$HEALTH_DIR/edge-vm-vzdump-backup.txt"
|
|
/usr/local/sbin/homelab-backup-edge-vm-vzdump:75:echo "STATUS=OK TS=$TS TYPE=edge-vm-vzdump-offhost NODE=$NODE VMID=$VMID HOST=pve02 DEST=$DEST_PATH SIZE=$SIZE SHA256=$SHA NOTES=timer_run" > "$HEALTH_DIR/edge-vm-vzdump-offhost.txt"
|
|
/usr/local/sbin/homelab-backup-edge-vm-vzdump:76:echo "STATUS=OK TS=$TS TYPE=edge-vm-vzdump-restore-verify HOST=pve02 VMID=$VMID ARCHIVE=$DEST_PATH SIZE=$SIZE SHA256=$SHA ZSTD_RC=0 VMA_RC=0 NOTES=timer_run" > "$HEALTH_DIR/edge-vm-vzdump-restore.txt"
|
|
/usr/local/sbin/homelab-backup-edge-vm-vzdump:77:chmod 0644 "$HEALTH_DIR/edge-vm-vzdump-backup.txt" "$HEALTH_DIR/edge-vm-vzdump-offhost.txt" "$HEALTH_DIR/edge-vm-vzdump-restore.txt"
|
|
/usr/local/sbin/homelab-backup-edge-vm-vzdump:79:remote "find '$BASE_DIR' -mindepth 1 -maxdepth 1 -type d -name 'edge-vm-*' -printf '%T@ %p\n' | sort -nr | awk 'NR>${KEEP_LOCAL}{print \$2}' | xargs -r rm -rf"
|
|
/usr/local/sbin/homelab-backup-edge-vm-vzdump:80:dest "find '$OFFHOST_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\n' | sort -nr | awk 'NR>${KEEP_OFFHOST}{print \$2}' | while read f; do rm -f \"\$f\" \"\$f.sha256\"; done"
|
|
/usr/local/sbin/appbackup-dockge-stacks:30:rm -rf "$PAYLOAD"
|
|
/usr/local/sbin/appbackup-dockge-stacks:51:echo "[STEP] local checksum"
|
|
/usr/local/sbin/appbackup-dockge-stacks:52:(cd "$STAGE" && sha256sum "$ARCHIVE" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e . manifest.json >/dev/null)
|
|
/usr/local/sbin/appbackup-dockge-stacks:54:echo "[STEP] upload cloud A metadata"; rclone copy "$STAGE" "$RA" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s
|
|
/usr/local/sbin/appbackup-dockge-stacks:56:echo "[STEP] upload cloud B metadata"; rclone copy "$STAGE" "$RB" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s
|
|
/usr/local/sbin/appbackup-dockge-stacks:60:echo "[STEP] upload latest index"; rclone copyto "$STAGE/latest-backup-index.json" "$RAL"; rclone copyto "$STAGE/latest-backup-index.json" "$RBL"; rclone cat "$RAL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null; rclone cat "$RBL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null
|
|
/usr/local/sbin/appbackup-dockge-stacks:76:rm -rf "$STAGE" "$VERIFY"
|
|
/usr/local/sbin/homelab-cluster-passport:65:need_ok /var/lib/homelab-health/drift-check.txt drift-check
|
|
/usr/local/sbin/backup-retention:18: if [[ "$MODE" == "apply" ]]; then rclone cat "$BASE/$LATEST/manifest.json" | jq -e --arg job "$LATEST" ".job_id == \$job" >/dev/null || { echo "REFUSE latest missing/bad on $BASE"; exit 4; }; fi
|
|
/usr/local/sbin/backup-retention:24: for ((i=0; i<DELETE_COUNT; i++)); do TARGET_NAME="${DIRS[$i]}"; TARGET="$BASE/$TARGET_NAME"; [[ -n "$LATEST" && "$TARGET_NAME" == "$LATEST" ]] && { echo "REFUSE delete latest $TARGET"; exit 5; }; if [[ "$MODE" == "apply" ]]; then echo "DELETE $TARGET"; rclone purge "$TARGET"; else echo "WOULD_DELETE $TARGET"; fi; done
|
|
/usr/local/sbin/homelab-external-probe-vps-health:49:closed_tail=smoke_checks_endpoint_reachability_and_status_prefix_not_overall_ok
|
|
/usr/local/bin/skladchik-status:11:check_health() {
|
|
/usr/local/bin/skladchik-status:24:check_timer() {
|
|
/usr/local/bin/skladchik-status:38:check_timer skladchik-reports-monitor-backup.timer
|
|
/usr/local/bin/skladchik-status:39:check_timer skladchik-reports-monitor-backup-restore-check.timer
|
|
/usr/local/bin/skladchik-status:40:check_timer skladchik-reports-monitor-supervisor.timer
|
|
/usr/local/bin/skladchik-status:41:check_timer skladchik-reports-monitor-monthly-selftest-watch.timer
|
|
/usr/local/bin/skladchik-status:43:check_health "PVE01: ежедневный бэкап" "/var/lib/skladchik-reports-monitor-backup/health.txt"
|
|
/usr/local/bin/skladchik-status:44:check_health "PVE01: проверка восстановления" "/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt"
|
|
/usr/local/bin/skladchik-status:45:check_health "PVE01: общий supervisor" "/var/lib/skladchik-reports-monitor-supervisor/health.txt"
|
|
/usr/local/bin/skladchik-status:46:check_health "PVE01: контроль ежемесячного самотеста" "/var/lib/skladchik-reports-monitor-selftest-watch/health.txt"
|
|
/usr/local/bin/hchat:36: echo "USAGE=hchat tx-run PLAN --confirm PLAN"
|
|
/usr/local/bin/hchat:112: --confirm \
|
|
/etc/systemd/system/homelab-private-vpn-hosts-health.timer:2:Description=Run Homelab private VPN hosts health check every minute
|
|
/etc/systemd/system/homelab-vm-full-backup.service:2:Description=Full vzdump backup of VM170 and VM171 to offhost and two clouds
|
|
/etc/systemd/system/homelab-mail-cloud-restore-drill.service.d/10-superseded-noop.conf:3:ExecStart=/usr/local/sbin/homelab-superseded-unit-ok homelab-mail-cloud-restore-drill.service superseded_by_safety_net_restore_check
|
|
/etc/systemd/system/homelab-drift-check.service:2:Description=Homelab drift check
|
|
/etc/systemd/system/homelab-drift-check.service:6:ExecStart=/usr/local/sbin/homelab-drift-check
|
|
/etc/systemd/system/homelab-safe-exec-health.timer:2:Description=Regular allowlisted homelab health checks
|
|
/etc/systemd/system/suspend.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
/etc/systemd/system/homelab-crypto-portfolio-chart-health.timer:2:Description=Recurring Crypto portfolio chart health check
|
|
/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:14:ExecStart=/usr/sbin/pve-ha-crm start
|
|
/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:15:ExecStop=/usr/sbin/pve-ha-crm stop
|
|
/etc/systemd/system/multi-user.target.wants/postfix.service:19:# perform 2-stage startup
|
|
/etc/systemd/system/multi-user.target.wants/postfix.service:20:ExecStartPre=+postfix check
|
|
/etc/systemd/system/multi-user.target.wants/postfix.service:30:# Individual Postfix daemons are started as root, optionally perform chroot
|
|
/etc/systemd/system/multi-user.target.wants/postfix.service:52:# CAP_CHOWN can be removed.
|
|
/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:22:ExecStart=/usr/sbin/pve-ha-lrm start
|
|
/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:23:ExecStop=/usr/sbin/pve-ha-lrm stop
|
|
/etc/systemd/system/multi-user.target.wants/homelab-scrutiny.service:10:ExecStartPre=-/usr/bin/podman rm -f scrutiny
|
|
/etc/systemd/system/multi-user.target.wants/homelab-scrutiny.service:11:ExecStart=/usr/bin/podman run --name scrutiny --rm --pull=never --privileged --device /dev/nvme0 --device /dev/nvme0n1 -v /run/udev:/run/udev:ro -v /opt/stacks/scrutiny/config:/opt/scrutiny/config -v /opt/stacks/scrutiny/influxdb:/opt/scrutiny/influxdb -p 127.0.0.1:8086:8080 ghcr.io/analogj/scrutiny:latest-omnibus
|
|
/etc/systemd/system/multi-user.target.wants/rrdcached.service:2:# To customize, copy this file to /etc/systemd/system/
|
|
/etc/systemd/system/multi-user.target.wants/pve-guests.service:22:ExecStop=-/usr/bin/vzdump -stop
|
|
/etc/systemd/system/multi-user.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
/etc/systemd/system/multi-user.target.wants/e2scrub_reap.service:2:Description=Remove Stale Online ext4 Metadata Check Snapshots
|
|
/etc/systemd/system/multi-user.target.wants/corosync.service:20:# as well; before editing this line, you may want to check DefaultStandardError
|
|
/etc/systemd/system/homelab-dependency-map-check.service:2:Description=Homelab dependency map check
|
|
/etc/systemd/system/homelab-dependency-map-check.service:6:ExecStart=/usr/local/sbin/homelab-dependency-map-check
|
|
/etc/systemd/system/homelab-edge-vm-offhost-freshness.timer:2:Description=Run Homelab edge-vm VM130 offhost freshness sync daily after nightly vzdump
|
|
/etc/systemd/system/homelab-dependency-map-check.timer:2:Description=Homelab dependency map check timer
|
|
/etc/systemd/system/pve-manager.service:22:ExecStop=-/usr/bin/vzdump -stop
|
|
/etc/systemd/system/homelab-alerting-health.service:2:Description=Homelab alerting health check
|
|
/etc/systemd/system/homelab-edge-vm-vzdump-backup.service:2:Description=Homelab edge-vm VM-level vzdump backup chain
|
|
/etc/systemd/system/homelab-edge-vm-vzdump-backup.service:8:ExecStart=/usr/local/sbin/homelab-backup-edge-vm-vzdump
|
|
/etc/systemd/system/default.target.wants/nvmefc-boot-connections.service:19:RemoveIPC=yes
|
|
/etc/systemd/system/default.target.wants/nvmf-autoconnect.service:21:RemoveIPC=yes
|
|
/etc/systemd/system/homelab-scrutiny.service:10:ExecStartPre=-/usr/bin/podman rm -f scrutiny
|
|
/etc/systemd/system/homelab-scrutiny.service:11:ExecStart=/usr/bin/podman run --name scrutiny --rm --pull=never --privileged --device /dev/nvme0 --device /dev/nvme0n1 -v /run/udev:/run/udev:ro -v /opt/stacks/scrutiny/config:/opt/scrutiny/config -v /opt/stacks/scrutiny/influxdb:/opt/scrutiny/influxdb -p 127.0.0.1:8086:8080 ghcr.io/analogj/scrutiny:latest-omnibus
|
|
/etc/systemd/system/skladchik-reports-monitor-backup-restore-check.service:2:Description=Restore-check Skladchik reports monitor cloud backups
|
|
/etc/systemd/system/skladchik-reports-monitor-backup-restore-check.service:8:ExecStart=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check
|
|
/etc/systemd/system/homelab-gitea-secondary-backup.service:2:Description=Create and copy a verified Gitea backup to pve03
|
|
/etc/systemd/system/homelab-crypto-portfolio-chart-health.service:2:Description=Crypto portfolio chart health check
|
|
/etc/systemd/system/homelab-crypto-portfolio-chart-health.service:8:ExecStart=/usr/local/sbin/crypto-portfolio-chart-health-check
|
|
/etc/systemd/system/skladchik-reports-monitor-supervisor.service:7:ExecStart=/usr/local/sbin/skladchik-reports-monitor-supervisor check
|
|
/etc/systemd/system/homelab-service-registry-check.service:2:Description=Homelab service registry health check
|
|
/etc/systemd/system/homelab-service-registry-check.service:6:ExecStart=/usr/local/sbin/homelab-service-registry-check
|
|
/etc/systemd/system/homelab-backup-healthcheck.service:2:Description=Homelab backup healthcheck
|
|
/etc/systemd/system/homelab-backup-healthcheck.service:6:ExecStart=/root/homelab-backup-healthcheck-wrapper.sh
|
|
/etc/systemd/system/nut-driver@cyberpower.service.d/nut-driver-enumerator-generated.conf:2:# DO NOT EDIT: This file would be removed or overwritten by that service
|
|
/etc/systemd/system/sysinit.target.wants/systemd-pstore.service:11:Description=Platform Persistent Storage Archival
|
|
/etc/systemd/system/sysinit.target.wants/pvenetcommit.service:8:ExecStartPre=-/bin/rm -f /etc/openvswitch/conf.db
|
|
/etc/systemd/system/hybrid-sleep.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
/etc/systemd/system/homelab-private-vpn-hosts-health.service:2:Description=Homelab private VPN hosts health check
|
|
/etc/systemd/system/homelab-private-vpn-hosts-health.service:8:ExecStart=/bin/bash -c 'set +e; /usr/local/sbin/homelab-check-private-vpn-hosts > /var/lib/homelab-health/private-vpn-hosts.txt.tmp; rc=$?; mv /var/lib/homelab-health/private-vpn-hosts.txt.tmp /var/lib/homelab-health/private-vpn-hosts.txt; chmod 0644 /var/lib/homelab-health/private-vpn-hosts.txt; exit $rc'
|
|
/etc/systemd/system/netbird-peers-health.service:2:Description=NetBird peers health check
|
|
/etc/systemd/system/homelab-pve03-staging-capacity-health.timer:2:Description=Run Homelab pve03 staging capacity health check
|
|
/etc/systemd/system/timers.target.wants/homelab-private-vpn-hosts-health.timer:2:Description=Run Homelab private VPN hosts health check every minute
|
|
/etc/systemd/system/timers.target.wants/homelab-crypto-portfolio-chart-health.timer:2:Description=Recurring Crypto portfolio chart health check
|
|
/etc/systemd/system/timers.target.wants/homelab-edge-vm-offhost-freshness.timer:2:Description=Run Homelab edge-vm VM130 offhost freshness sync daily after nightly vzdump
|
|
/etc/systemd/system/timers.target.wants/homelab-dependency-map-check.timer:2:Description=Homelab dependency map check timer
|
|
/etc/systemd/system/timers.target.wants/homelab-pve03-staging-capacity-health.timer:2:Description=Run Homelab pve03 staging capacity health check
|
|
/etc/systemd/system/timers.target.wants/homelab-drift-check.timer:2:Description=Homelab drift check timer
|
|
/etc/systemd/system/timers.target.wants/homelab-service-registry-check.timer:2:Description=Homelab service registry health check timer
|
|
/etc/systemd/system/timers.target.wants/homelab-backup-healthcheck.timer:2:Description=Run homelab backup healthcheck daily
|
|
/etc/systemd/system/timers.target.wants/homelab-backup-healthcheck.timer:8:Unit=homelab-backup-healthcheck.service
|
|
/etc/systemd/system/timers.target.wants/netbird-peers-health.timer:2:Description=Run NetBird peers health check
|
|
/etc/systemd/system/timers.target.wants/homelab-alerting-health.timer:2:Description=Homelab alerting health check timer
|
|
/etc/systemd/system/timers.target.wants/skladchik-reports-monitor-backup-restore-check.timer:2:Description=Weekly restore-check of Skladchik reports monitor cloud backups
|
|
/etc/systemd/system/timers.target.wants/skladchik-reports-monitor-backup-restore-check.timer:8:Unit=skladchik-reports-monitor-backup-restore-check.service
|
|
/etc/systemd/system/homelab-edge-vm-vzdump-backup.timer:8:Unit=homelab-edge-vm-vzdump-backup.service
|
|
/etc/systemd/system/homelab-drift-check.timer:2:Description=Homelab drift check timer
|
|
/etc/systemd/system/homelab-service-registry-check.timer:2:Description=Homelab service registry health check timer
|
|
/etc/systemd/system/homelab-pve03-staging-capacity-health.service:2:Description=Homelab pve03 staging capacity health check
|
|
/etc/systemd/system/homelab-backup-healthcheck.timer:2:Description=Run homelab backup healthcheck daily
|
|
/etc/systemd/system/homelab-backup-healthcheck.timer:8:Unit=homelab-backup-healthcheck.service
|
|
/etc/systemd/system/homelab-safe-exec-health.service:2:Description=Run allowlisted homelab health checks
|
|
/etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.service:2:Description=Homelab Mail Cloud Nextcloud VM vzdump upload
|
|
/etc/systemd/system/hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
/etc/systemd/system/netbird-peers-health.timer:2:Description=Run NetBird peers health check
|
|
/etc/systemd/system/homelab-alerting-health.timer:2:Description=Homelab alerting health check timer
|
|
/etc/systemd/system/skladchik-reports-monitor-backup-restore-check.timer:2:Description=Weekly restore-check of Skladchik reports monitor cloud backups
|
|
/etc/systemd/system/skladchik-reports-monitor-backup-restore-check.timer:8:Unit=skladchik-reports-monitor-backup-restore-check.service
|
|
/etc/systemd/system/suspend-then-hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
=== LOCAL_ARCHIVES ===
|
|
========== NODE=pve02 ==========
|
|
=== REMOTES ===
|
|
=== TIMERS ===
|
|
Tue 2026-07-21 18:41:03 MSK 3h 46min Tue 2026-07-21 12:41:22 MSK 2h 12min ago forum-prod-fresh5-cloud-backup.timer forum-prod-fresh5-cloud-backup.service
|
|
Wed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:22 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service
|
|
Wed 2026-07-22 04:20:00 MSK 13h Tue 2026-07-21 04:20:12 MSK 10h ago homelab-cluster-admin-vm-mail-cloud-backup.timer homelab-cluster-admin-vm-mail-cloud-backup.service
|
|
=== SERVICES ===
|
|
codevipe-cloud-backup.service static -
|
|
dpkg-db-backup.service static -
|
|
forum-prod-fresh5-cloud-backup.service static -
|
|
homelab-cluster-admin-vm-mail-cloud-backup.service static -
|
|
/usr/local/sbin/homelab-rclone-required-check.sh:4:RAW_FILE="/etc/homelab/rclone/required-remotes.txt"
|
|
/usr/local/sbin/homelab-rclone-required-check.sh:5:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt"
|
|
/usr/local/sbin/homelab-rclone-required-check.sh:17:CONFIGURED="$(rclone listremotes 2>/dev/null | sed 's/:$//' || true)"
|
|
/usr/local/sbin/homelab-rclone-required-check.sh:19: echo "No rclone remotes configured yet."
|
|
/usr/local/sbin/routerguard-watchdog-power-entrypoint:22:cleanup(){ rm -f "$PRE" "$ARM"; }
|
|
/usr/local/sbin/routerguard-watchdog-power-entrypoint:38:rm -f "$ARM"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:4:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:18: "checksums/"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:21:echo "=== homelab rclone crypt healthcheck: $(hostname) ==="
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:40: if ! rclone listremotes | sed 's/:$//' | grep -qx "$REMOTE"; then
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:47: if ! rclone about "${REMOTE}:"; then
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:48: echo "FAIL: rclone about failed: $REMOTE"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:54: ROOT_LIST="$(rclone lsf "${REMOTE}:/" || true)"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:76: if rclone cat "${REMOTE}:/latest-backup-index.json" | jq -e '.schema and .remote and .status and .items' >/dev/null; then
|
|
/usr/local/sbin/codevipe-cloud-backup.sh:28: rclone copy "$CACHE_ROOT/$BASE" "$R:$REMOTE_PATH/$BASE" --stats 30s
|
|
/usr/local/sbin/codevipe-cloud-backup.sh:31:find "$CACHE_ROOT" -mindepth 1 -maxdepth 1 -type d -mtime +14 -print -exec rm -rf {} \;
|
|
/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:26:rm -f "$STAGE/$base"
|
|
/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:34:format=tar.gz split_parts_1500M rclone_crypt_remote
|
|
/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:46: rclone copy "$STAGE" "$dest" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --stats-one-line >/tmp/forum-cloud-copy-${r}-$TS.log 2>&1
|
|
/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:49: remote_count=$(rclone lsf "$dest" --files-only 2>/tmp/forum-cloud-lsf-${r}-$TS.err | wc -l)
|
|
/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:55:find /root/backups/forum-prod-fresh5 -mindepth 1 -maxdepth 1 -type d -mtime +14 -exec rm -rf {} +
|
|
/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:73: retention_remote_count=$(rclone lsf "$r:$REMOTE_BASE/$retention_job" --files-only 2>/dev/null | wc -l)
|
|
/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:78: rm -rf --one-file-system "$retention_dir"
|
|
/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:83: rm -f "$retention_keep_tmp"
|
|
/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:24:rclone listremotes 2>/tmp/cluster-admin-rclone-listremotes.err | grep -E 'mail.*crypt|crypt.*mail' | sed 's/:$//' | sort > /tmp/cluster-admin-mail-remotes.txt
|
|
/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:41: rm -rf "$STAGE/$OLD_JOB"
|
|
/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:51: ls "$d"/vzdump-qemu-${VMID}-*.vma.zst >/dev/null 2>&1 || { echo "DELETE_EMPTY_LOCAL_JOB=$b"; rm -rf "$d"; }
|
|
/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:115: rm -f "$TMPVMA"
|
|
/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:124: rm -f "$TMPVMA"
|
|
/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:158: rclone copy "$JOBDIR" "$DEST" --checksum --transfers 2 --checkers 4 --retries 3 --low-level-retries 5 >/tmp/cluster-admin-rclone-copy-$R.log 2>&1
|
|
/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:161: rclone copyto "$JOBDIR/MANIFEST.txt" "$R:homelab-backups/pve02/cluster-admin-vm180/latest/MANIFEST.txt" --retries 3 >/tmp/cluster-admin-rclone-latest-$R.log 2>&1
|
|
/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:164: REMOTE_SHA="$(rclone cat "$DEST/$BASE.sha256" 2>/tmp/cluster-admin-rclone-catsha-$R.err | awk '{print $1}' | head -n1)"
|
|
/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:177:find "$STAGE" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort -r | tail -n +4 | while read old; do rm -rf "$STAGE/$old"; echo "LOCAL_STAGE_DELETED=$old"; done
|
|
/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:20: rm -rf -- "$WORK"
|
|
/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:117:for CMD in awk cat chmod date grep head mkdir mktemp rclone rm sed sha256sum sort sqlite3 tail tar wc; do
|
|
/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:131:rclone lsf "$REMOTE" --recursive >"$REMOTE_LIST" 2>"$ERROR_FILE" \
|
|
/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:149:rclone copyto "$REMOTE$CRITICAL_REMOTE_PATH" "$CRITICAL_ARCHIVE" 2>"$ERROR_FILE" \
|
|
/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:152:rclone copyto "$REMOTE$CRITICAL_REMOTE_SHA_PATH" "$CRITICAL_SHA_FILE" 2>"$ERROR_FILE" \
|
|
/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:251:INTEGRITY_RESULT="$(sqlite3 "$DB_PATH" 'PRAGMA integrity_check;' 2>"$ERROR_FILE")" \
|
|
/usr/local/sbin/routerguard-network:91: rm -f "$RUN/active"
|
|
/usr/local/sbin/routerguard-network:100: rm -rf "$RUN"
|
|
/etc/systemd/system/suspend.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
/etc/systemd/system/suspend-then-hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
/etc/systemd/system/hybrid-sleep.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
/etc/systemd/system/multi-user.target.wants/corosync.service:20:# as well; before editing this line, you may want to check DefaultStandardError
|
|
/etc/systemd/system/multi-user.target.wants/postfix.service:19:# perform 2-stage startup
|
|
/etc/systemd/system/multi-user.target.wants/postfix.service:20:ExecStartPre=+postfix check
|
|
/etc/systemd/system/multi-user.target.wants/postfix.service:30:# Individual Postfix daemons are started as root, optionally perform chroot
|
|
/etc/systemd/system/multi-user.target.wants/postfix.service:52:# CAP_CHOWN can be removed.
|
|
/etc/systemd/system/multi-user.target.wants/pve-guests.service:22:ExecStop=-/usr/bin/vzdump -stop
|
|
/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:22:ExecStart=/usr/sbin/pve-ha-lrm start
|
|
/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:23:ExecStop=/usr/sbin/pve-ha-lrm stop
|
|
/etc/systemd/system/multi-user.target.wants/rrdcached.service:2:# To customize, copy this file to /etc/systemd/system/
|
|
/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:14:ExecStart=/usr/sbin/pve-ha-crm start
|
|
/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:15:ExecStop=/usr/sbin/pve-ha-crm stop
|
|
/etc/systemd/system/multi-user.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
/etc/systemd/system/multi-user.target.wants/e2scrub_reap.service:2:Description=Remove Stale Online ext4 Metadata Check Snapshots
|
|
/etc/systemd/system/default.target.wants/nvmefc-boot-connections.service:19:RemoveIPC=yes
|
|
/etc/systemd/system/default.target.wants/nvmf-autoconnect.service:21:RemoveIPC=yes
|
|
/etc/systemd/system/sysinit.target.wants/pvenetcommit.service:8:ExecStartPre=-/bin/rm -f /etc/openvswitch/conf.db
|
|
/etc/systemd/system/sysinit.target.wants/systemd-pstore.service:11:Description=Platform Persistent Storage Archival
|
|
/etc/systemd/system/pve-manager.service:22:ExecStop=-/usr/bin/vzdump -stop
|
|
/etc/systemd/system/hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
=== LOCAL_ARCHIVES ===
|
|
========== NODE=pve03 ==========
|
|
=== REMOTES ===
|
|
=== TIMERS ===
|
|
Wed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:10 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service
|
|
=== SERVICES ===
|
|
dpkg-db-backup.service static -
|
|
/usr/local/sbin/homelab-immutable-offline-verify:11: rm -f "$TMP"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:4:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:18: "checksums/"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:21:echo "=== homelab rclone crypt healthcheck: $(hostname) ==="
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:40: if ! rclone listremotes | sed 's/:$//' | grep -qx "$REMOTE"; then
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:47: if ! rclone about "${REMOTE}:"; then
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:48: echo "FAIL: rclone about failed: $REMOTE"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:54: ROOT_LIST="$(rclone lsf "${REMOTE}:/" || true)"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:76: if rclone cat "${REMOTE}:/latest-backup-index.json" | jq -e '.schema and .remote and .status and .items' >/dev/null; then
|
|
/usr/local/sbin/homelab-rclone-required-check.sh:4:RAW_FILE="/etc/homelab/rclone/required-remotes.txt"
|
|
/usr/local/sbin/homelab-rclone-required-check.sh:5:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt"
|
|
/usr/local/sbin/homelab-rclone-required-check.sh:17:CONFIGURED="$(rclone listremotes 2>/dev/null | sed 's/:$//' || true)"
|
|
/usr/local/sbin/homelab-rclone-required-check.sh:19: echo "No rclone remotes configured yet."
|
|
/etc/systemd/system/hybrid-sleep.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
/etc/systemd/system/multi-user.target.wants/pve-guests.service:22:ExecStop=-/usr/bin/vzdump -stop
|
|
/etc/systemd/system/multi-user.target.wants/corosync.service:20:# as well; before editing this line, you may want to check DefaultStandardError
|
|
/etc/systemd/system/multi-user.target.wants/rrdcached.service:2:# To customize, copy this file to /etc/systemd/system/
|
|
/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:14:ExecStart=/usr/sbin/pve-ha-crm start
|
|
/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:15:ExecStop=/usr/sbin/pve-ha-crm stop
|
|
/etc/systemd/system/multi-user.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
/etc/systemd/system/multi-user.target.wants/postfix.service:19:# perform 2-stage startup
|
|
/etc/systemd/system/multi-user.target.wants/postfix.service:20:ExecStartPre=+postfix check
|
|
/etc/systemd/system/multi-user.target.wants/postfix.service:30:# Individual Postfix daemons are started as root, optionally perform chroot
|
|
/etc/systemd/system/multi-user.target.wants/postfix.service:52:# CAP_CHOWN can be removed.
|
|
/etc/systemd/system/multi-user.target.wants/e2scrub_reap.service:2:Description=Remove Stale Online ext4 Metadata Check Snapshots
|
|
/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:22:ExecStart=/usr/sbin/pve-ha-lrm start
|
|
/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:23:ExecStop=/usr/sbin/pve-ha-lrm stop
|
|
/etc/systemd/system/sysinit.target.wants/pvenetcommit.service:8:ExecStartPre=-/bin/rm -f /etc/openvswitch/conf.db
|
|
/etc/systemd/system/sysinit.target.wants/systemd-pstore.service:11:Description=Platform Persistent Storage Archival
|
|
/etc/systemd/system/hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
/etc/systemd/system/suspend-then-hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
/etc/systemd/system/default.target.wants/nvmf-autoconnect.service:21:RemoveIPC=yes
|
|
/etc/systemd/system/default.target.wants/nvmefc-boot-connections.service:19:RemoveIPC=yes
|
|
/etc/systemd/system/suspend.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
/etc/systemd/system/pve-manager.service:22:ExecStop=-/usr/bin/vzdump -stop
|
|
=== LOCAL_ARCHIVES ===
|
|
========== BACKUP-CLOUD-CONTENTS-065 ==========
|
|
========== BACKUP-PAIR-DELETE-GUARD-066 ==========
|
|
========== NODE=pve01 ==========
|
|
=== REMOTES ===
|
|
=== TIMERS ===
|
|
Wed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:14 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service
|
|
Wed 2026-07-22 00:09:43 MSK 9h Tue 2026-07-21 00:09:32 MSK 14h ago homelab-backup-freshness.timer homelab-backup-freshness.service
|
|
Wed 2026-07-22 02:15:00 MSK 11h Tue 2026-07-21 02:15:05 MSK 12h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service
|
|
Wed 2026-07-22 03:10:02 MSK 12h Tue 2026-07-21 03:13:14 MSK 11h ago homelab-emergency-backup.timer homelab-emergency-backup.service
|
|
Wed 2026-07-22 03:35:00 MSK 12h Tue 2026-07-21 03:35:14 MSK 11h ago filebrowser-backup.timer filebrowser-backup.service
|
|
Wed 2026-07-22 03:55:02 MSK 13h Tue 2026-07-21 03:51:25 MSK 11h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service
|
|
Wed 2026-07-22 04:17:29 MSK 13h Tue 2026-07-21 04:19:17 MSK 10h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service
|
|
Wed 2026-07-22 04:26:54 MSK 13h Tue 2026-07-21 04:28:14 MSK 10h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service
|
|
Wed 2026-07-22 04:38:04 MSK 13h Tue 2026-07-21 04:34:48 MSK 10h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service
|
|
Wed 2026-07-22 05:01:44 MSK 14h Tue 2026-07-21 04:57:14 MSK 9h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service
|
|
Wed 2026-07-22 05:23:56 MSK 14h Tue 2026-07-21 05:17:57 MSK 9h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service
|
|
Wed 2026-07-22 05:27:13 MSK 14h Tue 2026-07-21 05:29:05 MSK 9h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service
|
|
Wed 2026-07-22 05:47:19 MSK 14h Tue 2026-07-21 05:46:14 MSK 9h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service
|
|
Wed 2026-07-22 06:01:19 MSK 15h Tue 2026-07-21 05:57:30 MSK 8h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service
|
|
Wed 2026-07-22 06:27:43 MSK 15h Tue 2026-07-21 06:32:25 MSK 8h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service
|
|
Wed 2026-07-22 06:38:51 MSK 15h Tue 2026-07-21 06:43:54 MSK 8h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service
|
|
Wed 2026-07-22 06:39:22 MSK 15h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service
|
|
Wed 2026-07-22 06:44:16 MSK 15h Tue 2026-07-21 06:40:25 MSK 8h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service
|
|
Wed 2026-07-22 06:52:06 MSK 15h Tue 2026-07-21 06:47:47 MSK 8h ago homelab-beszel-backup.timer homelab-beszel-backup.service
|
|
Wed 2026-07-22 06:58:58 MSK 16h Tue 2026-07-21 06:57:30 MSK 7h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service
|
|
Wed 2026-07-22 07:02:27 MSK 16h Tue 2026-07-21 07:05:30 MSK 7h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service
|
|
Wed 2026-07-22 07:07:14 MSK 16h Tue 2026-07-21 07:18:55 MSK 7h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service
|
|
Wed 2026-07-22 07:35:06 MSK 16h Tue 2026-07-21 07:36:59 MSK 7h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service
|
|
Wed 2026-07-22 07:38:21 MSK 16h Tue 2026-07-21 07:36:14 MSK 7h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service
|
|
Wed 2026-07-22 07:41:49 MSK 16h Tue 2026-07-21 07:53:41 MSK 7h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service
|
|
Wed 2026-07-22 08:10:37 MSK 17h Tue 2026-07-21 08:07:25 MSK 6h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service
|
|
Wed 2026-07-22 08:37:02 MSK 17h Tue 2026-07-21 08:30:07 MSK 6h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service
|
|
Wed 2026-07-22 08:40:13 MSK 17h Tue 2026-07-21 08:50:14 MSK 6h ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service
|
|
Wed 2026-07-22 08:53:36 MSK 17h Tue 2026-07-21 08:41:14 MSK 6h ago memos-backup.timer memos-backup.service
|
|
Sun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service
|
|
Sun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service
|
|
Sun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service
|
|
Sun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service
|
|
Mon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 7h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service
|
|
=== SERVICES ===
|
|
dpkg-db-backup.service static -
|
|
filebrowser-backup.service static -
|
|
homelab-alertmanager-backup.service static -
|
|
homelab-appbackup-dockge-stacks.service static -
|
|
homelab-appbackup-mariadb-batch.service static -
|
|
homelab-appbackup-npmplus-edge.service static -
|
|
homelab-appbackup-postgres-batch.service static -
|
|
homelab-appbackup-sqlite-batch.service static -
|
|
homelab-authentik-backup.service static -
|
|
homelab-auto-backup.service static -
|
|
homelab-backup-coverage-matrix.service static -
|
|
homelab-backup-freshness.service static -
|
|
homelab-backup-healthcheck.service static -
|
|
homelab-backup-matrix-refresh.service static -
|
|
homelab-backup-sla-health.service static -
|
|
homelab-beszel-backup.service static -
|
|
homelab-blackbox-exporter-backup.service static -
|
|
homelab-cluster-admin-incident-engine-backup.service static -
|
|
homelab-crypto-portfolio-backup.service static -
|
|
homelab-edge-vm-vzdump-backup.service static -
|
|
homelab-emergency-backup.service static -
|
|
homelab-extended-appbackup.service static -
|
|
homelab-gitea-backup.service static -
|
|
homelab-gitea-secondary-backup.service static -
|
|
homelab-legacy-monitoring-backups.service static -
|
|
homelab-mail-cloud-critical-upload.service static -
|
|
homelab-mail-cloud-edge-vm-upload.service static -
|
|
homelab-mail-cloud-nextcloud-vm-upload.service static -
|
|
homelab-mail-cloud-restore-drill.service static -
|
|
homelab-npmplus-kuma-config-backup.service static -
|
|
homelab-router-running-config-mail-cloud.service static -
|
|
homelab-vaultwarden-backup.service static -
|
|
homelab-verified-backup-health.service static -
|
|
homelab-vm-backup-health.service static -
|
|
homelab-vm-backup-policy.service static -
|
|
homelab-vm-full-backup.service static -
|
|
homelab-vm-local-dumps-2cloud.service static -
|
|
homelab-vm160-remote-backup.service static -
|
|
immich-media-backup.service static -
|
|
memos-backup.service static -
|
|
netbird-vps-backup.service static -
|
|
paperless-backup.service static -
|
|
skladchik-reports-monitor-backup-restore-check.service static -
|
|
skladchik-reports-monitor-backup.service static -
|
|
/usr/local/sbin/homelab-vm160-remote-backup:8:PARENT=/mnt/staging/vzdump
|
|
/usr/local/sbin/homelab-vm160-remote-backup:22:SOURCE_ACTIVE="$(ssh "${SSH[@]}" "$SOURCE" "pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'")" || SOURCE_ACTIVE_RC=$?
|
|
/usr/local/sbin/homelab-vm160-remote-backup:27:NAME="vzdump-qemu-${VMID}-${TS}.vma.zst"
|
|
/usr/local/sbin/homelab-vm160-remote-backup:35: ssh "${SSH[@]}" "$TARGET" "rm -rf -- '$DIR'" >/dev/null 2>&1 || true
|
|
/usr/local/sbin/homelab-vm160-remote-backup:41: rm -f "$LOG"
|
|
/usr/local/sbin/homelab-vm160-remote-backup:46:LATEST_SIZE="$(ssh "${SSH[@]}" "$TARGET" "find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\n' | sort -nr | head -n1")"
|
|
/usr/local/sbin/homelab-vm160-remote-backup:55:ssh "${SSH[@]}" "$SOURCE" "vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7" 2>"$LOG" | ssh "${SSH[@]}" "$TARGET" "cat > '$PART'"
|
|
/usr/local/sbin/homelab-vm160-remote-backup:57:ssh "${SSH[@]}" "$TARGET" "cat > '$DIR/vzdump.log'" < "$LOG"
|
|
/usr/local/sbin/homelab-vm160-remote-backup:66:rm -f "$LOG"
|
|
/usr/local/sbin/skladchik-reports-monitor-status:11:check_health() {
|
|
/usr/local/sbin/skladchik-reports-monitor-status:24:check_timer() {
|
|
/usr/local/sbin/skladchik-reports-monitor-status:38:check_timer skladchik-reports-monitor-backup.timer
|
|
/usr/local/sbin/skladchik-reports-monitor-status:39:check_timer skladchik-reports-monitor-backup-restore-check.timer
|
|
/usr/local/sbin/skladchik-reports-monitor-status:40:check_timer skladchik-reports-monitor-supervisor.timer
|
|
/usr/local/sbin/skladchik-reports-monitor-status:41:check_timer skladchik-reports-monitor-monthly-selftest-watch.timer
|
|
/usr/local/sbin/skladchik-reports-monitor-status:43:check_health "PVE01: ежедневный бэкап" "/var/lib/skladchik-reports-monitor-backup/health.txt"
|
|
/usr/local/sbin/skladchik-reports-monitor-status:44:check_health "PVE01: проверка восстановления" "/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt"
|
|
/usr/local/sbin/skladchik-reports-monitor-status:45:check_health "PVE01: общий supervisor" "/var/lib/skladchik-reports-monitor-supervisor/health.txt"
|
|
/usr/local/sbin/skladchik-reports-monitor-status:46:check_health "PVE01: контроль ежемесячного самотеста" "/var/lib/skladchik-reports-monitor-selftest-watch/health.txt"
|
|
/usr/local/sbin/homelab-mail-cloud-restore-drill:83: rm -f "$OUT"
|
|
/usr/local/sbin/homelab-mail-cloud-restore-drill:108: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/MANIFEST.txt" "$WORK/edge/MANIFEST.txt" > "$WORK/logs/edge_manifest_download.out" 2>"$WORK/logs/edge_manifest_download.err"
|
|
/usr/local/sbin/homelab-mail-cloud-restore-drill:111: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/SHA256SUMS.txt" "$WORK/edge/SHA256SUMS.txt" > "$WORK/logs/edge_sums_download.out" 2>"$WORK/logs/edge_sums_download.err"
|
|
/usr/local/sbin/homelab-mail-cloud-restore-drill:114: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/$EDGE_ARCHIVE.sha256" "$WORK/edge/$EDGE_ARCHIVE.sha256" > "$WORK/logs/edge_archive_sha_download.out" 2>"$WORK/logs/edge_archive_sha_download.err"
|
|
/usr/local/sbin/homelab-mail-cloud-restore-drill:117: rclone lsf "${EDGE_REMOTE}:$EDGE_DEST/parts" --format ps > "$WORK/logs/edge_parts_lsf.txt" 2>"$WORK/logs/edge_parts_lsf.err"
|
|
/usr/local/sbin/homelab-mail-cloud-restore-drill:130: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/parts/$FIRST_PART" "$WORK/edge/$FIRST_PART" > "$WORK/logs/edge_first_part_download.out" 2>"$WORK/logs/edge_first_part_download.err"
|
|
/usr/local/sbin/homelab-mail-cloud-restore-drill:137: rclone copyto "${EDGE_REMOTE}:$EDGE_DEST/parts/$LAST_PART" "$WORK/edge/$LAST_PART" > "$WORK/logs/edge_last_part_download.out" 2>"$WORK/logs/edge_last_part_download.err"
|
|
/usr/local/sbin/homelab-mail-cloud-restore-drill:168: rm -f "$WORK/edge/$FIRST_PART" "$WORK/edge/$LAST_PART"
|
|
/usr/local/sbin/homelab-golden-state-extended-overlay:26:rclone copyto "$JSON" "$RA" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-golden-state-extended-overlay:27:rclone copyto "$JSON" "$RB" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-mail-cloud-critical-upload:80: rclone mkdir "${R}:$REMOTE_DIR" > "$WORK/logs/${SAFE}_mkdir.out" 2>"$WORK/logs/${SAFE}_mkdir.err"
|
|
/usr/local/sbin/homelab-mail-cloud-critical-upload:82: homelab-rclone-copy-chunked "$BUNDLE" "${R}:$REMOTE_DIR/$(basename "$BUNDLE")" > "$WORK/logs/${SAFE}_upload_bundle.out" 2>"$WORK/logs/${SAFE}_upload_bundle.err"
|
|
/usr/local/sbin/homelab-mail-cloud-critical-upload:84: rclone copyto "$BUNDLE_SHA" "${R}:$REMOTE_DIR/$(basename "$BUNDLE_SHA")" --checksum > "$WORK/logs/${SAFE}_upload_sha.out" 2>"$WORK/logs/${SAFE}_upload_sha.err"
|
|
/usr/local/sbin/homelab-mail-cloud-critical-upload:86: rclone copyto "$WORK/components/MANIFEST.txt" "${R}:$REMOTE_DIR/MANIFEST.txt" --checksum > "$WORK/logs/${SAFE}_upload_manifest.out" 2>"$WORK/logs/${SAFE}_upload_manifest.err"
|
|
/usr/local/sbin/homelab-mail-cloud-critical-upload:88: rclone copyto "$WORK/components/SHA256SUMS.txt" "${R}:$REMOTE_DIR/SHA256SUMS.txt" --checksum > "$WORK/logs/${SAFE}_upload_sums.out" 2>"$WORK/logs/${SAFE}_upload_sums.err"
|
|
/usr/local/sbin/homelab-mail-cloud-critical-upload:90: homelab-rclone-cat-chunked "${R}:$REMOTE_DIR/$(basename "$BUNDLE")" > "$WORK/download/${SAFE}-$(basename "$BUNDLE")" 2>"$WORK/logs/${SAFE}_download_bundle.err"
|
|
/usr/local/sbin/homelab-mail-cloud-critical-upload:108: rm -f "$WORK/download/${SAFE}-$(basename "$BUNDLE")"
|
|
/usr/local/sbin/homelab-mail-cloud-critical-upload:140: rm -rf --one-file-system "$OLD_DIR"
|
|
/usr/local/sbin/homelab-alertmanager-backup-run:16:ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$MON" "sudo -n docker exec alertmanager amtool check-config /etc/alertmanager/alertmanager.yml >/dev/null && sudo -n tar -C / -czf - opt/stacks/observability-lite/compose.yaml opt/stacks/observability-lite/compose.override.yaml opt/stacks/observability-lite/alertmanager/alertmanager.yml opt/stacks/observability-lite/alertmanager/data" > "$STAGE/remote.tgz"
|
|
/usr/local/sbin/homelab-alertmanager-backup-run:18:rm -f "$STAGE/remote.tgz"
|
|
/usr/local/sbin/homelab-alertmanager-backup-run:21:jq -n --arg app "$APP" --arg host "monitoring" --arg engine "files+alertmanager-config-check" --arg config "ok" --argjson files "$FILES" "{app:\$app,engine:\$engine,source_host:\$host,source_paths:[\"monitoring:/opt/stacks/observability-lite/alertmanager\",\"monitoring:/opt/stacks/observability-lite/compose.yaml\",\"monitoring:/opt/stacks/observability-lite/compose.override.yaml\"],config_check:\$config,file_count:\$files,note:\"Alertmanager configuration and runtime state in encrypted two-cloud backup\"}" > "$STAGE/metadata.json"
|
|
/usr/local/sbin/appbackupctl:21: *) echo "usage: appbackupctl run [dockge-stacks|cluster-admin-incident-engine]|check|restore-check|retention|report|audit|status|lastlog|timer"; exit 2 ;;
|
|
/usr/local/sbin/postgres-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@"$H" "sudo rm -f /tmp/postgres-app-${APP}-${JOB_ID}.tgz"
|
|
/usr/local/sbin/postgres-apps-backup-run:21: rm -rf "$STAGE/payload"
|
|
/usr/local/sbin/homelab-backup-gitea:27:rm -rf "$RESTORE"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:4:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:18: "checksums/"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:21:echo "=== homelab rclone crypt healthcheck: $(hostname) ==="
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:40: if ! rclone listremotes | sed 's/:$//' | grep -qx "$REMOTE"; then
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:47: if ! rclone about "${REMOTE}:"; then
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:48: echo "FAIL: rclone about failed: $REMOTE"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:54: ROOT_LIST="$(rclone lsf "${REMOTE}:/" || true)"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:76: if rclone cat "${REMOTE}:/latest-backup-index.json" | jq -e '.schema and .remote and .status and .items' >/dev/null; then
|
|
/usr/local/sbin/homelab-stage-verify-flat-chunked:7:rm -rf "$VERIFY"
|
|
/usr/local/sbin/homelab-stage-verify-flat-chunked:9:rclone copy "$REMOTE_DIR" "$VERIFY" --transfers 4 --checkers 8 --retries 5 --low-level-retries 20 --log-level ERROR
|
|
/usr/local/sbin/homelab-stage-verify-flat-chunked:11:(cd "$VERIFY" && test -s manifest.json && if [[ -n "$JOB" ]]; then jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null; fi && sha256sum -c checksums.sha256 >/dev/null && tar -tzf "$ARCHIVE" >/dev/null)
|
|
/usr/local/sbin/homelab-rclone-cat-chunked:7:cleanup(){ rm -rf "$WORK"; }
|
|
/usr/local/sbin/homelab-rclone-cat-chunked:10:rclone copyto "${REMOTE_FILE}.parts/MANIFEST.parts.txt" "$MAN" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR
|
|
/usr/local/sbin/homelab-rclone-cat-chunked:11:while read -r PART SIZE; do case "$PART" in part-*) rclone cat "${REMOTE_FILE}.parts/$PART" ;; esac; done < "$MAN"
|
|
/usr/local/sbin/homelab-stage-verify-chunked:7:rm -rf "$VERIFY"
|
|
/usr/local/sbin/homelab-stage-verify-chunked:9:rclone copy "$REMOTE_DIR" "$VERIFY" --exclude "$ARCHIVE.parts/**" --exclude "$ARCHIVE.parts/" --exclude "$ARCHIVE.sha256" --exclude "$ARCHIVE.manifest.txt" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --log-level ERROR
|
|
/usr/local/sbin/homelab-stage-verify-chunked:13:if [ -s checksums.sha256 ]; then grep -F -v " $ARCHIVE" checksums.sha256 > .checksums-no-archive.sha256 || true; if [ -s .checksums-no-archive.sha256 ]; then sha256sum -c .checksums-no-archive.sha256; fi; EXPECTED=$(grep -F " $ARCHIVE" checksums.sha256 | awk '{print $1; exit}'); else EXPECTED=$(rclone cat "${REMOTE_DIR}/${ARCHIVE}.sha256" | awk '{print $1; exit}'); fi
|
|
/usr/local/sbin/homelab-stage-verify-chunked:15:ACTUAL=$(homelab-rclone-cat-chunked "${REMOTE_DIR}/${ARCHIVE}" | sha256sum | awk '{print $1}')
|
|
/usr/local/sbin/homelab-stage-verify-chunked:17:homelab-rclone-cat-chunked "${REMOTE_DIR}/${ARCHIVE}" | tar -tzf - >/dev/null
|
|
/usr/local/sbin/crypto-portfolio-backup-run:15:ssh -n -o BatchMode=yes -o ConnectTimeout=10 "$CORE" 'H="$(hostname -f 2>/dev/null || hostname)"; { [ "$H" = "core-apps.gram1.ru" ] || [ "$H" = "core-apps" ]; } || exit 3; D=/opt/stacks/crypto-portfolio; T="$(sudo -n mktemp -d /tmp/crypto-portfolio-backup.XXXXXX)"; trap "sudo -n rm -rf \"$T\"" EXIT; sudo -n install -d -m 700 "$T/stack" "$T/stack/data"; for F in compose.yaml Dockerfile requirements.txt app.py worker.py .dockerignore; do sudo -n test -s "$D/$F" && sudo -n cp -a "$D/$F" "$T/stack/"; done; sudo -n cp -a "$D/templates" "$D/secrets" "$T/stack/"; sudo -n python3 -c "import sqlite3,sys;s=sqlite3.connect(sys.argv[1]);d=sqlite3.connect(sys.argv[2]);s.backup(d);d.close();s.close()" "$D/data/portfolio.db" "$T/stack/data/portfolio.db"; for F in market.json worker-status.json; do sudo -n test -s "$D/data/$F" && sudo -n cp -a "$D/data/$F" "$T/stack/data/"; done; sudo -n tar -C "$T" -czf - stack' > "$STAGE/remote.tgz"
|
|
/usr/local/sbin/crypto-portfolio-backup-run:17:rm -f "$STAGE/remote.tgz"
|
|
/usr/local/sbin/npmplus-edge-backup-run:20:ssh -o BatchMode=yes -o ConnectTimeout=8 "$EDGE" "sudo rm -f '/tmp/npmplus-edge-${JOB_ID}.tgz'"
|
|
/usr/local/sbin/npmplus-edge-backup-run:28:rm -rf "$STAGE/payload"
|
|
/usr/local/sbin/mariadb-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@"$H" "sudo rm -f /tmp/mariadb-app-${APP}-${JOB_ID}.tgz"
|
|
/usr/local/sbin/mariadb-apps-backup-run:20: rm -rf "$STAGE/payload"
|
|
/usr/local/sbin/homelab-blackbox-exporter-backup-run:17:rm -f "$STAGE/remote.tgz"
|
|
/usr/local/sbin/homelab-blackbox-exporter-backup-run:21:jq -n --arg app "$APP" --arg host "monitoring" --arg engine "files+runtime-health-check" --argjson files "$FILES" '{app:$app,engine:$engine,source_host:$host,source_paths:["monitoring:/opt/stacks/blackbox-exporter"],runtime_health:"ok",file_count:$files,note:"Blackbox Exporter compose and probe configuration in encrypted two-cloud backup"}' > "$STAGE/metadata.json"
|
|
/usr/local/sbin/homelab-ingress-standby-refresh:16: rm -f "$TMP_STATE" "$RELAY" "$RESOURCES"
|
|
/usr/local/sbin/homelab-ingress-standby-refresh:106:"chattr -i '$DEST_FILE' '$DEST_FILE.sha256' '$DEST_FILE.env' 2>/dev/null || true; rm -f '$DEST_TMP'"
|
|
/usr/local/sbin/homelab-ingress-standby-refresh:164: rm -f "$OLD" "$OLD.sha256" "$OLD.env"
|
|
/usr/local/sbin/homelab-ingress-standby-refresh:173:run_node "$SOURCE_NODE" rm -f "$SOURCE_BACKUP"
|
|
/usr/local/sbin/homelab-stage-2cloud-flat:33:(cd "$STAGE"; FILES=("$ARCHIVE" manifest.json latest-backup-index.json); [[ -f metadata.json ]] && FILES+=(metadata.json); sha256sum "${FILES[@]}" > checksums.sha256; sha256sum -c checksums.sha256 >/dev/null; tar -tzf "$ARCHIVE" >/dev/null; jq -e . manifest.json >/dev/null)
|
|
/usr/local/sbin/homelab-stage-2cloud-flat:36: if rclone lsf "$REMOTE_DIR" --max-depth 1 2>/dev/null | grep -q .; then echo "REFUSE NONEMPTY CANARY TARGET $REMOTE_DIR" >&2; exit 6; fi
|
|
/usr/local/sbin/homelab-stage-2cloud-flat:37: rclone mkdir "$REMOTE_DIR"
|
|
/usr/local/sbin/homelab-stage-2cloud-flat:38: for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f "$STAGE/$F" ]] && rclone copyto "$STAGE/$F" "$REMOTE_DIR/$F" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done
|
|
/usr/local/sbin/homelab-stage-2cloud-flat:39: CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-flat-chunked "$STAGE/$ARCHIVE" "$REMOTE_DIR/$ARCHIVE"
|
|
/usr/local/sbin/homelab-stage-2cloud-flat:41: rclone copyto "$STAGE/latest-backup-index.json" "$LATEST_FILE" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR
|
|
/usr/local/sbin/homelab-stage-2cloud-flat:47:rm -rf "$STAGE" "$VERIFY"
|
|
/usr/local/sbin/homelab-cluster-admin-webpanel-sync:7:copy_health() {
|
|
/usr/local/sbin/homelab-cluster-admin-webpanel-sync:17:copy_health /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt cluster-admin-vm-mail-cloud-backup.txt
|
|
/usr/local/sbin/homelab-cluster-admin-webpanel-sync:18:copy_health /var/lib/homelab-health/overall.txt overall.txt
|
|
/usr/local/sbin/homelab-cluster-admin-webpanel-sync:19:copy_health /var/lib/homelab-health/final-readiness.txt final-readiness.txt
|
|
/usr/local/sbin/homelab-cluster-admin-webpanel-sync:20:copy_health /var/lib/homelab-health/cluster-admin-strict-final-seal.txt cluster-admin-strict-final-seal.txt
|
|
/usr/local/sbin/homelab-backup-authentik:14:ssh debian@[PRIVATE_IP] "rm -rf /tmp/auth-small-snap; mkdir -p /tmp/auth-small-snap"
|
|
/usr/local/sbin/homelab-backup-authentik:27:ssh debian@[PRIVATE_IP] "sudo docker rm -f authentik-restore-test >/dev/null 2>&1 || true"
|
|
/usr/local/sbin/homelab-backup-authentik:51:ssh debian@[PRIVATE_IP] "sudo docker rm -f authentik-restore-test >/dev/null 2>&1 || true; rm -f /tmp/authentik-restore-from-archive.pg_dump"
|
|
/usr/local/sbin/homelab-safe-exec:621: latest_tmp.unlink(missing_ok=True)
|
|
/usr/local/sbin/homelab-golden-state-index:28:rclone copyto "$JSON" "$RA" >/tmp/golden-state-upload-a.log 2>&1 || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-golden-state-index:29:rclone copyto "$JSON" "$RB" >/tmp/golden-state-upload-b.log 2>&1 || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-golden-state-index:30:rclone cat "$RA" 2>/dev/null | jq -e --arg cp "$CP" --arg ext "$EXT" ".control_plane_job == \$cp and .health.extended_appbackup == \$ext and .extended_appbackup_health == \$ext" >/dev/null || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-golden-state-index:31:rclone cat "$RB" 2>/dev/null | jq -e --arg cp "$CP" --arg ext "$EXT" ".control_plane_job == \$cp and .health.extended_appbackup == \$ext and .extended_appbackup_health == \$ext" >/dev/null || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-vm-full-retention:35: rm -f \
|
|
/usr/local/sbin/homelab-vm-full-retention:73: rm -rf -- \"\$BASE/\$setname\"
|
|
/usr/local/sbin/homelab-stage-2cloud:22:echo "[STEP] local checksum"; (cd "$STAGE" && sha256sum "$ARCHIVE" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e . manifest.json >/dev/null)
|
|
/usr/local/sbin/homelab-stage-2cloud:23:echo "[STEP] upload cloud A metadata"; rclone copy "$STAGE" "$RA" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s
|
|
/usr/local/sbin/homelab-stage-2cloud:25:echo "[STEP] upload cloud B metadata"; rclone copy "$STAGE" "$RB" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s
|
|
/usr/local/sbin/homelab-stage-2cloud:29:echo "[STEP] upload latest index"; rclone copyto "$STAGE/latest-backup-index.json" "$RAL"; rclone copyto "$STAGE/latest-backup-index.json" "$RBL"; rclone cat "$RAL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null; rclone cat "$RBL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null
|
|
/usr/local/sbin/homelab-stage-2cloud:31:echo "[STEP] delete local staging"; rm -rf "$STAGE" "$VERIFY"; test ! -d "$STAGE" && test ! -d "$VERIFY"
|
|
/usr/local/sbin/homelab-safe-autoheal:17:SAFE_TIMERS="homelab-emergency-backup.timer homelab-appbackup-dockge-stacks.timer homelab-appbackup-npmplus-edge.timer homelab-appbackup-sqlite-batch.timer homelab-appbackup-postgres-batch.timer homelab-appbackup-mariadb-batch.timer homelab-verified-backup-health.timer homelab-drift-check.timer homelab-service-registry-check.timer homelab-dependency-map-check.timer homelab-vm-backup-health.timer homelab-alerting-health.timer homelab-runbook-generate.timer homelab-desired-state-sync.timer homelab-duty-admin-report.timer homelab-incident-journal.timer homelab-backup-sla-health.timer homelab-kuma-monitor-policy.timer homelab-safe-autoheal.timer homelab-overall-health.timer"
|
|
/usr/local/sbin/homelab-safe-autoheal:144:for cmd in homelab-alerting-health homelab-kuma-monitor-policy homelab-backup-sla-health homelab-service-registry-check homelab-dependency-map-check; do
|
|
/usr/local/sbin/homelab-backup-npmplus-kuma-configs:26: install -D -m 0755 /usr/local/sbin/homelab-check-private-vpn-hosts "$SNAP/pve01/usr/local/sbin/homelab-check-private-vpn-hosts" 2>>"$ERR" || fail "copy_private_vpn_script_failed"
|
|
/usr/local/sbin/homelab-backup-npmplus-kuma-configs:27: install -D -m 0644 /etc/systemd/system/homelab-private-vpn-hosts-health.service "$SNAP/pve01/etc/systemd/system/homelab-private-vpn-hosts-health.service" 2>>"$ERR" || fail "copy_private_vpn_service_failed"
|
|
/usr/local/sbin/homelab-backup-npmplus-kuma-configs:28: install -D -m 0644 /etc/systemd/system/homelab-private-vpn-hosts-health.timer "$SNAP/pve01/etc/systemd/system/homelab-private-vpn-hosts-health.timer" 2>>"$ERR" || fail "copy_private_vpn_timer_failed"
|
|
/usr/local/sbin/homelab-backup-npmplus-kuma-configs:29: install -D -m 0644 /var/lib/homelab-health/private-vpn-hosts.txt "$SNAP/pve01/var/lib/homelab-health/private-vpn-hosts.txt" 2>>"$ERR" || fail "copy_private_vpn_health_failed"
|
|
/usr/local/sbin/homelab-backup-npmplus-kuma-configs:42:sudo rm -rf "$TMP" "$EDGE_TGZ"
|
|
/usr/local/sbin/homelab-backup-npmplus-kuma-configs:82:sudo rm -rf "$TMP"
|
|
/usr/local/sbin/homelab-backup-npmplus-kuma-configs:90: ssh -o BatchMode=yes -o ConnectTimeout=8 debian@[PRIVATE_IP] "rm -f '$EDGE_TGZ'" 2>>"$ERR" || true
|
|
/usr/local/sbin/homelab-backup-npmplus-kuma-configs:99: NPM_INTEGRITY="$(sqlite3 "$SNAP/edge-vm/npmplus/db/database.sqlite" "PRAGMA integrity_check;" 2>>"$ERR" | tr '\n' ';' || echo error)"
|
|
/usr/local/sbin/homelab-backup-npmplus-kuma-configs:100: KUMA_INTEGRITY="$(sqlite3 "$SNAP/edge-vm/kuma/db/kuma.db" "PRAGMA integrity_check;" 2>>"$ERR" | tr '\n' ';' || echo error)"
|
|
/usr/local/sbin/homelab-backup-npmplus-kuma-configs:134: rm -rf "$RESTORE"
|
|
/usr/local/sbin/homelab-backup-npmplus-kuma-configs:138: RESTORE_NPM_INTEGRITY="$(sqlite3 "$RESTORE/edge-vm/npmplus/db/database.sqlite" "PRAGMA integrity_check;" 2>>"$ERR" | tr '\n' ';' || echo error)"
|
|
/usr/local/sbin/homelab-backup-npmplus-kuma-configs:139: RESTORE_KUMA_INTEGRITY="$(sqlite3 "$RESTORE/edge-vm/kuma/db/kuma.db" "PRAGMA integrity_check;" 2>>"$ERR" | tr '\n' ';' || echo error)"
|
|
/usr/local/sbin/homelab-stage-2cloud-flat-canary:31:(cd "$STAGE"; FILES=("$ARCHIVE" manifest.json latest-backup-index.json); [[ -f metadata.json ]] && FILES+=(metadata.json); sha256sum "${FILES[@]}" > checksums.sha256; sha256sum -c checksums.sha256 >/dev/null; tar -tzf "$ARCHIVE" >/dev/null; jq -e . manifest.json >/dev/null)
|
|
/usr/local/sbin/homelab-stage-2cloud-flat-canary:34: if rclone lsf "$REMOTE_DIR" --max-depth 1 2>/dev/null | grep -q .; then echo "REFUSE NONEMPTY CANARY TARGET $REMOTE_DIR" >&2; exit 6; fi
|
|
/usr/local/sbin/homelab-stage-2cloud-flat-canary:35: rclone mkdir "$REMOTE_DIR"
|
|
/usr/local/sbin/homelab-stage-2cloud-flat-canary:36: for F in metadata.json manifest.json latest-backup-index.json checksums.sha256; do [[ -f "$STAGE/$F" ]] && rclone copyto "$STAGE/$F" "$REMOTE_DIR/$F" --transfers 1 --checkers 2 --retries 5 --low-level-retries 20 --log-level ERROR; done
|
|
/usr/local/sbin/homelab-stage-2cloud-flat-canary:37: CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-flat-chunked "$STAGE/$ARCHIVE" "$REMOTE_DIR/$ARCHIVE"
|
|
/usr/local/sbin/homelab-stage-2cloud-flat-canary:39: rclone copyto "$STAGE/latest-backup-index.json" "$LATEST_FILE" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR
|
|
/usr/local/sbin/homelab-stage-2cloud-flat-canary:45:rm -rf "$STAGE" "$VERIFY"
|
|
/usr/local/sbin/homelab-router-running-config-mail-cloud:62: rm -f "$CONFIG"
|
|
/usr/local/sbin/homelab-router-running-config-mail-cloud:66: rm -f "$CONFIG"
|
|
/usr/local/sbin/homelab-router-running-config-mail-cloud:79: if rclone mkdir "$REMOTE:$DEST" >"$WORK/rclone-mkdir.out" 2>"$WORK/rclone-mkdir.err" && homelab-rclone-copy-chunked "$ARCHIVE" "$REMOTE:$DEST/$(basename "$ARCHIVE")" >"$WORK/rclone-upload.out" 2>"$WORK/rclone-upload.err" && rclone copyto "$SHA_FILE" "$REMOTE:$DEST/$(basename "$SHA_FILE")" --checksum >"$WORK/rclone-upload-sha.out" 2>"$WORK/rclone-upload-sha.err"; then
|
|
/usr/local/sbin/homelab-router-running-config-mail-cloud:86: if homelab-rclone-cat-chunked "$REMOTE:$DEST/$(basename "$ARCHIVE")" >"$DL" 2>"$WORK/rclone-download.err"; then
|
|
/usr/local/sbin/homelab-router-running-config-mail-cloud:93:rclone lsf "$REMOTE:$TARGET_BASE" --dirs-only 2>/dev/null | sed 's#/$##' | sort > "$DIR_LIST" || RETENTION_RC=$?
|
|
/usr/local/sbin/homelab-router-running-config-mail-cloud:98: [ -n "$OLD" ] && rclone purge "$REMOTE:$TARGET_BASE/$OLD" >>"$WORK/retention.out" 2>>"$WORK/retention.err"
|
|
/usr/local/sbin/homelab-router-running-config-mail-cloud:103: rm -f "$old" "$old.sha256"
|
|
/usr/local/sbin/homelab-router-running-config-mail-cloud:116:rm -rf "$WORK"
|
|
/usr/local/sbin/appbackup-restore-check:24:rm -rf "$ROOT" && mkdir -p "$ROOT/a" "$ROOT/b"
|
|
/usr/local/sbin/appbackup-restore-check:25:trap "echo \"ERROR app restore-check failed; kept: $ROOT\"; exit 1" ERR
|
|
/usr/local/sbin/appbackup-restore-check:27:rclone copy "$RA" "$ROOT/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >"${LOGBASE}-a.log" 2>&1
|
|
/usr/local/sbin/appbackup-restore-check:30:(cd "$ROOT/a" && sha256sum -c checksums.sha256 >"${LOGBASE}-a-sha.log" && ARCHIVE="$(jq -r ".archive_name" manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null)
|
|
/usr/local/sbin/appbackup-restore-check:32:rclone copy "$RB" "$ROOT/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >"${LOGBASE}-b.log" 2>&1
|
|
/usr/local/sbin/appbackup-restore-check:35:(cd "$ROOT/b" && sha256sum -c checksums.sha256 >"${LOGBASE}-b-sha.log" && ARCHIVE="$(jq -r ".archive_name" manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null)
|
|
/usr/local/sbin/appbackup-restore-check:37:rm -rf "$ROOT"
|
|
/usr/local/sbin/appbackup-restore-check:39:rm -f "${LOGBASE}-a.log" "${LOGBASE}-a-sha.log" "${LOGBASE}-b.log" "${LOGBASE}-b-sha.log"
|
|
/usr/local/sbin/backupctl:12: check) bash -n "$SCRIPT" && [[ -f "$STATE" ]] && jq -e ".status == \"success\" and .local_backup_deleted == true" "$STATE" >/dev/null && echo "OK backup framework" ;;
|
|
/usr/local/sbin/backupctl:16: remotes) source /etc/homelab-backup.env && NODE="$(hostname -s)" && echo "A=$REMOTE_A" && echo "B=$REMOTE_B" && rclone lsd "${REMOTE_A%/}/${NODE}/emergency-pack" | tail -n 5 && rclone lsd "${REMOTE_B%/}/${NODE}/emergency-pack" | tail -n 5 ;;
|
|
/usr/local/sbin/backupctl:19: *) echo "usage: backupctl run|status|lastlog|check|restore-check|audit|report|remotes|retention|timer"; exit 2 ;;
|
|
/usr/local/sbin/homelab-edge-vm-offhost-freshness:156:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-backup NODE=%s VMID=%s ARCHIVE=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\n' \
|
|
/usr/local/sbin/homelab-edge-vm-offhost-freshness:158: >"$HEALTH_DIR/edge-vm-vzdump-backup.txt"
|
|
/usr/local/sbin/homelab-edge-vm-offhost-freshness:159:printf 'STATUS=OK TS=%s TYPE=edge-vm-vzdump-offhost NODE=%s VMID=%s HOST=%s DEST=%s SIZE=%s SHA256=%s NOTES=nightly_existing_dump_freshness\n' \
|
|
/usr/local/sbin/homelab-edge-vm-offhost-freshness:161: >"$HEALTH_DIR/edge-vm-vzdump-offhost.txt"
|
|
/usr/local/sbin/homelab-edge-vm-offhost-freshness:164: >"$HEALTH_DIR/edge-vm-vzdump-restore.txt"
|
|
/usr/local/sbin/homelab-edge-vm-offhost-freshness:167: "$HEALTH_DIR/edge-vm-vzdump-backup.txt" \
|
|
/usr/local/sbin/homelab-edge-vm-offhost-freshness:168: "$HEALTH_DIR/edge-vm-vzdump-offhost.txt" \
|
|
/usr/local/sbin/homelab-edge-vm-offhost-freshness:169: "$HEALTH_DIR/edge-vm-vzdump-restore.txt"
|
|
/usr/local/sbin/homelab-external-probe-vps:20:rm -f "$TMP"
|
|
/usr/local/sbin/sqlite-apps-backup-run:14: ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@"$H" "sudo rm -f /tmp/sqlite-app-${APP}-${JOB_ID}.tgz"
|
|
/usr/local/sbin/sqlite-apps-backup-run:21: rm -rf "$STAGE/payload"
|
|
/usr/local/sbin/homelab-desired-state-sync:10:find "$REPO" -mindepth 1 -maxdepth 1 ! -name .git -exec rm -rf {} + 2>/dev/null
|
|
/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:12:HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt
|
|
/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:28: if rclone lsf "${remote_path}.parts/" > "$plist" 2>/dev/null && grep -q '^part-' "$plist"; then
|
|
/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:29: while IFS= read -r part; do [ -n "$part" ] && rclone cat "${remote_path}.parts/${part}" >> "$out"; done < <(grep '^part-' "$plist" | sort)
|
|
/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:31: rclone copyto "$remote_path" "$out" >/dev/null
|
|
/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:33: rm -f "$plist"
|
|
/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:36:check_one_remote(){
|
|
/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:97:check_one_remote CLOUD_1 "$CLOUD_1"
|
|
/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:98:check_one_remote CLOUD_2 "$CLOUD_2"
|
|
/usr/local/sbin/skladchik-reports-monitor-backup-restore-check:101:find "$WORK_ROOT" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_LOCAL}" | while read -r D; do [ -n "$D" ] && rm -rf "${WORK_ROOT}/${D}"; done
|
|
/usr/local/sbin/skladchik-reports-monitor-backup:29:[ -x /usr/local/sbin/homelab-rclone-copy-chunked ] || { echo "missing /usr/local/sbin/homelab-rclone-copy-chunked" >&2; exit 10; }
|
|
/usr/local/sbin/skladchik-reports-monitor-backup:36:cleanup(){ rm -rf "$TMP"; }
|
|
/usr/local/sbin/skladchik-reports-monitor-backup:82: mapfile -t OLD < <(rclone lsf --dirs-only "${REMOTE}${REMOTE_BASE}/daily/" 2>/dev/null | sed 's#/$##' | sort | head -n -"${KEEP_DAILY}" || true)
|
|
/usr/local/sbin/skladchik-reports-monitor-backup:83: for D in "${OLD[@]}"; do [ -n "$D" ] && rclone purge "${REMOTE}${REMOTE_BASE}/daily/${D}" >/dev/null 2>&1 || true; done
|
|
/usr/local/sbin/skladchik-reports-monitor-backup:85:find "${LOCAL_ROOT}/daily" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort | head -n -"${KEEP_DAILY}" | while read -r D; do [ -n "$D" ] && rm -rf "${LOCAL_ROOT}/daily/${D}"; done
|
|
/usr/local/sbin/homelab-vm-local-dumps-retention:56: rclone purge "$PATH_REMOTE" >> "$REPORT" 2>&1 && DELETED=$((DELETED+1)) || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-extended-appbackup-run:12:run_one(){ APP="$1"; HOST="$2"; PATHS="$3"; NOTE="$4"; CONS="$5"; JOB_ID="homelab-${NODE}-app-${APP}-${RUN_TS}"; export JOB_ID; SHORT_TS="${RUN_TS:2:6}-${RUN_TS:9:6}"; export BACKUP_PATH_REL="$(homelab-backup-path job a "$APP" "$SHORT_TS")"; STAGE="${STAGING_ROOT}/${JOB_ID}"; PAYLOAD="$STAGE/payload"; ARCHIVE="${JOB_ID}.tar.gz"; LOG="/tmp/extended-appbackup-${APP}-${RUN_TS}.log"; rm -rf "$STAGE"; mkdir -p "$PAYLOAD/local"; echo "### app=$APP" >> "$REPORT"; echo "host=$HOST" >> "$REPORT"; echo "source_note=$NOTE" >> "$REPORT"; ssh -n -o BatchMode=yes -o ConnectTimeout=30 "$HOST" "sudo tar -C / --ignore-failed-read --exclude=.env --exclude=*.env --exclude=.env.* --exclude=*.bak* -czf - $PATHS" > "$PAYLOAD/local/${APP}-data.tar.gz" 2>"$LOG"; TAR_RC=$?; if [ "$TAR_RC" -eq 0 ] && tar -tzf "$PAYLOAD/local/${APP}-data.tar.gz" >/dev/null 2>&1; then SIZE="$(stat -c%s "$PAYLOAD/local/${APP}-data.tar.gz" 2>/dev/null || echo 0)"; echo "archive_payload_size_bytes=$SIZE" >> "$REPORT"; if [ "$SIZE" -le 128 ]; then echo "backup=BAD reason=archive_too_small" >> "$REPORT"; BAD=$((BAD+1)); rm -rf "$STAGE"; else sha256sum "$PAYLOAD/local/${APP}-data.tar.gz" > "$PAYLOAD/local/${APP}-data.tar.gz.sha256"; jq -n --arg type "extended-appbackup" --arg app "$APP" --arg host "$HOST" --arg source_note "$NOTE" --arg archive_payload_size_bytes "$SIZE" --arg env_files_read "no" --arg secret_values_printed "no" --arg consistency "$CONS" "{type:\$type,app_name:\$app,host:\$host,source_note:\$source_note,archive_payload_size_bytes:(\$archive_payload_size_bytes|tonumber),env_files_read:\$env_files_read,secret_values_printed:\$secret_values_printed,consistency:\$consistency}" > "$STAGE/metadata.json"; tar -C "$PAYLOAD" -czf "$STAGE/$ARCHIVE" .; rm -rf "$PAYLOAD"; JOB_ID="$JOB_ID" homelab-stage-2cloud-flat "$APP" "$STAGE" "$ARCHIVE" > "$LOG" 2>&1; RC=$?; if [ "$RC" -eq 0 ]; then echo "backup=OK" >> "$REPORT"; grep -E "^\[STEP\]|^\[SUCCESS\]" "$LOG" >> "$REPORT"; appbackupctl restore-check "$APP" >> "$REPORT" 2>&1 && echo "restore_check=OK" >> "$REPORT" || echo "restore_check=WARN_non_registry" >> "$REPORT"; OK=$((OK+1)); else echo "backup=BAD rc=$RC" >> "$REPORT"; grep -E "^\[ERROR\]|\[STEP\]|ERROR|error|failed|Failed|required|missing|denied|forbidden|checksum|rclone|copy|mkdir|jq|tar" "$LOG" | tail -n 60 >> "$REPORT"; BAD=$((BAD+1)); fi; fi; else echo "tar=BAD rc=$TAR_RC" >> "$REPORT"; grep -E "ERROR|error|failed|Failed|denied|No such|Cannot|tar:" "$LOG" | tail -n 40 >> "$REPORT"; BAD=$((BAD+1)); rm -rf "$STAGE"; fi; echo >> "$REPORT"; }
|
|
/usr/local/sbin/homelab-extended-appbackup-run:21:run_one healthchecks "$MON" "var/lib/healthchecks opt/stacks/healthchecks/local_settings.py" "healthchecks data plus local settings" "emergency_data_config_archive"
|
|
/usr/local/sbin/homelab-mkdocs-auto-refresh:53: rm -f /tmp/homelab-mkdocs-auto-refresh-git-fetch.out /tmp/homelab-mkdocs-auto-refresh-git-fetch.err
|
|
/usr/local/sbin/homelab-mkdocs-auto-refresh:57:rm -f /tmp/homelab-mkdocs-auto-refresh-git-fetch.out /tmp/homelab-mkdocs-auto-refresh-git-fetch.err
|
|
/usr/local/sbin/homelab-mkdocs-auto-refresh:77:ssh "${EDGE_SSH_OPTS[@]}" "$EDGE_HOST" "rm -rf '$STAGE' && mkdir -p '$STAGE'"
|
|
/usr/local/sbin/homelab-mkdocs-auto-refresh:85:rm -rf "$BASE/site.new" "$BASE/site.old"
|
|
/usr/local/sbin/homelab-mkdocs-auto-refresh:93:rm -rf "$BASE/site.old" "$STAGE"
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:30:CHKS="$WORK/checksums.txt"
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:64: rm -rf "$CDIR"
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:126: rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:128: rclone copyto "$CDIR/chunks.sha256" "$DEST/files/$bn.chunks/chunks.sha256" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:131: rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:135: rclone copyto "$CHKS" "$DEST/checksums.txt" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:136: rclone copyto "$FILES_TSV" "$DEST/files.tsv" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:137: rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:144: rm -rf "$VBASE"
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:146: rclone copyto "$DEST/manifest.json" "$VBASE/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:154: rclone copy "$DEST/files/$bn.chunks" "$VDIR" --filter "+ part-*" --filter "+ chunks.sha256" --filter "- *" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:163: rm -rf "$VDIR"
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:166: rclone copyto "$DEST/files/$bn" "$VBASE/$bn" --transfers 1 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:168: rm -f "$VBASE/$bn"
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:171: rm -rf "$VBASE"
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:184:rm -rf "$VERIFY_ROOT/$JOB_ID"
|
|
/usr/local/sbin/homelab-vm-local-dumps-2cloud:185:rm -rf "$CHUNK_ROOT/$JOB_ID"
|
|
/usr/local/sbin/homelab-gitea-secondary-backup:18: rm -f "$TMP"
|
|
/usr/local/sbin/homelab-gitea-secondary-backup:31:docker exec -u git -w /tmp gitea sh -lc \"rm -f /tmp/\$NAME && gitea dump -c /data/gitea/conf/app.ini -f /tmp/\$NAME\"
|
|
/usr/local/sbin/homelab-gitea-secondary-backup:33:docker exec gitea rm -f \"/tmp/\$NAME\"
|
|
/usr/local/sbin/homelab-gitea-secondary-backup:100: rm -f -- "$FILE" "$FILE.sha256"
|
|
/usr/local/sbin/homelab-gitea-secondary-backup:123: rm -f -- "$FILE" "$FILE.sha256"
|
|
/usr/local/sbin/homelab-gitea-secondary-backup:136: rm -f -- "$FILE" "$FILE.sha256"
|
|
/usr/local/sbin/homelab-rclone-copy-chunked:13:cleanup(){ rm -rf "$WORK"; }
|
|
/usr/local/sbin/homelab-rclone-copy-chunked:26:rclone copy "$PARTS" "${DEST}.parts" --transfers 2 --checkers 4 --retries 5 --low-level-retries 20 --stats 30s --log-level INFO
|
|
/usr/local/sbin/homelab-rclone-copy-chunked:27:rclone copyto "$WORK/${NAME}.sha256" "${DEST}.sha256" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO
|
|
/usr/local/sbin/homelab-rclone-copy-chunked:28:rclone copyto "$WORK/MANIFEST.txt" "${DEST}.manifest.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO
|
|
/usr/local/sbin/homelab-rclone-copy-chunked:29:rclone copyto "$WORK/MANIFEST.parts.txt" "${DEST}.parts/MANIFEST.parts.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level INFO
|
|
/usr/local/sbin/homelab-rclone-copy-chunked:30:REMOTE_COUNT=$(rclone lsf "${DEST}.parts" --files-only | grep -c "^part-" || true)
|
|
/usr/local/sbin/homelab-router-hard-power-cycle:23:SAFE_OUT="$(printf '%s\n' "$OUT" | sed -E '/(PASSWORD|TOKEN|SECRET|PRIVATE_KEY|ROOT_PASSWORD|ENCRYPTION_KEY|COOKIE|SESSION|AUTHORIZATION|API_KEY|LOCAL_KEY|rclone[.]conf|[.]env)/Id')"
|
|
/usr/local/sbin/homelab-health-metrics-write:10:rm -f "$tmp"
|
|
/usr/local/sbin/homelab-rclone-cat-flat-chunked:5:trap "rm -f \"$TMP\"" EXIT
|
|
/usr/local/sbin/homelab-rclone-cat-flat-chunked:8:while read -r PART PART_SIZE; do [[ "$PART" =~ ^part-[0-9]{4}$ && "$PART_SIZE" =~ ^[0-9]+$ ]] || { echo "BAD FLAT PART MANIFEST" >&2; exit 2; }; rclone cat "${DEST}.${PART}"; COUNT=$((COUNT+1)); done < "$TMP"
|
|
/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:4:cat /var/lib/skladchik-reports-monitor-backup/restore-check-health.txt 2>/dev/null || true
|
|
/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:7:systemctl is-enabled skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/enabled=/' || true
|
|
/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:8:systemctl is-active skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/active=/' || true
|
|
/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status:9:systemctl list-timers --all | grep skladchik-reports-monitor-backup-restore-check || true
|
|
/usr/local/sbin/homelab-context-refresh-direct:5:trap "rm -rf \"$d\"" EXIT
|
|
/usr/local/sbin/homelab-context-refresh-direct:32:printf "{\"schema_version\":1,\"channel\":\"homelab-runtime\",\"command_id\":\"%s\",\"status\":\"OK\",\"rc\":0,\"host\":\"pve01\",\"mode\":\"read-only\",\"component\":\"cluster-context\",\"started_at_utc\":\"%s\",\"finished_at_utc\":\"%s\",\"reference_register_checked\":true,\"reference_sha256\":\"%s\",\"error_register_checked\":true,\"error_register_sha256\":\"%s\",\"changes_made\":false,\"sanitized\":true,\"secrets_included\":false,\"private_addresses_included\":false,\"output\":\"AUTOMATIC_CLUSTER_CONTEXT_REFRESH=OK\"}\n" "$id" "$started" "$finished" "$ref" "$err" >"$d/$id.json"
|
|
/usr/local/sbin/homelab-context-refresh-direct:34:ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] debian@[PRIVATE_IP] "sudo -n /usr/local/sbin/homelab-runtime-receive $id /tmp/$id.json /tmp/$id.txt && rm -f /tmp/$id.json /tmp/$id.txt"
|
|
/usr/local/sbin/homelab-evidence-seal:16: rm -f "$TMP"
|
|
/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:15:DUMP_SIZE="$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$REMOTE" "sudo -n rm -f \"$REMOTE_DUMP\"; sudo -n -u clusteradmin pg_dump --format=custom --no-owner --no-privileges --file=\"$REMOTE_DUMP\" clusteradmin; sudo -n test -s \"$REMOTE_DUMP\"; sudo -n -u clusteradmin pg_restore --list \"$REMOTE_DUMP\" >/dev/null; sudo -n chown debian:debian \"$REMOTE_DUMP\"; sudo -n chmod 600 \"$REMOTE_DUMP\"; stat -c %s \"$REMOTE_DUMP\"")"
|
|
/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:18:ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$REMOTE" "rm -f \"$REMOTE_DUMP\""
|
|
/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:23:rm -f "$STAGE/runtime.tgz"
|
|
/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run:25:jq -n --arg app "$APP" --arg host "cluster-admin" --arg engine "postgresql-custom-dump+files" --argjson files "$FILES" --argjson dump_size "$DUMP_SIZE" "{app:\$app,engine:\$engine,source_host:\$host,source_paths:[\"cluster-admin:/opt/cluster-admin-incident-engine\",\"cluster-admin:/etc/cluster-admin-incident-engine\",\"cluster-admin:/var/lib/cluster-admin-incident-engine\",\"cluster-admin:PostgreSQL clusteradmin\"],file_count:\$files,database_dump_size_bytes:\$dump_size,database_dump_format:\"custom\",database_restore_list_check:\"ok\",includes_runtime_credentials:true,note:\"Incident engine application database state and runtime configuration\"}" > "$STAGE/metadata.json"
|
|
/usr/local/sbin/appbackup-retention:32:trap "rm -rf \"$WORK\"" EXIT
|
|
/usr/local/sbin/appbackup-retention:35: rclone lsf "${REMOTE%/}" --dirs-only --max-depth 1 > "$ROOT_LIST"
|
|
/usr/local/sbin/appbackup-retention:36: rclone lsf "${REMOTE%/}/${NODE}/apps/${CANONICAL}" --dirs-only --max-depth 1 > "$LEGACY_LIST"
|
|
/usr/local/sbin/appbackup-retention:39: rclone lsf "${REMOTE%/}/hb" --dirs-only --max-depth 1 > "$HB_LIST"
|
|
/usr/local/sbin/appbackup-retention:40: if grep -Fxq "a/" "$HB_LIST"; then rclone lsf "${REMOTE%/}/hb/a" --dirs-only --max-depth 1 > "$NEW_LIST"; fi
|
|
/usr/local/sbin/appbackup-retention:73: rclone cat "${REMOTE%/}/${LATEST_REL}/manifest.json" | jq -e --arg job "$LATEST" ".job_id == \$job" >/dev/null || { echo "REFUSE latest missing/bad remote_index=$INDEX"; exit 5; }
|
|
/usr/local/sbin/homelab-change-transaction:195: latest_tmp.unlink(missing_ok=True)
|
|
/usr/local/sbin/homelab-change-transaction:421: " homelab-change-transaction run PLAN --confirm PLAN"
|
|
/usr/local/sbin/homelab-check-private-vpn-hosts:15:trap 'rm -f "$tmp"' EXIT
|
|
/usr/local/sbin/homelab-backup-vaultwarden:26:rm -rf "$RESTORE"
|
|
/usr/local/sbin/homelab-health-metrics:16:category_for_check(){
|
|
/usr/local/sbin/homelab-health-metrics:22: *backup*|*vzdump*) printf "%s" "backup" ;;
|
|
/usr/local/sbin/homelab-health-metrics:31:tier_for_check(){
|
|
/usr/local/sbin/homelab-health-metrics:34: npmplus-kuma-config-*|authentik-*|gitea-*|vaultwarden-*|netbox-*|mail-cloud-restore-drill|edge-vm-vzdump-*|mkdocs-runtime-publish|npmplus-certificate-expiry) printf "%s" "P0" ;;
|
|
/usr/local/sbin/homelab-health-metrics:69: check="${base%.txt}"
|
|
/usr/local/sbin/homelab-health-metrics:70: category="$(category_for_check "$check")"
|
|
/usr/local/sbin/homelab-health-metrics:71: tier="$(tier_for_check "$check")"
|
|
/usr/local/sbin/homelab-health-metrics:75: check_esc="$(esc_label "$check")"
|
|
/usr/local/sbin/homelab-health-metrics:79: labels="host=\"$host_esc\",check=\"$check_esc\",category=\"$cat_esc\",tier=\"$tier_esc\",file=\"$file_esc\""
|
|
/usr/local/sbin/homelab-backup-health-evidence:8:backupctl check >/tmp/backup-health-emergency.log 2>&1 && EMERGENCY_OK=1 || true
|
|
/usr/local/sbin/homelab-backup-health-evidence:9:appbackupctl check >/tmp/backup-health-appframework.log 2>&1 && APPS_OK=1 || true
|
|
/usr/local/sbin/homelab-drift-check:7:HEALTH=/var/lib/homelab-health/drift-check.txt
|
|
/usr/local/sbin/homelab-drift-check:24:echo "STATUS=$STATUS TS=$(date -u +%Y-%m-%dT%H:%M:%SZ) TYPE=drift-check NOTE=$NOTE DIFF=$DIFF BASE=$BASE CURRENT=$CUR" > "$HEALTH"
|
|
/usr/local/sbin/crypto-portfolio-chart-health-check:5:trap 'rm -f "$TMP"' EXIT
|
|
/usr/local/sbin/homelab-rclone-copy-flat-chunked:12:trap "rm -rf \"$WORK\"" EXIT
|
|
/usr/local/sbin/homelab-rclone-copy-flat-chunked:23:while read -r PART PART_SIZE; do [[ "$PART" =~ ^part-[0-9]{4}$ && "$PART_SIZE" =~ ^[0-9]+$ ]] || { echo "BAD PART MANIFEST" >&2; exit 4; }; rclone copyto "$WORK/parts/$PART" "${DEST}.${PART}" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR; REMOTE_BYTES="$(rclone size --json "${DEST}.${PART}" | jq -r ".bytes // -1")"; [[ "$REMOTE_BYTES" = "$PART_SIZE" ]] || { echo "REMOTE PART SIZE MISMATCH $PART" >&2; exit 5; }; UPLOADED=$((UPLOADED+1)); done < "$WORK/MANIFEST.parts.txt"
|
|
/usr/local/sbin/homelab-rclone-copy-flat-chunked:25:rclone copyto "$WORK/${NAME}.sha256" "${DEST}.sha256" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR
|
|
/usr/local/sbin/homelab-rclone-copy-flat-chunked:26:rclone copyto "$WORK/MANIFEST.txt" "${DEST}.manifest.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR
|
|
/usr/local/sbin/homelab-rclone-copy-flat-chunked:27:rclone copyto "$WORK/MANIFEST.parts.txt" "${DEST}.parts.txt" --transfers 1 --checkers 1 --retries 5 --low-level-retries 20 --log-level ERROR
|
|
/usr/local/sbin/homelab-final-readiness-gate:10:check(){ name="$1"; shift; echo "### $name" >> "$REPORT"; timeout "${CHECK_TIMEOUT_SECONDS:-300}" "$@" >> "$REPORT" 2>&1; rc=$?; if [ "$rc" -eq 0 ]; then echo "PASS $name" >> "$REPORT"; PASS=$((PASS+1)); else echo "FAIL $name rc=$rc" >> "$REPORT"; FAIL=$((FAIL+1)); BAD=$((BAD+1)); fi; echo "" >> "$REPORT"; }
|
|
/usr/local/sbin/homelab-final-readiness-gate:18:check "backup framework evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/backup-framework.txt && grep -q SOURCE_STATUS=OK /var/lib/homelab-health/backup-framework.txt && grep -q FRESHNESS=FRESH /var/lib/homelab-health/backup-framework.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:21:check "backup SLA evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/backup-sla.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:22:check "backup coverage matrix evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/backup-coverage-matrix.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:23:check "extended appbackup evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/extended-appbackup.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:24:check "residual review" bash -c "grep -q STATUS=OK /var/lib/homelab-health/residual-review.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:25:check "forum snuffleupagus" bash -c "grep -q STATUS=OK /var/lib/homelab-health/forum-snuffleupagus.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:26:check "vm170 vm171 full strategy" bash -c "grep -q STATUS=OK /var/lib/homelab-health/vm170-vm171-full-strategy.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:27:check "external probe vps" bash -c "grep -q STATUS=OK /var/lib/homelab-health/external-probe-vps.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:28:check "netbird vps trust" bash -c "grep -q STATUS=OK /var/lib/homelab-health/netbird-vps-trust.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:29:check "netbird vps trust closure" bash -c "grep -q STATUS=OK /var/lib/homelab-health/netbird-vps-trust-closure.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:30:check "external probe runner decision" bash -c "grep -q STATUS=OK /var/lib/homelab-health/external-probe-runner-decision.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:31:check "netbird vps trust clean seal" bash -c "grep -q STATUS=OK /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:32:check "live tail audit" bash -c "grep -q STATUS=OK /var/lib/homelab-health/live-tail-audit.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:34:check "duty admin v2" bash -c "grep -q STATUS=OK /var/lib/homelab-health/duty-admin-v2.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:35:check "cluster admin observer" bash -c "grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-observer.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:36:check "cluster admin restricted probes" bash -c "grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-restricted-probes.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:37:check "cluster admin full observer" bash -c "grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-full-observer.txt && grep -q BAD=0 /var/lib/homelab-health/cluster-admin-full-observer.txt && grep -q WARN=0 /var/lib/homelab-health/cluster-admin-full-observer.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:38:check "cluster admin VM Mail Cloud backup" bash -c "grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt && grep -q BAD=0 /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt && grep -q MAIL_CLOUD_2COPY_VERIFY_OK /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:39:check "cluster admin webpanel sync" bash -c "grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-webpanel-sync.txt && grep -q BAD=0 /var/lib/homelab-health/cluster-admin-webpanel-sync.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:40:check "cluster admin webpanel" bash -c "grep -q STATUS=OK /var/lib/homelab-health/cluster-admin-webpanel.txt && grep -q BAD=0 /var/lib/homelab-health/cluster-admin-webpanel.txt && curl -fsS --max-time 5 http://[PRIVATE_IP]:8080/status.txt | grep -q STATUS=OK"
|
|
/usr/local/sbin/homelab-final-readiness-gate:41:check "node loss readiness" bash -c "grep -q STATUS=OK /var/lib/homelab-health/node-loss-readiness.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:42:check "node loss runbooks" bash -c "grep -q STATUS=OK /var/lib/homelab-health/node-loss-runbooks.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:43:check "power loss readiness" bash -c "grep -q STATUS=OK /var/lib/homelab-health/power-loss-readiness.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:44:check "power loss runbook" bash -c "grep -q STATUS=OK /var/lib/homelab-health/power-loss-runbook.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:45:check "ungated health backlog" bash -c "grep -q STATUS=OK /var/lib/homelab-health/ungated-health-backlog.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:46:check "pve03 root cleanup" bash -c "grep -q STATUS=OK /var/lib/homelab-health/pve03-root-cleanup.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:47:check "pve03 staging retention" bash -c "grep -q STATUS=OK /var/lib/homelab-health/pve03-staging-retention.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:48:check "pve03 staging retention proof" bash -c "grep -q STATUS=OK /var/lib/homelab-health/pve03-staging-retention-proof.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:49:check "pve03 staging retention cleanup" bash -c "grep -q STATUS=OK /var/lib/homelab-health/pve03-staging-retention-cleanup.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:50:check "pve03 failed unit cleanup" bash -c "grep -q STATUS=OK /var/lib/homelab-health/pve03-failed-unit-cleanup.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:51:check "remote git sops age readiness" bash -c "grep -q STATUS=OK /var/lib/homelab-health/remote-git-sops-age-readiness.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:52:check "remote git sops age policy" bash -c "grep -q STATUS=OK /var/lib/homelab-health/remote-git-sops-age-policy.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:53:check "desired state remote target trace" bash -c "grep -q STATUS=OK /var/lib/homelab-health/desired-state-remote-target-trace.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:54:check "desired state remote target" bash -c "grep -q STATUS=OK /var/lib/homelab-health/desired-state-remote-target.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:55:check "Kuma monitor policy evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/kuma-monitor-policy.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:56:check "safe autoheal evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/safe-autoheal.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:57:check "desired-state clean evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/desired-state.txt && test -z "$(git -C /srv/homelab-desired-state status --short)""
|
|
/usr/local/sbin/homelab-final-readiness-gate:58:check "runbooks evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/runbooks.txt && test -s /var/lib/homelab-runbooks/INDEX.md"
|
|
/usr/local/sbin/homelab-final-readiness-gate:59:check "incident journal evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/incident-journal.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:60:check "alerting evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/alerting.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:61:check "VM backup health evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/vm-backup.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:62:check "VM backup policy evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/vm-backup-policy.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:63:check "VM local dumps cloud" bash -c "grep -q STATUS=OK /var/lib/homelab-health/vm-local-dumps-cloud.txt && [ $(jq -r .count /var/lib/homelab-vm-backup/latest-vm-local-dumps-cloud.json) -ge 3 ] && [ $(jq -r .chunked_count /var/lib/homelab-vm-backup/latest-vm-local-dumps-cloud.json) -ge 1 ]"
|
|
/usr/local/sbin/homelab-final-readiness-gate:64:check "VM local dumps retention" bash -c "grep -q STATUS=OK /var/lib/homelab-health/vm-local-dumps-retention.txt && grep -q ACTIVE_LEFT=0 /var/lib/homelab-health/kuma-monitor-policy.txt || grep -q STATUS=OK /var/lib/homelab-health/vm-local-dumps-retention.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:65:check "capacity risk evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/capacity-risk.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:66:check "secret exposure evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/secret-exposure.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:67:check "golden state evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/golden-state.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:68:check "service registry evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/service-registry.txt"
|
|
/usr/local/sbin/homelab-final-readiness-gate:69:check "dependency map evidence" bash -c "grep -q STATUS=OK /var/lib/homelab-health/dependency-map.txt"
|
|
/usr/local/sbin/homelab-docker-health.sh:44: echo "containers_without_healthcheck=$NOHEALTH"
|
|
/usr/local/sbin/homelab-docker-health.sh:63: echo "# HELP homelab_docker_health_container_without_healthcheck Docker containers without native healthcheck."
|
|
/usr/local/sbin/homelab-docker-health.sh:64: echo "# TYPE homelab_docker_health_container_without_healthcheck gauge"
|
|
/usr/local/sbin/homelab-docker-health.sh:65: echo "homelab_docker_health_container_without_healthcheck $NOHEALTH"
|
|
/usr/local/sbin/homelab-backup-coverage-matrix:17:P1_SAFE_APPS="actual-budget homebox linkding mealie memos node-red syncthing filebrowser healthchecks ntfy vikunja homeassistant karakeep minio n8n"
|
|
/usr/local/sbin/homelab-backup-coverage-matrix:38: while read -r norm raw; do
|
|
/usr/local/sbin/homelab-backup-coverage-matrix:50: while read -r norm raw; do
|
|
/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:93:rclone mkdir "$TARGET_REMOTE:$REMOTE_PARTS" > "$WORK/logs/mkdir.out" 2>"$WORK/logs/mkdir.err" || MKDIR_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:95:rclone copy "$PARTS_DIR" "$TARGET_REMOTE:$REMOTE_PARTS" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > "$WORK/logs/upload_parts.out" 2>"$WORK/logs/upload_parts.err" || UPLOAD_PARTS_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:97:rclone copyto "$WORK/$BASENAME.sha256" "$TARGET_REMOTE:$REMOTE_SHA" --checksum > "$WORK/logs/upload_sha.out" 2>"$WORK/logs/upload_sha.err" || UPLOAD_SHA_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:99:rclone copyto "$WORK/MANIFEST.txt" "$TARGET_REMOTE:$REMOTE_MANIFEST" --checksum > "$WORK/logs/upload_manifest.out" 2>"$WORK/logs/upload_manifest.err" || UPLOAD_MANIFEST_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:101:rclone copyto "$WORK/SHA256SUMS.txt" "$TARGET_REMOTE:$REMOTE_SUMS" --checksum > "$WORK/logs/upload_sums.out" 2>"$WORK/logs/upload_sums.err" || UPLOAD_SUMS_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:104:rclone lsf "$TARGET_REMOTE:$REMOTE_PARTS" --format ps > "$WORK/logs/remote_lsf_parts.txt" 2>"$WORK/logs/remote_lsf_parts.err" || LSF_PARTS_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:110:rclone copy "$TARGET_REMOTE:$REMOTE_PARTS" "$DL_DIR/parts" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 > "$WORK/logs/download_parts.out" 2>"$WORK/logs/download_parts.err" || DOWNLOAD_PARTS_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:120:rclone lsf "$TARGET_REMOTE:$TARGET_BASE" --dirs-only 2>/dev/null | sed 's#/$##' | sort > "$DIR_LIST" || RETENTION_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:126: [ -n "$OLD" ] && rclone purge "$TARGET_REMOTE:$TARGET_BASE/$OLD" >> "$WORK/logs/retention.out" 2>>"$WORK/logs/retention.err"
|
|
/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:140: rm -rf "$PARTS_DIR" "$DL_DIR"
|
|
/usr/local/sbin/homelab-rehydrate-flat-chunked-dir:10: rm -f "$TMP_TARGET"
|
|
/usr/local/sbin/homelab-disk-space-health.py:10:checks = [
|
|
/usr/local/sbin/homelab-disk-space-health.py:21:for label, cmd in checks:
|
|
/usr/local/sbin/homelab-rclone-required-check.sh:4:RAW_FILE="/etc/homelab/rclone/required-remotes.txt"
|
|
/usr/local/sbin/homelab-rclone-required-check.sh:5:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt"
|
|
/usr/local/sbin/homelab-rclone-required-check.sh:17:CONFIGURED="$(rclone listremotes 2>/dev/null | sed 's/:$//' || true)"
|
|
/usr/local/sbin/homelab-rclone-required-check.sh:19: echo "No rclone remotes configured yet."
|
|
/usr/local/sbin/homelab-legacy-monitoring-backups-run:24: rm -f "$STAGE/remote.tgz"
|
|
/usr/local/sbin/homelab-legacy-monitoring-backups-run:43: rm -f "$STAGE/pve01.tgz"
|
|
/usr/local/sbin/homelab-legacy-monitoring-backups-run:48: rm -f "$STAGE/$LABEL.tgz"
|
|
/usr/local/sbin/homelab-legacy-monitoring-backups-run:52: rm -f "$STAGE/monitoring.tgz"
|
|
/usr/local/sbin/homelab-legacy-monitoring-backups-run:71: rm -f "$STAGE/config.tgz"
|
|
/usr/local/sbin/homelab-cluster-admin-inventory-sync:19:ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$REMOTE" "rm -f $REMOTE_TMP"
|
|
/usr/local/sbin/homelab-chat-run:390: "error_register_checked": True,
|
|
/usr/local/sbin/homelab-chat-run:489: rm -f \
|
|
/usr/local/sbin/homelab-check-private-vpn-hosts.real-20260624T131702Z:10: rm -rf "$TMPD" 2>/dev/null || true
|
|
/usr/local/sbin/homelab-beszel-backup-run:33:rm -f "$STAGE/remote.tgz"
|
|
/usr/local/sbin/homelab-beszel-backup-run:37:while IFS= read -r DB; do DB_COUNT=$((DB_COUNT+1)); RESULT="$(python3 -c "import sqlite3,sys;print(sqlite3.connect(sys.argv[1]).execute(\"PRAGMA integrity_check\").fetchone()[0])" "$DB" 2>/dev/null || echo error)"; [ "$RESULT" = ok ] || exit 4; DB_OK=$((DB_OK+1)); done < <(find "$STAGE/payload" -type f \( -name "*.db" -o -name "*.sqlite" -o -name "*.sqlite3" \) 2>/dev/null)
|
|
/usr/local/sbin/homelab-runbook-generate:47: f"Do not restore over a live service blindly. First confirm both encrypted cloud copies pass restore-check, target VM/container is snapshotted or intentionally stopped, and dependency map is OK.\n\n"
|
|
/usr/local/sbin/homelab-runbook-generate:51: f"homelab-service-registry-check\n"
|
|
/usr/local/sbin/skladchik-reports-monitor-supervisor:25:PVE_RESTORE_HEALTH = Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt")
|
|
/usr/local/sbin/skladchik-reports-monitor-supervisor:254: "skladchik-reports-monitor-backup-restore-check.timer",
|
|
/usr/local/sbin/backup-restore-check:9:rm -rf "$ROOT" && mkdir -p "$ROOT/a" "$ROOT/b"
|
|
/usr/local/sbin/backup-restore-check:10:trap 'echo "ERROR restore-check failed; kept: "; exit 1' ERR
|
|
/usr/local/sbin/backup-restore-check:12:rclone copy "$RA" "$ROOT/a" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-a.log 2>&1; homelab-rehydrate-chunked-dir "$ROOT/a" >>/tmp/homelab-restore-a.log 2>&1
|
|
/usr/local/sbin/backup-restore-check:13:(cd "$ROOT/a" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-a-sha.log && ARCHIVE="$(jq -r .archive_name manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null)
|
|
/usr/local/sbin/backup-restore-check:15:rclone copy "$RB" "$ROOT/b" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-b.log 2>&1; homelab-rehydrate-chunked-dir "$ROOT/b" >>/tmp/homelab-restore-b.log 2>&1
|
|
/usr/local/sbin/backup-restore-check:16:(cd "$ROOT/b" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-b-sha.log && ARCHIVE="$(jq -r .archive_name manifest.json)" && tar -tzf "$ARCHIVE" >/dev/null && jq -e --arg job "$JOB" ".job_id == \$job" manifest.json >/dev/null)
|
|
/usr/local/sbin/backup-restore-check:18:rm -rf "$ROOT"
|
|
/usr/local/sbin/homelab-vps-identity-audit:8:trap 'rm -rf "$TMP"' EXIT
|
|
/usr/local/sbin/homelab-vm-full-restore-drill:184:check_container_health() {
|
|
/usr/local/sbin/homelab-vm-full-restore-drill:349:check_container_health() {
|
|
/usr/local/sbin/homelab-vm-full-restore-drill:386:check_container_health "$test_vmid"
|
|
/usr/local/sbin/homelab-vm-backup-health:8:check_file(){
|
|
/usr/local/sbin/homelab-vm-backup-health:15:check_file /var/lib/homelab-health/backup.txt auto-vmct
|
|
/usr/local/sbin/homelab-vm-backup-health:16:check_file /var/lib/homelab-health/edge-vm-vzdump-backup.txt edge-vm-backup
|
|
/usr/local/sbin/homelab-vm-backup-health:17:check_file /var/lib/homelab-health/edge-vm-vzdump-offhost.txt edge-vm-offhost
|
|
/usr/local/sbin/homelab-vm-backup-health:18:check_file /var/lib/homelab-health/edge-vm-vzdump-restore.txt edge-vm-restore
|
|
/usr/local/sbin/homelab-ingress-failover:37: rm -f "$RESOURCES"
|
|
/usr/local/sbin/homelab-ingress-failover:252: rm -f "$tmp"
|
|
/usr/local/sbin/homelab-overall-health:8: /var/lib/homelab-health/drift-check.txt
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:64:case "$SOURCE" in /mnt/staging/offhost/edge-vm-vzdump-from-pve03/vzdump-qemu-130-*.vma.zst) ;; *) fail source_path_bad 2 ;; esac
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:111:rclone about "$TARGET_REMOTE:" > "$WORK/logs/about.out" 2>"$WORK/logs/about.err" || ABOUT_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:114:DIRS="$(rclone lsf "$TARGET_REMOTE:$TARGET_BASE" --dirs-only 2>/dev/null | sed 's#/$##' | sort || true)"
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:116: if rclone cat "$TARGET_REMOTE:$TARGET_BASE/$D/$BASENAME.sha256" 2>/dev/null | grep -q "$SHA"; then
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:188:rclone mkdir "$TARGET_REMOTE:$REMOTE_PARTS" > "$WORK/logs/mkdir.out" 2>"$WORK/logs/mkdir.err" || MKDIR_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:190:rclone copy "$PARTS_DIR" "$TARGET_REMOTE:$REMOTE_PARTS" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > "$WORK/logs/upload_parts.out" 2>"$WORK/logs/upload_parts.err" || UPLOAD_PARTS_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:192:rclone copyto "$WORK/$BASENAME.sha256" "$TARGET_REMOTE:$REMOTE_SHA" --checksum > "$WORK/logs/upload_sha.out" 2>"$WORK/logs/upload_sha.err" || UPLOAD_SHA_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:194:rclone copyto "$WORK/MANIFEST.txt" "$TARGET_REMOTE:$REMOTE_MANIFEST" --checksum > "$WORK/logs/upload_manifest.out" 2>"$WORK/logs/upload_manifest.err" || UPLOAD_MANIFEST_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:196:rclone copyto "$WORK/SHA256SUMS.txt" "$TARGET_REMOTE:$REMOTE_SUMS" --checksum > "$WORK/logs/upload_sums.out" 2>"$WORK/logs/upload_sums.err" || UPLOAD_SUMS_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:199:rclone size "$TARGET_REMOTE:$TARGET_DIR" --json > "$WORK/logs/remote_size.json" 2>"$WORK/logs/remote_size.err" || SIZE_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:201:rclone lsf "$TARGET_REMOTE:$REMOTE_PARTS" --format "ps" > "$WORK/logs/remote_lsf_parts.txt" 2>"$WORK/logs/remote_lsf_parts.err" || LSF_PARTS_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:207:rclone copy "$TARGET_REMOTE:$REMOTE_PARTS" "$DL_DIR/parts" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --progress > "$WORK/logs/download_parts.out" 2>"$WORK/logs/download_parts.err" || DOWNLOAD_PARTS_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:222: rm -rf "$DL_DIR" "$PARTS_DIR"
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:234: rclone lsf "$TARGET_REMOTE:$TARGET_BASE" --dirs-only 2>/dev/null | sed "s#/$##" | sort > "$DIR_LIST" || RETENTION_RC=$?
|
|
/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:239: [ -n "$OLD" ] && rclone purge "$TARGET_REMOTE:$TARGET_BASE/$OLD" >> "$WORK/logs/retention.out" 2>>"$WORK/logs/retention.err"
|
|
/usr/local/sbin/homelab-backup-edge-vm-vzdump:40:REMOTE_OUT="$(remote "OUT=/tmp/edge_vm_vzdump_${VMID}_${TS}.out; ERR=/tmp/edge_vm_vzdump_${VMID}_${TS}.err; : > \$OUT; : > \$ERR; vzdump '$VMID' --mode snapshot --compress zstd --dumpdir '$RUN_DIR' --remove 0 >\$OUT 2>\$ERR; RC=\$?; ARCHIVE=\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\n' | sort -n | tail -n1 | cut -d' ' -f2-); LOGFILE=\$(find '$RUN_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.log' -printf '%T@ %p\n' | sort -n | tail -n1 | cut -d' ' -f2-); SIZE=0; SHA=none; if [ -n \"\$ARCHIVE\" ] && [ -f \"\$ARCHIVE\" ]; then SIZE=\$(stat -c %s \"\$ARCHIVE\"); SHA=\$(sha256sum \"\$ARCHIVE\" | awk '{print \$1}'); fi; echo VZDUMP_RC=\$RC; echo ARCHIVE=\$ARCHIVE; echo LOGFILE=\$LOGFILE; echo SIZE=\$SIZE; echo SHA=\$SHA; echo OUT_TAIL=\$(tail -n 6 \$OUT | tr '\n' '|'); echo ERR_TAIL=\$(tail -n 6 \$ERR | tr '\n' '|')")"
|
|
/usr/local/sbin/homelab-backup-edge-vm-vzdump:62:dest "mkdir -p '$OFFHOST_DIR' && rm -f '$DEST_PATH.tmp' '$DEST_PATH.sha256'"
|
|
/usr/local/sbin/homelab-backup-edge-vm-vzdump:74:echo "STATUS=OK TS=$TS TYPE=edge-vm-vzdump-backup NODE=$NODE VMID=$VMID ARCHIVE=$ARCHIVE SIZE=$SIZE SHA256=$SHA SCSI1_BACKUP0=yes NOTES=timer_run" > "$HEALTH_DIR/edge-vm-vzdump-backup.txt"
|
|
/usr/local/sbin/homelab-backup-edge-vm-vzdump:75:echo "STATUS=OK TS=$TS TYPE=edge-vm-vzdump-offhost NODE=$NODE VMID=$VMID HOST=pve02 DEST=$DEST_PATH SIZE=$SIZE SHA256=$SHA NOTES=timer_run" > "$HEALTH_DIR/edge-vm-vzdump-offhost.txt"
|
|
/usr/local/sbin/homelab-backup-edge-vm-vzdump:76:echo "STATUS=OK TS=$TS TYPE=edge-vm-vzdump-restore-verify HOST=pve02 VMID=$VMID ARCHIVE=$DEST_PATH SIZE=$SIZE SHA256=$SHA ZSTD_RC=0 VMA_RC=0 NOTES=timer_run" > "$HEALTH_DIR/edge-vm-vzdump-restore.txt"
|
|
/usr/local/sbin/homelab-backup-edge-vm-vzdump:77:chmod 0644 "$HEALTH_DIR/edge-vm-vzdump-backup.txt" "$HEALTH_DIR/edge-vm-vzdump-offhost.txt" "$HEALTH_DIR/edge-vm-vzdump-restore.txt"
|
|
/usr/local/sbin/homelab-backup-edge-vm-vzdump:79:remote "find '$BASE_DIR' -mindepth 1 -maxdepth 1 -type d -name 'edge-vm-*' -printf '%T@ %p\n' | sort -nr | awk 'NR>${KEEP_LOCAL}{print \$2}' | xargs -r rm -rf"
|
|
/usr/local/sbin/homelab-backup-edge-vm-vzdump:80:dest "find '$OFFHOST_DIR' -maxdepth 1 -type f -name 'vzdump-qemu-${VMID}-*.vma.zst' -printf '%T@ %p\n' | sort -nr | awk 'NR>${KEEP_OFFHOST}{print \$2}' | while read f; do rm -f \"\$f\" \"\$f.sha256\"; done"
|
|
/usr/local/sbin/appbackup-dockge-stacks:30:rm -rf "$PAYLOAD"
|
|
/usr/local/sbin/appbackup-dockge-stacks:51:echo "[STEP] local checksum"
|
|
/usr/local/sbin/appbackup-dockge-stacks:52:(cd "$STAGE" && sha256sum "$ARCHIVE" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e . manifest.json >/dev/null)
|
|
/usr/local/sbin/appbackup-dockge-stacks:54:echo "[STEP] upload cloud A metadata"; rclone copy "$STAGE" "$RA" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s
|
|
/usr/local/sbin/appbackup-dockge-stacks:56:echo "[STEP] upload cloud B metadata"; rclone copy "$STAGE" "$RB" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s
|
|
/usr/local/sbin/appbackup-dockge-stacks:60:echo "[STEP] upload latest index"; rclone copyto "$STAGE/latest-backup-index.json" "$RAL"; rclone copyto "$STAGE/latest-backup-index.json" "$RBL"; rclone cat "$RAL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null; rclone cat "$RBL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null
|
|
/usr/local/sbin/appbackup-dockge-stacks:76:rm -rf "$STAGE" "$VERIFY"
|
|
/usr/local/sbin/homelab-cluster-passport:65:need_ok /var/lib/homelab-health/drift-check.txt drift-check
|
|
/usr/local/sbin/backup-retention:18: if [[ "$MODE" == "apply" ]]; then rclone cat "$BASE/$LATEST/manifest.json" | jq -e --arg job "$LATEST" ".job_id == \$job" >/dev/null || { echo "REFUSE latest missing/bad on $BASE"; exit 4; }; fi
|
|
/usr/local/sbin/backup-retention:24: for ((i=0; i<DELETE_COUNT; i++)); do TARGET_NAME="${DIRS[$i]}"; TARGET="$BASE/$TARGET_NAME"; [[ -n "$LATEST" && "$TARGET_NAME" == "$LATEST" ]] && { echo "REFUSE delete latest $TARGET"; exit 5; }; if [[ "$MODE" == "apply" ]]; then echo "DELETE $TARGET"; rclone purge "$TARGET"; else echo "WOULD_DELETE $TARGET"; fi; done
|
|
/usr/local/sbin/homelab-external-probe-vps-health:49:closed_tail=smoke_checks_endpoint_reachability_and_status_prefix_not_overall_ok
|
|
/usr/local/bin/skladchik-status:11:check_health() {
|
|
/usr/local/bin/skladchik-status:24:check_timer() {
|
|
/usr/local/bin/skladchik-status:38:check_timer skladchik-reports-monitor-backup.timer
|
|
/usr/local/bin/skladchik-status:39:check_timer skladchik-reports-monitor-backup-restore-check.timer
|
|
/usr/local/bin/skladchik-status:40:check_timer skladchik-reports-monitor-supervisor.timer
|
|
/usr/local/bin/skladchik-status:41:check_timer skladchik-reports-monitor-monthly-selftest-watch.timer
|
|
/usr/local/bin/skladchik-status:43:check_health "PVE01: ежедневный бэкап" "/var/lib/skladchik-reports-monitor-backup/health.txt"
|
|
/usr/local/bin/skladchik-status:44:check_health "PVE01: проверка восстановления" "/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt"
|
|
/usr/local/bin/skladchik-status:45:check_health "PVE01: общий supervisor" "/var/lib/skladchik-reports-monitor-supervisor/health.txt"
|
|
/usr/local/bin/skladchik-status:46:check_health "PVE01: контроль ежемесячного самотеста" "/var/lib/skladchik-reports-monitor-selftest-watch/health.txt"
|
|
/usr/local/bin/hchat:36: echo "USAGE=hchat tx-run PLAN --confirm PLAN"
|
|
/usr/local/bin/hchat:112: --confirm \
|
|
/etc/systemd/system/homelab-private-vpn-hosts-health.timer:2:Description=Run Homelab private VPN hosts health check every minute
|
|
/etc/systemd/system/homelab-vm-full-backup.service:2:Description=Full vzdump backup of VM170 and VM171 to offhost and two clouds
|
|
/etc/systemd/system/homelab-mail-cloud-restore-drill.service.d/10-superseded-noop.conf:3:ExecStart=/usr/local/sbin/homelab-superseded-unit-ok homelab-mail-cloud-restore-drill.service superseded_by_safety_net_restore_check
|
|
/etc/systemd/system/homelab-drift-check.service:2:Description=Homelab drift check
|
|
/etc/systemd/system/homelab-drift-check.service:6:ExecStart=/usr/local/sbin/homelab-drift-check
|
|
/etc/systemd/system/homelab-safe-exec-health.timer:2:Description=Regular allowlisted homelab health checks
|
|
/etc/systemd/system/suspend.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
/etc/systemd/system/homelab-crypto-portfolio-chart-health.timer:2:Description=Recurring Crypto portfolio chart health check
|
|
/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:14:ExecStart=/usr/sbin/pve-ha-crm start
|
|
/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:15:ExecStop=/usr/sbin/pve-ha-crm stop
|
|
/etc/systemd/system/multi-user.target.wants/postfix.service:19:# perform 2-stage startup
|
|
/etc/systemd/system/multi-user.target.wants/postfix.service:20:ExecStartPre=+postfix check
|
|
/etc/systemd/system/multi-user.target.wants/postfix.service:30:# Individual Postfix daemons are started as root, optionally perform chroot
|
|
/etc/systemd/system/multi-user.target.wants/postfix.service:52:# CAP_CHOWN can be removed.
|
|
/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:22:ExecStart=/usr/sbin/pve-ha-lrm start
|
|
/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:23:ExecStop=/usr/sbin/pve-ha-lrm stop
|
|
/etc/systemd/system/multi-user.target.wants/homelab-scrutiny.service:10:ExecStartPre=-/usr/bin/podman rm -f scrutiny
|
|
/etc/systemd/system/multi-user.target.wants/homelab-scrutiny.service:11:ExecStart=/usr/bin/podman run --name scrutiny --rm --pull=never --privileged --device /dev/nvme0 --device /dev/nvme0n1 -v /run/udev:/run/udev:ro -v /opt/stacks/scrutiny/config:/opt/scrutiny/config -v /opt/stacks/scrutiny/influxdb:/opt/scrutiny/influxdb -p 127.0.0.1:8086:8080 ghcr.io/analogj/scrutiny:latest-omnibus
|
|
/etc/systemd/system/multi-user.target.wants/rrdcached.service:2:# To customize, copy this file to /etc/systemd/system/
|
|
/etc/systemd/system/multi-user.target.wants/pve-guests.service:22:ExecStop=-/usr/bin/vzdump -stop
|
|
/etc/systemd/system/multi-user.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
/etc/systemd/system/multi-user.target.wants/e2scrub_reap.service:2:Description=Remove Stale Online ext4 Metadata Check Snapshots
|
|
/etc/systemd/system/multi-user.target.wants/corosync.service:20:# as well; before editing this line, you may want to check DefaultStandardError
|
|
/etc/systemd/system/homelab-dependency-map-check.service:2:Description=Homelab dependency map check
|
|
/etc/systemd/system/homelab-dependency-map-check.service:6:ExecStart=/usr/local/sbin/homelab-dependency-map-check
|
|
/etc/systemd/system/homelab-edge-vm-offhost-freshness.timer:2:Description=Run Homelab edge-vm VM130 offhost freshness sync daily after nightly vzdump
|
|
/etc/systemd/system/homelab-dependency-map-check.timer:2:Description=Homelab dependency map check timer
|
|
/etc/systemd/system/pve-manager.service:22:ExecStop=-/usr/bin/vzdump -stop
|
|
/etc/systemd/system/homelab-alerting-health.service:2:Description=Homelab alerting health check
|
|
/etc/systemd/system/homelab-edge-vm-vzdump-backup.service:2:Description=Homelab edge-vm VM-level vzdump backup chain
|
|
/etc/systemd/system/homelab-edge-vm-vzdump-backup.service:8:ExecStart=/usr/local/sbin/homelab-backup-edge-vm-vzdump
|
|
/etc/systemd/system/default.target.wants/nvmefc-boot-connections.service:19:RemoveIPC=yes
|
|
/etc/systemd/system/default.target.wants/nvmf-autoconnect.service:21:RemoveIPC=yes
|
|
/etc/systemd/system/homelab-scrutiny.service:10:ExecStartPre=-/usr/bin/podman rm -f scrutiny
|
|
/etc/systemd/system/homelab-scrutiny.service:11:ExecStart=/usr/bin/podman run --name scrutiny --rm --pull=never --privileged --device /dev/nvme0 --device /dev/nvme0n1 -v /run/udev:/run/udev:ro -v /opt/stacks/scrutiny/config:/opt/scrutiny/config -v /opt/stacks/scrutiny/influxdb:/opt/scrutiny/influxdb -p 127.0.0.1:8086:8080 ghcr.io/analogj/scrutiny:latest-omnibus
|
|
/etc/systemd/system/skladchik-reports-monitor-backup-restore-check.service:2:Description=Restore-check Skladchik reports monitor cloud backups
|
|
/etc/systemd/system/skladchik-reports-monitor-backup-restore-check.service:8:ExecStart=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check
|
|
/etc/systemd/system/homelab-gitea-secondary-backup.service:2:Description=Create and copy a verified Gitea backup to pve03
|
|
/etc/systemd/system/homelab-crypto-portfolio-chart-health.service:2:Description=Crypto portfolio chart health check
|
|
/etc/systemd/system/homelab-crypto-portfolio-chart-health.service:8:ExecStart=/usr/local/sbin/crypto-portfolio-chart-health-check
|
|
/etc/systemd/system/skladchik-reports-monitor-supervisor.service:7:ExecStart=/usr/local/sbin/skladchik-reports-monitor-supervisor check
|
|
/etc/systemd/system/homelab-service-registry-check.service:2:Description=Homelab service registry health check
|
|
/etc/systemd/system/homelab-service-registry-check.service:6:ExecStart=/usr/local/sbin/homelab-service-registry-check
|
|
/etc/systemd/system/homelab-backup-healthcheck.service:2:Description=Homelab backup healthcheck
|
|
/etc/systemd/system/homelab-backup-healthcheck.service:6:ExecStart=/root/homelab-backup-healthcheck-wrapper.sh
|
|
/etc/systemd/system/nut-driver@cyberpower.service.d/nut-driver-enumerator-generated.conf:2:# DO NOT EDIT: This file would be removed or overwritten by that service
|
|
/etc/systemd/system/sysinit.target.wants/systemd-pstore.service:11:Description=Platform Persistent Storage Archival
|
|
/etc/systemd/system/sysinit.target.wants/pvenetcommit.service:8:ExecStartPre=-/bin/rm -f /etc/openvswitch/conf.db
|
|
/etc/systemd/system/hybrid-sleep.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
/etc/systemd/system/homelab-private-vpn-hosts-health.service:2:Description=Homelab private VPN hosts health check
|
|
/etc/systemd/system/homelab-private-vpn-hosts-health.service:8:ExecStart=/bin/bash -c 'set +e; /usr/local/sbin/homelab-check-private-vpn-hosts > /var/lib/homelab-health/private-vpn-hosts.txt.tmp; rc=$?; mv /var/lib/homelab-health/private-vpn-hosts.txt.tmp /var/lib/homelab-health/private-vpn-hosts.txt; chmod 0644 /var/lib/homelab-health/private-vpn-hosts.txt; exit $rc'
|
|
/etc/systemd/system/netbird-peers-health.service:2:Description=NetBird peers health check
|
|
/etc/systemd/system/homelab-pve03-staging-capacity-health.timer:2:Description=Run Homelab pve03 staging capacity health check
|
|
/etc/systemd/system/timers.target.wants/homelab-private-vpn-hosts-health.timer:2:Description=Run Homelab private VPN hosts health check every minute
|
|
/etc/systemd/system/timers.target.wants/homelab-crypto-portfolio-chart-health.timer:2:Description=Recurring Crypto portfolio chart health check
|
|
/etc/systemd/system/timers.target.wants/homelab-edge-vm-offhost-freshness.timer:2:Description=Run Homelab edge-vm VM130 offhost freshness sync daily after nightly vzdump
|
|
/etc/systemd/system/timers.target.wants/homelab-dependency-map-check.timer:2:Description=Homelab dependency map check timer
|
|
/etc/systemd/system/timers.target.wants/homelab-pve03-staging-capacity-health.timer:2:Description=Run Homelab pve03 staging capacity health check
|
|
/etc/systemd/system/timers.target.wants/homelab-drift-check.timer:2:Description=Homelab drift check timer
|
|
/etc/systemd/system/timers.target.wants/homelab-service-registry-check.timer:2:Description=Homelab service registry health check timer
|
|
/etc/systemd/system/timers.target.wants/homelab-backup-healthcheck.timer:2:Description=Run homelab backup healthcheck daily
|
|
/etc/systemd/system/timers.target.wants/homelab-backup-healthcheck.timer:8:Unit=homelab-backup-healthcheck.service
|
|
/etc/systemd/system/timers.target.wants/netbird-peers-health.timer:2:Description=Run NetBird peers health check
|
|
/etc/systemd/system/timers.target.wants/homelab-alerting-health.timer:2:Description=Homelab alerting health check timer
|
|
/etc/systemd/system/timers.target.wants/skladchik-reports-monitor-backup-restore-check.timer:2:Description=Weekly restore-check of Skladchik reports monitor cloud backups
|
|
/etc/systemd/system/timers.target.wants/skladchik-reports-monitor-backup-restore-check.timer:8:Unit=skladchik-reports-monitor-backup-restore-check.service
|
|
/etc/systemd/system/homelab-edge-vm-vzdump-backup.timer:8:Unit=homelab-edge-vm-vzdump-backup.service
|
|
/etc/systemd/system/homelab-drift-check.timer:2:Description=Homelab drift check timer
|
|
/etc/systemd/system/homelab-service-registry-check.timer:2:Description=Homelab service registry health check timer
|
|
/etc/systemd/system/homelab-pve03-staging-capacity-health.service:2:Description=Homelab pve03 staging capacity health check
|
|
/etc/systemd/system/homelab-backup-healthcheck.timer:2:Description=Run homelab backup healthcheck daily
|
|
/etc/systemd/system/homelab-backup-healthcheck.timer:8:Unit=homelab-backup-healthcheck.service
|
|
/etc/systemd/system/homelab-safe-exec-health.service:2:Description=Run allowlisted homelab health checks
|
|
/etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.service:2:Description=Homelab Mail Cloud Nextcloud VM vzdump upload
|
|
/etc/systemd/system/hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
/etc/systemd/system/netbird-peers-health.timer:2:Description=Run NetBird peers health check
|
|
/etc/systemd/system/homelab-alerting-health.timer:2:Description=Homelab alerting health check timer
|
|
/etc/systemd/system/skladchik-reports-monitor-backup-restore-check.timer:2:Description=Weekly restore-check of Skladchik reports monitor cloud backups
|
|
/etc/systemd/system/skladchik-reports-monitor-backup-restore-check.timer:8:Unit=skladchik-reports-monitor-backup-restore-check.service
|
|
/etc/systemd/system/suspend-then-hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
========== NODE=pve02 ==========
|
|
=== REMOTES ===
|
|
=== TIMERS ===
|
|
Tue 2026-07-21 18:41:03 MSK 3h 46min Tue 2026-07-21 12:41:22 MSK 2h 12min ago forum-prod-fresh5-cloud-backup.timer forum-prod-fresh5-cloud-backup.service
|
|
Wed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:22 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service
|
|
Wed 2026-07-22 04:20:00 MSK 13h Tue 2026-07-21 04:20:12 MSK 10h ago homelab-cluster-admin-vm-mail-cloud-backup.timer homelab-cluster-admin-vm-mail-cloud-backup.service
|
|
=== SERVICES ===
|
|
codevipe-cloud-backup.service static -
|
|
dpkg-db-backup.service static -
|
|
forum-prod-fresh5-cloud-backup.service static -
|
|
homelab-cluster-admin-vm-mail-cloud-backup.service static -
|
|
/usr/local/sbin/homelab-rclone-required-check.sh:4:RAW_FILE="/etc/homelab/rclone/required-remotes.txt"
|
|
/usr/local/sbin/homelab-rclone-required-check.sh:5:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt"
|
|
/usr/local/sbin/homelab-rclone-required-check.sh:17:CONFIGURED="$(rclone listremotes 2>/dev/null | sed 's/:$//' || true)"
|
|
/usr/local/sbin/homelab-rclone-required-check.sh:19: echo "No rclone remotes configured yet."
|
|
/usr/local/sbin/routerguard-watchdog-power-entrypoint:22:cleanup(){ rm -f "$PRE" "$ARM"; }
|
|
/usr/local/sbin/routerguard-watchdog-power-entrypoint:38:rm -f "$ARM"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:4:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:18: "checksums/"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:21:echo "=== homelab rclone crypt healthcheck: $(hostname) ==="
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:40: if ! rclone listremotes | sed 's/:$//' | grep -qx "$REMOTE"; then
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:47: if ! rclone about "${REMOTE}:"; then
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:48: echo "FAIL: rclone about failed: $REMOTE"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:54: ROOT_LIST="$(rclone lsf "${REMOTE}:/" || true)"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:76: if rclone cat "${REMOTE}:/latest-backup-index.json" | jq -e '.schema and .remote and .status and .items' >/dev/null; then
|
|
/usr/local/sbin/codevipe-cloud-backup.sh:28: rclone copy "$CACHE_ROOT/$BASE" "$R:$REMOTE_PATH/$BASE" --stats 30s
|
|
/usr/local/sbin/codevipe-cloud-backup.sh:31:find "$CACHE_ROOT" -mindepth 1 -maxdepth 1 -type d -mtime +14 -print -exec rm -rf {} \;
|
|
/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:26:rm -f "$STAGE/$base"
|
|
/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:34:format=tar.gz split_parts_1500M rclone_crypt_remote
|
|
/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:46: rclone copy "$STAGE" "$dest" --transfers 2 --checkers 4 --retries 3 --low-level-retries 10 --stats-one-line >/tmp/forum-cloud-copy-${r}-$TS.log 2>&1
|
|
/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:49: remote_count=$(rclone lsf "$dest" --files-only 2>/tmp/forum-cloud-lsf-${r}-$TS.err | wc -l)
|
|
/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:55:find /root/backups/forum-prod-fresh5 -mindepth 1 -maxdepth 1 -type d -mtime +14 -exec rm -rf {} +
|
|
/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:73: retention_remote_count=$(rclone lsf "$r:$REMOTE_BASE/$retention_job" --files-only 2>/dev/null | wc -l)
|
|
/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:78: rm -rf --one-file-system "$retention_dir"
|
|
/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh:83: rm -f "$retention_keep_tmp"
|
|
/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:24:rclone listremotes 2>/tmp/cluster-admin-rclone-listremotes.err | grep -E 'mail.*crypt|crypt.*mail' | sed 's/:$//' | sort > /tmp/cluster-admin-mail-remotes.txt
|
|
/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:41: rm -rf "$STAGE/$OLD_JOB"
|
|
/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:51: ls "$d"/vzdump-qemu-${VMID}-*.vma.zst >/dev/null 2>&1 || { echo "DELETE_EMPTY_LOCAL_JOB=$b"; rm -rf "$d"; }
|
|
/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:115: rm -f "$TMPVMA"
|
|
/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:124: rm -f "$TMPVMA"
|
|
/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:158: rclone copy "$JOBDIR" "$DEST" --checksum --transfers 2 --checkers 4 --retries 3 --low-level-retries 5 >/tmp/cluster-admin-rclone-copy-$R.log 2>&1
|
|
/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:161: rclone copyto "$JOBDIR/MANIFEST.txt" "$R:homelab-backups/pve02/cluster-admin-vm180/latest/MANIFEST.txt" --retries 3 >/tmp/cluster-admin-rclone-latest-$R.log 2>&1
|
|
/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:164: REMOTE_SHA="$(rclone cat "$DEST/$BASE.sha256" 2>/tmp/cluster-admin-rclone-catsha-$R.err | awk '{print $1}' | head -n1)"
|
|
/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup:177:find "$STAGE" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort -r | tail -n +4 | while read old; do rm -rf "$STAGE/$old"; echo "LOCAL_STAGE_DELETED=$old"; done
|
|
/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:20: rm -rf -- "$WORK"
|
|
/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:117:for CMD in awk cat chmod date grep head mkdir mktemp rclone rm sed sha256sum sort sqlite3 tail tar wc; do
|
|
/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:131:rclone lsf "$REMOTE" --recursive >"$REMOTE_LIST" 2>"$ERROR_FILE" \
|
|
/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:149:rclone copyto "$REMOTE$CRITICAL_REMOTE_PATH" "$CRITICAL_ARCHIVE" 2>"$ERROR_FILE" \
|
|
/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:152:rclone copyto "$REMOTE$CRITICAL_REMOTE_SHA_PATH" "$CRITICAL_SHA_FILE" 2>"$ERROR_FILE" \
|
|
/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill:251:INTEGRITY_RESULT="$(sqlite3 "$DB_PATH" 'PRAGMA integrity_check;' 2>"$ERROR_FILE")" \
|
|
/usr/local/sbin/routerguard-network:91: rm -f "$RUN/active"
|
|
/usr/local/sbin/routerguard-network:100: rm -rf "$RUN"
|
|
/etc/systemd/system/suspend.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
/etc/systemd/system/suspend-then-hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
/etc/systemd/system/hybrid-sleep.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
/etc/systemd/system/multi-user.target.wants/corosync.service:20:# as well; before editing this line, you may want to check DefaultStandardError
|
|
/etc/systemd/system/multi-user.target.wants/postfix.service:19:# perform 2-stage startup
|
|
/etc/systemd/system/multi-user.target.wants/postfix.service:20:ExecStartPre=+postfix check
|
|
/etc/systemd/system/multi-user.target.wants/postfix.service:30:# Individual Postfix daemons are started as root, optionally perform chroot
|
|
/etc/systemd/system/multi-user.target.wants/postfix.service:52:# CAP_CHOWN can be removed.
|
|
/etc/systemd/system/multi-user.target.wants/pve-guests.service:22:ExecStop=-/usr/bin/vzdump -stop
|
|
/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:22:ExecStart=/usr/sbin/pve-ha-lrm start
|
|
/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:23:ExecStop=/usr/sbin/pve-ha-lrm stop
|
|
/etc/systemd/system/multi-user.target.wants/rrdcached.service:2:# To customize, copy this file to /etc/systemd/system/
|
|
/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:14:ExecStart=/usr/sbin/pve-ha-crm start
|
|
/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:15:ExecStop=/usr/sbin/pve-ha-crm stop
|
|
/etc/systemd/system/multi-user.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
/etc/systemd/system/multi-user.target.wants/e2scrub_reap.service:2:Description=Remove Stale Online ext4 Metadata Check Snapshots
|
|
/etc/systemd/system/default.target.wants/nvmefc-boot-connections.service:19:RemoveIPC=yes
|
|
/etc/systemd/system/default.target.wants/nvmf-autoconnect.service:21:RemoveIPC=yes
|
|
/etc/systemd/system/sysinit.target.wants/pvenetcommit.service:8:ExecStartPre=-/bin/rm -f /etc/openvswitch/conf.db
|
|
/etc/systemd/system/sysinit.target.wants/systemd-pstore.service:11:Description=Platform Persistent Storage Archival
|
|
/etc/systemd/system/pve-manager.service:22:ExecStop=-/usr/bin/vzdump -stop
|
|
/etc/systemd/system/hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
========== NODE=pve03 ==========
|
|
=== REMOTES ===
|
|
=== TIMERS ===
|
|
Wed 2026-07-22 00:00:00 MSK 9h Tue 2026-07-21 00:00:10 MSK 14h ago dpkg-db-backup.timer dpkg-db-backup.service
|
|
=== SERVICES ===
|
|
dpkg-db-backup.service static -
|
|
/usr/local/sbin/homelab-immutable-offline-verify:11: rm -f "$TMP"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:4:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:18: "checksums/"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:21:echo "=== homelab rclone crypt healthcheck: $(hostname) ==="
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:40: if ! rclone listremotes | sed 's/:$//' | grep -qx "$REMOTE"; then
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:47: if ! rclone about "${REMOTE}:"; then
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:48: echo "FAIL: rclone about failed: $REMOTE"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:54: ROOT_LIST="$(rclone lsf "${REMOTE}:/" || true)"
|
|
/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh:76: if rclone cat "${REMOTE}:/latest-backup-index.json" | jq -e '.schema and .remote and .status and .items' >/dev/null; then
|
|
/usr/local/sbin/homelab-rclone-required-check.sh:4:RAW_FILE="/etc/homelab/rclone/required-remotes.txt"
|
|
/usr/local/sbin/homelab-rclone-required-check.sh:5:CRYPT_FILE="/etc/homelab/rclone/required-crypt-remotes.txt"
|
|
/usr/local/sbin/homelab-rclone-required-check.sh:17:CONFIGURED="$(rclone listremotes 2>/dev/null | sed 's/:$//' || true)"
|
|
/usr/local/sbin/homelab-rclone-required-check.sh:19: echo "No rclone remotes configured yet."
|
|
/etc/systemd/system/hybrid-sleep.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
/etc/systemd/system/multi-user.target.wants/pve-guests.service:22:ExecStop=-/usr/bin/vzdump -stop
|
|
/etc/systemd/system/multi-user.target.wants/corosync.service:20:# as well; before editing this line, you may want to check DefaultStandardError
|
|
/etc/systemd/system/multi-user.target.wants/rrdcached.service:2:# To customize, copy this file to /etc/systemd/system/
|
|
/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:14:ExecStart=/usr/sbin/pve-ha-crm start
|
|
/etc/systemd/system/multi-user.target.wants/pve-ha-crm.service:15:ExecStop=/usr/sbin/pve-ha-crm stop
|
|
/etc/systemd/system/multi-user.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
/etc/systemd/system/multi-user.target.wants/postfix.service:19:# perform 2-stage startup
|
|
/etc/systemd/system/multi-user.target.wants/postfix.service:20:ExecStartPre=+postfix check
|
|
/etc/systemd/system/multi-user.target.wants/postfix.service:30:# Individual Postfix daemons are started as root, optionally perform chroot
|
|
/etc/systemd/system/multi-user.target.wants/postfix.service:52:# CAP_CHOWN can be removed.
|
|
/etc/systemd/system/multi-user.target.wants/e2scrub_reap.service:2:Description=Remove Stale Online ext4 Metadata Check Snapshots
|
|
/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:22:ExecStart=/usr/sbin/pve-ha-lrm start
|
|
/etc/systemd/system/multi-user.target.wants/pve-ha-lrm.service:23:ExecStop=/usr/sbin/pve-ha-lrm stop
|
|
/etc/systemd/system/sysinit.target.wants/pvenetcommit.service:8:ExecStartPre=-/bin/rm -f /etc/openvswitch/conf.db
|
|
/etc/systemd/system/sysinit.target.wants/systemd-pstore.service:11:Description=Platform Persistent Storage Archival
|
|
/etc/systemd/system/hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
/etc/systemd/system/suspend-then-hibernate.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
/etc/systemd/system/default.target.wants/nvmf-autoconnect.service:21:RemoveIPC=yes
|
|
/etc/systemd/system/default.target.wants/nvmefc-boot-connections.service:19:RemoveIPC=yes
|
|
/etc/systemd/system/suspend.target.wants/grub-common.service:9:ExecStartPre=/bin/sh -c '[ -s /boot/grub/grubenv ] || rm -f /boot/grub/grubenv; mkdir -p /boot/grub'
|
|
/etc/systemd/system/pve-manager.service:22:ExecStop=-/usr/bin/vzdump -stop
|
|
=== CLOUD_HEALTH_AND_OBJECTS ===
|
|
STATUS=FAIL
|
|
JOB=filebrowser-backup STATUS=OK EVIDENCE=artifact_or_health AGE=14443 MAX_AGE=259200
|
|
JOB=filebrowser-restore-validate STATUS=STALE EVIDENCE=artifact_or_health AGE=1016424 MAX_AGE=864000
|
|
JOB=homelab-appbackup-dockge-stacks STATUS=OK EVIDENCE=artifact_or_health AGE=22248 MAX_AGE=172800
|
|
JOB=homelab-appbackup-npmplus-edge STATUS=OK EVIDENCE=artifact_or_health AGE=20573 MAX_AGE=172800
|
|
JOB=homelab-authentik-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987790 MAX_AGE=172800
|
|
JOB=homelab-backup-coverage-matrix STATUS=OK EVIDENCE=artifact_or_health AGE=4346 MAX_AGE=172800
|
|
JOB=homelab-backup-healthcheck STATUS=OK EVIDENCE=execution_only AGE=10631 MAX_AGE=172800
|
|
JOB=homelab-backup-sla-health STATUS=FAIL EVIDENCE=artifact_or_health AGE=144 MAX_AGE=172800
|
|
JOB=homelab-edge-vm-offhost-freshness STATUS=OK EVIDENCE=artifact_or_health AGE=18426 MAX_AGE=172800
|
|
JOB=homelab-emergency-backup STATUS=OK EVIDENCE=artifact_or_health AGE=24539 MAX_AGE=172800
|
|
JOB=homelab-extended-appbackup STATUS=OK EVIDENCE=artifact_or_health AGE=14156 MAX_AGE=172800
|
|
JOB=homelab-gitea-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987785 MAX_AGE=172800
|
|
JOB=homelab-golden-state-index STATUS=OK EVIDENCE=artifact_or_health AGE=4112 MAX_AGE=172800
|
|
JOB=homelab-mail-cloud-critical-upload STATUS=OK EVIDENCE=artifact_or_health AGE=11997 MAX_AGE=172800
|
|
JOB=homelab-mail-cloud-edge-vm-upload STATUS=OK EVIDENCE=artifact_or_health AGE=174045 MAX_AGE=864000
|
|
JOB=homelab-mail-cloud-nextcloud-vm-upload STATUS=OK EVIDENCE=artifact_or_health AGE=7575 MAX_AGE=864000
|
|
JOB=homelab-mail-cloud-restore-drill STATUS=STALE EVIDENCE=artifact_or_health AGE=988269 MAX_AGE=864000
|
|
JOB=homelab-npmplus-kuma-config-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=60071 MAX_AGE=172800
|
|
JOB=homelab-router-running-config-mail-cloud STATUS=OK EVIDENCE=artifact_or_health AGE=17076 MAX_AGE=172800
|
|
JOB=homelab-vaultwarden-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987768 MAX_AGE=172800
|
|
JOB=homelab-verified-backup-health STATUS=OK EVIDENCE=artifact_or_health AGE=11845 MAX_AGE=172800
|
|
JOB=homelab-vm-backup-health STATUS=OK EVIDENCE=artifact_or_health AGE=8742 MAX_AGE=172800
|
|
JOB=homelab-vm-backup-policy STATUS=OK EVIDENCE=artifact_or_health AGE=5548 MAX_AGE=172800
|
|
JOB=homelab-vm-full-backup STATUS=OK EVIDENCE=artifact_or_health AGE=24778 MAX_AGE=172800
|
|
JOB=homelab-vm-full-restore-drill STATUS=OK EVIDENCE=artifact_or_health AGE=523085 MAX_AGE=10368000
|
|
JOB=homelab-vm-local-dumps-2cloud STATUS=OK EVIDENCE=artifact_or_health AGE=172141 MAX_AGE=864000
|
|
JOB=homelab-vm-local-dumps-retention STATUS=OK EVIDENCE=artifact_or_health AGE=168839 MAX_AGE=864000
|
|
JOB=immich-media-offhost-sync STATUS=OK EVIDENCE=artifact_or_health AGE=4286 MAX_AGE=259200
|
|
JOB=memos-backup STATUS=OK EVIDENCE=artifact_or_health AGE=14547 MAX_AGE=259200
|
|
JOB=netbird-vps-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=1747896 MAX_AGE=864000
|
|
JOB=paperless-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=16347 MAX_AGE=259200
|
|
JOB=paperless-restore-dry-run STATUS=STALE EVIDENCE=artifact_or_health AGE=1001986 MAX_AGE=864000
|
|
JOB=skladchik-reports-monitor-backup STATUS=OK EVIDENCE=artifact_or_health AGE=19639 MAX_AGE=172800
|
|
JOB=skladchik-reports-monitor-backup-restore-check STATUS=OK EVIDENCE=artifact_or_health AGE=190273 MAX_AGE=864000
|
|
JOB=vm160-remote-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=26607 MAX_AGE=172800
|
|
========== BACKUP-FAILURE-ROOTCAUSE-067 ==========
|
|
STATUS=FAIL
|
|
JOB=filebrowser-backup STATUS=OK EVIDENCE=artifact_or_health AGE=14443 MAX_AGE=259200
|
|
JOB=filebrowser-restore-validate STATUS=STALE EVIDENCE=artifact_or_health AGE=1016424 MAX_AGE=864000
|
|
JOB=homelab-appbackup-dockge-stacks STATUS=OK EVIDENCE=artifact_or_health AGE=22248 MAX_AGE=172800
|
|
JOB=homelab-appbackup-npmplus-edge STATUS=OK EVIDENCE=artifact_or_health AGE=20573 MAX_AGE=172800
|
|
JOB=homelab-authentik-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987790 MAX_AGE=172800
|
|
JOB=homelab-backup-coverage-matrix STATUS=OK EVIDENCE=artifact_or_health AGE=4346 MAX_AGE=172800
|
|
JOB=homelab-backup-healthcheck STATUS=OK EVIDENCE=execution_only AGE=10631 MAX_AGE=172800
|
|
JOB=homelab-backup-sla-health STATUS=FAIL EVIDENCE=artifact_or_health AGE=144 MAX_AGE=172800
|
|
JOB=homelab-edge-vm-offhost-freshness STATUS=OK EVIDENCE=artifact_or_health AGE=18426 MAX_AGE=172800
|
|
JOB=homelab-emergency-backup STATUS=OK EVIDENCE=artifact_or_health AGE=24539 MAX_AGE=172800
|
|
JOB=homelab-extended-appbackup STATUS=OK EVIDENCE=artifact_or_health AGE=14156 MAX_AGE=172800
|
|
JOB=homelab-gitea-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987785 MAX_AGE=172800
|
|
JOB=homelab-golden-state-index STATUS=OK EVIDENCE=artifact_or_health AGE=4112 MAX_AGE=172800
|
|
JOB=homelab-mail-cloud-critical-upload STATUS=OK EVIDENCE=artifact_or_health AGE=11997 MAX_AGE=172800
|
|
JOB=homelab-mail-cloud-edge-vm-upload STATUS=OK EVIDENCE=artifact_or_health AGE=174045 MAX_AGE=864000
|
|
JOB=homelab-mail-cloud-nextcloud-vm-upload STATUS=OK EVIDENCE=artifact_or_health AGE=7575 MAX_AGE=864000
|
|
JOB=homelab-mail-cloud-restore-drill STATUS=STALE EVIDENCE=artifact_or_health AGE=988269 MAX_AGE=864000
|
|
JOB=homelab-npmplus-kuma-config-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=60071 MAX_AGE=172800
|
|
JOB=homelab-router-running-config-mail-cloud STATUS=OK EVIDENCE=artifact_or_health AGE=17076 MAX_AGE=172800
|
|
JOB=homelab-vaultwarden-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987768 MAX_AGE=172800
|
|
JOB=homelab-verified-backup-health STATUS=OK EVIDENCE=artifact_or_health AGE=11845 MAX_AGE=172800
|
|
JOB=homelab-vm-backup-health STATUS=OK EVIDENCE=artifact_or_health AGE=8742 MAX_AGE=172800
|
|
JOB=homelab-vm-backup-policy STATUS=OK EVIDENCE=artifact_or_health AGE=5548 MAX_AGE=172800
|
|
JOB=homelab-vm-full-backup STATUS=OK EVIDENCE=artifact_or_health AGE=24778 MAX_AGE=172800
|
|
JOB=homelab-vm-full-restore-drill STATUS=OK EVIDENCE=artifact_or_health AGE=523085 MAX_AGE=10368000
|
|
JOB=homelab-vm-local-dumps-2cloud STATUS=OK EVIDENCE=artifact_or_health AGE=172141 MAX_AGE=864000
|
|
JOB=homelab-vm-local-dumps-retention STATUS=OK EVIDENCE=artifact_or_health AGE=168839 MAX_AGE=864000
|
|
JOB=immich-media-offhost-sync STATUS=OK EVIDENCE=artifact_or_health AGE=4286 MAX_AGE=259200
|
|
JOB=memos-backup STATUS=OK EVIDENCE=artifact_or_health AGE=14547 MAX_AGE=259200
|
|
JOB=netbird-vps-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=1747896 MAX_AGE=864000
|
|
JOB=paperless-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=16347 MAX_AGE=259200
|
|
JOB=paperless-restore-dry-run STATUS=STALE EVIDENCE=artifact_or_health AGE=1001986 MAX_AGE=864000
|
|
JOB=skladchik-reports-monitor-backup STATUS=OK EVIDENCE=artifact_or_health AGE=19639 MAX_AGE=172800
|
|
JOB=skladchik-reports-monitor-backup-restore-check STATUS=OK EVIDENCE=artifact_or_health AGE=190273 MAX_AGE=864000
|
|
JOB=vm160-remote-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=26607 MAX_AGE=172800
|
|
========== NODE=pve01 ==========
|
|
=== FAILED_BACKUP_UNITS ===
|
|
=== STALE_OR_FAILED_HEALTH ===
|
|
2026-06-17T10:54:28.0506782690 /var/lib/homelab-health/netbird-vps-restore.txt
|
|
2026-06-17T22:53:44.6880508040 /var/lib/homelab-health/app-stacks-batch-offhost.txt
|
|
2026-06-17T22:55:09.3892995820 /var/lib/homelab-health/app-stacks-batch-restore.txt
|
|
2026-06-17T23:02:13.9525590360 /var/lib/homelab-health/homelab-backup-baseline.txt
|
|
2026-06-18T19:16:23.9155288070 /var/lib/homelab-health/immich-media-restore.txt
|
|
2026-06-18T20:45:46.5765458970 /var/lib/homelab-health/mail-cloud-canary.txt
|
|
2026-06-24T15:19:34.8276009230 /var/lib/homelab-health/npmplus-kuma-config-backup.txt.bak-20260624T121934Z
|
|
2026-06-24T16:02:38.7930804000 /var/lib/homelab-health/vps-identity-peer-discovery.txt
|
|
2026-06-24T16:23:03.5382758840 /var/lib/homelab-health/private-vpn-hosts.txt.bak-ok-wrapper-20260624T132705Z
|
|
2026-06-24T21:07:27.7414593140 /var/lib/homelab-health/pve01-failed-units.txt
|
|
2026-06-24T22:00:51.3972551230 /var/lib/homelab-health/edge-restore-proofs-offhost.txt
|
|
2026-06-24T22:03:31.7226417460 /var/lib/homelab-health/nextcloud-restore-proof-offhost.txt
|
|
2026-07-01T04:31:05.4336469050 /var/lib/homelab-health/netbird-vps-backup.txt
|
|
2026-07-02T11:54:39.6295399090 /var/lib/homelab-health/homelab-mailru-architecture-smoke.txt
|
|
2026-07-02T22:30:48.2601436500 /var/lib/homelab-health/crowdsec-capi-netbird-final.txt
|
|
2026-07-03T00:01:18.6769837810 /var/lib/homelab-health/homepage-final-status.txt
|
|
2026-07-03T00:21:03.3988362700 /var/lib/homelab-health/uptime-kuma-final-status.txt
|
|
2026-07-03T00:24:41.6771270360 /var/lib/homelab-health/uptime-kuma-npmplus-admin-fix.txt
|
|
2026-07-03T00:32:26.6061363170 /var/lib/homelab-health/uptime-kuma-approved-monitors-install.txt
|
|
2026-07-03T00:40:23.7603348270 /var/lib/homelab-health/uptime-kuma-pending-health-monitors-fix.txt
|
|
2026-07-03T00:45:30.9409011290 /var/lib/homelab-health/adguard-rewrite-sync.txt
|
|
2026-07-03T00:45:30.9411592050 /var/lib/homelab-health/restore-drill-index.txt
|
|
2026-07-03T00:45:30.9413791610 /var/lib/homelab-health/backup-restore-dashboard.txt
|
|
2026-07-03T00:48:36.3605124240 /var/lib/homelab-health/uptime-kuma-four-health-sources-repair.txt
|
|
2026-07-03T01:20:07.8445806940 /var/lib/homelab-health/uptime-kuma-four-heartbeat-final.txt
|
|
2026-07-03T01:47:00.7509517560 /var/lib/homelab-health/dockge-agent-install-register.txt
|
|
2026-07-03T02:09:14.1454067110 /var/lib/homelab-health/dockge-final-closure.txt
|
|
2026-07-03T02:20:50.3059631970 /var/lib/homelab-health/dockge-final-managed-labels.txt
|
|
2026-07-03T02:29:13.5615634610 /var/lib/homelab-health/service-inventory-matrix-phase00.tsv
|
|
2026-07-03T13:44:54.6056260110 /var/lib/homelab-health/health-url.env
|
|
2026-07-07T01:09:37.3328266250 /var/lib/homelab-health/p1-safe-data-backup.txt
|
|
2026-07-07T01:22:38.5488080160 /var/lib/homelab-health/p1-remaining-tranche2.txt
|
|
2026-07-07T03:35:26.7019788500 /var/lib/homelab-health/p1-final-tranche.txt
|
|
2026-07-07T03:57:33.0001246560 /var/lib/homelab-health/p2-media-backup.txt
|
|
2026-07-07T05:40:35.0716067090 /var/lib/homelab-health/residual-review.txt
|
|
2026-07-07T07:11:06.5481479160 /var/lib/homelab-health/vm170-vm171-full-strategy.txt
|
|
2026-07-07T08:36:26.0853658590 /var/lib/homelab-health/node-loss-readiness.txt
|
|
2026-07-07T08:37:36.0224235940 /var/lib/homelab-health/node-loss-runbooks.txt
|
|
2026-07-07T09:18:31.3375599320 /var/lib/homelab-health/node-loss-closure.txt
|
|
2026-07-07T09:21:33.8143735330 /var/lib/homelab-health/power-loss-readiness.txt
|
|
2026-07-07T09:22:45.8234102790 /var/lib/homelab-health/power-loss-runbook.txt
|
|
2026-07-07T10:27:44.2204118760 /var/lib/homelab-health/power-loss-closure.txt
|
|
2026-07-07T10:40:01.2285714610 /var/lib/homelab-health/pve03-root-cleanup.txt
|
|
2026-07-07T10:45:01.8591246400 /var/lib/homelab-health/ungated-health-backlog.txt
|
|
2026-07-07T11:24:07.6285928400 /var/lib/homelab-health/pve03-staging-retention-proof.txt
|
|
2026-07-07T11:25:41.5290134290 /var/lib/homelab-health/pve03-staging-retention-cleanup.txt
|
|
2026-07-07T11:29:49.2086910530 /var/lib/homelab-health/pve03-staging-retention.txt
|
|
2026-07-07T15:21:23.8337500480 /var/lib/homelab-health/pve03-staging-retention-closure.txt
|
|
2026-07-07T15:25:57.4458619660 /var/lib/homelab-health/pve03-failed-unit-cleanup.txt
|
|
2026-07-07T15:52:21.5095968040 /var/lib/homelab-health/pve03-failed-unit-cleanup-closure.txt
|
|
2026-07-07T15:59:30.7733163830 /var/lib/homelab-health/remote-git-sops-age-readiness.txt
|
|
2026-07-07T16:01:07.1647092920 /var/lib/homelab-health/remote-git-sops-age-policy.txt
|
|
2026-07-07T16:42:29.4222211440 /var/lib/homelab-health/remote-git-sops-age-closure.txt
|
|
2026-07-07T16:51:17.2367953280 /var/lib/homelab-health/desired-state-remote-target-trace.txt
|
|
2026-07-07T16:53:30.3894596920 /var/lib/homelab-health/desired-state-remote-target.txt
|
|
2026-07-07T17:33:08.7816787090 /var/lib/homelab-health/desired-state-remote-target-closure.txt
|
|
2026-07-07T18:29:49.1668160110 /var/lib/homelab-health/netbird-vps-trust-closure.txt
|
|
2026-07-07T18:29:49.1695821150 /var/lib/homelab-health/external-probe-runner-decision.txt
|
|
2026-07-07T18:55:43.9106665130 /var/lib/homelab-health/netbird-vps-trust-clean-seal.txt
|
|
2026-07-07T23:23:46.4871597760 /var/lib/homelab-health/final-cycle-stability-seal.txt
|
|
2026-07-08T07:58:18.9509332240 /var/lib/homelab-health/mail-cloud-restore-drill.txt.bak-refresh-20260709T203132Z
|
|
2026-07-08T07:59:58.6953604310 /var/lib/homelab-health/sops-edge-secret-coverage-audit.txt
|
|
2026-07-08T08:01:49.3036518230 /var/lib/homelab-health/sops-edge-secret-coverage-closure.txt
|
|
2026-07-08T08:20:29.6041867030 /var/lib/homelab-health/remote-failed-unit-audit.txt
|
|
2026-07-08T08:23:09.5176054980 /var/lib/homelab-health/remote-failed-unit-plan.txt
|
|
2026-07-08T08:25:25.7376658940 /var/lib/homelab-health/remote-failed-unit-apply.txt
|
|
2026-07-08T08:27:02.9011355390 /var/lib/homelab-health/paperless-core-guard-audit.txt
|
|
2026-07-08T08:38:32.4205675490 /var/lib/homelab-health/remote-failed-unit-cleanup-seal.txt
|
|
2026-07-08T08:40:40.2615024670 /var/lib/homelab-health/duty-admin-residual-audit.txt
|
|
2026-07-08T08:42:30.3881692640 /var/lib/homelab-health/pve03-capacity-forensics.txt
|
|
2026-07-08T08:42:31.7994319160 /var/lib/homelab-health/duty-admin-residual-cleanup.txt
|
|
2026-07-08T08:58:45.2482918440 /var/lib/homelab-health/duty-admin-residual-cleanup-seal.txt
|
|
2026-07-08T09:08:11.4945075240 /var/lib/homelab-health/post-cleanup-owner-timer-replay.txt
|
|
2026-07-08T09:17:26.8701684910 /var/lib/homelab-health/post-cleanup-final-seal.txt
|
|
2026-07-08T09:20:13.6454507410 /var/lib/homelab-health/review-surface-registry-audit.txt
|
|
2026-07-08T09:26:36.7402046120 /var/lib/homelab-health/dockge-agent-capability-analysis.txt
|
|
2026-07-08T09:26:36.7402750460 /var/lib/homelab-health/service-inventory-leftovers-phase00.txt
|
|
2026-07-08T09:26:36.7403618890 /var/lib/homelab-health/dockge-stale-stack-classification-after-agents.txt
|
|
2026-07-08T09:26:36.7403838590 /var/lib/homelab-health/dockge-remediation-plan.txt
|
|
2026-07-08T09:26:36.7404284930 /var/lib/homelab-health/failed-unit-legacy-inventory.txt
|
|
2026-07-08T09:26:36.7404732790 /var/lib/homelab-health/service-setup-backlog-exact-order.txt
|
|
2026-07-08T09:26:36.7405361960 /var/lib/homelab-health/ungated-health-backlog-closure.txt
|
|
2026-07-08T09:26:36.7406026680 /var/lib/homelab-health/homepage-missing-card-proposals.txt
|
|
2026-07-08T09:26:36.7406281280 /var/lib/homelab-health/dockge-agent-runtime-method.txt
|
|
2026-07-08T09:26:36.7406657810 /var/lib/homelab-health/homepage-card-recommendations.txt
|
|
2026-07-08T09:26:36.7408259230 /var/lib/homelab-health/homepage-missing-card-proposals-final.txt
|
|
2026-07-08T09:26:36.7408607340 /var/lib/homelab-health/service-inventory-summary-phase00.txt
|
|
2026-07-08T09:26:36.7409298280 /var/lib/homelab-health/failed-unit-legacy-classification.txt
|
|
2026-07-08T09:26:36.7409559880 /var/lib/homelab-health/failed-unit-legacy-local-cleanup.txt
|
|
2026-07-08T09:26:36.7409816770 /var/lib/homelab-health/phase00-freeze-final.txt
|
|
2026-07-08T09:26:36.7410552890 /var/lib/homelab-health/dockge-stale-archive-last.txt
|
|
2026-07-08T09:26:36.7410994270 /var/lib/homelab-health/dockge-agent-schema-and-register-method.txt
|
|
2026-07-08T09:26:36.7411245420 /var/lib/homelab-health/dockge-disconnected-containers-analysis.txt
|
|
2026-07-08T09:26:36.7411439090 /var/lib/homelab-health/final-readiness-cycle-break.txt
|
|
2026-07-08T09:26:36.7411631530 /var/lib/homelab-health/dockge-user-hash-agent-install-readiness.txt
|
|
2026-07-08T09:26:36.7411808970 /var/lib/homelab-health/service-setup-order-analysis.txt
|
|
2026-07-08T09:26:36.7412497980 /var/lib/homelab-health/dockge-exact-agent-table-inspect.txt
|
|
2026-07-08T09:26:36.7413054120 /var/lib/homelab-health/failed-unit-legacy-cleanup-seal.txt
|
|
2026-07-08T09:26:36.7413803250 /var/lib/homelab-health/service-setup-roadmap-deep.txt
|
|
2026-07-08T09:26:36.7414375920 /var/lib/homelab-health/uptime-kuma-monitoring-proposals.txt
|
|
2026-07-08T09:26:36.7414750460 /var/lib/homelab-health/kuma-legacy-monitor-quarantine.txt
|
|
2026-07-08T09:26:36.7415177720 /var/lib/homelab-health/dockge-agent-users-login-fix.txt
|
|
2026-07-08T09:36:40.2533945430 /var/lib/homelab-health/review-surface-registry-apply.txt
|
|
2026-07-08T09:44:44.0027202780 /var/lib/homelab-health/review-surface-registry-seal.txt
|
|
2026-07-08T10:27:09.8992163430 /var/lib/homelab-health/final-global-green-audit.txt
|
|
2026-07-08T10:52:21.5920659640 /var/lib/homelab-health/final-green-freeze-audit.txt
|
|
2026-07-08T11:20:32.9478120810 /var/lib/homelab-health/final-green-freeze-seal.txt
|
|
2026-07-08T11:31:18.9846514570 /var/lib/homelab-health/recovery-card.txt
|
|
2026-07-08T11:41:46.1321730550 /var/lib/homelab-health/recovery-card-seal.txt
|
|
2026-07-08T12:09:03.7459627090 /var/lib/homelab-health/final-closeout.txt
|
|
2026-07-08T12:13:28.4829706890 /var/lib/homelab-health/independent-external-runner-audit.txt
|
|
2026-07-08T12:24:03.1785809940 /var/lib/homelab-health/independent-external-runner.txt
|
|
2026-07-08T12:27:26.7886643440 /var/lib/homelab-health/desired-state-remote-git-audit.txt
|
|
2026-07-08T12:37:57.1142092610 /var/lib/homelab-health/desired-state-remote-git.txt
|
|
2026-07-08T12:50:05.9132437970 /var/lib/homelab-health/operator-handoff-pack.txt
|
|
2026-07-08T13:48:38.7184424100 /var/lib/homelab-health/live-tail-audit.txt
|
|
2026-07-08T15:04:03.6928977300 /var/lib/homelab-health/pve03-edge-vm-restore-drill.txt
|
|
2026-07-08T15:23:29.3375349030 /var/lib/homelab-health/pve03-edge-vm-offhost-freshness-seal.txt
|
|
2026-07-08T15:53:39.8149292590 /var/lib/homelab-health/pve02-forum-prod-restore-drill.txt
|
|
2026-07-08T17:10:02.1861577390 /var/lib/homelab-health/pve02-forum-prod-runtime-restore-drill.txt
|
|
2026-07-08T18:27:57.6728793210 /var/lib/homelab-health/pve01-nextcloud-dns-restore-drill.txt
|
|
2026-07-08T18:57:27.4506461580 /var/lib/homelab-health/pve01-core-monitoring-app-restorecheck.txt
|
|
2026-07-08T19:07:22.5726501490 /var/lib/homelab-health/pve01-core-monitoring-app-level-drill.txt
|
|
2026-07-08T19:35:56.6356041270 /var/lib/homelab-health/pve03-edge-vm-offhost-freshness.txt
|
|
2026-07-08T19:43:08.4121538910 /var/lib/homelab-health/node-loss-final-closure.txt
|
|
2026-07-08T20:49:00.3922878570 /var/lib/homelab-health/ideal-cluster-noise-seal.txt
|
|
2026-07-09T00:12:15.6152895130 /var/lib/homelab-health/ideal-cluster-final-seal.txt
|
|
2026-07-09T07:57:13.0808629150 /var/lib/homelab-health/cluster-admin-strict-final-seal.txt
|
|
2026-07-09T08:07:47.2134520460 /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt
|
|
2026-07-09T08:24:54.0720153050 /var/lib/homelab-health/cluster-admin-backup-webpanel-final-seal.txt
|
|
2026-07-09T09:02:55.0056277600 /var/lib/homelab-health/cluster-admin-homepage-card.txt
|
|
2026-07-09T09:05:58.6994207660 /var/lib/homelab-health/cluster-admin-service-readiness.txt
|
|
2026-07-09T09:14:02.6367788630 /var/lib/homelab-health/service-readiness-cluster-admin-webpanel.txt
|
|
2026-07-09T09:22:58.3799240000 /var/lib/homelab-health/service-readiness-vaultwarden.txt
|
|
2026-07-09T09:29:21.9787558270 /var/lib/homelab-health/service-readiness-authentik.txt
|
|
2026-07-09T09:37:47.4434368570 /var/lib/homelab-health/service-readiness-gitea.txt
|
|
2026-07-09T09:41:40.0979703420 /var/lib/homelab-health/service-readiness-backup-health.txt
|
|
2026-07-09T09:47:55.5636728080 /var/lib/homelab-health/service-readiness-uptime-kuma.txt
|
|
2026-07-09T09:52:06.0994778650 /var/lib/homelab-health/service-readiness-gotify.txt
|
|
2026-07-09T10:22:35.5302603170 /var/lib/homelab-health/app-stacks-batch-backup.txt
|
|
2026-07-09T10:22:35.8882657530 /var/lib/homelab-health/router-startup-config-mail-cloud.txt
|
|
2026-07-09T10:40:55.9699739050 /var/lib/homelab-health/service-readiness-prometheus.txt
|
|
2026-07-09T10:46:00.9056083920 /var/lib/homelab-health/grafana-backup.txt
|
|
2026-07-09T10:46:00.9056083920 /var/lib/homelab-health/grafana-offhost.txt
|
|
2026-07-09T10:46:00.9056083920 /var/lib/homelab-health/grafana-restore.txt
|
|
2026-07-09T10:47:19.2097991550 /var/lib/homelab-health/service-readiness-grafana.txt
|
|
2026-07-09T10:50:17.8267126930 /var/lib/homelab-health/alertmanager-backup.txt
|
|
2026-07-09T10:50:17.8267126930 /var/lib/homelab-health/alertmanager-offhost.txt
|
|
2026-07-09T10:50:17.8267126930 /var/lib/homelab-health/alertmanager-restore.txt
|
|
2026-07-09T10:51:33.5136663000 /var/lib/homelab-health/service-readiness-alertmanager.txt
|
|
2026-07-09T11:00:42.7700187710 /var/lib/homelab-health/service-readiness-ntfy.txt
|
|
2026-07-09T11:04:19.9613236750 /var/lib/homelab-health/blackbox-exporter-backup.txt
|
|
2026-07-09T11:04:19.9613236750 /var/lib/homelab-health/blackbox-exporter-offhost.txt
|
|
2026-07-09T11:04:19.9613236750 /var/lib/homelab-health/blackbox-exporter-restore.txt
|
|
2026-07-09T11:05:57.9168142140 /var/lib/homelab-health/service-readiness-blackbox-exporter.txt
|
|
2026-07-09T11:14:05.4352326180 /var/lib/homelab-health/node-exporter-backup.txt
|
|
2026-07-09T11:14:05.4352326180 /var/lib/homelab-health/node-exporter-offhost.txt
|
|
2026-07-09T11:14:05.4352326180 /var/lib/homelab-health/node-exporter-restore.txt
|
|
2026-07-09T11:17:45.6165830230 /var/lib/homelab-health/service-readiness-node-exporter.txt
|
|
2026-07-09T11:25:57.2660677240 /var/lib/homelab-health/cadvisor-backup.txt
|
|
2026-07-09T11:25:57.2660677240 /var/lib/homelab-health/cadvisor-offhost.txt
|
|
2026-07-09T11:25:57.2660677240 /var/lib/homelab-health/cadvisor-restore.txt
|
|
2026-07-09T11:28:23.8062985970 /var/lib/homelab-health/service-readiness-cadvisor.txt
|
|
2026-07-09T11:32:11.0687583470 /var/lib/homelab-health/dozzle-backup.txt
|
|
2026-07-09T11:32:11.0687583470 /var/lib/homelab-health/dozzle-offhost.txt
|
|
2026-07-09T11:32:11.0687583470 /var/lib/homelab-health/dozzle-restore.txt
|
|
2026-07-09T11:33:54.7913373940 /var/lib/homelab-health/service-readiness-dozzle.txt
|
|
2026-07-09T11:38:36.1186130800 /var/lib/homelab-health/beszel-backup.txt
|
|
2026-07-09T11:38:36.1186130800 /var/lib/homelab-health/beszel-offhost.txt
|
|
2026-07-09T11:38:36.1186130800 /var/lib/homelab-health/beszel-restore.txt
|
|
2026-07-09T11:44:52.1683257450 /var/lib/homelab-health/service-readiness-beszel.txt
|
|
2026-07-09T11:51:31.1083861380 /var/lib/homelab-health/healthchecks-backup.txt
|
|
2026-07-09T11:51:31.1083861380 /var/lib/homelab-health/healthchecks-offhost.txt
|
|
2026-07-09T11:51:31.1083861380 /var/lib/homelab-health/healthchecks-restore.txt
|
|
2026-07-09T12:39:46.4043274480 /var/lib/homelab-health/searxng-backup.txt
|
|
2026-07-09T12:39:46.4053274630 /var/lib/homelab-health/searxng-offhost.txt
|
|
2026-07-09T12:39:46.4053274630 /var/lib/homelab-health/searxng-restore.txt
|
|
2026-07-09T12:58:00.5629333130 /var/lib/homelab-health/service-readiness-searxng.txt
|
|
2026-07-09T13:02:56.5714256630 /var/lib/homelab-health/npmplus-edge-backup.txt
|
|
2026-07-09T13:02:56.5714256630 /var/lib/homelab-health/npmplus-edge-offhost.txt
|
|
2026-07-09T13:02:56.5714256630 /var/lib/homelab-health/npmplus-edge-restore.txt
|
|
2026-07-09T13:07:32.0966068350 /var/lib/homelab-health/service-readiness-npmplus.txt
|
|
2026-07-09T13:12:46.6093796530 /var/lib/homelab-health/dockge-backup.txt
|
|
2026-07-09T13:12:46.6093796530 /var/lib/homelab-health/dockge-offhost.txt
|
|
2026-07-09T13:12:46.6093796530 /var/lib/homelab-health/dockge-restore.txt
|
|
2026-07-09T13:12:46.6093796530 /var/lib/homelab-health/dockge-stacks-backup.txt
|
|
2026-07-09T13:12:46.6093796530 /var/lib/homelab-health/dockge-stacks-offhost.txt
|
|
2026-07-09T13:12:46.6093796530 /var/lib/homelab-health/dockge-stacks-restore.txt
|
|
2026-07-09T13:16:48.1830455990 /var/lib/homelab-health/service-readiness-dockge.txt
|
|
2026-07-09T13:23:55.6935309890 /var/lib/homelab-health/netbox-backup.txt
|
|
2026-07-09T13:23:55.6935309890 /var/lib/homelab-health/netbox-offhost.txt
|
|
2026-07-09T13:23:55.6935309890 /var/lib/homelab-health/netbox-restore.txt
|
|
2026-07-09T13:40:18.9044441000 /var/lib/homelab-health/service-readiness-netbox.txt
|
|
2026-07-09T15:13:16.8761117060 /var/lib/homelab-health/minio-backup.txt
|
|
2026-07-09T15:13:16.8761117060 /var/lib/homelab-health/minio-console-backup.txt
|
|
2026-07-09T15:13:16.8761117060 /var/lib/homelab-health/minio-console-offhost.txt
|
|
2026-07-09T15:13:16.8761117060 /var/lib/homelab-health/minio-console-restore.txt
|
|
2026-07-09T15:13:16.8761117060 /var/lib/homelab-health/minio-offhost.txt
|
|
2026-07-09T15:13:16.8761117060 /var/lib/homelab-health/minio-restore.txt
|
|
2026-07-09T15:18:29.6618645950 /var/lib/homelab-health/service-readiness-minio.txt
|
|
2026-07-09T15:29:41.9851382980 /var/lib/homelab-health/proxmox-pve01-backup.txt
|
|
2026-07-09T15:29:41.9851382980 /var/lib/homelab-health/proxmox-pve01-offhost.txt
|
|
2026-07-09T15:29:41.9851382980 /var/lib/homelab-health/proxmox-pve01-restore.txt
|
|
2026-07-09T15:29:41.9851382980 /var/lib/homelab-health/pve01-backup.txt
|
|
2026-07-09T15:29:41.9851382980 /var/lib/homelab-health/pve01-offhost.txt
|
|
2026-07-09T15:29:41.9851382980 /var/lib/homelab-health/pve01-restore.txt
|
|
2026-07-09T15:33:52.8128905890 /var/lib/homelab-health/service-readiness-proxmox-pve01.txt
|
|
2026-07-09T15:42:17.7925620460 /var/lib/homelab-health/filebrowser-offhost.txt
|
|
2026-07-09T15:47:18.8311350910 /var/lib/homelab-health/service-readiness-filebrowser.txt
|
|
2026-07-09T15:53:45.2480051850 /var/lib/homelab-health/actual-backup.txt
|
|
2026-07-09T15:53:45.2480051850 /var/lib/homelab-health/actual-budget-backup.txt
|
|
2026-07-09T15:53:45.2480051850 /var/lib/homelab-health/actual-budget-offhost.txt
|
|
2026-07-09T15:53:45.2480051850 /var/lib/homelab-health/actual-budget-restore.txt
|
|
2026-07-09T15:53:45.2480051850 /var/lib/homelab-health/actual-offhost.txt
|
|
2026-07-09T15:53:45.2480051850 /var/lib/homelab-health/actual-restore.txt
|
|
2026-07-09T16:04:50.0111067090 /var/lib/homelab-health/service-readiness-actual-budget.txt
|
|
2026-07-09T19:01:20.7901933280 /var/lib/homelab-health/homebox-backup.txt
|
|
2026-07-09T19:01:20.7901933280 /var/lib/homelab-health/homebox-offhost.txt
|
|
2026-07-09T19:01:20.7901933280 /var/lib/homelab-health/homebox-restore.txt
|
|
2026-07-09T19:06:43.1941052730 /var/lib/homelab-health/service-readiness-homebox.txt
|
|
2026-07-09T19:17:35.0170448070 /var/lib/homelab-health/mealie-backup.txt
|
|
2026-07-09T19:17:35.0170448070 /var/lib/homelab-health/mealie-offhost.txt
|
|
2026-07-09T19:17:35.0170448070 /var/lib/homelab-health/mealie-restore.txt
|
|
2026-07-09T19:23:39.3486047240 /var/lib/homelab-health/service-readiness-mealie.txt
|
|
2026-07-09T19:46:33.3515780290 /var/lib/homelab-health/service-readiness-paperless.txt
|
|
2026-07-09T19:49:28.4202504550 /var/lib/homelab-health/memos-restore.txt
|
|
2026-07-09T19:50:22.5980774800 /var/lib/homelab-health/service-readiness-memos.txt
|
|
2026-07-09T20:00:19.8381943590 /var/lib/homelab-health/linkding-backup.txt
|
|
2026-07-09T20:00:19.8381943590 /var/lib/homelab-health/linkding-offhost.txt
|
|
2026-07-09T20:00:19.8381943590 /var/lib/homelab-health/linkding-restore.txt
|
|
2026-07-09T20:40:58.8843463730 /var/lib/homelab-health/service-readiness-linkding.txt
|
|
2026-07-09T20:47:41.7774675140 /var/lib/homelab-health/healthchecks-canonical-domain-repair.txt
|
|
2026-07-09T20:47:41.7774675140 /var/lib/homelab-health/service-readiness-healthchecks.txt
|
|
2026-07-09T20:51:20.8507958870 /var/lib/homelab-health/vikunja-backup.txt
|
|
2026-07-09T20:51:20.8507958870 /var/lib/homelab-health/vikunja-offhost.txt
|
|
2026-07-09T20:51:20.8507958870 /var/lib/homelab-health/vikunja-restore.txt
|
|
2026-07-09T20:55:48.6988688880 /var/lib/homelab-health/service-readiness-vikunja.txt
|
|
2026-07-09T20:58:33.5284046710 /var/lib/homelab-health/bookstack-backup.txt
|
|
2026-07-09T20:58:33.5284046710 /var/lib/homelab-health/bookstack-offhost.txt
|
|
2026-07-09T20:58:33.5284046710 /var/lib/homelab-health/bookstack-restore.txt
|
|
2026-07-09T21:01:03.7247153370 /var/lib/homelab-health/service-readiness-bookstack.txt
|
|
2026-07-09T21:08:55.5329737450 /var/lib/homelab-health/stirling-pdf-backup.txt
|
|
2026-07-09T21:08:55.5329737450 /var/lib/homelab-health/stirling-pdf-offhost.txt
|
|
2026-07-09T21:08:55.5329737450 /var/lib/homelab-health/stirling-pdf-restore.txt
|
|
2026-07-09T21:13:22.9720909840 /var/lib/homelab-health/service-readiness-stirling-pdf.txt
|
|
2026-07-09T21:16:17.6627935760 /var/lib/homelab-health/jellyfin-backup.txt
|
|
2026-07-09T21:16:17.6627935760 /var/lib/homelab-health/jellyfin-offhost.txt
|
|
2026-07-09T21:16:17.6627935760 /var/lib/homelab-health/jellyfin-restore.txt
|
|
2026-07-09T21:19:11.7814873230 /var/lib/homelab-health/service-readiness-jellyfin.txt
|
|
2026-07-09T21:22:25.6704869300 /var/lib/homelab-health/audiobookshelf-backup.txt
|
|
2026-07-09T21:22:25.6704869300 /var/lib/homelab-health/audiobookshelf-offhost.txt
|
|
2026-07-09T21:22:25.6704869300 /var/lib/homelab-health/audiobookshelf-restore.txt
|
|
2026-07-09T21:25:22.5602235460 /var/lib/homelab-health/service-readiness-audiobookshelf.txt
|
|
2026-07-09T21:46:20.9338292150 /var/lib/homelab-health/calibre-web-backup.txt
|
|
2026-07-09T21:46:20.9338292150 /var/lib/homelab-health/calibre-web-offhost.txt
|
|
2026-07-09T21:46:20.9338292150 /var/lib/homelab-health/calibre-web-restore.txt
|
|
2026-07-09T21:57:20.3030989240 /var/lib/homelab-health/service-readiness-calibre-web.txt
|
|
2026-07-09T22:04:04.0733864250 /var/lib/homelab-health/homeassistant-backup.txt
|
|
2026-07-09T22:04:04.0733864250 /var/lib/homelab-health/homeassistant-offhost.txt
|
|
2026-07-09T22:04:04.0733864250 /var/lib/homelab-health/homeassistant-restore.txt
|
|
2026-07-09T22:07:41.2227931800 /var/lib/homelab-health/service-readiness-homeassistant.txt
|
|
2026-07-09T22:13:14.6980274280 /var/lib/homelab-health/it-tools-backup.txt
|
|
2026-07-09T22:13:14.6980274280 /var/lib/homelab-health/it-tools-offhost.txt
|
|
2026-07-09T22:13:14.6980274280 /var/lib/homelab-health/it-tools-restore.txt
|
|
2026-07-09T22:23:57.3350983150 /var/lib/homelab-health/service-readiness-it-tools.txt
|
|
2026-07-09T22:28:35.5724369860 /var/lib/homelab-health/karakeep-backup.txt
|
|
2026-07-09T22:28:35.5724369860 /var/lib/homelab-health/karakeep-offhost.txt
|
|
2026-07-09T22:28:35.5724369860 /var/lib/homelab-health/karakeep-restore.txt
|
|
2026-07-09T22:32:36.7481977520 /var/lib/homelab-health/service-readiness-karakeep.txt
|
|
2026-07-09T22:39:24.2375519320 /var/lib/homelab-health/n8n-backup.txt
|
|
2026-07-09T22:39:24.2375519320 /var/lib/homelab-health/n8n-offhost.txt
|
|
2026-07-09T22:39:24.2375519320 /var/lib/homelab-health/n8n-restore.txt
|
|
2026-07-09T22:42:27.8124145040 /var/lib/homelab-health/service-readiness-n8n.txt
|
|
2026-07-09T22:46:45.7324363630 /var/lib/homelab-health/node-red-backup.txt
|
|
2026-07-09T22:46:45.7324363630 /var/lib/homelab-health/node-red-offhost.txt
|
|
2026-07-09T22:46:45.7324363630 /var/lib/homelab-health/node-red-restore.txt
|
|
2026-07-09T22:50:18.7317577520 /var/lib/homelab-health/service-readiness-node-red.txt
|
|
2026-07-09T22:55:13.5023542440 /var/lib/homelab-health/syncthing-backup.txt
|
|
2026-07-09T22:55:13.5023542440 /var/lib/homelab-health/syncthing-offhost.txt
|
|
2026-07-09T22:55:13.5023542440 /var/lib/homelab-health/syncthing-restore.txt
|
|
2026-07-09T22:59:10.6320324800 /var/lib/homelab-health/service-readiness-syncthing.txt
|
|
2026-07-09T23:09:25.0745502100 /var/lib/homelab-health/immich-backup.txt
|
|
2026-07-09T23:09:25.0745502100 /var/lib/homelab-health/immich-offhost.txt
|
|
2026-07-09T23:09:25.0745502100 /var/lib/homelab-health/immich-restore.txt
|
|
2026-07-09T23:12:55.3968081120 /var/lib/homelab-health/service-readiness-immich.txt
|
|
2026-07-09T23:39:37.9114415990 /var/lib/homelab-health/gotify-backup.txt
|
|
2026-07-09T23:39:37.9124416140 /var/lib/homelab-health/gotify-offhost.txt
|
|
2026-07-09T23:39:37.9124416140 /var/lib/homelab-health/gotify-restore.txt
|
|
2026-07-09T23:39:41.3274938750 /var/lib/homelab-health/ntfy-backup.txt
|
|
2026-07-09T23:39:41.3274938750 /var/lib/homelab-health/ntfy-offhost.txt
|
|
2026-07-09T23:39:41.3274938750 /var/lib/homelab-health/ntfy-restore.txt
|
|
2026-07-09T23:39:50.5946356910 /var/lib/homelab-health/uptime-kuma-backup.txt
|
|
2026-07-09T23:39:50.5946356910 /var/lib/homelab-health/uptime-kuma-offhost.txt
|
|
2026-07-09T23:39:50.5946356910 /var/lib/homelab-health/uptime-kuma-restore.txt
|
|
2026-07-09T23:45:51.3911568860 /var/lib/homelab-health/service-readiness-nextcloud.txt
|
|
2026-07-09T23:55:38.2600376160 /var/lib/homelab-health/service-readiness-dns1.txt
|
|
2026-07-09T23:55:38.2600376160 /var/lib/homelab-health/service-readiness-dns2.txt
|
|
2026-07-09T23:55:38.2600376160 /var/lib/homelab-health/service-readiness-homepage.txt
|
|
2026-07-09T23:58:29.2076174390 /var/lib/homelab-health/external-link-aleisaevn-netcraze-5083.txt
|
|
2026-07-10T00:12:45.3264837560 /var/lib/homelab-health/deep-service-backup-audit.txt
|
|
2026-07-10T21:46:36.9609141470 /var/lib/homelab-health/alertmanager-two-cloud-backup.txt
|
|
2026-07-10T22:05:25.8570406280 /var/lib/homelab-health/beszel-two-cloud-backup.txt
|
|
2026-07-10T22:20:26.1397323180 /var/lib/homelab-health/blackbox-exporter-two-cloud-backup.txt
|
|
2026-07-10T23:00:03.1168349530 /var/lib/homelab-health/legacy-monitoring-backups-migrated.txt
|
|
2026-07-10T23:25:23.2350187140 /var/lib/homelab-health/drift-baseline-acceptance.txt
|
|
2026-07-11T00:52:36.4697413270 /var/lib/homelab-health/cluster-admin-inventory-sync.txt
|
|
2026-07-11T02:25:25.1542644770 /var/lib/homelab-health/cluster-admin-public-route.txt
|
|
2026-07-11T02:26:01.7218294760 /var/lib/homelab-health/cluster-admin-prometheus-integration.txt
|
|
2026-07-11T02:30:49.6252637080 /var/lib/homelab-health/cluster-admin-incident-engine-backup.txt
|
|
2026-07-11T04:07:23.3393000650 /var/lib/homelab-health/service-readiness-cluster-admin-incident-engine.txt
|
|
2026-07-11T04:39:28.9976540970 /var/lib/homelab-health/cluster-admin-webpanel.txt
|
|
2026-07-11T06:23:40.9114023280 /var/lib/homelab-health/cluster-mail-cloud-cleanup.txt
|
|
2026-07-13T01:15:40.0109283350 /var/lib/homelab-health/cluster-admin-stage0-remediation.txt
|
|
2026-07-13T02:08:25.4011133770 /var/lib/homelab-health/cluster-admin-stage0-desired-state-sync.txt
|
|
2026-07-13T02:14:59.9458929520 /var/lib/homelab-health/cluster-admin-stage0-convergence.txt
|
|
2026-07-13T02:22:57.9862108430 /var/lib/homelab-health/cluster-admin-stage0-readiness-fix.txt
|
|
2026-07-13T02:26:02.8616913120 /var/lib/homelab-health/cluster-admin-stage0-baseline.txt
|
|
2026-07-13T02:32:35.0280552640 /var/lib/homelab-health/cluster-admin-stage1-capture.txt
|
|
2026-07-13T02:35:09.6424265770 /var/lib/homelab-health/cluster-admin-stage1-design-inputs.txt
|
|
2026-07-13T02:46:43.8820757430 /var/lib/homelab-health/cluster-admin-stage1-data-shape.txt
|
|
2026-07-13T03:11:06.6805116280 /var/lib/homelab-health/cluster-admin-stage1-migration-compatibility.txt
|
|
2026-07-13T03:11:06.6805116280 /var/lib/homelab-health/cluster-admin-stage1-migration-dry-run.txt
|
|
2026-07-13T03:11:06.7005119340 /var/lib/homelab-health/cluster-admin-stage1-preapply-backup.txt
|
|
2026-07-13T03:22:22.4838707540 /var/lib/homelab-health/cluster-admin-stage1-foundation-apply.txt
|
|
2026-07-13T03:26:26.3016083230 /var/lib/homelab-health/cluster-admin-stage1-final-seal.txt
|
|
2026-07-13T03:29:40.0025776360 /var/lib/homelab-health/appbackupctl-routing.txt
|
|
2026-07-13T03:35:17.3427496850 /var/lib/homelab-health/cluster-admin-stage2-design.txt
|
|
2026-07-13T03:39:46.1788887390 /var/lib/homelab-health/cluster-admin-stage2-dry-run.txt
|
|
2026-07-13T04:05:34.7337002360 /var/lib/homelab-health/cluster-admin-stage2-recovery.txt
|
|
2026-07-13T04:15:16.8986427760 /var/lib/homelab-health/cluster-admin-stage2-placeholder-diagnosis.txt
|
|
2026-07-13T04:20:17.9572677960 /var/lib/homelab-health/cluster-admin-stage2-dynamic-redryrun.txt
|
|
2026-07-13T04:24:04.9957556580 /var/lib/homelab-health/cluster-admin-stage2-preapply-backup.txt
|
|
2026-07-13T04:39:27.6419256780 /var/lib/homelab-health/cluster-admin-stage2-apply.txt
|
|
2026-07-13T04:49:49.9254840660 /var/lib/homelab-health/cluster-admin-stage2-final-seal.txt
|
|
2026-07-13T05:04:04.9066160570 /var/lib/homelab-health/cluster-admin-stage3-api-design.txt
|
|
2026-07-13T05:36:13.0711962680 /var/lib/homelab-health/cluster-admin-stage3-api-auth-diagnosis.txt
|
|
2026-07-13T05:45:21.0465781570 /var/lib/homelab-health/cluster-admin-stage3-api-dry-run.txt
|
|
2026-07-13T05:53:43.0632562670 /var/lib/homelab-health/cluster-admin-stage3-api-runtime-dry-run.txt
|
|
2026-07-13T06:05:42.4882482970 /var/lib/homelab-health/cluster-admin-stage3-preapply-checkpoint.txt
|
|
2026-07-13T06:15:56.2396234100 /var/lib/homelab-health/cluster-admin-stage3-preapply-backup.txt
|
|
2026-07-13T06:20:26.7327543700 /var/lib/homelab-health/cluster-admin-stage3-api-candidate-normalized.txt
|
|
2026-07-13T06:22:40.5977987490 /var/lib/homelab-health/cluster-admin-stage3-api-apply.txt
|
|
2026-07-13T06:25:40.2405422500 /var/lib/homelab-health/cluster-admin-stage3-production-verifier-prepared.txt
|
|
2026-07-13T06:28:56.1285342090 /var/lib/homelab-health/cluster-admin-stage3-production-verify.txt
|
|
2026-07-13T06:44:42.7609957740 /var/lib/homelab-health/cluster-admin-stage3-postapply-backup.txt
|
|
2026-07-13T06:47:15.5353299170 /var/lib/homelab-health/cluster-admin-stage3-final-seal.txt
|
|
2026-07-13T06:50:50.2356102460 /var/lib/homelab-health/crypto-portfolio-remove-grt-discovery.txt
|
|
2026-07-13T06:52:07.9807980800 /var/lib/homelab-health/crypto-portfolio-remove-grt-target.txt
|
|
2026-07-13T07:00:02.5170483350 /var/lib/homelab-health/crypto-portfolio-remove-grt-current.txt
|
|
2026-07-13T07:05:43.6132589850 /var/lib/homelab-health/crypto-portfolio-remove-grt-surface.txt
|
|
2026-07-13T07:20:46.9160557760 /var/lib/homelab-health/crypto-portfolio-remove-grt-candidate.txt
|
|
2026-07-13T07:35:30.8895464510 /var/lib/homelab-health/crypto-portfolio-remove-grt-runtime-verifier-prepared.txt
|
|
2026-07-13T07:41:31.9430520860 /var/lib/homelab-health/crypto-portfolio-remove-grt-runtime-dry-run.txt
|
|
2026-07-13T07:52:08.1187504330 /var/lib/homelab-health/crypto-portfolio-remove-grt-browser-smoke.txt
|
|
2026-07-13T08:05:15.1507543740 /var/lib/homelab-health/crypto-portfolio-remove-grt-desired-state-diagnostic.txt
|
|
2026-07-13T08:07:21.2856783820 /var/lib/homelab-health/crypto-portfolio-remove-grt-desired-state-model.txt
|
|
2026-07-13T08:10:44.7277816120 /var/lib/homelab-health/crypto-portfolio-remove-grt-prechange-backup.txt
|
|
2026-07-13T08:39:10.8547948980 /var/lib/homelab-health/crypto-portfolio-remove-grt-recovery-audit.txt
|
|
2026-07-13T08:41:01.3664795150 /var/lib/homelab-health/crypto-portfolio-remove-grt-failed-units-diagnostic.txt
|
|
2026-07-13T08:48:13.2600633440 /var/lib/homelab-health/crypto-portfolio-remove-grt-production-apply.txt
|
|
2026-07-13T09:11:24.1542650490 /var/lib/homelab-health/crypto-portfolio-ui-center-alert-currency-candidate.txt
|
|
2026-07-13T10:04:47.2300377080 /var/lib/homelab-health/crypto-portfolio-ui-center-alert-currency-browser-smoke.txt
|
|
2026-07-13T10:42:57.1487691660 /var/lib/homelab-health/crypto-portfolio-ui-preapply-backup.txt
|
|
2026-07-13T11:12:13.3023976030 /var/lib/homelab-health/crypto-portfolio-backup.txt
|
|
2026-07-13T11:12:13.3023976030 /var/lib/homelab-health/crypto-portfolio-offhost.txt
|
|
2026-07-13T11:12:13.3023976030 /var/lib/homelab-health/crypto-portfolio-restore.txt
|
|
2026-07-13T11:12:13.3023976030 /var/lib/homelab-health/service-readiness-crypto-portfolio.txt
|
|
2026-07-13T11:12:14.9294222730 /var/lib/homelab-health/crypto-portfolio-ui-center-alert-currency-production-closure.txt
|
|
2026-07-13T11:37:59.5159486840 /var/lib/homelab-health/crypto-portfolio-ui-center-alert-currency-final-seal.txt
|
|
2026-07-13T11:37:59.5171037700 /var/lib/homelab-health/crypto-portfolio-runtime-final-seal.txt
|
|
2026-07-13T14:03:06.0152234300 /var/lib/homelab-health/hapusya-site-preapply-backup.txt
|
|
2026-07-13T14:34:59.2083653510 /var/lib/homelab-health/hapusya-site-v2-preapply-backup.txt
|
|
2026-07-13T14:45:27.4649272800 /var/lib/homelab-health/hapusya-site-production-apply.txt
|
|
2026-07-13T14:48:15.6134864600 /var/lib/homelab-health/hapusya-site-production-browser.txt
|
|
2026-07-13T15:10:58.1372215280 /var/lib/homelab-health/hapusya-site-backup.txt
|
|
2026-07-13T15:10:58.1372215280 /var/lib/homelab-health/hapusya-site-offhost.txt
|
|
2026-07-13T15:10:58.1372215280 /var/lib/homelab-health/hapusya-site-restore.txt
|
|
2026-07-13T15:10:58.1372215280 /var/lib/homelab-health/hapusya-site-retention.txt
|
|
2026-07-13T15:10:58.1382215430 /var/lib/homelab-health/hapusya-site-backup-path-mapping.txt
|
|
2026-07-13T15:26:19.5872390600 /var/lib/homelab-health/hapusya-site-desired-state-sync.txt
|
|
2026-07-13T15:33:08.6444630100 /var/lib/homelab-health/hapusya-site-final-seal.txt
|
|
2026-07-13T20:29:02.8967807310 /var/lib/homelab-health/backup-matrix-vm-health-feedback-fix.txt
|
|
2026-07-13T20:40:37.0653547720 /var/lib/homelab-health/backup-matrix-desired-state-persistence.txt
|
|
2026-07-13T20:48:14.6313253050 /var/lib/homelab-health/backup-matrix-cluster-preseal-convergence.txt
|
|
2026-07-13T20:51:32.4163384480 /var/lib/homelab-health/backup-matrix-cluster-desired-state-sync.txt
|
|
2026-07-13T20:51:32.4223385400 /var/lib/homelab-health/backup-matrix-cluster-final-seal.txt
|
|
2026-07-13T21:13:01.1129724320 /var/lib/homelab-health/mail-cloud-edge-vm-runtime-contract-fix.txt
|
|
2026-07-13T21:15:43.9664537840 /var/lib/homelab-health/mail-cloud-edge-vm-readonly-probe.txt
|
|
2026-07-13T21:27:22.2230931620 /var/lib/homelab-health/mail-cloud-edge-vm-controlled-no-retention-mode.txt
|
|
2026-07-13T21:39:22.4540680390 /var/lib/homelab-health/mail-cloud-edge-vm-controlled-upload.txt
|
|
2026-07-13T21:40:21.6099694610 /var/lib/homelab-health/mail-cloud-edge-vm-service-convergence.txt
|
|
2026-07-13T21:56:30.9657631360 /var/lib/homelab-health/mail-cloud-edge-vm-matrix-registration.txt
|
|
2026-07-14T04:02:58.3995631560 /var/lib/homelab-health/mail-cloud-edge-vm-final-seal.txt
|
|
2026-07-14T04:12:54.7747139510 /var/lib/homelab-health/remaining-special-families-matrix-registration.txt
|
|
2026-07-14T04:15:48.2773761880 /var/lib/homelab-health/special-vm-emergency-non-app-final-seal.txt
|
|
2026-07-14T04:47:22.1524259850 /var/lib/homelab-health/cluster-admin-node-online-green-dot-fix.txt
|
|
2026-07-14T04:54:55.1333752320 /var/lib/homelab-health/cluster-admin-collapsible-health-sections.txt
|
|
2026-07-14T08:33:26.7124814180 /var/lib/homelab-health/cluster-admin-stage4c-database-migration-dry-run.txt
|
|
2026-07-14T08:48:04.5210170900 /var/lib/homelab-health/homelab-admin-runner.txt
|
|
2026-07-14T09:14:58.6397106130 /var/lib/homelab-health/cluster-admin-stage4d-adapter-candidates-in-isolation.txt
|
|
2026-07-14T09:39:00.0749801060 /var/lib/homelab-health/cluster-admin-stage4e-restricted-probe-agent-design.txt
|
|
2026-07-14T10:39:53.2848761800 /var/lib/homelab-health/homelab-admin-guardrails.txt
|
|
2026-07-14T10:54:50.6815482590 /var/lib/homelab-health/cluster-admin-stage4f-preapply-backup.txt
|
|
2026-07-14T20:02:48.4191051180 /var/lib/homelab-health/cluster-admin-stage4g-production-apply-read-only.txt
|
|
2026-07-14T20:16:32.3610333130 /var/lib/homelab-health/operator-rule-close-tails-immediately.txt
|
|
2026-07-14T20:23:46.5434583710 /var/lib/homelab-health/cluster-admin-stage4h-scope-definition.txt
|
|
2026-07-14T20:29:41.0475984260 /var/lib/homelab-health/cluster-admin-stage4h-collector-baseline-audit.txt
|
|
2026-07-14T20:41:34.5353531810 /var/lib/homelab-health/cluster-admin-stage4h-collector-patch-candidate.txt
|
|
2026-07-14T21:26:54.1724254520 /var/lib/homelab-health/operator-command-size-guard.txt
|
|
2026-07-14T21:29:30.0818406610 /var/lib/homelab-health/cluster-admin-stage4h-final-acceptance-contract.txt
|
|
2026-07-14T21:33:40.0697208200 /var/lib/homelab-health/homelab-admin-cli-contract.txt
|
|
2026-07-14T21:53:16.5230291610 /var/lib/homelab-health/cluster-admin-stage4h-task-package-build.txt
|
|
2026-07-15T10:10:19.9159742040 /var/lib/homelab-health/cluster-admin-stage4h-final-acceptance.txt
|
|
2026-07-15T10:17:53.0215082300 /var/lib/homelab-health/cluster-admin-stage4i-scope-definition.txt
|
|
2026-07-16T01:32:11.0876121490 /var/lib/homelab-health/homelab-ops-bootstrap.txt
|
|
2026-07-16T09:20:34.5941094330 /var/lib/homelab-health/router-hard-power.txt
|
|
2026-07-16T12:14:29.4893285610 /var/lib/homelab-health/xenforo-branding-rollout.txt
|
|
2026-07-16T12:41:28.8422409190 /var/lib/homelab-health/xenforo-branding-header-visibility.txt
|
|
2026-07-16T12:48:31.4566023130 /var/lib/homelab-health/xenforo-branding-header-order.txt
|
|
2026-07-16T13:01:16.2035467580 /var/lib/homelab-health/homelab-ops-worker-vm190.txt
|
|
2026-07-17T04:59:42.6476420400 /var/lib/homelab-health/netbird-vps-offhost.txt
|
|
2026-07-17T09:31:15.5873108310 /var/lib/homelab-health/paperless-offhost.txt
|
|
2026-07-17T14:21:55.3567278930 /var/lib/homelab-health/cluster-storage-retention.txt
|
|
2026-07-18T01:52:29.8931662090 /var/lib/homelab-health/paperless-backup.txt
|
|
2026-07-18T01:54:49.4850009180 /var/lib/homelab-health/netbird-vps-trust.txt
|
|
2026-07-18T02:54:28.2167554790 /var/lib/homelab-health/full-handoff-recovery.txt
|
|
2026-07-19T09:41:56.0870397860 /var/lib/homelab-health/mail-cloud-edge-vm.txt
|
|
2026-07-21T02:37:04.7969419110 /var/lib/homelab-health/cluster-admin-full-observer.txt
|
|
2026-07-21T02:37:04.7969419110 /var/lib/homelab-health/cluster-admin-observer.txt
|
|
2026-07-21T02:37:04.7969419110 /var/lib/homelab-health/cluster-admin-restricted-probes.txt
|
|
2026-07-21T02:39:14.3120870600 /var/lib/homelab-health/vm160-remote-backup.txt
|
|
2026-07-21T04:55:35.3129057980 /var/lib/homelab-health/edge-vm-offhost-freshness.txt
|
|
2026-07-21T04:55:35.3129057980 /var/lib/homelab-health/edge-vm-vzdump-offhost.txt
|
|
2026-07-21T04:55:35.3129057980 /var/lib/homelab-health/edge-vm-vzdump-restore.txt
|
|
2026-07-21T04:55:35.3408368720 /var/lib/homelab-health/sops-edge-secret-coverage.txt
|
|
2026-07-21T05:18:05.9056051500 /var/lib/homelab-health/router-running-config-mail-cloud.txt
|
|
2026-07-21T06:42:44.4264139010 /var/lib/homelab-health/mail-cloud-critical.txt
|
|
2026-07-21T07:05:43.0783306490 /var/lib/homelab-health/backup-last-run.log
|
|
2026-07-21T07:05:43.0870410520 /var/lib/homelab-health/backup.txt
|
|
2026-07-21T07:05:43.0879592500 /var/lib/homelab-health/backup.json
|
|
2026-07-21T07:15:46.8155363020 /var/lib/homelab-health/drift-check.txt
|
|
2026-07-21T07:36:59.5343449800 /var/lib/homelab-health/vm-backup.txt
|
|
2026-07-21T07:49:15.8201883610 /var/lib/homelab-health/desired-state.txt
|
|
2026-07-21T07:56:14.9677282530 /var/lib/homelab-health/runbooks.txt
|
|
2026-07-21T07:56:26.4127609750 /var/lib/homelab-health/mail-cloud-nextcloud-vm.txt
|
|
2026-07-21T08:01:15.0701844650 /var/lib/homelab-health/overall.txt
|
|
2026-07-21T08:10:47.5199010040 /var/lib/homelab-health/duty-admin.txt
|
|
2026-07-21T08:29:16.9008054160 /var/lib/homelab-health/capacity-risk.txt
|
|
2026-07-21T08:31:47.8421084810 /var/lib/homelab-health/final-readiness.txt
|
|
2026-07-21T08:35:41.8646792150 /var/lib/homelab-health/secret-exposure.txt
|
|
2026-07-21T08:51:15.1419524730 /var/lib/homelab-health/cluster-passport.txt
|
|
2026-07-21T08:51:15.4733058530 /var/lib/homelab-health/immich-media-offhost.txt
|
|
2026-07-21T08:54:09.3706207380 /var/lib/homelab-health/golden-state.txt
|
|
2026-07-21T09:15:30.8626761250 /var/lib/homelab-health/memos-offhost.txt
|
|
2026-07-21T10:02:41.8774944390 /var/lib/homelab-health/filebrowser-backup.txt
|
|
2026-07-21T10:02:41.8983445980 /var/lib/homelab-health/filebrowser-restore.txt
|
|
2026-07-21T10:02:41.9637054430 /var/lib/homelab-health/npmplus-kuma-config-backup.txt
|
|
2026-07-21T10:02:41.9638165500 /var/lib/homelab-health/npmplus-kuma-config-offhost.txt
|
|
2026-07-21T10:02:41.9638965790 /var/lib/homelab-health/npmplus-kuma-config-restore.txt
|
|
2026-07-21T10:02:41.9880922600 /var/lib/homelab-health/authentik-backup.txt
|
|
2026-07-21T10:02:41.9883026710 /var/lib/homelab-health/authentik-offhost.txt
|
|
2026-07-21T10:02:41.9885226120 /var/lib/homelab-health/authentik-restore.txt
|
|
2026-07-21T10:02:42.0123817100 /var/lib/homelab-health/gitea-backup.txt
|
|
2026-07-21T10:02:42.0126132350 /var/lib/homelab-health/gitea-offhost.txt
|
|
2026-07-21T10:02:42.0128119330 /var/lib/homelab-health/gitea-restore.txt
|
|
2026-07-21T10:02:42.0130029950 /var/lib/homelab-health/vaultwarden-backup.txt
|
|
2026-07-21T10:02:42.0132144910 /var/lib/homelab-health/vaultwarden-offhost.txt
|
|
2026-07-21T10:02:42.0134265770 /var/lib/homelab-health/vaultwarden-restore.txt
|
|
2026-07-21T10:02:42.0850718970 /var/lib/homelab-health/backup-coverage-matrix.txt
|
|
2026-07-21T10:02:42.1723598770 /var/lib/homelab-health/edge-vm-vzdump-backup.txt
|
|
2026-07-21T10:02:42.2144238040 /var/lib/homelab-health/extended-appbackup.txt
|
|
2026-07-21T10:02:42.3608260970 /var/lib/homelab-health/mail-cloud-restore-drill.txt
|
|
2026-07-21T10:02:42.4582078560 /var/lib/homelab-health/backup-framework.txt
|
|
2026-07-21T10:02:42.5028324600 /var/lib/homelab-health/vm-backup-policy.txt
|
|
2026-07-21T10:02:42.5224863440 /var/lib/homelab-health/vm170-vm171-full-backup.txt
|
|
2026-07-21T10:02:42.5469238210 /var/lib/homelab-health/vm170-vm171-full-restore-drill.txt
|
|
2026-07-21T10:02:42.5676022180 /var/lib/homelab-health/vm-local-dumps-cloud.txt
|
|
2026-07-21T10:02:42.5851669640 /var/lib/homelab-health/vm-local-dumps-retention.txt
|
|
2026-07-21T10:02:42.6060015540 /var/lib/homelab-health/immich-media-backup.txt
|
|
2026-07-21T10:02:42.6265610130 /var/lib/homelab-health/memos-backup.txt
|
|
2026-07-21T10:02:42.6979119670 /var/lib/homelab-health/paperless-restore.txt
|
|
2026-07-21T11:42:40.5821734030 /var/lib/homelab-health/vps-identity-audit.txt
|
|
2026-07-21T15:00:05.7882680340 /var/lib/homelab-health/incident-journal.txt
|
|
2026-07-21T15:00:06.9770480650 /var/lib/homelab-health/alerting.txt
|
|
2026-07-21T15:00:07.0682870380 /var/lib/homelab-health/forum-snuffleupagus.txt
|
|
2026-07-21T15:00:07.4142921750 /var/lib/homelab-health/kuma-monitor-policy.txt
|
|
2026-07-21T15:00:07.9663003700 /var/lib/homelab-health/backup-sla.txt
|
|
2026-07-21T15:00:07.9818151670 /var/lib/homelab-health/service-registry.txt
|
|
2026-07-21T15:00:07.9999556330 /var/lib/homelab-health/dependency-map.txt
|
|
2026-07-21T15:00:08.0023009050 /var/lib/homelab-health/safe-autoheal.txt
|
|
2026-07-21T15:01:15.0323030250 /var/lib/homelab-health/external-probe-vps.txt
|
|
2026-07-21T15:01:18.6147317300 /var/lib/homelab-health/duty-admin-v2.txt
|
|
2026-07-21T15:01:30.7637101020 /var/lib/homelab-health/disk-space.txt
|
|
2026-07-21T15:03:07.4642298620 /var/lib/homelab-health/crypto-portfolio-chart-health.txt
|
|
2026-07-21T15:03:33.7894609500 /var/lib/homelab-health/netbird-peers.txt
|
|
2026-07-21T15:04:18.9282025240 /var/lib/homelab-health/mkdocs-auto-refresh.txt
|
|
2026-07-21T15:04:30.2509352380 /var/lib/homelab-health/pve03-staging-capacity.txt.err
|
|
2026-07-21T15:04:30.3813298740 /var/lib/homelab-health/pve03-staging-capacity.txt
|
|
2026-07-21T15:04:30.7515073700 /var/lib/homelab-health/cluster-admin-webpanel-sync.txt
|
|
2026-07-21T15:04:31.7613625120 /var/lib/homelab-health/router-watchdog.txt
|
|
2026-07-21T15:04:32.9953817040 /var/lib/homelab-health/private-vpn-hosts.txt
|
|
/var/lib/homelab-health/final-closeout.txt:1:STATUS=OK TS=2026-07-08T09:02:10Z TYPE=final-closeout BAD=0 WARN=0 MODE=FINAL_OPERATOR_CLOSEOUT CLOSEOUT_OK=YES LOCAL_FAILED=0 PVE02_FAILED=0 PVE03_FAILED=0 EDGE_FAILED=0 DUTY_V2_STATUS=OK DUTY_V2_GATED_NONOK=0 DUTY_V2_UNGATED_ERROR_REVIEW=0 DUTY_V2_UNGATED_WARN_REVIEW=0 DUTY_V2_UNGATED_REVIEW_ITEM=0 DUTY_V2_LEGACY_OR_PHASE_REVIEW=0 OVERALL_STATUS=OK OVERALL_BAD=0 FINAL_STATUS=OK FINAL_FAIL=0 CLUSTER_STATUS=OK CLUSTER_BAD=0 EXACT_WARN_REVIEW_FILES=0 ENDPOINT_LOCAL_OK=1 ENDPOINT_LAN_OK=1 ENDPOINT_PUBLIC_OK=1 RECOVERY_SEAL=YES FREEZE_SEAL=YES FINAL_GLOBAL_GREEN=YES REVIEW_SURFACE_REGISTRY=YES POST_CLEANUP=YES REMOTE_CLEANUP=YES BACKUP_SLA=OK BACKUP_COVERAGE=OK EXTENDED_APPBACKUP=OK VM_BACKUP=OK SECRET_EXPOSURE=OK EXTERNAL_PROBE=OK EXTERNAL_PUBLIC_SMOKE_BAD=0 RESTORE_CHECK_OK=YES GOLDEN_MATCH=YES LATEST_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260708T084146Z GOLDEN_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260708T084146Z EMERGENCY_JOB=homelab-pve01-emergency-pack-20260708T002739Z VM_DUMPS_JOB=homelab-pve01-vm-local-dumps-20260706T153606Z DIRTY_COUNT=0 REPORT=/var/lib/homelab-final-closeout/final-closeout-latest.md
|
|
/var/lib/homelab-health/backup-matrix-cluster-final-seal.txt:1:STATUS=OK TS=2026-07-13T17:51:31Z TYPE=backup-matrix-cluster-final-seal BAD=0 CURRENT_STAGE_COMPLETE=YES MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_OK=31 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 READ_WRITE_OVERLAP=0 VM_BACKUP_HEALTH=OK VM_FULL_BACKUP=OK EDGE_VM_CANONICAL_BACKUP=homelab-edge-vm-offhost-freshness.timer OLD_EDGE_TIMER_REENABLED=NO CLUSTER_PASSPORT=OK FINAL_READINESS=OK FINAL_READINESS_PASS=53 FINAL_READINESS_FAIL=0 OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 WEBPANEL=OK FAILED_UNITS=0 CLUSTER_NODES=3 QUORATE=YES GENERATED_SYNC_HEAD=597119391e7b265b85fd9a76644e4e33af47e469 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=14a31112aac5d577ae01df567921b093b2fedcf7189c4cc8f009216ee0749380 CLUSTER_PASSPORT_SHA256=1b8b924ae0e5ce6872fe5671318094ce494806c2db7efe28299b5ff49adc0d0d FINAL_READINESS_SHA256=ac5159fa8c497afe021b5cb232f08cfec0e1a4d516d8aae966ba03f7034f1412 OVERALL_SHA256=6ae529bfdf5811f686d3b53978f453ce4f8757c74fc1be105de219b13a61ba46 PRESEAL_SHA256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99 EVIDENCE_MANIFEST_SHA256=b4c68312026e1328b208a74090891f4809ab31ec69069e94a3d6902429bf3566 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO NEXT_ACTION=AUDIT_AND_MIGRATE_SPECIAL_VM_EMERGENCY_AND_NON_APP_BACKUP_FAMILIES EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-final-seal/20260713T175131Z
|
|
/var/lib/homelab-health/final-cycle-stability-seal.txt:1:STATUS=OK TS=2026-07-07T20:08:34Z TYPE=final-cycle-stability-seal BAD=0 WARN=0 MODE=FORCED_REPLAY_AFTER_CYCLE_BREAK STABLE=YES EXTERNAL_PROBE_STATUS=OK EXTERNAL_PROBE_WARN=0 EXTERNAL_PROBE_SMOKE_BAD=0 FINAL_STATUS=OK FINAL_BAD=0 FINAL_FAIL=0 CLUSTER_STATUS=OK CLUSTER_BAD=0 DUTY_V2_STATUS=OK DUTY_V2_GATED_NONOK=0 OVERALL_STATUS=OK OVERALL_BAD=0 OLD_EXTERNAL_PROBE_LIVE_MARKERS=0 OLD_EXTERNAL_PROBE_SOURCE_MARKERS=0 FINAL_CLUSTER_CHECKS=0 FINAL_LEGACY_DUTY_CHECKS=0 OVERALL_LEGACY_DUTY_GATE=0 CLUSTER_LEGACY_DUTY_GATE=0 SOURCE_CLUSTER_LEGACY_DUTY_GATE=0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-final-cycle-break/stability-seal-latest.md
|
|
/var/lib/homelab-health/crypto-portfolio-ui-center-alert-currency-production-closure.txt:1:STATUS=OK TS=2026-07-13T08:12:14Z TYPE=crypto-portfolio-ui-center-alert-currency-production-closure BAD=0 PRODUCTION_CHANGED=YES_UI_ONLY INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb SHARP_MOVES_CENTERED=YES ALERT_CURRENCY_USD_RUB=OK BROWSER_RUNS=2 BROWSER_ERRORS=0 GRT_ROWS=0|0|0 GRT_SOURCE_REFERENCES=0 POSTCHANGE_BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z RESTORE_CHECK=OK_BOTH_CLOUDS RETENTION_ROLLBACK_COUNT=3 RETENTION_OPTIONAL_OTHER_CANDIDATES=1 RETENTION_REMOVED_ROLLBACK_COUNT=0 DANGLING_IMAGES_REMAINING=2 UNREFERENCED_DANGLING_IMAGES=0 DESIRED_STATE_HEAD=2999910549eaff9b9431b9c656d318340eb95cbd DESIRED_STATE_CLEAN=YES PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES READY_FOR_UI_FINAL_SEAL=YES
|
|
/var/lib/homelab-health/desired-state-remote-git-audit.txt:1:STATUS=OK TS=2026-07-08T09:27:24Z TYPE=desired-state-remote-git-audit BAD=0 WARN=0 MODE=READ_ONLY_DESIRED_STATE_REMOTE_GIT_AUDIT AUDIT_OK=YES REMOTE_READY=NO ADD_REMOTE_DECISION=NO_CHANGE REPO_PRESENT=1 DIRTY_COUNT=0 REMOTE_COUNT=0 URL_CREDENTIAL_HINT=0 HTTPS_GITEA_OK=1 HTTPS_API_OK=0 BACKEND_OK=1 BACKEND_API_OK=0 LOCAL_FAILED=0 PVE02_FAILED=0 PVE03_FAILED=0 EDGE_FAILED=0 OVERALL_STATUS=OK OVERALL_BAD=0 DUTY_V2_STATUS=OK DUTY_V2_GATED_NONOK=0 DUTY_V2_UNGATED_ERROR_REVIEW=0 DUTY_V2_UNGATED_WARN_REVIEW=0 CLOSEOUT_OK=YES RECOVERY_SEAL_OK=YES INDEPENDENT_RUNNER_NO_DEPLOY=YES REPORT=/var/lib/homelab-desired-state-remote-git/audit-latest.md
|
|
/var/lib/homelab-health/cluster-admin-stage3-postapply-backup.txt:1:STATUS=OK TS=2026-07-13T03:43:59Z TYPE=cluster-admin-stage3-postapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=POSTAPPLY_BACKUP_AND_CONTROLLED_COLLECTOR_RUN PRODUCTION_DATABASE_CHANGED=COLLECTOR_NORMAL_OPERATION_ONLY PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=YES_CONTROLLED_POSTAPPLY_BACKUP LIVE_APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 APPBACKUP=VERIFIED APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T034400Z APPBACKUP_SIZE_BYTES=1791603 APPBACKUP_RESTORE_CHECK=OK APPBACKUP_ROUTING=VERIFIED COLLECTOR_CONTROL_RUN=OK COLLECTOR_RUN_ID=3113 CANONICAL_CONSISTENCY=0|0|0|0|0 INCIDENT_ENGINE=ACTIVE COLLECTOR_TIMER=ACTIVE VM180_FAILED_UNITS=0 READY_FOR_STAGE3_FINAL_SEAL=YES STAGE3_CLOSED=NO POSTAPPLY_ROOT=/var/lib/homelab-stage3-postapply/20260713T034359Z
|
|
/var/lib/homelab-health/cluster-admin-webpanel.txt:1:STATUS=OK TS=2026-07-11T01:39:28Z TYPE=cluster-admin-webpanel BAD=0 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=0 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK
|
|
/var/lib/homelab-health/post-cleanup-final-seal.txt:1:STATUS=OK TS=2026-07-08T06:10:30Z TYPE=post-cleanup-final-seal BAD=0 WARN=0 MODE=POST_CLEANUP_OWNER_TIMER_FINAL_SEAL FINAL_SEAL_OK=YES REPLAY_STATUS=OK REPLAY_OK=YES LOCAL_START_BAD=0 LOCAL_FAILED=0 PVE02_FAILED=0 PVE03_FAILED=0 EDGE_FAILED=0 DUTY_V2_STATUS=OK DUTY_V2_GATED_NONOK=0 DUTY_V2_UNGATED_ERROR_REVIEW=0 DUTY_V2_UNGATED_WARN_REVIEW=0 DUTY_V2_UNGATED_REVIEW_ITEM=22 DUTY_V2_LEGACY_OR_PHASE_REVIEW=5 REVIEW_ITEMS_INTENTIONALLY_LEFT=yes LEGACY_PHASE_ITEMS_INTENTIONALLY_LEFT=yes FINAL_STATUS=OK FINAL_BAD=0 FINAL_FAIL=0 CLUSTER_STATUS=OK CLUSTER_BAD=0 OVERALL_STATUS=OK OVERALL_BAD=0 LOCAL_CLEANUP_OK=YES REMOTE_CLEANUP_OK=YES RESIDUAL_SEAL_OK=YES SOPS_STATUS=OK EXTERNAL_PROBE_STATUS=OK EXTERNAL_PROBE_WARN=0 EXTERNAL_PROBE_SMOKE=0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-post-cleanup-replay/final-seal-latest.md
|
|
/var/lib/homelab-health/pve01-core-monitoring-app-restorecheck.txt:1:STATUS=OK TS=2026-07-08T15:57:27Z TYPE=pve01-core-monitoring-app-restorecheck BAD=0 MODE=APP_LEVEL_RESTORE_CHECKS_TWO_CLOUDS FULL_VM_RESTORE_AVAILABLE=NO TOTAL_APPS=30 OK_APPS=30 FAIL_LIST=none REPORT=/var/lib/homelab-node-loss/pve01-core-monitoring-app-level/app-restorecheck-latest.md
|
|
/var/lib/homelab-health/restore-drill-index.txt:5:MISSING_OR_NOT_OK=none
|
|
/var/lib/homelab-health/mail-cloud-edge-vm-final-seal.txt:1:STATUS=OK TS=2026-07-14T01:02:58Z TYPE=mail-cloud-edge-vm-final-seal BAD=0 WARN=0 VMID=130 SOURCE=/mnt/staging/offhost/edge-vm-vzdump-from-pve03/vzdump-qemu-130-2026_07_13-03_30_01.vma.zst SIZE=10942350973 SHA256=6979558850813d99881b20c559600ec1c8748cc4f59f18ef9875eee6e40c6757 CLOUD_FOUND_DIR=20260713T183122Z MATRIX_JOB_ID=homelab_mail_cloud_edge_vm_upload MATRIX_STATUS=OK JOBS_TOTAL=36 JOBS_OK=32 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 UPLOAD_VERIFIED=YES DOWNLOAD_VERIFY=YES SHA_MATCH=YES SIZE_MATCH=YES PART_COUNT_MATCH=YES CONTROLLED_UPLOAD_MODE=--upload-no-retention RETENTION_MODE=disabled RETENTION_RUN=NO RETENTION_DELETE_COUNT=0 CLOUD_PURGE_RUN=NO SOURCE_DELETE_RUN=NO EDGE_SERVICE_RESULT=success EDGE_EXEC_MAIN_STATUS=0 EDGE_TIMER_ENABLED=enabled EDGE_TIMER_ACTIVE=active CLUSTER_ADMIN_STATUS=OK PROBE_OK=4 PROBE_BAD=0 CLUSTER_PASSPORT=OK DUTY_ADMIN=OK OVERALL=OK OVERALL_BAD=0 FINAL_READINESS=OK FINAL_PASS=53 FINAL_FAIL=0 WEBPANEL=OK WEBPANEL_SYNC=OK PUBLIC_OVERALL=OK FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DESIRED_STATE_PRE_SEAL_HEAD=98dfaeb35683e18cd4e986d0121bde60daac1e7c CURRENT_EDGE_VM_CLOUD_SUBSTAGE_COMPLETE=YES EVIDENCE_ROOT=/var/lib/homelab-change-sets/mail-cloud-edge-vm-final-seal-idle-safe/20260714T010226Z
|
|
/var/lib/homelab-health/cluster-admin-stage2-dynamic-redryrun.txt:1:STATUS=OK TS=2026-07-13T01:20:06Z TYPE=cluster-admin-stage2-dynamic-redryrun BAD=0 PRODUCTION_CHANGED=TEMPORARY_DATABASE_AND_EVIDENCE_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO INCIDENT_ENGINE_RESTARTED=NO MIGRATION_ID=002_canonical_foundation_sync MIGRATION_SHA256=92a7f87ed6bf5215c81ec5f70ab1dc3c8339ecffa0b0f9fcab4eb6322651dc68 MIGRATION_PATCH_REQUIRED=NO COLLECTOR_PATCH_REQUIRED=NO APPLY_VERIFIER_PATCH_REQUIRED=YES FIXED_TOTAL_ENTITIES_REQUIRED=NO FIXED_TOTAL_ALIASES_REQUIRED=NO ACTIVE_CURRENT_SET_CONVERGENCE=OK NATURAL_KEY_CONVERGENCE=OK EPHEMERAL_ID_REFRESH=OK DYNAMIC_PLACEHOLDER_LIFECYCLE=OK RETIRED_PLACEHOLDERS_ALLOWED=YES ORPHANS=0 PREFLIGHT_TRANSACTION_ROLLBACK=OK DDL_ROLLBACK_TEST=OK ROLLBACK_AFTER_COMMITTED_SYNC=DO_NOT_PARTIAL_DDL_ROLLBACK PRESERVE_APPLIED_STATE_ON_POSTSYNC_VERIFIER_FAILURE=YES TEST_DATABASE_DROPPED=YES PRODUCTION_UNCHANGED=YES FRESH_PREAPPLY_BACKUP_REQUIRED=YES READY_FOR_FRESH_PREAPPLY_BACKUP=YES DESIRED_STATE_HEAD=4d0d03395d2baf4024ad7c9d5870f21bdaee9f99 CONTRACT=/var/lib/homelab-change-sets/stage2-dynamic-redryrun/20260713T012006Z/dynamic-acceptance-contract.json RESULT=/var/lib/homelab-change-sets/stage2-dynamic-redryrun/20260713T012006Z/dynamic-redryrun-result.txt CHANGE_SET=/var/lib/homelab-change-sets/stage2-dynamic-redryrun/20260713T012006Z
|
|
/var/lib/homelab-health/desired-state-remote-git.txt:1:STATUS=OK TS=2026-07-08T09:31:25Z TYPE=desired-state-remote-git BAD=0 WARN=0 MODE=NO_REMOTE_ADD_NO_PUSH_SEAL NO_REMOTE_SEAL_OK=YES AUDIT_STATUS=OK AUDIT_OK=YES REMOTE_READY=NO ADD_REMOTE_DECISION=NO_CHANGE GIT_REMOTE_CHANGED=NO GIT_PUSH_DONE=NO REPO_PRESENT=1 REPO_TOP=/srv/homelab-desired-state BRANCH=master DIRTY_COUNT=0 REMOTE_COUNT=0 URL_CREDENTIAL_HINT=0 HTTPS_GITEA_OK=1 HTTPS_API_OK=0 BACKEND_OK=1 BACKEND_API_OK=0 LOCAL_FAILED=0 PVE02_FAILED=0 PVE03_FAILED=0 EDGE_FAILED=0 OVERALL_STATUS=OK OVERALL_BAD=0 FINAL_STATUS=OK FINAL_FAIL=0 DUTY_V2_STATUS=OK DUTY_V2_GATED_NONOK=0 DUTY_V2_UNGATED_ERROR_REVIEW=0 DUTY_V2_UNGATED_WARN_REVIEW=0 DUTY_V2_UNGATED_REVIEW_ITEM=0 DUTY_V2_LEGACY_OR_PHASE_REVIEW=0 CLOSEOUT_OK=YES RECOVERY_SEAL_OK=YES INDEPENDENT_RUNNER_NO_DEPLOY=YES RESTORE_CHECK_OK=YES GOLDEN_MATCH=YES LATEST_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260708T092404Z GOLDEN_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260708T092404Z REPORT=/var/lib/homelab-desired-state-remote-git/no-change-seal-latest.md
|
|
/var/lib/homelab-health/service-registry.txt:1:STATUS=OK TS=2026-07-21T12:00:07Z TYPE=service-registry SERVICES=12 MISSING=0 REGISTRY=/var/lib/homelab-registry/service-registry-latest.tsv
|
|
/var/lib/homelab-health/crypto-portfolio-remove-grt-discovery.txt:1:STATUS=OK TS=2026-07-13T03:50:49Z TYPE=crypto-portfolio-remove-grt-discovery BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=NO DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO PREVIOUS_FAILURE_ROOT_CAUSE=UNVERIFIED_STACK_PATH_ASSUMPTION CANDIDATE_DIRECTORIES=4 CANDIDATE_DATABASES=0 CANDIDATE_CONTAINERS=0 CANDIDATE_REFERENCE_FILES=8 READY_FOR_GRT_TARGETED_CAPTURE=YES DISCOVERY_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-discovery/20260713T035049Z
|
|
/var/lib/homelab-health/post-cleanup-owner-timer-replay.txt:1:STATUS=OK TS=2026-07-08T06:01:13Z TYPE=post-cleanup-owner-timer-replay BAD=0 WARN=0 MODE=FORCED_REPLAY_CLEANUP_OWNERS REPLAY_OK=YES LOCAL_START_OK=10 LOCAL_START_BAD=0 EDGE_RC=0 EDGE_HEALTH=OK LOCAL_FAILED=0 PVE02_FAILED=0 PVE03_FAILED=0 EDGE_FAILED=0 EXTERNAL_PROBE_STATUS=OK EXTERNAL_PROBE_WARN=0 EXTERNAL_PROBE_SMOKE=0 SOPS_STATUS=OK DUTY_V2_STATUS=OK DUTY_V2_GATED_NONOK=0 DUTY_V2_UNGATED_ERROR_REVIEW=0 DUTY_V2_UNGATED_WARN_REVIEW=0 FINAL_STATUS=OK FINAL_FAIL=0 CLUSTER_STATUS=OK CLUSTER_BAD=0 OVERALL_STATUS=OK OVERALL_BAD=0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-post-cleanup-replay/owner-timer-replay-latest.md
|
|
/var/lib/homelab-health/paperless-core-guard-audit.txt:1:STATUS=OK TS=2026-07-08T05:26:59Z TYPE=paperless-core-guard-audit BAD=0 WARN=1 MODE=READ_ONLY_PAPERLESS_CORE_GUARD_AUDIT CLASS=REMOTE_DEFERRED_GUARD_NOT_HARD_GATE CONNECT_ERRORS=0 FAILED_ONLY_GUARD=YES IN_HARD_GATES=no OVERALL_BAD=0 FINAL_FAIL=0 DUTY_V2_GATED_NONOK=0 PAPERLESS_CONTAINER_LINES=0 CORE_RELATED_LINES=1 RESET_FAILED_DONE=no REPORT=/var/lib/homelab-remote-failed-unit-cleanup/paperless-core-guard-audit-latest.md
|
|
/var/lib/homelab-health/hapusya-site-final-seal.txt:1:STATUS=OK TS=2026-07-13T12:33:08Z TYPE=hapusya-site-final-seal BAD=0 APP=hapusya-site URL=https://hapusya.ru/ PRESEAL_DESIRED_STATE_HEAD=69966eb83fc065141df3c46df0d6c0f282275046 ROUTE_SHA256=02d65cb0082a04cae282126cc829a002f171e863291c18ba04d788dc2d6282ca UNIT_SHA256=b3b07137b25ac0a0b4bdadadad654cf209387ade86a2ef2cdca58e8a7c0cc212 RELATIVE_TREE_SHA256=9dadf21da041bc4d6ee4a42f5b3ac7719c16bcc39052de32af895c6cd28286cb APPLY_MANIFEST_SHA256=f44992e3e0cc443b372779cb35d9e899d904a9960a9de03c14559886fdb1fc75 PRODUCTION_BROWSER_MANIFEST_SHA256=9f3cdf3063f8724d1d1c1ad619f47d6f98d434c3422a74903585146858a8dfac BROWSER_SMOKE=OK BROWSER_RUNS=5 BROWSER_ERRORS=0 DESKTOP_VIDEO=OK MOBILE_VIDEO=OK STATIC_FALLBACK=OK BACKUP_JOB=homelab-pve01-app-hapusya-site-20260713T115259Z ARCHIVE_SHA256=bf284292974600384c43c47fc99be6fedf0ebd68f722b402a7260f1df680d167 TWO_CLOUDS=OK RESTORE_CHECK=OK_BOTH_CLOUDS RETENTION=OK DESIRED_STATE_SYNC=OK GENERATOR_SHA256=507bbbc2691f1f08cb1dd3660ed76cf13a8c36c23833d5fdec40cf6e07486cb8 MAP_SHA256=815b52dcedf7e3e92e07939bade32c7e8ed6acc0f9e064b65db6daec9f559c63 POLICY_SHA256=4bf996ff451ae3ec56eb2fcd42842606cd97295b4d5141aedb135c956395e4b5 EVIDENCE_ROOT=/var/lib/homelab-change-sets/hapusya-site-final-seal/20260713T123308Z EVIDENCE_MANIFEST_SHA256=7eb993a17ac18a1f7c9ae0b52d745d3778833d15626ed34c82496f74916eb26a PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES PRODUCTION_CHANGED=NO_FINAL_SEAL_ONLY DESIRED_STATE_CHANGED=YES_FINAL_SEAL_COMMIT_ONLY ALL_HAPUSYA_SITE_TAILS_CLOSED=YES TASK_COMPLETE=YES
|
|
/var/lib/homelab-health/router-watchdog.txt:1:STATUS=OK TS=2026-07-21T12:04:31Z TYPE=router-watchdog ACTION=none REASON=router_wan_and_outbound_healthy SSH_OK=1 WAN_OK=1 OUTBOUND_OK=2 FAIL_COUNT=0 THRESHOLD=5 LAST_REBOOT_EPOCH=0 REBOOTS_TODAY=0 HARD_POWER_HELPER_READY=yes
|
|
/var/lib/homelab-health/pve03-failed-unit-cleanup-closure.txt:1:STATUS=OK TS=2026-07-07T12:52:21Z TYPE=pve03-failed-unit-cleanup-closure BAD=0 WARN=0 MODE=READ_ONLY_CLOSURE LATEST_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260707T124031Z GOLDEN_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260707T124031Z GOLDEN_CP_MATCH=YES RESTORE_CHECK=YES PVE03_FAILED_UNITS=0 REPORT=/var/lib/homelab-pve03-failed-unit-cleanup/closure-latest.md
|
|
/var/lib/homelab-health/review-surface-registry-audit.txt:1:STATUS=OK TS=2026-07-08T06:20:10Z TYPE=review-surface-registry-audit BAD=0 WARN=1 MODE=READ_ONLY_REVIEW_SURFACE_REGISTRY_AUDIT LOCAL_FAILED=0 PVE02_FAILED=0 PVE03_FAILED=0 EDGE_FAILED=0 DUTY_V2_STATUS=OK DUTY_V2_GATED_NONOK=0 DUTY_V2_UNGATED_ERROR_REVIEW=0 DUTY_V2_UNGATED_WARN_REVIEW=0 DUTY_V2_UNGATED_REVIEW_ITEM=22 DUTY_V2_LEGACY_OR_PHASE_REVIEW=5 OVERALL_STATUS=OK OVERALL_BAD=0 REVIEW_COUNT=22 LEGACY_COUNT=5 HARD_GATE_COUNT=0 CONVERT_PLANNING=19 CONVERT_CLOSURE=4 ARCHIVE_LEGACY=4 FILES_CHANGED=no REGISTRY=/var/lib/homelab-review-surface-registry/review-surfaces-latest.tsv REPORT=/var/lib/homelab-review-surface-registry/audit-latest.md
|
|
/var/lib/homelab-health/nextcloud-restore-proof-offhost.txt:1:STATUS=OK TS=20260624T190330Z TYPE=nextcloud-restore-proof-offhost DEST_HOST=pve02 DEST_DIR=/mnt/staging/offhost/edge-restore-proofs-from-edge/nextcloud-aio/nextcloud-aio-restore-proof/20260624T120344Z ITEMS=1 OK=1 ERROR=0 BYTES=2135684 FILES=7 SECRET_PRINTED=NO
|
|
/var/lib/homelab-health/recovery-card-seal.txt:1:STATUS=OK TS=2026-07-08T08:34:39Z TYPE=recovery-card-seal BAD=0 WARN=0 MODE=RECOVERY_CARD_FINAL_SEAL RECOVERY_SEAL_OK=YES RECOVERY_CARD_STATUS=OK RECOVERY_CARD_OK=YES FREEZE_STATUS=OK FREEZE_OK=YES FINAL_GLOBAL_GREEN=YES REVIEW_SURFACE_REGISTRY=YES LOCAL_FAILED=0 PVE02_FAILED=0 PVE03_FAILED=0 EDGE_FAILED=0 DUTY_V2_STATUS=OK DUTY_V2_GATED_NONOK=0 DUTY_V2_UNGATED_ERROR_REVIEW=0 DUTY_V2_UNGATED_WARN_REVIEW=0 DUTY_V2_UNGATED_REVIEW_ITEM=0 DUTY_V2_LEGACY_OR_PHASE_REVIEW=0 OVERALL_STATUS=OK OVERALL_BAD=0 RESTORE_CHECK_OK=YES GOLDEN_MATCH=YES LATEST_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260708T082034Z GOLDEN_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260708T082034Z EMERGENCY_JOB=homelab-pve01-emergency-pack-20260708T002739Z VM_DUMPS_JOB=homelab-pve01-vm-local-dumps-20260706T153606Z ENDPOINT_LOCAL_OK=1 ENDPOINT_LAN_OK=1 ENDPOINT_PUBLIC_OK=1 BACKUP_SLA=OK BACKUP_COVERAGE=OK EXTENDED_APPBACKUP=OK VM_BACKUP=OK EXTERNAL_PROBE=OK EXTERNAL_PUBLIC_SMOKE_BAD=0 DIRTY_COUNT=0 CARD=/var/lib/homelab-recovery-card/recovery-card-latest.md SUMMARY=/var/lib/homelab-recovery-card/recovery-summary-latest.tsv REPORT=/var/lib/homelab-recovery-card/recovery-card-seal-latest.md
|
|
/var/lib/homelab-health/pve03-failed-unit-cleanup.txt:1:STATUS=OK TS=2026-07-07T12:25:56Z TYPE=pve03-failed-unit-cleanup BAD=0 WARN=0 MODE=RETRY_MANDB_AFTER_SPACE_RECOVERY START_RC=0 MAN_DB_FAILED_LEFT=0 AFTER_FAILED_COUNT=0 ROOT_USAGE_PERCENT=31 STAGING_USAGE_PERCENT=27 REPORT=/var/lib/homelab-pve03-failed-unit-cleanup/man-db-cleanup-latest.md
|
|
/var/lib/homelab-health/crypto-portfolio-remove-grt-candidate.txt:1:STATUS=OK TS=2026-07-13T04:20:46Z TYPE=crypto-portfolio-remove-grt-candidate BAD=0 COMMAND_PHYSICAL_LINES=1 PREVIOUS_FAILURE=CORRECTED PREVIOUS_FAILURE_ROOT_CAUSE=KLINE_CACHE_FILTER_APPLIED_TO_TOP_LEVEL_INSTEAD_OF_ENTRIES EXCLUSIVE_ROOT_CAUSE_PROOF=OK PRODUCTION_CHANGED=NO DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO TARGET_HOST=core-apps.gram1.ru DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 INPUT_APP_SHA256=684ad884119554503db0ed36252aeb017c0c04b54729f30c51e66a7ab293f8bd INPUT_WORKER_SHA256=40a1c81111d7097120125b140683a8604abcb4051db5073ab8af708a11854038 INPUT_INDEX_SHA256=2b1ab459cabf3050af1db4a7b293fff7c2a9ac18bf4a14faa1efc70a3a3f63e3 APP_CANDIDATE_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_CANDIDATE_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_CANDIDATE_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e MARKET_CANDIDATE_SHA256=eb6c21667d4a42a7a4df8a5d6c3a6808bf69dc5670390e5ee7bfb95687b25074 KLINE_CACHE_CANDIDATE_SHA256=f1a2fe26bedbe457ced26f4efdc605a16aaa1e26e0d5c3b5135533be6292d583 DATABASE_CANDIDATE_SHA256=ae04bf3e7a73568d1fe41813cf1ef224b324276e19a80da1d4b1ca65afcb3d87 CHART_HEALTH_CANDIDATE_SHA256=c16473f16390eeeff4d4eb701db604093564dd78d0be2d30b4cc8ffeeb22364a DATABASE_GRT_AFTER=0|0|0 DATABASE_INTEGRITY=ok CANDIDATE_GRT_REFERENCES=0 KLINE_CACHE_SCHEMA=ENTRIES_OBJECT KLINE_CACHE_GRT_ENTRIES_REMOVED=2 KLINE_CACHE_GRT_ENTRIES_AFTER=0 ACTIVE_COINS=HBAR,POL MARKET_COIN_COUNT=2 EXACT_COINS_SELECTOR_PATCH=OK MOVES_GRID_UNCHANGED=YES READY_FOR_GRT_ISOLATED_DRY_RUN=YES CANDIDATE_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-candidate-v3/20260713T042046Z
|
|
/var/lib/homelab-health/cluster-admin-stage0-readiness-fix.txt:1:STATUS=OK TS=2026-07-12T23:22:37Z TYPE=cluster-admin-stage0-readiness-fix BAD=0 RESULT=APPLIED FINAL_READINESS=OK FINAL_READINESS_DURATION_SECONDS=17 PASSPORT=OK OVERALL=OK DUTY_ADMIN=OK FAILED_UNITS=0 LIVE_MATCH_DESIRED_STATE=YES READY_FOR_FINAL_SEAL=YES OLD_HEAD=a6e09a8af3604e47d7cc78be4176e6d567e8dbfb NEW_HEAD=1379b4fb4e809e56e9bc16f0e0b19b0a509192d5 CHANGE_SET=/var/lib/homelab-change-sets/stage0-readiness/20260712T232237Z PRODUCTION_CHANGED=READINESS_GATE_LOGIC_ONLY PRODUCTION_SCHEMA_CHANGED=NO CLOUD_WRITE_RUN=NO
|
|
/var/lib/homelab-health/crypto-portfolio-remove-grt-recovery-audit.txt:1:STATUS=ERROR TS=2026-07-13T05:39:07Z TYPE=crypto-portfolio-remove-grt-recovery-audit BAD=1 COMMAND_PHYSICAL_LINES=1 ROOT_CAUSE=READINESS_ENDPOINT_MISMATCH RECOVERY_VERIFIED=NO TASK_COMPLETE=NO READY_FOR_GRT_APPLY_SCRIPT_FIX=NO AUDIT_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-recovery-audit-v2/20260713T053907Z
|
|
/var/lib/homelab-health/cluster-admin-backup-webpanel-final-seal.txt:1:STATUS=OK TS=2026-07-09T05:24:54Z TYPE=cluster-admin-backup-webpanel-final-seal BAD=0 VMID=180 URL=http://[PRIVATE_IP]:8080/ BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK WEBPANEL=OK WEBPANEL_SYNC=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-backup-webpanel-final-seal-latest.md
|
|
/var/lib/homelab-health/review-surface-registry-seal.txt:1:STATUS=OK TS=2026-07-08T06:36:39Z TYPE=review-surface-registry-seal BAD=0 WARN=0 MODE=REGISTERED_REVIEW_SURFACES_POLICY_AWARE_SEAL PRECHECK=YES REGISTRY_SEAL_OK=YES UPDATED=27 ARCHIVED=27 REGISTERED_COUNT=27 EXACT_WARN_REVIEW_FILES=0 LOCAL_FAILED=0 PVE02_FAILED=0 PVE03_FAILED=0 EDGE_FAILED=0 DUTY_V2_STATUS=OK DUTY_V2_GATED_NONOK=0 DUTY_V2_UNGATED_ERROR_REVIEW=0 DUTY_V2_UNGATED_WARN_REVIEW=0 DUTY_V2_UNGATED_REVIEW_ITEM=0 DUTY_V2_LEGACY_OR_PHASE_REVIEW=0 FINAL_STATUS=OK FINAL_BAD=0 FINAL_FAIL=0 CLUSTER_STATUS=OK CLUSTER_BAD=0 OVERALL_STATUS=OK OVERALL_BAD=0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-review-surface-registry/seal-latest.md
|
|
/var/lib/homelab-health/mail-cloud-critical.txt:1:STATUS=OK TS=20260721T034025Z TYPE=mail-cloud-critical MODE=recurring REMOTES=2 REMOTE_OK=2 REMOTE_FAIL=0 DEST=app/p0-critical-baseline/20260721T034025Z BUNDLE=p0-critical-baseline-20260721T034025Z.tar.gz SIZE=973275719 SHA256=e1681f82099b2430a46d47c3734063dce8322fd7cec37d4614b3d99ee293b45a RETENTION_KEEP=14 SECRET_PRINTED=no NOTES=none
|
|
/var/lib/homelab-health/duty-admin-residual-cleanup.txt:1:STATUS=OK TS=2026-07-08T05:42:29Z TYPE=duty-admin-residual-cleanup BAD=0 WARN=0 MODE=SUPERSEDE_STALE_PVE03_CAPACITY_FORENSICS PRECHECK=YES CLEANUP_OK=YES NEW_PVE03_CAPACITY_STATUS=OK FAILED_LOCAL=0 FAILED_PVE02=0 FAILED_PVE03=0 FAILED_EDGE=0 DUTY_V2_STATUS=OK DUTY_V2_GATED_NONOK=0 DUTY_V2_UNGATED_ERROR_REVIEW=0 DUTY_V2_UNGATED_WARN_REVIEW=0 DUTY_V2_UNGATED_REVIEW_ITEM=22 DUTY_V2_LEGACY_OR_PHASE_REVIEW=5 OVERALL_STATUS=OK OVERALL_BAD=0 WARN_REVIEW_FILES=19 REVIEW_ITEMS_INTENTIONALLY_LEFT=yes REPORT=/var/lib/homelab-duty-admin-residual-cleanup/pve03-capacity-forensics-closure-latest.md
|
|
/var/lib/homelab-health/appbackupctl-routing.txt:1:STATUS=OK TS=2026-07-13T00:29:36Z TYPE=appbackupctl-routing BAD=0 RESULT=APPLIED PRODUCTION_CHANGED=BACKUP_CLI_ROUTING_AND_DESIRED_STATE_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO DEFAULT_ROUTE=dockge-stacks CLUSTER_ADMIN_ROUTE=homelab-cluster-admin-incident-engine-backup.service UNSUPPORTED_TARGET_POLICY=REFUSE_RC_2 ROUTE_TEST_DOCKGE=OK ROUTE_TEST_CLUSTER_ADMIN=OK ROUTE_TEST_DEFAULT=OK ROUTE_TEST_UNSUPPORTED=OK FRAMEWORK_CHECK=OK UNSUPPORTED_ROUTE_BACKUP_CREATED=NO LIVE_MATCH_DESIRED_STATE=YES PVE01_FAILED_UNITS=0 OVERALL=OK DUTY_ADMIN=OK STAGE1_CLOSED=YES STAGE2_READY=YES OLD_HEAD=bd5eb856a8d61a5fb83c2bcaf74cb8ac98f075c5 NEW_HEAD=4d0d03395d2baf4024ad7c9d5870f21bdaee9f99 CHANGE_SET=/var/lib/homelab-change-sets/appbackupctl-routing/20260713T002936Z
|
|
/var/lib/homelab-health/final-readiness.txt:1:STATUS=WARN TS=2026-07-21T05:31:30Z TYPE=final-readiness BAD=5 PASS=48 FAIL=5 REPORT=/var/lib/homelab-final-readiness/latest.md
|
|
/var/lib/homelab-health/cluster-admin-stage3-api-apply.txt:1:STATUS=OK TS=2026-07-13T03:22:35Z TYPE=cluster-admin-stage3-api-apply BAD=0 COMMAND_PHYSICAL_LINES=1 RESULT=APPLIED PRODUCTION_CHANGED=LIVE_APP_AND_DESIRED_STATE PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO SERVICE_RESTARTED=YES CLOUD_WRITE_RUN=NO_REUSE_VERIFIED_PREAPPLY_BACKUP APP_PRECHANGE_SHA256=c2748b8f96df842f491b5d2790790594222549853d0e2336cf60ecd6d4de0020 APP_CANDIDATE_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 LIVE_APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 LIVE_MATCH_DESIRED_STATE=YES PREAPPLY_BACKUP_REUSED=YES AST_IDENTICAL_NORMALIZATION=YES EXISTING_ROUTES_BASELINE=OK NEW_GET_ROUTES=7 UNAUTHENTICATED_NEW_ROUTES_401=YES OPENAPI_DOCS_EXPOSURE_CHANGED=NO CORE_DATABASE_STATE=1|1|9|7|1 INCIDENT_ENGINE=ACTIVE VM180_FAILED_UNITS=0 ROLLBACK_CHECKPOINT=VERIFIED OLD_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a NEW_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 READY_FOR_STAGE3_AUTHENTICATED_PRODUCTION_VERIFY=YES STAGE3_CLOSED=NO CHANGE_SET=/var/lib/homelab-change-sets/stage3-api-production-apply/20260713T032235Z
|
|
/var/lib/homelab-health/cluster-admin-stage1-foundation-apply.txt:1:STATUS=OK TS=2026-07-13T00:21:21Z TYPE=cluster-admin-stage1-foundation-apply BAD=0 RESULT=APPLIED PRODUCTION_CHANGED=DATABASE_FOUNDATION_AND_DESIRED_STATE PRODUCTION_SCHEMA_CHANGED=PLANNED_ADDITIVE_ONLY SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO MIGRATION_ID=001_entity_source_policy_foundation NEW_TABLES=9 COLLECTOR_RUN_COLUMNS=7 TRIGGER=collector_runs_foundation_enrich_trg FUNCTION=cluster_admin_enrich_collector_run ENTITIES=83 PLACEHOLDERS=21 ENTITY_DEPENDENCIES=26 ENTITY_ALIASES=225 ALIAS_COLLISIONS=3 SOURCES=1 SOURCE_INSTANCES=1 SOURCE_EXPECTATIONS=1 POLICIES=1 POLICY_VERSIONS=1 ORPHANS=0 LIVE_COLLECTOR_COMPATIBILITY=OK API_HEALTH=OK VM180_FAILED_UNITS=0 ROLLBACK_POINT=VERIFIED OLD_HEAD=1379b4fb4e809e56e9bc16f0e0b19b0a509192d5 NEW_HEAD=bd5eb856a8d61a5fb83c2bcaf74cb8ac98f075c5 LIVE_MATCH_DESIRED_STATE=YES CHANGE_SET=/var/lib/homelab-change-sets/stage1-production-apply/20260713T002121Z
|
|
/var/lib/homelab-health/homelab-admin-guardrails.txt:1:STATUS=OK TS=2026-07-14T07:39:53Z TYPE=homelab-admin-machine-guardrails BAD=0 RUNNER_VERSION=0.2.0-guardrails LINTER_HOTFIX=2.1 RULES=13 FAIL_FAST=YES ERR_TRAP_REQUIRED=YES UNGUARDED_GREP_COUNT_BLOCKED=YES RUNNER_SHA256=bc76e629e434cc6406ae2f3e7886034a0cedbca43795b6f4197a1ee517c1f322 LINTER_SHA256=1d0cdc2c31adfb7446d9f253539b4004621c6d29a5ea6d1c99bbd9aed34957a2 POLICY_SHA256=ce6d46a01dfe14adc8b2a35a683f2771e4324b04fea611dd4f4bd0458b09d1af DESIRED_STATE_HEAD=69be75caeecaa242567d2d06c735ac5db3a9c048 EVIDENCE_ROOT=/var/lib/homelab-change-sets/homelab-admin-guardrails-v21-install/20260714T073953Z
|
|
/var/lib/homelab-health/cluster-admin-stage4i-scope-definition.txt:12:TASK_MODEL=VERSIONED_SCHEMA_V2_FAIL_FAST
|
|
/var/lib/homelab-health/cluster-admin-stage4h-final-acceptance.txt:15:ERROR_REGISTER_SHA256=ab8308549db90d235b591af0581580c3c7ba1b287b129cc6111c2eef0a0f9ecc
|
|
/var/lib/homelab-health/cluster-admin-stage4h-final-acceptance.txt:35:VM180_FAILED_UNITS=0
|
|
/var/lib/homelab-health/cluster-admin-stage4h-final-acceptance.txt:36:PVE01_FAILED_UNITS=0
|
|
/var/lib/homelab-health/ideal-cluster-final-seal.txt:1:STATUS=OK TS=2026-07-08T19:49:09Z TYPE=ideal-cluster-final-seal BAD=0 MODE=FINAL_GREEN_SEAL TOP_GATES_OK=YES NODE_LOSS_OK=YES NOISE_SEAL_OK=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 TEST_VMS_CLEANED=YES PROD_VMS_RUNNING=YES DNS_PAIR_OK=YES RESTORE_CHECK_OK=YES DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/final-seal-latest.md
|
|
/var/lib/homelab-health/vm160-remote-backup.txt:1:STATUS=FAIL
|
|
/var/lib/homelab-health/dockge-final-closure.txt:7:EDGE_STALE_REMAINING_MATCHES=0
|
|
/var/lib/homelab-health/disk-space.txt:10:ERRORS=none
|
|
/var/lib/homelab-health/remote-failed-unit-cleanup-seal.txt:1:STATUS=OK TS=2026-07-08T05:30:11Z TYPE=remote-failed-unit-cleanup-seal BAD=0 WARN=0 MODE=PAPERLESS_CORE_GUARD_SUPERSEDED_FINAL_SEAL PRECHECK=YES EDGE_APPLY=1 REMOTE_CLEANUP_OK=YES LOCAL_FAILED=0 PVE02_FAILED=0 PVE03_FAILED=0 EDGE_FAILED=0 EDGE_GUARD_STATE=inactive EDGE_MARKER_STATUS=OK DUTY_V2_STATUS=OK DUTY_V2_GATED_NONOK=0 DUTY_V2_UNGATED_ERROR_REVIEW=0 FINAL_STATUS=OK FINAL_FAIL=0 CLUSTER_STATUS=OK CLUSTER_BAD=0 OVERALL_STATUS=OK OVERALL_BAD=0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-remote-failed-unit-cleanup/seal-latest.md
|
|
/var/lib/homelab-health/special-vm-emergency-non-app-final-seal.txt:1:STATUS=OK TS=2026-07-14T01:15:48Z TYPE=special-vm-emergency-non-app-final-seal BAD=0 WARN=0 FAMILY_COVERAGE=11/11 MATRIX_STATUS=OK JOBS_TOTAL=40 JOBS_OK=36 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 COMPATIBILITY_OUTPUTS=31 REQUIRED_GREEN_JOBS=14 VM_FULL=OK VM_LOCAL_DUMPS=OK EDGE_VM_SOURCE=OK EDGE_VM_CLOUD=OK NEXTCLOUD_VM_SOURCE=homelab_vm_local_dumps_2cloud NEXTCLOUD_VM_CLOUD=OK EMERGENCY_PACK=OK SKLADCHIK_REPORTS_MONITOR=OK GOLDEN_STATE=OK ROUTER_RUNNING_CONFIG=OK MAIL_CLOUD_CRITICAL=OK MAIL_CLOUD_EDGE=OK MAIL_CLOUD_NEXTCLOUD=OK WRITE_MODE=legacy READ_MODE=dual MIGRATE_EXISTING=no SOURCE_BACKUP_RUN=NO CLOUD_OPERATIONS_RUN=NO RETENTION_RUN=NO CLOUD_PURGE_RUN=NO CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 CLUSTER_PASSPORT=OK DUTY_ADMIN=OK OVERALL=OK OVERALL_BAD=0 FINAL_READINESS=OK FINAL_PASS=53 FINAL_FAIL=0 WEBPANEL=OK PUBLIC_OVERALL=OK FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DESIRED_STATE_PRE_SEAL_HEAD=b26f498b55de5d80bcbb4b9fb3a9bccded9f99b4 CURRENT_SPECIAL_VM_EMERGENCY_NON_APP_STAGE_COMPLETE=YES EVIDENCE_ROOT=/var/lib/homelab-change-sets/special-vm-emergency-non-app-final-seal/20260714T011514Z
|
|
/var/lib/homelab-health/independent-external-runner.txt:1:STATUS=OK TS=2026-07-08T09:17:06Z TYPE=independent-external-runner BAD=0 WARN=0 MODE=NO_DEPLOY_SEAL NO_DEPLOY_SEAL_OK=YES AUDIT_STATUS=OK AUDIT_OK=YES INDEPENDENT_RUNNER_READY=NO DEPLOY_DECISION=NO_DEPLOY CURRENT_INDEPENDENT_RUNNER_AVAILABLE=no TRUSTED_NETBIRD_IDENTITY=edge-vm PUBLIC_VPS_DECISION=REJECTED_HOSTKEY_MISMATCH PUBLIC_VPS_SSH_OK=0 PUBLIC_VPS_REJECTED_HOSTKEY=YES EDGE_LAN_SSH_OK=2 EDGE_NETBIRD_SSH_OK=2 EDGE_DNS_SSH_OK=1 PUBLIC_OVERALL_OK=1 LOCAL_FAILED=0 PVE02_FAILED=0 PVE03_FAILED=0 EDGE_FAILED=0 DUTY_V2_STATUS=OK DUTY_V2_GATED_NONOK=0 DUTY_V2_UNGATED_ERROR_REVIEW=0 DUTY_V2_UNGATED_WARN_REVIEW=0 DUTY_V2_UNGATED_REVIEW_ITEM=0 DUTY_V2_LEGACY_OR_PHASE_REVIEW=0 OVERALL_STATUS=OK OVERALL_BAD=0 FINAL_STATUS=OK FINAL_FAIL=0 CLOSEOUT_OK=YES RECOVERY_SEAL_OK=YES RESTORE_CHECK_OK=YES GOLDEN_MATCH=YES LATEST_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260708T090905Z GOLDEN_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260708T090905Z DIRTY_COUNT=0 REPORT=/var/lib/homelab-independent-external-runner/no-deploy-seal-latest.md
|
|
/var/lib/homelab-health/node-exporter-restore.txt:1:STATUS=OK TS=2026-07-09T08:13:40Z TYPE=node-exporter-restore JOB=homelab-pve01-app-node-exporter-20260709T081340Z RESTORE=/tmp/node-exporter-restore-20260709T081340Z RESTORE_CHECK=PACKAGE_OK PVE01_TEXTFILE_SCRAPE_ERROR_AFTER=0 NOTE=service-readiness-restore-proof
|
|
/var/lib/homelab-health/cluster-admin-stage2-apply.txt:1:STATUS=OK TS=2026-07-13T01:38:37Z TYPE=cluster-admin-stage2-apply BAD=0 RESULT=APPLIED PRODUCTION_CHANGED=CANONICAL_SYNC_FUNCTION_COLLECTOR_MIGRATION_ARTIFACTS_AND_DESIRED_STATE PRODUCTION_SCHEMA_CHANGED=PLANNED_FUNCTION_AND_MIGRATION_REGISTRY_ONLY INCIDENT_ENGINE_RESTARTED=NO COLLECTOR_ONESHOT_RUN=YES COLLECTOR_TIMER_PAUSED_AND_RESUMED=YES CLOUD_WRITE_RUN=NO MIGRATION_ID=002_canonical_foundation_sync FUNCTION=cluster_admin_sync_foundation PREFLIGHT_NATURAL_KEY_DRIFT_ALLOWED=YES PREFLIGHT_NATURAL_KEY_MISSING_BEFORE=1 PREFLIGHT_NATURAL_KEY_STALE_BEFORE=1 PREFLIGHT_NATURAL_KEY_MISSING_AFTER=0 PREFLIGHT_NATURAL_KEY_STALE_AFTER=0 PREFLIGHT_EPHEMERAL_ID_DRIFT_ALLOWED=YES PREFLIGHT_EPHEMERAL_ID_DRIFT_BEFORE=25 PREFLIGHT_EPHEMERAL_ID_DRIFT_AFTER=0 PREFLIGHT_TRANSACTION_ROLLBACK=OK COMMITTED_SYNC=YES PARTIAL_ROLLBACK_AFTER_COMMIT=NOT_EXECUTED APPLIED_STATE_PRESERVED_POLICY=ACTIVE SAME_TRANSACTION=YES FIXED_TOTAL_ENTITIES_REQUIRED=NO FIXED_TOTAL_ALIASES_REQUIRED=NO FOUNDATION_COUNTS=85|21|2|229|26|0 DYNAMIC_INVARIANTS=0|0|21|21|0|0|0|0|0|0|0 ACTIVE_CURRENT_SET_CONVERGENCE=OK NATURAL_KEY_CONVERGENCE=OK EPHEMERAL_ID_REFRESH=OK RETIRED_PLACEHOLDERS_ALLOWED=YES ORPHANS=0 LIVE_COLLECTOR_RUN_ID=2992 LIVE_COLLECTOR_STATUS=OK SOURCE_INSTANCE_UPDATE=OK OBSERVER=OK FULL_OBSERVER=OK RESTRICTED_PROBES=OK FINAL_READINESS=OK CLUSTER_PASSPORT=OK OVERALL=OK DUTY_ADMIN=OK API_HEALTH=OK VM180_FAILED_UNITS=0 PVE01_FAILED_UNITS=0 ROLLBACK_POINT=VERIFIED LIVE_MATCH_DESIRED_STATE=YES OLD_HEAD=4d0d03395d2baf4024ad7c9d5870f21bdaee9f99 NEW_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a READY_FOR_POSTAPPLY_SEAL=YES STAGE2_CLOSED=NO CHANGE_SET=/var/lib/homelab-change-sets/stage2-production-apply/20260713T013837Z
|
|
/var/lib/homelab-health/cluster-admin-node-online-green-dot-fix.txt:1:STATUS=OK TS=2026-07-14T01:47:22Z TYPE=cluster-admin-node-online-green-dot-fix BAD=0 WARN=0 ROOT_CAUSE=STATUS_KIND_MISSING_ONLINE_AND_OFFLINE PRE_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 POST_SHA256=a068c1824943af182acbaebcd3ff89e84203a3fde17ccb8051a4680e37d9f5e5 NODE_STATUS=online NODE_COUNT=3 ONLINE_CLASSIFICATION=ok ONLINE_BADGE_CLASS=state-ok OFFLINE_CLASSIFICATION=bad OFFLINE_BADGE_CLASS=state-bad DATABASE_CHANGED=NO COLLECTOR_CHANGED=NO SERVICE_RESULT=success EXEC_MAIN_STATUS=0 API_HEALTH=OK BACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T014645Z BACKUP_STATUS=success BACKUP_TWO_CLOUDS=OK RESTORE_CHECK=OK_BOTH_CLOUDS REMOTE_ROLLBACK=/var/lib/cluster-admin-incident-engine/change-sets/node-online-green-dot-20260714T014640Z/app.py.before EVIDENCE_ROOT=/var/lib/homelab-change-sets/cluster-admin-node-online-green-dot-fix/20260714T014640Z
|
|
/var/lib/homelab-health/crypto-portfolio-ui-preapply-backup.txt:1:STATUS=OK TS=2026-07-13T07:42:53Z TYPE=crypto-portfolio-ui-preapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PREVIOUS_FAILURE=CORRECTED PREVIOUS_FAILURE_ROOT_CAUSE=INLINE_PYTHON_QUOTE_ERROR_IN_NONESSENTIAL_HELPER_PROOF BACKUP_SERVICE_RERUN=NO MANUAL_REHYDRATE_RERUN=NO CANONICAL_RESTORE_CHECK_RERUN=YES_READ_ONLY PRODUCTION_CHANGED=NO PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CONTAINER_RECREATED=NO CLOUD_WRITE_RUN=NO_READ_ONLY_RESTORE_CHECK TARGET_HOST=core-apps.gram1.ru INPUT_INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e CANDIDATE_INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T071127Z BACKUP_STATUS=success BACKUP_SCHEMA=homelab-app-backup-manifest-v2-flat BACKUP_LAYOUT=flat-v1 BACKUP_PATH_REL=hb/a/cp-260713-071127 BACKUP_DATABASE_INTEGRITY=ok BACKUP_FILE_COUNT=19 BACKUP_ARCHIVE_SIZE_BYTES=136632 BACKUP_ARCHIVE_SHA256=b29eb55820ec46c474f51e995e92e371251b6df717ac98182d807d1dbf8473f5 BACKUP_LOCAL_STAGING_DELETED=YES CANONICAL_TWO_CLOUD_RESTORE_CHECK=OK APPBACKUPCTL_RESTORE_CHECK=OK_BOTH_CLOUDS ROLLBACK_PATH=/var/lib/homelab-incidents/crypto-ui-center-alert-currency-prechange-20260713T071125Z ROLLBACK_MANIFEST_SHA256=0ef56c8f021f549777131ffe878cd5a263bdfb6f1dfb8bb080cefdd554573aca ROLLBACK_INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e PRODUCTION_UNCHANGED=YES PRODUCTION_HEALTH=200 VM170_FAILED_UNITS=0 PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES CHART_HEALTH_TIMER=ACTIVE BACKUP_TIMER=ACTIVE DESIRED_STATE_HEAD=80763a6c62b94cde4f3293f3b529980f76919d25 DESIRED_STATE_CLEAN=YES DESIRED_STATE_REMOTE_COUNT=0 READY_FOR_UI_PRODUCTION_APPLY=YES PREAPPLY_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-ui-preapply-backup-canonical-close/20260713T074253Z
|
|
/var/lib/homelab-health/router-hard-power.txt:1:STATUS=ERROR TS=2026-07-16T06:20:34Z MODE=cycle REMOTE=[PRIVATE_IP] RC=74
|
|
/var/lib/homelab-health/operator-handoff-pack.txt:1:STATUS=OK TS=2026-07-08T09:42:39Z TYPE=operator-handoff-pack BAD=0 WARN=0 MODE=FINAL_OPERATOR_HANDOFF_PACK HANDOFF_OK=YES PACK_OK=YES LOCAL_FAILED=0 PVE02_FAILED=0 PVE03_FAILED=0 EDGE_FAILED=0 DUTY_V2_STATUS=OK DUTY_V2_GATED_NONOK=0 DUTY_V2_UNGATED_ERROR_REVIEW=0 DUTY_V2_UNGATED_WARN_REVIEW=0 DUTY_V2_UNGATED_REVIEW_ITEM=0 DUTY_V2_LEGACY_OR_PHASE_REVIEW=0 OVERALL_STATUS=OK OVERALL_BAD=0 RESTORE_CHECK_OK=YES GOLDEN_MATCH=YES LATEST_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260708T093758Z GOLDEN_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260708T093758Z RECOVERY_CARD=YES FREEZE_PACK=YES FINAL_CLOSEOUT=YES INDEPENDENT_RUNNER_NO_DEPLOY=YES DESIRED_STATE_REMOTE_GIT=YES GIT_REMOTE_CHANGED=NO GIT_PUSH_DONE=NO DIRTY_COUNT=0 PACK=/var/lib/homelab-operator-handoff/operator-handoff-pack-20260708T094239Z.tar.gz README=/var/lib/homelab-operator-handoff/HANDOFF-latest.md STATUS_TSV=/var/lib/homelab-operator-handoff/handoff-status-latest.tsv NEXT=/var/lib/homelab-operator-handoff/NEXT-ACTIONS-latest.md
|
|
/var/lib/homelab-health/pve01-failed-units.txt:1:STATUS=OK TS=20260624T180727Z TYPE=pve01-failed-units FAILED_COUNT=0 ACTION=RESET_FAILED_ONLY SECRET_PRINTED=REDACTED
|
|
/var/lib/homelab-health/final-global-green-audit.txt:1:STATUS=OK TS=2026-07-08T07:20:01Z TYPE=final-global-green-audit BAD=0 WARN=0 MODE=SELF_REFERENCE_CLOSED_FINAL_GLOBAL_GREEN_SEAL AUDIT_OK=YES PRE_STATUS=WARN SELF_REFERENCE_CLOSED=yes LOCAL_FAILED=0 PVE02_FAILED=0 PVE03_FAILED=0 EDGE_FAILED=0 DUTY_V2_STATUS=OK DUTY_V2_GATED_NONOK=0 DUTY_V2_UNGATED_ERROR_REVIEW=0 DUTY_V2_UNGATED_WARN_REVIEW=0 DUTY_V2_UNGATED_REVIEW_ITEM=0 DUTY_V2_LEGACY_OR_PHASE_REVIEW=0 FINAL_STATUS=OK FINAL_FAIL=0 CLUSTER_STATUS=OK CLUSTER_BAD=0 OVERALL_STATUS=OK OVERALL_BAD=0 EXACT_WARN_REVIEW_FILES=0 LOCAL_OVERALL_OK=1 LAN_OVERALL_OK=1 PUBLIC_OVERALL_OK=1 EXTERNAL_PROBE_STATUS=OK EXTERNAL_PUBLIC_SMOKE_BAD=0 RESTORE_CHECK_OK=YES GOLDEN_MATCH=YES LATEST_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260708T064444Z GOLDEN_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260708T064444Z DIRTY_COUNT=0 REPORT=/var/lib/homelab-final-global-green-audit/final-seal-latest.md
|
|
/var/lib/homelab-health/remote-failed-unit-audit.txt:1:STATUS=OK TS=2026-07-08T05:20:16Z TYPE=remote-failed-unit-audit BAD=0 WARN=1 MODE=READ_ONLY_REMOTE_FAILED_UNIT_AUDIT LOCAL_FAILED_COUNT=1 REMOTE_FAILED_COUNT=8 CONNECT_ERRORS=0 JOURNAL_FAIL_HINTS=194 OVERALL_BAD=0 FINAL_FAIL=0 DUTY_V2_GATED_NONOK=0 RESET_FAILED_DONE=no REPORT=/var/lib/homelab-remote-failed-unit-cleanup/audit-latest.md
|
|
/var/lib/homelab-health/legacy-monitoring-backups-migrated.txt:1:STATUS=OK TS=2026-07-10T20:00:03Z TYPE=legacy-monitoring-backups-migrated BAD=0 APPS=6 RESTORE_FAILURES=0 ACTIVE_LEGACY_STATES=0 OFFHOST=OK RESTORE=OK
|
|
/var/lib/homelab-health/cluster-admin-stage4h-collector-patch-candidate.txt:1:STATUS=OK TS=2026-07-14T17:41:32Z TYPE=cluster-admin-stage4h-collector-patch-candidate FAILED_ATTEMPT_TAIL=CLOSED CANDIDATE_SUBSTEP=CLOSED STAGE4H_STATUS=OPEN PRODUCTION_CHANGE=NO UNRESOLVED_TAIL_COUNT=0 BLOCKER_COUNT=0 NEXT_ACTION=TEMPORARY_DATABASE_MIGRATION_AND_COLLECTOR_ACCEPTANCE_HARNESS CANDIDATE_SHA256=4204fe568d46fec84f643af974c82770cb5d5b08d3d8ceb4aad96cca57abc909 REPORT_SHA256=ecfdaab5438e1fc3bc7a90e08ec68c71d06a6c6a91a430dc1ea7b6a74b7699a9
|
|
/var/lib/homelab-health/cluster-admin-stage2-final-seal.txt:1:STATUS=OK TS=2026-07-13T01:45:29Z TYPE=cluster-admin-stage2-final-seal BAD=0 STAGE1_CLOSED=YES STAGE2_APPLY=OK PRODUCTION_CHANGED=BACKUP_HEALTH_AND_SEAL_ARTIFACTS_ONLY PRODUCTION_SCHEMA_CHANGED=NO_ADDITIONAL_CHANGE_DURING_SEAL INCIDENT_ENGINE_RESTARTED=NO MIGRATION_ID=002_canonical_foundation_sync FUNCTION=cluster_admin_sync_foundation COMMITTED_SYNC=YES MULTICYCLE_COLLECTOR_TEST=OK MULTICYCLE_NEW_RUNS=2 MULTICYCLE_GOOD_RUNS=2 ACTIVE_CURRENT_SET_CONVERGENCE=OK NATURAL_KEY_CONVERGENCE=OK EPHEMERAL_ID_REFRESH=OK RETIRED_PLACEHOLDERS_ALLOWED=YES FOUNDATION_COUNTS=85|21|2|229|26|0 DYNAMIC_INVARIANTS=0|0|21|21|0|0|0|0|0|0|0 ORPHANS=0 POSTAPPLY_DIRECT_SNAPSHOT=VERIFIED POSTAPPLY_RESTORE_TEST=OK POSTAPPLY_APPBACKUP=VERIFIED POSTAPPLY_APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T014757Z LIVE_MATCH_DESIRED_STATE=YES DESIRED_STATE_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a OBSERVER=OK FULL_OBSERVER=OK RESTRICTED_PROBES=OK FINAL_READINESS=OK CLUSTER_PASSPORT=OK OVERALL=OK DUTY_ADMIN=OK VM180_FAILED_UNITS=0 PVE01_FAILED_UNITS=0 CURRENT_HEALTH_RECHECKED=YES STAGE2_CLOSED=YES REPORT=/var/lib/homelab-cluster-admin-stage2/stage2-final-seal-latest.txt REPORT_SHA256=32d4d37d82aa25194d789083d0e7ad2e9052ee38a72df5312b711226898cf1a6 BLOCKERS=/var/lib/homelab-cluster-admin-stage2/20260713T014529Z/blockers.txt
|
|
/var/lib/homelab-health/remote-failed-unit-plan.txt:1:STATUS=OK TS=2026-07-08T05:22:47Z TYPE=remote-failed-unit-plan BAD=0 WARN=1 MODE=READ_ONLY_CLEANUP_PLAN LOCAL_FAILED_COUNT=1 REMOTE_FAILED_COUNT=7 LOCAL_NOOP_COUNT=4 REMOTE_RESET_ONLY_COUNT=1 REMOTE_NOOP_COUNT=5 REMOTE_DEFER_COUNT=1 OVERALL_BAD=0 FINAL_FAIL=0 RESET_FAILED_DONE=no PLAN=/var/lib/homelab-remote-failed-unit-cleanup/cleanup-plan-latest.tsv REPORT=/var/lib/homelab-remote-failed-unit-cleanup/plan-latest.md
|
|
/var/lib/homelab-health/review-surface-registry-apply.txt:1:STATUS=OK TS=2026-07-08T06:36:39Z TYPE=review-surface-registry-apply BAD=0 WARN=0 MODE=POLICY_AWARE_DUTY_ADMIN_REPAIR_SELF_CLOSED PRECHECK=YES APPLY_OK=YES REPAIR_OK=YES UPDATED=27 ARCHIVED=27 DUTY_V2_STATUS=OK DUTY_V2_GATED_NONOK=0 DUTY_V2_UNGATED_ERROR_REVIEW=0 DUTY_V2_UNGATED_WARN_REVIEW_WAS=1 DUTY_V2_UNGATED_REVIEW_ITEM=0 DUTY_V2_LEGACY_OR_PHASE_REVIEW=0 OVERALL_BAD=0 SELF_REFERENCE_CLOSED=YES REPORT=/var/lib/homelab-review-surface-registry/seal-latest.md
|
|
/var/lib/homelab-health/private-vpn-hosts.txt.bak-ok-wrapper-20260624T132705Z:16:STATUS=ERROR
|
|
/var/lib/homelab-health/cluster-admin-strict-final-seal.txt:1:STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md
|
|
/var/lib/homelab-health/paperless-restore.txt:1:STATUS=STALE
|
|
/var/lib/homelab-health/paperless-restore.txt:20:FRESHNESS=STALE
|
|
/var/lib/homelab-health/ideal-cluster-noise-seal.txt:1:STATUS=OK TS=2026-07-08T17:41:49Z TYPE=ideal-cluster-noise-seal BAD=0 MODE=POLICY_AWARE_NON_STATUS_ARTIFACT_SEAL_SELF_REFERENCE_CLOSED SELF_REFERENCE_CLOSED=YES NON_STATUS_TOTAL=8 NON_STATUS_OK_INFO=8 NON_STATUS_UNKNOWN=0 WARNBAD_ACCEPTED_POLICY_INFO=13 WARNBAD_NEEDS_REVIEW=0 TOP_GATES_OK=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/noise-seal-latest.md
|
|
/var/lib/homelab-health/crypto-portfolio-chart-health.txt:1:STATUS=OK TS=2026-07-21T12:03:07Z TYPE=crypto-portfolio-chart-health BAD=0 LIBRARY_OK=true DISK_CACHE_PRESENT=true SYMBOLS=2 STALE_COUNT=2
|
|
/var/lib/homelab-health/duty-admin-v2.txt:1:STATUS=OK TS=2026-07-21T12:01:18Z TYPE=duty-admin-v2 BAD=7 WARN=0 MODE=LOCAL_LONG_RUNNING_ANALYZER_POLICY_AWARE SEPARATE_VM=NO TOTAL_HEALTH_INPUTS=497 GATED_OK=45 GATED_NONOK=7 UNGATED_ERROR_REVIEW=2 UNGATED_WARN_REVIEW=3 UNGATED_REVIEW_ITEM=0 LEGACY_OR_PHASE_REVIEW=0 REPORT=/var/lib/homelab-duty-admin-v2/latest.md
|
|
/var/lib/homelab-health/cluster-admin-stage3-final-seal.txt:1:STATUS=OK TS=2026-07-13T03:46:30Z TYPE=cluster-admin-stage3-final-seal BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=HEALTH_REFRESH_AND_SEAL_ARTIFACTS_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO STAGE2_CLOSED=YES STAGE3_APPLY=OK AUTHENTICATED_PRODUCTION_VERIFY=OK AUTHENTICATED_PRODUCTION_TESTS=25 APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 LIVE_MATCH_DESIRED_STATE=YES DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 NEW_GET_ROUTES=7 DATABASE_TRANSACTION_MODE=BEGIN_READ_ONLY OPENAPI_DOCS_EXPOSURE=PRESERVED_DISABLED POSTAPPLY_APPBACKUP=VERIFIED POSTAPPLY_APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T034400Z POSTAPPLY_APPBACKUP_RESTORE_CHECK=OK COLLECTOR_STABILITY=OK LATEST_COLLECTOR_RUN_ID=3114 CANONICAL_CONSISTENCY=0|0|0|0|0 OBSERVER=OK FULL_OBSERVER=OK RESTRICTED_PROBES=OK FINAL_READINESS=OK CLUSTER_PASSPORT=OK OVERALL=OK DUTY_ADMIN=OK VM180_FAILED_UNITS=0 PVE01_FAILED_UNITS=0 CURRENT_HEALTH_RECHECKED=YES STAGE3_CLOSED=YES REPORT=/var/lib/homelab-cluster-admin-stage3/stage3-final-seal-latest.txt REPORT_SHA256=6f67545cd7f0a75b87146c1df74c61c3b88e142f9e53a96b083ec27209e6d265
|
|
/var/lib/homelab-health/hapusya-site-production-browser.txt:1:STATUS=OK TS=2026-07-13T11:48:15Z TYPE=hapusya-site-production-browser BAD=0 URL=https://hapusya.ru/ BROWSER_RUNS=5 BROWSER_ERRORS=0 DESKTOP_VIDEO=OK MOBILE_VIDEO=OK STATIC_FALLBACK=OK EVIDENCE_ROOT=/var/lib/homelab-change-sets/hapusya-site-production-browser-v2/20260713T114810Z MANIFEST_SHA256=9f3cdf3063f8724d1d1c1ad619f47d6f98d434c3422a74903585146858a8dfac ROUTE_SHA256=02d65cb0082a04cae282126cc829a002f171e863291c18ba04d788dc2d6282ca UNIT_SHA256=b3b07137b25ac0a0b4bdadadad654cf209387ade86a2ef2cdca58e8a7c0cc212 RELATIVE_TREE_SHA256=9dadf21da041bc4d6ee4a42f5b3ac7719c16bcc39052de32af895c6cd28286cb PRODUCTION_CHANGED=YES CLOSURE_PENDING=POSTCHANGE_BACKUP_RETENTION_DESIRED_STATE_FINAL_SEAL
|
|
/var/lib/homelab-health/backup-matrix-authoritative.txt:1:STATUS=FAIL
|
|
/var/lib/homelab-health/backup-matrix-desired-state-persistence.txt:1:STATUS=OK TS=2026-07-13T17:40:37Z TYPE=backup-matrix-desired-state-persistence BAD=0 GENERATED_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 GENERATED_PARENT=25568fab24c0594418b3676dc511c8234b4e1ae5 GENERATOR_SHA256=19da7f049b4f7f7286faf292c5419c37cc1e1768c2d8fe36b257ab0cf9c18874 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 DESIRED_MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATUS=OK JOBS_BAD=0 FAILED_UNITS=0 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO ROLLBACK=/var/lib/homelab-incidents/backup-matrix-desired-state-persistence-20260713T174036Z
|
|
/var/lib/homelab-health/duty-admin-residual-audit.txt:1:STATUS=OK TS=2026-07-08T05:40:37Z TYPE=duty-admin-residual-audit BAD=0 WARN=3 MODE=READ_ONLY_RESIDUAL_NOISE_AUDIT LOCAL_FAILED=0 PVE02_FAILED=0 PVE03_FAILED=0 EDGE_FAILED=0 DUTY_V2_STATUS=OK DUTY_V2_GATED_NONOK=0 DUTY_V2_UNGATED_ERROR_REVIEW=0 DUTY_V2_UNGATED_WARN_REVIEW=1 DUTY_V2_UNGATED_REVIEW_ITEM=22 DUTY_V2_LEGACY_OR_PHASE_REVIEW=5 OVERALL_STATUS=OK OVERALL_BAD=0 FINAL_FAIL=0 WARN_REVIEW_FILES=19 HARD_GATE_WARN_MATCHES=0 REPORT=/var/lib/homelab-duty-admin-residual-cleanup/audit-latest.md
|
|
/var/lib/homelab-health/ungated-health-backlog.txt:1:STATUS=OK TS=2026-07-07T07:45:01Z TYPE=ungated-health-backlog BAD=0 WARN=1 MODE=POST_PVE03_ROOT_CLEANUP_TRIAGE TOTAL=8 SUPERSEDED=4 STALE=2 POLICY_GAP_NEXT_STAGE=1 ACTIONABLE_CAPACITY=1 PVE03_ROOT_USAGE_PERCENT=31 PVE03_STAGING_USAGE_PERCENT=83 REPORT=/var/lib/homelab-ungated-health-backlog/latest.md
|
|
/var/lib/homelab-health/crypto-portfolio-ui-center-alert-currency-final-seal.txt:1:STATUS=OK TS=2026-07-13T08:37:54Z TYPE=crypto-portfolio-ui-center-alert-currency-final-seal BAD=0 DANGLING_IMAGE_REFERENCE_GATE=OK DESIRED_STATE_SYNC_ARTIFACT_GATE=OK APP=crypto-portfolio PRODUCTION_CHANGED=NO_FINAL_SEAL_ONLY APP_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb LIBRARY_SHA256=c0992580867c4912cc9385b3c2728315bcc1a76c7f1087dca908430fccdf31d7 ACTIVE_COINS=HBAR,POL GRT_REMOVED=YES GRT_ROWS=0|0|0 GRT_SOURCE_REFERENCES=0 GRT_CACHE_ENTRIES=0 SHARP_MOVES_CENTERED=YES ALERT_CURRENCY_USD_RUB=OK BROWSER_SMOKE=OK BROWSER_RUNS=2 BROWSER_ERRORS=0 BROWSER_RESULT_SHA256=b566f492cd80c8f99bc6b24cbef6d5f326c04f4d7168359d05eb1f189589e733 SCREENSHOT_SHA256=00b2853bee9c880146d0dd75fac60feb7e46d4c129ee06754047ef30d90bd4ba BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z TWO_CLOUDS=OK RESTORE_CHECK=OK DATABASE_INTEGRITY=ok RETENTION=OK ROLLBACK_COUNT=3 DANGLING_IMAGES=2 REFERENCED_DANGLING_IMAGES=2 UNREFERENCED_DANGLING_IMAGES=0 PRUNED_UNREFERENCED_DANGLING_IMAGES=0 CHART_HEALTH=OK PUBLIC=OK MONITOR_TIMER=active BACKUP_TIMER=active CLOSURE_DESIRED_STATE_HEAD=2999910549eaff9b9431b9c656d318340eb95cbd DESIRED_STATE=OK PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES ALL_CRYPTO_PORTFOLIO_UI_TAILS_CLOSED=YES
|
|
/var/lib/homelab-health/crypto-portfolio-remove-grt-runtime-verifier-prepared.txt:1:STATUS=OK TS=2026-07-13T04:35:29Z TYPE=crypto-portfolio-remove-grt-runtime-verifier-prepared BAD=0 COMMAND_PHYSICAL_LINES=1 VERIFIER_REVISION=V3 DEEP_ANALYSIS_RESULT=DIRECT_ROUTE_JSONRESPONSE_CONTRACT_CORRECTED FAILURE_CLASS=VERIFIER_ONLY PRODUCTION_CHANGED=NO PRODUCTION_DATABASE_CHANGED=NO CANDIDATE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CONTAINER_STARTED=NO CLOUD_WRITE_RUN=NO TARGET_HOST=core-apps.gram1.ru DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 INPUT_APP_SHA256=684ad884119554503db0ed36252aeb017c0c04b54729f30c51e66a7ab293f8bd INPUT_WORKER_SHA256=40a1c81111d7097120125b140683a8604abcb4051db5073ab8af708a11854038 INPUT_INDEX_SHA256=2b1ab459cabf3050af1db4a7b293fff7c2a9ac18bf4a14faa1efc70a3a3f63e3 CANDIDATE_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-candidate-v3/20260713T042046Z RUNTIME_POLICY=CONTAINER_PYTHON_ONLY PYTHONPATH_APP_INJECTED=YES ROUTE_RETURN_DECODER=DICT_OR_RESPONSE_BODY STATE_RETURN_DECODED=YES CHART_HEALTH_RETURN_DECODED=YES FETCH_KLINES_RETURN_DECODED=YES PROTECTED_KLINES_CODES=401_OR_403 LIVE_PRODUCTION_DATA_MOUNTED=NO ISOLATED_PORT=18030 FAILED_RUN_CLEANUP=VERIFIED REMOTE_RUNTIME_SCRIPT_SHA256=b60290b08f6bdb9801f4ccadbeadc4268399426996fd271527830fd399055448 VERIFIER_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-runtime-verifier-v3/20260713T043529Z READY_FOR_GRT_ISOLATED_RUNTIME_DRY_RUN_RUN=YES
|
|
/var/lib/homelab-health/crypto-portfolio-ui-center-alert-currency-candidate.txt:1:STATUS=OK TS=2026-07-13T06:11:23Z TYPE=crypto-portfolio-ui-center-alert-currency-candidate BAD=0 COMMAND_PHYSICAL_LINES=1 PREVIOUS_FAILURE=CORRECTED PREVIOUS_FAILURE_ROOT_CAUSE=EXPECTED_DIFF_RC_1_TRIGGERED_SET_E_PIPEFAIL PRODUCTION_CHANGED=NO PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO TARGET_HOST=core-apps.gram1.ru DESIRED_STATE_HEAD=80763a6c62b94cde4f3293f3b529980f76919d25 INPUT_INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e CANDIDATE_INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb BUILDER_DOUBLE_CHECK=OK CANDIDATE_DOUBLE_CHECK=OK SHARP_MOVES_POSITION=CENTERED SHARP_MOVES_MAX_WIDTH_PX=860 SHARP_MOVES_COLUMNS_DESKTOP=2 SHARP_MOVES_COLUMNS_MOBILE=1 ALERT_RENDERER=V5 ALERT_CURRENCY_SOURCE=TOP_USD_RUB_SWITCH ALERT_EXCHANGE_RATE_SOURCE=STATE_MARKET_USD_RUB ALERT_PRICE_THRESHOLD_CONVERTED=YES ALERT_CAP_THRESHOLD_CONVERTED=YES ALERT_SALE_VALUE_CONVERTED=YES CURRENCY_SWITCH_REDRAW=YES PUMP_DUMP_PERCENT_UNCHANGED=YES FIXED_USD_ALERT_FORMATTER=ABSENT RENDER_MOVES_COUNT=1 ALERT_RENDERER_MARKER_COUNT=1 GRT_REFERENCES=0 PRODUCTION_INDEX_UNCHANGED=YES PRODUCTION_HEALTH=200 READY_FOR_UI_ISOLATED_BROWSER_SMOKE=YES CANDIDATE_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-ui-center-alert-currency-candidate-v4/20260713T061123Z
|
|
/var/lib/homelab-health/cluster-admin-stage2-design.txt:1:STATUS=OK TS=2026-07-13T00:35:14Z TYPE=cluster-admin-stage2-design BAD=0 STAGE1_CLOSED=YES CAPTURE_REUSED=YES PRODUCTION_CHANGED=EVIDENCE_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO SERVICE_RESTARTED=NO FOUNDATION_STATE=9|7|1|1 EXPECTED_ENTITIES=62 MANAGED_ENTITIES=62 PLACEHOLDERS=21 MISSING_ENTITIES=0 STALE_ENTITIES=0 LEGACY_DEPENDENCIES=26 TYPED_DEPENDENCIES=26 DEPENDENCY_ID_MATCHES=0 DEPENDENCY_NATURAL_KEY_MATCHES=26 DEPENDENCY_NATURAL_KEY_MISSING=0 DEPENDENCY_NATURAL_KEY_STALE=0 EPHEMERAL_ID_MISSING=26 EPHEMERAL_ID_STALE=26 ROOT_CAUSE=LEGACY_DEPENDENCIES_DELETE_REINSERT_EACH_RUN CANONICAL_DEPENDENCY_IDENTITY=PARENT_CHILD_RELATION MIGRATION_ID=002_canonical_foundation_sync DATABASE_FUNCTION=cluster_admin_sync_foundation COLLECTOR_CALL_REQUIRED=YES SAME_TRANSACTION_REQUIRED=YES LEGACY_TABLES_PRESERVED=YES AUTOMATIC_REMEDIATION_ACTIONS=NO STAGE2_READY_FOR_DRY_RUN=YES DESIRED_STATE_HEAD=4d0d03395d2baf4024ad7c9d5870f21bdaee9f99 CHANGE_SET=/var/lib/homelab-change-sets/stage2-design/20260713T003514Z CONTRACT=/var/lib/homelab-change-sets/stage2-design/20260713T003514Z/stage2-contract.json REPORT=/var/lib/homelab-change-sets/stage2-design/20260713T003514Z/stage2-design-report.txt
|
|
/var/lib/homelab-health/final-green-freeze-seal.txt:1:STATUS=OK TS=2026-07-08T08:13:08Z TYPE=final-green-freeze-seal BAD=0 WARN=0 MODE=FINAL_FREEZE_EVIDENCE_SEAL FREEZE_SEAL_OK=YES AUDIT_STATUS=OK AUDIT_FREEZE_OK=YES LOCAL_FAILED=0 PVE02_FAILED=0 PVE03_FAILED=0 EDGE_FAILED=0 DUTY_V2_STATUS=OK DUTY_V2_GATED_NONOK=0 DUTY_V2_UNGATED_ERROR_REVIEW=0 DUTY_V2_UNGATED_WARN_REVIEW=0 DUTY_V2_UNGATED_REVIEW_ITEM=0 DUTY_V2_LEGACY_OR_PHASE_REVIEW=0 FINAL_STATUS=OK FINAL_FAIL=0 CLUSTER_STATUS=OK CLUSTER_BAD=0 OVERALL_STATUS=OK OVERALL_BAD=0 EXACT_WARN_REVIEW_FILES=0 RESTORE_CHECK_OK=YES GOLDEN_MATCH=YES LATEST_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260708T072711Z GOLDEN_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260708T072711Z DIRTY_COUNT=0 EVIDENCE=/var/lib/homelab-final-green-freeze/freeze-evidence-latest.tsv REPORT=/var/lib/homelab-final-green-freeze/final-seal-latest.md
|
|
/var/lib/homelab-health/cluster-admin-stage2-dry-run.txt:1:STATUS=OK TS=2026-07-13T00:39:40Z TYPE=cluster-admin-stage2-dry-run BAD=0 PRODUCTION_CHANGED=TEST_DATABASE_AND_CANDIDATE_ARTIFACTS_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO MIGRATION_ID=002_canonical_foundation_sync DATABASE_FUNCTION=cluster_admin_sync_foundation MIGRATION_APPLY_TEST=OK MIGRATION_IDEMPOTENCE_TEST=OK NATURAL_KEY_CONVERGENCE=OK EPHEMERAL_ID_REFRESH=OK ENTITY_ADD_SYNC=OK ENTITY_RETIRE_SYNC=OK PLACEHOLDER_ADD_SYNC=OK PLACEHOLDER_RETIRE_SYNC=OK STALE_TYPED_EDGE_DELETE=OK SOURCE_INSTANCE_UPDATE=OK COLLECTOR_CANDIDATE_SYNTAX=OK COLLECTOR_CALL_BEFORE_COMMIT=OK ROLLBACK_TEST=OK TEST_DATABASE_DROPPED=YES PRODUCTION_UNCHANGED=YES READY_FOR_PREAPPLY_BACKUP=YES DESIRED_STATE_HEAD=4d0d03395d2baf4024ad7c9d5870f21bdaee9f99 CHANGE_SET=/var/lib/homelab-change-sets/stage2-dry-run/20260713T003940Z MIGRATION=/var/lib/homelab-change-sets/stage2-dry-run/20260713T003940Z/002_canonical_foundation_sync.sql ROLLBACK=/var/lib/homelab-change-sets/stage2-dry-run/20260713T003940Z/002_canonical_foundation_sync.rollback.sql COLLECTOR=/var/lib/homelab-change-sets/stage2-dry-run/20260713T003940Z/collector.py.candidate PLAN=/var/lib/homelab-change-sets/stage2-dry-run/20260713T003940Z/stage2-migration-plan.json RESULT=/var/lib/homelab-change-sets/stage2-dry-run/20260713T003940Z/dry-run-result.txt
|
|
/var/lib/homelab-health/cluster-admin-stage3-production-verify.txt:1:STATUS=OK TS=2026-07-13T03:28:52Z TYPE=cluster-admin-stage3-production-verify BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=VERIFICATION_EVIDENCE_ONLY PRODUCTION_DATABASE_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO PREVIOUS_PERMISSION_FAILURE=CORRECTED VERIFIER_OWNER=CLUSTERADMIN VERIFIER_MODE=0700 LIVE_APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 AUTHENTICATED_PRODUCTION_TESTS=25 AUTHENTICATED_PRODUCTION_FAILED=0 LEGACY_AUTHENTICATED_SUMMARY=OK NEW_GET_ROUTES_AUTHENTICATED=OK NEW_GET_ROUTES_UNAUTHENTICATED_401=OK ENTITY_LIST=OK ENTITY_DETAIL=OK ENTITY_DEPENDENCIES=OK DEPENDENCIES=OK SOURCES=OK POLICIES=OK COLLECTOR_RUNS=OK VALIDATION_ERRORS=OK FORBIDDEN_RESPONSE_FIELDS=ABSENT OPENAPI_DOCS_EXPOSURE_CHANGED=NO SECURITY_HEADERS=OK PRODUCTION_RUNTIME_HEALTH=OK REMOTE_VERIFIER_CLEANUP=OK READY_FOR_STAGE3_POSTAPPLY_SEAL=YES STAGE3_CLOSED=NO RESULT=/var/lib/homelab-change-sets/stage3-api-production-verify-run/20260713T032852Z/production-result.json CHANGE_SET=/var/lib/homelab-change-sets/stage3-api-production-verify-run/20260713T032852Z
|
|
/var/lib/homelab-health/crypto-portfolio-ui-center-alert-currency-browser-smoke.txt:1:STATUS=OK TS=2026-07-13T07:04:45Z TYPE=crypto-portfolio-ui-center-alert-currency-browser-smoke BAD=0 COMMAND_PHYSICAL_LINES=1 PREVIOUS_FAILURE=CORRECTED PREVIOUS_FAILURE_ROOT_CAUSE=UNPRIVILEGED_REMOTE_RM_AFTER_SUDO_HARNESS PRODUCTION_CHANGED=NO_BROWSER_EVIDENCE_FINALIZATION_ONLY PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO TARGET_HOST=core-apps.gram1.ru CANDIDATE_INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb CANDIDATE_STATIC_GRT_DOUBLE_CHECK=OK BROWSER_ENGINE=CHROMIUM BROWSER_RUNS=2 BROWSER_RUNS_IDENTICAL=YES MOVES=2 SHARP_MOVES_CENTERED=YES SHARP_MOVES_MAX_WIDTH_OK=YES SHARP_MOVES_COLUMNS=2 ALERT_ROWS=2 USD_ALERT_RENDER=OK RUB_ALERT_RENDER=OK TOP_CURRENCY_SWITCH_REDRAW=OK RUB_PRICE_THRESHOLD=OK RUB_CAP_THRESHOLD=OK RUB_SALE_VALUE=OK ALERT_USD_RESIDUE_IN_RUB_MODE=ABSENT CHART_CARDS=2 CHART_CANVASES=14 CHARTS_BOTTOM=YES BROWSER_ERRORS=0 GRT_VISIBLE=NO RESULT_SHA256=b566f492cd80c8f99bc6b24cbef6d5f326c04f4d7168359d05eb1f189589e733 SCREENSHOT_SHA256=68a9a68d15673ef64c248a4fe886ac36f94923ec917daa2772fc9e7c1b5f4c16 ROOT_OWNED_STAGE_ENTRIES=11 ROOT_OWNED_STAGE_DIRECTORIES=2 REMOTE_STAGE_CLEANED=YES PRODUCTION_INDEX_UNCHANGED=YES PRODUCTION_HEALTH=200 VM170_FAILED_UNITS=0 PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES READY_FOR_UI_PREAPPLY_BACKUP_AND_PRODUCTION_APPLY=YES BROWSER_SMOKE_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-ui-center-alert-currency-browser-smoke-v2/20260713T070125Z
|
|
/var/lib/homelab-health/crypto-portfolio-remove-grt-surface.txt:1:STATUS=OK TS=2026-07-13T04:05:42Z TYPE=crypto-portfolio-remove-grt-surface BAD=0 COMMAND_PHYSICAL_LINES=1 PREVIOUS_FAILURE=CORRECTED PRODUCTION_CHANGED=NO DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO TARGET_HOST=core-apps.gram1.ru TARGET_STACK=/opt/stacks/crypto-portfolio DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 DESIRED_STATE_CLEAN=YES APP_SHA256=684ad884119554503db0ed36252aeb017c0c04b54729f30c51e66a7ab293f8bd WORKER_SHA256=40a1c81111d7097120125b140683a8604abcb4051db5073ab8af708a11854038 INDEX_SHA256=2b1ab459cabf3050af1db4a7b293fff7c2a9ac18bf4a14faa1efc70a3a3f63e3 APP_GRT_REFERENCES=5 WORKER_GRT_REFERENCES=2 INDEX_GRT_REFERENCES=2 LOCAL_GRT_REFERENCES=3 SQLITE_SCHEMA_CAPTURED=YES READY_FOR_GRT_CANDIDATE_PREPARE=YES SURFACE_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-surface/20260713T040542Z
|
|
/var/lib/homelab-health/cluster-admin-stage4d-adapter-candidates-in-isolation.txt:1:STATUS=OK TS=2026-07-14T06:14:58Z TYPE=cluster-admin-stage4d-adapter-candidates-in-isolation-final-seal BAD=0 STAGE4D_CLOSED=YES NEXT_STAGE=4E_RESTRICTED_PROBE_AGENT_DESIGN PRODUCTION_CHANGED=NO DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO APPLICATION_CHANGED=NO ENVELOPE_TOTAL=48 PROMETHEUS_ENVELOPES=5 BACKUP_MATRIX_ENVELOPES=40 HEALTH_ENVELOPES=3 LOCAL_REMOTE_OUTPUT_MATCH=OK PRODUCTION_STATE=0|0|OK DESIRED_STATE_HEAD=e77fb37a0fa0033fe46558045e31dd92b6d1238a DESIRED_STATE_CLEAN=YES PVE01_FAILED_UNITS=0 REPORT=/var/lib/homelab-cluster-admin-stage4/20260714T061458Z/stage4d-adapter-candidates-in-isolation-seal.txt REPORT_SHA256=da6244c0a8a458b508fd58a0fec9cb1e6fa8da98a36f44d1368cbe947f4b5835
|
|
/var/lib/homelab-health/backup-matrix-cluster-desired-state-sync.txt:1:STATUS=OK TS=2026-07-13T17:51:31Z TYPE=backup-matrix-cluster-desired-state-sync BAD=0 PRE_SYNC_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 GENERATED_SYNC_HEAD=597119391e7b265b85fd9a76644e4e33af47e469 GENERATED_SYNC_MODE=NEW_GENERATED_COMMIT GENERATED_SYNC_COMMIT_DELTA=1 MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_OK=31 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 CLUSTER_PASSPORT=OK FINAL_READINESS=OK OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 FAILED_UNITS=0 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=14a31112aac5d577ae01df567921b093b2fedcf7189c4cc8f009216ee0749380 PRESEAL_SHA256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-final-seal/20260713T175131Z
|
|
/var/lib/homelab-health/cluster-admin-stage2-recovery.txt:1:STATUS=OK TS=2026-07-13T01:04:59Z TYPE=cluster-admin-stage2-recovery BAD=0 RESULT=RECOVERED PRODUCTION_CHANGED=HEALTH_REGENERATION_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO INCIDENT_ENGINE_RESTARTED=NO DATABASE_ROLLBACK=OK COLLECTOR_ROLLBACK=OK CORE_DATABASE_STATE=0|0|9|7|1 LIVE_COLLECTOR_SHA256=a83abde2290d8392b1c631122671bcb680f9271d0b576fafacb8876d3c8034f1 OBSERVER=OK FULL_OBSERVER=OK RESTRICTED_PROBES=OK RESTRICTED_PROBES_AUTHORITY=CENTRAL_SYNTHESIZED_FROM_REMOTE_FULL_OBSERVER VERIFIER_BUG=REMOTE_RESTRICTED_PROBES_FILE_INCORRECTLY_REQUIRED FINAL_READINESS=OK CLUSTER_PASSPORT=OK OVERALL=OK DUTY_ADMIN=OK VM180_FAILED_UNITS=0 PVE01_FAILED_UNITS=0 DESIRED_STATE_UNCHANGED=YES ROLLBACK_POINT=VERIFIED READY_FOR_STAGE2_RETRY=YES STAGE2_CLOSED=NO CHANGE_SET=/var/lib/homelab-change-sets/stage2-recovery-final/20260713T010459Z
|
|
/var/lib/homelab-health/cluster-admin-stage3-api-runtime-dry-run.txt:1:STATUS=OK TS=2026-07-13T02:53:34Z TYPE=cluster-admin-stage3-api-runtime-dry-run BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=TEMPORARY_ISOLATED_RUNTIME_AND_EVIDENCE_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO PRODUCTION_SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO APP_PRECHANGE_SHA256=c2748b8f96df842f491b5d2790790594222549853d0e2336cf60ecd6d4de0020 APP_CANDIDATE_SHA256=672d6a2975dd0e093788e7eb81a76e7f901ea6374ca4092a25ba0091aba1cb50 ISOLATED_BIND=127.0.0.1:18081 TEMPORARY_DATABASE_RESTORED=YES TEMPORARY_DATABASE_DROPPED=YES TEMPORARY_DATABASE_UNCHANGED=YES RUNTIME_TESTS=41 RUNTIME_FAILED_TESTS=0 EXISTING_ROUTES_BACKWARD_COMPATIBLE=YES NEW_GET_ROUTES=7 UNAUTHENTICATED_NEW_ROUTES_401=YES AUTHENTICATED_NEW_ROUTES=OK PAGINATION=OK FILTER_VALIDATION=OK ENTITY_DETAIL=OK ENTITY_DEPENDENCIES=OK SOURCES=OK POLICIES=OK COLLECTOR_RUNS=OK DATABASE_TRANSACTION_MODE=BEGIN_READ_ONLY FORBIDDEN_RESPONSE_FIELDS=ABSENT OPENAPI_DOCS_EXPOSURE_CHANGED=NO SECURITY_HEADERS=OK ISOLATED_RUNTIME_STOPPED=YES ISOLATED_PORT_CLOSED=YES LIVE_APP_UNCHANGED=YES PRODUCTION_RUNTIME_HEALTH=OK READY_FOR_STAGE3_PREAPPLY_BACKUP=YES DESIRED_STATE_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a CHANGE_SET=/var/lib/homelab-change-sets/stage3-api-runtime-dry-run/20260713T025334Z RESULT=/var/lib/homelab-change-sets/stage3-api-runtime-dry-run/20260713T025334Z/runtime-result.json
|
|
/var/lib/homelab-health/crypto-portfolio-remove-grt-failed-units-diagnostic.txt:1:STATUS=OK TS=2026-07-13T05:41:00Z TYPE=crypto-portfolio-remove-grt-failed-units-diagnostic BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=NO_DIAGNOSTIC_ONLY PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO FAILED_STATE_RESET=NO CLOUD_WRITE_RUN=NO PVE01_FAILED_UNIT_COUNT=2 PVE01_FAILED_UNIT_NAMES=homelab-backup-matrix-refresh.service,homelab-cluster-admin-pull.service CRYPTO_PORTFOLIO_FAILED_UNIT_COUNT=0 OTHER_FAILED_UNIT_COUNT=2 CURRENT_CRYPTO_PRECHANGE_HEALTH=YES UNIT_DETAILS_CAPTURED=YES READY_FOR_TARGETED_FAILED_UNIT_DECISION=YES DIAGNOSTIC_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-failed-units-diagnostic/20260713T054100Z
|
|
/var/lib/homelab-health/crypto-portfolio-remove-grt-production-apply.txt:1:STATUS=OK TS=2026-07-13T05:47:54Z TYPE=crypto-portfolio-remove-grt-production-apply BAD=0 COMMAND_PHYSICAL_LINES=1 APPLY_REVISION=V3 FIX_UID_READONLY=YES FIX_READINESS_ENDPOINT=YES FIX_ROLLBACK_DATABASE_METADATA=YES PRODUCTION_CHANGED=YES_REMOVE_GRT DESIRED_STATE_CHANGED=NO_BY_THIS_COMMAND PRODUCTION_CONTAINER_RECREATED=YES CLOUD_WRITE_RUN=NO ROLLBACK_ON_FAILURE=ARMED_NOT_NEEDED TARGET_HOST=core-apps.gram1.ru DESIRED_STATE_HEAD=80763a6c62b94cde4f3293f3b529980f76919d25 PRECHANGE_BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T051015Z ROLLBACK_PATH=/var/lib/homelab-incidents/crypto-remove-grt-prechange-20260713T051013Z APP_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e CHART_HEALTH_SCRIPT_SHA256=c16473f16390eeeff4d4eb701db604093564dd78d0be2d30b4cc8ffeeb22364a ACTIVE_COINS=HBAR,POL GRT_SOURCE_REFERENCES=0 GRT_DATABASE_ROWS=0|0|0 GRT_MARKET_ENTRY=ABSENT GRT_CACHE_ENTRIES=ABSENT DATABASE_MODE=0644 DATABASE_UID=10001 DATABASE_GID=10001 DATABASE_INTEGRITY=ok WORKER_STATUS=OK WORKER_COIN_COUNT=2 PUBLIC_CODES=200|303|401|200|200|401 CHART_HEALTH=OK CHART_SYMBOLS=HBAR,POL CHART_HEALTH_TIMER=ACTIVE VM170_FAILED_UNITS=0 PVE01_UNRELATED_FAILED_UNITS=homelab-backup-matrix-refresh.service,homelab-cluster-admin-pull.service PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES REMOTE_STAGE_CLEANED=YES READY_FOR_GRT_POSTAPPLY_VERIFY_BACKUP_SYNC=YES APPLY_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-production-apply-v3/20260713T054754Z
|
|
/var/lib/homelab-health/mail-cloud-restore-drill.txt:1:STATUS=STALE
|
|
/var/lib/homelab-health/mail-cloud-restore-drill.txt:20:FRESHNESS=STALE
|
|
/var/lib/homelab-health/edge-restore-proofs-offhost.txt:1:STATUS=OK TS=20260624T190040Z TYPE=edge-restore-proofs-offhost DEST_HOST=pve02 DEST_DIR=/mnt/staging/offhost/edge-restore-proofs-from-edge ITEMS=14 OK=14 ERROR=0 BYTES=226024287 SECRET_PRINTED=NO
|
|
/var/lib/homelab-health/crypto-portfolio-runtime-final-seal.txt:1:STATUS=OK TS=2026-07-13T08:37:54Z TYPE=crypto-portfolio-ui-center-alert-currency-final-seal BAD=0 DANGLING_IMAGE_REFERENCE_GATE=OK DESIRED_STATE_SYNC_ARTIFACT_GATE=OK APP=crypto-portfolio PRODUCTION_CHANGED=NO_FINAL_SEAL_ONLY APP_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb LIBRARY_SHA256=c0992580867c4912cc9385b3c2728315bcc1a76c7f1087dca908430fccdf31d7 ACTIVE_COINS=HBAR,POL GRT_REMOVED=YES GRT_ROWS=0|0|0 GRT_SOURCE_REFERENCES=0 GRT_CACHE_ENTRIES=0 SHARP_MOVES_CENTERED=YES ALERT_CURRENCY_USD_RUB=OK BROWSER_SMOKE=OK BROWSER_RUNS=2 BROWSER_ERRORS=0 BROWSER_RESULT_SHA256=b566f492cd80c8f99bc6b24cbef6d5f326c04f4d7168359d05eb1f189589e733 SCREENSHOT_SHA256=00b2853bee9c880146d0dd75fac60feb7e46d4c129ee06754047ef30d90bd4ba BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z TWO_CLOUDS=OK RESTORE_CHECK=OK DATABASE_INTEGRITY=ok RETENTION=OK ROLLBACK_COUNT=3 DANGLING_IMAGES=2 REFERENCED_DANGLING_IMAGES=2 UNREFERENCED_DANGLING_IMAGES=0 PRUNED_UNREFERENCED_DANGLING_IMAGES=0 CHART_HEALTH=OK PUBLIC=OK MONITOR_TIMER=active BACKUP_TIMER=active CLOSURE_DESIRED_STATE_HEAD=2999910549eaff9b9431b9c656d318340eb95cbd DESIRED_STATE=OK PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES ALL_CRYPTO_PORTFOLIO_UI_TAILS_CLOSED=YES
|
|
/var/lib/homelab-health/backup-matrix-cluster-preseal-convergence.txt:1:STATUS=OK TS=2026-07-13T17:48:14Z TYPE=backup-matrix-cluster-preseal-convergence BAD=0 MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 CLUSTER_PASSPORT=OK FINAL_READINESS=OK OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 WEBPANEL=OK FAILED_UNITS=0 DESIRED_STATE_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=3791fa78cdfb56cd7027f89350dff245abcffc4f935adfc336ceeea00854ac89 CLUSTER_PASSPORT_SHA256=1b8b924ae0e5ce6872fe5671318094ce494806c2db7efe28299b5ff49adc0d0d FINAL_READINESS_SHA256=ac5159fa8c497afe021b5cb232f08cfec0e1a4d516d8aae966ba03f7034f1412 OVERALL_SHA256=6ae529bfdf5811f686d3b53978f453ce4f8757c74fc1be105de219b13a61ba46 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-preseal-resume/20260713T174748Z
|
|
/var/lib/homelab-health/uptime-kuma-pending-health-monitors-fix.txt:8:NOTE=Paperless Restore may become red if its health file contains STATUS=ERROR; that is a real restore-health issue, not a Kuma pending issue.
|
|
/var/lib/homelab-health/pve03-capacity-forensics.txt:1:STATUS=OK TS=2026-07-08T05:42:29Z TYPE=pve03-capacity-forensics MODE=SUPERSEDED_BY_PVE03_ROOT_STAGING_CLEANUP BAD=0 WARN=0 OLD_STATUS=WARN_ARCHIVED ROOT_CLEANUP=OK STAGING_RETENTION=OK STAGING_CLEANUP=OK FAILED_UNIT_CLEANUP=OK REPORT=/var/lib/homelab-duty-admin-residual-cleanup/pve03-capacity-forensics-closure-latest.md
|
|
/var/lib/homelab-health/remote-failed-unit-apply.txt:1:STATUS=OK TS=2026-07-08T05:25:22Z TYPE=remote-failed-unit-apply BAD=0 WARN=0 MODE=APPLY_CLEANUP_PLAN_EXCEPT_PAPERLESS_CORE_GUARD PRECHECK=YES APPLY_OK=YES LOCAL_OVERRIDE_COUNT=4 PVE02_RESET=1 EDGE_APPLY=1 LOCAL_FAILED_AFTER=0 PVE02_FAILED_AFTER=0 EDGE_FAILED_AFTER=1 EDGE_DEFER_ONLY=YES PVE02_CODEVIPE_FAILED=inactive DUTY_V2_STATUS=OK DUTY_V2_GATED_NONOK=0 OVERALL_STATUS=OK OVERALL_BAD=0 DEFERRED_UNIT=homelab-paperless-core-guard.service REPORT=/var/lib/homelab-remote-failed-unit-cleanup/apply-latest.md
|
|
/var/lib/homelab-health/recovery-card.txt:1:STATUS=OK TS=2026-07-08T08:24:05Z TYPE=recovery-card BAD=0 WARN=0 MODE=READ_ONLY_RECOVERY_CARD_GENERATION RECOVERY_CARD_OK=YES FREEZE_STATUS=OK FREEZE_OK=YES LOCAL_FAILED=0 PVE02_FAILED=0 PVE03_FAILED=0 EDGE_FAILED=0 DUTY_V2_STATUS=OK DUTY_V2_GATED_NONOK=0 DUTY_V2_UNGATED_ERROR_REVIEW=0 DUTY_V2_UNGATED_WARN_REVIEW=0 DUTY_V2_UNGATED_REVIEW_ITEM=0 DUTY_V2_LEGACY_OR_PHASE_REVIEW=0 OVERALL_STATUS=OK OVERALL_BAD=0 RESTORE_CHECK_OK=YES GOLDEN_MATCH=YES LATEST_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260708T082034Z GOLDEN_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260708T082034Z EMERGENCY_JOB=homelab-pve01-emergency-pack-20260708T002739Z VM_DUMPS_JOB=homelab-pve01-vm-local-dumps-20260706T153606Z ENDPOINT_LOCAL_OK=1 ENDPOINT_LAN_OK=1 ENDPOINT_PUBLIC_OK=1 BACKUP_SLA=OK BACKUP_COVERAGE=OK EXTENDED_APPBACKUP=OK VM_BACKUP=OK EXTERNAL_PROBE=OK EXTERNAL_PUBLIC_SMOKE_BAD=0 DIRTY_COUNT=0 CARD=/var/lib/homelab-recovery-card/recovery-card-latest.md SUMMARY=/var/lib/homelab-recovery-card/recovery-summary-latest.tsv REPORT=/var/lib/homelab-recovery-card/recovery-card-latest.md
|
|
/var/lib/homelab-health/cluster-admin-stage0-convergence.txt:1:STATUS=WARN TS=2026-07-12T23:12:59Z TYPE=cluster-admin-stage0-targeted-convergence BAD=2 PROFILE_OK=NO CLUSTER_PASSPORT=FAIL FINAL_READINESS=FAIL OVERALL=FAIL DUTY_ADMIN=FAIL FAILED_UNITS=0 LIVE_MATCH_DESIRED_STATE=YES READY_FOR_FINAL_SEAL=NO REPORT_DIR=/var/lib/homelab-cluster-admin-stage0/readiness-profile-20260712T231259Z PRODUCTION_CHANGED=HEALTH_REGENERATION_ONLY PRODUCTION_SCHEMA_CHANGED=NO
|
|
/var/lib/homelab-health/cluster-admin-stage4c-database-migration-dry-run.txt:1:STATUS=OK TS=2026-07-14T05:33:26Z TYPE=cluster-admin-stage4c-database-migration-dry-run-final-seal BAD=0 STAGE4C_CLOSED=YES NEXT_STAGE=4D_ADAPTER_CANDIDATES_IN_ISOLATION MIGRATION_SHA256=a0bd6a50c6ad9b90528c2e90e6d9cb4279b09811b559b2504c83d7527b19e578 FINAL_READINESS=OK OVERALL=OK PUBLIC_ENDPOINT=OK WEBPANEL_SYNC=OK PVE01_FAILED_UNITS=0 VM180_FAILED_UNITS=0 DESIRED_STATE_HEAD=df86e4d27a64613b60ae55253b6348866dd34fa9 DESIRED_STATE_CLEAN=YES REPORT=/var/lib/homelab-cluster-admin-stage4/20260714T053326Z/stage4c-database-migration-dry-run-seal.txt REPORT_SHA256=3368c302ca0ee76fab8fbc97caa65adf4cf57e61c5b5cdb9a1f3829a05af2e2a
|
|
/var/lib/homelab-health/operator-command-size-guard.txt:1:STATUS=OK TS=2026-07-14T18:26:54Z TYPE=operator-command-size-guard RULE_ACTIVE=YES PARSE_ERROR_TAIL=CLOSED UNRESOLVED_TAIL_COUNT=0 BLOCKER_COUNT=0 REPORT_SHA256=2ed7949dde9d5f8346e46af362e27cb8982d65f00ec602546efab506509e47af
|
|
/var/lib/homelab-health/node-loss-final-closure.txt:1:STATUS=OK TS=2026-07-08T16:35:56Z TYPE=node-loss-final-closure BAD=0 MODE=ALL_NODE_LOSS_DRILLS_SEALED PVE03_EDGE_RESTORE=OK PVE03_EDGE_FRESHNESS=OK PVE02_FORUM_RUNTIME=OK PVE01_NEXTCLOUD_DNS=OK PVE01_CORE_MONITORING_APPLEVEL=OK FULL_VM_GAP_VM170_VM171=ACKNOWLEDGED_APP_LEVEL_ONLY TEST_VMS_CLEANED=YES DNS_PAIR_OK=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-node-loss/final-closure-latest.md
|
|
/var/lib/homelab-health/crypto-portfolio-remove-grt-browser-smoke.txt:1:STATUS=OK TS=2026-07-13T04:52:06Z TYPE=crypto-portfolio-remove-grt-browser-smoke BAD=0 COMMAND_PHYSICAL_LINES=1 PREVIOUS_FAILURE=CORRECTED PREVIOUS_FAILURE_ROOT_CAUSE=OPTIONAL_XXD_DEPENDENCY_MISSING_WITH_STDERR_SUPPRESSED BROWSER_RERUN=NO EVIDENCE_SALVAGED=YES PRODUCTION_CHANGED=NO PRODUCTION_DATABASE_CHANGED=NO CANDIDATE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CONTAINER_STARTED=NO CLOUD_WRITE_RUN=NO TARGET_HOST=core-apps.gram1.ru DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 INPUT_APP_SHA256=684ad884119554503db0ed36252aeb017c0c04b54729f30c51e66a7ab293f8bd INPUT_WORKER_SHA256=40a1c81111d7097120125b140683a8604abcb4051db5073ab8af708a11854038 INPUT_INDEX_SHA256=2b1ab459cabf3050af1db4a7b293fff7c2a9ac18bf4a14faa1efc70a3a3f63e3 ACTIVE_COINS=HBAR,POL GRT_VISIBLE=NO BROWSER_CARDS=2 BROWSER_CANVASES=14 BROWSER_MOVES=2 BROWSER_COIN_CARDS=2 BROWSER_POSITION_ROWS=2 BROWSER_ALERT_OPTIONS=2 BROWSER_ACTIVE_COIN=HBAR BROWSER_CHART_SYMBOLS=HBAR,POL BROWSER_ERRORS=0 BROWSER_LOADING_TEXT=ABSENT BROWSER_ERROR_TEXT=ABSENT CHARTS_POSITION=BOTTOM PNG_VALIDATION=PYTHON_STDLIB PNG_SIGNATURE=89504e470d0a1a0a SCREENSHOT_SHA256=3d132b477a382487e34db2f2feaf7e4f43af8d30fd8e1530225782da5d7336b9 RESULT_SHA256=0deadbf38aeef86726035da66eb1991c9494f59172b1c1a5c502c50c2de95c60 LIVE_PRODUCTION_DATA_MOUNTED=NO NPM_INSTALL_RUN=NO DOCKER_PULL_RUN=NO PRODUCTION_SOURCE_HASHES_UNCHANGED=YES PRODUCTION_CONTAINER_IDS_AND_RESTARTS_UNCHANGED=YES PRODUCTION_DATABASE_INTEGRITY=ok PRODUCTION_GRT_STATE_PRESERVED=YES PUBLIC_PRODUCTION_STATE_PRESERVED=YES REMOTE_RESIDUE=NONE READY_FOR_GRT_PRECHANGE_BACKUP=YES BROWSER_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-browser-smoke-salvage/20260713T045206Z
|
|
/var/lib/homelab-health/independent-external-runner-audit.txt:1:STATUS=OK TS=2026-07-08T09:13:25Z TYPE=independent-external-runner-audit BAD=0 WARN=0 MODE=READ_ONLY_INDEPENDENT_EXTERNAL_RUNNER_AUDIT AUDIT_OK=YES INDEPENDENT_RUNNER_READY=NO DEPLOY_DECISION=NO_DEPLOY CURRENT_INDEPENDENT_RUNNER_AVAILABLE=no TRUSTED_NETBIRD_IDENTITY=edge-vm PUBLIC_VPS_DECISION=REJECTED_HOSTKEY_MISMATCH PUBLIC_VPS_SSH_OK=0 PUBLIC_VPS_REJECTED_HOSTKEY=YES EDGE_LAN_SSH_OK=2 EDGE_NETBIRD_SSH_OK=2 EDGE_DNS_SSH_OK=1 PUBLIC_OVERALL_OK=1 LOCAL_FAILED=0 PVE02_FAILED=0 PVE03_FAILED=0 EDGE_FAILED=0 OVERALL_STATUS=OK OVERALL_BAD=0 DUTY_V2_STATUS=OK DUTY_V2_GATED_NONOK=0 DUTY_V2_UNGATED_ERROR_REVIEW=0 DUTY_V2_UNGATED_WARN_REVIEW=0 CLOSEOUT_OK=YES RECOVERY_SEAL_OK=YES DIRTY_COUNT=0 REPORT=/var/lib/homelab-independent-external-runner/audit-latest.md
|
|
/var/lib/homelab-health/sops-edge-secret-coverage-closure.txt:1:STATUS=OK TS=2026-07-08T05:01:47Z TYPE=sops-edge-secret-coverage-closure BAD=0 WARN=0 MODE=POLICY_SKELETON_SUPERSEDED_NOOP PRECHECK=YES POLICY_MODE=POLICY_SKELETON_NOT_ACTIVE AUDIT_CLASS=UNGATED_FAILED_STATE_REVIEW IN_HARD_GATES=0 OVERRIDE_WRITTEN=1 FAILED_UNITS_AFTER=0 UNIT_FAILED_AFTER=inactive MARKER_STATUS=OK DUTY_V2_STATUS=OK DUTY_V2_GATED_NONOK=0 DUTY_V2_UNGATED_ERROR_REVIEW=0 OVERALL_STATUS=OK OVERALL_BAD=0 DIRTY_COUNT=0 SOPS_CLOSURE_OK=YES REPORT=/var/lib/homelab-failed-unit-legacy-cleanup/sops-edge-secret-coverage-closure-latest.md
|
|
/var/lib/homelab-health/mail-cloud-canary.txt:1:STATUS=OK TS=20260618T174444Z TYPE=mail-cloud-canary REMOTE_TOTAL=13 REMOTE_OK=13 REMOTE_FAIL=0 DEST=restore-test/p1-canary/20260618T174444Z SHA256=a6bf6a1493a2556b92c8e6547163500345d4e0e32688937370b942b881c47182 SECRET_PRINTED=no NOTES=none
|
|
/var/lib/homelab-health/filebrowser-restore.txt:1:STATUS=STALE
|
|
/var/lib/homelab-health/filebrowser-restore.txt:20:FRESHNESS=STALE
|
|
/var/lib/homelab-health/evidence-seal/latest.env:6:TOTAL_FAILURES=0
|
|
/var/lib/homelab-health/final-green-freeze-audit.txt:1:STATUS=OK TS=2026-07-08T07:45:35Z TYPE=final-green-freeze-audit BAD=0 WARN=0 MODE=READ_ONLY_FREEZE_EVIDENCE_AUDIT FREEZE_OK=YES LOCAL_FAILED=0 PVE02_FAILED=0 PVE03_FAILED=0 EDGE_FAILED=0 DUTY_V2_STATUS=OK DUTY_V2_GATED_NONOK=0 DUTY_V2_UNGATED_ERROR_REVIEW=0 DUTY_V2_UNGATED_WARN_REVIEW=0 DUTY_V2_UNGATED_REVIEW_ITEM=0 DUTY_V2_LEGACY_OR_PHASE_REVIEW=0 OVERALL_STATUS=OK OVERALL_BAD=0 EXACT_WARN_REVIEW_FILES=0 LOCAL_OVERALL_OK=1 LAN_OVERALL_OK=1 PUBLIC_OVERALL_OK=1 RESTORE_CHECK_OK=YES GOLDEN_MATCH=YES LATEST_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260708T072711Z GOLDEN_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260708T072711Z BACKUP_SLA=OK BACKUP_COVERAGE=OK EXTENDED_APPBACKUP=OK VM_BACKUP=OK SECRET_EXPOSURE=OK REVIEW_SURFACE_REGISTRY=YES FINAL_GLOBAL_GREEN=YES EXTERNAL_PROBE=OK EXTERNAL_PUBLIC_SMOKE_BAD=0 DIRTY_COUNT=0 EVIDENCE=/var/lib/homelab-final-green-freeze/freeze-evidence-latest.tsv REPORT=/var/lib/homelab-final-green-freeze/freeze-audit-latest.md
|
|
/var/lib/homelab-health/crypto-portfolio-remove-grt-runtime-dry-run.txt:1:STATUS=OK TS=2026-07-13T04:41:24Z TYPE=crypto-portfolio-remove-grt-runtime-dry-run BAD=0 COMMAND_PHYSICAL_LINES=1 VERIFIER_REVISION=V3 ANALYSIS_SOURCES=FULL_CHAT_CRYPTO_HANDOFF_CLUSTER_REFERENCE_ERROR_HISTORY PRODUCTION_CHANGED=NO PRODUCTION_DATABASE_CHANGED=NO CANDIDATE_DATABASE_CHANGED=ISOLATED_COPY_RUNTIME_CHECK_ONLY DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO PRODUCTION_CONTAINER_RECREATED=NO ISOLATED_CONTAINER_STARTED=YES ISOLATED_CONTAINER_CLEANED=YES CLOUD_WRITE_RUN=NO TARGET_HOST=core-apps.gram1.ru DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 INPUT_APP_SHA256=684ad884119554503db0ed36252aeb017c0c04b54729f30c51e66a7ab293f8bd INPUT_WORKER_SHA256=40a1c81111d7097120125b140683a8604abcb4051db5073ab8af708a11854038 INPUT_INDEX_SHA256=2b1ab459cabf3050af1db4a7b293fff7c2a9ac18bf4a14faa1efc70a3a3f63e3 ACTIVE_COINS=HBAR,POL CANDIDATE_GRT_REFERENCES=0 DATABASE_GRT_AFTER=0|0|0 WORKER_RUNTIME_CHECK=OK WEB_RUNTIME_CHECK=OK HTTP_RUNTIME_CHECK=OK HTTP_CODES=200|303|401|200|200|401 CHART_HEALTH=OK CHART_SYMBOLS=HBAR,POL ROUTE_RETURN_DECODER=DICT_OR_RESPONSE_BODY LIVE_PRODUCTION_DATA_MOUNTED=NO PRODUCTION_SOURCE_HASHES_UNCHANGED=YES PRODUCTION_CONTAINER_IDS_AND_RESTARTS_UNCHANGED=YES PRODUCTION_DATABASE_INTEGRITY=ok PRODUCTION_GRT_STATE_PRESERVED=YES PUBLIC_PRODUCTION_STATE_PRESERVED=YES REMOTE_RESIDUE=NONE BROWSER_SMOKE=NOT_RUN READY_FOR_GRT_ISOLATED_BROWSER_SMOKE=YES RUNTIME_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-runtime-run-v3/20260713T044124Z
|
|
/var/lib/homelab-health/sops-edge-secret-coverage-audit.txt:1:STATUS=OK TS=2026-07-08T04:59:58Z TYPE=sops-edge-secret-coverage-audit BAD=0 WARN=1 MODE=READ_ONLY_SOPS_EDGE_SECRET_COVERAGE_AUDIT UNIT_FAILED=failed CLASS=UNGATED_FAILED_STATE_REVIEW IN_HARD_GATES=no OVERALL_BAD=0 FINAL_FAIL=0 DUTY_V2_GATED_NONOK=0 DUTY_V2_UNGATED_ERROR_REVIEW=1 RELATED_WARN_OR_ERROR_HEALTH=3 RESET_FAILED_DONE=no REPORT=/var/lib/homelab-failed-unit-legacy-cleanup/sops-edge-secret-coverage-audit-latest.md
|
|
/var/lib/homelab-health/duty-admin-residual-cleanup-seal.txt:1:STATUS=OK TS=2026-07-08T05:44:22Z TYPE=duty-admin-residual-cleanup-seal BAD=0 WARN=0 MODE=FINAL_RESIDUAL_NOISE_SEAL RESIDUAL_SEAL_OK=YES LOCAL_FAILED=0 PVE02_FAILED=0 PVE03_FAILED=0 EDGE_FAILED=0 DUTY_V2_STATUS=OK DUTY_V2_GATED_NONOK=0 DUTY_V2_UNGATED_ERROR_REVIEW=0 DUTY_V2_UNGATED_WARN_REVIEW=0 DUTY_V2_UNGATED_REVIEW_ITEM=22 DUTY_V2_LEGACY_OR_PHASE_REVIEW=5 REVIEW_ITEMS_INTENTIONALLY_LEFT=yes LEGACY_PHASE_ITEMS_INTENTIONALLY_LEFT=yes FINAL_STATUS=OK FINAL_FAIL=0 CLUSTER_STATUS=OK CLUSTER_BAD=0 OVERALL_STATUS=OK OVERALL_BAD=0 PVE03_CAPACITY_FORENSICS_STATUS=OK LOCAL_CLEANUP_OK=YES REMOTE_CLEANUP_OK=YES DIRTY_COUNT=0 REPORT=/var/lib/homelab-duty-admin-residual-cleanup/seal-latest.md
|
|
/var/lib/homelab-health/prometheus-npmplus-kuma-supersede-repair-20260709T073019Z/npmplus-kuma-config-backup.txt.bak:1:STATUS=ERROR TS=20260709T072619Z TYPE=npmplus-kuma-config-backup NOTES=edge_backup_failed
|
|
========== NODE=pve02 ==========
|
|
=== FAILED_BACKUP_UNITS ===
|
|
=== STALE_OR_FAILED_HEALTH ===
|
|
2026-07-08T08:25:22.7370010450 /var/lib/homelab-health/codevipe-cloud-backup-reset.txt
|
|
2026-07-12T01:29:08.8140509520 /var/lib/homelab-health/routerguard-hard-power-cycle.txt
|
|
2026-07-14T06:42:03.4801032090 /var/lib/homelab-health/routerguard-ap.txt.tmp.3936294
|
|
2026-07-19T02:59:29.0591803690 /var/lib/homelab-health/routerguard-ap.txt.tmp.1662197
|
|
2026-07-19T02:59:46.2495844810 /var/lib/homelab-health/routerguard-ap.txt.tmp.1662387
|
|
2026-07-21T04:21:29.8655959860 /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup-local.txt
|
|
2026-07-21T08:03:00.2596892130 /var/lib/homelab-health/vaultwarden-isolated-restore-drill.txt
|
|
2026-07-21T15:04:23.0497723490 /var/lib/homelab-health/routerguard-ap.txt
|
|
/var/lib/homelab-health/codevipe-cloud-backup-reset.txt:1:STATUS=OK TS=2026-07-08T05:25:22Z TYPE=remote-failed-unit-codevipe-reset BAD=0 WARN=0 MODE=RESET_STALE_FAILED_STATE UNIT=codevipe-cloud-backup.service REASON=last_journal_success_and_no_timer ACTION=reset_failed_only
|
|
========== NODE=pve03 ==========
|
|
=== FAILED_BACKUP_UNITS ===
|
|
=== STALE_OR_FAILED_HEALTH ===
|
|
BACKUP_FAILURE_ROOTCAUSE=COMPLETE
|
|
========== CURRENT_MATRIX ==========
|
|
STATUS=FAIL
|
|
TS=2026-07-21T07:02:41Z
|
|
TYPE=unified-backup-matrix
|
|
MATRIX_MODE=authoritative
|
|
JOBS_TOTAL=41
|
|
JOBS_OK=30
|
|
JOBS_BAD=6
|
|
JOBS_UNMAPPED=0
|
|
JOBS_DISABLED=5
|
|
STATE_JSON=/var/lib/homelab-backup-matrix/state.json
|
|
MATRIX_JSON=/etc/homelab-backup-matrix/matrix.json
|
|
AUTHORITY=canonical
|
|
COMPAT_OUTPUTS=29
|
|
COMPAT_MAP=/var/lib/homelab-backup-matrix/compat-map.json
|
|
JOB=dpkg-db-backup STATUS=OK EVIDENCE=execution_only AGE=36147 MAX_AGE=172800
|
|
JOB=filebrowser-backup STATUS=OK EVIDENCE=artifact_or_health AGE=14443 MAX_AGE=259200
|
|
JOB=filebrowser-restore-validate STATUS=STALE EVIDENCE=artifact_or_health AGE=1016424 MAX_AGE=864000
|
|
JOB=homelab-appbackup-dockge-stacks STATUS=OK EVIDENCE=artifact_or_health AGE=22248 MAX_AGE=172800
|
|
JOB=homelab-appbackup-mariadb-batch STATUS=OK EVIDENCE=execution_only AGE=15387 MAX_AGE=172800
|
|
JOB=homelab-appbackup-npmplus-edge STATUS=OK EVIDENCE=artifact_or_health AGE=20573 MAX_AGE=172800
|
|
JOB=homelab-appbackup-postgres-batch STATUS=OK EVIDENCE=execution_only AGE=16416 MAX_AGE=172800
|
|
JOB=homelab-appbackup-sqlite-batch STATUS=OK EVIDENCE=execution_only AGE=18327 MAX_AGE=172800
|
|
JOB=homelab-authentik-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987790 MAX_AGE=172800
|
|
JOB=homelab-auto-backup STATUS=OK EVIDENCE=execution_only AGE=195424 MAX_AGE=864000
|
|
JOB=homelab-backup-coverage-matrix STATUS=OK EVIDENCE=artifact_or_health AGE=4346 MAX_AGE=172800
|
|
JOB=homelab-backup-freshness STATUS=OK EVIDENCE=execution_only AGE=35589 MAX_AGE=172800
|
|
JOB=homelab-backup-healthcheck STATUS=OK EVIDENCE=execution_only AGE=10631 MAX_AGE=172800
|
|
JOB=homelab-backup-sla-health STATUS=FAIL EVIDENCE=artifact_or_health AGE=144 MAX_AGE=172800
|
|
JOB=homelab-edge-vm-offhost-freshness STATUS=OK EVIDENCE=artifact_or_health AGE=18426 MAX_AGE=172800
|
|
JOB=homelab-emergency-backup STATUS=OK EVIDENCE=artifact_or_health AGE=24539 MAX_AGE=172800
|
|
JOB=homelab-extended-appbackup STATUS=OK EVIDENCE=artifact_or_health AGE=14156 MAX_AGE=172800
|
|
JOB=homelab-gitea-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987785 MAX_AGE=172800
|
|
JOB=homelab-golden-state-index STATUS=OK EVIDENCE=artifact_or_health AGE=4112 MAX_AGE=172800
|
|
JOB=homelab-mail-cloud-critical-upload STATUS=OK EVIDENCE=artifact_or_health AGE=11997 MAX_AGE=172800
|
|
JOB=homelab-mail-cloud-edge-vm-upload STATUS=OK EVIDENCE=artifact_or_health AGE=174045 MAX_AGE=864000
|
|
JOB=homelab-mail-cloud-nextcloud-vm-upload STATUS=OK EVIDENCE=artifact_or_health AGE=7575 MAX_AGE=864000
|
|
JOB=homelab-mail-cloud-restore-drill STATUS=STALE EVIDENCE=artifact_or_health AGE=988269 MAX_AGE=864000
|
|
JOB=homelab-npmplus-kuma-config-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=60071 MAX_AGE=172800
|
|
JOB=homelab-router-running-config-mail-cloud STATUS=OK EVIDENCE=artifact_or_health AGE=17076 MAX_AGE=172800
|
|
JOB=homelab-vaultwarden-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987768 MAX_AGE=172800
|
|
JOB=homelab-verified-backup-health STATUS=OK EVIDENCE=artifact_or_health AGE=11845 MAX_AGE=172800
|
|
JOB=homelab-vm-backup-health STATUS=OK EVIDENCE=artifact_or_health AGE=8742 MAX_AGE=172800
|
|
JOB=homelab-vm-backup-policy STATUS=OK EVIDENCE=artifact_or_health AGE=5548 MAX_AGE=172800
|
|
JOB=homelab-vm-full-backup STATUS=OK EVIDENCE=artifact_or_health AGE=24778 MAX_AGE=172800
|
|
JOB=homelab-vm-full-restore-drill STATUS=OK EVIDENCE=artifact_or_health AGE=523085 MAX_AGE=10368000
|
|
JOB=homelab-vm-local-dumps-2cloud STATUS=OK EVIDENCE=artifact_or_health AGE=172141 MAX_AGE=864000
|
|
JOB=homelab-vm-local-dumps-retention STATUS=OK EVIDENCE=artifact_or_health AGE=168839 MAX_AGE=864000
|
|
JOB=immich-media-offhost-sync STATUS=OK EVIDENCE=artifact_or_health AGE=4286 MAX_AGE=259200
|
|
JOB=memos-backup STATUS=OK EVIDENCE=artifact_or_health AGE=14547 MAX_AGE=259200
|
|
JOB=netbird-vps-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=1747896 MAX_AGE=864000
|
|
JOB=paperless-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=16347 MAX_AGE=259200
|
|
JOB=paperless-restore-dry-run STATUS=STALE EVIDENCE=artifact_or_health AGE=1001986 MAX_AGE=864000
|
|
JOB=skladchik-reports-monitor-backup STATUS=OK EVIDENCE=artifact_or_health AGE=19639 MAX_AGE=172800
|
|
JOB=skladchik-reports-monitor-backup-restore-check STATUS=OK EVIDENCE=artifact_or_health AGE=190273 MAX_AGE=864000
|
|
JOB=vm160-remote-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=26607 MAX_AGE=172800
|
|
BACKUP_REMEDIATION_PLAN_DATA=COMPLETE
|
|
|
|
OUTPUT_END
|
|
CHAT_OUTPUT_END
|