From 566fb6d435ec739b01721c6e9307d97da18b5073 Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Fri, 24 Jul 2026 10:17:02 +0000 Subject: [PATCH] runtime: publish CONTEXT-AUTO-20260724T101701Z --- .../CONTEXT-AUTO-20260724T101701Z.json | 1 + .../history/CONTEXT-AUTO-20260724T101701Z.txt | 200 +++++++ runtime/latest.json | 31 +- runtime/latest.txt | 519 +++++++----------- 4 files changed, 394 insertions(+), 357 deletions(-) create mode 100644 runtime/history/CONTEXT-AUTO-20260724T101701Z.json create mode 100644 runtime/history/CONTEXT-AUTO-20260724T101701Z.txt diff --git a/runtime/history/CONTEXT-AUTO-20260724T101701Z.json b/runtime/history/CONTEXT-AUTO-20260724T101701Z.json new file mode 100644 index 0000000..aa5cc70 --- /dev/null +++ b/runtime/history/CONTEXT-AUTO-20260724T101701Z.json @@ -0,0 +1 @@ +{"schema_version":1,"channel":"homelab-runtime","command_id":"CONTEXT-AUTO-20260724T101701Z","status":"OK","rc":0,"host":"pve01","mode":"read-only","component":"cluster-context","started_at_utc":"2026-07-24T10:17:01Z","finished_at_utc":"2026-07-24T10:17:02Z","reference_register_checked":true,"reference_sha256":"5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e","error_register_checked":true,"error_register_sha256":"08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752","changes_made":false,"sanitized":true,"secrets_included":false,"private_addresses_included":false,"output":"AUTOMATIC_CLUSTER_CONTEXT_REFRESH=OK"} diff --git a/runtime/history/CONTEXT-AUTO-20260724T101701Z.txt b/runtime/history/CONTEXT-AUTO-20260724T101701Z.txt new file mode 100644 index 0000000..8036d07 --- /dev/null +++ b/runtime/history/CONTEXT-AUTO-20260724T101701Z.txt @@ -0,0 +1,200 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=CONTEXT-AUTO-[PRIVATE_IP]701Z +STATUS=OK +RC=0 +HOST=pve01 +COMPONENT=cluster-context +REFERENCE_SHA[PRIVATE_IP]f8edc75fcf[PRIVATE_IP]c[PRIVATE_IP]cc112ceccb6cf[PRIVATE_IP]a4deb8e +ERROR_REGISTER_SHA[PRIVATE_IP]b8e38eaa1acec915352fb[PRIVATE_IP]caa7ff6cd9d1ec0da[PRIVATE_IP] +OUTPUT_BEGIN +GENERATED_AT_UTC=[PRIVATE_IP]T10:17:01Z +Cluster information +------------------- +Name: homelab +Config Version: 3 +Transport: knet +Secure auth: on + +Quorum information +------------------ +Date: Fri Jul [PRIVATE_IP] 2026 +Quorum provider: corosync_votequorum +Nodes: 3 +Node ID: [PRIVATE_IP] +Ring ID: 1.ba +Quorate: Yes + +Votequorum information +---------------------- +Expected votes: 3 +Highest expected: 3 +Total votes: 3 +Quorum: 2 +Flags: Quorate + +Membership information +---------------------- + Nodeid Votes Name +[PRIVATE_IP] [PRIVATE_IP].11 (local) +[PRIVATE_IP] [PRIVATE_IP].13 +[PRIVATE_IP] [PRIVATE_IP].12 + +Membership information +---------------------- + Nodeid Votes Name + 1 1 pve01 (local) + 2 1 pve03 + 3 1 pve02 +[{"cpu":[PRIVATE_IP][PRIVATE_IP],"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/110","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"dns1","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"lxc","uptime":75356,"vmid":110},{"cpu":[PRIVATE_IP][PRIVATE_IP],"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/111","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"dns2","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve02","status":"running","template":0,"type":"lxc","uptime":75188,"vmid":111},{"cpu":0,"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/112","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"unbound1","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"lxc","uptime":75359,"vmid":112},{"cpu":[PRIVATE_IP]559e-05,"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/113","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"unbound2","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve02","status":"running","template":0,"type":"lxc","uptime":75191,"vmid":113},{"cpu":[PRIVATE_IP]20445,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/130","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"edge-vm","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve03","status":"running","template":0,"type":"qemu","uptime":75297,"vmid":130},{"cpu":[PRIVATE_IP]7415,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/150","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"nextcloud","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"qemu","uptime":75353,"vmid":150},{"cpu":[PRIVATE_IP]555957,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/160","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"forum-prod","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve02","status":"running","template":0,"type":"qemu","uptime":75185,"vmid":160},{"cpu":[PRIVATE_IP]70144,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/170","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"core-apps","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"qemu","uptime":75349,"vmid":170},{"cpu":[PRIVATE_IP]20509,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/171","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"monitoring","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"qemu","uptime":75346,"vmid":171},{"cpu":[PRIVATE_IP]667871,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/180","maxcpu":2,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"cluster-admin","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve02","status":"running","template":0,"type":"qemu","uptime":75182,"vmid":180},{"cpu":[PRIVATE_IP]634515,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/190","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"homelab-ops-worker","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve03","status":"running","template":0,"type":"qemu","uptime":75293,"vmid":190},{"cpu":0,"disk":0,"diskread":0,"diskwrite":0,"id":"qemu/9130","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":0,"memhost":0,"name":"edge-cold-standby","netin":0,"netout":0,"node":"pve02","status":"stopped","template":0,"type":"qemu","uptime":0,"vmid":9130},{"cgroup-mode":2,"cpu":[PRIVATE_IP]35108,"disk":[PRIVATE_IP],"id":"node/pve03","level":"","maxcpu":8,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"node":"pve03","status":"online","type":"node","uptime":75315},{"cgroup-mode":2,"cpu":[PRIVATE_IP]78104,"disk":[PRIVATE_IP],"id":"node/pve02","level":"","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"node":"pve02","status":"online","type":"node","uptime":75252},{"cgroup-mode":2,"cpu":[PRIVATE_IP]76872,"disk":[PRIVATE_IP],"id":"node/pve01","level":"","maxcpu":24,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"node":"pve01","status":"online","type":"node","uptime":75386},{"content":"rootdir,images","disk":[PRIVATE_IP],"id":"storage/pve03/local-lvm","maxdisk":[PRIVATE_IP],"node":"pve03","plugintype":"lvmthin","shared":0,"status":"available","storage":"local-lvm","type":"storage"},{"content":"rootdir,images","disk":[PRIVATE_IP],"id":"storage/pve02/local-lvm","maxdisk":[PRIVATE_IP],"node":"pve02","plugintype":"lvmthin","shared":0,"status":"available","storage":"local-lvm","type":"storage"},{"content":"rootdir,images","disk":[PRIVATE_IP],"id":"storage/pve01/local-lvm","maxdisk":[PRIVATE_IP],"node":"pve01","plugintype":"lvmthin","shared":0,"status":"available","storage":"local-lvm","type":"storage"},{"content":"import,backup,iso,vztmpl","disk":[PRIVATE_IP],"id":"storage/pve03/local","maxdisk":[PRIVATE_IP],"node":"pve03","plugintype":"dir","shared":0,"status":"available","storage":"local","type":"storage"},{"content":"import,backup,iso,vztmpl","disk":[PRIVATE_IP],"id":"storage/pve02/local","maxdisk":[PRIVATE_IP],"node":"pve02","plugintype":"dir","shared":0,"status":"available","storage":"local","type":"storage"},{"content":"import,backup,iso,vztmpl","disk":[PRIVATE_IP],"id":"storage/pve01/local","maxdisk":[PRIVATE_IP],"node":"pve01","plugintype":"dir","shared":0,"status":"available","storage":"local","type":"storage"},{"id":"network/pve03/zone/localnetwork","network":"localnetwork","network-type":"zone","node":"pve03","status":"ok","type":"network"},{"id":"network/pve02/zone/localnetwork","network":"localnetwork","network-type":"zone","node":"pve02","status":"ok","type":"network"},{"id":"network/pve01/zone/localnetwork","network":"localnetwork","network-type":"zone","node":"pve01","status":"ok","type":"network"}] +Name Type Status Total (KiB) Used (KiB) Available (KiB) % +local dir active [PRIVATE_IP] [PRIVATE_IP] [PRIVATE_IP] 41.27% +local-lvm lvmthin active [PRIVATE_IP] [PRIVATE_IP] [PRIVATE_IP] 38.40% + UNIT LOAD ACTIVE SUB DESCRIPTION +● homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state + +Legend: LOAD → Reflects whether the unit definition was properly loaded. + ACTIVE → The high-level unit activation state, i.e. generalization of SUB. + SUB → The low-level unit activation state, values depend on unit type. + +1 loaded units listed. +NEXT LEFT LAST PASSED UNIT ACTIVATES +Fri [PRIVATE_IP] 13:17:33 MSK 31s Fri [PRIVATE_IP] 13:16:29 MSK 32s ago homelab-router-watchdog.timer homelab-router-watchdog.service +Fri [PRIVATE_IP] 13:18:15 MSK 1min 12s Fri [PRIVATE_IP] 13:08:15 MSK 8min ago skladchik-reports-monitor-supervisor.timer skladchik-reports-monitor-supervisor.service +Fri [PRIVATE_IP] 13:18:39 MSK 1min 37s Fri [PRIVATE_IP] 13:13:38 MSK 3min 24s ago homelab-crypto-portfolio-chart-health.timer homelab-crypto-portfolio-chart-health.service +Fri [PRIVATE_IP] 13:18:41 MSK 1min 38s Fri [PRIVATE_IP] 13:16:28 MSK 33s ago homelab-cluster-admin-inventory-sync.timer homelab-cluster-admin-inventory-sync.service +Fri [PRIVATE_IP] 13:19:28 MSK 2min 26s Fri [PRIVATE_IP] 13:14:28 MSK 2min 33s ago homelab-smartctl-textfile.timer homelab-smartctl-textfile.service +Fri [PRIVATE_IP] 13:21:28 MSK 4min 26s Fri [PRIVATE_IP] 13:16:28 MSK 33s ago netbird-peers-health.timer netbird-peers-health.service +Fri [PRIVATE_IP] 13:21:49 MSK 4min 47s Fri [PRIVATE_IP] 13:16:28 MSK 33s ago homelab-health-metrics.timer homelab-health-metrics.service +Fri [PRIVATE_IP] 13:22:28 MSK 5min Fri [PRIVATE_IP] 13:07:28 MSK 9min ago prometheus-node-exporter-apt.timer prometheus-node-exporter-apt.service +Fri [PRIVATE_IP] 13:22:28 MSK 5min Fri [PRIVATE_IP] 13:07:28 MSK 9min ago prometheus-node-exporter-nvme.timer prometheus-node-exporter-nvme.service +Fri [PRIVATE_IP] 13:24:15 MSK 7min Fri [PRIVATE_IP] 13:09:15 MSK 7min ago homelab-disk-space-health.timer homelab-disk-space-health.service +Fri [PRIVATE_IP] 13:24:15 MSK 7min Fri [PRIVATE_IP] 13:09:15 MSK 7min ago homelab-pve03-staging-capacity-health.timer homelab-pve03-staging-capacity-health.service +Fri [PRIVATE_IP] 13:30:00 MSK 12min Fri [PRIVATE_IP] 13:15:04 MSK 1min 58s ago homelab-incident-journal.timer homelab-incident-journal.service +Fri [PRIVATE_IP] 13:30:00 MSK 12min Fri [PRIVATE_IP] 13:15:04 MSK 1min 58s ago homelab-safe-autoheal.timer homelab-safe-autoheal.service +Fri [PRIVATE_IP] 13:30:00 MSK 12min Fri [PRIVATE_IP] 13:15:36 MSK 1min 25s ago homelab-forum-snuffleupagus-health.timer homelab-forum-snuffleupagus-health.service +Fri [PRIVATE_IP] 13:30:45 MSK 13min Fri [PRIVATE_IP] 13:15:58 MSK 1min 4s ago homelab-duty-admin-v2.timer homelab-duty-admin-v2.service +Fri [PRIVATE_IP] 13:30:49 MSK 13min Fri [PRIVATE_IP] 13:15:36 MSK 1min 25s ago homelab-external-probe-vps-health.timer homelab-external-probe-vps-health.service +Fri [PRIVATE_IP] 13:32:12 MSK 15min Fri [PRIVATE_IP] 13:00:28 MSK 16min ago homelab-mkdocs-auto-refresh.timer homelab-mkdocs-auto-refresh.service +Fri [PRIVATE_IP] 13:33:30 MSK 16min Fri [PRIVATE_IP] 13:02:38 MSK 14min ago homelab-reference-refresh.timer homelab-reference-refresh.service +Fri [PRIVATE_IP] 14:48:04 MSK 1h 31min Thu [PRIVATE_IP] 21:43:04 MSK 15h ago apt-daily.timer apt-daily.service +Fri [PRIVATE_IP] 16:23:38 MSK 3h 6min Fri [PRIVATE_IP] 10:23:38 MSK 2h 53min ago homelab-vps-identity-audit.timer homelab-vps-identity-audit.service +Fri [PRIVATE_IP] 16:35:37 MSK 3h 18min Thu [PRIVATE_IP] 16:35:37 MSK 20h ago systemd-tmpfiles-clean.timer systemd-tmpfiles-clean.service +Sat [PRIVATE_IP] 00:00:00 MSK 10h Fri [PRIVATE_IP] 00:00:04 MSK 13h ago dpkg-db-backup.timer dpkg-db-backup.service +Sat [PRIVATE_IP] 00:00:00 MSK 10h Fri [PRIVATE_IP] 00:20:00 MSK 12h ago homelab-docker-health.timer homelab-docker-health.service +Sat [PRIVATE_IP] 00:02:57 MSK 10h Fri [PRIVATE_IP] 00:03:04 MSK 13h ago homelab-evidence-catalog.timer homelab-evidence-catalog.service +Sat [PRIVATE_IP] 00:06:10 MSK 10h Fri [PRIVATE_IP] 00:19:28 MSK 12h ago homelab-storage-capacity.timer homelab-storage-capacity.service +Sat [PRIVATE_IP] 00:14:20 MSK 10h Fri [PRIVATE_IP] 00:03:38 MSK 13h ago homelab-quality-gate.timer homelab-quality-gate.service +Sat [PRIVATE_IP] 00:15:16 MSK 10h Fri [PRIVATE_IP] 00:05:04 MSK 13h ago homelab-backup-freshness.timer homelab-backup-freshness.service +Sat [PRIVATE_IP] 00:15:30 MSK 10h Fri [PRIVATE_IP] 07:47:04 MSK 5h 29min ago man-db.timer man-db.service +Sat [PRIVATE_IP] 00:25:03 MSK 11h Fri [PRIVATE_IP] 00:52:04 MSK 12h ago logrotate.timer logrotate.service +Sat [PRIVATE_IP] 02:15:00 MSK 12h Fri [PRIVATE_IP] 02:15:04 MSK 11h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service +Sat [PRIVATE_IP] 03:18:38 MSK 14h Fri [PRIVATE_IP] 03:13:28 MSK 10h ago homelab-emergency-backup.timer homelab-emergency-backup.service +Sat [PRIVATE_IP] 03:35:00 MSK 14h Fri [PRIVATE_IP] 03:35:00 MSK 9h ago filebrowser-backup.timer filebrowser-backup.service +Sat [PRIVATE_IP] 03:50:00 MSK 14h Fri [PRIVATE_IP] 03:50:04 MSK 9h ago filebrowser-offhost-sync.timer filebrowser-offhost-sync.service +Sat [PRIVATE_IP] 03:50:16 MSK 14h Fri [PRIVATE_IP] 03:48:15 MSK 9h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service +Sat [PRIVATE_IP] 03:55:37 MSK 14h Fri [PRIVATE_IP] 01:38:48 MSK 11h ago pve-daily-update.timer pve-daily-update.service +Sat [PRIVATE_IP] 04:10:00 MSK 14h Fri [PRIVATE_IP] 04:10:04 MSK 9h ago filebrowser-restore-validate.timer filebrowser-restore-validate.service +Sat [PRIVATE_IP] 04:22:27 MSK 15h Fri [PRIVATE_IP] 04:27:33 MSK 8h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service +Sat [PRIVATE_IP] 04:22:44 MSK 15h Fri [PRIVATE_IP] 04:17:28 MSK 8h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service +Sat [PRIVATE_IP] 04:39:05 MSK 15h Fri [PRIVATE_IP] 04:28:08 MSK 8h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service +Sat [PRIVATE_IP] 04:45:41 MSK 15h Fri [PRIVATE_IP] 04:46:28 MSK 8h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service +Sat [PRIVATE_IP] 04:50:23 MSK 15h Fri [PRIVATE_IP] 04:49:40 MSK 8h ago homelab-edge-vm-offhost-freshness.timer homelab-edge-vm-offhost-freshness.service +Sat [PRIVATE_IP] 04:58:22 MSK 15h Fri [PRIVATE_IP] 05:12:28 MSK 8h ago homelab-sops-edge-secret-coverage.timer homelab-sops-edge-secret-coverage.service +Sat [PRIVATE_IP] 05:17:01 MSK 15h Fri [PRIVATE_IP] 05:15:28 MSK 8h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service +Sat [PRIVATE_IP] 05:24:41 MSK 16h Fri [PRIVATE_IP] 05:18:04 MSK 7h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service +Sat [PRIVATE_IP] 05:46:41 MSK 16h Fri [PRIVATE_IP] 05:54:15 MSK 7h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service +Sat [PRIVATE_IP] 06:09:38 MSK 16h Fri [PRIVATE_IP] 05:58:49 MSK 7h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service +Sat [PRIVATE_IP] 06:12:31 MSK 16h Fri [PRIVATE_IP] 06:34:50 MSK 6h ago apt-daily-upgrade.timer apt-daily-upgrade.service +Sat [PRIVATE_IP] 06:30:50 MSK 17h Fri [PRIVATE_IP] 06:35:48 MSK 6h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service +Sat [PRIVATE_IP] 06:44:52 MSK 17h Fri [PRIVATE_IP] 06:39:49 MSK 6h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service +Sat [PRIVATE_IP] 06:49:50 MSK 17h Fri [PRIVATE_IP] 06:45:28 MSK 6h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service +Sat [PRIVATE_IP] 06:52:38 MSK 17h Fri [PRIVATE_IP] 06:50:28 MSK 6h ago homelab-beszel-backup.timer homelab-beszel-backup.service +Sat [PRIVATE_IP] 06:54:17 MSK 17h Fri [PRIVATE_IP] 06:41:38 MSK 6h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service +Sat [PRIVATE_IP] 06:56:53 MSK 17h Fri [PRIVATE_IP] 07:04:58 MSK 6h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service +Sat [PRIVATE_IP] 07:01:19 MSK 17h Fri [PRIVATE_IP] 07:10:51 MSK 6h ago homelab-drift-check.timer homelab-drift-check.service +Sat [PRIVATE_IP] 07:02:50 MSK 17h Fri [PRIVATE_IP] 07:04:28 MSK 6h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service +Sat [PRIVATE_IP] 07:14:04 MSK 17h Fri [PRIVATE_IP] 07:15:31 MSK 6h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service +Sat [PRIVATE_IP] 07:33:57 MSK 18h Fri [PRIVATE_IP] 07:24:04 MSK 5h 52min ago homelab-service-registry-check.timer homelab-service-registry-check.service +Sat [PRIVATE_IP] 07:37:08 MSK 18h Fri [PRIVATE_IP] 07:28:04 MSK 5h 48min ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service +Sat [PRIVATE_IP] 07:46:01 MSK 18h Fri [PRIVATE_IP] 07:41:38 MSK 5h 35min ago homelab-dependency-map-check.timer homelab-dependency-map-check.service +Sat [PRIVATE_IP] 07:47:12 MSK 18h Fri [PRIVATE_IP] 07:46:04 MSK 5h 30min ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service +Sat [PRIVATE_IP] 07:47:48 MSK 18h Fri [PRIVATE_IP] 07:57:48 MSK 5h 19min ago homelab-runbook-generate.timer homelab-runbook-generate.service +Sat [PRIVATE_IP] 07:48:02 MSK 18h Fri [PRIVATE_IP] 07:48:15 MSK 5h 28min ago homelab-alerting-health.timer homelab-alerting-health.service +Sat [PRIVATE_IP] 07:49:43 MSK 18h Fri [PRIVATE_IP] 07:43:04 MSK 5h 33min ago homelab-vm-backup-health.timer homelab-vm-backup-health.service +Sat [PRIVATE_IP] 07:51:40 MSK 18h Fri [PRIVATE_IP] 08:05:28 MSK 5h 11min ago homelab-desired-state-sync.timer homelab-desired-state-sync.service +Sat [PRIVATE_IP] 08:01:29 MSK 18h Fri [PRIVATE_IP] 08:01:26 MSK 5h 15min ago homelab-duty-admin-report.timer homelab-duty-admin-report.service +Sat [PRIVATE_IP] 08:05:31 MSK 18h Fri [PRIVATE_IP] 08:10:04 MSK 5h 6min ago homelab-overall-health.timer homelab-overall-health.service +Sat [PRIVATE_IP] 08:08:57 MSK 18h Fri [PRIVATE_IP] 08:20:04 MSK 4h 56min ago homelab-backup-sla-health.timer homelab-backup-sla-health.service +Sat [PRIVATE_IP] 08:15:04 MSK 18h Fri [PRIVATE_IP] 08:25:48 MSK 4h 51min ago homelab-final-readiness-gate.timer homelab-final-readiness-gate.service +Sat [PRIVATE_IP] 08:19:25 MSK 19h Fri [PRIVATE_IP] 08:11:15 MSK 5h 5min ago homelab-kuma-monitor-policy.timer homelab-kuma-monitor-policy.service +Sat [PRIVATE_IP] 08:27:04 MSK 19h Fri [PRIVATE_IP] 08:34:28 MSK 4h 42min ago homelab-capacity-risk.timer homelab-capacity-risk.service +Sat [PRIVATE_IP] 08:30:30 MSK 19h Fri [PRIVATE_IP] 08:26:18 MSK 4h 50min ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service +Sat [PRIVATE_IP] 08:31:42 MSK 19h Fri [PRIVATE_IP] 08:46:05 MSK 4h 30min ago homelab-secret-exposure-guard.timer homelab-secret-exposure-guard.service +Sat [PRIVATE_IP] 08:35:48 MSK 19h Fri [PRIVATE_IP] 08:37:50 MSK 4h 39min ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service +Sat [PRIVATE_IP] 08:36:51 MSK 19h Fri [PRIVATE_IP] 08:35:15 MSK 4h 41min ago memos-backup.timer memos-backup.service +Sat [PRIVATE_IP] 08:41:06 MSK 19h Fri [PRIVATE_IP] 08:44:57 MSK 4h 32min ago homelab-cluster-passport.timer homelab-cluster-passport.service +Sat [PRIVATE_IP] 08:45:03 MSK 19h Fri [PRIVATE_IP] 08:54:27 MSK 4h 22min ago immich-media-offhost-sync.timer immich-media-offhost-sync.service +Sat [PRIVATE_IP] 08:55:59 MSK 19h Fri [PRIVATE_IP] 08:56:28 MSK 4h 20min ago homelab-golden-state-index.timer homelab-golden-state-index.service +Sat [PRIVATE_IP] 09:23:17 MSK 20h Fri [PRIVATE_IP] 09:11:15 MSK 4h 5min ago memos-offhost-sync.timer memos-offhost-sync.service +Sat [PRIVATE_IP] 10:03:38 MSK 20h Fri [PRIVATE_IP] 10:06:04 MSK 3h 10min ago paperless-restore-dry-run.timer paperless-restore-dry-run.service +Sat [PRIVATE_IP] 10:15:00 MSK 20h Fri [PRIVATE_IP] 10:15:04 MSK 3h 1min ago skladchik-reports-monitor-monthly-selftest-watch.timer skladchik-reports-monitor-monthly-selftest-watch.service +Sun [PRIVATE_IP] 03:10:03 MSK 1 day 13h Sun [PRIVATE_IP] 03:10:29 MSK - e2scrub_all.timer e2scrub_all.service +Sun [PRIVATE_IP] 03:10:35 MSK 1 day 13h Sun [PRIVATE_IP] 03:10:14 MSK - xfs_scrub_all.timer xfs_scrub_all.service +Sun [PRIVATE_IP] 03:34:48 MSK 1 day 14h Sun [PRIVATE_IP] 03:45:37 MSK - homelab-auto-backup.timer homelab-auto-backup.service +Sun [PRIVATE_IP] 05:15:13 MSK 1 day 15h Sun [PRIVATE_IP] 05:11:25 MSK - skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service +Sun [PRIVATE_IP] 09:39:32 MSK 1 day 20h Sun [PRIVATE_IP] 09:32:29 MSK - homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service +Sun [PRIVATE_IP] 10:11:29 MSK 1 day 20h Sun [PRIVATE_IP] 10:06:07 MSK - homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service +Sun [PRIVATE_IP] 11:28:24 MSK 1 day 22h Sun [PRIVATE_IP] 11:08:41 MSK - homelab-vm-local-dumps-retention.timer homelab-vm-local-dumps-retention.service +Mon [PRIVATE_IP] 00:07:15 MSK 2 days Mon [PRIVATE_IP] 00:00:29 MSK - homelab-secret-sanity.timer homelab-secret-sanity.service +Mon [PRIVATE_IP] 01:33:11 MSK 2 days Mon [PRIVATE_IP] 00:51:14 MSK - fstrim.timer fstrim.service +Mon [PRIVATE_IP] 07:34:52 MSK 2 days Mon [PRIVATE_IP] 07:25:14 MSK - homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service +Thu [PRIVATE_IP] 06:00:00 MSK 2 months 21 days Wed [PRIVATE_IP] 06:00:00 MSK - homelab-vm-full-restore-drill.timer homelab-vm-full-restore-drill.service +- - Fri [PRIVATE_IP] 13:17:02 MSK 21ms ago homelab-cluster-admin-webpanel-sync.timer homelab-cluster-admin-webpanel-sync.service +- - Fri [PRIVATE_IP] 13:17:02 MSK 20ms ago homelab-private-vpn-hosts-health.timer homelab-private-vpn-hosts-health.service +- - - - prometheus-node-exporter-ipmitool-sensor.timer prometheus-node-exporter-ipmitool-sensor.service +- - - - prometheus-node-exporter-mellanox-hca-temp.timer prometheus-node-exporter-mellanox-hca-temp.service +- - - - prometheus-node-exporter-smartmon.timer prometheus-node-exporter-smartmon.service + +96 timers listed. +- Обрезанная, повреждённая или частично вставленная команда считается невыполненной; сначала проводится side-effect audit. +- Хвост parse-ошибки закрывается только после доказательства отсутствия изменений и временных артефактов. +- Proof: /var/lib/homelab-change-sets/operator-command-size-guard-v[PRIVATE_IP]T182653Z/report.txt + +## HOMELAB_ADMIN_CLI_CONTRACT_[PRIVATE_IP] +- Поддерживаемые команды: version, status, lint TASK_DIR, validate TASK_DIR, run TASK_DIR PHASE. +- Отдельная команда --help отсутствует. Неизвестный аргумент печатает usage и штатно возвращает RC=64. +- Нельзя трактовать RC=64 от --help как неисправность runner. +- Для проверки доступности runner использовать homelab-admin version; для структуры CLI использовать зафиксированную usage-строку. +- Proof: /var/lib/homelab-change-sets/homelab-admin-cli-contract-fix-v[PRIVATE_IP]T183338Z/report.txt + +AUTOMATIC_CLUSTER_CONTEXT_REFRESH_[PRIVATE_IP] +Collector: /usr/local/sbin/homelab-context-collect +Scheduler: hourly at minute 17 via /etc/cron.d/homelab-context-refresh +Destination: immutable Gitea runtime history with CONTEXT-AUTO UTC command IDs. + +AUTOMATIC_CLUSTER_CONTEXT_REFRESH_DIRECT_[PRIVATE_IP] +Publisher: /usr/local/sbin/homelab-context-refresh-direct +Schedule: hourly at minute 17. +Destination: immutable assistant-runtime history with CONTEXT-AUTO UTC IDs. +- Registered at: [PRIVATE_IP]T06:27:06Z + +AUTOREFRESH_INLINE_FAILURE_[PRIVATE_IP] +Attempts 048 and 049 did not appear in immutable history. +Rule: use validated local helper scripts and a simple scheduler entry; never embed the full collector in cron or nested SSH quoting. + +AUTOREFRESH_WRAPPER_PUBLICATION_FAILURE_[PRIVATE_IP] +Commands [PRIVATE_IP] did not reach immutable history through homelab-chat-run. +Resolution: hourly snapshots use direct scp plus homelab-runtime-receive publication. + +AUTOREFRESH_052_NOT_PUBLISHED_[PRIVATE_IP] +CONTEXT-AUTOREFRESH-ACTIVATE-052 did not appear in immutable history; direct publisher retry requires sanitization before transmission. + +IMMUTABLE_HISTORY_FALSE_NEGATIVE_[PRIVATE_IP] +BACKUP-AUDIT-CONTROLPLANE-054 was successfully published at commit 2b966acfe[PRIVATE_IP]d4a62bf[PRIVATE_IP]bdd1a073, but was incorrectly reported missing because a stale directory listing was treated as authoritative. +Rule: verify the exact COMMAND_ID result, wrapper publication status and commit SHA; never infer absence from a cached directory listing. + +BACKUP_UNSAFE_SOURCE_CAPTURE_071_PARSE_ERROR_[PRIVATE_IP] +BACKUP-UNSAFE-SOURCE-CAPTURE-071 failed before execution: shell syntax error near unexpected token (. +Resolution: avoid nested quote layers; parse targets first and pass file paths as simple SSH arguments. +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 4506ee1..aa5cc70 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,30 +1 @@ -{ - "schema_version": 1, - "channel": "homelab-runtime", - "command_id": "HOMELAB-VM-2CLOUD-NOLOCAL-PREFLIGHT-435", - "status": "OK", - "rc": 0, - "host": "pve01", - "mode": "read-only", - "component": "cluster-audit", - "started_at_utc": "2026-07-24T10:15:43Z", - "finished_at_utc": "2026-07-24T10:15:43Z", - "reference_register_checked": true, - "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", - "error_register_checked": true, - "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", - "command_sha256": "86526459d447c1f1c45038a4d8dd89cb4cd68c9982e5bf88c20e2cacfb516c8b", - "duplicate_failed_command_blocked": false, - "block_reason": null, - "execution_started": true, - "changes_made": false, - "sanitized": true, - "secrets_included": false, - "private_addresses_included": false, - "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "90b33c793e8f02c3bbc5dba5e5d5e93c7d4ca4e647513b4cf189ba5db0cd05d7", - "sanitized_output_sha256": "90b33c793e8f02c3bbc5dba5e5d5e93c7d4ca4e647513b4cf189ba5db0cd05d7", - "output_truncated_in_json": false, - "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "DETAIL=RUNTIME_SHA256=265aa081c8c9981ec0724a38febf8d0f4269ed62bb853fce181ae809c291ed17\nDETAIL=DESIRED_SHA256=265aa081c8c9981ec0724a38febf8d0f4269ed62bb853fce181ae809c291ed17\nDETAIL=INSTALLED_LOGIC_BEGIN\nDETAIL=SOURCE=17-BAD=0\nDETAIL=SOURCE=18-COUNT=0\nDETAIL=SOURCE=19-TOTAL=0\nDETAIL=SOURCE=20-CHUNKED=0\nDETAIL=SOURCE=21-VMIDS=\"${VMIDS:-110 112 150}\"\nDETAIL=SOURCE=22-\nDETAIL=SOURCE=23-source /etc/homelab-backup.env 2>/dev/null || BAD=1\nDETAIL=SOURCE=24-\nDETAIL=SOURCE=25-JOB_ID=\"homelab-${NODE}-vm-local-dumps-${JOB_TS}\"\nDETAIL=SOURCE=26-WORK=\"$ROOT/$JOB_ID\"\nDETAIL=SOURCE=27-mkdir -p \"$WORK\"\nDETAIL=SOURCE=28-\nDETAIL=SOURCE=29:FILES_TSV=\"$WORK/files.tsv\"\nDETAIL=SOURCE=30-CHKS=\"$WORK/checksums.txt\"\nDETAIL=SOURCE=31-MAN=\"$WORK/manifest.json\"\nDETAIL=SOURCE=32-\nDETAIL=SOURCE=33:: > \"$FILES_TSV\"\nDETAIL=SOURCE=34-: > \"$CHKS\"\nDETAIL=SOURCE=35-: > \"$LOG\"\nDETAIL=SOURCE=36-\nDETAIL=SOURCE=37-echo \"job_id=$JOB_ID\" >> \"$LOG\"\nDETAIL=SOURCE=38-echo \"vmids=$VMIDS\" >> \"$LOG\"\nDETAIL=SOURCE=39-echo \"chunk_size_bytes=$CHUNK_SIZE_BYTES\" >> \"$LOG\"\nDETAIL=SOURCE=40-\nDETAIL=SOURCE=41-for id in $VMIDS; do\nDETAIL=SOURCE=42: F=\"$(find /var/lib/vz/dump -maxdepth 1 -type f \\( -name \"vzdump-qemu-${id}-*.vma.zst\" -o -name \"vzdump-lxc-${id}-*.tar.zst\" \\) -printf \"%T@ %p\\n\" 2>/dev/null | sort -n | tail -n 1 | cut -d\" \" -f2-)\"\nDETAIL=SOURCE=43- if [ -z \"$F\" ] || [ ! -f \"$F\" ]; then\nDETAIL=SOURCE=44- echo \"MISSING vmid=$id\" >> \"$LOG\"\nDETAIL=SOURCE=45- BAD=$((BAD+1))\nDETAIL=SOURCE=46- continue\nDETAIL=SOURCE=47- fi\nDETAIL=SOURCE=48-\nDETAIL=SOURCE=49- BN=\"$(basename \"$F\")\"\nDETAIL=SOURCE=50- SIZE=\"$(stat -c %s \"$F\" 2>/dev/null || echo 0)\"\nDETAIL=SOURCE=51- MT=\"$(date -u -d \"@$(stat -c %Y \"$F\" 2>/dev/null || echo 0)\" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)\"\nDETAIL=SOURCE=52- KIND=\"unknown\"\nDETAIL=SOURCE=53- echo \"$BN\" | grep -q \"^vzdump-qemu-\" && KIND=\"qemu\"\nDETAIL=SOURCE=54- echo \"$BN\" | grep -q \"^vzdump-lxc-\" && KIND=\"lxc\"\nDETAIL=SOURCE=55- SHA=\"$(sha256sum \"$F\" | cut -d\" \" -f1)\"\nDETAIL=SOURCE=56- printf \"%s %s\\n\" \"$SHA\" \"$BN\" >> \"$CHKS\"\nDETAIL=SOURCE=57-\nDETAIL=SOURCE=58- MODE=\"direct\"\nDETAIL=SOURCE=59- PARTS=0\nDETAIL=SOURCE=60- if [ \"$SIZE\" -gt \"$CHUNK_SIZE_BYTES\" ]; then\nDETAIL=SOURCE=61- MODE=\"chunked\"\nDETAIL=SOURCE=62- CHUNKED=$((CHUNKED+1))\nDETAIL=SOURCE=--\nDETAIL=SOURCE=64- rm -rf \"$CDIR\"\nDETAIL=SOURCE=65- mkdir -p \"$CDIR\"\nDETAIL=SOURCE=66- echo \"SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR\" >> \"$LOG\"\nDETAIL=SOURCE=67- split -b \"$CHUNK_SIZE_SPLIT\" -d -a 4 \"$F\" \"$CDIR/part-\" >> \"$LOG\" 2>&1\nDETAIL=SOURCE=68- PARTS=\"$(find \"$CDIR\" -maxdepth 1 -type f -name 'part-*' | wc -l)\"\nDETAIL=SOURCE=69- (cd \"$CDIR\" && sha256sum part-* > chunks.sha256)\nDETAIL=SOURCE=70- if [ \"$PARTS\" -le 0 ]; then\nDETAIL=SOURCE=71- echo \"BAD split produced no chunks file=$BN\" >> \"$LOG\"\nDETAIL=SOURCE=72- BAD=$((BAD+1))\nDETAIL=SOURCE=73- fi\nDETAIL=SOURCE=74- fi\nDETAIL=SOURCE=75-\nDETAIL=SOURCE=76: printf \"%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n\" \"$id\" \"$KIND\" \"$SIZE\" \"$SHA\" \"$MT\" \"$BN\" \"$F\" \"$MODE\" \"$PARTS\" >> \"$FILES_TSV\"\nDETAIL=SOURCE=77- COUNT=$((COUNT+1))\nDETAIL=SOURCE=78- TOTAL=$((TOTAL+SIZE))\nDETAIL=SOURCE=79-done\nDETAIL=SOURCE=80-\nDETAIL=SOURCE=81:python3 - \"$JOB_ID\" \"$NODE\" \"$TS\" \"$VMIDS\" \"$COUNT\" \"$TOTAL\" \"$CHUNKED\" \"$CHUNK_SIZE_BYTES\" \"$FILES_TSV\" > \"$MAN\" <<'PY'\nDETAIL=SOURCE=82-import json, sys, pathlib\nDETAIL=SOURCE=83-job,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:]\nDETAIL=SOURCE=84-items=[]\nDETAIL=SOURCE=85-for line in pathlib.Path(tsv).read_text().splitlines():\nDETAIL=SOURCE=86- p=line.split(\"\\t\")\nDETAIL=SOURCE=87- if len(p)>=9:\nDETAIL=SOURCE=88- items.append({\nDETAIL=SOURCE=89- \"vmid\":p[0],\nDETAIL=SOURCE=90- \"kind\":p[1],\nDETAIL=SOURCE=91- \"size_bytes\":int(p[2]),\nDETAIL=SOURCE=92- \"sha256\":p[3],\nDETAIL=SOURCE=93- \"mtime_utc\":p[4],\nDETAIL=SOURCE=94- \"filename\":p[5],\nDETAIL=SOURCE=95- \"source_path\":p[6],\nDETAIL=SOURCE=96- \"mode\":p[7],\nDETAIL=SOURCE=97- \"parts\":int(p[8]),\nDETAIL=SOURCE=98- })\nDETAIL=SOURCE=99-print(json.dumps({\nDETAIL=SOURCE=100- \"job_id\":job,\nDETAIL=SOURCE=101- \"node\":node,\nDETAIL=SOURCE=--\nDETAIL=SOURCE=103- \"vmids\":vmids.split(),\nDETAIL=SOURCE=104- \"count\":int(count),\nDETAIL=SOURCE=105- \"total_bytes\":int(total),\nDETAIL=SOURCE=106- \"chunked_count\":int(chunked),\nDETAIL=SOURCE=107- \"chunk_size_bytes\":int(chunk_size),\nDETAIL=SOURCE=108- \"items\":items,\nDETAIL=SOURCE=109-}, indent=2))\nDETAIL=SOURCE=110-PY\nDETAIL=SOURCE=111-\nDETAIL=SOURCE=112-DEST_A=\"${REMOTE_A%/}/${NODE}/vm-local-dumps/${JOB_ID}\"\nDETAIL=SOURCE=113-DEST_B=\"${REMOTE_B%/}/${NODE}/vm-local-dumps/${JOB_ID}\"\nDETAIL=SOURCE=114-\nDETAIL=SOURCE=115:upload_one_dest() {\nDETAIL=SOURCE=116- DEST=\"$1\"\nDETAIL=SOURCE=117- rclone mkdir \"$DEST/files\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE=118- while IFS=\"$(printf \"\\t\")\" read -r id kind size sha mt bn path mode parts; do\nDETAIL=SOURCE=119- if [ \"$mode\" = \"chunked\" ]; then\nDETAIL=SOURCE=120- CDIR=\"$CHUNK_ROOT/$JOB_ID/$bn.chunks\"\nDETAIL=SOURCE=121- echo \"UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts\" >> \"$LOG\"\nDETAIL=SOURCE=122- rclone mkdir \"$DEST/files/$bn.chunks\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE=123- for part in \"$CDIR\"/part-*; do\nDETAIL=SOURCE=124- [ -f \"$part\" ] || continue\nDETAIL=SOURCE=125- pn=\"$(basename \"$part\")\"\nDETAIL=SOURCE=126- rclone copyto \"$part\" \"$DEST/files/$bn.chunks/$pn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE=127- done\nDETAIL=SOURCE=128- rclone copyto \"$CDIR/chunks.sha256\" \"$DEST/files/$bn.chunks/chunks.sha256\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE=129- else\nDETAIL=SOURCE=130- echo \"UPLOAD_DIRECT dest=$DEST file=$bn size=$size\" >> \"$LOG\"\nDETAIL=SOURCE=131- rclone copyto \"$path\" \"$DEST/files/$bn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE=132- fi\nDETAIL=SOURCE=133: done < \"$FILES_TSV\"\nDETAIL=SOURCE=134-\nDETAIL=SOURCE=135- rclone copyto \"$CHKS\" \"$DEST/checksums.txt\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE=136: rclone copyto \"$FILES_TSV\" \"$DEST/files.tsv\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE=137- rclone copyto \"$MAN\" \"$DEST/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE=138-}\nDETAIL=SOURCE=139-\nDETAIL=SOURCE=140:verify_one_dest() {\nDETAIL=SOURCE=141- LABEL=\"$1\"\nDETAIL=SOURCE=142- DEST=\"$2\"\nDETAIL=SOURCE=143- VBASE=\"$VERIFY_ROOT/$JOB_ID/$LABEL\"\nDETAIL=SOURCE=144- rm -rf \"$VBASE\"\nDETAIL=SOURCE=145- mkdir -p \"$VBASE\"\nDETAIL=SOURCE=146- rclone copyto \"$DEST/manifest.json\" \"$VBASE/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE=147- test -s \"$VBASE/manifest.json\" || BAD=$((BAD+1))\nDETAIL=SOURCE=148-\nDETAIL=SOURCE=149- while IFS=\"$(printf \"\\t\")\" read -r id kind size sha mt bn path mode parts; do\nDETAIL=SOURCE=150- if [ \"$mode\" = \"chunked\" ]; then\nDETAIL=SOURCE=151- VDIR=\"$VBASE/$bn.chunks\"\nDETAIL=SOURCE=152- mkdir -p \"$VDIR\"\nDETAIL=SOURCE=153- echo \"VERIFY_CHUNKED label=$LABEL file=$bn parts=$parts\" >> \"$LOG\"\nDETAIL=SOURCE=154- rclone copy \"$DEST/files/$bn.chunks\" \"$VDIR\" --filter \"+ part-*\" --filter \"+ chunks.sha256\" --filter \"- *\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE=155- (cd \"$VDIR\" && sha256sum -c chunks.sha256) >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE=156- CALC=\"$(find \"$VDIR\" -maxdepth 1 -type f -name 'part-*' | sort | xargs cat | sha256sum | cut -d\" \" -f1)\"\nDETAIL=SOURCE=157- if [ \"$CALC\" = \"$sha\" ]; then\nDETAIL=SOURCE=158- echo \"OK combined_sha label=$LABEL file=$bn sha=$sha\" >> \"$LOG\"\nDETAIL=SOURCE=159- else\nDETAIL=SOURCE=160- echo \"BAD combined_sha label=$LABEL file=$bn expected=$sha got=$CALC\" >> \"$LOG\"\nDETAIL=SOURCE=161- BAD=$((BAD+1))\nDETAIL=SOURCE=162- fi\nDETAIL=SOURCE=163- rm -rf \"$VDIR\"\nDETAIL=SOURCE=164- else\nDETAIL=SOURCE=165- echo \"VERIFY_DIRECT label=$LABEL file=$bn\" >> \"$LOG\"\nDETAIL=SOURCE=166- rclone copyto \"$DEST/files/$bn\" \"$VBASE/$bn\" --transfers 1 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE=167- printf \"%s %s\\n\" \"$sha\" \"$VBASE/$bn\" | sha256sum -c - >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE=168- rm -f \"$VBASE/$bn\"\nDETAIL=SOURCE=169- fi\nDETAIL=SOURCE=170: done < \"$FILES_TSV\"\nDETAIL=SOURCE=171- rm -rf \"$VBASE\"\nDETAIL=SOURCE=172-}\nDETAIL=SOURCE=173-\nDETAIL=SOURCE=174-if [ \"$BAD\" -eq 0 ]; then\nDETAIL=SOURCE=175: upload_one_dest \"$DEST_A\"\nDETAIL=SOURCE=176: upload_one_dest \"$DEST_B\"\nDETAIL=SOURCE=177-fi\nDETAIL=SOURCE=178-\nDETAIL=SOURCE=179-if [ \"$BAD\" -eq 0 ]; then\nDETAIL=SOURCE=180: verify_one_dest \"A\" \"$DEST_A\"\nDETAIL=SOURCE=181: verify_one_dest \"B\" \"$DEST_B\"\nDETAIL=SOURCE=182-fi\nDETAIL=SOURCE=183-\nDETAIL=SOURCE=184-rm -rf \"$VERIFY_ROOT/$JOB_ID\"\nDETAIL=SOURCE=185-rm -rf \"$CHUNK_ROOT/$JOB_ID\"\nDETAIL=SOURCE=186-\nDETAIL=SOURCE=187:# HOMELAB_NO_LOCAL_AFTER_DUAL_CLOUD_434\nDETAIL=SOURCE=188-LOCAL_DUMPS_DELETED=false\nDETAIL=SOURCE=189:DELETED_COUNT=0\nDETAIL=SOURCE=190:DELETED_BYTES=0\nDETAIL=SOURCE=191-\nDETAIL=SOURCE=192-if [ \"$BAD\" -eq 0 ]; then\nDETAIL=SOURCE=193: echo \"PREDELETE_VALIDATION_BEGIN\" >> \"$LOG\"\nDETAIL=SOURCE=194-\nDETAIL=SOURCE=195- while IFS=\"$(printf \"\t\")\" read -r id kind size sha mt bn path mode parts; do\nDETAIL=SOURCE=196- case \"$path\" in\nDETAIL=SOURCE=197- /var/lib/vz/dump/*)\nDETAIL=SOURCE=198- ;;\nDETAIL=SOURCE=199- *)\nDETAIL=SOURCE=200- echo \"REFUSE_DELETE_OUTSIDE_ALLOWED_ROOT path=$path\" >> \"$LOG\"\nDETAIL=SOURCE=201- BAD=$((BAD+1))\nDETAIL=SOURCE=202- continue\nDETAIL=SOURCE=203- ;;\nDETAIL=SOURCE=204- esac\nDETAIL=SOURCE=205-\nDETAIL=SOURCE=206- if [ \"$(basename \"$path\")\" != \"$bn\" ]; then\nDETAIL=SOURCE=207- echo \"REFUSE_DELETE_BASENAME_MISMATCH path=$path expected=$bn\" >> \"$LOG\"\nDETAIL=SOURCE=208- BAD=$((BAD+1))\nDETAIL=SOURCE=209- continue\nDETAIL=SOURCE=210- fi\nDETAIL=SOURCE=211-\nDETAIL=SOURCE=212- if [ ! -f \"$path\" ] || [ -L \"$path\" ]; then\nDETAIL=SOURCE=213- echo \"REFUSE_DELETE_INVALID_SOURCE path=$path\" >> \"$LOG\"\nDETAIL=SOURCE=--\nDETAIL=SOURCE=222- BAD=$((BAD+1))\nDETAIL=SOURCE=223- continue\nDETAIL=SOURCE=224- fi\nDETAIL=SOURCE=225-\nDETAIL=SOURCE=226- actual_sha=\"$(sha256sum \"$path\" 2>>\"$LOG\" | cut -d \" \" -f1 || true)\"\nDETAIL=SOURCE=227-\nDETAIL=SOURCE=228- if [ \"$actual_sha\" != \"$sha\" ]; then\nDETAIL=SOURCE=229- echo \"REFUSE_DELETE_SHA_MISMATCH path=$path\" >> \"$LOG\"\nDETAIL=SOURCE=230- BAD=$((BAD+1))\nDETAIL=SOURCE=231- continue\nDETAIL=SOURCE=232- fi\nDETAIL=SOURCE=233-\nDETAIL=SOURCE=234: echo \"PREDELETE_VALIDATION_OK path=$path size=$size sha=$sha\" >> \"$LOG\"\nDETAIL=SOURCE=235: done < \"$FILES_TSV\"\nDETAIL=SOURCE=236-\nDETAIL=SOURCE=237: echo \"PREDELETE_VALIDATION_END bad=$BAD\" >> \"$LOG\"\nDETAIL=SOURCE=238-fi\nDETAIL=SOURCE=239-\nDETAIL=SOURCE=240-if [ \"$BAD\" -eq 0 ]; then\nDETAIL=SOURCE=241: echo \"LOCAL_PAYLOAD_DELETE_BEGIN\" >> \"$LOG\"\nDETAIL=SOURCE=242-\nDETAIL=SOURCE=243- while IFS=\"$(printf \"\t\")\" read -r id kind size sha mt bn path mode parts; do\nDETAIL=SOURCE=244: if rm -f -- \"$path\" 2>>\"$LOG\"; then\nDETAIL=SOURCE=245- if [ ! -e \"$path\" ] && [ ! -L \"$path\" ]; then\nDETAIL=SOURCE=246: DELETED_COUNT=$((DELETED_COUNT+1))\nDETAIL=SOURCE=247: DELETED_BYTES=$((DELETED_BYTES+size))\nDETAIL=SOURCE=248: echo \"LOCAL_PAYLOAD_DELETED path=$path bytes=$size\" >> \"$LOG\"\nDETAIL=SOURCE=249- else\nDETAIL=SOURCE=250: echo \"LOCAL_PAYLOAD_DELETE_CONFIRM_FAILED path=$path\" >> \"$LOG\"\nDETAIL=SOURCE=251- BAD=$((BAD+1))\nDETAIL=SOURCE=252- fi\nDETAIL=SOURCE=253- else\nDETAIL=SOURCE=254: echo \"LOCAL_PAYLOAD_DELETE_FAILED path=$path\" >> \"$LOG\"\nDETAIL=SOURCE=255- BAD=$((BAD+1))\nDETAIL=SOURCE=256- fi\nDETAIL=SOURCE=257: done < \"$FILES_TSV\"\nDETAIL=SOURCE=258-\nDETAIL=SOURCE=259: if [ \"$BAD\" -eq 0 ] && [ \"$DELETED_COUNT\" -eq \"$COUNT\" ]; then\nDETAIL=SOURCE=260- LOCAL_DUMPS_DELETED=true\nDETAIL=SOURCE=261- fi\nDETAIL=SOURCE=262-\nDETAIL=SOURCE=263: echo \"LOCAL_PAYLOAD_DELETE_END deleted_count=$DELETED_COUNT deleted_bytes=$DELETED_BYTES bad=$BAD\" >> \"$LOG\"\nDETAIL=SOURCE=264-fi\nDETAIL=SOURCE=265-\nDETAIL=SOURCE=266-if [ \"$BAD\" -eq 0 ] && [ \"$LOCAL_DUMPS_DELETED\" = true ]; then\nDETAIL=SOURCE=267- STATUS=\"OK\"\nDETAIL=SOURCE=268-else\nDETAIL=SOURCE=269- STATUS=\"WARN\"\nDETAIL=SOURCE=270-fi\nDETAIL=SOURCE=271-\nDETAIL=SOURCE=272-jq -n \\\nDETAIL=SOURCE=273- --arg job_id \"$JOB_ID\" \\\nDETAIL=SOURCE=274- --arg status \"$STATUS\" \\\nDETAIL=SOURCE=275- --arg node \"$NODE\" \\\nDETAIL=SOURCE=276- --arg remote_a \"$DEST_A\" \\\nDETAIL=SOURCE=277- --arg remote_b \"$DEST_B\" \\\nDETAIL=SOURCE=278- --argjson count \"$COUNT\" \\\nDETAIL=SOURCE=279- --argjson total_bytes \"$TOTAL\" \\\nDETAIL=SOURCE=280- --argjson chunked_count \"$CHUNKED\" \\\nDETAIL=SOURCE=281- --argjson chunk_size_bytes \"$CHUNK_SIZE_BYTES\" \\\nDETAIL=SOURCE=282- --arg log \"$LOG\" \\\nDETAIL=SOURCE=283: --argjson local_dumps_deleted \"$LOCAL_DUMPS_DELETED\" \\\nDETAIL=SOURCE=284: --argjson deleted_count \"$DELETED_COUNT\" \\\nDETAIL=SOURCE=285: --argjson deleted_bytes \"$DELETED_BYTES\" \\\nDETAIL=SOURCE=286: \"{job_id:\\$job_id,status:\\$status,node:\\$node,remote_a:\\$remote_a,remote_b:\\$remote_b,count:\\$count,total_bytes:\\$total_bytes,chunked_count:\\$chunked_count,chunk_size_bytes:\\$chunk_size_bytes,log:\\$log,local_dumps_deleted:$local_dumps_deleted,deleted_count:$deleted_count,deleted_bytes:$deleted_bytes}\" \\\nDETAIL=SOURCE=287- > \"$ROOT/latest-vm-local-dumps-cloud.json\"\nDETAIL=SOURCE=288-\nDETAIL=SOURCE=289:echo \"STATUS=$STATUS TS=$TS TYPE=vm-local-dumps-cloud BAD=$BAD COUNT=$COUNT CHUNKED=$CHUNKED TOTAL_BYTES=$TOTAL JOB=$JOB_ID REMOTE_A=$DEST_A REMOTE_B=$DEST_B LOCAL_DUMPS_DELETED=$LOCAL_DUMPS_DELETED DELETED_COUNT=$DELETED_COUNT DELETED_BYTES=$DELETED_BYTES LOG=$LOG\" > \"$OUT\"\nDETAIL=SOURCE=290-cat \"$OUT\"\nDETAIL=INSTALLED_LOGIC_END\nDETAIL=UNIT_STATE_BEGIN\nDETAIL=SERVICE_PROPERTY=Result=success\nDETAIL=SERVICE_PROPERTY=ExecMainStatus=0\nDETAIL=SERVICE_PROPERTY=ExecStart={ path=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; argv[]=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nDETAIL=SERVICE_PROPERTY=LoadState=loaded\nDETAIL=SERVICE_PROPERTY=ActiveState=inactive\nDETAIL=SERVICE_PROPERTY=SubState=dead\nDETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/homelab-vm-local-dumps-2cloud.service\nDETAIL=TIMER_PROPERTY=Unit=homelab-vm-local-dumps-2cloud.service\nDETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Sun 2026-07-26 10:11:29 MSK\nDETAIL=TIMER_PROPERTY=LastTriggerUSec=Sun 2026-07-19 10:06:07 MSK\nDETAIL=TIMER_PROPERTY=LoadState=loaded\nDETAIL=TIMER_PROPERTY=ActiveState=active\nDETAIL=TIMER_PROPERTY=SubState=waiting\nDETAIL=TIMER_ENABLED=enabled\nDETAIL=TIMER_ACTIVE=active\nDETAIL=UNIT_STATE_END\nDETAIL=CURRENT_DUMP_INVENTORY_BEGIN\nDETAIL=CURRENT_DUMP_COUNT=12\nDETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst|BYTES:329541843|MODE:644|OWNER:root:root|MTIME:2026-07-21 03:30:09.768869633 +0300\nDETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-110-2026_07_22-03_30_01.tar.zst|BYTES:332951369|MODE:644|OWNER:root:root|MTIME:2026-07-22 03:30:08.768607650 +0300\nDETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-110-2026_07_23-03_30_02.tar.zst|BYTES:336245409|MODE:644|OWNER:root:root|MTIME:2026-07-23 03:30:10.836949127 +0300\nDETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-110-2026_07_24-03_30_02.tar.zst|BYTES:339140622|MODE:644|OWNER:root:root|MTIME:2026-07-24 03:30:10.776974593 +0300\nDETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst|BYTES:222777732|MODE:644|OWNER:root:root|MTIME:2026-07-21 03:30:15.905963829 +0300\nDETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-112-2026_07_22-03_30_09.tar.zst|BYTES:222794054|MODE:644|OWNER:root:root|MTIME:2026-07-22 03:30:14.284693312 +0300\nDETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-112-2026_07_23-03_30_11.tar.zst|BYTES:222808025|MODE:644|OWNER:root:root|MTIME:2026-07-23 03:30:16.408035215 +0300\nDETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-112-2026_07_24-03_30_11.tar.zst|BYTES:222859457|MODE:644|OWNER:root:root|MTIME:2026-07-24 03:30:16.202058480 +0300\nDETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst|BYTES:7021952392|MODE:644|OWNER:root:root|MTIME:2026-07-21 03:31:06.571741495 +0300\nDETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-qemu-150-2026_07_22-03_30_14.vma.zst|BYTES:7052176114|MODE:644|OWNER:root:root|MTIME:2026-07-22 03:31:02.895448212 +0300\nDETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-qemu-150-2026_07_23-03_30_16.vma.zst|BYTES:7024165999|MODE:644|OWNER:root:root|MTIME:2026-07-23 03:31:08.099833995 +0300\nDETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-qemu-150-2026_07_24-03_30_16.vma.zst|BYTES:7016624549|MODE:644|OWNER:root:root|MTIME:2026-07-24 03:31:04.670807942 +0300\nDETAIL=CURRENT_DUMP_TOTAL_BYTES=30344037565\nDETAIL=CURRENT_DUMP_INVENTORY_END\nDETAIL=LAST_STATE_BEGIN\nDETAIL=LAST_STATE={\"status\":\"OK\",\"job_id\":\"homelab-pve01-vm-local-dumps-20260719T070607Z\",\"count\":3,\"total_bytes\":7567711895,\"local_dumps_deleted\":false,\"deleted_count\":null,\"deleted_bytes\":null,\"remote_a\":\"pve01-mail-01-crypt:homelab-backups/pve01/vm-local-dumps/homelab-pve01-vm-local-dumps-20260719T070607Z\",\"remote_b\":\"pve01-mail-02-crypt:homelab-backups/pve01/vm-local-dumps/homelab-pve01-vm-local-dumps-20260719T070607Z\"}\nDETAIL=LAST_STATE_END\nDETAIL=DELETE_POLICY_VALIDATION=PASS\nDETAIL=BACKUP_PAYLOAD_CHANGED=NO\nDETAIL=SECRET_VALUES_OUTPUT=NONE\nCHANGES_MADE=NO\n" -} +{"schema_version":1,"channel":"homelab-runtime","command_id":"CONTEXT-AUTO-20260724T101701Z","status":"OK","rc":0,"host":"pve01","mode":"read-only","component":"cluster-context","started_at_utc":"2026-07-24T10:17:01Z","finished_at_utc":"2026-07-24T10:17:02Z","reference_register_checked":true,"reference_sha256":"5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e","error_register_checked":true,"error_register_sha256":"08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752","changes_made":false,"sanitized":true,"secrets_included":false,"private_addresses_included":false,"output":"AUTOMATIC_CLUSTER_CONTEXT_REFRESH=OK"} diff --git a/runtime/latest.txt b/runtime/latest.txt index 10def39..8036d07 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,335 +1,200 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=HOMELAB-VM-2CLOUD-NOLOCAL-PREFLIGHT-435 +COMMAND_ID=CONTEXT-AUTO-[PRIVATE_IP]701Z STATUS=OK RC=0 HOST=pve01 -MODE=read-only -COMPONENT=cluster-audit -REFERENCE_REGISTER_CHECK=OK -REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e -ERROR_REGISTER_CHECK=OK -ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 -COMMAND_SHA256=86526459d447c1f1c45038a4d8dd89cb4cd68c9982e5bf88c20e2cacfb516c8b -DUPLICATE_FAILED_COMMAND_BLOCKED=false -EXECUTION_STARTED=true -CHANGES_MADE=false -SANITIZED=yes -SECRETS_INCLUDED=no -PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=90b33c793e8f02c3bbc5dba5e5d5e93c7d4ca4e647513b4cf189ba5db0cd05d7 -SANITIZED_OUTPUT_SHA256=90b33c793e8f02c3bbc5dba5e5d5e93c7d4ca4e647513b4cf189ba5db0cd05d7 +COMPONENT=cluster-context +REFERENCE_SHA[PRIVATE_IP]f8edc75fcf[PRIVATE_IP]c[PRIVATE_IP]cc112ceccb6cf[PRIVATE_IP]a4deb8e +ERROR_REGISTER_SHA[PRIVATE_IP]b8e38eaa1acec915352fb[PRIVATE_IP]caa7ff6cd9d1ec0da[PRIVATE_IP] OUTPUT_BEGIN -DETAIL=RUNTIME_SHA256=265aa081c8c9981ec0724a38febf8d0f4269ed62bb853fce181ae809c291ed17 -DETAIL=DESIRED_SHA256=265aa081c8c9981ec0724a38febf8d0f4269ed62bb853fce181ae809c291ed17 -DETAIL=INSTALLED_LOGIC_BEGIN -DETAIL=SOURCE=17-BAD=0 -DETAIL=SOURCE=18-COUNT=0 -DETAIL=SOURCE=19-TOTAL=0 -DETAIL=SOURCE=20-CHUNKED=0 -DETAIL=SOURCE=21-VMIDS="${VMIDS:-110 112 150}" -DETAIL=SOURCE=22- -DETAIL=SOURCE=23-source /etc/homelab-backup.env 2>/dev/null || BAD=1 -DETAIL=SOURCE=24- -DETAIL=SOURCE=25-JOB_ID="homelab-${NODE}-vm-local-dumps-${JOB_TS}" -DETAIL=SOURCE=26-WORK="$ROOT/$JOB_ID" -DETAIL=SOURCE=27-mkdir -p "$WORK" -DETAIL=SOURCE=28- -DETAIL=SOURCE=29:FILES_TSV="$WORK/files.tsv" -DETAIL=SOURCE=30-CHKS="$WORK/checksums.txt" -DETAIL=SOURCE=31-MAN="$WORK/manifest.json" -DETAIL=SOURCE=32- -DETAIL=SOURCE=33:: > "$FILES_TSV" -DETAIL=SOURCE=34-: > "$CHKS" -DETAIL=SOURCE=35-: > "$LOG" -DETAIL=SOURCE=36- -DETAIL=SOURCE=37-echo "job_id=$JOB_ID" >> "$LOG" -DETAIL=SOURCE=38-echo "vmids=$VMIDS" >> "$LOG" -DETAIL=SOURCE=39-echo "chunk_size_bytes=$CHUNK_SIZE_BYTES" >> "$LOG" -DETAIL=SOURCE=40- -DETAIL=SOURCE=41-for id in $VMIDS; do -DETAIL=SOURCE=42: F="$(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-${id}-*.vma.zst" -o -name "vzdump-lxc-${id}-*.tar.zst" \) -printf "%T@ %p\n" 2>/dev/null | sort -n | tail -n 1 | cut -d" " -f2-)" -DETAIL=SOURCE=43- if [ -z "$F" ] || [ ! -f "$F" ]; then -DETAIL=SOURCE=44- echo "MISSING vmid=$id" >> "$LOG" -DETAIL=SOURCE=45- BAD=$((BAD+1)) -DETAIL=SOURCE=46- continue -DETAIL=SOURCE=47- fi -DETAIL=SOURCE=48- -DETAIL=SOURCE=49- BN="$(basename "$F")" -DETAIL=SOURCE=50- SIZE="$(stat -c %s "$F" 2>/dev/null || echo 0)" -DETAIL=SOURCE=51- MT="$(date -u -d "@$(stat -c %Y "$F" 2>/dev/null || echo 0)" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)" -DETAIL=SOURCE=52- KIND="unknown" -DETAIL=SOURCE=53- echo "$BN" | grep -q "^vzdump-qemu-" && KIND="qemu" -DETAIL=SOURCE=54- echo "$BN" | grep -q "^vzdump-lxc-" && KIND="lxc" -DETAIL=SOURCE=55- SHA="$(sha256sum "$F" | cut -d" " -f1)" -DETAIL=SOURCE=56- printf "%s %s\n" "$SHA" "$BN" >> "$CHKS" -DETAIL=SOURCE=57- -DETAIL=SOURCE=58- MODE="direct" -DETAIL=SOURCE=59- PARTS=0 -DETAIL=SOURCE=60- if [ "$SIZE" -gt "$CHUNK_SIZE_BYTES" ]; then -DETAIL=SOURCE=61- MODE="chunked" -DETAIL=SOURCE=62- CHUNKED=$((CHUNKED+1)) -DETAIL=SOURCE=-- -DETAIL=SOURCE=64- rm -rf "$CDIR" -DETAIL=SOURCE=65- mkdir -p "$CDIR" -DETAIL=SOURCE=66- echo "SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR" >> "$LOG" -DETAIL=SOURCE=67- split -b "$CHUNK_SIZE_SPLIT" -d -a 4 "$F" "$CDIR/part-" >> "$LOG" 2>&1 -DETAIL=SOURCE=68- PARTS="$(find "$CDIR" -maxdepth 1 -type f -name 'part-*' | wc -l)" -DETAIL=SOURCE=69- (cd "$CDIR" && sha256sum part-* > chunks.sha256) -DETAIL=SOURCE=70- if [ "$PARTS" -le 0 ]; then -DETAIL=SOURCE=71- echo "BAD split produced no chunks file=$BN" >> "$LOG" -DETAIL=SOURCE=72- BAD=$((BAD+1)) -DETAIL=SOURCE=73- fi -DETAIL=SOURCE=74- fi -DETAIL=SOURCE=75- -DETAIL=SOURCE=76: printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" "$id" "$KIND" "$SIZE" "$SHA" "$MT" "$BN" "$F" "$MODE" "$PARTS" >> "$FILES_TSV" -DETAIL=SOURCE=77- COUNT=$((COUNT+1)) -DETAIL=SOURCE=78- TOTAL=$((TOTAL+SIZE)) -DETAIL=SOURCE=79-done -DETAIL=SOURCE=80- -DETAIL=SOURCE=81:python3 - "$JOB_ID" "$NODE" "$TS" "$VMIDS" "$COUNT" "$TOTAL" "$CHUNKED" "$CHUNK_SIZE_BYTES" "$FILES_TSV" > "$MAN" <<'PY' -DETAIL=SOURCE=82-import json, sys, pathlib -DETAIL=SOURCE=83-job,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:] -DETAIL=SOURCE=84-items=[] -DETAIL=SOURCE=85-for line in pathlib.Path(tsv).read_text().splitlines(): -DETAIL=SOURCE=86- p=line.split("\t") -DETAIL=SOURCE=87- if len(p)>=9: -DETAIL=SOURCE=88- items.append({ -DETAIL=SOURCE=89- "vmid":p[0], -DETAIL=SOURCE=90- "kind":p[1], -DETAIL=SOURCE=91- "size_bytes":int(p[2]), -DETAIL=SOURCE=92- "sha256":p[3], -DETAIL=SOURCE=93- "mtime_utc":p[4], -DETAIL=SOURCE=94- "filename":p[5], -DETAIL=SOURCE=95- "source_path":p[6], -DETAIL=SOURCE=96- "mode":p[7], -DETAIL=SOURCE=97- "parts":int(p[8]), -DETAIL=SOURCE=98- }) -DETAIL=SOURCE=99-print(json.dumps({ -DETAIL=SOURCE=100- "job_id":job, -DETAIL=SOURCE=101- "node":node, -DETAIL=SOURCE=-- -DETAIL=SOURCE=103- "vmids":vmids.split(), -DETAIL=SOURCE=104- "count":int(count), -DETAIL=SOURCE=105- "total_bytes":int(total), -DETAIL=SOURCE=106- "chunked_count":int(chunked), -DETAIL=SOURCE=107- "chunk_size_bytes":int(chunk_size), -DETAIL=SOURCE=108- "items":items, -DETAIL=SOURCE=109-}, indent=2)) -DETAIL=SOURCE=110-PY -DETAIL=SOURCE=111- -DETAIL=SOURCE=112-DEST_A="${REMOTE_A%/}/${NODE}/vm-local-dumps/${JOB_ID}" -DETAIL=SOURCE=113-DEST_B="${REMOTE_B%/}/${NODE}/vm-local-dumps/${JOB_ID}" -DETAIL=SOURCE=114- -DETAIL=SOURCE=115:upload_one_dest() { -DETAIL=SOURCE=116- DEST="$1" -DETAIL=SOURCE=117- rclone mkdir "$DEST/files" >> "$LOG" 2>&1 || BAD=$((BAD+1)) -DETAIL=SOURCE=118- while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do -DETAIL=SOURCE=119- if [ "$mode" = "chunked" ]; then -DETAIL=SOURCE=120- CDIR="$CHUNK_ROOT/$JOB_ID/$bn.chunks" -DETAIL=SOURCE=121- echo "UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts" >> "$LOG" -DETAIL=SOURCE=122- rclone mkdir "$DEST/files/$bn.chunks" >> "$LOG" 2>&1 || BAD=$((BAD+1)) -DETAIL=SOURCE=123- for part in "$CDIR"/part-*; do -DETAIL=SOURCE=124- [ -f "$part" ] || continue -DETAIL=SOURCE=125- pn="$(basename "$part")" -DETAIL=SOURCE=126- rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) -DETAIL=SOURCE=127- done -DETAIL=SOURCE=128- rclone copyto "$CDIR/chunks.sha256" "$DEST/files/$bn.chunks/chunks.sha256" >> "$LOG" 2>&1 || BAD=$((BAD+1)) -DETAIL=SOURCE=129- else -DETAIL=SOURCE=130- echo "UPLOAD_DIRECT dest=$DEST file=$bn size=$size" >> "$LOG" -DETAIL=SOURCE=131- rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) -DETAIL=SOURCE=132- fi -DETAIL=SOURCE=133: done < "$FILES_TSV" -DETAIL=SOURCE=134- -DETAIL=SOURCE=135- rclone copyto "$CHKS" "$DEST/checksums.txt" >> "$LOG" 2>&1 || BAD=$((BAD+1)) -DETAIL=SOURCE=136: rclone copyto "$FILES_TSV" "$DEST/files.tsv" >> "$LOG" 2>&1 || BAD=$((BAD+1)) -DETAIL=SOURCE=137- rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1)) -DETAIL=SOURCE=138-} -DETAIL=SOURCE=139- -DETAIL=SOURCE=140:verify_one_dest() { -DETAIL=SOURCE=141- LABEL="$1" -DETAIL=SOURCE=142- DEST="$2" -DETAIL=SOURCE=143- VBASE="$VERIFY_ROOT/$JOB_ID/$LABEL" -DETAIL=SOURCE=144- rm -rf "$VBASE" -DETAIL=SOURCE=145- mkdir -p "$VBASE" -DETAIL=SOURCE=146- rclone copyto "$DEST/manifest.json" "$VBASE/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1)) -DETAIL=SOURCE=147- test -s "$VBASE/manifest.json" || BAD=$((BAD+1)) -DETAIL=SOURCE=148- -DETAIL=SOURCE=149- while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do -DETAIL=SOURCE=150- if [ "$mode" = "chunked" ]; then -DETAIL=SOURCE=151- VDIR="$VBASE/$bn.chunks" -DETAIL=SOURCE=152- mkdir -p "$VDIR" -DETAIL=SOURCE=153- echo "VERIFY_CHUNKED label=$LABEL file=$bn parts=$parts" >> "$LOG" -DETAIL=SOURCE=154- rclone copy "$DEST/files/$bn.chunks" "$VDIR" --filter "+ part-*" --filter "+ chunks.sha256" --filter "- *" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) -DETAIL=SOURCE=155- (cd "$VDIR" && sha256sum -c chunks.sha256) >> "$LOG" 2>&1 || BAD=$((BAD+1)) -DETAIL=SOURCE=156- CALC="$(find "$VDIR" -maxdepth 1 -type f -name 'part-*' | sort | xargs cat | sha256sum | cut -d" " -f1)" -DETAIL=SOURCE=157- if [ "$CALC" = "$sha" ]; then -DETAIL=SOURCE=158- echo "OK combined_sha label=$LABEL file=$bn sha=$sha" >> "$LOG" -DETAIL=SOURCE=159- else -DETAIL=SOURCE=160- echo "BAD combined_sha label=$LABEL file=$bn expected=$sha got=$CALC" >> "$LOG" -DETAIL=SOURCE=161- BAD=$((BAD+1)) -DETAIL=SOURCE=162- fi -DETAIL=SOURCE=163- rm -rf "$VDIR" -DETAIL=SOURCE=164- else -DETAIL=SOURCE=165- echo "VERIFY_DIRECT label=$LABEL file=$bn" >> "$LOG" -DETAIL=SOURCE=166- rclone copyto "$DEST/files/$bn" "$VBASE/$bn" --transfers 1 >> "$LOG" 2>&1 || BAD=$((BAD+1)) -DETAIL=SOURCE=167- printf "%s %s\n" "$sha" "$VBASE/$bn" | sha256sum -c - >> "$LOG" 2>&1 || BAD=$((BAD+1)) -DETAIL=SOURCE=168- rm -f "$VBASE/$bn" -DETAIL=SOURCE=169- fi -DETAIL=SOURCE=170: done < "$FILES_TSV" -DETAIL=SOURCE=171- rm -rf "$VBASE" -DETAIL=SOURCE=172-} -DETAIL=SOURCE=173- -DETAIL=SOURCE=174-if [ "$BAD" -eq 0 ]; then -DETAIL=SOURCE=175: upload_one_dest "$DEST_A" -DETAIL=SOURCE=176: upload_one_dest "$DEST_B" -DETAIL=SOURCE=177-fi -DETAIL=SOURCE=178- -DETAIL=SOURCE=179-if [ "$BAD" -eq 0 ]; then -DETAIL=SOURCE=180: verify_one_dest "A" "$DEST_A" -DETAIL=SOURCE=181: verify_one_dest "B" "$DEST_B" -DETAIL=SOURCE=182-fi -DETAIL=SOURCE=183- -DETAIL=SOURCE=184-rm -rf "$VERIFY_ROOT/$JOB_ID" -DETAIL=SOURCE=185-rm -rf "$CHUNK_ROOT/$JOB_ID" -DETAIL=SOURCE=186- -DETAIL=SOURCE=187:# HOMELAB_NO_LOCAL_AFTER_DUAL_CLOUD_434 -DETAIL=SOURCE=188-LOCAL_DUMPS_DELETED=false -DETAIL=SOURCE=189:DELETED_COUNT=0 -DETAIL=SOURCE=190:DELETED_BYTES=0 -DETAIL=SOURCE=191- -DETAIL=SOURCE=192-if [ "$BAD" -eq 0 ]; then -DETAIL=SOURCE=193: echo "PREDELETE_VALIDATION_BEGIN" >> "$LOG" -DETAIL=SOURCE=194- -DETAIL=SOURCE=195- while IFS="$(printf " ")" read -r id kind size sha mt bn path mode parts; do -DETAIL=SOURCE=196- case "$path" in -DETAIL=SOURCE=197- /var/lib/vz/dump/*) -DETAIL=SOURCE=198- ;; -DETAIL=SOURCE=199- *) -DETAIL=SOURCE=200- echo "REFUSE_DELETE_OUTSIDE_ALLOWED_ROOT path=$path" >> "$LOG" -DETAIL=SOURCE=201- BAD=$((BAD+1)) -DETAIL=SOURCE=202- continue -DETAIL=SOURCE=203- ;; -DETAIL=SOURCE=204- esac -DETAIL=SOURCE=205- -DETAIL=SOURCE=206- if [ "$(basename "$path")" != "$bn" ]; then -DETAIL=SOURCE=207- echo "REFUSE_DELETE_BASENAME_MISMATCH path=$path expected=$bn" >> "$LOG" -DETAIL=SOURCE=208- BAD=$((BAD+1)) -DETAIL=SOURCE=209- continue -DETAIL=SOURCE=210- fi -DETAIL=SOURCE=211- -DETAIL=SOURCE=212- if [ ! -f "$path" ] || [ -L "$path" ]; then -DETAIL=SOURCE=213- echo "REFUSE_DELETE_INVALID_SOURCE path=$path" >> "$LOG" -DETAIL=SOURCE=-- -DETAIL=SOURCE=222- BAD=$((BAD+1)) -DETAIL=SOURCE=223- continue -DETAIL=SOURCE=224- fi -DETAIL=SOURCE=225- -DETAIL=SOURCE=226- actual_sha="$(sha256sum "$path" 2>>"$LOG" | cut -d " " -f1 || true)" -DETAIL=SOURCE=227- -DETAIL=SOURCE=228- if [ "$actual_sha" != "$sha" ]; then -DETAIL=SOURCE=229- echo "REFUSE_DELETE_SHA_MISMATCH path=$path" >> "$LOG" -DETAIL=SOURCE=230- BAD=$((BAD+1)) -DETAIL=SOURCE=231- continue -DETAIL=SOURCE=232- fi -DETAIL=SOURCE=233- -DETAIL=SOURCE=234: echo "PREDELETE_VALIDATION_OK path=$path size=$size sha=$sha" >> "$LOG" -DETAIL=SOURCE=235: done < "$FILES_TSV" -DETAIL=SOURCE=236- -DETAIL=SOURCE=237: echo "PREDELETE_VALIDATION_END bad=$BAD" >> "$LOG" -DETAIL=SOURCE=238-fi -DETAIL=SOURCE=239- -DETAIL=SOURCE=240-if [ "$BAD" -eq 0 ]; then -DETAIL=SOURCE=241: echo "LOCAL_PAYLOAD_DELETE_BEGIN" >> "$LOG" -DETAIL=SOURCE=242- -DETAIL=SOURCE=243- while IFS="$(printf " ")" read -r id kind size sha mt bn path mode parts; do -DETAIL=SOURCE=244: if rm -f -- "$path" 2>>"$LOG"; then -DETAIL=SOURCE=245- if [ ! -e "$path" ] && [ ! -L "$path" ]; then -DETAIL=SOURCE=246: DELETED_COUNT=$((DELETED_COUNT+1)) -DETAIL=SOURCE=247: DELETED_BYTES=$((DELETED_BYTES+size)) -DETAIL=SOURCE=248: echo "LOCAL_PAYLOAD_DELETED path=$path bytes=$size" >> "$LOG" -DETAIL=SOURCE=249- else -DETAIL=SOURCE=250: echo "LOCAL_PAYLOAD_DELETE_CONFIRM_FAILED path=$path" >> "$LOG" -DETAIL=SOURCE=251- BAD=$((BAD+1)) -DETAIL=SOURCE=252- fi -DETAIL=SOURCE=253- else -DETAIL=SOURCE=254: echo "LOCAL_PAYLOAD_DELETE_FAILED path=$path" >> "$LOG" -DETAIL=SOURCE=255- BAD=$((BAD+1)) -DETAIL=SOURCE=256- fi -DETAIL=SOURCE=257: done < "$FILES_TSV" -DETAIL=SOURCE=258- -DETAIL=SOURCE=259: if [ "$BAD" -eq 0 ] && [ "$DELETED_COUNT" -eq "$COUNT" ]; then -DETAIL=SOURCE=260- LOCAL_DUMPS_DELETED=true -DETAIL=SOURCE=261- fi -DETAIL=SOURCE=262- -DETAIL=SOURCE=263: echo "LOCAL_PAYLOAD_DELETE_END deleted_count=$DELETED_COUNT deleted_bytes=$DELETED_BYTES bad=$BAD" >> "$LOG" -DETAIL=SOURCE=264-fi -DETAIL=SOURCE=265- -DETAIL=SOURCE=266-if [ "$BAD" -eq 0 ] && [ "$LOCAL_DUMPS_DELETED" = true ]; then -DETAIL=SOURCE=267- STATUS="OK" -DETAIL=SOURCE=268-else -DETAIL=SOURCE=269- STATUS="WARN" -DETAIL=SOURCE=270-fi -DETAIL=SOURCE=271- -DETAIL=SOURCE=272-jq -n \ -DETAIL=SOURCE=273- --arg job_id "$JOB_ID" \ -DETAIL=SOURCE=274- --arg status "$STATUS" \ -DETAIL=SOURCE=275- --arg node "$NODE" \ -DETAIL=SOURCE=276- --arg remote_a "$DEST_A" \ -DETAIL=SOURCE=277- --arg remote_b "$DEST_B" \ -DETAIL=SOURCE=278- --argjson count "$COUNT" \ -DETAIL=SOURCE=279- --argjson total_bytes "$TOTAL" \ -DETAIL=SOURCE=280- --argjson chunked_count "$CHUNKED" \ -DETAIL=SOURCE=281- --argjson chunk_size_bytes "$CHUNK_SIZE_BYTES" \ -DETAIL=SOURCE=282- --arg log "$LOG" \ -DETAIL=SOURCE=283: --argjson local_dumps_deleted "$LOCAL_DUMPS_DELETED" \ -DETAIL=SOURCE=284: --argjson deleted_count "$DELETED_COUNT" \ -DETAIL=SOURCE=285: --argjson deleted_bytes "$DELETED_BYTES" \ -DETAIL=SOURCE=286: "{job_id:\$job_id,status:\$status,node:\$node,remote_a:\$remote_a,remote_b:\$remote_b,count:\$count,total_bytes:\$total_bytes,chunked_count:\$chunked_count,chunk_size_bytes:\$chunk_size_bytes,log:\$log,local_dumps_deleted:$local_dumps_deleted,deleted_count:$deleted_count,deleted_bytes:$deleted_bytes}" \ -DETAIL=SOURCE=287- > "$ROOT/latest-vm-local-dumps-cloud.json" -DETAIL=SOURCE=288- -DETAIL=SOURCE=289:echo "STATUS=$STATUS TS=$TS TYPE=vm-local-dumps-cloud BAD=$BAD COUNT=$COUNT CHUNKED=$CHUNKED TOTAL_BYTES=$TOTAL JOB=$JOB_ID REMOTE_A=$DEST_A REMOTE_B=$DEST_B LOCAL_DUMPS_DELETED=$LOCAL_DUMPS_DELETED DELETED_COUNT=$DELETED_COUNT DELETED_BYTES=$DELETED_BYTES LOG=$LOG" > "$OUT" -DETAIL=SOURCE=290-cat "$OUT" -DETAIL=INSTALLED_LOGIC_END -DETAIL=UNIT_STATE_BEGIN -DETAIL=SERVICE_PROPERTY=Result=success -DETAIL=SERVICE_PROPERTY=ExecMainStatus=0 -DETAIL=SERVICE_PROPERTY=ExecStart={ path=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; argv[]=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } -DETAIL=SERVICE_PROPERTY=LoadState=loaded -DETAIL=SERVICE_PROPERTY=ActiveState=inactive -DETAIL=SERVICE_PROPERTY=SubState=dead -DETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/homelab-vm-local-dumps-2cloud.service -DETAIL=TIMER_PROPERTY=Unit=homelab-vm-local-dumps-2cloud.service -DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Sun 2026-07-26 10:11:29 MSK -DETAIL=TIMER_PROPERTY=LastTriggerUSec=Sun 2026-07-19 10:06:07 MSK -DETAIL=TIMER_PROPERTY=LoadState=loaded -DETAIL=TIMER_PROPERTY=ActiveState=active -DETAIL=TIMER_PROPERTY=SubState=waiting -DETAIL=TIMER_ENABLED=enabled -DETAIL=TIMER_ACTIVE=active -DETAIL=UNIT_STATE_END -DETAIL=CURRENT_DUMP_INVENTORY_BEGIN -DETAIL=CURRENT_DUMP_COUNT=12 -DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst|BYTES:329541843|MODE:644|OWNER:root:root|MTIME:2026-07-21 03:30:09.768869633 +0300 -DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-110-2026_07_22-03_30_01.tar.zst|BYTES:332951369|MODE:644|OWNER:root:root|MTIME:2026-07-22 03:30:08.768607650 +0300 -DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-110-2026_07_23-03_30_02.tar.zst|BYTES:336245409|MODE:644|OWNER:root:root|MTIME:2026-07-23 03:30:10.836949127 +0300 -DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-110-2026_07_24-03_30_02.tar.zst|BYTES:339140622|MODE:644|OWNER:root:root|MTIME:2026-07-24 03:30:10.776974593 +0300 -DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst|BYTES:222777732|MODE:644|OWNER:root:root|MTIME:2026-07-21 03:30:15.905963829 +0300 -DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-112-2026_07_22-03_30_09.tar.zst|BYTES:222794054|MODE:644|OWNER:root:root|MTIME:2026-07-22 03:30:14.284693312 +0300 -DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-112-2026_07_23-03_30_11.tar.zst|BYTES:222808025|MODE:644|OWNER:root:root|MTIME:2026-07-23 03:30:16.408035215 +0300 -DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-112-2026_07_24-03_30_11.tar.zst|BYTES:222859457|MODE:644|OWNER:root:root|MTIME:2026-07-24 03:30:16.202058480 +0300 -DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst|BYTES:7021952392|MODE:644|OWNER:root:root|MTIME:2026-07-21 03:31:06.571741495 +0300 -DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-qemu-150-2026_07_22-03_30_14.vma.zst|BYTES:7052176114|MODE:644|OWNER:root:root|MTIME:2026-07-22 03:31:02.895448212 +0300 -DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-qemu-150-2026_07_23-03_30_16.vma.zst|BYTES:7024165999|MODE:644|OWNER:root:root|MTIME:2026-07-23 03:31:08.099833995 +0300 -DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-qemu-150-2026_07_24-03_30_16.vma.zst|BYTES:7016624549|MODE:644|OWNER:root:root|MTIME:2026-07-24 03:31:04.670807942 +0300 -DETAIL=CURRENT_DUMP_TOTAL_BYTES=30344037565 -DETAIL=CURRENT_DUMP_INVENTORY_END -DETAIL=LAST_STATE_BEGIN -DETAIL=LAST_STATE={"status":"OK","job_id":"homelab-pve01-vm-local-dumps-20260719T070607Z","count":3,"total_bytes":7567711895,"local_dumps_deleted":false,"deleted_count":null,"deleted_bytes":null,"remote_a":"pve01-mail-01-crypt:homelab-backups/pve01/vm-local-dumps/homelab-pve01-vm-local-dumps-20260719T070607Z","remote_b":"pve01-mail-02-crypt:homelab-backups/pve01/vm-local-dumps/homelab-pve01-vm-local-dumps-20260719T070607Z"} -DETAIL=LAST_STATE_END -DETAIL=DELETE_POLICY_VALIDATION=PASS -DETAIL=BACKUP_PAYLOAD_CHANGED=NO -DETAIL=SECRET_VALUES_OUTPUT=NONE -CHANGES_MADE=NO +GENERATED_AT_UTC=[PRIVATE_IP]T10:17:01Z +Cluster information +------------------- +Name: homelab +Config Version: 3 +Transport: knet +Secure auth: on +Quorum information +------------------ +Date: Fri Jul [PRIVATE_IP] 2026 +Quorum provider: corosync_votequorum +Nodes: 3 +Node ID: [PRIVATE_IP] +Ring ID: 1.ba +Quorate: Yes + +Votequorum information +---------------------- +Expected votes: 3 +Highest expected: 3 +Total votes: 3 +Quorum: 2 +Flags: Quorate + +Membership information +---------------------- + Nodeid Votes Name +[PRIVATE_IP] [PRIVATE_IP].11 (local) +[PRIVATE_IP] [PRIVATE_IP].13 +[PRIVATE_IP] [PRIVATE_IP].12 + +Membership information +---------------------- + Nodeid Votes Name + 1 1 pve01 (local) + 2 1 pve03 + 3 1 pve02 +[{"cpu":[PRIVATE_IP][PRIVATE_IP],"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/110","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"dns1","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"lxc","uptime":75356,"vmid":110},{"cpu":[PRIVATE_IP][PRIVATE_IP],"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/111","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"dns2","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve02","status":"running","template":0,"type":"lxc","uptime":75188,"vmid":111},{"cpu":0,"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/112","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"unbound1","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"lxc","uptime":75359,"vmid":112},{"cpu":[PRIVATE_IP]559e-05,"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/113","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"unbound2","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve02","status":"running","template":0,"type":"lxc","uptime":75191,"vmid":113},{"cpu":[PRIVATE_IP]20445,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/130","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"edge-vm","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve03","status":"running","template":0,"type":"qemu","uptime":75297,"vmid":130},{"cpu":[PRIVATE_IP]7415,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/150","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"nextcloud","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"qemu","uptime":75353,"vmid":150},{"cpu":[PRIVATE_IP]555957,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/160","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"forum-prod","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve02","status":"running","template":0,"type":"qemu","uptime":75185,"vmid":160},{"cpu":[PRIVATE_IP]70144,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/170","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"core-apps","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"qemu","uptime":75349,"vmid":170},{"cpu":[PRIVATE_IP]20509,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/171","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"monitoring","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"qemu","uptime":75346,"vmid":171},{"cpu":[PRIVATE_IP]667871,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/180","maxcpu":2,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"cluster-admin","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve02","status":"running","template":0,"type":"qemu","uptime":75182,"vmid":180},{"cpu":[PRIVATE_IP]634515,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/190","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"homelab-ops-worker","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve03","status":"running","template":0,"type":"qemu","uptime":75293,"vmid":190},{"cpu":0,"disk":0,"diskread":0,"diskwrite":0,"id":"qemu/9130","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":0,"memhost":0,"name":"edge-cold-standby","netin":0,"netout":0,"node":"pve02","status":"stopped","template":0,"type":"qemu","uptime":0,"vmid":9130},{"cgroup-mode":2,"cpu":[PRIVATE_IP]35108,"disk":[PRIVATE_IP],"id":"node/pve03","level":"","maxcpu":8,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"node":"pve03","status":"online","type":"node","uptime":75315},{"cgroup-mode":2,"cpu":[PRIVATE_IP]78104,"disk":[PRIVATE_IP],"id":"node/pve02","level":"","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"node":"pve02","status":"online","type":"node","uptime":75252},{"cgroup-mode":2,"cpu":[PRIVATE_IP]76872,"disk":[PRIVATE_IP],"id":"node/pve01","level":"","maxcpu":24,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"node":"pve01","status":"online","type":"node","uptime":75386},{"content":"rootdir,images","disk":[PRIVATE_IP],"id":"storage/pve03/local-lvm","maxdisk":[PRIVATE_IP],"node":"pve03","plugintype":"lvmthin","shared":0,"status":"available","storage":"local-lvm","type":"storage"},{"content":"rootdir,images","disk":[PRIVATE_IP],"id":"storage/pve02/local-lvm","maxdisk":[PRIVATE_IP],"node":"pve02","plugintype":"lvmthin","shared":0,"status":"available","storage":"local-lvm","type":"storage"},{"content":"rootdir,images","disk":[PRIVATE_IP],"id":"storage/pve01/local-lvm","maxdisk":[PRIVATE_IP],"node":"pve01","plugintype":"lvmthin","shared":0,"status":"available","storage":"local-lvm","type":"storage"},{"content":"import,backup,iso,vztmpl","disk":[PRIVATE_IP],"id":"storage/pve03/local","maxdisk":[PRIVATE_IP],"node":"pve03","plugintype":"dir","shared":0,"status":"available","storage":"local","type":"storage"},{"content":"import,backup,iso,vztmpl","disk":[PRIVATE_IP],"id":"storage/pve02/local","maxdisk":[PRIVATE_IP],"node":"pve02","plugintype":"dir","shared":0,"status":"available","storage":"local","type":"storage"},{"content":"import,backup,iso,vztmpl","disk":[PRIVATE_IP],"id":"storage/pve01/local","maxdisk":[PRIVATE_IP],"node":"pve01","plugintype":"dir","shared":0,"status":"available","storage":"local","type":"storage"},{"id":"network/pve03/zone/localnetwork","network":"localnetwork","network-type":"zone","node":"pve03","status":"ok","type":"network"},{"id":"network/pve02/zone/localnetwork","network":"localnetwork","network-type":"zone","node":"pve02","status":"ok","type":"network"},{"id":"network/pve01/zone/localnetwork","network":"localnetwork","network-type":"zone","node":"pve01","status":"ok","type":"network"}] +Name Type Status Total (KiB) Used (KiB) Available (KiB) % +local dir active [PRIVATE_IP] [PRIVATE_IP] [PRIVATE_IP] 41.27% +local-lvm lvmthin active [PRIVATE_IP] [PRIVATE_IP] [PRIVATE_IP] 38.40% + UNIT LOAD ACTIVE SUB DESCRIPTION +● homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state + +Legend: LOAD → Reflects whether the unit definition was properly loaded. + ACTIVE → The high-level unit activation state, i.e. generalization of SUB. + SUB → The low-level unit activation state, values depend on unit type. + +1 loaded units listed. +NEXT LEFT LAST PASSED UNIT ACTIVATES +Fri [PRIVATE_IP] 13:17:33 MSK 31s Fri [PRIVATE_IP] 13:16:29 MSK 32s ago homelab-router-watchdog.timer homelab-router-watchdog.service +Fri [PRIVATE_IP] 13:18:15 MSK 1min 12s Fri [PRIVATE_IP] 13:08:15 MSK 8min ago skladchik-reports-monitor-supervisor.timer skladchik-reports-monitor-supervisor.service +Fri [PRIVATE_IP] 13:18:39 MSK 1min 37s Fri [PRIVATE_IP] 13:13:38 MSK 3min 24s ago homelab-crypto-portfolio-chart-health.timer homelab-crypto-portfolio-chart-health.service +Fri [PRIVATE_IP] 13:18:41 MSK 1min 38s Fri [PRIVATE_IP] 13:16:28 MSK 33s ago homelab-cluster-admin-inventory-sync.timer homelab-cluster-admin-inventory-sync.service +Fri [PRIVATE_IP] 13:19:28 MSK 2min 26s Fri [PRIVATE_IP] 13:14:28 MSK 2min 33s ago homelab-smartctl-textfile.timer homelab-smartctl-textfile.service +Fri [PRIVATE_IP] 13:21:28 MSK 4min 26s Fri [PRIVATE_IP] 13:16:28 MSK 33s ago netbird-peers-health.timer netbird-peers-health.service +Fri [PRIVATE_IP] 13:21:49 MSK 4min 47s Fri [PRIVATE_IP] 13:16:28 MSK 33s ago homelab-health-metrics.timer homelab-health-metrics.service +Fri [PRIVATE_IP] 13:22:28 MSK 5min Fri [PRIVATE_IP] 13:07:28 MSK 9min ago prometheus-node-exporter-apt.timer prometheus-node-exporter-apt.service +Fri [PRIVATE_IP] 13:22:28 MSK 5min Fri [PRIVATE_IP] 13:07:28 MSK 9min ago prometheus-node-exporter-nvme.timer prometheus-node-exporter-nvme.service +Fri [PRIVATE_IP] 13:24:15 MSK 7min Fri [PRIVATE_IP] 13:09:15 MSK 7min ago homelab-disk-space-health.timer homelab-disk-space-health.service +Fri [PRIVATE_IP] 13:24:15 MSK 7min Fri [PRIVATE_IP] 13:09:15 MSK 7min ago homelab-pve03-staging-capacity-health.timer homelab-pve03-staging-capacity-health.service +Fri [PRIVATE_IP] 13:30:00 MSK 12min Fri [PRIVATE_IP] 13:15:04 MSK 1min 58s ago homelab-incident-journal.timer homelab-incident-journal.service +Fri [PRIVATE_IP] 13:30:00 MSK 12min Fri [PRIVATE_IP] 13:15:04 MSK 1min 58s ago homelab-safe-autoheal.timer homelab-safe-autoheal.service +Fri [PRIVATE_IP] 13:30:00 MSK 12min Fri [PRIVATE_IP] 13:15:36 MSK 1min 25s ago homelab-forum-snuffleupagus-health.timer homelab-forum-snuffleupagus-health.service +Fri [PRIVATE_IP] 13:30:45 MSK 13min Fri [PRIVATE_IP] 13:15:58 MSK 1min 4s ago homelab-duty-admin-v2.timer homelab-duty-admin-v2.service +Fri [PRIVATE_IP] 13:30:49 MSK 13min Fri [PRIVATE_IP] 13:15:36 MSK 1min 25s ago homelab-external-probe-vps-health.timer homelab-external-probe-vps-health.service +Fri [PRIVATE_IP] 13:32:12 MSK 15min Fri [PRIVATE_IP] 13:00:28 MSK 16min ago homelab-mkdocs-auto-refresh.timer homelab-mkdocs-auto-refresh.service +Fri [PRIVATE_IP] 13:33:30 MSK 16min Fri [PRIVATE_IP] 13:02:38 MSK 14min ago homelab-reference-refresh.timer homelab-reference-refresh.service +Fri [PRIVATE_IP] 14:48:04 MSK 1h 31min Thu [PRIVATE_IP] 21:43:04 MSK 15h ago apt-daily.timer apt-daily.service +Fri [PRIVATE_IP] 16:23:38 MSK 3h 6min Fri [PRIVATE_IP] 10:23:38 MSK 2h 53min ago homelab-vps-identity-audit.timer homelab-vps-identity-audit.service +Fri [PRIVATE_IP] 16:35:37 MSK 3h 18min Thu [PRIVATE_IP] 16:35:37 MSK 20h ago systemd-tmpfiles-clean.timer systemd-tmpfiles-clean.service +Sat [PRIVATE_IP] 00:00:00 MSK 10h Fri [PRIVATE_IP] 00:00:04 MSK 13h ago dpkg-db-backup.timer dpkg-db-backup.service +Sat [PRIVATE_IP] 00:00:00 MSK 10h Fri [PRIVATE_IP] 00:20:00 MSK 12h ago homelab-docker-health.timer homelab-docker-health.service +Sat [PRIVATE_IP] 00:02:57 MSK 10h Fri [PRIVATE_IP] 00:03:04 MSK 13h ago homelab-evidence-catalog.timer homelab-evidence-catalog.service +Sat [PRIVATE_IP] 00:06:10 MSK 10h Fri [PRIVATE_IP] 00:19:28 MSK 12h ago homelab-storage-capacity.timer homelab-storage-capacity.service +Sat [PRIVATE_IP] 00:14:20 MSK 10h Fri [PRIVATE_IP] 00:03:38 MSK 13h ago homelab-quality-gate.timer homelab-quality-gate.service +Sat [PRIVATE_IP] 00:15:16 MSK 10h Fri [PRIVATE_IP] 00:05:04 MSK 13h ago homelab-backup-freshness.timer homelab-backup-freshness.service +Sat [PRIVATE_IP] 00:15:30 MSK 10h Fri [PRIVATE_IP] 07:47:04 MSK 5h 29min ago man-db.timer man-db.service +Sat [PRIVATE_IP] 00:25:03 MSK 11h Fri [PRIVATE_IP] 00:52:04 MSK 12h ago logrotate.timer logrotate.service +Sat [PRIVATE_IP] 02:15:00 MSK 12h Fri [PRIVATE_IP] 02:15:04 MSK 11h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service +Sat [PRIVATE_IP] 03:18:38 MSK 14h Fri [PRIVATE_IP] 03:13:28 MSK 10h ago homelab-emergency-backup.timer homelab-emergency-backup.service +Sat [PRIVATE_IP] 03:35:00 MSK 14h Fri [PRIVATE_IP] 03:35:00 MSK 9h ago filebrowser-backup.timer filebrowser-backup.service +Sat [PRIVATE_IP] 03:50:00 MSK 14h Fri [PRIVATE_IP] 03:50:04 MSK 9h ago filebrowser-offhost-sync.timer filebrowser-offhost-sync.service +Sat [PRIVATE_IP] 03:50:16 MSK 14h Fri [PRIVATE_IP] 03:48:15 MSK 9h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service +Sat [PRIVATE_IP] 03:55:37 MSK 14h Fri [PRIVATE_IP] 01:38:48 MSK 11h ago pve-daily-update.timer pve-daily-update.service +Sat [PRIVATE_IP] 04:10:00 MSK 14h Fri [PRIVATE_IP] 04:10:04 MSK 9h ago filebrowser-restore-validate.timer filebrowser-restore-validate.service +Sat [PRIVATE_IP] 04:22:27 MSK 15h Fri [PRIVATE_IP] 04:27:33 MSK 8h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service +Sat [PRIVATE_IP] 04:22:44 MSK 15h Fri [PRIVATE_IP] 04:17:28 MSK 8h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service +Sat [PRIVATE_IP] 04:39:05 MSK 15h Fri [PRIVATE_IP] 04:28:08 MSK 8h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service +Sat [PRIVATE_IP] 04:45:41 MSK 15h Fri [PRIVATE_IP] 04:46:28 MSK 8h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service +Sat [PRIVATE_IP] 04:50:23 MSK 15h Fri [PRIVATE_IP] 04:49:40 MSK 8h ago homelab-edge-vm-offhost-freshness.timer homelab-edge-vm-offhost-freshness.service +Sat [PRIVATE_IP] 04:58:22 MSK 15h Fri [PRIVATE_IP] 05:12:28 MSK 8h ago homelab-sops-edge-secret-coverage.timer homelab-sops-edge-secret-coverage.service +Sat [PRIVATE_IP] 05:17:01 MSK 15h Fri [PRIVATE_IP] 05:15:28 MSK 8h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service +Sat [PRIVATE_IP] 05:24:41 MSK 16h Fri [PRIVATE_IP] 05:18:04 MSK 7h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service +Sat [PRIVATE_IP] 05:46:41 MSK 16h Fri [PRIVATE_IP] 05:54:15 MSK 7h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service +Sat [PRIVATE_IP] 06:09:38 MSK 16h Fri [PRIVATE_IP] 05:58:49 MSK 7h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service +Sat [PRIVATE_IP] 06:12:31 MSK 16h Fri [PRIVATE_IP] 06:34:50 MSK 6h ago apt-daily-upgrade.timer apt-daily-upgrade.service +Sat [PRIVATE_IP] 06:30:50 MSK 17h Fri [PRIVATE_IP] 06:35:48 MSK 6h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service +Sat [PRIVATE_IP] 06:44:52 MSK 17h Fri [PRIVATE_IP] 06:39:49 MSK 6h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service +Sat [PRIVATE_IP] 06:49:50 MSK 17h Fri [PRIVATE_IP] 06:45:28 MSK 6h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service +Sat [PRIVATE_IP] 06:52:38 MSK 17h Fri [PRIVATE_IP] 06:50:28 MSK 6h ago homelab-beszel-backup.timer homelab-beszel-backup.service +Sat [PRIVATE_IP] 06:54:17 MSK 17h Fri [PRIVATE_IP] 06:41:38 MSK 6h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service +Sat [PRIVATE_IP] 06:56:53 MSK 17h Fri [PRIVATE_IP] 07:04:58 MSK 6h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service +Sat [PRIVATE_IP] 07:01:19 MSK 17h Fri [PRIVATE_IP] 07:10:51 MSK 6h ago homelab-drift-check.timer homelab-drift-check.service +Sat [PRIVATE_IP] 07:02:50 MSK 17h Fri [PRIVATE_IP] 07:04:28 MSK 6h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service +Sat [PRIVATE_IP] 07:14:04 MSK 17h Fri [PRIVATE_IP] 07:15:31 MSK 6h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service +Sat [PRIVATE_IP] 07:33:57 MSK 18h Fri [PRIVATE_IP] 07:24:04 MSK 5h 52min ago homelab-service-registry-check.timer homelab-service-registry-check.service +Sat [PRIVATE_IP] 07:37:08 MSK 18h Fri [PRIVATE_IP] 07:28:04 MSK 5h 48min ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service +Sat [PRIVATE_IP] 07:46:01 MSK 18h Fri [PRIVATE_IP] 07:41:38 MSK 5h 35min ago homelab-dependency-map-check.timer homelab-dependency-map-check.service +Sat [PRIVATE_IP] 07:47:12 MSK 18h Fri [PRIVATE_IP] 07:46:04 MSK 5h 30min ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service +Sat [PRIVATE_IP] 07:47:48 MSK 18h Fri [PRIVATE_IP] 07:57:48 MSK 5h 19min ago homelab-runbook-generate.timer homelab-runbook-generate.service +Sat [PRIVATE_IP] 07:48:02 MSK 18h Fri [PRIVATE_IP] 07:48:15 MSK 5h 28min ago homelab-alerting-health.timer homelab-alerting-health.service +Sat [PRIVATE_IP] 07:49:43 MSK 18h Fri [PRIVATE_IP] 07:43:04 MSK 5h 33min ago homelab-vm-backup-health.timer homelab-vm-backup-health.service +Sat [PRIVATE_IP] 07:51:40 MSK 18h Fri [PRIVATE_IP] 08:05:28 MSK 5h 11min ago homelab-desired-state-sync.timer homelab-desired-state-sync.service +Sat [PRIVATE_IP] 08:01:29 MSK 18h Fri [PRIVATE_IP] 08:01:26 MSK 5h 15min ago homelab-duty-admin-report.timer homelab-duty-admin-report.service +Sat [PRIVATE_IP] 08:05:31 MSK 18h Fri [PRIVATE_IP] 08:10:04 MSK 5h 6min ago homelab-overall-health.timer homelab-overall-health.service +Sat [PRIVATE_IP] 08:08:57 MSK 18h Fri [PRIVATE_IP] 08:20:04 MSK 4h 56min ago homelab-backup-sla-health.timer homelab-backup-sla-health.service +Sat [PRIVATE_IP] 08:15:04 MSK 18h Fri [PRIVATE_IP] 08:25:48 MSK 4h 51min ago homelab-final-readiness-gate.timer homelab-final-readiness-gate.service +Sat [PRIVATE_IP] 08:19:25 MSK 19h Fri [PRIVATE_IP] 08:11:15 MSK 5h 5min ago homelab-kuma-monitor-policy.timer homelab-kuma-monitor-policy.service +Sat [PRIVATE_IP] 08:27:04 MSK 19h Fri [PRIVATE_IP] 08:34:28 MSK 4h 42min ago homelab-capacity-risk.timer homelab-capacity-risk.service +Sat [PRIVATE_IP] 08:30:30 MSK 19h Fri [PRIVATE_IP] 08:26:18 MSK 4h 50min ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service +Sat [PRIVATE_IP] 08:31:42 MSK 19h Fri [PRIVATE_IP] 08:46:05 MSK 4h 30min ago homelab-secret-exposure-guard.timer homelab-secret-exposure-guard.service +Sat [PRIVATE_IP] 08:35:48 MSK 19h Fri [PRIVATE_IP] 08:37:50 MSK 4h 39min ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service +Sat [PRIVATE_IP] 08:36:51 MSK 19h Fri [PRIVATE_IP] 08:35:15 MSK 4h 41min ago memos-backup.timer memos-backup.service +Sat [PRIVATE_IP] 08:41:06 MSK 19h Fri [PRIVATE_IP] 08:44:57 MSK 4h 32min ago homelab-cluster-passport.timer homelab-cluster-passport.service +Sat [PRIVATE_IP] 08:45:03 MSK 19h Fri [PRIVATE_IP] 08:54:27 MSK 4h 22min ago immich-media-offhost-sync.timer immich-media-offhost-sync.service +Sat [PRIVATE_IP] 08:55:59 MSK 19h Fri [PRIVATE_IP] 08:56:28 MSK 4h 20min ago homelab-golden-state-index.timer homelab-golden-state-index.service +Sat [PRIVATE_IP] 09:23:17 MSK 20h Fri [PRIVATE_IP] 09:11:15 MSK 4h 5min ago memos-offhost-sync.timer memos-offhost-sync.service +Sat [PRIVATE_IP] 10:03:38 MSK 20h Fri [PRIVATE_IP] 10:06:04 MSK 3h 10min ago paperless-restore-dry-run.timer paperless-restore-dry-run.service +Sat [PRIVATE_IP] 10:15:00 MSK 20h Fri [PRIVATE_IP] 10:15:04 MSK 3h 1min ago skladchik-reports-monitor-monthly-selftest-watch.timer skladchik-reports-monitor-monthly-selftest-watch.service +Sun [PRIVATE_IP] 03:10:03 MSK 1 day 13h Sun [PRIVATE_IP] 03:10:29 MSK - e2scrub_all.timer e2scrub_all.service +Sun [PRIVATE_IP] 03:10:35 MSK 1 day 13h Sun [PRIVATE_IP] 03:10:14 MSK - xfs_scrub_all.timer xfs_scrub_all.service +Sun [PRIVATE_IP] 03:34:48 MSK 1 day 14h Sun [PRIVATE_IP] 03:45:37 MSK - homelab-auto-backup.timer homelab-auto-backup.service +Sun [PRIVATE_IP] 05:15:13 MSK 1 day 15h Sun [PRIVATE_IP] 05:11:25 MSK - skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service +Sun [PRIVATE_IP] 09:39:32 MSK 1 day 20h Sun [PRIVATE_IP] 09:32:29 MSK - homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service +Sun [PRIVATE_IP] 10:11:29 MSK 1 day 20h Sun [PRIVATE_IP] 10:06:07 MSK - homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service +Sun [PRIVATE_IP] 11:28:24 MSK 1 day 22h Sun [PRIVATE_IP] 11:08:41 MSK - homelab-vm-local-dumps-retention.timer homelab-vm-local-dumps-retention.service +Mon [PRIVATE_IP] 00:07:15 MSK 2 days Mon [PRIVATE_IP] 00:00:29 MSK - homelab-secret-sanity.timer homelab-secret-sanity.service +Mon [PRIVATE_IP] 01:33:11 MSK 2 days Mon [PRIVATE_IP] 00:51:14 MSK - fstrim.timer fstrim.service +Mon [PRIVATE_IP] 07:34:52 MSK 2 days Mon [PRIVATE_IP] 07:25:14 MSK - homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service +Thu [PRIVATE_IP] 06:00:00 MSK 2 months 21 days Wed [PRIVATE_IP] 06:00:00 MSK - homelab-vm-full-restore-drill.timer homelab-vm-full-restore-drill.service +- - Fri [PRIVATE_IP] 13:17:02 MSK 21ms ago homelab-cluster-admin-webpanel-sync.timer homelab-cluster-admin-webpanel-sync.service +- - Fri [PRIVATE_IP] 13:17:02 MSK 20ms ago homelab-private-vpn-hosts-health.timer homelab-private-vpn-hosts-health.service +- - - - prometheus-node-exporter-ipmitool-sensor.timer prometheus-node-exporter-ipmitool-sensor.service +- - - - prometheus-node-exporter-mellanox-hca-temp.timer prometheus-node-exporter-mellanox-hca-temp.service +- - - - prometheus-node-exporter-smartmon.timer prometheus-node-exporter-smartmon.service + +96 timers listed. +- Обрезанная, повреждённая или частично вставленная команда считается невыполненной; сначала проводится side-effect audit. +- Хвост parse-ошибки закрывается только после доказательства отсутствия изменений и временных артефактов. +- Proof: /var/lib/homelab-change-sets/operator-command-size-guard-v[PRIVATE_IP]T182653Z/report.txt + +## HOMELAB_ADMIN_CLI_CONTRACT_[PRIVATE_IP] +- Поддерживаемые команды: version, status, lint TASK_DIR, validate TASK_DIR, run TASK_DIR PHASE. +- Отдельная команда --help отсутствует. Неизвестный аргумент печатает usage и штатно возвращает RC=64. +- Нельзя трактовать RC=64 от --help как неисправность runner. +- Для проверки доступности runner использовать homelab-admin version; для структуры CLI использовать зафиксированную usage-строку. +- Proof: /var/lib/homelab-change-sets/homelab-admin-cli-contract-fix-v[PRIVATE_IP]T183338Z/report.txt + +AUTOMATIC_CLUSTER_CONTEXT_REFRESH_[PRIVATE_IP] +Collector: /usr/local/sbin/homelab-context-collect +Scheduler: hourly at minute 17 via /etc/cron.d/homelab-context-refresh +Destination: immutable Gitea runtime history with CONTEXT-AUTO UTC command IDs. + +AUTOMATIC_CLUSTER_CONTEXT_REFRESH_DIRECT_[PRIVATE_IP] +Publisher: /usr/local/sbin/homelab-context-refresh-direct +Schedule: hourly at minute 17. +Destination: immutable assistant-runtime history with CONTEXT-AUTO UTC IDs. +- Registered at: [PRIVATE_IP]T06:27:06Z + +AUTOREFRESH_INLINE_FAILURE_[PRIVATE_IP] +Attempts 048 and 049 did not appear in immutable history. +Rule: use validated local helper scripts and a simple scheduler entry; never embed the full collector in cron or nested SSH quoting. + +AUTOREFRESH_WRAPPER_PUBLICATION_FAILURE_[PRIVATE_IP] +Commands [PRIVATE_IP] did not reach immutable history through homelab-chat-run. +Resolution: hourly snapshots use direct scp plus homelab-runtime-receive publication. + +AUTOREFRESH_052_NOT_PUBLISHED_[PRIVATE_IP] +CONTEXT-AUTOREFRESH-ACTIVATE-052 did not appear in immutable history; direct publisher retry requires sanitization before transmission. + +IMMUTABLE_HISTORY_FALSE_NEGATIVE_[PRIVATE_IP] +BACKUP-AUDIT-CONTROLPLANE-054 was successfully published at commit 2b966acfe[PRIVATE_IP]d4a62bf[PRIVATE_IP]bdd1a073, but was incorrectly reported missing because a stale directory listing was treated as authoritative. +Rule: verify the exact COMMAND_ID result, wrapper publication status and commit SHA; never infer absence from a cached directory listing. + +BACKUP_UNSAFE_SOURCE_CAPTURE_071_PARSE_ERROR_[PRIVATE_IP] +BACKUP-UNSAFE-SOURCE-CAPTURE-071 failed before execution: shell syntax error near unexpected token (. +Resolution: avoid nested quote layers; parse targets first and pass file paths as simple SSH arguments. OUTPUT_END CHAT_OUTPUT_END