diff --git a/runtime/history/HOMELAB-BACKUP-CURRENT-STATE-SEAL-459.json b/runtime/history/HOMELAB-BACKUP-CURRENT-STATE-SEAL-459.json new file mode 100644 index 0000000..fb63ebd --- /dev/null +++ b/runtime/history/HOMELAB-BACKUP-CURRENT-STATE-SEAL-459.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "HOMELAB-BACKUP-CURRENT-STATE-SEAL-459", + "status": "FAIL", + "rc": 1, + "host": "pve01", + "mode": "read-only", + "component": "cluster-backup-reset", + "started_at_utc": "2026-07-24T15:02:05Z", + "finished_at_utc": "2026-07-24T15:03:37Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83", + "command_sha256": "035aca8551e77c67944c13906d92f66396f48d9ed2512f5d1e22631d5b372a3f", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "d4e26d689b466295aa4c8a5ad17e6003bc98db2ade45005316dac02896af8ffa", + "sanitized_output_sha256": "d4e26d689b466295aa4c8a5ad17e6003bc98db2ade45005316dac02896af8ffa", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "DETAIL=NODE_ACCESS|NODE=pve01|STATUS=OK|HOSTNAME=pve01\nDETAIL=NODE_ACCESS|NODE=pve02|STATUS=OK|HOSTNAME=pve02\nDETAIL=NODE_ACCESS|NODE=pve03|STATUS=OK|HOSTNAME=pve03\nDETAIL=PROXMOX_BACKUP_JOB|ID=homelab-nightly-all|ENABLED=0\nDETAIL=TIMER_INVENTORY|NODE=pve01|COUNT=47|EXPECTED_COUNT=37|SHA256=045feaa9c0e86d768f22ae0571023abe4dee718b1f7e6bf0cc7c2607717f9a4e|EXPECTED_SHA256=09810394f8315232c93f8b529fb44d417862525747c912fb628995012c91e9da\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|filebrowser-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|filebrowser-restore-validate.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-alertmanager-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-appbackup-dockge-stacks.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-appbackup-mariadb-batch.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-appbackup-npmplus-edge.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-appbackup-postgres-batch.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-appbackup-sqlite-batch.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-authentik-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-auto-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-backup-coverage-matrix.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-backup-freshness.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-backup-healthcheck.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-backup-matrix-refresh.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-backup-sla-health.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-beszel-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-blackbox-exporter-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-cluster-admin-incident-engine-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-crypto-portfolio-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-edge-vm-vzdump-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-emergency-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-extended-appbackup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-gitea-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-gitea-secondary-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-legacy-monitoring-backups.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-mail-cloud-critical-upload.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-mail-cloud-edge-vm-upload.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-mail-cloud-nextcloud-vm-upload.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-mail-cloud-restore-drill.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-npmplus-kuma-config-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-router-running-config-mail-cloud.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vaultwarden-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-verified-backup-health.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm-backup-health.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm-backup-policy.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm-full-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm-full-restore-drill.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm-local-dumps-2cloud.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm-local-dumps-retention.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm160-remote-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|immich-media-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|memos-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|netbird-vps-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|paperless-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|paperless-restore-dry-run.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|skladchik-reports-monitor-backup-restore-check.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|skladchik-reports-monitor-backup.timer\nDETAIL=SEAL_MISMATCH|CHECK=TIMER_INVENTORY_pve01\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=filebrowser-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=filebrowser-restore-validate.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-alertmanager-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-appbackup-dockge-stacks.timer|ENABLED=enabled|ACTIVE=active\nDETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-appbackup-dockge-stacks.timer\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-appbackup-mariadb-batch.timer|ENABLED=enabled|ACTIVE=active\nDETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-appbackup-mariadb-batch.timer\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-appbackup-npmplus-edge.timer|ENABLED=enabled|ACTIVE=active\nDETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-appbackup-npmplus-edge.timer\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-appbackup-postgres-batch.timer|ENABLED=enabled|ACTIVE=active\nDETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-appbackup-postgres-batch.timer\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-appbackup-sqlite-batch.timer|ENABLED=enabled|ACTIVE=active\nDETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-appbackup-sqlite-batch.timer\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-authentik-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-auto-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-backup-coverage-matrix.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-backup-freshness.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-backup-healthcheck.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-backup-matrix-refresh.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-backup-sla-health.timer|ENABLED=enabled|ACTIVE=active\nDETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-backup-sla-health.timer\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-beszel-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-blackbox-exporter-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-cluster-admin-incident-engine-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-crypto-portfolio-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-edge-vm-vzdump-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-emergency-backup.timer|ENABLED=enabled|ACTIVE=active\nDETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-emergency-backup.timer\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-extended-appbackup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-gitea-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-gitea-secondary-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-legacy-monitoring-backups.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-mail-cloud-critical-upload.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-mail-cloud-edge-vm-upload.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-mail-cloud-nextcloud-vm-upload.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-mail-cloud-restore-drill.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-npmplus-kuma-config-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-router-running-config-mail-cloud.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vaultwarden-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-verified-backup-health.timer|ENABLED=enabled|ACTIVE=active\nDETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-verified-backup-health.timer\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm-backup-health.timer|ENABLED=enabled|ACTIVE=active\nDETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-vm-backup-health.timer\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm-backup-policy.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm-full-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm-full-restore-drill.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm-local-dumps-2cloud.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm-local-dumps-retention.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm160-remote-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=immich-media-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=memos-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=netbird-vps-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=paperless-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=paperless-restore-dry-run.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=skladchik-reports-monitor-backup-restore-check.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=skladchik-reports-monitor-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=TIMER_INVENTORY|NODE=pve02|COUNT=4|EXPECTED_COUNT=3|SHA256=47b2eff72008dceac8ccfb35a90eca550b4baf2e6340b225ddd8c9cb3e052c46|EXPECTED_SHA256=913306cc221152a7414dbb923b4e3526fbaec4afcc7d072e114d7ea9a87903ca\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve02|codevipe-cloud-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve02|forum-prod-fresh5-cloud-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve02|homelab-cluster-admin-vm-mail-cloud-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve02|homelab-vaultwarden-isolated-restore-drill.timer\nDETAIL=SEAL_MISMATCH|CHECK=TIMER_INVENTORY_pve02\nDETAIL=KNOWN_TIMER|NODE=pve02|TIMER=codevipe-cloud-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve02|TIMER=forum-prod-fresh5-cloud-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve02|TIMER=homelab-cluster-admin-vm-mail-cloud-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve02|TIMER=homelab-vaultwarden-isolated-restore-drill.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=TIMER_INVENTORY|NODE=pve03|COUNT=0|EXPECTED_COUNT=0|SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|EXPECTED_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nDETAIL=SEAL_MISMATCH|CHECK=TIMER_PASS_COUNT\nDETAIL=KNOWN_TIMER_SUMMARY|EXPECTED=40|PASS=42\nDETAIL=WRITER_PROCESS|NODE=pve01|PROCESS=vzdump|STATUS=NONE\nDETAIL=WRITER_PROCESS|NODE=pve01|PROCESS=rclone|STATUS=NONE\nDETAIL=WRITER_PROCESS|NODE=pve02|PROCESS=vzdump|STATUS=NONE\nDETAIL=WRITER_PROCESS|NODE=pve02|PROCESS=rclone|STATUS=NONE\nDETAIL=WRITER_PROCESS|NODE=pve03|PROCESS=vzdump|STATUS=NONE\nDETAIL=WRITER_PROCESS|NODE=pve03|PROCESS=rclone|STATUS=NONE\nDETAIL=GUEST_CONFIG_SHA256|CURRENT=121518389af6a5a6445b2657e2d836fa2a50849f73d19fd50efae703574ba2fc|EXPECTED=121518389af6a5a6445b2657e2d836fa2a50849f73d19fd50efae703574ba2fc\nDETAIL=STORAGE_CONFIG_SHA256|CURRENT=35f32ffa86ebc6a711aa1a44d8a0d3a89872fca686752c9db1fd0270049727b0|EXPECTED=35f32ffa86ebc6a711aa1a44d8a0d3a89872fca686752c9db1fd0270049727b0\nDETAIL=LOCAL_ARCHIVES|NODE=pve01|COUNT=0|BYTES=0|EXPECTED_COUNT=0|EXPECTED_BYTES=0\nDETAIL=LOCAL_STAGING|NODE=pve01|BYTES=158150895639|MIN_EXPECTED=158150895639\nDETAIL=LOCAL_ARCHIVES|NODE=pve02|COUNT=10|BYTES=3910297912|EXPECTED_COUNT=10|EXPECTED_BYTES=3910297912\nDETAIL=LOCAL_STAGING|NODE=pve02|BYTES=111110930032|MIN_EXPECTED=111110930032\nDETAIL=LOCAL_ARCHIVES|NODE=pve03|COUNT=8|BYTES=49287554728|EXPECTED_COUNT=8|EXPECTED_BYTES=49287554728\nDETAIL=LOCAL_STAGING|NODE=pve03|BYTES=106852877874|MIN_EXPECTED=106852877874\nDETAIL=SEAL_MISMATCH|CHECK=CLOUD_READ_pve01-mail-01-crypt\nDETAIL=CLOUD_PAYLOAD|REMOTE=pve01-mail-02-crypt|FILES=9840|BYTES=194612949642|MIN_FILES=9840|MIN_BYTES=194612949642\nDETAIL=CLOUD_PAYLOAD|REMOTE=pve01-mail-03-crypt|FILES=178|BYTES=84995529177|MIN_FILES=178|MIN_BYTES=84995529177\nDETAIL=CLOUD_PAYLOAD|REMOTE=pve01-mail-04-crypt|FILES=178|BYTES=84995529207|MIN_FILES=178|MIN_BYTES=84995529207\nDETAIL=CLOUD_PAYLOAD|REMOTE=pve01-mail-05-crypt|FILES=113|BYTES=13626007377|MIN_FILES=113|MIN_BYTES=13626007377\nDETAIL=CLOUD_PAYLOAD|REMOTE=pve02-mail-01-crypt|FILES=605|BYTES=218239204595|MIN_FILES=605|MIN_BYTES=218239204595\nDETAIL=CLOUD_PAYLOAD|REMOTE=pve02-mail-02-crypt|FILES=605|BYTES=218239204573|MIN_FILES=605|MIN_BYTES=218239204573\nDETAIL=CLOUD_PAYLOAD|REMOTE=pve02-mail-03-crypt|FILES=540|BYTES=207146226209|MIN_FILES=540|MIN_BYTES=207146226209\nDETAIL=CLOUD_PAYLOAD|REMOTE=pve02-mail-04-crypt|FILES=540|BYTES=207146226239|MIN_FILES=540|MIN_BYTES=207146226239\nDETAIL=CLOUD_PAYLOAD|REMOTE=pve03-mail-01-crypt|FILES=15|BYTES=293253|MIN_FILES=15|MIN_BYTES=293253\nDETAIL=CLOUD_PAYLOAD|REMOTE=pve03-mail-02-crypt|FILES=15|BYTES=293279|MIN_FILES=15|MIN_BYTES=293279\nDETAIL=CLOUD_PAYLOAD|REMOTE=pve03-mail-03-crypt|FILES=203|BYTES=91297553097|MIN_FILES=203|MIN_BYTES=91297553097\nDETAIL=CLOUD_PAYLOAD|REMOTE=pve03-mail-04-crypt|FILES=113|BYTES=13626007377|MIN_FILES=113|MIN_BYTES=13626007377\nDETAIL=CLOUD_TOTAL|FILES=12945|BYTES=1333925024025|BASELINE_FILES=22797|BASELINE_BYTES=1556652840494\nDETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 03:10:18 MSK 9h Fri 2026-07-24 03:13:28 MSK - homelab-emergency-backup.timer homelab-emergency-backup.service\nDETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 03:45:47 MSK 9h Fri 2026-07-24 03:48:15 MSK - homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service\nDETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 04:11:30 MSK 10h Fri 2026-07-24 04:17:28 MSK - homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service\nDETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 04:55:12 MSK 10h Fri 2026-07-24 04:46:28 MSK - homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service\nDETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 05:11:48 MSK 11h Fri 2026-07-24 05:18:04 MSK - homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service\nDETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 05:42:29 MSK 11h Fri 2026-07-24 05:54:15 MSK - homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service\nDETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 06:33:52 MSK 12h Fri 2026-07-24 06:45:28 MSK - homelab-verified-backup-health.timer homelab-verified-backup-health.service\nDETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 07:37:34 MSK 13h Fri 2026-07-24 07:43:04 MSK - homelab-vm-backup-health.timer homelab-vm-backup-health.service\nDETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 08:05:50 MSK 14h Fri 2026-07-24 08:20:04 MSK - homelab-backup-sla-health.timer homelab-backup-sla-health.service\nDETAIL=RC458_EXACT_REASON=PVE01_BROAD_TIMER_FILTER_NONEMPTY\nDETAIL=CURRENT_STATE_BAD=13\nDETAIL=CURRENT_STATE_SEAL=FAIL\nCHANGES_MADE=NO\n" +} diff --git a/runtime/history/HOMELAB-BACKUP-CURRENT-STATE-SEAL-459.txt b/runtime/history/HOMELAB-BACKUP-CURRENT-STATE-SEAL-459.txt new file mode 100644 index 0000000..4131665 --- /dev/null +++ b/runtime/history/HOMELAB-BACKUP-CURRENT-STATE-SEAL-459.txt @@ -0,0 +1,187 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=HOMELAB-BACKUP-CURRENT-STATE-SEAL-459 +STATUS=FAIL +RC=1 +HOST=pve01 +MODE=read-only +COMPONENT=cluster-backup-reset +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83 +COMMAND_SHA256=035aca8551e77c67944c13906d92f66396f48d9ed2512f5d1e22631d5b372a3f +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=false +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=d4e26d689b466295aa4c8a5ad17e6003bc98db2ade45005316dac02896af8ffa +SANITIZED_OUTPUT_SHA256=d4e26d689b466295aa4c8a5ad17e6003bc98db2ade45005316dac02896af8ffa +OUTPUT_BEGIN +DETAIL=NODE_ACCESS|NODE=pve01|STATUS=OK|HOSTNAME=pve01 +DETAIL=NODE_ACCESS|NODE=pve02|STATUS=OK|HOSTNAME=pve02 +DETAIL=NODE_ACCESS|NODE=pve03|STATUS=OK|HOSTNAME=pve03 +DETAIL=PROXMOX_BACKUP_JOB|ID=homelab-nightly-all|ENABLED=0 +DETAIL=TIMER_INVENTORY|NODE=pve01|COUNT=47|EXPECTED_COUNT=37|SHA256=045feaa9c0e86d768f22ae0571023abe4dee718b1f7e6bf0cc7c2607717f9a4e|EXPECTED_SHA256=09810394f8315232c93f8b529fb44d417862525747c912fb628995012c91e9da +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|filebrowser-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|filebrowser-restore-validate.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-alertmanager-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-appbackup-dockge-stacks.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-appbackup-mariadb-batch.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-appbackup-npmplus-edge.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-appbackup-postgres-batch.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-appbackup-sqlite-batch.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-authentik-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-auto-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-backup-coverage-matrix.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-backup-freshness.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-backup-healthcheck.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-backup-matrix-refresh.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-backup-sla-health.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-beszel-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-blackbox-exporter-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-cluster-admin-incident-engine-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-crypto-portfolio-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-edge-vm-vzdump-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-emergency-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-extended-appbackup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-gitea-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-gitea-secondary-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-legacy-monitoring-backups.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-mail-cloud-critical-upload.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-mail-cloud-edge-vm-upload.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-mail-cloud-nextcloud-vm-upload.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-mail-cloud-restore-drill.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-npmplus-kuma-config-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-router-running-config-mail-cloud.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vaultwarden-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-verified-backup-health.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm-backup-health.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm-backup-policy.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm-full-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm-full-restore-drill.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm-local-dumps-2cloud.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm-local-dumps-retention.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm160-remote-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|immich-media-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|memos-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|netbird-vps-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|paperless-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|paperless-restore-dry-run.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|skladchik-reports-monitor-backup-restore-check.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|skladchik-reports-monitor-backup.timer +DETAIL=SEAL_MISMATCH|CHECK=TIMER_INVENTORY_pve01 +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=filebrowser-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=filebrowser-restore-validate.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-alertmanager-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-appbackup-dockge-stacks.timer|ENABLED=enabled|ACTIVE=active +DETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-appbackup-dockge-stacks.timer +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-appbackup-mariadb-batch.timer|ENABLED=enabled|ACTIVE=active +DETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-appbackup-mariadb-batch.timer +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-appbackup-npmplus-edge.timer|ENABLED=enabled|ACTIVE=active +DETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-appbackup-npmplus-edge.timer +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-appbackup-postgres-batch.timer|ENABLED=enabled|ACTIVE=active +DETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-appbackup-postgres-batch.timer +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-appbackup-sqlite-batch.timer|ENABLED=enabled|ACTIVE=active +DETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-appbackup-sqlite-batch.timer +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-authentik-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-auto-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-backup-coverage-matrix.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-backup-freshness.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-backup-healthcheck.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-backup-matrix-refresh.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-backup-sla-health.timer|ENABLED=enabled|ACTIVE=active +DETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-backup-sla-health.timer +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-beszel-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-blackbox-exporter-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-cluster-admin-incident-engine-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-crypto-portfolio-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-edge-vm-vzdump-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-emergency-backup.timer|ENABLED=enabled|ACTIVE=active +DETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-emergency-backup.timer +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-extended-appbackup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-gitea-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-gitea-secondary-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-legacy-monitoring-backups.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-mail-cloud-critical-upload.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-mail-cloud-edge-vm-upload.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-mail-cloud-nextcloud-vm-upload.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-mail-cloud-restore-drill.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-npmplus-kuma-config-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-router-running-config-mail-cloud.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vaultwarden-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-verified-backup-health.timer|ENABLED=enabled|ACTIVE=active +DETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-verified-backup-health.timer +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm-backup-health.timer|ENABLED=enabled|ACTIVE=active +DETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-vm-backup-health.timer +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm-backup-policy.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm-full-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm-full-restore-drill.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm-local-dumps-2cloud.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm-local-dumps-retention.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm160-remote-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=immich-media-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=memos-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=netbird-vps-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=paperless-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=paperless-restore-dry-run.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=skladchik-reports-monitor-backup-restore-check.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=skladchik-reports-monitor-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=TIMER_INVENTORY|NODE=pve02|COUNT=4|EXPECTED_COUNT=3|SHA256=47b2eff72008dceac8ccfb35a90eca550b4baf2e6340b225ddd8c9cb3e052c46|EXPECTED_SHA256=913306cc221152a7414dbb923b4e3526fbaec4afcc7d072e114d7ea9a87903ca +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve02|codevipe-cloud-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve02|forum-prod-fresh5-cloud-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve02|homelab-cluster-admin-vm-mail-cloud-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve02|homelab-vaultwarden-isolated-restore-drill.timer +DETAIL=SEAL_MISMATCH|CHECK=TIMER_INVENTORY_pve02 +DETAIL=KNOWN_TIMER|NODE=pve02|TIMER=codevipe-cloud-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve02|TIMER=forum-prod-fresh5-cloud-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve02|TIMER=homelab-cluster-admin-vm-mail-cloud-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve02|TIMER=homelab-vaultwarden-isolated-restore-drill.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=TIMER_INVENTORY|NODE=pve03|COUNT=0|EXPECTED_COUNT=0|SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|EXPECTED_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +DETAIL=SEAL_MISMATCH|CHECK=TIMER_PASS_COUNT +DETAIL=KNOWN_TIMER_SUMMARY|EXPECTED=40|PASS=42 +DETAIL=WRITER_PROCESS|NODE=pve01|PROCESS=vzdump|STATUS=NONE +DETAIL=WRITER_PROCESS|NODE=pve01|PROCESS=rclone|STATUS=NONE +DETAIL=WRITER_PROCESS|NODE=pve02|PROCESS=vzdump|STATUS=NONE +DETAIL=WRITER_PROCESS|NODE=pve02|PROCESS=rclone|STATUS=NONE +DETAIL=WRITER_PROCESS|NODE=pve03|PROCESS=vzdump|STATUS=NONE +DETAIL=WRITER_PROCESS|NODE=pve03|PROCESS=rclone|STATUS=NONE +DETAIL=GUEST_CONFIG_SHA256|CURRENT=121518389af6a5a6445b2657e2d836fa2a50849f73d19fd50efae703574ba2fc|EXPECTED=121518389af6a5a6445b2657e2d836fa2a50849f73d19fd50efae703574ba2fc +DETAIL=STORAGE_CONFIG_SHA256|CURRENT=35f32ffa86ebc6a711aa1a44d8a0d3a89872fca686752c9db1fd0270049727b0|EXPECTED=35f32ffa86ebc6a711aa1a44d8a0d3a89872fca686752c9db1fd0270049727b0 +DETAIL=LOCAL_ARCHIVES|NODE=pve01|COUNT=0|BYTES=0|EXPECTED_COUNT=0|EXPECTED_BYTES=0 +DETAIL=LOCAL_STAGING|NODE=pve01|BYTES=158150895639|MIN_EXPECTED=158150895639 +DETAIL=LOCAL_ARCHIVES|NODE=pve02|COUNT=10|BYTES=3910297912|EXPECTED_COUNT=10|EXPECTED_BYTES=3910297912 +DETAIL=LOCAL_STAGING|NODE=pve02|BYTES=111110930032|MIN_EXPECTED=111110930032 +DETAIL=LOCAL_ARCHIVES|NODE=pve03|COUNT=8|BYTES=49287554728|EXPECTED_COUNT=8|EXPECTED_BYTES=49287554728 +DETAIL=LOCAL_STAGING|NODE=pve03|BYTES=106852877874|MIN_EXPECTED=106852877874 +DETAIL=SEAL_MISMATCH|CHECK=CLOUD_READ_pve01-mail-01-crypt +DETAIL=CLOUD_PAYLOAD|REMOTE=pve01-mail-02-crypt|FILES=9840|BYTES=194612949642|MIN_FILES=9840|MIN_BYTES=194612949642 +DETAIL=CLOUD_PAYLOAD|REMOTE=pve01-mail-03-crypt|FILES=178|BYTES=84995529177|MIN_FILES=178|MIN_BYTES=84995529177 +DETAIL=CLOUD_PAYLOAD|REMOTE=pve01-mail-04-crypt|FILES=178|BYTES=84995529207|MIN_FILES=178|MIN_BYTES=84995529207 +DETAIL=CLOUD_PAYLOAD|REMOTE=pve01-mail-05-crypt|FILES=113|BYTES=13626007377|MIN_FILES=113|MIN_BYTES=13626007377 +DETAIL=CLOUD_PAYLOAD|REMOTE=pve02-mail-01-crypt|FILES=605|BYTES=218239204595|MIN_FILES=605|MIN_BYTES=218239204595 +DETAIL=CLOUD_PAYLOAD|REMOTE=pve02-mail-02-crypt|FILES=605|BYTES=218239204573|MIN_FILES=605|MIN_BYTES=218239204573 +DETAIL=CLOUD_PAYLOAD|REMOTE=pve02-mail-03-crypt|FILES=540|BYTES=207146226209|MIN_FILES=540|MIN_BYTES=207146226209 +DETAIL=CLOUD_PAYLOAD|REMOTE=pve02-mail-04-crypt|FILES=540|BYTES=207146226239|MIN_FILES=540|MIN_BYTES=207146226239 +DETAIL=CLOUD_PAYLOAD|REMOTE=pve03-mail-01-crypt|FILES=15|BYTES=293253|MIN_FILES=15|MIN_BYTES=293253 +DETAIL=CLOUD_PAYLOAD|REMOTE=pve03-mail-02-crypt|FILES=15|BYTES=293279|MIN_FILES=15|MIN_BYTES=293279 +DETAIL=CLOUD_PAYLOAD|REMOTE=pve03-mail-03-crypt|FILES=203|BYTES=91297553097|MIN_FILES=203|MIN_BYTES=91297553097 +DETAIL=CLOUD_PAYLOAD|REMOTE=pve03-mail-04-crypt|FILES=113|BYTES=13626007377|MIN_FILES=113|MIN_BYTES=13626007377 +DETAIL=CLOUD_TOTAL|FILES=12945|BYTES=1333925024025|BASELINE_FILES=22797|BASELINE_BYTES=1556652840494 +DETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 03:10:18 MSK 9h Fri 2026-07-24 03:13:28 MSK - homelab-emergency-backup.timer homelab-emergency-backup.service +DETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 03:45:47 MSK 9h Fri 2026-07-24 03:48:15 MSK - homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service +DETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 04:11:30 MSK 10h Fri 2026-07-24 04:17:28 MSK - homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service +DETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 04:55:12 MSK 10h Fri 2026-07-24 04:46:28 MSK - homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service +DETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 05:11:48 MSK 11h Fri 2026-07-24 05:18:04 MSK - homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service +DETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 05:42:29 MSK 11h Fri 2026-07-24 05:54:15 MSK - homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service +DETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 06:33:52 MSK 12h Fri 2026-07-24 06:45:28 MSK - homelab-verified-backup-health.timer homelab-verified-backup-health.service +DETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 07:37:34 MSK 13h Fri 2026-07-24 07:43:04 MSK - homelab-vm-backup-health.timer homelab-vm-backup-health.service +DETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 08:05:50 MSK 14h Fri 2026-07-24 08:20:04 MSK - homelab-backup-sla-health.timer homelab-backup-sla-health.service +DETAIL=RC458_EXACT_REASON=PVE01_BROAD_TIMER_FILTER_NONEMPTY +DETAIL=CURRENT_STATE_BAD=13 +DETAIL=CURRENT_STATE_SEAL=FAIL +CHANGES_MADE=NO + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 5b193c3..fb63ebd 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,30 +1,30 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "HOMELAB-BACKUP-PVE02-FREEZE-CORRECTION-458", + "command_id": "HOMELAB-BACKUP-CURRENT-STATE-SEAL-459", "status": "FAIL", "rc": 1, "host": "pve01", - "mode": "change", + "mode": "read-only", "component": "cluster-backup-reset", - "started_at_utc": "2026-07-24T14:17:49Z", - "finished_at_utc": "2026-07-24T14:17:52Z", + "started_at_utc": "2026-07-24T15:02:05Z", + "finished_at_utc": "2026-07-24T15:03:37Z", "reference_register_checked": true, "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", "error_register_checked": true, "error_register_sha256": "24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83", - "command_sha256": "d4fb4a4c10dbf820f364b75704379477e963a9973b89b27ad80b0ec9acc5af89", + "command_sha256": "035aca8551e77c67944c13906d92f66396f48d9ed2512f5d1e22631d5b372a3f", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, - "changes_made": true, + "changes_made": false, "sanitized": true, "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "7b1b9bad6844f66bf2fa7d7ffadef2678eedfcc4a7b7626965f5faf5ed2a621a", - "sanitized_output_sha256": "7b1b9bad6844f66bf2fa7d7ffadef2678eedfcc4a7b7626965f5faf5ed2a621a", + "raw_evidence_sha256": "d4e26d689b466295aa4c8a5ad17e6003bc98db2ade45005316dac02896af8ffa", + "sanitized_output_sha256": "d4e26d689b466295aa4c8a5ad17e6003bc98db2ade45005316dac02896af8ffa", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "Removed '/etc/systemd/system/timers.target.wants/homelab-vaultwarden-isolated-restore-drill.timer'.\nRemoved '/etc/systemd/system/timers.target.wants/homelab-cluster-admin-vm-mail-cloud-backup.timer'.\nDETAIL=PVE02_TIMER|TIMER=forum-prod-fresh5-cloud-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=PVE02_TIMER|TIMER=homelab-cluster-admin-vm-mail-cloud-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=PVE02_TIMER|TIMER=homelab-vaultwarden-isolated-restore-drill.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=PVE02_SERVICE|SERVICE=homelab-cluster-admin-vm-mail-cloud-backup.service|ACTIVE=inactive\nDETAIL=PVE02_SERVICE|SERVICE=homelab-vaultwarden-isolated-restore-drill.service|ACTIVE=inactive\nDETAIL=CORRECTION_FAIL|STAGE=postcheck|RC=1\n" + "output": "DETAIL=NODE_ACCESS|NODE=pve01|STATUS=OK|HOSTNAME=pve01\nDETAIL=NODE_ACCESS|NODE=pve02|STATUS=OK|HOSTNAME=pve02\nDETAIL=NODE_ACCESS|NODE=pve03|STATUS=OK|HOSTNAME=pve03\nDETAIL=PROXMOX_BACKUP_JOB|ID=homelab-nightly-all|ENABLED=0\nDETAIL=TIMER_INVENTORY|NODE=pve01|COUNT=47|EXPECTED_COUNT=37|SHA256=045feaa9c0e86d768f22ae0571023abe4dee718b1f7e6bf0cc7c2607717f9a4e|EXPECTED_SHA256=09810394f8315232c93f8b529fb44d417862525747c912fb628995012c91e9da\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|filebrowser-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|filebrowser-restore-validate.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-alertmanager-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-appbackup-dockge-stacks.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-appbackup-mariadb-batch.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-appbackup-npmplus-edge.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-appbackup-postgres-batch.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-appbackup-sqlite-batch.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-authentik-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-auto-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-backup-coverage-matrix.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-backup-freshness.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-backup-healthcheck.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-backup-matrix-refresh.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-backup-sla-health.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-beszel-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-blackbox-exporter-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-cluster-admin-incident-engine-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-crypto-portfolio-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-edge-vm-vzdump-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-emergency-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-extended-appbackup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-gitea-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-gitea-secondary-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-legacy-monitoring-backups.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-mail-cloud-critical-upload.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-mail-cloud-edge-vm-upload.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-mail-cloud-nextcloud-vm-upload.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-mail-cloud-restore-drill.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-npmplus-kuma-config-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-router-running-config-mail-cloud.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vaultwarden-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-verified-backup-health.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm-backup-health.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm-backup-policy.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm-full-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm-full-restore-drill.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm-local-dumps-2cloud.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm-local-dumps-retention.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm160-remote-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|immich-media-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|memos-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|netbird-vps-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|paperless-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|paperless-restore-dry-run.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|skladchik-reports-monitor-backup-restore-check.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|skladchik-reports-monitor-backup.timer\nDETAIL=SEAL_MISMATCH|CHECK=TIMER_INVENTORY_pve01\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=filebrowser-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=filebrowser-restore-validate.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-alertmanager-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-appbackup-dockge-stacks.timer|ENABLED=enabled|ACTIVE=active\nDETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-appbackup-dockge-stacks.timer\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-appbackup-mariadb-batch.timer|ENABLED=enabled|ACTIVE=active\nDETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-appbackup-mariadb-batch.timer\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-appbackup-npmplus-edge.timer|ENABLED=enabled|ACTIVE=active\nDETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-appbackup-npmplus-edge.timer\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-appbackup-postgres-batch.timer|ENABLED=enabled|ACTIVE=active\nDETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-appbackup-postgres-batch.timer\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-appbackup-sqlite-batch.timer|ENABLED=enabled|ACTIVE=active\nDETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-appbackup-sqlite-batch.timer\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-authentik-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-auto-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-backup-coverage-matrix.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-backup-freshness.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-backup-healthcheck.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-backup-matrix-refresh.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-backup-sla-health.timer|ENABLED=enabled|ACTIVE=active\nDETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-backup-sla-health.timer\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-beszel-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-blackbox-exporter-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-cluster-admin-incident-engine-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-crypto-portfolio-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-edge-vm-vzdump-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-emergency-backup.timer|ENABLED=enabled|ACTIVE=active\nDETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-emergency-backup.timer\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-extended-appbackup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-gitea-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-gitea-secondary-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-legacy-monitoring-backups.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-mail-cloud-critical-upload.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-mail-cloud-edge-vm-upload.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-mail-cloud-nextcloud-vm-upload.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-mail-cloud-restore-drill.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-npmplus-kuma-config-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-router-running-config-mail-cloud.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vaultwarden-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-verified-backup-health.timer|ENABLED=enabled|ACTIVE=active\nDETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-verified-backup-health.timer\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm-backup-health.timer|ENABLED=enabled|ACTIVE=active\nDETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-vm-backup-health.timer\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm-backup-policy.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm-full-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm-full-restore-drill.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm-local-dumps-2cloud.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm-local-dumps-retention.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm160-remote-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=immich-media-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=memos-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=netbird-vps-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=paperless-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=paperless-restore-dry-run.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=skladchik-reports-monitor-backup-restore-check.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve01|TIMER=skladchik-reports-monitor-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=TIMER_INVENTORY|NODE=pve02|COUNT=4|EXPECTED_COUNT=3|SHA256=47b2eff72008dceac8ccfb35a90eca550b4baf2e6340b225ddd8c9cb3e052c46|EXPECTED_SHA256=913306cc221152a7414dbb923b4e3526fbaec4afcc7d072e114d7ea9a87903ca\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve02|codevipe-cloud-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve02|forum-prod-fresh5-cloud-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve02|homelab-cluster-admin-vm-mail-cloud-backup.timer\nDETAIL=TIMER_INVENTORY_MEMBER|NODE=pve02|homelab-vaultwarden-isolated-restore-drill.timer\nDETAIL=SEAL_MISMATCH|CHECK=TIMER_INVENTORY_pve02\nDETAIL=KNOWN_TIMER|NODE=pve02|TIMER=codevipe-cloud-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve02|TIMER=forum-prod-fresh5-cloud-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve02|TIMER=homelab-cluster-admin-vm-mail-cloud-backup.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=KNOWN_TIMER|NODE=pve02|TIMER=homelab-vaultwarden-isolated-restore-drill.timer|ENABLED=disabled|ACTIVE=inactive\nDETAIL=TIMER_INVENTORY|NODE=pve03|COUNT=0|EXPECTED_COUNT=0|SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|EXPECTED_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nDETAIL=SEAL_MISMATCH|CHECK=TIMER_PASS_COUNT\nDETAIL=KNOWN_TIMER_SUMMARY|EXPECTED=40|PASS=42\nDETAIL=WRITER_PROCESS|NODE=pve01|PROCESS=vzdump|STATUS=NONE\nDETAIL=WRITER_PROCESS|NODE=pve01|PROCESS=rclone|STATUS=NONE\nDETAIL=WRITER_PROCESS|NODE=pve02|PROCESS=vzdump|STATUS=NONE\nDETAIL=WRITER_PROCESS|NODE=pve02|PROCESS=rclone|STATUS=NONE\nDETAIL=WRITER_PROCESS|NODE=pve03|PROCESS=vzdump|STATUS=NONE\nDETAIL=WRITER_PROCESS|NODE=pve03|PROCESS=rclone|STATUS=NONE\nDETAIL=GUEST_CONFIG_SHA256|CURRENT=121518389af6a5a6445b2657e2d836fa2a50849f73d19fd50efae703574ba2fc|EXPECTED=121518389af6a5a6445b2657e2d836fa2a50849f73d19fd50efae703574ba2fc\nDETAIL=STORAGE_CONFIG_SHA256|CURRENT=35f32ffa86ebc6a711aa1a44d8a0d3a89872fca686752c9db1fd0270049727b0|EXPECTED=35f32ffa86ebc6a711aa1a44d8a0d3a89872fca686752c9db1fd0270049727b0\nDETAIL=LOCAL_ARCHIVES|NODE=pve01|COUNT=0|BYTES=0|EXPECTED_COUNT=0|EXPECTED_BYTES=0\nDETAIL=LOCAL_STAGING|NODE=pve01|BYTES=158150895639|MIN_EXPECTED=158150895639\nDETAIL=LOCAL_ARCHIVES|NODE=pve02|COUNT=10|BYTES=3910297912|EXPECTED_COUNT=10|EXPECTED_BYTES=3910297912\nDETAIL=LOCAL_STAGING|NODE=pve02|BYTES=111110930032|MIN_EXPECTED=111110930032\nDETAIL=LOCAL_ARCHIVES|NODE=pve03|COUNT=8|BYTES=49287554728|EXPECTED_COUNT=8|EXPECTED_BYTES=49287554728\nDETAIL=LOCAL_STAGING|NODE=pve03|BYTES=106852877874|MIN_EXPECTED=106852877874\nDETAIL=SEAL_MISMATCH|CHECK=CLOUD_READ_pve01-mail-01-crypt\nDETAIL=CLOUD_PAYLOAD|REMOTE=pve01-mail-02-crypt|FILES=9840|BYTES=194612949642|MIN_FILES=9840|MIN_BYTES=194612949642\nDETAIL=CLOUD_PAYLOAD|REMOTE=pve01-mail-03-crypt|FILES=178|BYTES=84995529177|MIN_FILES=178|MIN_BYTES=84995529177\nDETAIL=CLOUD_PAYLOAD|REMOTE=pve01-mail-04-crypt|FILES=178|BYTES=84995529207|MIN_FILES=178|MIN_BYTES=84995529207\nDETAIL=CLOUD_PAYLOAD|REMOTE=pve01-mail-05-crypt|FILES=113|BYTES=13626007377|MIN_FILES=113|MIN_BYTES=13626007377\nDETAIL=CLOUD_PAYLOAD|REMOTE=pve02-mail-01-crypt|FILES=605|BYTES=218239204595|MIN_FILES=605|MIN_BYTES=218239204595\nDETAIL=CLOUD_PAYLOAD|REMOTE=pve02-mail-02-crypt|FILES=605|BYTES=218239204573|MIN_FILES=605|MIN_BYTES=218239204573\nDETAIL=CLOUD_PAYLOAD|REMOTE=pve02-mail-03-crypt|FILES=540|BYTES=207146226209|MIN_FILES=540|MIN_BYTES=207146226209\nDETAIL=CLOUD_PAYLOAD|REMOTE=pve02-mail-04-crypt|FILES=540|BYTES=207146226239|MIN_FILES=540|MIN_BYTES=207146226239\nDETAIL=CLOUD_PAYLOAD|REMOTE=pve03-mail-01-crypt|FILES=15|BYTES=293253|MIN_FILES=15|MIN_BYTES=293253\nDETAIL=CLOUD_PAYLOAD|REMOTE=pve03-mail-02-crypt|FILES=15|BYTES=293279|MIN_FILES=15|MIN_BYTES=293279\nDETAIL=CLOUD_PAYLOAD|REMOTE=pve03-mail-03-crypt|FILES=203|BYTES=91297553097|MIN_FILES=203|MIN_BYTES=91297553097\nDETAIL=CLOUD_PAYLOAD|REMOTE=pve03-mail-04-crypt|FILES=113|BYTES=13626007377|MIN_FILES=113|MIN_BYTES=13626007377\nDETAIL=CLOUD_TOTAL|FILES=12945|BYTES=1333925024025|BASELINE_FILES=22797|BASELINE_BYTES=1556652840494\nDETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 03:10:18 MSK 9h Fri 2026-07-24 03:13:28 MSK - homelab-emergency-backup.timer homelab-emergency-backup.service\nDETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 03:45:47 MSK 9h Fri 2026-07-24 03:48:15 MSK - homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service\nDETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 04:11:30 MSK 10h Fri 2026-07-24 04:17:28 MSK - homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service\nDETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 04:55:12 MSK 10h Fri 2026-07-24 04:46:28 MSK - homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service\nDETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 05:11:48 MSK 11h Fri 2026-07-24 05:18:04 MSK - homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service\nDETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 05:42:29 MSK 11h Fri 2026-07-24 05:54:15 MSK - homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service\nDETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 06:33:52 MSK 12h Fri 2026-07-24 06:45:28 MSK - homelab-verified-backup-health.timer homelab-verified-backup-health.service\nDETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 07:37:34 MSK 13h Fri 2026-07-24 07:43:04 MSK - homelab-vm-backup-health.timer homelab-vm-backup-health.service\nDETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 08:05:50 MSK 14h Fri 2026-07-24 08:20:04 MSK - homelab-backup-sla-health.timer homelab-backup-sla-health.service\nDETAIL=RC458_EXACT_REASON=PVE01_BROAD_TIMER_FILTER_NONEMPTY\nDETAIL=CURRENT_STATE_BAD=13\nDETAIL=CURRENT_STATE_SEAL=FAIL\nCHANGES_MADE=NO\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 647cc27..4131665 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,32 +1,187 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=HOMELAB-BACKUP-PVE02-FREEZE-CORRECTION-458 +COMMAND_ID=HOMELAB-BACKUP-CURRENT-STATE-SEAL-459 STATUS=FAIL RC=1 HOST=pve01 -MODE=change +MODE=read-only COMPONENT=cluster-backup-reset REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83 -COMMAND_SHA256=d4fb4a4c10dbf820f364b75704379477e963a9973b89b27ad80b0ec9acc5af89 +COMMAND_SHA256=035aca8551e77c67944c13906d92f66396f48d9ed2512f5d1e22631d5b372a3f DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true -CHANGES_MADE=true +CHANGES_MADE=false SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=7b1b9bad6844f66bf2fa7d7ffadef2678eedfcc4a7b7626965f5faf5ed2a621a -SANITIZED_OUTPUT_SHA256=7b1b9bad6844f66bf2fa7d7ffadef2678eedfcc4a7b7626965f5faf5ed2a621a +RAW_EVIDENCE_SHA256=d4e26d689b466295aa4c8a5ad17e6003bc98db2ade45005316dac02896af8ffa +SANITIZED_OUTPUT_SHA256=d4e26d689b466295aa4c8a5ad17e6003bc98db2ade45005316dac02896af8ffa OUTPUT_BEGIN -Removed '/etc/systemd/system/timers.target.wants/homelab-vaultwarden-isolated-restore-drill.timer'. -Removed '/etc/systemd/system/timers.target.wants/homelab-cluster-admin-vm-mail-cloud-backup.timer'. -DETAIL=PVE02_TIMER|TIMER=forum-prod-fresh5-cloud-backup.timer|ENABLED=disabled|ACTIVE=inactive -DETAIL=PVE02_TIMER|TIMER=homelab-cluster-admin-vm-mail-cloud-backup.timer|ENABLED=disabled|ACTIVE=inactive -DETAIL=PVE02_TIMER|TIMER=homelab-vaultwarden-isolated-restore-drill.timer|ENABLED=disabled|ACTIVE=inactive -DETAIL=PVE02_SERVICE|SERVICE=homelab-cluster-admin-vm-mail-cloud-backup.service|ACTIVE=inactive -DETAIL=PVE02_SERVICE|SERVICE=homelab-vaultwarden-isolated-restore-drill.service|ACTIVE=inactive -DETAIL=CORRECTION_FAIL|STAGE=postcheck|RC=1 +DETAIL=NODE_ACCESS|NODE=pve01|STATUS=OK|HOSTNAME=pve01 +DETAIL=NODE_ACCESS|NODE=pve02|STATUS=OK|HOSTNAME=pve02 +DETAIL=NODE_ACCESS|NODE=pve03|STATUS=OK|HOSTNAME=pve03 +DETAIL=PROXMOX_BACKUP_JOB|ID=homelab-nightly-all|ENABLED=0 +DETAIL=TIMER_INVENTORY|NODE=pve01|COUNT=47|EXPECTED_COUNT=37|SHA256=045feaa9c0e86d768f22ae0571023abe4dee718b1f7e6bf0cc7c2607717f9a4e|EXPECTED_SHA256=09810394f8315232c93f8b529fb44d417862525747c912fb628995012c91e9da +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|filebrowser-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|filebrowser-restore-validate.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-alertmanager-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-appbackup-dockge-stacks.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-appbackup-mariadb-batch.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-appbackup-npmplus-edge.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-appbackup-postgres-batch.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-appbackup-sqlite-batch.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-authentik-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-auto-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-backup-coverage-matrix.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-backup-freshness.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-backup-healthcheck.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-backup-matrix-refresh.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-backup-sla-health.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-beszel-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-blackbox-exporter-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-cluster-admin-incident-engine-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-crypto-portfolio-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-edge-vm-vzdump-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-emergency-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-extended-appbackup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-gitea-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-gitea-secondary-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-legacy-monitoring-backups.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-mail-cloud-critical-upload.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-mail-cloud-edge-vm-upload.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-mail-cloud-nextcloud-vm-upload.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-mail-cloud-restore-drill.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-npmplus-kuma-config-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-router-running-config-mail-cloud.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vaultwarden-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-verified-backup-health.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm-backup-health.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm-backup-policy.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm-full-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm-full-restore-drill.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm-local-dumps-2cloud.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm-local-dumps-retention.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|homelab-vm160-remote-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|immich-media-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|memos-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|netbird-vps-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|paperless-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|paperless-restore-dry-run.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|skladchik-reports-monitor-backup-restore-check.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve01|skladchik-reports-monitor-backup.timer +DETAIL=SEAL_MISMATCH|CHECK=TIMER_INVENTORY_pve01 +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=filebrowser-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=filebrowser-restore-validate.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-alertmanager-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-appbackup-dockge-stacks.timer|ENABLED=enabled|ACTIVE=active +DETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-appbackup-dockge-stacks.timer +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-appbackup-mariadb-batch.timer|ENABLED=enabled|ACTIVE=active +DETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-appbackup-mariadb-batch.timer +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-appbackup-npmplus-edge.timer|ENABLED=enabled|ACTIVE=active +DETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-appbackup-npmplus-edge.timer +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-appbackup-postgres-batch.timer|ENABLED=enabled|ACTIVE=active +DETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-appbackup-postgres-batch.timer +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-appbackup-sqlite-batch.timer|ENABLED=enabled|ACTIVE=active +DETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-appbackup-sqlite-batch.timer +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-authentik-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-auto-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-backup-coverage-matrix.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-backup-freshness.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-backup-healthcheck.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-backup-matrix-refresh.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-backup-sla-health.timer|ENABLED=enabled|ACTIVE=active +DETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-backup-sla-health.timer +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-beszel-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-blackbox-exporter-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-cluster-admin-incident-engine-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-crypto-portfolio-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-edge-vm-vzdump-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-emergency-backup.timer|ENABLED=enabled|ACTIVE=active +DETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-emergency-backup.timer +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-extended-appbackup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-gitea-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-gitea-secondary-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-legacy-monitoring-backups.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-mail-cloud-critical-upload.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-mail-cloud-edge-vm-upload.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-mail-cloud-nextcloud-vm-upload.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-mail-cloud-restore-drill.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-npmplus-kuma-config-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-router-running-config-mail-cloud.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vaultwarden-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-verified-backup-health.timer|ENABLED=enabled|ACTIVE=active +DETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-verified-backup-health.timer +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm-backup-health.timer|ENABLED=enabled|ACTIVE=active +DETAIL=SEAL_MISMATCH|CHECK=KNOWN_TIMER_pve01_homelab-vm-backup-health.timer +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm-backup-policy.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm-full-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm-full-restore-drill.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm-local-dumps-2cloud.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm-local-dumps-retention.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=homelab-vm160-remote-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=immich-media-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=memos-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=netbird-vps-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=paperless-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=paperless-restore-dry-run.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=skladchik-reports-monitor-backup-restore-check.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve01|TIMER=skladchik-reports-monitor-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=TIMER_INVENTORY|NODE=pve02|COUNT=4|EXPECTED_COUNT=3|SHA256=47b2eff72008dceac8ccfb35a90eca550b4baf2e6340b225ddd8c9cb3e052c46|EXPECTED_SHA256=913306cc221152a7414dbb923b4e3526fbaec4afcc7d072e114d7ea9a87903ca +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve02|codevipe-cloud-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve02|forum-prod-fresh5-cloud-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve02|homelab-cluster-admin-vm-mail-cloud-backup.timer +DETAIL=TIMER_INVENTORY_MEMBER|NODE=pve02|homelab-vaultwarden-isolated-restore-drill.timer +DETAIL=SEAL_MISMATCH|CHECK=TIMER_INVENTORY_pve02 +DETAIL=KNOWN_TIMER|NODE=pve02|TIMER=codevipe-cloud-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve02|TIMER=forum-prod-fresh5-cloud-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve02|TIMER=homelab-cluster-admin-vm-mail-cloud-backup.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=KNOWN_TIMER|NODE=pve02|TIMER=homelab-vaultwarden-isolated-restore-drill.timer|ENABLED=disabled|ACTIVE=inactive +DETAIL=TIMER_INVENTORY|NODE=pve03|COUNT=0|EXPECTED_COUNT=0|SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|EXPECTED_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +DETAIL=SEAL_MISMATCH|CHECK=TIMER_PASS_COUNT +DETAIL=KNOWN_TIMER_SUMMARY|EXPECTED=40|PASS=42 +DETAIL=WRITER_PROCESS|NODE=pve01|PROCESS=vzdump|STATUS=NONE +DETAIL=WRITER_PROCESS|NODE=pve01|PROCESS=rclone|STATUS=NONE +DETAIL=WRITER_PROCESS|NODE=pve02|PROCESS=vzdump|STATUS=NONE +DETAIL=WRITER_PROCESS|NODE=pve02|PROCESS=rclone|STATUS=NONE +DETAIL=WRITER_PROCESS|NODE=pve03|PROCESS=vzdump|STATUS=NONE +DETAIL=WRITER_PROCESS|NODE=pve03|PROCESS=rclone|STATUS=NONE +DETAIL=GUEST_CONFIG_SHA256|CURRENT=121518389af6a5a6445b2657e2d836fa2a50849f73d19fd50efae703574ba2fc|EXPECTED=121518389af6a5a6445b2657e2d836fa2a50849f73d19fd50efae703574ba2fc +DETAIL=STORAGE_CONFIG_SHA256|CURRENT=35f32ffa86ebc6a711aa1a44d8a0d3a89872fca686752c9db1fd0270049727b0|EXPECTED=35f32ffa86ebc6a711aa1a44d8a0d3a89872fca686752c9db1fd0270049727b0 +DETAIL=LOCAL_ARCHIVES|NODE=pve01|COUNT=0|BYTES=0|EXPECTED_COUNT=0|EXPECTED_BYTES=0 +DETAIL=LOCAL_STAGING|NODE=pve01|BYTES=158150895639|MIN_EXPECTED=158150895639 +DETAIL=LOCAL_ARCHIVES|NODE=pve02|COUNT=10|BYTES=3910297912|EXPECTED_COUNT=10|EXPECTED_BYTES=3910297912 +DETAIL=LOCAL_STAGING|NODE=pve02|BYTES=111110930032|MIN_EXPECTED=111110930032 +DETAIL=LOCAL_ARCHIVES|NODE=pve03|COUNT=8|BYTES=49287554728|EXPECTED_COUNT=8|EXPECTED_BYTES=49287554728 +DETAIL=LOCAL_STAGING|NODE=pve03|BYTES=106852877874|MIN_EXPECTED=106852877874 +DETAIL=SEAL_MISMATCH|CHECK=CLOUD_READ_pve01-mail-01-crypt +DETAIL=CLOUD_PAYLOAD|REMOTE=pve01-mail-02-crypt|FILES=9840|BYTES=194612949642|MIN_FILES=9840|MIN_BYTES=194612949642 +DETAIL=CLOUD_PAYLOAD|REMOTE=pve01-mail-03-crypt|FILES=178|BYTES=84995529177|MIN_FILES=178|MIN_BYTES=84995529177 +DETAIL=CLOUD_PAYLOAD|REMOTE=pve01-mail-04-crypt|FILES=178|BYTES=84995529207|MIN_FILES=178|MIN_BYTES=84995529207 +DETAIL=CLOUD_PAYLOAD|REMOTE=pve01-mail-05-crypt|FILES=113|BYTES=13626007377|MIN_FILES=113|MIN_BYTES=13626007377 +DETAIL=CLOUD_PAYLOAD|REMOTE=pve02-mail-01-crypt|FILES=605|BYTES=218239204595|MIN_FILES=605|MIN_BYTES=218239204595 +DETAIL=CLOUD_PAYLOAD|REMOTE=pve02-mail-02-crypt|FILES=605|BYTES=218239204573|MIN_FILES=605|MIN_BYTES=218239204573 +DETAIL=CLOUD_PAYLOAD|REMOTE=pve02-mail-03-crypt|FILES=540|BYTES=207146226209|MIN_FILES=540|MIN_BYTES=207146226209 +DETAIL=CLOUD_PAYLOAD|REMOTE=pve02-mail-04-crypt|FILES=540|BYTES=207146226239|MIN_FILES=540|MIN_BYTES=207146226239 +DETAIL=CLOUD_PAYLOAD|REMOTE=pve03-mail-01-crypt|FILES=15|BYTES=293253|MIN_FILES=15|MIN_BYTES=293253 +DETAIL=CLOUD_PAYLOAD|REMOTE=pve03-mail-02-crypt|FILES=15|BYTES=293279|MIN_FILES=15|MIN_BYTES=293279 +DETAIL=CLOUD_PAYLOAD|REMOTE=pve03-mail-03-crypt|FILES=203|BYTES=91297553097|MIN_FILES=203|MIN_BYTES=91297553097 +DETAIL=CLOUD_PAYLOAD|REMOTE=pve03-mail-04-crypt|FILES=113|BYTES=13626007377|MIN_FILES=113|MIN_BYTES=13626007377 +DETAIL=CLOUD_TOTAL|FILES=12945|BYTES=1333925024025|BASELINE_FILES=22797|BASELINE_BYTES=1556652840494 +DETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 03:10:18 MSK 9h Fri 2026-07-24 03:13:28 MSK - homelab-emergency-backup.timer homelab-emergency-backup.service +DETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 03:45:47 MSK 9h Fri 2026-07-24 03:48:15 MSK - homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service +DETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 04:11:30 MSK 10h Fri 2026-07-24 04:17:28 MSK - homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service +DETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 04:55:12 MSK 10h Fri 2026-07-24 04:46:28 MSK - homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service +DETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 05:11:48 MSK 11h Fri 2026-07-24 05:18:04 MSK - homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service +DETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 05:42:29 MSK 11h Fri 2026-07-24 05:54:15 MSK - homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service +DETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 06:33:52 MSK 12h Fri 2026-07-24 06:45:28 MSK - homelab-verified-backup-health.timer homelab-verified-backup-health.service +DETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 07:37:34 MSK 13h Fri 2026-07-24 07:43:04 MSK - homelab-vm-backup-health.timer homelab-vm-backup-health.service +DETAIL=RC458_REPLAY|NODE=pve01|Sat 2026-07-25 08:05:50 MSK 14h Fri 2026-07-24 08:20:04 MSK - homelab-backup-sla-health.timer homelab-backup-sla-health.service +DETAIL=RC458_EXACT_REASON=PVE01_BROAD_TIMER_FILTER_NONEMPTY +DETAIL=CURRENT_STATE_BAD=13 +DETAIL=CURRENT_STATE_SEAL=FAIL +CHANGES_MADE=NO OUTPUT_END CHAT_OUTPUT_END