From b9f573979a9f3cf71a0d59cd903ca013a4060888 Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Fri, 24 Jul 2026 09:55:31 +0000 Subject: [PATCH] runtime: publish HOMELAB-BACKUP-BAD-JOBS-ROOTCAUSE-431 --- ...HOMELAB-BACKUP-BAD-JOBS-ROOTCAUSE-431.json | 30 + .../HOMELAB-BACKUP-BAD-JOBS-ROOTCAUSE-431.txt | 557 ++++++++++++++++ runtime/latest.json | 14 +- runtime/latest.txt | 606 ++++++++++++++++-- 4 files changed, 1129 insertions(+), 78 deletions(-) create mode 100644 runtime/history/HOMELAB-BACKUP-BAD-JOBS-ROOTCAUSE-431.json create mode 100644 runtime/history/HOMELAB-BACKUP-BAD-JOBS-ROOTCAUSE-431.txt diff --git a/runtime/history/HOMELAB-BACKUP-BAD-JOBS-ROOTCAUSE-431.json b/runtime/history/HOMELAB-BACKUP-BAD-JOBS-ROOTCAUSE-431.json new file mode 100644 index 0000000..31daf1c --- /dev/null +++ b/runtime/history/HOMELAB-BACKUP-BAD-JOBS-ROOTCAUSE-431.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "HOMELAB-BACKUP-BAD-JOBS-ROOTCAUSE-431", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "cluster-audit", + "started_at_utc": "2026-07-24T09:55:30Z", + "finished_at_utc": "2026-07-24T09:55:30Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", + "command_sha256": "8635e368c9b3e168a7302affcd2c133d8f85547164b3f3352fc0cbad679edc98", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "e39cc1d76af62d7050936445c27d21728aaeb6ca00ab2430c6cb3963a7da3907", + "sanitized_output_sha256": "e39cc1d76af62d7050936445c27d21728aaeb6ca00ab2430c6cb3963a7da3907", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "DETAIL=MATRIX_GENERATED_AT=2026-07-24T07:06:04Z\nDETAIL=TARGET_JOB_COUNT=6\nDETAIL=JOB_BEGIN=filebrowser_restore_validate\nDETAIL=JOB_FIELD=id=filebrowser_restore_validate\nDETAIL=JOB_FIELD=name=filebrowser-restore-validate\nDETAIL=JOB_FIELD=category=restore\nDETAIL=JOB_FIELD=critical=True\nDETAIL=JOB_FIELD=status=STALE\nDETAIL=JOB_FIELD=freshness=STALE\nDETAIL=JOB_FIELD=age_seconds=1275827\nDETAIL=JOB_FIELD=max_age_seconds=864000\nDETAIL=JOB_FIELD=source_kind=health_file\nDETAIL=JOB_FIELD=source_status=OK\nDETAIL=JOB_FIELD=source_mtime=2026-07-09T12:42:17Z\nDETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt\nDETAIL=JOB_FIELD=service=filebrowser-restore-validate.service\nDETAIL=JOB_FIELD=service_result=success\nDETAIL=JOB_FIELD=service_exec_main_status=0\nDETAIL=JOB_FIELD=timer=filebrowser-restore-validate.timer\nDETAIL=JOB_FIELD=timer_enabled=enabled\nDETAIL=JOB_FIELD=timer_active=active\nDETAIL=SERVICE_SHOW_RC=0\nDETAIL=SERVICE_PROPERTY=Result=success\nDETAIL=SERVICE_PROPERTY=ExecMainCode=1\nDETAIL=SERVICE_PROPERTY=ExecMainStatus=0\nDETAIL=SERVICE_PROPERTY=LoadState=loaded\nDETAIL=SERVICE_PROPERTY=ActiveState=inactive\nDETAIL=SERVICE_PROPERTY=SubState=dead\nDETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/filebrowser-restore-validate.service\nDETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/filebrowser-restore-validate.service.d/10-superseded-noop.conf /etc/systemd/system/filebrowser-restore-validate.service.d/90-backup-matrix-refresh.conf\nDETAIL=SERVICE_JOURNAL_MATCH_COUNT=56\nDETAIL=SERVICE_JOURNAL=2026-07-11T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-11T04:10:00+03:00 pve01 homelab-superseded-unit-ok[1686870]: STATUS=OK TS=2026-07-11T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-11T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-11T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-12T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-12T04:10:00+03:00 pve01 homelab-superseded-unit-ok[1699058]: STATUS=OK TS=2026-07-12T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-12T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-12T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-13T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-13T04:10:00+03:00 pve01 homelab-superseded-unit-ok[1681015]: STATUS=OK TS=2026-07-13T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-13T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-13T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-14T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-14T04:10:00+03:00 pve01 homelab-superseded-unit-ok[2132068]: STATUS=OK TS=2026-07-14T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-14T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-14T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-15T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-15T04:10:00+03:00 pve01 homelab-superseded-unit-ok[2783319]: STATUS=OK TS=2026-07-15T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-15T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-15T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-16T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-16T04:10:00+03:00 pve01 homelab-superseded-unit-ok[3557730]: STATUS=OK TS=2026-07-16T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-16T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-16T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-17T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-17T04:10:14+03:00 pve01 homelab-superseded-unit-ok[3346747]: STATUS=OK TS=2026-07-17T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-17T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-17T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-18T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-18T04:10:14+03:00 pve01 homelab-superseded-unit-ok[4129958]: STATUS=OK TS=2026-07-18T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-18T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-18T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-19T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-19T04:10:14+03:00 pve01 homelab-superseded-unit-ok[738797]: STATUS=OK TS=2026-07-19T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-19T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-19T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-20T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-20T04:10:14+03:00 pve01 homelab-superseded-unit-ok[1448291]: STATUS=OK TS=2026-07-20T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-20T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-20T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-21T04:10:07+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-21T04:10:07+03:00 pve01 homelab-superseded-unit-ok[2324924]: STATUS=OK TS=2026-07-21T01:10:07Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-21T04:10:07+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-21T04:10:07+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-22T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-22T04:10:14+03:00 pve01 homelab-superseded-unit-ok[3008751]: STATUS=OK TS=2026-07-22T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-22T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-22T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-23T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-23T04:10:14+03:00 pve01 homelab-superseded-unit-ok[3700126]: STATUS=OK TS=2026-07-23T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-23T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-23T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-24T04:10:04+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-24T04:10:04+03:00 pve01 homelab-superseded-unit-ok[2416005]: STATUS=OK TS=2026-07-24T01:10:04Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-24T04:10:04+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-24T04:10:04+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=TIMER_SHOW_RC=0\nDETAIL=TIMER_PROPERTY=Unit=filebrowser-restore-validate.service\nDETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Sat 2026-07-25 04:10:00 MSK\nDETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 04:10:04 MSK\nDETAIL=TIMER_PROPERTY=LoadState=loaded\nDETAIL=TIMER_PROPERTY=ActiveState=active\nDETAIL=TIMER_PROPERTY=SubState=waiting\nDETAIL=TIMER_ENABLED_RESULT=enabled|RC=0\nDETAIL=HEALTH_METADATA=BYTES:209|MTIME_NS:1783600937792562046\nDETAIL=HEALTH_MATCH_COUNT=1\nDETAIL=HEALTH_LINE=STATUS=OK TS=2026-07-09T12:42:13Z TYPE=filebrowser-restore JOB=homelab-pve01-app-filebrowser-20260709T051713Z SOURCE=appbackupctl-restore-check RESTORECHECK=OK_BOTH_CLOUDS NOTE=service-readiness-restore-proof\nDETAIL=JOB_END=filebrowser_restore_validate\nDETAIL=JOB_BEGIN=homelab_backup_sla_health\nDETAIL=JOB_FIELD=id=homelab_backup_sla_health\nDETAIL=JOB_FIELD=name=homelab-backup-sla-health\nDETAIL=JOB_FIELD=category=application\nDETAIL=JOB_FIELD=critical=True\nDETAIL=JOB_FIELD=status=FAIL\nDETAIL=JOB_FIELD=freshness=FRESH\nDETAIL=JOB_FIELD=age_seconds=358\nDETAIL=JOB_FIELD=max_age_seconds=172800\nDETAIL=JOB_FIELD=source_kind=health_file\nDETAIL=JOB_FIELD=source_status=WARN\nDETAIL=JOB_FIELD=source_mtime=2026-07-24T07:00:06Z\nDETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt\nDETAIL=JOB_FIELD=service=homelab-backup-sla-health.service\nDETAIL=JOB_FIELD=service_result=success\nDETAIL=JOB_FIELD=service_exec_main_status=0\nDETAIL=JOB_FIELD=timer=homelab-backup-sla-health.timer\nDETAIL=JOB_FIELD=timer_enabled=enabled\nDETAIL=JOB_FIELD=timer_active=active\nDETAIL=SERVICE_SHOW_RC=0\nDETAIL=SERVICE_PROPERTY=Result=success\nDETAIL=SERVICE_PROPERTY=ExecMainCode=1\nDETAIL=SERVICE_PROPERTY=ExecMainStatus=0\nDETAIL=SERVICE_PROPERTY=LoadState=loaded\nDETAIL=SERVICE_PROPERTY=ActiveState=inactive\nDETAIL=SERVICE_PROPERTY=SubState=dead\nDETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/homelab-backup-sla-health.service\nDETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/homelab-backup-sla-health.service.d/90-backup-matrix-refresh.conf\nDETAIL=SERVICE_JOURNAL_MATCH_COUNT=76\nDETAIL=SERVICE_JOURNAL=2026-07-11T06:24:11+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-11T06:24:11+03:00 pve01 homelab-backup-sla-health[2238839]: STATUS=OK TS=2026-07-11T03:24:11Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-11T06:24:11+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-11T06:24:11+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-11T07:39:35+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-11T07:39:36+03:00 pve01 homelab-backup-sla-health[2476861]: STATUS=OK TS=2026-07-11T04:39:35Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-11T07:39:36+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-11T07:39:36+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-11T08:15:32+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-11T08:15:32+03:00 pve01 homelab-backup-sla-health[2616083]: STATUS=OK TS=2026-07-11T05:15:32Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-11T08:15:33+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-11T08:15:33+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-12T08:04:09+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-12T08:04:09+03:00 pve01 homelab-backup-sla-health[2414445]: STATUS=WARN TS=2026-07-12T05:04:09Z TYPE=backup-sla BAD=0 WARN=1 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-12T08:04:09+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-12T08:04:09+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-13T01:03:10+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-13T01:03:11+03:00 pve01 homelab-backup-sla-health[980591]: STATUS=OK TS=2026-07-12T22:03:10Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-13T01:03:11+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-13T01:03:11+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-13T01:08:07+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-13T01:08:08+03:00 pve01 homelab-backup-sla-health[994479]: STATUS=OK TS=2026-07-12T22:08:07Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-13T01:08:08+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-13T01:08:08+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-13T08:15:21+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-13T08:15:21+03:00 pve01 homelab-backup-sla-health[2521375]: STATUS=OK TS=2026-07-13T05:15:21Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-13T08:15:21+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-13T08:15:21+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-14T08:05:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-14T08:05:15+03:00 pve01 homelab-backup-sla-health[2983241]: STATUS=WARN TS=2026-07-14T05:05:14Z TYPE=backup-sla BAD=0 WARN=1 APPS=46 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-14T08:05:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-14T08:05:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-14T08:23:27+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-14T08:23:28+03:00 pve01 homelab-backup-sla-health[3042266]: STATUS=OK TS=2026-07-14T05:23:27Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-14T08:23:28+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-14T08:23:28+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-15T08:19:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-15T08:19:15+03:00 pve01 homelab-backup-sla-health[3670619]: STATUS=OK TS=2026-07-15T05:19:14Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-15T08:19:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-15T08:19:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-16T08:09:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-16T08:09:15+03:00 pve01 homelab-backup-sla-health[225757]: STATUS=WARN TS=2026-07-16T05:09:14Z TYPE=backup-sla BAD=0 WARN=1 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-16T08:09:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-16T08:09:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-17T08:07:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-17T08:07:15+03:00 pve01 homelab-backup-sla-health[4188049]: STATUS=WARN TS=2026-07-17T05:07:14Z TYPE=backup-sla BAD=0 WARN=4 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-17T08:07:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-17T08:07:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-18T08:05:29+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-18T08:05:29+03:00 pve01 homelab-backup-sla-health[806098]: STATUS=OK TS=2026-07-18T05:05:29Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-18T08:05:29+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-18T08:05:29+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-19T08:05:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-19T08:05:15+03:00 pve01 homelab-backup-sla-health[1545067]: STATUS=OK TS=2026-07-19T05:05:14Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-19T08:05:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-19T08:05:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-20T08:10:25+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-20T08:10:26+03:00 pve01 homelab-backup-sla-health[2331602]: STATUS=OK TS=2026-07-20T05:10:25Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-20T08:10:26+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-20T08:10:26+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-21T08:07:25+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-21T08:07:26+03:00 pve01 homelab-backup-sla-health[3122692]: STATUS=WARN TS=2026-07-21T05:07:25Z TYPE=backup-sla BAD=0 WARN=1 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-21T08:07:26+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-21T08:07:26+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-22T08:10:41+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-22T08:10:42+03:00 pve01 homelab-backup-sla-health[3806055]: STATUS=WARN TS=2026-07-22T05:10:41Z TYPE=backup-sla BAD=0 WARN=1 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-22T08:10:42+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-22T08:10:42+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-23T08:06:25+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-23T08:06:26+03:00 pve01 homelab-backup-sla-health[310318]: STATUS=WARN TS=2026-07-23T05:06:25Z TYPE=backup-sla BAD=0 WARN=2 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-23T08:06:26+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-23T08:06:26+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-24T08:20:04+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-24T08:20:04+03:00 pve01 homelab-backup-sla-health[3253899]: STATUS=WARN TS=2026-07-24T05:20:04Z TYPE=backup-sla BAD=0 WARN=4 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-24T08:20:04+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-24T08:20:04+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=TIMER_SHOW_RC=0\nDETAIL=TIMER_PROPERTY=Unit=homelab-backup-sla-health.service\nDETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Sat 2026-07-25 08:08:57 MSK\nDETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 08:20:04 MSK\nDETAIL=TIMER_PROPERTY=LoadState=loaded\nDETAIL=TIMER_PROPERTY=ActiveState=active\nDETAIL=TIMER_PROPERTY=SubState=waiting\nDETAIL=TIMER_ENABLED_RESULT=enabled|RC=0\nDETAIL=HEALTH_METADATA=BYTES:154|MTIME_NS:1784876406380685648\nDETAIL=HEALTH_MATCH_COUNT=1\nDETAIL=HEALTH_LINE=STATUS=WARN TS=2026-07-24T07:00:05Z TYPE=backup-sla BAD=0 WARN=4 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=JOB_END=homelab_backup_sla_health\nDETAIL=JOB_BEGIN=homelab_mail_cloud_restore_drill\nDETAIL=JOB_FIELD=id=homelab_mail_cloud_restore_drill\nDETAIL=JOB_FIELD=name=homelab-mail-cloud-restore-drill\nDETAIL=JOB_FIELD=category=restore\nDETAIL=JOB_FIELD=critical=True\nDETAIL=JOB_FIELD=status=STALE\nDETAIL=JOB_FIELD=freshness=STALE\nDETAIL=JOB_FIELD=age_seconds=1247672\nDETAIL=JOB_FIELD=max_age_seconds=864000\nDETAIL=JOB_FIELD=source_kind=health_file\nDETAIL=JOB_FIELD=source_status=OK\nDETAIL=JOB_FIELD=source_mtime=2026-07-09T20:31:32Z\nDETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/homelab_mail_cloud_restore_drill/0-mail-cloud-restore-drill.txt\nDETAIL=JOB_FIELD=service=homelab-mail-cloud-restore-drill.service\nDETAIL=JOB_FIELD=service_result=success\nDETAIL=JOB_FIELD=service_exec_main_status=0\nDETAIL=JOB_FIELD=timer=homelab-mail-cloud-restore-drill.timer\nDETAIL=JOB_FIELD=timer_enabled=enabled\nDETAIL=JOB_FIELD=timer_active=active\nDETAIL=SERVICE_SHOW_RC=0\nDETAIL=SERVICE_PROPERTY=Result=success\nDETAIL=SERVICE_PROPERTY=ExecMainCode=0\nDETAIL=SERVICE_PROPERTY=ExecMainStatus=0\nDETAIL=SERVICE_PROPERTY=LoadState=loaded\nDETAIL=SERVICE_PROPERTY=ActiveState=inactive\nDETAIL=SERVICE_PROPERTY=SubState=dead\nDETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/homelab-mail-cloud-restore-drill.service\nDETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/homelab-mail-cloud-restore-drill.service.d/10-superseded-noop.conf /etc/systemd/system/homelab-mail-cloud-restore-drill.service.d/90-backup-matrix-refresh.conf\nDETAIL=SERVICE_JOURNAL_MATCH_COUNT=8\nDETAIL=SERVICE_JOURNAL=2026-07-13T07:38:36+03:00 pve01 systemd[1]: Starting homelab-mail-cloud-restore-drill.service - Homelab Mail Cloud restore drill...\nDETAIL=SERVICE_JOURNAL=2026-07-13T07:38:36+03:00 pve01 homelab-superseded-unit-ok[2401599]: STATUS=OK TS=2026-07-13T04:38:36Z TYPE=superseded-unit UNIT=homelab-mail-cloud-restore-drill.service REASON=superseded_by_safety_net_restore_check ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-13T07:38:36+03:00 pve01 systemd[1]: homelab-mail-cloud-restore-drill.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-13T07:38:36+03:00 pve01 systemd[1]: Finished homelab-mail-cloud-restore-drill.service - Homelab Mail Cloud restore drill.\nDETAIL=SERVICE_JOURNAL=2026-07-20T07:25:14+03:00 pve01 systemd[1]: Starting homelab-mail-cloud-restore-drill.service - Homelab Mail Cloud restore drill...\nDETAIL=SERVICE_JOURNAL=2026-07-20T07:25:14+03:00 pve01 homelab-superseded-unit-ok[2171079]: STATUS=OK TS=2026-07-20T04:25:14Z TYPE=superseded-unit UNIT=homelab-mail-cloud-restore-drill.service REASON=superseded_by_safety_net_restore_check ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-20T07:25:14+03:00 pve01 systemd[1]: homelab-mail-cloud-restore-drill.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-20T07:25:14+03:00 pve01 systemd[1]: Finished homelab-mail-cloud-restore-drill.service - Homelab Mail Cloud restore drill.\nDETAIL=TIMER_SHOW_RC=0\nDETAIL=TIMER_PROPERTY=Unit=homelab-mail-cloud-restore-drill.service\nDETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Mon 2026-07-27 07:34:52 MSK\nDETAIL=TIMER_PROPERTY=LastTriggerUSec=Mon 2026-07-20 07:25:14 MSK\nDETAIL=TIMER_PROPERTY=LoadState=loaded\nDETAIL=TIMER_PROPERTY=ActiveState=active\nDETAIL=TIMER_PROPERTY=SubState=waiting\nDETAIL=TIMER_ENABLED_RESULT=enabled|RC=0\nDETAIL=HEALTH_METADATA=BYTES:353|MTIME_NS:1783629092051018609\nDETAIL=HEALTH_MATCH_COUNT=1\nDETAIL=HEALTH_LINE=STATUS=OK TS=2026-07-09T20:31:32Z TYPE=legacy-mail-cloud-restore-drill-superseded MODE=SUPERSEDED_BY_CURRENT_PROOF BAD=0 WARN=0 PROOF=safety_net_control_plane_restore_check_OK ACTION=stale_health_refresh_only REASON=legacy_drill_superseded_but_health_file_mtime_must_stay_fresh REPORT=/var/lib/homelab-failed-unit-legacy-cleanup/local-cleanup-latest.md\nDETAIL=JOB_END=homelab_mail_cloud_restore_drill\nDETAIL=JOB_BEGIN=netbird_vps_backup\nDETAIL=JOB_FIELD=id=netbird_vps_backup\nDETAIL=JOB_FIELD=name=netbird-vps-backup\nDETAIL=JOB_FIELD=category=application\nDETAIL=JOB_FIELD=critical=True\nDETAIL=JOB_FIELD=status=FAIL\nDETAIL=JOB_FIELD=freshness=STALE\nDETAIL=JOB_FIELD=age_seconds=2007299\nDETAIL=JOB_FIELD=max_age_seconds=864000\nDETAIL=JOB_FIELD=source_kind=health_file\nDETAIL=JOB_FIELD=source_status=OK\nDETAIL=JOB_FIELD=source_mtime=2026-07-01T01:31:05Z\nDETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt\nDETAIL=JOB_FIELD=service=netbird-vps-backup.service\nDETAIL=JOB_FIELD=service_result=success\nDETAIL=JOB_FIELD=service_exec_main_status=0\nDETAIL=JOB_FIELD=timer=netbird-vps-backup.timer\nDETAIL=JOB_FIELD=timer_enabled=disabled\nDETAIL=JOB_FIELD=timer_active=inactive\nDETAIL=SERVICE_SHOW_RC=0\nDETAIL=SERVICE_PROPERTY=Result=success\nDETAIL=SERVICE_PROPERTY=ExecMainCode=0\nDETAIL=SERVICE_PROPERTY=ExecMainStatus=0\nDETAIL=SERVICE_PROPERTY=LoadState=loaded\nDETAIL=SERVICE_PROPERTY=ActiveState=inactive\nDETAIL=SERVICE_PROPERTY=SubState=dead\nDETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/netbird-vps-backup.service\nDETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/netbird-vps-backup.service.d/10-superseded-noop.conf /etc/systemd/system/netbird-vps-backup.service.d/90-backup-matrix-refresh.conf\nDETAIL=SERVICE_JOURNAL_MATCH_COUNT=28\nDETAIL=SERVICE_JOURNAL=2026-07-11T04:30:35+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging...\nDETAIL=SERVICE_JOURNAL=2026-07-11T04:30:35+03:00 pve01 homelab-superseded-unit-ok[1772638]: STATUS=OK TS=2026-07-11T01:30:35Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-11T04:30:35+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-11T04:30:35+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging.\nDETAIL=SERVICE_JOURNAL=2026-07-12T04:39:35+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging...\nDETAIL=SERVICE_JOURNAL=2026-07-12T04:39:35+03:00 pve01 homelab-superseded-unit-ok[1789866]: STATUS=OK TS=2026-07-12T01:39:35Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-12T04:39:35+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-12T04:39:35+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging.\nDETAIL=SERVICE_JOURNAL=2026-07-13T04:28:00+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging...\nDETAIL=SERVICE_JOURNAL=2026-07-13T04:28:00+03:00 pve01 homelab-superseded-unit-ok[1739137]: STATUS=OK TS=2026-07-13T01:28:00Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-13T04:28:00+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-13T04:28:00+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging.\nDETAIL=SERVICE_JOURNAL=2026-07-14T04:28:39+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging...\nDETAIL=SERVICE_JOURNAL=2026-07-14T04:28:39+03:00 pve01 homelab-superseded-unit-ok[2199841]: STATUS=OK TS=2026-07-14T01:28:39Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-14T04:28:39+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-14T04:28:39+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging.\nDETAIL=SERVICE_JOURNAL=2026-07-15T04:39:00+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging...\nDETAIL=SERVICE_JOURNAL=2026-07-15T04:39:00+03:00 pve01 homelab-superseded-unit-ok[2872498]: STATUS=OK TS=2026-07-15T01:39:00Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-15T04:39:00+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-15T04:39:00+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging.\nDETAIL=SERVICE_JOURNAL=2026-07-16T04:25:14+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging...\nDETAIL=SERVICE_JOURNAL=2026-07-16T04:25:14+03:00 pve01 homelab-superseded-unit-ok[3611661]: STATUS=OK TS=2026-07-16T01:25:14Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-16T04:25:14+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-16T04:25:14+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging.\nDETAIL=SERVICE_JOURNAL=2026-07-17T04:31:14+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging...\nDETAIL=SERVICE_JOURNAL=2026-07-17T04:31:14+03:00 pve01 homelab-superseded-unit-ok[3412653]: STATUS=OK TS=2026-07-17T01:31:14Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-17T04:31:14+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-17T04:31:14+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging.\nDETAIL=TIMER_SHOW_RC=0\nDETAIL=TIMER_PROPERTY=Unit=netbird-vps-backup.service\nDETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=\nDETAIL=TIMER_PROPERTY=LastTriggerUSec=\nDETAIL=TIMER_PROPERTY=LoadState=loaded\nDETAIL=TIMER_PROPERTY=ActiveState=inactive\nDETAIL=TIMER_PROPERTY=SubState=dead\nDETAIL=TIMER_ENABLED_RESULT=disabled|RC=1\nDETAIL=HEALTH_METADATA=BYTES:312|MTIME_NS:1782869465433646905\nDETAIL=HEALTH_MATCH_COUNT=7\nDETAIL=HEALTH_LINE=STATUS=OK\nDETAIL=HEALTH_LINE=SERVICE=netbird-vps-backup\nDETAIL=HEALTH_LINE=TIMESTAMP_UTC=20260701T013054Z\nDETAIL=HEALTH_LINE=SNAPSHOT=/mnt/staging/netbird-vps-backups/snapshots/20260701T013054Z\nDETAIL=HEALTH_LINE=LATEST_SYMLINK=/mnt/staging/netbird-vps-backups/snapshots/latest\nDETAIL=HEALTH_LINE=SIZE=32M\nDETAIL=HEALTH_LINE=SHA256=0226dd820cb9ede0ea9e8ea4af6cef656ffa54cfb819048c222a88f4c14d8db9\nDETAIL=JOB_END=netbird_vps_backup\nDETAIL=JOB_BEGIN=paperless_restore_dry_run\nDETAIL=JOB_FIELD=id=paperless_restore_dry_run\nDETAIL=JOB_FIELD=name=paperless-restore-dry-run\nDETAIL=JOB_FIELD=category=restore\nDETAIL=JOB_FIELD=critical=True\nDETAIL=JOB_FIELD=status=STALE\nDETAIL=JOB_FIELD=freshness=STALE\nDETAIL=JOB_FIELD=age_seconds=1261389\nDETAIL=JOB_FIELD=max_age_seconds=864000\nDETAIL=JOB_FIELD=source_kind=health_file\nDETAIL=JOB_FIELD=source_status=OK\nDETAIL=JOB_FIELD=source_mtime=2026-07-09T16:42:55Z\nDETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/paperless_restore_dry_run/0-paperless-restore.txt\nDETAIL=JOB_FIELD=service=paperless-restore-dry-run.service\nDETAIL=JOB_FIELD=service_result=success\nDETAIL=JOB_FIELD=service_exec_main_status=0\nDETAIL=JOB_FIELD=timer=paperless-restore-dry-run.timer\nDETAIL=JOB_FIELD=timer_enabled=enabled\nDETAIL=JOB_FIELD=timer_active=active\nDETAIL=SERVICE_SHOW_RC=0\nDETAIL=SERVICE_PROPERTY=Result=success\nDETAIL=SERVICE_PROPERTY=ExecMainCode=1\nDETAIL=SERVICE_PROPERTY=ExecMainStatus=0\nDETAIL=SERVICE_PROPERTY=LoadState=loaded\nDETAIL=SERVICE_PROPERTY=ActiveState=inactive\nDETAIL=SERVICE_PROPERTY=SubState=dead\nDETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/paperless-restore-dry-run.service\nDETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/paperless-restore-dry-run.service.d/10-superseded-noop.conf /etc/systemd/system/paperless-restore-dry-run.service.d/90-backup-matrix-refresh.conf\nDETAIL=SERVICE_JOURNAL_MATCH_COUNT=56\nDETAIL=SERVICE_JOURNAL=2026-07-11T10:08:13+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-11T10:08:13+03:00 pve01 homelab-superseded-unit-ok[2988079]: STATUS=OK TS=2026-07-11T07:08:13Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-11T10:08:13+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-11T10:08:13+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-12T10:04:59+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-12T10:04:59+03:00 pve01 homelab-superseded-unit-ok[2757392]: STATUS=OK TS=2026-07-12T07:04:59Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-12T10:04:59+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-12T10:04:59+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-13T10:07:25+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-13T10:07:25+03:00 pve01 homelab-superseded-unit-ok[2886298]: STATUS=OK TS=2026-07-13T07:07:25Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-13T10:07:25+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-13T10:07:25+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-14T09:56:36+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-14T09:56:36+03:00 pve01 homelab-superseded-unit-ok[3367368]: STATUS=OK TS=2026-07-14T06:56:36Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-14T09:56:36+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-14T09:56:36+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-15T10:12:54+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-15T10:12:54+03:00 pve01 homelab-superseded-unit-ok[4062186]: STATUS=OK TS=2026-07-15T07:12:54Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-15T10:12:54+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-15T10:12:54+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-16T11:40:40+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-16T11:40:40+03:00 pve01 homelab-superseded-unit-ok[12176]: STATUS=OK TS=2026-07-16T08:40:40Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-16T11:40:40+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-16T11:40:40+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-17T10:00:14+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-17T10:00:14+03:00 pve01 homelab-superseded-unit-ok[390020]: STATUS=OK TS=2026-07-17T07:00:14Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-17T10:00:14+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-17T10:00:14+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-18T09:57:16+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-18T09:57:16+03:00 pve01 homelab-superseded-unit-ok[1201647]: STATUS=OK TS=2026-07-18T06:57:16Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-18T09:57:16+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-18T09:57:16+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-19T10:09:17+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-19T10:09:17+03:00 pve01 homelab-superseded-unit-ok[1945203]: STATUS=OK TS=2026-07-19T07:09:17Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-19T10:09:17+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-19T10:09:17+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-20T10:04:06+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-20T10:04:06+03:00 pve01 homelab-superseded-unit-ok[2734120]: STATUS=OK TS=2026-07-20T07:04:06Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-20T10:04:06+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-20T10:04:06+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-21T10:02:41+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-21T10:02:41+03:00 pve01 homelab-superseded-unit-ok[3525432]: STATUS=OK TS=2026-07-21T07:02:41Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-21T10:02:41+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-21T10:02:41+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-22T10:11:14+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-22T10:11:14+03:00 pve01 homelab-superseded-unit-ok[4192753]: STATUS=OK TS=2026-07-22T07:11:14Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-22T10:11:14+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-22T10:11:14+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-23T10:12:14+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-23T10:12:14+03:00 pve01 homelab-superseded-unit-ok[752982]: STATUS=OK TS=2026-07-23T07:12:14Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-23T10:12:14+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-23T10:12:14+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-24T10:06:04+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-24T10:06:04+03:00 pve01 homelab-superseded-unit-ok[3581148]: STATUS=OK TS=2026-07-24T07:06:04Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-24T10:06:04+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-24T10:06:04+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=TIMER_SHOW_RC=0\nDETAIL=TIMER_PROPERTY=Unit=paperless-restore-dry-run.service\nDETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Sat 2026-07-25 10:03:38 MSK\nDETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 10:06:04 MSK\nDETAIL=TIMER_PROPERTY=LoadState=loaded\nDETAIL=TIMER_PROPERTY=ActiveState=active\nDETAIL=TIMER_PROPERTY=SubState=waiting\nDETAIL=TIMER_ENABLED_RESULT=enabled|RC=0\nDETAIL=HEALTH_METADATA=BYTES:232|MTIME_NS:1783615375842257624\nDETAIL=HEALTH_MATCH_COUNT=1\nDETAIL=HEALTH_LINE=STATUS=OK TS=2026-07-09T16:42:53Z TYPE=paperless-restore BAD=0 APP=paperless JOB=homelab-pve01-app-paperless-20260709T071949Z RESTORE_VERIFIED=true RESTORE_CHECK_APP=paperless JSON=/var/lib/homelab-backup/apps/latest-paperless.json\nDETAIL=JOB_END=paperless_restore_dry_run\nDETAIL=JOB_BEGIN=vm160_remote_backup\nDETAIL=JOB_FIELD=id=vm160_remote_backup\nDETAIL=JOB_FIELD=name=vm160-remote-backup\nDETAIL=JOB_FIELD=category=virtual-machine\nDETAIL=JOB_FIELD=critical=True\nDETAIL=JOB_FIELD=status=FAIL\nDETAIL=JOB_FIELD=freshness=STALE\nDETAIL=JOB_FIELD=age_seconds=286010\nDETAIL=JOB_FIELD=max_age_seconds=172800\nDETAIL=JOB_FIELD=source_kind=health_file\nDETAIL=JOB_FIELD=source_status=FAIL\nDETAIL=JOB_FIELD=source_mtime=2026-07-20T23:39:14Z\nDETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/vm160_remote_backup/0-vm160-remote-backup.txt\nDETAIL=JOB_FIELD=service=homelab-vm160-remote-backup.service\nDETAIL=JOB_FIELD=service_result=success\nDETAIL=JOB_FIELD=service_exec_main_status=0\nDETAIL=JOB_FIELD=timer=homelab-vm160-remote-backup.timer\nDETAIL=JOB_FIELD=timer_enabled=disabled\nDETAIL=JOB_FIELD=timer_active=inactive\nDETAIL=SERVICE_SHOW_RC=0\nDETAIL=SERVICE_PROPERTY=Result=success\nDETAIL=SERVICE_PROPERTY=ExecMainCode=0\nDETAIL=SERVICE_PROPERTY=ExecMainStatus=0\nDETAIL=SERVICE_PROPERTY=LoadState=loaded\nDETAIL=SERVICE_PROPERTY=ActiveState=inactive\nDETAIL=SERVICE_PROPERTY=SubState=dead\nDETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/homelab-vm160-remote-backup.service\nDETAIL=SERVICE_PROPERTY=DropInPaths=\nDETAIL=SERVICE_JOURNAL_MATCH_COUNT=50\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:02:41+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:15:43+03:00 pve01 homelab-vm160-remote-backup[3491061]: VM160_TARGET_FINALIZE_STATUS=OK\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3530071]: VM160_TARGET_PROMOTE_STATUS=OK\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: VM160_REMOTE_BACKUP_STATUS=OK\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260717T220242Z\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: ARCHIVE=/mnt/staging/vzdump/vm160-pve02-20260717T220242Z/vzdump-qemu-160-20260717T220242Z.vma.zst\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: SIZE_BYTES=84993482488\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: SHA256=a9562d8b60d7fd5e7ec73bc43d252cfea80ed12c05b32cca0a73d7bb88532411\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 systemd[1]: Finished homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Consumed 2min 44.280s CPU time, 15.8M memory peak.\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:38:18+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:51:18+03:00 pve01 homelab-vm160-remote-backup[3609200]: VM160_TARGET_FINALIZE_STATUS=OK\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3648126]: VM160_TARGET_PROMOTE_STATUS=OK\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: VM160_REMOTE_BACKUP_STATUS=OK\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: ARCHIVE=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z/vzdump-qemu-160-20260717T223819Z.vma.zst\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: SIZE_BYTES=84995216465\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: SHA256=257e10821e62e3e2f9318b238a5302a6db601dd597bfa3e0d77aba07f3b85e72\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 systemd[1]: Finished homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Consumed 2min 44.842s CPU time, 16.1M memory peak.\nDETAIL=SERVICE_JOURNAL=2026-07-18T04:16:25+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nDETAIL=SERVICE_JOURNAL=2026-07-18T04:16:27+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=SERVICE_JOURNAL=2026-07-18T04:16:27+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nDETAIL=SERVICE_JOURNAL=2026-07-18T04:16:27+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nDETAIL=SERVICE_JOURNAL=2026-07-19T04:19:14+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nDETAIL=SERVICE_JOURNAL=2026-07-19T04:19:16+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=SERVICE_JOURNAL=2026-07-19T04:19:16+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nDETAIL=SERVICE_JOURNAL=2026-07-19T04:19:16+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nDETAIL=SERVICE_JOURNAL=2026-07-20T04:14:14+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nDETAIL=SERVICE_JOURNAL=2026-07-20T04:14:16+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=SERVICE_JOURNAL=2026-07-20T04:14:16+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nDETAIL=SERVICE_JOURNAL=2026-07-20T04:14:16+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nDETAIL=SERVICE_JOURNAL=2026-07-20T19:28:57+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nDETAIL=SERVICE_JOURNAL=2026-07-20T19:28:59+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=SERVICE_JOURNAL=2026-07-20T19:28:59+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nDETAIL=SERVICE_JOURNAL=2026-07-20T19:28:59+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:14:11+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:14:13+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:14:13+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:14:13+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:20:20+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:20:21+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:20:21+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:20:21+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:39:12+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:39:14+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:39:14+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:39:14+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nDETAIL=TIMER_SHOW_RC=0\nDETAIL=TIMER_PROPERTY=Unit=homelab-vm160-remote-backup.service\nDETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=\nDETAIL=TIMER_PROPERTY=LastTriggerUSec=\nDETAIL=TIMER_PROPERTY=LoadState=loaded\nDETAIL=TIMER_PROPERTY=ActiveState=inactive\nDETAIL=TIMER_PROPERTY=SubState=dead\nDETAIL=TIMER_ENABLED_RESULT=disabled|RC=1\nDETAIL=HEALTH_METADATA=BYTES:153|MTIME_NS:1784590754312087060\nDETAIL=HEALTH_MATCH_COUNT=3\nDETAIL=HEALTH_LINE=STATUS=FAIL\nDETAIL=HEALTH_LINE=SERVICE=homelab-vm160-remote-backup\nDETAIL=HEALTH_LINE=TIMESTAMP_UTC=20260720T233913Z\nDETAIL=JOB_END=vm160_remote_backup\nDETAIL=SECRET_VALUES_OUTPUT=NONE\nDETAIL=BACKUP_BAD_JOBS_ROOTCAUSE=PASS\nCHANGES_MADE=NO\n" +} diff --git a/runtime/history/HOMELAB-BACKUP-BAD-JOBS-ROOTCAUSE-431.txt b/runtime/history/HOMELAB-BACKUP-BAD-JOBS-ROOTCAUSE-431.txt new file mode 100644 index 0000000..d54e648 --- /dev/null +++ b/runtime/history/HOMELAB-BACKUP-BAD-JOBS-ROOTCAUSE-431.txt @@ -0,0 +1,557 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=HOMELAB-BACKUP-BAD-JOBS-ROOTCAUSE-431 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=cluster-audit +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 +COMMAND_SHA256=8635e368c9b3e168a7302affcd2c133d8f85547164b3f3352fc0cbad679edc98 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=false +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=e39cc1d76af62d7050936445c27d21728aaeb6ca00ab2430c6cb3963a7da3907 +SANITIZED_OUTPUT_SHA256=e39cc1d76af62d7050936445c27d21728aaeb6ca00ab2430c6cb3963a7da3907 +OUTPUT_BEGIN +DETAIL=MATRIX_GENERATED_AT=2026-07-24T07:06:04Z +DETAIL=TARGET_JOB_COUNT=6 +DETAIL=JOB_BEGIN=filebrowser_restore_validate +DETAIL=JOB_FIELD=id=filebrowser_restore_validate +DETAIL=JOB_FIELD=name=filebrowser-restore-validate +DETAIL=JOB_FIELD=category=restore +DETAIL=JOB_FIELD=critical=True +DETAIL=JOB_FIELD=status=STALE +DETAIL=JOB_FIELD=freshness=STALE +DETAIL=JOB_FIELD=age_seconds=1275827 +DETAIL=JOB_FIELD=max_age_seconds=864000 +DETAIL=JOB_FIELD=source_kind=health_file +DETAIL=JOB_FIELD=source_status=OK +DETAIL=JOB_FIELD=source_mtime=2026-07-09T12:42:17Z +DETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt +DETAIL=JOB_FIELD=service=filebrowser-restore-validate.service +DETAIL=JOB_FIELD=service_result=success +DETAIL=JOB_FIELD=service_exec_main_status=0 +DETAIL=JOB_FIELD=timer=filebrowser-restore-validate.timer +DETAIL=JOB_FIELD=timer_enabled=enabled +DETAIL=JOB_FIELD=timer_active=active +DETAIL=SERVICE_SHOW_RC=0 +DETAIL=SERVICE_PROPERTY=Result=success +DETAIL=SERVICE_PROPERTY=ExecMainCode=1 +DETAIL=SERVICE_PROPERTY=ExecMainStatus=0 +DETAIL=SERVICE_PROPERTY=LoadState=loaded +DETAIL=SERVICE_PROPERTY=ActiveState=inactive +DETAIL=SERVICE_PROPERTY=SubState=dead +DETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/filebrowser-restore-validate.service +DETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/filebrowser-restore-validate.service.d/10-superseded-noop.conf /etc/systemd/system/filebrowser-restore-validate.service.d/90-backup-matrix-refresh.conf +DETAIL=SERVICE_JOURNAL_MATCH_COUNT=56 +DETAIL=SERVICE_JOURNAL=2026-07-11T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-11T04:10:00+03:00 pve01 homelab-superseded-unit-ok[1686870]: STATUS=OK TS=2026-07-11T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-11T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-11T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=SERVICE_JOURNAL=2026-07-12T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-12T04:10:00+03:00 pve01 homelab-superseded-unit-ok[1699058]: STATUS=OK TS=2026-07-12T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-12T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-12T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=SERVICE_JOURNAL=2026-07-13T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-13T04:10:00+03:00 pve01 homelab-superseded-unit-ok[1681015]: STATUS=OK TS=2026-07-13T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-13T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-13T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=SERVICE_JOURNAL=2026-07-14T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-14T04:10:00+03:00 pve01 homelab-superseded-unit-ok[2132068]: STATUS=OK TS=2026-07-14T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-14T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-14T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=SERVICE_JOURNAL=2026-07-15T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-15T04:10:00+03:00 pve01 homelab-superseded-unit-ok[2783319]: STATUS=OK TS=2026-07-15T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-15T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-15T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=SERVICE_JOURNAL=2026-07-16T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-16T04:10:00+03:00 pve01 homelab-superseded-unit-ok[3557730]: STATUS=OK TS=2026-07-16T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-16T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-16T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=SERVICE_JOURNAL=2026-07-17T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-17T04:10:14+03:00 pve01 homelab-superseded-unit-ok[3346747]: STATUS=OK TS=2026-07-17T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-17T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-17T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=SERVICE_JOURNAL=2026-07-18T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-18T04:10:14+03:00 pve01 homelab-superseded-unit-ok[4129958]: STATUS=OK TS=2026-07-18T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-18T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-18T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=SERVICE_JOURNAL=2026-07-19T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-19T04:10:14+03:00 pve01 homelab-superseded-unit-ok[738797]: STATUS=OK TS=2026-07-19T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-19T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-19T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=SERVICE_JOURNAL=2026-07-20T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-20T04:10:14+03:00 pve01 homelab-superseded-unit-ok[1448291]: STATUS=OK TS=2026-07-20T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-20T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-20T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=SERVICE_JOURNAL=2026-07-21T04:10:07+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-21T04:10:07+03:00 pve01 homelab-superseded-unit-ok[2324924]: STATUS=OK TS=2026-07-21T01:10:07Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-21T04:10:07+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-21T04:10:07+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=SERVICE_JOURNAL=2026-07-22T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-22T04:10:14+03:00 pve01 homelab-superseded-unit-ok[3008751]: STATUS=OK TS=2026-07-22T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-22T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-22T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=SERVICE_JOURNAL=2026-07-23T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-23T04:10:14+03:00 pve01 homelab-superseded-unit-ok[3700126]: STATUS=OK TS=2026-07-23T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-23T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-23T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=SERVICE_JOURNAL=2026-07-24T04:10:04+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-24T04:10:04+03:00 pve01 homelab-superseded-unit-ok[2416005]: STATUS=OK TS=2026-07-24T01:10:04Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-24T04:10:04+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-24T04:10:04+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=TIMER_SHOW_RC=0 +DETAIL=TIMER_PROPERTY=Unit=filebrowser-restore-validate.service +DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Sat 2026-07-25 04:10:00 MSK +DETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 04:10:04 MSK +DETAIL=TIMER_PROPERTY=LoadState=loaded +DETAIL=TIMER_PROPERTY=ActiveState=active +DETAIL=TIMER_PROPERTY=SubState=waiting +DETAIL=TIMER_ENABLED_RESULT=enabled|RC=0 +DETAIL=HEALTH_METADATA=BYTES:209|MTIME_NS:1783600937792562046 +DETAIL=HEALTH_MATCH_COUNT=1 +DETAIL=HEALTH_LINE=STATUS=OK TS=2026-07-09T12:42:13Z TYPE=filebrowser-restore JOB=homelab-pve01-app-filebrowser-20260709T051713Z SOURCE=appbackupctl-restore-check RESTORECHECK=OK_BOTH_CLOUDS NOTE=service-readiness-restore-proof +DETAIL=JOB_END=filebrowser_restore_validate +DETAIL=JOB_BEGIN=homelab_backup_sla_health +DETAIL=JOB_FIELD=id=homelab_backup_sla_health +DETAIL=JOB_FIELD=name=homelab-backup-sla-health +DETAIL=JOB_FIELD=category=application +DETAIL=JOB_FIELD=critical=True +DETAIL=JOB_FIELD=status=FAIL +DETAIL=JOB_FIELD=freshness=FRESH +DETAIL=JOB_FIELD=age_seconds=358 +DETAIL=JOB_FIELD=max_age_seconds=172800 +DETAIL=JOB_FIELD=source_kind=health_file +DETAIL=JOB_FIELD=source_status=WARN +DETAIL=JOB_FIELD=source_mtime=2026-07-24T07:00:06Z +DETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt +DETAIL=JOB_FIELD=service=homelab-backup-sla-health.service +DETAIL=JOB_FIELD=service_result=success +DETAIL=JOB_FIELD=service_exec_main_status=0 +DETAIL=JOB_FIELD=timer=homelab-backup-sla-health.timer +DETAIL=JOB_FIELD=timer_enabled=enabled +DETAIL=JOB_FIELD=timer_active=active +DETAIL=SERVICE_SHOW_RC=0 +DETAIL=SERVICE_PROPERTY=Result=success +DETAIL=SERVICE_PROPERTY=ExecMainCode=1 +DETAIL=SERVICE_PROPERTY=ExecMainStatus=0 +DETAIL=SERVICE_PROPERTY=LoadState=loaded +DETAIL=SERVICE_PROPERTY=ActiveState=inactive +DETAIL=SERVICE_PROPERTY=SubState=dead +DETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/homelab-backup-sla-health.service +DETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/homelab-backup-sla-health.service.d/90-backup-matrix-refresh.conf +DETAIL=SERVICE_JOURNAL_MATCH_COUNT=76 +DETAIL=SERVICE_JOURNAL=2026-07-11T06:24:11+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-11T06:24:11+03:00 pve01 homelab-backup-sla-health[2238839]: STATUS=OK TS=2026-07-11T03:24:11Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-11T06:24:11+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-11T06:24:11+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-11T07:39:35+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-11T07:39:36+03:00 pve01 homelab-backup-sla-health[2476861]: STATUS=OK TS=2026-07-11T04:39:35Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-11T07:39:36+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-11T07:39:36+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-11T08:15:32+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-11T08:15:32+03:00 pve01 homelab-backup-sla-health[2616083]: STATUS=OK TS=2026-07-11T05:15:32Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-11T08:15:33+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-11T08:15:33+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-12T08:04:09+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-12T08:04:09+03:00 pve01 homelab-backup-sla-health[2414445]: STATUS=WARN TS=2026-07-12T05:04:09Z TYPE=backup-sla BAD=0 WARN=1 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-12T08:04:09+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-12T08:04:09+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-13T01:03:10+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-13T01:03:11+03:00 pve01 homelab-backup-sla-health[980591]: STATUS=OK TS=2026-07-12T22:03:10Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-13T01:03:11+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-13T01:03:11+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-13T01:08:07+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-13T01:08:08+03:00 pve01 homelab-backup-sla-health[994479]: STATUS=OK TS=2026-07-12T22:08:07Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-13T01:08:08+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-13T01:08:08+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-13T08:15:21+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-13T08:15:21+03:00 pve01 homelab-backup-sla-health[2521375]: STATUS=OK TS=2026-07-13T05:15:21Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-13T08:15:21+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-13T08:15:21+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-14T08:05:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-14T08:05:15+03:00 pve01 homelab-backup-sla-health[2983241]: STATUS=WARN TS=2026-07-14T05:05:14Z TYPE=backup-sla BAD=0 WARN=1 APPS=46 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-14T08:05:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-14T08:05:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-14T08:23:27+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-14T08:23:28+03:00 pve01 homelab-backup-sla-health[3042266]: STATUS=OK TS=2026-07-14T05:23:27Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-14T08:23:28+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-14T08:23:28+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-15T08:19:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-15T08:19:15+03:00 pve01 homelab-backup-sla-health[3670619]: STATUS=OK TS=2026-07-15T05:19:14Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-15T08:19:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-15T08:19:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-16T08:09:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-16T08:09:15+03:00 pve01 homelab-backup-sla-health[225757]: STATUS=WARN TS=2026-07-16T05:09:14Z TYPE=backup-sla BAD=0 WARN=1 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-16T08:09:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-16T08:09:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-17T08:07:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-17T08:07:15+03:00 pve01 homelab-backup-sla-health[4188049]: STATUS=WARN TS=2026-07-17T05:07:14Z TYPE=backup-sla BAD=0 WARN=4 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-17T08:07:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-17T08:07:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-18T08:05:29+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-18T08:05:29+03:00 pve01 homelab-backup-sla-health[806098]: STATUS=OK TS=2026-07-18T05:05:29Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-18T08:05:29+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-18T08:05:29+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-19T08:05:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-19T08:05:15+03:00 pve01 homelab-backup-sla-health[1545067]: STATUS=OK TS=2026-07-19T05:05:14Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-19T08:05:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-19T08:05:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-20T08:10:25+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-20T08:10:26+03:00 pve01 homelab-backup-sla-health[2331602]: STATUS=OK TS=2026-07-20T05:10:25Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-20T08:10:26+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-20T08:10:26+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-21T08:07:25+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-21T08:07:26+03:00 pve01 homelab-backup-sla-health[3122692]: STATUS=WARN TS=2026-07-21T05:07:25Z TYPE=backup-sla BAD=0 WARN=1 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-21T08:07:26+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-21T08:07:26+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-22T08:10:41+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-22T08:10:42+03:00 pve01 homelab-backup-sla-health[3806055]: STATUS=WARN TS=2026-07-22T05:10:41Z TYPE=backup-sla BAD=0 WARN=1 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-22T08:10:42+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-22T08:10:42+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-23T08:06:25+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-23T08:06:26+03:00 pve01 homelab-backup-sla-health[310318]: STATUS=WARN TS=2026-07-23T05:06:25Z TYPE=backup-sla BAD=0 WARN=2 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-23T08:06:26+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-23T08:06:26+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-24T08:20:04+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-24T08:20:04+03:00 pve01 homelab-backup-sla-health[3253899]: STATUS=WARN TS=2026-07-24T05:20:04Z TYPE=backup-sla BAD=0 WARN=4 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-24T08:20:04+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-24T08:20:04+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=TIMER_SHOW_RC=0 +DETAIL=TIMER_PROPERTY=Unit=homelab-backup-sla-health.service +DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Sat 2026-07-25 08:08:57 MSK +DETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 08:20:04 MSK +DETAIL=TIMER_PROPERTY=LoadState=loaded +DETAIL=TIMER_PROPERTY=ActiveState=active +DETAIL=TIMER_PROPERTY=SubState=waiting +DETAIL=TIMER_ENABLED_RESULT=enabled|RC=0 +DETAIL=HEALTH_METADATA=BYTES:154|MTIME_NS:1784876406380685648 +DETAIL=HEALTH_MATCH_COUNT=1 +DETAIL=HEALTH_LINE=STATUS=WARN TS=2026-07-24T07:00:05Z TYPE=backup-sla BAD=0 WARN=4 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=JOB_END=homelab_backup_sla_health +DETAIL=JOB_BEGIN=homelab_mail_cloud_restore_drill +DETAIL=JOB_FIELD=id=homelab_mail_cloud_restore_drill +DETAIL=JOB_FIELD=name=homelab-mail-cloud-restore-drill +DETAIL=JOB_FIELD=category=restore +DETAIL=JOB_FIELD=critical=True +DETAIL=JOB_FIELD=status=STALE +DETAIL=JOB_FIELD=freshness=STALE +DETAIL=JOB_FIELD=age_seconds=1247672 +DETAIL=JOB_FIELD=max_age_seconds=864000 +DETAIL=JOB_FIELD=source_kind=health_file +DETAIL=JOB_FIELD=source_status=OK +DETAIL=JOB_FIELD=source_mtime=2026-07-09T20:31:32Z +DETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/homelab_mail_cloud_restore_drill/0-mail-cloud-restore-drill.txt +DETAIL=JOB_FIELD=service=homelab-mail-cloud-restore-drill.service +DETAIL=JOB_FIELD=service_result=success +DETAIL=JOB_FIELD=service_exec_main_status=0 +DETAIL=JOB_FIELD=timer=homelab-mail-cloud-restore-drill.timer +DETAIL=JOB_FIELD=timer_enabled=enabled +DETAIL=JOB_FIELD=timer_active=active +DETAIL=SERVICE_SHOW_RC=0 +DETAIL=SERVICE_PROPERTY=Result=success +DETAIL=SERVICE_PROPERTY=ExecMainCode=0 +DETAIL=SERVICE_PROPERTY=ExecMainStatus=0 +DETAIL=SERVICE_PROPERTY=LoadState=loaded +DETAIL=SERVICE_PROPERTY=ActiveState=inactive +DETAIL=SERVICE_PROPERTY=SubState=dead +DETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/homelab-mail-cloud-restore-drill.service +DETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/homelab-mail-cloud-restore-drill.service.d/10-superseded-noop.conf /etc/systemd/system/homelab-mail-cloud-restore-drill.service.d/90-backup-matrix-refresh.conf +DETAIL=SERVICE_JOURNAL_MATCH_COUNT=8 +DETAIL=SERVICE_JOURNAL=2026-07-13T07:38:36+03:00 pve01 systemd[1]: Starting homelab-mail-cloud-restore-drill.service - Homelab Mail Cloud restore drill... +DETAIL=SERVICE_JOURNAL=2026-07-13T07:38:36+03:00 pve01 homelab-superseded-unit-ok[2401599]: STATUS=OK TS=2026-07-13T04:38:36Z TYPE=superseded-unit UNIT=homelab-mail-cloud-restore-drill.service REASON=superseded_by_safety_net_restore_check ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-13T07:38:36+03:00 pve01 systemd[1]: homelab-mail-cloud-restore-drill.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-13T07:38:36+03:00 pve01 systemd[1]: Finished homelab-mail-cloud-restore-drill.service - Homelab Mail Cloud restore drill. +DETAIL=SERVICE_JOURNAL=2026-07-20T07:25:14+03:00 pve01 systemd[1]: Starting homelab-mail-cloud-restore-drill.service - Homelab Mail Cloud restore drill... +DETAIL=SERVICE_JOURNAL=2026-07-20T07:25:14+03:00 pve01 homelab-superseded-unit-ok[2171079]: STATUS=OK TS=2026-07-20T04:25:14Z TYPE=superseded-unit UNIT=homelab-mail-cloud-restore-drill.service REASON=superseded_by_safety_net_restore_check ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-20T07:25:14+03:00 pve01 systemd[1]: homelab-mail-cloud-restore-drill.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-20T07:25:14+03:00 pve01 systemd[1]: Finished homelab-mail-cloud-restore-drill.service - Homelab Mail Cloud restore drill. +DETAIL=TIMER_SHOW_RC=0 +DETAIL=TIMER_PROPERTY=Unit=homelab-mail-cloud-restore-drill.service +DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Mon 2026-07-27 07:34:52 MSK +DETAIL=TIMER_PROPERTY=LastTriggerUSec=Mon 2026-07-20 07:25:14 MSK +DETAIL=TIMER_PROPERTY=LoadState=loaded +DETAIL=TIMER_PROPERTY=ActiveState=active +DETAIL=TIMER_PROPERTY=SubState=waiting +DETAIL=TIMER_ENABLED_RESULT=enabled|RC=0 +DETAIL=HEALTH_METADATA=BYTES:353|MTIME_NS:1783629092051018609 +DETAIL=HEALTH_MATCH_COUNT=1 +DETAIL=HEALTH_LINE=STATUS=OK TS=2026-07-09T20:31:32Z TYPE=legacy-mail-cloud-restore-drill-superseded MODE=SUPERSEDED_BY_CURRENT_PROOF BAD=0 WARN=0 PROOF=safety_net_control_plane_restore_check_OK ACTION=stale_health_refresh_only REASON=legacy_drill_superseded_but_health_file_mtime_must_stay_fresh REPORT=/var/lib/homelab-failed-unit-legacy-cleanup/local-cleanup-latest.md +DETAIL=JOB_END=homelab_mail_cloud_restore_drill +DETAIL=JOB_BEGIN=netbird_vps_backup +DETAIL=JOB_FIELD=id=netbird_vps_backup +DETAIL=JOB_FIELD=name=netbird-vps-backup +DETAIL=JOB_FIELD=category=application +DETAIL=JOB_FIELD=critical=True +DETAIL=JOB_FIELD=status=FAIL +DETAIL=JOB_FIELD=freshness=STALE +DETAIL=JOB_FIELD=age_seconds=2007299 +DETAIL=JOB_FIELD=max_age_seconds=864000 +DETAIL=JOB_FIELD=source_kind=health_file +DETAIL=JOB_FIELD=source_status=OK +DETAIL=JOB_FIELD=source_mtime=2026-07-01T01:31:05Z +DETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt +DETAIL=JOB_FIELD=service=netbird-vps-backup.service +DETAIL=JOB_FIELD=service_result=success +DETAIL=JOB_FIELD=service_exec_main_status=0 +DETAIL=JOB_FIELD=timer=netbird-vps-backup.timer +DETAIL=JOB_FIELD=timer_enabled=disabled +DETAIL=JOB_FIELD=timer_active=inactive +DETAIL=SERVICE_SHOW_RC=0 +DETAIL=SERVICE_PROPERTY=Result=success +DETAIL=SERVICE_PROPERTY=ExecMainCode=0 +DETAIL=SERVICE_PROPERTY=ExecMainStatus=0 +DETAIL=SERVICE_PROPERTY=LoadState=loaded +DETAIL=SERVICE_PROPERTY=ActiveState=inactive +DETAIL=SERVICE_PROPERTY=SubState=dead +DETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/netbird-vps-backup.service +DETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/netbird-vps-backup.service.d/10-superseded-noop.conf /etc/systemd/system/netbird-vps-backup.service.d/90-backup-matrix-refresh.conf +DETAIL=SERVICE_JOURNAL_MATCH_COUNT=28 +DETAIL=SERVICE_JOURNAL=2026-07-11T04:30:35+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging... +DETAIL=SERVICE_JOURNAL=2026-07-11T04:30:35+03:00 pve01 homelab-superseded-unit-ok[1772638]: STATUS=OK TS=2026-07-11T01:30:35Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-11T04:30:35+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-11T04:30:35+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging. +DETAIL=SERVICE_JOURNAL=2026-07-12T04:39:35+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging... +DETAIL=SERVICE_JOURNAL=2026-07-12T04:39:35+03:00 pve01 homelab-superseded-unit-ok[1789866]: STATUS=OK TS=2026-07-12T01:39:35Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-12T04:39:35+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-12T04:39:35+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging. +DETAIL=SERVICE_JOURNAL=2026-07-13T04:28:00+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging... +DETAIL=SERVICE_JOURNAL=2026-07-13T04:28:00+03:00 pve01 homelab-superseded-unit-ok[1739137]: STATUS=OK TS=2026-07-13T01:28:00Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-13T04:28:00+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-13T04:28:00+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging. +DETAIL=SERVICE_JOURNAL=2026-07-14T04:28:39+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging... +DETAIL=SERVICE_JOURNAL=2026-07-14T04:28:39+03:00 pve01 homelab-superseded-unit-ok[2199841]: STATUS=OK TS=2026-07-14T01:28:39Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-14T04:28:39+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-14T04:28:39+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging. +DETAIL=SERVICE_JOURNAL=2026-07-15T04:39:00+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging... +DETAIL=SERVICE_JOURNAL=2026-07-15T04:39:00+03:00 pve01 homelab-superseded-unit-ok[2872498]: STATUS=OK TS=2026-07-15T01:39:00Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-15T04:39:00+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-15T04:39:00+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging. +DETAIL=SERVICE_JOURNAL=2026-07-16T04:25:14+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging... +DETAIL=SERVICE_JOURNAL=2026-07-16T04:25:14+03:00 pve01 homelab-superseded-unit-ok[3611661]: STATUS=OK TS=2026-07-16T01:25:14Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-16T04:25:14+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-16T04:25:14+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging. +DETAIL=SERVICE_JOURNAL=2026-07-17T04:31:14+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging... +DETAIL=SERVICE_JOURNAL=2026-07-17T04:31:14+03:00 pve01 homelab-superseded-unit-ok[3412653]: STATUS=OK TS=2026-07-17T01:31:14Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-17T04:31:14+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-17T04:31:14+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging. +DETAIL=TIMER_SHOW_RC=0 +DETAIL=TIMER_PROPERTY=Unit=netbird-vps-backup.service +DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime= +DETAIL=TIMER_PROPERTY=LastTriggerUSec= +DETAIL=TIMER_PROPERTY=LoadState=loaded +DETAIL=TIMER_PROPERTY=ActiveState=inactive +DETAIL=TIMER_PROPERTY=SubState=dead +DETAIL=TIMER_ENABLED_RESULT=disabled|RC=1 +DETAIL=HEALTH_METADATA=BYTES:312|MTIME_NS:1782869465433646905 +DETAIL=HEALTH_MATCH_COUNT=7 +DETAIL=HEALTH_LINE=STATUS=OK +DETAIL=HEALTH_LINE=SERVICE=netbird-vps-backup +DETAIL=HEALTH_LINE=TIMESTAMP_UTC=20260701T013054Z +DETAIL=HEALTH_LINE=SNAPSHOT=/mnt/staging/netbird-vps-backups/snapshots/20260701T013054Z +DETAIL=HEALTH_LINE=LATEST_SYMLINK=/mnt/staging/netbird-vps-backups/snapshots/latest +DETAIL=HEALTH_LINE=SIZE=32M +DETAIL=HEALTH_LINE=SHA256=0226dd820cb9ede0ea9e8ea4af6cef656ffa54cfb819048c222a88f4c14d8db9 +DETAIL=JOB_END=netbird_vps_backup +DETAIL=JOB_BEGIN=paperless_restore_dry_run +DETAIL=JOB_FIELD=id=paperless_restore_dry_run +DETAIL=JOB_FIELD=name=paperless-restore-dry-run +DETAIL=JOB_FIELD=category=restore +DETAIL=JOB_FIELD=critical=True +DETAIL=JOB_FIELD=status=STALE +DETAIL=JOB_FIELD=freshness=STALE +DETAIL=JOB_FIELD=age_seconds=1261389 +DETAIL=JOB_FIELD=max_age_seconds=864000 +DETAIL=JOB_FIELD=source_kind=health_file +DETAIL=JOB_FIELD=source_status=OK +DETAIL=JOB_FIELD=source_mtime=2026-07-09T16:42:55Z +DETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/paperless_restore_dry_run/0-paperless-restore.txt +DETAIL=JOB_FIELD=service=paperless-restore-dry-run.service +DETAIL=JOB_FIELD=service_result=success +DETAIL=JOB_FIELD=service_exec_main_status=0 +DETAIL=JOB_FIELD=timer=paperless-restore-dry-run.timer +DETAIL=JOB_FIELD=timer_enabled=enabled +DETAIL=JOB_FIELD=timer_active=active +DETAIL=SERVICE_SHOW_RC=0 +DETAIL=SERVICE_PROPERTY=Result=success +DETAIL=SERVICE_PROPERTY=ExecMainCode=1 +DETAIL=SERVICE_PROPERTY=ExecMainStatus=0 +DETAIL=SERVICE_PROPERTY=LoadState=loaded +DETAIL=SERVICE_PROPERTY=ActiveState=inactive +DETAIL=SERVICE_PROPERTY=SubState=dead +DETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/paperless-restore-dry-run.service +DETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/paperless-restore-dry-run.service.d/10-superseded-noop.conf /etc/systemd/system/paperless-restore-dry-run.service.d/90-backup-matrix-refresh.conf +DETAIL=SERVICE_JOURNAL_MATCH_COUNT=56 +DETAIL=SERVICE_JOURNAL=2026-07-11T10:08:13+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-11T10:08:13+03:00 pve01 homelab-superseded-unit-ok[2988079]: STATUS=OK TS=2026-07-11T07:08:13Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-11T10:08:13+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-11T10:08:13+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=SERVICE_JOURNAL=2026-07-12T10:04:59+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-12T10:04:59+03:00 pve01 homelab-superseded-unit-ok[2757392]: STATUS=OK TS=2026-07-12T07:04:59Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-12T10:04:59+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-12T10:04:59+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=SERVICE_JOURNAL=2026-07-13T10:07:25+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-13T10:07:25+03:00 pve01 homelab-superseded-unit-ok[2886298]: STATUS=OK TS=2026-07-13T07:07:25Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-13T10:07:25+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-13T10:07:25+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=SERVICE_JOURNAL=2026-07-14T09:56:36+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-14T09:56:36+03:00 pve01 homelab-superseded-unit-ok[3367368]: STATUS=OK TS=2026-07-14T06:56:36Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-14T09:56:36+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-14T09:56:36+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=SERVICE_JOURNAL=2026-07-15T10:12:54+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-15T10:12:54+03:00 pve01 homelab-superseded-unit-ok[4062186]: STATUS=OK TS=2026-07-15T07:12:54Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-15T10:12:54+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-15T10:12:54+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=SERVICE_JOURNAL=2026-07-16T11:40:40+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-16T11:40:40+03:00 pve01 homelab-superseded-unit-ok[12176]: STATUS=OK TS=2026-07-16T08:40:40Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-16T11:40:40+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-16T11:40:40+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=SERVICE_JOURNAL=2026-07-17T10:00:14+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-17T10:00:14+03:00 pve01 homelab-superseded-unit-ok[390020]: STATUS=OK TS=2026-07-17T07:00:14Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-17T10:00:14+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-17T10:00:14+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=SERVICE_JOURNAL=2026-07-18T09:57:16+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-18T09:57:16+03:00 pve01 homelab-superseded-unit-ok[1201647]: STATUS=OK TS=2026-07-18T06:57:16Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-18T09:57:16+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-18T09:57:16+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=SERVICE_JOURNAL=2026-07-19T10:09:17+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-19T10:09:17+03:00 pve01 homelab-superseded-unit-ok[1945203]: STATUS=OK TS=2026-07-19T07:09:17Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-19T10:09:17+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-19T10:09:17+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=SERVICE_JOURNAL=2026-07-20T10:04:06+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-20T10:04:06+03:00 pve01 homelab-superseded-unit-ok[2734120]: STATUS=OK TS=2026-07-20T07:04:06Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-20T10:04:06+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-20T10:04:06+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=SERVICE_JOURNAL=2026-07-21T10:02:41+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-21T10:02:41+03:00 pve01 homelab-superseded-unit-ok[3525432]: STATUS=OK TS=2026-07-21T07:02:41Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-21T10:02:41+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-21T10:02:41+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=SERVICE_JOURNAL=2026-07-22T10:11:14+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-22T10:11:14+03:00 pve01 homelab-superseded-unit-ok[4192753]: STATUS=OK TS=2026-07-22T07:11:14Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-22T10:11:14+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-22T10:11:14+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=SERVICE_JOURNAL=2026-07-23T10:12:14+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-23T10:12:14+03:00 pve01 homelab-superseded-unit-ok[752982]: STATUS=OK TS=2026-07-23T07:12:14Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-23T10:12:14+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-23T10:12:14+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=SERVICE_JOURNAL=2026-07-24T10:06:04+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-24T10:06:04+03:00 pve01 homelab-superseded-unit-ok[3581148]: STATUS=OK TS=2026-07-24T07:06:04Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-24T10:06:04+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-24T10:06:04+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=TIMER_SHOW_RC=0 +DETAIL=TIMER_PROPERTY=Unit=paperless-restore-dry-run.service +DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Sat 2026-07-25 10:03:38 MSK +DETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 10:06:04 MSK +DETAIL=TIMER_PROPERTY=LoadState=loaded +DETAIL=TIMER_PROPERTY=ActiveState=active +DETAIL=TIMER_PROPERTY=SubState=waiting +DETAIL=TIMER_ENABLED_RESULT=enabled|RC=0 +DETAIL=HEALTH_METADATA=BYTES:232|MTIME_NS:1783615375842257624 +DETAIL=HEALTH_MATCH_COUNT=1 +DETAIL=HEALTH_LINE=STATUS=OK TS=2026-07-09T16:42:53Z TYPE=paperless-restore BAD=0 APP=paperless JOB=homelab-pve01-app-paperless-20260709T071949Z RESTORE_VERIFIED=true RESTORE_CHECK_APP=paperless JSON=/var/lib/homelab-backup/apps/latest-paperless.json +DETAIL=JOB_END=paperless_restore_dry_run +DETAIL=JOB_BEGIN=vm160_remote_backup +DETAIL=JOB_FIELD=id=vm160_remote_backup +DETAIL=JOB_FIELD=name=vm160-remote-backup +DETAIL=JOB_FIELD=category=virtual-machine +DETAIL=JOB_FIELD=critical=True +DETAIL=JOB_FIELD=status=FAIL +DETAIL=JOB_FIELD=freshness=STALE +DETAIL=JOB_FIELD=age_seconds=286010 +DETAIL=JOB_FIELD=max_age_seconds=172800 +DETAIL=JOB_FIELD=source_kind=health_file +DETAIL=JOB_FIELD=source_status=FAIL +DETAIL=JOB_FIELD=source_mtime=2026-07-20T23:39:14Z +DETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/vm160_remote_backup/0-vm160-remote-backup.txt +DETAIL=JOB_FIELD=service=homelab-vm160-remote-backup.service +DETAIL=JOB_FIELD=service_result=success +DETAIL=JOB_FIELD=service_exec_main_status=0 +DETAIL=JOB_FIELD=timer=homelab-vm160-remote-backup.timer +DETAIL=JOB_FIELD=timer_enabled=disabled +DETAIL=JOB_FIELD=timer_active=inactive +DETAIL=SERVICE_SHOW_RC=0 +DETAIL=SERVICE_PROPERTY=Result=success +DETAIL=SERVICE_PROPERTY=ExecMainCode=0 +DETAIL=SERVICE_PROPERTY=ExecMainStatus=0 +DETAIL=SERVICE_PROPERTY=LoadState=loaded +DETAIL=SERVICE_PROPERTY=ActiveState=inactive +DETAIL=SERVICE_PROPERTY=SubState=dead +DETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/homelab-vm160-remote-backup.service +DETAIL=SERVICE_PROPERTY=DropInPaths= +DETAIL=SERVICE_JOURNAL_MATCH_COUNT=50 +DETAIL=SERVICE_JOURNAL=2026-07-18T01:02:41+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... +DETAIL=SERVICE_JOURNAL=2026-07-18T01:15:43+03:00 pve01 homelab-vm160-remote-backup[3491061]: VM160_TARGET_FINALIZE_STATUS=OK +DETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3530071]: VM160_TARGET_PROMOTE_STATUS=OK +DETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: VM160_REMOTE_BACKUP_STATUS=OK +DETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260717T220242Z +DETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: ARCHIVE=/mnt/staging/vzdump/vm160-pve02-20260717T220242Z/vzdump-qemu-160-20260717T220242Z.vma.zst +DETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: SIZE_BYTES=84993482488 +DETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: SHA256=a9562d8b60d7fd5e7ec73bc43d252cfea80ed12c05b32cca0a73d7bb88532411 +DETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 systemd[1]: Finished homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. +DETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Consumed 2min 44.280s CPU time, 15.8M memory peak. +DETAIL=SERVICE_JOURNAL=2026-07-18T01:38:18+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... +DETAIL=SERVICE_JOURNAL=2026-07-18T01:51:18+03:00 pve01 homelab-vm160-remote-backup[3609200]: VM160_TARGET_FINALIZE_STATUS=OK +DETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3648126]: VM160_TARGET_PROMOTE_STATUS=OK +DETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: VM160_REMOTE_BACKUP_STATUS=OK +DETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z +DETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: ARCHIVE=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z/vzdump-qemu-160-20260717T223819Z.vma.zst +DETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: SIZE_BYTES=84995216465 +DETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: SHA256=257e10821e62e3e2f9318b238a5302a6db601dd597bfa3e0d77aba07f3b85e72 +DETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 systemd[1]: Finished homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. +DETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Consumed 2min 44.842s CPU time, 16.1M memory peak. +DETAIL=SERVICE_JOURNAL=2026-07-18T04:16:25+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... +DETAIL=SERVICE_JOURNAL=2026-07-18T04:16:27+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=SERVICE_JOURNAL=2026-07-18T04:16:27+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. +DETAIL=SERVICE_JOURNAL=2026-07-18T04:16:27+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. +DETAIL=SERVICE_JOURNAL=2026-07-19T04:19:14+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... +DETAIL=SERVICE_JOURNAL=2026-07-19T04:19:16+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=SERVICE_JOURNAL=2026-07-19T04:19:16+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. +DETAIL=SERVICE_JOURNAL=2026-07-19T04:19:16+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. +DETAIL=SERVICE_JOURNAL=2026-07-20T04:14:14+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... +DETAIL=SERVICE_JOURNAL=2026-07-20T04:14:16+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=SERVICE_JOURNAL=2026-07-20T04:14:16+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. +DETAIL=SERVICE_JOURNAL=2026-07-20T04:14:16+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. +DETAIL=SERVICE_JOURNAL=2026-07-20T19:28:57+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... +DETAIL=SERVICE_JOURNAL=2026-07-20T19:28:59+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=SERVICE_JOURNAL=2026-07-20T19:28:59+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. +DETAIL=SERVICE_JOURNAL=2026-07-20T19:28:59+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. +DETAIL=SERVICE_JOURNAL=2026-07-21T02:14:11+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... +DETAIL=SERVICE_JOURNAL=2026-07-21T02:14:13+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=SERVICE_JOURNAL=2026-07-21T02:14:13+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. +DETAIL=SERVICE_JOURNAL=2026-07-21T02:14:13+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. +DETAIL=SERVICE_JOURNAL=2026-07-21T02:20:20+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... +DETAIL=SERVICE_JOURNAL=2026-07-21T02:20:21+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=SERVICE_JOURNAL=2026-07-21T02:20:21+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. +DETAIL=SERVICE_JOURNAL=2026-07-21T02:20:21+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. +DETAIL=SERVICE_JOURNAL=2026-07-21T02:39:12+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... +DETAIL=SERVICE_JOURNAL=2026-07-21T02:39:14+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=SERVICE_JOURNAL=2026-07-21T02:39:14+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. +DETAIL=SERVICE_JOURNAL=2026-07-21T02:39:14+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. +DETAIL=TIMER_SHOW_RC=0 +DETAIL=TIMER_PROPERTY=Unit=homelab-vm160-remote-backup.service +DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime= +DETAIL=TIMER_PROPERTY=LastTriggerUSec= +DETAIL=TIMER_PROPERTY=LoadState=loaded +DETAIL=TIMER_PROPERTY=ActiveState=inactive +DETAIL=TIMER_PROPERTY=SubState=dead +DETAIL=TIMER_ENABLED_RESULT=disabled|RC=1 +DETAIL=HEALTH_METADATA=BYTES:153|MTIME_NS:1784590754312087060 +DETAIL=HEALTH_MATCH_COUNT=3 +DETAIL=HEALTH_LINE=STATUS=FAIL +DETAIL=HEALTH_LINE=SERVICE=homelab-vm160-remote-backup +DETAIL=HEALTH_LINE=TIMESTAMP_UTC=20260720T233913Z +DETAIL=JOB_END=vm160_remote_backup +DETAIL=SECRET_VALUES_OUTPUT=NONE +DETAIL=BACKUP_BAD_JOBS_ROOTCAUSE=PASS +CHANGES_MADE=NO + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index f0a781f..31daf1c 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "HOMELAB-BACKUP-MATRIX-BAD-JOBS-DISCOVERY-430", + "command_id": "HOMELAB-BACKUP-BAD-JOBS-ROOTCAUSE-431", "status": "OK", "rc": 0, "host": "pve01", "mode": "read-only", "component": "cluster-audit", - "started_at_utc": "2026-07-24T09:53:00Z", - "finished_at_utc": "2026-07-24T09:53:00Z", + "started_at_utc": "2026-07-24T09:55:30Z", + "finished_at_utc": "2026-07-24T09:55:30Z", "reference_register_checked": true, "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", "error_register_checked": true, "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", - "command_sha256": "8a21cde408577f47c4d88ff6dcf0a4449ef7a77e0c064419260504b794cdc2e6", + "command_sha256": "8635e368c9b3e168a7302affcd2c133d8f85547164b3f3352fc0cbad679edc98", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -22,9 +22,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "de359611c59c55b28e546d3099a9a49187e6e8523948be5d2bf6ea5b633cb2ac", - "sanitized_output_sha256": "de359611c59c55b28e546d3099a9a49187e6e8523948be5d2bf6ea5b633cb2ac", + "raw_evidence_sha256": "e39cc1d76af62d7050936445c27d21728aaeb6ca00ab2430c6cb3963a7da3907", + "sanitized_output_sha256": "e39cc1d76af62d7050936445c27d21728aaeb6ca00ab2430c6cb3963a7da3907", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "DETAIL=STATE_PATH=/var/lib/homelab-backup-matrix/state.json\nDETAIL=STATE_BYTES=36840\nDETAIL=STATE_SHA256=08d7536cb2c11d036726c4bc0c59d573f6c02e8b3e136e9872128a380ab88807\nDETAIL=ROOT_KEYS=authority,compatibility_map,generated_at,jobs,matrix_mode,matrix_sha256,summary,version\nDETAIL=SUMMARY=$.jobs[0].status=OK\nDETAIL=SUMMARY=$.jobs[10].status=OK\nDETAIL=SUMMARY=$.jobs[11].status=OK\nDETAIL=SUMMARY=$.jobs[12].status=OK\nDETAIL=SUMMARY=$.jobs[13].status=FAIL\nDETAIL=SUMMARY=$.jobs[14].status=OK\nDETAIL=SUMMARY=$.jobs[15].status=OK\nDETAIL=SUMMARY=$.jobs[16].status=OK\nDETAIL=SUMMARY=$.jobs[17].status=DISABLED\nDETAIL=SUMMARY=$.jobs[18].status=OK\nDETAIL=SUMMARY=$.jobs[19].status=OK\nDETAIL=SUMMARY=$.jobs[1].status=OK\nDETAIL=SUMMARY=$.jobs[20].status=OK\nDETAIL=SUMMARY=$.jobs[21].status=OK\nDETAIL=SUMMARY=$.jobs[22].status=STALE\nDETAIL=SUMMARY=$.jobs[23].status=DISABLED\nDETAIL=SUMMARY=$.jobs[24].status=OK\nDETAIL=SUMMARY=$.jobs[25].status=DISABLED\nDETAIL=SUMMARY=$.jobs[26].status=OK\nDETAIL=SUMMARY=$.jobs[27].status=OK\nDETAIL=SUMMARY=$.jobs[28].status=OK\nDETAIL=SUMMARY=$.jobs[29].status=OK\nDETAIL=SUMMARY=$.jobs[2].status=STALE\nDETAIL=SUMMARY=$.jobs[30].status=OK\nDETAIL=SUMMARY=$.jobs[31].status=OK\nDETAIL=SUMMARY=$.jobs[32].status=OK\nDETAIL=SUMMARY=$.jobs[33].status=OK\nDETAIL=SUMMARY=$.jobs[34].status=OK\nDETAIL=SUMMARY=$.jobs[35].status=FAIL\nDETAIL=SUMMARY=$.jobs[36].status=DISABLED\nDETAIL=SUMMARY=$.jobs[37].status=STALE\nDETAIL=SUMMARY=$.jobs[38].status=OK\nDETAIL=SUMMARY=$.jobs[39].status=OK\nDETAIL=SUMMARY=$.jobs[3].status=OK\nDETAIL=SUMMARY=$.jobs[40].status=FAIL\nDETAIL=SUMMARY=$.jobs[4].status=OK\nDETAIL=SUMMARY=$.jobs[5].status=OK\nDETAIL=SUMMARY=$.jobs[6].status=OK\nDETAIL=SUMMARY=$.jobs[7].status=OK\nDETAIL=SUMMARY=$.jobs[8].status=DISABLED\nDETAIL=SUMMARY=$.jobs[9].status=OK\nDETAIL=SUMMARY=$.summary.jobs_bad=6\nDETAIL=SUMMARY=$.summary.jobs_disabled=5\nDETAIL=SUMMARY=$.summary.jobs_ok=30\nDETAIL=SUMMARY=$.summary.jobs_total=41\nDETAIL=SUMMARY=$.summary.jobs_unmapped=0\nDETAIL=SUMMARY=$.summary.status=FAIL\nDETAIL=BAD_RECORD_COUNT=7\nDETAIL=BAD_RECORD=PATH:$.jobs[2]|IDENTITY:filebrowser_restore_validate|BAD:status=stale|DISABLED:none|FIELDS:id=filebrowser_restore_validate;name=filebrowser-restore-validate;service=filebrowser-restore-validate.service;status=STALE\nDETAIL=BAD_RECORD=PATH:$.jobs[13]|IDENTITY:homelab_backup_sla_health|BAD:status=fail|DISABLED:none|FIELDS:id=homelab_backup_sla_health;name=homelab-backup-sla-health;service=homelab-backup-sla-health.service;status=FAIL\nDETAIL=BAD_RECORD=PATH:$.jobs[22]|IDENTITY:homelab_mail_cloud_restore_drill|BAD:status=stale|DISABLED:none|FIELDS:id=homelab_mail_cloud_restore_drill;name=homelab-mail-cloud-restore-drill;service=homelab-mail-cloud-restore-drill.service;status=STALE\nDETAIL=BAD_RECORD=PATH:$.jobs[35]|IDENTITY:netbird_vps_backup|BAD:status=fail|DISABLED:none|FIELDS:id=netbird_vps_backup;name=netbird-vps-backup;service=netbird-vps-backup.service;status=FAIL\nDETAIL=BAD_RECORD=PATH:$.jobs[37]|IDENTITY:paperless_restore_dry_run|BAD:status=stale|DISABLED:none|FIELDS:id=paperless_restore_dry_run;name=paperless-restore-dry-run;service=paperless-restore-dry-run.service;status=STALE\nDETAIL=BAD_RECORD=PATH:$.jobs[40]|IDENTITY:vm160_remote_backup|BAD:status=fail|DISABLED:none|FIELDS:id=vm160_remote_backup;name=vm160-remote-backup;service=homelab-vm160-remote-backup.service;status=FAIL\nDETAIL=BAD_RECORD=PATH:$.summary|IDENTITY:|BAD:status=fail|DISABLED:none|FIELDS:status=FAIL\nDETAIL=DISABLED_ONLY_COUNT=5\nDETAIL=DISABLED_RECORD=PATH:$.jobs[8]|IDENTITY:homelab_authentik_backup|SIGNALS:status=disabled\nDETAIL=DISABLED_RECORD=PATH:$.jobs[17]|IDENTITY:homelab_gitea_backup|SIGNALS:status=disabled\nDETAIL=DISABLED_RECORD=PATH:$.jobs[23]|IDENTITY:homelab_npmplus_kuma_config_backup|SIGNALS:status=disabled\nDETAIL=DISABLED_RECORD=PATH:$.jobs[25]|IDENTITY:homelab_vaultwarden_backup|SIGNALS:status=disabled\nDETAIL=DISABLED_RECORD=PATH:$.jobs[36]|IDENTITY:paperless_backup|SIGNALS:status=disabled\nDETAIL=LIST_SCHEMA=PATH:$.jobs|ITEMS:41|DICT_ITEMS:41|KEYS:age_seconds,category,critical,evidence_level,freshness,generated_health_output,id,max_age_seconds,name,service,service_exec_main_status,service_result,source_health,source_kind,source_mtime,source_status,status,timer,timer_active,timer_enabled\nDETAIL=SECRET_VALUES_OUTPUT=NONE\nDETAIL=BACKUP_MATRIX_BAD_JOB_DISCOVERY=PASS\nCHANGES_MADE=NO\n" + "output": "DETAIL=MATRIX_GENERATED_AT=2026-07-24T07:06:04Z\nDETAIL=TARGET_JOB_COUNT=6\nDETAIL=JOB_BEGIN=filebrowser_restore_validate\nDETAIL=JOB_FIELD=id=filebrowser_restore_validate\nDETAIL=JOB_FIELD=name=filebrowser-restore-validate\nDETAIL=JOB_FIELD=category=restore\nDETAIL=JOB_FIELD=critical=True\nDETAIL=JOB_FIELD=status=STALE\nDETAIL=JOB_FIELD=freshness=STALE\nDETAIL=JOB_FIELD=age_seconds=1275827\nDETAIL=JOB_FIELD=max_age_seconds=864000\nDETAIL=JOB_FIELD=source_kind=health_file\nDETAIL=JOB_FIELD=source_status=OK\nDETAIL=JOB_FIELD=source_mtime=2026-07-09T12:42:17Z\nDETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt\nDETAIL=JOB_FIELD=service=filebrowser-restore-validate.service\nDETAIL=JOB_FIELD=service_result=success\nDETAIL=JOB_FIELD=service_exec_main_status=0\nDETAIL=JOB_FIELD=timer=filebrowser-restore-validate.timer\nDETAIL=JOB_FIELD=timer_enabled=enabled\nDETAIL=JOB_FIELD=timer_active=active\nDETAIL=SERVICE_SHOW_RC=0\nDETAIL=SERVICE_PROPERTY=Result=success\nDETAIL=SERVICE_PROPERTY=ExecMainCode=1\nDETAIL=SERVICE_PROPERTY=ExecMainStatus=0\nDETAIL=SERVICE_PROPERTY=LoadState=loaded\nDETAIL=SERVICE_PROPERTY=ActiveState=inactive\nDETAIL=SERVICE_PROPERTY=SubState=dead\nDETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/filebrowser-restore-validate.service\nDETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/filebrowser-restore-validate.service.d/10-superseded-noop.conf /etc/systemd/system/filebrowser-restore-validate.service.d/90-backup-matrix-refresh.conf\nDETAIL=SERVICE_JOURNAL_MATCH_COUNT=56\nDETAIL=SERVICE_JOURNAL=2026-07-11T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-11T04:10:00+03:00 pve01 homelab-superseded-unit-ok[1686870]: STATUS=OK TS=2026-07-11T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-11T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-11T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-12T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-12T04:10:00+03:00 pve01 homelab-superseded-unit-ok[1699058]: STATUS=OK TS=2026-07-12T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-12T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-12T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-13T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-13T04:10:00+03:00 pve01 homelab-superseded-unit-ok[1681015]: STATUS=OK TS=2026-07-13T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-13T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-13T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-14T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-14T04:10:00+03:00 pve01 homelab-superseded-unit-ok[2132068]: STATUS=OK TS=2026-07-14T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-14T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-14T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-15T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-15T04:10:00+03:00 pve01 homelab-superseded-unit-ok[2783319]: STATUS=OK TS=2026-07-15T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-15T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-15T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-16T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-16T04:10:00+03:00 pve01 homelab-superseded-unit-ok[3557730]: STATUS=OK TS=2026-07-16T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-16T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-16T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-17T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-17T04:10:14+03:00 pve01 homelab-superseded-unit-ok[3346747]: STATUS=OK TS=2026-07-17T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-17T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-17T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-18T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-18T04:10:14+03:00 pve01 homelab-superseded-unit-ok[4129958]: STATUS=OK TS=2026-07-18T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-18T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-18T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-19T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-19T04:10:14+03:00 pve01 homelab-superseded-unit-ok[738797]: STATUS=OK TS=2026-07-19T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-19T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-19T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-20T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-20T04:10:14+03:00 pve01 homelab-superseded-unit-ok[1448291]: STATUS=OK TS=2026-07-20T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-20T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-20T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-21T04:10:07+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-21T04:10:07+03:00 pve01 homelab-superseded-unit-ok[2324924]: STATUS=OK TS=2026-07-21T01:10:07Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-21T04:10:07+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-21T04:10:07+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-22T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-22T04:10:14+03:00 pve01 homelab-superseded-unit-ok[3008751]: STATUS=OK TS=2026-07-22T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-22T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-22T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-23T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-23T04:10:14+03:00 pve01 homelab-superseded-unit-ok[3700126]: STATUS=OK TS=2026-07-23T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-23T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-23T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-24T04:10:04+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-24T04:10:04+03:00 pve01 homelab-superseded-unit-ok[2416005]: STATUS=OK TS=2026-07-24T01:10:04Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-24T04:10:04+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-24T04:10:04+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=TIMER_SHOW_RC=0\nDETAIL=TIMER_PROPERTY=Unit=filebrowser-restore-validate.service\nDETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Sat 2026-07-25 04:10:00 MSK\nDETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 04:10:04 MSK\nDETAIL=TIMER_PROPERTY=LoadState=loaded\nDETAIL=TIMER_PROPERTY=ActiveState=active\nDETAIL=TIMER_PROPERTY=SubState=waiting\nDETAIL=TIMER_ENABLED_RESULT=enabled|RC=0\nDETAIL=HEALTH_METADATA=BYTES:209|MTIME_NS:1783600937792562046\nDETAIL=HEALTH_MATCH_COUNT=1\nDETAIL=HEALTH_LINE=STATUS=OK TS=2026-07-09T12:42:13Z TYPE=filebrowser-restore JOB=homelab-pve01-app-filebrowser-20260709T051713Z SOURCE=appbackupctl-restore-check RESTORECHECK=OK_BOTH_CLOUDS NOTE=service-readiness-restore-proof\nDETAIL=JOB_END=filebrowser_restore_validate\nDETAIL=JOB_BEGIN=homelab_backup_sla_health\nDETAIL=JOB_FIELD=id=homelab_backup_sla_health\nDETAIL=JOB_FIELD=name=homelab-backup-sla-health\nDETAIL=JOB_FIELD=category=application\nDETAIL=JOB_FIELD=critical=True\nDETAIL=JOB_FIELD=status=FAIL\nDETAIL=JOB_FIELD=freshness=FRESH\nDETAIL=JOB_FIELD=age_seconds=358\nDETAIL=JOB_FIELD=max_age_seconds=172800\nDETAIL=JOB_FIELD=source_kind=health_file\nDETAIL=JOB_FIELD=source_status=WARN\nDETAIL=JOB_FIELD=source_mtime=2026-07-24T07:00:06Z\nDETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt\nDETAIL=JOB_FIELD=service=homelab-backup-sla-health.service\nDETAIL=JOB_FIELD=service_result=success\nDETAIL=JOB_FIELD=service_exec_main_status=0\nDETAIL=JOB_FIELD=timer=homelab-backup-sla-health.timer\nDETAIL=JOB_FIELD=timer_enabled=enabled\nDETAIL=JOB_FIELD=timer_active=active\nDETAIL=SERVICE_SHOW_RC=0\nDETAIL=SERVICE_PROPERTY=Result=success\nDETAIL=SERVICE_PROPERTY=ExecMainCode=1\nDETAIL=SERVICE_PROPERTY=ExecMainStatus=0\nDETAIL=SERVICE_PROPERTY=LoadState=loaded\nDETAIL=SERVICE_PROPERTY=ActiveState=inactive\nDETAIL=SERVICE_PROPERTY=SubState=dead\nDETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/homelab-backup-sla-health.service\nDETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/homelab-backup-sla-health.service.d/90-backup-matrix-refresh.conf\nDETAIL=SERVICE_JOURNAL_MATCH_COUNT=76\nDETAIL=SERVICE_JOURNAL=2026-07-11T06:24:11+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-11T06:24:11+03:00 pve01 homelab-backup-sla-health[2238839]: STATUS=OK TS=2026-07-11T03:24:11Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-11T06:24:11+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-11T06:24:11+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-11T07:39:35+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-11T07:39:36+03:00 pve01 homelab-backup-sla-health[2476861]: STATUS=OK TS=2026-07-11T04:39:35Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-11T07:39:36+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-11T07:39:36+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-11T08:15:32+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-11T08:15:32+03:00 pve01 homelab-backup-sla-health[2616083]: STATUS=OK TS=2026-07-11T05:15:32Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-11T08:15:33+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-11T08:15:33+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-12T08:04:09+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-12T08:04:09+03:00 pve01 homelab-backup-sla-health[2414445]: STATUS=WARN TS=2026-07-12T05:04:09Z TYPE=backup-sla BAD=0 WARN=1 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-12T08:04:09+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-12T08:04:09+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-13T01:03:10+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-13T01:03:11+03:00 pve01 homelab-backup-sla-health[980591]: STATUS=OK TS=2026-07-12T22:03:10Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-13T01:03:11+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-13T01:03:11+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-13T01:08:07+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-13T01:08:08+03:00 pve01 homelab-backup-sla-health[994479]: STATUS=OK TS=2026-07-12T22:08:07Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-13T01:08:08+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-13T01:08:08+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-13T08:15:21+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-13T08:15:21+03:00 pve01 homelab-backup-sla-health[2521375]: STATUS=OK TS=2026-07-13T05:15:21Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-13T08:15:21+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-13T08:15:21+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-14T08:05:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-14T08:05:15+03:00 pve01 homelab-backup-sla-health[2983241]: STATUS=WARN TS=2026-07-14T05:05:14Z TYPE=backup-sla BAD=0 WARN=1 APPS=46 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-14T08:05:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-14T08:05:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-14T08:23:27+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-14T08:23:28+03:00 pve01 homelab-backup-sla-health[3042266]: STATUS=OK TS=2026-07-14T05:23:27Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-14T08:23:28+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-14T08:23:28+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-15T08:19:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-15T08:19:15+03:00 pve01 homelab-backup-sla-health[3670619]: STATUS=OK TS=2026-07-15T05:19:14Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-15T08:19:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-15T08:19:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-16T08:09:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-16T08:09:15+03:00 pve01 homelab-backup-sla-health[225757]: STATUS=WARN TS=2026-07-16T05:09:14Z TYPE=backup-sla BAD=0 WARN=1 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-16T08:09:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-16T08:09:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-17T08:07:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-17T08:07:15+03:00 pve01 homelab-backup-sla-health[4188049]: STATUS=WARN TS=2026-07-17T05:07:14Z TYPE=backup-sla BAD=0 WARN=4 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-17T08:07:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-17T08:07:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-18T08:05:29+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-18T08:05:29+03:00 pve01 homelab-backup-sla-health[806098]: STATUS=OK TS=2026-07-18T05:05:29Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-18T08:05:29+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-18T08:05:29+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-19T08:05:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-19T08:05:15+03:00 pve01 homelab-backup-sla-health[1545067]: STATUS=OK TS=2026-07-19T05:05:14Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-19T08:05:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-19T08:05:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-20T08:10:25+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-20T08:10:26+03:00 pve01 homelab-backup-sla-health[2331602]: STATUS=OK TS=2026-07-20T05:10:25Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-20T08:10:26+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-20T08:10:26+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-21T08:07:25+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-21T08:07:26+03:00 pve01 homelab-backup-sla-health[3122692]: STATUS=WARN TS=2026-07-21T05:07:25Z TYPE=backup-sla BAD=0 WARN=1 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-21T08:07:26+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-21T08:07:26+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-22T08:10:41+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-22T08:10:42+03:00 pve01 homelab-backup-sla-health[3806055]: STATUS=WARN TS=2026-07-22T05:10:41Z TYPE=backup-sla BAD=0 WARN=1 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-22T08:10:42+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-22T08:10:42+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-23T08:06:25+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-23T08:06:26+03:00 pve01 homelab-backup-sla-health[310318]: STATUS=WARN TS=2026-07-23T05:06:25Z TYPE=backup-sla BAD=0 WARN=2 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-23T08:06:26+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-23T08:06:26+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-24T08:20:04+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-24T08:20:04+03:00 pve01 homelab-backup-sla-health[3253899]: STATUS=WARN TS=2026-07-24T05:20:04Z TYPE=backup-sla BAD=0 WARN=4 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-24T08:20:04+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-24T08:20:04+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=TIMER_SHOW_RC=0\nDETAIL=TIMER_PROPERTY=Unit=homelab-backup-sla-health.service\nDETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Sat 2026-07-25 08:08:57 MSK\nDETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 08:20:04 MSK\nDETAIL=TIMER_PROPERTY=LoadState=loaded\nDETAIL=TIMER_PROPERTY=ActiveState=active\nDETAIL=TIMER_PROPERTY=SubState=waiting\nDETAIL=TIMER_ENABLED_RESULT=enabled|RC=0\nDETAIL=HEALTH_METADATA=BYTES:154|MTIME_NS:1784876406380685648\nDETAIL=HEALTH_MATCH_COUNT=1\nDETAIL=HEALTH_LINE=STATUS=WARN TS=2026-07-24T07:00:05Z TYPE=backup-sla BAD=0 WARN=4 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=JOB_END=homelab_backup_sla_health\nDETAIL=JOB_BEGIN=homelab_mail_cloud_restore_drill\nDETAIL=JOB_FIELD=id=homelab_mail_cloud_restore_drill\nDETAIL=JOB_FIELD=name=homelab-mail-cloud-restore-drill\nDETAIL=JOB_FIELD=category=restore\nDETAIL=JOB_FIELD=critical=True\nDETAIL=JOB_FIELD=status=STALE\nDETAIL=JOB_FIELD=freshness=STALE\nDETAIL=JOB_FIELD=age_seconds=1247672\nDETAIL=JOB_FIELD=max_age_seconds=864000\nDETAIL=JOB_FIELD=source_kind=health_file\nDETAIL=JOB_FIELD=source_status=OK\nDETAIL=JOB_FIELD=source_mtime=2026-07-09T20:31:32Z\nDETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/homelab_mail_cloud_restore_drill/0-mail-cloud-restore-drill.txt\nDETAIL=JOB_FIELD=service=homelab-mail-cloud-restore-drill.service\nDETAIL=JOB_FIELD=service_result=success\nDETAIL=JOB_FIELD=service_exec_main_status=0\nDETAIL=JOB_FIELD=timer=homelab-mail-cloud-restore-drill.timer\nDETAIL=JOB_FIELD=timer_enabled=enabled\nDETAIL=JOB_FIELD=timer_active=active\nDETAIL=SERVICE_SHOW_RC=0\nDETAIL=SERVICE_PROPERTY=Result=success\nDETAIL=SERVICE_PROPERTY=ExecMainCode=0\nDETAIL=SERVICE_PROPERTY=ExecMainStatus=0\nDETAIL=SERVICE_PROPERTY=LoadState=loaded\nDETAIL=SERVICE_PROPERTY=ActiveState=inactive\nDETAIL=SERVICE_PROPERTY=SubState=dead\nDETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/homelab-mail-cloud-restore-drill.service\nDETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/homelab-mail-cloud-restore-drill.service.d/10-superseded-noop.conf /etc/systemd/system/homelab-mail-cloud-restore-drill.service.d/90-backup-matrix-refresh.conf\nDETAIL=SERVICE_JOURNAL_MATCH_COUNT=8\nDETAIL=SERVICE_JOURNAL=2026-07-13T07:38:36+03:00 pve01 systemd[1]: Starting homelab-mail-cloud-restore-drill.service - Homelab Mail Cloud restore drill...\nDETAIL=SERVICE_JOURNAL=2026-07-13T07:38:36+03:00 pve01 homelab-superseded-unit-ok[2401599]: STATUS=OK TS=2026-07-13T04:38:36Z TYPE=superseded-unit UNIT=homelab-mail-cloud-restore-drill.service REASON=superseded_by_safety_net_restore_check ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-13T07:38:36+03:00 pve01 systemd[1]: homelab-mail-cloud-restore-drill.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-13T07:38:36+03:00 pve01 systemd[1]: Finished homelab-mail-cloud-restore-drill.service - Homelab Mail Cloud restore drill.\nDETAIL=SERVICE_JOURNAL=2026-07-20T07:25:14+03:00 pve01 systemd[1]: Starting homelab-mail-cloud-restore-drill.service - Homelab Mail Cloud restore drill...\nDETAIL=SERVICE_JOURNAL=2026-07-20T07:25:14+03:00 pve01 homelab-superseded-unit-ok[2171079]: STATUS=OK TS=2026-07-20T04:25:14Z TYPE=superseded-unit UNIT=homelab-mail-cloud-restore-drill.service REASON=superseded_by_safety_net_restore_check ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-20T07:25:14+03:00 pve01 systemd[1]: homelab-mail-cloud-restore-drill.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-20T07:25:14+03:00 pve01 systemd[1]: Finished homelab-mail-cloud-restore-drill.service - Homelab Mail Cloud restore drill.\nDETAIL=TIMER_SHOW_RC=0\nDETAIL=TIMER_PROPERTY=Unit=homelab-mail-cloud-restore-drill.service\nDETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Mon 2026-07-27 07:34:52 MSK\nDETAIL=TIMER_PROPERTY=LastTriggerUSec=Mon 2026-07-20 07:25:14 MSK\nDETAIL=TIMER_PROPERTY=LoadState=loaded\nDETAIL=TIMER_PROPERTY=ActiveState=active\nDETAIL=TIMER_PROPERTY=SubState=waiting\nDETAIL=TIMER_ENABLED_RESULT=enabled|RC=0\nDETAIL=HEALTH_METADATA=BYTES:353|MTIME_NS:1783629092051018609\nDETAIL=HEALTH_MATCH_COUNT=1\nDETAIL=HEALTH_LINE=STATUS=OK TS=2026-07-09T20:31:32Z TYPE=legacy-mail-cloud-restore-drill-superseded MODE=SUPERSEDED_BY_CURRENT_PROOF BAD=0 WARN=0 PROOF=safety_net_control_plane_restore_check_OK ACTION=stale_health_refresh_only REASON=legacy_drill_superseded_but_health_file_mtime_must_stay_fresh REPORT=/var/lib/homelab-failed-unit-legacy-cleanup/local-cleanup-latest.md\nDETAIL=JOB_END=homelab_mail_cloud_restore_drill\nDETAIL=JOB_BEGIN=netbird_vps_backup\nDETAIL=JOB_FIELD=id=netbird_vps_backup\nDETAIL=JOB_FIELD=name=netbird-vps-backup\nDETAIL=JOB_FIELD=category=application\nDETAIL=JOB_FIELD=critical=True\nDETAIL=JOB_FIELD=status=FAIL\nDETAIL=JOB_FIELD=freshness=STALE\nDETAIL=JOB_FIELD=age_seconds=2007299\nDETAIL=JOB_FIELD=max_age_seconds=864000\nDETAIL=JOB_FIELD=source_kind=health_file\nDETAIL=JOB_FIELD=source_status=OK\nDETAIL=JOB_FIELD=source_mtime=2026-07-01T01:31:05Z\nDETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt\nDETAIL=JOB_FIELD=service=netbird-vps-backup.service\nDETAIL=JOB_FIELD=service_result=success\nDETAIL=JOB_FIELD=service_exec_main_status=0\nDETAIL=JOB_FIELD=timer=netbird-vps-backup.timer\nDETAIL=JOB_FIELD=timer_enabled=disabled\nDETAIL=JOB_FIELD=timer_active=inactive\nDETAIL=SERVICE_SHOW_RC=0\nDETAIL=SERVICE_PROPERTY=Result=success\nDETAIL=SERVICE_PROPERTY=ExecMainCode=0\nDETAIL=SERVICE_PROPERTY=ExecMainStatus=0\nDETAIL=SERVICE_PROPERTY=LoadState=loaded\nDETAIL=SERVICE_PROPERTY=ActiveState=inactive\nDETAIL=SERVICE_PROPERTY=SubState=dead\nDETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/netbird-vps-backup.service\nDETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/netbird-vps-backup.service.d/10-superseded-noop.conf /etc/systemd/system/netbird-vps-backup.service.d/90-backup-matrix-refresh.conf\nDETAIL=SERVICE_JOURNAL_MATCH_COUNT=28\nDETAIL=SERVICE_JOURNAL=2026-07-11T04:30:35+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging...\nDETAIL=SERVICE_JOURNAL=2026-07-11T04:30:35+03:00 pve01 homelab-superseded-unit-ok[1772638]: STATUS=OK TS=2026-07-11T01:30:35Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-11T04:30:35+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-11T04:30:35+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging.\nDETAIL=SERVICE_JOURNAL=2026-07-12T04:39:35+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging...\nDETAIL=SERVICE_JOURNAL=2026-07-12T04:39:35+03:00 pve01 homelab-superseded-unit-ok[1789866]: STATUS=OK TS=2026-07-12T01:39:35Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-12T04:39:35+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-12T04:39:35+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging.\nDETAIL=SERVICE_JOURNAL=2026-07-13T04:28:00+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging...\nDETAIL=SERVICE_JOURNAL=2026-07-13T04:28:00+03:00 pve01 homelab-superseded-unit-ok[1739137]: STATUS=OK TS=2026-07-13T01:28:00Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-13T04:28:00+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-13T04:28:00+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging.\nDETAIL=SERVICE_JOURNAL=2026-07-14T04:28:39+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging...\nDETAIL=SERVICE_JOURNAL=2026-07-14T04:28:39+03:00 pve01 homelab-superseded-unit-ok[2199841]: STATUS=OK TS=2026-07-14T01:28:39Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-14T04:28:39+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-14T04:28:39+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging.\nDETAIL=SERVICE_JOURNAL=2026-07-15T04:39:00+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging...\nDETAIL=SERVICE_JOURNAL=2026-07-15T04:39:00+03:00 pve01 homelab-superseded-unit-ok[2872498]: STATUS=OK TS=2026-07-15T01:39:00Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-15T04:39:00+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-15T04:39:00+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging.\nDETAIL=SERVICE_JOURNAL=2026-07-16T04:25:14+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging...\nDETAIL=SERVICE_JOURNAL=2026-07-16T04:25:14+03:00 pve01 homelab-superseded-unit-ok[3611661]: STATUS=OK TS=2026-07-16T01:25:14Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-16T04:25:14+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-16T04:25:14+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging.\nDETAIL=SERVICE_JOURNAL=2026-07-17T04:31:14+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging...\nDETAIL=SERVICE_JOURNAL=2026-07-17T04:31:14+03:00 pve01 homelab-superseded-unit-ok[3412653]: STATUS=OK TS=2026-07-17T01:31:14Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-17T04:31:14+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-17T04:31:14+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging.\nDETAIL=TIMER_SHOW_RC=0\nDETAIL=TIMER_PROPERTY=Unit=netbird-vps-backup.service\nDETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=\nDETAIL=TIMER_PROPERTY=LastTriggerUSec=\nDETAIL=TIMER_PROPERTY=LoadState=loaded\nDETAIL=TIMER_PROPERTY=ActiveState=inactive\nDETAIL=TIMER_PROPERTY=SubState=dead\nDETAIL=TIMER_ENABLED_RESULT=disabled|RC=1\nDETAIL=HEALTH_METADATA=BYTES:312|MTIME_NS:1782869465433646905\nDETAIL=HEALTH_MATCH_COUNT=7\nDETAIL=HEALTH_LINE=STATUS=OK\nDETAIL=HEALTH_LINE=SERVICE=netbird-vps-backup\nDETAIL=HEALTH_LINE=TIMESTAMP_UTC=20260701T013054Z\nDETAIL=HEALTH_LINE=SNAPSHOT=/mnt/staging/netbird-vps-backups/snapshots/20260701T013054Z\nDETAIL=HEALTH_LINE=LATEST_SYMLINK=/mnt/staging/netbird-vps-backups/snapshots/latest\nDETAIL=HEALTH_LINE=SIZE=32M\nDETAIL=HEALTH_LINE=SHA256=0226dd820cb9ede0ea9e8ea4af6cef656ffa54cfb819048c222a88f4c14d8db9\nDETAIL=JOB_END=netbird_vps_backup\nDETAIL=JOB_BEGIN=paperless_restore_dry_run\nDETAIL=JOB_FIELD=id=paperless_restore_dry_run\nDETAIL=JOB_FIELD=name=paperless-restore-dry-run\nDETAIL=JOB_FIELD=category=restore\nDETAIL=JOB_FIELD=critical=True\nDETAIL=JOB_FIELD=status=STALE\nDETAIL=JOB_FIELD=freshness=STALE\nDETAIL=JOB_FIELD=age_seconds=1261389\nDETAIL=JOB_FIELD=max_age_seconds=864000\nDETAIL=JOB_FIELD=source_kind=health_file\nDETAIL=JOB_FIELD=source_status=OK\nDETAIL=JOB_FIELD=source_mtime=2026-07-09T16:42:55Z\nDETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/paperless_restore_dry_run/0-paperless-restore.txt\nDETAIL=JOB_FIELD=service=paperless-restore-dry-run.service\nDETAIL=JOB_FIELD=service_result=success\nDETAIL=JOB_FIELD=service_exec_main_status=0\nDETAIL=JOB_FIELD=timer=paperless-restore-dry-run.timer\nDETAIL=JOB_FIELD=timer_enabled=enabled\nDETAIL=JOB_FIELD=timer_active=active\nDETAIL=SERVICE_SHOW_RC=0\nDETAIL=SERVICE_PROPERTY=Result=success\nDETAIL=SERVICE_PROPERTY=ExecMainCode=1\nDETAIL=SERVICE_PROPERTY=ExecMainStatus=0\nDETAIL=SERVICE_PROPERTY=LoadState=loaded\nDETAIL=SERVICE_PROPERTY=ActiveState=inactive\nDETAIL=SERVICE_PROPERTY=SubState=dead\nDETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/paperless-restore-dry-run.service\nDETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/paperless-restore-dry-run.service.d/10-superseded-noop.conf /etc/systemd/system/paperless-restore-dry-run.service.d/90-backup-matrix-refresh.conf\nDETAIL=SERVICE_JOURNAL_MATCH_COUNT=56\nDETAIL=SERVICE_JOURNAL=2026-07-11T10:08:13+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-11T10:08:13+03:00 pve01 homelab-superseded-unit-ok[2988079]: STATUS=OK TS=2026-07-11T07:08:13Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-11T10:08:13+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-11T10:08:13+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-12T10:04:59+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-12T10:04:59+03:00 pve01 homelab-superseded-unit-ok[2757392]: STATUS=OK TS=2026-07-12T07:04:59Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-12T10:04:59+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-12T10:04:59+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-13T10:07:25+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-13T10:07:25+03:00 pve01 homelab-superseded-unit-ok[2886298]: STATUS=OK TS=2026-07-13T07:07:25Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-13T10:07:25+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-13T10:07:25+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-14T09:56:36+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-14T09:56:36+03:00 pve01 homelab-superseded-unit-ok[3367368]: STATUS=OK TS=2026-07-14T06:56:36Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-14T09:56:36+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-14T09:56:36+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-15T10:12:54+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-15T10:12:54+03:00 pve01 homelab-superseded-unit-ok[4062186]: STATUS=OK TS=2026-07-15T07:12:54Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-15T10:12:54+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-15T10:12:54+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-16T11:40:40+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-16T11:40:40+03:00 pve01 homelab-superseded-unit-ok[12176]: STATUS=OK TS=2026-07-16T08:40:40Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-16T11:40:40+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-16T11:40:40+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-17T10:00:14+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-17T10:00:14+03:00 pve01 homelab-superseded-unit-ok[390020]: STATUS=OK TS=2026-07-17T07:00:14Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-17T10:00:14+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-17T10:00:14+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-18T09:57:16+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-18T09:57:16+03:00 pve01 homelab-superseded-unit-ok[1201647]: STATUS=OK TS=2026-07-18T06:57:16Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-18T09:57:16+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-18T09:57:16+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-19T10:09:17+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-19T10:09:17+03:00 pve01 homelab-superseded-unit-ok[1945203]: STATUS=OK TS=2026-07-19T07:09:17Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-19T10:09:17+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-19T10:09:17+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-20T10:04:06+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-20T10:04:06+03:00 pve01 homelab-superseded-unit-ok[2734120]: STATUS=OK TS=2026-07-20T07:04:06Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-20T10:04:06+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-20T10:04:06+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-21T10:02:41+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-21T10:02:41+03:00 pve01 homelab-superseded-unit-ok[3525432]: STATUS=OK TS=2026-07-21T07:02:41Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-21T10:02:41+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-21T10:02:41+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-22T10:11:14+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-22T10:11:14+03:00 pve01 homelab-superseded-unit-ok[4192753]: STATUS=OK TS=2026-07-22T07:11:14Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-22T10:11:14+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-22T10:11:14+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-23T10:12:14+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-23T10:12:14+03:00 pve01 homelab-superseded-unit-ok[752982]: STATUS=OK TS=2026-07-23T07:12:14Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-23T10:12:14+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-23T10:12:14+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-24T10:06:04+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-24T10:06:04+03:00 pve01 homelab-superseded-unit-ok[3581148]: STATUS=OK TS=2026-07-24T07:06:04Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-24T10:06:04+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-24T10:06:04+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=TIMER_SHOW_RC=0\nDETAIL=TIMER_PROPERTY=Unit=paperless-restore-dry-run.service\nDETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Sat 2026-07-25 10:03:38 MSK\nDETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 10:06:04 MSK\nDETAIL=TIMER_PROPERTY=LoadState=loaded\nDETAIL=TIMER_PROPERTY=ActiveState=active\nDETAIL=TIMER_PROPERTY=SubState=waiting\nDETAIL=TIMER_ENABLED_RESULT=enabled|RC=0\nDETAIL=HEALTH_METADATA=BYTES:232|MTIME_NS:1783615375842257624\nDETAIL=HEALTH_MATCH_COUNT=1\nDETAIL=HEALTH_LINE=STATUS=OK TS=2026-07-09T16:42:53Z TYPE=paperless-restore BAD=0 APP=paperless JOB=homelab-pve01-app-paperless-20260709T071949Z RESTORE_VERIFIED=true RESTORE_CHECK_APP=paperless JSON=/var/lib/homelab-backup/apps/latest-paperless.json\nDETAIL=JOB_END=paperless_restore_dry_run\nDETAIL=JOB_BEGIN=vm160_remote_backup\nDETAIL=JOB_FIELD=id=vm160_remote_backup\nDETAIL=JOB_FIELD=name=vm160-remote-backup\nDETAIL=JOB_FIELD=category=virtual-machine\nDETAIL=JOB_FIELD=critical=True\nDETAIL=JOB_FIELD=status=FAIL\nDETAIL=JOB_FIELD=freshness=STALE\nDETAIL=JOB_FIELD=age_seconds=286010\nDETAIL=JOB_FIELD=max_age_seconds=172800\nDETAIL=JOB_FIELD=source_kind=health_file\nDETAIL=JOB_FIELD=source_status=FAIL\nDETAIL=JOB_FIELD=source_mtime=2026-07-20T23:39:14Z\nDETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/vm160_remote_backup/0-vm160-remote-backup.txt\nDETAIL=JOB_FIELD=service=homelab-vm160-remote-backup.service\nDETAIL=JOB_FIELD=service_result=success\nDETAIL=JOB_FIELD=service_exec_main_status=0\nDETAIL=JOB_FIELD=timer=homelab-vm160-remote-backup.timer\nDETAIL=JOB_FIELD=timer_enabled=disabled\nDETAIL=JOB_FIELD=timer_active=inactive\nDETAIL=SERVICE_SHOW_RC=0\nDETAIL=SERVICE_PROPERTY=Result=success\nDETAIL=SERVICE_PROPERTY=ExecMainCode=0\nDETAIL=SERVICE_PROPERTY=ExecMainStatus=0\nDETAIL=SERVICE_PROPERTY=LoadState=loaded\nDETAIL=SERVICE_PROPERTY=ActiveState=inactive\nDETAIL=SERVICE_PROPERTY=SubState=dead\nDETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/homelab-vm160-remote-backup.service\nDETAIL=SERVICE_PROPERTY=DropInPaths=\nDETAIL=SERVICE_JOURNAL_MATCH_COUNT=50\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:02:41+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:15:43+03:00 pve01 homelab-vm160-remote-backup[3491061]: VM160_TARGET_FINALIZE_STATUS=OK\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3530071]: VM160_TARGET_PROMOTE_STATUS=OK\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: VM160_REMOTE_BACKUP_STATUS=OK\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260717T220242Z\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: ARCHIVE=/mnt/staging/vzdump/vm160-pve02-20260717T220242Z/vzdump-qemu-160-20260717T220242Z.vma.zst\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: SIZE_BYTES=84993482488\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: SHA256=a9562d8b60d7fd5e7ec73bc43d252cfea80ed12c05b32cca0a73d7bb88532411\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 systemd[1]: Finished homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Consumed 2min 44.280s CPU time, 15.8M memory peak.\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:38:18+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:51:18+03:00 pve01 homelab-vm160-remote-backup[3609200]: VM160_TARGET_FINALIZE_STATUS=OK\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3648126]: VM160_TARGET_PROMOTE_STATUS=OK\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: VM160_REMOTE_BACKUP_STATUS=OK\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: ARCHIVE=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z/vzdump-qemu-160-20260717T223819Z.vma.zst\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: SIZE_BYTES=84995216465\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: SHA256=257e10821e62e3e2f9318b238a5302a6db601dd597bfa3e0d77aba07f3b85e72\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 systemd[1]: Finished homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Consumed 2min 44.842s CPU time, 16.1M memory peak.\nDETAIL=SERVICE_JOURNAL=2026-07-18T04:16:25+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nDETAIL=SERVICE_JOURNAL=2026-07-18T04:16:27+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=SERVICE_JOURNAL=2026-07-18T04:16:27+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nDETAIL=SERVICE_JOURNAL=2026-07-18T04:16:27+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nDETAIL=SERVICE_JOURNAL=2026-07-19T04:19:14+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nDETAIL=SERVICE_JOURNAL=2026-07-19T04:19:16+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=SERVICE_JOURNAL=2026-07-19T04:19:16+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nDETAIL=SERVICE_JOURNAL=2026-07-19T04:19:16+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nDETAIL=SERVICE_JOURNAL=2026-07-20T04:14:14+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nDETAIL=SERVICE_JOURNAL=2026-07-20T04:14:16+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=SERVICE_JOURNAL=2026-07-20T04:14:16+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nDETAIL=SERVICE_JOURNAL=2026-07-20T04:14:16+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nDETAIL=SERVICE_JOURNAL=2026-07-20T19:28:57+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nDETAIL=SERVICE_JOURNAL=2026-07-20T19:28:59+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=SERVICE_JOURNAL=2026-07-20T19:28:59+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nDETAIL=SERVICE_JOURNAL=2026-07-20T19:28:59+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:14:11+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:14:13+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:14:13+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:14:13+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:20:20+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:20:21+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:20:21+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:20:21+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:39:12+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:39:14+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:39:14+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:39:14+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nDETAIL=TIMER_SHOW_RC=0\nDETAIL=TIMER_PROPERTY=Unit=homelab-vm160-remote-backup.service\nDETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=\nDETAIL=TIMER_PROPERTY=LastTriggerUSec=\nDETAIL=TIMER_PROPERTY=LoadState=loaded\nDETAIL=TIMER_PROPERTY=ActiveState=inactive\nDETAIL=TIMER_PROPERTY=SubState=dead\nDETAIL=TIMER_ENABLED_RESULT=disabled|RC=1\nDETAIL=HEALTH_METADATA=BYTES:153|MTIME_NS:1784590754312087060\nDETAIL=HEALTH_MATCH_COUNT=3\nDETAIL=HEALTH_LINE=STATUS=FAIL\nDETAIL=HEALTH_LINE=SERVICE=homelab-vm160-remote-backup\nDETAIL=HEALTH_LINE=TIMESTAMP_UTC=20260720T233913Z\nDETAIL=JOB_END=vm160_remote_backup\nDETAIL=SECRET_VALUES_OUTPUT=NONE\nDETAIL=BACKUP_BAD_JOBS_ROOTCAUSE=PASS\nCHANGES_MADE=NO\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index d4fc116..d54e648 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,5 +1,5 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=HOMELAB-BACKUP-MATRIX-BAD-JOBS-DISCOVERY-430 +COMMAND_ID=HOMELAB-BACKUP-BAD-JOBS-ROOTCAUSE-431 STATUS=OK RC=0 HOST=pve01 @@ -9,84 +9,548 @@ REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 -COMMAND_SHA256=8a21cde408577f47c4d88ff6dcf0a4449ef7a77e0c064419260504b794cdc2e6 +COMMAND_SHA256=8635e368c9b3e168a7302affcd2c133d8f85547164b3f3352fc0cbad679edc98 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGES_MADE=false SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=de359611c59c55b28e546d3099a9a49187e6e8523948be5d2bf6ea5b633cb2ac -SANITIZED_OUTPUT_SHA256=de359611c59c55b28e546d3099a9a49187e6e8523948be5d2bf6ea5b633cb2ac +RAW_EVIDENCE_SHA256=e39cc1d76af62d7050936445c27d21728aaeb6ca00ab2430c6cb3963a7da3907 +SANITIZED_OUTPUT_SHA256=e39cc1d76af62d7050936445c27d21728aaeb6ca00ab2430c6cb3963a7da3907 OUTPUT_BEGIN -DETAIL=STATE_PATH=/var/lib/homelab-backup-matrix/state.json -DETAIL=STATE_BYTES=36840 -DETAIL=STATE_SHA256=08d7536cb2c11d036726c4bc0c59d573f6c02e8b3e136e9872128a380ab88807 -DETAIL=ROOT_KEYS=authority,compatibility_map,generated_at,jobs,matrix_mode,matrix_sha256,summary,version -DETAIL=SUMMARY=$.jobs[0].status=OK -DETAIL=SUMMARY=$.jobs[10].status=OK -DETAIL=SUMMARY=$.jobs[11].status=OK -DETAIL=SUMMARY=$.jobs[12].status=OK -DETAIL=SUMMARY=$.jobs[13].status=FAIL -DETAIL=SUMMARY=$.jobs[14].status=OK -DETAIL=SUMMARY=$.jobs[15].status=OK -DETAIL=SUMMARY=$.jobs[16].status=OK -DETAIL=SUMMARY=$.jobs[17].status=DISABLED -DETAIL=SUMMARY=$.jobs[18].status=OK -DETAIL=SUMMARY=$.jobs[19].status=OK -DETAIL=SUMMARY=$.jobs[1].status=OK -DETAIL=SUMMARY=$.jobs[20].status=OK -DETAIL=SUMMARY=$.jobs[21].status=OK -DETAIL=SUMMARY=$.jobs[22].status=STALE -DETAIL=SUMMARY=$.jobs[23].status=DISABLED -DETAIL=SUMMARY=$.jobs[24].status=OK -DETAIL=SUMMARY=$.jobs[25].status=DISABLED -DETAIL=SUMMARY=$.jobs[26].status=OK -DETAIL=SUMMARY=$.jobs[27].status=OK -DETAIL=SUMMARY=$.jobs[28].status=OK -DETAIL=SUMMARY=$.jobs[29].status=OK -DETAIL=SUMMARY=$.jobs[2].status=STALE -DETAIL=SUMMARY=$.jobs[30].status=OK -DETAIL=SUMMARY=$.jobs[31].status=OK -DETAIL=SUMMARY=$.jobs[32].status=OK -DETAIL=SUMMARY=$.jobs[33].status=OK -DETAIL=SUMMARY=$.jobs[34].status=OK -DETAIL=SUMMARY=$.jobs[35].status=FAIL -DETAIL=SUMMARY=$.jobs[36].status=DISABLED -DETAIL=SUMMARY=$.jobs[37].status=STALE -DETAIL=SUMMARY=$.jobs[38].status=OK -DETAIL=SUMMARY=$.jobs[39].status=OK -DETAIL=SUMMARY=$.jobs[3].status=OK -DETAIL=SUMMARY=$.jobs[40].status=FAIL -DETAIL=SUMMARY=$.jobs[4].status=OK -DETAIL=SUMMARY=$.jobs[5].status=OK -DETAIL=SUMMARY=$.jobs[6].status=OK -DETAIL=SUMMARY=$.jobs[7].status=OK -DETAIL=SUMMARY=$.jobs[8].status=DISABLED -DETAIL=SUMMARY=$.jobs[9].status=OK -DETAIL=SUMMARY=$.summary.jobs_bad=6 -DETAIL=SUMMARY=$.summary.jobs_disabled=5 -DETAIL=SUMMARY=$.summary.jobs_ok=30 -DETAIL=SUMMARY=$.summary.jobs_total=41 -DETAIL=SUMMARY=$.summary.jobs_unmapped=0 -DETAIL=SUMMARY=$.summary.status=FAIL -DETAIL=BAD_RECORD_COUNT=7 -DETAIL=BAD_RECORD=PATH:$.jobs[2]|IDENTITY:filebrowser_restore_validate|BAD:status=stale|DISABLED:none|FIELDS:id=filebrowser_restore_validate;name=filebrowser-restore-validate;service=filebrowser-restore-validate.service;status=STALE -DETAIL=BAD_RECORD=PATH:$.jobs[13]|IDENTITY:homelab_backup_sla_health|BAD:status=fail|DISABLED:none|FIELDS:id=homelab_backup_sla_health;name=homelab-backup-sla-health;service=homelab-backup-sla-health.service;status=FAIL -DETAIL=BAD_RECORD=PATH:$.jobs[22]|IDENTITY:homelab_mail_cloud_restore_drill|BAD:status=stale|DISABLED:none|FIELDS:id=homelab_mail_cloud_restore_drill;name=homelab-mail-cloud-restore-drill;service=homelab-mail-cloud-restore-drill.service;status=STALE -DETAIL=BAD_RECORD=PATH:$.jobs[35]|IDENTITY:netbird_vps_backup|BAD:status=fail|DISABLED:none|FIELDS:id=netbird_vps_backup;name=netbird-vps-backup;service=netbird-vps-backup.service;status=FAIL -DETAIL=BAD_RECORD=PATH:$.jobs[37]|IDENTITY:paperless_restore_dry_run|BAD:status=stale|DISABLED:none|FIELDS:id=paperless_restore_dry_run;name=paperless-restore-dry-run;service=paperless-restore-dry-run.service;status=STALE -DETAIL=BAD_RECORD=PATH:$.jobs[40]|IDENTITY:vm160_remote_backup|BAD:status=fail|DISABLED:none|FIELDS:id=vm160_remote_backup;name=vm160-remote-backup;service=homelab-vm160-remote-backup.service;status=FAIL -DETAIL=BAD_RECORD=PATH:$.summary|IDENTITY:|BAD:status=fail|DISABLED:none|FIELDS:status=FAIL -DETAIL=DISABLED_ONLY_COUNT=5 -DETAIL=DISABLED_RECORD=PATH:$.jobs[8]|IDENTITY:homelab_authentik_backup|SIGNALS:status=disabled -DETAIL=DISABLED_RECORD=PATH:$.jobs[17]|IDENTITY:homelab_gitea_backup|SIGNALS:status=disabled -DETAIL=DISABLED_RECORD=PATH:$.jobs[23]|IDENTITY:homelab_npmplus_kuma_config_backup|SIGNALS:status=disabled -DETAIL=DISABLED_RECORD=PATH:$.jobs[25]|IDENTITY:homelab_vaultwarden_backup|SIGNALS:status=disabled -DETAIL=DISABLED_RECORD=PATH:$.jobs[36]|IDENTITY:paperless_backup|SIGNALS:status=disabled -DETAIL=LIST_SCHEMA=PATH:$.jobs|ITEMS:41|DICT_ITEMS:41|KEYS:age_seconds,category,critical,evidence_level,freshness,generated_health_output,id,max_age_seconds,name,service,service_exec_main_status,service_result,source_health,source_kind,source_mtime,source_status,status,timer,timer_active,timer_enabled +DETAIL=MATRIX_GENERATED_AT=2026-07-24T07:06:04Z +DETAIL=TARGET_JOB_COUNT=6 +DETAIL=JOB_BEGIN=filebrowser_restore_validate +DETAIL=JOB_FIELD=id=filebrowser_restore_validate +DETAIL=JOB_FIELD=name=filebrowser-restore-validate +DETAIL=JOB_FIELD=category=restore +DETAIL=JOB_FIELD=critical=True +DETAIL=JOB_FIELD=status=STALE +DETAIL=JOB_FIELD=freshness=STALE +DETAIL=JOB_FIELD=age_seconds=1275827 +DETAIL=JOB_FIELD=max_age_seconds=864000 +DETAIL=JOB_FIELD=source_kind=health_file +DETAIL=JOB_FIELD=source_status=OK +DETAIL=JOB_FIELD=source_mtime=2026-07-09T12:42:17Z +DETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt +DETAIL=JOB_FIELD=service=filebrowser-restore-validate.service +DETAIL=JOB_FIELD=service_result=success +DETAIL=JOB_FIELD=service_exec_main_status=0 +DETAIL=JOB_FIELD=timer=filebrowser-restore-validate.timer +DETAIL=JOB_FIELD=timer_enabled=enabled +DETAIL=JOB_FIELD=timer_active=active +DETAIL=SERVICE_SHOW_RC=0 +DETAIL=SERVICE_PROPERTY=Result=success +DETAIL=SERVICE_PROPERTY=ExecMainCode=1 +DETAIL=SERVICE_PROPERTY=ExecMainStatus=0 +DETAIL=SERVICE_PROPERTY=LoadState=loaded +DETAIL=SERVICE_PROPERTY=ActiveState=inactive +DETAIL=SERVICE_PROPERTY=SubState=dead +DETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/filebrowser-restore-validate.service +DETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/filebrowser-restore-validate.service.d/10-superseded-noop.conf /etc/systemd/system/filebrowser-restore-validate.service.d/90-backup-matrix-refresh.conf +DETAIL=SERVICE_JOURNAL_MATCH_COUNT=56 +DETAIL=SERVICE_JOURNAL=2026-07-11T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-11T04:10:00+03:00 pve01 homelab-superseded-unit-ok[1686870]: STATUS=OK TS=2026-07-11T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-11T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-11T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=SERVICE_JOURNAL=2026-07-12T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-12T04:10:00+03:00 pve01 homelab-superseded-unit-ok[1699058]: STATUS=OK TS=2026-07-12T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-12T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-12T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=SERVICE_JOURNAL=2026-07-13T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-13T04:10:00+03:00 pve01 homelab-superseded-unit-ok[1681015]: STATUS=OK TS=2026-07-13T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-13T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-13T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=SERVICE_JOURNAL=2026-07-14T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-14T04:10:00+03:00 pve01 homelab-superseded-unit-ok[2132068]: STATUS=OK TS=2026-07-14T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-14T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-14T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=SERVICE_JOURNAL=2026-07-15T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-15T04:10:00+03:00 pve01 homelab-superseded-unit-ok[2783319]: STATUS=OK TS=2026-07-15T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-15T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-15T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=SERVICE_JOURNAL=2026-07-16T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-16T04:10:00+03:00 pve01 homelab-superseded-unit-ok[3557730]: STATUS=OK TS=2026-07-16T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-16T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-16T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=SERVICE_JOURNAL=2026-07-17T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-17T04:10:14+03:00 pve01 homelab-superseded-unit-ok[3346747]: STATUS=OK TS=2026-07-17T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-17T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-17T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=SERVICE_JOURNAL=2026-07-18T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-18T04:10:14+03:00 pve01 homelab-superseded-unit-ok[4129958]: STATUS=OK TS=2026-07-18T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-18T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-18T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=SERVICE_JOURNAL=2026-07-19T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-19T04:10:14+03:00 pve01 homelab-superseded-unit-ok[738797]: STATUS=OK TS=2026-07-19T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-19T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-19T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=SERVICE_JOURNAL=2026-07-20T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-20T04:10:14+03:00 pve01 homelab-superseded-unit-ok[1448291]: STATUS=OK TS=2026-07-20T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-20T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-20T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=SERVICE_JOURNAL=2026-07-21T04:10:07+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-21T04:10:07+03:00 pve01 homelab-superseded-unit-ok[2324924]: STATUS=OK TS=2026-07-21T01:10:07Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-21T04:10:07+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-21T04:10:07+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=SERVICE_JOURNAL=2026-07-22T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-22T04:10:14+03:00 pve01 homelab-superseded-unit-ok[3008751]: STATUS=OK TS=2026-07-22T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-22T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-22T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=SERVICE_JOURNAL=2026-07-23T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-23T04:10:14+03:00 pve01 homelab-superseded-unit-ok[3700126]: STATUS=OK TS=2026-07-23T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-23T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-23T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=SERVICE_JOURNAL=2026-07-24T04:10:04+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... +DETAIL=SERVICE_JOURNAL=2026-07-24T04:10:04+03:00 pve01 homelab-superseded-unit-ok[2416005]: STATUS=OK TS=2026-07-24T01:10:04Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-24T04:10:04+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-24T04:10:04+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. +DETAIL=TIMER_SHOW_RC=0 +DETAIL=TIMER_PROPERTY=Unit=filebrowser-restore-validate.service +DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Sat 2026-07-25 04:10:00 MSK +DETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 04:10:04 MSK +DETAIL=TIMER_PROPERTY=LoadState=loaded +DETAIL=TIMER_PROPERTY=ActiveState=active +DETAIL=TIMER_PROPERTY=SubState=waiting +DETAIL=TIMER_ENABLED_RESULT=enabled|RC=0 +DETAIL=HEALTH_METADATA=BYTES:209|MTIME_NS:1783600937792562046 +DETAIL=HEALTH_MATCH_COUNT=1 +DETAIL=HEALTH_LINE=STATUS=OK TS=2026-07-09T12:42:13Z TYPE=filebrowser-restore JOB=homelab-pve01-app-filebrowser-20260709T051713Z SOURCE=appbackupctl-restore-check RESTORECHECK=OK_BOTH_CLOUDS NOTE=service-readiness-restore-proof +DETAIL=JOB_END=filebrowser_restore_validate +DETAIL=JOB_BEGIN=homelab_backup_sla_health +DETAIL=JOB_FIELD=id=homelab_backup_sla_health +DETAIL=JOB_FIELD=name=homelab-backup-sla-health +DETAIL=JOB_FIELD=category=application +DETAIL=JOB_FIELD=critical=True +DETAIL=JOB_FIELD=status=FAIL +DETAIL=JOB_FIELD=freshness=FRESH +DETAIL=JOB_FIELD=age_seconds=358 +DETAIL=JOB_FIELD=max_age_seconds=172800 +DETAIL=JOB_FIELD=source_kind=health_file +DETAIL=JOB_FIELD=source_status=WARN +DETAIL=JOB_FIELD=source_mtime=2026-07-24T07:00:06Z +DETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt +DETAIL=JOB_FIELD=service=homelab-backup-sla-health.service +DETAIL=JOB_FIELD=service_result=success +DETAIL=JOB_FIELD=service_exec_main_status=0 +DETAIL=JOB_FIELD=timer=homelab-backup-sla-health.timer +DETAIL=JOB_FIELD=timer_enabled=enabled +DETAIL=JOB_FIELD=timer_active=active +DETAIL=SERVICE_SHOW_RC=0 +DETAIL=SERVICE_PROPERTY=Result=success +DETAIL=SERVICE_PROPERTY=ExecMainCode=1 +DETAIL=SERVICE_PROPERTY=ExecMainStatus=0 +DETAIL=SERVICE_PROPERTY=LoadState=loaded +DETAIL=SERVICE_PROPERTY=ActiveState=inactive +DETAIL=SERVICE_PROPERTY=SubState=dead +DETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/homelab-backup-sla-health.service +DETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/homelab-backup-sla-health.service.d/90-backup-matrix-refresh.conf +DETAIL=SERVICE_JOURNAL_MATCH_COUNT=76 +DETAIL=SERVICE_JOURNAL=2026-07-11T06:24:11+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-11T06:24:11+03:00 pve01 homelab-backup-sla-health[2238839]: STATUS=OK TS=2026-07-11T03:24:11Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-11T06:24:11+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-11T06:24:11+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-11T07:39:35+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-11T07:39:36+03:00 pve01 homelab-backup-sla-health[2476861]: STATUS=OK TS=2026-07-11T04:39:35Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-11T07:39:36+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-11T07:39:36+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-11T08:15:32+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-11T08:15:32+03:00 pve01 homelab-backup-sla-health[2616083]: STATUS=OK TS=2026-07-11T05:15:32Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-11T08:15:33+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-11T08:15:33+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-12T08:04:09+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-12T08:04:09+03:00 pve01 homelab-backup-sla-health[2414445]: STATUS=WARN TS=2026-07-12T05:04:09Z TYPE=backup-sla BAD=0 WARN=1 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-12T08:04:09+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-12T08:04:09+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-13T01:03:10+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-13T01:03:11+03:00 pve01 homelab-backup-sla-health[980591]: STATUS=OK TS=2026-07-12T22:03:10Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-13T01:03:11+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-13T01:03:11+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-13T01:08:07+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-13T01:08:08+03:00 pve01 homelab-backup-sla-health[994479]: STATUS=OK TS=2026-07-12T22:08:07Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-13T01:08:08+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-13T01:08:08+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-13T08:15:21+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-13T08:15:21+03:00 pve01 homelab-backup-sla-health[2521375]: STATUS=OK TS=2026-07-13T05:15:21Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-13T08:15:21+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-13T08:15:21+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-14T08:05:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-14T08:05:15+03:00 pve01 homelab-backup-sla-health[2983241]: STATUS=WARN TS=2026-07-14T05:05:14Z TYPE=backup-sla BAD=0 WARN=1 APPS=46 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-14T08:05:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-14T08:05:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-14T08:23:27+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-14T08:23:28+03:00 pve01 homelab-backup-sla-health[3042266]: STATUS=OK TS=2026-07-14T05:23:27Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-14T08:23:28+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-14T08:23:28+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-15T08:19:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-15T08:19:15+03:00 pve01 homelab-backup-sla-health[3670619]: STATUS=OK TS=2026-07-15T05:19:14Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-15T08:19:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-15T08:19:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-16T08:09:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-16T08:09:15+03:00 pve01 homelab-backup-sla-health[225757]: STATUS=WARN TS=2026-07-16T05:09:14Z TYPE=backup-sla BAD=0 WARN=1 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-16T08:09:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-16T08:09:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-17T08:07:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-17T08:07:15+03:00 pve01 homelab-backup-sla-health[4188049]: STATUS=WARN TS=2026-07-17T05:07:14Z TYPE=backup-sla BAD=0 WARN=4 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-17T08:07:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-17T08:07:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-18T08:05:29+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-18T08:05:29+03:00 pve01 homelab-backup-sla-health[806098]: STATUS=OK TS=2026-07-18T05:05:29Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-18T08:05:29+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-18T08:05:29+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-19T08:05:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-19T08:05:15+03:00 pve01 homelab-backup-sla-health[1545067]: STATUS=OK TS=2026-07-19T05:05:14Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-19T08:05:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-19T08:05:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-20T08:10:25+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-20T08:10:26+03:00 pve01 homelab-backup-sla-health[2331602]: STATUS=OK TS=2026-07-20T05:10:25Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-20T08:10:26+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-20T08:10:26+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-21T08:07:25+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-21T08:07:26+03:00 pve01 homelab-backup-sla-health[3122692]: STATUS=WARN TS=2026-07-21T05:07:25Z TYPE=backup-sla BAD=0 WARN=1 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-21T08:07:26+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-21T08:07:26+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-22T08:10:41+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-22T08:10:42+03:00 pve01 homelab-backup-sla-health[3806055]: STATUS=WARN TS=2026-07-22T05:10:41Z TYPE=backup-sla BAD=0 WARN=1 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-22T08:10:42+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-22T08:10:42+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-23T08:06:25+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-23T08:06:26+03:00 pve01 homelab-backup-sla-health[310318]: STATUS=WARN TS=2026-07-23T05:06:25Z TYPE=backup-sla BAD=0 WARN=2 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-23T08:06:26+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-23T08:06:26+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=SERVICE_JOURNAL=2026-07-24T08:20:04+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... +DETAIL=SERVICE_JOURNAL=2026-07-24T08:20:04+03:00 pve01 homelab-backup-sla-health[3253899]: STATUS=WARN TS=2026-07-24T05:20:04Z TYPE=backup-sla BAD=0 WARN=4 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=SERVICE_JOURNAL=2026-07-24T08:20:04+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-24T08:20:04+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. +DETAIL=TIMER_SHOW_RC=0 +DETAIL=TIMER_PROPERTY=Unit=homelab-backup-sla-health.service +DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Sat 2026-07-25 08:08:57 MSK +DETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 08:20:04 MSK +DETAIL=TIMER_PROPERTY=LoadState=loaded +DETAIL=TIMER_PROPERTY=ActiveState=active +DETAIL=TIMER_PROPERTY=SubState=waiting +DETAIL=TIMER_ENABLED_RESULT=enabled|RC=0 +DETAIL=HEALTH_METADATA=BYTES:154|MTIME_NS:1784876406380685648 +DETAIL=HEALTH_MATCH_COUNT=1 +DETAIL=HEALTH_LINE=STATUS=WARN TS=2026-07-24T07:00:05Z TYPE=backup-sla BAD=0 WARN=4 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +DETAIL=JOB_END=homelab_backup_sla_health +DETAIL=JOB_BEGIN=homelab_mail_cloud_restore_drill +DETAIL=JOB_FIELD=id=homelab_mail_cloud_restore_drill +DETAIL=JOB_FIELD=name=homelab-mail-cloud-restore-drill +DETAIL=JOB_FIELD=category=restore +DETAIL=JOB_FIELD=critical=True +DETAIL=JOB_FIELD=status=STALE +DETAIL=JOB_FIELD=freshness=STALE +DETAIL=JOB_FIELD=age_seconds=1247672 +DETAIL=JOB_FIELD=max_age_seconds=864000 +DETAIL=JOB_FIELD=source_kind=health_file +DETAIL=JOB_FIELD=source_status=OK +DETAIL=JOB_FIELD=source_mtime=2026-07-09T20:31:32Z +DETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/homelab_mail_cloud_restore_drill/0-mail-cloud-restore-drill.txt +DETAIL=JOB_FIELD=service=homelab-mail-cloud-restore-drill.service +DETAIL=JOB_FIELD=service_result=success +DETAIL=JOB_FIELD=service_exec_main_status=0 +DETAIL=JOB_FIELD=timer=homelab-mail-cloud-restore-drill.timer +DETAIL=JOB_FIELD=timer_enabled=enabled +DETAIL=JOB_FIELD=timer_active=active +DETAIL=SERVICE_SHOW_RC=0 +DETAIL=SERVICE_PROPERTY=Result=success +DETAIL=SERVICE_PROPERTY=ExecMainCode=0 +DETAIL=SERVICE_PROPERTY=ExecMainStatus=0 +DETAIL=SERVICE_PROPERTY=LoadState=loaded +DETAIL=SERVICE_PROPERTY=ActiveState=inactive +DETAIL=SERVICE_PROPERTY=SubState=dead +DETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/homelab-mail-cloud-restore-drill.service +DETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/homelab-mail-cloud-restore-drill.service.d/10-superseded-noop.conf /etc/systemd/system/homelab-mail-cloud-restore-drill.service.d/90-backup-matrix-refresh.conf +DETAIL=SERVICE_JOURNAL_MATCH_COUNT=8 +DETAIL=SERVICE_JOURNAL=2026-07-13T07:38:36+03:00 pve01 systemd[1]: Starting homelab-mail-cloud-restore-drill.service - Homelab Mail Cloud restore drill... +DETAIL=SERVICE_JOURNAL=2026-07-13T07:38:36+03:00 pve01 homelab-superseded-unit-ok[2401599]: STATUS=OK TS=2026-07-13T04:38:36Z TYPE=superseded-unit UNIT=homelab-mail-cloud-restore-drill.service REASON=superseded_by_safety_net_restore_check ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-13T07:38:36+03:00 pve01 systemd[1]: homelab-mail-cloud-restore-drill.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-13T07:38:36+03:00 pve01 systemd[1]: Finished homelab-mail-cloud-restore-drill.service - Homelab Mail Cloud restore drill. +DETAIL=SERVICE_JOURNAL=2026-07-20T07:25:14+03:00 pve01 systemd[1]: Starting homelab-mail-cloud-restore-drill.service - Homelab Mail Cloud restore drill... +DETAIL=SERVICE_JOURNAL=2026-07-20T07:25:14+03:00 pve01 homelab-superseded-unit-ok[2171079]: STATUS=OK TS=2026-07-20T04:25:14Z TYPE=superseded-unit UNIT=homelab-mail-cloud-restore-drill.service REASON=superseded_by_safety_net_restore_check ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-20T07:25:14+03:00 pve01 systemd[1]: homelab-mail-cloud-restore-drill.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-20T07:25:14+03:00 pve01 systemd[1]: Finished homelab-mail-cloud-restore-drill.service - Homelab Mail Cloud restore drill. +DETAIL=TIMER_SHOW_RC=0 +DETAIL=TIMER_PROPERTY=Unit=homelab-mail-cloud-restore-drill.service +DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Mon 2026-07-27 07:34:52 MSK +DETAIL=TIMER_PROPERTY=LastTriggerUSec=Mon 2026-07-20 07:25:14 MSK +DETAIL=TIMER_PROPERTY=LoadState=loaded +DETAIL=TIMER_PROPERTY=ActiveState=active +DETAIL=TIMER_PROPERTY=SubState=waiting +DETAIL=TIMER_ENABLED_RESULT=enabled|RC=0 +DETAIL=HEALTH_METADATA=BYTES:353|MTIME_NS:1783629092051018609 +DETAIL=HEALTH_MATCH_COUNT=1 +DETAIL=HEALTH_LINE=STATUS=OK TS=2026-07-09T20:31:32Z TYPE=legacy-mail-cloud-restore-drill-superseded MODE=SUPERSEDED_BY_CURRENT_PROOF BAD=0 WARN=0 PROOF=safety_net_control_plane_restore_check_OK ACTION=stale_health_refresh_only REASON=legacy_drill_superseded_but_health_file_mtime_must_stay_fresh REPORT=/var/lib/homelab-failed-unit-legacy-cleanup/local-cleanup-latest.md +DETAIL=JOB_END=homelab_mail_cloud_restore_drill +DETAIL=JOB_BEGIN=netbird_vps_backup +DETAIL=JOB_FIELD=id=netbird_vps_backup +DETAIL=JOB_FIELD=name=netbird-vps-backup +DETAIL=JOB_FIELD=category=application +DETAIL=JOB_FIELD=critical=True +DETAIL=JOB_FIELD=status=FAIL +DETAIL=JOB_FIELD=freshness=STALE +DETAIL=JOB_FIELD=age_seconds=2007299 +DETAIL=JOB_FIELD=max_age_seconds=864000 +DETAIL=JOB_FIELD=source_kind=health_file +DETAIL=JOB_FIELD=source_status=OK +DETAIL=JOB_FIELD=source_mtime=2026-07-01T01:31:05Z +DETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt +DETAIL=JOB_FIELD=service=netbird-vps-backup.service +DETAIL=JOB_FIELD=service_result=success +DETAIL=JOB_FIELD=service_exec_main_status=0 +DETAIL=JOB_FIELD=timer=netbird-vps-backup.timer +DETAIL=JOB_FIELD=timer_enabled=disabled +DETAIL=JOB_FIELD=timer_active=inactive +DETAIL=SERVICE_SHOW_RC=0 +DETAIL=SERVICE_PROPERTY=Result=success +DETAIL=SERVICE_PROPERTY=ExecMainCode=0 +DETAIL=SERVICE_PROPERTY=ExecMainStatus=0 +DETAIL=SERVICE_PROPERTY=LoadState=loaded +DETAIL=SERVICE_PROPERTY=ActiveState=inactive +DETAIL=SERVICE_PROPERTY=SubState=dead +DETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/netbird-vps-backup.service +DETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/netbird-vps-backup.service.d/10-superseded-noop.conf /etc/systemd/system/netbird-vps-backup.service.d/90-backup-matrix-refresh.conf +DETAIL=SERVICE_JOURNAL_MATCH_COUNT=28 +DETAIL=SERVICE_JOURNAL=2026-07-11T04:30:35+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging... +DETAIL=SERVICE_JOURNAL=2026-07-11T04:30:35+03:00 pve01 homelab-superseded-unit-ok[1772638]: STATUS=OK TS=2026-07-11T01:30:35Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-11T04:30:35+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-11T04:30:35+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging. +DETAIL=SERVICE_JOURNAL=2026-07-12T04:39:35+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging... +DETAIL=SERVICE_JOURNAL=2026-07-12T04:39:35+03:00 pve01 homelab-superseded-unit-ok[1789866]: STATUS=OK TS=2026-07-12T01:39:35Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-12T04:39:35+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-12T04:39:35+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging. +DETAIL=SERVICE_JOURNAL=2026-07-13T04:28:00+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging... +DETAIL=SERVICE_JOURNAL=2026-07-13T04:28:00+03:00 pve01 homelab-superseded-unit-ok[1739137]: STATUS=OK TS=2026-07-13T01:28:00Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-13T04:28:00+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-13T04:28:00+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging. +DETAIL=SERVICE_JOURNAL=2026-07-14T04:28:39+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging... +DETAIL=SERVICE_JOURNAL=2026-07-14T04:28:39+03:00 pve01 homelab-superseded-unit-ok[2199841]: STATUS=OK TS=2026-07-14T01:28:39Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-14T04:28:39+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-14T04:28:39+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging. +DETAIL=SERVICE_JOURNAL=2026-07-15T04:39:00+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging... +DETAIL=SERVICE_JOURNAL=2026-07-15T04:39:00+03:00 pve01 homelab-superseded-unit-ok[2872498]: STATUS=OK TS=2026-07-15T01:39:00Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-15T04:39:00+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-15T04:39:00+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging. +DETAIL=SERVICE_JOURNAL=2026-07-16T04:25:14+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging... +DETAIL=SERVICE_JOURNAL=2026-07-16T04:25:14+03:00 pve01 homelab-superseded-unit-ok[3611661]: STATUS=OK TS=2026-07-16T01:25:14Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-16T04:25:14+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-16T04:25:14+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging. +DETAIL=SERVICE_JOURNAL=2026-07-17T04:31:14+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging... +DETAIL=SERVICE_JOURNAL=2026-07-17T04:31:14+03:00 pve01 homelab-superseded-unit-ok[3412653]: STATUS=OK TS=2026-07-17T01:31:14Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-17T04:31:14+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-17T04:31:14+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging. +DETAIL=TIMER_SHOW_RC=0 +DETAIL=TIMER_PROPERTY=Unit=netbird-vps-backup.service +DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime= +DETAIL=TIMER_PROPERTY=LastTriggerUSec= +DETAIL=TIMER_PROPERTY=LoadState=loaded +DETAIL=TIMER_PROPERTY=ActiveState=inactive +DETAIL=TIMER_PROPERTY=SubState=dead +DETAIL=TIMER_ENABLED_RESULT=disabled|RC=1 +DETAIL=HEALTH_METADATA=BYTES:312|MTIME_NS:1782869465433646905 +DETAIL=HEALTH_MATCH_COUNT=7 +DETAIL=HEALTH_LINE=STATUS=OK +DETAIL=HEALTH_LINE=SERVICE=netbird-vps-backup +DETAIL=HEALTH_LINE=TIMESTAMP_UTC=20260701T013054Z +DETAIL=HEALTH_LINE=SNAPSHOT=/mnt/staging/netbird-vps-backups/snapshots/20260701T013054Z +DETAIL=HEALTH_LINE=LATEST_SYMLINK=/mnt/staging/netbird-vps-backups/snapshots/latest +DETAIL=HEALTH_LINE=SIZE=32M +DETAIL=HEALTH_LINE=SHA256=0226dd820cb9ede0ea9e8ea4af6cef656ffa54cfb819048c222a88f4c14d8db9 +DETAIL=JOB_END=netbird_vps_backup +DETAIL=JOB_BEGIN=paperless_restore_dry_run +DETAIL=JOB_FIELD=id=paperless_restore_dry_run +DETAIL=JOB_FIELD=name=paperless-restore-dry-run +DETAIL=JOB_FIELD=category=restore +DETAIL=JOB_FIELD=critical=True +DETAIL=JOB_FIELD=status=STALE +DETAIL=JOB_FIELD=freshness=STALE +DETAIL=JOB_FIELD=age_seconds=1261389 +DETAIL=JOB_FIELD=max_age_seconds=864000 +DETAIL=JOB_FIELD=source_kind=health_file +DETAIL=JOB_FIELD=source_status=OK +DETAIL=JOB_FIELD=source_mtime=2026-07-09T16:42:55Z +DETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/paperless_restore_dry_run/0-paperless-restore.txt +DETAIL=JOB_FIELD=service=paperless-restore-dry-run.service +DETAIL=JOB_FIELD=service_result=success +DETAIL=JOB_FIELD=service_exec_main_status=0 +DETAIL=JOB_FIELD=timer=paperless-restore-dry-run.timer +DETAIL=JOB_FIELD=timer_enabled=enabled +DETAIL=JOB_FIELD=timer_active=active +DETAIL=SERVICE_SHOW_RC=0 +DETAIL=SERVICE_PROPERTY=Result=success +DETAIL=SERVICE_PROPERTY=ExecMainCode=1 +DETAIL=SERVICE_PROPERTY=ExecMainStatus=0 +DETAIL=SERVICE_PROPERTY=LoadState=loaded +DETAIL=SERVICE_PROPERTY=ActiveState=inactive +DETAIL=SERVICE_PROPERTY=SubState=dead +DETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/paperless-restore-dry-run.service +DETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/paperless-restore-dry-run.service.d/10-superseded-noop.conf /etc/systemd/system/paperless-restore-dry-run.service.d/90-backup-matrix-refresh.conf +DETAIL=SERVICE_JOURNAL_MATCH_COUNT=56 +DETAIL=SERVICE_JOURNAL=2026-07-11T10:08:13+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-11T10:08:13+03:00 pve01 homelab-superseded-unit-ok[2988079]: STATUS=OK TS=2026-07-11T07:08:13Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-11T10:08:13+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-11T10:08:13+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=SERVICE_JOURNAL=2026-07-12T10:04:59+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-12T10:04:59+03:00 pve01 homelab-superseded-unit-ok[2757392]: STATUS=OK TS=2026-07-12T07:04:59Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-12T10:04:59+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-12T10:04:59+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=SERVICE_JOURNAL=2026-07-13T10:07:25+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-13T10:07:25+03:00 pve01 homelab-superseded-unit-ok[2886298]: STATUS=OK TS=2026-07-13T07:07:25Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-13T10:07:25+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-13T10:07:25+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=SERVICE_JOURNAL=2026-07-14T09:56:36+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-14T09:56:36+03:00 pve01 homelab-superseded-unit-ok[3367368]: STATUS=OK TS=2026-07-14T06:56:36Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-14T09:56:36+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-14T09:56:36+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=SERVICE_JOURNAL=2026-07-15T10:12:54+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-15T10:12:54+03:00 pve01 homelab-superseded-unit-ok[4062186]: STATUS=OK TS=2026-07-15T07:12:54Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-15T10:12:54+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-15T10:12:54+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=SERVICE_JOURNAL=2026-07-16T11:40:40+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-16T11:40:40+03:00 pve01 homelab-superseded-unit-ok[12176]: STATUS=OK TS=2026-07-16T08:40:40Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-16T11:40:40+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-16T11:40:40+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=SERVICE_JOURNAL=2026-07-17T10:00:14+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-17T10:00:14+03:00 pve01 homelab-superseded-unit-ok[390020]: STATUS=OK TS=2026-07-17T07:00:14Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-17T10:00:14+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-17T10:00:14+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=SERVICE_JOURNAL=2026-07-18T09:57:16+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-18T09:57:16+03:00 pve01 homelab-superseded-unit-ok[1201647]: STATUS=OK TS=2026-07-18T06:57:16Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-18T09:57:16+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-18T09:57:16+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=SERVICE_JOURNAL=2026-07-19T10:09:17+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-19T10:09:17+03:00 pve01 homelab-superseded-unit-ok[1945203]: STATUS=OK TS=2026-07-19T07:09:17Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-19T10:09:17+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-19T10:09:17+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=SERVICE_JOURNAL=2026-07-20T10:04:06+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-20T10:04:06+03:00 pve01 homelab-superseded-unit-ok[2734120]: STATUS=OK TS=2026-07-20T07:04:06Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-20T10:04:06+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-20T10:04:06+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=SERVICE_JOURNAL=2026-07-21T10:02:41+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-21T10:02:41+03:00 pve01 homelab-superseded-unit-ok[3525432]: STATUS=OK TS=2026-07-21T07:02:41Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-21T10:02:41+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-21T10:02:41+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=SERVICE_JOURNAL=2026-07-22T10:11:14+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-22T10:11:14+03:00 pve01 homelab-superseded-unit-ok[4192753]: STATUS=OK TS=2026-07-22T07:11:14Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-22T10:11:14+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-22T10:11:14+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=SERVICE_JOURNAL=2026-07-23T10:12:14+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-23T10:12:14+03:00 pve01 homelab-superseded-unit-ok[752982]: STATUS=OK TS=2026-07-23T07:12:14Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-23T10:12:14+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-23T10:12:14+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=SERVICE_JOURNAL=2026-07-24T10:06:04+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... +DETAIL=SERVICE_JOURNAL=2026-07-24T10:06:04+03:00 pve01 homelab-superseded-unit-ok[3581148]: STATUS=OK TS=2026-07-24T07:06:04Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit +DETAIL=SERVICE_JOURNAL=2026-07-24T10:06:04+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-24T10:06:04+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. +DETAIL=TIMER_SHOW_RC=0 +DETAIL=TIMER_PROPERTY=Unit=paperless-restore-dry-run.service +DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Sat 2026-07-25 10:03:38 MSK +DETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 10:06:04 MSK +DETAIL=TIMER_PROPERTY=LoadState=loaded +DETAIL=TIMER_PROPERTY=ActiveState=active +DETAIL=TIMER_PROPERTY=SubState=waiting +DETAIL=TIMER_ENABLED_RESULT=enabled|RC=0 +DETAIL=HEALTH_METADATA=BYTES:232|MTIME_NS:1783615375842257624 +DETAIL=HEALTH_MATCH_COUNT=1 +DETAIL=HEALTH_LINE=STATUS=OK TS=2026-07-09T16:42:53Z TYPE=paperless-restore BAD=0 APP=paperless JOB=homelab-pve01-app-paperless-20260709T071949Z RESTORE_VERIFIED=true RESTORE_CHECK_APP=paperless JSON=/var/lib/homelab-backup/apps/latest-paperless.json +DETAIL=JOB_END=paperless_restore_dry_run +DETAIL=JOB_BEGIN=vm160_remote_backup +DETAIL=JOB_FIELD=id=vm160_remote_backup +DETAIL=JOB_FIELD=name=vm160-remote-backup +DETAIL=JOB_FIELD=category=virtual-machine +DETAIL=JOB_FIELD=critical=True +DETAIL=JOB_FIELD=status=FAIL +DETAIL=JOB_FIELD=freshness=STALE +DETAIL=JOB_FIELD=age_seconds=286010 +DETAIL=JOB_FIELD=max_age_seconds=172800 +DETAIL=JOB_FIELD=source_kind=health_file +DETAIL=JOB_FIELD=source_status=FAIL +DETAIL=JOB_FIELD=source_mtime=2026-07-20T23:39:14Z +DETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/vm160_remote_backup/0-vm160-remote-backup.txt +DETAIL=JOB_FIELD=service=homelab-vm160-remote-backup.service +DETAIL=JOB_FIELD=service_result=success +DETAIL=JOB_FIELD=service_exec_main_status=0 +DETAIL=JOB_FIELD=timer=homelab-vm160-remote-backup.timer +DETAIL=JOB_FIELD=timer_enabled=disabled +DETAIL=JOB_FIELD=timer_active=inactive +DETAIL=SERVICE_SHOW_RC=0 +DETAIL=SERVICE_PROPERTY=Result=success +DETAIL=SERVICE_PROPERTY=ExecMainCode=0 +DETAIL=SERVICE_PROPERTY=ExecMainStatus=0 +DETAIL=SERVICE_PROPERTY=LoadState=loaded +DETAIL=SERVICE_PROPERTY=ActiveState=inactive +DETAIL=SERVICE_PROPERTY=SubState=dead +DETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/homelab-vm160-remote-backup.service +DETAIL=SERVICE_PROPERTY=DropInPaths= +DETAIL=SERVICE_JOURNAL_MATCH_COUNT=50 +DETAIL=SERVICE_JOURNAL=2026-07-18T01:02:41+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... +DETAIL=SERVICE_JOURNAL=2026-07-18T01:15:43+03:00 pve01 homelab-vm160-remote-backup[3491061]: VM160_TARGET_FINALIZE_STATUS=OK +DETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3530071]: VM160_TARGET_PROMOTE_STATUS=OK +DETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: VM160_REMOTE_BACKUP_STATUS=OK +DETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260717T220242Z +DETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: ARCHIVE=/mnt/staging/vzdump/vm160-pve02-20260717T220242Z/vzdump-qemu-160-20260717T220242Z.vma.zst +DETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: SIZE_BYTES=84993482488 +DETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: SHA256=a9562d8b60d7fd5e7ec73bc43d252cfea80ed12c05b32cca0a73d7bb88532411 +DETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 systemd[1]: Finished homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. +DETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Consumed 2min 44.280s CPU time, 15.8M memory peak. +DETAIL=SERVICE_JOURNAL=2026-07-18T01:38:18+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... +DETAIL=SERVICE_JOURNAL=2026-07-18T01:51:18+03:00 pve01 homelab-vm160-remote-backup[3609200]: VM160_TARGET_FINALIZE_STATUS=OK +DETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3648126]: VM160_TARGET_PROMOTE_STATUS=OK +DETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: VM160_REMOTE_BACKUP_STATUS=OK +DETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z +DETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: ARCHIVE=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z/vzdump-qemu-160-20260717T223819Z.vma.zst +DETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: SIZE_BYTES=84995216465 +DETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: SHA256=257e10821e62e3e2f9318b238a5302a6db601dd597bfa3e0d77aba07f3b85e72 +DETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Deactivated successfully. +DETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 systemd[1]: Finished homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. +DETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Consumed 2min 44.842s CPU time, 16.1M memory peak. +DETAIL=SERVICE_JOURNAL=2026-07-18T04:16:25+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... +DETAIL=SERVICE_JOURNAL=2026-07-18T04:16:27+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=SERVICE_JOURNAL=2026-07-18T04:16:27+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. +DETAIL=SERVICE_JOURNAL=2026-07-18T04:16:27+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. +DETAIL=SERVICE_JOURNAL=2026-07-19T04:19:14+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... +DETAIL=SERVICE_JOURNAL=2026-07-19T04:19:16+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=SERVICE_JOURNAL=2026-07-19T04:19:16+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. +DETAIL=SERVICE_JOURNAL=2026-07-19T04:19:16+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. +DETAIL=SERVICE_JOURNAL=2026-07-20T04:14:14+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... +DETAIL=SERVICE_JOURNAL=2026-07-20T04:14:16+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=SERVICE_JOURNAL=2026-07-20T04:14:16+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. +DETAIL=SERVICE_JOURNAL=2026-07-20T04:14:16+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. +DETAIL=SERVICE_JOURNAL=2026-07-20T19:28:57+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... +DETAIL=SERVICE_JOURNAL=2026-07-20T19:28:59+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=SERVICE_JOURNAL=2026-07-20T19:28:59+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. +DETAIL=SERVICE_JOURNAL=2026-07-20T19:28:59+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. +DETAIL=SERVICE_JOURNAL=2026-07-21T02:14:11+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... +DETAIL=SERVICE_JOURNAL=2026-07-21T02:14:13+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=SERVICE_JOURNAL=2026-07-21T02:14:13+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. +DETAIL=SERVICE_JOURNAL=2026-07-21T02:14:13+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. +DETAIL=SERVICE_JOURNAL=2026-07-21T02:20:20+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... +DETAIL=SERVICE_JOURNAL=2026-07-21T02:20:21+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=SERVICE_JOURNAL=2026-07-21T02:20:21+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. +DETAIL=SERVICE_JOURNAL=2026-07-21T02:20:21+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. +DETAIL=SERVICE_JOURNAL=2026-07-21T02:39:12+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... +DETAIL=SERVICE_JOURNAL=2026-07-21T02:39:14+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=SERVICE_JOURNAL=2026-07-21T02:39:14+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. +DETAIL=SERVICE_JOURNAL=2026-07-21T02:39:14+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. +DETAIL=TIMER_SHOW_RC=0 +DETAIL=TIMER_PROPERTY=Unit=homelab-vm160-remote-backup.service +DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime= +DETAIL=TIMER_PROPERTY=LastTriggerUSec= +DETAIL=TIMER_PROPERTY=LoadState=loaded +DETAIL=TIMER_PROPERTY=ActiveState=inactive +DETAIL=TIMER_PROPERTY=SubState=dead +DETAIL=TIMER_ENABLED_RESULT=disabled|RC=1 +DETAIL=HEALTH_METADATA=BYTES:153|MTIME_NS:1784590754312087060 +DETAIL=HEALTH_MATCH_COUNT=3 +DETAIL=HEALTH_LINE=STATUS=FAIL +DETAIL=HEALTH_LINE=SERVICE=homelab-vm160-remote-backup +DETAIL=HEALTH_LINE=TIMESTAMP_UTC=20260720T233913Z +DETAIL=JOB_END=vm160_remote_backup DETAIL=SECRET_VALUES_OUTPUT=NONE -DETAIL=BACKUP_MATRIX_BAD_JOB_DISCOVERY=PASS +DETAIL=BACKUP_BAD_JOBS_ROOTCAUSE=PASS CHANGES_MADE=NO OUTPUT_END