diff --git a/runtime/history/HOMELAB-BACKUP-NO-LOCAL-INVARIANT-AUDIT-432.json b/runtime/history/HOMELAB-BACKUP-NO-LOCAL-INVARIANT-AUDIT-432.json new file mode 100644 index 0000000..74085bb --- /dev/null +++ b/runtime/history/HOMELAB-BACKUP-NO-LOCAL-INVARIANT-AUDIT-432.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "HOMELAB-BACKUP-NO-LOCAL-INVARIANT-AUDIT-432", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "cluster-audit", + "started_at_utc": "2026-07-24T10:02:43Z", + "finished_at_utc": "2026-07-24T10:02:45Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", + "command_sha256": "3d74c7e2671b37a475e371e668d24c5a96cd5db6b4171efbae25b123278b6883", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "cc2f90a2cae34633773e4468f017e38bc9d8d85abc7d3c929e39aa9f0e9d278b", + "sanitized_output_sha256": "cc2f90a2cae34633773e4468f017e38bc9d8d85abc7d3c929e39aa9f0e9d278b", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "DETAIL=POLICY=DUAL_CLOUD_VERIFY_THEN_DELETE_ALL_LOCAL_PAYLOADS\nDETAIL=CHANGES_PLANNED=NO\nDETAIL=STATE_FILE=/var/lib/homelab-vm-backup/latest-vm-local-dumps-cloud.json\nDETAIL=STATE_FILE_STATUS=READABLE\nDETAIL=STATE_FIELD=$.job_id=homelab-pve01-vm-local-dumps-20260719T070607Z\nDETAIL=STATE_FIELD=$.status=OK\nDETAIL=STATE_FIELD=$.remote_a=pve01-mail-01-crypt:homelab-backups/pve01/vm-local-dumps/homelab-pve01-vm-local-dumps-20260719T070607Z\nDETAIL=STATE_FIELD=$.remote_b=pve01-mail-02-crypt:homelab-backups/pve01/vm-local-dumps/homelab-pve01-vm-local-dumps-20260719T070607Z\nDETAIL=STATE_FIELD=$.chunk_size_bytes=1073741824\nDETAIL=STATE_FIELD=$.local_dumps_deleted=False\nDETAIL=STATE_FILE=/var/lib/homelab-backup-matrix/state.json\nDETAIL=STATE_FILE_STATUS=READABLE\nDETAIL=STATE_FIELD=$.jobs[0].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[0].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[0].status=OK\nDETAIL=STATE_FIELD=$.jobs[1].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[1].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[1].status=OK\nDETAIL=STATE_FIELD=$.jobs[2].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[2].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[2].status=STALE\nDETAIL=STATE_FIELD=$.jobs[3].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[3].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[3].status=OK\nDETAIL=STATE_FIELD=$.jobs[4].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[4].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[4].status=OK\nDETAIL=STATE_FIELD=$.jobs[5].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[5].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[5].status=OK\nDETAIL=STATE_FIELD=$.jobs[6].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[6].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[6].status=OK\nDETAIL=STATE_FIELD=$.jobs[7].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[7].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[7].status=OK\nDETAIL=STATE_FIELD=$.jobs[8].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[8].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[8].status=DISABLED\nDETAIL=STATE_FIELD=$.jobs[9].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[9].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[9].status=OK\nDETAIL=STATE_FIELD=$.jobs[10].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[10].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[10].status=OK\nDETAIL=STATE_FIELD=$.jobs[11].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[11].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[11].status=OK\nDETAIL=STATE_FIELD=$.jobs[12].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[12].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[12].status=OK\nDETAIL=STATE_FIELD=$.jobs[13].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[13].source_status=WARN\nDETAIL=STATE_FIELD=$.jobs[13].status=FAIL\nDETAIL=STATE_FIELD=$.jobs[14].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[14].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[14].status=OK\nDETAIL=STATE_FIELD=$.jobs[15].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[15].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[15].status=OK\nDETAIL=STATE_FIELD=$.jobs[16].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[16].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[16].status=OK\nDETAIL=STATE_FIELD=$.jobs[17].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[17].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[17].status=DISABLED\nDETAIL=STATE_FIELD=$.jobs[18].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[18].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[18].status=OK\nDETAIL=STATE_FIELD=$.jobs[19].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[19].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[19].status=OK\nDETAIL=STATE_FIELD=$.jobs[20].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[20].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[20].status=OK\nDETAIL=STATE_FIELD=$.jobs[21].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[21].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[21].status=OK\nDETAIL=STATE_FIELD=$.jobs[22].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[22].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[22].status=STALE\nDETAIL=STATE_FIELD=$.jobs[23].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[23].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[23].status=DISABLED\nDETAIL=STATE_FIELD=$.jobs[24].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[24].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[24].status=OK\nDETAIL=STATE_FIELD=$.jobs[25].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[25].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[25].status=DISABLED\nDETAIL=STATE_FIELD=$.jobs[26].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[26].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[26].status=OK\nDETAIL=STATE_FIELD=$.jobs[27].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[27].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[27].status=OK\nDETAIL=STATE_FIELD=$.jobs[28].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[28].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[28].status=OK\nDETAIL=STATE_FIELD=$.jobs[29].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[29].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[29].status=OK\nDETAIL=STATE_FIELD=$.jobs[30].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[30].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[30].status=OK\nDETAIL=STATE_FIELD=$.jobs[31].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[31].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[31].status=OK\nDETAIL=STATE_FIELD=$.jobs[32].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[32].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[32].status=OK\nDETAIL=STATE_FIELD=$.jobs[33].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[33].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[33].status=OK\nDETAIL=STATE_FIELD=$.jobs[34].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[34].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[34].status=OK\nDETAIL=STATE_FIELD=$.jobs[35].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[35].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[35].status=FAIL\nDETAIL=STATE_FIELD=$.jobs[36].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[36].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[36].status=DISABLED\nDETAIL=STATE_FIELD=$.jobs[37].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[37].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[37].status=STALE\nDETAIL=STATE_FIELD=$.jobs[38].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[38].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[38].status=OK\nDETAIL=STATE_FIELD=$.jobs[39].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[39].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[39].status=OK\nDETAIL=STATE_FIELD=$.jobs[40].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[40].source_status=FAIL\nDETAIL=STATE_FIELD=$.jobs[40].status=FAIL\nDETAIL=STATE_FIELD=$.matrix_sha256=6b7c5df66c411d1e9630fb336c55007bcd923f1a0bfcc64d687fb1105011eceb\nDETAIL=STATE_FIELD=$.summary.jobs_bad=6\nDETAIL=STATE_FIELD=$.summary.jobs_disabled=5\nDETAIL=STATE_FIELD=$.summary.jobs_ok=30\nDETAIL=STATE_FIELD=$.summary.jobs_total=41\nDETAIL=STATE_FIELD=$.summary.jobs_unmapped=0\nDETAIL=STATE_FIELD=$.summary.status=FAIL\nDETAIL=UNIT_BEGIN=homelab-vm160-remote-backup.service\nDETAIL=UNIT_SHOW_RC=0\nDETAIL=UNIT_PROPERTY=Result=success\nDETAIL=UNIT_PROPERTY=ExecMainStatus=0\nDETAIL=UNIT_PROPERTY=ExecStart={ path=/usr/local/sbin/homelab-vm160-remote-backup ; argv[]=/usr/local/sbin/homelab-vm160-remote-backup ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nDETAIL=UNIT_PROPERTY=LoadState=loaded\nDETAIL=UNIT_PROPERTY=ActiveState=inactive\nDETAIL=UNIT_PROPERTY=SubState=dead\nDETAIL=UNIT_PROPERTY=FragmentPath=/etc/systemd/system/homelab-vm160-remote-backup.service\nDETAIL=SCRIPT_PATH=/usr/local/sbin/homelab-vm160-remote-backup\nDETAIL=SCRIPT_LINE=3:export SYSTEMD_PAGER=cat PAGER=cat SYSTEMD_LESS=FRXMK\nDETAIL=SCRIPT_LINE=8:PARENT=/mnt/staging/vzdump\nDETAIL=SCRIPT_LINE=10:HEALTH=/var/lib/homelab-health/vm160-remote-backup.txt\nDETAIL=SCRIPT_LINE=11:LOCK=/run/lock/homelab-vm160-remote-backup.lock\nDETAIL=SCRIPT_LINE=30:LOG=\"$(mktemp /tmp/vm160-remote-backup.XXXXXX.log)\"\nDETAIL=SCRIPT_LINE=31:cleanup(){\nDETAIL=SCRIPT_LINE=35:ssh \"${SSH[@]}\" \"$TARGET\" \"rm -rf -- '$DIR'\" >/dev/null 2>&1 || true\nDETAIL=SCRIPT_LINE=37:printf 'STATUS=FAIL\\nSERVICE=homelab-vm160-remote-backup\\nTIMESTAMP_UTC=%s\\nVMID=%s\\nTARGET_DIR=%s\\nRC=%s\\n' \"$TS\" \"$VMID\" \"$DIR\" \"$rc\" > \"$tmp\"\nDETAIL=SCRIPT_LINE=41:rm -f \"$LOG\"\nDETAIL=SCRIPT_LINE=44:trap cleanup EXIT\nDETAIL=SCRIPT_LINE=45:REMOTE_FREE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"df -B1 --output=avail '$PARENT' | awk 'NR==2{print \\$1}'\")\"\nDETAIL=SCRIPT_LINE=53:test \"$REMOTE_FREE\" -ge \"$REQUIRED\"\nDETAIL=SCRIPT_LINE=55:ssh \"${SSH[@]}\" \"$SOURCE\" \"vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7\" 2>\"$LOG\" | ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$PART'\"\nDETAIL=SCRIPT_LINE=57:ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$DIR/vzdump.log'\" < \"$LOG\"\nDETAIL=SCRIPT_LINE=59:SHA=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"sha256sum '$FINAL' | awk '{print \\$1}'\")\"\nDETAIL=SCRIPT_LINE=62:printf 'STATUS=OK\\nSERVICE=homelab-vm160-remote-backup\\nTIMESTAMP_UTC=%s\\nVMID=%s\\nSOURCE_NODE=pve02\\nTARGET_NODE=pve03\\nTARGET_DIR=%s\\nARCHIVE=%s\\nSIZE_BYTES=%s\\nSHA256=%s\\nREMOTE_RETENTION_GENERATIONS=1\\n' \"$TS\" \"$VMID\" \"$DIR\" \"$FINAL\" \"$SIZE\" \"$SHA\" > \"$tmp\"\nDETAIL=SCRIPT_LINE=66:rm -f \"$LOG\"\nDETAIL=SCRIPT_LINE=67:printf 'VM160_REMOTE_BACKUP_STATUS=OK\\nTARGET_DIR=%s\\nARCHIVE=%s\\nSIZE_BYTES=%s\\nSHA256=%s\\n' \"$DIR\" \"$FINAL\" \"$SIZE\" \"$SHA\"\nDETAIL=UNIT_END=homelab-vm160-remote-backup.service\nDETAIL=UNIT_BEGIN=homelab-vm-local-dumps-2cloud.service\nDETAIL=UNIT_SHOW_RC=0\nDETAIL=UNIT_PROPERTY=Result=success\nDETAIL=UNIT_PROPERTY=ExecMainStatus=0\nDETAIL=UNIT_PROPERTY=ExecStart={ path=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; argv[]=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nDETAIL=UNIT_PROPERTY=LoadState=loaded\nDETAIL=UNIT_PROPERTY=ActiveState=inactive\nDETAIL=UNIT_PROPERTY=SubState=dead\nDETAIL=UNIT_PROPERTY=FragmentPath=/etc/systemd/system/homelab-vm-local-dumps-2cloud.service\nDETAIL=SCRIPT_PATH=/usr/local/sbin/homelab-vm-local-dumps-2cloud\nDETAIL=SCRIPT_LINE=4:OUT=\"/var/lib/homelab-health/vm-local-dumps-cloud.txt\"\nDETAIL=SCRIPT_LINE=6:LOG=\"$ROOT/vm-local-dumps-2cloud-latest.log\"\nDETAIL=SCRIPT_LINE=7:VERIFY_ROOT=\"/var/tmp/homelab-vm-local-dumps-verify\"\nDETAIL=SCRIPT_LINE=12:mkdir -p /var/lib/homelab-health \"$ROOT\" \"$VERIFY_ROOT\" \"$CHUNK_ROOT\"\nDETAIL=SCRIPT_LINE=55:SHA=\"$(sha256sum \"$F\" | cut -d\" \" -f1)\"\nDETAIL=SCRIPT_LINE=64:rm -rf \"$CDIR\"\nDETAIL=SCRIPT_LINE=69:(cd \"$CDIR\" && sha256sum part-* > chunks.sha256)\nDETAIL=SCRIPT_LINE=92:\"sha256\":p[3],\nDETAIL=SCRIPT_LINE=112:DEST_A=\"${REMOTE_A%/}/${NODE}/vm-local-dumps/${JOB_ID}\"\nDETAIL=SCRIPT_LINE=113:DEST_B=\"${REMOTE_B%/}/${NODE}/vm-local-dumps/${JOB_ID}\"\nDETAIL=SCRIPT_LINE=115:upload_one_dest() {\nDETAIL=SCRIPT_LINE=117:rclone mkdir \"$DEST/files\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SCRIPT_LINE=121:echo \"UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts\" >> \"$LOG\"\nDETAIL=SCRIPT_LINE=122:rclone mkdir \"$DEST/files/$bn.chunks\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SCRIPT_LINE=126:rclone copyto \"$part\" \"$DEST/files/$bn.chunks/$pn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SCRIPT_LINE=128:rclone copyto \"$CDIR/chunks.sha256\" \"$DEST/files/$bn.chunks/chunks.sha256\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SCRIPT_LINE=130:echo \"UPLOAD_DIRECT dest=$DEST file=$bn size=$size\" >> \"$LOG\"\nDETAIL=SCRIPT_LINE=131:rclone copyto \"$path\" \"$DEST/files/$bn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SCRIPT_LINE=135:rclone copyto \"$CHKS\" \"$DEST/checksums.txt\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SCRIPT_LINE=136:rclone copyto \"$FILES_TSV\" \"$DEST/files.tsv\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SCRIPT_LINE=137:rclone copyto \"$MAN\" \"$DEST/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SCRIPT_LINE=140:verify_one_dest() {\nDETAIL=SCRIPT_LINE=143:VBASE=\"$VERIFY_ROOT/$JOB_ID/$LABEL\"\nDETAIL=SCRIPT_LINE=144:rm -rf \"$VBASE\"\nDETAIL=SCRIPT_LINE=146:rclone copyto \"$DEST/manifest.json\" \"$VBASE/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SCRIPT_LINE=153:echo \"VERIFY_CHUNKED label=$LABEL file=$bn parts=$parts\" >> \"$LOG\"\nDETAIL=SCRIPT_LINE=154:rclone copy \"$DEST/files/$bn.chunks\" \"$VDIR\" --filter \"+ part-*\" --filter \"+ chunks.sha256\" --filter \"- *\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SCRIPT_LINE=155:(cd \"$VDIR\" && sha256sum -c chunks.sha256) >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SCRIPT_LINE=156:CALC=\"$(find \"$VDIR\" -maxdepth 1 -type f -name 'part-*' | sort | xargs cat | sha256sum | cut -d\" \" -f1)\"\nDETAIL=SCRIPT_LINE=163:rm -rf \"$VDIR\"\nDETAIL=SCRIPT_LINE=165:echo \"VERIFY_DIRECT label=$LABEL file=$bn\" >> \"$LOG\"\nDETAIL=SCRIPT_LINE=166:rclone copyto \"$DEST/files/$bn\" \"$VBASE/$bn\" --transfers 1 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SCRIPT_LINE=167:printf \"%s %s\\n\" \"$sha\" \"$VBASE/$bn\" | sha256sum -c - >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SCRIPT_LINE=168:rm -f \"$VBASE/$bn\"\nDETAIL=SCRIPT_LINE=171:rm -rf \"$VBASE\"\nDETAIL=SCRIPT_LINE=175:upload_one_dest \"$DEST_A\"\nDETAIL=SCRIPT_LINE=176:upload_one_dest \"$DEST_B\"\nDETAIL=SCRIPT_LINE=180:verify_one_dest \"A\" \"$DEST_A\"\nDETAIL=SCRIPT_LINE=181:verify_one_dest \"B\" \"$DEST_B\"\nDETAIL=SCRIPT_LINE=184:rm -rf \"$VERIFY_ROOT/$JOB_ID\"\nDETAIL=SCRIPT_LINE=185:rm -rf \"$CHUNK_ROOT/$JOB_ID\"\nDETAIL=SCRIPT_LINE=193:--arg remote_a \"$DEST_A\" \\\nDETAIL=SCRIPT_LINE=194:--arg remote_b \"$DEST_B\" \\\nDETAIL=SCRIPT_LINE=200:\"{job_id:\\$job_id,status:\\$status,node:\\$node,remote_a:\\$remote_a,remote_b:\\$remote_b,count:\\$count,total_bytes:\\$total_bytes,chunked_count:\\$chunked_count,chunk_size_bytes:\\$chunk_size_bytes,log:\\$log,local_dumps_deleted:false}\" \\\nDETAIL=SCRIPT_LINE=201:> \"$ROOT/latest-vm-local-dumps-cloud.json\"\nDETAIL=SCRIPT_LINE=203:echo \"STATUS=$STATUS TS=$TS TYPE=vm-local-dumps-cloud BAD=$BAD COUNT=$COUNT CHUNKED=$CHUNKED TOTAL_BYTES=$TOTAL JOB=$JOB_ID REMOTE_A=$DEST_A REMOTE_B=$DEST_B LOG=$LOG\" > \"$OUT\"\nDETAIL=SCRIPT_LINE=204:cat \"$OUT\"\nDETAIL=UNIT_END=homelab-vm-local-dumps-2cloud.service\nDETAIL=PAYLOAD_HOST_BEGIN=pve01\nDETAIL=PAYLOAD_ROOT=pve01:/var/lib/vz/dump\nDETAIL=PAYLOAD_ROOT_STATUS=PRESENT\nDETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst|BYTES=329541843|AGE_SECONDS=293553\nDETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-110-2026_07_24-03_30_02.tar.zst|BYTES=339140622|AGE_SECONDS=34352\nDETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-112-2026_07_22-03_30_09.tar.zst|BYTES=222794054|AGE_SECONDS=207149\nDETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-110-2026_07_22-03_30_01.tar.zst|BYTES=332951369|AGE_SECONDS=207154\nDETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst|BYTES=7021952392|AGE_SECONDS=293496\nDETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-qemu-150-2026_07_24-03_30_16.vma.zst|BYTES=7016624549|AGE_SECONDS=34298\nDETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-qemu-150-2026_07_22-03_30_14.vma.zst|BYTES=7052176114|AGE_SECONDS=207100\nDETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst|BYTES=222777732|AGE_SECONDS=293547\nDETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-112-2026_07_24-03_30_11.tar.zst|BYTES=222859457|AGE_SECONDS=34347\nDETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-qemu-150-2026_07_23-03_30_16.vma.zst|BYTES=7024165999|AGE_SECONDS=120695\nDETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-110-2026_07_23-03_30_02.tar.zst|BYTES=336245409|AGE_SECONDS=120752\nDETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-112-2026_07_23-03_30_11.tar.zst|BYTES=222808025|AGE_SECONDS=120747\nDETAIL=PAYLOAD_ROOT=pve01:/mnt/staging/vzdump\nDETAIL=PAYLOAD_ROOT_STATUS=PRESENT\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260705T004739Z/pve03/vzdump-qemu-130-2026_07_05-03_49_34.vma.zst|BYTES=11242736440|AGE_SECONDS=1674545\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260705T004739Z/pve02/vzdump-lxc-111-2026_07_05-03_48_52.tar.zst|BYTES=255533195|AGE_SECONDS=1674805\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260705T004739Z/pve02/vzdump-lxc-113-2026_07_05-03_49_19.tar.zst|BYTES=220380760|AGE_SECONDS=1674791\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260705T004739Z/pve01/vzdump-qemu-150-2026_07_05-03_47_52.vma.zst|BYTES=7004920660|AGE_SECONDS=1674842\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260705T004739Z/pve01/vzdump-lxc-112-2026_07_05-03_47_46.tar.zst|BYTES=222468077|AGE_SECONDS=1674891\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260705T004739Z/pve01/vzdump-lxc-110-2026_07_05-03_47_40.tar.zst|BYTES=269221740|AGE_SECONDS=1674897\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260628T004739Z/pve03/vzdump-qemu-130-2026_06_28-03_49_15.vma.zst|BYTES=27024990491|AGE_SECONDS=2279192\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260628T004739Z/pve02/vzdump-lxc-113-2026_06_28-03_49_00.tar.zst|BYTES=220326595|AGE_SECONDS=2279610\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260628T004739Z/pve02/vzdump-lxc-111-2026_06_28-03_48_46.tar.zst|BYTES=242676818|AGE_SECONDS=2279624\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260628T004739Z/pve01/vzdump-lxc-112-2026_06_28-03_47_46.tar.zst|BYTES=222365246|AGE_SECONDS=2279692\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260628T004739Z/pve01/vzdump-lxc-110-2026_06_28-03_47_40.tar.zst|BYTES=253314990|AGE_SECONDS=2279698\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260628T004739Z/pve01/vzdump-qemu-150-2026_06_28-03_47_51.vma.zst|BYTES=7003671925|AGE_SECONDS=2279646\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260719T004537Z/pve03/vzdump-qemu-130-2026_07_19-03_48_01.vma.zst|BYTES=11043726130|AGE_SECONDS=465037\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260719T004537Z/pve02/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst|BYTES=220049041|AGE_SECONDS=465284\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260719T004537Z/pve02/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst|BYTES=284443608|AGE_SECONDS=465301\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260719T004537Z/pve01/vzdump-qemu-150-2026_07_19-03_45_58.vma.zst|BYTES=7008027143|AGE_SECONDS=465328\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260719T004537Z/pve01/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst|BYTES=323668231|AGE_SECONDS=465417\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260719T004537Z/pve01/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst|BYTES=222745965|AGE_SECONDS=465406\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260712T004231Z/pve03/vzdump-qemu-130-2026_07_12-03_44_28.vma.zst|BYTES=10813915936|AGE_SECONDS=1070055\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260712T004231Z/pve02/vzdump-lxc-111-2026_07_12-03_43_45.tar.zst|BYTES=270924730|AGE_SECONDS=1070317\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260712T004231Z/pve02/vzdump-lxc-113-2026_07_12-03_44_07.tar.zst|BYTES=220482178|AGE_SECONDS=1070297\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260712T004231Z/pve01/vzdump-lxc-112-2026_07_12-03_42_39.tar.zst|BYTES=222566793|AGE_SECONDS=1070398\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260712T004231Z/pve01/vzdump-qemu-150-2026_07_12-03_42_45.vma.zst|BYTES=7014397787|AGE_SECONDS=1070350\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260712T004231Z/pve01/vzdump-lxc-110-2026_07_12-03_42_31.tar.zst|BYTES=302840996|AGE_SECONDS=1070404\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/full-manual-backup-20260615T092720Z/pve03/vzdump-qemu-130-2026_06_15-12_27_45.vma.zst|BYTES=829658873|AGE_SECONDS=3371676\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/full-manual-backup-20260615T092720Z/pve02/vzdump-lxc-113-2026_06_15-12_27_32.tar.zst|BYTES=220018661|AGE_SECONDS=3371701\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/full-manual-backup-20260615T092720Z/pve02/vzdump-lxc-111-2026_06_15-12_27_22.tar.zst|BYTES=228398861|AGE_SECONDS=3371712\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/full-manual-backup-20260615T092720Z/pve01/vzdump-qemu-150-2026_06_15-12_19_05.vma.zst|BYTES=6957596210|AGE_SECONDS=3372173\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/full-manual-backup-20260615T092720Z/pve01/vzdump-lxc-112-2026_06_15-12_23_56.tar.zst|BYTES=222110027|AGE_SECONDS=3371924\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/full-manual-backup-20260615T092720Z/pve01/vzdump-lxc-110-2026_06_15-12_23_52.tar.zst|BYTES=230562036|AGE_SECONDS=3371928\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/vm170-vm171/vzdump-qemu-170-2026_07_24-02_15_07.vma.zst|BYTES=11828704880|AGE_SECONDS=37618\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/vm170-vm171/vzdump-qemu-171-2026_07_23-02_36_37.vma.zst|BYTES=2157597134|AGE_SECONDS=122728\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/vm170-vm171/vzdump-qemu-170-2026_07_23-02_15_17.vma.zst|BYTES=11842330422|AGE_SECONDS=124008\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/vm170-vm171/vzdump-qemu-171-2026_07_24-02_36_27.vma.zst|BYTES=2158869638|AGE_SECONDS=36338\nDETAIL=PAYLOAD_ROOT=pve01:/var/lib/homelab-vm-backup/staging\nDETAIL=PAYLOAD_ROOT_STATUS=ABSENT\nDETAIL=PAYLOAD_HOST_SUMMARY=pve01|COUNT=46|BYTES=159150279782\nDETAIL=PAYLOAD_HOST_END=pve01\nDETAIL=PAYLOAD_HOST_BEGIN=pve02\nDETAIL=PAYLOAD_ROOT=pve02:/var/lib/vz/dump\nDETAIL=PAYLOAD_ROOT_STATUS=PRESENT\nDETAIL=LOCAL_PAYLOAD=pve02:/var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.vma.zst::997503422::1782865871.1700059310/var/lib/vz/dump/vzdump-lxc-111-2026_07_24-03_30_03.tar.zst::293632136::1784853018.7398517980/var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.vma.zst::868048369::1782779468.4903256780/var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.tar.zst::220080970::1784593833.3910787180/var/lib/vz/dump/vzdump-lxc-111-2026_07_22-03_30_01.tar.zst::290059742::1784680222.8748248510/var/lib/vz/dump/vzdump-lxc-111-2026_07_23-03_30_02.tar.zst::292185437::1784766624.6563558990/var/lib/vz/dump/vzdump-lxc-113-2026_07_23-03_30_24.tar.zst::220108944::1784766637.5724249240/var/lib/vz/dump/vzdump-lxc-113-2026_07_22-03_30_23.tar.zst::220095608::1784680240.6809207660/var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.tar.zst::288349157::1784593821.2480085730/var/lib/vz/dump/vzdump-lxc-113-2026_07_24-03_30_19.tar.zst|BYTES=220234127|AGE_SECONDS=34333\nDETAIL=PAYLOAD_ROOT=pve02:/mnt/staging/vzdump\nDETAIL=PAYLOAD_ROOT_STATUS=PRESENT\nDETAIL=LOCAL_PAYLOAD=pve02:/mnt/staging/vzdump/auto-pve02-20260615T121549Z/vzdump-lxc-113-2026_06_15-15_16_01.tar.zst::220085886::1781525770.6722751820/mnt/staging/vzdump/auto-pve02-20260615T121549Z/vzdump-lxc-111-2026_06_15-15_15_50.tar.zst::228433321::1781525759.9832231610/mnt/staging/vzdump/auto-pve02-20260615T184325Z/vzdump-lxc-111-2026_06_15-21_43_27.tar.zst::228695245::1781549016.3828865780/mnt/staging/vzdump/auto-pve02-20260615T184325Z/vzdump-lxc-113-2026_06_15-21_43_37.tar.zst::220088590::1781549027.0039397060/mnt/staging/vzdump/auto-pve02-20260615T145425Z/vzdump-lxc-111-2026_06_15-17_54_26.tar.zst::228364545::1781535276.0106286580/mnt/staging/vzdump/auto-pve02-20260615T145425Z/vzdump-lxc-113-2026_06_15-17_54_37.tar.zst::220087147::1781535286.6446806260/mnt/staging/vzdump/auto-pve02-20260615T125928Z/vzdump-lxc-111-2026_06_15-15_59_30.tar.zst::228433824::1781528379.7579730690/mnt/staging/vzdump/auto-pve02-20260615T125928Z/vzdump-lxc-113-2026_06_15-15_59_40.tar.zst::220086313::1781528390.6340260000/mnt/staging/vzdump/auto-pve02-20260628T004845Z/vzdump-lxc-113-2026_06_28-03_49_00.tar.zst::220326595::1782607753.1428569490/mnt/staging/vzdump/auto-pve02-20260628T004845Z/vzdump-lxc-111-2026_06_28-03_48_46.tar.zst::242676818::1782607739.4867871320/mnt/staging/vzdump/auto-pve02-20260615T201446Z/vzdump-lxc-111-2026_06_15-23_14_48.tar.zst::228706014::1781554497.6364731750/mnt/staging/vzdump/auto-pve02-20260615T201446Z/vzdump-lxc-113-2026_06_15-23_14_58.tar.zst::220089008::1781554508.5315278350/mnt/staging/vzdump/auto-pve02-20260705T004851Z/vzdump-lxc-111-2026_07_05-03_48_52.tar.zst::255533195::1783212558.3825236020/mnt/staging/vzdump/auto-pve02-20260705T004851Z/vzdump-lxc-113-2026_07_05-03_49_19.tar.zst::220380760::1783212571.8805933550/mnt/staging/vzdump/auto-pve02-20260719T004725Z/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst::284443608::1784422061.8525503200/mnt/staging/vzdump/auto-pve02-20260719T004725Z/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst::220049041::1784422078.9256407960/mnt/staging/vzdump/auto-pve02-20260712T004344Z/vzdump-lxc-111-2026_07_12-03_43_45.tar.zst::270924730::1783817046.4081420260/mnt/staging/vzdump/auto-pve02-20260712T004344Z/vzdump-lxc-113-2026_07_12-03_44_07.tar.zst::220482178::1783817065.7622435280/mnt/staging/vzdump/auto-pve02-20260621T004841Z/vzdump-lxc-113-2026_06_21-03_48_54.tar.zst::220165475::1782002947.0683211180/mnt/staging/vzdump/auto-pve02-20260621T004841Z/vzdump-lxc-111-2026_06_21-03_48_42.tar.zst::232731208::1782002933.6692546410/mnt/staging/vzdump/auto-pve02-20260615T161514Z/vzdump-lxc-111-2026_06_15-19_15_16.tar.zst::228462340::1781540125.7713503770/mnt/staging/vzdump/auto-pve02-20260615T161514Z/vzdump-lxc-113-2026_06_15-19_15_26.tar.zst::220087434::1781540136.7614046640/mnt/staging/vzdump/manual-pve02-20260615T092720Z/vzdump-lxc-113-2026_06_15-12_27_32.tar.zst::220018661::1781515662.0924756330/mnt/staging/vzdump/manual-pve02-20260615T092720Z/vzdump-lxc-111-2026_06_15-12_27_22.tar.zst::228398861::1781515651.4234214190/mnt/staging/vzdump/auto-pve02-20260616T023537Z/vzdump-lxc-111-2026_06_16-05_35_38.tar.zst::228693413::1781577348.0265431350/mnt/staging/vzdump/auto-pve02-20260616T023537Z/vzdump-lxc-113-2026_06_16-05_35_49.tar.zst::220093717::1781577358.9785981020/mnt/staging/vzdump/auto-pve02-20260615T095831Z/vzdump-lxc-111-2026_06_15-12_58_32.tar.zst::228398872::1781517522.2069308430/mnt/staging/vzdump/auto-pve02-20260615T095831Z/vzdump-lxc-113-2026_06_15-12_58_43.tar.zst|BYTES=220019157|AGE_SECONDS=3369831\nDETAIL=PAYLOAD_HOST_SUMMARY=pve02|COUNT=2|BYTES=440253284\nDETAIL=PAYLOAD_HOST_END=pve02\nDETAIL=PAYLOAD_HOST_BEGIN=pve03\nDETAIL=PAYLOAD_ROOT=pve03:/var/lib/vz/dump\nDETAIL=PAYLOAD_ROOT_STATUS=PRESENT\nDETAIL=LOCAL_PAYLOAD=pve03:/var/lib/vz/dump/vzdump-qemu-130-2026_07_23-03_30_03.vma.zst::11139455731::1784766836.0161157970/var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.vma.zst::11043834894::1784594034.3725716230/var/lib/vz/dump/vzdump-qemu-130-2026_07_22-03_30_02.vma.zst::11072513783::1784680438.8961071480/var/lib/vz/dump/vzdump-qemu-190-2026_07_23-03_33_56.vma.zst::1268419395::1784766881.2902288360/var/lib/vz/dump/vzdump-qemu-190-2026_07_24-03_34_00.vma.zst::1278418128::1784853285.0606354470/var/lib/vz/dump/vzdump-qemu-130-2026_07_24-03_30_02.vma.zst::11170433004::1784853240.1424086410/var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.vma.zst::1137072350::1784594077.6276811360/var/lib/vz/dump/vzdump-qemu-190-2026_07_22-03_33_59.vma.zst|BYTES=1177407443|AGE_SECONDS=206880\nDETAIL=PAYLOAD_ROOT=pve03:/mnt/staging/vzdump\nDETAIL=PAYLOAD_ROOT_STATUS=PRESENT\nDETAIL=LOCAL_PAYLOAD=pve03:/mnt/staging/vzdump/auto-pve03-20260719T004800Z/vzdump-qemu-130-2026_07_19-03_48_01.vma.zst::11043726130::1784422326.4811235470/mnt/staging/vzdump/auto-pve03-20260712T004427Z/vzdump-qemu-130-2026_07_12-03_44_28.vma.zst::10813915936::1783817308.4882503530/mnt/staging/vzdump/vm160-pve02-20260717T223819Z/vzdump-qemu-160-20260717T223819Z.vma.zst|BYTES=84995216465|AGE_SECONDS=559051\nDETAIL=PAYLOAD_HOST_SUMMARY=pve03|COUNT=2|BYTES=86172623908\nDETAIL=PAYLOAD_HOST_END=pve03\nDETAIL=POLICY_SUCCESS_REQUIRES=CLOUD1_VERIFIED,CLOUD2_VERIFIED,LOCAL_PAYLOAD_COUNT_ZERO\nDETAIL=SECRET_VALUES_OUTPUT=NONE\nDETAIL=BACKUP_NO_LOCAL_INVARIANT_AUDIT=PASS\nCHANGES_MADE=NO\n" +} diff --git a/runtime/history/HOMELAB-BACKUP-NO-LOCAL-INVARIANT-AUDIT-432.txt b/runtime/history/HOMELAB-BACKUP-NO-LOCAL-INVARIANT-AUDIT-432.txt new file mode 100644 index 0000000..446552e --- /dev/null +++ b/runtime/history/HOMELAB-BACKUP-NO-LOCAL-INVARIANT-AUDIT-432.txt @@ -0,0 +1,330 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=HOMELAB-BACKUP-NO-LOCAL-INVARIANT-AUDIT-432 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=cluster-audit +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 +COMMAND_SHA256=3d74c7e2671b37a475e371e668d24c5a96cd5db6b4171efbae25b123278b6883 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=false +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=cc2f90a2cae34633773e4468f017e38bc9d8d85abc7d3c929e39aa9f0e9d278b +SANITIZED_OUTPUT_SHA256=cc2f90a2cae34633773e4468f017e38bc9d8d85abc7d3c929e39aa9f0e9d278b +OUTPUT_BEGIN +DETAIL=POLICY=DUAL_CLOUD_VERIFY_THEN_DELETE_ALL_LOCAL_PAYLOADS +DETAIL=CHANGES_PLANNED=NO +DETAIL=STATE_FILE=/var/lib/homelab-vm-backup/latest-vm-local-dumps-cloud.json +DETAIL=STATE_FILE_STATUS=READABLE +DETAIL=STATE_FIELD=$.job_id=homelab-pve01-vm-local-dumps-20260719T070607Z +DETAIL=STATE_FIELD=$.status=OK +DETAIL=STATE_FIELD=$.remote_a=pve01-mail-01-crypt:homelab-backups/pve01/vm-local-dumps/homelab-pve01-vm-local-dumps-20260719T070607Z +DETAIL=STATE_FIELD=$.remote_b=pve01-mail-02-crypt:homelab-backups/pve01/vm-local-dumps/homelab-pve01-vm-local-dumps-20260719T070607Z +DETAIL=STATE_FIELD=$.chunk_size_bytes=1073741824 +DETAIL=STATE_FIELD=$.local_dumps_deleted=False +DETAIL=STATE_FILE=/var/lib/homelab-backup-matrix/state.json +DETAIL=STATE_FILE_STATUS=READABLE +DETAIL=STATE_FIELD=$.jobs[0].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[0].source_status=OK +DETAIL=STATE_FIELD=$.jobs[0].status=OK +DETAIL=STATE_FIELD=$.jobs[1].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[1].source_status=OK +DETAIL=STATE_FIELD=$.jobs[1].status=OK +DETAIL=STATE_FIELD=$.jobs[2].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[2].source_status=OK +DETAIL=STATE_FIELD=$.jobs[2].status=STALE +DETAIL=STATE_FIELD=$.jobs[3].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[3].source_status=OK +DETAIL=STATE_FIELD=$.jobs[3].status=OK +DETAIL=STATE_FIELD=$.jobs[4].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[4].source_status=OK +DETAIL=STATE_FIELD=$.jobs[4].status=OK +DETAIL=STATE_FIELD=$.jobs[5].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[5].source_status=OK +DETAIL=STATE_FIELD=$.jobs[5].status=OK +DETAIL=STATE_FIELD=$.jobs[6].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[6].source_status=OK +DETAIL=STATE_FIELD=$.jobs[6].status=OK +DETAIL=STATE_FIELD=$.jobs[7].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[7].source_status=OK +DETAIL=STATE_FIELD=$.jobs[7].status=OK +DETAIL=STATE_FIELD=$.jobs[8].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[8].source_status=OK +DETAIL=STATE_FIELD=$.jobs[8].status=DISABLED +DETAIL=STATE_FIELD=$.jobs[9].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[9].source_status=OK +DETAIL=STATE_FIELD=$.jobs[9].status=OK +DETAIL=STATE_FIELD=$.jobs[10].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[10].source_status=OK +DETAIL=STATE_FIELD=$.jobs[10].status=OK +DETAIL=STATE_FIELD=$.jobs[11].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[11].source_status=OK +DETAIL=STATE_FIELD=$.jobs[11].status=OK +DETAIL=STATE_FIELD=$.jobs[12].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[12].source_status=OK +DETAIL=STATE_FIELD=$.jobs[12].status=OK +DETAIL=STATE_FIELD=$.jobs[13].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[13].source_status=WARN +DETAIL=STATE_FIELD=$.jobs[13].status=FAIL +DETAIL=STATE_FIELD=$.jobs[14].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[14].source_status=OK +DETAIL=STATE_FIELD=$.jobs[14].status=OK +DETAIL=STATE_FIELD=$.jobs[15].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[15].source_status=OK +DETAIL=STATE_FIELD=$.jobs[15].status=OK +DETAIL=STATE_FIELD=$.jobs[16].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[16].source_status=OK +DETAIL=STATE_FIELD=$.jobs[16].status=OK +DETAIL=STATE_FIELD=$.jobs[17].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[17].source_status=OK +DETAIL=STATE_FIELD=$.jobs[17].status=DISABLED +DETAIL=STATE_FIELD=$.jobs[18].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[18].source_status=OK +DETAIL=STATE_FIELD=$.jobs[18].status=OK +DETAIL=STATE_FIELD=$.jobs[19].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[19].source_status=OK +DETAIL=STATE_FIELD=$.jobs[19].status=OK +DETAIL=STATE_FIELD=$.jobs[20].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[20].source_status=OK +DETAIL=STATE_FIELD=$.jobs[20].status=OK +DETAIL=STATE_FIELD=$.jobs[21].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[21].source_status=OK +DETAIL=STATE_FIELD=$.jobs[21].status=OK +DETAIL=STATE_FIELD=$.jobs[22].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[22].source_status=OK +DETAIL=STATE_FIELD=$.jobs[22].status=STALE +DETAIL=STATE_FIELD=$.jobs[23].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[23].source_status=OK +DETAIL=STATE_FIELD=$.jobs[23].status=DISABLED +DETAIL=STATE_FIELD=$.jobs[24].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[24].source_status=OK +DETAIL=STATE_FIELD=$.jobs[24].status=OK +DETAIL=STATE_FIELD=$.jobs[25].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[25].source_status=OK +DETAIL=STATE_FIELD=$.jobs[25].status=DISABLED +DETAIL=STATE_FIELD=$.jobs[26].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[26].source_status=OK +DETAIL=STATE_FIELD=$.jobs[26].status=OK +DETAIL=STATE_FIELD=$.jobs[27].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[27].source_status=OK +DETAIL=STATE_FIELD=$.jobs[27].status=OK +DETAIL=STATE_FIELD=$.jobs[28].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[28].source_status=OK +DETAIL=STATE_FIELD=$.jobs[28].status=OK +DETAIL=STATE_FIELD=$.jobs[29].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[29].source_status=OK +DETAIL=STATE_FIELD=$.jobs[29].status=OK +DETAIL=STATE_FIELD=$.jobs[30].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[30].source_status=OK +DETAIL=STATE_FIELD=$.jobs[30].status=OK +DETAIL=STATE_FIELD=$.jobs[31].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[31].source_status=OK +DETAIL=STATE_FIELD=$.jobs[31].status=OK +DETAIL=STATE_FIELD=$.jobs[32].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[32].source_status=OK +DETAIL=STATE_FIELD=$.jobs[32].status=OK +DETAIL=STATE_FIELD=$.jobs[33].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[33].source_status=OK +DETAIL=STATE_FIELD=$.jobs[33].status=OK +DETAIL=STATE_FIELD=$.jobs[34].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[34].source_status=OK +DETAIL=STATE_FIELD=$.jobs[34].status=OK +DETAIL=STATE_FIELD=$.jobs[35].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[35].source_status=OK +DETAIL=STATE_FIELD=$.jobs[35].status=FAIL +DETAIL=STATE_FIELD=$.jobs[36].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[36].source_status=OK +DETAIL=STATE_FIELD=$.jobs[36].status=DISABLED +DETAIL=STATE_FIELD=$.jobs[37].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[37].source_status=OK +DETAIL=STATE_FIELD=$.jobs[37].status=STALE +DETAIL=STATE_FIELD=$.jobs[38].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[38].source_status=OK +DETAIL=STATE_FIELD=$.jobs[38].status=OK +DETAIL=STATE_FIELD=$.jobs[39].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[39].source_status=OK +DETAIL=STATE_FIELD=$.jobs[39].status=OK +DETAIL=STATE_FIELD=$.jobs[40].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[40].source_status=FAIL +DETAIL=STATE_FIELD=$.jobs[40].status=FAIL +DETAIL=STATE_FIELD=$.matrix_sha256=6b7c5df66c411d1e9630fb336c55007bcd923f1a0bfcc64d687fb1105011eceb +DETAIL=STATE_FIELD=$.summary.jobs_bad=6 +DETAIL=STATE_FIELD=$.summary.jobs_disabled=5 +DETAIL=STATE_FIELD=$.summary.jobs_ok=30 +DETAIL=STATE_FIELD=$.summary.jobs_total=41 +DETAIL=STATE_FIELD=$.summary.jobs_unmapped=0 +DETAIL=STATE_FIELD=$.summary.status=FAIL +DETAIL=UNIT_BEGIN=homelab-vm160-remote-backup.service +DETAIL=UNIT_SHOW_RC=0 +DETAIL=UNIT_PROPERTY=Result=success +DETAIL=UNIT_PROPERTY=ExecMainStatus=0 +DETAIL=UNIT_PROPERTY=ExecStart={ path=/usr/local/sbin/homelab-vm160-remote-backup ; argv[]=/usr/local/sbin/homelab-vm160-remote-backup ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } +DETAIL=UNIT_PROPERTY=LoadState=loaded +DETAIL=UNIT_PROPERTY=ActiveState=inactive +DETAIL=UNIT_PROPERTY=SubState=dead +DETAIL=UNIT_PROPERTY=FragmentPath=/etc/systemd/system/homelab-vm160-remote-backup.service +DETAIL=SCRIPT_PATH=/usr/local/sbin/homelab-vm160-remote-backup +DETAIL=SCRIPT_LINE=3:export SYSTEMD_PAGER=cat PAGER=cat SYSTEMD_LESS=FRXMK +DETAIL=SCRIPT_LINE=8:PARENT=/mnt/staging/vzdump +DETAIL=SCRIPT_LINE=10:HEALTH=/var/lib/homelab-health/vm160-remote-backup.txt +DETAIL=SCRIPT_LINE=11:LOCK=/run/lock/homelab-vm160-remote-backup.lock +DETAIL=SCRIPT_LINE=30:LOG="$(mktemp /tmp/vm160-remote-backup.XXXXXX.log)" +DETAIL=SCRIPT_LINE=31:cleanup(){ +DETAIL=SCRIPT_LINE=35:ssh "${SSH[@]}" "$TARGET" "rm -rf -- '$DIR'" >/dev/null 2>&1 || true +DETAIL=SCRIPT_LINE=37:printf 'STATUS=FAIL\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=%s\nVMID=%s\nTARGET_DIR=%s\nRC=%s\n' "$TS" "$VMID" "$DIR" "$rc" > "$tmp" +DETAIL=SCRIPT_LINE=41:rm -f "$LOG" +DETAIL=SCRIPT_LINE=44:trap cleanup EXIT +DETAIL=SCRIPT_LINE=45:REMOTE_FREE="$(ssh "${SSH[@]}" "$TARGET" "df -B1 --output=avail '$PARENT' | awk 'NR==2{print \$1}'")" +DETAIL=SCRIPT_LINE=53:test "$REMOTE_FREE" -ge "$REQUIRED" +DETAIL=SCRIPT_LINE=55:ssh "${SSH[@]}" "$SOURCE" "vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7" 2>"$LOG" | ssh "${SSH[@]}" "$TARGET" "cat > '$PART'" +DETAIL=SCRIPT_LINE=57:ssh "${SSH[@]}" "$TARGET" "cat > '$DIR/vzdump.log'" < "$LOG" +DETAIL=SCRIPT_LINE=59:SHA="$(ssh "${SSH[@]}" "$TARGET" "sha256sum '$FINAL' | awk '{print \$1}'")" +DETAIL=SCRIPT_LINE=62:printf 'STATUS=OK\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=%s\nVMID=%s\nSOURCE_NODE=pve02\nTARGET_NODE=pve03\nTARGET_DIR=%s\nARCHIVE=%s\nSIZE_BYTES=%s\nSHA256=%s\nREMOTE_RETENTION_GENERATIONS=1\n' "$TS" "$VMID" "$DIR" "$FINAL" "$SIZE" "$SHA" > "$tmp" +DETAIL=SCRIPT_LINE=66:rm -f "$LOG" +DETAIL=SCRIPT_LINE=67:printf 'VM160_REMOTE_BACKUP_STATUS=OK\nTARGET_DIR=%s\nARCHIVE=%s\nSIZE_BYTES=%s\nSHA256=%s\n' "$DIR" "$FINAL" "$SIZE" "$SHA" +DETAIL=UNIT_END=homelab-vm160-remote-backup.service +DETAIL=UNIT_BEGIN=homelab-vm-local-dumps-2cloud.service +DETAIL=UNIT_SHOW_RC=0 +DETAIL=UNIT_PROPERTY=Result=success +DETAIL=UNIT_PROPERTY=ExecMainStatus=0 +DETAIL=UNIT_PROPERTY=ExecStart={ path=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; argv[]=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } +DETAIL=UNIT_PROPERTY=LoadState=loaded +DETAIL=UNIT_PROPERTY=ActiveState=inactive +DETAIL=UNIT_PROPERTY=SubState=dead +DETAIL=UNIT_PROPERTY=FragmentPath=/etc/systemd/system/homelab-vm-local-dumps-2cloud.service +DETAIL=SCRIPT_PATH=/usr/local/sbin/homelab-vm-local-dumps-2cloud +DETAIL=SCRIPT_LINE=4:OUT="/var/lib/homelab-health/vm-local-dumps-cloud.txt" +DETAIL=SCRIPT_LINE=6:LOG="$ROOT/vm-local-dumps-2cloud-latest.log" +DETAIL=SCRIPT_LINE=7:VERIFY_ROOT="/var/tmp/homelab-vm-local-dumps-verify" +DETAIL=SCRIPT_LINE=12:mkdir -p /var/lib/homelab-health "$ROOT" "$VERIFY_ROOT" "$CHUNK_ROOT" +DETAIL=SCRIPT_LINE=55:SHA="$(sha256sum "$F" | cut -d" " -f1)" +DETAIL=SCRIPT_LINE=64:rm -rf "$CDIR" +DETAIL=SCRIPT_LINE=69:(cd "$CDIR" && sha256sum part-* > chunks.sha256) +DETAIL=SCRIPT_LINE=92:"sha256":p[3], +DETAIL=SCRIPT_LINE=112:DEST_A="${REMOTE_A%/}/${NODE}/vm-local-dumps/${JOB_ID}" +DETAIL=SCRIPT_LINE=113:DEST_B="${REMOTE_B%/}/${NODE}/vm-local-dumps/${JOB_ID}" +DETAIL=SCRIPT_LINE=115:upload_one_dest() { +DETAIL=SCRIPT_LINE=117:rclone mkdir "$DEST/files" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SCRIPT_LINE=121:echo "UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts" >> "$LOG" +DETAIL=SCRIPT_LINE=122:rclone mkdir "$DEST/files/$bn.chunks" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SCRIPT_LINE=126:rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SCRIPT_LINE=128:rclone copyto "$CDIR/chunks.sha256" "$DEST/files/$bn.chunks/chunks.sha256" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SCRIPT_LINE=130:echo "UPLOAD_DIRECT dest=$DEST file=$bn size=$size" >> "$LOG" +DETAIL=SCRIPT_LINE=131:rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SCRIPT_LINE=135:rclone copyto "$CHKS" "$DEST/checksums.txt" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SCRIPT_LINE=136:rclone copyto "$FILES_TSV" "$DEST/files.tsv" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SCRIPT_LINE=137:rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SCRIPT_LINE=140:verify_one_dest() { +DETAIL=SCRIPT_LINE=143:VBASE="$VERIFY_ROOT/$JOB_ID/$LABEL" +DETAIL=SCRIPT_LINE=144:rm -rf "$VBASE" +DETAIL=SCRIPT_LINE=146:rclone copyto "$DEST/manifest.json" "$VBASE/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SCRIPT_LINE=153:echo "VERIFY_CHUNKED label=$LABEL file=$bn parts=$parts" >> "$LOG" +DETAIL=SCRIPT_LINE=154:rclone copy "$DEST/files/$bn.chunks" "$VDIR" --filter "+ part-*" --filter "+ chunks.sha256" --filter "- *" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SCRIPT_LINE=155:(cd "$VDIR" && sha256sum -c chunks.sha256) >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SCRIPT_LINE=156:CALC="$(find "$VDIR" -maxdepth 1 -type f -name 'part-*' | sort | xargs cat | sha256sum | cut -d" " -f1)" +DETAIL=SCRIPT_LINE=163:rm -rf "$VDIR" +DETAIL=SCRIPT_LINE=165:echo "VERIFY_DIRECT label=$LABEL file=$bn" >> "$LOG" +DETAIL=SCRIPT_LINE=166:rclone copyto "$DEST/files/$bn" "$VBASE/$bn" --transfers 1 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SCRIPT_LINE=167:printf "%s %s\n" "$sha" "$VBASE/$bn" | sha256sum -c - >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SCRIPT_LINE=168:rm -f "$VBASE/$bn" +DETAIL=SCRIPT_LINE=171:rm -rf "$VBASE" +DETAIL=SCRIPT_LINE=175:upload_one_dest "$DEST_A" +DETAIL=SCRIPT_LINE=176:upload_one_dest "$DEST_B" +DETAIL=SCRIPT_LINE=180:verify_one_dest "A" "$DEST_A" +DETAIL=SCRIPT_LINE=181:verify_one_dest "B" "$DEST_B" +DETAIL=SCRIPT_LINE=184:rm -rf "$VERIFY_ROOT/$JOB_ID" +DETAIL=SCRIPT_LINE=185:rm -rf "$CHUNK_ROOT/$JOB_ID" +DETAIL=SCRIPT_LINE=193:--arg remote_a "$DEST_A" \ +DETAIL=SCRIPT_LINE=194:--arg remote_b "$DEST_B" \ +DETAIL=SCRIPT_LINE=200:"{job_id:\$job_id,status:\$status,node:\$node,remote_a:\$remote_a,remote_b:\$remote_b,count:\$count,total_bytes:\$total_bytes,chunked_count:\$chunked_count,chunk_size_bytes:\$chunk_size_bytes,log:\$log,local_dumps_deleted:false}" \ +DETAIL=SCRIPT_LINE=201:> "$ROOT/latest-vm-local-dumps-cloud.json" +DETAIL=SCRIPT_LINE=203:echo "STATUS=$STATUS TS=$TS TYPE=vm-local-dumps-cloud BAD=$BAD COUNT=$COUNT CHUNKED=$CHUNKED TOTAL_BYTES=$TOTAL JOB=$JOB_ID REMOTE_A=$DEST_A REMOTE_B=$DEST_B LOG=$LOG" > "$OUT" +DETAIL=SCRIPT_LINE=204:cat "$OUT" +DETAIL=UNIT_END=homelab-vm-local-dumps-2cloud.service +DETAIL=PAYLOAD_HOST_BEGIN=pve01 +DETAIL=PAYLOAD_ROOT=pve01:/var/lib/vz/dump +DETAIL=PAYLOAD_ROOT_STATUS=PRESENT +DETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst|BYTES=329541843|AGE_SECONDS=293553 +DETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-110-2026_07_24-03_30_02.tar.zst|BYTES=339140622|AGE_SECONDS=34352 +DETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-112-2026_07_22-03_30_09.tar.zst|BYTES=222794054|AGE_SECONDS=207149 +DETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-110-2026_07_22-03_30_01.tar.zst|BYTES=332951369|AGE_SECONDS=207154 +DETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst|BYTES=7021952392|AGE_SECONDS=293496 +DETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-qemu-150-2026_07_24-03_30_16.vma.zst|BYTES=7016624549|AGE_SECONDS=34298 +DETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-qemu-150-2026_07_22-03_30_14.vma.zst|BYTES=7052176114|AGE_SECONDS=207100 +DETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst|BYTES=222777732|AGE_SECONDS=293547 +DETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-112-2026_07_24-03_30_11.tar.zst|BYTES=222859457|AGE_SECONDS=34347 +DETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-qemu-150-2026_07_23-03_30_16.vma.zst|BYTES=7024165999|AGE_SECONDS=120695 +DETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-110-2026_07_23-03_30_02.tar.zst|BYTES=336245409|AGE_SECONDS=120752 +DETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-112-2026_07_23-03_30_11.tar.zst|BYTES=222808025|AGE_SECONDS=120747 +DETAIL=PAYLOAD_ROOT=pve01:/mnt/staging/vzdump +DETAIL=PAYLOAD_ROOT_STATUS=PRESENT +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260705T004739Z/pve03/vzdump-qemu-130-2026_07_05-03_49_34.vma.zst|BYTES=11242736440|AGE_SECONDS=1674545 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260705T004739Z/pve02/vzdump-lxc-111-2026_07_05-03_48_52.tar.zst|BYTES=255533195|AGE_SECONDS=1674805 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260705T004739Z/pve02/vzdump-lxc-113-2026_07_05-03_49_19.tar.zst|BYTES=220380760|AGE_SECONDS=1674791 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260705T004739Z/pve01/vzdump-qemu-150-2026_07_05-03_47_52.vma.zst|BYTES=7004920660|AGE_SECONDS=1674842 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260705T004739Z/pve01/vzdump-lxc-112-2026_07_05-03_47_46.tar.zst|BYTES=222468077|AGE_SECONDS=1674891 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260705T004739Z/pve01/vzdump-lxc-110-2026_07_05-03_47_40.tar.zst|BYTES=269221740|AGE_SECONDS=1674897 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260628T004739Z/pve03/vzdump-qemu-130-2026_06_28-03_49_15.vma.zst|BYTES=27024990491|AGE_SECONDS=2279192 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260628T004739Z/pve02/vzdump-lxc-113-2026_06_28-03_49_00.tar.zst|BYTES=220326595|AGE_SECONDS=2279610 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260628T004739Z/pve02/vzdump-lxc-111-2026_06_28-03_48_46.tar.zst|BYTES=242676818|AGE_SECONDS=2279624 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260628T004739Z/pve01/vzdump-lxc-112-2026_06_28-03_47_46.tar.zst|BYTES=222365246|AGE_SECONDS=2279692 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260628T004739Z/pve01/vzdump-lxc-110-2026_06_28-03_47_40.tar.zst|BYTES=253314990|AGE_SECONDS=2279698 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260628T004739Z/pve01/vzdump-qemu-150-2026_06_28-03_47_51.vma.zst|BYTES=7003671925|AGE_SECONDS=2279646 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260719T004537Z/pve03/vzdump-qemu-130-2026_07_19-03_48_01.vma.zst|BYTES=11043726130|AGE_SECONDS=465037 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260719T004537Z/pve02/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst|BYTES=220049041|AGE_SECONDS=465284 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260719T004537Z/pve02/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst|BYTES=284443608|AGE_SECONDS=465301 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260719T004537Z/pve01/vzdump-qemu-150-2026_07_19-03_45_58.vma.zst|BYTES=7008027143|AGE_SECONDS=465328 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260719T004537Z/pve01/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst|BYTES=323668231|AGE_SECONDS=465417 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260719T004537Z/pve01/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst|BYTES=222745965|AGE_SECONDS=465406 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260712T004231Z/pve03/vzdump-qemu-130-2026_07_12-03_44_28.vma.zst|BYTES=10813915936|AGE_SECONDS=1070055 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260712T004231Z/pve02/vzdump-lxc-111-2026_07_12-03_43_45.tar.zst|BYTES=270924730|AGE_SECONDS=1070317 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260712T004231Z/pve02/vzdump-lxc-113-2026_07_12-03_44_07.tar.zst|BYTES=220482178|AGE_SECONDS=1070297 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260712T004231Z/pve01/vzdump-lxc-112-2026_07_12-03_42_39.tar.zst|BYTES=222566793|AGE_SECONDS=1070398 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260712T004231Z/pve01/vzdump-qemu-150-2026_07_12-03_42_45.vma.zst|BYTES=7014397787|AGE_SECONDS=1070350 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260712T004231Z/pve01/vzdump-lxc-110-2026_07_12-03_42_31.tar.zst|BYTES=302840996|AGE_SECONDS=1070404 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/full-manual-backup-20260615T092720Z/pve03/vzdump-qemu-130-2026_06_15-12_27_45.vma.zst|BYTES=829658873|AGE_SECONDS=3371676 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/full-manual-backup-20260615T092720Z/pve02/vzdump-lxc-113-2026_06_15-12_27_32.tar.zst|BYTES=220018661|AGE_SECONDS=3371701 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/full-manual-backup-20260615T092720Z/pve02/vzdump-lxc-111-2026_06_15-12_27_22.tar.zst|BYTES=228398861|AGE_SECONDS=3371712 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/full-manual-backup-20260615T092720Z/pve01/vzdump-qemu-150-2026_06_15-12_19_05.vma.zst|BYTES=6957596210|AGE_SECONDS=3372173 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/full-manual-backup-20260615T092720Z/pve01/vzdump-lxc-112-2026_06_15-12_23_56.tar.zst|BYTES=222110027|AGE_SECONDS=3371924 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/full-manual-backup-20260615T092720Z/pve01/vzdump-lxc-110-2026_06_15-12_23_52.tar.zst|BYTES=230562036|AGE_SECONDS=3371928 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/vm170-vm171/vzdump-qemu-170-2026_07_24-02_15_07.vma.zst|BYTES=11828704880|AGE_SECONDS=37618 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/vm170-vm171/vzdump-qemu-171-2026_07_23-02_36_37.vma.zst|BYTES=2157597134|AGE_SECONDS=122728 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/vm170-vm171/vzdump-qemu-170-2026_07_23-02_15_17.vma.zst|BYTES=11842330422|AGE_SECONDS=124008 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/vm170-vm171/vzdump-qemu-171-2026_07_24-02_36_27.vma.zst|BYTES=2158869638|AGE_SECONDS=36338 +DETAIL=PAYLOAD_ROOT=pve01:/var/lib/homelab-vm-backup/staging +DETAIL=PAYLOAD_ROOT_STATUS=ABSENT +DETAIL=PAYLOAD_HOST_SUMMARY=pve01|COUNT=46|BYTES=159150279782 +DETAIL=PAYLOAD_HOST_END=pve01 +DETAIL=PAYLOAD_HOST_BEGIN=pve02 +DETAIL=PAYLOAD_ROOT=pve02:/var/lib/vz/dump +DETAIL=PAYLOAD_ROOT_STATUS=PRESENT +DETAIL=LOCAL_PAYLOAD=pve02:/var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.vma.zst::997503422::1782865871.1700059310/var/lib/vz/dump/vzdump-lxc-111-2026_07_24-03_30_03.tar.zst::293632136::1784853018.7398517980/var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.vma.zst::868048369::1782779468.4903256780/var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.tar.zst::220080970::1784593833.3910787180/var/lib/vz/dump/vzdump-lxc-111-2026_07_22-03_30_01.tar.zst::290059742::1784680222.8748248510/var/lib/vz/dump/vzdump-lxc-111-2026_07_23-03_30_02.tar.zst::292185437::1784766624.6563558990/var/lib/vz/dump/vzdump-lxc-113-2026_07_23-03_30_24.tar.zst::220108944::1784766637.5724249240/var/lib/vz/dump/vzdump-lxc-113-2026_07_22-03_30_23.tar.zst::220095608::1784680240.6809207660/var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.tar.zst::288349157::1784593821.2480085730/var/lib/vz/dump/vzdump-lxc-113-2026_07_24-03_30_19.tar.zst|BYTES=220234127|AGE_SECONDS=34333 +DETAIL=PAYLOAD_ROOT=pve02:/mnt/staging/vzdump +DETAIL=PAYLOAD_ROOT_STATUS=PRESENT +DETAIL=LOCAL_PAYLOAD=pve02:/mnt/staging/vzdump/auto-pve02-20260615T121549Z/vzdump-lxc-113-2026_06_15-15_16_01.tar.zst::220085886::1781525770.6722751820/mnt/staging/vzdump/auto-pve02-20260615T121549Z/vzdump-lxc-111-2026_06_15-15_15_50.tar.zst::228433321::1781525759.9832231610/mnt/staging/vzdump/auto-pve02-20260615T184325Z/vzdump-lxc-111-2026_06_15-21_43_27.tar.zst::228695245::1781549016.3828865780/mnt/staging/vzdump/auto-pve02-20260615T184325Z/vzdump-lxc-113-2026_06_15-21_43_37.tar.zst::220088590::1781549027.0039397060/mnt/staging/vzdump/auto-pve02-20260615T145425Z/vzdump-lxc-111-2026_06_15-17_54_26.tar.zst::228364545::1781535276.0106286580/mnt/staging/vzdump/auto-pve02-20260615T145425Z/vzdump-lxc-113-2026_06_15-17_54_37.tar.zst::220087147::1781535286.6446806260/mnt/staging/vzdump/auto-pve02-20260615T125928Z/vzdump-lxc-111-2026_06_15-15_59_30.tar.zst::228433824::1781528379.7579730690/mnt/staging/vzdump/auto-pve02-20260615T125928Z/vzdump-lxc-113-2026_06_15-15_59_40.tar.zst::220086313::1781528390.6340260000/mnt/staging/vzdump/auto-pve02-20260628T004845Z/vzdump-lxc-113-2026_06_28-03_49_00.tar.zst::220326595::1782607753.1428569490/mnt/staging/vzdump/auto-pve02-20260628T004845Z/vzdump-lxc-111-2026_06_28-03_48_46.tar.zst::242676818::1782607739.4867871320/mnt/staging/vzdump/auto-pve02-20260615T201446Z/vzdump-lxc-111-2026_06_15-23_14_48.tar.zst::228706014::1781554497.6364731750/mnt/staging/vzdump/auto-pve02-20260615T201446Z/vzdump-lxc-113-2026_06_15-23_14_58.tar.zst::220089008::1781554508.5315278350/mnt/staging/vzdump/auto-pve02-20260705T004851Z/vzdump-lxc-111-2026_07_05-03_48_52.tar.zst::255533195::1783212558.3825236020/mnt/staging/vzdump/auto-pve02-20260705T004851Z/vzdump-lxc-113-2026_07_05-03_49_19.tar.zst::220380760::1783212571.8805933550/mnt/staging/vzdump/auto-pve02-20260719T004725Z/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst::284443608::1784422061.8525503200/mnt/staging/vzdump/auto-pve02-20260719T004725Z/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst::220049041::1784422078.9256407960/mnt/staging/vzdump/auto-pve02-20260712T004344Z/vzdump-lxc-111-2026_07_12-03_43_45.tar.zst::270924730::1783817046.4081420260/mnt/staging/vzdump/auto-pve02-20260712T004344Z/vzdump-lxc-113-2026_07_12-03_44_07.tar.zst::220482178::1783817065.7622435280/mnt/staging/vzdump/auto-pve02-20260621T004841Z/vzdump-lxc-113-2026_06_21-03_48_54.tar.zst::220165475::1782002947.0683211180/mnt/staging/vzdump/auto-pve02-20260621T004841Z/vzdump-lxc-111-2026_06_21-03_48_42.tar.zst::232731208::1782002933.6692546410/mnt/staging/vzdump/auto-pve02-20260615T161514Z/vzdump-lxc-111-2026_06_15-19_15_16.tar.zst::228462340::1781540125.7713503770/mnt/staging/vzdump/auto-pve02-20260615T161514Z/vzdump-lxc-113-2026_06_15-19_15_26.tar.zst::220087434::1781540136.7614046640/mnt/staging/vzdump/manual-pve02-20260615T092720Z/vzdump-lxc-113-2026_06_15-12_27_32.tar.zst::220018661::1781515662.0924756330/mnt/staging/vzdump/manual-pve02-20260615T092720Z/vzdump-lxc-111-2026_06_15-12_27_22.tar.zst::228398861::1781515651.4234214190/mnt/staging/vzdump/auto-pve02-20260616T023537Z/vzdump-lxc-111-2026_06_16-05_35_38.tar.zst::228693413::1781577348.0265431350/mnt/staging/vzdump/auto-pve02-20260616T023537Z/vzdump-lxc-113-2026_06_16-05_35_49.tar.zst::220093717::1781577358.9785981020/mnt/staging/vzdump/auto-pve02-20260615T095831Z/vzdump-lxc-111-2026_06_15-12_58_32.tar.zst::228398872::1781517522.2069308430/mnt/staging/vzdump/auto-pve02-20260615T095831Z/vzdump-lxc-113-2026_06_15-12_58_43.tar.zst|BYTES=220019157|AGE_SECONDS=3369831 +DETAIL=PAYLOAD_HOST_SUMMARY=pve02|COUNT=2|BYTES=440253284 +DETAIL=PAYLOAD_HOST_END=pve02 +DETAIL=PAYLOAD_HOST_BEGIN=pve03 +DETAIL=PAYLOAD_ROOT=pve03:/var/lib/vz/dump +DETAIL=PAYLOAD_ROOT_STATUS=PRESENT +DETAIL=LOCAL_PAYLOAD=pve03:/var/lib/vz/dump/vzdump-qemu-130-2026_07_23-03_30_03.vma.zst::11139455731::1784766836.0161157970/var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.vma.zst::11043834894::1784594034.3725716230/var/lib/vz/dump/vzdump-qemu-130-2026_07_22-03_30_02.vma.zst::11072513783::1784680438.8961071480/var/lib/vz/dump/vzdump-qemu-190-2026_07_23-03_33_56.vma.zst::1268419395::1784766881.2902288360/var/lib/vz/dump/vzdump-qemu-190-2026_07_24-03_34_00.vma.zst::1278418128::1784853285.0606354470/var/lib/vz/dump/vzdump-qemu-130-2026_07_24-03_30_02.vma.zst::11170433004::1784853240.1424086410/var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.vma.zst::1137072350::1784594077.6276811360/var/lib/vz/dump/vzdump-qemu-190-2026_07_22-03_33_59.vma.zst|BYTES=1177407443|AGE_SECONDS=206880 +DETAIL=PAYLOAD_ROOT=pve03:/mnt/staging/vzdump +DETAIL=PAYLOAD_ROOT_STATUS=PRESENT +DETAIL=LOCAL_PAYLOAD=pve03:/mnt/staging/vzdump/auto-pve03-20260719T004800Z/vzdump-qemu-130-2026_07_19-03_48_01.vma.zst::11043726130::1784422326.4811235470/mnt/staging/vzdump/auto-pve03-20260712T004427Z/vzdump-qemu-130-2026_07_12-03_44_28.vma.zst::10813915936::1783817308.4882503530/mnt/staging/vzdump/vm160-pve02-20260717T223819Z/vzdump-qemu-160-20260717T223819Z.vma.zst|BYTES=84995216465|AGE_SECONDS=559051 +DETAIL=PAYLOAD_HOST_SUMMARY=pve03|COUNT=2|BYTES=86172623908 +DETAIL=PAYLOAD_HOST_END=pve03 +DETAIL=POLICY_SUCCESS_REQUIRES=CLOUD1_VERIFIED,CLOUD2_VERIFIED,LOCAL_PAYLOAD_COUNT_ZERO +DETAIL=SECRET_VALUES_OUTPUT=NONE +DETAIL=BACKUP_NO_LOCAL_INVARIANT_AUDIT=PASS +CHANGES_MADE=NO + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 31daf1c..74085bb 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "HOMELAB-BACKUP-BAD-JOBS-ROOTCAUSE-431", + "command_id": "HOMELAB-BACKUP-NO-LOCAL-INVARIANT-AUDIT-432", "status": "OK", "rc": 0, "host": "pve01", "mode": "read-only", "component": "cluster-audit", - "started_at_utc": "2026-07-24T09:55:30Z", - "finished_at_utc": "2026-07-24T09:55:30Z", + "started_at_utc": "2026-07-24T10:02:43Z", + "finished_at_utc": "2026-07-24T10:02:45Z", "reference_register_checked": true, "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", "error_register_checked": true, "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", - "command_sha256": "8635e368c9b3e168a7302affcd2c133d8f85547164b3f3352fc0cbad679edc98", + "command_sha256": "3d74c7e2671b37a475e371e668d24c5a96cd5db6b4171efbae25b123278b6883", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -22,9 +22,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "e39cc1d76af62d7050936445c27d21728aaeb6ca00ab2430c6cb3963a7da3907", - "sanitized_output_sha256": "e39cc1d76af62d7050936445c27d21728aaeb6ca00ab2430c6cb3963a7da3907", + "raw_evidence_sha256": "cc2f90a2cae34633773e4468f017e38bc9d8d85abc7d3c929e39aa9f0e9d278b", + "sanitized_output_sha256": "cc2f90a2cae34633773e4468f017e38bc9d8d85abc7d3c929e39aa9f0e9d278b", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "DETAIL=MATRIX_GENERATED_AT=2026-07-24T07:06:04Z\nDETAIL=TARGET_JOB_COUNT=6\nDETAIL=JOB_BEGIN=filebrowser_restore_validate\nDETAIL=JOB_FIELD=id=filebrowser_restore_validate\nDETAIL=JOB_FIELD=name=filebrowser-restore-validate\nDETAIL=JOB_FIELD=category=restore\nDETAIL=JOB_FIELD=critical=True\nDETAIL=JOB_FIELD=status=STALE\nDETAIL=JOB_FIELD=freshness=STALE\nDETAIL=JOB_FIELD=age_seconds=1275827\nDETAIL=JOB_FIELD=max_age_seconds=864000\nDETAIL=JOB_FIELD=source_kind=health_file\nDETAIL=JOB_FIELD=source_status=OK\nDETAIL=JOB_FIELD=source_mtime=2026-07-09T12:42:17Z\nDETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt\nDETAIL=JOB_FIELD=service=filebrowser-restore-validate.service\nDETAIL=JOB_FIELD=service_result=success\nDETAIL=JOB_FIELD=service_exec_main_status=0\nDETAIL=JOB_FIELD=timer=filebrowser-restore-validate.timer\nDETAIL=JOB_FIELD=timer_enabled=enabled\nDETAIL=JOB_FIELD=timer_active=active\nDETAIL=SERVICE_SHOW_RC=0\nDETAIL=SERVICE_PROPERTY=Result=success\nDETAIL=SERVICE_PROPERTY=ExecMainCode=1\nDETAIL=SERVICE_PROPERTY=ExecMainStatus=0\nDETAIL=SERVICE_PROPERTY=LoadState=loaded\nDETAIL=SERVICE_PROPERTY=ActiveState=inactive\nDETAIL=SERVICE_PROPERTY=SubState=dead\nDETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/filebrowser-restore-validate.service\nDETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/filebrowser-restore-validate.service.d/10-superseded-noop.conf /etc/systemd/system/filebrowser-restore-validate.service.d/90-backup-matrix-refresh.conf\nDETAIL=SERVICE_JOURNAL_MATCH_COUNT=56\nDETAIL=SERVICE_JOURNAL=2026-07-11T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-11T04:10:00+03:00 pve01 homelab-superseded-unit-ok[1686870]: STATUS=OK TS=2026-07-11T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-11T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-11T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-12T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-12T04:10:00+03:00 pve01 homelab-superseded-unit-ok[1699058]: STATUS=OK TS=2026-07-12T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-12T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-12T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-13T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-13T04:10:00+03:00 pve01 homelab-superseded-unit-ok[1681015]: STATUS=OK TS=2026-07-13T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-13T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-13T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-14T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-14T04:10:00+03:00 pve01 homelab-superseded-unit-ok[2132068]: STATUS=OK TS=2026-07-14T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-14T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-14T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-15T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-15T04:10:00+03:00 pve01 homelab-superseded-unit-ok[2783319]: STATUS=OK TS=2026-07-15T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-15T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-15T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-16T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-16T04:10:00+03:00 pve01 homelab-superseded-unit-ok[3557730]: STATUS=OK TS=2026-07-16T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-16T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-16T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-17T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-17T04:10:14+03:00 pve01 homelab-superseded-unit-ok[3346747]: STATUS=OK TS=2026-07-17T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-17T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-17T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-18T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-18T04:10:14+03:00 pve01 homelab-superseded-unit-ok[4129958]: STATUS=OK TS=2026-07-18T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-18T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-18T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-19T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-19T04:10:14+03:00 pve01 homelab-superseded-unit-ok[738797]: STATUS=OK TS=2026-07-19T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-19T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-19T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-20T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-20T04:10:14+03:00 pve01 homelab-superseded-unit-ok[1448291]: STATUS=OK TS=2026-07-20T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-20T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-20T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-21T04:10:07+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-21T04:10:07+03:00 pve01 homelab-superseded-unit-ok[2324924]: STATUS=OK TS=2026-07-21T01:10:07Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-21T04:10:07+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-21T04:10:07+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-22T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-22T04:10:14+03:00 pve01 homelab-superseded-unit-ok[3008751]: STATUS=OK TS=2026-07-22T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-22T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-22T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-23T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-23T04:10:14+03:00 pve01 homelab-superseded-unit-ok[3700126]: STATUS=OK TS=2026-07-23T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-23T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-23T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=SERVICE_JOURNAL=2026-07-24T04:10:04+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation...\nDETAIL=SERVICE_JOURNAL=2026-07-24T04:10:04+03:00 pve01 homelab-superseded-unit-ok[2416005]: STATUS=OK TS=2026-07-24T01:10:04Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-24T04:10:04+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-24T04:10:04+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation.\nDETAIL=TIMER_SHOW_RC=0\nDETAIL=TIMER_PROPERTY=Unit=filebrowser-restore-validate.service\nDETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Sat 2026-07-25 04:10:00 MSK\nDETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 04:10:04 MSK\nDETAIL=TIMER_PROPERTY=LoadState=loaded\nDETAIL=TIMER_PROPERTY=ActiveState=active\nDETAIL=TIMER_PROPERTY=SubState=waiting\nDETAIL=TIMER_ENABLED_RESULT=enabled|RC=0\nDETAIL=HEALTH_METADATA=BYTES:209|MTIME_NS:1783600937792562046\nDETAIL=HEALTH_MATCH_COUNT=1\nDETAIL=HEALTH_LINE=STATUS=OK TS=2026-07-09T12:42:13Z TYPE=filebrowser-restore JOB=homelab-pve01-app-filebrowser-20260709T051713Z SOURCE=appbackupctl-restore-check RESTORECHECK=OK_BOTH_CLOUDS NOTE=service-readiness-restore-proof\nDETAIL=JOB_END=filebrowser_restore_validate\nDETAIL=JOB_BEGIN=homelab_backup_sla_health\nDETAIL=JOB_FIELD=id=homelab_backup_sla_health\nDETAIL=JOB_FIELD=name=homelab-backup-sla-health\nDETAIL=JOB_FIELD=category=application\nDETAIL=JOB_FIELD=critical=True\nDETAIL=JOB_FIELD=status=FAIL\nDETAIL=JOB_FIELD=freshness=FRESH\nDETAIL=JOB_FIELD=age_seconds=358\nDETAIL=JOB_FIELD=max_age_seconds=172800\nDETAIL=JOB_FIELD=source_kind=health_file\nDETAIL=JOB_FIELD=source_status=WARN\nDETAIL=JOB_FIELD=source_mtime=2026-07-24T07:00:06Z\nDETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt\nDETAIL=JOB_FIELD=service=homelab-backup-sla-health.service\nDETAIL=JOB_FIELD=service_result=success\nDETAIL=JOB_FIELD=service_exec_main_status=0\nDETAIL=JOB_FIELD=timer=homelab-backup-sla-health.timer\nDETAIL=JOB_FIELD=timer_enabled=enabled\nDETAIL=JOB_FIELD=timer_active=active\nDETAIL=SERVICE_SHOW_RC=0\nDETAIL=SERVICE_PROPERTY=Result=success\nDETAIL=SERVICE_PROPERTY=ExecMainCode=1\nDETAIL=SERVICE_PROPERTY=ExecMainStatus=0\nDETAIL=SERVICE_PROPERTY=LoadState=loaded\nDETAIL=SERVICE_PROPERTY=ActiveState=inactive\nDETAIL=SERVICE_PROPERTY=SubState=dead\nDETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/homelab-backup-sla-health.service\nDETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/homelab-backup-sla-health.service.d/90-backup-matrix-refresh.conf\nDETAIL=SERVICE_JOURNAL_MATCH_COUNT=76\nDETAIL=SERVICE_JOURNAL=2026-07-11T06:24:11+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-11T06:24:11+03:00 pve01 homelab-backup-sla-health[2238839]: STATUS=OK TS=2026-07-11T03:24:11Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-11T06:24:11+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-11T06:24:11+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-11T07:39:35+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-11T07:39:36+03:00 pve01 homelab-backup-sla-health[2476861]: STATUS=OK TS=2026-07-11T04:39:35Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-11T07:39:36+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-11T07:39:36+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-11T08:15:32+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-11T08:15:32+03:00 pve01 homelab-backup-sla-health[2616083]: STATUS=OK TS=2026-07-11T05:15:32Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-11T08:15:33+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-11T08:15:33+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-12T08:04:09+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-12T08:04:09+03:00 pve01 homelab-backup-sla-health[2414445]: STATUS=WARN TS=2026-07-12T05:04:09Z TYPE=backup-sla BAD=0 WARN=1 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-12T08:04:09+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-12T08:04:09+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-13T01:03:10+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-13T01:03:11+03:00 pve01 homelab-backup-sla-health[980591]: STATUS=OK TS=2026-07-12T22:03:10Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-13T01:03:11+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-13T01:03:11+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-13T01:08:07+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-13T01:08:08+03:00 pve01 homelab-backup-sla-health[994479]: STATUS=OK TS=2026-07-12T22:08:07Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-13T01:08:08+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-13T01:08:08+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-13T08:15:21+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-13T08:15:21+03:00 pve01 homelab-backup-sla-health[2521375]: STATUS=OK TS=2026-07-13T05:15:21Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-13T08:15:21+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-13T08:15:21+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-14T08:05:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-14T08:05:15+03:00 pve01 homelab-backup-sla-health[2983241]: STATUS=WARN TS=2026-07-14T05:05:14Z TYPE=backup-sla BAD=0 WARN=1 APPS=46 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-14T08:05:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-14T08:05:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-14T08:23:27+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-14T08:23:28+03:00 pve01 homelab-backup-sla-health[3042266]: STATUS=OK TS=2026-07-14T05:23:27Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-14T08:23:28+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-14T08:23:28+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-15T08:19:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-15T08:19:15+03:00 pve01 homelab-backup-sla-health[3670619]: STATUS=OK TS=2026-07-15T05:19:14Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-15T08:19:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-15T08:19:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-16T08:09:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-16T08:09:15+03:00 pve01 homelab-backup-sla-health[225757]: STATUS=WARN TS=2026-07-16T05:09:14Z TYPE=backup-sla BAD=0 WARN=1 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-16T08:09:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-16T08:09:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-17T08:07:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-17T08:07:15+03:00 pve01 homelab-backup-sla-health[4188049]: STATUS=WARN TS=2026-07-17T05:07:14Z TYPE=backup-sla BAD=0 WARN=4 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-17T08:07:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-17T08:07:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-18T08:05:29+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-18T08:05:29+03:00 pve01 homelab-backup-sla-health[806098]: STATUS=OK TS=2026-07-18T05:05:29Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-18T08:05:29+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-18T08:05:29+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-19T08:05:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-19T08:05:15+03:00 pve01 homelab-backup-sla-health[1545067]: STATUS=OK TS=2026-07-19T05:05:14Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-19T08:05:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-19T08:05:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-20T08:10:25+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-20T08:10:26+03:00 pve01 homelab-backup-sla-health[2331602]: STATUS=OK TS=2026-07-20T05:10:25Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-20T08:10:26+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-20T08:10:26+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-21T08:07:25+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-21T08:07:26+03:00 pve01 homelab-backup-sla-health[3122692]: STATUS=WARN TS=2026-07-21T05:07:25Z TYPE=backup-sla BAD=0 WARN=1 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-21T08:07:26+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-21T08:07:26+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-22T08:10:41+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-22T08:10:42+03:00 pve01 homelab-backup-sla-health[3806055]: STATUS=WARN TS=2026-07-22T05:10:41Z TYPE=backup-sla BAD=0 WARN=1 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-22T08:10:42+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-22T08:10:42+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-23T08:06:25+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-23T08:06:26+03:00 pve01 homelab-backup-sla-health[310318]: STATUS=WARN TS=2026-07-23T05:06:25Z TYPE=backup-sla BAD=0 WARN=2 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-23T08:06:26+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-23T08:06:26+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=SERVICE_JOURNAL=2026-07-24T08:20:04+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health...\nDETAIL=SERVICE_JOURNAL=2026-07-24T08:20:04+03:00 pve01 homelab-backup-sla-health[3253899]: STATUS=WARN TS=2026-07-24T05:20:04Z TYPE=backup-sla BAD=0 WARN=4 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=SERVICE_JOURNAL=2026-07-24T08:20:04+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-24T08:20:04+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health.\nDETAIL=TIMER_SHOW_RC=0\nDETAIL=TIMER_PROPERTY=Unit=homelab-backup-sla-health.service\nDETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Sat 2026-07-25 08:08:57 MSK\nDETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 08:20:04 MSK\nDETAIL=TIMER_PROPERTY=LoadState=loaded\nDETAIL=TIMER_PROPERTY=ActiveState=active\nDETAIL=TIMER_PROPERTY=SubState=waiting\nDETAIL=TIMER_ENABLED_RESULT=enabled|RC=0\nDETAIL=HEALTH_METADATA=BYTES:154|MTIME_NS:1784876406380685648\nDETAIL=HEALTH_MATCH_COUNT=1\nDETAIL=HEALTH_LINE=STATUS=WARN TS=2026-07-24T07:00:05Z TYPE=backup-sla BAD=0 WARN=4 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nDETAIL=JOB_END=homelab_backup_sla_health\nDETAIL=JOB_BEGIN=homelab_mail_cloud_restore_drill\nDETAIL=JOB_FIELD=id=homelab_mail_cloud_restore_drill\nDETAIL=JOB_FIELD=name=homelab-mail-cloud-restore-drill\nDETAIL=JOB_FIELD=category=restore\nDETAIL=JOB_FIELD=critical=True\nDETAIL=JOB_FIELD=status=STALE\nDETAIL=JOB_FIELD=freshness=STALE\nDETAIL=JOB_FIELD=age_seconds=1247672\nDETAIL=JOB_FIELD=max_age_seconds=864000\nDETAIL=JOB_FIELD=source_kind=health_file\nDETAIL=JOB_FIELD=source_status=OK\nDETAIL=JOB_FIELD=source_mtime=2026-07-09T20:31:32Z\nDETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/homelab_mail_cloud_restore_drill/0-mail-cloud-restore-drill.txt\nDETAIL=JOB_FIELD=service=homelab-mail-cloud-restore-drill.service\nDETAIL=JOB_FIELD=service_result=success\nDETAIL=JOB_FIELD=service_exec_main_status=0\nDETAIL=JOB_FIELD=timer=homelab-mail-cloud-restore-drill.timer\nDETAIL=JOB_FIELD=timer_enabled=enabled\nDETAIL=JOB_FIELD=timer_active=active\nDETAIL=SERVICE_SHOW_RC=0\nDETAIL=SERVICE_PROPERTY=Result=success\nDETAIL=SERVICE_PROPERTY=ExecMainCode=0\nDETAIL=SERVICE_PROPERTY=ExecMainStatus=0\nDETAIL=SERVICE_PROPERTY=LoadState=loaded\nDETAIL=SERVICE_PROPERTY=ActiveState=inactive\nDETAIL=SERVICE_PROPERTY=SubState=dead\nDETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/homelab-mail-cloud-restore-drill.service\nDETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/homelab-mail-cloud-restore-drill.service.d/10-superseded-noop.conf /etc/systemd/system/homelab-mail-cloud-restore-drill.service.d/90-backup-matrix-refresh.conf\nDETAIL=SERVICE_JOURNAL_MATCH_COUNT=8\nDETAIL=SERVICE_JOURNAL=2026-07-13T07:38:36+03:00 pve01 systemd[1]: Starting homelab-mail-cloud-restore-drill.service - Homelab Mail Cloud restore drill...\nDETAIL=SERVICE_JOURNAL=2026-07-13T07:38:36+03:00 pve01 homelab-superseded-unit-ok[2401599]: STATUS=OK TS=2026-07-13T04:38:36Z TYPE=superseded-unit UNIT=homelab-mail-cloud-restore-drill.service REASON=superseded_by_safety_net_restore_check ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-13T07:38:36+03:00 pve01 systemd[1]: homelab-mail-cloud-restore-drill.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-13T07:38:36+03:00 pve01 systemd[1]: Finished homelab-mail-cloud-restore-drill.service - Homelab Mail Cloud restore drill.\nDETAIL=SERVICE_JOURNAL=2026-07-20T07:25:14+03:00 pve01 systemd[1]: Starting homelab-mail-cloud-restore-drill.service - Homelab Mail Cloud restore drill...\nDETAIL=SERVICE_JOURNAL=2026-07-20T07:25:14+03:00 pve01 homelab-superseded-unit-ok[2171079]: STATUS=OK TS=2026-07-20T04:25:14Z TYPE=superseded-unit UNIT=homelab-mail-cloud-restore-drill.service REASON=superseded_by_safety_net_restore_check ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-20T07:25:14+03:00 pve01 systemd[1]: homelab-mail-cloud-restore-drill.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-20T07:25:14+03:00 pve01 systemd[1]: Finished homelab-mail-cloud-restore-drill.service - Homelab Mail Cloud restore drill.\nDETAIL=TIMER_SHOW_RC=0\nDETAIL=TIMER_PROPERTY=Unit=homelab-mail-cloud-restore-drill.service\nDETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Mon 2026-07-27 07:34:52 MSK\nDETAIL=TIMER_PROPERTY=LastTriggerUSec=Mon 2026-07-20 07:25:14 MSK\nDETAIL=TIMER_PROPERTY=LoadState=loaded\nDETAIL=TIMER_PROPERTY=ActiveState=active\nDETAIL=TIMER_PROPERTY=SubState=waiting\nDETAIL=TIMER_ENABLED_RESULT=enabled|RC=0\nDETAIL=HEALTH_METADATA=BYTES:353|MTIME_NS:1783629092051018609\nDETAIL=HEALTH_MATCH_COUNT=1\nDETAIL=HEALTH_LINE=STATUS=OK TS=2026-07-09T20:31:32Z TYPE=legacy-mail-cloud-restore-drill-superseded MODE=SUPERSEDED_BY_CURRENT_PROOF BAD=0 WARN=0 PROOF=safety_net_control_plane_restore_check_OK ACTION=stale_health_refresh_only REASON=legacy_drill_superseded_but_health_file_mtime_must_stay_fresh REPORT=/var/lib/homelab-failed-unit-legacy-cleanup/local-cleanup-latest.md\nDETAIL=JOB_END=homelab_mail_cloud_restore_drill\nDETAIL=JOB_BEGIN=netbird_vps_backup\nDETAIL=JOB_FIELD=id=netbird_vps_backup\nDETAIL=JOB_FIELD=name=netbird-vps-backup\nDETAIL=JOB_FIELD=category=application\nDETAIL=JOB_FIELD=critical=True\nDETAIL=JOB_FIELD=status=FAIL\nDETAIL=JOB_FIELD=freshness=STALE\nDETAIL=JOB_FIELD=age_seconds=2007299\nDETAIL=JOB_FIELD=max_age_seconds=864000\nDETAIL=JOB_FIELD=source_kind=health_file\nDETAIL=JOB_FIELD=source_status=OK\nDETAIL=JOB_FIELD=source_mtime=2026-07-01T01:31:05Z\nDETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt\nDETAIL=JOB_FIELD=service=netbird-vps-backup.service\nDETAIL=JOB_FIELD=service_result=success\nDETAIL=JOB_FIELD=service_exec_main_status=0\nDETAIL=JOB_FIELD=timer=netbird-vps-backup.timer\nDETAIL=JOB_FIELD=timer_enabled=disabled\nDETAIL=JOB_FIELD=timer_active=inactive\nDETAIL=SERVICE_SHOW_RC=0\nDETAIL=SERVICE_PROPERTY=Result=success\nDETAIL=SERVICE_PROPERTY=ExecMainCode=0\nDETAIL=SERVICE_PROPERTY=ExecMainStatus=0\nDETAIL=SERVICE_PROPERTY=LoadState=loaded\nDETAIL=SERVICE_PROPERTY=ActiveState=inactive\nDETAIL=SERVICE_PROPERTY=SubState=dead\nDETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/netbird-vps-backup.service\nDETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/netbird-vps-backup.service.d/10-superseded-noop.conf /etc/systemd/system/netbird-vps-backup.service.d/90-backup-matrix-refresh.conf\nDETAIL=SERVICE_JOURNAL_MATCH_COUNT=28\nDETAIL=SERVICE_JOURNAL=2026-07-11T04:30:35+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging...\nDETAIL=SERVICE_JOURNAL=2026-07-11T04:30:35+03:00 pve01 homelab-superseded-unit-ok[1772638]: STATUS=OK TS=2026-07-11T01:30:35Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-11T04:30:35+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-11T04:30:35+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging.\nDETAIL=SERVICE_JOURNAL=2026-07-12T04:39:35+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging...\nDETAIL=SERVICE_JOURNAL=2026-07-12T04:39:35+03:00 pve01 homelab-superseded-unit-ok[1789866]: STATUS=OK TS=2026-07-12T01:39:35Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-12T04:39:35+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-12T04:39:35+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging.\nDETAIL=SERVICE_JOURNAL=2026-07-13T04:28:00+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging...\nDETAIL=SERVICE_JOURNAL=2026-07-13T04:28:00+03:00 pve01 homelab-superseded-unit-ok[1739137]: STATUS=OK TS=2026-07-13T01:28:00Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-13T04:28:00+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-13T04:28:00+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging.\nDETAIL=SERVICE_JOURNAL=2026-07-14T04:28:39+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging...\nDETAIL=SERVICE_JOURNAL=2026-07-14T04:28:39+03:00 pve01 homelab-superseded-unit-ok[2199841]: STATUS=OK TS=2026-07-14T01:28:39Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-14T04:28:39+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-14T04:28:39+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging.\nDETAIL=SERVICE_JOURNAL=2026-07-15T04:39:00+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging...\nDETAIL=SERVICE_JOURNAL=2026-07-15T04:39:00+03:00 pve01 homelab-superseded-unit-ok[2872498]: STATUS=OK TS=2026-07-15T01:39:00Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-15T04:39:00+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-15T04:39:00+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging.\nDETAIL=SERVICE_JOURNAL=2026-07-16T04:25:14+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging...\nDETAIL=SERVICE_JOURNAL=2026-07-16T04:25:14+03:00 pve01 homelab-superseded-unit-ok[3611661]: STATUS=OK TS=2026-07-16T01:25:14Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-16T04:25:14+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-16T04:25:14+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging.\nDETAIL=SERVICE_JOURNAL=2026-07-17T04:31:14+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging...\nDETAIL=SERVICE_JOURNAL=2026-07-17T04:31:14+03:00 pve01 homelab-superseded-unit-ok[3412653]: STATUS=OK TS=2026-07-17T01:31:14Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-17T04:31:14+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-17T04:31:14+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging.\nDETAIL=TIMER_SHOW_RC=0\nDETAIL=TIMER_PROPERTY=Unit=netbird-vps-backup.service\nDETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=\nDETAIL=TIMER_PROPERTY=LastTriggerUSec=\nDETAIL=TIMER_PROPERTY=LoadState=loaded\nDETAIL=TIMER_PROPERTY=ActiveState=inactive\nDETAIL=TIMER_PROPERTY=SubState=dead\nDETAIL=TIMER_ENABLED_RESULT=disabled|RC=1\nDETAIL=HEALTH_METADATA=BYTES:312|MTIME_NS:1782869465433646905\nDETAIL=HEALTH_MATCH_COUNT=7\nDETAIL=HEALTH_LINE=STATUS=OK\nDETAIL=HEALTH_LINE=SERVICE=netbird-vps-backup\nDETAIL=HEALTH_LINE=TIMESTAMP_UTC=20260701T013054Z\nDETAIL=HEALTH_LINE=SNAPSHOT=/mnt/staging/netbird-vps-backups/snapshots/20260701T013054Z\nDETAIL=HEALTH_LINE=LATEST_SYMLINK=/mnt/staging/netbird-vps-backups/snapshots/latest\nDETAIL=HEALTH_LINE=SIZE=32M\nDETAIL=HEALTH_LINE=SHA256=0226dd820cb9ede0ea9e8ea4af6cef656ffa54cfb819048c222a88f4c14d8db9\nDETAIL=JOB_END=netbird_vps_backup\nDETAIL=JOB_BEGIN=paperless_restore_dry_run\nDETAIL=JOB_FIELD=id=paperless_restore_dry_run\nDETAIL=JOB_FIELD=name=paperless-restore-dry-run\nDETAIL=JOB_FIELD=category=restore\nDETAIL=JOB_FIELD=critical=True\nDETAIL=JOB_FIELD=status=STALE\nDETAIL=JOB_FIELD=freshness=STALE\nDETAIL=JOB_FIELD=age_seconds=1261389\nDETAIL=JOB_FIELD=max_age_seconds=864000\nDETAIL=JOB_FIELD=source_kind=health_file\nDETAIL=JOB_FIELD=source_status=OK\nDETAIL=JOB_FIELD=source_mtime=2026-07-09T16:42:55Z\nDETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/paperless_restore_dry_run/0-paperless-restore.txt\nDETAIL=JOB_FIELD=service=paperless-restore-dry-run.service\nDETAIL=JOB_FIELD=service_result=success\nDETAIL=JOB_FIELD=service_exec_main_status=0\nDETAIL=JOB_FIELD=timer=paperless-restore-dry-run.timer\nDETAIL=JOB_FIELD=timer_enabled=enabled\nDETAIL=JOB_FIELD=timer_active=active\nDETAIL=SERVICE_SHOW_RC=0\nDETAIL=SERVICE_PROPERTY=Result=success\nDETAIL=SERVICE_PROPERTY=ExecMainCode=1\nDETAIL=SERVICE_PROPERTY=ExecMainStatus=0\nDETAIL=SERVICE_PROPERTY=LoadState=loaded\nDETAIL=SERVICE_PROPERTY=ActiveState=inactive\nDETAIL=SERVICE_PROPERTY=SubState=dead\nDETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/paperless-restore-dry-run.service\nDETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/paperless-restore-dry-run.service.d/10-superseded-noop.conf /etc/systemd/system/paperless-restore-dry-run.service.d/90-backup-matrix-refresh.conf\nDETAIL=SERVICE_JOURNAL_MATCH_COUNT=56\nDETAIL=SERVICE_JOURNAL=2026-07-11T10:08:13+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-11T10:08:13+03:00 pve01 homelab-superseded-unit-ok[2988079]: STATUS=OK TS=2026-07-11T07:08:13Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-11T10:08:13+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-11T10:08:13+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-12T10:04:59+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-12T10:04:59+03:00 pve01 homelab-superseded-unit-ok[2757392]: STATUS=OK TS=2026-07-12T07:04:59Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-12T10:04:59+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-12T10:04:59+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-13T10:07:25+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-13T10:07:25+03:00 pve01 homelab-superseded-unit-ok[2886298]: STATUS=OK TS=2026-07-13T07:07:25Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-13T10:07:25+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-13T10:07:25+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-14T09:56:36+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-14T09:56:36+03:00 pve01 homelab-superseded-unit-ok[3367368]: STATUS=OK TS=2026-07-14T06:56:36Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-14T09:56:36+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-14T09:56:36+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-15T10:12:54+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-15T10:12:54+03:00 pve01 homelab-superseded-unit-ok[4062186]: STATUS=OK TS=2026-07-15T07:12:54Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-15T10:12:54+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-15T10:12:54+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-16T11:40:40+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-16T11:40:40+03:00 pve01 homelab-superseded-unit-ok[12176]: STATUS=OK TS=2026-07-16T08:40:40Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-16T11:40:40+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-16T11:40:40+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-17T10:00:14+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-17T10:00:14+03:00 pve01 homelab-superseded-unit-ok[390020]: STATUS=OK TS=2026-07-17T07:00:14Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-17T10:00:14+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-17T10:00:14+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-18T09:57:16+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-18T09:57:16+03:00 pve01 homelab-superseded-unit-ok[1201647]: STATUS=OK TS=2026-07-18T06:57:16Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-18T09:57:16+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-18T09:57:16+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-19T10:09:17+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-19T10:09:17+03:00 pve01 homelab-superseded-unit-ok[1945203]: STATUS=OK TS=2026-07-19T07:09:17Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-19T10:09:17+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-19T10:09:17+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-20T10:04:06+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-20T10:04:06+03:00 pve01 homelab-superseded-unit-ok[2734120]: STATUS=OK TS=2026-07-20T07:04:06Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-20T10:04:06+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-20T10:04:06+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-21T10:02:41+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-21T10:02:41+03:00 pve01 homelab-superseded-unit-ok[3525432]: STATUS=OK TS=2026-07-21T07:02:41Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-21T10:02:41+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-21T10:02:41+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-22T10:11:14+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-22T10:11:14+03:00 pve01 homelab-superseded-unit-ok[4192753]: STATUS=OK TS=2026-07-22T07:11:14Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-22T10:11:14+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-22T10:11:14+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-23T10:12:14+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-23T10:12:14+03:00 pve01 homelab-superseded-unit-ok[752982]: STATUS=OK TS=2026-07-23T07:12:14Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-23T10:12:14+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-23T10:12:14+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=SERVICE_JOURNAL=2026-07-24T10:06:04+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker...\nDETAIL=SERVICE_JOURNAL=2026-07-24T10:06:04+03:00 pve01 homelab-superseded-unit-ok[3581148]: STATUS=OK TS=2026-07-24T07:06:04Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit\nDETAIL=SERVICE_JOURNAL=2026-07-24T10:06:04+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-24T10:06:04+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker.\nDETAIL=TIMER_SHOW_RC=0\nDETAIL=TIMER_PROPERTY=Unit=paperless-restore-dry-run.service\nDETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Sat 2026-07-25 10:03:38 MSK\nDETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 10:06:04 MSK\nDETAIL=TIMER_PROPERTY=LoadState=loaded\nDETAIL=TIMER_PROPERTY=ActiveState=active\nDETAIL=TIMER_PROPERTY=SubState=waiting\nDETAIL=TIMER_ENABLED_RESULT=enabled|RC=0\nDETAIL=HEALTH_METADATA=BYTES:232|MTIME_NS:1783615375842257624\nDETAIL=HEALTH_MATCH_COUNT=1\nDETAIL=HEALTH_LINE=STATUS=OK TS=2026-07-09T16:42:53Z TYPE=paperless-restore BAD=0 APP=paperless JOB=homelab-pve01-app-paperless-20260709T071949Z RESTORE_VERIFIED=true RESTORE_CHECK_APP=paperless JSON=/var/lib/homelab-backup/apps/latest-paperless.json\nDETAIL=JOB_END=paperless_restore_dry_run\nDETAIL=JOB_BEGIN=vm160_remote_backup\nDETAIL=JOB_FIELD=id=vm160_remote_backup\nDETAIL=JOB_FIELD=name=vm160-remote-backup\nDETAIL=JOB_FIELD=category=virtual-machine\nDETAIL=JOB_FIELD=critical=True\nDETAIL=JOB_FIELD=status=FAIL\nDETAIL=JOB_FIELD=freshness=STALE\nDETAIL=JOB_FIELD=age_seconds=286010\nDETAIL=JOB_FIELD=max_age_seconds=172800\nDETAIL=JOB_FIELD=source_kind=health_file\nDETAIL=JOB_FIELD=source_status=FAIL\nDETAIL=JOB_FIELD=source_mtime=2026-07-20T23:39:14Z\nDETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/vm160_remote_backup/0-vm160-remote-backup.txt\nDETAIL=JOB_FIELD=service=homelab-vm160-remote-backup.service\nDETAIL=JOB_FIELD=service_result=success\nDETAIL=JOB_FIELD=service_exec_main_status=0\nDETAIL=JOB_FIELD=timer=homelab-vm160-remote-backup.timer\nDETAIL=JOB_FIELD=timer_enabled=disabled\nDETAIL=JOB_FIELD=timer_active=inactive\nDETAIL=SERVICE_SHOW_RC=0\nDETAIL=SERVICE_PROPERTY=Result=success\nDETAIL=SERVICE_PROPERTY=ExecMainCode=0\nDETAIL=SERVICE_PROPERTY=ExecMainStatus=0\nDETAIL=SERVICE_PROPERTY=LoadState=loaded\nDETAIL=SERVICE_PROPERTY=ActiveState=inactive\nDETAIL=SERVICE_PROPERTY=SubState=dead\nDETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/homelab-vm160-remote-backup.service\nDETAIL=SERVICE_PROPERTY=DropInPaths=\nDETAIL=SERVICE_JOURNAL_MATCH_COUNT=50\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:02:41+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:15:43+03:00 pve01 homelab-vm160-remote-backup[3491061]: VM160_TARGET_FINALIZE_STATUS=OK\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3530071]: VM160_TARGET_PROMOTE_STATUS=OK\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: VM160_REMOTE_BACKUP_STATUS=OK\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260717T220242Z\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: ARCHIVE=/mnt/staging/vzdump/vm160-pve02-20260717T220242Z/vzdump-qemu-160-20260717T220242Z.vma.zst\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: SIZE_BYTES=84993482488\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: SHA256=a9562d8b60d7fd5e7ec73bc43d252cfea80ed12c05b32cca0a73d7bb88532411\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 systemd[1]: Finished homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Consumed 2min 44.280s CPU time, 15.8M memory peak.\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:38:18+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:51:18+03:00 pve01 homelab-vm160-remote-backup[3609200]: VM160_TARGET_FINALIZE_STATUS=OK\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3648126]: VM160_TARGET_PROMOTE_STATUS=OK\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: VM160_REMOTE_BACKUP_STATUS=OK\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: ARCHIVE=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z/vzdump-qemu-160-20260717T223819Z.vma.zst\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: SIZE_BYTES=84995216465\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: SHA256=257e10821e62e3e2f9318b238a5302a6db601dd597bfa3e0d77aba07f3b85e72\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Deactivated successfully.\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 systemd[1]: Finished homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nDETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Consumed 2min 44.842s CPU time, 16.1M memory peak.\nDETAIL=SERVICE_JOURNAL=2026-07-18T04:16:25+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nDETAIL=SERVICE_JOURNAL=2026-07-18T04:16:27+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=SERVICE_JOURNAL=2026-07-18T04:16:27+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nDETAIL=SERVICE_JOURNAL=2026-07-18T04:16:27+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nDETAIL=SERVICE_JOURNAL=2026-07-19T04:19:14+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nDETAIL=SERVICE_JOURNAL=2026-07-19T04:19:16+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=SERVICE_JOURNAL=2026-07-19T04:19:16+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nDETAIL=SERVICE_JOURNAL=2026-07-19T04:19:16+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nDETAIL=SERVICE_JOURNAL=2026-07-20T04:14:14+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nDETAIL=SERVICE_JOURNAL=2026-07-20T04:14:16+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=SERVICE_JOURNAL=2026-07-20T04:14:16+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nDETAIL=SERVICE_JOURNAL=2026-07-20T04:14:16+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nDETAIL=SERVICE_JOURNAL=2026-07-20T19:28:57+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nDETAIL=SERVICE_JOURNAL=2026-07-20T19:28:59+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=SERVICE_JOURNAL=2026-07-20T19:28:59+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nDETAIL=SERVICE_JOURNAL=2026-07-20T19:28:59+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:14:11+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:14:13+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:14:13+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:14:13+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:20:20+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:20:21+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:20:21+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:20:21+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:39:12+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:39:14+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:39:14+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nDETAIL=SERVICE_JOURNAL=2026-07-21T02:39:14+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nDETAIL=TIMER_SHOW_RC=0\nDETAIL=TIMER_PROPERTY=Unit=homelab-vm160-remote-backup.service\nDETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=\nDETAIL=TIMER_PROPERTY=LastTriggerUSec=\nDETAIL=TIMER_PROPERTY=LoadState=loaded\nDETAIL=TIMER_PROPERTY=ActiveState=inactive\nDETAIL=TIMER_PROPERTY=SubState=dead\nDETAIL=TIMER_ENABLED_RESULT=disabled|RC=1\nDETAIL=HEALTH_METADATA=BYTES:153|MTIME_NS:1784590754312087060\nDETAIL=HEALTH_MATCH_COUNT=3\nDETAIL=HEALTH_LINE=STATUS=FAIL\nDETAIL=HEALTH_LINE=SERVICE=homelab-vm160-remote-backup\nDETAIL=HEALTH_LINE=TIMESTAMP_UTC=20260720T233913Z\nDETAIL=JOB_END=vm160_remote_backup\nDETAIL=SECRET_VALUES_OUTPUT=NONE\nDETAIL=BACKUP_BAD_JOBS_ROOTCAUSE=PASS\nCHANGES_MADE=NO\n" + "output": "DETAIL=POLICY=DUAL_CLOUD_VERIFY_THEN_DELETE_ALL_LOCAL_PAYLOADS\nDETAIL=CHANGES_PLANNED=NO\nDETAIL=STATE_FILE=/var/lib/homelab-vm-backup/latest-vm-local-dumps-cloud.json\nDETAIL=STATE_FILE_STATUS=READABLE\nDETAIL=STATE_FIELD=$.job_id=homelab-pve01-vm-local-dumps-20260719T070607Z\nDETAIL=STATE_FIELD=$.status=OK\nDETAIL=STATE_FIELD=$.remote_a=pve01-mail-01-crypt:homelab-backups/pve01/vm-local-dumps/homelab-pve01-vm-local-dumps-20260719T070607Z\nDETAIL=STATE_FIELD=$.remote_b=pve01-mail-02-crypt:homelab-backups/pve01/vm-local-dumps/homelab-pve01-vm-local-dumps-20260719T070607Z\nDETAIL=STATE_FIELD=$.chunk_size_bytes=1073741824\nDETAIL=STATE_FIELD=$.local_dumps_deleted=False\nDETAIL=STATE_FILE=/var/lib/homelab-backup-matrix/state.json\nDETAIL=STATE_FILE_STATUS=READABLE\nDETAIL=STATE_FIELD=$.jobs[0].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[0].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[0].status=OK\nDETAIL=STATE_FIELD=$.jobs[1].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[1].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[1].status=OK\nDETAIL=STATE_FIELD=$.jobs[2].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[2].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[2].status=STALE\nDETAIL=STATE_FIELD=$.jobs[3].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[3].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[3].status=OK\nDETAIL=STATE_FIELD=$.jobs[4].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[4].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[4].status=OK\nDETAIL=STATE_FIELD=$.jobs[5].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[5].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[5].status=OK\nDETAIL=STATE_FIELD=$.jobs[6].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[6].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[6].status=OK\nDETAIL=STATE_FIELD=$.jobs[7].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[7].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[7].status=OK\nDETAIL=STATE_FIELD=$.jobs[8].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[8].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[8].status=DISABLED\nDETAIL=STATE_FIELD=$.jobs[9].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[9].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[9].status=OK\nDETAIL=STATE_FIELD=$.jobs[10].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[10].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[10].status=OK\nDETAIL=STATE_FIELD=$.jobs[11].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[11].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[11].status=OK\nDETAIL=STATE_FIELD=$.jobs[12].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[12].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[12].status=OK\nDETAIL=STATE_FIELD=$.jobs[13].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[13].source_status=WARN\nDETAIL=STATE_FIELD=$.jobs[13].status=FAIL\nDETAIL=STATE_FIELD=$.jobs[14].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[14].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[14].status=OK\nDETAIL=STATE_FIELD=$.jobs[15].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[15].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[15].status=OK\nDETAIL=STATE_FIELD=$.jobs[16].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[16].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[16].status=OK\nDETAIL=STATE_FIELD=$.jobs[17].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[17].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[17].status=DISABLED\nDETAIL=STATE_FIELD=$.jobs[18].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[18].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[18].status=OK\nDETAIL=STATE_FIELD=$.jobs[19].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[19].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[19].status=OK\nDETAIL=STATE_FIELD=$.jobs[20].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[20].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[20].status=OK\nDETAIL=STATE_FIELD=$.jobs[21].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[21].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[21].status=OK\nDETAIL=STATE_FIELD=$.jobs[22].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[22].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[22].status=STALE\nDETAIL=STATE_FIELD=$.jobs[23].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[23].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[23].status=DISABLED\nDETAIL=STATE_FIELD=$.jobs[24].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[24].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[24].status=OK\nDETAIL=STATE_FIELD=$.jobs[25].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[25].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[25].status=DISABLED\nDETAIL=STATE_FIELD=$.jobs[26].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[26].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[26].status=OK\nDETAIL=STATE_FIELD=$.jobs[27].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[27].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[27].status=OK\nDETAIL=STATE_FIELD=$.jobs[28].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[28].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[28].status=OK\nDETAIL=STATE_FIELD=$.jobs[29].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[29].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[29].status=OK\nDETAIL=STATE_FIELD=$.jobs[30].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[30].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[30].status=OK\nDETAIL=STATE_FIELD=$.jobs[31].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[31].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[31].status=OK\nDETAIL=STATE_FIELD=$.jobs[32].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[32].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[32].status=OK\nDETAIL=STATE_FIELD=$.jobs[33].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[33].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[33].status=OK\nDETAIL=STATE_FIELD=$.jobs[34].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[34].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[34].status=OK\nDETAIL=STATE_FIELD=$.jobs[35].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[35].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[35].status=FAIL\nDETAIL=STATE_FIELD=$.jobs[36].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[36].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[36].status=DISABLED\nDETAIL=STATE_FIELD=$.jobs[37].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[37].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[37].status=STALE\nDETAIL=STATE_FIELD=$.jobs[38].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[38].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[38].status=OK\nDETAIL=STATE_FIELD=$.jobs[39].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[39].source_status=OK\nDETAIL=STATE_FIELD=$.jobs[39].status=OK\nDETAIL=STATE_FIELD=$.jobs[40].service_exec_main_status=0\nDETAIL=STATE_FIELD=$.jobs[40].source_status=FAIL\nDETAIL=STATE_FIELD=$.jobs[40].status=FAIL\nDETAIL=STATE_FIELD=$.matrix_sha256=6b7c5df66c411d1e9630fb336c55007bcd923f1a0bfcc64d687fb1105011eceb\nDETAIL=STATE_FIELD=$.summary.jobs_bad=6\nDETAIL=STATE_FIELD=$.summary.jobs_disabled=5\nDETAIL=STATE_FIELD=$.summary.jobs_ok=30\nDETAIL=STATE_FIELD=$.summary.jobs_total=41\nDETAIL=STATE_FIELD=$.summary.jobs_unmapped=0\nDETAIL=STATE_FIELD=$.summary.status=FAIL\nDETAIL=UNIT_BEGIN=homelab-vm160-remote-backup.service\nDETAIL=UNIT_SHOW_RC=0\nDETAIL=UNIT_PROPERTY=Result=success\nDETAIL=UNIT_PROPERTY=ExecMainStatus=0\nDETAIL=UNIT_PROPERTY=ExecStart={ path=/usr/local/sbin/homelab-vm160-remote-backup ; argv[]=/usr/local/sbin/homelab-vm160-remote-backup ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nDETAIL=UNIT_PROPERTY=LoadState=loaded\nDETAIL=UNIT_PROPERTY=ActiveState=inactive\nDETAIL=UNIT_PROPERTY=SubState=dead\nDETAIL=UNIT_PROPERTY=FragmentPath=/etc/systemd/system/homelab-vm160-remote-backup.service\nDETAIL=SCRIPT_PATH=/usr/local/sbin/homelab-vm160-remote-backup\nDETAIL=SCRIPT_LINE=3:export SYSTEMD_PAGER=cat PAGER=cat SYSTEMD_LESS=FRXMK\nDETAIL=SCRIPT_LINE=8:PARENT=/mnt/staging/vzdump\nDETAIL=SCRIPT_LINE=10:HEALTH=/var/lib/homelab-health/vm160-remote-backup.txt\nDETAIL=SCRIPT_LINE=11:LOCK=/run/lock/homelab-vm160-remote-backup.lock\nDETAIL=SCRIPT_LINE=30:LOG=\"$(mktemp /tmp/vm160-remote-backup.XXXXXX.log)\"\nDETAIL=SCRIPT_LINE=31:cleanup(){\nDETAIL=SCRIPT_LINE=35:ssh \"${SSH[@]}\" \"$TARGET\" \"rm -rf -- '$DIR'\" >/dev/null 2>&1 || true\nDETAIL=SCRIPT_LINE=37:printf 'STATUS=FAIL\\nSERVICE=homelab-vm160-remote-backup\\nTIMESTAMP_UTC=%s\\nVMID=%s\\nTARGET_DIR=%s\\nRC=%s\\n' \"$TS\" \"$VMID\" \"$DIR\" \"$rc\" > \"$tmp\"\nDETAIL=SCRIPT_LINE=41:rm -f \"$LOG\"\nDETAIL=SCRIPT_LINE=44:trap cleanup EXIT\nDETAIL=SCRIPT_LINE=45:REMOTE_FREE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"df -B1 --output=avail '$PARENT' | awk 'NR==2{print \\$1}'\")\"\nDETAIL=SCRIPT_LINE=53:test \"$REMOTE_FREE\" -ge \"$REQUIRED\"\nDETAIL=SCRIPT_LINE=55:ssh \"${SSH[@]}\" \"$SOURCE\" \"vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7\" 2>\"$LOG\" | ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$PART'\"\nDETAIL=SCRIPT_LINE=57:ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$DIR/vzdump.log'\" < \"$LOG\"\nDETAIL=SCRIPT_LINE=59:SHA=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"sha256sum '$FINAL' | awk '{print \\$1}'\")\"\nDETAIL=SCRIPT_LINE=62:printf 'STATUS=OK\\nSERVICE=homelab-vm160-remote-backup\\nTIMESTAMP_UTC=%s\\nVMID=%s\\nSOURCE_NODE=pve02\\nTARGET_NODE=pve03\\nTARGET_DIR=%s\\nARCHIVE=%s\\nSIZE_BYTES=%s\\nSHA256=%s\\nREMOTE_RETENTION_GENERATIONS=1\\n' \"$TS\" \"$VMID\" \"$DIR\" \"$FINAL\" \"$SIZE\" \"$SHA\" > \"$tmp\"\nDETAIL=SCRIPT_LINE=66:rm -f \"$LOG\"\nDETAIL=SCRIPT_LINE=67:printf 'VM160_REMOTE_BACKUP_STATUS=OK\\nTARGET_DIR=%s\\nARCHIVE=%s\\nSIZE_BYTES=%s\\nSHA256=%s\\n' \"$DIR\" \"$FINAL\" \"$SIZE\" \"$SHA\"\nDETAIL=UNIT_END=homelab-vm160-remote-backup.service\nDETAIL=UNIT_BEGIN=homelab-vm-local-dumps-2cloud.service\nDETAIL=UNIT_SHOW_RC=0\nDETAIL=UNIT_PROPERTY=Result=success\nDETAIL=UNIT_PROPERTY=ExecMainStatus=0\nDETAIL=UNIT_PROPERTY=ExecStart={ path=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; argv[]=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nDETAIL=UNIT_PROPERTY=LoadState=loaded\nDETAIL=UNIT_PROPERTY=ActiveState=inactive\nDETAIL=UNIT_PROPERTY=SubState=dead\nDETAIL=UNIT_PROPERTY=FragmentPath=/etc/systemd/system/homelab-vm-local-dumps-2cloud.service\nDETAIL=SCRIPT_PATH=/usr/local/sbin/homelab-vm-local-dumps-2cloud\nDETAIL=SCRIPT_LINE=4:OUT=\"/var/lib/homelab-health/vm-local-dumps-cloud.txt\"\nDETAIL=SCRIPT_LINE=6:LOG=\"$ROOT/vm-local-dumps-2cloud-latest.log\"\nDETAIL=SCRIPT_LINE=7:VERIFY_ROOT=\"/var/tmp/homelab-vm-local-dumps-verify\"\nDETAIL=SCRIPT_LINE=12:mkdir -p /var/lib/homelab-health \"$ROOT\" \"$VERIFY_ROOT\" \"$CHUNK_ROOT\"\nDETAIL=SCRIPT_LINE=55:SHA=\"$(sha256sum \"$F\" | cut -d\" \" -f1)\"\nDETAIL=SCRIPT_LINE=64:rm -rf \"$CDIR\"\nDETAIL=SCRIPT_LINE=69:(cd \"$CDIR\" && sha256sum part-* > chunks.sha256)\nDETAIL=SCRIPT_LINE=92:\"sha256\":p[3],\nDETAIL=SCRIPT_LINE=112:DEST_A=\"${REMOTE_A%/}/${NODE}/vm-local-dumps/${JOB_ID}\"\nDETAIL=SCRIPT_LINE=113:DEST_B=\"${REMOTE_B%/}/${NODE}/vm-local-dumps/${JOB_ID}\"\nDETAIL=SCRIPT_LINE=115:upload_one_dest() {\nDETAIL=SCRIPT_LINE=117:rclone mkdir \"$DEST/files\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SCRIPT_LINE=121:echo \"UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts\" >> \"$LOG\"\nDETAIL=SCRIPT_LINE=122:rclone mkdir \"$DEST/files/$bn.chunks\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SCRIPT_LINE=126:rclone copyto \"$part\" \"$DEST/files/$bn.chunks/$pn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SCRIPT_LINE=128:rclone copyto \"$CDIR/chunks.sha256\" \"$DEST/files/$bn.chunks/chunks.sha256\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SCRIPT_LINE=130:echo \"UPLOAD_DIRECT dest=$DEST file=$bn size=$size\" >> \"$LOG\"\nDETAIL=SCRIPT_LINE=131:rclone copyto \"$path\" \"$DEST/files/$bn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SCRIPT_LINE=135:rclone copyto \"$CHKS\" \"$DEST/checksums.txt\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SCRIPT_LINE=136:rclone copyto \"$FILES_TSV\" \"$DEST/files.tsv\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SCRIPT_LINE=137:rclone copyto \"$MAN\" \"$DEST/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SCRIPT_LINE=140:verify_one_dest() {\nDETAIL=SCRIPT_LINE=143:VBASE=\"$VERIFY_ROOT/$JOB_ID/$LABEL\"\nDETAIL=SCRIPT_LINE=144:rm -rf \"$VBASE\"\nDETAIL=SCRIPT_LINE=146:rclone copyto \"$DEST/manifest.json\" \"$VBASE/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SCRIPT_LINE=153:echo \"VERIFY_CHUNKED label=$LABEL file=$bn parts=$parts\" >> \"$LOG\"\nDETAIL=SCRIPT_LINE=154:rclone copy \"$DEST/files/$bn.chunks\" \"$VDIR\" --filter \"+ part-*\" --filter \"+ chunks.sha256\" --filter \"- *\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SCRIPT_LINE=155:(cd \"$VDIR\" && sha256sum -c chunks.sha256) >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SCRIPT_LINE=156:CALC=\"$(find \"$VDIR\" -maxdepth 1 -type f -name 'part-*' | sort | xargs cat | sha256sum | cut -d\" \" -f1)\"\nDETAIL=SCRIPT_LINE=163:rm -rf \"$VDIR\"\nDETAIL=SCRIPT_LINE=165:echo \"VERIFY_DIRECT label=$LABEL file=$bn\" >> \"$LOG\"\nDETAIL=SCRIPT_LINE=166:rclone copyto \"$DEST/files/$bn\" \"$VBASE/$bn\" --transfers 1 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SCRIPT_LINE=167:printf \"%s %s\\n\" \"$sha\" \"$VBASE/$bn\" | sha256sum -c - >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SCRIPT_LINE=168:rm -f \"$VBASE/$bn\"\nDETAIL=SCRIPT_LINE=171:rm -rf \"$VBASE\"\nDETAIL=SCRIPT_LINE=175:upload_one_dest \"$DEST_A\"\nDETAIL=SCRIPT_LINE=176:upload_one_dest \"$DEST_B\"\nDETAIL=SCRIPT_LINE=180:verify_one_dest \"A\" \"$DEST_A\"\nDETAIL=SCRIPT_LINE=181:verify_one_dest \"B\" \"$DEST_B\"\nDETAIL=SCRIPT_LINE=184:rm -rf \"$VERIFY_ROOT/$JOB_ID\"\nDETAIL=SCRIPT_LINE=185:rm -rf \"$CHUNK_ROOT/$JOB_ID\"\nDETAIL=SCRIPT_LINE=193:--arg remote_a \"$DEST_A\" \\\nDETAIL=SCRIPT_LINE=194:--arg remote_b \"$DEST_B\" \\\nDETAIL=SCRIPT_LINE=200:\"{job_id:\\$job_id,status:\\$status,node:\\$node,remote_a:\\$remote_a,remote_b:\\$remote_b,count:\\$count,total_bytes:\\$total_bytes,chunked_count:\\$chunked_count,chunk_size_bytes:\\$chunk_size_bytes,log:\\$log,local_dumps_deleted:false}\" \\\nDETAIL=SCRIPT_LINE=201:> \"$ROOT/latest-vm-local-dumps-cloud.json\"\nDETAIL=SCRIPT_LINE=203:echo \"STATUS=$STATUS TS=$TS TYPE=vm-local-dumps-cloud BAD=$BAD COUNT=$COUNT CHUNKED=$CHUNKED TOTAL_BYTES=$TOTAL JOB=$JOB_ID REMOTE_A=$DEST_A REMOTE_B=$DEST_B LOG=$LOG\" > \"$OUT\"\nDETAIL=SCRIPT_LINE=204:cat \"$OUT\"\nDETAIL=UNIT_END=homelab-vm-local-dumps-2cloud.service\nDETAIL=PAYLOAD_HOST_BEGIN=pve01\nDETAIL=PAYLOAD_ROOT=pve01:/var/lib/vz/dump\nDETAIL=PAYLOAD_ROOT_STATUS=PRESENT\nDETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst|BYTES=329541843|AGE_SECONDS=293553\nDETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-110-2026_07_24-03_30_02.tar.zst|BYTES=339140622|AGE_SECONDS=34352\nDETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-112-2026_07_22-03_30_09.tar.zst|BYTES=222794054|AGE_SECONDS=207149\nDETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-110-2026_07_22-03_30_01.tar.zst|BYTES=332951369|AGE_SECONDS=207154\nDETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst|BYTES=7021952392|AGE_SECONDS=293496\nDETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-qemu-150-2026_07_24-03_30_16.vma.zst|BYTES=7016624549|AGE_SECONDS=34298\nDETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-qemu-150-2026_07_22-03_30_14.vma.zst|BYTES=7052176114|AGE_SECONDS=207100\nDETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst|BYTES=222777732|AGE_SECONDS=293547\nDETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-112-2026_07_24-03_30_11.tar.zst|BYTES=222859457|AGE_SECONDS=34347\nDETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-qemu-150-2026_07_23-03_30_16.vma.zst|BYTES=7024165999|AGE_SECONDS=120695\nDETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-110-2026_07_23-03_30_02.tar.zst|BYTES=336245409|AGE_SECONDS=120752\nDETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-112-2026_07_23-03_30_11.tar.zst|BYTES=222808025|AGE_SECONDS=120747\nDETAIL=PAYLOAD_ROOT=pve01:/mnt/staging/vzdump\nDETAIL=PAYLOAD_ROOT_STATUS=PRESENT\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260705T004739Z/pve03/vzdump-qemu-130-2026_07_05-03_49_34.vma.zst|BYTES=11242736440|AGE_SECONDS=1674545\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260705T004739Z/pve02/vzdump-lxc-111-2026_07_05-03_48_52.tar.zst|BYTES=255533195|AGE_SECONDS=1674805\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260705T004739Z/pve02/vzdump-lxc-113-2026_07_05-03_49_19.tar.zst|BYTES=220380760|AGE_SECONDS=1674791\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260705T004739Z/pve01/vzdump-qemu-150-2026_07_05-03_47_52.vma.zst|BYTES=7004920660|AGE_SECONDS=1674842\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260705T004739Z/pve01/vzdump-lxc-112-2026_07_05-03_47_46.tar.zst|BYTES=222468077|AGE_SECONDS=1674891\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260705T004739Z/pve01/vzdump-lxc-110-2026_07_05-03_47_40.tar.zst|BYTES=269221740|AGE_SECONDS=1674897\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260628T004739Z/pve03/vzdump-qemu-130-2026_06_28-03_49_15.vma.zst|BYTES=27024990491|AGE_SECONDS=2279192\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260628T004739Z/pve02/vzdump-lxc-113-2026_06_28-03_49_00.tar.zst|BYTES=220326595|AGE_SECONDS=2279610\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260628T004739Z/pve02/vzdump-lxc-111-2026_06_28-03_48_46.tar.zst|BYTES=242676818|AGE_SECONDS=2279624\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260628T004739Z/pve01/vzdump-lxc-112-2026_06_28-03_47_46.tar.zst|BYTES=222365246|AGE_SECONDS=2279692\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260628T004739Z/pve01/vzdump-lxc-110-2026_06_28-03_47_40.tar.zst|BYTES=253314990|AGE_SECONDS=2279698\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260628T004739Z/pve01/vzdump-qemu-150-2026_06_28-03_47_51.vma.zst|BYTES=7003671925|AGE_SECONDS=2279646\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260719T004537Z/pve03/vzdump-qemu-130-2026_07_19-03_48_01.vma.zst|BYTES=11043726130|AGE_SECONDS=465037\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260719T004537Z/pve02/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst|BYTES=220049041|AGE_SECONDS=465284\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260719T004537Z/pve02/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst|BYTES=284443608|AGE_SECONDS=465301\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260719T004537Z/pve01/vzdump-qemu-150-2026_07_19-03_45_58.vma.zst|BYTES=7008027143|AGE_SECONDS=465328\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260719T004537Z/pve01/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst|BYTES=323668231|AGE_SECONDS=465417\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260719T004537Z/pve01/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst|BYTES=222745965|AGE_SECONDS=465406\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260712T004231Z/pve03/vzdump-qemu-130-2026_07_12-03_44_28.vma.zst|BYTES=10813915936|AGE_SECONDS=1070055\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260712T004231Z/pve02/vzdump-lxc-111-2026_07_12-03_43_45.tar.zst|BYTES=270924730|AGE_SECONDS=1070317\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260712T004231Z/pve02/vzdump-lxc-113-2026_07_12-03_44_07.tar.zst|BYTES=220482178|AGE_SECONDS=1070297\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260712T004231Z/pve01/vzdump-lxc-112-2026_07_12-03_42_39.tar.zst|BYTES=222566793|AGE_SECONDS=1070398\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260712T004231Z/pve01/vzdump-qemu-150-2026_07_12-03_42_45.vma.zst|BYTES=7014397787|AGE_SECONDS=1070350\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260712T004231Z/pve01/vzdump-lxc-110-2026_07_12-03_42_31.tar.zst|BYTES=302840996|AGE_SECONDS=1070404\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/full-manual-backup-20260615T092720Z/pve03/vzdump-qemu-130-2026_06_15-12_27_45.vma.zst|BYTES=829658873|AGE_SECONDS=3371676\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/full-manual-backup-20260615T092720Z/pve02/vzdump-lxc-113-2026_06_15-12_27_32.tar.zst|BYTES=220018661|AGE_SECONDS=3371701\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/full-manual-backup-20260615T092720Z/pve02/vzdump-lxc-111-2026_06_15-12_27_22.tar.zst|BYTES=228398861|AGE_SECONDS=3371712\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/full-manual-backup-20260615T092720Z/pve01/vzdump-qemu-150-2026_06_15-12_19_05.vma.zst|BYTES=6957596210|AGE_SECONDS=3372173\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/full-manual-backup-20260615T092720Z/pve01/vzdump-lxc-112-2026_06_15-12_23_56.tar.zst|BYTES=222110027|AGE_SECONDS=3371924\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/full-manual-backup-20260615T092720Z/pve01/vzdump-lxc-110-2026_06_15-12_23_52.tar.zst|BYTES=230562036|AGE_SECONDS=3371928\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/vm170-vm171/vzdump-qemu-170-2026_07_24-02_15_07.vma.zst|BYTES=11828704880|AGE_SECONDS=37618\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/vm170-vm171/vzdump-qemu-171-2026_07_23-02_36_37.vma.zst|BYTES=2157597134|AGE_SECONDS=122728\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/vm170-vm171/vzdump-qemu-170-2026_07_23-02_15_17.vma.zst|BYTES=11842330422|AGE_SECONDS=124008\nDETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/vm170-vm171/vzdump-qemu-171-2026_07_24-02_36_27.vma.zst|BYTES=2158869638|AGE_SECONDS=36338\nDETAIL=PAYLOAD_ROOT=pve01:/var/lib/homelab-vm-backup/staging\nDETAIL=PAYLOAD_ROOT_STATUS=ABSENT\nDETAIL=PAYLOAD_HOST_SUMMARY=pve01|COUNT=46|BYTES=159150279782\nDETAIL=PAYLOAD_HOST_END=pve01\nDETAIL=PAYLOAD_HOST_BEGIN=pve02\nDETAIL=PAYLOAD_ROOT=pve02:/var/lib/vz/dump\nDETAIL=PAYLOAD_ROOT_STATUS=PRESENT\nDETAIL=LOCAL_PAYLOAD=pve02:/var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.vma.zst::997503422::1782865871.1700059310/var/lib/vz/dump/vzdump-lxc-111-2026_07_24-03_30_03.tar.zst::293632136::1784853018.7398517980/var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.vma.zst::868048369::1782779468.4903256780/var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.tar.zst::220080970::1784593833.3910787180/var/lib/vz/dump/vzdump-lxc-111-2026_07_22-03_30_01.tar.zst::290059742::1784680222.8748248510/var/lib/vz/dump/vzdump-lxc-111-2026_07_23-03_30_02.tar.zst::292185437::1784766624.6563558990/var/lib/vz/dump/vzdump-lxc-113-2026_07_23-03_30_24.tar.zst::220108944::1784766637.5724249240/var/lib/vz/dump/vzdump-lxc-113-2026_07_22-03_30_23.tar.zst::220095608::1784680240.6809207660/var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.tar.zst::288349157::1784593821.2480085730/var/lib/vz/dump/vzdump-lxc-113-2026_07_24-03_30_19.tar.zst|BYTES=220234127|AGE_SECONDS=34333\nDETAIL=PAYLOAD_ROOT=pve02:/mnt/staging/vzdump\nDETAIL=PAYLOAD_ROOT_STATUS=PRESENT\nDETAIL=LOCAL_PAYLOAD=pve02:/mnt/staging/vzdump/auto-pve02-20260615T121549Z/vzdump-lxc-113-2026_06_15-15_16_01.tar.zst::220085886::1781525770.6722751820/mnt/staging/vzdump/auto-pve02-20260615T121549Z/vzdump-lxc-111-2026_06_15-15_15_50.tar.zst::228433321::1781525759.9832231610/mnt/staging/vzdump/auto-pve02-20260615T184325Z/vzdump-lxc-111-2026_06_15-21_43_27.tar.zst::228695245::1781549016.3828865780/mnt/staging/vzdump/auto-pve02-20260615T184325Z/vzdump-lxc-113-2026_06_15-21_43_37.tar.zst::220088590::1781549027.0039397060/mnt/staging/vzdump/auto-pve02-20260615T145425Z/vzdump-lxc-111-2026_06_15-17_54_26.tar.zst::228364545::1781535276.0106286580/mnt/staging/vzdump/auto-pve02-20260615T145425Z/vzdump-lxc-113-2026_06_15-17_54_37.tar.zst::220087147::1781535286.6446806260/mnt/staging/vzdump/auto-pve02-20260615T125928Z/vzdump-lxc-111-2026_06_15-15_59_30.tar.zst::228433824::1781528379.7579730690/mnt/staging/vzdump/auto-pve02-20260615T125928Z/vzdump-lxc-113-2026_06_15-15_59_40.tar.zst::220086313::1781528390.6340260000/mnt/staging/vzdump/auto-pve02-20260628T004845Z/vzdump-lxc-113-2026_06_28-03_49_00.tar.zst::220326595::1782607753.1428569490/mnt/staging/vzdump/auto-pve02-20260628T004845Z/vzdump-lxc-111-2026_06_28-03_48_46.tar.zst::242676818::1782607739.4867871320/mnt/staging/vzdump/auto-pve02-20260615T201446Z/vzdump-lxc-111-2026_06_15-23_14_48.tar.zst::228706014::1781554497.6364731750/mnt/staging/vzdump/auto-pve02-20260615T201446Z/vzdump-lxc-113-2026_06_15-23_14_58.tar.zst::220089008::1781554508.5315278350/mnt/staging/vzdump/auto-pve02-20260705T004851Z/vzdump-lxc-111-2026_07_05-03_48_52.tar.zst::255533195::1783212558.3825236020/mnt/staging/vzdump/auto-pve02-20260705T004851Z/vzdump-lxc-113-2026_07_05-03_49_19.tar.zst::220380760::1783212571.8805933550/mnt/staging/vzdump/auto-pve02-20260719T004725Z/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst::284443608::1784422061.8525503200/mnt/staging/vzdump/auto-pve02-20260719T004725Z/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst::220049041::1784422078.9256407960/mnt/staging/vzdump/auto-pve02-20260712T004344Z/vzdump-lxc-111-2026_07_12-03_43_45.tar.zst::270924730::1783817046.4081420260/mnt/staging/vzdump/auto-pve02-20260712T004344Z/vzdump-lxc-113-2026_07_12-03_44_07.tar.zst::220482178::1783817065.7622435280/mnt/staging/vzdump/auto-pve02-20260621T004841Z/vzdump-lxc-113-2026_06_21-03_48_54.tar.zst::220165475::1782002947.0683211180/mnt/staging/vzdump/auto-pve02-20260621T004841Z/vzdump-lxc-111-2026_06_21-03_48_42.tar.zst::232731208::1782002933.6692546410/mnt/staging/vzdump/auto-pve02-20260615T161514Z/vzdump-lxc-111-2026_06_15-19_15_16.tar.zst::228462340::1781540125.7713503770/mnt/staging/vzdump/auto-pve02-20260615T161514Z/vzdump-lxc-113-2026_06_15-19_15_26.tar.zst::220087434::1781540136.7614046640/mnt/staging/vzdump/manual-pve02-20260615T092720Z/vzdump-lxc-113-2026_06_15-12_27_32.tar.zst::220018661::1781515662.0924756330/mnt/staging/vzdump/manual-pve02-20260615T092720Z/vzdump-lxc-111-2026_06_15-12_27_22.tar.zst::228398861::1781515651.4234214190/mnt/staging/vzdump/auto-pve02-20260616T023537Z/vzdump-lxc-111-2026_06_16-05_35_38.tar.zst::228693413::1781577348.0265431350/mnt/staging/vzdump/auto-pve02-20260616T023537Z/vzdump-lxc-113-2026_06_16-05_35_49.tar.zst::220093717::1781577358.9785981020/mnt/staging/vzdump/auto-pve02-20260615T095831Z/vzdump-lxc-111-2026_06_15-12_58_32.tar.zst::228398872::1781517522.2069308430/mnt/staging/vzdump/auto-pve02-20260615T095831Z/vzdump-lxc-113-2026_06_15-12_58_43.tar.zst|BYTES=220019157|AGE_SECONDS=3369831\nDETAIL=PAYLOAD_HOST_SUMMARY=pve02|COUNT=2|BYTES=440253284\nDETAIL=PAYLOAD_HOST_END=pve02\nDETAIL=PAYLOAD_HOST_BEGIN=pve03\nDETAIL=PAYLOAD_ROOT=pve03:/var/lib/vz/dump\nDETAIL=PAYLOAD_ROOT_STATUS=PRESENT\nDETAIL=LOCAL_PAYLOAD=pve03:/var/lib/vz/dump/vzdump-qemu-130-2026_07_23-03_30_03.vma.zst::11139455731::1784766836.0161157970/var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.vma.zst::11043834894::1784594034.3725716230/var/lib/vz/dump/vzdump-qemu-130-2026_07_22-03_30_02.vma.zst::11072513783::1784680438.8961071480/var/lib/vz/dump/vzdump-qemu-190-2026_07_23-03_33_56.vma.zst::1268419395::1784766881.2902288360/var/lib/vz/dump/vzdump-qemu-190-2026_07_24-03_34_00.vma.zst::1278418128::1784853285.0606354470/var/lib/vz/dump/vzdump-qemu-130-2026_07_24-03_30_02.vma.zst::11170433004::1784853240.1424086410/var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.vma.zst::1137072350::1784594077.6276811360/var/lib/vz/dump/vzdump-qemu-190-2026_07_22-03_33_59.vma.zst|BYTES=1177407443|AGE_SECONDS=206880\nDETAIL=PAYLOAD_ROOT=pve03:/mnt/staging/vzdump\nDETAIL=PAYLOAD_ROOT_STATUS=PRESENT\nDETAIL=LOCAL_PAYLOAD=pve03:/mnt/staging/vzdump/auto-pve03-20260719T004800Z/vzdump-qemu-130-2026_07_19-03_48_01.vma.zst::11043726130::1784422326.4811235470/mnt/staging/vzdump/auto-pve03-20260712T004427Z/vzdump-qemu-130-2026_07_12-03_44_28.vma.zst::10813915936::1783817308.4882503530/mnt/staging/vzdump/vm160-pve02-20260717T223819Z/vzdump-qemu-160-20260717T223819Z.vma.zst|BYTES=84995216465|AGE_SECONDS=559051\nDETAIL=PAYLOAD_HOST_SUMMARY=pve03|COUNT=2|BYTES=86172623908\nDETAIL=PAYLOAD_HOST_END=pve03\nDETAIL=POLICY_SUCCESS_REQUIRES=CLOUD1_VERIFIED,CLOUD2_VERIFIED,LOCAL_PAYLOAD_COUNT_ZERO\nDETAIL=SECRET_VALUES_OUTPUT=NONE\nDETAIL=BACKUP_NO_LOCAL_INVARIANT_AUDIT=PASS\nCHANGES_MADE=NO\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index d54e648..446552e 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,5 +1,5 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=HOMELAB-BACKUP-BAD-JOBS-ROOTCAUSE-431 +COMMAND_ID=HOMELAB-BACKUP-NO-LOCAL-INVARIANT-AUDIT-432 STATUS=OK RC=0 HOST=pve01 @@ -9,548 +9,321 @@ REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 -COMMAND_SHA256=8635e368c9b3e168a7302affcd2c133d8f85547164b3f3352fc0cbad679edc98 +COMMAND_SHA256=3d74c7e2671b37a475e371e668d24c5a96cd5db6b4171efbae25b123278b6883 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGES_MADE=false SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=e39cc1d76af62d7050936445c27d21728aaeb6ca00ab2430c6cb3963a7da3907 -SANITIZED_OUTPUT_SHA256=e39cc1d76af62d7050936445c27d21728aaeb6ca00ab2430c6cb3963a7da3907 +RAW_EVIDENCE_SHA256=cc2f90a2cae34633773e4468f017e38bc9d8d85abc7d3c929e39aa9f0e9d278b +SANITIZED_OUTPUT_SHA256=cc2f90a2cae34633773e4468f017e38bc9d8d85abc7d3c929e39aa9f0e9d278b OUTPUT_BEGIN -DETAIL=MATRIX_GENERATED_AT=2026-07-24T07:06:04Z -DETAIL=TARGET_JOB_COUNT=6 -DETAIL=JOB_BEGIN=filebrowser_restore_validate -DETAIL=JOB_FIELD=id=filebrowser_restore_validate -DETAIL=JOB_FIELD=name=filebrowser-restore-validate -DETAIL=JOB_FIELD=category=restore -DETAIL=JOB_FIELD=critical=True -DETAIL=JOB_FIELD=status=STALE -DETAIL=JOB_FIELD=freshness=STALE -DETAIL=JOB_FIELD=age_seconds=1275827 -DETAIL=JOB_FIELD=max_age_seconds=864000 -DETAIL=JOB_FIELD=source_kind=health_file -DETAIL=JOB_FIELD=source_status=OK -DETAIL=JOB_FIELD=source_mtime=2026-07-09T12:42:17Z -DETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt -DETAIL=JOB_FIELD=service=filebrowser-restore-validate.service -DETAIL=JOB_FIELD=service_result=success -DETAIL=JOB_FIELD=service_exec_main_status=0 -DETAIL=JOB_FIELD=timer=filebrowser-restore-validate.timer -DETAIL=JOB_FIELD=timer_enabled=enabled -DETAIL=JOB_FIELD=timer_active=active -DETAIL=SERVICE_SHOW_RC=0 -DETAIL=SERVICE_PROPERTY=Result=success -DETAIL=SERVICE_PROPERTY=ExecMainCode=1 -DETAIL=SERVICE_PROPERTY=ExecMainStatus=0 -DETAIL=SERVICE_PROPERTY=LoadState=loaded -DETAIL=SERVICE_PROPERTY=ActiveState=inactive -DETAIL=SERVICE_PROPERTY=SubState=dead -DETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/filebrowser-restore-validate.service -DETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/filebrowser-restore-validate.service.d/10-superseded-noop.conf /etc/systemd/system/filebrowser-restore-validate.service.d/90-backup-matrix-refresh.conf -DETAIL=SERVICE_JOURNAL_MATCH_COUNT=56 -DETAIL=SERVICE_JOURNAL=2026-07-11T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... -DETAIL=SERVICE_JOURNAL=2026-07-11T04:10:00+03:00 pve01 homelab-superseded-unit-ok[1686870]: STATUS=OK TS=2026-07-11T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-11T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-11T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. -DETAIL=SERVICE_JOURNAL=2026-07-12T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... -DETAIL=SERVICE_JOURNAL=2026-07-12T04:10:00+03:00 pve01 homelab-superseded-unit-ok[1699058]: STATUS=OK TS=2026-07-12T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-12T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-12T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. -DETAIL=SERVICE_JOURNAL=2026-07-13T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... -DETAIL=SERVICE_JOURNAL=2026-07-13T04:10:00+03:00 pve01 homelab-superseded-unit-ok[1681015]: STATUS=OK TS=2026-07-13T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-13T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-13T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. -DETAIL=SERVICE_JOURNAL=2026-07-14T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... -DETAIL=SERVICE_JOURNAL=2026-07-14T04:10:00+03:00 pve01 homelab-superseded-unit-ok[2132068]: STATUS=OK TS=2026-07-14T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-14T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-14T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. -DETAIL=SERVICE_JOURNAL=2026-07-15T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... -DETAIL=SERVICE_JOURNAL=2026-07-15T04:10:00+03:00 pve01 homelab-superseded-unit-ok[2783319]: STATUS=OK TS=2026-07-15T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-15T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-15T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. -DETAIL=SERVICE_JOURNAL=2026-07-16T04:10:00+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... -DETAIL=SERVICE_JOURNAL=2026-07-16T04:10:00+03:00 pve01 homelab-superseded-unit-ok[3557730]: STATUS=OK TS=2026-07-16T01:10:00Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-16T04:10:00+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-16T04:10:00+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. -DETAIL=SERVICE_JOURNAL=2026-07-17T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... -DETAIL=SERVICE_JOURNAL=2026-07-17T04:10:14+03:00 pve01 homelab-superseded-unit-ok[3346747]: STATUS=OK TS=2026-07-17T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-17T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-17T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. -DETAIL=SERVICE_JOURNAL=2026-07-18T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... -DETAIL=SERVICE_JOURNAL=2026-07-18T04:10:14+03:00 pve01 homelab-superseded-unit-ok[4129958]: STATUS=OK TS=2026-07-18T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-18T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-18T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. -DETAIL=SERVICE_JOURNAL=2026-07-19T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... -DETAIL=SERVICE_JOURNAL=2026-07-19T04:10:14+03:00 pve01 homelab-superseded-unit-ok[738797]: STATUS=OK TS=2026-07-19T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-19T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-19T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. -DETAIL=SERVICE_JOURNAL=2026-07-20T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... -DETAIL=SERVICE_JOURNAL=2026-07-20T04:10:14+03:00 pve01 homelab-superseded-unit-ok[1448291]: STATUS=OK TS=2026-07-20T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-20T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-20T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. -DETAIL=SERVICE_JOURNAL=2026-07-21T04:10:07+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... -DETAIL=SERVICE_JOURNAL=2026-07-21T04:10:07+03:00 pve01 homelab-superseded-unit-ok[2324924]: STATUS=OK TS=2026-07-21T01:10:07Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-21T04:10:07+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-21T04:10:07+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. -DETAIL=SERVICE_JOURNAL=2026-07-22T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... -DETAIL=SERVICE_JOURNAL=2026-07-22T04:10:14+03:00 pve01 homelab-superseded-unit-ok[3008751]: STATUS=OK TS=2026-07-22T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-22T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-22T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. -DETAIL=SERVICE_JOURNAL=2026-07-23T04:10:14+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... -DETAIL=SERVICE_JOURNAL=2026-07-23T04:10:14+03:00 pve01 homelab-superseded-unit-ok[3700126]: STATUS=OK TS=2026-07-23T01:10:14Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-23T04:10:14+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-23T04:10:14+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. -DETAIL=SERVICE_JOURNAL=2026-07-24T04:10:04+03:00 pve01 systemd[1]: Starting filebrowser-restore-validate.service - File Browser restore validation... -DETAIL=SERVICE_JOURNAL=2026-07-24T04:10:04+03:00 pve01 homelab-superseded-unit-ok[2416005]: STATUS=OK TS=2026-07-24T01:10:04Z TYPE=superseded-unit UNIT=filebrowser-restore-validate.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-24T04:10:04+03:00 pve01 systemd[1]: filebrowser-restore-validate.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-24T04:10:04+03:00 pve01 systemd[1]: Finished filebrowser-restore-validate.service - File Browser restore validation. -DETAIL=TIMER_SHOW_RC=0 -DETAIL=TIMER_PROPERTY=Unit=filebrowser-restore-validate.service -DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Sat 2026-07-25 04:10:00 MSK -DETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 04:10:04 MSK -DETAIL=TIMER_PROPERTY=LoadState=loaded -DETAIL=TIMER_PROPERTY=ActiveState=active -DETAIL=TIMER_PROPERTY=SubState=waiting -DETAIL=TIMER_ENABLED_RESULT=enabled|RC=0 -DETAIL=HEALTH_METADATA=BYTES:209|MTIME_NS:1783600937792562046 -DETAIL=HEALTH_MATCH_COUNT=1 -DETAIL=HEALTH_LINE=STATUS=OK TS=2026-07-09T12:42:13Z TYPE=filebrowser-restore JOB=homelab-pve01-app-filebrowser-20260709T051713Z SOURCE=appbackupctl-restore-check RESTORECHECK=OK_BOTH_CLOUDS NOTE=service-readiness-restore-proof -DETAIL=JOB_END=filebrowser_restore_validate -DETAIL=JOB_BEGIN=homelab_backup_sla_health -DETAIL=JOB_FIELD=id=homelab_backup_sla_health -DETAIL=JOB_FIELD=name=homelab-backup-sla-health -DETAIL=JOB_FIELD=category=application -DETAIL=JOB_FIELD=critical=True -DETAIL=JOB_FIELD=status=FAIL -DETAIL=JOB_FIELD=freshness=FRESH -DETAIL=JOB_FIELD=age_seconds=358 -DETAIL=JOB_FIELD=max_age_seconds=172800 -DETAIL=JOB_FIELD=source_kind=health_file -DETAIL=JOB_FIELD=source_status=WARN -DETAIL=JOB_FIELD=source_mtime=2026-07-24T07:00:06Z -DETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt -DETAIL=JOB_FIELD=service=homelab-backup-sla-health.service -DETAIL=JOB_FIELD=service_result=success -DETAIL=JOB_FIELD=service_exec_main_status=0 -DETAIL=JOB_FIELD=timer=homelab-backup-sla-health.timer -DETAIL=JOB_FIELD=timer_enabled=enabled -DETAIL=JOB_FIELD=timer_active=active -DETAIL=SERVICE_SHOW_RC=0 -DETAIL=SERVICE_PROPERTY=Result=success -DETAIL=SERVICE_PROPERTY=ExecMainCode=1 -DETAIL=SERVICE_PROPERTY=ExecMainStatus=0 -DETAIL=SERVICE_PROPERTY=LoadState=loaded -DETAIL=SERVICE_PROPERTY=ActiveState=inactive -DETAIL=SERVICE_PROPERTY=SubState=dead -DETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/homelab-backup-sla-health.service -DETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/homelab-backup-sla-health.service.d/90-backup-matrix-refresh.conf -DETAIL=SERVICE_JOURNAL_MATCH_COUNT=76 -DETAIL=SERVICE_JOURNAL=2026-07-11T06:24:11+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... -DETAIL=SERVICE_JOURNAL=2026-07-11T06:24:11+03:00 pve01 homelab-backup-sla-health[2238839]: STATUS=OK TS=2026-07-11T03:24:11Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md -DETAIL=SERVICE_JOURNAL=2026-07-11T06:24:11+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-11T06:24:11+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. -DETAIL=SERVICE_JOURNAL=2026-07-11T07:39:35+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... -DETAIL=SERVICE_JOURNAL=2026-07-11T07:39:36+03:00 pve01 homelab-backup-sla-health[2476861]: STATUS=OK TS=2026-07-11T04:39:35Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md -DETAIL=SERVICE_JOURNAL=2026-07-11T07:39:36+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-11T07:39:36+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. -DETAIL=SERVICE_JOURNAL=2026-07-11T08:15:32+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... -DETAIL=SERVICE_JOURNAL=2026-07-11T08:15:32+03:00 pve01 homelab-backup-sla-health[2616083]: STATUS=OK TS=2026-07-11T05:15:32Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md -DETAIL=SERVICE_JOURNAL=2026-07-11T08:15:33+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-11T08:15:33+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. -DETAIL=SERVICE_JOURNAL=2026-07-12T08:04:09+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... -DETAIL=SERVICE_JOURNAL=2026-07-12T08:04:09+03:00 pve01 homelab-backup-sla-health[2414445]: STATUS=WARN TS=2026-07-12T05:04:09Z TYPE=backup-sla BAD=0 WARN=1 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md -DETAIL=SERVICE_JOURNAL=2026-07-12T08:04:09+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-12T08:04:09+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. -DETAIL=SERVICE_JOURNAL=2026-07-13T01:03:10+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... -DETAIL=SERVICE_JOURNAL=2026-07-13T01:03:11+03:00 pve01 homelab-backup-sla-health[980591]: STATUS=OK TS=2026-07-12T22:03:10Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md -DETAIL=SERVICE_JOURNAL=2026-07-13T01:03:11+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-13T01:03:11+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. -DETAIL=SERVICE_JOURNAL=2026-07-13T01:08:07+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... -DETAIL=SERVICE_JOURNAL=2026-07-13T01:08:08+03:00 pve01 homelab-backup-sla-health[994479]: STATUS=OK TS=2026-07-12T22:08:07Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md -DETAIL=SERVICE_JOURNAL=2026-07-13T01:08:08+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-13T01:08:08+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. -DETAIL=SERVICE_JOURNAL=2026-07-13T08:15:21+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... -DETAIL=SERVICE_JOURNAL=2026-07-13T08:15:21+03:00 pve01 homelab-backup-sla-health[2521375]: STATUS=OK TS=2026-07-13T05:15:21Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md -DETAIL=SERVICE_JOURNAL=2026-07-13T08:15:21+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-13T08:15:21+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. -DETAIL=SERVICE_JOURNAL=2026-07-14T08:05:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... -DETAIL=SERVICE_JOURNAL=2026-07-14T08:05:15+03:00 pve01 homelab-backup-sla-health[2983241]: STATUS=WARN TS=2026-07-14T05:05:14Z TYPE=backup-sla BAD=0 WARN=1 APPS=46 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md -DETAIL=SERVICE_JOURNAL=2026-07-14T08:05:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-14T08:05:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. -DETAIL=SERVICE_JOURNAL=2026-07-14T08:23:27+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... -DETAIL=SERVICE_JOURNAL=2026-07-14T08:23:28+03:00 pve01 homelab-backup-sla-health[3042266]: STATUS=OK TS=2026-07-14T05:23:27Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md -DETAIL=SERVICE_JOURNAL=2026-07-14T08:23:28+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-14T08:23:28+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. -DETAIL=SERVICE_JOURNAL=2026-07-15T08:19:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... -DETAIL=SERVICE_JOURNAL=2026-07-15T08:19:15+03:00 pve01 homelab-backup-sla-health[3670619]: STATUS=OK TS=2026-07-15T05:19:14Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md -DETAIL=SERVICE_JOURNAL=2026-07-15T08:19:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-15T08:19:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. -DETAIL=SERVICE_JOURNAL=2026-07-16T08:09:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... -DETAIL=SERVICE_JOURNAL=2026-07-16T08:09:15+03:00 pve01 homelab-backup-sla-health[225757]: STATUS=WARN TS=2026-07-16T05:09:14Z TYPE=backup-sla BAD=0 WARN=1 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md -DETAIL=SERVICE_JOURNAL=2026-07-16T08:09:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-16T08:09:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. -DETAIL=SERVICE_JOURNAL=2026-07-17T08:07:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... -DETAIL=SERVICE_JOURNAL=2026-07-17T08:07:15+03:00 pve01 homelab-backup-sla-health[4188049]: STATUS=WARN TS=2026-07-17T05:07:14Z TYPE=backup-sla BAD=0 WARN=4 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md -DETAIL=SERVICE_JOURNAL=2026-07-17T08:07:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-17T08:07:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. -DETAIL=SERVICE_JOURNAL=2026-07-18T08:05:29+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... -DETAIL=SERVICE_JOURNAL=2026-07-18T08:05:29+03:00 pve01 homelab-backup-sla-health[806098]: STATUS=OK TS=2026-07-18T05:05:29Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md -DETAIL=SERVICE_JOURNAL=2026-07-18T08:05:29+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-18T08:05:29+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. -DETAIL=SERVICE_JOURNAL=2026-07-19T08:05:14+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... -DETAIL=SERVICE_JOURNAL=2026-07-19T08:05:15+03:00 pve01 homelab-backup-sla-health[1545067]: STATUS=OK TS=2026-07-19T05:05:14Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md -DETAIL=SERVICE_JOURNAL=2026-07-19T08:05:15+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-19T08:05:15+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. -DETAIL=SERVICE_JOURNAL=2026-07-20T08:10:25+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... -DETAIL=SERVICE_JOURNAL=2026-07-20T08:10:26+03:00 pve01 homelab-backup-sla-health[2331602]: STATUS=OK TS=2026-07-20T05:10:25Z TYPE=backup-sla BAD=0 WARN=0 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md -DETAIL=SERVICE_JOURNAL=2026-07-20T08:10:26+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-20T08:10:26+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. -DETAIL=SERVICE_JOURNAL=2026-07-21T08:07:25+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... -DETAIL=SERVICE_JOURNAL=2026-07-21T08:07:26+03:00 pve01 homelab-backup-sla-health[3122692]: STATUS=WARN TS=2026-07-21T05:07:25Z TYPE=backup-sla BAD=0 WARN=1 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md -DETAIL=SERVICE_JOURNAL=2026-07-21T08:07:26+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-21T08:07:26+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. -DETAIL=SERVICE_JOURNAL=2026-07-22T08:10:41+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... -DETAIL=SERVICE_JOURNAL=2026-07-22T08:10:42+03:00 pve01 homelab-backup-sla-health[3806055]: STATUS=WARN TS=2026-07-22T05:10:41Z TYPE=backup-sla BAD=0 WARN=1 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md -DETAIL=SERVICE_JOURNAL=2026-07-22T08:10:42+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-22T08:10:42+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. -DETAIL=SERVICE_JOURNAL=2026-07-23T08:06:25+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... -DETAIL=SERVICE_JOURNAL=2026-07-23T08:06:26+03:00 pve01 homelab-backup-sla-health[310318]: STATUS=WARN TS=2026-07-23T05:06:25Z TYPE=backup-sla BAD=0 WARN=2 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md -DETAIL=SERVICE_JOURNAL=2026-07-23T08:06:26+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-23T08:06:26+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. -DETAIL=SERVICE_JOURNAL=2026-07-24T08:20:04+03:00 pve01 systemd[1]: Starting homelab-backup-sla-health.service - Homelab backup freshness SLA health... -DETAIL=SERVICE_JOURNAL=2026-07-24T08:20:04+03:00 pve01 homelab-backup-sla-health[3253899]: STATUS=WARN TS=2026-07-24T05:20:04Z TYPE=backup-sla BAD=0 WARN=4 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md -DETAIL=SERVICE_JOURNAL=2026-07-24T08:20:04+03:00 pve01 systemd[1]: homelab-backup-sla-health.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-24T08:20:04+03:00 pve01 systemd[1]: Finished homelab-backup-sla-health.service - Homelab backup freshness SLA health. -DETAIL=TIMER_SHOW_RC=0 -DETAIL=TIMER_PROPERTY=Unit=homelab-backup-sla-health.service -DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Sat 2026-07-25 08:08:57 MSK -DETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 08:20:04 MSK -DETAIL=TIMER_PROPERTY=LoadState=loaded -DETAIL=TIMER_PROPERTY=ActiveState=active -DETAIL=TIMER_PROPERTY=SubState=waiting -DETAIL=TIMER_ENABLED_RESULT=enabled|RC=0 -DETAIL=HEALTH_METADATA=BYTES:154|MTIME_NS:1784876406380685648 -DETAIL=HEALTH_MATCH_COUNT=1 -DETAIL=HEALTH_LINE=STATUS=WARN TS=2026-07-24T07:00:05Z TYPE=backup-sla BAD=0 WARN=4 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md -DETAIL=JOB_END=homelab_backup_sla_health -DETAIL=JOB_BEGIN=homelab_mail_cloud_restore_drill -DETAIL=JOB_FIELD=id=homelab_mail_cloud_restore_drill -DETAIL=JOB_FIELD=name=homelab-mail-cloud-restore-drill -DETAIL=JOB_FIELD=category=restore -DETAIL=JOB_FIELD=critical=True -DETAIL=JOB_FIELD=status=STALE -DETAIL=JOB_FIELD=freshness=STALE -DETAIL=JOB_FIELD=age_seconds=1247672 -DETAIL=JOB_FIELD=max_age_seconds=864000 -DETAIL=JOB_FIELD=source_kind=health_file -DETAIL=JOB_FIELD=source_status=OK -DETAIL=JOB_FIELD=source_mtime=2026-07-09T20:31:32Z -DETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/homelab_mail_cloud_restore_drill/0-mail-cloud-restore-drill.txt -DETAIL=JOB_FIELD=service=homelab-mail-cloud-restore-drill.service -DETAIL=JOB_FIELD=service_result=success -DETAIL=JOB_FIELD=service_exec_main_status=0 -DETAIL=JOB_FIELD=timer=homelab-mail-cloud-restore-drill.timer -DETAIL=JOB_FIELD=timer_enabled=enabled -DETAIL=JOB_FIELD=timer_active=active -DETAIL=SERVICE_SHOW_RC=0 -DETAIL=SERVICE_PROPERTY=Result=success -DETAIL=SERVICE_PROPERTY=ExecMainCode=0 -DETAIL=SERVICE_PROPERTY=ExecMainStatus=0 -DETAIL=SERVICE_PROPERTY=LoadState=loaded -DETAIL=SERVICE_PROPERTY=ActiveState=inactive -DETAIL=SERVICE_PROPERTY=SubState=dead -DETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/homelab-mail-cloud-restore-drill.service -DETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/homelab-mail-cloud-restore-drill.service.d/10-superseded-noop.conf /etc/systemd/system/homelab-mail-cloud-restore-drill.service.d/90-backup-matrix-refresh.conf -DETAIL=SERVICE_JOURNAL_MATCH_COUNT=8 -DETAIL=SERVICE_JOURNAL=2026-07-13T07:38:36+03:00 pve01 systemd[1]: Starting homelab-mail-cloud-restore-drill.service - Homelab Mail Cloud restore drill... -DETAIL=SERVICE_JOURNAL=2026-07-13T07:38:36+03:00 pve01 homelab-superseded-unit-ok[2401599]: STATUS=OK TS=2026-07-13T04:38:36Z TYPE=superseded-unit UNIT=homelab-mail-cloud-restore-drill.service REASON=superseded_by_safety_net_restore_check ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-13T07:38:36+03:00 pve01 systemd[1]: homelab-mail-cloud-restore-drill.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-13T07:38:36+03:00 pve01 systemd[1]: Finished homelab-mail-cloud-restore-drill.service - Homelab Mail Cloud restore drill. -DETAIL=SERVICE_JOURNAL=2026-07-20T07:25:14+03:00 pve01 systemd[1]: Starting homelab-mail-cloud-restore-drill.service - Homelab Mail Cloud restore drill... -DETAIL=SERVICE_JOURNAL=2026-07-20T07:25:14+03:00 pve01 homelab-superseded-unit-ok[2171079]: STATUS=OK TS=2026-07-20T04:25:14Z TYPE=superseded-unit UNIT=homelab-mail-cloud-restore-drill.service REASON=superseded_by_safety_net_restore_check ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-20T07:25:14+03:00 pve01 systemd[1]: homelab-mail-cloud-restore-drill.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-20T07:25:14+03:00 pve01 systemd[1]: Finished homelab-mail-cloud-restore-drill.service - Homelab Mail Cloud restore drill. -DETAIL=TIMER_SHOW_RC=0 -DETAIL=TIMER_PROPERTY=Unit=homelab-mail-cloud-restore-drill.service -DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Mon 2026-07-27 07:34:52 MSK -DETAIL=TIMER_PROPERTY=LastTriggerUSec=Mon 2026-07-20 07:25:14 MSK -DETAIL=TIMER_PROPERTY=LoadState=loaded -DETAIL=TIMER_PROPERTY=ActiveState=active -DETAIL=TIMER_PROPERTY=SubState=waiting -DETAIL=TIMER_ENABLED_RESULT=enabled|RC=0 -DETAIL=HEALTH_METADATA=BYTES:353|MTIME_NS:1783629092051018609 -DETAIL=HEALTH_MATCH_COUNT=1 -DETAIL=HEALTH_LINE=STATUS=OK TS=2026-07-09T20:31:32Z TYPE=legacy-mail-cloud-restore-drill-superseded MODE=SUPERSEDED_BY_CURRENT_PROOF BAD=0 WARN=0 PROOF=safety_net_control_plane_restore_check_OK ACTION=stale_health_refresh_only REASON=legacy_drill_superseded_but_health_file_mtime_must_stay_fresh REPORT=/var/lib/homelab-failed-unit-legacy-cleanup/local-cleanup-latest.md -DETAIL=JOB_END=homelab_mail_cloud_restore_drill -DETAIL=JOB_BEGIN=netbird_vps_backup -DETAIL=JOB_FIELD=id=netbird_vps_backup -DETAIL=JOB_FIELD=name=netbird-vps-backup -DETAIL=JOB_FIELD=category=application -DETAIL=JOB_FIELD=critical=True -DETAIL=JOB_FIELD=status=FAIL -DETAIL=JOB_FIELD=freshness=STALE -DETAIL=JOB_FIELD=age_seconds=2007299 -DETAIL=JOB_FIELD=max_age_seconds=864000 -DETAIL=JOB_FIELD=source_kind=health_file -DETAIL=JOB_FIELD=source_status=OK -DETAIL=JOB_FIELD=source_mtime=2026-07-01T01:31:05Z -DETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt -DETAIL=JOB_FIELD=service=netbird-vps-backup.service -DETAIL=JOB_FIELD=service_result=success -DETAIL=JOB_FIELD=service_exec_main_status=0 -DETAIL=JOB_FIELD=timer=netbird-vps-backup.timer -DETAIL=JOB_FIELD=timer_enabled=disabled -DETAIL=JOB_FIELD=timer_active=inactive -DETAIL=SERVICE_SHOW_RC=0 -DETAIL=SERVICE_PROPERTY=Result=success -DETAIL=SERVICE_PROPERTY=ExecMainCode=0 -DETAIL=SERVICE_PROPERTY=ExecMainStatus=0 -DETAIL=SERVICE_PROPERTY=LoadState=loaded -DETAIL=SERVICE_PROPERTY=ActiveState=inactive -DETAIL=SERVICE_PROPERTY=SubState=dead -DETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/netbird-vps-backup.service -DETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/netbird-vps-backup.service.d/10-superseded-noop.conf /etc/systemd/system/netbird-vps-backup.service.d/90-backup-matrix-refresh.conf -DETAIL=SERVICE_JOURNAL_MATCH_COUNT=28 -DETAIL=SERVICE_JOURNAL=2026-07-11T04:30:35+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging... -DETAIL=SERVICE_JOURNAL=2026-07-11T04:30:35+03:00 pve01 homelab-superseded-unit-ok[1772638]: STATUS=OK TS=2026-07-11T01:30:35Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-11T04:30:35+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-11T04:30:35+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging. -DETAIL=SERVICE_JOURNAL=2026-07-12T04:39:35+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging... -DETAIL=SERVICE_JOURNAL=2026-07-12T04:39:35+03:00 pve01 homelab-superseded-unit-ok[1789866]: STATUS=OK TS=2026-07-12T01:39:35Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-12T04:39:35+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-12T04:39:35+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging. -DETAIL=SERVICE_JOURNAL=2026-07-13T04:28:00+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging... -DETAIL=SERVICE_JOURNAL=2026-07-13T04:28:00+03:00 pve01 homelab-superseded-unit-ok[1739137]: STATUS=OK TS=2026-07-13T01:28:00Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-13T04:28:00+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-13T04:28:00+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging. -DETAIL=SERVICE_JOURNAL=2026-07-14T04:28:39+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging... -DETAIL=SERVICE_JOURNAL=2026-07-14T04:28:39+03:00 pve01 homelab-superseded-unit-ok[2199841]: STATUS=OK TS=2026-07-14T01:28:39Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-14T04:28:39+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-14T04:28:39+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging. -DETAIL=SERVICE_JOURNAL=2026-07-15T04:39:00+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging... -DETAIL=SERVICE_JOURNAL=2026-07-15T04:39:00+03:00 pve01 homelab-superseded-unit-ok[2872498]: STATUS=OK TS=2026-07-15T01:39:00Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-15T04:39:00+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-15T04:39:00+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging. -DETAIL=SERVICE_JOURNAL=2026-07-16T04:25:14+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging... -DETAIL=SERVICE_JOURNAL=2026-07-16T04:25:14+03:00 pve01 homelab-superseded-unit-ok[3611661]: STATUS=OK TS=2026-07-16T01:25:14Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-16T04:25:14+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-16T04:25:14+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging. -DETAIL=SERVICE_JOURNAL=2026-07-17T04:31:14+03:00 pve01 systemd[1]: Starting netbird-vps-backup.service - NetBird VPS backup to pve01 staging... -DETAIL=SERVICE_JOURNAL=2026-07-17T04:31:14+03:00 pve01 homelab-superseded-unit-ok[3412653]: STATUS=OK TS=2026-07-17T01:31:14Z TYPE=superseded-unit UNIT=netbird-vps-backup.service REASON=superseded_by_current_appbackup_or_trust_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-17T04:31:14+03:00 pve01 systemd[1]: netbird-vps-backup.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-17T04:31:14+03:00 pve01 systemd[1]: Finished netbird-vps-backup.service - NetBird VPS backup to pve01 staging. -DETAIL=TIMER_SHOW_RC=0 -DETAIL=TIMER_PROPERTY=Unit=netbird-vps-backup.service -DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime= -DETAIL=TIMER_PROPERTY=LastTriggerUSec= -DETAIL=TIMER_PROPERTY=LoadState=loaded -DETAIL=TIMER_PROPERTY=ActiveState=inactive -DETAIL=TIMER_PROPERTY=SubState=dead -DETAIL=TIMER_ENABLED_RESULT=disabled|RC=1 -DETAIL=HEALTH_METADATA=BYTES:312|MTIME_NS:1782869465433646905 -DETAIL=HEALTH_MATCH_COUNT=7 -DETAIL=HEALTH_LINE=STATUS=OK -DETAIL=HEALTH_LINE=SERVICE=netbird-vps-backup -DETAIL=HEALTH_LINE=TIMESTAMP_UTC=20260701T013054Z -DETAIL=HEALTH_LINE=SNAPSHOT=/mnt/staging/netbird-vps-backups/snapshots/20260701T013054Z -DETAIL=HEALTH_LINE=LATEST_SYMLINK=/mnt/staging/netbird-vps-backups/snapshots/latest -DETAIL=HEALTH_LINE=SIZE=32M -DETAIL=HEALTH_LINE=SHA256=0226dd820cb9ede0ea9e8ea4af6cef656ffa54cfb819048c222a88f4c14d8db9 -DETAIL=JOB_END=netbird_vps_backup -DETAIL=JOB_BEGIN=paperless_restore_dry_run -DETAIL=JOB_FIELD=id=paperless_restore_dry_run -DETAIL=JOB_FIELD=name=paperless-restore-dry-run -DETAIL=JOB_FIELD=category=restore -DETAIL=JOB_FIELD=critical=True -DETAIL=JOB_FIELD=status=STALE -DETAIL=JOB_FIELD=freshness=STALE -DETAIL=JOB_FIELD=age_seconds=1261389 -DETAIL=JOB_FIELD=max_age_seconds=864000 -DETAIL=JOB_FIELD=source_kind=health_file -DETAIL=JOB_FIELD=source_status=OK -DETAIL=JOB_FIELD=source_mtime=2026-07-09T16:42:55Z -DETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/paperless_restore_dry_run/0-paperless-restore.txt -DETAIL=JOB_FIELD=service=paperless-restore-dry-run.service -DETAIL=JOB_FIELD=service_result=success -DETAIL=JOB_FIELD=service_exec_main_status=0 -DETAIL=JOB_FIELD=timer=paperless-restore-dry-run.timer -DETAIL=JOB_FIELD=timer_enabled=enabled -DETAIL=JOB_FIELD=timer_active=active -DETAIL=SERVICE_SHOW_RC=0 -DETAIL=SERVICE_PROPERTY=Result=success -DETAIL=SERVICE_PROPERTY=ExecMainCode=1 -DETAIL=SERVICE_PROPERTY=ExecMainStatus=0 -DETAIL=SERVICE_PROPERTY=LoadState=loaded -DETAIL=SERVICE_PROPERTY=ActiveState=inactive -DETAIL=SERVICE_PROPERTY=SubState=dead -DETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/paperless-restore-dry-run.service -DETAIL=SERVICE_PROPERTY=DropInPaths=/etc/systemd/system/paperless-restore-dry-run.service.d/10-superseded-noop.conf /etc/systemd/system/paperless-restore-dry-run.service.d/90-backup-matrix-refresh.conf -DETAIL=SERVICE_JOURNAL_MATCH_COUNT=56 -DETAIL=SERVICE_JOURNAL=2026-07-11T10:08:13+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... -DETAIL=SERVICE_JOURNAL=2026-07-11T10:08:13+03:00 pve01 homelab-superseded-unit-ok[2988079]: STATUS=OK TS=2026-07-11T07:08:13Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-11T10:08:13+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-11T10:08:13+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. -DETAIL=SERVICE_JOURNAL=2026-07-12T10:04:59+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... -DETAIL=SERVICE_JOURNAL=2026-07-12T10:04:59+03:00 pve01 homelab-superseded-unit-ok[2757392]: STATUS=OK TS=2026-07-12T07:04:59Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-12T10:04:59+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-12T10:04:59+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. -DETAIL=SERVICE_JOURNAL=2026-07-13T10:07:25+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... -DETAIL=SERVICE_JOURNAL=2026-07-13T10:07:25+03:00 pve01 homelab-superseded-unit-ok[2886298]: STATUS=OK TS=2026-07-13T07:07:25Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-13T10:07:25+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-13T10:07:25+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. -DETAIL=SERVICE_JOURNAL=2026-07-14T09:56:36+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... -DETAIL=SERVICE_JOURNAL=2026-07-14T09:56:36+03:00 pve01 homelab-superseded-unit-ok[3367368]: STATUS=OK TS=2026-07-14T06:56:36Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-14T09:56:36+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-14T09:56:36+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. -DETAIL=SERVICE_JOURNAL=2026-07-15T10:12:54+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... -DETAIL=SERVICE_JOURNAL=2026-07-15T10:12:54+03:00 pve01 homelab-superseded-unit-ok[4062186]: STATUS=OK TS=2026-07-15T07:12:54Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-15T10:12:54+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-15T10:12:54+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. -DETAIL=SERVICE_JOURNAL=2026-07-16T11:40:40+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... -DETAIL=SERVICE_JOURNAL=2026-07-16T11:40:40+03:00 pve01 homelab-superseded-unit-ok[12176]: STATUS=OK TS=2026-07-16T08:40:40Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-16T11:40:40+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-16T11:40:40+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. -DETAIL=SERVICE_JOURNAL=2026-07-17T10:00:14+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... -DETAIL=SERVICE_JOURNAL=2026-07-17T10:00:14+03:00 pve01 homelab-superseded-unit-ok[390020]: STATUS=OK TS=2026-07-17T07:00:14Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-17T10:00:14+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-17T10:00:14+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. -DETAIL=SERVICE_JOURNAL=2026-07-18T09:57:16+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... -DETAIL=SERVICE_JOURNAL=2026-07-18T09:57:16+03:00 pve01 homelab-superseded-unit-ok[1201647]: STATUS=OK TS=2026-07-18T06:57:16Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-18T09:57:16+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-18T09:57:16+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. -DETAIL=SERVICE_JOURNAL=2026-07-19T10:09:17+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... -DETAIL=SERVICE_JOURNAL=2026-07-19T10:09:17+03:00 pve01 homelab-superseded-unit-ok[1945203]: STATUS=OK TS=2026-07-19T07:09:17Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-19T10:09:17+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-19T10:09:17+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. -DETAIL=SERVICE_JOURNAL=2026-07-20T10:04:06+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... -DETAIL=SERVICE_JOURNAL=2026-07-20T10:04:06+03:00 pve01 homelab-superseded-unit-ok[2734120]: STATUS=OK TS=2026-07-20T07:04:06Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-20T10:04:06+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-20T10:04:06+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. -DETAIL=SERVICE_JOURNAL=2026-07-21T10:02:41+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... -DETAIL=SERVICE_JOURNAL=2026-07-21T10:02:41+03:00 pve01 homelab-superseded-unit-ok[3525432]: STATUS=OK TS=2026-07-21T07:02:41Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-21T10:02:41+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-21T10:02:41+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. -DETAIL=SERVICE_JOURNAL=2026-07-22T10:11:14+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... -DETAIL=SERVICE_JOURNAL=2026-07-22T10:11:14+03:00 pve01 homelab-superseded-unit-ok[4192753]: STATUS=OK TS=2026-07-22T07:11:14Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-22T10:11:14+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-22T10:11:14+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. -DETAIL=SERVICE_JOURNAL=2026-07-23T10:12:14+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... -DETAIL=SERVICE_JOURNAL=2026-07-23T10:12:14+03:00 pve01 homelab-superseded-unit-ok[752982]: STATUS=OK TS=2026-07-23T07:12:14Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-23T10:12:14+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-23T10:12:14+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. -DETAIL=SERVICE_JOURNAL=2026-07-24T10:06:04+03:00 pve01 systemd[1]: Starting paperless-restore-dry-run.service - Paperless restore dry-run health marker... -DETAIL=SERVICE_JOURNAL=2026-07-24T10:06:04+03:00 pve01 homelab-superseded-unit-ok[3581148]: STATUS=OK TS=2026-07-24T07:06:04Z TYPE=superseded-unit UNIT=paperless-restore-dry-run.service REASON=superseded_by_current_appbackup_restore_proof ACTION=noop_superseded_unit -DETAIL=SERVICE_JOURNAL=2026-07-24T10:06:04+03:00 pve01 systemd[1]: paperless-restore-dry-run.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-24T10:06:04+03:00 pve01 systemd[1]: Finished paperless-restore-dry-run.service - Paperless restore dry-run health marker. -DETAIL=TIMER_SHOW_RC=0 -DETAIL=TIMER_PROPERTY=Unit=paperless-restore-dry-run.service -DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Sat 2026-07-25 10:03:38 MSK -DETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 10:06:04 MSK -DETAIL=TIMER_PROPERTY=LoadState=loaded -DETAIL=TIMER_PROPERTY=ActiveState=active -DETAIL=TIMER_PROPERTY=SubState=waiting -DETAIL=TIMER_ENABLED_RESULT=enabled|RC=0 -DETAIL=HEALTH_METADATA=BYTES:232|MTIME_NS:1783615375842257624 -DETAIL=HEALTH_MATCH_COUNT=1 -DETAIL=HEALTH_LINE=STATUS=OK TS=2026-07-09T16:42:53Z TYPE=paperless-restore BAD=0 APP=paperless JOB=homelab-pve01-app-paperless-20260709T071949Z RESTORE_VERIFIED=true RESTORE_CHECK_APP=paperless JSON=/var/lib/homelab-backup/apps/latest-paperless.json -DETAIL=JOB_END=paperless_restore_dry_run -DETAIL=JOB_BEGIN=vm160_remote_backup -DETAIL=JOB_FIELD=id=vm160_remote_backup -DETAIL=JOB_FIELD=name=vm160-remote-backup -DETAIL=JOB_FIELD=category=virtual-machine -DETAIL=JOB_FIELD=critical=True -DETAIL=JOB_FIELD=status=FAIL -DETAIL=JOB_FIELD=freshness=STALE -DETAIL=JOB_FIELD=age_seconds=286010 -DETAIL=JOB_FIELD=max_age_seconds=172800 -DETAIL=JOB_FIELD=source_kind=health_file -DETAIL=JOB_FIELD=source_status=FAIL -DETAIL=JOB_FIELD=source_mtime=2026-07-20T23:39:14Z -DETAIL=JOB_FIELD=source_health=/var/lib/homelab-backup-matrix/evidence/vm160_remote_backup/0-vm160-remote-backup.txt -DETAIL=JOB_FIELD=service=homelab-vm160-remote-backup.service -DETAIL=JOB_FIELD=service_result=success -DETAIL=JOB_FIELD=service_exec_main_status=0 -DETAIL=JOB_FIELD=timer=homelab-vm160-remote-backup.timer -DETAIL=JOB_FIELD=timer_enabled=disabled -DETAIL=JOB_FIELD=timer_active=inactive -DETAIL=SERVICE_SHOW_RC=0 -DETAIL=SERVICE_PROPERTY=Result=success -DETAIL=SERVICE_PROPERTY=ExecMainCode=0 -DETAIL=SERVICE_PROPERTY=ExecMainStatus=0 -DETAIL=SERVICE_PROPERTY=LoadState=loaded -DETAIL=SERVICE_PROPERTY=ActiveState=inactive -DETAIL=SERVICE_PROPERTY=SubState=dead -DETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/homelab-vm160-remote-backup.service -DETAIL=SERVICE_PROPERTY=DropInPaths= -DETAIL=SERVICE_JOURNAL_MATCH_COUNT=50 -DETAIL=SERVICE_JOURNAL=2026-07-18T01:02:41+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... -DETAIL=SERVICE_JOURNAL=2026-07-18T01:15:43+03:00 pve01 homelab-vm160-remote-backup[3491061]: VM160_TARGET_FINALIZE_STATUS=OK -DETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3530071]: VM160_TARGET_PROMOTE_STATUS=OK -DETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: VM160_REMOTE_BACKUP_STATUS=OK -DETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260717T220242Z -DETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: ARCHIVE=/mnt/staging/vzdump/vm160-pve02-20260717T220242Z/vzdump-qemu-160-20260717T220242Z.vma.zst -DETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: SIZE_BYTES=84993482488 -DETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 homelab-vm160-remote-backup[3473847]: SHA256=a9562d8b60d7fd5e7ec73bc43d252cfea80ed12c05b32cca0a73d7bb88532411 -DETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 systemd[1]: Finished homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. -DETAIL=SERVICE_JOURNAL=2026-07-18T01:19:10+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Consumed 2min 44.280s CPU time, 15.8M memory peak. -DETAIL=SERVICE_JOURNAL=2026-07-18T01:38:18+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... -DETAIL=SERVICE_JOURNAL=2026-07-18T01:51:18+03:00 pve01 homelab-vm160-remote-backup[3609200]: VM160_TARGET_FINALIZE_STATUS=OK -DETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3648126]: VM160_TARGET_PROMOTE_STATUS=OK -DETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: VM160_REMOTE_BACKUP_STATUS=OK -DETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z -DETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: ARCHIVE=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z/vzdump-qemu-160-20260717T223819Z.vma.zst -DETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: SIZE_BYTES=84995216465 -DETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 homelab-vm160-remote-backup[3592132]: SHA256=257e10821e62e3e2f9318b238a5302a6db601dd597bfa3e0d77aba07f3b85e72 -DETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Deactivated successfully. -DETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 systemd[1]: Finished homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. -DETAIL=SERVICE_JOURNAL=2026-07-18T01:54:48+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Consumed 2min 44.842s CPU time, 16.1M memory peak. -DETAIL=SERVICE_JOURNAL=2026-07-18T04:16:25+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... -DETAIL=SERVICE_JOURNAL=2026-07-18T04:16:27+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE -DETAIL=SERVICE_JOURNAL=2026-07-18T04:16:27+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. -DETAIL=SERVICE_JOURNAL=2026-07-18T04:16:27+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. -DETAIL=SERVICE_JOURNAL=2026-07-19T04:19:14+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... -DETAIL=SERVICE_JOURNAL=2026-07-19T04:19:16+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE -DETAIL=SERVICE_JOURNAL=2026-07-19T04:19:16+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. -DETAIL=SERVICE_JOURNAL=2026-07-19T04:19:16+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. -DETAIL=SERVICE_JOURNAL=2026-07-20T04:14:14+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... -DETAIL=SERVICE_JOURNAL=2026-07-20T04:14:16+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE -DETAIL=SERVICE_JOURNAL=2026-07-20T04:14:16+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. -DETAIL=SERVICE_JOURNAL=2026-07-20T04:14:16+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. -DETAIL=SERVICE_JOURNAL=2026-07-20T19:28:57+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... -DETAIL=SERVICE_JOURNAL=2026-07-20T19:28:59+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE -DETAIL=SERVICE_JOURNAL=2026-07-20T19:28:59+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. -DETAIL=SERVICE_JOURNAL=2026-07-20T19:28:59+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. -DETAIL=SERVICE_JOURNAL=2026-07-21T02:14:11+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... -DETAIL=SERVICE_JOURNAL=2026-07-21T02:14:13+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE -DETAIL=SERVICE_JOURNAL=2026-07-21T02:14:13+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. -DETAIL=SERVICE_JOURNAL=2026-07-21T02:14:13+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. -DETAIL=SERVICE_JOURNAL=2026-07-21T02:20:20+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... -DETAIL=SERVICE_JOURNAL=2026-07-21T02:20:21+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE -DETAIL=SERVICE_JOURNAL=2026-07-21T02:20:21+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. -DETAIL=SERVICE_JOURNAL=2026-07-21T02:20:21+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. -DETAIL=SERVICE_JOURNAL=2026-07-21T02:39:12+03:00 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... -DETAIL=SERVICE_JOURNAL=2026-07-21T02:39:14+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE -DETAIL=SERVICE_JOURNAL=2026-07-21T02:39:14+03:00 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. -DETAIL=SERVICE_JOURNAL=2026-07-21T02:39:14+03:00 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. -DETAIL=TIMER_SHOW_RC=0 -DETAIL=TIMER_PROPERTY=Unit=homelab-vm160-remote-backup.service -DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime= -DETAIL=TIMER_PROPERTY=LastTriggerUSec= -DETAIL=TIMER_PROPERTY=LoadState=loaded -DETAIL=TIMER_PROPERTY=ActiveState=inactive -DETAIL=TIMER_PROPERTY=SubState=dead -DETAIL=TIMER_ENABLED_RESULT=disabled|RC=1 -DETAIL=HEALTH_METADATA=BYTES:153|MTIME_NS:1784590754312087060 -DETAIL=HEALTH_MATCH_COUNT=3 -DETAIL=HEALTH_LINE=STATUS=FAIL -DETAIL=HEALTH_LINE=SERVICE=homelab-vm160-remote-backup -DETAIL=HEALTH_LINE=TIMESTAMP_UTC=20260720T233913Z -DETAIL=JOB_END=vm160_remote_backup +DETAIL=POLICY=DUAL_CLOUD_VERIFY_THEN_DELETE_ALL_LOCAL_PAYLOADS +DETAIL=CHANGES_PLANNED=NO +DETAIL=STATE_FILE=/var/lib/homelab-vm-backup/latest-vm-local-dumps-cloud.json +DETAIL=STATE_FILE_STATUS=READABLE +DETAIL=STATE_FIELD=$.job_id=homelab-pve01-vm-local-dumps-20260719T070607Z +DETAIL=STATE_FIELD=$.status=OK +DETAIL=STATE_FIELD=$.remote_a=pve01-mail-01-crypt:homelab-backups/pve01/vm-local-dumps/homelab-pve01-vm-local-dumps-20260719T070607Z +DETAIL=STATE_FIELD=$.remote_b=pve01-mail-02-crypt:homelab-backups/pve01/vm-local-dumps/homelab-pve01-vm-local-dumps-20260719T070607Z +DETAIL=STATE_FIELD=$.chunk_size_bytes=1073741824 +DETAIL=STATE_FIELD=$.local_dumps_deleted=False +DETAIL=STATE_FILE=/var/lib/homelab-backup-matrix/state.json +DETAIL=STATE_FILE_STATUS=READABLE +DETAIL=STATE_FIELD=$.jobs[0].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[0].source_status=OK +DETAIL=STATE_FIELD=$.jobs[0].status=OK +DETAIL=STATE_FIELD=$.jobs[1].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[1].source_status=OK +DETAIL=STATE_FIELD=$.jobs[1].status=OK +DETAIL=STATE_FIELD=$.jobs[2].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[2].source_status=OK +DETAIL=STATE_FIELD=$.jobs[2].status=STALE +DETAIL=STATE_FIELD=$.jobs[3].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[3].source_status=OK +DETAIL=STATE_FIELD=$.jobs[3].status=OK +DETAIL=STATE_FIELD=$.jobs[4].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[4].source_status=OK +DETAIL=STATE_FIELD=$.jobs[4].status=OK +DETAIL=STATE_FIELD=$.jobs[5].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[5].source_status=OK +DETAIL=STATE_FIELD=$.jobs[5].status=OK +DETAIL=STATE_FIELD=$.jobs[6].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[6].source_status=OK +DETAIL=STATE_FIELD=$.jobs[6].status=OK +DETAIL=STATE_FIELD=$.jobs[7].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[7].source_status=OK +DETAIL=STATE_FIELD=$.jobs[7].status=OK +DETAIL=STATE_FIELD=$.jobs[8].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[8].source_status=OK +DETAIL=STATE_FIELD=$.jobs[8].status=DISABLED +DETAIL=STATE_FIELD=$.jobs[9].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[9].source_status=OK +DETAIL=STATE_FIELD=$.jobs[9].status=OK +DETAIL=STATE_FIELD=$.jobs[10].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[10].source_status=OK +DETAIL=STATE_FIELD=$.jobs[10].status=OK +DETAIL=STATE_FIELD=$.jobs[11].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[11].source_status=OK +DETAIL=STATE_FIELD=$.jobs[11].status=OK +DETAIL=STATE_FIELD=$.jobs[12].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[12].source_status=OK +DETAIL=STATE_FIELD=$.jobs[12].status=OK +DETAIL=STATE_FIELD=$.jobs[13].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[13].source_status=WARN +DETAIL=STATE_FIELD=$.jobs[13].status=FAIL +DETAIL=STATE_FIELD=$.jobs[14].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[14].source_status=OK +DETAIL=STATE_FIELD=$.jobs[14].status=OK +DETAIL=STATE_FIELD=$.jobs[15].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[15].source_status=OK +DETAIL=STATE_FIELD=$.jobs[15].status=OK +DETAIL=STATE_FIELD=$.jobs[16].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[16].source_status=OK +DETAIL=STATE_FIELD=$.jobs[16].status=OK +DETAIL=STATE_FIELD=$.jobs[17].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[17].source_status=OK +DETAIL=STATE_FIELD=$.jobs[17].status=DISABLED +DETAIL=STATE_FIELD=$.jobs[18].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[18].source_status=OK +DETAIL=STATE_FIELD=$.jobs[18].status=OK +DETAIL=STATE_FIELD=$.jobs[19].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[19].source_status=OK +DETAIL=STATE_FIELD=$.jobs[19].status=OK +DETAIL=STATE_FIELD=$.jobs[20].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[20].source_status=OK +DETAIL=STATE_FIELD=$.jobs[20].status=OK +DETAIL=STATE_FIELD=$.jobs[21].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[21].source_status=OK +DETAIL=STATE_FIELD=$.jobs[21].status=OK +DETAIL=STATE_FIELD=$.jobs[22].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[22].source_status=OK +DETAIL=STATE_FIELD=$.jobs[22].status=STALE +DETAIL=STATE_FIELD=$.jobs[23].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[23].source_status=OK +DETAIL=STATE_FIELD=$.jobs[23].status=DISABLED +DETAIL=STATE_FIELD=$.jobs[24].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[24].source_status=OK +DETAIL=STATE_FIELD=$.jobs[24].status=OK +DETAIL=STATE_FIELD=$.jobs[25].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[25].source_status=OK +DETAIL=STATE_FIELD=$.jobs[25].status=DISABLED +DETAIL=STATE_FIELD=$.jobs[26].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[26].source_status=OK +DETAIL=STATE_FIELD=$.jobs[26].status=OK +DETAIL=STATE_FIELD=$.jobs[27].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[27].source_status=OK +DETAIL=STATE_FIELD=$.jobs[27].status=OK +DETAIL=STATE_FIELD=$.jobs[28].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[28].source_status=OK +DETAIL=STATE_FIELD=$.jobs[28].status=OK +DETAIL=STATE_FIELD=$.jobs[29].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[29].source_status=OK +DETAIL=STATE_FIELD=$.jobs[29].status=OK +DETAIL=STATE_FIELD=$.jobs[30].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[30].source_status=OK +DETAIL=STATE_FIELD=$.jobs[30].status=OK +DETAIL=STATE_FIELD=$.jobs[31].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[31].source_status=OK +DETAIL=STATE_FIELD=$.jobs[31].status=OK +DETAIL=STATE_FIELD=$.jobs[32].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[32].source_status=OK +DETAIL=STATE_FIELD=$.jobs[32].status=OK +DETAIL=STATE_FIELD=$.jobs[33].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[33].source_status=OK +DETAIL=STATE_FIELD=$.jobs[33].status=OK +DETAIL=STATE_FIELD=$.jobs[34].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[34].source_status=OK +DETAIL=STATE_FIELD=$.jobs[34].status=OK +DETAIL=STATE_FIELD=$.jobs[35].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[35].source_status=OK +DETAIL=STATE_FIELD=$.jobs[35].status=FAIL +DETAIL=STATE_FIELD=$.jobs[36].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[36].source_status=OK +DETAIL=STATE_FIELD=$.jobs[36].status=DISABLED +DETAIL=STATE_FIELD=$.jobs[37].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[37].source_status=OK +DETAIL=STATE_FIELD=$.jobs[37].status=STALE +DETAIL=STATE_FIELD=$.jobs[38].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[38].source_status=OK +DETAIL=STATE_FIELD=$.jobs[38].status=OK +DETAIL=STATE_FIELD=$.jobs[39].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[39].source_status=OK +DETAIL=STATE_FIELD=$.jobs[39].status=OK +DETAIL=STATE_FIELD=$.jobs[40].service_exec_main_status=0 +DETAIL=STATE_FIELD=$.jobs[40].source_status=FAIL +DETAIL=STATE_FIELD=$.jobs[40].status=FAIL +DETAIL=STATE_FIELD=$.matrix_sha256=6b7c5df66c411d1e9630fb336c55007bcd923f1a0bfcc64d687fb1105011eceb +DETAIL=STATE_FIELD=$.summary.jobs_bad=6 +DETAIL=STATE_FIELD=$.summary.jobs_disabled=5 +DETAIL=STATE_FIELD=$.summary.jobs_ok=30 +DETAIL=STATE_FIELD=$.summary.jobs_total=41 +DETAIL=STATE_FIELD=$.summary.jobs_unmapped=0 +DETAIL=STATE_FIELD=$.summary.status=FAIL +DETAIL=UNIT_BEGIN=homelab-vm160-remote-backup.service +DETAIL=UNIT_SHOW_RC=0 +DETAIL=UNIT_PROPERTY=Result=success +DETAIL=UNIT_PROPERTY=ExecMainStatus=0 +DETAIL=UNIT_PROPERTY=ExecStart={ path=/usr/local/sbin/homelab-vm160-remote-backup ; argv[]=/usr/local/sbin/homelab-vm160-remote-backup ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } +DETAIL=UNIT_PROPERTY=LoadState=loaded +DETAIL=UNIT_PROPERTY=ActiveState=inactive +DETAIL=UNIT_PROPERTY=SubState=dead +DETAIL=UNIT_PROPERTY=FragmentPath=/etc/systemd/system/homelab-vm160-remote-backup.service +DETAIL=SCRIPT_PATH=/usr/local/sbin/homelab-vm160-remote-backup +DETAIL=SCRIPT_LINE=3:export SYSTEMD_PAGER=cat PAGER=cat SYSTEMD_LESS=FRXMK +DETAIL=SCRIPT_LINE=8:PARENT=/mnt/staging/vzdump +DETAIL=SCRIPT_LINE=10:HEALTH=/var/lib/homelab-health/vm160-remote-backup.txt +DETAIL=SCRIPT_LINE=11:LOCK=/run/lock/homelab-vm160-remote-backup.lock +DETAIL=SCRIPT_LINE=30:LOG="$(mktemp /tmp/vm160-remote-backup.XXXXXX.log)" +DETAIL=SCRIPT_LINE=31:cleanup(){ +DETAIL=SCRIPT_LINE=35:ssh "${SSH[@]}" "$TARGET" "rm -rf -- '$DIR'" >/dev/null 2>&1 || true +DETAIL=SCRIPT_LINE=37:printf 'STATUS=FAIL\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=%s\nVMID=%s\nTARGET_DIR=%s\nRC=%s\n' "$TS" "$VMID" "$DIR" "$rc" > "$tmp" +DETAIL=SCRIPT_LINE=41:rm -f "$LOG" +DETAIL=SCRIPT_LINE=44:trap cleanup EXIT +DETAIL=SCRIPT_LINE=45:REMOTE_FREE="$(ssh "${SSH[@]}" "$TARGET" "df -B1 --output=avail '$PARENT' | awk 'NR==2{print \$1}'")" +DETAIL=SCRIPT_LINE=53:test "$REMOTE_FREE" -ge "$REQUIRED" +DETAIL=SCRIPT_LINE=55:ssh "${SSH[@]}" "$SOURCE" "vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7" 2>"$LOG" | ssh "${SSH[@]}" "$TARGET" "cat > '$PART'" +DETAIL=SCRIPT_LINE=57:ssh "${SSH[@]}" "$TARGET" "cat > '$DIR/vzdump.log'" < "$LOG" +DETAIL=SCRIPT_LINE=59:SHA="$(ssh "${SSH[@]}" "$TARGET" "sha256sum '$FINAL' | awk '{print \$1}'")" +DETAIL=SCRIPT_LINE=62:printf 'STATUS=OK\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=%s\nVMID=%s\nSOURCE_NODE=pve02\nTARGET_NODE=pve03\nTARGET_DIR=%s\nARCHIVE=%s\nSIZE_BYTES=%s\nSHA256=%s\nREMOTE_RETENTION_GENERATIONS=1\n' "$TS" "$VMID" "$DIR" "$FINAL" "$SIZE" "$SHA" > "$tmp" +DETAIL=SCRIPT_LINE=66:rm -f "$LOG" +DETAIL=SCRIPT_LINE=67:printf 'VM160_REMOTE_BACKUP_STATUS=OK\nTARGET_DIR=%s\nARCHIVE=%s\nSIZE_BYTES=%s\nSHA256=%s\n' "$DIR" "$FINAL" "$SIZE" "$SHA" +DETAIL=UNIT_END=homelab-vm160-remote-backup.service +DETAIL=UNIT_BEGIN=homelab-vm-local-dumps-2cloud.service +DETAIL=UNIT_SHOW_RC=0 +DETAIL=UNIT_PROPERTY=Result=success +DETAIL=UNIT_PROPERTY=ExecMainStatus=0 +DETAIL=UNIT_PROPERTY=ExecStart={ path=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; argv[]=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } +DETAIL=UNIT_PROPERTY=LoadState=loaded +DETAIL=UNIT_PROPERTY=ActiveState=inactive +DETAIL=UNIT_PROPERTY=SubState=dead +DETAIL=UNIT_PROPERTY=FragmentPath=/etc/systemd/system/homelab-vm-local-dumps-2cloud.service +DETAIL=SCRIPT_PATH=/usr/local/sbin/homelab-vm-local-dumps-2cloud +DETAIL=SCRIPT_LINE=4:OUT="/var/lib/homelab-health/vm-local-dumps-cloud.txt" +DETAIL=SCRIPT_LINE=6:LOG="$ROOT/vm-local-dumps-2cloud-latest.log" +DETAIL=SCRIPT_LINE=7:VERIFY_ROOT="/var/tmp/homelab-vm-local-dumps-verify" +DETAIL=SCRIPT_LINE=12:mkdir -p /var/lib/homelab-health "$ROOT" "$VERIFY_ROOT" "$CHUNK_ROOT" +DETAIL=SCRIPT_LINE=55:SHA="$(sha256sum "$F" | cut -d" " -f1)" +DETAIL=SCRIPT_LINE=64:rm -rf "$CDIR" +DETAIL=SCRIPT_LINE=69:(cd "$CDIR" && sha256sum part-* > chunks.sha256) +DETAIL=SCRIPT_LINE=92:"sha256":p[3], +DETAIL=SCRIPT_LINE=112:DEST_A="${REMOTE_A%/}/${NODE}/vm-local-dumps/${JOB_ID}" +DETAIL=SCRIPT_LINE=113:DEST_B="${REMOTE_B%/}/${NODE}/vm-local-dumps/${JOB_ID}" +DETAIL=SCRIPT_LINE=115:upload_one_dest() { +DETAIL=SCRIPT_LINE=117:rclone mkdir "$DEST/files" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SCRIPT_LINE=121:echo "UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts" >> "$LOG" +DETAIL=SCRIPT_LINE=122:rclone mkdir "$DEST/files/$bn.chunks" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SCRIPT_LINE=126:rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SCRIPT_LINE=128:rclone copyto "$CDIR/chunks.sha256" "$DEST/files/$bn.chunks/chunks.sha256" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SCRIPT_LINE=130:echo "UPLOAD_DIRECT dest=$DEST file=$bn size=$size" >> "$LOG" +DETAIL=SCRIPT_LINE=131:rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SCRIPT_LINE=135:rclone copyto "$CHKS" "$DEST/checksums.txt" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SCRIPT_LINE=136:rclone copyto "$FILES_TSV" "$DEST/files.tsv" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SCRIPT_LINE=137:rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SCRIPT_LINE=140:verify_one_dest() { +DETAIL=SCRIPT_LINE=143:VBASE="$VERIFY_ROOT/$JOB_ID/$LABEL" +DETAIL=SCRIPT_LINE=144:rm -rf "$VBASE" +DETAIL=SCRIPT_LINE=146:rclone copyto "$DEST/manifest.json" "$VBASE/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SCRIPT_LINE=153:echo "VERIFY_CHUNKED label=$LABEL file=$bn parts=$parts" >> "$LOG" +DETAIL=SCRIPT_LINE=154:rclone copy "$DEST/files/$bn.chunks" "$VDIR" --filter "+ part-*" --filter "+ chunks.sha256" --filter "- *" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SCRIPT_LINE=155:(cd "$VDIR" && sha256sum -c chunks.sha256) >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SCRIPT_LINE=156:CALC="$(find "$VDIR" -maxdepth 1 -type f -name 'part-*' | sort | xargs cat | sha256sum | cut -d" " -f1)" +DETAIL=SCRIPT_LINE=163:rm -rf "$VDIR" +DETAIL=SCRIPT_LINE=165:echo "VERIFY_DIRECT label=$LABEL file=$bn" >> "$LOG" +DETAIL=SCRIPT_LINE=166:rclone copyto "$DEST/files/$bn" "$VBASE/$bn" --transfers 1 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SCRIPT_LINE=167:printf "%s %s\n" "$sha" "$VBASE/$bn" | sha256sum -c - >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SCRIPT_LINE=168:rm -f "$VBASE/$bn" +DETAIL=SCRIPT_LINE=171:rm -rf "$VBASE" +DETAIL=SCRIPT_LINE=175:upload_one_dest "$DEST_A" +DETAIL=SCRIPT_LINE=176:upload_one_dest "$DEST_B" +DETAIL=SCRIPT_LINE=180:verify_one_dest "A" "$DEST_A" +DETAIL=SCRIPT_LINE=181:verify_one_dest "B" "$DEST_B" +DETAIL=SCRIPT_LINE=184:rm -rf "$VERIFY_ROOT/$JOB_ID" +DETAIL=SCRIPT_LINE=185:rm -rf "$CHUNK_ROOT/$JOB_ID" +DETAIL=SCRIPT_LINE=193:--arg remote_a "$DEST_A" \ +DETAIL=SCRIPT_LINE=194:--arg remote_b "$DEST_B" \ +DETAIL=SCRIPT_LINE=200:"{job_id:\$job_id,status:\$status,node:\$node,remote_a:\$remote_a,remote_b:\$remote_b,count:\$count,total_bytes:\$total_bytes,chunked_count:\$chunked_count,chunk_size_bytes:\$chunk_size_bytes,log:\$log,local_dumps_deleted:false}" \ +DETAIL=SCRIPT_LINE=201:> "$ROOT/latest-vm-local-dumps-cloud.json" +DETAIL=SCRIPT_LINE=203:echo "STATUS=$STATUS TS=$TS TYPE=vm-local-dumps-cloud BAD=$BAD COUNT=$COUNT CHUNKED=$CHUNKED TOTAL_BYTES=$TOTAL JOB=$JOB_ID REMOTE_A=$DEST_A REMOTE_B=$DEST_B LOG=$LOG" > "$OUT" +DETAIL=SCRIPT_LINE=204:cat "$OUT" +DETAIL=UNIT_END=homelab-vm-local-dumps-2cloud.service +DETAIL=PAYLOAD_HOST_BEGIN=pve01 +DETAIL=PAYLOAD_ROOT=pve01:/var/lib/vz/dump +DETAIL=PAYLOAD_ROOT_STATUS=PRESENT +DETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst|BYTES=329541843|AGE_SECONDS=293553 +DETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-110-2026_07_24-03_30_02.tar.zst|BYTES=339140622|AGE_SECONDS=34352 +DETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-112-2026_07_22-03_30_09.tar.zst|BYTES=222794054|AGE_SECONDS=207149 +DETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-110-2026_07_22-03_30_01.tar.zst|BYTES=332951369|AGE_SECONDS=207154 +DETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst|BYTES=7021952392|AGE_SECONDS=293496 +DETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-qemu-150-2026_07_24-03_30_16.vma.zst|BYTES=7016624549|AGE_SECONDS=34298 +DETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-qemu-150-2026_07_22-03_30_14.vma.zst|BYTES=7052176114|AGE_SECONDS=207100 +DETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst|BYTES=222777732|AGE_SECONDS=293547 +DETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-112-2026_07_24-03_30_11.tar.zst|BYTES=222859457|AGE_SECONDS=34347 +DETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-qemu-150-2026_07_23-03_30_16.vma.zst|BYTES=7024165999|AGE_SECONDS=120695 +DETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-110-2026_07_23-03_30_02.tar.zst|BYTES=336245409|AGE_SECONDS=120752 +DETAIL=LOCAL_PAYLOAD=pve01:/var/lib/vz/dump/vzdump-lxc-112-2026_07_23-03_30_11.tar.zst|BYTES=222808025|AGE_SECONDS=120747 +DETAIL=PAYLOAD_ROOT=pve01:/mnt/staging/vzdump +DETAIL=PAYLOAD_ROOT_STATUS=PRESENT +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260705T004739Z/pve03/vzdump-qemu-130-2026_07_05-03_49_34.vma.zst|BYTES=11242736440|AGE_SECONDS=1674545 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260705T004739Z/pve02/vzdump-lxc-111-2026_07_05-03_48_52.tar.zst|BYTES=255533195|AGE_SECONDS=1674805 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260705T004739Z/pve02/vzdump-lxc-113-2026_07_05-03_49_19.tar.zst|BYTES=220380760|AGE_SECONDS=1674791 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260705T004739Z/pve01/vzdump-qemu-150-2026_07_05-03_47_52.vma.zst|BYTES=7004920660|AGE_SECONDS=1674842 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260705T004739Z/pve01/vzdump-lxc-112-2026_07_05-03_47_46.tar.zst|BYTES=222468077|AGE_SECONDS=1674891 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260705T004739Z/pve01/vzdump-lxc-110-2026_07_05-03_47_40.tar.zst|BYTES=269221740|AGE_SECONDS=1674897 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260628T004739Z/pve03/vzdump-qemu-130-2026_06_28-03_49_15.vma.zst|BYTES=27024990491|AGE_SECONDS=2279192 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260628T004739Z/pve02/vzdump-lxc-113-2026_06_28-03_49_00.tar.zst|BYTES=220326595|AGE_SECONDS=2279610 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260628T004739Z/pve02/vzdump-lxc-111-2026_06_28-03_48_46.tar.zst|BYTES=242676818|AGE_SECONDS=2279624 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260628T004739Z/pve01/vzdump-lxc-112-2026_06_28-03_47_46.tar.zst|BYTES=222365246|AGE_SECONDS=2279692 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260628T004739Z/pve01/vzdump-lxc-110-2026_06_28-03_47_40.tar.zst|BYTES=253314990|AGE_SECONDS=2279698 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260628T004739Z/pve01/vzdump-qemu-150-2026_06_28-03_47_51.vma.zst|BYTES=7003671925|AGE_SECONDS=2279646 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260719T004537Z/pve03/vzdump-qemu-130-2026_07_19-03_48_01.vma.zst|BYTES=11043726130|AGE_SECONDS=465037 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260719T004537Z/pve02/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst|BYTES=220049041|AGE_SECONDS=465284 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260719T004537Z/pve02/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst|BYTES=284443608|AGE_SECONDS=465301 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260719T004537Z/pve01/vzdump-qemu-150-2026_07_19-03_45_58.vma.zst|BYTES=7008027143|AGE_SECONDS=465328 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260719T004537Z/pve01/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst|BYTES=323668231|AGE_SECONDS=465417 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260719T004537Z/pve01/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst|BYTES=222745965|AGE_SECONDS=465406 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260712T004231Z/pve03/vzdump-qemu-130-2026_07_12-03_44_28.vma.zst|BYTES=10813915936|AGE_SECONDS=1070055 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260712T004231Z/pve02/vzdump-lxc-111-2026_07_12-03_43_45.tar.zst|BYTES=270924730|AGE_SECONDS=1070317 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260712T004231Z/pve02/vzdump-lxc-113-2026_07_12-03_44_07.tar.zst|BYTES=220482178|AGE_SECONDS=1070297 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260712T004231Z/pve01/vzdump-lxc-112-2026_07_12-03_42_39.tar.zst|BYTES=222566793|AGE_SECONDS=1070398 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260712T004231Z/pve01/vzdump-qemu-150-2026_07_12-03_42_45.vma.zst|BYTES=7014397787|AGE_SECONDS=1070350 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/auto-full-backup-20260712T004231Z/pve01/vzdump-lxc-110-2026_07_12-03_42_31.tar.zst|BYTES=302840996|AGE_SECONDS=1070404 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/full-manual-backup-20260615T092720Z/pve03/vzdump-qemu-130-2026_06_15-12_27_45.vma.zst|BYTES=829658873|AGE_SECONDS=3371676 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/full-manual-backup-20260615T092720Z/pve02/vzdump-lxc-113-2026_06_15-12_27_32.tar.zst|BYTES=220018661|AGE_SECONDS=3371701 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/full-manual-backup-20260615T092720Z/pve02/vzdump-lxc-111-2026_06_15-12_27_22.tar.zst|BYTES=228398861|AGE_SECONDS=3371712 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/full-manual-backup-20260615T092720Z/pve01/vzdump-qemu-150-2026_06_15-12_19_05.vma.zst|BYTES=6957596210|AGE_SECONDS=3372173 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/full-manual-backup-20260615T092720Z/pve01/vzdump-lxc-112-2026_06_15-12_23_56.tar.zst|BYTES=222110027|AGE_SECONDS=3371924 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/full-manual-backup-20260615T092720Z/pve01/vzdump-lxc-110-2026_06_15-12_23_52.tar.zst|BYTES=230562036|AGE_SECONDS=3371928 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/vm170-vm171/vzdump-qemu-170-2026_07_24-02_15_07.vma.zst|BYTES=11828704880|AGE_SECONDS=37618 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/vm170-vm171/vzdump-qemu-171-2026_07_23-02_36_37.vma.zst|BYTES=2157597134|AGE_SECONDS=122728 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/vm170-vm171/vzdump-qemu-170-2026_07_23-02_15_17.vma.zst|BYTES=11842330422|AGE_SECONDS=124008 +DETAIL=LOCAL_PAYLOAD=pve01:/mnt/staging/vzdump/vm170-vm171/vzdump-qemu-171-2026_07_24-02_36_27.vma.zst|BYTES=2158869638|AGE_SECONDS=36338 +DETAIL=PAYLOAD_ROOT=pve01:/var/lib/homelab-vm-backup/staging +DETAIL=PAYLOAD_ROOT_STATUS=ABSENT +DETAIL=PAYLOAD_HOST_SUMMARY=pve01|COUNT=46|BYTES=159150279782 +DETAIL=PAYLOAD_HOST_END=pve01 +DETAIL=PAYLOAD_HOST_BEGIN=pve02 +DETAIL=PAYLOAD_ROOT=pve02:/var/lib/vz/dump +DETAIL=PAYLOAD_ROOT_STATUS=PRESENT +DETAIL=LOCAL_PAYLOAD=pve02:/var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.vma.zst::997503422::1782865871.1700059310/var/lib/vz/dump/vzdump-lxc-111-2026_07_24-03_30_03.tar.zst::293632136::1784853018.7398517980/var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.vma.zst::868048369::1782779468.4903256780/var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.tar.zst::220080970::1784593833.3910787180/var/lib/vz/dump/vzdump-lxc-111-2026_07_22-03_30_01.tar.zst::290059742::1784680222.8748248510/var/lib/vz/dump/vzdump-lxc-111-2026_07_23-03_30_02.tar.zst::292185437::1784766624.6563558990/var/lib/vz/dump/vzdump-lxc-113-2026_07_23-03_30_24.tar.zst::220108944::1784766637.5724249240/var/lib/vz/dump/vzdump-lxc-113-2026_07_22-03_30_23.tar.zst::220095608::1784680240.6809207660/var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.tar.zst::288349157::1784593821.2480085730/var/lib/vz/dump/vzdump-lxc-113-2026_07_24-03_30_19.tar.zst|BYTES=220234127|AGE_SECONDS=34333 +DETAIL=PAYLOAD_ROOT=pve02:/mnt/staging/vzdump +DETAIL=PAYLOAD_ROOT_STATUS=PRESENT +DETAIL=LOCAL_PAYLOAD=pve02:/mnt/staging/vzdump/auto-pve02-20260615T121549Z/vzdump-lxc-113-2026_06_15-15_16_01.tar.zst::220085886::1781525770.6722751820/mnt/staging/vzdump/auto-pve02-20260615T121549Z/vzdump-lxc-111-2026_06_15-15_15_50.tar.zst::228433321::1781525759.9832231610/mnt/staging/vzdump/auto-pve02-20260615T184325Z/vzdump-lxc-111-2026_06_15-21_43_27.tar.zst::228695245::1781549016.3828865780/mnt/staging/vzdump/auto-pve02-20260615T184325Z/vzdump-lxc-113-2026_06_15-21_43_37.tar.zst::220088590::1781549027.0039397060/mnt/staging/vzdump/auto-pve02-20260615T145425Z/vzdump-lxc-111-2026_06_15-17_54_26.tar.zst::228364545::1781535276.0106286580/mnt/staging/vzdump/auto-pve02-20260615T145425Z/vzdump-lxc-113-2026_06_15-17_54_37.tar.zst::220087147::1781535286.6446806260/mnt/staging/vzdump/auto-pve02-20260615T125928Z/vzdump-lxc-111-2026_06_15-15_59_30.tar.zst::228433824::1781528379.7579730690/mnt/staging/vzdump/auto-pve02-20260615T125928Z/vzdump-lxc-113-2026_06_15-15_59_40.tar.zst::220086313::1781528390.6340260000/mnt/staging/vzdump/auto-pve02-20260628T004845Z/vzdump-lxc-113-2026_06_28-03_49_00.tar.zst::220326595::1782607753.1428569490/mnt/staging/vzdump/auto-pve02-20260628T004845Z/vzdump-lxc-111-2026_06_28-03_48_46.tar.zst::242676818::1782607739.4867871320/mnt/staging/vzdump/auto-pve02-20260615T201446Z/vzdump-lxc-111-2026_06_15-23_14_48.tar.zst::228706014::1781554497.6364731750/mnt/staging/vzdump/auto-pve02-20260615T201446Z/vzdump-lxc-113-2026_06_15-23_14_58.tar.zst::220089008::1781554508.5315278350/mnt/staging/vzdump/auto-pve02-20260705T004851Z/vzdump-lxc-111-2026_07_05-03_48_52.tar.zst::255533195::1783212558.3825236020/mnt/staging/vzdump/auto-pve02-20260705T004851Z/vzdump-lxc-113-2026_07_05-03_49_19.tar.zst::220380760::1783212571.8805933550/mnt/staging/vzdump/auto-pve02-20260719T004725Z/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst::284443608::1784422061.8525503200/mnt/staging/vzdump/auto-pve02-20260719T004725Z/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst::220049041::1784422078.9256407960/mnt/staging/vzdump/auto-pve02-20260712T004344Z/vzdump-lxc-111-2026_07_12-03_43_45.tar.zst::270924730::1783817046.4081420260/mnt/staging/vzdump/auto-pve02-20260712T004344Z/vzdump-lxc-113-2026_07_12-03_44_07.tar.zst::220482178::1783817065.7622435280/mnt/staging/vzdump/auto-pve02-20260621T004841Z/vzdump-lxc-113-2026_06_21-03_48_54.tar.zst::220165475::1782002947.0683211180/mnt/staging/vzdump/auto-pve02-20260621T004841Z/vzdump-lxc-111-2026_06_21-03_48_42.tar.zst::232731208::1782002933.6692546410/mnt/staging/vzdump/auto-pve02-20260615T161514Z/vzdump-lxc-111-2026_06_15-19_15_16.tar.zst::228462340::1781540125.7713503770/mnt/staging/vzdump/auto-pve02-20260615T161514Z/vzdump-lxc-113-2026_06_15-19_15_26.tar.zst::220087434::1781540136.7614046640/mnt/staging/vzdump/manual-pve02-20260615T092720Z/vzdump-lxc-113-2026_06_15-12_27_32.tar.zst::220018661::1781515662.0924756330/mnt/staging/vzdump/manual-pve02-20260615T092720Z/vzdump-lxc-111-2026_06_15-12_27_22.tar.zst::228398861::1781515651.4234214190/mnt/staging/vzdump/auto-pve02-20260616T023537Z/vzdump-lxc-111-2026_06_16-05_35_38.tar.zst::228693413::1781577348.0265431350/mnt/staging/vzdump/auto-pve02-20260616T023537Z/vzdump-lxc-113-2026_06_16-05_35_49.tar.zst::220093717::1781577358.9785981020/mnt/staging/vzdump/auto-pve02-20260615T095831Z/vzdump-lxc-111-2026_06_15-12_58_32.tar.zst::228398872::1781517522.2069308430/mnt/staging/vzdump/auto-pve02-20260615T095831Z/vzdump-lxc-113-2026_06_15-12_58_43.tar.zst|BYTES=220019157|AGE_SECONDS=3369831 +DETAIL=PAYLOAD_HOST_SUMMARY=pve02|COUNT=2|BYTES=440253284 +DETAIL=PAYLOAD_HOST_END=pve02 +DETAIL=PAYLOAD_HOST_BEGIN=pve03 +DETAIL=PAYLOAD_ROOT=pve03:/var/lib/vz/dump +DETAIL=PAYLOAD_ROOT_STATUS=PRESENT +DETAIL=LOCAL_PAYLOAD=pve03:/var/lib/vz/dump/vzdump-qemu-130-2026_07_23-03_30_03.vma.zst::11139455731::1784766836.0161157970/var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.vma.zst::11043834894::1784594034.3725716230/var/lib/vz/dump/vzdump-qemu-130-2026_07_22-03_30_02.vma.zst::11072513783::1784680438.8961071480/var/lib/vz/dump/vzdump-qemu-190-2026_07_23-03_33_56.vma.zst::1268419395::1784766881.2902288360/var/lib/vz/dump/vzdump-qemu-190-2026_07_24-03_34_00.vma.zst::1278418128::1784853285.0606354470/var/lib/vz/dump/vzdump-qemu-130-2026_07_24-03_30_02.vma.zst::11170433004::1784853240.1424086410/var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.vma.zst::1137072350::1784594077.6276811360/var/lib/vz/dump/vzdump-qemu-190-2026_07_22-03_33_59.vma.zst|BYTES=1177407443|AGE_SECONDS=206880 +DETAIL=PAYLOAD_ROOT=pve03:/mnt/staging/vzdump +DETAIL=PAYLOAD_ROOT_STATUS=PRESENT +DETAIL=LOCAL_PAYLOAD=pve03:/mnt/staging/vzdump/auto-pve03-20260719T004800Z/vzdump-qemu-130-2026_07_19-03_48_01.vma.zst::11043726130::1784422326.4811235470/mnt/staging/vzdump/auto-pve03-20260712T004427Z/vzdump-qemu-130-2026_07_12-03_44_28.vma.zst::10813915936::1783817308.4882503530/mnt/staging/vzdump/vm160-pve02-20260717T223819Z/vzdump-qemu-160-20260717T223819Z.vma.zst|BYTES=84995216465|AGE_SECONDS=559051 +DETAIL=PAYLOAD_HOST_SUMMARY=pve03|COUNT=2|BYTES=86172623908 +DETAIL=PAYLOAD_HOST_END=pve03 +DETAIL=POLICY_SUCCESS_REQUIRES=CLOUD1_VERIFIED,CLOUD2_VERIFIED,LOCAL_PAYLOAD_COUNT_ZERO DETAIL=SECRET_VALUES_OUTPUT=NONE -DETAIL=BACKUP_BAD_JOBS_ROOTCAUSE=PASS +DETAIL=BACKUP_NO_LOCAL_INVARIANT_AUDIT=PASS CHANGES_MADE=NO OUTPUT_END