1
0

runtime: publish HOMELAB-VM-2CLOUD-NOLOCAL-PREFLIGHT-435

This commit is contained in:
homelab-runtime-publisher
2026-07-24 10:15:43 +00:00
parent ab719b8b0b
commit 8fcd6b59e2
4 changed files with 691 additions and 29 deletions
File diff suppressed because one or more lines are too long
@@ -0,0 +1,335 @@
CHAT_OUTPUT_BEGIN
COMMAND_ID=HOMELAB-VM-2CLOUD-NOLOCAL-PREFLIGHT-435
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=cluster-audit
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
COMMAND_SHA256=86526459d447c1f1c45038a4d8dd89cb4cd68c9982e5bf88c20e2cacfb516c8b
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGES_MADE=false
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=90b33c793e8f02c3bbc5dba5e5d5e93c7d4ca4e647513b4cf189ba5db0cd05d7
SANITIZED_OUTPUT_SHA256=90b33c793e8f02c3bbc5dba5e5d5e93c7d4ca4e647513b4cf189ba5db0cd05d7
OUTPUT_BEGIN
DETAIL=RUNTIME_SHA256=265aa081c8c9981ec0724a38febf8d0f4269ed62bb853fce181ae809c291ed17
DETAIL=DESIRED_SHA256=265aa081c8c9981ec0724a38febf8d0f4269ed62bb853fce181ae809c291ed17
DETAIL=INSTALLED_LOGIC_BEGIN
DETAIL=SOURCE=17-BAD=0
DETAIL=SOURCE=18-COUNT=0
DETAIL=SOURCE=19-TOTAL=0
DETAIL=SOURCE=20-CHUNKED=0
DETAIL=SOURCE=21-VMIDS="${VMIDS:-110 112 150}"
DETAIL=SOURCE=22-
DETAIL=SOURCE=23-source /etc/homelab-backup.env 2>/dev/null || BAD=1
DETAIL=SOURCE=24-
DETAIL=SOURCE=25-JOB_ID="homelab-${NODE}-vm-local-dumps-${JOB_TS}"
DETAIL=SOURCE=26-WORK="$ROOT/$JOB_ID"
DETAIL=SOURCE=27-mkdir -p "$WORK"
DETAIL=SOURCE=28-
DETAIL=SOURCE=29:FILES_TSV="$WORK/files.tsv"
DETAIL=SOURCE=30-CHKS="$WORK/checksums.txt"
DETAIL=SOURCE=31-MAN="$WORK/manifest.json"
DETAIL=SOURCE=32-
DETAIL=SOURCE=33:: > "$FILES_TSV"
DETAIL=SOURCE=34-: > "$CHKS"
DETAIL=SOURCE=35-: > "$LOG"
DETAIL=SOURCE=36-
DETAIL=SOURCE=37-echo "job_id=$JOB_ID" >> "$LOG"
DETAIL=SOURCE=38-echo "vmids=$VMIDS" >> "$LOG"
DETAIL=SOURCE=39-echo "chunk_size_bytes=$CHUNK_SIZE_BYTES" >> "$LOG"
DETAIL=SOURCE=40-
DETAIL=SOURCE=41-for id in $VMIDS; do
DETAIL=SOURCE=42: F="$(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-${id}-*.vma.zst" -o -name "vzdump-lxc-${id}-*.tar.zst" \) -printf "%T@ %p\n" 2>/dev/null | sort -n | tail -n 1 | cut -d" " -f2-)"
DETAIL=SOURCE=43- if [ -z "$F" ] || [ ! -f "$F" ]; then
DETAIL=SOURCE=44- echo "MISSING vmid=$id" >> "$LOG"
DETAIL=SOURCE=45- BAD=$((BAD+1))
DETAIL=SOURCE=46- continue
DETAIL=SOURCE=47- fi
DETAIL=SOURCE=48-
DETAIL=SOURCE=49- BN="$(basename "$F")"
DETAIL=SOURCE=50- SIZE="$(stat -c %s "$F" 2>/dev/null || echo 0)"
DETAIL=SOURCE=51- MT="$(date -u -d "@$(stat -c %Y "$F" 2>/dev/null || echo 0)" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)"
DETAIL=SOURCE=52- KIND="unknown"
DETAIL=SOURCE=53- echo "$BN" | grep -q "^vzdump-qemu-" && KIND="qemu"
DETAIL=SOURCE=54- echo "$BN" | grep -q "^vzdump-lxc-" && KIND="lxc"
DETAIL=SOURCE=55- SHA="$(sha256sum "$F" | cut -d" " -f1)"
DETAIL=SOURCE=56- printf "%s %s\n" "$SHA" "$BN" >> "$CHKS"
DETAIL=SOURCE=57-
DETAIL=SOURCE=58- MODE="direct"
DETAIL=SOURCE=59- PARTS=0
DETAIL=SOURCE=60- if [ "$SIZE" -gt "$CHUNK_SIZE_BYTES" ]; then
DETAIL=SOURCE=61- MODE="chunked"
DETAIL=SOURCE=62- CHUNKED=$((CHUNKED+1))
DETAIL=SOURCE=--
DETAIL=SOURCE=64- rm -rf "$CDIR"
DETAIL=SOURCE=65- mkdir -p "$CDIR"
DETAIL=SOURCE=66- echo "SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR" >> "$LOG"
DETAIL=SOURCE=67- split -b "$CHUNK_SIZE_SPLIT" -d -a 4 "$F" "$CDIR/part-" >> "$LOG" 2>&1
DETAIL=SOURCE=68- PARTS="$(find "$CDIR" -maxdepth 1 -type f -name 'part-*' | wc -l)"
DETAIL=SOURCE=69- (cd "$CDIR" && sha256sum part-* > chunks.sha256)
DETAIL=SOURCE=70- if [ "$PARTS" -le 0 ]; then
DETAIL=SOURCE=71- echo "BAD split produced no chunks file=$BN" >> "$LOG"
DETAIL=SOURCE=72- BAD=$((BAD+1))
DETAIL=SOURCE=73- fi
DETAIL=SOURCE=74- fi
DETAIL=SOURCE=75-
DETAIL=SOURCE=76: printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" "$id" "$KIND" "$SIZE" "$SHA" "$MT" "$BN" "$F" "$MODE" "$PARTS" >> "$FILES_TSV"
DETAIL=SOURCE=77- COUNT=$((COUNT+1))
DETAIL=SOURCE=78- TOTAL=$((TOTAL+SIZE))
DETAIL=SOURCE=79-done
DETAIL=SOURCE=80-
DETAIL=SOURCE=81:python3 - "$JOB_ID" "$NODE" "$TS" "$VMIDS" "$COUNT" "$TOTAL" "$CHUNKED" "$CHUNK_SIZE_BYTES" "$FILES_TSV" > "$MAN" <<'PY'
DETAIL=SOURCE=82-import json, sys, pathlib
DETAIL=SOURCE=83-job,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:]
DETAIL=SOURCE=84-items=[]
DETAIL=SOURCE=85-for line in pathlib.Path(tsv).read_text().splitlines():
DETAIL=SOURCE=86- p=line.split("\t")
DETAIL=SOURCE=87- if len(p)>=9:
DETAIL=SOURCE=88- items.append({
DETAIL=SOURCE=89- "vmid":p[0],
DETAIL=SOURCE=90- "kind":p[1],
DETAIL=SOURCE=91- "size_bytes":int(p[2]),
DETAIL=SOURCE=92- "sha256":p[3],
DETAIL=SOURCE=93- "mtime_utc":p[4],
DETAIL=SOURCE=94- "filename":p[5],
DETAIL=SOURCE=95- "source_path":p[6],
DETAIL=SOURCE=96- "mode":p[7],
DETAIL=SOURCE=97- "parts":int(p[8]),
DETAIL=SOURCE=98- })
DETAIL=SOURCE=99-print(json.dumps({
DETAIL=SOURCE=100- "job_id":job,
DETAIL=SOURCE=101- "node":node,
DETAIL=SOURCE=--
DETAIL=SOURCE=103- "vmids":vmids.split(),
DETAIL=SOURCE=104- "count":int(count),
DETAIL=SOURCE=105- "total_bytes":int(total),
DETAIL=SOURCE=106- "chunked_count":int(chunked),
DETAIL=SOURCE=107- "chunk_size_bytes":int(chunk_size),
DETAIL=SOURCE=108- "items":items,
DETAIL=SOURCE=109-}, indent=2))
DETAIL=SOURCE=110-PY
DETAIL=SOURCE=111-
DETAIL=SOURCE=112-DEST_A="${REMOTE_A%/}/${NODE}/vm-local-dumps/${JOB_ID}"
DETAIL=SOURCE=113-DEST_B="${REMOTE_B%/}/${NODE}/vm-local-dumps/${JOB_ID}"
DETAIL=SOURCE=114-
DETAIL=SOURCE=115:upload_one_dest() {
DETAIL=SOURCE=116- DEST="$1"
DETAIL=SOURCE=117- rclone mkdir "$DEST/files" >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE=118- while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do
DETAIL=SOURCE=119- if [ "$mode" = "chunked" ]; then
DETAIL=SOURCE=120- CDIR="$CHUNK_ROOT/$JOB_ID/$bn.chunks"
DETAIL=SOURCE=121- echo "UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts" >> "$LOG"
DETAIL=SOURCE=122- rclone mkdir "$DEST/files/$bn.chunks" >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE=123- for part in "$CDIR"/part-*; do
DETAIL=SOURCE=124- [ -f "$part" ] || continue
DETAIL=SOURCE=125- pn="$(basename "$part")"
DETAIL=SOURCE=126- rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE=127- done
DETAIL=SOURCE=128- rclone copyto "$CDIR/chunks.sha256" "$DEST/files/$bn.chunks/chunks.sha256" >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE=129- else
DETAIL=SOURCE=130- echo "UPLOAD_DIRECT dest=$DEST file=$bn size=$size" >> "$LOG"
DETAIL=SOURCE=131- rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE=132- fi
DETAIL=SOURCE=133: done < "$FILES_TSV"
DETAIL=SOURCE=134-
DETAIL=SOURCE=135- rclone copyto "$CHKS" "$DEST/checksums.txt" >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE=136: rclone copyto "$FILES_TSV" "$DEST/files.tsv" >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE=137- rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE=138-}
DETAIL=SOURCE=139-
DETAIL=SOURCE=140:verify_one_dest() {
DETAIL=SOURCE=141- LABEL="$1"
DETAIL=SOURCE=142- DEST="$2"
DETAIL=SOURCE=143- VBASE="$VERIFY_ROOT/$JOB_ID/$LABEL"
DETAIL=SOURCE=144- rm -rf "$VBASE"
DETAIL=SOURCE=145- mkdir -p "$VBASE"
DETAIL=SOURCE=146- rclone copyto "$DEST/manifest.json" "$VBASE/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE=147- test -s "$VBASE/manifest.json" || BAD=$((BAD+1))
DETAIL=SOURCE=148-
DETAIL=SOURCE=149- while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do
DETAIL=SOURCE=150- if [ "$mode" = "chunked" ]; then
DETAIL=SOURCE=151- VDIR="$VBASE/$bn.chunks"
DETAIL=SOURCE=152- mkdir -p "$VDIR"
DETAIL=SOURCE=153- echo "VERIFY_CHUNKED label=$LABEL file=$bn parts=$parts" >> "$LOG"
DETAIL=SOURCE=154- rclone copy "$DEST/files/$bn.chunks" "$VDIR" --filter "+ part-*" --filter "+ chunks.sha256" --filter "- *" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE=155- (cd "$VDIR" && sha256sum -c chunks.sha256) >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE=156- CALC="$(find "$VDIR" -maxdepth 1 -type f -name 'part-*' | sort | xargs cat | sha256sum | cut -d" " -f1)"
DETAIL=SOURCE=157- if [ "$CALC" = "$sha" ]; then
DETAIL=SOURCE=158- echo "OK combined_sha label=$LABEL file=$bn sha=$sha" >> "$LOG"
DETAIL=SOURCE=159- else
DETAIL=SOURCE=160- echo "BAD combined_sha label=$LABEL file=$bn expected=$sha got=$CALC" >> "$LOG"
DETAIL=SOURCE=161- BAD=$((BAD+1))
DETAIL=SOURCE=162- fi
DETAIL=SOURCE=163- rm -rf "$VDIR"
DETAIL=SOURCE=164- else
DETAIL=SOURCE=165- echo "VERIFY_DIRECT label=$LABEL file=$bn" >> "$LOG"
DETAIL=SOURCE=166- rclone copyto "$DEST/files/$bn" "$VBASE/$bn" --transfers 1 >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE=167- printf "%s %s\n" "$sha" "$VBASE/$bn" | sha256sum -c - >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE=168- rm -f "$VBASE/$bn"
DETAIL=SOURCE=169- fi
DETAIL=SOURCE=170: done < "$FILES_TSV"
DETAIL=SOURCE=171- rm -rf "$VBASE"
DETAIL=SOURCE=172-}
DETAIL=SOURCE=173-
DETAIL=SOURCE=174-if [ "$BAD" -eq 0 ]; then
DETAIL=SOURCE=175: upload_one_dest "$DEST_A"
DETAIL=SOURCE=176: upload_one_dest "$DEST_B"
DETAIL=SOURCE=177-fi
DETAIL=SOURCE=178-
DETAIL=SOURCE=179-if [ "$BAD" -eq 0 ]; then
DETAIL=SOURCE=180: verify_one_dest "A" "$DEST_A"
DETAIL=SOURCE=181: verify_one_dest "B" "$DEST_B"
DETAIL=SOURCE=182-fi
DETAIL=SOURCE=183-
DETAIL=SOURCE=184-rm -rf "$VERIFY_ROOT/$JOB_ID"
DETAIL=SOURCE=185-rm -rf "$CHUNK_ROOT/$JOB_ID"
DETAIL=SOURCE=186-
DETAIL=SOURCE=187:# HOMELAB_NO_LOCAL_AFTER_DUAL_CLOUD_434
DETAIL=SOURCE=188-LOCAL_DUMPS_DELETED=false
DETAIL=SOURCE=189:DELETED_COUNT=0
DETAIL=SOURCE=190:DELETED_BYTES=0
DETAIL=SOURCE=191-
DETAIL=SOURCE=192-if [ "$BAD" -eq 0 ]; then
DETAIL=SOURCE=193: echo "PREDELETE_VALIDATION_BEGIN" >> "$LOG"
DETAIL=SOURCE=194-
DETAIL=SOURCE=195- while IFS="$(printf " ")" read -r id kind size sha mt bn path mode parts; do
DETAIL=SOURCE=196- case "$path" in
DETAIL=SOURCE=197- /var/lib/vz/dump/*)
DETAIL=SOURCE=198- ;;
DETAIL=SOURCE=199- *)
DETAIL=SOURCE=200- echo "REFUSE_DELETE_OUTSIDE_ALLOWED_ROOT path=$path" >> "$LOG"
DETAIL=SOURCE=201- BAD=$((BAD+1))
DETAIL=SOURCE=202- continue
DETAIL=SOURCE=203- ;;
DETAIL=SOURCE=204- esac
DETAIL=SOURCE=205-
DETAIL=SOURCE=206- if [ "$(basename "$path")" != "$bn" ]; then
DETAIL=SOURCE=207- echo "REFUSE_DELETE_BASENAME_MISMATCH path=$path expected=$bn" >> "$LOG"
DETAIL=SOURCE=208- BAD=$((BAD+1))
DETAIL=SOURCE=209- continue
DETAIL=SOURCE=210- fi
DETAIL=SOURCE=211-
DETAIL=SOURCE=212- if [ ! -f "$path" ] || [ -L "$path" ]; then
DETAIL=SOURCE=213- echo "REFUSE_DELETE_INVALID_SOURCE path=$path" >> "$LOG"
DETAIL=SOURCE=--
DETAIL=SOURCE=222- BAD=$((BAD+1))
DETAIL=SOURCE=223- continue
DETAIL=SOURCE=224- fi
DETAIL=SOURCE=225-
DETAIL=SOURCE=226- actual_sha="$(sha256sum "$path" 2>>"$LOG" | cut -d " " -f1 || true)"
DETAIL=SOURCE=227-
DETAIL=SOURCE=228- if [ "$actual_sha" != "$sha" ]; then
DETAIL=SOURCE=229- echo "REFUSE_DELETE_SHA_MISMATCH path=$path" >> "$LOG"
DETAIL=SOURCE=230- BAD=$((BAD+1))
DETAIL=SOURCE=231- continue
DETAIL=SOURCE=232- fi
DETAIL=SOURCE=233-
DETAIL=SOURCE=234: echo "PREDELETE_VALIDATION_OK path=$path size=$size sha=$sha" >> "$LOG"
DETAIL=SOURCE=235: done < "$FILES_TSV"
DETAIL=SOURCE=236-
DETAIL=SOURCE=237: echo "PREDELETE_VALIDATION_END bad=$BAD" >> "$LOG"
DETAIL=SOURCE=238-fi
DETAIL=SOURCE=239-
DETAIL=SOURCE=240-if [ "$BAD" -eq 0 ]; then
DETAIL=SOURCE=241: echo "LOCAL_PAYLOAD_DELETE_BEGIN" >> "$LOG"
DETAIL=SOURCE=242-
DETAIL=SOURCE=243- while IFS="$(printf " ")" read -r id kind size sha mt bn path mode parts; do
DETAIL=SOURCE=244: if rm -f -- "$path" 2>>"$LOG"; then
DETAIL=SOURCE=245- if [ ! -e "$path" ] && [ ! -L "$path" ]; then
DETAIL=SOURCE=246: DELETED_COUNT=$((DELETED_COUNT+1))
DETAIL=SOURCE=247: DELETED_BYTES=$((DELETED_BYTES+size))
DETAIL=SOURCE=248: echo "LOCAL_PAYLOAD_DELETED path=$path bytes=$size" >> "$LOG"
DETAIL=SOURCE=249- else
DETAIL=SOURCE=250: echo "LOCAL_PAYLOAD_DELETE_CONFIRM_FAILED path=$path" >> "$LOG"
DETAIL=SOURCE=251- BAD=$((BAD+1))
DETAIL=SOURCE=252- fi
DETAIL=SOURCE=253- else
DETAIL=SOURCE=254: echo "LOCAL_PAYLOAD_DELETE_FAILED path=$path" >> "$LOG"
DETAIL=SOURCE=255- BAD=$((BAD+1))
DETAIL=SOURCE=256- fi
DETAIL=SOURCE=257: done < "$FILES_TSV"
DETAIL=SOURCE=258-
DETAIL=SOURCE=259: if [ "$BAD" -eq 0 ] && [ "$DELETED_COUNT" -eq "$COUNT" ]; then
DETAIL=SOURCE=260- LOCAL_DUMPS_DELETED=true
DETAIL=SOURCE=261- fi
DETAIL=SOURCE=262-
DETAIL=SOURCE=263: echo "LOCAL_PAYLOAD_DELETE_END deleted_count=$DELETED_COUNT deleted_bytes=$DELETED_BYTES bad=$BAD" >> "$LOG"
DETAIL=SOURCE=264-fi
DETAIL=SOURCE=265-
DETAIL=SOURCE=266-if [ "$BAD" -eq 0 ] && [ "$LOCAL_DUMPS_DELETED" = true ]; then
DETAIL=SOURCE=267- STATUS="OK"
DETAIL=SOURCE=268-else
DETAIL=SOURCE=269- STATUS="WARN"
DETAIL=SOURCE=270-fi
DETAIL=SOURCE=271-
DETAIL=SOURCE=272-jq -n \
DETAIL=SOURCE=273- --arg job_id "$JOB_ID" \
DETAIL=SOURCE=274- --arg status "$STATUS" \
DETAIL=SOURCE=275- --arg node "$NODE" \
DETAIL=SOURCE=276- --arg remote_a "$DEST_A" \
DETAIL=SOURCE=277- --arg remote_b "$DEST_B" \
DETAIL=SOURCE=278- --argjson count "$COUNT" \
DETAIL=SOURCE=279- --argjson total_bytes "$TOTAL" \
DETAIL=SOURCE=280- --argjson chunked_count "$CHUNKED" \
DETAIL=SOURCE=281- --argjson chunk_size_bytes "$CHUNK_SIZE_BYTES" \
DETAIL=SOURCE=282- --arg log "$LOG" \
DETAIL=SOURCE=283: --argjson local_dumps_deleted "$LOCAL_DUMPS_DELETED" \
DETAIL=SOURCE=284: --argjson deleted_count "$DELETED_COUNT" \
DETAIL=SOURCE=285: --argjson deleted_bytes "$DELETED_BYTES" \
DETAIL=SOURCE=286: "{job_id:\$job_id,status:\$status,node:\$node,remote_a:\$remote_a,remote_b:\$remote_b,count:\$count,total_bytes:\$total_bytes,chunked_count:\$chunked_count,chunk_size_bytes:\$chunk_size_bytes,log:\$log,local_dumps_deleted:$local_dumps_deleted,deleted_count:$deleted_count,deleted_bytes:$deleted_bytes}" \
DETAIL=SOURCE=287- > "$ROOT/latest-vm-local-dumps-cloud.json"
DETAIL=SOURCE=288-
DETAIL=SOURCE=289:echo "STATUS=$STATUS TS=$TS TYPE=vm-local-dumps-cloud BAD=$BAD COUNT=$COUNT CHUNKED=$CHUNKED TOTAL_BYTES=$TOTAL JOB=$JOB_ID REMOTE_A=$DEST_A REMOTE_B=$DEST_B LOCAL_DUMPS_DELETED=$LOCAL_DUMPS_DELETED DELETED_COUNT=$DELETED_COUNT DELETED_BYTES=$DELETED_BYTES LOG=$LOG" > "$OUT"
DETAIL=SOURCE=290-cat "$OUT"
DETAIL=INSTALLED_LOGIC_END
DETAIL=UNIT_STATE_BEGIN
DETAIL=SERVICE_PROPERTY=Result=success
DETAIL=SERVICE_PROPERTY=ExecMainStatus=0
DETAIL=SERVICE_PROPERTY=ExecStart={ path=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; argv[]=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
DETAIL=SERVICE_PROPERTY=LoadState=loaded
DETAIL=SERVICE_PROPERTY=ActiveState=inactive
DETAIL=SERVICE_PROPERTY=SubState=dead
DETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/homelab-vm-local-dumps-2cloud.service
DETAIL=TIMER_PROPERTY=Unit=homelab-vm-local-dumps-2cloud.service
DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Sun 2026-07-26 10:11:29 MSK
DETAIL=TIMER_PROPERTY=LastTriggerUSec=Sun 2026-07-19 10:06:07 MSK
DETAIL=TIMER_PROPERTY=LoadState=loaded
DETAIL=TIMER_PROPERTY=ActiveState=active
DETAIL=TIMER_PROPERTY=SubState=waiting
DETAIL=TIMER_ENABLED=enabled
DETAIL=TIMER_ACTIVE=active
DETAIL=UNIT_STATE_END
DETAIL=CURRENT_DUMP_INVENTORY_BEGIN
DETAIL=CURRENT_DUMP_COUNT=12
DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst|BYTES:329541843|MODE:644|OWNER:root:root|MTIME:2026-07-21 03:30:09.768869633 +0300
DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-110-2026_07_22-03_30_01.tar.zst|BYTES:332951369|MODE:644|OWNER:root:root|MTIME:2026-07-22 03:30:08.768607650 +0300
DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-110-2026_07_23-03_30_02.tar.zst|BYTES:336245409|MODE:644|OWNER:root:root|MTIME:2026-07-23 03:30:10.836949127 +0300
DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-110-2026_07_24-03_30_02.tar.zst|BYTES:339140622|MODE:644|OWNER:root:root|MTIME:2026-07-24 03:30:10.776974593 +0300
DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst|BYTES:222777732|MODE:644|OWNER:root:root|MTIME:2026-07-21 03:30:15.905963829 +0300
DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-112-2026_07_22-03_30_09.tar.zst|BYTES:222794054|MODE:644|OWNER:root:root|MTIME:2026-07-22 03:30:14.284693312 +0300
DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-112-2026_07_23-03_30_11.tar.zst|BYTES:222808025|MODE:644|OWNER:root:root|MTIME:2026-07-23 03:30:16.408035215 +0300
DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-112-2026_07_24-03_30_11.tar.zst|BYTES:222859457|MODE:644|OWNER:root:root|MTIME:2026-07-24 03:30:16.202058480 +0300
DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst|BYTES:7021952392|MODE:644|OWNER:root:root|MTIME:2026-07-21 03:31:06.571741495 +0300
DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-qemu-150-2026_07_22-03_30_14.vma.zst|BYTES:7052176114|MODE:644|OWNER:root:root|MTIME:2026-07-22 03:31:02.895448212 +0300
DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-qemu-150-2026_07_23-03_30_16.vma.zst|BYTES:7024165999|MODE:644|OWNER:root:root|MTIME:2026-07-23 03:31:08.099833995 +0300
DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-qemu-150-2026_07_24-03_30_16.vma.zst|BYTES:7016624549|MODE:644|OWNER:root:root|MTIME:2026-07-24 03:31:04.670807942 +0300
DETAIL=CURRENT_DUMP_TOTAL_BYTES=30344037565
DETAIL=CURRENT_DUMP_INVENTORY_END
DETAIL=LAST_STATE_BEGIN
DETAIL=LAST_STATE={"status":"OK","job_id":"homelab-pve01-vm-local-dumps-20260719T070607Z","count":3,"total_bytes":7567711895,"local_dumps_deleted":false,"deleted_count":null,"deleted_bytes":null,"remote_a":"pve01-mail-01-crypt:homelab-backups/pve01/vm-local-dumps/homelab-pve01-vm-local-dumps-20260719T070607Z","remote_b":"pve01-mail-02-crypt:homelab-backups/pve01/vm-local-dumps/homelab-pve01-vm-local-dumps-20260719T070607Z"}
DETAIL=LAST_STATE_END
DETAIL=DELETE_POLICY_VALIDATION=PASS
DETAIL=BACKUP_PAYLOAD_CHANGED=NO
DETAIL=SECRET_VALUES_OUTPUT=NONE
CHANGES_MADE=NO
OUTPUT_END
CHAT_OUTPUT_END
+9 -9
View File
File diff suppressed because one or more lines are too long
+317 -20
View File
@@ -1,38 +1,335 @@
CHAT_OUTPUT_BEGIN CHAT_OUTPUT_BEGIN
COMMAND_ID=HOMELAB-VM-2CLOUD-NOLOCAL-FIX-434 COMMAND_ID=HOMELAB-VM-2CLOUD-NOLOCAL-PREFLIGHT-435
STATUS=OK STATUS=OK
RC=0 RC=0
HOST=pve01 HOST=pve01
MODE=change MODE=read-only
COMPONENT=cluster-audit COMPONENT=cluster-audit
REFERENCE_REGISTER_CHECK=OK REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
ERROR_REGISTER_CHECK=OK ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
COMMAND_SHA256=c04ecc684d6f03eeb05b48ac78557a165d4e70b028cda281900362ff571501be COMMAND_SHA256=86526459d447c1f1c45038a4d8dd89cb4cd68c9982e5bf88c20e2cacfb516c8b
DUPLICATE_FAILED_COMMAND_BLOCKED=false DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true EXECUTION_STARTED=true
CHANGES_MADE=true CHANGES_MADE=false
SANITIZED=yes SANITIZED=yes
SECRETS_INCLUDED=no SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=17d8b5b7583799851b97b71bf6047d0d5178c48fa761842d8a77dbb5e07456a3 RAW_EVIDENCE_SHA256=90b33c793e8f02c3bbc5dba5e5d5e93c7d4ca4e647513b4cf189ba5db0cd05d7
SANITIZED_OUTPUT_SHA256=17d8b5b7583799851b97b71bf6047d0d5178c48fa761842d8a77dbb5e07456a3 SANITIZED_OUTPUT_SHA256=90b33c793e8f02c3bbc5dba5e5d5e93c7d4ca4e647513b4cf189ba5db0cd05d7
OUTPUT_BEGIN OUTPUT_BEGIN
DETAIL=RUNTIME_ORIGINAL_SHA256=1de52eb4ddb17b0ecff2cf3f3021e65b92e91fb44416f97943c51e248c512ed6 DETAIL=RUNTIME_SHA256=265aa081c8c9981ec0724a38febf8d0f4269ed62bb853fce181ae809c291ed17
DETAIL=DESIRED_ORIGINAL_SHA256=1de52eb4ddb17b0ecff2cf3f3021e65b92e91fb44416f97943c51e248c512ed6 DETAIL=DESIRED_SHA256=265aa081c8c9981ec0724a38febf8d0f4269ed62bb853fce181ae809c291ed17
DETAIL=RUNTIME_UPDATED_SHA256=265aa081c8c9981ec0724a38febf8d0f4269ed62bb853fce181ae809c291ed17 DETAIL=INSTALLED_LOGIC_BEGIN
DETAIL=DESIRED_UPDATED_SHA256=265aa081c8c9981ec0724a38febf8d0f4269ed62bb853fce181ae809c291ed17 DETAIL=SOURCE=17-BAD=0
DETAIL=DELETE_ALLOWED_ROOT=/var/lib/vz/dump DETAIL=SOURCE=18-COUNT=0
DETAIL=DELETE_REQUIRES_CLOUD_A_VERIFY=YES DETAIL=SOURCE=19-TOTAL=0
DETAIL=DELETE_REQUIRES_CLOUD_B_VERIFY=YES DETAIL=SOURCE=20-CHUNKED=0
DETAIL=DELETE_REQUIRES_LOCAL_SIZE_MATCH=YES DETAIL=SOURCE=21-VMIDS="${VMIDS:-110 112 150}"
DETAIL=DELETE_REQUIRES_LOCAL_SHA256_MATCH=YES DETAIL=SOURCE=22-
DETAIL=BACKUP_PAYLOAD_DELETED_DURING_THIS_COMMAND=NO DETAIL=SOURCE=23-source /etc/homelab-backup.env 2>/dev/null || BAD=1
DETAIL=SCRIPT_BACKUP_BYTES=14544 DETAIL=SOURCE=24-
DETAIL=ROLLBACK_STATE=NOT_REQUIRED DETAIL=SOURCE=25-JOB_ID="homelab-${NODE}-vm-local-dumps-${JOB_TS}"
DETAIL=VM_2CLOUD_NOLOCAL_FIX=PASS DETAIL=SOURCE=26-WORK="$ROOT/$JOB_ID"
CHANGES_MADE=YES DETAIL=SOURCE=27-mkdir -p "$WORK"
DETAIL=SOURCE=28-
DETAIL=SOURCE=29:FILES_TSV="$WORK/files.tsv"
DETAIL=SOURCE=30-CHKS="$WORK/checksums.txt"
DETAIL=SOURCE=31-MAN="$WORK/manifest.json"
DETAIL=SOURCE=32-
DETAIL=SOURCE=33:: > "$FILES_TSV"
DETAIL=SOURCE=34-: > "$CHKS"
DETAIL=SOURCE=35-: > "$LOG"
DETAIL=SOURCE=36-
DETAIL=SOURCE=37-echo "job_id=$JOB_ID" >> "$LOG"
DETAIL=SOURCE=38-echo "vmids=$VMIDS" >> "$LOG"
DETAIL=SOURCE=39-echo "chunk_size_bytes=$CHUNK_SIZE_BYTES" >> "$LOG"
DETAIL=SOURCE=40-
DETAIL=SOURCE=41-for id in $VMIDS; do
DETAIL=SOURCE=42: F="$(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-${id}-*.vma.zst" -o -name "vzdump-lxc-${id}-*.tar.zst" \) -printf "%T@ %p\n" 2>/dev/null | sort -n | tail -n 1 | cut -d" " -f2-)"
DETAIL=SOURCE=43- if [ -z "$F" ] || [ ! -f "$F" ]; then
DETAIL=SOURCE=44- echo "MISSING vmid=$id" >> "$LOG"
DETAIL=SOURCE=45- BAD=$((BAD+1))
DETAIL=SOURCE=46- continue
DETAIL=SOURCE=47- fi
DETAIL=SOURCE=48-
DETAIL=SOURCE=49- BN="$(basename "$F")"
DETAIL=SOURCE=50- SIZE="$(stat -c %s "$F" 2>/dev/null || echo 0)"
DETAIL=SOURCE=51- MT="$(date -u -d "@$(stat -c %Y "$F" 2>/dev/null || echo 0)" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)"
DETAIL=SOURCE=52- KIND="unknown"
DETAIL=SOURCE=53- echo "$BN" | grep -q "^vzdump-qemu-" && KIND="qemu"
DETAIL=SOURCE=54- echo "$BN" | grep -q "^vzdump-lxc-" && KIND="lxc"
DETAIL=SOURCE=55- SHA="$(sha256sum "$F" | cut -d" " -f1)"
DETAIL=SOURCE=56- printf "%s %s\n" "$SHA" "$BN" >> "$CHKS"
DETAIL=SOURCE=57-
DETAIL=SOURCE=58- MODE="direct"
DETAIL=SOURCE=59- PARTS=0
DETAIL=SOURCE=60- if [ "$SIZE" -gt "$CHUNK_SIZE_BYTES" ]; then
DETAIL=SOURCE=61- MODE="chunked"
DETAIL=SOURCE=62- CHUNKED=$((CHUNKED+1))
DETAIL=SOURCE=--
DETAIL=SOURCE=64- rm -rf "$CDIR"
DETAIL=SOURCE=65- mkdir -p "$CDIR"
DETAIL=SOURCE=66- echo "SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR" >> "$LOG"
DETAIL=SOURCE=67- split -b "$CHUNK_SIZE_SPLIT" -d -a 4 "$F" "$CDIR/part-" >> "$LOG" 2>&1
DETAIL=SOURCE=68- PARTS="$(find "$CDIR" -maxdepth 1 -type f -name 'part-*' | wc -l)"
DETAIL=SOURCE=69- (cd "$CDIR" && sha256sum part-* > chunks.sha256)
DETAIL=SOURCE=70- if [ "$PARTS" -le 0 ]; then
DETAIL=SOURCE=71- echo "BAD split produced no chunks file=$BN" >> "$LOG"
DETAIL=SOURCE=72- BAD=$((BAD+1))
DETAIL=SOURCE=73- fi
DETAIL=SOURCE=74- fi
DETAIL=SOURCE=75-
DETAIL=SOURCE=76: printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" "$id" "$KIND" "$SIZE" "$SHA" "$MT" "$BN" "$F" "$MODE" "$PARTS" >> "$FILES_TSV"
DETAIL=SOURCE=77- COUNT=$((COUNT+1))
DETAIL=SOURCE=78- TOTAL=$((TOTAL+SIZE))
DETAIL=SOURCE=79-done
DETAIL=SOURCE=80-
DETAIL=SOURCE=81:python3 - "$JOB_ID" "$NODE" "$TS" "$VMIDS" "$COUNT" "$TOTAL" "$CHUNKED" "$CHUNK_SIZE_BYTES" "$FILES_TSV" > "$MAN" <<'PY'
DETAIL=SOURCE=82-import json, sys, pathlib
DETAIL=SOURCE=83-job,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:]
DETAIL=SOURCE=84-items=[]
DETAIL=SOURCE=85-for line in pathlib.Path(tsv).read_text().splitlines():
DETAIL=SOURCE=86- p=line.split("\t")
DETAIL=SOURCE=87- if len(p)>=9:
DETAIL=SOURCE=88- items.append({
DETAIL=SOURCE=89- "vmid":p[0],
DETAIL=SOURCE=90- "kind":p[1],
DETAIL=SOURCE=91- "size_bytes":int(p[2]),
DETAIL=SOURCE=92- "sha256":p[3],
DETAIL=SOURCE=93- "mtime_utc":p[4],
DETAIL=SOURCE=94- "filename":p[5],
DETAIL=SOURCE=95- "source_path":p[6],
DETAIL=SOURCE=96- "mode":p[7],
DETAIL=SOURCE=97- "parts":int(p[8]),
DETAIL=SOURCE=98- })
DETAIL=SOURCE=99-print(json.dumps({
DETAIL=SOURCE=100- "job_id":job,
DETAIL=SOURCE=101- "node":node,
DETAIL=SOURCE=--
DETAIL=SOURCE=103- "vmids":vmids.split(),
DETAIL=SOURCE=104- "count":int(count),
DETAIL=SOURCE=105- "total_bytes":int(total),
DETAIL=SOURCE=106- "chunked_count":int(chunked),
DETAIL=SOURCE=107- "chunk_size_bytes":int(chunk_size),
DETAIL=SOURCE=108- "items":items,
DETAIL=SOURCE=109-}, indent=2))
DETAIL=SOURCE=110-PY
DETAIL=SOURCE=111-
DETAIL=SOURCE=112-DEST_A="${REMOTE_A%/}/${NODE}/vm-local-dumps/${JOB_ID}"
DETAIL=SOURCE=113-DEST_B="${REMOTE_B%/}/${NODE}/vm-local-dumps/${JOB_ID}"
DETAIL=SOURCE=114-
DETAIL=SOURCE=115:upload_one_dest() {
DETAIL=SOURCE=116- DEST="$1"
DETAIL=SOURCE=117- rclone mkdir "$DEST/files" >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE=118- while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do
DETAIL=SOURCE=119- if [ "$mode" = "chunked" ]; then
DETAIL=SOURCE=120- CDIR="$CHUNK_ROOT/$JOB_ID/$bn.chunks"
DETAIL=SOURCE=121- echo "UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts" >> "$LOG"
DETAIL=SOURCE=122- rclone mkdir "$DEST/files/$bn.chunks" >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE=123- for part in "$CDIR"/part-*; do
DETAIL=SOURCE=124- [ -f "$part" ] || continue
DETAIL=SOURCE=125- pn="$(basename "$part")"
DETAIL=SOURCE=126- rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE=127- done
DETAIL=SOURCE=128- rclone copyto "$CDIR/chunks.sha256" "$DEST/files/$bn.chunks/chunks.sha256" >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE=129- else
DETAIL=SOURCE=130- echo "UPLOAD_DIRECT dest=$DEST file=$bn size=$size" >> "$LOG"
DETAIL=SOURCE=131- rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE=132- fi
DETAIL=SOURCE=133: done < "$FILES_TSV"
DETAIL=SOURCE=134-
DETAIL=SOURCE=135- rclone copyto "$CHKS" "$DEST/checksums.txt" >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE=136: rclone copyto "$FILES_TSV" "$DEST/files.tsv" >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE=137- rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE=138-}
DETAIL=SOURCE=139-
DETAIL=SOURCE=140:verify_one_dest() {
DETAIL=SOURCE=141- LABEL="$1"
DETAIL=SOURCE=142- DEST="$2"
DETAIL=SOURCE=143- VBASE="$VERIFY_ROOT/$JOB_ID/$LABEL"
DETAIL=SOURCE=144- rm -rf "$VBASE"
DETAIL=SOURCE=145- mkdir -p "$VBASE"
DETAIL=SOURCE=146- rclone copyto "$DEST/manifest.json" "$VBASE/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE=147- test -s "$VBASE/manifest.json" || BAD=$((BAD+1))
DETAIL=SOURCE=148-
DETAIL=SOURCE=149- while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do
DETAIL=SOURCE=150- if [ "$mode" = "chunked" ]; then
DETAIL=SOURCE=151- VDIR="$VBASE/$bn.chunks"
DETAIL=SOURCE=152- mkdir -p "$VDIR"
DETAIL=SOURCE=153- echo "VERIFY_CHUNKED label=$LABEL file=$bn parts=$parts" >> "$LOG"
DETAIL=SOURCE=154- rclone copy "$DEST/files/$bn.chunks" "$VDIR" --filter "+ part-*" --filter "+ chunks.sha256" --filter "- *" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE=155- (cd "$VDIR" && sha256sum -c chunks.sha256) >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE=156- CALC="$(find "$VDIR" -maxdepth 1 -type f -name 'part-*' | sort | xargs cat | sha256sum | cut -d" " -f1)"
DETAIL=SOURCE=157- if [ "$CALC" = "$sha" ]; then
DETAIL=SOURCE=158- echo "OK combined_sha label=$LABEL file=$bn sha=$sha" >> "$LOG"
DETAIL=SOURCE=159- else
DETAIL=SOURCE=160- echo "BAD combined_sha label=$LABEL file=$bn expected=$sha got=$CALC" >> "$LOG"
DETAIL=SOURCE=161- BAD=$((BAD+1))
DETAIL=SOURCE=162- fi
DETAIL=SOURCE=163- rm -rf "$VDIR"
DETAIL=SOURCE=164- else
DETAIL=SOURCE=165- echo "VERIFY_DIRECT label=$LABEL file=$bn" >> "$LOG"
DETAIL=SOURCE=166- rclone copyto "$DEST/files/$bn" "$VBASE/$bn" --transfers 1 >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE=167- printf "%s %s\n" "$sha" "$VBASE/$bn" | sha256sum -c - >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE=168- rm -f "$VBASE/$bn"
DETAIL=SOURCE=169- fi
DETAIL=SOURCE=170: done < "$FILES_TSV"
DETAIL=SOURCE=171- rm -rf "$VBASE"
DETAIL=SOURCE=172-}
DETAIL=SOURCE=173-
DETAIL=SOURCE=174-if [ "$BAD" -eq 0 ]; then
DETAIL=SOURCE=175: upload_one_dest "$DEST_A"
DETAIL=SOURCE=176: upload_one_dest "$DEST_B"
DETAIL=SOURCE=177-fi
DETAIL=SOURCE=178-
DETAIL=SOURCE=179-if [ "$BAD" -eq 0 ]; then
DETAIL=SOURCE=180: verify_one_dest "A" "$DEST_A"
DETAIL=SOURCE=181: verify_one_dest "B" "$DEST_B"
DETAIL=SOURCE=182-fi
DETAIL=SOURCE=183-
DETAIL=SOURCE=184-rm -rf "$VERIFY_ROOT/$JOB_ID"
DETAIL=SOURCE=185-rm -rf "$CHUNK_ROOT/$JOB_ID"
DETAIL=SOURCE=186-
DETAIL=SOURCE=187:# HOMELAB_NO_LOCAL_AFTER_DUAL_CLOUD_434
DETAIL=SOURCE=188-LOCAL_DUMPS_DELETED=false
DETAIL=SOURCE=189:DELETED_COUNT=0
DETAIL=SOURCE=190:DELETED_BYTES=0
DETAIL=SOURCE=191-
DETAIL=SOURCE=192-if [ "$BAD" -eq 0 ]; then
DETAIL=SOURCE=193: echo "PREDELETE_VALIDATION_BEGIN" >> "$LOG"
DETAIL=SOURCE=194-
DETAIL=SOURCE=195- while IFS="$(printf " ")" read -r id kind size sha mt bn path mode parts; do
DETAIL=SOURCE=196- case "$path" in
DETAIL=SOURCE=197- /var/lib/vz/dump/*)
DETAIL=SOURCE=198- ;;
DETAIL=SOURCE=199- *)
DETAIL=SOURCE=200- echo "REFUSE_DELETE_OUTSIDE_ALLOWED_ROOT path=$path" >> "$LOG"
DETAIL=SOURCE=201- BAD=$((BAD+1))
DETAIL=SOURCE=202- continue
DETAIL=SOURCE=203- ;;
DETAIL=SOURCE=204- esac
DETAIL=SOURCE=205-
DETAIL=SOURCE=206- if [ "$(basename "$path")" != "$bn" ]; then
DETAIL=SOURCE=207- echo "REFUSE_DELETE_BASENAME_MISMATCH path=$path expected=$bn" >> "$LOG"
DETAIL=SOURCE=208- BAD=$((BAD+1))
DETAIL=SOURCE=209- continue
DETAIL=SOURCE=210- fi
DETAIL=SOURCE=211-
DETAIL=SOURCE=212- if [ ! -f "$path" ] || [ -L "$path" ]; then
DETAIL=SOURCE=213- echo "REFUSE_DELETE_INVALID_SOURCE path=$path" >> "$LOG"
DETAIL=SOURCE=--
DETAIL=SOURCE=222- BAD=$((BAD+1))
DETAIL=SOURCE=223- continue
DETAIL=SOURCE=224- fi
DETAIL=SOURCE=225-
DETAIL=SOURCE=226- actual_sha="$(sha256sum "$path" 2>>"$LOG" | cut -d " " -f1 || true)"
DETAIL=SOURCE=227-
DETAIL=SOURCE=228- if [ "$actual_sha" != "$sha" ]; then
DETAIL=SOURCE=229- echo "REFUSE_DELETE_SHA_MISMATCH path=$path" >> "$LOG"
DETAIL=SOURCE=230- BAD=$((BAD+1))
DETAIL=SOURCE=231- continue
DETAIL=SOURCE=232- fi
DETAIL=SOURCE=233-
DETAIL=SOURCE=234: echo "PREDELETE_VALIDATION_OK path=$path size=$size sha=$sha" >> "$LOG"
DETAIL=SOURCE=235: done < "$FILES_TSV"
DETAIL=SOURCE=236-
DETAIL=SOURCE=237: echo "PREDELETE_VALIDATION_END bad=$BAD" >> "$LOG"
DETAIL=SOURCE=238-fi
DETAIL=SOURCE=239-
DETAIL=SOURCE=240-if [ "$BAD" -eq 0 ]; then
DETAIL=SOURCE=241: echo "LOCAL_PAYLOAD_DELETE_BEGIN" >> "$LOG"
DETAIL=SOURCE=242-
DETAIL=SOURCE=243- while IFS="$(printf " ")" read -r id kind size sha mt bn path mode parts; do
DETAIL=SOURCE=244: if rm -f -- "$path" 2>>"$LOG"; then
DETAIL=SOURCE=245- if [ ! -e "$path" ] && [ ! -L "$path" ]; then
DETAIL=SOURCE=246: DELETED_COUNT=$((DELETED_COUNT+1))
DETAIL=SOURCE=247: DELETED_BYTES=$((DELETED_BYTES+size))
DETAIL=SOURCE=248: echo "LOCAL_PAYLOAD_DELETED path=$path bytes=$size" >> "$LOG"
DETAIL=SOURCE=249- else
DETAIL=SOURCE=250: echo "LOCAL_PAYLOAD_DELETE_CONFIRM_FAILED path=$path" >> "$LOG"
DETAIL=SOURCE=251- BAD=$((BAD+1))
DETAIL=SOURCE=252- fi
DETAIL=SOURCE=253- else
DETAIL=SOURCE=254: echo "LOCAL_PAYLOAD_DELETE_FAILED path=$path" >> "$LOG"
DETAIL=SOURCE=255- BAD=$((BAD+1))
DETAIL=SOURCE=256- fi
DETAIL=SOURCE=257: done < "$FILES_TSV"
DETAIL=SOURCE=258-
DETAIL=SOURCE=259: if [ "$BAD" -eq 0 ] && [ "$DELETED_COUNT" -eq "$COUNT" ]; then
DETAIL=SOURCE=260- LOCAL_DUMPS_DELETED=true
DETAIL=SOURCE=261- fi
DETAIL=SOURCE=262-
DETAIL=SOURCE=263: echo "LOCAL_PAYLOAD_DELETE_END deleted_count=$DELETED_COUNT deleted_bytes=$DELETED_BYTES bad=$BAD" >> "$LOG"
DETAIL=SOURCE=264-fi
DETAIL=SOURCE=265-
DETAIL=SOURCE=266-if [ "$BAD" -eq 0 ] && [ "$LOCAL_DUMPS_DELETED" = true ]; then
DETAIL=SOURCE=267- STATUS="OK"
DETAIL=SOURCE=268-else
DETAIL=SOURCE=269- STATUS="WARN"
DETAIL=SOURCE=270-fi
DETAIL=SOURCE=271-
DETAIL=SOURCE=272-jq -n \
DETAIL=SOURCE=273- --arg job_id "$JOB_ID" \
DETAIL=SOURCE=274- --arg status "$STATUS" \
DETAIL=SOURCE=275- --arg node "$NODE" \
DETAIL=SOURCE=276- --arg remote_a "$DEST_A" \
DETAIL=SOURCE=277- --arg remote_b "$DEST_B" \
DETAIL=SOURCE=278- --argjson count "$COUNT" \
DETAIL=SOURCE=279- --argjson total_bytes "$TOTAL" \
DETAIL=SOURCE=280- --argjson chunked_count "$CHUNKED" \
DETAIL=SOURCE=281- --argjson chunk_size_bytes "$CHUNK_SIZE_BYTES" \
DETAIL=SOURCE=282- --arg log "$LOG" \
DETAIL=SOURCE=283: --argjson local_dumps_deleted "$LOCAL_DUMPS_DELETED" \
DETAIL=SOURCE=284: --argjson deleted_count "$DELETED_COUNT" \
DETAIL=SOURCE=285: --argjson deleted_bytes "$DELETED_BYTES" \
DETAIL=SOURCE=286: "{job_id:\$job_id,status:\$status,node:\$node,remote_a:\$remote_a,remote_b:\$remote_b,count:\$count,total_bytes:\$total_bytes,chunked_count:\$chunked_count,chunk_size_bytes:\$chunk_size_bytes,log:\$log,local_dumps_deleted:$local_dumps_deleted,deleted_count:$deleted_count,deleted_bytes:$deleted_bytes}" \
DETAIL=SOURCE=287- > "$ROOT/latest-vm-local-dumps-cloud.json"
DETAIL=SOURCE=288-
DETAIL=SOURCE=289:echo "STATUS=$STATUS TS=$TS TYPE=vm-local-dumps-cloud BAD=$BAD COUNT=$COUNT CHUNKED=$CHUNKED TOTAL_BYTES=$TOTAL JOB=$JOB_ID REMOTE_A=$DEST_A REMOTE_B=$DEST_B LOCAL_DUMPS_DELETED=$LOCAL_DUMPS_DELETED DELETED_COUNT=$DELETED_COUNT DELETED_BYTES=$DELETED_BYTES LOG=$LOG" > "$OUT"
DETAIL=SOURCE=290-cat "$OUT"
DETAIL=INSTALLED_LOGIC_END
DETAIL=UNIT_STATE_BEGIN
DETAIL=SERVICE_PROPERTY=Result=success
DETAIL=SERVICE_PROPERTY=ExecMainStatus=0
DETAIL=SERVICE_PROPERTY=ExecStart={ path=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; argv[]=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
DETAIL=SERVICE_PROPERTY=LoadState=loaded
DETAIL=SERVICE_PROPERTY=ActiveState=inactive
DETAIL=SERVICE_PROPERTY=SubState=dead
DETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/homelab-vm-local-dumps-2cloud.service
DETAIL=TIMER_PROPERTY=Unit=homelab-vm-local-dumps-2cloud.service
DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=Sun 2026-07-26 10:11:29 MSK
DETAIL=TIMER_PROPERTY=LastTriggerUSec=Sun 2026-07-19 10:06:07 MSK
DETAIL=TIMER_PROPERTY=LoadState=loaded
DETAIL=TIMER_PROPERTY=ActiveState=active
DETAIL=TIMER_PROPERTY=SubState=waiting
DETAIL=TIMER_ENABLED=enabled
DETAIL=TIMER_ACTIVE=active
DETAIL=UNIT_STATE_END
DETAIL=CURRENT_DUMP_INVENTORY_BEGIN
DETAIL=CURRENT_DUMP_COUNT=12
DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst|BYTES:329541843|MODE:644|OWNER:root:root|MTIME:2026-07-21 03:30:09.768869633 +0300
DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-110-2026_07_22-03_30_01.tar.zst|BYTES:332951369|MODE:644|OWNER:root:root|MTIME:2026-07-22 03:30:08.768607650 +0300
DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-110-2026_07_23-03_30_02.tar.zst|BYTES:336245409|MODE:644|OWNER:root:root|MTIME:2026-07-23 03:30:10.836949127 +0300
DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-110-2026_07_24-03_30_02.tar.zst|BYTES:339140622|MODE:644|OWNER:root:root|MTIME:2026-07-24 03:30:10.776974593 +0300
DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst|BYTES:222777732|MODE:644|OWNER:root:root|MTIME:2026-07-21 03:30:15.905963829 +0300
DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-112-2026_07_22-03_30_09.tar.zst|BYTES:222794054|MODE:644|OWNER:root:root|MTIME:2026-07-22 03:30:14.284693312 +0300
DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-112-2026_07_23-03_30_11.tar.zst|BYTES:222808025|MODE:644|OWNER:root:root|MTIME:2026-07-23 03:30:16.408035215 +0300
DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-lxc-112-2026_07_24-03_30_11.tar.zst|BYTES:222859457|MODE:644|OWNER:root:root|MTIME:2026-07-24 03:30:16.202058480 +0300
DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst|BYTES:7021952392|MODE:644|OWNER:root:root|MTIME:2026-07-21 03:31:06.571741495 +0300
DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-qemu-150-2026_07_22-03_30_14.vma.zst|BYTES:7052176114|MODE:644|OWNER:root:root|MTIME:2026-07-22 03:31:02.895448212 +0300
DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-qemu-150-2026_07_23-03_30_16.vma.zst|BYTES:7024165999|MODE:644|OWNER:root:root|MTIME:2026-07-23 03:31:08.099833995 +0300
DETAIL=CURRENT_DUMP=PATH:/var/lib/vz/dump/vzdump-qemu-150-2026_07_24-03_30_16.vma.zst|BYTES:7016624549|MODE:644|OWNER:root:root|MTIME:2026-07-24 03:31:04.670807942 +0300
DETAIL=CURRENT_DUMP_TOTAL_BYTES=30344037565
DETAIL=CURRENT_DUMP_INVENTORY_END
DETAIL=LAST_STATE_BEGIN
DETAIL=LAST_STATE={"status":"OK","job_id":"homelab-pve01-vm-local-dumps-20260719T070607Z","count":3,"total_bytes":7567711895,"local_dumps_deleted":false,"deleted_count":null,"deleted_bytes":null,"remote_a":"pve01-mail-01-crypt:homelab-backups/pve01/vm-local-dumps/homelab-pve01-vm-local-dumps-20260719T070607Z","remote_b":"pve01-mail-02-crypt:homelab-backups/pve01/vm-local-dumps/homelab-pve01-vm-local-dumps-20260719T070607Z"}
DETAIL=LAST_STATE_END
DETAIL=DELETE_POLICY_VALIDATION=PASS
DETAIL=BACKUP_PAYLOAD_CHANGED=NO
DETAIL=SECRET_VALUES_OUTPUT=NONE
CHANGES_MADE=NO
OUTPUT_END OUTPUT_END
CHAT_OUTPUT_END CHAT_OUTPUT_END