CHAT_OUTPUT_BEGIN
COMMAND_ID=BACKUP-VM-DUMP-SOURCE-077
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=backup-audit
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
COMMAND_SHA256=b835d1210c5520c9d8d56eac7575cc679e5d0a22ce04f879cf8af953e2bb9b9e
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGES_MADE=false
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=9dd20a7d68dce96c8f08bcb4407101b10e80a36077f51706b0ba8b709099ba58
SANITIZED_OUTPUT_SHA256=9dd20a7d68dce96c8f08bcb4407101b10e80a36077f51706b0ba8b709099ba58
OUTPUT_BEGIN
=== SCRIPT_METADATA ===
1de52eb4ddb17b0ecff2cf3f3021e65b92e91fb44416f97943c51e248c512ed6  /usr/local/sbin/homelab-vm-local-dumps-2cloud
  File: /usr/local/sbin/homelab-vm-local-dumps-2cloud
  Size: 7272      	Blocks: 16         IO Block: 4096   regular file
Device: 252,1	Inode: 5148408     Links: 1
Access: (0700/-rwx------)  Uid: (    0/    root)   Gid: (    0/    root)
Access: 2026-07-21 00:05:00.471248283 +0300
Modify: 2026-07-06 18:49:49.740645050 +0300
Change: 2026-07-06 18:49:49.740645050 +0300
 Birth: 2026-07-06 18:27:29.152531375 +0300
=== SCRIPT_SOURCE ===
#!/usr/bin/env bash
set +u

OUT="/var/lib/homelab-health/vm-local-dumps-cloud.txt"
ROOT="/var/lib/homelab-vm-backup"
LOG="$ROOT/vm-local-dumps-2cloud-latest.log"
VERIFY_ROOT="/var/tmp/homelab-vm-local-dumps-verify"
CHUNK_ROOT="/var/tmp/homelab-vm-local-dumps-chunks"
CHUNK_SIZE_BYTES="${CHUNK_SIZE_BYTES:-1073741824}"
CHUNK_SIZE_SPLIT="${CHUNK_SIZE_SPLIT:-1024M}"

mkdir -p /var/lib/homelab-health "$ROOT" "$VERIFY_ROOT" "$CHUNK_ROOT"

TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
JOB_TS="$(date -u +%Y%m%dT%H%M%SZ)"
NODE="$(hostname -s)"
BAD=0
COUNT=0
TOTAL=0
CHUNKED=0
VMIDS="${VMIDS:-110 112 150}"

source /etc/homelab-backup.env 2>/dev/null || BAD=1

JOB_ID="homelab-${NODE}-vm-local-dumps-${JOB_TS}"
WORK="$ROOT/$JOB_ID"
mkdir -p "$WORK"

FILES_TSV="$WORK/files.tsv"
CHKS="$WORK/checksums.txt"
MAN="$WORK/manifest.json"

: > "$FILES_TSV"
: > "$CHKS"
: > "$LOG"

echo "job_id=$JOB_ID" >> "$LOG"
echo "vmids=$VMIDS" >> "$LOG"
echo "chunk_size_bytes=$CHUNK_SIZE_BYTES" >> "$LOG"

for id in $VMIDS; do
  F="$(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-${id}-*.vma.zst" -o -name "vzdump-lxc-${id}-*.tar.zst" \) -printf "%T@ %p\n" 2>/dev/null | sort -n | tail -n 1 | cut -d" " -f2-)"
  if [ -z "$F" ] || [ ! -f "$F" ]; then
    echo "MISSING vmid=$id" >> "$LOG"
    BAD=$((BAD+1))
    continue
  fi

  BN="$(basename "$F")"
  SIZE="$(stat -c %s "$F" 2>/dev/null || echo 0)"
  MT="$(date -u -d "@$(stat -c %Y "$F" 2>/dev/null || echo 0)" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)"
  KIND="unknown"
  echo "$BN" | grep -q "^vzdump-qemu-" && KIND="qemu"
  echo "$BN" | grep -q "^vzdump-lxc-" && KIND="lxc"
  SHA="$(sha256sum "$F" | cut -d" " -f1)"
  printf "%s  %s\n" "$SHA" "$BN" >> "$CHKS"

  MODE="direct"
  PARTS=0
  if [ "$SIZE" -gt "$CHUNK_SIZE_BYTES" ]; then
    MODE="chunked"
    CHUNKED=$((CHUNKED+1))
    CDIR="$CHUNK_ROOT/$JOB_ID/$BN.chunks"
    rm -rf "$CDIR"
    mkdir -p "$CDIR"
    echo "SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR" >> "$LOG"
    split -b "$CHUNK_SIZE_SPLIT" -d -a 4 "$F" "$CDIR/part-" >> "$LOG" 2>&1
    PARTS="$(find "$CDIR" -maxdepth 1 -type f -name 'part-*' | wc -l)"
    (cd "$CDIR" && sha256sum part-* > chunks.sha256)
    if [ "$PARTS" -le 0 ]; then
      echo "BAD split produced no chunks file=$BN" >> "$LOG"
      BAD=$((BAD+1))
    fi
  fi

  printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" "$id" "$KIND" "$SIZE" "$SHA" "$MT" "$BN" "$F" "$MODE" "$PARTS" >> "$FILES_TSV"
  COUNT=$((COUNT+1))
  TOTAL=$((TOTAL+SIZE))
done

python3 - "$JOB_ID" "$NODE" "$TS" "$VMIDS" "$COUNT" "$TOTAL" "$CHUNKED" "$CHUNK_SIZE_BYTES" "$FILES_TSV" > "$MAN" <<'PY'
import json, sys, pathlib
job,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:]
items=[]
for line in pathlib.Path(tsv).read_text().splitlines():
    p=line.split("\t")
    if len(p)>=9:
        items.append({
            "vmid":p[0],
            "kind":p[1],
            "size_bytes":int(p[2]),
            "sha256":p[3],
            "mtime_utc":p[4],
            "filename":p[5],
            "source_path":p[6],
            "mode":p[7],
            "parts":int(p[8]),
        })
print(json.dumps({
    "job_id":job,
    "node":node,
    "generated_utc":ts,
    "vmids":vmids.split(),
    "count":int(count),
    "total_bytes":int(total),
    "chunked_count":int(chunked),
    "chunk_size_bytes":int(chunk_size),
    "items":items,
}, indent=2))
PY

DEST_A="${REMOTE_A%/}/${NODE}/vm-local-dumps/${JOB_ID}"
DEST_B="${REMOTE_B%/}/${NODE}/vm-local-dumps/${JOB_ID}"

upload_one_dest() {
  DEST="$1"
  rclone mkdir "$DEST/files" >> "$LOG" 2>&1 || BAD=$((BAD+1))
  while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do
    if [ "$mode" = "chunked" ]; then
      CDIR="$CHUNK_ROOT/$JOB_ID/$bn.chunks"
      echo "UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts" >> "$LOG"
      rclone mkdir "$DEST/files/$bn.chunks" >> "$LOG" 2>&1 || BAD=$((BAD+1))
      for part in "$CDIR"/part-*; do
        [ -f "$part" ] || continue
        pn="$(basename "$part")"
        rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
      done
      rclone copyto "$CDIR/chunks.sha256" "$DEST/files/$bn.chunks/chunks.sha256" >> "$LOG" 2>&1 || BAD=$((BAD+1))
    else
      echo "UPLOAD_DIRECT dest=$DEST file=$bn size=$size" >> "$LOG"
      rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
    fi
  done < "$FILES_TSV"

  rclone copyto "$CHKS" "$DEST/checksums.txt" >> "$LOG" 2>&1 || BAD=$((BAD+1))
  rclone copyto "$FILES_TSV" "$DEST/files.tsv" >> "$LOG" 2>&1 || BAD=$((BAD+1))
  rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
}

verify_one_dest() {
  LABEL="$1"
  DEST="$2"
  VBASE="$VERIFY_ROOT/$JOB_ID/$LABEL"
  rm -rf "$VBASE"
  mkdir -p "$VBASE"
  rclone copyto "$DEST/manifest.json" "$VBASE/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
  test -s "$VBASE/manifest.json" || BAD=$((BAD+1))

  while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do
    if [ "$mode" = "chunked" ]; then
      VDIR="$VBASE/$bn.chunks"
      mkdir -p "$VDIR"
      echo "VERIFY_CHUNKED label=$LABEL file=$bn parts=$parts" >> "$LOG"
      rclone copy "$DEST/files/$bn.chunks" "$VDIR" --filter "+ part-*" --filter "+ chunks.sha256" --filter "- *" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
      (cd "$VDIR" && sha256sum -c chunks.sha256) >> "$LOG" 2>&1 || BAD=$((BAD+1))
      CALC="$(find "$VDIR" -maxdepth 1 -type f -name 'part-*' | sort | xargs cat | sha256sum | cut -d" " -f1)"
      if [ "$CALC" = "$sha" ]; then
        echo "OK combined_sha label=$LABEL file=$bn sha=$sha" >> "$LOG"
      else
        echo "BAD combined_sha label=$LABEL file=$bn expected=$sha got=$CALC" >> "$LOG"
        BAD=$((BAD+1))
      fi
      rm -rf "$VDIR"
    else
      echo "VERIFY_DIRECT label=$LABEL file=$bn" >> "$LOG"
      rclone copyto "$DEST/files/$bn" "$VBASE/$bn" --transfers 1 >> "$LOG" 2>&1 || BAD=$((BAD+1))
      printf "%s  %s\n" "$sha" "$VBASE/$bn" | sha256sum -c - >> "$LOG" 2>&1 || BAD=$((BAD+1))
      rm -f "$VBASE/$bn"
    fi
  done < "$FILES_TSV"
  rm -rf "$VBASE"
}

if [ "$BAD" -eq 0 ]; then
  upload_one_dest "$DEST_A"
  upload_one_dest "$DEST_B"
fi

if [ "$BAD" -eq 0 ]; then
  verify_one_dest "A" "$DEST_A"
  verify_one_dest "B" "$DEST_B"
fi

rm -rf "$VERIFY_ROOT/$JOB_ID"
rm -rf "$CHUNK_ROOT/$JOB_ID"

if [ "$BAD" -eq 0 ]; then STATUS="OK"; else STATUS="WARN"; fi

jq -n \
  --arg job_id "$JOB_ID" \
  --arg status "$STATUS" \
  --arg node "$NODE" \
  --arg remote_a "$DEST_A" \
  --arg remote_b "$DEST_B" \
  --argjson count "$COUNT" \
  --argjson total_bytes "$TOTAL" \
  --argjson chunked_count "$CHUNKED" \
  --argjson chunk_size_bytes "$CHUNK_SIZE_BYTES" \
  --arg log "$LOG" \
  "{job_id:\$job_id,status:\$status,node:\$node,remote_a:\$remote_a,remote_b:\$remote_b,count:\$count,total_bytes:\$total_bytes,chunked_count:\$chunked_count,chunk_size_bytes:\$chunk_size_bytes,log:\$log,local_dumps_deleted:false}" \
  > "$ROOT/latest-vm-local-dumps-cloud.json"

echo "STATUS=$STATUS TS=$TS TYPE=vm-local-dumps-cloud BAD=$BAD COUNT=$COUNT CHUNKED=$CHUNKED TOTAL_BYTES=$TOTAL JOB=$JOB_ID REMOTE_A=$DEST_A REMOTE_B=$DEST_B LOG=$LOG" > "$OUT"
cat "$OUT"
=== SERVICE ===
# /etc/systemd/system/homelab-vm-local-dumps-2cloud.service
[Unit]
Description=Homelab VM/CT existing local dumps two-cloud verifier
After=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-vm-local-dumps-2cloud

# /etc/systemd/system/homelab-vm-local-dumps-2cloud.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; argv[]=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
ActiveState=inactive
SubState=dead
=== TIMER ===
# /etc/systemd/system/homelab-vm-local-dumps-2cloud.timer
[Unit]
Description=Homelab VM/CT existing local dumps two-cloud verifier timer

[Timer]
OnCalendar=Sun *-*-* 10:00:00
RandomizedDelaySec=30m
Persistent=true

[Install]
WantedBy=timers.target
NextElapseUSecRealtime=Sun 2026-07-26 10:29:15 MSK
ActiveState=active
UnitFileState=enabled

OUTPUT_END
CHAT_OUTPUT_END