CHAT_OUTPUT_BEGIN
COMMAND_ID=HOMELAB-VM-2CLOUD-FILESET-TRACE-439
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=cluster-audit
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
COMMAND_SHA256=83fd063427bb5f7d0ab471a7f689bc234ac30842988be5c364a0bf8ce043f395
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGES_MADE=false
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=fbfb933d600dfcd687148901187dd377972d1044a5d915a6f8f9bc1e06e64ad1
SANITIZED_OUTPUT_SHA256=fbfb933d600dfcd687148901187dd377972d1044a5d915a6f8f9bc1e06e64ad1
OUTPUT_BEGIN
DETAIL=SCRIPT_SHA256=0ded6ba72e49b9cf7522c7f3d8840cc1026d2a173d4909dd3b4cfc5ff8831115
DETAIL=FILESET_SOURCE_BEGIN
DETAIL=SOURCE_LINE=0026:WORK="$ROOT/$JOB_ID"
DETAIL=SOURCE_LINE=0027:mkdir -p "$WORK"
DETAIL=SOURCE_LINE=0028:
DETAIL=SOURCE_LINE=0029:FILES_TSV="$WORK/files.tsv"
DETAIL=SOURCE_LINE=0030:CHKS="$WORK/checksums.txt"
DETAIL=SOURCE_LINE=0031:MAN="$WORK/manifest.json"
DETAIL=SOURCE_LINE=0032:
DETAIL=SOURCE_LINE=0033:: > "$FILES_TSV"
DETAIL=SOURCE_LINE=0034:: > "$CHKS"
DETAIL=SOURCE_LINE=0035:: > "$LOG"
DETAIL=SOURCE_LINE=0036:
DETAIL=SOURCE_LINE=0037:echo "job_id=$JOB_ID" >> "$LOG"
DETAIL=SOURCE_LINE=0038:echo "vmids=$VMIDS" >> "$LOG"
DETAIL=SOURCE_LINE=0039:echo "chunk_size_bytes=$CHUNK_SIZE_BYTES" >> "$LOG"
DETAIL=SOURCE_LINE=0040:
DETAIL=SOURCE_LINE=0041:for id in $VMIDS; do
DETAIL=SOURCE_LINE=0042:  F="$(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-${id}-*.vma.zst" -o -name "vzdump-lxc-${id}-*.tar.zst" \) -printf "%T@ %p\n" 2>/dev/null | sort -n | tail -n 1 | cut -d" " -f2-)"
DETAIL=SOURCE_LINE=0043:  if [ -z "$F" ] || [ ! -f "$F" ]; then
DETAIL=SOURCE_LINE=0044:    echo "MISSING vmid=$id" >> "$LOG"
DETAIL=SOURCE_LINE=0045:    BAD=$((BAD+1))
DETAIL=SOURCE_LINE=0046:    continue
DETAIL=SOURCE_LINE=0047:  fi
DETAIL=SOURCE_LINE=0048:
DETAIL=SOURCE_LINE=0049:  BN="$(basename "$F")"
DETAIL=SOURCE_LINE=0050:  SIZE="$(stat -c %s "$F" 2>/dev/null || echo 0)"
DETAIL=SOURCE_LINE=0051:  MT="$(date -u -d "@$(stat -c %Y "$F" 2>/dev/null || echo 0)" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)"
DETAIL=SOURCE_LINE=0052:  KIND="unknown"
DETAIL=SOURCE_LINE=0053:  echo "$BN" | grep -q "^vzdump-qemu-" && KIND="qemu"
DETAIL=SOURCE_LINE=0054:  echo "$BN" | grep -q "^vzdump-lxc-" && KIND="lxc"
DETAIL=SOURCE_LINE=0055:  SHA="$(sha256sum "$F" | cut -d" " -f1)"
DETAIL=SOURCE_LINE=0056:  printf "%s  %s\n" "$SHA" "$BN" >> "$CHKS"
DETAIL=SOURCE_LINE=0057:
DETAIL=SOURCE_LINE=0058:  MODE="direct"
DETAIL=SOURCE_LINE=0059:  PARTS=0
DETAIL=SOURCE_LINE=...
DETAIL=SOURCE_LINE=0065:    mkdir -p "$CDIR"
DETAIL=SOURCE_LINE=0066:    echo "SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR" >> "$LOG"
DETAIL=SOURCE_LINE=0067:    split -b "$CHUNK_SIZE_SPLIT" -d -a 4 "$F" "$CDIR/part-" >> "$LOG" 2>&1
DETAIL=SOURCE_LINE=0068:    PARTS="$(find "$CDIR" -maxdepth 1 -type f -name 'part-*' | wc -l)"
DETAIL=SOURCE_LINE=0069:    (cd "$CDIR" && sha256sum part-* > chunks.sha256)
DETAIL=SOURCE_LINE=0070:    if [ "$PARTS" -le 0 ]; then
DETAIL=SOURCE_LINE=0071:      echo "BAD split produced no chunks file=$BN" >> "$LOG"
DETAIL=SOURCE_LINE=0072:      BAD=$((BAD+1))
DETAIL=SOURCE_LINE=0073:    fi
DETAIL=SOURCE_LINE=0074:  fi
DETAIL=SOURCE_LINE=0075:
DETAIL=SOURCE_LINE=0076:  printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" "$id" "$KIND" "$SIZE" "$SHA" "$MT" "$BN" "$F" "$MODE" "$PARTS" >> "$FILES_TSV"
DETAIL=SOURCE_LINE=0077:  COUNT=$((COUNT+1))
DETAIL=SOURCE_LINE=0078:  TOTAL=$((TOTAL+SIZE))
DETAIL=SOURCE_LINE=0079:done
DETAIL=SOURCE_LINE=0080:
DETAIL=SOURCE_LINE=0081:python3 - "$JOB_ID" "$NODE" "$TS" "$VMIDS" "$COUNT" "$TOTAL" "$CHUNKED" "$CHUNK_SIZE_BYTES" "$FILES_TSV" > "$MAN" <<'PY'
DETAIL=SOURCE_LINE=0082:import json, sys, pathlib
DETAIL=SOURCE_LINE=0083:job,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:]
DETAIL=SOURCE_LINE=0084:items=[]
DETAIL=SOURCE_LINE=0085:for line in pathlib.Path(tsv).read_text().splitlines():
DETAIL=SOURCE_LINE=0086:    p=line.split("\t")
DETAIL=SOURCE_LINE=0087:    if len(p)>=9:
DETAIL=SOURCE_LINE=0088:        items.append({
DETAIL=SOURCE_LINE=0089:            "vmid":p[0],
DETAIL=SOURCE_LINE=...
DETAIL=SOURCE_LINE=0120:      CDIR="$CHUNK_ROOT/$JOB_ID/$bn.chunks"
DETAIL=SOURCE_LINE=0121:      echo "UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts" >> "$LOG"
DETAIL=SOURCE_LINE=0122:      rclone mkdir "$DEST/files/$bn.chunks" >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE_LINE=0123:      for part in "$CDIR"/part-*; do
DETAIL=SOURCE_LINE=0124:        [ -f "$part" ] || continue
DETAIL=SOURCE_LINE=0125:        pn="$(basename "$part")"
DETAIL=SOURCE_LINE=0126:        rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE_LINE=0127:      done
DETAIL=SOURCE_LINE=...
DETAIL=SOURCE_LINE=0130:      echo "UPLOAD_DIRECT dest=$DEST file=$bn size=$size" >> "$LOG"
DETAIL=SOURCE_LINE=0131:      rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE_LINE=0132:    fi
DETAIL=SOURCE_LINE=0133:  done < "$FILES_TSV"
DETAIL=SOURCE_LINE=0134:
DETAIL=SOURCE_LINE=0135:  rclone copyto "$CHKS" "$DEST/checksums.txt" >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE_LINE=0136:  rclone copyto "$FILES_TSV" "$DEST/files.tsv" >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE_LINE=0137:  rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
DETAIL=SOURCE_LINE=0138:}
DETAIL=SOURCE_LINE=0139:
DETAIL=SOURCE_LINE=0140:verify_one_dest() {
DETAIL=FILESET_SOURCE_END
DETAIL=ARCHIVES_CHANGED=NO
DETAIL=SERVICE_STARTED=NO
DETAIL=SECRET_VALUES_OUTPUT=NONE
DETAIL=FILESET_TRACE=PASS
CHANGES_MADE=NO

OUTPUT_END
CHAT_OUTPUT_END