homelab-runtime-publisher
114 lines
5.8 KiB
Plaintext
114 lines
5.8 KiB
Plaintext
CHAT_OUTPUT_BEGIN
|
|
COMMAND_ID=HOMELAB-VM-2CLOUD-FILESET-TRACE-439
|
|
STATUS=OK
|
|
RC=0
|
|
HOST=pve01
|
|
MODE=read-only
|
|
COMPONENT=cluster-audit
|
|
REFERENCE_REGISTER_CHECK=OK
|
|
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
|
|
ERROR_REGISTER_CHECK=OK
|
|
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
|
|
COMMAND_SHA256=83fd063427bb5f7d0ab471a7f689bc234ac30842988be5c364a0bf8ce043f395
|
|
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
|
EXECUTION_STARTED=true
|
|
CHANGES_MADE=false
|
|
SANITIZED=yes
|
|
SECRETS_INCLUDED=no
|
|
PRIVATE_ADDRESSES_INCLUDED=no
|
|
RAW_EVIDENCE_SHA256=fbfb933d600dfcd687148901187dd377972d1044a5d915a6f8f9bc1e06e64ad1
|
|
SANITIZED_OUTPUT_SHA256=fbfb933d600dfcd687148901187dd377972d1044a5d915a6f8f9bc1e06e64ad1
|
|
OUTPUT_BEGIN
|
|
DETAIL=SCRIPT_SHA256=0ded6ba72e49b9cf7522c7f3d8840cc1026d2a173d4909dd3b4cfc5ff8831115
|
|
DETAIL=FILESET_SOURCE_BEGIN
|
|
DETAIL=SOURCE_LINE=0026:WORK="$ROOT/$JOB_ID"
|
|
DETAIL=SOURCE_LINE=0027:mkdir -p "$WORK"
|
|
DETAIL=SOURCE_LINE=0028:
|
|
DETAIL=SOURCE_LINE=0029:FILES_TSV="$WORK/files.tsv"
|
|
DETAIL=SOURCE_LINE=0030:CHKS="$WORK/checksums.txt"
|
|
DETAIL=SOURCE_LINE=0031:MAN="$WORK/manifest.json"
|
|
DETAIL=SOURCE_LINE=0032:
|
|
DETAIL=SOURCE_LINE=0033:: > "$FILES_TSV"
|
|
DETAIL=SOURCE_LINE=0034:: > "$CHKS"
|
|
DETAIL=SOURCE_LINE=0035:: > "$LOG"
|
|
DETAIL=SOURCE_LINE=0036:
|
|
DETAIL=SOURCE_LINE=0037:echo "job_id=$JOB_ID" >> "$LOG"
|
|
DETAIL=SOURCE_LINE=0038:echo "vmids=$VMIDS" >> "$LOG"
|
|
DETAIL=SOURCE_LINE=0039:echo "chunk_size_bytes=$CHUNK_SIZE_BYTES" >> "$LOG"
|
|
DETAIL=SOURCE_LINE=0040:
|
|
DETAIL=SOURCE_LINE=0041:for id in $VMIDS; do
|
|
DETAIL=SOURCE_LINE=0042: F="$(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-${id}-*.vma.zst" -o -name "vzdump-lxc-${id}-*.tar.zst" \) -printf "%T@ %p\n" 2>/dev/null | sort -n | tail -n 1 | cut -d" " -f2-)"
|
|
DETAIL=SOURCE_LINE=0043: if [ -z "$F" ] || [ ! -f "$F" ]; then
|
|
DETAIL=SOURCE_LINE=0044: echo "MISSING vmid=$id" >> "$LOG"
|
|
DETAIL=SOURCE_LINE=0045: BAD=$((BAD+1))
|
|
DETAIL=SOURCE_LINE=0046: continue
|
|
DETAIL=SOURCE_LINE=0047: fi
|
|
DETAIL=SOURCE_LINE=0048:
|
|
DETAIL=SOURCE_LINE=0049: BN="$(basename "$F")"
|
|
DETAIL=SOURCE_LINE=0050: SIZE="$(stat -c %s "$F" 2>/dev/null || echo 0)"
|
|
DETAIL=SOURCE_LINE=0051: MT="$(date -u -d "@$(stat -c %Y "$F" 2>/dev/null || echo 0)" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)"
|
|
DETAIL=SOURCE_LINE=0052: KIND="unknown"
|
|
DETAIL=SOURCE_LINE=0053: echo "$BN" | grep -q "^vzdump-qemu-" && KIND="qemu"
|
|
DETAIL=SOURCE_LINE=0054: echo "$BN" | grep -q "^vzdump-lxc-" && KIND="lxc"
|
|
DETAIL=SOURCE_LINE=0055: SHA="$(sha256sum "$F" | cut -d" " -f1)"
|
|
DETAIL=SOURCE_LINE=0056: printf "%s %s\n" "$SHA" "$BN" >> "$CHKS"
|
|
DETAIL=SOURCE_LINE=0057:
|
|
DETAIL=SOURCE_LINE=0058: MODE="direct"
|
|
DETAIL=SOURCE_LINE=0059: PARTS=0
|
|
DETAIL=SOURCE_LINE=...
|
|
DETAIL=SOURCE_LINE=0065: mkdir -p "$CDIR"
|
|
DETAIL=SOURCE_LINE=0066: echo "SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR" >> "$LOG"
|
|
DETAIL=SOURCE_LINE=0067: split -b "$CHUNK_SIZE_SPLIT" -d -a 4 "$F" "$CDIR/part-" >> "$LOG" 2>&1
|
|
DETAIL=SOURCE_LINE=0068: PARTS="$(find "$CDIR" -maxdepth 1 -type f -name 'part-*' | wc -l)"
|
|
DETAIL=SOURCE_LINE=0069: (cd "$CDIR" && sha256sum part-* > chunks.sha256)
|
|
DETAIL=SOURCE_LINE=0070: if [ "$PARTS" -le 0 ]; then
|
|
DETAIL=SOURCE_LINE=0071: echo "BAD split produced no chunks file=$BN" >> "$LOG"
|
|
DETAIL=SOURCE_LINE=0072: BAD=$((BAD+1))
|
|
DETAIL=SOURCE_LINE=0073: fi
|
|
DETAIL=SOURCE_LINE=0074: fi
|
|
DETAIL=SOURCE_LINE=0075:
|
|
DETAIL=SOURCE_LINE=0076: printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" "$id" "$KIND" "$SIZE" "$SHA" "$MT" "$BN" "$F" "$MODE" "$PARTS" >> "$FILES_TSV"
|
|
DETAIL=SOURCE_LINE=0077: COUNT=$((COUNT+1))
|
|
DETAIL=SOURCE_LINE=0078: TOTAL=$((TOTAL+SIZE))
|
|
DETAIL=SOURCE_LINE=0079:done
|
|
DETAIL=SOURCE_LINE=0080:
|
|
DETAIL=SOURCE_LINE=0081:python3 - "$JOB_ID" "$NODE" "$TS" "$VMIDS" "$COUNT" "$TOTAL" "$CHUNKED" "$CHUNK_SIZE_BYTES" "$FILES_TSV" > "$MAN" <<'PY'
|
|
DETAIL=SOURCE_LINE=0082:import json, sys, pathlib
|
|
DETAIL=SOURCE_LINE=0083:job,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:]
|
|
DETAIL=SOURCE_LINE=0084:items=[]
|
|
DETAIL=SOURCE_LINE=0085:for line in pathlib.Path(tsv).read_text().splitlines():
|
|
DETAIL=SOURCE_LINE=0086: p=line.split("\t")
|
|
DETAIL=SOURCE_LINE=0087: if len(p)>=9:
|
|
DETAIL=SOURCE_LINE=0088: items.append({
|
|
DETAIL=SOURCE_LINE=0089: "vmid":p[0],
|
|
DETAIL=SOURCE_LINE=...
|
|
DETAIL=SOURCE_LINE=0120: CDIR="$CHUNK_ROOT/$JOB_ID/$bn.chunks"
|
|
DETAIL=SOURCE_LINE=0121: echo "UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts" >> "$LOG"
|
|
DETAIL=SOURCE_LINE=0122: rclone mkdir "$DEST/files/$bn.chunks" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
DETAIL=SOURCE_LINE=0123: for part in "$CDIR"/part-*; do
|
|
DETAIL=SOURCE_LINE=0124: [ -f "$part" ] || continue
|
|
DETAIL=SOURCE_LINE=0125: pn="$(basename "$part")"
|
|
DETAIL=SOURCE_LINE=0126: rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
DETAIL=SOURCE_LINE=0127: done
|
|
DETAIL=SOURCE_LINE=...
|
|
DETAIL=SOURCE_LINE=0130: echo "UPLOAD_DIRECT dest=$DEST file=$bn size=$size" >> "$LOG"
|
|
DETAIL=SOURCE_LINE=0131: rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
DETAIL=SOURCE_LINE=0132: fi
|
|
DETAIL=SOURCE_LINE=0133: done < "$FILES_TSV"
|
|
DETAIL=SOURCE_LINE=0134:
|
|
DETAIL=SOURCE_LINE=0135: rclone copyto "$CHKS" "$DEST/checksums.txt" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
DETAIL=SOURCE_LINE=0136: rclone copyto "$FILES_TSV" "$DEST/files.tsv" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
DETAIL=SOURCE_LINE=0137: rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
DETAIL=SOURCE_LINE=0138:}
|
|
DETAIL=SOURCE_LINE=0139:
|
|
DETAIL=SOURCE_LINE=0140:verify_one_dest() {
|
|
DETAIL=FILESET_SOURCE_END
|
|
DETAIL=ARCHIVES_CHANGED=NO
|
|
DETAIL=SERVICE_STARTED=NO
|
|
DETAIL=SECRET_VALUES_OUTPUT=NONE
|
|
DETAIL=FILESET_TRACE=PASS
|
|
CHANGES_MADE=NO
|
|
|
|
OUTPUT_END
|
|
CHAT_OUTPUT_END
|