runtime: publish HOMELAB-VM-2CLOUD-FILESET-TRACE-439
This commit is contained in:
homelab-runtime-publisher
File diff suppressed because one or more lines are too long
@@ -0,0 +1,113 @@
|
|||||||
|
CHAT_OUTPUT_BEGIN
|
||||||
|
COMMAND_ID=HOMELAB-VM-2CLOUD-FILESET-TRACE-439
|
||||||
|
STATUS=OK
|
||||||
|
RC=0
|
||||||
|
HOST=pve01
|
||||||
|
MODE=read-only
|
||||||
|
COMPONENT=cluster-audit
|
||||||
|
REFERENCE_REGISTER_CHECK=OK
|
||||||
|
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
|
||||||
|
ERROR_REGISTER_CHECK=OK
|
||||||
|
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
|
||||||
|
COMMAND_SHA256=83fd063427bb5f7d0ab471a7f689bc234ac30842988be5c364a0bf8ce043f395
|
||||||
|
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||||
|
EXECUTION_STARTED=true
|
||||||
|
CHANGES_MADE=false
|
||||||
|
SANITIZED=yes
|
||||||
|
SECRETS_INCLUDED=no
|
||||||
|
PRIVATE_ADDRESSES_INCLUDED=no
|
||||||
|
RAW_EVIDENCE_SHA256=fbfb933d600dfcd687148901187dd377972d1044a5d915a6f8f9bc1e06e64ad1
|
||||||
|
SANITIZED_OUTPUT_SHA256=fbfb933d600dfcd687148901187dd377972d1044a5d915a6f8f9bc1e06e64ad1
|
||||||
|
OUTPUT_BEGIN
|
||||||
|
DETAIL=SCRIPT_SHA256=0ded6ba72e49b9cf7522c7f3d8840cc1026d2a173d4909dd3b4cfc5ff8831115
|
||||||
|
DETAIL=FILESET_SOURCE_BEGIN
|
||||||
|
DETAIL=SOURCE_LINE=0026:WORK="$ROOT/$JOB_ID"
|
||||||
|
DETAIL=SOURCE_LINE=0027:mkdir -p "$WORK"
|
||||||
|
DETAIL=SOURCE_LINE=0028:
|
||||||
|
DETAIL=SOURCE_LINE=0029:FILES_TSV="$WORK/files.tsv"
|
||||||
|
DETAIL=SOURCE_LINE=0030:CHKS="$WORK/checksums.txt"
|
||||||
|
DETAIL=SOURCE_LINE=0031:MAN="$WORK/manifest.json"
|
||||||
|
DETAIL=SOURCE_LINE=0032:
|
||||||
|
DETAIL=SOURCE_LINE=0033:: > "$FILES_TSV"
|
||||||
|
DETAIL=SOURCE_LINE=0034:: > "$CHKS"
|
||||||
|
DETAIL=SOURCE_LINE=0035:: > "$LOG"
|
||||||
|
DETAIL=SOURCE_LINE=0036:
|
||||||
|
DETAIL=SOURCE_LINE=0037:echo "job_id=$JOB_ID" >> "$LOG"
|
||||||
|
DETAIL=SOURCE_LINE=0038:echo "vmids=$VMIDS" >> "$LOG"
|
||||||
|
DETAIL=SOURCE_LINE=0039:echo "chunk_size_bytes=$CHUNK_SIZE_BYTES" >> "$LOG"
|
||||||
|
DETAIL=SOURCE_LINE=0040:
|
||||||
|
DETAIL=SOURCE_LINE=0041:for id in $VMIDS; do
|
||||||
|
DETAIL=SOURCE_LINE=0042: F="$(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-${id}-*.vma.zst" -o -name "vzdump-lxc-${id}-*.tar.zst" \) -printf "%T@ %p\n" 2>/dev/null | sort -n | tail -n 1 | cut -d" " -f2-)"
|
||||||
|
DETAIL=SOURCE_LINE=0043: if [ -z "$F" ] || [ ! -f "$F" ]; then
|
||||||
|
DETAIL=SOURCE_LINE=0044: echo "MISSING vmid=$id" >> "$LOG"
|
||||||
|
DETAIL=SOURCE_LINE=0045: BAD=$((BAD+1))
|
||||||
|
DETAIL=SOURCE_LINE=0046: continue
|
||||||
|
DETAIL=SOURCE_LINE=0047: fi
|
||||||
|
DETAIL=SOURCE_LINE=0048:
|
||||||
|
DETAIL=SOURCE_LINE=0049: BN="$(basename "$F")"
|
||||||
|
DETAIL=SOURCE_LINE=0050: SIZE="$(stat -c %s "$F" 2>/dev/null || echo 0)"
|
||||||
|
DETAIL=SOURCE_LINE=0051: MT="$(date -u -d "@$(stat -c %Y "$F" 2>/dev/null || echo 0)" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)"
|
||||||
|
DETAIL=SOURCE_LINE=0052: KIND="unknown"
|
||||||
|
DETAIL=SOURCE_LINE=0053: echo "$BN" | grep -q "^vzdump-qemu-" && KIND="qemu"
|
||||||
|
DETAIL=SOURCE_LINE=0054: echo "$BN" | grep -q "^vzdump-lxc-" && KIND="lxc"
|
||||||
|
DETAIL=SOURCE_LINE=0055: SHA="$(sha256sum "$F" | cut -d" " -f1)"
|
||||||
|
DETAIL=SOURCE_LINE=0056: printf "%s %s\n" "$SHA" "$BN" >> "$CHKS"
|
||||||
|
DETAIL=SOURCE_LINE=0057:
|
||||||
|
DETAIL=SOURCE_LINE=0058: MODE="direct"
|
||||||
|
DETAIL=SOURCE_LINE=0059: PARTS=0
|
||||||
|
DETAIL=SOURCE_LINE=...
|
||||||
|
DETAIL=SOURCE_LINE=0065: mkdir -p "$CDIR"
|
||||||
|
DETAIL=SOURCE_LINE=0066: echo "SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR" >> "$LOG"
|
||||||
|
DETAIL=SOURCE_LINE=0067: split -b "$CHUNK_SIZE_SPLIT" -d -a 4 "$F" "$CDIR/part-" >> "$LOG" 2>&1
|
||||||
|
DETAIL=SOURCE_LINE=0068: PARTS="$(find "$CDIR" -maxdepth 1 -type f -name 'part-*' | wc -l)"
|
||||||
|
DETAIL=SOURCE_LINE=0069: (cd "$CDIR" && sha256sum part-* > chunks.sha256)
|
||||||
|
DETAIL=SOURCE_LINE=0070: if [ "$PARTS" -le 0 ]; then
|
||||||
|
DETAIL=SOURCE_LINE=0071: echo "BAD split produced no chunks file=$BN" >> "$LOG"
|
||||||
|
DETAIL=SOURCE_LINE=0072: BAD=$((BAD+1))
|
||||||
|
DETAIL=SOURCE_LINE=0073: fi
|
||||||
|
DETAIL=SOURCE_LINE=0074: fi
|
||||||
|
DETAIL=SOURCE_LINE=0075:
|
||||||
|
DETAIL=SOURCE_LINE=0076: printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" "$id" "$KIND" "$SIZE" "$SHA" "$MT" "$BN" "$F" "$MODE" "$PARTS" >> "$FILES_TSV"
|
||||||
|
DETAIL=SOURCE_LINE=0077: COUNT=$((COUNT+1))
|
||||||
|
DETAIL=SOURCE_LINE=0078: TOTAL=$((TOTAL+SIZE))
|
||||||
|
DETAIL=SOURCE_LINE=0079:done
|
||||||
|
DETAIL=SOURCE_LINE=0080:
|
||||||
|
DETAIL=SOURCE_LINE=0081:python3 - "$JOB_ID" "$NODE" "$TS" "$VMIDS" "$COUNT" "$TOTAL" "$CHUNKED" "$CHUNK_SIZE_BYTES" "$FILES_TSV" > "$MAN" <<'PY'
|
||||||
|
DETAIL=SOURCE_LINE=0082:import json, sys, pathlib
|
||||||
|
DETAIL=SOURCE_LINE=0083:job,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:]
|
||||||
|
DETAIL=SOURCE_LINE=0084:items=[]
|
||||||
|
DETAIL=SOURCE_LINE=0085:for line in pathlib.Path(tsv).read_text().splitlines():
|
||||||
|
DETAIL=SOURCE_LINE=0086: p=line.split("\t")
|
||||||
|
DETAIL=SOURCE_LINE=0087: if len(p)>=9:
|
||||||
|
DETAIL=SOURCE_LINE=0088: items.append({
|
||||||
|
DETAIL=SOURCE_LINE=0089: "vmid":p[0],
|
||||||
|
DETAIL=SOURCE_LINE=...
|
||||||
|
DETAIL=SOURCE_LINE=0120: CDIR="$CHUNK_ROOT/$JOB_ID/$bn.chunks"
|
||||||
|
DETAIL=SOURCE_LINE=0121: echo "UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts" >> "$LOG"
|
||||||
|
DETAIL=SOURCE_LINE=0122: rclone mkdir "$DEST/files/$bn.chunks" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||||
|
DETAIL=SOURCE_LINE=0123: for part in "$CDIR"/part-*; do
|
||||||
|
DETAIL=SOURCE_LINE=0124: [ -f "$part" ] || continue
|
||||||
|
DETAIL=SOURCE_LINE=0125: pn="$(basename "$part")"
|
||||||
|
DETAIL=SOURCE_LINE=0126: rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||||
|
DETAIL=SOURCE_LINE=0127: done
|
||||||
|
DETAIL=SOURCE_LINE=...
|
||||||
|
DETAIL=SOURCE_LINE=0130: echo "UPLOAD_DIRECT dest=$DEST file=$bn size=$size" >> "$LOG"
|
||||||
|
DETAIL=SOURCE_LINE=0131: rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||||
|
DETAIL=SOURCE_LINE=0132: fi
|
||||||
|
DETAIL=SOURCE_LINE=0133: done < "$FILES_TSV"
|
||||||
|
DETAIL=SOURCE_LINE=0134:
|
||||||
|
DETAIL=SOURCE_LINE=0135: rclone copyto "$CHKS" "$DEST/checksums.txt" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||||
|
DETAIL=SOURCE_LINE=0136: rclone copyto "$FILES_TSV" "$DEST/files.tsv" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||||
|
DETAIL=SOURCE_LINE=0137: rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||||
|
DETAIL=SOURCE_LINE=0138:}
|
||||||
|
DETAIL=SOURCE_LINE=0139:
|
||||||
|
DETAIL=SOURCE_LINE=0140:verify_one_dest() {
|
||||||
|
DETAIL=FILESET_SOURCE_END
|
||||||
|
DETAIL=ARCHIVES_CHANGED=NO
|
||||||
|
DETAIL=SERVICE_STARTED=NO
|
||||||
|
DETAIL=SECRET_VALUES_OUTPUT=NONE
|
||||||
|
DETAIL=FILESET_TRACE=PASS
|
||||||
|
CHANGES_MADE=NO
|
||||||
|
|
||||||
|
OUTPUT_END
|
||||||
|
CHAT_OUTPUT_END
|
||||||
+7
-7
File diff suppressed because one or more lines are too long
+10
-211
@@ -1,5 +1,5 @@
|
|||||||
CHAT_OUTPUT_BEGIN
|
CHAT_OUTPUT_BEGIN
|
||||||
COMMAND_ID=HOMELAB-VM-2CLOUD-EXACT-SELECTION-438
|
COMMAND_ID=HOMELAB-VM-2CLOUD-FILESET-TRACE-439
|
||||||
STATUS=OK
|
STATUS=OK
|
||||||
RC=0
|
RC=0
|
||||||
HOST=pve01
|
HOST=pve01
|
||||||
@@ -9,35 +9,18 @@ REFERENCE_REGISTER_CHECK=OK
|
|||||||
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
|
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
|
||||||
ERROR_REGISTER_CHECK=OK
|
ERROR_REGISTER_CHECK=OK
|
||||||
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
|
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
|
||||||
COMMAND_SHA256=458a61bfe476e4aeab9fdefc2057f75cf06c246702c653458e91f8c84eb823f2
|
COMMAND_SHA256=83fd063427bb5f7d0ab471a7f689bc234ac30842988be5c364a0bf8ce043f395
|
||||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||||
EXECUTION_STARTED=true
|
EXECUTION_STARTED=true
|
||||||
CHANGES_MADE=false
|
CHANGES_MADE=false
|
||||||
SANITIZED=yes
|
SANITIZED=yes
|
||||||
SECRETS_INCLUDED=no
|
SECRETS_INCLUDED=no
|
||||||
PRIVATE_ADDRESSES_INCLUDED=no
|
PRIVATE_ADDRESSES_INCLUDED=no
|
||||||
RAW_EVIDENCE_SHA256=1a2e8ce33d51e8861acf1434ee15eee384f0fcb6316b36d3b3fd87c946570531
|
RAW_EVIDENCE_SHA256=fbfb933d600dfcd687148901187dd377972d1044a5d915a6f8f9bc1e06e64ad1
|
||||||
SANITIZED_OUTPUT_SHA256=1a2e8ce33d51e8861acf1434ee15eee384f0fcb6316b36d3b3fd87c946570531
|
SANITIZED_OUTPUT_SHA256=fbfb933d600dfcd687148901187dd377972d1044a5d915a6f8f9bc1e06e64ad1
|
||||||
OUTPUT_BEGIN
|
OUTPUT_BEGIN
|
||||||
<string>:33: FutureWarning: Possible nested set at position 2
|
|
||||||
<string>:33: FutureWarning: Possible nested set at position 29
|
|
||||||
<string>:43: FutureWarning: Possible nested set at position 2
|
|
||||||
<string>:43: FutureWarning: Possible nested set at position 29
|
|
||||||
DETAIL=SCRIPT_SHA256=0ded6ba72e49b9cf7522c7f3d8840cc1026d2a173d4909dd3b4cfc5ff8831115
|
DETAIL=SCRIPT_SHA256=0ded6ba72e49b9cf7522c7f3d8840cc1026d2a173d4909dd3b4cfc5ff8831115
|
||||||
DETAIL=SELECTION_SOURCE_RANGE=14-290
|
DETAIL=FILESET_SOURCE_BEGIN
|
||||||
DETAIL=SELECTION_SOURCE_BEGIN
|
|
||||||
DETAIL=SOURCE_LINE=0014:TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
||||||
DETAIL=SOURCE_LINE=0015:JOB_TS="$(date -u +%Y%m%dT%H%M%SZ)"
|
|
||||||
DETAIL=SOURCE_LINE=0016:NODE="$(hostname -s)"
|
|
||||||
DETAIL=SOURCE_LINE=0017:BAD=0
|
|
||||||
DETAIL=SOURCE_LINE=0018:COUNT=0
|
|
||||||
DETAIL=SOURCE_LINE=0019:TOTAL=0
|
|
||||||
DETAIL=SOURCE_LINE=0020:CHUNKED=0
|
|
||||||
DETAIL=SOURCE_LINE=0021:VMIDS="${VMIDS:-110 112 150}"
|
|
||||||
DETAIL=SOURCE_LINE=0022:
|
|
||||||
DETAIL=SOURCE_LINE=0023:source /etc/homelab-backup.env 2>/dev/null || BAD=1
|
|
||||||
DETAIL=SOURCE_LINE=0024:
|
|
||||||
DETAIL=SOURCE_LINE=0025:JOB_ID="homelab-${NODE}-vm-local-dumps-${JOB_TS}"
|
|
||||||
DETAIL=SOURCE_LINE=0026:WORK="$ROOT/$JOB_ID"
|
DETAIL=SOURCE_LINE=0026:WORK="$ROOT/$JOB_ID"
|
||||||
DETAIL=SOURCE_LINE=0027:mkdir -p "$WORK"
|
DETAIL=SOURCE_LINE=0027:mkdir -p "$WORK"
|
||||||
DETAIL=SOURCE_LINE=0028:
|
DETAIL=SOURCE_LINE=0028:
|
||||||
@@ -72,11 +55,7 @@ DETAIL=SOURCE_LINE=0056: printf "%s %s\n" "$SHA" "$BN" >> "$CHKS"
|
|||||||
DETAIL=SOURCE_LINE=0057:
|
DETAIL=SOURCE_LINE=0057:
|
||||||
DETAIL=SOURCE_LINE=0058: MODE="direct"
|
DETAIL=SOURCE_LINE=0058: MODE="direct"
|
||||||
DETAIL=SOURCE_LINE=0059: PARTS=0
|
DETAIL=SOURCE_LINE=0059: PARTS=0
|
||||||
DETAIL=SOURCE_LINE=0060: if [ "$SIZE" -gt "$CHUNK_SIZE_BYTES" ]; then
|
DETAIL=SOURCE_LINE=...
|
||||||
DETAIL=SOURCE_LINE=0061: MODE="chunked"
|
|
||||||
DETAIL=SOURCE_LINE=0062: CHUNKED=$((CHUNKED+1))
|
|
||||||
DETAIL=SOURCE_LINE=0063: CDIR="$CHUNK_ROOT/$JOB_ID/$BN.chunks"
|
|
||||||
DETAIL=SOURCE_LINE=0064: rm -rf "$CDIR"
|
|
||||||
DETAIL=SOURCE_LINE=0065: mkdir -p "$CDIR"
|
DETAIL=SOURCE_LINE=0065: mkdir -p "$CDIR"
|
||||||
DETAIL=SOURCE_LINE=0066: echo "SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR" >> "$LOG"
|
DETAIL=SOURCE_LINE=0066: echo "SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR" >> "$LOG"
|
||||||
DETAIL=SOURCE_LINE=0067: split -b "$CHUNK_SIZE_SPLIT" -d -a 4 "$F" "$CDIR/part-" >> "$LOG" 2>&1
|
DETAIL=SOURCE_LINE=0067: split -b "$CHUNK_SIZE_SPLIT" -d -a 4 "$F" "$CDIR/part-" >> "$LOG" 2>&1
|
||||||
@@ -102,36 +81,7 @@ DETAIL=SOURCE_LINE=0086: p=line.split("\t")
|
|||||||
DETAIL=SOURCE_LINE=0087: if len(p)>=9:
|
DETAIL=SOURCE_LINE=0087: if len(p)>=9:
|
||||||
DETAIL=SOURCE_LINE=0088: items.append({
|
DETAIL=SOURCE_LINE=0088: items.append({
|
||||||
DETAIL=SOURCE_LINE=0089: "vmid":p[0],
|
DETAIL=SOURCE_LINE=0089: "vmid":p[0],
|
||||||
DETAIL=SOURCE_LINE=0090: "kind":p[1],
|
DETAIL=SOURCE_LINE=...
|
||||||
DETAIL=SOURCE_LINE=0091: "size_bytes":int(p[2]),
|
|
||||||
DETAIL=SOURCE_LINE=0092: "sha256":p[3],
|
|
||||||
DETAIL=SOURCE_LINE=0093: "mtime_utc":p[4],
|
|
||||||
DETAIL=SOURCE_LINE=0094: "filename":p[5],
|
|
||||||
DETAIL=SOURCE_LINE=0095: "source_path":p[6],
|
|
||||||
DETAIL=SOURCE_LINE=0096: "mode":p[7],
|
|
||||||
DETAIL=SOURCE_LINE=0097: "parts":int(p[8]),
|
|
||||||
DETAIL=SOURCE_LINE=0098: })
|
|
||||||
DETAIL=SOURCE_LINE=0099:print(json.dumps({
|
|
||||||
DETAIL=SOURCE_LINE=0100: "job_id":job,
|
|
||||||
DETAIL=SOURCE_LINE=0101: "node":node,
|
|
||||||
DETAIL=SOURCE_LINE=0102: "generated_utc":ts,
|
|
||||||
DETAIL=SOURCE_LINE=0103: "vmids":vmids.split(),
|
|
||||||
DETAIL=SOURCE_LINE=0104: "count":int(count),
|
|
||||||
DETAIL=SOURCE_LINE=0105: "total_bytes":int(total),
|
|
||||||
DETAIL=SOURCE_LINE=0106: "chunked_count":int(chunked),
|
|
||||||
DETAIL=SOURCE_LINE=0107: "chunk_size_bytes":int(chunk_size),
|
|
||||||
DETAIL=SOURCE_LINE=0108: "items":items,
|
|
||||||
DETAIL=SOURCE_LINE=0109:}, indent=2))
|
|
||||||
DETAIL=SOURCE_LINE=0110:PY
|
|
||||||
DETAIL=SOURCE_LINE=0111:
|
|
||||||
DETAIL=SOURCE_LINE=0112:DEST_A="${REMOTE_A%/}/${NODE}/vm-local-dumps/${JOB_ID}"
|
|
||||||
DETAIL=SOURCE_LINE=0113:DEST_B="${REMOTE_B%/}/${NODE}/vm-local-dumps/${JOB_ID}"
|
|
||||||
DETAIL=SOURCE_LINE=0114:
|
|
||||||
DETAIL=SOURCE_LINE=0115:upload_one_dest() {
|
|
||||||
DETAIL=SOURCE_LINE=0116: DEST="$1"
|
|
||||||
DETAIL=SOURCE_LINE=0117: rclone mkdir "$DEST/files" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
||||||
DETAIL=SOURCE_LINE=0118: while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do
|
|
||||||
DETAIL=SOURCE_LINE=0119: if [ "$mode" = "chunked" ]; then
|
|
||||||
DETAIL=SOURCE_LINE=0120: CDIR="$CHUNK_ROOT/$JOB_ID/$bn.chunks"
|
DETAIL=SOURCE_LINE=0120: CDIR="$CHUNK_ROOT/$JOB_ID/$bn.chunks"
|
||||||
DETAIL=SOURCE_LINE=0121: echo "UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts" >> "$LOG"
|
DETAIL=SOURCE_LINE=0121: echo "UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts" >> "$LOG"
|
||||||
DETAIL=SOURCE_LINE=0122: rclone mkdir "$DEST/files/$bn.chunks" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
DETAIL=SOURCE_LINE=0122: rclone mkdir "$DEST/files/$bn.chunks" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||||
@@ -140,8 +90,7 @@ DETAIL=SOURCE_LINE=0124: [ -f "$part" ] || continue
|
|||||||
DETAIL=SOURCE_LINE=0125: pn="$(basename "$part")"
|
DETAIL=SOURCE_LINE=0125: pn="$(basename "$part")"
|
||||||
DETAIL=SOURCE_LINE=0126: rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
DETAIL=SOURCE_LINE=0126: rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||||
DETAIL=SOURCE_LINE=0127: done
|
DETAIL=SOURCE_LINE=0127: done
|
||||||
DETAIL=SOURCE_LINE=0128: rclone copyto "$CDIR/chunks.sha256" "$DEST/files/$bn.chunks/chunks.sha256" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
DETAIL=SOURCE_LINE=...
|
||||||
DETAIL=SOURCE_LINE=0129: else
|
|
||||||
DETAIL=SOURCE_LINE=0130: echo "UPLOAD_DIRECT dest=$DEST file=$bn size=$size" >> "$LOG"
|
DETAIL=SOURCE_LINE=0130: echo "UPLOAD_DIRECT dest=$DEST file=$bn size=$size" >> "$LOG"
|
||||||
DETAIL=SOURCE_LINE=0131: rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
DETAIL=SOURCE_LINE=0131: rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||||
DETAIL=SOURCE_LINE=0132: fi
|
DETAIL=SOURCE_LINE=0132: fi
|
||||||
@@ -153,161 +102,11 @@ DETAIL=SOURCE_LINE=0137: rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2
|
|||||||
DETAIL=SOURCE_LINE=0138:}
|
DETAIL=SOURCE_LINE=0138:}
|
||||||
DETAIL=SOURCE_LINE=0139:
|
DETAIL=SOURCE_LINE=0139:
|
||||||
DETAIL=SOURCE_LINE=0140:verify_one_dest() {
|
DETAIL=SOURCE_LINE=0140:verify_one_dest() {
|
||||||
DETAIL=SOURCE_LINE=0141: LABEL="$1"
|
DETAIL=FILESET_SOURCE_END
|
||||||
DETAIL=SOURCE_LINE=0142: DEST="$2"
|
|
||||||
DETAIL=SOURCE_LINE=0143: VBASE="$VERIFY_ROOT/$JOB_ID/$LABEL"
|
|
||||||
DETAIL=SOURCE_LINE=0144: rm -rf "$VBASE"
|
|
||||||
DETAIL=SOURCE_LINE=0145: mkdir -p "$VBASE"
|
|
||||||
DETAIL=SOURCE_LINE=0146: rclone copyto "$DEST/manifest.json" "$VBASE/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
||||||
DETAIL=SOURCE_LINE=0147: test -s "$VBASE/manifest.json" || BAD=$((BAD+1))
|
|
||||||
DETAIL=SOURCE_LINE=0148:
|
|
||||||
DETAIL=SOURCE_LINE=0149: while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do
|
|
||||||
DETAIL=SOURCE_LINE=0150: if [ "$mode" = "chunked" ]; then
|
|
||||||
DETAIL=SOURCE_LINE=0151: VDIR="$VBASE/$bn.chunks"
|
|
||||||
DETAIL=SOURCE_LINE=0152: mkdir -p "$VDIR"
|
|
||||||
DETAIL=SOURCE_LINE=0153: echo "VERIFY_CHUNKED label=$LABEL file=$bn parts=$parts" >> "$LOG"
|
|
||||||
DETAIL=SOURCE_LINE=0154: rclone copy "$DEST/files/$bn.chunks" "$VDIR" --filter "+ part-*" --filter "+ chunks.sha256" --filter "- *" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
||||||
DETAIL=SOURCE_LINE=0155: (cd "$VDIR" && sha256sum -c chunks.sha256) >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
||||||
DETAIL=SOURCE_LINE=0156: CALC="$(find "$VDIR" -maxdepth 1 -type f -name 'part-*' | sort | xargs cat | sha256sum | cut -d" " -f1)"
|
|
||||||
DETAIL=SOURCE_LINE=0157: if [ "$CALC" = "$sha" ]; then
|
|
||||||
DETAIL=SOURCE_LINE=0158: echo "OK combined_sha label=$LABEL file=$bn sha=$sha" >> "$LOG"
|
|
||||||
DETAIL=SOURCE_LINE=0159: else
|
|
||||||
DETAIL=SOURCE_LINE=0160: echo "BAD combined_sha label=$LABEL file=$bn expected=$sha got=$CALC" >> "$LOG"
|
|
||||||
DETAIL=SOURCE_LINE=0161: BAD=$((BAD+1))
|
|
||||||
DETAIL=SOURCE_LINE=0162: fi
|
|
||||||
DETAIL=SOURCE_LINE=0163: rm -rf "$VDIR"
|
|
||||||
DETAIL=SOURCE_LINE=0164: else
|
|
||||||
DETAIL=SOURCE_LINE=0165: echo "VERIFY_DIRECT label=$LABEL file=$bn" >> "$LOG"
|
|
||||||
DETAIL=SOURCE_LINE=0166: rclone copyto "$DEST/files/$bn" "$VBASE/$bn" --transfers 1 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
||||||
DETAIL=SOURCE_LINE=0167: printf "%s %s\n" "$sha" "$VBASE/$bn" | sha256sum -c - >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
|
||||||
DETAIL=SOURCE_LINE=0168: rm -f "$VBASE/$bn"
|
|
||||||
DETAIL=SOURCE_LINE=0169: fi
|
|
||||||
DETAIL=SOURCE_LINE=0170: done < "$FILES_TSV"
|
|
||||||
DETAIL=SOURCE_LINE=0171: rm -rf "$VBASE"
|
|
||||||
DETAIL=SOURCE_LINE=0172:}
|
|
||||||
DETAIL=SOURCE_LINE=0173:
|
|
||||||
DETAIL=SOURCE_LINE=0174:if [ "$BAD" -eq 0 ]; then
|
|
||||||
DETAIL=SOURCE_LINE=0175: upload_one_dest "$DEST_A"
|
|
||||||
DETAIL=SOURCE_LINE=0176: upload_one_dest "$DEST_B"
|
|
||||||
DETAIL=SOURCE_LINE=0177:fi
|
|
||||||
DETAIL=SOURCE_LINE=0178:
|
|
||||||
DETAIL=SOURCE_LINE=0179:if [ "$BAD" -eq 0 ]; then
|
|
||||||
DETAIL=SOURCE_LINE=0180: verify_one_dest "A" "$DEST_A"
|
|
||||||
DETAIL=SOURCE_LINE=0181: verify_one_dest "B" "$DEST_B"
|
|
||||||
DETAIL=SOURCE_LINE=0182:fi
|
|
||||||
DETAIL=SOURCE_LINE=0183:
|
|
||||||
DETAIL=SOURCE_LINE=0184:rm -rf "$VERIFY_ROOT/$JOB_ID"
|
|
||||||
DETAIL=SOURCE_LINE=0185:rm -rf "$CHUNK_ROOT/$JOB_ID"
|
|
||||||
DETAIL=SOURCE_LINE=0186:
|
|
||||||
DETAIL=SOURCE_LINE=0187:# HOMELAB_NO_LOCAL_AFTER_DUAL_CLOUD_434
|
|
||||||
DETAIL=SOURCE_LINE=0188:LOCAL_DUMPS_DELETED=false
|
|
||||||
DETAIL=SOURCE_LINE=0189:DELETED_COUNT=0
|
|
||||||
DETAIL=SOURCE_LINE=0190:DELETED_BYTES=0
|
|
||||||
DETAIL=SOURCE_LINE=0191:
|
|
||||||
DETAIL=SOURCE_LINE=0192:if [ "$BAD" -eq 0 ]; then
|
|
||||||
DETAIL=SOURCE_LINE=0193: echo "PREDELETE_VALIDATION_BEGIN" >> "$LOG"
|
|
||||||
DETAIL=SOURCE_LINE=0194:
|
|
||||||
DETAIL=SOURCE_LINE=0195: while IFS="$(printf " ")" read -r id kind size sha mt bn path mode parts; do
|
|
||||||
DETAIL=SOURCE_LINE=0196: case "$path" in
|
|
||||||
DETAIL=SOURCE_LINE=0197: /var/lib/vz/dump/*)
|
|
||||||
DETAIL=SOURCE_LINE=0198: ;;
|
|
||||||
DETAIL=SOURCE_LINE=0199: *)
|
|
||||||
DETAIL=SOURCE_LINE=0200: echo "REFUSE_DELETE_OUTSIDE_ALLOWED_ROOT path=$path" >> "$LOG"
|
|
||||||
DETAIL=SOURCE_LINE=0201: BAD=$((BAD+1))
|
|
||||||
DETAIL=SOURCE_LINE=0202: continue
|
|
||||||
DETAIL=SOURCE_LINE=0203: ;;
|
|
||||||
DETAIL=SOURCE_LINE=0204: esac
|
|
||||||
DETAIL=SOURCE_LINE=0205:
|
|
||||||
DETAIL=SOURCE_LINE=0206: if [ "$(basename "$path")" != "$bn" ]; then
|
|
||||||
DETAIL=SOURCE_LINE=0207: echo "REFUSE_DELETE_BASENAME_MISMATCH path=$path expected=$bn" >> "$LOG"
|
|
||||||
DETAIL=SOURCE_LINE=0208: BAD=$((BAD+1))
|
|
||||||
DETAIL=SOURCE_LINE=0209: continue
|
|
||||||
DETAIL=SOURCE_LINE=0210: fi
|
|
||||||
DETAIL=SOURCE_LINE=0211:
|
|
||||||
DETAIL=SOURCE_LINE=0212: if [ ! -f "$path" ] || [ -L "$path" ]; then
|
|
||||||
DETAIL=SOURCE_LINE=0213: echo "REFUSE_DELETE_INVALID_SOURCE path=$path" >> "$LOG"
|
|
||||||
DETAIL=SOURCE_LINE=0214: BAD=$((BAD+1))
|
|
||||||
DETAIL=SOURCE_LINE=0215: continue
|
|
||||||
DETAIL=SOURCE_LINE=0216: fi
|
|
||||||
DETAIL=SOURCE_LINE=0217:
|
|
||||||
DETAIL=SOURCE_LINE=0218: actual_size="$(stat -c %s "$path" 2>/dev/null || echo 0)"
|
|
||||||
DETAIL=SOURCE_LINE=0219:
|
|
||||||
DETAIL=SOURCE_LINE=0220: if [ "$actual_size" != "$size" ]; then
|
|
||||||
DETAIL=SOURCE_LINE=0221: echo "REFUSE_DELETE_SIZE_MISMATCH path=$path expected=$size actual=$actual_size" >> "$LOG"
|
|
||||||
DETAIL=SOURCE_LINE=0222: BAD=$((BAD+1))
|
|
||||||
DETAIL=SOURCE_LINE=0223: continue
|
|
||||||
DETAIL=SOURCE_LINE=0224: fi
|
|
||||||
DETAIL=SOURCE_LINE=0225:
|
|
||||||
DETAIL=SOURCE_LINE=0226: actual_sha="$(sha256sum "$path" 2>>"$LOG" | cut -d " " -f1 || true)"
|
|
||||||
DETAIL=SOURCE_LINE=0227:
|
|
||||||
DETAIL=SOURCE_LINE=0228: if [ "$actual_sha" != "$sha" ]; then
|
|
||||||
DETAIL=SOURCE_LINE=0229: echo "REFUSE_DELETE_SHA_MISMATCH path=$path" >> "$LOG"
|
|
||||||
DETAIL=SOURCE_LINE=0230: BAD=$((BAD+1))
|
|
||||||
DETAIL=SOURCE_LINE=0231: continue
|
|
||||||
DETAIL=SOURCE_LINE=0232: fi
|
|
||||||
DETAIL=SOURCE_LINE=0233:
|
|
||||||
DETAIL=SOURCE_LINE=0234: echo "PREDELETE_VALIDATION_OK path=$path size=$size sha=$sha" >> "$LOG"
|
|
||||||
DETAIL=SOURCE_LINE=0235: done < "$FILES_TSV"
|
|
||||||
DETAIL=SOURCE_LINE=0236:
|
|
||||||
DETAIL=SOURCE_LINE=0237: echo "PREDELETE_VALIDATION_END bad=$BAD" >> "$LOG"
|
|
||||||
DETAIL=SOURCE_LINE=0238:fi
|
|
||||||
DETAIL=SOURCE_LINE=0239:
|
|
||||||
DETAIL=SOURCE_LINE=0240:if [ "$BAD" -eq 0 ]; then
|
|
||||||
DETAIL=SOURCE_LINE=0241: echo "LOCAL_PAYLOAD_DELETE_BEGIN" >> "$LOG"
|
|
||||||
DETAIL=SOURCE_LINE=0242:
|
|
||||||
DETAIL=SOURCE_LINE=0243: while IFS="$(printf " ")" read -r id kind size sha mt bn path mode parts; do
|
|
||||||
DETAIL=SOURCE_LINE=0244: if rm -f -- "$path" 2>>"$LOG"; then
|
|
||||||
DETAIL=SOURCE_LINE=0245: if [ ! -e "$path" ] && [ ! -L "$path" ]; then
|
|
||||||
DETAIL=SOURCE_LINE=0246: DELETED_COUNT=$((DELETED_COUNT+1))
|
|
||||||
DETAIL=SOURCE_LINE=0247: DELETED_BYTES=$((DELETED_BYTES+size))
|
|
||||||
DETAIL=SOURCE_LINE=0248: echo "LOCAL_PAYLOAD_DELETED path=$path bytes=$size" >> "$LOG"
|
|
||||||
DETAIL=SOURCE_LINE=0249: else
|
|
||||||
DETAIL=SOURCE_LINE=0250: echo "LOCAL_PAYLOAD_DELETE_CONFIRM_FAILED path=$path" >> "$LOG"
|
|
||||||
DETAIL=SOURCE_LINE=0251: BAD=$((BAD+1))
|
|
||||||
DETAIL=SOURCE_LINE=0252: fi
|
|
||||||
DETAIL=SOURCE_LINE=0253: else
|
|
||||||
DETAIL=SOURCE_LINE=0254: echo "LOCAL_PAYLOAD_DELETE_FAILED path=$path" >> "$LOG"
|
|
||||||
DETAIL=SOURCE_LINE=0255: BAD=$((BAD+1))
|
|
||||||
DETAIL=SOURCE_LINE=0256: fi
|
|
||||||
DETAIL=SOURCE_LINE=0257: done < "$FILES_TSV"
|
|
||||||
DETAIL=SOURCE_LINE=0258:
|
|
||||||
DETAIL=SOURCE_LINE=0259: if [ "$BAD" -eq 0 ] && [ "$DELETED_COUNT" -eq "$COUNT" ]; then
|
|
||||||
DETAIL=SOURCE_LINE=0260: LOCAL_DUMPS_DELETED=true
|
|
||||||
DETAIL=SOURCE_LINE=0261: fi
|
|
||||||
DETAIL=SOURCE_LINE=0262:
|
|
||||||
DETAIL=SOURCE_LINE=0263: echo "LOCAL_PAYLOAD_DELETE_END deleted_count=$DELETED_COUNT deleted_bytes=$DELETED_BYTES bad=$BAD" >> "$LOG"
|
|
||||||
DETAIL=SOURCE_LINE=0264:fi
|
|
||||||
DETAIL=SOURCE_LINE=0265:
|
|
||||||
DETAIL=SOURCE_LINE=0266:if [ "$BAD" -eq 0 ] && [ "$LOCAL_DUMPS_DELETED" = true ]; then
|
|
||||||
DETAIL=SOURCE_LINE=0267: STATUS="OK"
|
|
||||||
DETAIL=SOURCE_LINE=0268:else
|
|
||||||
DETAIL=SOURCE_LINE=0269: STATUS="WARN"
|
|
||||||
DETAIL=SOURCE_LINE=0270:fi
|
|
||||||
DETAIL=SOURCE_LINE=0271:
|
|
||||||
DETAIL=SOURCE_LINE=0272:jq -n \
|
|
||||||
DETAIL=SOURCE_LINE=0273: --arg job_id "$JOB_ID" \
|
|
||||||
DETAIL=SOURCE_LINE=0274: --arg status "$STATUS" \
|
|
||||||
DETAIL=SOURCE_LINE=0275: --arg node "$NODE" \
|
|
||||||
DETAIL=SOURCE_LINE=0276: --arg remote_a "$DEST_A" \
|
|
||||||
DETAIL=SOURCE_LINE=0277: --arg remote_b "$DEST_B" \
|
|
||||||
DETAIL=SOURCE_LINE=0278: --argjson count "$COUNT" \
|
|
||||||
DETAIL=SOURCE_LINE=0279: --argjson total_bytes "$TOTAL" \
|
|
||||||
DETAIL=SOURCE_LINE=0280: --argjson chunked_count "$CHUNKED" \
|
|
||||||
DETAIL=SOURCE_LINE=0281: --argjson chunk_size_bytes "$CHUNK_SIZE_BYTES" \
|
|
||||||
DETAIL=SOURCE_LINE=0282: --arg log "$LOG" \
|
|
||||||
DETAIL=SOURCE_LINE=0283: --argjson local_dumps_deleted "$LOCAL_DUMPS_DELETED" \
|
|
||||||
DETAIL=SOURCE_LINE=0284: --argjson deleted_count "$DELETED_COUNT" \
|
|
||||||
DETAIL=SOURCE_LINE=0285: --argjson deleted_bytes "$DELETED_BYTES" \
|
|
||||||
DETAIL=SOURCE_LINE=0286: "{job_id:\$job_id,status:\$status,node:\$node,remote_a:\$remote_a,remote_b:\$remote_b,count:\$count,total_bytes:\$total_bytes,chunked_count:\$chunked_count,chunk_size_bytes:\$chunk_size_bytes,log:\$log,local_dumps_deleted:\$local_dumps_deleted,deleted_count:\$deleted_count,deleted_bytes:\$deleted_bytes}" \
|
|
||||||
DETAIL=SOURCE_LINE=0287: > "$ROOT/latest-vm-local-dumps-cloud.json"
|
|
||||||
DETAIL=SOURCE_LINE=0288:
|
|
||||||
DETAIL=SOURCE_LINE=0289:echo "STATUS=$STATUS TS=$TS TYPE=vm-local-dumps-cloud BAD=$BAD COUNT=$COUNT CHUNKED=$CHUNKED TOTAL_BYTES=$TOTAL JOB=$JOB_ID REMOTE_A=$DEST_A REMOTE_B=$DEST_B LOCAL_DUMPS_DELETED=$LOCAL_DUMPS_DELETED DELETED_COUNT=$DELETED_COUNT DELETED_BYTES=$DELETED_BYTES LOG=$LOG" > "$OUT"
|
|
||||||
DETAIL=SOURCE_LINE=0290:cat "$OUT"
|
|
||||||
DETAIL=SELECTION_SOURCE_END
|
|
||||||
DETAIL=ARCHIVES_CHANGED=NO
|
DETAIL=ARCHIVES_CHANGED=NO
|
||||||
DETAIL=SERVICE_STARTED=NO
|
DETAIL=SERVICE_STARTED=NO
|
||||||
DETAIL=SECRET_VALUES_OUTPUT=NONE
|
DETAIL=SECRET_VALUES_OUTPUT=NONE
|
||||||
DETAIL=EXACT_SELECTION_AUDIT=PASS
|
DETAIL=FILESET_TRACE=PASS
|
||||||
CHANGES_MADE=NO
|
CHANGES_MADE=NO
|
||||||
|
|
||||||
OUTPUT_END
|
OUTPUT_END
|
||||||
|
|||||||
Reference in New Issue
Block a user