homelab-admin/homelab-public-context
1
0
Fork 0
Code Activity

runtime: publish BACKUP-VM-DUMP-SOURCE-077

Browse Source
This commit is contained in:
homelab-runtime-publisher
2026-07-21 12:47:45 +00:00
parent 03190db0aa
commit f21d35151d
4 changed files with 561 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/history/BACKUP-VM-DUMP-SOURCE-077.json
+30
View File
File diff suppressed because one or more lines are too long
runtime/history/BACKUP-VM-DUMP-SOURCE-077.txt
+272
View File
@@ -0,0 +1,272 @@
CHAT_OUTPUT_BEGIN
COMMAND_ID=BACKUP-VM-DUMP-SOURCE-077
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=backup-audit
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
COMMAND_SHA256=b835d1210c5520c9d8d56eac7575cc679e5d0a22ce04f879cf8af953e2bb9b9e
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGES_MADE=false
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=9dd20a7d68dce96c8f08bcb4407101b10e80a36077f51706b0ba8b709099ba58
SANITIZED_OUTPUT_SHA256=9dd20a7d68dce96c8f08bcb4407101b10e80a36077f51706b0ba8b709099ba58
OUTPUT_BEGIN
=== SCRIPT_METADATA ===
1de52eb4ddb17b0ecff2cf3f3021e65b92e91fb44416f97943c51e248c512ed6 /usr/local/sbin/homelab-vm-local-dumps-2cloud
File: /usr/local/sbin/homelab-vm-local-dumps-2cloud
Size: 7272 Blocks: 16 IO Block: 4096 regular file
Device: 252,1 Inode: 5148408 Links: 1
Access: (0700/-rwx------) Uid: ( 0/ root) Gid: ( 0/ root)
Access: 2026-07-21 00:05:00.471248283 +0300
Modify: 2026-07-06 18:49:49.740645050 +0300
Change: 2026-07-06 18:49:49.740645050 +0300
Birth: 2026-07-06 18:27:29.152531375 +0300
=== SCRIPT_SOURCE ===
#!/usr/bin/env bash
set +u
OUT="/var/lib/homelab-health/vm-local-dumps-cloud.txt"
ROOT="/var/lib/homelab-vm-backup"
LOG="$ROOT/vm-local-dumps-2cloud-latest.log"
VERIFY_ROOT="/var/tmp/homelab-vm-local-dumps-verify"
CHUNK_ROOT="/var/tmp/homelab-vm-local-dumps-chunks"
CHUNK_SIZE_BYTES="${CHUNK_SIZE_BYTES:-1073741824}"
CHUNK_SIZE_SPLIT="${CHUNK_SIZE_SPLIT:-1024M}"
mkdir -p /var/lib/homelab-health "$ROOT" "$VERIFY_ROOT" "$CHUNK_ROOT"
TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
JOB_TS="$(date -u +%Y%m%dT%H%M%SZ)"
NODE="$(hostname -s)"
BAD=0
COUNT=0
TOTAL=0
CHUNKED=0
VMIDS="${VMIDS:-110 112 150}"
source /etc/homelab-backup.env 2>/dev/null || BAD=1
JOB_ID="homelab-${NODE}-vm-local-dumps-${JOB_TS}"
WORK="$ROOT/$JOB_ID"
mkdir -p "$WORK"
FILES_TSV="$WORK/files.tsv"
CHKS="$WORK/checksums.txt"
MAN="$WORK/manifest.json"
: > "$FILES_TSV"
: > "$CHKS"
: > "$LOG"
echo "job_id=$JOB_ID" >> "$LOG"
echo "vmids=$VMIDS" >> "$LOG"
echo "chunk_size_bytes=$CHUNK_SIZE_BYTES" >> "$LOG"
for id in $VMIDS; do
F="$(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-${id}-*.vma.zst" -o -name "vzdump-lxc-${id}-*.tar.zst" \) -printf "%T@ %p\n" 2>/dev/null | sort -n | tail -n 1 | cut -d" " -f2-)"
if [ -z "$F" ] || [ ! -f "$F" ]; then
echo "MISSING vmid=$id" >> "$LOG"
BAD=$((BAD+1))
continue
fi
BN="$(basename "$F")"
SIZE="$(stat -c %s "$F" 2>/dev/null || echo 0)"
MT="$(date -u -d "@$(stat -c %Y "$F" 2>/dev/null || echo 0)" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)"
KIND="unknown"
echo "$BN" | grep -q "^vzdump-qemu-" && KIND="qemu"
echo "$BN" | grep -q "^vzdump-lxc-" && KIND="lxc"
SHA="$(sha256sum "$F" | cut -d" " -f1)"
printf "%s %s\n" "$SHA" "$BN" >> "$CHKS"
MODE="direct"
PARTS=0
if [ "$SIZE" -gt "$CHUNK_SIZE_BYTES" ]; then
MODE="chunked"
CHUNKED=$((CHUNKED+1))
CDIR="$CHUNK_ROOT/$JOB_ID/$BN.chunks"
rm -rf "$CDIR"
mkdir -p "$CDIR"
echo "SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR" >> "$LOG"
split -b "$CHUNK_SIZE_SPLIT" -d -a 4 "$F" "$CDIR/part-" >> "$LOG" 2>&1
PARTS="$(find "$CDIR" -maxdepth 1 -type f -name 'part-*' | wc -l)"
(cd "$CDIR" && sha256sum part-* > chunks.sha256)
if [ "$PARTS" -le 0 ]; then
echo "BAD split produced no chunks file=$BN" >> "$LOG"
BAD=$((BAD+1))
fi
fi
printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" "$id" "$KIND" "$SIZE" "$SHA" "$MT" "$BN" "$F" "$MODE" "$PARTS" >> "$FILES_TSV"
COUNT=$((COUNT+1))
TOTAL=$((TOTAL+SIZE))
done
python3 - "$JOB_ID" "$NODE" "$TS" "$VMIDS" "$COUNT" "$TOTAL" "$CHUNKED" "$CHUNK_SIZE_BYTES" "$FILES_TSV" > "$MAN" <<'PY'
import json, sys, pathlib
job,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:]
items=[]
for line in pathlib.Path(tsv).read_text().splitlines():
p=line.split("\t")
if len(p)>=9:
items.append({
"vmid":p[0],
"kind":p[1],
"size_bytes":int(p[2]),
"sha256":p[3],
"mtime_utc":p[4],
"filename":p[5],
"source_path":p[6],
"mode":p[7],
"parts":int(p[8]),
})
print(json.dumps({
"job_id":job,
"node":node,
"generated_utc":ts,
"vmids":vmids.split(),
"count":int(count),
"total_bytes":int(total),
"chunked_count":int(chunked),
"chunk_size_bytes":int(chunk_size),
"items":items,
}, indent=2))
PY
DEST_A="${REMOTE_A%/}/${NODE}/vm-local-dumps/${JOB_ID}"
DEST_B="${REMOTE_B%/}/${NODE}/vm-local-dumps/${JOB_ID}"
upload_one_dest() {
DEST="$1"
rclone mkdir "$DEST/files" >> "$LOG" 2>&1 || BAD=$((BAD+1))
while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do
if [ "$mode" = "chunked" ]; then
CDIR="$CHUNK_ROOT/$JOB_ID/$bn.chunks"
echo "UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts" >> "$LOG"
rclone mkdir "$DEST/files/$bn.chunks" >> "$LOG" 2>&1 || BAD=$((BAD+1))
for part in "$CDIR"/part-*; do
[ -f "$part" ] || continue
pn="$(basename "$part")"
rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
done
rclone copyto "$CDIR/chunks.sha256" "$DEST/files/$bn.chunks/chunks.sha256" >> "$LOG" 2>&1 || BAD=$((BAD+1))
else
echo "UPLOAD_DIRECT dest=$DEST file=$bn size=$size" >> "$LOG"
rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
fi
done < "$FILES_TSV"
rclone copyto "$CHKS" "$DEST/checksums.txt" >> "$LOG" 2>&1 || BAD=$((BAD+1))
rclone copyto "$FILES_TSV" "$DEST/files.tsv" >> "$LOG" 2>&1 || BAD=$((BAD+1))
rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
}
verify_one_dest() {
LABEL="$1"
DEST="$2"
VBASE="$VERIFY_ROOT/$JOB_ID/$LABEL"
rm -rf "$VBASE"
mkdir -p "$VBASE"
rclone copyto "$DEST/manifest.json" "$VBASE/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
test -s "$VBASE/manifest.json" || BAD=$((BAD+1))
while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do
if [ "$mode" = "chunked" ]; then
VDIR="$VBASE/$bn.chunks"
mkdir -p "$VDIR"
echo "VERIFY_CHUNKED label=$LABEL file=$bn parts=$parts" >> "$LOG"
rclone copy "$DEST/files/$bn.chunks" "$VDIR" --filter "+ part-*" --filter "+ chunks.sha256" --filter "- *" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
(cd "$VDIR" && sha256sum -c chunks.sha256) >> "$LOG" 2>&1 || BAD=$((BAD+1))
CALC="$(find "$VDIR" -maxdepth 1 -type f -name 'part-*' | sort | xargs cat | sha256sum | cut -d" " -f1)"
if [ "$CALC" = "$sha" ]; then
echo "OK combined_sha label=$LABEL file=$bn sha=$sha" >> "$LOG"
else
echo "BAD combined_sha label=$LABEL file=$bn expected=$sha got=$CALC" >> "$LOG"
BAD=$((BAD+1))
fi
rm -rf "$VDIR"
else
echo "VERIFY_DIRECT label=$LABEL file=$bn" >> "$LOG"
rclone copyto "$DEST/files/$bn" "$VBASE/$bn" --transfers 1 >> "$LOG" 2>&1 || BAD=$((BAD+1))
printf "%s %s\n" "$sha" "$VBASE/$bn" | sha256sum -c - >> "$LOG" 2>&1 || BAD=$((BAD+1))
rm -f "$VBASE/$bn"
fi
done < "$FILES_TSV"
rm -rf "$VBASE"
}
if [ "$BAD" -eq 0 ]; then
upload_one_dest "$DEST_A"
upload_one_dest "$DEST_B"
fi
if [ "$BAD" -eq 0 ]; then
verify_one_dest "A" "$DEST_A"
verify_one_dest "B" "$DEST_B"
fi
rm -rf "$VERIFY_ROOT/$JOB_ID"
rm -rf "$CHUNK_ROOT/$JOB_ID"
if [ "$BAD" -eq 0 ]; then STATUS="OK"; else STATUS="WARN"; fi
jq -n \
--arg job_id "$JOB_ID" \
--arg status "$STATUS" \
--arg node "$NODE" \
--arg remote_a "$DEST_A" \
--arg remote_b "$DEST_B" \
--argjson count "$COUNT" \
--argjson total_bytes "$TOTAL" \
--argjson chunked_count "$CHUNKED" \
--argjson chunk_size_bytes "$CHUNK_SIZE_BYTES" \
--arg log "$LOG" \
"{job_id:\$job_id,status:\$status,node:\$node,remote_a:\$remote_a,remote_b:\$remote_b,count:\$count,total_bytes:\$total_bytes,chunked_count:\$chunked_count,chunk_size_bytes:\$chunk_size_bytes,log:\$log,local_dumps_deleted:false}" \
> "$ROOT/latest-vm-local-dumps-cloud.json"
echo "STATUS=$STATUS TS=$TS TYPE=vm-local-dumps-cloud BAD=$BAD COUNT=$COUNT CHUNKED=$CHUNKED TOTAL_BYTES=$TOTAL JOB=$JOB_ID REMOTE_A=$DEST_A REMOTE_B=$DEST_B LOG=$LOG" > "$OUT"
cat "$OUT"
=== SERVICE ===
# /etc/systemd/system/homelab-vm-local-dumps-2cloud.service
[Unit]
Description=Homelab VM/CT existing local dumps two-cloud verifier
After=network-online.target
[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-vm-local-dumps-2cloud
# /etc/systemd/system/homelab-vm-local-dumps-2cloud.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; argv[]=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
ActiveState=inactive
SubState=dead
=== TIMER ===
# /etc/systemd/system/homelab-vm-local-dumps-2cloud.timer
[Unit]
Description=Homelab VM/CT existing local dumps two-cloud verifier timer
[Timer]
OnCalendar=Sun *-*-* 10:00:00
RandomizedDelaySec=30m
Persistent=true
[Install]
WantedBy=timers.target
NextElapseUSecRealtime=Sun 2026-07-26 10:29:15 MSK
ActiveState=active
UnitFileState=enabled
OUTPUT_END
CHAT_OUTPUT_END
runtime/latest.json
+7 -7
View File
File diff suppressed because one or more lines are too long
runtime/latest.txt
+252 -7
View File
@@ -1,5 +1,5 @@
CHAT_OUTPUT_BEGIN
COMMAND_ID=BACKUP-VM-DUMP-DECISION-076
COMMAND_ID=BACKUP-VM-DUMP-SOURCE-077
STATUS=OK
RC=0
HOST=pve01
@@ -9,19 +9,264 @@ REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
COMMAND_SHA256=79b8d619ea551eca9abbf92c24d3e8954f464302759ebb8adf24b308c705df6a
COMMAND_SHA256=b835d1210c5520c9d8d56eac7575cc679e5d0a22ce04f879cf8af953e2bb9b9e
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGES_MADE=false
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=b7a0c0af997d0aa6c682407daef9f7a3855cc52d0e4df08c53a1f4013d9eca72
SANITIZED_OUTPUT_SHA256=b7a0c0af997d0aa6c682407daef9f7a3855cc52d0e4df08c53a1f4013d9eca72
RAW_EVIDENCE_SHA256=9dd20a7d68dce96c8f08bcb4407101b10e80a36077f51706b0ba8b709099ba58
SANITIZED_OUTPUT_SHA256=9dd20a7d68dce96c8f08bcb4407101b10e80a36077f51706b0ba8b709099ba58
OUTPUT_BEGIN
=== NODE_PIPELINE_STATES ===
========== NODE=pve01 ==========
VM_DUMP_DECISION_DATA=COMPLETE
=== SCRIPT_METADATA ===
1de52eb4ddb17b0ecff2cf3f3021e65b92e91fb44416f97943c51e248c512ed6 /usr/local/sbin/homelab-vm-local-dumps-2cloud
File: /usr/local/sbin/homelab-vm-local-dumps-2cloud
Size: 7272 Blocks: 16 IO Block: 4096 regular file
Device: 252,1 Inode: 5148408 Links: 1
Access: (0700/-rwx------) Uid: ( 0/ root) Gid: ( 0/ root)
Access: 2026-07-21 00:05:00.471248283 +0300
Modify: 2026-07-06 18:49:49.740645050 +0300
Change: 2026-07-06 18:49:49.740645050 +0300
Birth: 2026-07-06 18:27:29.152531375 +0300
=== SCRIPT_SOURCE ===
#!/usr/bin/env bash
set +u
OUT="/var/lib/homelab-health/vm-local-dumps-cloud.txt"
ROOT="/var/lib/homelab-vm-backup"
LOG="$ROOT/vm-local-dumps-2cloud-latest.log"
VERIFY_ROOT="/var/tmp/homelab-vm-local-dumps-verify"
CHUNK_ROOT="/var/tmp/homelab-vm-local-dumps-chunks"
CHUNK_SIZE_BYTES="${CHUNK_SIZE_BYTES:-1073741824}"
CHUNK_SIZE_SPLIT="${CHUNK_SIZE_SPLIT:-1024M}"
mkdir -p /var/lib/homelab-health "$ROOT" "$VERIFY_ROOT" "$CHUNK_ROOT"
TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
JOB_TS="$(date -u +%Y%m%dT%H%M%SZ)"
NODE="$(hostname -s)"
BAD=0
COUNT=0
TOTAL=0
CHUNKED=0
VMIDS="${VMIDS:-110 112 150}"
source /etc/homelab-backup.env 2>/dev/null || BAD=1
JOB_ID="homelab-${NODE}-vm-local-dumps-${JOB_TS}"
WORK="$ROOT/$JOB_ID"
mkdir -p "$WORK"
FILES_TSV="$WORK/files.tsv"
CHKS="$WORK/checksums.txt"
MAN="$WORK/manifest.json"
: > "$FILES_TSV"
: > "$CHKS"
: > "$LOG"
echo "job_id=$JOB_ID" >> "$LOG"
echo "vmids=$VMIDS" >> "$LOG"
echo "chunk_size_bytes=$CHUNK_SIZE_BYTES" >> "$LOG"
for id in $VMIDS; do
F="$(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-${id}-*.vma.zst" -o -name "vzdump-lxc-${id}-*.tar.zst" \) -printf "%T@ %p\n" 2>/dev/null | sort -n | tail -n 1 | cut -d" " -f2-)"
if [ -z "$F" ] || [ ! -f "$F" ]; then
echo "MISSING vmid=$id" >> "$LOG"
BAD=$((BAD+1))
continue
fi
BN="$(basename "$F")"
SIZE="$(stat -c %s "$F" 2>/dev/null || echo 0)"
MT="$(date -u -d "@$(stat -c %Y "$F" 2>/dev/null || echo 0)" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)"
KIND="unknown"
echo "$BN" | grep -q "^vzdump-qemu-" && KIND="qemu"
echo "$BN" | grep -q "^vzdump-lxc-" && KIND="lxc"
SHA="$(sha256sum "$F" | cut -d" " -f1)"
printf "%s %s\n" "$SHA" "$BN" >> "$CHKS"
MODE="direct"
PARTS=0
if [ "$SIZE" -gt "$CHUNK_SIZE_BYTES" ]; then
MODE="chunked"
CHUNKED=$((CHUNKED+1))
CDIR="$CHUNK_ROOT/$JOB_ID/$BN.chunks"
rm -rf "$CDIR"
mkdir -p "$CDIR"
echo "SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR" >> "$LOG"
split -b "$CHUNK_SIZE_SPLIT" -d -a 4 "$F" "$CDIR/part-" >> "$LOG" 2>&1
PARTS="$(find "$CDIR" -maxdepth 1 -type f -name 'part-*' | wc -l)"
(cd "$CDIR" && sha256sum part-* > chunks.sha256)
if [ "$PARTS" -le 0 ]; then
echo "BAD split produced no chunks file=$BN" >> "$LOG"
BAD=$((BAD+1))
fi
fi
printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" "$id" "$KIND" "$SIZE" "$SHA" "$MT" "$BN" "$F" "$MODE" "$PARTS" >> "$FILES_TSV"
COUNT=$((COUNT+1))
TOTAL=$((TOTAL+SIZE))
done
python3 - "$JOB_ID" "$NODE" "$TS" "$VMIDS" "$COUNT" "$TOTAL" "$CHUNKED" "$CHUNK_SIZE_BYTES" "$FILES_TSV" > "$MAN" <<'PY'
import json, sys, pathlib
job,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:]
items=[]
for line in pathlib.Path(tsv).read_text().splitlines():
p=line.split("\t")
if len(p)>=9:
items.append({
"vmid":p[0],
"kind":p[1],
"size_bytes":int(p[2]),
"sha256":p[3],
"mtime_utc":p[4],
"filename":p[5],
"source_path":p[6],
"mode":p[7],
"parts":int(p[8]),
})
print(json.dumps({
"job_id":job,
"node":node,
"generated_utc":ts,
"vmids":vmids.split(),
"count":int(count),
"total_bytes":int(total),
"chunked_count":int(chunked),
"chunk_size_bytes":int(chunk_size),
"items":items,
}, indent=2))
PY
DEST_A="${REMOTE_A%/}/${NODE}/vm-local-dumps/${JOB_ID}"
DEST_B="${REMOTE_B%/}/${NODE}/vm-local-dumps/${JOB_ID}"
upload_one_dest() {
DEST="$1"
rclone mkdir "$DEST/files" >> "$LOG" 2>&1 || BAD=$((BAD+1))
while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do
if [ "$mode" = "chunked" ]; then
CDIR="$CHUNK_ROOT/$JOB_ID/$bn.chunks"
echo "UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts" >> "$LOG"
rclone mkdir "$DEST/files/$bn.chunks" >> "$LOG" 2>&1 || BAD=$((BAD+1))
for part in "$CDIR"/part-*; do
[ -f "$part" ] || continue
pn="$(basename "$part")"
rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
done
rclone copyto "$CDIR/chunks.sha256" "$DEST/files/$bn.chunks/chunks.sha256" >> "$LOG" 2>&1 || BAD=$((BAD+1))
else
echo "UPLOAD_DIRECT dest=$DEST file=$bn size=$size" >> "$LOG"
rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
fi
done < "$FILES_TSV"
rclone copyto "$CHKS" "$DEST/checksums.txt" >> "$LOG" 2>&1 || BAD=$((BAD+1))
rclone copyto "$FILES_TSV" "$DEST/files.tsv" >> "$LOG" 2>&1 || BAD=$((BAD+1))
rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
}
verify_one_dest() {
LABEL="$1"
DEST="$2"
VBASE="$VERIFY_ROOT/$JOB_ID/$LABEL"
rm -rf "$VBASE"
mkdir -p "$VBASE"
rclone copyto "$DEST/manifest.json" "$VBASE/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
test -s "$VBASE/manifest.json" || BAD=$((BAD+1))
while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do
if [ "$mode" = "chunked" ]; then
VDIR="$VBASE/$bn.chunks"
mkdir -p "$VDIR"
echo "VERIFY_CHUNKED label=$LABEL file=$bn parts=$parts" >> "$LOG"
rclone copy "$DEST/files/$bn.chunks" "$VDIR" --filter "+ part-*" --filter "+ chunks.sha256" --filter "- *" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
(cd "$VDIR" && sha256sum -c chunks.sha256) >> "$LOG" 2>&1 || BAD=$((BAD+1))
CALC="$(find "$VDIR" -maxdepth 1 -type f -name 'part-*' | sort | xargs cat | sha256sum | cut -d" " -f1)"
if [ "$CALC" = "$sha" ]; then
echo "OK combined_sha label=$LABEL file=$bn sha=$sha" >> "$LOG"
else
echo "BAD combined_sha label=$LABEL file=$bn expected=$sha got=$CALC" >> "$LOG"
BAD=$((BAD+1))
fi
rm -rf "$VDIR"
else
echo "VERIFY_DIRECT label=$LABEL file=$bn" >> "$LOG"
rclone copyto "$DEST/files/$bn" "$VBASE/$bn" --transfers 1 >> "$LOG" 2>&1 || BAD=$((BAD+1))
printf "%s %s\n" "$sha" "$VBASE/$bn" | sha256sum -c - >> "$LOG" 2>&1 || BAD=$((BAD+1))
rm -f "$VBASE/$bn"
fi
done < "$FILES_TSV"
rm -rf "$VBASE"
}
if [ "$BAD" -eq 0 ]; then
upload_one_dest "$DEST_A"
upload_one_dest "$DEST_B"
fi
if [ "$BAD" -eq 0 ]; then
verify_one_dest "A" "$DEST_A"
verify_one_dest "B" "$DEST_B"
fi
rm -rf "$VERIFY_ROOT/$JOB_ID"
rm -rf "$CHUNK_ROOT/$JOB_ID"
if [ "$BAD" -eq 0 ]; then STATUS="OK"; else STATUS="WARN"; fi
jq -n \
--arg job_id "$JOB_ID" \
--arg status "$STATUS" \
--arg node "$NODE" \
--arg remote_a "$DEST_A" \
--arg remote_b "$DEST_B" \
--argjson count "$COUNT" \
--argjson total_bytes "$TOTAL" \
--argjson chunked_count "$CHUNKED" \
--argjson chunk_size_bytes "$CHUNK_SIZE_BYTES" \
--arg log "$LOG" \
"{job_id:\$job_id,status:\$status,node:\$node,remote_a:\$remote_a,remote_b:\$remote_b,count:\$count,total_bytes:\$total_bytes,chunked_count:\$chunked_count,chunk_size_bytes:\$chunk_size_bytes,log:\$log,local_dumps_deleted:false}" \
> "$ROOT/latest-vm-local-dumps-cloud.json"
echo "STATUS=$STATUS TS=$TS TYPE=vm-local-dumps-cloud BAD=$BAD COUNT=$COUNT CHUNKED=$CHUNKED TOTAL_BYTES=$TOTAL JOB=$JOB_ID REMOTE_A=$DEST_A REMOTE_B=$DEST_B LOG=$LOG" > "$OUT"
cat "$OUT"
=== SERVICE ===
# /etc/systemd/system/homelab-vm-local-dumps-2cloud.service
[Unit]
Description=Homelab VM/CT existing local dumps two-cloud verifier
After=network-online.target
[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-vm-local-dumps-2cloud
# /etc/systemd/system/homelab-vm-local-dumps-2cloud.service.d/90-backup-matrix-refresh.conf
[Service]
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; argv[]=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
ActiveState=inactive
SubState=dead
=== TIMER ===
# /etc/systemd/system/homelab-vm-local-dumps-2cloud.timer
[Unit]
Description=Homelab VM/CT existing local dumps two-cloud verifier timer
[Timer]
OnCalendar=Sun *-*-* 10:00:00
RandomizedDelaySec=30m
Persistent=true
[Install]
WantedBy=timers.target
NextElapseUSecRealtime=Sun 2026-07-26 10:29:15 MSK
ActiveState=active
UnitFileState=enabled
OUTPUT_END
CHAT_OUTPUT_END