runtime: publish BACKUP-VM-DUMP-CONSISTENCY-078
This commit is contained in:
homelab-runtime-publisher
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"channel": "homelab-runtime",
|
||||
"command_id": "BACKUP-VM-DUMP-CONSISTENCY-078",
|
||||
"status": "OK",
|
||||
"rc": 0,
|
||||
"host": "pve01",
|
||||
"mode": "read-only",
|
||||
"component": "backup-audit",
|
||||
"started_at_utc": "2026-07-21T12:55:03Z",
|
||||
"finished_at_utc": "2026-07-21T12:55:04Z",
|
||||
"reference_register_checked": true,
|
||||
"reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
|
||||
"error_register_checked": true,
|
||||
"error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752",
|
||||
"command_sha256": "876c832ac6a555af28ff29ccdceb28c79e7244a361f277e9278a8f75b65caad5",
|
||||
"duplicate_failed_command_blocked": false,
|
||||
"block_reason": null,
|
||||
"execution_started": true,
|
||||
"changes_made": false,
|
||||
"sanitized": true,
|
||||
"secrets_included": false,
|
||||
"private_addresses_included": false,
|
||||
"raw_evidence_retained_locally": true,
|
||||
"raw_evidence_sha256": "02a6668400353ad78630dc31cad6ae25943bc9bd92bc8aff5a46cc2f4dc89d8f",
|
||||
"sanitized_output_sha256": "02a6668400353ad78630dc31cad6ae25943bc9bd92bc8aff5a46cc2f4dc89d8f",
|
||||
"output_truncated_in_json": false,
|
||||
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
|
||||
"output": "========== NODE=pve01 ==========\n1de52eb4ddb17b0ecff2cf3f3021e65b92e91fb44416f97943c51e248c512ed6 /usr/local/sbin/homelab-vm-local-dumps-2cloud\n30:CHKS=\"$WORK/checksums.txt\"\n39:echo \"chunk_size_bytes=$CHUNK_SIZE_BYTES\" >> \"$LOG\"\n50: SIZE=\"$(stat -c %s \"$F\" 2>/dev/null || echo 0)\"\n51: MT=\"$(date -u -d \"@$(stat -c %Y \"$F\" 2>/dev/null || echo 0)\" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)\"\n55: SHA=\"$(sha256sum \"$F\" | cut -d\" \" -f1)\"\n64: rm -rf \"$CDIR\"\n66: echo \"SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR\" >> \"$LOG\"\n69: (cd \"$CDIR\" && sha256sum part-* > chunks.sha256)\n83:job,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:]\n91: \"size_bytes\":int(p[2]),\n92: \"sha256\":p[3],\n107: \"chunk_size_bytes\":int(chunk_size),\n112:DEST_A=\"${REMOTE_A%/}/${NODE}/vm-local-dumps/${JOB_ID}\"\n113:DEST_B=\"${REMOTE_B%/}/${NODE}/vm-local-dumps/${JOB_ID}\"\n116: DEST=\"$1\"\n117: rclone mkdir \"$DEST/files\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n118: while IFS=\"$(printf \"\\t\")\" read -r id kind size sha mt bn path mode parts; do\n121: echo \"UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts\" >> \"$LOG\"\n122: rclone mkdir \"$DEST/files/$bn.chunks\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n126: rclone copyto \"$part\" \"$DEST/files/$bn.chunks/$pn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n128: rclone copyto \"$CDIR/chunks.sha256\" \"$DEST/files/$bn.chunks/chunks.sha256\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n130: echo \"UPLOAD_DIRECT dest=$DEST file=$bn size=$size\" >> \"$LOG\"\n131: rclone copyto \"$path\" \"$DEST/files/$bn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n135: rclone copyto \"$CHKS\" \"$DEST/checksums.txt\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n136: rclone copyto \"$FILES_TSV\" \"$DEST/files.tsv\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n137: rclone copyto \"$MAN\" \"$DEST/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n142: DEST=\"$2\"\n144: rm -rf \"$VBASE\"\n146: rclone copyto \"$DEST/manifest.json\" \"$VBASE/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n149: while IFS=\"$(printf \"\\t\")\" read -r id kind size sha mt bn path mode parts; do\n154: rclone copy \"$DEST/files/$bn.chunks\" \"$VDIR\" --filter \"+ part-*\" --filter \"+ chunks.sha256\" --filter \"- *\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n155: (cd \"$VDIR\" && sha256sum -c chunks.sha256) >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n156: CALC=\"$(find \"$VDIR\" -maxdepth 1 -type f -name 'part-*' | sort | xargs cat | sha256sum | cut -d\" \" -f1)\"\n163: rm -rf \"$VDIR\"\n166: rclone copyto \"$DEST/files/$bn\" \"$VBASE/$bn\" --transfers 1 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n167: printf \"%s %s\\n\" \"$sha\" \"$VBASE/$bn\" | sha256sum -c - >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n168: rm -f \"$VBASE/$bn\"\n171: rm -rf \"$VBASE\"\n175: upload_one_dest \"$DEST_A\"\n176: upload_one_dest \"$DEST_B\"\n180: verify_one_dest \"A\" \"$DEST_A\"\n181: verify_one_dest \"B\" \"$DEST_B\"\n184:rm -rf \"$VERIFY_ROOT/$JOB_ID\"\n185:rm -rf \"$CHUNK_ROOT/$JOB_ID\"\n191: --arg status \"$STATUS\" \\\n193: --arg remote_a \"$DEST_A\" \\\n194: --arg remote_b \"$DEST_B\" \\\n198: --argjson chunk_size_bytes \"$CHUNK_SIZE_BYTES\" \\\n200: \"{job_id:\\$job_id,status:\\$status,node:\\$node,remote_a:\\$remote_a,remote_b:\\$remote_b,count:\\$count,total_bytes:\\$total_bytes,chunked_count:\\$chunked_count,chunk_size_bytes:\\$chunk_size_bytes,log:\\$log,local_dumps_deleted:false}\" \\\n203:echo \"STATUS=$STATUS TS=$TS TYPE=vm-local-dumps-cloud BAD=$BAD COUNT=$COUNT CHUNKED=$CHUNKED TOTAL_BYTES=$TOTAL JOB=$JOB_ID REMOTE_A=$DEST_A REMOTE_B=$DEST_B LOG=$LOG\" > \"$OUT\"\nResult=success\nExecMainStatus=0\nNextElapseUSecRealtime=Sun 2026-07-26 10:29:15 MSK\nActiveState=active\nUnitFileState=enabled\nLOCAL_FILES=24 LOCAL_BYTES=30273877574\n========== NODE=pve02 ==========\nbash: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory\nsha256sum: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory\ngrep: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory\nResult=success\nExecMainStatus=0\nNextElapseUSecRealtime=\nActiveState=inactive\nUnitFileState=\nLOCAL_FILES=23 LOCAL_BYTES=3888168125\n========== NODE=pve03 ==========\nbash: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory\nsha256sum: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory\ngrep: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory\nResult=success\nExecMainStatus=0\nNextElapseUSecRealtime=\nActiveState=inactive\nUnitFileState=\nLOCAL_FILES=20 LOCAL_BYTES=47769002980\nVM_DUMP_CONSISTENCY=COMPLETE\n"
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
CHAT_OUTPUT_BEGIN
|
||||
COMMAND_ID=BACKUP-VM-DUMP-CONSISTENCY-078
|
||||
STATUS=OK
|
||||
RC=0
|
||||
HOST=pve01
|
||||
MODE=read-only
|
||||
COMPONENT=backup-audit
|
||||
REFERENCE_REGISTER_CHECK=OK
|
||||
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
|
||||
COMMAND_SHA256=876c832ac6a555af28ff29ccdceb28c79e7244a361f277e9278a8f75b65caad5
|
||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||
EXECUTION_STARTED=true
|
||||
CHANGES_MADE=false
|
||||
SANITIZED=yes
|
||||
SECRETS_INCLUDED=no
|
||||
PRIVATE_ADDRESSES_INCLUDED=no
|
||||
RAW_EVIDENCE_SHA256=02a6668400353ad78630dc31cad6ae25943bc9bd92bc8aff5a46cc2f4dc89d8f
|
||||
SANITIZED_OUTPUT_SHA256=02a6668400353ad78630dc31cad6ae25943bc9bd92bc8aff5a46cc2f4dc89d8f
|
||||
OUTPUT_BEGIN
|
||||
========== NODE=pve01 ==========
|
||||
1de52eb4ddb17b0ecff2cf3f3021e65b92e91fb44416f97943c51e248c512ed6 /usr/local/sbin/homelab-vm-local-dumps-2cloud
|
||||
30:CHKS="$WORK/checksums.txt"
|
||||
39:echo "chunk_size_bytes=$CHUNK_SIZE_BYTES" >> "$LOG"
|
||||
50: SIZE="$(stat -c %s "$F" 2>/dev/null || echo 0)"
|
||||
51: MT="$(date -u -d "@$(stat -c %Y "$F" 2>/dev/null || echo 0)" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)"
|
||||
55: SHA="$(sha256sum "$F" | cut -d" " -f1)"
|
||||
64: rm -rf "$CDIR"
|
||||
66: echo "SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR" >> "$LOG"
|
||||
69: (cd "$CDIR" && sha256sum part-* > chunks.sha256)
|
||||
83:job,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:]
|
||||
91: "size_bytes":int(p[2]),
|
||||
92: "sha256":p[3],
|
||||
107: "chunk_size_bytes":int(chunk_size),
|
||||
112:DEST_A="${REMOTE_A%/}/${NODE}/vm-local-dumps/${JOB_ID}"
|
||||
113:DEST_B="${REMOTE_B%/}/${NODE}/vm-local-dumps/${JOB_ID}"
|
||||
116: DEST="$1"
|
||||
117: rclone mkdir "$DEST/files" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
118: while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do
|
||||
121: echo "UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts" >> "$LOG"
|
||||
122: rclone mkdir "$DEST/files/$bn.chunks" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
126: rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
128: rclone copyto "$CDIR/chunks.sha256" "$DEST/files/$bn.chunks/chunks.sha256" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
130: echo "UPLOAD_DIRECT dest=$DEST file=$bn size=$size" >> "$LOG"
|
||||
131: rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
135: rclone copyto "$CHKS" "$DEST/checksums.txt" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
136: rclone copyto "$FILES_TSV" "$DEST/files.tsv" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
137: rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
142: DEST="$2"
|
||||
144: rm -rf "$VBASE"
|
||||
146: rclone copyto "$DEST/manifest.json" "$VBASE/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
149: while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do
|
||||
154: rclone copy "$DEST/files/$bn.chunks" "$VDIR" --filter "+ part-*" --filter "+ chunks.sha256" --filter "- *" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
155: (cd "$VDIR" && sha256sum -c chunks.sha256) >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
156: CALC="$(find "$VDIR" -maxdepth 1 -type f -name 'part-*' | sort | xargs cat | sha256sum | cut -d" " -f1)"
|
||||
163: rm -rf "$VDIR"
|
||||
166: rclone copyto "$DEST/files/$bn" "$VBASE/$bn" --transfers 1 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
167: printf "%s %s\n" "$sha" "$VBASE/$bn" | sha256sum -c - >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
168: rm -f "$VBASE/$bn"
|
||||
171: rm -rf "$VBASE"
|
||||
175: upload_one_dest "$DEST_A"
|
||||
176: upload_one_dest "$DEST_B"
|
||||
180: verify_one_dest "A" "$DEST_A"
|
||||
181: verify_one_dest "B" "$DEST_B"
|
||||
184:rm -rf "$VERIFY_ROOT/$JOB_ID"
|
||||
185:rm -rf "$CHUNK_ROOT/$JOB_ID"
|
||||
191: --arg status "$STATUS" \
|
||||
193: --arg remote_a "$DEST_A" \
|
||||
194: --arg remote_b "$DEST_B" \
|
||||
198: --argjson chunk_size_bytes "$CHUNK_SIZE_BYTES" \
|
||||
200: "{job_id:\$job_id,status:\$status,node:\$node,remote_a:\$remote_a,remote_b:\$remote_b,count:\$count,total_bytes:\$total_bytes,chunked_count:\$chunked_count,chunk_size_bytes:\$chunk_size_bytes,log:\$log,local_dumps_deleted:false}" \
|
||||
203:echo "STATUS=$STATUS TS=$TS TYPE=vm-local-dumps-cloud BAD=$BAD COUNT=$COUNT CHUNKED=$CHUNKED TOTAL_BYTES=$TOTAL JOB=$JOB_ID REMOTE_A=$DEST_A REMOTE_B=$DEST_B LOG=$LOG" > "$OUT"
|
||||
Result=success
|
||||
ExecMainStatus=0
|
||||
NextElapseUSecRealtime=Sun 2026-07-26 10:29:15 MSK
|
||||
ActiveState=active
|
||||
UnitFileState=enabled
|
||||
LOCAL_FILES=24 LOCAL_BYTES=30273877574
|
||||
========== NODE=pve02 ==========
|
||||
bash: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory
|
||||
sha256sum: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory
|
||||
grep: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory
|
||||
Result=success
|
||||
ExecMainStatus=0
|
||||
NextElapseUSecRealtime=
|
||||
ActiveState=inactive
|
||||
UnitFileState=
|
||||
LOCAL_FILES=23 LOCAL_BYTES=3888168125
|
||||
========== NODE=pve03 ==========
|
||||
bash: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory
|
||||
sha256sum: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory
|
||||
grep: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory
|
||||
Result=success
|
||||
ExecMainStatus=0
|
||||
NextElapseUSecRealtime=
|
||||
ActiveState=inactive
|
||||
UnitFileState=
|
||||
LOCAL_FILES=20 LOCAL_BYTES=47769002980
|
||||
VM_DUMP_CONSISTENCY=COMPLETE
|
||||
|
||||
OUTPUT_END
|
||||
CHAT_OUTPUT_END
|
||||
+7
-7
File diff suppressed because one or more lines are too long
+77
-246
@@ -1,5 +1,5 @@
|
||||
CHAT_OUTPUT_BEGIN
|
||||
COMMAND_ID=BACKUP-VM-DUMP-SOURCE-077
|
||||
COMMAND_ID=BACKUP-VM-DUMP-CONSISTENCY-078
|
||||
STATUS=OK
|
||||
RC=0
|
||||
HOST=pve01
|
||||
@@ -9,264 +9,95 @@ REFERENCE_REGISTER_CHECK=OK
|
||||
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
|
||||
COMMAND_SHA256=b835d1210c5520c9d8d56eac7575cc679e5d0a22ce04f879cf8af953e2bb9b9e
|
||||
COMMAND_SHA256=876c832ac6a555af28ff29ccdceb28c79e7244a361f277e9278a8f75b65caad5
|
||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||
EXECUTION_STARTED=true
|
||||
CHANGES_MADE=false
|
||||
SANITIZED=yes
|
||||
SECRETS_INCLUDED=no
|
||||
PRIVATE_ADDRESSES_INCLUDED=no
|
||||
RAW_EVIDENCE_SHA256=9dd20a7d68dce96c8f08bcb4407101b10e80a36077f51706b0ba8b709099ba58
|
||||
SANITIZED_OUTPUT_SHA256=9dd20a7d68dce96c8f08bcb4407101b10e80a36077f51706b0ba8b709099ba58
|
||||
RAW_EVIDENCE_SHA256=02a6668400353ad78630dc31cad6ae25943bc9bd92bc8aff5a46cc2f4dc89d8f
|
||||
SANITIZED_OUTPUT_SHA256=02a6668400353ad78630dc31cad6ae25943bc9bd92bc8aff5a46cc2f4dc89d8f
|
||||
OUTPUT_BEGIN
|
||||
=== SCRIPT_METADATA ===
|
||||
========== NODE=pve01 ==========
|
||||
1de52eb4ddb17b0ecff2cf3f3021e65b92e91fb44416f97943c51e248c512ed6 /usr/local/sbin/homelab-vm-local-dumps-2cloud
|
||||
File: /usr/local/sbin/homelab-vm-local-dumps-2cloud
|
||||
Size: 7272 Blocks: 16 IO Block: 4096 regular file
|
||||
Device: 252,1 Inode: 5148408 Links: 1
|
||||
Access: (0700/-rwx------) Uid: ( 0/ root) Gid: ( 0/ root)
|
||||
Access: 2026-07-21 00:05:00.471248283 +0300
|
||||
Modify: 2026-07-06 18:49:49.740645050 +0300
|
||||
Change: 2026-07-06 18:49:49.740645050 +0300
|
||||
Birth: 2026-07-06 18:27:29.152531375 +0300
|
||||
=== SCRIPT_SOURCE ===
|
||||
#!/usr/bin/env bash
|
||||
set +u
|
||||
|
||||
OUT="/var/lib/homelab-health/vm-local-dumps-cloud.txt"
|
||||
ROOT="/var/lib/homelab-vm-backup"
|
||||
LOG="$ROOT/vm-local-dumps-2cloud-latest.log"
|
||||
VERIFY_ROOT="/var/tmp/homelab-vm-local-dumps-verify"
|
||||
CHUNK_ROOT="/var/tmp/homelab-vm-local-dumps-chunks"
|
||||
CHUNK_SIZE_BYTES="${CHUNK_SIZE_BYTES:-1073741824}"
|
||||
CHUNK_SIZE_SPLIT="${CHUNK_SIZE_SPLIT:-1024M}"
|
||||
|
||||
mkdir -p /var/lib/homelab-health "$ROOT" "$VERIFY_ROOT" "$CHUNK_ROOT"
|
||||
|
||||
TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
JOB_TS="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
NODE="$(hostname -s)"
|
||||
BAD=0
|
||||
COUNT=0
|
||||
TOTAL=0
|
||||
CHUNKED=0
|
||||
VMIDS="${VMIDS:-110 112 150}"
|
||||
|
||||
source /etc/homelab-backup.env 2>/dev/null || BAD=1
|
||||
|
||||
JOB_ID="homelab-${NODE}-vm-local-dumps-${JOB_TS}"
|
||||
WORK="$ROOT/$JOB_ID"
|
||||
mkdir -p "$WORK"
|
||||
|
||||
FILES_TSV="$WORK/files.tsv"
|
||||
CHKS="$WORK/checksums.txt"
|
||||
MAN="$WORK/manifest.json"
|
||||
|
||||
: > "$FILES_TSV"
|
||||
: > "$CHKS"
|
||||
: > "$LOG"
|
||||
|
||||
echo "job_id=$JOB_ID" >> "$LOG"
|
||||
echo "vmids=$VMIDS" >> "$LOG"
|
||||
echo "chunk_size_bytes=$CHUNK_SIZE_BYTES" >> "$LOG"
|
||||
|
||||
for id in $VMIDS; do
|
||||
F="$(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-${id}-*.vma.zst" -o -name "vzdump-lxc-${id}-*.tar.zst" \) -printf "%T@ %p\n" 2>/dev/null | sort -n | tail -n 1 | cut -d" " -f2-)"
|
||||
if [ -z "$F" ] || [ ! -f "$F" ]; then
|
||||
echo "MISSING vmid=$id" >> "$LOG"
|
||||
BAD=$((BAD+1))
|
||||
continue
|
||||
fi
|
||||
|
||||
BN="$(basename "$F")"
|
||||
SIZE="$(stat -c %s "$F" 2>/dev/null || echo 0)"
|
||||
MT="$(date -u -d "@$(stat -c %Y "$F" 2>/dev/null || echo 0)" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)"
|
||||
KIND="unknown"
|
||||
echo "$BN" | grep -q "^vzdump-qemu-" && KIND="qemu"
|
||||
echo "$BN" | grep -q "^vzdump-lxc-" && KIND="lxc"
|
||||
SHA="$(sha256sum "$F" | cut -d" " -f1)"
|
||||
printf "%s %s\n" "$SHA" "$BN" >> "$CHKS"
|
||||
|
||||
MODE="direct"
|
||||
PARTS=0
|
||||
if [ "$SIZE" -gt "$CHUNK_SIZE_BYTES" ]; then
|
||||
MODE="chunked"
|
||||
CHUNKED=$((CHUNKED+1))
|
||||
CDIR="$CHUNK_ROOT/$JOB_ID/$BN.chunks"
|
||||
rm -rf "$CDIR"
|
||||
mkdir -p "$CDIR"
|
||||
echo "SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR" >> "$LOG"
|
||||
split -b "$CHUNK_SIZE_SPLIT" -d -a 4 "$F" "$CDIR/part-" >> "$LOG" 2>&1
|
||||
PARTS="$(find "$CDIR" -maxdepth 1 -type f -name 'part-*' | wc -l)"
|
||||
(cd "$CDIR" && sha256sum part-* > chunks.sha256)
|
||||
if [ "$PARTS" -le 0 ]; then
|
||||
echo "BAD split produced no chunks file=$BN" >> "$LOG"
|
||||
BAD=$((BAD+1))
|
||||
fi
|
||||
fi
|
||||
|
||||
printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" "$id" "$KIND" "$SIZE" "$SHA" "$MT" "$BN" "$F" "$MODE" "$PARTS" >> "$FILES_TSV"
|
||||
COUNT=$((COUNT+1))
|
||||
TOTAL=$((TOTAL+SIZE))
|
||||
done
|
||||
|
||||
python3 - "$JOB_ID" "$NODE" "$TS" "$VMIDS" "$COUNT" "$TOTAL" "$CHUNKED" "$CHUNK_SIZE_BYTES" "$FILES_TSV" > "$MAN" <<'PY'
|
||||
import json, sys, pathlib
|
||||
job,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:]
|
||||
items=[]
|
||||
for line in pathlib.Path(tsv).read_text().splitlines():
|
||||
p=line.split("\t")
|
||||
if len(p)>=9:
|
||||
items.append({
|
||||
"vmid":p[0],
|
||||
"kind":p[1],
|
||||
"size_bytes":int(p[2]),
|
||||
"sha256":p[3],
|
||||
"mtime_utc":p[4],
|
||||
"filename":p[5],
|
||||
"source_path":p[6],
|
||||
"mode":p[7],
|
||||
"parts":int(p[8]),
|
||||
})
|
||||
print(json.dumps({
|
||||
"job_id":job,
|
||||
"node":node,
|
||||
"generated_utc":ts,
|
||||
"vmids":vmids.split(),
|
||||
"count":int(count),
|
||||
"total_bytes":int(total),
|
||||
"chunked_count":int(chunked),
|
||||
"chunk_size_bytes":int(chunk_size),
|
||||
"items":items,
|
||||
}, indent=2))
|
||||
PY
|
||||
|
||||
DEST_A="${REMOTE_A%/}/${NODE}/vm-local-dumps/${JOB_ID}"
|
||||
DEST_B="${REMOTE_B%/}/${NODE}/vm-local-dumps/${JOB_ID}"
|
||||
|
||||
upload_one_dest() {
|
||||
DEST="$1"
|
||||
rclone mkdir "$DEST/files" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do
|
||||
if [ "$mode" = "chunked" ]; then
|
||||
CDIR="$CHUNK_ROOT/$JOB_ID/$bn.chunks"
|
||||
echo "UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts" >> "$LOG"
|
||||
rclone mkdir "$DEST/files/$bn.chunks" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
for part in "$CDIR"/part-*; do
|
||||
[ -f "$part" ] || continue
|
||||
pn="$(basename "$part")"
|
||||
rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
done
|
||||
rclone copyto "$CDIR/chunks.sha256" "$DEST/files/$bn.chunks/chunks.sha256" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
else
|
||||
echo "UPLOAD_DIRECT dest=$DEST file=$bn size=$size" >> "$LOG"
|
||||
rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
fi
|
||||
done < "$FILES_TSV"
|
||||
|
||||
rclone copyto "$CHKS" "$DEST/checksums.txt" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
rclone copyto "$FILES_TSV" "$DEST/files.tsv" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
}
|
||||
|
||||
verify_one_dest() {
|
||||
LABEL="$1"
|
||||
DEST="$2"
|
||||
VBASE="$VERIFY_ROOT/$JOB_ID/$LABEL"
|
||||
rm -rf "$VBASE"
|
||||
mkdir -p "$VBASE"
|
||||
rclone copyto "$DEST/manifest.json" "$VBASE/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
test -s "$VBASE/manifest.json" || BAD=$((BAD+1))
|
||||
|
||||
while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do
|
||||
if [ "$mode" = "chunked" ]; then
|
||||
VDIR="$VBASE/$bn.chunks"
|
||||
mkdir -p "$VDIR"
|
||||
echo "VERIFY_CHUNKED label=$LABEL file=$bn parts=$parts" >> "$LOG"
|
||||
rclone copy "$DEST/files/$bn.chunks" "$VDIR" --filter "+ part-*" --filter "+ chunks.sha256" --filter "- *" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
(cd "$VDIR" && sha256sum -c chunks.sha256) >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
CALC="$(find "$VDIR" -maxdepth 1 -type f -name 'part-*' | sort | xargs cat | sha256sum | cut -d" " -f1)"
|
||||
if [ "$CALC" = "$sha" ]; then
|
||||
echo "OK combined_sha label=$LABEL file=$bn sha=$sha" >> "$LOG"
|
||||
else
|
||||
echo "BAD combined_sha label=$LABEL file=$bn expected=$sha got=$CALC" >> "$LOG"
|
||||
BAD=$((BAD+1))
|
||||
fi
|
||||
rm -rf "$VDIR"
|
||||
else
|
||||
echo "VERIFY_DIRECT label=$LABEL file=$bn" >> "$LOG"
|
||||
rclone copyto "$DEST/files/$bn" "$VBASE/$bn" --transfers 1 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
printf "%s %s\n" "$sha" "$VBASE/$bn" | sha256sum -c - >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
rm -f "$VBASE/$bn"
|
||||
fi
|
||||
done < "$FILES_TSV"
|
||||
rm -rf "$VBASE"
|
||||
}
|
||||
|
||||
if [ "$BAD" -eq 0 ]; then
|
||||
upload_one_dest "$DEST_A"
|
||||
upload_one_dest "$DEST_B"
|
||||
fi
|
||||
|
||||
if [ "$BAD" -eq 0 ]; then
|
||||
verify_one_dest "A" "$DEST_A"
|
||||
verify_one_dest "B" "$DEST_B"
|
||||
fi
|
||||
|
||||
rm -rf "$VERIFY_ROOT/$JOB_ID"
|
||||
rm -rf "$CHUNK_ROOT/$JOB_ID"
|
||||
|
||||
if [ "$BAD" -eq 0 ]; then STATUS="OK"; else STATUS="WARN"; fi
|
||||
|
||||
jq -n \
|
||||
--arg job_id "$JOB_ID" \
|
||||
--arg status "$STATUS" \
|
||||
--arg node "$NODE" \
|
||||
--arg remote_a "$DEST_A" \
|
||||
--arg remote_b "$DEST_B" \
|
||||
--argjson count "$COUNT" \
|
||||
--argjson total_bytes "$TOTAL" \
|
||||
--argjson chunked_count "$CHUNKED" \
|
||||
--argjson chunk_size_bytes "$CHUNK_SIZE_BYTES" \
|
||||
--arg log "$LOG" \
|
||||
"{job_id:\$job_id,status:\$status,node:\$node,remote_a:\$remote_a,remote_b:\$remote_b,count:\$count,total_bytes:\$total_bytes,chunked_count:\$chunked_count,chunk_size_bytes:\$chunk_size_bytes,log:\$log,local_dumps_deleted:false}" \
|
||||
> "$ROOT/latest-vm-local-dumps-cloud.json"
|
||||
|
||||
echo "STATUS=$STATUS TS=$TS TYPE=vm-local-dumps-cloud BAD=$BAD COUNT=$COUNT CHUNKED=$CHUNKED TOTAL_BYTES=$TOTAL JOB=$JOB_ID REMOTE_A=$DEST_A REMOTE_B=$DEST_B LOG=$LOG" > "$OUT"
|
||||
cat "$OUT"
|
||||
=== SERVICE ===
|
||||
# /etc/systemd/system/homelab-vm-local-dumps-2cloud.service
|
||||
[Unit]
|
||||
Description=Homelab VM/CT existing local dumps two-cloud verifier
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/homelab-vm-local-dumps-2cloud
|
||||
|
||||
# /etc/systemd/system/homelab-vm-local-dumps-2cloud.service.d/90-backup-matrix-refresh.conf
|
||||
[Service]
|
||||
ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service
|
||||
30:CHKS="$WORK/checksums.txt"
|
||||
39:echo "chunk_size_bytes=$CHUNK_SIZE_BYTES" >> "$LOG"
|
||||
50: SIZE="$(stat -c %s "$F" 2>/dev/null || echo 0)"
|
||||
51: MT="$(date -u -d "@$(stat -c %Y "$F" 2>/dev/null || echo 0)" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)"
|
||||
55: SHA="$(sha256sum "$F" | cut -d" " -f1)"
|
||||
64: rm -rf "$CDIR"
|
||||
66: echo "SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR" >> "$LOG"
|
||||
69: (cd "$CDIR" && sha256sum part-* > chunks.sha256)
|
||||
83:job,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:]
|
||||
91: "size_bytes":int(p[2]),
|
||||
92: "sha256":p[3],
|
||||
107: "chunk_size_bytes":int(chunk_size),
|
||||
112:DEST_A="${REMOTE_A%/}/${NODE}/vm-local-dumps/${JOB_ID}"
|
||||
113:DEST_B="${REMOTE_B%/}/${NODE}/vm-local-dumps/${JOB_ID}"
|
||||
116: DEST="$1"
|
||||
117: rclone mkdir "$DEST/files" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
118: while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do
|
||||
121: echo "UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts" >> "$LOG"
|
||||
122: rclone mkdir "$DEST/files/$bn.chunks" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
126: rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
128: rclone copyto "$CDIR/chunks.sha256" "$DEST/files/$bn.chunks/chunks.sha256" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
130: echo "UPLOAD_DIRECT dest=$DEST file=$bn size=$size" >> "$LOG"
|
||||
131: rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
135: rclone copyto "$CHKS" "$DEST/checksums.txt" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
136: rclone copyto "$FILES_TSV" "$DEST/files.tsv" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
137: rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
142: DEST="$2"
|
||||
144: rm -rf "$VBASE"
|
||||
146: rclone copyto "$DEST/manifest.json" "$VBASE/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
149: while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do
|
||||
154: rclone copy "$DEST/files/$bn.chunks" "$VDIR" --filter "+ part-*" --filter "+ chunks.sha256" --filter "- *" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
155: (cd "$VDIR" && sha256sum -c chunks.sha256) >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
156: CALC="$(find "$VDIR" -maxdepth 1 -type f -name 'part-*' | sort | xargs cat | sha256sum | cut -d" " -f1)"
|
||||
163: rm -rf "$VDIR"
|
||||
166: rclone copyto "$DEST/files/$bn" "$VBASE/$bn" --transfers 1 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
167: printf "%s %s\n" "$sha" "$VBASE/$bn" | sha256sum -c - >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
168: rm -f "$VBASE/$bn"
|
||||
171: rm -rf "$VBASE"
|
||||
175: upload_one_dest "$DEST_A"
|
||||
176: upload_one_dest "$DEST_B"
|
||||
180: verify_one_dest "A" "$DEST_A"
|
||||
181: verify_one_dest "B" "$DEST_B"
|
||||
184:rm -rf "$VERIFY_ROOT/$JOB_ID"
|
||||
185:rm -rf "$CHUNK_ROOT/$JOB_ID"
|
||||
191: --arg status "$STATUS" \
|
||||
193: --arg remote_a "$DEST_A" \
|
||||
194: --arg remote_b "$DEST_B" \
|
||||
198: --argjson chunk_size_bytes "$CHUNK_SIZE_BYTES" \
|
||||
200: "{job_id:\$job_id,status:\$status,node:\$node,remote_a:\$remote_a,remote_b:\$remote_b,count:\$count,total_bytes:\$total_bytes,chunked_count:\$chunked_count,chunk_size_bytes:\$chunk_size_bytes,log:\$log,local_dumps_deleted:false}" \
|
||||
203:echo "STATUS=$STATUS TS=$TS TYPE=vm-local-dumps-cloud BAD=$BAD COUNT=$COUNT CHUNKED=$CHUNKED TOTAL_BYTES=$TOTAL JOB=$JOB_ID REMOTE_A=$DEST_A REMOTE_B=$DEST_B LOG=$LOG" > "$OUT"
|
||||
Result=success
|
||||
ExecMainStatus=0
|
||||
ExecStart={ path=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; argv[]=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
|
||||
ActiveState=inactive
|
||||
SubState=dead
|
||||
=== TIMER ===
|
||||
# /etc/systemd/system/homelab-vm-local-dumps-2cloud.timer
|
||||
[Unit]
|
||||
Description=Homelab VM/CT existing local dumps two-cloud verifier timer
|
||||
|
||||
[Timer]
|
||||
OnCalendar=Sun *-*-* 10:00:00
|
||||
RandomizedDelaySec=30m
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
NextElapseUSecRealtime=Sun 2026-07-26 10:29:15 MSK
|
||||
ActiveState=active
|
||||
UnitFileState=enabled
|
||||
LOCAL_FILES=24 LOCAL_BYTES=30273877574
|
||||
========== NODE=pve02 ==========
|
||||
bash: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory
|
||||
sha256sum: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory
|
||||
grep: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory
|
||||
Result=success
|
||||
ExecMainStatus=0
|
||||
NextElapseUSecRealtime=
|
||||
ActiveState=inactive
|
||||
UnitFileState=
|
||||
LOCAL_FILES=23 LOCAL_BYTES=3888168125
|
||||
========== NODE=pve03 ==========
|
||||
bash: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory
|
||||
sha256sum: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory
|
||||
grep: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory
|
||||
Result=success
|
||||
ExecMainStatus=0
|
||||
NextElapseUSecRealtime=
|
||||
ActiveState=inactive
|
||||
UnitFileState=
|
||||
LOCAL_FILES=20 LOCAL_BYTES=47769002980
|
||||
VM_DUMP_CONSISTENCY=COMPLETE
|
||||
|
||||
OUTPUT_END
|
||||
CHAT_OUTPUT_END
|
||||
|
||||
Reference in New Issue
Block a user