diff --git a/runtime/history/BACKUP-VM-DUMP-SOURCE-077.json b/runtime/history/BACKUP-VM-DUMP-SOURCE-077.json new file mode 100644 index 0000000..9a7524d --- /dev/null +++ b/runtime/history/BACKUP-VM-DUMP-SOURCE-077.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "BACKUP-VM-DUMP-SOURCE-077", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "backup-audit", + "started_at_utc": "2026-07-21T12:47:44Z", + "finished_at_utc": "2026-07-21T12:47:44Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", + "command_sha256": "b835d1210c5520c9d8d56eac7575cc679e5d0a22ce04f879cf8af953e2bb9b9e", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "9dd20a7d68dce96c8f08bcb4407101b10e80a36077f51706b0ba8b709099ba58", + "sanitized_output_sha256": "9dd20a7d68dce96c8f08bcb4407101b10e80a36077f51706b0ba8b709099ba58", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "=== SCRIPT_METADATA ===\n1de52eb4ddb17b0ecff2cf3f3021e65b92e91fb44416f97943c51e248c512ed6 /usr/local/sbin/homelab-vm-local-dumps-2cloud\n File: /usr/local/sbin/homelab-vm-local-dumps-2cloud\n Size: 7272 \tBlocks: 16 IO Block: 4096 regular file\nDevice: 252,1\tInode: 5148408 Links: 1\nAccess: (0700/-rwx------) Uid: ( 0/ root) Gid: ( 0/ root)\nAccess: 2026-07-21 00:05:00.471248283 +0300\nModify: 2026-07-06 18:49:49.740645050 +0300\nChange: 2026-07-06 18:49:49.740645050 +0300\n Birth: 2026-07-06 18:27:29.152531375 +0300\n=== SCRIPT_SOURCE ===\n#!/usr/bin/env bash\nset +u\n\nOUT=\"/var/lib/homelab-health/vm-local-dumps-cloud.txt\"\nROOT=\"/var/lib/homelab-vm-backup\"\nLOG=\"$ROOT/vm-local-dumps-2cloud-latest.log\"\nVERIFY_ROOT=\"/var/tmp/homelab-vm-local-dumps-verify\"\nCHUNK_ROOT=\"/var/tmp/homelab-vm-local-dumps-chunks\"\nCHUNK_SIZE_BYTES=\"${CHUNK_SIZE_BYTES:-1073741824}\"\nCHUNK_SIZE_SPLIT=\"${CHUNK_SIZE_SPLIT:-1024M}\"\n\nmkdir -p /var/lib/homelab-health \"$ROOT\" \"$VERIFY_ROOT\" \"$CHUNK_ROOT\"\n\nTS=\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"\nJOB_TS=\"$(date -u +%Y%m%dT%H%M%SZ)\"\nNODE=\"$(hostname -s)\"\nBAD=0\nCOUNT=0\nTOTAL=0\nCHUNKED=0\nVMIDS=\"${VMIDS:-110 112 150}\"\n\nsource /etc/homelab-backup.env 2>/dev/null || BAD=1\n\nJOB_ID=\"homelab-${NODE}-vm-local-dumps-${JOB_TS}\"\nWORK=\"$ROOT/$JOB_ID\"\nmkdir -p \"$WORK\"\n\nFILES_TSV=\"$WORK/files.tsv\"\nCHKS=\"$WORK/checksums.txt\"\nMAN=\"$WORK/manifest.json\"\n\n: > \"$FILES_TSV\"\n: > \"$CHKS\"\n: > \"$LOG\"\n\necho \"job_id=$JOB_ID\" >> \"$LOG\"\necho \"vmids=$VMIDS\" >> \"$LOG\"\necho \"chunk_size_bytes=$CHUNK_SIZE_BYTES\" >> \"$LOG\"\n\nfor id in $VMIDS; do\n F=\"$(find /var/lib/vz/dump -maxdepth 1 -type f \\( -name \"vzdump-qemu-${id}-*.vma.zst\" -o -name \"vzdump-lxc-${id}-*.tar.zst\" \\) -printf \"%T@ %p\\n\" 2>/dev/null | sort -n | tail -n 1 | cut -d\" \" -f2-)\"\n if [ -z \"$F\" ] || [ ! -f \"$F\" ]; then\n echo \"MISSING vmid=$id\" >> \"$LOG\"\n BAD=$((BAD+1))\n continue\n fi\n\n BN=\"$(basename \"$F\")\"\n SIZE=\"$(stat -c %s \"$F\" 2>/dev/null || echo 0)\"\n MT=\"$(date -u -d \"@$(stat -c %Y \"$F\" 2>/dev/null || echo 0)\" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)\"\n KIND=\"unknown\"\n echo \"$BN\" | grep -q \"^vzdump-qemu-\" && KIND=\"qemu\"\n echo \"$BN\" | grep -q \"^vzdump-lxc-\" && KIND=\"lxc\"\n SHA=\"$(sha256sum \"$F\" | cut -d\" \" -f1)\"\n printf \"%s %s\\n\" \"$SHA\" \"$BN\" >> \"$CHKS\"\n\n MODE=\"direct\"\n PARTS=0\n if [ \"$SIZE\" -gt \"$CHUNK_SIZE_BYTES\" ]; then\n MODE=\"chunked\"\n CHUNKED=$((CHUNKED+1))\n CDIR=\"$CHUNK_ROOT/$JOB_ID/$BN.chunks\"\n rm -rf \"$CDIR\"\n mkdir -p \"$CDIR\"\n echo \"SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR\" >> \"$LOG\"\n split -b \"$CHUNK_SIZE_SPLIT\" -d -a 4 \"$F\" \"$CDIR/part-\" >> \"$LOG\" 2>&1\n PARTS=\"$(find \"$CDIR\" -maxdepth 1 -type f -name 'part-*' | wc -l)\"\n (cd \"$CDIR\" && sha256sum part-* > chunks.sha256)\n if [ \"$PARTS\" -le 0 ]; then\n echo \"BAD split produced no chunks file=$BN\" >> \"$LOG\"\n BAD=$((BAD+1))\n fi\n fi\n\n printf \"%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n\" \"$id\" \"$KIND\" \"$SIZE\" \"$SHA\" \"$MT\" \"$BN\" \"$F\" \"$MODE\" \"$PARTS\" >> \"$FILES_TSV\"\n COUNT=$((COUNT+1))\n TOTAL=$((TOTAL+SIZE))\ndone\n\npython3 - \"$JOB_ID\" \"$NODE\" \"$TS\" \"$VMIDS\" \"$COUNT\" \"$TOTAL\" \"$CHUNKED\" \"$CHUNK_SIZE_BYTES\" \"$FILES_TSV\" > \"$MAN\" <<'PY'\nimport json, sys, pathlib\njob,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:]\nitems=[]\nfor line in pathlib.Path(tsv).read_text().splitlines():\n p=line.split(\"\\t\")\n if len(p)>=9:\n items.append({\n \"vmid\":p[0],\n \"kind\":p[1],\n \"size_bytes\":int(p[2]),\n \"sha256\":p[3],\n \"mtime_utc\":p[4],\n \"filename\":p[5],\n \"source_path\":p[6],\n \"mode\":p[7],\n \"parts\":int(p[8]),\n })\nprint(json.dumps({\n \"job_id\":job,\n \"node\":node,\n \"generated_utc\":ts,\n \"vmids\":vmids.split(),\n \"count\":int(count),\n \"total_bytes\":int(total),\n \"chunked_count\":int(chunked),\n \"chunk_size_bytes\":int(chunk_size),\n \"items\":items,\n}, indent=2))\nPY\n\nDEST_A=\"${REMOTE_A%/}/${NODE}/vm-local-dumps/${JOB_ID}\"\nDEST_B=\"${REMOTE_B%/}/${NODE}/vm-local-dumps/${JOB_ID}\"\n\nupload_one_dest() {\n DEST=\"$1\"\n rclone mkdir \"$DEST/files\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n while IFS=\"$(printf \"\\t\")\" read -r id kind size sha mt bn path mode parts; do\n if [ \"$mode\" = \"chunked\" ]; then\n CDIR=\"$CHUNK_ROOT/$JOB_ID/$bn.chunks\"\n echo \"UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts\" >> \"$LOG\"\n rclone mkdir \"$DEST/files/$bn.chunks\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n for part in \"$CDIR\"/part-*; do\n [ -f \"$part\" ] || continue\n pn=\"$(basename \"$part\")\"\n rclone copyto \"$part\" \"$DEST/files/$bn.chunks/$pn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n done\n rclone copyto \"$CDIR/chunks.sha256\" \"$DEST/files/$bn.chunks/chunks.sha256\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n else\n echo \"UPLOAD_DIRECT dest=$DEST file=$bn size=$size\" >> \"$LOG\"\n rclone copyto \"$path\" \"$DEST/files/$bn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n fi\n done < \"$FILES_TSV\"\n\n rclone copyto \"$CHKS\" \"$DEST/checksums.txt\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n rclone copyto \"$FILES_TSV\" \"$DEST/files.tsv\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n rclone copyto \"$MAN\" \"$DEST/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n}\n\nverify_one_dest() {\n LABEL=\"$1\"\n DEST=\"$2\"\n VBASE=\"$VERIFY_ROOT/$JOB_ID/$LABEL\"\n rm -rf \"$VBASE\"\n mkdir -p \"$VBASE\"\n rclone copyto \"$DEST/manifest.json\" \"$VBASE/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n test -s \"$VBASE/manifest.json\" || BAD=$((BAD+1))\n\n while IFS=\"$(printf \"\\t\")\" read -r id kind size sha mt bn path mode parts; do\n if [ \"$mode\" = \"chunked\" ]; then\n VDIR=\"$VBASE/$bn.chunks\"\n mkdir -p \"$VDIR\"\n echo \"VERIFY_CHUNKED label=$LABEL file=$bn parts=$parts\" >> \"$LOG\"\n rclone copy \"$DEST/files/$bn.chunks\" \"$VDIR\" --filter \"+ part-*\" --filter \"+ chunks.sha256\" --filter \"- *\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n (cd \"$VDIR\" && sha256sum -c chunks.sha256) >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n CALC=\"$(find \"$VDIR\" -maxdepth 1 -type f -name 'part-*' | sort | xargs cat | sha256sum | cut -d\" \" -f1)\"\n if [ \"$CALC\" = \"$sha\" ]; then\n echo \"OK combined_sha label=$LABEL file=$bn sha=$sha\" >> \"$LOG\"\n else\n echo \"BAD combined_sha label=$LABEL file=$bn expected=$sha got=$CALC\" >> \"$LOG\"\n BAD=$((BAD+1))\n fi\n rm -rf \"$VDIR\"\n else\n echo \"VERIFY_DIRECT label=$LABEL file=$bn\" >> \"$LOG\"\n rclone copyto \"$DEST/files/$bn\" \"$VBASE/$bn\" --transfers 1 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n printf \"%s %s\\n\" \"$sha\" \"$VBASE/$bn\" | sha256sum -c - >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n rm -f \"$VBASE/$bn\"\n fi\n done < \"$FILES_TSV\"\n rm -rf \"$VBASE\"\n}\n\nif [ \"$BAD\" -eq 0 ]; then\n upload_one_dest \"$DEST_A\"\n upload_one_dest \"$DEST_B\"\nfi\n\nif [ \"$BAD\" -eq 0 ]; then\n verify_one_dest \"A\" \"$DEST_A\"\n verify_one_dest \"B\" \"$DEST_B\"\nfi\n\nrm -rf \"$VERIFY_ROOT/$JOB_ID\"\nrm -rf \"$CHUNK_ROOT/$JOB_ID\"\n\nif [ \"$BAD\" -eq 0 ]; then STATUS=\"OK\"; else STATUS=\"WARN\"; fi\n\njq -n \\\n --arg job_id \"$JOB_ID\" \\\n --arg status \"$STATUS\" \\\n --arg node \"$NODE\" \\\n --arg remote_a \"$DEST_A\" \\\n --arg remote_b \"$DEST_B\" \\\n --argjson count \"$COUNT\" \\\n --argjson total_bytes \"$TOTAL\" \\\n --argjson chunked_count \"$CHUNKED\" \\\n --argjson chunk_size_bytes \"$CHUNK_SIZE_BYTES\" \\\n --arg log \"$LOG\" \\\n \"{job_id:\\$job_id,status:\\$status,node:\\$node,remote_a:\\$remote_a,remote_b:\\$remote_b,count:\\$count,total_bytes:\\$total_bytes,chunked_count:\\$chunked_count,chunk_size_bytes:\\$chunk_size_bytes,log:\\$log,local_dumps_deleted:false}\" \\\n > \"$ROOT/latest-vm-local-dumps-cloud.json\"\n\necho \"STATUS=$STATUS TS=$TS TYPE=vm-local-dumps-cloud BAD=$BAD COUNT=$COUNT CHUNKED=$CHUNKED TOTAL_BYTES=$TOTAL JOB=$JOB_ID REMOTE_A=$DEST_A REMOTE_B=$DEST_B LOG=$LOG\" > \"$OUT\"\ncat \"$OUT\"\n=== SERVICE ===\n# /etc/systemd/system/homelab-vm-local-dumps-2cloud.service\n[Unit]\nDescription=Homelab VM/CT existing local dumps two-cloud verifier\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-vm-local-dumps-2cloud\n\n# /etc/systemd/system/homelab-vm-local-dumps-2cloud.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; argv[]=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\nSubState=dead\n=== TIMER ===\n# /etc/systemd/system/homelab-vm-local-dumps-2cloud.timer\n[Unit]\nDescription=Homelab VM/CT existing local dumps two-cloud verifier timer\n\n[Timer]\nOnCalendar=Sun *-*-* 10:00:00\nRandomizedDelaySec=30m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\nNextElapseUSecRealtime=Sun 2026-07-26 10:29:15 MSK\nActiveState=active\nUnitFileState=enabled\n" +} diff --git a/runtime/history/BACKUP-VM-DUMP-SOURCE-077.txt b/runtime/history/BACKUP-VM-DUMP-SOURCE-077.txt new file mode 100644 index 0000000..6599c4b --- /dev/null +++ b/runtime/history/BACKUP-VM-DUMP-SOURCE-077.txt @@ -0,0 +1,272 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=BACKUP-VM-DUMP-SOURCE-077 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=backup-audit +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 +COMMAND_SHA256=b835d1210c5520c9d8d56eac7575cc679e5d0a22ce04f879cf8af953e2bb9b9e +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=false +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=9dd20a7d68dce96c8f08bcb4407101b10e80a36077f51706b0ba8b709099ba58 +SANITIZED_OUTPUT_SHA256=9dd20a7d68dce96c8f08bcb4407101b10e80a36077f51706b0ba8b709099ba58 +OUTPUT_BEGIN +=== SCRIPT_METADATA === +1de52eb4ddb17b0ecff2cf3f3021e65b92e91fb44416f97943c51e248c512ed6 /usr/local/sbin/homelab-vm-local-dumps-2cloud + File: /usr/local/sbin/homelab-vm-local-dumps-2cloud + Size: 7272 Blocks: 16 IO Block: 4096 regular file +Device: 252,1 Inode: 5148408 Links: 1 +Access: (0700/-rwx------) Uid: ( 0/ root) Gid: ( 0/ root) +Access: 2026-07-21 00:05:00.471248283 +0300 +Modify: 2026-07-06 18:49:49.740645050 +0300 +Change: 2026-07-06 18:49:49.740645050 +0300 + Birth: 2026-07-06 18:27:29.152531375 +0300 +=== SCRIPT_SOURCE === +#!/usr/bin/env bash +set +u + +OUT="/var/lib/homelab-health/vm-local-dumps-cloud.txt" +ROOT="/var/lib/homelab-vm-backup" +LOG="$ROOT/vm-local-dumps-2cloud-latest.log" +VERIFY_ROOT="/var/tmp/homelab-vm-local-dumps-verify" +CHUNK_ROOT="/var/tmp/homelab-vm-local-dumps-chunks" +CHUNK_SIZE_BYTES="${CHUNK_SIZE_BYTES:-1073741824}" +CHUNK_SIZE_SPLIT="${CHUNK_SIZE_SPLIT:-1024M}" + +mkdir -p /var/lib/homelab-health "$ROOT" "$VERIFY_ROOT" "$CHUNK_ROOT" + +TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +JOB_TS="$(date -u +%Y%m%dT%H%M%SZ)" +NODE="$(hostname -s)" +BAD=0 +COUNT=0 +TOTAL=0 +CHUNKED=0 +VMIDS="${VMIDS:-110 112 150}" + +source /etc/homelab-backup.env 2>/dev/null || BAD=1 + +JOB_ID="homelab-${NODE}-vm-local-dumps-${JOB_TS}" +WORK="$ROOT/$JOB_ID" +mkdir -p "$WORK" + +FILES_TSV="$WORK/files.tsv" +CHKS="$WORK/checksums.txt" +MAN="$WORK/manifest.json" + +: > "$FILES_TSV" +: > "$CHKS" +: > "$LOG" + +echo "job_id=$JOB_ID" >> "$LOG" +echo "vmids=$VMIDS" >> "$LOG" +echo "chunk_size_bytes=$CHUNK_SIZE_BYTES" >> "$LOG" + +for id in $VMIDS; do + F="$(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-${id}-*.vma.zst" -o -name "vzdump-lxc-${id}-*.tar.zst" \) -printf "%T@ %p\n" 2>/dev/null | sort -n | tail -n 1 | cut -d" " -f2-)" + if [ -z "$F" ] || [ ! -f "$F" ]; then + echo "MISSING vmid=$id" >> "$LOG" + BAD=$((BAD+1)) + continue + fi + + BN="$(basename "$F")" + SIZE="$(stat -c %s "$F" 2>/dev/null || echo 0)" + MT="$(date -u -d "@$(stat -c %Y "$F" 2>/dev/null || echo 0)" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)" + KIND="unknown" + echo "$BN" | grep -q "^vzdump-qemu-" && KIND="qemu" + echo "$BN" | grep -q "^vzdump-lxc-" && KIND="lxc" + SHA="$(sha256sum "$F" | cut -d" " -f1)" + printf "%s %s\n" "$SHA" "$BN" >> "$CHKS" + + MODE="direct" + PARTS=0 + if [ "$SIZE" -gt "$CHUNK_SIZE_BYTES" ]; then + MODE="chunked" + CHUNKED=$((CHUNKED+1)) + CDIR="$CHUNK_ROOT/$JOB_ID/$BN.chunks" + rm -rf "$CDIR" + mkdir -p "$CDIR" + echo "SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR" >> "$LOG" + split -b "$CHUNK_SIZE_SPLIT" -d -a 4 "$F" "$CDIR/part-" >> "$LOG" 2>&1 + PARTS="$(find "$CDIR" -maxdepth 1 -type f -name 'part-*' | wc -l)" + (cd "$CDIR" && sha256sum part-* > chunks.sha256) + if [ "$PARTS" -le 0 ]; then + echo "BAD split produced no chunks file=$BN" >> "$LOG" + BAD=$((BAD+1)) + fi + fi + + printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" "$id" "$KIND" "$SIZE" "$SHA" "$MT" "$BN" "$F" "$MODE" "$PARTS" >> "$FILES_TSV" + COUNT=$((COUNT+1)) + TOTAL=$((TOTAL+SIZE)) +done + +python3 - "$JOB_ID" "$NODE" "$TS" "$VMIDS" "$COUNT" "$TOTAL" "$CHUNKED" "$CHUNK_SIZE_BYTES" "$FILES_TSV" > "$MAN" <<'PY' +import json, sys, pathlib +job,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:] +items=[] +for line in pathlib.Path(tsv).read_text().splitlines(): + p=line.split("\t") + if len(p)>=9: + items.append({ + "vmid":p[0], + "kind":p[1], + "size_bytes":int(p[2]), + "sha256":p[3], + "mtime_utc":p[4], + "filename":p[5], + "source_path":p[6], + "mode":p[7], + "parts":int(p[8]), + }) +print(json.dumps({ + "job_id":job, + "node":node, + "generated_utc":ts, + "vmids":vmids.split(), + "count":int(count), + "total_bytes":int(total), + "chunked_count":int(chunked), + "chunk_size_bytes":int(chunk_size), + "items":items, +}, indent=2)) +PY + +DEST_A="${REMOTE_A%/}/${NODE}/vm-local-dumps/${JOB_ID}" +DEST_B="${REMOTE_B%/}/${NODE}/vm-local-dumps/${JOB_ID}" + +upload_one_dest() { + DEST="$1" + rclone mkdir "$DEST/files" >> "$LOG" 2>&1 || BAD=$((BAD+1)) + while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do + if [ "$mode" = "chunked" ]; then + CDIR="$CHUNK_ROOT/$JOB_ID/$bn.chunks" + echo "UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts" >> "$LOG" + rclone mkdir "$DEST/files/$bn.chunks" >> "$LOG" 2>&1 || BAD=$((BAD+1)) + for part in "$CDIR"/part-*; do + [ -f "$part" ] || continue + pn="$(basename "$part")" + rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) + done + rclone copyto "$CDIR/chunks.sha256" "$DEST/files/$bn.chunks/chunks.sha256" >> "$LOG" 2>&1 || BAD=$((BAD+1)) + else + echo "UPLOAD_DIRECT dest=$DEST file=$bn size=$size" >> "$LOG" + rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) + fi + done < "$FILES_TSV" + + rclone copyto "$CHKS" "$DEST/checksums.txt" >> "$LOG" 2>&1 || BAD=$((BAD+1)) + rclone copyto "$FILES_TSV" "$DEST/files.tsv" >> "$LOG" 2>&1 || BAD=$((BAD+1)) + rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +} + +verify_one_dest() { + LABEL="$1" + DEST="$2" + VBASE="$VERIFY_ROOT/$JOB_ID/$LABEL" + rm -rf "$VBASE" + mkdir -p "$VBASE" + rclone copyto "$DEST/manifest.json" "$VBASE/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1)) + test -s "$VBASE/manifest.json" || BAD=$((BAD+1)) + + while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do + if [ "$mode" = "chunked" ]; then + VDIR="$VBASE/$bn.chunks" + mkdir -p "$VDIR" + echo "VERIFY_CHUNKED label=$LABEL file=$bn parts=$parts" >> "$LOG" + rclone copy "$DEST/files/$bn.chunks" "$VDIR" --filter "+ part-*" --filter "+ chunks.sha256" --filter "- *" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) + (cd "$VDIR" && sha256sum -c chunks.sha256) >> "$LOG" 2>&1 || BAD=$((BAD+1)) + CALC="$(find "$VDIR" -maxdepth 1 -type f -name 'part-*' | sort | xargs cat | sha256sum | cut -d" " -f1)" + if [ "$CALC" = "$sha" ]; then + echo "OK combined_sha label=$LABEL file=$bn sha=$sha" >> "$LOG" + else + echo "BAD combined_sha label=$LABEL file=$bn expected=$sha got=$CALC" >> "$LOG" + BAD=$((BAD+1)) + fi + rm -rf "$VDIR" + else + echo "VERIFY_DIRECT label=$LABEL file=$bn" >> "$LOG" + rclone copyto "$DEST/files/$bn" "$VBASE/$bn" --transfers 1 >> "$LOG" 2>&1 || BAD=$((BAD+1)) + printf "%s %s\n" "$sha" "$VBASE/$bn" | sha256sum -c - >> "$LOG" 2>&1 || BAD=$((BAD+1)) + rm -f "$VBASE/$bn" + fi + done < "$FILES_TSV" + rm -rf "$VBASE" +} + +if [ "$BAD" -eq 0 ]; then + upload_one_dest "$DEST_A" + upload_one_dest "$DEST_B" +fi + +if [ "$BAD" -eq 0 ]; then + verify_one_dest "A" "$DEST_A" + verify_one_dest "B" "$DEST_B" +fi + +rm -rf "$VERIFY_ROOT/$JOB_ID" +rm -rf "$CHUNK_ROOT/$JOB_ID" + +if [ "$BAD" -eq 0 ]; then STATUS="OK"; else STATUS="WARN"; fi + +jq -n \ + --arg job_id "$JOB_ID" \ + --arg status "$STATUS" \ + --arg node "$NODE" \ + --arg remote_a "$DEST_A" \ + --arg remote_b "$DEST_B" \ + --argjson count "$COUNT" \ + --argjson total_bytes "$TOTAL" \ + --argjson chunked_count "$CHUNKED" \ + --argjson chunk_size_bytes "$CHUNK_SIZE_BYTES" \ + --arg log "$LOG" \ + "{job_id:\$job_id,status:\$status,node:\$node,remote_a:\$remote_a,remote_b:\$remote_b,count:\$count,total_bytes:\$total_bytes,chunked_count:\$chunked_count,chunk_size_bytes:\$chunk_size_bytes,log:\$log,local_dumps_deleted:false}" \ + > "$ROOT/latest-vm-local-dumps-cloud.json" + +echo "STATUS=$STATUS TS=$TS TYPE=vm-local-dumps-cloud BAD=$BAD COUNT=$COUNT CHUNKED=$CHUNKED TOTAL_BYTES=$TOTAL JOB=$JOB_ID REMOTE_A=$DEST_A REMOTE_B=$DEST_B LOG=$LOG" > "$OUT" +cat "$OUT" +=== SERVICE === +# /etc/systemd/system/homelab-vm-local-dumps-2cloud.service +[Unit] +Description=Homelab VM/CT existing local dumps two-cloud verifier +After=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/homelab-vm-local-dumps-2cloud + +# /etc/systemd/system/homelab-vm-local-dumps-2cloud.service.d/90-backup-matrix-refresh.conf +[Service] +ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service +Result=success +ExecMainStatus=0 +ExecStart={ path=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; argv[]=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } +ActiveState=inactive +SubState=dead +=== TIMER === +# /etc/systemd/system/homelab-vm-local-dumps-2cloud.timer +[Unit] +Description=Homelab VM/CT existing local dumps two-cloud verifier timer + +[Timer] +OnCalendar=Sun *-*-* 10:00:00 +RandomizedDelaySec=30m +Persistent=true + +[Install] +WantedBy=timers.target +NextElapseUSecRealtime=Sun 2026-07-26 10:29:15 MSK +ActiveState=active +UnitFileState=enabled + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 976574a..9a7524d 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "BACKUP-VM-DUMP-DECISION-076", + "command_id": "BACKUP-VM-DUMP-SOURCE-077", "status": "OK", "rc": 0, "host": "pve01", "mode": "read-only", "component": "backup-audit", - "started_at_utc": "2026-07-21T12:44:41Z", - "finished_at_utc": "2026-07-21T12:44:41Z", + "started_at_utc": "2026-07-21T12:47:44Z", + "finished_at_utc": "2026-07-21T12:47:44Z", "reference_register_checked": true, "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", "error_register_checked": true, "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", - "command_sha256": "79b8d619ea551eca9abbf92c24d3e8954f464302759ebb8adf24b308c705df6a", + "command_sha256": "b835d1210c5520c9d8d56eac7575cc679e5d0a22ce04f879cf8af953e2bb9b9e", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -22,9 +22,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "b7a0c0af997d0aa6c682407daef9f7a3855cc52d0e4df08c53a1f4013d9eca72", - "sanitized_output_sha256": "b7a0c0af997d0aa6c682407daef9f7a3855cc52d0e4df08c53a1f4013d9eca72", + "raw_evidence_sha256": "9dd20a7d68dce96c8f08bcb4407101b10e80a36077f51706b0ba8b709099ba58", + "sanitized_output_sha256": "9dd20a7d68dce96c8f08bcb4407101b10e80a36077f51706b0ba8b709099ba58", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "=== NODE_PIPELINE_STATES ===\n========== NODE=pve01 ==========\nVM_DUMP_DECISION_DATA=COMPLETE\n" + "output": "=== SCRIPT_METADATA ===\n1de52eb4ddb17b0ecff2cf3f3021e65b92e91fb44416f97943c51e248c512ed6 /usr/local/sbin/homelab-vm-local-dumps-2cloud\n File: /usr/local/sbin/homelab-vm-local-dumps-2cloud\n Size: 7272 \tBlocks: 16 IO Block: 4096 regular file\nDevice: 252,1\tInode: 5148408 Links: 1\nAccess: (0700/-rwx------) Uid: ( 0/ root) Gid: ( 0/ root)\nAccess: 2026-07-21 00:05:00.471248283 +0300\nModify: 2026-07-06 18:49:49.740645050 +0300\nChange: 2026-07-06 18:49:49.740645050 +0300\n Birth: 2026-07-06 18:27:29.152531375 +0300\n=== SCRIPT_SOURCE ===\n#!/usr/bin/env bash\nset +u\n\nOUT=\"/var/lib/homelab-health/vm-local-dumps-cloud.txt\"\nROOT=\"/var/lib/homelab-vm-backup\"\nLOG=\"$ROOT/vm-local-dumps-2cloud-latest.log\"\nVERIFY_ROOT=\"/var/tmp/homelab-vm-local-dumps-verify\"\nCHUNK_ROOT=\"/var/tmp/homelab-vm-local-dumps-chunks\"\nCHUNK_SIZE_BYTES=\"${CHUNK_SIZE_BYTES:-1073741824}\"\nCHUNK_SIZE_SPLIT=\"${CHUNK_SIZE_SPLIT:-1024M}\"\n\nmkdir -p /var/lib/homelab-health \"$ROOT\" \"$VERIFY_ROOT\" \"$CHUNK_ROOT\"\n\nTS=\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"\nJOB_TS=\"$(date -u +%Y%m%dT%H%M%SZ)\"\nNODE=\"$(hostname -s)\"\nBAD=0\nCOUNT=0\nTOTAL=0\nCHUNKED=0\nVMIDS=\"${VMIDS:-110 112 150}\"\n\nsource /etc/homelab-backup.env 2>/dev/null || BAD=1\n\nJOB_ID=\"homelab-${NODE}-vm-local-dumps-${JOB_TS}\"\nWORK=\"$ROOT/$JOB_ID\"\nmkdir -p \"$WORK\"\n\nFILES_TSV=\"$WORK/files.tsv\"\nCHKS=\"$WORK/checksums.txt\"\nMAN=\"$WORK/manifest.json\"\n\n: > \"$FILES_TSV\"\n: > \"$CHKS\"\n: > \"$LOG\"\n\necho \"job_id=$JOB_ID\" >> \"$LOG\"\necho \"vmids=$VMIDS\" >> \"$LOG\"\necho \"chunk_size_bytes=$CHUNK_SIZE_BYTES\" >> \"$LOG\"\n\nfor id in $VMIDS; do\n F=\"$(find /var/lib/vz/dump -maxdepth 1 -type f \\( -name \"vzdump-qemu-${id}-*.vma.zst\" -o -name \"vzdump-lxc-${id}-*.tar.zst\" \\) -printf \"%T@ %p\\n\" 2>/dev/null | sort -n | tail -n 1 | cut -d\" \" -f2-)\"\n if [ -z \"$F\" ] || [ ! -f \"$F\" ]; then\n echo \"MISSING vmid=$id\" >> \"$LOG\"\n BAD=$((BAD+1))\n continue\n fi\n\n BN=\"$(basename \"$F\")\"\n SIZE=\"$(stat -c %s \"$F\" 2>/dev/null || echo 0)\"\n MT=\"$(date -u -d \"@$(stat -c %Y \"$F\" 2>/dev/null || echo 0)\" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)\"\n KIND=\"unknown\"\n echo \"$BN\" | grep -q \"^vzdump-qemu-\" && KIND=\"qemu\"\n echo \"$BN\" | grep -q \"^vzdump-lxc-\" && KIND=\"lxc\"\n SHA=\"$(sha256sum \"$F\" | cut -d\" \" -f1)\"\n printf \"%s %s\\n\" \"$SHA\" \"$BN\" >> \"$CHKS\"\n\n MODE=\"direct\"\n PARTS=0\n if [ \"$SIZE\" -gt \"$CHUNK_SIZE_BYTES\" ]; then\n MODE=\"chunked\"\n CHUNKED=$((CHUNKED+1))\n CDIR=\"$CHUNK_ROOT/$JOB_ID/$BN.chunks\"\n rm -rf \"$CDIR\"\n mkdir -p \"$CDIR\"\n echo \"SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR\" >> \"$LOG\"\n split -b \"$CHUNK_SIZE_SPLIT\" -d -a 4 \"$F\" \"$CDIR/part-\" >> \"$LOG\" 2>&1\n PARTS=\"$(find \"$CDIR\" -maxdepth 1 -type f -name 'part-*' | wc -l)\"\n (cd \"$CDIR\" && sha256sum part-* > chunks.sha256)\n if [ \"$PARTS\" -le 0 ]; then\n echo \"BAD split produced no chunks file=$BN\" >> \"$LOG\"\n BAD=$((BAD+1))\n fi\n fi\n\n printf \"%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n\" \"$id\" \"$KIND\" \"$SIZE\" \"$SHA\" \"$MT\" \"$BN\" \"$F\" \"$MODE\" \"$PARTS\" >> \"$FILES_TSV\"\n COUNT=$((COUNT+1))\n TOTAL=$((TOTAL+SIZE))\ndone\n\npython3 - \"$JOB_ID\" \"$NODE\" \"$TS\" \"$VMIDS\" \"$COUNT\" \"$TOTAL\" \"$CHUNKED\" \"$CHUNK_SIZE_BYTES\" \"$FILES_TSV\" > \"$MAN\" <<'PY'\nimport json, sys, pathlib\njob,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:]\nitems=[]\nfor line in pathlib.Path(tsv).read_text().splitlines():\n p=line.split(\"\\t\")\n if len(p)>=9:\n items.append({\n \"vmid\":p[0],\n \"kind\":p[1],\n \"size_bytes\":int(p[2]),\n \"sha256\":p[3],\n \"mtime_utc\":p[4],\n \"filename\":p[5],\n \"source_path\":p[6],\n \"mode\":p[7],\n \"parts\":int(p[8]),\n })\nprint(json.dumps({\n \"job_id\":job,\n \"node\":node,\n \"generated_utc\":ts,\n \"vmids\":vmids.split(),\n \"count\":int(count),\n \"total_bytes\":int(total),\n \"chunked_count\":int(chunked),\n \"chunk_size_bytes\":int(chunk_size),\n \"items\":items,\n}, indent=2))\nPY\n\nDEST_A=\"${REMOTE_A%/}/${NODE}/vm-local-dumps/${JOB_ID}\"\nDEST_B=\"${REMOTE_B%/}/${NODE}/vm-local-dumps/${JOB_ID}\"\n\nupload_one_dest() {\n DEST=\"$1\"\n rclone mkdir \"$DEST/files\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n while IFS=\"$(printf \"\\t\")\" read -r id kind size sha mt bn path mode parts; do\n if [ \"$mode\" = \"chunked\" ]; then\n CDIR=\"$CHUNK_ROOT/$JOB_ID/$bn.chunks\"\n echo \"UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts\" >> \"$LOG\"\n rclone mkdir \"$DEST/files/$bn.chunks\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n for part in \"$CDIR\"/part-*; do\n [ -f \"$part\" ] || continue\n pn=\"$(basename \"$part\")\"\n rclone copyto \"$part\" \"$DEST/files/$bn.chunks/$pn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n done\n rclone copyto \"$CDIR/chunks.sha256\" \"$DEST/files/$bn.chunks/chunks.sha256\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n else\n echo \"UPLOAD_DIRECT dest=$DEST file=$bn size=$size\" >> \"$LOG\"\n rclone copyto \"$path\" \"$DEST/files/$bn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n fi\n done < \"$FILES_TSV\"\n\n rclone copyto \"$CHKS\" \"$DEST/checksums.txt\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n rclone copyto \"$FILES_TSV\" \"$DEST/files.tsv\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n rclone copyto \"$MAN\" \"$DEST/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n}\n\nverify_one_dest() {\n LABEL=\"$1\"\n DEST=\"$2\"\n VBASE=\"$VERIFY_ROOT/$JOB_ID/$LABEL\"\n rm -rf \"$VBASE\"\n mkdir -p \"$VBASE\"\n rclone copyto \"$DEST/manifest.json\" \"$VBASE/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n test -s \"$VBASE/manifest.json\" || BAD=$((BAD+1))\n\n while IFS=\"$(printf \"\\t\")\" read -r id kind size sha mt bn path mode parts; do\n if [ \"$mode\" = \"chunked\" ]; then\n VDIR=\"$VBASE/$bn.chunks\"\n mkdir -p \"$VDIR\"\n echo \"VERIFY_CHUNKED label=$LABEL file=$bn parts=$parts\" >> \"$LOG\"\n rclone copy \"$DEST/files/$bn.chunks\" \"$VDIR\" --filter \"+ part-*\" --filter \"+ chunks.sha256\" --filter \"- *\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n (cd \"$VDIR\" && sha256sum -c chunks.sha256) >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n CALC=\"$(find \"$VDIR\" -maxdepth 1 -type f -name 'part-*' | sort | xargs cat | sha256sum | cut -d\" \" -f1)\"\n if [ \"$CALC\" = \"$sha\" ]; then\n echo \"OK combined_sha label=$LABEL file=$bn sha=$sha\" >> \"$LOG\"\n else\n echo \"BAD combined_sha label=$LABEL file=$bn expected=$sha got=$CALC\" >> \"$LOG\"\n BAD=$((BAD+1))\n fi\n rm -rf \"$VDIR\"\n else\n echo \"VERIFY_DIRECT label=$LABEL file=$bn\" >> \"$LOG\"\n rclone copyto \"$DEST/files/$bn\" \"$VBASE/$bn\" --transfers 1 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n printf \"%s %s\\n\" \"$sha\" \"$VBASE/$bn\" | sha256sum -c - >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n rm -f \"$VBASE/$bn\"\n fi\n done < \"$FILES_TSV\"\n rm -rf \"$VBASE\"\n}\n\nif [ \"$BAD\" -eq 0 ]; then\n upload_one_dest \"$DEST_A\"\n upload_one_dest \"$DEST_B\"\nfi\n\nif [ \"$BAD\" -eq 0 ]; then\n verify_one_dest \"A\" \"$DEST_A\"\n verify_one_dest \"B\" \"$DEST_B\"\nfi\n\nrm -rf \"$VERIFY_ROOT/$JOB_ID\"\nrm -rf \"$CHUNK_ROOT/$JOB_ID\"\n\nif [ \"$BAD\" -eq 0 ]; then STATUS=\"OK\"; else STATUS=\"WARN\"; fi\n\njq -n \\\n --arg job_id \"$JOB_ID\" \\\n --arg status \"$STATUS\" \\\n --arg node \"$NODE\" \\\n --arg remote_a \"$DEST_A\" \\\n --arg remote_b \"$DEST_B\" \\\n --argjson count \"$COUNT\" \\\n --argjson total_bytes \"$TOTAL\" \\\n --argjson chunked_count \"$CHUNKED\" \\\n --argjson chunk_size_bytes \"$CHUNK_SIZE_BYTES\" \\\n --arg log \"$LOG\" \\\n \"{job_id:\\$job_id,status:\\$status,node:\\$node,remote_a:\\$remote_a,remote_b:\\$remote_b,count:\\$count,total_bytes:\\$total_bytes,chunked_count:\\$chunked_count,chunk_size_bytes:\\$chunk_size_bytes,log:\\$log,local_dumps_deleted:false}\" \\\n > \"$ROOT/latest-vm-local-dumps-cloud.json\"\n\necho \"STATUS=$STATUS TS=$TS TYPE=vm-local-dumps-cloud BAD=$BAD COUNT=$COUNT CHUNKED=$CHUNKED TOTAL_BYTES=$TOTAL JOB=$JOB_ID REMOTE_A=$DEST_A REMOTE_B=$DEST_B LOG=$LOG\" > \"$OUT\"\ncat \"$OUT\"\n=== SERVICE ===\n# /etc/systemd/system/homelab-vm-local-dumps-2cloud.service\n[Unit]\nDescription=Homelab VM/CT existing local dumps two-cloud verifier\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-vm-local-dumps-2cloud\n\n# /etc/systemd/system/homelab-vm-local-dumps-2cloud.service.d/90-backup-matrix-refresh.conf\n[Service]\nExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; argv[]=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nActiveState=inactive\nSubState=dead\n=== TIMER ===\n# /etc/systemd/system/homelab-vm-local-dumps-2cloud.timer\n[Unit]\nDescription=Homelab VM/CT existing local dumps two-cloud verifier timer\n\n[Timer]\nOnCalendar=Sun *-*-* 10:00:00\nRandomizedDelaySec=30m\nPersistent=true\n\n[Install]\nWantedBy=timers.target\nNextElapseUSecRealtime=Sun 2026-07-26 10:29:15 MSK\nActiveState=active\nUnitFileState=enabled\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 7c93257..6599c4b 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,5 +1,5 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=BACKUP-VM-DUMP-DECISION-076 +COMMAND_ID=BACKUP-VM-DUMP-SOURCE-077 STATUS=OK RC=0 HOST=pve01 @@ -9,19 +9,264 @@ REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 -COMMAND_SHA256=79b8d619ea551eca9abbf92c24d3e8954f464302759ebb8adf24b308c705df6a +COMMAND_SHA256=b835d1210c5520c9d8d56eac7575cc679e5d0a22ce04f879cf8af953e2bb9b9e DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGES_MADE=false SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=b7a0c0af997d0aa6c682407daef9f7a3855cc52d0e4df08c53a1f4013d9eca72 -SANITIZED_OUTPUT_SHA256=b7a0c0af997d0aa6c682407daef9f7a3855cc52d0e4df08c53a1f4013d9eca72 +RAW_EVIDENCE_SHA256=9dd20a7d68dce96c8f08bcb4407101b10e80a36077f51706b0ba8b709099ba58 +SANITIZED_OUTPUT_SHA256=9dd20a7d68dce96c8f08bcb4407101b10e80a36077f51706b0ba8b709099ba58 OUTPUT_BEGIN -=== NODE_PIPELINE_STATES === -========== NODE=pve01 ========== -VM_DUMP_DECISION_DATA=COMPLETE +=== SCRIPT_METADATA === +1de52eb4ddb17b0ecff2cf3f3021e65b92e91fb44416f97943c51e248c512ed6 /usr/local/sbin/homelab-vm-local-dumps-2cloud + File: /usr/local/sbin/homelab-vm-local-dumps-2cloud + Size: 7272 Blocks: 16 IO Block: 4096 regular file +Device: 252,1 Inode: 5148408 Links: 1 +Access: (0700/-rwx------) Uid: ( 0/ root) Gid: ( 0/ root) +Access: 2026-07-21 00:05:00.471248283 +0300 +Modify: 2026-07-06 18:49:49.740645050 +0300 +Change: 2026-07-06 18:49:49.740645050 +0300 + Birth: 2026-07-06 18:27:29.152531375 +0300 +=== SCRIPT_SOURCE === +#!/usr/bin/env bash +set +u + +OUT="/var/lib/homelab-health/vm-local-dumps-cloud.txt" +ROOT="/var/lib/homelab-vm-backup" +LOG="$ROOT/vm-local-dumps-2cloud-latest.log" +VERIFY_ROOT="/var/tmp/homelab-vm-local-dumps-verify" +CHUNK_ROOT="/var/tmp/homelab-vm-local-dumps-chunks" +CHUNK_SIZE_BYTES="${CHUNK_SIZE_BYTES:-1073741824}" +CHUNK_SIZE_SPLIT="${CHUNK_SIZE_SPLIT:-1024M}" + +mkdir -p /var/lib/homelab-health "$ROOT" "$VERIFY_ROOT" "$CHUNK_ROOT" + +TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +JOB_TS="$(date -u +%Y%m%dT%H%M%SZ)" +NODE="$(hostname -s)" +BAD=0 +COUNT=0 +TOTAL=0 +CHUNKED=0 +VMIDS="${VMIDS:-110 112 150}" + +source /etc/homelab-backup.env 2>/dev/null || BAD=1 + +JOB_ID="homelab-${NODE}-vm-local-dumps-${JOB_TS}" +WORK="$ROOT/$JOB_ID" +mkdir -p "$WORK" + +FILES_TSV="$WORK/files.tsv" +CHKS="$WORK/checksums.txt" +MAN="$WORK/manifest.json" + +: > "$FILES_TSV" +: > "$CHKS" +: > "$LOG" + +echo "job_id=$JOB_ID" >> "$LOG" +echo "vmids=$VMIDS" >> "$LOG" +echo "chunk_size_bytes=$CHUNK_SIZE_BYTES" >> "$LOG" + +for id in $VMIDS; do + F="$(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-${id}-*.vma.zst" -o -name "vzdump-lxc-${id}-*.tar.zst" \) -printf "%T@ %p\n" 2>/dev/null | sort -n | tail -n 1 | cut -d" " -f2-)" + if [ -z "$F" ] || [ ! -f "$F" ]; then + echo "MISSING vmid=$id" >> "$LOG" + BAD=$((BAD+1)) + continue + fi + + BN="$(basename "$F")" + SIZE="$(stat -c %s "$F" 2>/dev/null || echo 0)" + MT="$(date -u -d "@$(stat -c %Y "$F" 2>/dev/null || echo 0)" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)" + KIND="unknown" + echo "$BN" | grep -q "^vzdump-qemu-" && KIND="qemu" + echo "$BN" | grep -q "^vzdump-lxc-" && KIND="lxc" + SHA="$(sha256sum "$F" | cut -d" " -f1)" + printf "%s %s\n" "$SHA" "$BN" >> "$CHKS" + + MODE="direct" + PARTS=0 + if [ "$SIZE" -gt "$CHUNK_SIZE_BYTES" ]; then + MODE="chunked" + CHUNKED=$((CHUNKED+1)) + CDIR="$CHUNK_ROOT/$JOB_ID/$BN.chunks" + rm -rf "$CDIR" + mkdir -p "$CDIR" + echo "SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR" >> "$LOG" + split -b "$CHUNK_SIZE_SPLIT" -d -a 4 "$F" "$CDIR/part-" >> "$LOG" 2>&1 + PARTS="$(find "$CDIR" -maxdepth 1 -type f -name 'part-*' | wc -l)" + (cd "$CDIR" && sha256sum part-* > chunks.sha256) + if [ "$PARTS" -le 0 ]; then + echo "BAD split produced no chunks file=$BN" >> "$LOG" + BAD=$((BAD+1)) + fi + fi + + printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" "$id" "$KIND" "$SIZE" "$SHA" "$MT" "$BN" "$F" "$MODE" "$PARTS" >> "$FILES_TSV" + COUNT=$((COUNT+1)) + TOTAL=$((TOTAL+SIZE)) +done + +python3 - "$JOB_ID" "$NODE" "$TS" "$VMIDS" "$COUNT" "$TOTAL" "$CHUNKED" "$CHUNK_SIZE_BYTES" "$FILES_TSV" > "$MAN" <<'PY' +import json, sys, pathlib +job,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:] +items=[] +for line in pathlib.Path(tsv).read_text().splitlines(): + p=line.split("\t") + if len(p)>=9: + items.append({ + "vmid":p[0], + "kind":p[1], + "size_bytes":int(p[2]), + "sha256":p[3], + "mtime_utc":p[4], + "filename":p[5], + "source_path":p[6], + "mode":p[7], + "parts":int(p[8]), + }) +print(json.dumps({ + "job_id":job, + "node":node, + "generated_utc":ts, + "vmids":vmids.split(), + "count":int(count), + "total_bytes":int(total), + "chunked_count":int(chunked), + "chunk_size_bytes":int(chunk_size), + "items":items, +}, indent=2)) +PY + +DEST_A="${REMOTE_A%/}/${NODE}/vm-local-dumps/${JOB_ID}" +DEST_B="${REMOTE_B%/}/${NODE}/vm-local-dumps/${JOB_ID}" + +upload_one_dest() { + DEST="$1" + rclone mkdir "$DEST/files" >> "$LOG" 2>&1 || BAD=$((BAD+1)) + while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do + if [ "$mode" = "chunked" ]; then + CDIR="$CHUNK_ROOT/$JOB_ID/$bn.chunks" + echo "UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts" >> "$LOG" + rclone mkdir "$DEST/files/$bn.chunks" >> "$LOG" 2>&1 || BAD=$((BAD+1)) + for part in "$CDIR"/part-*; do + [ -f "$part" ] || continue + pn="$(basename "$part")" + rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) + done + rclone copyto "$CDIR/chunks.sha256" "$DEST/files/$bn.chunks/chunks.sha256" >> "$LOG" 2>&1 || BAD=$((BAD+1)) + else + echo "UPLOAD_DIRECT dest=$DEST file=$bn size=$size" >> "$LOG" + rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) + fi + done < "$FILES_TSV" + + rclone copyto "$CHKS" "$DEST/checksums.txt" >> "$LOG" 2>&1 || BAD=$((BAD+1)) + rclone copyto "$FILES_TSV" "$DEST/files.tsv" >> "$LOG" 2>&1 || BAD=$((BAD+1)) + rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +} + +verify_one_dest() { + LABEL="$1" + DEST="$2" + VBASE="$VERIFY_ROOT/$JOB_ID/$LABEL" + rm -rf "$VBASE" + mkdir -p "$VBASE" + rclone copyto "$DEST/manifest.json" "$VBASE/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1)) + test -s "$VBASE/manifest.json" || BAD=$((BAD+1)) + + while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do + if [ "$mode" = "chunked" ]; then + VDIR="$VBASE/$bn.chunks" + mkdir -p "$VDIR" + echo "VERIFY_CHUNKED label=$LABEL file=$bn parts=$parts" >> "$LOG" + rclone copy "$DEST/files/$bn.chunks" "$VDIR" --filter "+ part-*" --filter "+ chunks.sha256" --filter "- *" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) + (cd "$VDIR" && sha256sum -c chunks.sha256) >> "$LOG" 2>&1 || BAD=$((BAD+1)) + CALC="$(find "$VDIR" -maxdepth 1 -type f -name 'part-*' | sort | xargs cat | sha256sum | cut -d" " -f1)" + if [ "$CALC" = "$sha" ]; then + echo "OK combined_sha label=$LABEL file=$bn sha=$sha" >> "$LOG" + else + echo "BAD combined_sha label=$LABEL file=$bn expected=$sha got=$CALC" >> "$LOG" + BAD=$((BAD+1)) + fi + rm -rf "$VDIR" + else + echo "VERIFY_DIRECT label=$LABEL file=$bn" >> "$LOG" + rclone copyto "$DEST/files/$bn" "$VBASE/$bn" --transfers 1 >> "$LOG" 2>&1 || BAD=$((BAD+1)) + printf "%s %s\n" "$sha" "$VBASE/$bn" | sha256sum -c - >> "$LOG" 2>&1 || BAD=$((BAD+1)) + rm -f "$VBASE/$bn" + fi + done < "$FILES_TSV" + rm -rf "$VBASE" +} + +if [ "$BAD" -eq 0 ]; then + upload_one_dest "$DEST_A" + upload_one_dest "$DEST_B" +fi + +if [ "$BAD" -eq 0 ]; then + verify_one_dest "A" "$DEST_A" + verify_one_dest "B" "$DEST_B" +fi + +rm -rf "$VERIFY_ROOT/$JOB_ID" +rm -rf "$CHUNK_ROOT/$JOB_ID" + +if [ "$BAD" -eq 0 ]; then STATUS="OK"; else STATUS="WARN"; fi + +jq -n \ + --arg job_id "$JOB_ID" \ + --arg status "$STATUS" \ + --arg node "$NODE" \ + --arg remote_a "$DEST_A" \ + --arg remote_b "$DEST_B" \ + --argjson count "$COUNT" \ + --argjson total_bytes "$TOTAL" \ + --argjson chunked_count "$CHUNKED" \ + --argjson chunk_size_bytes "$CHUNK_SIZE_BYTES" \ + --arg log "$LOG" \ + "{job_id:\$job_id,status:\$status,node:\$node,remote_a:\$remote_a,remote_b:\$remote_b,count:\$count,total_bytes:\$total_bytes,chunked_count:\$chunked_count,chunk_size_bytes:\$chunk_size_bytes,log:\$log,local_dumps_deleted:false}" \ + > "$ROOT/latest-vm-local-dumps-cloud.json" + +echo "STATUS=$STATUS TS=$TS TYPE=vm-local-dumps-cloud BAD=$BAD COUNT=$COUNT CHUNKED=$CHUNKED TOTAL_BYTES=$TOTAL JOB=$JOB_ID REMOTE_A=$DEST_A REMOTE_B=$DEST_B LOG=$LOG" > "$OUT" +cat "$OUT" +=== SERVICE === +# /etc/systemd/system/homelab-vm-local-dumps-2cloud.service +[Unit] +Description=Homelab VM/CT existing local dumps two-cloud verifier +After=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/homelab-vm-local-dumps-2cloud + +# /etc/systemd/system/homelab-vm-local-dumps-2cloud.service.d/90-backup-matrix-refresh.conf +[Service] +ExecStartPost=-/usr/bin/systemctl start --no-block homelab-backup-matrix-refresh.service +Result=success +ExecMainStatus=0 +ExecStart={ path=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; argv[]=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } +ActiveState=inactive +SubState=dead +=== TIMER === +# /etc/systemd/system/homelab-vm-local-dumps-2cloud.timer +[Unit] +Description=Homelab VM/CT existing local dumps two-cloud verifier timer + +[Timer] +OnCalendar=Sun *-*-* 10:00:00 +RandomizedDelaySec=30m +Persistent=true + +[Install] +WantedBy=timers.target +NextElapseUSecRealtime=Sun 2026-07-26 10:29:15 MSK +ActiveState=active +UnitFileState=enabled OUTPUT_END CHAT_OUTPUT_END