runtime: publish BACKUP-DRAIN-LOCAL-ARCHIVES-079
This commit is contained in:
homelab-runtime-publisher
+15
-87
@@ -1,103 +1,31 @@
|
||||
CHAT_OUTPUT_BEGIN
|
||||
COMMAND_ID=BACKUP-VM-DUMP-CONSISTENCY-078
|
||||
STATUS=OK
|
||||
RC=0
|
||||
COMMAND_ID=BACKUP-DRAIN-LOCAL-ARCHIVES-079
|
||||
STATUS=FAIL
|
||||
RC=75
|
||||
HOST=pve01
|
||||
MODE=read-only
|
||||
COMPONENT=backup-audit
|
||||
MODE=change
|
||||
COMPONENT=backup-remediation
|
||||
REFERENCE_REGISTER_CHECK=OK
|
||||
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
|
||||
COMMAND_SHA256=876c832ac6a555af28ff29ccdceb28c79e7244a361f277e9278a8f75b65caad5
|
||||
COMMAND_SHA256=531e1da7ce1681eed084b22ced7da87eb1ee8995f662dc7559f4f9dde3c0ac32
|
||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||
EXECUTION_STARTED=true
|
||||
CHANGES_MADE=false
|
||||
CHANGES_MADE=true
|
||||
SANITIZED=yes
|
||||
SECRETS_INCLUDED=no
|
||||
PRIVATE_ADDRESSES_INCLUDED=no
|
||||
RAW_EVIDENCE_SHA256=02a6668400353ad78630dc31cad6ae25943bc9bd92bc8aff5a46cc2f4dc89d8f
|
||||
SANITIZED_OUTPUT_SHA256=02a6668400353ad78630dc31cad6ae25943bc9bd92bc8aff5a46cc2f4dc89d8f
|
||||
RAW_EVIDENCE_SHA256=891e5c92df1afab4a71ce59fcd6e1cf23bf3d9473f8db7942404c001a4d1661b
|
||||
SANITIZED_OUTPUT_SHA256=891e5c92df1afab4a71ce59fcd6e1cf23bf3d9473f8db7942404c001a4d1661b
|
||||
OUTPUT_BEGIN
|
||||
========== NODE=pve01 ==========
|
||||
1de52eb4ddb17b0ecff2cf3f3021e65b92e91fb44416f97943c51e248c512ed6 /usr/local/sbin/homelab-vm-local-dumps-2cloud
|
||||
30:CHKS="$WORK/checksums.txt"
|
||||
39:echo "chunk_size_bytes=$CHUNK_SIZE_BYTES" >> "$LOG"
|
||||
50: SIZE="$(stat -c %s "$F" 2>/dev/null || echo 0)"
|
||||
51: MT="$(date -u -d "@$(stat -c %Y "$F" 2>/dev/null || echo 0)" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)"
|
||||
55: SHA="$(sha256sum "$F" | cut -d" " -f1)"
|
||||
64: rm -rf "$CDIR"
|
||||
66: echo "SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR" >> "$LOG"
|
||||
69: (cd "$CDIR" && sha256sum part-* > chunks.sha256)
|
||||
83:job,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:]
|
||||
91: "size_bytes":int(p[2]),
|
||||
92: "sha256":p[3],
|
||||
107: "chunk_size_bytes":int(chunk_size),
|
||||
112:DEST_A="${REMOTE_A%/}/${NODE}/vm-local-dumps/${JOB_ID}"
|
||||
113:DEST_B="${REMOTE_B%/}/${NODE}/vm-local-dumps/${JOB_ID}"
|
||||
116: DEST="$1"
|
||||
117: rclone mkdir "$DEST/files" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
118: while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do
|
||||
121: echo "UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts" >> "$LOG"
|
||||
122: rclone mkdir "$DEST/files/$bn.chunks" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
126: rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
128: rclone copyto "$CDIR/chunks.sha256" "$DEST/files/$bn.chunks/chunks.sha256" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
130: echo "UPLOAD_DIRECT dest=$DEST file=$bn size=$size" >> "$LOG"
|
||||
131: rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
135: rclone copyto "$CHKS" "$DEST/checksums.txt" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
136: rclone copyto "$FILES_TSV" "$DEST/files.tsv" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
137: rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
142: DEST="$2"
|
||||
144: rm -rf "$VBASE"
|
||||
146: rclone copyto "$DEST/manifest.json" "$VBASE/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
149: while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do
|
||||
154: rclone copy "$DEST/files/$bn.chunks" "$VDIR" --filter "+ part-*" --filter "+ chunks.sha256" --filter "- *" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
155: (cd "$VDIR" && sha256sum -c chunks.sha256) >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
156: CALC="$(find "$VDIR" -maxdepth 1 -type f -name 'part-*' | sort | xargs cat | sha256sum | cut -d" " -f1)"
|
||||
163: rm -rf "$VDIR"
|
||||
166: rclone copyto "$DEST/files/$bn" "$VBASE/$bn" --transfers 1 >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
167: printf "%s %s\n" "$sha" "$VBASE/$bn" | sha256sum -c - >> "$LOG" 2>&1 || BAD=$((BAD+1))
|
||||
168: rm -f "$VBASE/$bn"
|
||||
171: rm -rf "$VBASE"
|
||||
175: upload_one_dest "$DEST_A"
|
||||
176: upload_one_dest "$DEST_B"
|
||||
180: verify_one_dest "A" "$DEST_A"
|
||||
181: verify_one_dest "B" "$DEST_B"
|
||||
184:rm -rf "$VERIFY_ROOT/$JOB_ID"
|
||||
185:rm -rf "$CHUNK_ROOT/$JOB_ID"
|
||||
191: --arg status "$STATUS" \
|
||||
193: --arg remote_a "$DEST_A" \
|
||||
194: --arg remote_b "$DEST_B" \
|
||||
198: --argjson chunk_size_bytes "$CHUNK_SIZE_BYTES" \
|
||||
200: "{job_id:\$job_id,status:\$status,node:\$node,remote_a:\$remote_a,remote_b:\$remote_b,count:\$count,total_bytes:\$total_bytes,chunked_count:\$chunked_count,chunk_size_bytes:\$chunk_size_bytes,log:\$log,local_dumps_deleted:false}" \
|
||||
203:echo "STATUS=$STATUS TS=$TS TYPE=vm-local-dumps-cloud BAD=$BAD COUNT=$COUNT CHUNKED=$CHUNKED TOTAL_BYTES=$TOTAL JOB=$JOB_ID REMOTE_A=$DEST_A REMOTE_B=$DEST_B LOG=$LOG" > "$OUT"
|
||||
Result=success
|
||||
ExecMainStatus=0
|
||||
NextElapseUSecRealtime=Sun 2026-07-26 10:29:15 MSK
|
||||
ActiveState=active
|
||||
UnitFileState=enabled
|
||||
LOCAL_FILES=24 LOCAL_BYTES=30273877574
|
||||
========== NODE=pve02 ==========
|
||||
bash: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory
|
||||
sha256sum: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory
|
||||
grep: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory
|
||||
Result=success
|
||||
ExecMainStatus=0
|
||||
NextElapseUSecRealtime=
|
||||
ActiveState=inactive
|
||||
UnitFileState=
|
||||
LOCAL_FILES=23 LOCAL_BYTES=3888168125
|
||||
========== NODE=pve03 ==========
|
||||
bash: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory
|
||||
sha256sum: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory
|
||||
grep: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory
|
||||
Result=success
|
||||
ExecMainStatus=0
|
||||
NextElapseUSecRealtime=
|
||||
ActiveState=inactive
|
||||
UnitFileState=
|
||||
LOCAL_FILES=20 LOCAL_BYTES=47769002980
|
||||
VM_DUMP_CONSISTENCY=COMPLETE
|
||||
559463 bash /usr/local/sbin/homelab-chat-run BACKUP-DRAIN-LOCAL-ARCHIVES-079 change backup-remediation -- /usr/bin/bash -c set -Eeuo pipefail; for n in $(pvesh get /nodes --output-format json | jq -r ".[].node" | sort); do echo "========== NODE=$n =========="; ssh -o BatchMode=yes -o ConnectTimeout=10 root@"$n" "set -Eeuo pipefail; p=/usr/local/sbin/homelab-vm-local-dumps-2cloud; u=homelab-vm-local-dumps-2cloud.service; test -x \$p; bash -n \$p; systemctl cat \$u >/dev/null; if pgrep -af \"[v]zdump\"; then echo ACTIVE_VZDUMP=YES; exit 75; fi; timeout 7000 systemctl start \$u; systemctl show \$u -p Result -p ExecMainStatus --no-pager; echo --- LOCAL_AFTER ---; find /var/lib/vz/dump /mnt/pve -type f -name \"vzdump-*\" -printf \"%s %TY-%Tm-%TdT%TH:%TM:%TS %p\n\" 2>/dev/null | sort -nr; left=\$(find /var/lib/vz/dump /mnt/pve -type f -name \"vzdump-*\" -print -quit 2>/dev/null); if test -n \"\$left\"; then echo LOCAL_ARCHIVES_REMAIN=YES; exit 1; fi; echo LOCAL_ARCHIVES_REMAIN=NO"; done; echo BACKUP_DRAIN=COMPLETE
|
||||
559486 timeout --foreground --preserve-status 7200 /usr/bin/bash -c set -Eeuo pipefail; for n in $(pvesh get /nodes --output-format json | jq -r ".[].node" | sort); do echo "========== NODE=$n =========="; ssh -o BatchMode=yes -o ConnectTimeout=10 root@"$n" "set -Eeuo pipefail; p=/usr/local/sbin/homelab-vm-local-dumps-2cloud; u=homelab-vm-local-dumps-2cloud.service; test -x \$p; bash -n \$p; systemctl cat \$u >/dev/null; if pgrep -af \"[v]zdump\"; then echo ACTIVE_VZDUMP=YES; exit 75; fi; timeout 7000 systemctl start \$u; systemctl show \$u -p Result -p ExecMainStatus --no-pager; echo --- LOCAL_AFTER ---; find /var/lib/vz/dump /mnt/pve -type f -name \"vzdump-*\" -printf \"%s %TY-%Tm-%TdT%TH:%TM:%TS %p\n\" 2>/dev/null | sort -nr; left=\$(find /var/lib/vz/dump /mnt/pve -type f -name \"vzdump-*\" -print -quit 2>/dev/null); if test -n \"\$left\"; then echo LOCAL_ARCHIVES_REMAIN=YES; exit 1; fi; echo LOCAL_ARCHIVES_REMAIN=NO"; done; echo BACKUP_DRAIN=COMPLETE
|
||||
559487 /usr/bin/bash -c set -Eeuo pipefail; for n in $(pvesh get /nodes --output-format json | jq -r ".[].node" | sort); do echo "========== NODE=$n =========="; ssh -o BatchMode=yes -o ConnectTimeout=10 root@"$n" "set -Eeuo pipefail; p=/usr/local/sbin/homelab-vm-local-dumps-2cloud; u=homelab-vm-local-dumps-2cloud.service; test -x \$p; bash -n \$p; systemctl cat \$u >/dev/null; if pgrep -af \"[v]zdump\"; then echo ACTIVE_VZDUMP=YES; exit 75; fi; timeout 7000 systemctl start \$u; systemctl show \$u -p Result -p ExecMainStatus --no-pager; echo --- LOCAL_AFTER ---; find /var/lib/vz/dump /mnt/pve -type f -name \"vzdump-*\" -printf \"%s %TY-%Tm-%TdT%TH:%TM:%TS %p\n\" 2>/dev/null | sort -nr; left=\$(find /var/lib/vz/dump /mnt/pve -type f -name \"vzdump-*\" -print -quit 2>/dev/null); if test -n \"\$left\"; then echo LOCAL_ARCHIVES_REMAIN=YES; exit 1; fi; echo LOCAL_ARCHIVES_REMAIN=NO"; done; echo BACKUP_DRAIN=COMPLETE
|
||||
559499 ssh -o BatchMode=yes -o ConnectTimeout=10 root@pve01 set -Eeuo pipefail; p=/usr/local/sbin/homelab-vm-local-dumps-2cloud; u=homelab-vm-local-dumps-2cloud.service; test -x $p; bash -n $p; systemctl cat $u >/dev/null; if pgrep -af "[v]zdump"; then echo ACTIVE_VZDUMP=YES; exit 75; fi; timeout 7000 systemctl start $u; systemctl show $u -p Result -p ExecMainStatus --no-pager; echo --- LOCAL_AFTER ---; find /var/lib/vz/dump /mnt/pve -type f -name "vzdump-*" -printf "%s %TY-%Tm-%TdT%TH:%TM:%TS %p\n" 2>/dev/null | sort -nr; left=$(find /var/lib/vz/dump /mnt/pve -type f -name "vzdump-*" -print -quit 2>/dev/null); if test -n "$left"; then echo LOCAL_ARCHIVES_REMAIN=YES; exit 1; fi; echo LOCAL_ARCHIVES_REMAIN=NO
|
||||
559508 bash -c set -Eeuo pipefail; p=/usr/local/sbin/homelab-vm-local-dumps-2cloud; u=homelab-vm-local-dumps-2cloud.service; test -x $p; bash -n $p; systemctl cat $u >/dev/null; if pgrep -af "[v]zdump"; then echo ACTIVE_VZDUMP=YES; exit 75; fi; timeout 7000 systemctl start $u; systemctl show $u -p Result -p ExecMainStatus --no-pager; echo --- LOCAL_AFTER ---; find /var/lib/vz/dump /mnt/pve -type f -name "vzdump-*" -printf "%s %TY-%Tm-%TdT%TH:%TM:%TS %p\n" 2>/dev/null | sort -nr; left=$(find /var/lib/vz/dump /mnt/pve -type f -name "vzdump-*" -print -quit 2>/dev/null); if test -n "$left"; then echo LOCAL_ARCHIVES_REMAIN=YES; exit 1; fi; echo LOCAL_ARCHIVES_REMAIN=NO
|
||||
ACTIVE_VZDUMP=YES
|
||||
|
||||
OUTPUT_END
|
||||
CHAT_OUTPUT_END
|
||||
|
||||
Reference in New Issue
Block a user