From 67b43b3496b5664729e6687114c01ddfd04ab5a7 Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Tue, 21 Jul 2026 12:56:52 +0000 Subject: [PATCH] runtime: publish BACKUP-DRAIN-LOCAL-ARCHIVES-079 --- .../BACKUP-DRAIN-LOCAL-ARCHIVES-079.json | 30 ++++++ .../BACKUP-DRAIN-LOCAL-ARCHIVES-079.txt | 31 ++++++ runtime/latest.json | 24 ++--- runtime/latest.txt | 102 +++--------------- 4 files changed, 88 insertions(+), 99 deletions(-) create mode 100644 runtime/history/BACKUP-DRAIN-LOCAL-ARCHIVES-079.json create mode 100644 runtime/history/BACKUP-DRAIN-LOCAL-ARCHIVES-079.txt diff --git a/runtime/history/BACKUP-DRAIN-LOCAL-ARCHIVES-079.json b/runtime/history/BACKUP-DRAIN-LOCAL-ARCHIVES-079.json new file mode 100644 index 0000000..01cc13b --- /dev/null +++ b/runtime/history/BACKUP-DRAIN-LOCAL-ARCHIVES-079.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "BACKUP-DRAIN-LOCAL-ARCHIVES-079", + "status": "FAIL", + "rc": 75, + "host": "pve01", + "mode": "change", + "component": "backup-remediation", + "started_at_utc": "2026-07-21T12:56:51Z", + "finished_at_utc": "2026-07-21T12:56:52Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", + "command_sha256": "531e1da7ce1681eed084b22ced7da87eb1ee8995f662dc7559f4f9dde3c0ac32", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": true, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "891e5c92df1afab4a71ce59fcd6e1cf23bf3d9473f8db7942404c001a4d1661b", + "sanitized_output_sha256": "891e5c92df1afab4a71ce59fcd6e1cf23bf3d9473f8db7942404c001a4d1661b", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "========== NODE=pve01 ==========\n559463 bash /usr/local/sbin/homelab-chat-run BACKUP-DRAIN-LOCAL-ARCHIVES-079 change backup-remediation -- /usr/bin/bash -c set -Eeuo pipefail; for n in $(pvesh get /nodes --output-format json | jq -r \".[].node\" | sort); do echo \"========== NODE=$n ==========\"; ssh -o BatchMode=yes -o ConnectTimeout=10 root@\"$n\" \"set -Eeuo pipefail; p=/usr/local/sbin/homelab-vm-local-dumps-2cloud; u=homelab-vm-local-dumps-2cloud.service; test -x \\$p; bash -n \\$p; systemctl cat \\$u >/dev/null; if pgrep -af \\\"[v]zdump\\\"; then echo ACTIVE_VZDUMP=YES; exit 75; fi; timeout 7000 systemctl start \\$u; systemctl show \\$u -p Result -p ExecMainStatus --no-pager; echo --- LOCAL_AFTER ---; find /var/lib/vz/dump /mnt/pve -type f -name \\\"vzdump-*\\\" -printf \\\"%s %TY-%Tm-%TdT%TH:%TM:%TS %p\\n\\\" 2>/dev/null | sort -nr; left=\\$(find /var/lib/vz/dump /mnt/pve -type f -name \\\"vzdump-*\\\" -print -quit 2>/dev/null); if test -n \\\"\\$left\\\"; then echo LOCAL_ARCHIVES_REMAIN=YES; exit 1; fi; echo LOCAL_ARCHIVES_REMAIN=NO\"; done; echo BACKUP_DRAIN=COMPLETE\n559486 timeout --foreground --preserve-status 7200 /usr/bin/bash -c set -Eeuo pipefail; for n in $(pvesh get /nodes --output-format json | jq -r \".[].node\" | sort); do echo \"========== NODE=$n ==========\"; ssh -o BatchMode=yes -o ConnectTimeout=10 root@\"$n\" \"set -Eeuo pipefail; p=/usr/local/sbin/homelab-vm-local-dumps-2cloud; u=homelab-vm-local-dumps-2cloud.service; test -x \\$p; bash -n \\$p; systemctl cat \\$u >/dev/null; if pgrep -af \\\"[v]zdump\\\"; then echo ACTIVE_VZDUMP=YES; exit 75; fi; timeout 7000 systemctl start \\$u; systemctl show \\$u -p Result -p ExecMainStatus --no-pager; echo --- LOCAL_AFTER ---; find /var/lib/vz/dump /mnt/pve -type f -name \\\"vzdump-*\\\" -printf \\\"%s %TY-%Tm-%TdT%TH:%TM:%TS %p\\n\\\" 2>/dev/null | sort -nr; left=\\$(find /var/lib/vz/dump /mnt/pve -type f -name \\\"vzdump-*\\\" -print -quit 2>/dev/null); if test -n \\\"\\$left\\\"; then echo LOCAL_ARCHIVES_REMAIN=YES; exit 1; fi; echo LOCAL_ARCHIVES_REMAIN=NO\"; done; echo BACKUP_DRAIN=COMPLETE\n559487 /usr/bin/bash -c set -Eeuo pipefail; for n in $(pvesh get /nodes --output-format json | jq -r \".[].node\" | sort); do echo \"========== NODE=$n ==========\"; ssh -o BatchMode=yes -o ConnectTimeout=10 root@\"$n\" \"set -Eeuo pipefail; p=/usr/local/sbin/homelab-vm-local-dumps-2cloud; u=homelab-vm-local-dumps-2cloud.service; test -x \\$p; bash -n \\$p; systemctl cat \\$u >/dev/null; if pgrep -af \\\"[v]zdump\\\"; then echo ACTIVE_VZDUMP=YES; exit 75; fi; timeout 7000 systemctl start \\$u; systemctl show \\$u -p Result -p ExecMainStatus --no-pager; echo --- LOCAL_AFTER ---; find /var/lib/vz/dump /mnt/pve -type f -name \\\"vzdump-*\\\" -printf \\\"%s %TY-%Tm-%TdT%TH:%TM:%TS %p\\n\\\" 2>/dev/null | sort -nr; left=\\$(find /var/lib/vz/dump /mnt/pve -type f -name \\\"vzdump-*\\\" -print -quit 2>/dev/null); if test -n \\\"\\$left\\\"; then echo LOCAL_ARCHIVES_REMAIN=YES; exit 1; fi; echo LOCAL_ARCHIVES_REMAIN=NO\"; done; echo BACKUP_DRAIN=COMPLETE\n559499 ssh -o BatchMode=yes -o ConnectTimeout=10 root@pve01 set -Eeuo pipefail; p=/usr/local/sbin/homelab-vm-local-dumps-2cloud; u=homelab-vm-local-dumps-2cloud.service; test -x $p; bash -n $p; systemctl cat $u >/dev/null; if pgrep -af \"[v]zdump\"; then echo ACTIVE_VZDUMP=YES; exit 75; fi; timeout 7000 systemctl start $u; systemctl show $u -p Result -p ExecMainStatus --no-pager; echo --- LOCAL_AFTER ---; find /var/lib/vz/dump /mnt/pve -type f -name \"vzdump-*\" -printf \"%s %TY-%Tm-%TdT%TH:%TM:%TS %p\\n\" 2>/dev/null | sort -nr; left=$(find /var/lib/vz/dump /mnt/pve -type f -name \"vzdump-*\" -print -quit 2>/dev/null); if test -n \"$left\"; then echo LOCAL_ARCHIVES_REMAIN=YES; exit 1; fi; echo LOCAL_ARCHIVES_REMAIN=NO\n559508 bash -c set -Eeuo pipefail; p=/usr/local/sbin/homelab-vm-local-dumps-2cloud; u=homelab-vm-local-dumps-2cloud.service; test -x $p; bash -n $p; systemctl cat $u >/dev/null; if pgrep -af \"[v]zdump\"; then echo ACTIVE_VZDUMP=YES; exit 75; fi; timeout 7000 systemctl start $u; systemctl show $u -p Result -p ExecMainStatus --no-pager; echo --- LOCAL_AFTER ---; find /var/lib/vz/dump /mnt/pve -type f -name \"vzdump-*\" -printf \"%s %TY-%Tm-%TdT%TH:%TM:%TS %p\\n\" 2>/dev/null | sort -nr; left=$(find /var/lib/vz/dump /mnt/pve -type f -name \"vzdump-*\" -print -quit 2>/dev/null); if test -n \"$left\"; then echo LOCAL_ARCHIVES_REMAIN=YES; exit 1; fi; echo LOCAL_ARCHIVES_REMAIN=NO\nACTIVE_VZDUMP=YES\n" +} diff --git a/runtime/history/BACKUP-DRAIN-LOCAL-ARCHIVES-079.txt b/runtime/history/BACKUP-DRAIN-LOCAL-ARCHIVES-079.txt new file mode 100644 index 0000000..c87a268 --- /dev/null +++ b/runtime/history/BACKUP-DRAIN-LOCAL-ARCHIVES-079.txt @@ -0,0 +1,31 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=BACKUP-DRAIN-LOCAL-ARCHIVES-079 +STATUS=FAIL +RC=75 +HOST=pve01 +MODE=change +COMPONENT=backup-remediation +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 +COMMAND_SHA256=531e1da7ce1681eed084b22ced7da87eb1ee8995f662dc7559f4f9dde3c0ac32 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=true +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=891e5c92df1afab4a71ce59fcd6e1cf23bf3d9473f8db7942404c001a4d1661b +SANITIZED_OUTPUT_SHA256=891e5c92df1afab4a71ce59fcd6e1cf23bf3d9473f8db7942404c001a4d1661b +OUTPUT_BEGIN +========== NODE=pve01 ========== +559463 bash /usr/local/sbin/homelab-chat-run BACKUP-DRAIN-LOCAL-ARCHIVES-079 change backup-remediation -- /usr/bin/bash -c set -Eeuo pipefail; for n in $(pvesh get /nodes --output-format json | jq -r ".[].node" | sort); do echo "========== NODE=$n =========="; ssh -o BatchMode=yes -o ConnectTimeout=10 root@"$n" "set -Eeuo pipefail; p=/usr/local/sbin/homelab-vm-local-dumps-2cloud; u=homelab-vm-local-dumps-2cloud.service; test -x \$p; bash -n \$p; systemctl cat \$u >/dev/null; if pgrep -af \"[v]zdump\"; then echo ACTIVE_VZDUMP=YES; exit 75; fi; timeout 7000 systemctl start \$u; systemctl show \$u -p Result -p ExecMainStatus --no-pager; echo --- LOCAL_AFTER ---; find /var/lib/vz/dump /mnt/pve -type f -name \"vzdump-*\" -printf \"%s %TY-%Tm-%TdT%TH:%TM:%TS %p\n\" 2>/dev/null | sort -nr; left=\$(find /var/lib/vz/dump /mnt/pve -type f -name \"vzdump-*\" -print -quit 2>/dev/null); if test -n \"\$left\"; then echo LOCAL_ARCHIVES_REMAIN=YES; exit 1; fi; echo LOCAL_ARCHIVES_REMAIN=NO"; done; echo BACKUP_DRAIN=COMPLETE +559486 timeout --foreground --preserve-status 7200 /usr/bin/bash -c set -Eeuo pipefail; for n in $(pvesh get /nodes --output-format json | jq -r ".[].node" | sort); do echo "========== NODE=$n =========="; ssh -o BatchMode=yes -o ConnectTimeout=10 root@"$n" "set -Eeuo pipefail; p=/usr/local/sbin/homelab-vm-local-dumps-2cloud; u=homelab-vm-local-dumps-2cloud.service; test -x \$p; bash -n \$p; systemctl cat \$u >/dev/null; if pgrep -af \"[v]zdump\"; then echo ACTIVE_VZDUMP=YES; exit 75; fi; timeout 7000 systemctl start \$u; systemctl show \$u -p Result -p ExecMainStatus --no-pager; echo --- LOCAL_AFTER ---; find /var/lib/vz/dump /mnt/pve -type f -name \"vzdump-*\" -printf \"%s %TY-%Tm-%TdT%TH:%TM:%TS %p\n\" 2>/dev/null | sort -nr; left=\$(find /var/lib/vz/dump /mnt/pve -type f -name \"vzdump-*\" -print -quit 2>/dev/null); if test -n \"\$left\"; then echo LOCAL_ARCHIVES_REMAIN=YES; exit 1; fi; echo LOCAL_ARCHIVES_REMAIN=NO"; done; echo BACKUP_DRAIN=COMPLETE +559487 /usr/bin/bash -c set -Eeuo pipefail; for n in $(pvesh get /nodes --output-format json | jq -r ".[].node" | sort); do echo "========== NODE=$n =========="; ssh -o BatchMode=yes -o ConnectTimeout=10 root@"$n" "set -Eeuo pipefail; p=/usr/local/sbin/homelab-vm-local-dumps-2cloud; u=homelab-vm-local-dumps-2cloud.service; test -x \$p; bash -n \$p; systemctl cat \$u >/dev/null; if pgrep -af \"[v]zdump\"; then echo ACTIVE_VZDUMP=YES; exit 75; fi; timeout 7000 systemctl start \$u; systemctl show \$u -p Result -p ExecMainStatus --no-pager; echo --- LOCAL_AFTER ---; find /var/lib/vz/dump /mnt/pve -type f -name \"vzdump-*\" -printf \"%s %TY-%Tm-%TdT%TH:%TM:%TS %p\n\" 2>/dev/null | sort -nr; left=\$(find /var/lib/vz/dump /mnt/pve -type f -name \"vzdump-*\" -print -quit 2>/dev/null); if test -n \"\$left\"; then echo LOCAL_ARCHIVES_REMAIN=YES; exit 1; fi; echo LOCAL_ARCHIVES_REMAIN=NO"; done; echo BACKUP_DRAIN=COMPLETE +559499 ssh -o BatchMode=yes -o ConnectTimeout=10 root@pve01 set -Eeuo pipefail; p=/usr/local/sbin/homelab-vm-local-dumps-2cloud; u=homelab-vm-local-dumps-2cloud.service; test -x $p; bash -n $p; systemctl cat $u >/dev/null; if pgrep -af "[v]zdump"; then echo ACTIVE_VZDUMP=YES; exit 75; fi; timeout 7000 systemctl start $u; systemctl show $u -p Result -p ExecMainStatus --no-pager; echo --- LOCAL_AFTER ---; find /var/lib/vz/dump /mnt/pve -type f -name "vzdump-*" -printf "%s %TY-%Tm-%TdT%TH:%TM:%TS %p\n" 2>/dev/null | sort -nr; left=$(find /var/lib/vz/dump /mnt/pve -type f -name "vzdump-*" -print -quit 2>/dev/null); if test -n "$left"; then echo LOCAL_ARCHIVES_REMAIN=YES; exit 1; fi; echo LOCAL_ARCHIVES_REMAIN=NO +559508 bash -c set -Eeuo pipefail; p=/usr/local/sbin/homelab-vm-local-dumps-2cloud; u=homelab-vm-local-dumps-2cloud.service; test -x $p; bash -n $p; systemctl cat $u >/dev/null; if pgrep -af "[v]zdump"; then echo ACTIVE_VZDUMP=YES; exit 75; fi; timeout 7000 systemctl start $u; systemctl show $u -p Result -p ExecMainStatus --no-pager; echo --- LOCAL_AFTER ---; find /var/lib/vz/dump /mnt/pve -type f -name "vzdump-*" -printf "%s %TY-%Tm-%TdT%TH:%TM:%TS %p\n" 2>/dev/null | sort -nr; left=$(find /var/lib/vz/dump /mnt/pve -type f -name "vzdump-*" -print -quit 2>/dev/null); if test -n "$left"; then echo LOCAL_ARCHIVES_REMAIN=YES; exit 1; fi; echo LOCAL_ARCHIVES_REMAIN=NO +ACTIVE_VZDUMP=YES + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 5b6eef1..01cc13b 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,30 +1,30 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "BACKUP-VM-DUMP-CONSISTENCY-078", - "status": "OK", - "rc": 0, + "command_id": "BACKUP-DRAIN-LOCAL-ARCHIVES-079", + "status": "FAIL", + "rc": 75, "host": "pve01", - "mode": "read-only", - "component": "backup-audit", - "started_at_utc": "2026-07-21T12:55:03Z", - "finished_at_utc": "2026-07-21T12:55:04Z", + "mode": "change", + "component": "backup-remediation", + "started_at_utc": "2026-07-21T12:56:51Z", + "finished_at_utc": "2026-07-21T12:56:52Z", "reference_register_checked": true, "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", "error_register_checked": true, "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", - "command_sha256": "876c832ac6a555af28ff29ccdceb28c79e7244a361f277e9278a8f75b65caad5", + "command_sha256": "531e1da7ce1681eed084b22ced7da87eb1ee8995f662dc7559f4f9dde3c0ac32", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, - "changes_made": false, + "changes_made": true, "sanitized": true, "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "02a6668400353ad78630dc31cad6ae25943bc9bd92bc8aff5a46cc2f4dc89d8f", - "sanitized_output_sha256": "02a6668400353ad78630dc31cad6ae25943bc9bd92bc8aff5a46cc2f4dc89d8f", + "raw_evidence_sha256": "891e5c92df1afab4a71ce59fcd6e1cf23bf3d9473f8db7942404c001a4d1661b", + "sanitized_output_sha256": "891e5c92df1afab4a71ce59fcd6e1cf23bf3d9473f8db7942404c001a4d1661b", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "========== NODE=pve01 ==========\n1de52eb4ddb17b0ecff2cf3f3021e65b92e91fb44416f97943c51e248c512ed6 /usr/local/sbin/homelab-vm-local-dumps-2cloud\n30:CHKS=\"$WORK/checksums.txt\"\n39:echo \"chunk_size_bytes=$CHUNK_SIZE_BYTES\" >> \"$LOG\"\n50: SIZE=\"$(stat -c %s \"$F\" 2>/dev/null || echo 0)\"\n51: MT=\"$(date -u -d \"@$(stat -c %Y \"$F\" 2>/dev/null || echo 0)\" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)\"\n55: SHA=\"$(sha256sum \"$F\" | cut -d\" \" -f1)\"\n64: rm -rf \"$CDIR\"\n66: echo \"SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR\" >> \"$LOG\"\n69: (cd \"$CDIR\" && sha256sum part-* > chunks.sha256)\n83:job,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:]\n91: \"size_bytes\":int(p[2]),\n92: \"sha256\":p[3],\n107: \"chunk_size_bytes\":int(chunk_size),\n112:DEST_A=\"${REMOTE_A%/}/${NODE}/vm-local-dumps/${JOB_ID}\"\n113:DEST_B=\"${REMOTE_B%/}/${NODE}/vm-local-dumps/${JOB_ID}\"\n116: DEST=\"$1\"\n117: rclone mkdir \"$DEST/files\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n118: while IFS=\"$(printf \"\\t\")\" read -r id kind size sha mt bn path mode parts; do\n121: echo \"UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts\" >> \"$LOG\"\n122: rclone mkdir \"$DEST/files/$bn.chunks\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n126: rclone copyto \"$part\" \"$DEST/files/$bn.chunks/$pn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n128: rclone copyto \"$CDIR/chunks.sha256\" \"$DEST/files/$bn.chunks/chunks.sha256\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n130: echo \"UPLOAD_DIRECT dest=$DEST file=$bn size=$size\" >> \"$LOG\"\n131: rclone copyto \"$path\" \"$DEST/files/$bn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n135: rclone copyto \"$CHKS\" \"$DEST/checksums.txt\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n136: rclone copyto \"$FILES_TSV\" \"$DEST/files.tsv\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n137: rclone copyto \"$MAN\" \"$DEST/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n142: DEST=\"$2\"\n144: rm -rf \"$VBASE\"\n146: rclone copyto \"$DEST/manifest.json\" \"$VBASE/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n149: while IFS=\"$(printf \"\\t\")\" read -r id kind size sha mt bn path mode parts; do\n154: rclone copy \"$DEST/files/$bn.chunks\" \"$VDIR\" --filter \"+ part-*\" --filter \"+ chunks.sha256\" --filter \"- *\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n155: (cd \"$VDIR\" && sha256sum -c chunks.sha256) >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n156: CALC=\"$(find \"$VDIR\" -maxdepth 1 -type f -name 'part-*' | sort | xargs cat | sha256sum | cut -d\" \" -f1)\"\n163: rm -rf \"$VDIR\"\n166: rclone copyto \"$DEST/files/$bn\" \"$VBASE/$bn\" --transfers 1 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n167: printf \"%s %s\\n\" \"$sha\" \"$VBASE/$bn\" | sha256sum -c - >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\n168: rm -f \"$VBASE/$bn\"\n171: rm -rf \"$VBASE\"\n175: upload_one_dest \"$DEST_A\"\n176: upload_one_dest \"$DEST_B\"\n180: verify_one_dest \"A\" \"$DEST_A\"\n181: verify_one_dest \"B\" \"$DEST_B\"\n184:rm -rf \"$VERIFY_ROOT/$JOB_ID\"\n185:rm -rf \"$CHUNK_ROOT/$JOB_ID\"\n191: --arg status \"$STATUS\" \\\n193: --arg remote_a \"$DEST_A\" \\\n194: --arg remote_b \"$DEST_B\" \\\n198: --argjson chunk_size_bytes \"$CHUNK_SIZE_BYTES\" \\\n200: \"{job_id:\\$job_id,status:\\$status,node:\\$node,remote_a:\\$remote_a,remote_b:\\$remote_b,count:\\$count,total_bytes:\\$total_bytes,chunked_count:\\$chunked_count,chunk_size_bytes:\\$chunk_size_bytes,log:\\$log,local_dumps_deleted:false}\" \\\n203:echo \"STATUS=$STATUS TS=$TS TYPE=vm-local-dumps-cloud BAD=$BAD COUNT=$COUNT CHUNKED=$CHUNKED TOTAL_BYTES=$TOTAL JOB=$JOB_ID REMOTE_A=$DEST_A REMOTE_B=$DEST_B LOG=$LOG\" > \"$OUT\"\nResult=success\nExecMainStatus=0\nNextElapseUSecRealtime=Sun 2026-07-26 10:29:15 MSK\nActiveState=active\nUnitFileState=enabled\nLOCAL_FILES=24 LOCAL_BYTES=30273877574\n========== NODE=pve02 ==========\nbash: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory\nsha256sum: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory\ngrep: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory\nResult=success\nExecMainStatus=0\nNextElapseUSecRealtime=\nActiveState=inactive\nUnitFileState=\nLOCAL_FILES=23 LOCAL_BYTES=3888168125\n========== NODE=pve03 ==========\nbash: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory\nsha256sum: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory\ngrep: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory\nResult=success\nExecMainStatus=0\nNextElapseUSecRealtime=\nActiveState=inactive\nUnitFileState=\nLOCAL_FILES=20 LOCAL_BYTES=47769002980\nVM_DUMP_CONSISTENCY=COMPLETE\n" + "output": "========== NODE=pve01 ==========\n559463 bash /usr/local/sbin/homelab-chat-run BACKUP-DRAIN-LOCAL-ARCHIVES-079 change backup-remediation -- /usr/bin/bash -c set -Eeuo pipefail; for n in $(pvesh get /nodes --output-format json | jq -r \".[].node\" | sort); do echo \"========== NODE=$n ==========\"; ssh -o BatchMode=yes -o ConnectTimeout=10 root@\"$n\" \"set -Eeuo pipefail; p=/usr/local/sbin/homelab-vm-local-dumps-2cloud; u=homelab-vm-local-dumps-2cloud.service; test -x \\$p; bash -n \\$p; systemctl cat \\$u >/dev/null; if pgrep -af \\\"[v]zdump\\\"; then echo ACTIVE_VZDUMP=YES; exit 75; fi; timeout 7000 systemctl start \\$u; systemctl show \\$u -p Result -p ExecMainStatus --no-pager; echo --- LOCAL_AFTER ---; find /var/lib/vz/dump /mnt/pve -type f -name \\\"vzdump-*\\\" -printf \\\"%s %TY-%Tm-%TdT%TH:%TM:%TS %p\\n\\\" 2>/dev/null | sort -nr; left=\\$(find /var/lib/vz/dump /mnt/pve -type f -name \\\"vzdump-*\\\" -print -quit 2>/dev/null); if test -n \\\"\\$left\\\"; then echo LOCAL_ARCHIVES_REMAIN=YES; exit 1; fi; echo LOCAL_ARCHIVES_REMAIN=NO\"; done; echo BACKUP_DRAIN=COMPLETE\n559486 timeout --foreground --preserve-status 7200 /usr/bin/bash -c set -Eeuo pipefail; for n in $(pvesh get /nodes --output-format json | jq -r \".[].node\" | sort); do echo \"========== NODE=$n ==========\"; ssh -o BatchMode=yes -o ConnectTimeout=10 root@\"$n\" \"set -Eeuo pipefail; p=/usr/local/sbin/homelab-vm-local-dumps-2cloud; u=homelab-vm-local-dumps-2cloud.service; test -x \\$p; bash -n \\$p; systemctl cat \\$u >/dev/null; if pgrep -af \\\"[v]zdump\\\"; then echo ACTIVE_VZDUMP=YES; exit 75; fi; timeout 7000 systemctl start \\$u; systemctl show \\$u -p Result -p ExecMainStatus --no-pager; echo --- LOCAL_AFTER ---; find /var/lib/vz/dump /mnt/pve -type f -name \\\"vzdump-*\\\" -printf \\\"%s %TY-%Tm-%TdT%TH:%TM:%TS %p\\n\\\" 2>/dev/null | sort -nr; left=\\$(find /var/lib/vz/dump /mnt/pve -type f -name \\\"vzdump-*\\\" -print -quit 2>/dev/null); if test -n \\\"\\$left\\\"; then echo LOCAL_ARCHIVES_REMAIN=YES; exit 1; fi; echo LOCAL_ARCHIVES_REMAIN=NO\"; done; echo BACKUP_DRAIN=COMPLETE\n559487 /usr/bin/bash -c set -Eeuo pipefail; for n in $(pvesh get /nodes --output-format json | jq -r \".[].node\" | sort); do echo \"========== NODE=$n ==========\"; ssh -o BatchMode=yes -o ConnectTimeout=10 root@\"$n\" \"set -Eeuo pipefail; p=/usr/local/sbin/homelab-vm-local-dumps-2cloud; u=homelab-vm-local-dumps-2cloud.service; test -x \\$p; bash -n \\$p; systemctl cat \\$u >/dev/null; if pgrep -af \\\"[v]zdump\\\"; then echo ACTIVE_VZDUMP=YES; exit 75; fi; timeout 7000 systemctl start \\$u; systemctl show \\$u -p Result -p ExecMainStatus --no-pager; echo --- LOCAL_AFTER ---; find /var/lib/vz/dump /mnt/pve -type f -name \\\"vzdump-*\\\" -printf \\\"%s %TY-%Tm-%TdT%TH:%TM:%TS %p\\n\\\" 2>/dev/null | sort -nr; left=\\$(find /var/lib/vz/dump /mnt/pve -type f -name \\\"vzdump-*\\\" -print -quit 2>/dev/null); if test -n \\\"\\$left\\\"; then echo LOCAL_ARCHIVES_REMAIN=YES; exit 1; fi; echo LOCAL_ARCHIVES_REMAIN=NO\"; done; echo BACKUP_DRAIN=COMPLETE\n559499 ssh -o BatchMode=yes -o ConnectTimeout=10 root@pve01 set -Eeuo pipefail; p=/usr/local/sbin/homelab-vm-local-dumps-2cloud; u=homelab-vm-local-dumps-2cloud.service; test -x $p; bash -n $p; systemctl cat $u >/dev/null; if pgrep -af \"[v]zdump\"; then echo ACTIVE_VZDUMP=YES; exit 75; fi; timeout 7000 systemctl start $u; systemctl show $u -p Result -p ExecMainStatus --no-pager; echo --- LOCAL_AFTER ---; find /var/lib/vz/dump /mnt/pve -type f -name \"vzdump-*\" -printf \"%s %TY-%Tm-%TdT%TH:%TM:%TS %p\\n\" 2>/dev/null | sort -nr; left=$(find /var/lib/vz/dump /mnt/pve -type f -name \"vzdump-*\" -print -quit 2>/dev/null); if test -n \"$left\"; then echo LOCAL_ARCHIVES_REMAIN=YES; exit 1; fi; echo LOCAL_ARCHIVES_REMAIN=NO\n559508 bash -c set -Eeuo pipefail; p=/usr/local/sbin/homelab-vm-local-dumps-2cloud; u=homelab-vm-local-dumps-2cloud.service; test -x $p; bash -n $p; systemctl cat $u >/dev/null; if pgrep -af \"[v]zdump\"; then echo ACTIVE_VZDUMP=YES; exit 75; fi; timeout 7000 systemctl start $u; systemctl show $u -p Result -p ExecMainStatus --no-pager; echo --- LOCAL_AFTER ---; find /var/lib/vz/dump /mnt/pve -type f -name \"vzdump-*\" -printf \"%s %TY-%Tm-%TdT%TH:%TM:%TS %p\\n\" 2>/dev/null | sort -nr; left=$(find /var/lib/vz/dump /mnt/pve -type f -name \"vzdump-*\" -print -quit 2>/dev/null); if test -n \"$left\"; then echo LOCAL_ARCHIVES_REMAIN=YES; exit 1; fi; echo LOCAL_ARCHIVES_REMAIN=NO\nACTIVE_VZDUMP=YES\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 173afb3..c87a268 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,103 +1,31 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=BACKUP-VM-DUMP-CONSISTENCY-078 -STATUS=OK -RC=0 +COMMAND_ID=BACKUP-DRAIN-LOCAL-ARCHIVES-079 +STATUS=FAIL +RC=75 HOST=pve01 -MODE=read-only -COMPONENT=backup-audit +MODE=change +COMPONENT=backup-remediation REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 -COMMAND_SHA256=876c832ac6a555af28ff29ccdceb28c79e7244a361f277e9278a8f75b65caad5 +COMMAND_SHA256=531e1da7ce1681eed084b22ced7da87eb1ee8995f662dc7559f4f9dde3c0ac32 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true -CHANGES_MADE=false +CHANGES_MADE=true SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=02a6668400353ad78630dc31cad6ae25943bc9bd92bc8aff5a46cc2f4dc89d8f -SANITIZED_OUTPUT_SHA256=02a6668400353ad78630dc31cad6ae25943bc9bd92bc8aff5a46cc2f4dc89d8f +RAW_EVIDENCE_SHA256=891e5c92df1afab4a71ce59fcd6e1cf23bf3d9473f8db7942404c001a4d1661b +SANITIZED_OUTPUT_SHA256=891e5c92df1afab4a71ce59fcd6e1cf23bf3d9473f8db7942404c001a4d1661b OUTPUT_BEGIN ========== NODE=pve01 ========== -1de52eb4ddb17b0ecff2cf3f3021e65b92e91fb44416f97943c51e248c512ed6 /usr/local/sbin/homelab-vm-local-dumps-2cloud -30:CHKS="$WORK/checksums.txt" -39:echo "chunk_size_bytes=$CHUNK_SIZE_BYTES" >> "$LOG" -50: SIZE="$(stat -c %s "$F" 2>/dev/null || echo 0)" -51: MT="$(date -u -d "@$(stat -c %Y "$F" 2>/dev/null || echo 0)" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)" -55: SHA="$(sha256sum "$F" | cut -d" " -f1)" -64: rm -rf "$CDIR" -66: echo "SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR" >> "$LOG" -69: (cd "$CDIR" && sha256sum part-* > chunks.sha256) -83:job,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:] -91: "size_bytes":int(p[2]), -92: "sha256":p[3], -107: "chunk_size_bytes":int(chunk_size), -112:DEST_A="${REMOTE_A%/}/${NODE}/vm-local-dumps/${JOB_ID}" -113:DEST_B="${REMOTE_B%/}/${NODE}/vm-local-dumps/${JOB_ID}" -116: DEST="$1" -117: rclone mkdir "$DEST/files" >> "$LOG" 2>&1 || BAD=$((BAD+1)) -118: while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do -121: echo "UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts" >> "$LOG" -122: rclone mkdir "$DEST/files/$bn.chunks" >> "$LOG" 2>&1 || BAD=$((BAD+1)) -126: rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) -128: rclone copyto "$CDIR/chunks.sha256" "$DEST/files/$bn.chunks/chunks.sha256" >> "$LOG" 2>&1 || BAD=$((BAD+1)) -130: echo "UPLOAD_DIRECT dest=$DEST file=$bn size=$size" >> "$LOG" -131: rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) -135: rclone copyto "$CHKS" "$DEST/checksums.txt" >> "$LOG" 2>&1 || BAD=$((BAD+1)) -136: rclone copyto "$FILES_TSV" "$DEST/files.tsv" >> "$LOG" 2>&1 || BAD=$((BAD+1)) -137: rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1)) -142: DEST="$2" -144: rm -rf "$VBASE" -146: rclone copyto "$DEST/manifest.json" "$VBASE/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1)) -149: while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do -154: rclone copy "$DEST/files/$bn.chunks" "$VDIR" --filter "+ part-*" --filter "+ chunks.sha256" --filter "- *" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) -155: (cd "$VDIR" && sha256sum -c chunks.sha256) >> "$LOG" 2>&1 || BAD=$((BAD+1)) -156: CALC="$(find "$VDIR" -maxdepth 1 -type f -name 'part-*' | sort | xargs cat | sha256sum | cut -d" " -f1)" -163: rm -rf "$VDIR" -166: rclone copyto "$DEST/files/$bn" "$VBASE/$bn" --transfers 1 >> "$LOG" 2>&1 || BAD=$((BAD+1)) -167: printf "%s %s\n" "$sha" "$VBASE/$bn" | sha256sum -c - >> "$LOG" 2>&1 || BAD=$((BAD+1)) -168: rm -f "$VBASE/$bn" -171: rm -rf "$VBASE" -175: upload_one_dest "$DEST_A" -176: upload_one_dest "$DEST_B" -180: verify_one_dest "A" "$DEST_A" -181: verify_one_dest "B" "$DEST_B" -184:rm -rf "$VERIFY_ROOT/$JOB_ID" -185:rm -rf "$CHUNK_ROOT/$JOB_ID" -191: --arg status "$STATUS" \ -193: --arg remote_a "$DEST_A" \ -194: --arg remote_b "$DEST_B" \ -198: --argjson chunk_size_bytes "$CHUNK_SIZE_BYTES" \ -200: "{job_id:\$job_id,status:\$status,node:\$node,remote_a:\$remote_a,remote_b:\$remote_b,count:\$count,total_bytes:\$total_bytes,chunked_count:\$chunked_count,chunk_size_bytes:\$chunk_size_bytes,log:\$log,local_dumps_deleted:false}" \ -203:echo "STATUS=$STATUS TS=$TS TYPE=vm-local-dumps-cloud BAD=$BAD COUNT=$COUNT CHUNKED=$CHUNKED TOTAL_BYTES=$TOTAL JOB=$JOB_ID REMOTE_A=$DEST_A REMOTE_B=$DEST_B LOG=$LOG" > "$OUT" -Result=success -ExecMainStatus=0 -NextElapseUSecRealtime=Sun 2026-07-26 10:29:15 MSK -ActiveState=active -UnitFileState=enabled -LOCAL_FILES=24 LOCAL_BYTES=30273877574 -========== NODE=pve02 ========== -bash: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory -sha256sum: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory -grep: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory -Result=success -ExecMainStatus=0 -NextElapseUSecRealtime= -ActiveState=inactive -UnitFileState= -LOCAL_FILES=23 LOCAL_BYTES=3888168125 -========== NODE=pve03 ========== -bash: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory -sha256sum: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory -grep: /usr/local/sbin/homelab-vm-local-dumps-2cloud: No such file or directory -Result=success -ExecMainStatus=0 -NextElapseUSecRealtime= -ActiveState=inactive -UnitFileState= -LOCAL_FILES=20 LOCAL_BYTES=47769002980 -VM_DUMP_CONSISTENCY=COMPLETE +559463 bash /usr/local/sbin/homelab-chat-run BACKUP-DRAIN-LOCAL-ARCHIVES-079 change backup-remediation -- /usr/bin/bash -c set -Eeuo pipefail; for n in $(pvesh get /nodes --output-format json | jq -r ".[].node" | sort); do echo "========== NODE=$n =========="; ssh -o BatchMode=yes -o ConnectTimeout=10 root@"$n" "set -Eeuo pipefail; p=/usr/local/sbin/homelab-vm-local-dumps-2cloud; u=homelab-vm-local-dumps-2cloud.service; test -x \$p; bash -n \$p; systemctl cat \$u >/dev/null; if pgrep -af \"[v]zdump\"; then echo ACTIVE_VZDUMP=YES; exit 75; fi; timeout 7000 systemctl start \$u; systemctl show \$u -p Result -p ExecMainStatus --no-pager; echo --- LOCAL_AFTER ---; find /var/lib/vz/dump /mnt/pve -type f -name \"vzdump-*\" -printf \"%s %TY-%Tm-%TdT%TH:%TM:%TS %p\n\" 2>/dev/null | sort -nr; left=\$(find /var/lib/vz/dump /mnt/pve -type f -name \"vzdump-*\" -print -quit 2>/dev/null); if test -n \"\$left\"; then echo LOCAL_ARCHIVES_REMAIN=YES; exit 1; fi; echo LOCAL_ARCHIVES_REMAIN=NO"; done; echo BACKUP_DRAIN=COMPLETE +559486 timeout --foreground --preserve-status 7200 /usr/bin/bash -c set -Eeuo pipefail; for n in $(pvesh get /nodes --output-format json | jq -r ".[].node" | sort); do echo "========== NODE=$n =========="; ssh -o BatchMode=yes -o ConnectTimeout=10 root@"$n" "set -Eeuo pipefail; p=/usr/local/sbin/homelab-vm-local-dumps-2cloud; u=homelab-vm-local-dumps-2cloud.service; test -x \$p; bash -n \$p; systemctl cat \$u >/dev/null; if pgrep -af \"[v]zdump\"; then echo ACTIVE_VZDUMP=YES; exit 75; fi; timeout 7000 systemctl start \$u; systemctl show \$u -p Result -p ExecMainStatus --no-pager; echo --- LOCAL_AFTER ---; find /var/lib/vz/dump /mnt/pve -type f -name \"vzdump-*\" -printf \"%s %TY-%Tm-%TdT%TH:%TM:%TS %p\n\" 2>/dev/null | sort -nr; left=\$(find /var/lib/vz/dump /mnt/pve -type f -name \"vzdump-*\" -print -quit 2>/dev/null); if test -n \"\$left\"; then echo LOCAL_ARCHIVES_REMAIN=YES; exit 1; fi; echo LOCAL_ARCHIVES_REMAIN=NO"; done; echo BACKUP_DRAIN=COMPLETE +559487 /usr/bin/bash -c set -Eeuo pipefail; for n in $(pvesh get /nodes --output-format json | jq -r ".[].node" | sort); do echo "========== NODE=$n =========="; ssh -o BatchMode=yes -o ConnectTimeout=10 root@"$n" "set -Eeuo pipefail; p=/usr/local/sbin/homelab-vm-local-dumps-2cloud; u=homelab-vm-local-dumps-2cloud.service; test -x \$p; bash -n \$p; systemctl cat \$u >/dev/null; if pgrep -af \"[v]zdump\"; then echo ACTIVE_VZDUMP=YES; exit 75; fi; timeout 7000 systemctl start \$u; systemctl show \$u -p Result -p ExecMainStatus --no-pager; echo --- LOCAL_AFTER ---; find /var/lib/vz/dump /mnt/pve -type f -name \"vzdump-*\" -printf \"%s %TY-%Tm-%TdT%TH:%TM:%TS %p\n\" 2>/dev/null | sort -nr; left=\$(find /var/lib/vz/dump /mnt/pve -type f -name \"vzdump-*\" -print -quit 2>/dev/null); if test -n \"\$left\"; then echo LOCAL_ARCHIVES_REMAIN=YES; exit 1; fi; echo LOCAL_ARCHIVES_REMAIN=NO"; done; echo BACKUP_DRAIN=COMPLETE +559499 ssh -o BatchMode=yes -o ConnectTimeout=10 root@pve01 set -Eeuo pipefail; p=/usr/local/sbin/homelab-vm-local-dumps-2cloud; u=homelab-vm-local-dumps-2cloud.service; test -x $p; bash -n $p; systemctl cat $u >/dev/null; if pgrep -af "[v]zdump"; then echo ACTIVE_VZDUMP=YES; exit 75; fi; timeout 7000 systemctl start $u; systemctl show $u -p Result -p ExecMainStatus --no-pager; echo --- LOCAL_AFTER ---; find /var/lib/vz/dump /mnt/pve -type f -name "vzdump-*" -printf "%s %TY-%Tm-%TdT%TH:%TM:%TS %p\n" 2>/dev/null | sort -nr; left=$(find /var/lib/vz/dump /mnt/pve -type f -name "vzdump-*" -print -quit 2>/dev/null); if test -n "$left"; then echo LOCAL_ARCHIVES_REMAIN=YES; exit 1; fi; echo LOCAL_ARCHIVES_REMAIN=NO +559508 bash -c set -Eeuo pipefail; p=/usr/local/sbin/homelab-vm-local-dumps-2cloud; u=homelab-vm-local-dumps-2cloud.service; test -x $p; bash -n $p; systemctl cat $u >/dev/null; if pgrep -af "[v]zdump"; then echo ACTIVE_VZDUMP=YES; exit 75; fi; timeout 7000 systemctl start $u; systemctl show $u -p Result -p ExecMainStatus --no-pager; echo --- LOCAL_AFTER ---; find /var/lib/vz/dump /mnt/pve -type f -name "vzdump-*" -printf "%s %TY-%Tm-%TdT%TH:%TM:%TS %p\n" 2>/dev/null | sort -nr; left=$(find /var/lib/vz/dump /mnt/pve -type f -name "vzdump-*" -print -quit 2>/dev/null); if test -n "$left"; then echo LOCAL_ARCHIVES_REMAIN=YES; exit 1; fi; echo LOCAL_ARCHIVES_REMAIN=NO +ACTIVE_VZDUMP=YES OUTPUT_END CHAT_OUTPUT_END