From 2b9546075614e744fc9e9aa67ef5ed1162ca1498 Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Fri, 24 Jul 2026 10:21:27 +0000 Subject: [PATCH] runtime: publish HOMELAB-VM-2CLOUD-EXACT-SELECTION-438 --- ...HOMELAB-VM-2CLOUD-EXACT-SELECTION-438.json | 30 ++ .../HOMELAB-VM-2CLOUD-EXACT-SELECTION-438.txt | 314 ++++++++++++++++++ runtime/latest.json | 14 +- runtime/latest.txt | 107 ++++-- 4 files changed, 439 insertions(+), 26 deletions(-) create mode 100644 runtime/history/HOMELAB-VM-2CLOUD-EXACT-SELECTION-438.json create mode 100644 runtime/history/HOMELAB-VM-2CLOUD-EXACT-SELECTION-438.txt diff --git a/runtime/history/HOMELAB-VM-2CLOUD-EXACT-SELECTION-438.json b/runtime/history/HOMELAB-VM-2CLOUD-EXACT-SELECTION-438.json new file mode 100644 index 0000000..17b5527 --- /dev/null +++ b/runtime/history/HOMELAB-VM-2CLOUD-EXACT-SELECTION-438.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "HOMELAB-VM-2CLOUD-EXACT-SELECTION-438", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "cluster-audit", + "started_at_utc": "2026-07-24T10:21:26Z", + "finished_at_utc": "2026-07-24T10:21:26Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", + "command_sha256": "458a61bfe476e4aeab9fdefc2057f75cf06c246702c653458e91f8c84eb823f2", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "1a2e8ce33d51e8861acf1434ee15eee384f0fcb6316b36d3b3fd87c946570531", + "sanitized_output_sha256": "1a2e8ce33d51e8861acf1434ee15eee384f0fcb6316b36d3b3fd87c946570531", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": ":33: FutureWarning: Possible nested set at position 2\n:33: FutureWarning: Possible nested set at position 29\n:43: FutureWarning: Possible nested set at position 2\n:43: FutureWarning: Possible nested set at position 29\nDETAIL=SCRIPT_SHA256=0ded6ba72e49b9cf7522c7f3d8840cc1026d2a173d4909dd3b4cfc5ff8831115\nDETAIL=SELECTION_SOURCE_RANGE=14-290\nDETAIL=SELECTION_SOURCE_BEGIN\nDETAIL=SOURCE_LINE=0014:TS=\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"\nDETAIL=SOURCE_LINE=0015:JOB_TS=\"$(date -u +%Y%m%dT%H%M%SZ)\"\nDETAIL=SOURCE_LINE=0016:NODE=\"$(hostname -s)\"\nDETAIL=SOURCE_LINE=0017:BAD=0\nDETAIL=SOURCE_LINE=0018:COUNT=0\nDETAIL=SOURCE_LINE=0019:TOTAL=0\nDETAIL=SOURCE_LINE=0020:CHUNKED=0\nDETAIL=SOURCE_LINE=0021:VMIDS=\"${VMIDS:-110 112 150}\"\nDETAIL=SOURCE_LINE=0022:\nDETAIL=SOURCE_LINE=0023:source /etc/homelab-backup.env 2>/dev/null || BAD=1\nDETAIL=SOURCE_LINE=0024:\nDETAIL=SOURCE_LINE=0025:JOB_ID=\"homelab-${NODE}-vm-local-dumps-${JOB_TS}\"\nDETAIL=SOURCE_LINE=0026:WORK=\"$ROOT/$JOB_ID\"\nDETAIL=SOURCE_LINE=0027:mkdir -p \"$WORK\"\nDETAIL=SOURCE_LINE=0028:\nDETAIL=SOURCE_LINE=0029:FILES_TSV=\"$WORK/files.tsv\"\nDETAIL=SOURCE_LINE=0030:CHKS=\"$WORK/checksums.txt\"\nDETAIL=SOURCE_LINE=0031:MAN=\"$WORK/manifest.json\"\nDETAIL=SOURCE_LINE=0032:\nDETAIL=SOURCE_LINE=0033:: > \"$FILES_TSV\"\nDETAIL=SOURCE_LINE=0034:: > \"$CHKS\"\nDETAIL=SOURCE_LINE=0035:: > \"$LOG\"\nDETAIL=SOURCE_LINE=0036:\nDETAIL=SOURCE_LINE=0037:echo \"job_id=$JOB_ID\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0038:echo \"vmids=$VMIDS\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0039:echo \"chunk_size_bytes=$CHUNK_SIZE_BYTES\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0040:\nDETAIL=SOURCE_LINE=0041:for id in $VMIDS; do\nDETAIL=SOURCE_LINE=0042: F=\"$(find /var/lib/vz/dump -maxdepth 1 -type f \\( -name \"vzdump-qemu-${id}-*.vma.zst\" -o -name \"vzdump-lxc-${id}-*.tar.zst\" \\) -printf \"%T@ %p\\n\" 2>/dev/null | sort -n | tail -n 1 | cut -d\" \" -f2-)\"\nDETAIL=SOURCE_LINE=0043: if [ -z \"$F\" ] || [ ! -f \"$F\" ]; then\nDETAIL=SOURCE_LINE=0044: echo \"MISSING vmid=$id\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0045: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0046: continue\nDETAIL=SOURCE_LINE=0047: fi\nDETAIL=SOURCE_LINE=0048:\nDETAIL=SOURCE_LINE=0049: BN=\"$(basename \"$F\")\"\nDETAIL=SOURCE_LINE=0050: SIZE=\"$(stat -c %s \"$F\" 2>/dev/null || echo 0)\"\nDETAIL=SOURCE_LINE=0051: MT=\"$(date -u -d \"@$(stat -c %Y \"$F\" 2>/dev/null || echo 0)\" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)\"\nDETAIL=SOURCE_LINE=0052: KIND=\"unknown\"\nDETAIL=SOURCE_LINE=0053: echo \"$BN\" | grep -q \"^vzdump-qemu-\" && KIND=\"qemu\"\nDETAIL=SOURCE_LINE=0054: echo \"$BN\" | grep -q \"^vzdump-lxc-\" && KIND=\"lxc\"\nDETAIL=SOURCE_LINE=0055: SHA=\"$(sha256sum \"$F\" | cut -d\" \" -f1)\"\nDETAIL=SOURCE_LINE=0056: printf \"%s %s\\n\" \"$SHA\" \"$BN\" >> \"$CHKS\"\nDETAIL=SOURCE_LINE=0057:\nDETAIL=SOURCE_LINE=0058: MODE=\"direct\"\nDETAIL=SOURCE_LINE=0059: PARTS=0\nDETAIL=SOURCE_LINE=0060: if [ \"$SIZE\" -gt \"$CHUNK_SIZE_BYTES\" ]; then\nDETAIL=SOURCE_LINE=0061: MODE=\"chunked\"\nDETAIL=SOURCE_LINE=0062: CHUNKED=$((CHUNKED+1))\nDETAIL=SOURCE_LINE=0063: CDIR=\"$CHUNK_ROOT/$JOB_ID/$BN.chunks\"\nDETAIL=SOURCE_LINE=0064: rm -rf \"$CDIR\"\nDETAIL=SOURCE_LINE=0065: mkdir -p \"$CDIR\"\nDETAIL=SOURCE_LINE=0066: echo \"SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0067: split -b \"$CHUNK_SIZE_SPLIT\" -d -a 4 \"$F\" \"$CDIR/part-\" >> \"$LOG\" 2>&1\nDETAIL=SOURCE_LINE=0068: PARTS=\"$(find \"$CDIR\" -maxdepth 1 -type f -name 'part-*' | wc -l)\"\nDETAIL=SOURCE_LINE=0069: (cd \"$CDIR\" && sha256sum part-* > chunks.sha256)\nDETAIL=SOURCE_LINE=0070: if [ \"$PARTS\" -le 0 ]; then\nDETAIL=SOURCE_LINE=0071: echo \"BAD split produced no chunks file=$BN\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0072: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0073: fi\nDETAIL=SOURCE_LINE=0074: fi\nDETAIL=SOURCE_LINE=0075:\nDETAIL=SOURCE_LINE=0076: printf \"%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n\" \"$id\" \"$KIND\" \"$SIZE\" \"$SHA\" \"$MT\" \"$BN\" \"$F\" \"$MODE\" \"$PARTS\" >> \"$FILES_TSV\"\nDETAIL=SOURCE_LINE=0077: COUNT=$((COUNT+1))\nDETAIL=SOURCE_LINE=0078: TOTAL=$((TOTAL+SIZE))\nDETAIL=SOURCE_LINE=0079:done\nDETAIL=SOURCE_LINE=0080:\nDETAIL=SOURCE_LINE=0081:python3 - \"$JOB_ID\" \"$NODE\" \"$TS\" \"$VMIDS\" \"$COUNT\" \"$TOTAL\" \"$CHUNKED\" \"$CHUNK_SIZE_BYTES\" \"$FILES_TSV\" > \"$MAN\" <<'PY'\nDETAIL=SOURCE_LINE=0082:import json, sys, pathlib\nDETAIL=SOURCE_LINE=0083:job,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:]\nDETAIL=SOURCE_LINE=0084:items=[]\nDETAIL=SOURCE_LINE=0085:for line in pathlib.Path(tsv).read_text().splitlines():\nDETAIL=SOURCE_LINE=0086: p=line.split(\"\\t\")\nDETAIL=SOURCE_LINE=0087: if len(p)>=9:\nDETAIL=SOURCE_LINE=0088: items.append({\nDETAIL=SOURCE_LINE=0089: \"vmid\":p[0],\nDETAIL=SOURCE_LINE=0090: \"kind\":p[1],\nDETAIL=SOURCE_LINE=0091: \"size_bytes\":int(p[2]),\nDETAIL=SOURCE_LINE=0092: \"sha256\":p[3],\nDETAIL=SOURCE_LINE=0093: \"mtime_utc\":p[4],\nDETAIL=SOURCE_LINE=0094: \"filename\":p[5],\nDETAIL=SOURCE_LINE=0095: \"source_path\":p[6],\nDETAIL=SOURCE_LINE=0096: \"mode\":p[7],\nDETAIL=SOURCE_LINE=0097: \"parts\":int(p[8]),\nDETAIL=SOURCE_LINE=0098: })\nDETAIL=SOURCE_LINE=0099:print(json.dumps({\nDETAIL=SOURCE_LINE=0100: \"job_id\":job,\nDETAIL=SOURCE_LINE=0101: \"node\":node,\nDETAIL=SOURCE_LINE=0102: \"generated_utc\":ts,\nDETAIL=SOURCE_LINE=0103: \"vmids\":vmids.split(),\nDETAIL=SOURCE_LINE=0104: \"count\":int(count),\nDETAIL=SOURCE_LINE=0105: \"total_bytes\":int(total),\nDETAIL=SOURCE_LINE=0106: \"chunked_count\":int(chunked),\nDETAIL=SOURCE_LINE=0107: \"chunk_size_bytes\":int(chunk_size),\nDETAIL=SOURCE_LINE=0108: \"items\":items,\nDETAIL=SOURCE_LINE=0109:}, indent=2))\nDETAIL=SOURCE_LINE=0110:PY\nDETAIL=SOURCE_LINE=0111:\nDETAIL=SOURCE_LINE=0112:DEST_A=\"${REMOTE_A%/}/${NODE}/vm-local-dumps/${JOB_ID}\"\nDETAIL=SOURCE_LINE=0113:DEST_B=\"${REMOTE_B%/}/${NODE}/vm-local-dumps/${JOB_ID}\"\nDETAIL=SOURCE_LINE=0114:\nDETAIL=SOURCE_LINE=0115:upload_one_dest() {\nDETAIL=SOURCE_LINE=0116: DEST=\"$1\"\nDETAIL=SOURCE_LINE=0117: rclone mkdir \"$DEST/files\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0118: while IFS=\"$(printf \"\\t\")\" read -r id kind size sha mt bn path mode parts; do\nDETAIL=SOURCE_LINE=0119: if [ \"$mode\" = \"chunked\" ]; then\nDETAIL=SOURCE_LINE=0120: CDIR=\"$CHUNK_ROOT/$JOB_ID/$bn.chunks\"\nDETAIL=SOURCE_LINE=0121: echo \"UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0122: rclone mkdir \"$DEST/files/$bn.chunks\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0123: for part in \"$CDIR\"/part-*; do\nDETAIL=SOURCE_LINE=0124: [ -f \"$part\" ] || continue\nDETAIL=SOURCE_LINE=0125: pn=\"$(basename \"$part\")\"\nDETAIL=SOURCE_LINE=0126: rclone copyto \"$part\" \"$DEST/files/$bn.chunks/$pn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0127: done\nDETAIL=SOURCE_LINE=0128: rclone copyto \"$CDIR/chunks.sha256\" \"$DEST/files/$bn.chunks/chunks.sha256\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0129: else\nDETAIL=SOURCE_LINE=0130: echo \"UPLOAD_DIRECT dest=$DEST file=$bn size=$size\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0131: rclone copyto \"$path\" \"$DEST/files/$bn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0132: fi\nDETAIL=SOURCE_LINE=0133: done < \"$FILES_TSV\"\nDETAIL=SOURCE_LINE=0134:\nDETAIL=SOURCE_LINE=0135: rclone copyto \"$CHKS\" \"$DEST/checksums.txt\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0136: rclone copyto \"$FILES_TSV\" \"$DEST/files.tsv\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0137: rclone copyto \"$MAN\" \"$DEST/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0138:}\nDETAIL=SOURCE_LINE=0139:\nDETAIL=SOURCE_LINE=0140:verify_one_dest() {\nDETAIL=SOURCE_LINE=0141: LABEL=\"$1\"\nDETAIL=SOURCE_LINE=0142: DEST=\"$2\"\nDETAIL=SOURCE_LINE=0143: VBASE=\"$VERIFY_ROOT/$JOB_ID/$LABEL\"\nDETAIL=SOURCE_LINE=0144: rm -rf \"$VBASE\"\nDETAIL=SOURCE_LINE=0145: mkdir -p \"$VBASE\"\nDETAIL=SOURCE_LINE=0146: rclone copyto \"$DEST/manifest.json\" \"$VBASE/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0147: test -s \"$VBASE/manifest.json\" || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0148:\nDETAIL=SOURCE_LINE=0149: while IFS=\"$(printf \"\\t\")\" read -r id kind size sha mt bn path mode parts; do\nDETAIL=SOURCE_LINE=0150: if [ \"$mode\" = \"chunked\" ]; then\nDETAIL=SOURCE_LINE=0151: VDIR=\"$VBASE/$bn.chunks\"\nDETAIL=SOURCE_LINE=0152: mkdir -p \"$VDIR\"\nDETAIL=SOURCE_LINE=0153: echo \"VERIFY_CHUNKED label=$LABEL file=$bn parts=$parts\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0154: rclone copy \"$DEST/files/$bn.chunks\" \"$VDIR\" --filter \"+ part-*\" --filter \"+ chunks.sha256\" --filter \"- *\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0155: (cd \"$VDIR\" && sha256sum -c chunks.sha256) >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0156: CALC=\"$(find \"$VDIR\" -maxdepth 1 -type f -name 'part-*' | sort | xargs cat | sha256sum | cut -d\" \" -f1)\"\nDETAIL=SOURCE_LINE=0157: if [ \"$CALC\" = \"$sha\" ]; then\nDETAIL=SOURCE_LINE=0158: echo \"OK combined_sha label=$LABEL file=$bn sha=$sha\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0159: else\nDETAIL=SOURCE_LINE=0160: echo \"BAD combined_sha label=$LABEL file=$bn expected=$sha got=$CALC\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0161: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0162: fi\nDETAIL=SOURCE_LINE=0163: rm -rf \"$VDIR\"\nDETAIL=SOURCE_LINE=0164: else\nDETAIL=SOURCE_LINE=0165: echo \"VERIFY_DIRECT label=$LABEL file=$bn\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0166: rclone copyto \"$DEST/files/$bn\" \"$VBASE/$bn\" --transfers 1 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0167: printf \"%s %s\\n\" \"$sha\" \"$VBASE/$bn\" | sha256sum -c - >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0168: rm -f \"$VBASE/$bn\"\nDETAIL=SOURCE_LINE=0169: fi\nDETAIL=SOURCE_LINE=0170: done < \"$FILES_TSV\"\nDETAIL=SOURCE_LINE=0171: rm -rf \"$VBASE\"\nDETAIL=SOURCE_LINE=0172:}\nDETAIL=SOURCE_LINE=0173:\nDETAIL=SOURCE_LINE=0174:if [ \"$BAD\" -eq 0 ]; then\nDETAIL=SOURCE_LINE=0175: upload_one_dest \"$DEST_A\"\nDETAIL=SOURCE_LINE=0176: upload_one_dest \"$DEST_B\"\nDETAIL=SOURCE_LINE=0177:fi\nDETAIL=SOURCE_LINE=0178:\nDETAIL=SOURCE_LINE=0179:if [ \"$BAD\" -eq 0 ]; then\nDETAIL=SOURCE_LINE=0180: verify_one_dest \"A\" \"$DEST_A\"\nDETAIL=SOURCE_LINE=0181: verify_one_dest \"B\" \"$DEST_B\"\nDETAIL=SOURCE_LINE=0182:fi\nDETAIL=SOURCE_LINE=0183:\nDETAIL=SOURCE_LINE=0184:rm -rf \"$VERIFY_ROOT/$JOB_ID\"\nDETAIL=SOURCE_LINE=0185:rm -rf \"$CHUNK_ROOT/$JOB_ID\"\nDETAIL=SOURCE_LINE=0186:\nDETAIL=SOURCE_LINE=0187:# HOMELAB_NO_LOCAL_AFTER_DUAL_CLOUD_434\nDETAIL=SOURCE_LINE=0188:LOCAL_DUMPS_DELETED=false\nDETAIL=SOURCE_LINE=0189:DELETED_COUNT=0\nDETAIL=SOURCE_LINE=0190:DELETED_BYTES=0\nDETAIL=SOURCE_LINE=0191:\nDETAIL=SOURCE_LINE=0192:if [ \"$BAD\" -eq 0 ]; then\nDETAIL=SOURCE_LINE=0193: echo \"PREDELETE_VALIDATION_BEGIN\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0194:\nDETAIL=SOURCE_LINE=0195: while IFS=\"$(printf \"\t\")\" read -r id kind size sha mt bn path mode parts; do\nDETAIL=SOURCE_LINE=0196: case \"$path\" in\nDETAIL=SOURCE_LINE=0197: /var/lib/vz/dump/*)\nDETAIL=SOURCE_LINE=0198: ;;\nDETAIL=SOURCE_LINE=0199: *)\nDETAIL=SOURCE_LINE=0200: echo \"REFUSE_DELETE_OUTSIDE_ALLOWED_ROOT path=$path\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0201: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0202: continue\nDETAIL=SOURCE_LINE=0203: ;;\nDETAIL=SOURCE_LINE=0204: esac\nDETAIL=SOURCE_LINE=0205:\nDETAIL=SOURCE_LINE=0206: if [ \"$(basename \"$path\")\" != \"$bn\" ]; then\nDETAIL=SOURCE_LINE=0207: echo \"REFUSE_DELETE_BASENAME_MISMATCH path=$path expected=$bn\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0208: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0209: continue\nDETAIL=SOURCE_LINE=0210: fi\nDETAIL=SOURCE_LINE=0211:\nDETAIL=SOURCE_LINE=0212: if [ ! -f \"$path\" ] || [ -L \"$path\" ]; then\nDETAIL=SOURCE_LINE=0213: echo \"REFUSE_DELETE_INVALID_SOURCE path=$path\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0214: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0215: continue\nDETAIL=SOURCE_LINE=0216: fi\nDETAIL=SOURCE_LINE=0217:\nDETAIL=SOURCE_LINE=0218: actual_size=\"$(stat -c %s \"$path\" 2>/dev/null || echo 0)\"\nDETAIL=SOURCE_LINE=0219:\nDETAIL=SOURCE_LINE=0220: if [ \"$actual_size\" != \"$size\" ]; then\nDETAIL=SOURCE_LINE=0221: echo \"REFUSE_DELETE_SIZE_MISMATCH path=$path expected=$size actual=$actual_size\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0222: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0223: continue\nDETAIL=SOURCE_LINE=0224: fi\nDETAIL=SOURCE_LINE=0225:\nDETAIL=SOURCE_LINE=0226: actual_sha=\"$(sha256sum \"$path\" 2>>\"$LOG\" | cut -d \" \" -f1 || true)\"\nDETAIL=SOURCE_LINE=0227:\nDETAIL=SOURCE_LINE=0228: if [ \"$actual_sha\" != \"$sha\" ]; then\nDETAIL=SOURCE_LINE=0229: echo \"REFUSE_DELETE_SHA_MISMATCH path=$path\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0230: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0231: continue\nDETAIL=SOURCE_LINE=0232: fi\nDETAIL=SOURCE_LINE=0233:\nDETAIL=SOURCE_LINE=0234: echo \"PREDELETE_VALIDATION_OK path=$path size=$size sha=$sha\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0235: done < \"$FILES_TSV\"\nDETAIL=SOURCE_LINE=0236:\nDETAIL=SOURCE_LINE=0237: echo \"PREDELETE_VALIDATION_END bad=$BAD\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0238:fi\nDETAIL=SOURCE_LINE=0239:\nDETAIL=SOURCE_LINE=0240:if [ \"$BAD\" -eq 0 ]; then\nDETAIL=SOURCE_LINE=0241: echo \"LOCAL_PAYLOAD_DELETE_BEGIN\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0242:\nDETAIL=SOURCE_LINE=0243: while IFS=\"$(printf \"\t\")\" read -r id kind size sha mt bn path mode parts; do\nDETAIL=SOURCE_LINE=0244: if rm -f -- \"$path\" 2>>\"$LOG\"; then\nDETAIL=SOURCE_LINE=0245: if [ ! -e \"$path\" ] && [ ! -L \"$path\" ]; then\nDETAIL=SOURCE_LINE=0246: DELETED_COUNT=$((DELETED_COUNT+1))\nDETAIL=SOURCE_LINE=0247: DELETED_BYTES=$((DELETED_BYTES+size))\nDETAIL=SOURCE_LINE=0248: echo \"LOCAL_PAYLOAD_DELETED path=$path bytes=$size\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0249: else\nDETAIL=SOURCE_LINE=0250: echo \"LOCAL_PAYLOAD_DELETE_CONFIRM_FAILED path=$path\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0251: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0252: fi\nDETAIL=SOURCE_LINE=0253: else\nDETAIL=SOURCE_LINE=0254: echo \"LOCAL_PAYLOAD_DELETE_FAILED path=$path\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0255: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0256: fi\nDETAIL=SOURCE_LINE=0257: done < \"$FILES_TSV\"\nDETAIL=SOURCE_LINE=0258:\nDETAIL=SOURCE_LINE=0259: if [ \"$BAD\" -eq 0 ] && [ \"$DELETED_COUNT\" -eq \"$COUNT\" ]; then\nDETAIL=SOURCE_LINE=0260: LOCAL_DUMPS_DELETED=true\nDETAIL=SOURCE_LINE=0261: fi\nDETAIL=SOURCE_LINE=0262:\nDETAIL=SOURCE_LINE=0263: echo \"LOCAL_PAYLOAD_DELETE_END deleted_count=$DELETED_COUNT deleted_bytes=$DELETED_BYTES bad=$BAD\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0264:fi\nDETAIL=SOURCE_LINE=0265:\nDETAIL=SOURCE_LINE=0266:if [ \"$BAD\" -eq 0 ] && [ \"$LOCAL_DUMPS_DELETED\" = true ]; then\nDETAIL=SOURCE_LINE=0267: STATUS=\"OK\"\nDETAIL=SOURCE_LINE=0268:else\nDETAIL=SOURCE_LINE=0269: STATUS=\"WARN\"\nDETAIL=SOURCE_LINE=0270:fi\nDETAIL=SOURCE_LINE=0271:\nDETAIL=SOURCE_LINE=0272:jq -n \\\nDETAIL=SOURCE_LINE=0273: --arg job_id \"$JOB_ID\" \\\nDETAIL=SOURCE_LINE=0274: --arg status \"$STATUS\" \\\nDETAIL=SOURCE_LINE=0275: --arg node \"$NODE\" \\\nDETAIL=SOURCE_LINE=0276: --arg remote_a \"$DEST_A\" \\\nDETAIL=SOURCE_LINE=0277: --arg remote_b \"$DEST_B\" \\\nDETAIL=SOURCE_LINE=0278: --argjson count \"$COUNT\" \\\nDETAIL=SOURCE_LINE=0279: --argjson total_bytes \"$TOTAL\" \\\nDETAIL=SOURCE_LINE=0280: --argjson chunked_count \"$CHUNKED\" \\\nDETAIL=SOURCE_LINE=0281: --argjson chunk_size_bytes \"$CHUNK_SIZE_BYTES\" \\\nDETAIL=SOURCE_LINE=0282: --arg log \"$LOG\" \\\nDETAIL=SOURCE_LINE=0283: --argjson local_dumps_deleted \"$LOCAL_DUMPS_DELETED\" \\\nDETAIL=SOURCE_LINE=0284: --argjson deleted_count \"$DELETED_COUNT\" \\\nDETAIL=SOURCE_LINE=0285: --argjson deleted_bytes \"$DELETED_BYTES\" \\\nDETAIL=SOURCE_LINE=0286: \"{job_id:\\$job_id,status:\\$status,node:\\$node,remote_a:\\$remote_a,remote_b:\\$remote_b,count:\\$count,total_bytes:\\$total_bytes,chunked_count:\\$chunked_count,chunk_size_bytes:\\$chunk_size_bytes,log:\\$log,local_dumps_deleted:\\$local_dumps_deleted,deleted_count:\\$deleted_count,deleted_bytes:\\$deleted_bytes}\" \\\nDETAIL=SOURCE_LINE=0287: > \"$ROOT/latest-vm-local-dumps-cloud.json\"\nDETAIL=SOURCE_LINE=0288:\nDETAIL=SOURCE_LINE=0289:echo \"STATUS=$STATUS TS=$TS TYPE=vm-local-dumps-cloud BAD=$BAD COUNT=$COUNT CHUNKED=$CHUNKED TOTAL_BYTES=$TOTAL JOB=$JOB_ID REMOTE_A=$DEST_A REMOTE_B=$DEST_B LOCAL_DUMPS_DELETED=$LOCAL_DUMPS_DELETED DELETED_COUNT=$DELETED_COUNT DELETED_BYTES=$DELETED_BYTES LOG=$LOG\" > \"$OUT\"\nDETAIL=SOURCE_LINE=0290:cat \"$OUT\"\nDETAIL=SELECTION_SOURCE_END\nDETAIL=ARCHIVES_CHANGED=NO\nDETAIL=SERVICE_STARTED=NO\nDETAIL=SECRET_VALUES_OUTPUT=NONE\nDETAIL=EXACT_SELECTION_AUDIT=PASS\nCHANGES_MADE=NO\n" +} diff --git a/runtime/history/HOMELAB-VM-2CLOUD-EXACT-SELECTION-438.txt b/runtime/history/HOMELAB-VM-2CLOUD-EXACT-SELECTION-438.txt new file mode 100644 index 0000000..3e2f4a4 --- /dev/null +++ b/runtime/history/HOMELAB-VM-2CLOUD-EXACT-SELECTION-438.txt @@ -0,0 +1,314 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=HOMELAB-VM-2CLOUD-EXACT-SELECTION-438 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=cluster-audit +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 +COMMAND_SHA256=458a61bfe476e4aeab9fdefc2057f75cf06c246702c653458e91f8c84eb823f2 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=false +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=1a2e8ce33d51e8861acf1434ee15eee384f0fcb6316b36d3b3fd87c946570531 +SANITIZED_OUTPUT_SHA256=1a2e8ce33d51e8861acf1434ee15eee384f0fcb6316b36d3b3fd87c946570531 +OUTPUT_BEGIN +:33: FutureWarning: Possible nested set at position 2 +:33: FutureWarning: Possible nested set at position 29 +:43: FutureWarning: Possible nested set at position 2 +:43: FutureWarning: Possible nested set at position 29 +DETAIL=SCRIPT_SHA256=0ded6ba72e49b9cf7522c7f3d8840cc1026d2a173d4909dd3b4cfc5ff8831115 +DETAIL=SELECTION_SOURCE_RANGE=14-290 +DETAIL=SELECTION_SOURCE_BEGIN +DETAIL=SOURCE_LINE=0014:TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +DETAIL=SOURCE_LINE=0015:JOB_TS="$(date -u +%Y%m%dT%H%M%SZ)" +DETAIL=SOURCE_LINE=0016:NODE="$(hostname -s)" +DETAIL=SOURCE_LINE=0017:BAD=0 +DETAIL=SOURCE_LINE=0018:COUNT=0 +DETAIL=SOURCE_LINE=0019:TOTAL=0 +DETAIL=SOURCE_LINE=0020:CHUNKED=0 +DETAIL=SOURCE_LINE=0021:VMIDS="${VMIDS:-110 112 150}" +DETAIL=SOURCE_LINE=0022: +DETAIL=SOURCE_LINE=0023:source /etc/homelab-backup.env 2>/dev/null || BAD=1 +DETAIL=SOURCE_LINE=0024: +DETAIL=SOURCE_LINE=0025:JOB_ID="homelab-${NODE}-vm-local-dumps-${JOB_TS}" +DETAIL=SOURCE_LINE=0026:WORK="$ROOT/$JOB_ID" +DETAIL=SOURCE_LINE=0027:mkdir -p "$WORK" +DETAIL=SOURCE_LINE=0028: +DETAIL=SOURCE_LINE=0029:FILES_TSV="$WORK/files.tsv" +DETAIL=SOURCE_LINE=0030:CHKS="$WORK/checksums.txt" +DETAIL=SOURCE_LINE=0031:MAN="$WORK/manifest.json" +DETAIL=SOURCE_LINE=0032: +DETAIL=SOURCE_LINE=0033:: > "$FILES_TSV" +DETAIL=SOURCE_LINE=0034:: > "$CHKS" +DETAIL=SOURCE_LINE=0035:: > "$LOG" +DETAIL=SOURCE_LINE=0036: +DETAIL=SOURCE_LINE=0037:echo "job_id=$JOB_ID" >> "$LOG" +DETAIL=SOURCE_LINE=0038:echo "vmids=$VMIDS" >> "$LOG" +DETAIL=SOURCE_LINE=0039:echo "chunk_size_bytes=$CHUNK_SIZE_BYTES" >> "$LOG" +DETAIL=SOURCE_LINE=0040: +DETAIL=SOURCE_LINE=0041:for id in $VMIDS; do +DETAIL=SOURCE_LINE=0042: F="$(find /var/lib/vz/dump -maxdepth 1 -type f \( -name "vzdump-qemu-${id}-*.vma.zst" -o -name "vzdump-lxc-${id}-*.tar.zst" \) -printf "%T@ %p\n" 2>/dev/null | sort -n | tail -n 1 | cut -d" " -f2-)" +DETAIL=SOURCE_LINE=0043: if [ -z "$F" ] || [ ! -f "$F" ]; then +DETAIL=SOURCE_LINE=0044: echo "MISSING vmid=$id" >> "$LOG" +DETAIL=SOURCE_LINE=0045: BAD=$((BAD+1)) +DETAIL=SOURCE_LINE=0046: continue +DETAIL=SOURCE_LINE=0047: fi +DETAIL=SOURCE_LINE=0048: +DETAIL=SOURCE_LINE=0049: BN="$(basename "$F")" +DETAIL=SOURCE_LINE=0050: SIZE="$(stat -c %s "$F" 2>/dev/null || echo 0)" +DETAIL=SOURCE_LINE=0051: MT="$(date -u -d "@$(stat -c %Y "$F" 2>/dev/null || echo 0)" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)" +DETAIL=SOURCE_LINE=0052: KIND="unknown" +DETAIL=SOURCE_LINE=0053: echo "$BN" | grep -q "^vzdump-qemu-" && KIND="qemu" +DETAIL=SOURCE_LINE=0054: echo "$BN" | grep -q "^vzdump-lxc-" && KIND="lxc" +DETAIL=SOURCE_LINE=0055: SHA="$(sha256sum "$F" | cut -d" " -f1)" +DETAIL=SOURCE_LINE=0056: printf "%s %s\n" "$SHA" "$BN" >> "$CHKS" +DETAIL=SOURCE_LINE=0057: +DETAIL=SOURCE_LINE=0058: MODE="direct" +DETAIL=SOURCE_LINE=0059: PARTS=0 +DETAIL=SOURCE_LINE=0060: if [ "$SIZE" -gt "$CHUNK_SIZE_BYTES" ]; then +DETAIL=SOURCE_LINE=0061: MODE="chunked" +DETAIL=SOURCE_LINE=0062: CHUNKED=$((CHUNKED+1)) +DETAIL=SOURCE_LINE=0063: CDIR="$CHUNK_ROOT/$JOB_ID/$BN.chunks" +DETAIL=SOURCE_LINE=0064: rm -rf "$CDIR" +DETAIL=SOURCE_LINE=0065: mkdir -p "$CDIR" +DETAIL=SOURCE_LINE=0066: echo "SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR" >> "$LOG" +DETAIL=SOURCE_LINE=0067: split -b "$CHUNK_SIZE_SPLIT" -d -a 4 "$F" "$CDIR/part-" >> "$LOG" 2>&1 +DETAIL=SOURCE_LINE=0068: PARTS="$(find "$CDIR" -maxdepth 1 -type f -name 'part-*' | wc -l)" +DETAIL=SOURCE_LINE=0069: (cd "$CDIR" && sha256sum part-* > chunks.sha256) +DETAIL=SOURCE_LINE=0070: if [ "$PARTS" -le 0 ]; then +DETAIL=SOURCE_LINE=0071: echo "BAD split produced no chunks file=$BN" >> "$LOG" +DETAIL=SOURCE_LINE=0072: BAD=$((BAD+1)) +DETAIL=SOURCE_LINE=0073: fi +DETAIL=SOURCE_LINE=0074: fi +DETAIL=SOURCE_LINE=0075: +DETAIL=SOURCE_LINE=0076: printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" "$id" "$KIND" "$SIZE" "$SHA" "$MT" "$BN" "$F" "$MODE" "$PARTS" >> "$FILES_TSV" +DETAIL=SOURCE_LINE=0077: COUNT=$((COUNT+1)) +DETAIL=SOURCE_LINE=0078: TOTAL=$((TOTAL+SIZE)) +DETAIL=SOURCE_LINE=0079:done +DETAIL=SOURCE_LINE=0080: +DETAIL=SOURCE_LINE=0081:python3 - "$JOB_ID" "$NODE" "$TS" "$VMIDS" "$COUNT" "$TOTAL" "$CHUNKED" "$CHUNK_SIZE_BYTES" "$FILES_TSV" > "$MAN" <<'PY' +DETAIL=SOURCE_LINE=0082:import json, sys, pathlib +DETAIL=SOURCE_LINE=0083:job,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:] +DETAIL=SOURCE_LINE=0084:items=[] +DETAIL=SOURCE_LINE=0085:for line in pathlib.Path(tsv).read_text().splitlines(): +DETAIL=SOURCE_LINE=0086: p=line.split("\t") +DETAIL=SOURCE_LINE=0087: if len(p)>=9: +DETAIL=SOURCE_LINE=0088: items.append({ +DETAIL=SOURCE_LINE=0089: "vmid":p[0], +DETAIL=SOURCE_LINE=0090: "kind":p[1], +DETAIL=SOURCE_LINE=0091: "size_bytes":int(p[2]), +DETAIL=SOURCE_LINE=0092: "sha256":p[3], +DETAIL=SOURCE_LINE=0093: "mtime_utc":p[4], +DETAIL=SOURCE_LINE=0094: "filename":p[5], +DETAIL=SOURCE_LINE=0095: "source_path":p[6], +DETAIL=SOURCE_LINE=0096: "mode":p[7], +DETAIL=SOURCE_LINE=0097: "parts":int(p[8]), +DETAIL=SOURCE_LINE=0098: }) +DETAIL=SOURCE_LINE=0099:print(json.dumps({ +DETAIL=SOURCE_LINE=0100: "job_id":job, +DETAIL=SOURCE_LINE=0101: "node":node, +DETAIL=SOURCE_LINE=0102: "generated_utc":ts, +DETAIL=SOURCE_LINE=0103: "vmids":vmids.split(), +DETAIL=SOURCE_LINE=0104: "count":int(count), +DETAIL=SOURCE_LINE=0105: "total_bytes":int(total), +DETAIL=SOURCE_LINE=0106: "chunked_count":int(chunked), +DETAIL=SOURCE_LINE=0107: "chunk_size_bytes":int(chunk_size), +DETAIL=SOURCE_LINE=0108: "items":items, +DETAIL=SOURCE_LINE=0109:}, indent=2)) +DETAIL=SOURCE_LINE=0110:PY +DETAIL=SOURCE_LINE=0111: +DETAIL=SOURCE_LINE=0112:DEST_A="${REMOTE_A%/}/${NODE}/vm-local-dumps/${JOB_ID}" +DETAIL=SOURCE_LINE=0113:DEST_B="${REMOTE_B%/}/${NODE}/vm-local-dumps/${JOB_ID}" +DETAIL=SOURCE_LINE=0114: +DETAIL=SOURCE_LINE=0115:upload_one_dest() { +DETAIL=SOURCE_LINE=0116: DEST="$1" +DETAIL=SOURCE_LINE=0117: rclone mkdir "$DEST/files" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SOURCE_LINE=0118: while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do +DETAIL=SOURCE_LINE=0119: if [ "$mode" = "chunked" ]; then +DETAIL=SOURCE_LINE=0120: CDIR="$CHUNK_ROOT/$JOB_ID/$bn.chunks" +DETAIL=SOURCE_LINE=0121: echo "UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts" >> "$LOG" +DETAIL=SOURCE_LINE=0122: rclone mkdir "$DEST/files/$bn.chunks" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SOURCE_LINE=0123: for part in "$CDIR"/part-*; do +DETAIL=SOURCE_LINE=0124: [ -f "$part" ] || continue +DETAIL=SOURCE_LINE=0125: pn="$(basename "$part")" +DETAIL=SOURCE_LINE=0126: rclone copyto "$part" "$DEST/files/$bn.chunks/$pn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SOURCE_LINE=0127: done +DETAIL=SOURCE_LINE=0128: rclone copyto "$CDIR/chunks.sha256" "$DEST/files/$bn.chunks/chunks.sha256" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SOURCE_LINE=0129: else +DETAIL=SOURCE_LINE=0130: echo "UPLOAD_DIRECT dest=$DEST file=$bn size=$size" >> "$LOG" +DETAIL=SOURCE_LINE=0131: rclone copyto "$path" "$DEST/files/$bn" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SOURCE_LINE=0132: fi +DETAIL=SOURCE_LINE=0133: done < "$FILES_TSV" +DETAIL=SOURCE_LINE=0134: +DETAIL=SOURCE_LINE=0135: rclone copyto "$CHKS" "$DEST/checksums.txt" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SOURCE_LINE=0136: rclone copyto "$FILES_TSV" "$DEST/files.tsv" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SOURCE_LINE=0137: rclone copyto "$MAN" "$DEST/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SOURCE_LINE=0138:} +DETAIL=SOURCE_LINE=0139: +DETAIL=SOURCE_LINE=0140:verify_one_dest() { +DETAIL=SOURCE_LINE=0141: LABEL="$1" +DETAIL=SOURCE_LINE=0142: DEST="$2" +DETAIL=SOURCE_LINE=0143: VBASE="$VERIFY_ROOT/$JOB_ID/$LABEL" +DETAIL=SOURCE_LINE=0144: rm -rf "$VBASE" +DETAIL=SOURCE_LINE=0145: mkdir -p "$VBASE" +DETAIL=SOURCE_LINE=0146: rclone copyto "$DEST/manifest.json" "$VBASE/manifest.json" >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SOURCE_LINE=0147: test -s "$VBASE/manifest.json" || BAD=$((BAD+1)) +DETAIL=SOURCE_LINE=0148: +DETAIL=SOURCE_LINE=0149: while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do +DETAIL=SOURCE_LINE=0150: if [ "$mode" = "chunked" ]; then +DETAIL=SOURCE_LINE=0151: VDIR="$VBASE/$bn.chunks" +DETAIL=SOURCE_LINE=0152: mkdir -p "$VDIR" +DETAIL=SOURCE_LINE=0153: echo "VERIFY_CHUNKED label=$LABEL file=$bn parts=$parts" >> "$LOG" +DETAIL=SOURCE_LINE=0154: rclone copy "$DEST/files/$bn.chunks" "$VDIR" --filter "+ part-*" --filter "+ chunks.sha256" --filter "- *" --transfers 1 --checkers 4 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SOURCE_LINE=0155: (cd "$VDIR" && sha256sum -c chunks.sha256) >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SOURCE_LINE=0156: CALC="$(find "$VDIR" -maxdepth 1 -type f -name 'part-*' | sort | xargs cat | sha256sum | cut -d" " -f1)" +DETAIL=SOURCE_LINE=0157: if [ "$CALC" = "$sha" ]; then +DETAIL=SOURCE_LINE=0158: echo "OK combined_sha label=$LABEL file=$bn sha=$sha" >> "$LOG" +DETAIL=SOURCE_LINE=0159: else +DETAIL=SOURCE_LINE=0160: echo "BAD combined_sha label=$LABEL file=$bn expected=$sha got=$CALC" >> "$LOG" +DETAIL=SOURCE_LINE=0161: BAD=$((BAD+1)) +DETAIL=SOURCE_LINE=0162: fi +DETAIL=SOURCE_LINE=0163: rm -rf "$VDIR" +DETAIL=SOURCE_LINE=0164: else +DETAIL=SOURCE_LINE=0165: echo "VERIFY_DIRECT label=$LABEL file=$bn" >> "$LOG" +DETAIL=SOURCE_LINE=0166: rclone copyto "$DEST/files/$bn" "$VBASE/$bn" --transfers 1 >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SOURCE_LINE=0167: printf "%s %s\n" "$sha" "$VBASE/$bn" | sha256sum -c - >> "$LOG" 2>&1 || BAD=$((BAD+1)) +DETAIL=SOURCE_LINE=0168: rm -f "$VBASE/$bn" +DETAIL=SOURCE_LINE=0169: fi +DETAIL=SOURCE_LINE=0170: done < "$FILES_TSV" +DETAIL=SOURCE_LINE=0171: rm -rf "$VBASE" +DETAIL=SOURCE_LINE=0172:} +DETAIL=SOURCE_LINE=0173: +DETAIL=SOURCE_LINE=0174:if [ "$BAD" -eq 0 ]; then +DETAIL=SOURCE_LINE=0175: upload_one_dest "$DEST_A" +DETAIL=SOURCE_LINE=0176: upload_one_dest "$DEST_B" +DETAIL=SOURCE_LINE=0177:fi +DETAIL=SOURCE_LINE=0178: +DETAIL=SOURCE_LINE=0179:if [ "$BAD" -eq 0 ]; then +DETAIL=SOURCE_LINE=0180: verify_one_dest "A" "$DEST_A" +DETAIL=SOURCE_LINE=0181: verify_one_dest "B" "$DEST_B" +DETAIL=SOURCE_LINE=0182:fi +DETAIL=SOURCE_LINE=0183: +DETAIL=SOURCE_LINE=0184:rm -rf "$VERIFY_ROOT/$JOB_ID" +DETAIL=SOURCE_LINE=0185:rm -rf "$CHUNK_ROOT/$JOB_ID" +DETAIL=SOURCE_LINE=0186: +DETAIL=SOURCE_LINE=0187:# HOMELAB_NO_LOCAL_AFTER_DUAL_CLOUD_434 +DETAIL=SOURCE_LINE=0188:LOCAL_DUMPS_DELETED=false +DETAIL=SOURCE_LINE=0189:DELETED_COUNT=0 +DETAIL=SOURCE_LINE=0190:DELETED_BYTES=0 +DETAIL=SOURCE_LINE=0191: +DETAIL=SOURCE_LINE=0192:if [ "$BAD" -eq 0 ]; then +DETAIL=SOURCE_LINE=0193: echo "PREDELETE_VALIDATION_BEGIN" >> "$LOG" +DETAIL=SOURCE_LINE=0194: +DETAIL=SOURCE_LINE=0195: while IFS="$(printf " ")" read -r id kind size sha mt bn path mode parts; do +DETAIL=SOURCE_LINE=0196: case "$path" in +DETAIL=SOURCE_LINE=0197: /var/lib/vz/dump/*) +DETAIL=SOURCE_LINE=0198: ;; +DETAIL=SOURCE_LINE=0199: *) +DETAIL=SOURCE_LINE=0200: echo "REFUSE_DELETE_OUTSIDE_ALLOWED_ROOT path=$path" >> "$LOG" +DETAIL=SOURCE_LINE=0201: BAD=$((BAD+1)) +DETAIL=SOURCE_LINE=0202: continue +DETAIL=SOURCE_LINE=0203: ;; +DETAIL=SOURCE_LINE=0204: esac +DETAIL=SOURCE_LINE=0205: +DETAIL=SOURCE_LINE=0206: if [ "$(basename "$path")" != "$bn" ]; then +DETAIL=SOURCE_LINE=0207: echo "REFUSE_DELETE_BASENAME_MISMATCH path=$path expected=$bn" >> "$LOG" +DETAIL=SOURCE_LINE=0208: BAD=$((BAD+1)) +DETAIL=SOURCE_LINE=0209: continue +DETAIL=SOURCE_LINE=0210: fi +DETAIL=SOURCE_LINE=0211: +DETAIL=SOURCE_LINE=0212: if [ ! -f "$path" ] || [ -L "$path" ]; then +DETAIL=SOURCE_LINE=0213: echo "REFUSE_DELETE_INVALID_SOURCE path=$path" >> "$LOG" +DETAIL=SOURCE_LINE=0214: BAD=$((BAD+1)) +DETAIL=SOURCE_LINE=0215: continue +DETAIL=SOURCE_LINE=0216: fi +DETAIL=SOURCE_LINE=0217: +DETAIL=SOURCE_LINE=0218: actual_size="$(stat -c %s "$path" 2>/dev/null || echo 0)" +DETAIL=SOURCE_LINE=0219: +DETAIL=SOURCE_LINE=0220: if [ "$actual_size" != "$size" ]; then +DETAIL=SOURCE_LINE=0221: echo "REFUSE_DELETE_SIZE_MISMATCH path=$path expected=$size actual=$actual_size" >> "$LOG" +DETAIL=SOURCE_LINE=0222: BAD=$((BAD+1)) +DETAIL=SOURCE_LINE=0223: continue +DETAIL=SOURCE_LINE=0224: fi +DETAIL=SOURCE_LINE=0225: +DETAIL=SOURCE_LINE=0226: actual_sha="$(sha256sum "$path" 2>>"$LOG" | cut -d " " -f1 || true)" +DETAIL=SOURCE_LINE=0227: +DETAIL=SOURCE_LINE=0228: if [ "$actual_sha" != "$sha" ]; then +DETAIL=SOURCE_LINE=0229: echo "REFUSE_DELETE_SHA_MISMATCH path=$path" >> "$LOG" +DETAIL=SOURCE_LINE=0230: BAD=$((BAD+1)) +DETAIL=SOURCE_LINE=0231: continue +DETAIL=SOURCE_LINE=0232: fi +DETAIL=SOURCE_LINE=0233: +DETAIL=SOURCE_LINE=0234: echo "PREDELETE_VALIDATION_OK path=$path size=$size sha=$sha" >> "$LOG" +DETAIL=SOURCE_LINE=0235: done < "$FILES_TSV" +DETAIL=SOURCE_LINE=0236: +DETAIL=SOURCE_LINE=0237: echo "PREDELETE_VALIDATION_END bad=$BAD" >> "$LOG" +DETAIL=SOURCE_LINE=0238:fi +DETAIL=SOURCE_LINE=0239: +DETAIL=SOURCE_LINE=0240:if [ "$BAD" -eq 0 ]; then +DETAIL=SOURCE_LINE=0241: echo "LOCAL_PAYLOAD_DELETE_BEGIN" >> "$LOG" +DETAIL=SOURCE_LINE=0242: +DETAIL=SOURCE_LINE=0243: while IFS="$(printf " ")" read -r id kind size sha mt bn path mode parts; do +DETAIL=SOURCE_LINE=0244: if rm -f -- "$path" 2>>"$LOG"; then +DETAIL=SOURCE_LINE=0245: if [ ! -e "$path" ] && [ ! -L "$path" ]; then +DETAIL=SOURCE_LINE=0246: DELETED_COUNT=$((DELETED_COUNT+1)) +DETAIL=SOURCE_LINE=0247: DELETED_BYTES=$((DELETED_BYTES+size)) +DETAIL=SOURCE_LINE=0248: echo "LOCAL_PAYLOAD_DELETED path=$path bytes=$size" >> "$LOG" +DETAIL=SOURCE_LINE=0249: else +DETAIL=SOURCE_LINE=0250: echo "LOCAL_PAYLOAD_DELETE_CONFIRM_FAILED path=$path" >> "$LOG" +DETAIL=SOURCE_LINE=0251: BAD=$((BAD+1)) +DETAIL=SOURCE_LINE=0252: fi +DETAIL=SOURCE_LINE=0253: else +DETAIL=SOURCE_LINE=0254: echo "LOCAL_PAYLOAD_DELETE_FAILED path=$path" >> "$LOG" +DETAIL=SOURCE_LINE=0255: BAD=$((BAD+1)) +DETAIL=SOURCE_LINE=0256: fi +DETAIL=SOURCE_LINE=0257: done < "$FILES_TSV" +DETAIL=SOURCE_LINE=0258: +DETAIL=SOURCE_LINE=0259: if [ "$BAD" -eq 0 ] && [ "$DELETED_COUNT" -eq "$COUNT" ]; then +DETAIL=SOURCE_LINE=0260: LOCAL_DUMPS_DELETED=true +DETAIL=SOURCE_LINE=0261: fi +DETAIL=SOURCE_LINE=0262: +DETAIL=SOURCE_LINE=0263: echo "LOCAL_PAYLOAD_DELETE_END deleted_count=$DELETED_COUNT deleted_bytes=$DELETED_BYTES bad=$BAD" >> "$LOG" +DETAIL=SOURCE_LINE=0264:fi +DETAIL=SOURCE_LINE=0265: +DETAIL=SOURCE_LINE=0266:if [ "$BAD" -eq 0 ] && [ "$LOCAL_DUMPS_DELETED" = true ]; then +DETAIL=SOURCE_LINE=0267: STATUS="OK" +DETAIL=SOURCE_LINE=0268:else +DETAIL=SOURCE_LINE=0269: STATUS="WARN" +DETAIL=SOURCE_LINE=0270:fi +DETAIL=SOURCE_LINE=0271: +DETAIL=SOURCE_LINE=0272:jq -n \ +DETAIL=SOURCE_LINE=0273: --arg job_id "$JOB_ID" \ +DETAIL=SOURCE_LINE=0274: --arg status "$STATUS" \ +DETAIL=SOURCE_LINE=0275: --arg node "$NODE" \ +DETAIL=SOURCE_LINE=0276: --arg remote_a "$DEST_A" \ +DETAIL=SOURCE_LINE=0277: --arg remote_b "$DEST_B" \ +DETAIL=SOURCE_LINE=0278: --argjson count "$COUNT" \ +DETAIL=SOURCE_LINE=0279: --argjson total_bytes "$TOTAL" \ +DETAIL=SOURCE_LINE=0280: --argjson chunked_count "$CHUNKED" \ +DETAIL=SOURCE_LINE=0281: --argjson chunk_size_bytes "$CHUNK_SIZE_BYTES" \ +DETAIL=SOURCE_LINE=0282: --arg log "$LOG" \ +DETAIL=SOURCE_LINE=0283: --argjson local_dumps_deleted "$LOCAL_DUMPS_DELETED" \ +DETAIL=SOURCE_LINE=0284: --argjson deleted_count "$DELETED_COUNT" \ +DETAIL=SOURCE_LINE=0285: --argjson deleted_bytes "$DELETED_BYTES" \ +DETAIL=SOURCE_LINE=0286: "{job_id:\$job_id,status:\$status,node:\$node,remote_a:\$remote_a,remote_b:\$remote_b,count:\$count,total_bytes:\$total_bytes,chunked_count:\$chunked_count,chunk_size_bytes:\$chunk_size_bytes,log:\$log,local_dumps_deleted:\$local_dumps_deleted,deleted_count:\$deleted_count,deleted_bytes:\$deleted_bytes}" \ +DETAIL=SOURCE_LINE=0287: > "$ROOT/latest-vm-local-dumps-cloud.json" +DETAIL=SOURCE_LINE=0288: +DETAIL=SOURCE_LINE=0289:echo "STATUS=$STATUS TS=$TS TYPE=vm-local-dumps-cloud BAD=$BAD COUNT=$COUNT CHUNKED=$CHUNKED TOTAL_BYTES=$TOTAL JOB=$JOB_ID REMOTE_A=$DEST_A REMOTE_B=$DEST_B LOCAL_DUMPS_DELETED=$LOCAL_DUMPS_DELETED DELETED_COUNT=$DELETED_COUNT DELETED_BYTES=$DELETED_BYTES LOG=$LOG" > "$OUT" +DETAIL=SOURCE_LINE=0290:cat "$OUT" +DETAIL=SELECTION_SOURCE_END +DETAIL=ARCHIVES_CHANGED=NO +DETAIL=SERVICE_STARTED=NO +DETAIL=SECRET_VALUES_OUTPUT=NONE +DETAIL=EXACT_SELECTION_AUDIT=PASS +CHANGES_MADE=NO + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 2eb9e82..17b5527 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "HOMELAB-VM-2CLOUD-SELECTION-LOGIC-437", + "command_id": "HOMELAB-VM-2CLOUD-EXACT-SELECTION-438", "status": "OK", "rc": 0, "host": "pve01", "mode": "read-only", "component": "cluster-audit", - "started_at_utc": "2026-07-24T10:19:52Z", - "finished_at_utc": "2026-07-24T10:19:52Z", + "started_at_utc": "2026-07-24T10:21:26Z", + "finished_at_utc": "2026-07-24T10:21:26Z", "reference_register_checked": true, "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", "error_register_checked": true, "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", - "command_sha256": "ad4b5c1c52b4eaffa790b5ebe2bda18624ef1905c2e4259c94c24b9c7d9c1165", + "command_sha256": "458a61bfe476e4aeab9fdefc2057f75cf06c246702c653458e91f8c84eb823f2", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -22,9 +22,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "51b679ab8afb22764728c1e4338d954989e990db1de09b0c0e331ebc32674a6a", - "sanitized_output_sha256": "51b679ab8afb22764728c1e4338d954989e990db1de09b0c0e331ebc32674a6a", + "raw_evidence_sha256": "1a2e8ce33d51e8861acf1434ee15eee384f0fcb6316b36d3b3fd87c946570531", + "sanitized_output_sha256": "1a2e8ce33d51e8861acf1434ee15eee384f0fcb6316b36d3b3fd87c946570531", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "DETAIL=RUNTIME_SHA256=0ded6ba72e49b9cf7522c7f3d8840cc1026d2a173d4909dd3b4cfc5ff8831115\nDETAIL=DESIRED_SHA256=0ded6ba72e49b9cf7522c7f3d8840cc1026d2a173d4909dd3b4cfc5ff8831115\nDETAIL=SELECTION_SOURCE_BEGIN\nDETAIL=SOURCE_LINE=0021:VMIDS=\"${VMIDS:-110 112 150}\"\nDETAIL=SOURCE_LINE=0022:\nDETAIL=SOURCE_LINE=0023:source /etc/homelab-backup.env 2>/dev/null || BAD=1\nDETAIL=SOURCE_LINE=0024:\nDETAIL=SOURCE_LINE=0025:JOB_ID=\"homelab-${NODE}-vm-local-dumps-${JOB_TS}\"\nDETAIL=SOURCE_LINE=0026:WORK=\"$ROOT/$JOB_ID\"\nDETAIL=SOURCE_LINE=0027:mkdir -p \"$WORK\"\nDETAIL=SOURCE_LINE=0028:\nDETAIL=SOURCE_LINE=0029:FILES_TSV=\"$WORK/files.tsv\"\nDETAIL=SOURCE_LINE=0030:CHKS=\"$WORK/checksums.txt\"\nDETAIL=SOURCE_LINE=0031:MAN=\"$WORK/manifest.json\"\nDETAIL=SOURCE_LINE=0032:\nDETAIL=SOURCE_LINE=0033:: > \"$FILES_TSV\"\nDETAIL=SOURCE_LINE=0034:: > \"$CHKS\"\nDETAIL=SOURCE_LINE=0035:: > \"$LOG\"\nDETAIL=SOURCE_LINE=0036:\nDETAIL=SOURCE_LINE=0037:echo \"job_id=$JOB_ID\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0038:echo \"vmids=$VMIDS\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0039:echo \"chunk_size_bytes=$CHUNK_SIZE_BYTES\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0040:\nDETAIL=SOURCE_LINE=0041:for id in $VMIDS; do\nDETAIL=SOURCE_LINE=0042: F=\"$(find /var/lib/vz/dump -maxdepth 1 -type f \\( -name \"vzdump-qemu-${id}-*.vma.zst\" -o -name \"vzdump-lxc-${id}-*.tar.zst\" \\) -printf \"%T@ %p\\n\" 2>/dev/null | sort -n | tail -n 1 | cut -d\" \" -f2-)\"\nDETAIL=SOURCE_LINE=0043: if [ -z \"$F\" ] || [ ! -f \"$F\" ]; then\nDETAIL=SOURCE_LINE=0044: echo \"MISSING vmid=$id\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0045: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0046: continue\nDETAIL=SOURCE_LINE=0047: fi\nDETAIL=SOURCE_LINE=0048:\nDETAIL=SOURCE_LINE=0049: BN=\"$(basename \"$F\")\"\nDETAIL=SOURCE_LINE=0050: SIZE=\"$(stat -c %s \"$F\" 2>/dev/null || echo 0)\"\nDETAIL=SOURCE_LINE=0051: MT=\"$(date -u -d \"@$(stat -c %Y \"$F\" 2>/dev/null || echo 0)\" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)\"\nDETAIL=SOURCE_LINE=0052: KIND=\"unknown\"\nDETAIL=SOURCE_LINE=0053: echo \"$BN\" | grep -q \"^vzdump-qemu-\" && KIND=\"qemu\"\nDETAIL=SOURCE_LINE=0054: echo \"$BN\" | grep -q \"^vzdump-lxc-\" && KIND=\"lxc\"\nDETAIL=SOURCE_LINE=0055: SHA=\"$(sha256sum \"$F\" | cut -d\" \" -f1)\"\nDETAIL=SOURCE_LINE=0056: printf \"%s %s\\n\" \"$SHA\" \"$BN\" >> \"$CHKS\"\nDETAIL=SOURCE_LINE=0057:\nDETAIL=SOURCE_LINE=0058: MODE=\"direct\"\nDETAIL=SOURCE_LINE=0059: PARTS=0\nDETAIL=SOURCE_LINE=0060: if [ \"$SIZE\" -gt \"$CHUNK_SIZE_BYTES\" ]; then\nDETAIL=SOURCE_LINE=0061: MODE=\"chunked\"\nDETAIL=SOURCE_LINE=0062: CHUNKED=$((CHUNKED+1))\nDETAIL=SOURCE_LINE=0063: CDIR=\"$CHUNK_ROOT/$JOB_ID/$BN.chunks\"\nDETAIL=SOURCE_LINE=0064: rm -rf \"$CDIR\"\nDETAIL=SOURCE_LINE=0065: mkdir -p \"$CDIR\"\nDETAIL=SOURCE_LINE=0066: echo \"SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0067: split -b \"$CHUNK_SIZE_SPLIT\" -d -a 4 \"$F\" \"$CDIR/part-\" >> \"$LOG\" 2>&1\nDETAIL=SOURCE_LINE=0068: PARTS=\"$(find \"$CDIR\" -maxdepth 1 -type f -name 'part-*' | wc -l)\"\nDETAIL=SOURCE_LINE=0069: (cd \"$CDIR\" && sha256sum part-* > chunks.sha256)\nDETAIL=SOURCE_LINE=0070: if [ \"$PARTS\" -le 0 ]; then\nDETAIL=SOURCE_LINE=0071: echo \"BAD split produced no chunks file=$BN\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0072: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0073: fi\nDETAIL=SOURCE_LINE=0074: fi\nDETAIL=SOURCE_LINE=0075:\nDETAIL=SOURCE_LINE=0076: printf \"%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n\" \"$id\" \"$KIND\" \"$SIZE\" \"$SHA\" \"$MT\" \"$BN\" \"$F\" \"$MODE\" \"$PARTS\" >> \"$FILES_TSV\"\nDETAIL=SOURCE_LINE=0077: COUNT=$((COUNT+1))\nDETAIL=SOURCE_LINE=0078: TOTAL=$((TOTAL+SIZE))\nDETAIL=SOURCE_LINE=0079:done\nDETAIL=SOURCE_LINE=0080:\nDETAIL=SOURCE_LINE=0081:python3 - \"$JOB_ID\" \"$NODE\" \"$TS\" \"$VMIDS\" \"$COUNT\" \"$TOTAL\" \"$CHUNKED\" \"$CHUNK_SIZE_BYTES\" \"$FILES_TSV\" > \"$MAN\" <<'PY'\nDETAIL=SOURCE_LINE=0082:import json, sys, pathlib\nDETAIL=SOURCE_LINE=0083:job,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:]\nDETAIL=SOURCE_LINE=0084:items=[]\nDETAIL=SOURCE_LINE=0085:for line in pathlib.Path(tsv).read_text().splitlines():\nDETAIL=SOURCE_LINE=0086: p=line.split(\"\\t\")\nDETAIL=SOURCE_LINE=0087: if len(p)>=9:\nDETAIL=SOURCE_LINE=0088: items.append({\nDETAIL=SOURCE_LINE=0089: \"vmid\":p[0],\nDETAIL=SOURCE_LINE=0090: \"kind\":p[1],\nDETAIL=SOURCE_LINE=0091: \"size_bytes\":int(p[2]),\nDETAIL=SOURCE_LINE=...\nDETAIL=SOURCE_LINE=0117: rclone mkdir \"$DEST/files\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0118: while IFS=\"$(printf \"\\t\")\" read -r id kind size sha mt bn path mode parts; do\nDETAIL=SOURCE_LINE=0119: if [ \"$mode\" = \"chunked\" ]; then\nDETAIL=SOURCE_LINE=0120: CDIR=\"$CHUNK_ROOT/$JOB_ID/$bn.chunks\"\nDETAIL=SOURCE_LINE=0121: echo \"UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0122: rclone mkdir \"$DEST/files/$bn.chunks\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0123: for part in \"$CDIR\"/part-*; do\nDETAIL=SOURCE_LINE=0124: [ -f \"$part\" ] || continue\nDETAIL=SOURCE_LINE=0125: pn=\"$(basename \"$part\")\"\nDETAIL=SOURCE_LINE=0126: rclone copyto \"$part\" \"$DEST/files/$bn.chunks/$pn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0127: done\nDETAIL=SOURCE_LINE=0128: rclone copyto \"$CDIR/chunks.sha256\" \"$DEST/files/$bn.chunks/chunks.sha256\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0129: else\nDETAIL=SOURCE_LINE=0130: echo \"UPLOAD_DIRECT dest=$DEST file=$bn size=$size\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0131: rclone copyto \"$path\" \"$DEST/files/$bn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0132: fi\nDETAIL=SOURCE_LINE=0133: done < \"$FILES_TSV\"\nDETAIL=SOURCE_LINE=0134:\nDETAIL=SOURCE_LINE=0135: rclone copyto \"$CHKS\" \"$DEST/checksums.txt\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0136: rclone copyto \"$FILES_TSV\" \"$DEST/files.tsv\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0137: rclone copyto \"$MAN\" \"$DEST/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0138:}\nDETAIL=SOURCE_LINE=0139:\nDETAIL=SOURCE_LINE=0140:verify_one_dest() {\nDETAIL=SOURCE_LINE=0141: LABEL=\"$1\"\nDETAIL=SOURCE_LINE=0142: DEST=\"$2\"\nDETAIL=SOURCE_LINE=0143: VBASE=\"$VERIFY_ROOT/$JOB_ID/$LABEL\"\nDETAIL=SOURCE_LINE=0144: rm -rf \"$VBASE\"\nDETAIL=SOURCE_LINE=0145: mkdir -p \"$VBASE\"\nDETAIL=SOURCE_LINE=0146: rclone copyto \"$DEST/manifest.json\" \"$VBASE/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0147: test -s \"$VBASE/manifest.json\" || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0148:\nDETAIL=SOURCE_LINE=0149: while IFS=\"$(printf \"\\t\")\" read -r id kind size sha mt bn path mode parts; do\nDETAIL=SOURCE_LINE=0150: if [ \"$mode\" = \"chunked\" ]; then\nDETAIL=SOURCE_LINE=0151: VDIR=\"$VBASE/$bn.chunks\"\nDETAIL=SOURCE_LINE=0152: mkdir -p \"$VDIR\"\nDETAIL=SOURCE_LINE=0153: echo \"VERIFY_CHUNKED label=$LABEL file=$bn parts=$parts\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0154: rclone copy \"$DEST/files/$bn.chunks\" \"$VDIR\" --filter \"+ part-*\" --filter \"+ chunks.sha256\" --filter \"- *\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0155: (cd \"$VDIR\" && sha256sum -c chunks.sha256) >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0156: CALC=\"$(find \"$VDIR\" -maxdepth 1 -type f -name 'part-*' | sort | xargs cat | sha256sum | cut -d\" \" -f1)\"\nDETAIL=SOURCE_LINE=0157: if [ \"$CALC\" = \"$sha\" ]; then\nDETAIL=SOURCE_LINE=0158: echo \"OK combined_sha label=$LABEL file=$bn sha=$sha\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0159: else\nDETAIL=SOURCE_LINE=0160: echo \"BAD combined_sha label=$LABEL file=$bn expected=$sha got=$CALC\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0161: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0162: fi\nDETAIL=SOURCE_LINE=0163: rm -rf \"$VDIR\"\nDETAIL=SOURCE_LINE=0164: else\nDETAIL=SOURCE_LINE=0165: echo \"VERIFY_DIRECT label=$LABEL file=$bn\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0166: rclone copyto \"$DEST/files/$bn\" \"$VBASE/$bn\" --transfers 1 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0167: printf \"%s %s\\n\" \"$sha\" \"$VBASE/$bn\" | sha256sum -c - >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0168: rm -f \"$VBASE/$bn\"\nDETAIL=SOURCE_LINE=0169: fi\nDETAIL=SOURCE_LINE=0170: done < \"$FILES_TSV\"\nDETAIL=SOURCE_LINE=0171: rm -rf \"$VBASE\"\nDETAIL=SOURCE_LINE=0172:}\nDETAIL=SOURCE_LINE=0173:\nDETAIL=SOURCE_LINE=0174:if [ \"$BAD\" -eq 0 ]; then\nDETAIL=SOURCE_LINE=0175: upload_one_dest \"$DEST_A\"\nDETAIL=SOURCE_LINE=0176: upload_one_dest \"$DEST_B\"\nDETAIL=SOURCE_LINE=0177:fi\nDETAIL=SOURCE_LINE=0178:\nDETAIL=SOURCE_LINE=0179:if [ \"$BAD\" -eq 0 ]; then\nDETAIL=SOURCE_LINE=0180: verify_one_dest \"A\" \"$DEST_A\"\nDETAIL=SOURCE_LINE=...\nDETAIL=SOURCE_LINE=0198: ;;\nDETAIL=SOURCE_LINE=0199: *)\nDETAIL=SOURCE_LINE=0200: echo \"REFUSE_DELETE_OUTSIDE_ALLOWED_ROOT path=$path\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0201: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0202: continue\nDETAIL=SOURCE_LINE=0203: ;;\nDETAIL=SOURCE_LINE=0204: esac\nDETAIL=SOURCE_LINE=0205:\nDETAIL=SOURCE_LINE=0206: if [ \"$(basename \"$path\")\" != \"$bn\" ]; then\nDETAIL=SOURCE_LINE=0207: echo \"REFUSE_DELETE_BASENAME_MISMATCH path=$path expected=$bn\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0208: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0209: continue\nDETAIL=SOURCE_LINE=0210: fi\nDETAIL=SOURCE_LINE=0211:\nDETAIL=SOURCE_LINE=0212: if [ ! -f \"$path\" ] || [ -L \"$path\" ]; then\nDETAIL=SOURCE_LINE=0213: echo \"REFUSE_DELETE_INVALID_SOURCE path=$path\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0214: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0215: continue\nDETAIL=SOURCE_LINE=0216: fi\nDETAIL=SOURCE_LINE=0217:\nDETAIL=SOURCE_LINE=0218: actual_size=\"$(stat -c %s \"$path\" 2>/dev/null || echo 0)\"\nDETAIL=SOURCE_LINE=0219:\nDETAIL=SOURCE_LINE=0220: if [ \"$actual_size\" != \"$size\" ]; then\nDETAIL=SOURCE_LINE=0221: echo \"REFUSE_DELETE_SIZE_MISMATCH path=$path expected=$size actual=$actual_size\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0222: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0223: continue\nDETAIL=SOURCE_LINE=0224: fi\nDETAIL=SOURCE_LINE=0225:\nDETAIL=SOURCE_LINE=0226: actual_sha=\"$(sha256sum \"$path\" 2>>\"$LOG\" | cut -d \" \" -f1 || true)\"\nDETAIL=SOURCE_LINE=0227:\nDETAIL=SOURCE_LINE=0228: if [ \"$actual_sha\" != \"$sha\" ]; then\nDETAIL=SOURCE_LINE=0229: echo \"REFUSE_DELETE_SHA_MISMATCH path=$path\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0230: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0231: continue\nDETAIL=SOURCE_LINE=0232: fi\nDETAIL=SOURCE_LINE=0233:\nDETAIL=SOURCE_LINE=0234: echo \"PREDELETE_VALIDATION_OK path=$path size=$size sha=$sha\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0235: done < \"$FILES_TSV\"\nDETAIL=SOURCE_LINE=0236:\nDETAIL=SOURCE_LINE=0237: echo \"PREDELETE_VALIDATION_END bad=$BAD\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0238:fi\nDETAIL=SOURCE_LINE=0239:\nDETAIL=SOURCE_LINE=0240:if [ \"$BAD\" -eq 0 ]; then\nDETAIL=SOURCE_LINE=0241: echo \"LOCAL_PAYLOAD_DELETE_BEGIN\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0242:\nDETAIL=SOURCE_LINE=0243: while IFS=\"$(printf \"\t\")\" read -r id kind size sha mt bn path mode parts; do\nDETAIL=SOURCE_LINE=0244: if rm -f -- \"$path\" 2>>\"$LOG\"; then\nDETAIL=SOURCE_LINE=0245: if [ ! -e \"$path\" ] && [ ! -L \"$path\" ]; then\nDETAIL=SOURCE_LINE=...\nDETAIL=SOURCE_LINE=0249: else\nDETAIL=SOURCE_LINE=0250: echo \"LOCAL_PAYLOAD_DELETE_CONFIRM_FAILED path=$path\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0251: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0252: fi\nDETAIL=SOURCE_LINE=0253: else\nDETAIL=SOURCE_LINE=0254: echo \"LOCAL_PAYLOAD_DELETE_FAILED path=$path\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0255: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0256: fi\nDETAIL=SOURCE_LINE=0257: done < \"$FILES_TSV\"\nDETAIL=SOURCE_LINE=0258:\nDETAIL=SOURCE_LINE=0259: if [ \"$BAD\" -eq 0 ] && [ \"$DELETED_COUNT\" -eq \"$COUNT\" ]; then\nDETAIL=SOURCE_LINE=0260: LOCAL_DUMPS_DELETED=true\nDETAIL=SOURCE_LINE=0261: fi\nDETAIL=SOURCE_LINE=0262:\nDETAIL=SOURCE_LINE=0263: echo \"LOCAL_PAYLOAD_DELETE_END deleted_count=$DELETED_COUNT deleted_bytes=$DELETED_BYTES bad=$BAD\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0264:fi\nDETAIL=SOURCE_LINE=0265:\nDETAIL=SOURCE_LINE=0266:if [ \"$BAD\" -eq 0 ] && [ \"$LOCAL_DUMPS_DELETED\" = true ]; then\nDETAIL=SOURCE_LINE=0267: STATUS=\"OK\"\nDETAIL=SELECTION_SOURCE_END\nDETAIL=INVENTORY_GROUP_COUNT=3\nDETAIL=INVENTORY_GROUP=KIND:lxc|ID:110|COUNT:4|TOTAL_BYTES:1337879243|LATEST_PATH:/var/lib/vz/dump/vzdump-lxc-110-2026_07_24-03_30_02.tar.zst|LATEST_BYTES:339140622\nDETAIL=INVENTORY_GROUP=KIND:lxc|ID:112|COUNT:4|TOTAL_BYTES:891239268|LATEST_PATH:/var/lib/vz/dump/vzdump-lxc-112-2026_07_24-03_30_11.tar.zst|LATEST_BYTES:222859457\nDETAIL=INVENTORY_GROUP=KIND:qemu|ID:150|COUNT:4|TOTAL_BYTES:28114919054|LATEST_PATH:/var/lib/vz/dump/vzdump-qemu-150-2026_07_24-03_30_16.vma.zst|LATEST_BYTES:7016624549\nDETAIL=ALL_ARCHIVE_COUNT=12\nDETAIL=ALL_ARCHIVE_BYTES=30344037565\nDETAIL=LATEST_PER_OBJECT_COUNT=3\nDETAIL=LATEST_PER_OBJECT_BYTES=7578624628\nDETAIL=BACKUP_PAYLOAD_CHANGED=NO\nDETAIL=SECRET_VALUES_OUTPUT=NONE\nDETAIL=VM_2CLOUD_SELECTION_LOGIC_AUDIT=PASS\nCHANGES_MADE=NO\n" + "output": ":33: FutureWarning: Possible nested set at position 2\n:33: FutureWarning: Possible nested set at position 29\n:43: FutureWarning: Possible nested set at position 2\n:43: FutureWarning: Possible nested set at position 29\nDETAIL=SCRIPT_SHA256=0ded6ba72e49b9cf7522c7f3d8840cc1026d2a173d4909dd3b4cfc5ff8831115\nDETAIL=SELECTION_SOURCE_RANGE=14-290\nDETAIL=SELECTION_SOURCE_BEGIN\nDETAIL=SOURCE_LINE=0014:TS=\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"\nDETAIL=SOURCE_LINE=0015:JOB_TS=\"$(date -u +%Y%m%dT%H%M%SZ)\"\nDETAIL=SOURCE_LINE=0016:NODE=\"$(hostname -s)\"\nDETAIL=SOURCE_LINE=0017:BAD=0\nDETAIL=SOURCE_LINE=0018:COUNT=0\nDETAIL=SOURCE_LINE=0019:TOTAL=0\nDETAIL=SOURCE_LINE=0020:CHUNKED=0\nDETAIL=SOURCE_LINE=0021:VMIDS=\"${VMIDS:-110 112 150}\"\nDETAIL=SOURCE_LINE=0022:\nDETAIL=SOURCE_LINE=0023:source /etc/homelab-backup.env 2>/dev/null || BAD=1\nDETAIL=SOURCE_LINE=0024:\nDETAIL=SOURCE_LINE=0025:JOB_ID=\"homelab-${NODE}-vm-local-dumps-${JOB_TS}\"\nDETAIL=SOURCE_LINE=0026:WORK=\"$ROOT/$JOB_ID\"\nDETAIL=SOURCE_LINE=0027:mkdir -p \"$WORK\"\nDETAIL=SOURCE_LINE=0028:\nDETAIL=SOURCE_LINE=0029:FILES_TSV=\"$WORK/files.tsv\"\nDETAIL=SOURCE_LINE=0030:CHKS=\"$WORK/checksums.txt\"\nDETAIL=SOURCE_LINE=0031:MAN=\"$WORK/manifest.json\"\nDETAIL=SOURCE_LINE=0032:\nDETAIL=SOURCE_LINE=0033:: > \"$FILES_TSV\"\nDETAIL=SOURCE_LINE=0034:: > \"$CHKS\"\nDETAIL=SOURCE_LINE=0035:: > \"$LOG\"\nDETAIL=SOURCE_LINE=0036:\nDETAIL=SOURCE_LINE=0037:echo \"job_id=$JOB_ID\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0038:echo \"vmids=$VMIDS\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0039:echo \"chunk_size_bytes=$CHUNK_SIZE_BYTES\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0040:\nDETAIL=SOURCE_LINE=0041:for id in $VMIDS; do\nDETAIL=SOURCE_LINE=0042: F=\"$(find /var/lib/vz/dump -maxdepth 1 -type f \\( -name \"vzdump-qemu-${id}-*.vma.zst\" -o -name \"vzdump-lxc-${id}-*.tar.zst\" \\) -printf \"%T@ %p\\n\" 2>/dev/null | sort -n | tail -n 1 | cut -d\" \" -f2-)\"\nDETAIL=SOURCE_LINE=0043: if [ -z \"$F\" ] || [ ! -f \"$F\" ]; then\nDETAIL=SOURCE_LINE=0044: echo \"MISSING vmid=$id\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0045: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0046: continue\nDETAIL=SOURCE_LINE=0047: fi\nDETAIL=SOURCE_LINE=0048:\nDETAIL=SOURCE_LINE=0049: BN=\"$(basename \"$F\")\"\nDETAIL=SOURCE_LINE=0050: SIZE=\"$(stat -c %s \"$F\" 2>/dev/null || echo 0)\"\nDETAIL=SOURCE_LINE=0051: MT=\"$(date -u -d \"@$(stat -c %Y \"$F\" 2>/dev/null || echo 0)\" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)\"\nDETAIL=SOURCE_LINE=0052: KIND=\"unknown\"\nDETAIL=SOURCE_LINE=0053: echo \"$BN\" | grep -q \"^vzdump-qemu-\" && KIND=\"qemu\"\nDETAIL=SOURCE_LINE=0054: echo \"$BN\" | grep -q \"^vzdump-lxc-\" && KIND=\"lxc\"\nDETAIL=SOURCE_LINE=0055: SHA=\"$(sha256sum \"$F\" | cut -d\" \" -f1)\"\nDETAIL=SOURCE_LINE=0056: printf \"%s %s\\n\" \"$SHA\" \"$BN\" >> \"$CHKS\"\nDETAIL=SOURCE_LINE=0057:\nDETAIL=SOURCE_LINE=0058: MODE=\"direct\"\nDETAIL=SOURCE_LINE=0059: PARTS=0\nDETAIL=SOURCE_LINE=0060: if [ \"$SIZE\" -gt \"$CHUNK_SIZE_BYTES\" ]; then\nDETAIL=SOURCE_LINE=0061: MODE=\"chunked\"\nDETAIL=SOURCE_LINE=0062: CHUNKED=$((CHUNKED+1))\nDETAIL=SOURCE_LINE=0063: CDIR=\"$CHUNK_ROOT/$JOB_ID/$BN.chunks\"\nDETAIL=SOURCE_LINE=0064: rm -rf \"$CDIR\"\nDETAIL=SOURCE_LINE=0065: mkdir -p \"$CDIR\"\nDETAIL=SOURCE_LINE=0066: echo \"SPLIT file=$BN size=$SIZE chunk_size=$CHUNK_SIZE_SPLIT dir=$CDIR\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0067: split -b \"$CHUNK_SIZE_SPLIT\" -d -a 4 \"$F\" \"$CDIR/part-\" >> \"$LOG\" 2>&1\nDETAIL=SOURCE_LINE=0068: PARTS=\"$(find \"$CDIR\" -maxdepth 1 -type f -name 'part-*' | wc -l)\"\nDETAIL=SOURCE_LINE=0069: (cd \"$CDIR\" && sha256sum part-* > chunks.sha256)\nDETAIL=SOURCE_LINE=0070: if [ \"$PARTS\" -le 0 ]; then\nDETAIL=SOURCE_LINE=0071: echo \"BAD split produced no chunks file=$BN\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0072: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0073: fi\nDETAIL=SOURCE_LINE=0074: fi\nDETAIL=SOURCE_LINE=0075:\nDETAIL=SOURCE_LINE=0076: printf \"%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n\" \"$id\" \"$KIND\" \"$SIZE\" \"$SHA\" \"$MT\" \"$BN\" \"$F\" \"$MODE\" \"$PARTS\" >> \"$FILES_TSV\"\nDETAIL=SOURCE_LINE=0077: COUNT=$((COUNT+1))\nDETAIL=SOURCE_LINE=0078: TOTAL=$((TOTAL+SIZE))\nDETAIL=SOURCE_LINE=0079:done\nDETAIL=SOURCE_LINE=0080:\nDETAIL=SOURCE_LINE=0081:python3 - \"$JOB_ID\" \"$NODE\" \"$TS\" \"$VMIDS\" \"$COUNT\" \"$TOTAL\" \"$CHUNKED\" \"$CHUNK_SIZE_BYTES\" \"$FILES_TSV\" > \"$MAN\" <<'PY'\nDETAIL=SOURCE_LINE=0082:import json, sys, pathlib\nDETAIL=SOURCE_LINE=0083:job,node,ts,vmids,count,total,chunked,chunk_size,tsv=sys.argv[1:]\nDETAIL=SOURCE_LINE=0084:items=[]\nDETAIL=SOURCE_LINE=0085:for line in pathlib.Path(tsv).read_text().splitlines():\nDETAIL=SOURCE_LINE=0086: p=line.split(\"\\t\")\nDETAIL=SOURCE_LINE=0087: if len(p)>=9:\nDETAIL=SOURCE_LINE=0088: items.append({\nDETAIL=SOURCE_LINE=0089: \"vmid\":p[0],\nDETAIL=SOURCE_LINE=0090: \"kind\":p[1],\nDETAIL=SOURCE_LINE=0091: \"size_bytes\":int(p[2]),\nDETAIL=SOURCE_LINE=0092: \"sha256\":p[3],\nDETAIL=SOURCE_LINE=0093: \"mtime_utc\":p[4],\nDETAIL=SOURCE_LINE=0094: \"filename\":p[5],\nDETAIL=SOURCE_LINE=0095: \"source_path\":p[6],\nDETAIL=SOURCE_LINE=0096: \"mode\":p[7],\nDETAIL=SOURCE_LINE=0097: \"parts\":int(p[8]),\nDETAIL=SOURCE_LINE=0098: })\nDETAIL=SOURCE_LINE=0099:print(json.dumps({\nDETAIL=SOURCE_LINE=0100: \"job_id\":job,\nDETAIL=SOURCE_LINE=0101: \"node\":node,\nDETAIL=SOURCE_LINE=0102: \"generated_utc\":ts,\nDETAIL=SOURCE_LINE=0103: \"vmids\":vmids.split(),\nDETAIL=SOURCE_LINE=0104: \"count\":int(count),\nDETAIL=SOURCE_LINE=0105: \"total_bytes\":int(total),\nDETAIL=SOURCE_LINE=0106: \"chunked_count\":int(chunked),\nDETAIL=SOURCE_LINE=0107: \"chunk_size_bytes\":int(chunk_size),\nDETAIL=SOURCE_LINE=0108: \"items\":items,\nDETAIL=SOURCE_LINE=0109:}, indent=2))\nDETAIL=SOURCE_LINE=0110:PY\nDETAIL=SOURCE_LINE=0111:\nDETAIL=SOURCE_LINE=0112:DEST_A=\"${REMOTE_A%/}/${NODE}/vm-local-dumps/${JOB_ID}\"\nDETAIL=SOURCE_LINE=0113:DEST_B=\"${REMOTE_B%/}/${NODE}/vm-local-dumps/${JOB_ID}\"\nDETAIL=SOURCE_LINE=0114:\nDETAIL=SOURCE_LINE=0115:upload_one_dest() {\nDETAIL=SOURCE_LINE=0116: DEST=\"$1\"\nDETAIL=SOURCE_LINE=0117: rclone mkdir \"$DEST/files\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0118: while IFS=\"$(printf \"\\t\")\" read -r id kind size sha mt bn path mode parts; do\nDETAIL=SOURCE_LINE=0119: if [ \"$mode\" = \"chunked\" ]; then\nDETAIL=SOURCE_LINE=0120: CDIR=\"$CHUNK_ROOT/$JOB_ID/$bn.chunks\"\nDETAIL=SOURCE_LINE=0121: echo \"UPLOAD_CHUNKED dest=$DEST file=$bn size=$size parts=$parts\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0122: rclone mkdir \"$DEST/files/$bn.chunks\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0123: for part in \"$CDIR\"/part-*; do\nDETAIL=SOURCE_LINE=0124: [ -f \"$part\" ] || continue\nDETAIL=SOURCE_LINE=0125: pn=\"$(basename \"$part\")\"\nDETAIL=SOURCE_LINE=0126: rclone copyto \"$part\" \"$DEST/files/$bn.chunks/$pn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0127: done\nDETAIL=SOURCE_LINE=0128: rclone copyto \"$CDIR/chunks.sha256\" \"$DEST/files/$bn.chunks/chunks.sha256\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0129: else\nDETAIL=SOURCE_LINE=0130: echo \"UPLOAD_DIRECT dest=$DEST file=$bn size=$size\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0131: rclone copyto \"$path\" \"$DEST/files/$bn\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0132: fi\nDETAIL=SOURCE_LINE=0133: done < \"$FILES_TSV\"\nDETAIL=SOURCE_LINE=0134:\nDETAIL=SOURCE_LINE=0135: rclone copyto \"$CHKS\" \"$DEST/checksums.txt\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0136: rclone copyto \"$FILES_TSV\" \"$DEST/files.tsv\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0137: rclone copyto \"$MAN\" \"$DEST/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0138:}\nDETAIL=SOURCE_LINE=0139:\nDETAIL=SOURCE_LINE=0140:verify_one_dest() {\nDETAIL=SOURCE_LINE=0141: LABEL=\"$1\"\nDETAIL=SOURCE_LINE=0142: DEST=\"$2\"\nDETAIL=SOURCE_LINE=0143: VBASE=\"$VERIFY_ROOT/$JOB_ID/$LABEL\"\nDETAIL=SOURCE_LINE=0144: rm -rf \"$VBASE\"\nDETAIL=SOURCE_LINE=0145: mkdir -p \"$VBASE\"\nDETAIL=SOURCE_LINE=0146: rclone copyto \"$DEST/manifest.json\" \"$VBASE/manifest.json\" >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0147: test -s \"$VBASE/manifest.json\" || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0148:\nDETAIL=SOURCE_LINE=0149: while IFS=\"$(printf \"\\t\")\" read -r id kind size sha mt bn path mode parts; do\nDETAIL=SOURCE_LINE=0150: if [ \"$mode\" = \"chunked\" ]; then\nDETAIL=SOURCE_LINE=0151: VDIR=\"$VBASE/$bn.chunks\"\nDETAIL=SOURCE_LINE=0152: mkdir -p \"$VDIR\"\nDETAIL=SOURCE_LINE=0153: echo \"VERIFY_CHUNKED label=$LABEL file=$bn parts=$parts\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0154: rclone copy \"$DEST/files/$bn.chunks\" \"$VDIR\" --filter \"+ part-*\" --filter \"+ chunks.sha256\" --filter \"- *\" --transfers 1 --checkers 4 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0155: (cd \"$VDIR\" && sha256sum -c chunks.sha256) >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0156: CALC=\"$(find \"$VDIR\" -maxdepth 1 -type f -name 'part-*' | sort | xargs cat | sha256sum | cut -d\" \" -f1)\"\nDETAIL=SOURCE_LINE=0157: if [ \"$CALC\" = \"$sha\" ]; then\nDETAIL=SOURCE_LINE=0158: echo \"OK combined_sha label=$LABEL file=$bn sha=$sha\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0159: else\nDETAIL=SOURCE_LINE=0160: echo \"BAD combined_sha label=$LABEL file=$bn expected=$sha got=$CALC\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0161: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0162: fi\nDETAIL=SOURCE_LINE=0163: rm -rf \"$VDIR\"\nDETAIL=SOURCE_LINE=0164: else\nDETAIL=SOURCE_LINE=0165: echo \"VERIFY_DIRECT label=$LABEL file=$bn\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0166: rclone copyto \"$DEST/files/$bn\" \"$VBASE/$bn\" --transfers 1 >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0167: printf \"%s %s\\n\" \"$sha\" \"$VBASE/$bn\" | sha256sum -c - >> \"$LOG\" 2>&1 || BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0168: rm -f \"$VBASE/$bn\"\nDETAIL=SOURCE_LINE=0169: fi\nDETAIL=SOURCE_LINE=0170: done < \"$FILES_TSV\"\nDETAIL=SOURCE_LINE=0171: rm -rf \"$VBASE\"\nDETAIL=SOURCE_LINE=0172:}\nDETAIL=SOURCE_LINE=0173:\nDETAIL=SOURCE_LINE=0174:if [ \"$BAD\" -eq 0 ]; then\nDETAIL=SOURCE_LINE=0175: upload_one_dest \"$DEST_A\"\nDETAIL=SOURCE_LINE=0176: upload_one_dest \"$DEST_B\"\nDETAIL=SOURCE_LINE=0177:fi\nDETAIL=SOURCE_LINE=0178:\nDETAIL=SOURCE_LINE=0179:if [ \"$BAD\" -eq 0 ]; then\nDETAIL=SOURCE_LINE=0180: verify_one_dest \"A\" \"$DEST_A\"\nDETAIL=SOURCE_LINE=0181: verify_one_dest \"B\" \"$DEST_B\"\nDETAIL=SOURCE_LINE=0182:fi\nDETAIL=SOURCE_LINE=0183:\nDETAIL=SOURCE_LINE=0184:rm -rf \"$VERIFY_ROOT/$JOB_ID\"\nDETAIL=SOURCE_LINE=0185:rm -rf \"$CHUNK_ROOT/$JOB_ID\"\nDETAIL=SOURCE_LINE=0186:\nDETAIL=SOURCE_LINE=0187:# HOMELAB_NO_LOCAL_AFTER_DUAL_CLOUD_434\nDETAIL=SOURCE_LINE=0188:LOCAL_DUMPS_DELETED=false\nDETAIL=SOURCE_LINE=0189:DELETED_COUNT=0\nDETAIL=SOURCE_LINE=0190:DELETED_BYTES=0\nDETAIL=SOURCE_LINE=0191:\nDETAIL=SOURCE_LINE=0192:if [ \"$BAD\" -eq 0 ]; then\nDETAIL=SOURCE_LINE=0193: echo \"PREDELETE_VALIDATION_BEGIN\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0194:\nDETAIL=SOURCE_LINE=0195: while IFS=\"$(printf \"\t\")\" read -r id kind size sha mt bn path mode parts; do\nDETAIL=SOURCE_LINE=0196: case \"$path\" in\nDETAIL=SOURCE_LINE=0197: /var/lib/vz/dump/*)\nDETAIL=SOURCE_LINE=0198: ;;\nDETAIL=SOURCE_LINE=0199: *)\nDETAIL=SOURCE_LINE=0200: echo \"REFUSE_DELETE_OUTSIDE_ALLOWED_ROOT path=$path\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0201: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0202: continue\nDETAIL=SOURCE_LINE=0203: ;;\nDETAIL=SOURCE_LINE=0204: esac\nDETAIL=SOURCE_LINE=0205:\nDETAIL=SOURCE_LINE=0206: if [ \"$(basename \"$path\")\" != \"$bn\" ]; then\nDETAIL=SOURCE_LINE=0207: echo \"REFUSE_DELETE_BASENAME_MISMATCH path=$path expected=$bn\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0208: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0209: continue\nDETAIL=SOURCE_LINE=0210: fi\nDETAIL=SOURCE_LINE=0211:\nDETAIL=SOURCE_LINE=0212: if [ ! -f \"$path\" ] || [ -L \"$path\" ]; then\nDETAIL=SOURCE_LINE=0213: echo \"REFUSE_DELETE_INVALID_SOURCE path=$path\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0214: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0215: continue\nDETAIL=SOURCE_LINE=0216: fi\nDETAIL=SOURCE_LINE=0217:\nDETAIL=SOURCE_LINE=0218: actual_size=\"$(stat -c %s \"$path\" 2>/dev/null || echo 0)\"\nDETAIL=SOURCE_LINE=0219:\nDETAIL=SOURCE_LINE=0220: if [ \"$actual_size\" != \"$size\" ]; then\nDETAIL=SOURCE_LINE=0221: echo \"REFUSE_DELETE_SIZE_MISMATCH path=$path expected=$size actual=$actual_size\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0222: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0223: continue\nDETAIL=SOURCE_LINE=0224: fi\nDETAIL=SOURCE_LINE=0225:\nDETAIL=SOURCE_LINE=0226: actual_sha=\"$(sha256sum \"$path\" 2>>\"$LOG\" | cut -d \" \" -f1 || true)\"\nDETAIL=SOURCE_LINE=0227:\nDETAIL=SOURCE_LINE=0228: if [ \"$actual_sha\" != \"$sha\" ]; then\nDETAIL=SOURCE_LINE=0229: echo \"REFUSE_DELETE_SHA_MISMATCH path=$path\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0230: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0231: continue\nDETAIL=SOURCE_LINE=0232: fi\nDETAIL=SOURCE_LINE=0233:\nDETAIL=SOURCE_LINE=0234: echo \"PREDELETE_VALIDATION_OK path=$path size=$size sha=$sha\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0235: done < \"$FILES_TSV\"\nDETAIL=SOURCE_LINE=0236:\nDETAIL=SOURCE_LINE=0237: echo \"PREDELETE_VALIDATION_END bad=$BAD\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0238:fi\nDETAIL=SOURCE_LINE=0239:\nDETAIL=SOURCE_LINE=0240:if [ \"$BAD\" -eq 0 ]; then\nDETAIL=SOURCE_LINE=0241: echo \"LOCAL_PAYLOAD_DELETE_BEGIN\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0242:\nDETAIL=SOURCE_LINE=0243: while IFS=\"$(printf \"\t\")\" read -r id kind size sha mt bn path mode parts; do\nDETAIL=SOURCE_LINE=0244: if rm -f -- \"$path\" 2>>\"$LOG\"; then\nDETAIL=SOURCE_LINE=0245: if [ ! -e \"$path\" ] && [ ! -L \"$path\" ]; then\nDETAIL=SOURCE_LINE=0246: DELETED_COUNT=$((DELETED_COUNT+1))\nDETAIL=SOURCE_LINE=0247: DELETED_BYTES=$((DELETED_BYTES+size))\nDETAIL=SOURCE_LINE=0248: echo \"LOCAL_PAYLOAD_DELETED path=$path bytes=$size\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0249: else\nDETAIL=SOURCE_LINE=0250: echo \"LOCAL_PAYLOAD_DELETE_CONFIRM_FAILED path=$path\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0251: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0252: fi\nDETAIL=SOURCE_LINE=0253: else\nDETAIL=SOURCE_LINE=0254: echo \"LOCAL_PAYLOAD_DELETE_FAILED path=$path\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0255: BAD=$((BAD+1))\nDETAIL=SOURCE_LINE=0256: fi\nDETAIL=SOURCE_LINE=0257: done < \"$FILES_TSV\"\nDETAIL=SOURCE_LINE=0258:\nDETAIL=SOURCE_LINE=0259: if [ \"$BAD\" -eq 0 ] && [ \"$DELETED_COUNT\" -eq \"$COUNT\" ]; then\nDETAIL=SOURCE_LINE=0260: LOCAL_DUMPS_DELETED=true\nDETAIL=SOURCE_LINE=0261: fi\nDETAIL=SOURCE_LINE=0262:\nDETAIL=SOURCE_LINE=0263: echo \"LOCAL_PAYLOAD_DELETE_END deleted_count=$DELETED_COUNT deleted_bytes=$DELETED_BYTES bad=$BAD\" >> \"$LOG\"\nDETAIL=SOURCE_LINE=0264:fi\nDETAIL=SOURCE_LINE=0265:\nDETAIL=SOURCE_LINE=0266:if [ \"$BAD\" -eq 0 ] && [ \"$LOCAL_DUMPS_DELETED\" = true ]; then\nDETAIL=SOURCE_LINE=0267: STATUS=\"OK\"\nDETAIL=SOURCE_LINE=0268:else\nDETAIL=SOURCE_LINE=0269: STATUS=\"WARN\"\nDETAIL=SOURCE_LINE=0270:fi\nDETAIL=SOURCE_LINE=0271:\nDETAIL=SOURCE_LINE=0272:jq -n \\\nDETAIL=SOURCE_LINE=0273: --arg job_id \"$JOB_ID\" \\\nDETAIL=SOURCE_LINE=0274: --arg status \"$STATUS\" \\\nDETAIL=SOURCE_LINE=0275: --arg node \"$NODE\" \\\nDETAIL=SOURCE_LINE=0276: --arg remote_a \"$DEST_A\" \\\nDETAIL=SOURCE_LINE=0277: --arg remote_b \"$DEST_B\" \\\nDETAIL=SOURCE_LINE=0278: --argjson count \"$COUNT\" \\\nDETAIL=SOURCE_LINE=0279: --argjson total_bytes \"$TOTAL\" \\\nDETAIL=SOURCE_LINE=0280: --argjson chunked_count \"$CHUNKED\" \\\nDETAIL=SOURCE_LINE=0281: --argjson chunk_size_bytes \"$CHUNK_SIZE_BYTES\" \\\nDETAIL=SOURCE_LINE=0282: --arg log \"$LOG\" \\\nDETAIL=SOURCE_LINE=0283: --argjson local_dumps_deleted \"$LOCAL_DUMPS_DELETED\" \\\nDETAIL=SOURCE_LINE=0284: --argjson deleted_count \"$DELETED_COUNT\" \\\nDETAIL=SOURCE_LINE=0285: --argjson deleted_bytes \"$DELETED_BYTES\" \\\nDETAIL=SOURCE_LINE=0286: \"{job_id:\\$job_id,status:\\$status,node:\\$node,remote_a:\\$remote_a,remote_b:\\$remote_b,count:\\$count,total_bytes:\\$total_bytes,chunked_count:\\$chunked_count,chunk_size_bytes:\\$chunk_size_bytes,log:\\$log,local_dumps_deleted:\\$local_dumps_deleted,deleted_count:\\$deleted_count,deleted_bytes:\\$deleted_bytes}\" \\\nDETAIL=SOURCE_LINE=0287: > \"$ROOT/latest-vm-local-dumps-cloud.json\"\nDETAIL=SOURCE_LINE=0288:\nDETAIL=SOURCE_LINE=0289:echo \"STATUS=$STATUS TS=$TS TYPE=vm-local-dumps-cloud BAD=$BAD COUNT=$COUNT CHUNKED=$CHUNKED TOTAL_BYTES=$TOTAL JOB=$JOB_ID REMOTE_A=$DEST_A REMOTE_B=$DEST_B LOCAL_DUMPS_DELETED=$LOCAL_DUMPS_DELETED DELETED_COUNT=$DELETED_COUNT DELETED_BYTES=$DELETED_BYTES LOG=$LOG\" > \"$OUT\"\nDETAIL=SOURCE_LINE=0290:cat \"$OUT\"\nDETAIL=SELECTION_SOURCE_END\nDETAIL=ARCHIVES_CHANGED=NO\nDETAIL=SERVICE_STARTED=NO\nDETAIL=SECRET_VALUES_OUTPUT=NONE\nDETAIL=EXACT_SELECTION_AUDIT=PASS\nCHANGES_MADE=NO\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 0e4a0b8..3e2f4a4 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,5 +1,5 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=HOMELAB-VM-2CLOUD-SELECTION-LOGIC-437 +COMMAND_ID=HOMELAB-VM-2CLOUD-EXACT-SELECTION-438 STATUS=OK RC=0 HOST=pve01 @@ -9,19 +9,30 @@ REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 -COMMAND_SHA256=ad4b5c1c52b4eaffa790b5ebe2bda18624ef1905c2e4259c94c24b9c7d9c1165 +COMMAND_SHA256=458a61bfe476e4aeab9fdefc2057f75cf06c246702c653458e91f8c84eb823f2 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGES_MADE=false SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=51b679ab8afb22764728c1e4338d954989e990db1de09b0c0e331ebc32674a6a -SANITIZED_OUTPUT_SHA256=51b679ab8afb22764728c1e4338d954989e990db1de09b0c0e331ebc32674a6a +RAW_EVIDENCE_SHA256=1a2e8ce33d51e8861acf1434ee15eee384f0fcb6316b36d3b3fd87c946570531 +SANITIZED_OUTPUT_SHA256=1a2e8ce33d51e8861acf1434ee15eee384f0fcb6316b36d3b3fd87c946570531 OUTPUT_BEGIN -DETAIL=RUNTIME_SHA256=0ded6ba72e49b9cf7522c7f3d8840cc1026d2a173d4909dd3b4cfc5ff8831115 -DETAIL=DESIRED_SHA256=0ded6ba72e49b9cf7522c7f3d8840cc1026d2a173d4909dd3b4cfc5ff8831115 +:33: FutureWarning: Possible nested set at position 2 +:33: FutureWarning: Possible nested set at position 29 +:43: FutureWarning: Possible nested set at position 2 +:43: FutureWarning: Possible nested set at position 29 +DETAIL=SCRIPT_SHA256=0ded6ba72e49b9cf7522c7f3d8840cc1026d2a173d4909dd3b4cfc5ff8831115 +DETAIL=SELECTION_SOURCE_RANGE=14-290 DETAIL=SELECTION_SOURCE_BEGIN +DETAIL=SOURCE_LINE=0014:TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +DETAIL=SOURCE_LINE=0015:JOB_TS="$(date -u +%Y%m%dT%H%M%SZ)" +DETAIL=SOURCE_LINE=0016:NODE="$(hostname -s)" +DETAIL=SOURCE_LINE=0017:BAD=0 +DETAIL=SOURCE_LINE=0018:COUNT=0 +DETAIL=SOURCE_LINE=0019:TOTAL=0 +DETAIL=SOURCE_LINE=0020:CHUNKED=0 DETAIL=SOURCE_LINE=0021:VMIDS="${VMIDS:-110 112 150}" DETAIL=SOURCE_LINE=0022: DETAIL=SOURCE_LINE=0023:source /etc/homelab-backup.env 2>/dev/null || BAD=1 @@ -93,7 +104,31 @@ DETAIL=SOURCE_LINE=0088: items.append({ DETAIL=SOURCE_LINE=0089: "vmid":p[0], DETAIL=SOURCE_LINE=0090: "kind":p[1], DETAIL=SOURCE_LINE=0091: "size_bytes":int(p[2]), -DETAIL=SOURCE_LINE=... +DETAIL=SOURCE_LINE=0092: "sha256":p[3], +DETAIL=SOURCE_LINE=0093: "mtime_utc":p[4], +DETAIL=SOURCE_LINE=0094: "filename":p[5], +DETAIL=SOURCE_LINE=0095: "source_path":p[6], +DETAIL=SOURCE_LINE=0096: "mode":p[7], +DETAIL=SOURCE_LINE=0097: "parts":int(p[8]), +DETAIL=SOURCE_LINE=0098: }) +DETAIL=SOURCE_LINE=0099:print(json.dumps({ +DETAIL=SOURCE_LINE=0100: "job_id":job, +DETAIL=SOURCE_LINE=0101: "node":node, +DETAIL=SOURCE_LINE=0102: "generated_utc":ts, +DETAIL=SOURCE_LINE=0103: "vmids":vmids.split(), +DETAIL=SOURCE_LINE=0104: "count":int(count), +DETAIL=SOURCE_LINE=0105: "total_bytes":int(total), +DETAIL=SOURCE_LINE=0106: "chunked_count":int(chunked), +DETAIL=SOURCE_LINE=0107: "chunk_size_bytes":int(chunk_size), +DETAIL=SOURCE_LINE=0108: "items":items, +DETAIL=SOURCE_LINE=0109:}, indent=2)) +DETAIL=SOURCE_LINE=0110:PY +DETAIL=SOURCE_LINE=0111: +DETAIL=SOURCE_LINE=0112:DEST_A="${REMOTE_A%/}/${NODE}/vm-local-dumps/${JOB_ID}" +DETAIL=SOURCE_LINE=0113:DEST_B="${REMOTE_B%/}/${NODE}/vm-local-dumps/${JOB_ID}" +DETAIL=SOURCE_LINE=0114: +DETAIL=SOURCE_LINE=0115:upload_one_dest() { +DETAIL=SOURCE_LINE=0116: DEST="$1" DETAIL=SOURCE_LINE=0117: rclone mkdir "$DEST/files" >> "$LOG" 2>&1 || BAD=$((BAD+1)) DETAIL=SOURCE_LINE=0118: while IFS="$(printf "\t")" read -r id kind size sha mt bn path mode parts; do DETAIL=SOURCE_LINE=0119: if [ "$mode" = "chunked" ]; then @@ -158,7 +193,23 @@ DETAIL=SOURCE_LINE=0177:fi DETAIL=SOURCE_LINE=0178: DETAIL=SOURCE_LINE=0179:if [ "$BAD" -eq 0 ]; then DETAIL=SOURCE_LINE=0180: verify_one_dest "A" "$DEST_A" -DETAIL=SOURCE_LINE=... +DETAIL=SOURCE_LINE=0181: verify_one_dest "B" "$DEST_B" +DETAIL=SOURCE_LINE=0182:fi +DETAIL=SOURCE_LINE=0183: +DETAIL=SOURCE_LINE=0184:rm -rf "$VERIFY_ROOT/$JOB_ID" +DETAIL=SOURCE_LINE=0185:rm -rf "$CHUNK_ROOT/$JOB_ID" +DETAIL=SOURCE_LINE=0186: +DETAIL=SOURCE_LINE=0187:# HOMELAB_NO_LOCAL_AFTER_DUAL_CLOUD_434 +DETAIL=SOURCE_LINE=0188:LOCAL_DUMPS_DELETED=false +DETAIL=SOURCE_LINE=0189:DELETED_COUNT=0 +DETAIL=SOURCE_LINE=0190:DELETED_BYTES=0 +DETAIL=SOURCE_LINE=0191: +DETAIL=SOURCE_LINE=0192:if [ "$BAD" -eq 0 ]; then +DETAIL=SOURCE_LINE=0193: echo "PREDELETE_VALIDATION_BEGIN" >> "$LOG" +DETAIL=SOURCE_LINE=0194: +DETAIL=SOURCE_LINE=0195: while IFS="$(printf " ")" read -r id kind size sha mt bn path mode parts; do +DETAIL=SOURCE_LINE=0196: case "$path" in +DETAIL=SOURCE_LINE=0197: /var/lib/vz/dump/*) DETAIL=SOURCE_LINE=0198: ;; DETAIL=SOURCE_LINE=0199: *) DETAIL=SOURCE_LINE=0200: echo "REFUSE_DELETE_OUTSIDE_ALLOWED_ROOT path=$path" >> "$LOG" @@ -207,7 +258,9 @@ DETAIL=SOURCE_LINE=0242: DETAIL=SOURCE_LINE=0243: while IFS="$(printf " ")" read -r id kind size sha mt bn path mode parts; do DETAIL=SOURCE_LINE=0244: if rm -f -- "$path" 2>>"$LOG"; then DETAIL=SOURCE_LINE=0245: if [ ! -e "$path" ] && [ ! -L "$path" ]; then -DETAIL=SOURCE_LINE=... +DETAIL=SOURCE_LINE=0246: DELETED_COUNT=$((DELETED_COUNT+1)) +DETAIL=SOURCE_LINE=0247: DELETED_BYTES=$((DELETED_BYTES+size)) +DETAIL=SOURCE_LINE=0248: echo "LOCAL_PAYLOAD_DELETED path=$path bytes=$size" >> "$LOG" DETAIL=SOURCE_LINE=0249: else DETAIL=SOURCE_LINE=0250: echo "LOCAL_PAYLOAD_DELETE_CONFIRM_FAILED path=$path" >> "$LOG" DETAIL=SOURCE_LINE=0251: BAD=$((BAD+1)) @@ -227,18 +280,34 @@ DETAIL=SOURCE_LINE=0264:fi DETAIL=SOURCE_LINE=0265: DETAIL=SOURCE_LINE=0266:if [ "$BAD" -eq 0 ] && [ "$LOCAL_DUMPS_DELETED" = true ]; then DETAIL=SOURCE_LINE=0267: STATUS="OK" +DETAIL=SOURCE_LINE=0268:else +DETAIL=SOURCE_LINE=0269: STATUS="WARN" +DETAIL=SOURCE_LINE=0270:fi +DETAIL=SOURCE_LINE=0271: +DETAIL=SOURCE_LINE=0272:jq -n \ +DETAIL=SOURCE_LINE=0273: --arg job_id "$JOB_ID" \ +DETAIL=SOURCE_LINE=0274: --arg status "$STATUS" \ +DETAIL=SOURCE_LINE=0275: --arg node "$NODE" \ +DETAIL=SOURCE_LINE=0276: --arg remote_a "$DEST_A" \ +DETAIL=SOURCE_LINE=0277: --arg remote_b "$DEST_B" \ +DETAIL=SOURCE_LINE=0278: --argjson count "$COUNT" \ +DETAIL=SOURCE_LINE=0279: --argjson total_bytes "$TOTAL" \ +DETAIL=SOURCE_LINE=0280: --argjson chunked_count "$CHUNKED" \ +DETAIL=SOURCE_LINE=0281: --argjson chunk_size_bytes "$CHUNK_SIZE_BYTES" \ +DETAIL=SOURCE_LINE=0282: --arg log "$LOG" \ +DETAIL=SOURCE_LINE=0283: --argjson local_dumps_deleted "$LOCAL_DUMPS_DELETED" \ +DETAIL=SOURCE_LINE=0284: --argjson deleted_count "$DELETED_COUNT" \ +DETAIL=SOURCE_LINE=0285: --argjson deleted_bytes "$DELETED_BYTES" \ +DETAIL=SOURCE_LINE=0286: "{job_id:\$job_id,status:\$status,node:\$node,remote_a:\$remote_a,remote_b:\$remote_b,count:\$count,total_bytes:\$total_bytes,chunked_count:\$chunked_count,chunk_size_bytes:\$chunk_size_bytes,log:\$log,local_dumps_deleted:\$local_dumps_deleted,deleted_count:\$deleted_count,deleted_bytes:\$deleted_bytes}" \ +DETAIL=SOURCE_LINE=0287: > "$ROOT/latest-vm-local-dumps-cloud.json" +DETAIL=SOURCE_LINE=0288: +DETAIL=SOURCE_LINE=0289:echo "STATUS=$STATUS TS=$TS TYPE=vm-local-dumps-cloud BAD=$BAD COUNT=$COUNT CHUNKED=$CHUNKED TOTAL_BYTES=$TOTAL JOB=$JOB_ID REMOTE_A=$DEST_A REMOTE_B=$DEST_B LOCAL_DUMPS_DELETED=$LOCAL_DUMPS_DELETED DELETED_COUNT=$DELETED_COUNT DELETED_BYTES=$DELETED_BYTES LOG=$LOG" > "$OUT" +DETAIL=SOURCE_LINE=0290:cat "$OUT" DETAIL=SELECTION_SOURCE_END -DETAIL=INVENTORY_GROUP_COUNT=3 -DETAIL=INVENTORY_GROUP=KIND:lxc|ID:110|COUNT:4|TOTAL_BYTES:1337879243|LATEST_PATH:/var/lib/vz/dump/vzdump-lxc-110-2026_07_24-03_30_02.tar.zst|LATEST_BYTES:339140622 -DETAIL=INVENTORY_GROUP=KIND:lxc|ID:112|COUNT:4|TOTAL_BYTES:891239268|LATEST_PATH:/var/lib/vz/dump/vzdump-lxc-112-2026_07_24-03_30_11.tar.zst|LATEST_BYTES:222859457 -DETAIL=INVENTORY_GROUP=KIND:qemu|ID:150|COUNT:4|TOTAL_BYTES:28114919054|LATEST_PATH:/var/lib/vz/dump/vzdump-qemu-150-2026_07_24-03_30_16.vma.zst|LATEST_BYTES:7016624549 -DETAIL=ALL_ARCHIVE_COUNT=12 -DETAIL=ALL_ARCHIVE_BYTES=30344037565 -DETAIL=LATEST_PER_OBJECT_COUNT=3 -DETAIL=LATEST_PER_OBJECT_BYTES=7578624628 -DETAIL=BACKUP_PAYLOAD_CHANGED=NO +DETAIL=ARCHIVES_CHANGED=NO +DETAIL=SERVICE_STARTED=NO DETAIL=SECRET_VALUES_OUTPUT=NONE -DETAIL=VM_2CLOUD_SELECTION_LOGIC_AUDIT=PASS +DETAIL=EXACT_SELECTION_AUDIT=PASS CHANGES_MADE=NO OUTPUT_END