diff --git a/runtime/history/BACKUP-VM-DUMP-PIPELINE-073.json b/runtime/history/BACKUP-VM-DUMP-PIPELINE-073.json new file mode 100644 index 0000000..17c1b6e --- /dev/null +++ b/runtime/history/BACKUP-VM-DUMP-PIPELINE-073.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "BACKUP-VM-DUMP-PIPELINE-073", + "status": "FAIL", + "rc": 1, + "host": "pve01", + "mode": "read-only", + "component": "backup-audit", + "started_at_utc": "2026-07-21T12:28:25Z", + "finished_at_utc": "2026-07-21T12:28:26Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", + "command_sha256": "cd8291d0f14a37241e6062408603455d1ebd483091bebfa46c2b7a768f21308b", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "7db6efc577fe3d66f942203bcbeb45d4fb7f84204740483e04018721d4e4d9f4", + "sanitized_output_sha256": "7db6efc577fe3d66f942203bcbeb45d4fb7f84204740483e04018721d4e4d9f4", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "=== PVE_BACKUP_JOBS ===\n[{\"compress\":\"zstd\",\"enabled\":1,\"id\":\"homelab-nightly-all\",\"mailnotification\":\"failure\",\"mode\":\"snapshot\",\"next-run\":1784680200,\"prune-backups\":{\"keep-daily\":\"4\"},\"schedule\":\"03:30\",\"storage\":\"local\",\"type\":\"vzdump\",\"vmid\":\"110,111,112,113,130,150,190\"}]\n========== NODE=pve01 ==========\nFILE=/usr/local/sbin/homelab-vm-local-dumps-2cloud\nstat: cannot statx 'MODE=%a': No such file or directory\nstat: cannot statx 'OWNER=%U:%G': No such file or directory\nstat: cannot statx 'SIZE=%s': No such file or directory\nstat: cannot statx 'MTIME=%y': No such file or directory\n" +} diff --git a/runtime/history/BACKUP-VM-DUMP-PIPELINE-073.txt b/runtime/history/BACKUP-VM-DUMP-PIPELINE-073.txt new file mode 100644 index 0000000..8fd1727 --- /dev/null +++ b/runtime/history/BACKUP-VM-DUMP-PIPELINE-073.txt @@ -0,0 +1,32 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=BACKUP-VM-DUMP-PIPELINE-073 +STATUS=FAIL +RC=1 +HOST=pve01 +MODE=read-only +COMPONENT=backup-audit +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 +COMMAND_SHA256=cd8291d0f14a37241e6062408603455d1ebd483091bebfa46c2b7a768f21308b +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=false +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=7db6efc577fe3d66f942203bcbeb45d4fb7f84204740483e04018721d4e4d9f4 +SANITIZED_OUTPUT_SHA256=7db6efc577fe3d66f942203bcbeb45d4fb7f84204740483e04018721d4e4d9f4 +OUTPUT_BEGIN +=== PVE_BACKUP_JOBS === +[{"compress":"zstd","enabled":1,"id":"homelab-nightly-all","mailnotification":"failure","mode":"snapshot","next-run":1784680200,"prune-backups":{"keep-daily":"4"},"schedule":"03:30","storage":"local","type":"vzdump","vmid":"110,111,112,113,130,150,190"}] +========== NODE=pve01 ========== +FILE=/usr/local/sbin/homelab-vm-local-dumps-2cloud +stat: cannot statx 'MODE=%a': No such file or directory +stat: cannot statx 'OWNER=%U:%G': No such file or directory +stat: cannot statx 'SIZE=%s': No such file or directory +stat: cannot statx 'MTIME=%y': No such file or directory + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index b7c59ab..17c1b6e 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,30 +1,30 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "BACKUP-UNSAFE-SOURCE-CAPTURE-072", - "status": "OK", - "rc": 0, + "command_id": "BACKUP-VM-DUMP-PIPELINE-073", + "status": "FAIL", + "rc": 1, "host": "pve01", - "mode": "change", + "mode": "read-only", "component": "backup-audit", - "started_at_utc": "2026-07-21T12:25:10Z", - "finished_at_utc": "2026-07-21T12:25:10Z", + "started_at_utc": "2026-07-21T12:28:25Z", + "finished_at_utc": "2026-07-21T12:28:26Z", "reference_register_checked": true, "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", "error_register_checked": true, - "error_register_sha256": "f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1", - "command_sha256": "f79a127b4f27e1cdf886c681d5c7257244ec31a6c903e96cd53fc75d7a1ed8d4", + "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", + "command_sha256": "cd8291d0f14a37241e6062408603455d1ebd483091bebfa46c2b7a768f21308b", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, - "changes_made": true, + "changes_made": false, "sanitized": true, "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "b3a164a5b5260930b012305a47a24bc11db166e2bf6a6f25bfd1e1914941c5c7", - "sanitized_output_sha256": "b3a164a5b5260930b012305a47a24bc11db166e2bf6a6f25bfd1e1914941c5c7", + "raw_evidence_sha256": "7db6efc577fe3d66f942203bcbeb45d4fb7f84204740483e04018721d4e4d9f4", + "sanitized_output_sha256": "7db6efc577fe3d66f942203bcbeb45d4fb7f84204740483e04018721d4e4d9f4", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "========== NODE=pve01 PIPELINE=/usr/local/sbin/appbackup-dockge-stacks ==========\nFILE=/usr/local/sbin/appbackup-dockge-stacks,MODE=700,OWNER=root:root,SIZE=4097,MTIME=2026-07-10 04:14:10.050376764 +0300\n--- RELEVANT_LINES ---\n14:RA=\"${REMOTE_A%/}/${NODE}/apps/${APP}/${JOB}\"\n15:RB=\"${REMOTE_B%/}/${NODE}/apps/${APP}/${JOB}\"\n16:RAL=\"${REMOTE_A%/}/${NODE}/apps/${APP}/latest-backup-index.json\"\n17:RBL=\"${REMOTE_B%/}/${NODE}/apps/${APP}/latest-backup-index.json\"\n30:rm -rf \"$PAYLOAD\"\n32:SHA=\"$(sha256sum \"$STAGE/$ARCHIVE\" | cut -d\" \" -f1)\"\n51:echo \"[STEP] local checksum\"\n52:(cd \"$STAGE\" && sha256sum \"$ARCHIVE\" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e . manifest.json >/dev/null)\n54:echo \"[STEP] upload cloud A metadata\"; rclone copy \"$STAGE\" \"$RA\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n55:echo \"[STEP] upload cloud A archive chunks\"; rclone deletefile \"$RA/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RA/$ARCHIVE\"\n56:echo \"[STEP] upload cloud B metadata\"; rclone copy \"$STAGE\" \"$RB\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n57:echo \"[STEP] upload cloud B archive chunks\"; rclone deletefile \"$RB/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RB/$ARCHIVE\"\n58:echo \"[STEP] verify cloud A chunked\"; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-stage-verify-chunked \"$RA\" \"$ARCHIVE\" \"$VERIFY/a\" \"$JOB\"\n59:echo \"[STEP] verify cloud B chunked\"; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-stage-verify-chunked \"$RB\" \"$ARCHIVE\" \"$VERIFY/b\" \"$JOB\"\n60:echo \"[STEP] upload latest index\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RAL\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RBL\"; rclone cat \"$RAL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null; rclone cat \"$RBL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n76:rm -rf \"$STAGE\" \"$VERIFY\"\n--- UNIT_REFERENCES ---\nUNSAFE_SOURCE_CAPTURE=COMPLETE\n" + "output": "=== PVE_BACKUP_JOBS ===\n[{\"compress\":\"zstd\",\"enabled\":1,\"id\":\"homelab-nightly-all\",\"mailnotification\":\"failure\",\"mode\":\"snapshot\",\"next-run\":1784680200,\"prune-backups\":{\"keep-daily\":\"4\"},\"schedule\":\"03:30\",\"storage\":\"local\",\"type\":\"vzdump\",\"vmid\":\"110,111,112,113,130,150,190\"}]\n========== NODE=pve01 ==========\nFILE=/usr/local/sbin/homelab-vm-local-dumps-2cloud\nstat: cannot statx 'MODE=%a': No such file or directory\nstat: cannot statx 'OWNER=%U:%G': No such file or directory\nstat: cannot statx 'SIZE=%s': No such file or directory\nstat: cannot statx 'MTIME=%y': No such file or directory\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 5882f30..8fd1727 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,45 +1,32 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=BACKUP-UNSAFE-SOURCE-CAPTURE-072 -STATUS=OK -RC=0 +COMMAND_ID=BACKUP-VM-DUMP-PIPELINE-073 +STATUS=FAIL +RC=1 HOST=pve01 -MODE=change +MODE=read-only COMPONENT=backup-audit REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e ERROR_REGISTER_CHECK=OK -ERROR_REGISTER_SHA256=f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1 -COMMAND_SHA256=f79a127b4f27e1cdf886c681d5c7257244ec31a6c903e96cd53fc75d7a1ed8d4 +ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 +COMMAND_SHA256=cd8291d0f14a37241e6062408603455d1ebd483091bebfa46c2b7a768f21308b DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true -CHANGES_MADE=true +CHANGES_MADE=false SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=b3a164a5b5260930b012305a47a24bc11db166e2bf6a6f25bfd1e1914941c5c7 -SANITIZED_OUTPUT_SHA256=b3a164a5b5260930b012305a47a24bc11db166e2bf6a6f25bfd1e1914941c5c7 +RAW_EVIDENCE_SHA256=7db6efc577fe3d66f942203bcbeb45d4fb7f84204740483e04018721d4e4d9f4 +SANITIZED_OUTPUT_SHA256=7db6efc577fe3d66f942203bcbeb45d4fb7f84204740483e04018721d4e4d9f4 OUTPUT_BEGIN -========== NODE=pve01 PIPELINE=/usr/local/sbin/appbackup-dockge-stacks ========== -FILE=/usr/local/sbin/appbackup-dockge-stacks,MODE=700,OWNER=root:root,SIZE=4097,MTIME=2026-07-10 04:14:10.050376764 +0300 ---- RELEVANT_LINES --- -14:RA="${REMOTE_A%/}/${NODE}/apps/${APP}/${JOB}" -15:RB="${REMOTE_B%/}/${NODE}/apps/${APP}/${JOB}" -16:RAL="${REMOTE_A%/}/${NODE}/apps/${APP}/latest-backup-index.json" -17:RBL="${REMOTE_B%/}/${NODE}/apps/${APP}/latest-backup-index.json" -30:rm -rf "$PAYLOAD" -32:SHA="$(sha256sum "$STAGE/$ARCHIVE" | cut -d" " -f1)" -51:echo "[STEP] local checksum" -52:(cd "$STAGE" && sha256sum "$ARCHIVE" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e . manifest.json >/dev/null) -54:echo "[STEP] upload cloud A metadata"; rclone copy "$STAGE" "$RA" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s -55:echo "[STEP] upload cloud A archive chunks"; rclone deletefile "$RA/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RA/$ARCHIVE" -56:echo "[STEP] upload cloud B metadata"; rclone copy "$STAGE" "$RB" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s -57:echo "[STEP] upload cloud B archive chunks"; rclone deletefile "$RB/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RB/$ARCHIVE" -58:echo "[STEP] verify cloud A chunked"; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-stage-verify-chunked "$RA" "$ARCHIVE" "$VERIFY/a" "$JOB" -59:echo "[STEP] verify cloud B chunked"; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-stage-verify-chunked "$RB" "$ARCHIVE" "$VERIFY/b" "$JOB" -60:echo "[STEP] upload latest index"; rclone copyto "$STAGE/latest-backup-index.json" "$RAL"; rclone copyto "$STAGE/latest-backup-index.json" "$RBL"; rclone cat "$RAL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null; rclone cat "$RBL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null -76:rm -rf "$STAGE" "$VERIFY" ---- UNIT_REFERENCES --- -UNSAFE_SOURCE_CAPTURE=COMPLETE +=== PVE_BACKUP_JOBS === +[{"compress":"zstd","enabled":1,"id":"homelab-nightly-all","mailnotification":"failure","mode":"snapshot","next-run":1784680200,"prune-backups":{"keep-daily":"4"},"schedule":"03:30","storage":"local","type":"vzdump","vmid":"110,111,112,113,130,150,190"}] +========== NODE=pve01 ========== +FILE=/usr/local/sbin/homelab-vm-local-dumps-2cloud +stat: cannot statx 'MODE=%a': No such file or directory +stat: cannot statx 'OWNER=%U:%G': No such file or directory +stat: cannot statx 'SIZE=%s': No such file or directory +stat: cannot statx 'MTIME=%y': No such file or directory OUTPUT_END CHAT_OUTPUT_END