From 72cf78774e0c1e71cb0f12feb4306e6721f13f9e Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Tue, 21 Jul 2026 12:25:11 +0000 Subject: [PATCH] runtime: publish BACKUP-UNSAFE-SOURCE-CAPTURE-072 --- .../BACKUP-UNSAFE-SOURCE-CAPTURE-072.json | 30 ++ .../BACKUP-UNSAFE-SOURCE-CAPTURE-072.txt | 45 +++ runtime/latest.json | 18 +- runtime/latest.txt | 278 ++---------------- 4 files changed, 111 insertions(+), 260 deletions(-) create mode 100644 runtime/history/BACKUP-UNSAFE-SOURCE-CAPTURE-072.json create mode 100644 runtime/history/BACKUP-UNSAFE-SOURCE-CAPTURE-072.txt diff --git a/runtime/history/BACKUP-UNSAFE-SOURCE-CAPTURE-072.json b/runtime/history/BACKUP-UNSAFE-SOURCE-CAPTURE-072.json new file mode 100644 index 0000000..b7c59ab --- /dev/null +++ b/runtime/history/BACKUP-UNSAFE-SOURCE-CAPTURE-072.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "BACKUP-UNSAFE-SOURCE-CAPTURE-072", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "change", + "component": "backup-audit", + "started_at_utc": "2026-07-21T12:25:10Z", + "finished_at_utc": "2026-07-21T12:25:10Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1", + "command_sha256": "f79a127b4f27e1cdf886c681d5c7257244ec31a6c903e96cd53fc75d7a1ed8d4", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": true, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "b3a164a5b5260930b012305a47a24bc11db166e2bf6a6f25bfd1e1914941c5c7", + "sanitized_output_sha256": "b3a164a5b5260930b012305a47a24bc11db166e2bf6a6f25bfd1e1914941c5c7", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "========== NODE=pve01 PIPELINE=/usr/local/sbin/appbackup-dockge-stacks ==========\nFILE=/usr/local/sbin/appbackup-dockge-stacks,MODE=700,OWNER=root:root,SIZE=4097,MTIME=2026-07-10 04:14:10.050376764 +0300\n--- RELEVANT_LINES ---\n14:RA=\"${REMOTE_A%/}/${NODE}/apps/${APP}/${JOB}\"\n15:RB=\"${REMOTE_B%/}/${NODE}/apps/${APP}/${JOB}\"\n16:RAL=\"${REMOTE_A%/}/${NODE}/apps/${APP}/latest-backup-index.json\"\n17:RBL=\"${REMOTE_B%/}/${NODE}/apps/${APP}/latest-backup-index.json\"\n30:rm -rf \"$PAYLOAD\"\n32:SHA=\"$(sha256sum \"$STAGE/$ARCHIVE\" | cut -d\" \" -f1)\"\n51:echo \"[STEP] local checksum\"\n52:(cd \"$STAGE\" && sha256sum \"$ARCHIVE\" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e . manifest.json >/dev/null)\n54:echo \"[STEP] upload cloud A metadata\"; rclone copy \"$STAGE\" \"$RA\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n55:echo \"[STEP] upload cloud A archive chunks\"; rclone deletefile \"$RA/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RA/$ARCHIVE\"\n56:echo \"[STEP] upload cloud B metadata\"; rclone copy \"$STAGE\" \"$RB\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n57:echo \"[STEP] upload cloud B archive chunks\"; rclone deletefile \"$RB/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RB/$ARCHIVE\"\n58:echo \"[STEP] verify cloud A chunked\"; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-stage-verify-chunked \"$RA\" \"$ARCHIVE\" \"$VERIFY/a\" \"$JOB\"\n59:echo \"[STEP] verify cloud B chunked\"; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-stage-verify-chunked \"$RB\" \"$ARCHIVE\" \"$VERIFY/b\" \"$JOB\"\n60:echo \"[STEP] upload latest index\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RAL\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RBL\"; rclone cat \"$RAL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null; rclone cat \"$RBL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n76:rm -rf \"$STAGE\" \"$VERIFY\"\n--- UNIT_REFERENCES ---\nUNSAFE_SOURCE_CAPTURE=COMPLETE\n" +} diff --git a/runtime/history/BACKUP-UNSAFE-SOURCE-CAPTURE-072.txt b/runtime/history/BACKUP-UNSAFE-SOURCE-CAPTURE-072.txt new file mode 100644 index 0000000..5882f30 --- /dev/null +++ b/runtime/history/BACKUP-UNSAFE-SOURCE-CAPTURE-072.txt @@ -0,0 +1,45 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=BACKUP-UNSAFE-SOURCE-CAPTURE-072 +STATUS=OK +RC=0 +HOST=pve01 +MODE=change +COMPONENT=backup-audit +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1 +COMMAND_SHA256=f79a127b4f27e1cdf886c681d5c7257244ec31a6c903e96cd53fc75d7a1ed8d4 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=true +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=b3a164a5b5260930b012305a47a24bc11db166e2bf6a6f25bfd1e1914941c5c7 +SANITIZED_OUTPUT_SHA256=b3a164a5b5260930b012305a47a24bc11db166e2bf6a6f25bfd1e1914941c5c7 +OUTPUT_BEGIN +========== NODE=pve01 PIPELINE=/usr/local/sbin/appbackup-dockge-stacks ========== +FILE=/usr/local/sbin/appbackup-dockge-stacks,MODE=700,OWNER=root:root,SIZE=4097,MTIME=2026-07-10 04:14:10.050376764 +0300 +--- RELEVANT_LINES --- +14:RA="${REMOTE_A%/}/${NODE}/apps/${APP}/${JOB}" +15:RB="${REMOTE_B%/}/${NODE}/apps/${APP}/${JOB}" +16:RAL="${REMOTE_A%/}/${NODE}/apps/${APP}/latest-backup-index.json" +17:RBL="${REMOTE_B%/}/${NODE}/apps/${APP}/latest-backup-index.json" +30:rm -rf "$PAYLOAD" +32:SHA="$(sha256sum "$STAGE/$ARCHIVE" | cut -d" " -f1)" +51:echo "[STEP] local checksum" +52:(cd "$STAGE" && sha256sum "$ARCHIVE" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e . manifest.json >/dev/null) +54:echo "[STEP] upload cloud A metadata"; rclone copy "$STAGE" "$RA" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s +55:echo "[STEP] upload cloud A archive chunks"; rclone deletefile "$RA/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RA/$ARCHIVE" +56:echo "[STEP] upload cloud B metadata"; rclone copy "$STAGE" "$RB" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s +57:echo "[STEP] upload cloud B archive chunks"; rclone deletefile "$RB/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RB/$ARCHIVE" +58:echo "[STEP] verify cloud A chunked"; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-stage-verify-chunked "$RA" "$ARCHIVE" "$VERIFY/a" "$JOB" +59:echo "[STEP] verify cloud B chunked"; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-stage-verify-chunked "$RB" "$ARCHIVE" "$VERIFY/b" "$JOB" +60:echo "[STEP] upload latest index"; rclone copyto "$STAGE/latest-backup-index.json" "$RAL"; rclone copyto "$STAGE/latest-backup-index.json" "$RBL"; rclone cat "$RAL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null; rclone cat "$RBL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null +76:rm -rf "$STAGE" "$VERIFY" +--- UNIT_REFERENCES --- +UNSAFE_SOURCE_CAPTURE=COMPLETE + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index ae77377..b7c59ab 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,30 +1,30 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "BACKUP-REMEDIATION-TARGETS-070", + "command_id": "BACKUP-UNSAFE-SOURCE-CAPTURE-072", "status": "OK", "rc": 0, "host": "pve01", - "mode": "read-only", + "mode": "change", "component": "backup-audit", - "started_at_utc": "2026-07-21T12:21:13Z", - "finished_at_utc": "2026-07-21T12:21:13Z", + "started_at_utc": "2026-07-21T12:25:10Z", + "finished_at_utc": "2026-07-21T12:25:10Z", "reference_register_checked": true, "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", "error_register_checked": true, "error_register_sha256": "f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1", - "command_sha256": "dccff9fba1741a41e62299cfbb57f3bb2f082b09a654d7649453becbeb1d9924", + "command_sha256": "f79a127b4f27e1cdf886c681d5c7257244ec31a6c903e96cd53fc75d7a1ed8d4", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, - "changes_made": false, + "changes_made": true, "sanitized": true, "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "06d66f8abbe177ea07ddaf08dcd812189601ea92a92a65db53baf5aa80867097", - "sanitized_output_sha256": "06d66f8abbe177ea07ddaf08dcd812189601ea92a92a65db53baf5aa80867097", + "raw_evidence_sha256": "b3a164a5b5260930b012305a47a24bc11db166e2bf6a6f25bfd1e1914941c5c7", + "sanitized_output_sha256": "b3a164a5b5260930b012305a47a24bc11db166e2bf6a6f25bfd1e1914941c5c7", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "=== UNSAFE_PIPELINES ===\nPIPELINE=/usr/local/sbin/appbackup-dockge-stacks REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/appbackup-restore-check REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/appbackup-retention REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/backup-restore-check REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-backup-edge-vm-vzdump REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-chat-run REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-extended-appbackup-run REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-ingress-standby-refresh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-mail-cloud-critical-upload REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-mail-cloud-restore-drill REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-rclone-cat-chunked REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=NO LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-rclone-cat-flat-chunked REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-rclone-copy-chunked REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-rclone-copy-flat-chunked REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-router-running-config-mail-cloud REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-stage-2cloud REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-stage-2cloud-flat REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-stage-2cloud-flat-canary REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-stage-verify-chunked REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-stage-verify-flat-chunked REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-vm160-remote-backup REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-vm-full-retention REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-vm-local-dumps-2cloud REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/codevipe-cloud-backup.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill REMOTE_COUNT=1 REMOTES=pve01-mail-05-crypt: UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\n=== ALL_PIPELINES ===\nPIPELINE=/etc/cron.d/vzdump REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/etc/systemd/system/homelab-edge-vm-offhost-freshness.timer REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/etc/systemd/system/homelab-edge-vm-vzdump-backup.service REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/etc/systemd/system/homelab-edge-vm-vzdump-backup.timer REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.service REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/etc/systemd/system/homelab-vm-full-backup.service REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/etc/systemd/system/multi-user.target.wants/pve-guests.service REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/etc/systemd/system/pve-manager.service REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/etc/systemd/system/timers.target.wants/homelab-edge-vm-offhost-freshness.timer REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/appbackup-dockge-stacks REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/appbackup-restore-check REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/appbackup-retention REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/backupctl REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/backup-restore-check REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/backup-retention REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/homelab-backup-edge-vm-vzdump REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-capacity-risk REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/homelab-chat-run REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-edge-vm-offhost-freshness REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=YES LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/homelab-extended-appbackup-run REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-golden-state-extended-overlay REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/homelab-golden-state-index REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/homelab-health-metrics REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/homelab-ingress-standby-refresh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-mail-cloud-critical-upload REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-mail-cloud-restore-drill REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-rclone-cat-chunked REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=NO LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-rclone-cat-flat-chunked REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-rclone-copy-chunked REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-rclone-copy-flat-chunked REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/homelab-rclone-required-check.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/homelab-router-hard-power-cycle REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/homelab-router-running-config-mail-cloud REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-secret-exposure-guard REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/homelab-stage-2cloud REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-stage-2cloud-flat REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-stage-2cloud-flat-canary REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-stage-verify-chunked REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-stage-verify-flat-chunked REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-vm160-remote-backup REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-vm-backup-health REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/homelab-vm-backup-policy REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/homelab-vm-full-backup REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=YES LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/homelab-vm-full-restore-drill REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=YES LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/homelab-vm-full-retention REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-vm-local-dumps-2cloud REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-vm-local-dumps-retention REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/skladchik-reports-monitor-backup REMOTE_COUNT=2 REMOTES=pve01-mail-01-crypt:,pve01-mail-02-crypt: UPLOAD=NO VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check REMOTE_COUNT=2 REMOTES=pve01-mail-01-crypt:,pve01-mail-02-crypt: UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=NO\nPIPELINE=/etc/cron.d/vzdump REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/etc/systemd/system/multi-user.target.wants/pve-guests.service REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/etc/systemd/system/pve-manager.service REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/codevipe-cloud-backup.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/homelab-rclone-required-check.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill REMOTE_COUNT=1 REMOTES=pve01-mail-05-crypt: UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/etc/cron.d/vzdump REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/etc/systemd/system/multi-user.target.wants/pve-guests.service REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/etc/systemd/system/pve-manager.service REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/homelab-rclone-required-check.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\n=== LOCAL_ARCHIVES ===\n--- LOCAL_ARCHIVES ---\n7022386770\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.vma.zst\n7021952392\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst\n7021297699\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst\n7016275481\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst\n329541843\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst\n326601118\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst\n323668231\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst\n321121649\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.tar.zst\n222777732\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst\n222761273\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst\n222745965\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst\n222727760\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.tar.zst\n3021\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.log\n3020\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.log\n3018\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.log\n3017\t/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.log\n1089\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.log\n1086\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.log\n989\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.log\n989\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.log\n986\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.log\n986\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.log\n731\t/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.log\n729\t/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.log\n========== NODE=pve02 ==========\nPIPELINE=/etc/cron.d/vzdump REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/etc/systemd/system/multi-user.target.wants/pve-guests.service REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/etc/systemd/system/pve-manager.service REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/codevipe-cloud-backup.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\nPIPELINE=/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/homelab-rclone-required-check.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill REMOTE_COUNT=1 REMOTES=pve01-mail-05-crypt: UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES\n--- LOCAL_ARCHIVES ---\n997503422\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.vma.zst\n868048369\t/var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.vma.zst\n288349157\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.tar.zst\n286538526\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.tar.zst\n284443608\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst\n283022034\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_18-03_30_03.tar.zst\n220080970\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.tar.zst\n220065825\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.tar.zst\n220049041\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst\n220033534\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_18-03_30_20.tar.zst\n7547\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_18-01_02_44.log\n7443\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_18-01_38_20.log\n5796\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_17-21_30_20.log\n2685\t/var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.log\n2590\t/var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.log\n1088\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.log\n1085\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.log\n988\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.log\n988\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_18-03_30_20.log\n985\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.log\n985\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_18-03_30_03.log\n731\t/var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.log\n728\t/var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.log\n========== NODE=pve03 ==========\nPIPELINE=/etc/cron.d/vzdump REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/etc/systemd/system/multi-user.target.wants/pve-guests.service REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/etc/systemd/system/pve-manager.service REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\nPIPELINE=/usr/local/sbin/homelab-rclone-required-check.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO\n--- LOCAL_ARCHIVES ---\n11116472459\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.vma.zst\n11043834894\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.vma.zst\n11013723404\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.vma.zst\n10994820121\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_18-03_30_02.vma.zst\n1137072350\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.vma.zst\n838951976\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.vma.zst\n838817925\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.vma.zst\n785249297\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_18-03_33_48.vma.zst\n7106\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_02-03_30_02.log\n7102\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_03-03_30_05.log\n6564\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_18-03_30_02.log\n6560\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.log\n6559\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.log\n6459\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.log\n6246\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_04-03_30_02.log\n4107\t/var/lib/vz/dump/vzdump-qemu-130-2026_07_07-03_30_01.log\n2642\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.log\n2500\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.log\n2404\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.log\n2305\t/var/lib/vz/dump/vzdump-qemu-190-2026_07_18-03_33_48.log\n========== PVE_RETENTION ==========\n=== PVE_RETENTION ===\n========== PVE_RETENTION ==========\n{\"id\":\"homelab-nightly-all\",\"node\":null,\"vmid\":\"110,111,112,113,130,150,190\",\"storage\":\"local\",\"prune_backups\":null,\"enabled\":1,\"schedule\":\"03:30\"}\nDELETE_GUARD_ANALYSIS=COMPLETE\n=== CURRENT_MATRIX ===\nSTATUS=FAIL\nTS=2026-07-21T07:02:41Z\nTYPE=unified-backup-matrix\nMATRIX_MODE=authoritative\nJOBS_TOTAL=41\nJOBS_OK=30\nJOBS_BAD=6\nJOBS_UNMAPPED=0\nJOBS_DISABLED=5\nSTATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_JSON=/etc/homelab-backup-matrix/matrix.json\nAUTHORITY=canonical\nCOMPAT_OUTPUTS=29\nCOMPAT_MAP=/var/lib/homelab-backup-matrix/compat-map.json\nJOB=dpkg-db-backup STATUS=OK EVIDENCE=execution_only AGE=36147 MAX_AGE=172800\nJOB=filebrowser-backup STATUS=OK EVIDENCE=artifact_or_health AGE=14443 MAX_AGE=259200\nJOB=filebrowser-restore-validate STATUS=STALE EVIDENCE=artifact_or_health AGE=1016424 MAX_AGE=864000\nJOB=homelab-appbackup-dockge-stacks STATUS=OK EVIDENCE=artifact_or_health AGE=22248 MAX_AGE=172800\nJOB=homelab-appbackup-mariadb-batch STATUS=OK EVIDENCE=execution_only AGE=15387 MAX_AGE=172800\nJOB=homelab-appbackup-npmplus-edge STATUS=OK EVIDENCE=artifact_or_health AGE=20573 MAX_AGE=172800\nJOB=homelab-appbackup-postgres-batch STATUS=OK EVIDENCE=execution_only AGE=16416 MAX_AGE=172800\nJOB=homelab-appbackup-sqlite-batch STATUS=OK EVIDENCE=execution_only AGE=18327 MAX_AGE=172800\nJOB=homelab-authentik-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987790 MAX_AGE=172800\nJOB=homelab-auto-backup STATUS=OK EVIDENCE=execution_only AGE=195424 MAX_AGE=864000\nJOB=homelab-backup-coverage-matrix STATUS=OK EVIDENCE=artifact_or_health AGE=4346 MAX_AGE=172800\nJOB=homelab-backup-freshness STATUS=OK EVIDENCE=execution_only AGE=35589 MAX_AGE=172800\nJOB=homelab-backup-healthcheck STATUS=OK EVIDENCE=execution_only AGE=10631 MAX_AGE=172800\nJOB=homelab-backup-sla-health STATUS=FAIL EVIDENCE=artifact_or_health AGE=144 MAX_AGE=172800\nJOB=homelab-edge-vm-offhost-freshness STATUS=OK EVIDENCE=artifact_or_health AGE=18426 MAX_AGE=172800\nJOB=homelab-emergency-backup STATUS=OK EVIDENCE=artifact_or_health AGE=24539 MAX_AGE=172800\nJOB=homelab-extended-appbackup STATUS=OK EVIDENCE=artifact_or_health AGE=14156 MAX_AGE=172800\nJOB=homelab-gitea-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987785 MAX_AGE=172800\nJOB=homelab-golden-state-index STATUS=OK EVIDENCE=artifact_or_health AGE=4112 MAX_AGE=172800\nJOB=homelab-mail-cloud-critical-upload STATUS=OK EVIDENCE=artifact_or_health AGE=11997 MAX_AGE=172800\nJOB=homelab-mail-cloud-edge-vm-upload STATUS=OK EVIDENCE=artifact_or_health AGE=174045 MAX_AGE=864000\nJOB=homelab-mail-cloud-nextcloud-vm-upload STATUS=OK EVIDENCE=artifact_or_health AGE=7575 MAX_AGE=864000\nJOB=homelab-mail-cloud-restore-drill STATUS=STALE EVIDENCE=artifact_or_health AGE=988269 MAX_AGE=864000\nJOB=homelab-npmplus-kuma-config-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=60071 MAX_AGE=172800\nJOB=homelab-router-running-config-mail-cloud STATUS=OK EVIDENCE=artifact_or_health AGE=17076 MAX_AGE=172800\nJOB=homelab-vaultwarden-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987768 MAX_AGE=172800\nJOB=homelab-verified-backup-health STATUS=OK EVIDENCE=artifact_or_health AGE=11845 MAX_AGE=172800\nJOB=homelab-vm-backup-health STATUS=OK EVIDENCE=artifact_or_health AGE=8742 MAX_AGE=172800\nJOB=homelab-vm-backup-policy STATUS=OK EVIDENCE=artifact_or_health AGE=5548 MAX_AGE=172800\nJOB=homelab-vm-full-backup STATUS=OK EVIDENCE=artifact_or_health AGE=24778 MAX_AGE=172800\nJOB=homelab-vm-full-restore-drill STATUS=OK EVIDENCE=artifact_or_health AGE=523085 MAX_AGE=10368000\nJOB=homelab-vm-local-dumps-2cloud STATUS=OK EVIDENCE=artifact_or_health AGE=172141 MAX_AGE=864000\nJOB=homelab-vm-local-dumps-retention STATUS=OK EVIDENCE=artifact_or_health AGE=168839 MAX_AGE=864000\nJOB=immich-media-offhost-sync STATUS=OK EVIDENCE=artifact_or_health AGE=4286 MAX_AGE=259200\nJOB=memos-backup STATUS=OK EVIDENCE=artifact_or_health AGE=14547 MAX_AGE=259200\nJOB=netbird-vps-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=1747896 MAX_AGE=864000\nJOB=paperless-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=16347 MAX_AGE=259200\nJOB=paperless-restore-dry-run STATUS=STALE EVIDENCE=artifact_or_health AGE=1001986 MAX_AGE=864000\nJOB=skladchik-reports-monitor-backup STATUS=OK EVIDENCE=artifact_or_health AGE=19639 MAX_AGE=172800\nJOB=skladchik-reports-monitor-backup-restore-check STATUS=OK EVIDENCE=artifact_or_health AGE=190273 MAX_AGE=864000\nJOB=vm160-remote-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=26607 MAX_AGE=172800\n" + "output": "========== NODE=pve01 PIPELINE=/usr/local/sbin/appbackup-dockge-stacks ==========\nFILE=/usr/local/sbin/appbackup-dockge-stacks,MODE=700,OWNER=root:root,SIZE=4097,MTIME=2026-07-10 04:14:10.050376764 +0300\n--- RELEVANT_LINES ---\n14:RA=\"${REMOTE_A%/}/${NODE}/apps/${APP}/${JOB}\"\n15:RB=\"${REMOTE_B%/}/${NODE}/apps/${APP}/${JOB}\"\n16:RAL=\"${REMOTE_A%/}/${NODE}/apps/${APP}/latest-backup-index.json\"\n17:RBL=\"${REMOTE_B%/}/${NODE}/apps/${APP}/latest-backup-index.json\"\n30:rm -rf \"$PAYLOAD\"\n32:SHA=\"$(sha256sum \"$STAGE/$ARCHIVE\" | cut -d\" \" -f1)\"\n51:echo \"[STEP] local checksum\"\n52:(cd \"$STAGE\" && sha256sum \"$ARCHIVE\" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf \"$ARCHIVE\" >/dev/null && jq -e . manifest.json >/dev/null)\n54:echo \"[STEP] upload cloud A metadata\"; rclone copy \"$STAGE\" \"$RA\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n55:echo \"[STEP] upload cloud A archive chunks\"; rclone deletefile \"$RA/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RA/$ARCHIVE\"\n56:echo \"[STEP] upload cloud B metadata\"; rclone copy \"$STAGE\" \"$RB\" --exclude \"$ARCHIVE\" --exclude \"$ARCHIVE.sha256\" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s\n57:echo \"[STEP] upload cloud B archive chunks\"; rclone deletefile \"$RB/$ARCHIVE\" >/dev/null 2>&1 || true; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-rclone-copy-chunked \"$STAGE/$ARCHIVE\" \"$RB/$ARCHIVE\"\n58:echo \"[STEP] verify cloud A chunked\"; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-stage-verify-chunked \"$RA\" \"$ARCHIVE\" \"$VERIFY/a\" \"$JOB\"\n59:echo \"[STEP] verify cloud B chunked\"; CHUNK_TMP_BASE=\"${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}\" homelab-stage-verify-chunked \"$RB\" \"$ARCHIVE\" \"$VERIFY/b\" \"$JOB\"\n60:echo \"[STEP] upload latest index\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RAL\"; rclone copyto \"$STAGE/latest-backup-index.json\" \"$RBL\"; rclone cat \"$RAL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null; rclone cat \"$RBL\" | jq -e --arg job \"$JOB\" \".job_id == \\$job\" >/dev/null\n76:rm -rf \"$STAGE\" \"$VERIFY\"\n--- UNIT_REFERENCES ---\nUNSAFE_SOURCE_CAPTURE=COMPLETE\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 4e54d8e..5882f30 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,269 +1,45 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=BACKUP-REMEDIATION-TARGETS-070 +COMMAND_ID=BACKUP-UNSAFE-SOURCE-CAPTURE-072 STATUS=OK RC=0 HOST=pve01 -MODE=read-only +MODE=change COMPONENT=backup-audit REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=f6d592c8d53021fe9feeb81a7e0cca33f4b44501143f6378290486e46d250bb1 -COMMAND_SHA256=dccff9fba1741a41e62299cfbb57f3bb2f082b09a654d7649453becbeb1d9924 +COMMAND_SHA256=f79a127b4f27e1cdf886c681d5c7257244ec31a6c903e96cd53fc75d7a1ed8d4 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true -CHANGES_MADE=false +CHANGES_MADE=true SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=06d66f8abbe177ea07ddaf08dcd812189601ea92a92a65db53baf5aa80867097 -SANITIZED_OUTPUT_SHA256=06d66f8abbe177ea07ddaf08dcd812189601ea92a92a65db53baf5aa80867097 +RAW_EVIDENCE_SHA256=b3a164a5b5260930b012305a47a24bc11db166e2bf6a6f25bfd1e1914941c5c7 +SANITIZED_OUTPUT_SHA256=b3a164a5b5260930b012305a47a24bc11db166e2bf6a6f25bfd1e1914941c5c7 OUTPUT_BEGIN -=== UNSAFE_PIPELINES === -PIPELINE=/usr/local/sbin/appbackup-dockge-stacks REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/appbackup-restore-check REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/appbackup-retention REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/backup-restore-check REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-backup-edge-vm-vzdump REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-chat-run REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-extended-appbackup-run REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-ingress-standby-refresh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-mail-cloud-critical-upload REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-mail-cloud-restore-drill REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-rclone-cat-chunked REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=NO LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-rclone-cat-flat-chunked REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-rclone-copy-chunked REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-rclone-copy-flat-chunked REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-router-running-config-mail-cloud REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-stage-2cloud REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-stage-2cloud-flat REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-stage-2cloud-flat-canary REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-stage-verify-chunked REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-stage-verify-flat-chunked REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-vm160-remote-backup REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-vm-full-retention REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-vm-local-dumps-2cloud REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/codevipe-cloud-backup.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill REMOTE_COUNT=1 REMOTES=pve01-mail-05-crypt: UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -=== ALL_PIPELINES === -PIPELINE=/etc/cron.d/vzdump REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/etc/systemd/system/homelab-edge-vm-offhost-freshness.timer REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/etc/systemd/system/homelab-edge-vm-vzdump-backup.service REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/etc/systemd/system/homelab-edge-vm-vzdump-backup.timer REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/etc/systemd/system/homelab-mail-cloud-nextcloud-vm-upload.service REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/etc/systemd/system/homelab-vm-full-backup.service REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/etc/systemd/system/multi-user.target.wants/pve-guests.service REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/etc/systemd/system/pve-manager.service REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/etc/systemd/system/timers.target.wants/homelab-edge-vm-offhost-freshness.timer REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/usr/local/sbin/appbackup-dockge-stacks REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/appbackup-restore-check REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/appbackup-retention REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/backupctl REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/usr/local/sbin/backup-restore-check REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/backup-retention REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/usr/local/sbin/homelab-backup-edge-vm-vzdump REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-capacity-risk REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/usr/local/sbin/homelab-chat-run REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-edge-vm-offhost-freshness REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=YES LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/usr/local/sbin/homelab-extended-appbackup-run REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-golden-state-extended-overlay REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/usr/local/sbin/homelab-golden-state-index REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/usr/local/sbin/homelab-health-metrics REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/usr/local/sbin/homelab-ingress-standby-refresh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-mail-cloud-critical-upload REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-mail-cloud-restore-drill REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-rclone-cat-chunked REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=NO LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-rclone-cat-flat-chunked REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-rclone-copy-chunked REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-rclone-copy-flat-chunked REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/usr/local/sbin/homelab-rclone-required-check.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/usr/local/sbin/homelab-router-hard-power-cycle REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/usr/local/sbin/homelab-router-running-config-mail-cloud REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-secret-exposure-guard REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/usr/local/sbin/homelab-stage-2cloud REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-stage-2cloud-flat REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-stage-2cloud-flat-canary REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-stage-verify-chunked REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-stage-verify-flat-chunked REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-vm160-remote-backup REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-vm-backup-health REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/usr/local/sbin/homelab-vm-backup-policy REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/usr/local/sbin/homelab-vm-full-backup REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=YES LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/usr/local/sbin/homelab-vm-full-restore-drill REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=YES LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/usr/local/sbin/homelab-vm-full-retention REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-vm-local-dumps-2cloud REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-vm-local-dumps-retention REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/usr/local/sbin/skladchik-reports-monitor-backup REMOTE_COUNT=2 REMOTES=pve01-mail-01-crypt:,pve01-mail-02-crypt: UPLOAD=NO VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=NO -PIPELINE=/usr/local/sbin/skladchik-reports-monitor-backup-restore-check REMOTE_COUNT=2 REMOTES=pve01-mail-01-crypt:,pve01-mail-02-crypt: UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=NO -PIPELINE=/etc/cron.d/vzdump REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/etc/systemd/system/multi-user.target.wants/pve-guests.service REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/etc/systemd/system/pve-manager.service REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/usr/local/sbin/codevipe-cloud-backup.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/usr/local/sbin/homelab-rclone-required-check.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill REMOTE_COUNT=1 REMOTES=pve01-mail-05-crypt: UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/etc/cron.d/vzdump REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/etc/systemd/system/multi-user.target.wants/pve-guests.service REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/etc/systemd/system/pve-manager.service REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/usr/local/sbin/homelab-rclone-required-check.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -=== LOCAL_ARCHIVES === ---- LOCAL_ARCHIVES --- -7022386770 /var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.vma.zst -7021952392 /var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst -7021297699 /var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst -7016275481 /var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst -329541843 /var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst -326601118 /var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst -323668231 /var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst -321121649 /var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.tar.zst -222777732 /var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst -222761273 /var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst -222745965 /var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst -222727760 /var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.tar.zst -3021 /var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.log -3020 /var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.log -3018 /var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.log -3017 /var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.log -1089 /var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.log -1086 /var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.log -989 /var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.log -989 /var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.log -986 /var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.log -986 /var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.log -731 /var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.log -729 /var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.log -========== NODE=pve02 ========== -PIPELINE=/etc/cron.d/vzdump REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/etc/systemd/system/multi-user.target.wants/pve-guests.service REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/etc/systemd/system/pve-manager.service REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/usr/local/sbin/codevipe-cloud-backup.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/forum-prod-fresh5-cloud-backup.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-cluster-admin-vm-mail-cloud-backup REMOTE_COUNT=0 REMOTES=NONE UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES -PIPELINE=/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/usr/local/sbin/homelab-rclone-required-check.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/usr/local/sbin/homelab-vaultwarden-isolated-restore-drill REMOTE_COUNT=1 REMOTES=pve01-mail-05-crypt: UPLOAD=YES VERIFY=YES LOCAL_DELETE=YES DELETE_RISK=YES ---- LOCAL_ARCHIVES --- -997503422 /var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.vma.zst -868048369 /var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.vma.zst -288349157 /var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.tar.zst -286538526 /var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.tar.zst -284443608 /var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst -283022034 /var/lib/vz/dump/vzdump-lxc-111-2026_07_18-03_30_03.tar.zst -220080970 /var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.tar.zst -220065825 /var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.tar.zst -220049041 /var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst -220033534 /var/lib/vz/dump/vzdump-lxc-113-2026_07_18-03_30_20.tar.zst -7547 /var/lib/vz/dump/vzdump-qemu-160-2026_07_18-01_02_44.log -7443 /var/lib/vz/dump/vzdump-qemu-160-2026_07_18-01_38_20.log -5796 /var/lib/vz/dump/vzdump-qemu-160-2026_07_17-21_30_20.log -2685 /var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.log -2590 /var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.log -1088 /var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.log -1085 /var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.log -988 /var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.log -988 /var/lib/vz/dump/vzdump-lxc-113-2026_07_18-03_30_20.log -985 /var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.log -985 /var/lib/vz/dump/vzdump-lxc-111-2026_07_18-03_30_03.log -731 /var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.log -728 /var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.log -========== NODE=pve03 ========== -PIPELINE=/etc/cron.d/vzdump REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/etc/systemd/system/multi-user.target.wants/pve-guests.service REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/etc/systemd/system/pve-manager.service REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/usr/local/sbin/homelab-rclone-crypt-healthcheck.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO -PIPELINE=/usr/local/sbin/homelab-rclone-required-check.sh REMOTE_COUNT=0 REMOTES=NONE UPLOAD=NO VERIFY=NO LOCAL_DELETE=NO DELETE_RISK=NO ---- LOCAL_ARCHIVES --- -11116472459 /var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.vma.zst -11043834894 /var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.vma.zst -11013723404 /var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.vma.zst -10994820121 /var/lib/vz/dump/vzdump-qemu-130-2026_07_18-03_30_02.vma.zst -1137072350 /var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.vma.zst -838951976 /var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.vma.zst -838817925 /var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.vma.zst -785249297 /var/lib/vz/dump/vzdump-qemu-190-2026_07_18-03_33_48.vma.zst -7106 /var/lib/vz/dump/vzdump-qemu-130-2026_07_02-03_30_02.log -7102 /var/lib/vz/dump/vzdump-qemu-130-2026_07_03-03_30_05.log -6564 /var/lib/vz/dump/vzdump-qemu-130-2026_07_18-03_30_02.log -6560 /var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.log -6559 /var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.log -6459 /var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.log -6246 /var/lib/vz/dump/vzdump-qemu-130-2026_07_04-03_30_02.log -4107 /var/lib/vz/dump/vzdump-qemu-130-2026_07_07-03_30_01.log -2642 /var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.log -2500 /var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.log -2404 /var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.log -2305 /var/lib/vz/dump/vzdump-qemu-190-2026_07_18-03_33_48.log -========== PVE_RETENTION ========== -=== PVE_RETENTION === -========== PVE_RETENTION ========== -{"id":"homelab-nightly-all","node":null,"vmid":"110,111,112,113,130,150,190","storage":"local","prune_backups":null,"enabled":1,"schedule":"03:30"} -DELETE_GUARD_ANALYSIS=COMPLETE -=== CURRENT_MATRIX === -STATUS=FAIL -TS=2026-07-21T07:02:41Z -TYPE=unified-backup-matrix -MATRIX_MODE=authoritative -JOBS_TOTAL=41 -JOBS_OK=30 -JOBS_BAD=6 -JOBS_UNMAPPED=0 -JOBS_DISABLED=5 -STATE_JSON=/var/lib/homelab-backup-matrix/state.json -MATRIX_JSON=/etc/homelab-backup-matrix/matrix.json -AUTHORITY=canonical -COMPAT_OUTPUTS=29 -COMPAT_MAP=/var/lib/homelab-backup-matrix/compat-map.json -JOB=dpkg-db-backup STATUS=OK EVIDENCE=execution_only AGE=36147 MAX_AGE=172800 -JOB=filebrowser-backup STATUS=OK EVIDENCE=artifact_or_health AGE=14443 MAX_AGE=259200 -JOB=filebrowser-restore-validate STATUS=STALE EVIDENCE=artifact_or_health AGE=1016424 MAX_AGE=864000 -JOB=homelab-appbackup-dockge-stacks STATUS=OK EVIDENCE=artifact_or_health AGE=22248 MAX_AGE=172800 -JOB=homelab-appbackup-mariadb-batch STATUS=OK EVIDENCE=execution_only AGE=15387 MAX_AGE=172800 -JOB=homelab-appbackup-npmplus-edge STATUS=OK EVIDENCE=artifact_or_health AGE=20573 MAX_AGE=172800 -JOB=homelab-appbackup-postgres-batch STATUS=OK EVIDENCE=execution_only AGE=16416 MAX_AGE=172800 -JOB=homelab-appbackup-sqlite-batch STATUS=OK EVIDENCE=execution_only AGE=18327 MAX_AGE=172800 -JOB=homelab-authentik-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987790 MAX_AGE=172800 -JOB=homelab-auto-backup STATUS=OK EVIDENCE=execution_only AGE=195424 MAX_AGE=864000 -JOB=homelab-backup-coverage-matrix STATUS=OK EVIDENCE=artifact_or_health AGE=4346 MAX_AGE=172800 -JOB=homelab-backup-freshness STATUS=OK EVIDENCE=execution_only AGE=35589 MAX_AGE=172800 -JOB=homelab-backup-healthcheck STATUS=OK EVIDENCE=execution_only AGE=10631 MAX_AGE=172800 -JOB=homelab-backup-sla-health STATUS=FAIL EVIDENCE=artifact_or_health AGE=144 MAX_AGE=172800 -JOB=homelab-edge-vm-offhost-freshness STATUS=OK EVIDENCE=artifact_or_health AGE=18426 MAX_AGE=172800 -JOB=homelab-emergency-backup STATUS=OK EVIDENCE=artifact_or_health AGE=24539 MAX_AGE=172800 -JOB=homelab-extended-appbackup STATUS=OK EVIDENCE=artifact_or_health AGE=14156 MAX_AGE=172800 -JOB=homelab-gitea-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987785 MAX_AGE=172800 -JOB=homelab-golden-state-index STATUS=OK EVIDENCE=artifact_or_health AGE=4112 MAX_AGE=172800 -JOB=homelab-mail-cloud-critical-upload STATUS=OK EVIDENCE=artifact_or_health AGE=11997 MAX_AGE=172800 -JOB=homelab-mail-cloud-edge-vm-upload STATUS=OK EVIDENCE=artifact_or_health AGE=174045 MAX_AGE=864000 -JOB=homelab-mail-cloud-nextcloud-vm-upload STATUS=OK EVIDENCE=artifact_or_health AGE=7575 MAX_AGE=864000 -JOB=homelab-mail-cloud-restore-drill STATUS=STALE EVIDENCE=artifact_or_health AGE=988269 MAX_AGE=864000 -JOB=homelab-npmplus-kuma-config-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=60071 MAX_AGE=172800 -JOB=homelab-router-running-config-mail-cloud STATUS=OK EVIDENCE=artifact_or_health AGE=17076 MAX_AGE=172800 -JOB=homelab-vaultwarden-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=987768 MAX_AGE=172800 -JOB=homelab-verified-backup-health STATUS=OK EVIDENCE=artifact_or_health AGE=11845 MAX_AGE=172800 -JOB=homelab-vm-backup-health STATUS=OK EVIDENCE=artifact_or_health AGE=8742 MAX_AGE=172800 -JOB=homelab-vm-backup-policy STATUS=OK EVIDENCE=artifact_or_health AGE=5548 MAX_AGE=172800 -JOB=homelab-vm-full-backup STATUS=OK EVIDENCE=artifact_or_health AGE=24778 MAX_AGE=172800 -JOB=homelab-vm-full-restore-drill STATUS=OK EVIDENCE=artifact_or_health AGE=523085 MAX_AGE=10368000 -JOB=homelab-vm-local-dumps-2cloud STATUS=OK EVIDENCE=artifact_or_health AGE=172141 MAX_AGE=864000 -JOB=homelab-vm-local-dumps-retention STATUS=OK EVIDENCE=artifact_or_health AGE=168839 MAX_AGE=864000 -JOB=immich-media-offhost-sync STATUS=OK EVIDENCE=artifact_or_health AGE=4286 MAX_AGE=259200 -JOB=memos-backup STATUS=OK EVIDENCE=artifact_or_health AGE=14547 MAX_AGE=259200 -JOB=netbird-vps-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=1747896 MAX_AGE=864000 -JOB=paperless-backup STATUS=DISABLED EVIDENCE=artifact_or_health AGE=16347 MAX_AGE=259200 -JOB=paperless-restore-dry-run STATUS=STALE EVIDENCE=artifact_or_health AGE=1001986 MAX_AGE=864000 -JOB=skladchik-reports-monitor-backup STATUS=OK EVIDENCE=artifact_or_health AGE=19639 MAX_AGE=172800 -JOB=skladchik-reports-monitor-backup-restore-check STATUS=OK EVIDENCE=artifact_or_health AGE=190273 MAX_AGE=864000 -JOB=vm160-remote-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=26607 MAX_AGE=172800 +========== NODE=pve01 PIPELINE=/usr/local/sbin/appbackup-dockge-stacks ========== +FILE=/usr/local/sbin/appbackup-dockge-stacks,MODE=700,OWNER=root:root,SIZE=4097,MTIME=2026-07-10 04:14:10.050376764 +0300 +--- RELEVANT_LINES --- +14:RA="${REMOTE_A%/}/${NODE}/apps/${APP}/${JOB}" +15:RB="${REMOTE_B%/}/${NODE}/apps/${APP}/${JOB}" +16:RAL="${REMOTE_A%/}/${NODE}/apps/${APP}/latest-backup-index.json" +17:RBL="${REMOTE_B%/}/${NODE}/apps/${APP}/latest-backup-index.json" +30:rm -rf "$PAYLOAD" +32:SHA="$(sha256sum "$STAGE/$ARCHIVE" | cut -d" " -f1)" +51:echo "[STEP] local checksum" +52:(cd "$STAGE" && sha256sum "$ARCHIVE" manifest.json latest-backup-index.json > checksums.sha256 && sha256sum -c checksums.sha256 && tar -tzf "$ARCHIVE" >/dev/null && jq -e . manifest.json >/dev/null) +54:echo "[STEP] upload cloud A metadata"; rclone copy "$STAGE" "$RA" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s +55:echo "[STEP] upload cloud A archive chunks"; rclone deletefile "$RA/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RA/$ARCHIVE" +56:echo "[STEP] upload cloud B metadata"; rclone copy "$STAGE" "$RB" --exclude "$ARCHIVE" --exclude "$ARCHIVE.sha256" --transfers=4 --checkers=8 --retries=5 --low-level-retries=20 --stats=30s +57:echo "[STEP] upload cloud B archive chunks"; rclone deletefile "$RB/$ARCHIVE" >/dev/null 2>&1 || true; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-rclone-copy-chunked "$STAGE/$ARCHIVE" "$RB/$ARCHIVE" +58:echo "[STEP] verify cloud A chunked"; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-stage-verify-chunked "$RA" "$ARCHIVE" "$VERIFY/a" "$JOB" +59:echo "[STEP] verify cloud B chunked"; CHUNK_TMP_BASE="${CHUNK_TMP_BASE:-/mnt/staging/rclone-chunks}" homelab-stage-verify-chunked "$RB" "$ARCHIVE" "$VERIFY/b" "$JOB" +60:echo "[STEP] upload latest index"; rclone copyto "$STAGE/latest-backup-index.json" "$RAL"; rclone copyto "$STAGE/latest-backup-index.json" "$RBL"; rclone cat "$RAL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null; rclone cat "$RBL" | jq -e --arg job "$JOB" ".job_id == \$job" >/dev/null +76:rm -rf "$STAGE" "$VERIFY" +--- UNIT_REFERENCES --- +UNSAFE_SOURCE_CAPTURE=COMPLETE OUTPUT_END CHAT_OUTPUT_END