diff --git a/runtime/history/GITEA-PUSH-AUTH-001.json b/runtime/history/GITEA-PUSH-AUTH-001.json new file mode 100644 index 0000000..2d79e6e --- /dev/null +++ b/runtime/history/GITEA-PUSH-AUTH-001.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "GITEA-PUSH-AUTH-001", + "status": "FAIL", + "rc": 1, + "host": "pve01", + "mode": "change", + "component": "gitea-runtime", + "started_at_utc": "2026-07-21T08:03:07Z", + "finished_at_utc": "2026-07-21T08:03:07Z", + "reference_register_checked": true, + "reference_sha256": "f6f7749fda946015116cdbbd754c7e7f20356b4a6cf4df330a1b0b8fdf59cbb2", + "error_register_checked": true, + "error_register_sha256": "ab8308549db90d235b591af0581580c3c7ba1b287b129cc6111c2eef0a0f9ecc", + "command_sha256": "fcfc28bcc5acb6382e895d5ed7cf51d7d5a3e2f5d8f8332770c58b42039f8a94", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": true, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "sanitized_output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "" +} diff --git a/runtime/history/GITEA-PUSH-AUTH-001.txt b/runtime/history/GITEA-PUSH-AUTH-001.txt new file mode 100644 index 0000000..dbb438c --- /dev/null +++ b/runtime/history/GITEA-PUSH-AUTH-001.txt @@ -0,0 +1,24 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=GITEA-PUSH-AUTH-001 +STATUS=FAIL +RC=1 +HOST=pve01 +MODE=change +COMPONENT=gitea-runtime +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=f6f7749fda946015116cdbbd754c7e7f20356b4a6cf4df330a1b0b8fdf59cbb2 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=ab8308549db90d235b591af0581580c3c7ba1b287b129cc6111c2eef0a0f9ecc +COMMAND_SHA256=fcfc28bcc5acb6382e895d5ed7cf51d7d5a3e2f5d8f8332770c58b42039f8a94 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=true +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +SANITIZED_OUTPUT_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +OUTPUT_BEGIN + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 964cfec..2d79e6e 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,30 +1,30 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "VM160-LOCAL-ROOTCAUSE-001", - "status": "OK", - "rc": 0, + "command_id": "GITEA-PUSH-AUTH-001", + "status": "FAIL", + "rc": 1, "host": "pve01", - "mode": "read-only", - "component": "vm160-backup", - "started_at_utc": "2026-07-21T07:55:39Z", - "finished_at_utc": "2026-07-21T07:55:39Z", + "mode": "change", + "component": "gitea-runtime", + "started_at_utc": "2026-07-21T08:03:07Z", + "finished_at_utc": "2026-07-21T08:03:07Z", "reference_register_checked": true, "reference_sha256": "f6f7749fda946015116cdbbd754c7e7f20356b4a6cf4df330a1b0b8fdf59cbb2", "error_register_checked": true, "error_register_sha256": "ab8308549db90d235b591af0581580c3c7ba1b287b129cc6111c2eef0a0f9ecc", - "command_sha256": "abe456bec06dfed260177703a3c57c723a6d506d59a2103104a1bbae3a1523c4", + "command_sha256": "fcfc28bcc5acb6382e895d5ed7cf51d7d5a3e2f5d8f8332770c58b42039f8a94", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, - "changes_made": false, + "changes_made": true, "sanitized": true, "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "a6f5c40d21e4083664d8d699d1e2a10ad4f47e53c2716db0fde0148120fdaa97", - "sanitized_output_sha256": "196da65bcc79aa66809cd6ec0bf42a1f59ea83cb1b63cad59e6d86e2ff08409f", + "raw_evidence_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "sanitized_output_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "===== VM160 LOCAL ROOT CAUSE =====\nTIME=2026-07-21T10:55:39+03:00\n===== HEALTH =====\nSTATUS=FAIL\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=20260720T233913Z\nVMID=160\nTARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260720T233913Z\nRC=1\n===== TEMP LOG =====\nTEMP_LOG_MISSING\n===== SCRIPT FAILURE PATHS =====\n2:set -Eeuo pipefail\n7:KH=[SENSITIVE_PATH]\n8:PARENT=/mnt/staging/vzdump\n14:target_worker(){ ssh \"${SSH[@]}\" \"$TARGET\" bash -s -- \"$@\" < \"$TARGET_WORKER\"; }\n17:flock -n 9\n20:test \"$(ssh \"${SSH[@]}\" \"$SOURCE\" \"qm status $VMID\" | awk '{print $2}')\" = running\n22:SOURCE_ACTIVE=\"$(ssh \"${SSH[@]}\" \"$SOURCE\" \"pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'\")\" || SOURCE_ACTIVE_RC=$?\n27:NAME=\"vzdump-qemu-${VMID}-${TS}.vma.zst\"\n30:LOG=\"$(mktemp /tmp/vm160-remote-backup.XXXXXX.log)\"\n33: trap - EXIT\n35: ssh \"${SSH[@]}\" \"$TARGET\" \"rm -rf -- '$DIR'\" >/dev/null 2>&1 || true\n37: printf 'STATUS=FAIL\\nSERVICE=homelab-vm160-remote-backup\\nTIMESTAMP_UTC=%s\\nVMID=%s\\nTARGET_DIR=%s\\nRC=%s\\n' \"$TS\" \"$VMID\" \"$DIR\" \"$rc\" > \"$tmp\"\n42: exit \"$rc\"\n44:trap cleanup EXIT\n45:REMOTE_FREE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"df -B1 --output=avail '$PARENT' | awk 'NR==2{print \\$1}'\")\"\n46:LATEST_SIZE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\\n' | sort -nr | head -n1\")\"\n54:ssh \"${SSH[@]}\" \"$TARGET\" \"install -d -m 0700 '$DIR'\"\n55:ssh \"${SSH[@]}\" \"$SOURCE\" \"vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7\" 2>\"$LOG\" | ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$PART'\"\n57:ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$DIR/vzdump.log'\" < \"$LOG\"\n58:SIZE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"stat -c %s '$FINAL'\")\"\n59:SHA=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"sha256sum '$FINAL' | awk '{print \\$1}'\")\"\n65:trap - EXIT\n===== CURRENT PROCESS =====\n3706440 00:00 S+ bash /usr/local/sbin/homelab-chat-run VM160-LOCAL-ROOTCAUSE-001 read-only vm160-backup -- /usr/bin/bash -lc echo \"===== VM160 LOCAL ROOT CAUSE =====\"; echo \"TIME=$(date -Is)\"; echo \"===== HEALTH =====\"; cat /var/lib/homelab-health/vm160-remote-backup.txt 2>/dev/null || echo \"HEALTH_FILE_MISSING\"; echo \"===== TEMP LOG =====\"; f=\"$(find /tmp -maxdepth 1 -type f -name \"vm160-remote-backup.*.log\" -printf \"%T@|%p\\n\" 2>/dev/null | sort -t\"|\" -k1,1nr | head -n1 | cut -d\"|\" -f2-)\"; if [ -n \"$f\" ] && [ -r \"$f\" ]; then stat -c \"FILE=%n SIZE=%s MTIME=%y\" \"$f\"; tail -n 200 \"$f\"; else echo \"TEMP_LOG_MISSING\"; fi; echo \"===== SCRIPT FAILURE PATHS =====\"; grep -nE \"set -|trap|flock|ssh|vzdump|exit|return|STATUS=FAIL|rm -rf|mktemp\" /usr/local/sbin/homelab-vm160-remote-backup 2>/dev/null; echo \"===== CURRENT PROCESS =====\"; ps -eo pid,etime,stat,args | grep -E \"[h]omelab-vm160-remote-backup|[v]zdump.*160\" || echo \"PROCESS_PRESENT=no\"; echo \"CHANGES_MADE=NO\"; true\n3706463 00:00 S+ timeout --foreground --preserve-status 90 /usr/bin/bash -lc echo \"===== VM160 LOCAL ROOT CAUSE =====\"; echo \"TIME=$(date -Is)\"; echo \"===== HEALTH =====\"; cat /var/lib/homelab-health/vm160-remote-backup.txt 2>/dev/null || echo \"HEALTH_FILE_MISSING\"; echo \"===== TEMP LOG =====\"; f=\"$(find /tmp -maxdepth 1 -type f -name \"vm160-remote-backup.*.log\" -printf \"%T@|%p\\n\" 2>/dev/null | sort -t\"|\" -k1,1nr | head -n1 | cut -d\"|\" -f2-)\"; if [ -n \"$f\" ] && [ -r \"$f\" ]; then stat -c \"FILE=%n SIZE=%s MTIME=%y\" \"$f\"; tail -n 200 \"$f\"; else echo \"TEMP_LOG_MISSING\"; fi; echo \"===== SCRIPT FAILURE PATHS =====\"; grep -nE \"set -|trap|flock|ssh|vzdump|exit|return|STATUS=FAIL|rm -rf|mktemp\" /usr/local/sbin/homelab-vm160-remote-backup 2>/dev/null; echo \"===== CURRENT PROCESS =====\"; ps -eo pid,etime,stat,args | grep -E \"[h]omelab-vm160-remote-backup|[v]zdump.*160\" || echo \"PROCESS_PRESENT=no\"; echo \"CHANGES_MADE=NO\"; true\n3706464 00:00 S+ /usr/bin/bash -lc echo \"===== VM160 LOCAL ROOT CAUSE =====\"; echo \"TIME=$(date -Is)\"; echo \"===== HEALTH =====\"; cat /var/lib/homelab-health/vm160-remote-backup.txt 2>/dev/null || echo \"HEALTH_FILE_MISSING\"; echo \"===== TEMP LOG =====\"; f=\"$(find /tmp -maxdepth 1 -type f -name \"vm160-remote-backup.*.log\" -printf \"%T@|%p\\n\" 2>/dev/null | sort -t\"|\" -k1,1nr | head -n1 | cut -d\"|\" -f2-)\"; if [ -n \"$f\" ] && [ -r \"$f\" ]; then stat -c \"FILE=%n SIZE=%s MTIME=%y\" \"$f\"; tail -n 200 \"$f\"; else echo \"TEMP_LOG_MISSING\"; fi; echo \"===== SCRIPT FAILURE PATHS =====\"; grep -nE \"set -|trap|flock|ssh|vzdump|exit|return|STATUS=FAIL|rm -rf|mktemp\" /usr/local/sbin/homelab-vm160-remote-backup 2>/dev/null; echo \"===== CURRENT PROCESS =====\"; ps -eo pid,etime,stat,args | grep -E \"[h]omelab-vm160-remote-backup|[v]zdump.*160\" || echo \"PROCESS_PRESENT=no\"; echo \"CHANGES_MADE=NO\"; true\nCHANGES_MADE=NO\n" + "output": "" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 2a30440..dbb438c 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,63 +1,24 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=VM160-LOCAL-ROOTCAUSE-001 -STATUS=OK -RC=0 +COMMAND_ID=GITEA-PUSH-AUTH-001 +STATUS=FAIL +RC=1 HOST=pve01 -MODE=read-only -COMPONENT=vm160-backup +MODE=change +COMPONENT=gitea-runtime REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=f6f7749fda946015116cdbbd754c7e7f20356b4a6cf4df330a1b0b8fdf59cbb2 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=ab8308549db90d235b591af0581580c3c7ba1b287b129cc6111c2eef0a0f9ecc -COMMAND_SHA256=abe456bec06dfed260177703a3c57c723a6d506d59a2103104a1bbae3a1523c4 +COMMAND_SHA256=fcfc28bcc5acb6382e895d5ed7cf51d7d5a3e2f5d8f8332770c58b42039f8a94 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true -CHANGES_MADE=false +CHANGES_MADE=true SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=a6f5c40d21e4083664d8d699d1e2a10ad4f47e53c2716db0fde0148120fdaa97 -SANITIZED_OUTPUT_SHA256=196da65bcc79aa66809cd6ec0bf42a1f59ea83cb1b63cad59e6d86e2ff08409f +RAW_EVIDENCE_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +SANITIZED_OUTPUT_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 OUTPUT_BEGIN -===== VM160 LOCAL ROOT CAUSE ===== -TIME=2026-07-21T10:55:39+03:00 -===== HEALTH ===== -STATUS=FAIL -SERVICE=homelab-vm160-remote-backup -TIMESTAMP_UTC=20260720T233913Z -VMID=160 -TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260720T233913Z -RC=1 -===== TEMP LOG ===== -TEMP_LOG_MISSING -===== SCRIPT FAILURE PATHS ===== -2:set -Eeuo pipefail -7:KH=[SENSITIVE_PATH] -8:PARENT=/mnt/staging/vzdump -14:target_worker(){ ssh "${SSH[@]}" "$TARGET" bash -s -- "$@" < "$TARGET_WORKER"; } -17:flock -n 9 -20:test "$(ssh "${SSH[@]}" "$SOURCE" "qm status $VMID" | awk '{print $2}')" = running -22:SOURCE_ACTIVE="$(ssh "${SSH[@]}" "$SOURCE" "pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'")" || SOURCE_ACTIVE_RC=$? -27:NAME="vzdump-qemu-${VMID}-${TS}.vma.zst" -30:LOG="$(mktemp /tmp/vm160-remote-backup.XXXXXX.log)" -33: trap - EXIT -35: ssh "${SSH[@]}" "$TARGET" "rm -rf -- '$DIR'" >/dev/null 2>&1 || true -37: printf 'STATUS=FAIL\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=%s\nVMID=%s\nTARGET_DIR=%s\nRC=%s\n' "$TS" "$VMID" "$DIR" "$rc" > "$tmp" -42: exit "$rc" -44:trap cleanup EXIT -45:REMOTE_FREE="$(ssh "${SSH[@]}" "$TARGET" "df -B1 --output=avail '$PARENT' | awk 'NR==2{print \$1}'")" -46:LATEST_SIZE="$(ssh "${SSH[@]}" "$TARGET" "find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\n' | sort -nr | head -n1")" -54:ssh "${SSH[@]}" "$TARGET" "install -d -m 0700 '$DIR'" -55:ssh "${SSH[@]}" "$SOURCE" "vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7" 2>"$LOG" | ssh "${SSH[@]}" "$TARGET" "cat > '$PART'" -57:ssh "${SSH[@]}" "$TARGET" "cat > '$DIR/vzdump.log'" < "$LOG" -58:SIZE="$(ssh "${SSH[@]}" "$TARGET" "stat -c %s '$FINAL'")" -59:SHA="$(ssh "${SSH[@]}" "$TARGET" "sha256sum '$FINAL' | awk '{print \$1}'")" -65:trap - EXIT -===== CURRENT PROCESS ===== -3706440 00:00 S+ bash /usr/local/sbin/homelab-chat-run VM160-LOCAL-ROOTCAUSE-001 read-only vm160-backup -- /usr/bin/bash -lc echo "===== VM160 LOCAL ROOT CAUSE ====="; echo "TIME=$(date -Is)"; echo "===== HEALTH ====="; cat /var/lib/homelab-health/vm160-remote-backup.txt 2>/dev/null || echo "HEALTH_FILE_MISSING"; echo "===== TEMP LOG ====="; f="$(find /tmp -maxdepth 1 -type f -name "vm160-remote-backup.*.log" -printf "%T@|%p\n" 2>/dev/null | sort -t"|" -k1,1nr | head -n1 | cut -d"|" -f2-)"; if [ -n "$f" ] && [ -r "$f" ]; then stat -c "FILE=%n SIZE=%s MTIME=%y" "$f"; tail -n 200 "$f"; else echo "TEMP_LOG_MISSING"; fi; echo "===== SCRIPT FAILURE PATHS ====="; grep -nE "set -|trap|flock|ssh|vzdump|exit|return|STATUS=FAIL|rm -rf|mktemp" /usr/local/sbin/homelab-vm160-remote-backup 2>/dev/null; echo "===== CURRENT PROCESS ====="; ps -eo pid,etime,stat,args | grep -E "[h]omelab-vm160-remote-backup|[v]zdump.*160" || echo "PROCESS_PRESENT=no"; echo "CHANGES_MADE=NO"; true -3706463 00:00 S+ timeout --foreground --preserve-status 90 /usr/bin/bash -lc echo "===== VM160 LOCAL ROOT CAUSE ====="; echo "TIME=$(date -Is)"; echo "===== HEALTH ====="; cat /var/lib/homelab-health/vm160-remote-backup.txt 2>/dev/null || echo "HEALTH_FILE_MISSING"; echo "===== TEMP LOG ====="; f="$(find /tmp -maxdepth 1 -type f -name "vm160-remote-backup.*.log" -printf "%T@|%p\n" 2>/dev/null | sort -t"|" -k1,1nr | head -n1 | cut -d"|" -f2-)"; if [ -n "$f" ] && [ -r "$f" ]; then stat -c "FILE=%n SIZE=%s MTIME=%y" "$f"; tail -n 200 "$f"; else echo "TEMP_LOG_MISSING"; fi; echo "===== SCRIPT FAILURE PATHS ====="; grep -nE "set -|trap|flock|ssh|vzdump|exit|return|STATUS=FAIL|rm -rf|mktemp" /usr/local/sbin/homelab-vm160-remote-backup 2>/dev/null; echo "===== CURRENT PROCESS ====="; ps -eo pid,etime,stat,args | grep -E "[h]omelab-vm160-remote-backup|[v]zdump.*160" || echo "PROCESS_PRESENT=no"; echo "CHANGES_MADE=NO"; true -3706464 00:00 S+ /usr/bin/bash -lc echo "===== VM160 LOCAL ROOT CAUSE ====="; echo "TIME=$(date -Is)"; echo "===== HEALTH ====="; cat /var/lib/homelab-health/vm160-remote-backup.txt 2>/dev/null || echo "HEALTH_FILE_MISSING"; echo "===== TEMP LOG ====="; f="$(find /tmp -maxdepth 1 -type f -name "vm160-remote-backup.*.log" -printf "%T@|%p\n" 2>/dev/null | sort -t"|" -k1,1nr | head -n1 | cut -d"|" -f2-)"; if [ -n "$f" ] && [ -r "$f" ]; then stat -c "FILE=%n SIZE=%s MTIME=%y" "$f"; tail -n 200 "$f"; else echo "TEMP_LOG_MISSING"; fi; echo "===== SCRIPT FAILURE PATHS ====="; grep -nE "set -|trap|flock|ssh|vzdump|exit|return|STATUS=FAIL|rm -rf|mktemp" /usr/local/sbin/homelab-vm160-remote-backup 2>/dev/null; echo "===== CURRENT PROCESS ====="; ps -eo pid,etime,stat,args | grep -E "[h]omelab-vm160-remote-backup|[v]zdump.*160" || echo "PROCESS_PRESENT=no"; echo "CHANGES_MADE=NO"; true -CHANGES_MADE=NO OUTPUT_END CHAT_OUTPUT_END