From 1defed251d7bf343b57a1703efb90a4457b4b295 Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Wed, 22 Jul 2026 08:27:15 +0000 Subject: [PATCH] runtime: publish CLUSTER-REFERENCE-PUBLICATION-ROOTCAUSE-157 --- ...R-REFERENCE-PUBLICATION-ROOTCAUSE-157.json | 30 + ...ER-REFERENCE-PUBLICATION-ROOTCAUSE-157.txt | 549 ++++++++++++++++++ runtime/latest.json | 24 +- runtime/latest.txt | 547 ++++++++++++++++- 4 files changed, 1125 insertions(+), 25 deletions(-) create mode 100644 runtime/history/CLUSTER-REFERENCE-PUBLICATION-ROOTCAUSE-157.json create mode 100644 runtime/history/CLUSTER-REFERENCE-PUBLICATION-ROOTCAUSE-157.txt diff --git a/runtime/history/CLUSTER-REFERENCE-PUBLICATION-ROOTCAUSE-157.json b/runtime/history/CLUSTER-REFERENCE-PUBLICATION-ROOTCAUSE-157.json new file mode 100644 index 0000000..1c91cab --- /dev/null +++ b/runtime/history/CLUSTER-REFERENCE-PUBLICATION-ROOTCAUSE-157.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "CLUSTER-REFERENCE-PUBLICATION-ROOTCAUSE-157", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "cluster-audit", + "started_at_utc": "2026-07-22T08:27:15Z", + "finished_at_utc": "2026-07-22T08:27:15Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", + "command_sha256": "4aab341221543b56b703380a36a56615e9c1b8083637dd486284e765228f35cc", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "d7f4d6f0c5fbb56a2635094046fe7bc28593aa8c1a32a616dba0306229034bfb", + "sanitized_output_sha256": "40566782503f51e3d02dabefddb0d16e849da8324d97462c4230bdd109a92e41", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "=== LOCAL_COMMAND_TRACES ===\n--- COMMAND_ID=CLUSTER-REFERENCE-FOUNDATION-151 ---\nFILE=/var/lib/homelab-chat-run/raw/CLUSTER-REFERENCE-FOUNDATION-151-20260722T081210Z.log SIZE=226 MTIME=2026-07-22 11:12:10.952034889 +0300\n1:REFERENCE_ROOT=/etc/homelab/reference\n2:REFERENCE_SECTIONS=24\n3:REFERENCE_JSON_FILES=76\n4:REFERENCE_BACKUP=/var/backups/homelab-reference/20260722T081210949547Z\n5:REFERENCE_SECRET_POLICY=ENFORCED\n6:CLUSTER_REFERENCE_FOUNDATION=COMPLETE\n--- COMMAND_ID=CLUSTER-REFERENCE-MODE-SEMANTICS-153 ---\nFILE=/var/lib/homelab-chat-run/raw/CLUSTER-REFERENCE-MODE-SEMANTICS-153-20260722T075606Z.log SIZE=8757 MTIME=2026-07-22 10:56:06.971072094 +0300\n1:PREVIOUS_RESULT=VERIFIED\n52:36- echo 'RESULT=FAIL'\n53:37- echo 'REASON=INVALID_COMMAND_ID'\n60:44- echo 'RESULT=FAIL'\n61:45- echo 'REASON=INVALID_COMPONENT'\n70:54- echo 'RESULT=FAIL'\n71:55: echo 'REASON=INVALID_MODE'\n86:70- echo 'RESULT=FAIL'\n87:71- echo \"REASON=REQUIRED_TOOL_MISSING:$TOOL\"\n96:80- echo 'RESULT=FAIL'\n97:81- echo 'REASON=REFERENCE_NOT_READABLE'\n102:86- echo 'RESULT=FAIL'\n103:87- echo 'REASON=ERROR_REGISTER_NOT_READABLE'\n114:132-PREVIOUS_FAILED_RC=\"$(\n128:146:BLOCK_REASON=''\n142:160: BLOCK_REASON='EXACT_FAILED_COMMAND_REPEAT'\n143:161- RC=73\n144:162- STATUS='BLOCKED'\n148:166- echo 'REASON=EXACT_FAILED_COMMAND_REPEAT'\n149:167- echo \"PREVIOUS_RC=$PREVIOUS_FAILED_RC\"\n160:178- echo 'RESULT=FAIL'\n161:179- echo 'REASON=INVALID_HOMELAB_TIMEOUT'\n171:189- RC=$?\n174:192- RC=$?\n180:198- STATUS='OK'\n182:200- STATUS='FAIL'\n192:325- \"$REFERENCE_SHA256\" \\\n292:425- echo \"STATUS=$STATUS\"\n293:426- echo \"RC=$RC\"\n297:430- echo 'REFERENCE_REGISTER_CHECK=OK'\n298:431- echo \"REFERENCE_SHA256=$REFERENCE_SHA256\"\n354:CLUSTER_REFERENCE_MODE_SEMANTICS=COMPLETE\n--- COMMAND_ID=CLUSTER-REFERENCE-FOUNDATION-154 ---\nFILE=/var/lib/homelab-chat-run/raw/CLUSTER-REFERENCE-FOUNDATION-154-20260722T080015Z.log SIZE=382 MTIME=2026-07-22 11:00:15.407910031 +0300\n4:REFERENCE_ROOT=/etc/homelab/reference\n5:REFERENCE_SECTIONS=24\n6:REFERENCE_BACKUP=/var/backups/homelab-reference/20260722T080015404968Z\n7:REFERENCE_SECRET_POLICY=ENFORCED\n8:CLUSTER_REFERENCE_FOUNDATION=COMPLETE\n--- COMMAND_ID=CLUSTER-REFERENCE-FOUNDATION-155 ---\nFILE=/var/lib/homelab-chat-run/raw/CLUSTER-REFERENCE-FOUNDATION-155-20260722T080346Z.log SIZE=454 MTIME=2026-07-22 11:03:47.117217539 +0300\n3:PREVIOUS_STATUS=OK\n4:PREVIOUS_RC=0\n5:PREVIOUS_REASON=None\n7:REFERENCE_ROOT=/etc/homelab/reference\n8:REFERENCE_SECTIONS=24\n9:REFERENCE_JSON_FILES=76\n10:REFERENCE_VALIDATION=PASS\n11:REFERENCE_SECRET_POLICY=ENFORCED\n12:REFERENCE_BACKUP=/var/backups/homelab-reference/20260722T080346Z\n13:CLUSTER_REFERENCE_FOUNDATION=COMPLETE\n--- COMMAND_ID=CLUSTER-REFERENCE-CORE-INVENTORY-156 ---\nFILE=/var/lib/homelab-chat-run/raw/CLUSTER-REFERENCE-CORE-INVENTORY-156-20260722T082023Z.log SIZE=3515 MTIME=2026-07-22 11:20:25.544704562 +0300\n4:SyntaxError: invalid syntax\n=== RUNNER_HELP ===\nUsage:\n homelab-chat-run COMMAND_ID MODE COMPONENT -- command [arguments...]\n\nMODE:\n read-only\n verify\n change\n transport-test\n\nEnvironment:\n HOMELAB_ALLOW_REPEAT=1 Allow exact repeat of a previously failed command.\n HOMELAB_TIMEOUT=SECONDS Optional command timeout.\n=== RUNNER_VALIDATION_RULES ===\n1-#!/usr/bin/env bash\n2-set -Eeuo pipefail\n3-umask 077\n4-\n5-usage() {\n6- cat >&2 <<'USAGE'\n7-Usage:\n8: homelab-chat-run COMMAND_ID MODE COMPONENT -- command [arguments...]\n9-\n10-MODE:\n11: read-only\n12: verify\n13: change\n14: transport-test\n15-\n16-Environment:\n17- HOMELAB_ALLOW_REPEAT=1 Allow exact repeat of a previously failed command.\n18- HOMELAB_TIMEOUT=SECONDS Optional command timeout.\n19-USAGE\n20- exit 64\n21-}\n22-\n23-[ \"$#\" -ge 5 ] || usage\n24-\n25-COMMAND_ID=\"$1\"\n26-MODE=\"$2\"\n27:COMPONENT=\"$3\"\n28-shift 3\n29-\n30-[ \"${1:-}\" = '--' ] || usage\n31-shift\n32-[ \"$#\" -ge 1 ] || usage\n33-\n34-case \"$COMMAND_ID\" in\n35- ''|*[!A-Za-z0-9._-]*)\n36- echo 'RESULT=FAIL'\n37- echo 'REASON=INVALID_COMMAND_ID'\n38- exit 65\n39- ;;\n40-esac\n41-\n42:case \"$COMPONENT\" in\n43- ''|*[!A-Za-z0-9._-]*)\n44- echo 'RESULT=FAIL'\n45: echo 'REASON=INVALID_COMPONENT'\n46- exit 66\n47- ;;\n48-esac\n49-\n50-case \"$MODE\" in\n51: read-only|verify|change|transport-test)\n52- ;;\n53- *)\n54- echo 'RESULT=FAIL'\n55: echo 'REASON=INVALID_MODE'\n56- exit 67\n57- ;;\n58-esac\n59-\n60-for TOOL in \\\n61- awk \\\n62- curl \\\n63- flock \\\n64- python3 \\\n65- scp \\\n66- sha256sum \\\n67- ssh\n68-do\n69- command -v \"$TOOL\" >/dev/null 2>&1 || {\n70- echo 'RESULT=FAIL'\n71- echo \"REASON=REQUIRED_TOOL_MISSING:$TOOL\"\n72- exit 68\n73- }\n74-done\n75-\n76-REFERENCE=/etc/pve/31_HOMELAB_REFERENCE.md\n77-ERROR_REGISTER=/etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md\n78-\n79-[ -r \"$REFERENCE\" ] || {\n80- echo 'RESULT=FAIL'\n81- echo 'REASON=REFERENCE_NOT_READABLE'\n82- exit 69\n83-}\n84-\n85-[ -r \"$ERROR_REGISTER\" ] || {\n86- echo 'RESULT=FAIL'\n87- echo 'REASON=ERROR_REGISTER_NOT_READABLE'\n88- exit 70\n89-}\n90-\n--\n113-\n114-RAW_FILE=\"$RAW_DIR/${COMMAND_ID}-${STAMP}.log\"\n115-SANITIZED_FILE=\"$PUBLIC_DIR/${COMMAND_ID}-${STAMP}.txt\"\n116-JSON_FILE=\"$PUBLIC_DIR/${COMMAND_ID}-${STAMP}.json\"\n117-TXT_FILE=\"$PUBLIC_DIR/${COMMAND_ID}-${STAMP}-latest.txt\"\n118-\n119-REFERENCE_SHA256=\"$(sha256sum \"$REFERENCE\" | awk '{print $1}')\"\n120-ERROR_REGISTER_SHA256=\"$(sha256sum \"$ERROR_REGISTER\" | awk '{print $1}')\"\n121-\n122-COMMAND_SHA256=\"$(\n123- {\n124- printf '%s\\0' \"$MODE\"\n125: printf '%s\\0' \"$COMPONENT\"\n126- printf '%s\\0' \"$@\"\n127- } |\n128- sha256sum |\n129- awk '{print $1}'\n130-)\"\n131-\n132-PREVIOUS_FAILED_RC=\"$(\n133- awk -F '\\t' -v hash=\"$COMMAND_SHA256\" '\n134- $3 == hash && $4 != \"0\" {\n135- value=$4\n136- }\n137-\n138- END {\n139- print value\n140- }\n141- ' \"$LEDGER\"\n142-)\"\n143-\n144-RUN_STARTED=true\n145-REPEAT_BLOCKED=false\n146-BLOCK_REASON=''\n147:CHANGES_MADE=false\n148-\n149-case \"$MODE\" in\n150: change)\n151: CHANGES_MADE=true\n152- ;;\n153-esac\n154-\n155-if [ -n \"$PREVIOUS_FAILED_RC\" ] &&\n156- [ \"${HOMELAB_ALLOW_REPEAT:-0}\" != 1 ]; then\n157-\n158- RUN_STARTED=false\n159- REPEAT_BLOCKED=true\n160- BLOCK_REASON='EXACT_FAILED_COMMAND_REPEAT'\n161- RC=73\n162- STATUS='BLOCKED'\n163-\n164- {\n165- echo 'COMMAND_EXECUTION_BLOCKED=yes'\n166- echo 'REASON=EXACT_FAILED_COMMAND_REPEAT'\n167- echo \"PREVIOUS_RC=$PREVIOUS_FAILED_RC\"\n168- echo \"COMMAND_SHA256=$COMMAND_SHA256\"\n169- echo\n170- echo 'Set HOMELAB_ALLOW_REPEAT=1 only after reviewing the previous failure.'\n171- } >\"$RAW_FILE\"\n172-else\n173- set +e\n174-\n175- if [ -n \"${HOMELAB_TIMEOUT:-}\" ]; then\n176- case \"$HOMELAB_TIMEOUT\" in\n177- *[!0-9]*|'')\n178- echo 'RESULT=FAIL'\n179- echo 'REASON=INVALID_HOMELAB_TIMEOUT'\n180- exit 74\n181- ;;\n182- esac\n183-\n184- timeout \\\n185- --foreground \\\n186- --preserve-status \\\n--\n310- sha256sum \"$SANITIZED_FILE\" |\n311- awk '{print $1}'\n312-)\"\n313-\n314-python3 - \\\n315- \"$JSON_FILE\" \\\n316- \"$SANITIZED_FILE\" \\\n317- \"$COMMAND_ID\" \\\n318- \"$STATUS\" \\\n319- \"$RC\" \\\n320- \"$HOST_SHORT\" \\\n321- \"$MODE\" \\\n322: \"$COMPONENT\" \\\n323- \"$STARTED_AT\" \\\n324- \"$FINISHED_AT\" \\\n325- \"$REFERENCE_SHA256\" \\\n326- \"$ERROR_REGISTER_SHA256\" \\\n327- \"$COMMAND_SHA256\" \\\n328- \"$RAW_SHA256\" \\\n329- \"$SANITIZED_SHA256\" \\\n330- \"$RUN_STARTED\" \\\n331- \"$REPEAT_BLOCKED\" \\\n332- \"$BLOCK_REASON\" \\\n333: \"$CHANGES_MADE\" <<'PY'\n334-import json\n335-import sys\n336-from pathlib import Path\n337-\n338-(\n339- json_path,\n340- output_path,\n341- command_id,\n342- status,\n343- rc,\n344- host,\n345- mode,\n346: component,\n347- started,\n348- finished,\n349- reference_sha,\n350- error_sha,\n351- command_sha,\n352- raw_sha,\n353- sanitized_sha,\n354- run_started,\n355- repeat_blocked,\n356- block_reason,\n357: changes_made,\n358-) = sys.argv[1:]\n359-\n360-output = Path(output_path).read_text(\n361- encoding=\"utf-8\",\n362- errors=\"replace\",\n363-)\n364-\n365-encoded = output.encode(\"utf-8\")\n366-json_limit = 300_000\n367-output_truncated = len(encoded) > json_limit\n368-\n369-if output_truncated:\n370- output_preview = encoded[:json_limit].decode(\n371- \"utf-8\",\n372- errors=\"ignore\",\n373- )\n374-else:\n375- output_preview = output\n376-\n377-document = {\n378- \"schema_version\": 1,\n379- \"channel\": \"homelab-runtime\",\n380- \"command_id\": command_id,\n381- \"status\": status,\n382- \"rc\": int(rc),\n383- \"host\": host,\n384- \"mode\": mode,\n385: \"component\": component,\n386- \"started_at_utc\": started,\n387- \"finished_at_utc\": finished,\n388- \"reference_register_checked\": True,\n389- \"reference_sha256\": reference_sha,\n390- \"error_register_checked\": True,\n391- \"error_register_sha256\": error_sha,\n392- \"command_sha256\": command_sha,\n393- \"duplicate_failed_command_blocked\": repeat_blocked == \"true\",\n394- \"block_reason\": block_reason or None,\n395- \"execution_started\": run_started == \"true\",\n396: \"changes_made\": changes_made == \"true\",\n397- \"sanitized\": True,\n398- \"secrets_included\": False,\n399- \"private_addresses_included\": False,\n400- \"raw_evidence_retained_locally\": True,\n401- \"raw_evidence_sha256\": raw_sha,\n402- \"sanitized_output_sha256\": sanitized_sha,\n403- \"output_truncated_in_json\": output_truncated,\n404- \"full_sanitized_output_url\": (\n405- \"https://git.gram1.ru/.well-known/\"\n406- \"homelab-runtime/latest.txt\"\n407- ),\n408- \"output\": output_preview,\n409-}\n410-\n411-Path(json_path).write_text(\n412- json.dumps(\n413- document,\n414- ensure_ascii=False,\n415- indent=2,\n416- )\n417- + \"\\n\",\n418- encoding=\"utf-8\",\n419-)\n420-PY\n421-\n422-{\n423- echo 'CHAT_OUTPUT_BEGIN'\n424- echo \"COMMAND_ID=$COMMAND_ID\"\n425- echo \"STATUS=$STATUS\"\n426- echo \"RC=$RC\"\n427- echo \"HOST=$HOST_SHORT\"\n428- echo \"MODE=$MODE\"\n429: echo \"COMPONENT=$COMPONENT\"\n430- echo 'REFERENCE_REGISTER_CHECK=OK'\n431- echo \"REFERENCE_SHA256=$REFERENCE_SHA256\"\n432- echo 'ERROR_REGISTER_CHECK=OK'\n433- echo \"ERROR_REGISTER_SHA256=$ERROR_REGISTER_SHA256\"\n434- echo \"COMMAND_SHA256=$COMMAND_SHA256\"\n435- echo \"DUPLICATE_FAILED_COMMAND_BLOCKED=$REPEAT_BLOCKED\"\n436- echo \"EXECUTION_STARTED=$RUN_STARTED\"\n437: echo \"CHANGES_MADE=$CHANGES_MADE\"\n438- echo 'SANITIZED=yes'\n439- echo 'SECRETS_INCLUDED=no'\n440- echo 'PRIVATE_ADDRESSES_INCLUDED=no'\n441- echo \"RAW_EVIDENCE_SHA256=$RAW_SHA256\"\n442- echo \"SANITIZED_OUTPUT_SHA256=$SANITIZED_SHA256\"\n443- echo 'OUTPUT_BEGIN'\n444- cat \"$SANITIZED_FILE\"\n445- echo\n446- echo 'OUTPUT_END'\n447- echo 'CHAT_OUTPUT_END'\n448-} >\"$TXT_FILE\"\n449-\n450-chmod 0600 \\\n451- \"$RAW_FILE\" \\\n452- \"$SANITIZED_FILE\" \\\n453- \"$JSON_FILE\" \\\n454- \"$TXT_FILE\"\n455-\n456-TARGET=debian@[PRIVATE_IP]\n457-REMOTE_PREFIX=\"/tmp/homelab-runtime-${COMMAND_ID}-$$\"\n458-\n459-SSH_OPTS=(\n460- -o BatchMode=yes\n461- -o ConnectTimeout=10\n462- -o ConnectionAttempts=1\n463- -o StrictHostKeyChecking=yes\n464- -o UserKnownHostsFile=[SENSITIVE_PATH]\n465-)\n466-\n467-scp \\\n468- -q \\\n469- \"${SSH_OPTS[@]}\" \\\n470- \"$JSON_FILE\" \\\n471- \"${TARGET}:${REMOTE_PREFIX}.json\"\n472-\n--\n518- echo 'REASON=PUBLIC_RUNTIME_RESULT_NOT_VISIBLE'\n519- echo \"COMMAND_ID=$COMMAND_ID\"\n520- echo \"LOCAL_RAW_FILE=$RAW_FILE\"\n521- exit 75\n522-fi\n523-\n524-printf '%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n' \\\n525- \"$FINISHED_AT\" \\\n526- \"$COMMAND_ID\" \\\n527- \"$COMMAND_SHA256\" \\\n528- \"$RC\" \\\n529- \"$STATUS\" \\\n530: \"$COMPONENT\" >>\"$LEDGER\"\n531-\n532-echo\n533-echo '===== HOMELAB CHAT RUN COMPLETE ====='\n534-echo \"COMMAND_ID=$COMMAND_ID\"\n535-echo \"STATUS=$STATUS\"\n536-echo \"RC=$RC\"\n537-echo 'REFERENCE_REGISTER_CHECK=OK'\n538-echo 'ERROR_REGISTER_CHECK=OK'\n539-echo \"DUPLICATE_FAILED_COMMAND_BLOCKED=$REPEAT_BLOCKED\"\n540-echo 'GITEA_PUBLICATION=OK'\n541-echo \"PUBLIC_URL=$PUBLIC_URL\"\n542-echo \"LOCAL_RAW_FILE=$RAW_FILE\"\n543-echo 'SHELL_SESSION_PRESERVED=yes'\n544-\n545-exit \"$RC\"\n=== PARTIAL_REFERENCE_STATE ===\nd 750 4096 2026-07-22T10:34:58.6964026990 /etc/homelab/reference/public\nd 750 4096 2026-07-22T10:34:58.6964026990 /etc/homelab/reference/runbooks\nd 750 4096 2026-07-22T10:34:58.6964026990 /etc/homelab/reference/snapshots\nd 750 4096 2026-07-22T10:34:58.6970459770 /etc/homelab/reference/declared\nd 750 4096 2026-07-22T10:34:58.6974681490 /etc/homelab/reference/observed\nd 750 4096 2026-07-22T10:34:58.6978512600 /etc/homelab/reference/effective\nd 750 4096 2026-07-22T10:34:58.6978691140 /etc/homelab/reference\nd 750 4096 2026-07-22T10:34:58.6978834310 /etc/homelab/reference/state\nd 750 4096 2026-07-22T11:03:47.1132880790 /etc/homelab/reference/schema\nf 640 153 2026-07-22T10:34:58.6968014830 /etc/homelab/reference/declared/dns.json\nf 640 153 2026-07-22T10:34:58.6976137370 /etc/homelab/reference/effective/dns.json\nf 640 155 2026-07-22T10:34:58.6966969670 /etc/homelab/reference/declared/nodes.json\nf 640 155 2026-07-22T10:34:58.6975198720 /etc/homelab/reference/effective/nodes.json\nf 640 156 2026-07-22T10:34:58.6967158380 /etc/homelab/reference/declared/guests.json\nf 640 156 2026-07-22T10:34:58.6969484530 /etc/homelab/reference/declared/access.json\nf 640 156 2026-07-22T10:34:58.6969644820 /etc/homelab/reference/declared/owners.json\nf 640 156 2026-07-22T10:34:58.6971898020 /etc/homelab/reference/observed/dns.json\nf 640 156 2026-07-22T10:34:58.6975362380 /etc/homelab/reference/effective/guests.json\nf 640 156 2026-07-22T10:34:58.6977531080 /etc/homelab/reference/effective/access.json\nf 640 156 2026-07-22T10:34:58.6977693510 /etc/homelab/reference/effective/owners.json\nf 640 157 2026-07-22T10:34:58.6966738300 /etc/homelab/reference/declared/cluster.json\nf 640 157 2026-07-22T10:34:58.6968176220 /etc/homelab/reference/declared/domains.json\nf 640 157 2026-07-22T10:34:58.6968507170 /etc/homelab/reference/declared/ingress.json\nf 640 157 2026-07-22T10:34:58.6968665330 /etc/homelab/reference/declared/storage.json\nf 640 157 2026-07-22T10:34:58.6968815380 /etc/homelab/reference/declared/backups.json\nf 640 157 2026-07-22T10:34:58.6969804230 /etc/homelab/reference/declared/vendors.json\nf 640 157 2026-07-22T10:34:58.6975026350 /etc/homelab/reference/effective/cluster.json\nf 640 157 2026-07-22T10:34:58.6976296950 /etc/homelab/reference/effective/domains.json\nf 640 157 2026-07-22T10:34:58.6976621460 /etc/homelab/reference/effective/ingress.json\nf 640 157 2026-07-22T10:34:58.6976775090 /etc/homelab/reference/effective/storage.json\nf 640 157 2026-07-22T10:34:58.6976922400 /etc/homelab/reference/effective/backups.json\nf 640 157 2026-07-22T10:34:58.6977857550 /etc/homelab/reference/effective/vendors.json\nf 640 158 2026-07-22T10:34:58.6967496500 /etc/homelab/reference/declared/services.json\nf 640 158 2026-07-22T10:34:58.6967835450 /etc/homelab/reference/declared/networks.json\nf 640 158 2026-07-22T10:34:58.6970122440 /etc/homelab/reference/declared/security.json\nf 640 158 2026-07-22T10:34:58.6970626700 /etc/homelab/reference/declared/runbooks.json\nf 640 158 2026-07-22T10:34:58.6970953450 /etc/homelab/reference/observed/nodes.json\nf 640 158 2026-07-22T10:34:58.6975661010 /etc/homelab/reference/effective/services.json\nf 640 158 2026-07-22T10:34:58.6975973330 /etc/homelab/reference/effective/networks.json\nf 640 158 2026-07-22T10:34:58.6978179760 /etc/homelab/reference/effective/security.json\nf 640 158 2026-07-22T10:34:58.6978691140 /etc/homelab/reference/effective/runbooks.json\nf 640 159 2026-07-22T10:34:58.6969966040 /etc/homelab/reference/declared/contracts.json\nf 640 159 2026-07-22T10:34:58.6971108280 /etc/homelab/reference/observed/guests.json\nf 640 159 2026-07-22T10:34:58.6973313430 /etc/homelab/reference/observed/access.json\nf 640 159 2026-07-22T10:34:58.6973523380 /etc/homelab/reference/observed/owners.json\nf 640 159 2026-07-22T10:34:58.6978023050 /etc/homelab/reference/effective/contracts.json\nf 640 160 2026-07-22T10:34:58.6967330550 /etc/homelab/reference/declared/containers.json\nf 640 160 2026-07-22T10:34:58.6968977560 /etc/homelab/reference/declared/monitoring.json\nf 640 160 2026-07-22T10:34:58.6970793370 /etc/homelab/reference/observed/cluster.json\nf 640 160 2026-07-22T10:34:58.6972051460 /etc/homelab/reference/observed/domains.json\nf 640 160 2026-07-22T10:34:58.6972368890 /etc/homelab/reference/observed/ingress.json\nf 640 160 2026-07-22T10:34:58.6972525750 /etc/homelab/reference/observed/storage.json\nf 640 160 2026-07-22T10:34:58.6972679110 /etc/homelab/reference/observed/backups.json\nf 640 160 2026-07-22T10:34:58.6973681500 /etc/homelab/reference/observed/vendors.json\nf 640 160 2026-07-22T10:34:58.6975511580 /etc/homelab/reference/effective/containers.json\nf 640 160 2026-07-22T10:34:58.6977071740 /etc/homelab/reference/effective/monitoring.json\nf 640 161 2026-07-22T10:34:58.6971427070 /etc/homelab/reference/observed/services.json\nf 640 161 2026-07-22T10:34:58.6971740390 /etc/homelab/reference/observed/networks.json\nf 640 161 2026-07-22T10:34:58.6974304020 /etc/homelab/reference/observed/security.json\nf 640 161 2026-07-22T10:34:58.6974849460 /etc/homelab/reference/observed/runbooks.json\nf 640 162 2026-07-22T10:34:58.6967668110 /etc/homelab/reference/declared/applications.json\nf 640 162 2026-07-22T10:34:58.6968346450 /etc/homelab/reference/declared/certificates.json\nf 640 162 2026-07-22T10:34:58.6969325220 /etc/homelab/reference/declared/dependencies.json\nf 640 162 2026-07-22T10:34:58.6974142710 /etc/homelab/reference/observed/contracts.json\nf 640 162 2026-07-22T10:34:58.6975824200 /etc/homelab/reference/effective/applications.json\nf 640 162 2026-07-22T10:34:58.6976463310 /etc/homelab/reference/effective/certificates.json\nf 640 162 2026-07-22T10:34:58.6977380340 /etc/homelab/reference/effective/dependencies.json\nf 640 163 2026-07-22T10:34:58.6971273900 /etc/homelab/reference/observed/containers.json\nf 640 163 2026-07-22T10:34:58.6972830790 /etc/homelab/reference/observed/monitoring.json\nf 640 165 2026-07-22T10:34:58.6971583070 /etc/homelab/reference/observed/applications.json\nf 640 165 2026-07-22T10:34:58.6972204840 /etc/homelab/reference/observed/certificates.json\nf 640 165 2026-07-22T10:34:58.6973154820 /etc/homelab/reference/observed/dependencies.json\nf 640 167 2026-07-22T10:34:58.6969164820 /etc/homelab/reference/declared/external_services.json\nf 640 167 2026-07-22T10:34:58.6970278950 /etc/homelab/reference/declared/disaster_recovery.json\nf 640 167 2026-07-22T10:34:58.6970459770 /etc/homelab/reference/declared/change_management.json\nf 640 167 2026-07-22T10:34:58.6977227780 /etc/homelab/reference/effective/external_services.json\nf 640 167 2026-07-22T10:34:58.6978337330 /etc/homelab/reference/effective/disaster_recovery.json\nf 640 167 2026-07-22T10:34:58.6978512600 /etc/homelab/reference/effective/change_management.json\nf 640 170 2026-07-22T10:34:58.6972985790 /etc/homelab/reference/observed/external_services.json\nf 640 170 2026-07-22T10:34:58.6974490020 /etc/homelab/reference/observed/disaster_recovery.json\nf 640 170 2026-07-22T10:34:58.6974681490 /etc/homelab/reference/observed/change_management.json\nf 640 1780 2026-07-22T11:12:10.9514643580 /etc/homelab/reference/manifest.json\nf 640 199 2026-07-22T11:12:10.9518855430 /etc/homelab/reference/state/current.json\nf 640 262 2026-07-22T11:12:10.9515215670 /etc/homelab/reference/schema/record.schema.json\nf 640 440 2026-07-22T11:03:47.1133590400 /etc/homelab/reference/schema/edit-policy.json\nf 640 593 2026-07-22T11:12:10.9518574460 /etc/homelab/reference/README.md\nCLUSTER_REFERENCE_PUBLICATION_ROOTCAUSE=COMPLETE\n" +} diff --git a/runtime/history/CLUSTER-REFERENCE-PUBLICATION-ROOTCAUSE-157.txt b/runtime/history/CLUSTER-REFERENCE-PUBLICATION-ROOTCAUSE-157.txt new file mode 100644 index 0000000..6cb20c1 --- /dev/null +++ b/runtime/history/CLUSTER-REFERENCE-PUBLICATION-ROOTCAUSE-157.txt @@ -0,0 +1,549 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=CLUSTER-REFERENCE-PUBLICATION-ROOTCAUSE-157 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=cluster-audit +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 +COMMAND_SHA256=4aab341221543b56b703380a36a56615e9c1b8083637dd486284e765228f35cc +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=false +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=d7f4d6f0c5fbb56a2635094046fe7bc28593aa8c1a32a616dba0306229034bfb +SANITIZED_OUTPUT_SHA256=40566782503f51e3d02dabefddb0d16e849da8324d97462c4230bdd109a92e41 +OUTPUT_BEGIN +=== LOCAL_COMMAND_TRACES === +--- COMMAND_ID=CLUSTER-REFERENCE-FOUNDATION-151 --- +FILE=/var/lib/homelab-chat-run/raw/CLUSTER-REFERENCE-FOUNDATION-151-20260722T081210Z.log SIZE=226 MTIME=2026-07-22 11:12:10.952034889 +0300 +1:REFERENCE_ROOT=/etc/homelab/reference +2:REFERENCE_SECTIONS=24 +3:REFERENCE_JSON_FILES=76 +4:REFERENCE_BACKUP=/var/backups/homelab-reference/20260722T081210949547Z +5:REFERENCE_SECRET_POLICY=ENFORCED +6:CLUSTER_REFERENCE_FOUNDATION=COMPLETE +--- COMMAND_ID=CLUSTER-REFERENCE-MODE-SEMANTICS-153 --- +FILE=/var/lib/homelab-chat-run/raw/CLUSTER-REFERENCE-MODE-SEMANTICS-153-20260722T075606Z.log SIZE=8757 MTIME=2026-07-22 10:56:06.971072094 +0300 +1:PREVIOUS_RESULT=VERIFIED +52:36- echo 'RESULT=FAIL' +53:37- echo 'REASON=INVALID_COMMAND_ID' +60:44- echo 'RESULT=FAIL' +61:45- echo 'REASON=INVALID_COMPONENT' +70:54- echo 'RESULT=FAIL' +71:55: echo 'REASON=INVALID_MODE' +86:70- echo 'RESULT=FAIL' +87:71- echo "REASON=REQUIRED_TOOL_MISSING:$TOOL" +96:80- echo 'RESULT=FAIL' +97:81- echo 'REASON=REFERENCE_NOT_READABLE' +102:86- echo 'RESULT=FAIL' +103:87- echo 'REASON=ERROR_REGISTER_NOT_READABLE' +114:132-PREVIOUS_FAILED_RC="$( +128:146:BLOCK_REASON='' +142:160: BLOCK_REASON='EXACT_FAILED_COMMAND_REPEAT' +143:161- RC=73 +144:162- STATUS='BLOCKED' +148:166- echo 'REASON=EXACT_FAILED_COMMAND_REPEAT' +149:167- echo "PREVIOUS_RC=$PREVIOUS_FAILED_RC" +160:178- echo 'RESULT=FAIL' +161:179- echo 'REASON=INVALID_HOMELAB_TIMEOUT' +171:189- RC=$? +174:192- RC=$? +180:198- STATUS='OK' +182:200- STATUS='FAIL' +192:325- "$REFERENCE_SHA256" \ +292:425- echo "STATUS=$STATUS" +293:426- echo "RC=$RC" +297:430- echo 'REFERENCE_REGISTER_CHECK=OK' +298:431- echo "REFERENCE_SHA256=$REFERENCE_SHA256" +354:CLUSTER_REFERENCE_MODE_SEMANTICS=COMPLETE +--- COMMAND_ID=CLUSTER-REFERENCE-FOUNDATION-154 --- +FILE=/var/lib/homelab-chat-run/raw/CLUSTER-REFERENCE-FOUNDATION-154-20260722T080015Z.log SIZE=382 MTIME=2026-07-22 11:00:15.407910031 +0300 +4:REFERENCE_ROOT=/etc/homelab/reference +5:REFERENCE_SECTIONS=24 +6:REFERENCE_BACKUP=/var/backups/homelab-reference/20260722T080015404968Z +7:REFERENCE_SECRET_POLICY=ENFORCED +8:CLUSTER_REFERENCE_FOUNDATION=COMPLETE +--- COMMAND_ID=CLUSTER-REFERENCE-FOUNDATION-155 --- +FILE=/var/lib/homelab-chat-run/raw/CLUSTER-REFERENCE-FOUNDATION-155-20260722T080346Z.log SIZE=454 MTIME=2026-07-22 11:03:47.117217539 +0300 +3:PREVIOUS_STATUS=OK +4:PREVIOUS_RC=0 +5:PREVIOUS_REASON=None +7:REFERENCE_ROOT=/etc/homelab/reference +8:REFERENCE_SECTIONS=24 +9:REFERENCE_JSON_FILES=76 +10:REFERENCE_VALIDATION=PASS +11:REFERENCE_SECRET_POLICY=ENFORCED +12:REFERENCE_BACKUP=/var/backups/homelab-reference/20260722T080346Z +13:CLUSTER_REFERENCE_FOUNDATION=COMPLETE +--- COMMAND_ID=CLUSTER-REFERENCE-CORE-INVENTORY-156 --- +FILE=/var/lib/homelab-chat-run/raw/CLUSTER-REFERENCE-CORE-INVENTORY-156-20260722T082023Z.log SIZE=3515 MTIME=2026-07-22 11:20:25.544704562 +0300 +4:SyntaxError: invalid syntax +=== RUNNER_HELP === +Usage: + homelab-chat-run COMMAND_ID MODE COMPONENT -- command [arguments...] + +MODE: + read-only + verify + change + transport-test + +Environment: + HOMELAB_ALLOW_REPEAT=1 Allow exact repeat of a previously failed command. + HOMELAB_TIMEOUT=SECONDS Optional command timeout. +=== RUNNER_VALIDATION_RULES === +1-#!/usr/bin/env bash +2-set -Eeuo pipefail +3-umask 077 +4- +5-usage() { +6- cat >&2 <<'USAGE' +7-Usage: +8: homelab-chat-run COMMAND_ID MODE COMPONENT -- command [arguments...] +9- +10-MODE: +11: read-only +12: verify +13: change +14: transport-test +15- +16-Environment: +17- HOMELAB_ALLOW_REPEAT=1 Allow exact repeat of a previously failed command. +18- HOMELAB_TIMEOUT=SECONDS Optional command timeout. +19-USAGE +20- exit 64 +21-} +22- +23-[ "$#" -ge 5 ] || usage +24- +25-COMMAND_ID="$1" +26-MODE="$2" +27:COMPONENT="$3" +28-shift 3 +29- +30-[ "${1:-}" = '--' ] || usage +31-shift +32-[ "$#" -ge 1 ] || usage +33- +34-case "$COMMAND_ID" in +35- ''|*[!A-Za-z0-9._-]*) +36- echo 'RESULT=FAIL' +37- echo 'REASON=INVALID_COMMAND_ID' +38- exit 65 +39- ;; +40-esac +41- +42:case "$COMPONENT" in +43- ''|*[!A-Za-z0-9._-]*) +44- echo 'RESULT=FAIL' +45: echo 'REASON=INVALID_COMPONENT' +46- exit 66 +47- ;; +48-esac +49- +50-case "$MODE" in +51: read-only|verify|change|transport-test) +52- ;; +53- *) +54- echo 'RESULT=FAIL' +55: echo 'REASON=INVALID_MODE' +56- exit 67 +57- ;; +58-esac +59- +60-for TOOL in \ +61- awk \ +62- curl \ +63- flock \ +64- python3 \ +65- scp \ +66- sha256sum \ +67- ssh +68-do +69- command -v "$TOOL" >/dev/null 2>&1 || { +70- echo 'RESULT=FAIL' +71- echo "REASON=REQUIRED_TOOL_MISSING:$TOOL" +72- exit 68 +73- } +74-done +75- +76-REFERENCE=/etc/pve/31_HOMELAB_REFERENCE.md +77-ERROR_REGISTER=/etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md +78- +79-[ -r "$REFERENCE" ] || { +80- echo 'RESULT=FAIL' +81- echo 'REASON=REFERENCE_NOT_READABLE' +82- exit 69 +83-} +84- +85-[ -r "$ERROR_REGISTER" ] || { +86- echo 'RESULT=FAIL' +87- echo 'REASON=ERROR_REGISTER_NOT_READABLE' +88- exit 70 +89-} +90- +-- +113- +114-RAW_FILE="$RAW_DIR/${COMMAND_ID}-${STAMP}.log" +115-SANITIZED_FILE="$PUBLIC_DIR/${COMMAND_ID}-${STAMP}.txt" +116-JSON_FILE="$PUBLIC_DIR/${COMMAND_ID}-${STAMP}.json" +117-TXT_FILE="$PUBLIC_DIR/${COMMAND_ID}-${STAMP}-latest.txt" +118- +119-REFERENCE_SHA256="$(sha256sum "$REFERENCE" | awk '{print $1}')" +120-ERROR_REGISTER_SHA256="$(sha256sum "$ERROR_REGISTER" | awk '{print $1}')" +121- +122-COMMAND_SHA256="$( +123- { +124- printf '%s\0' "$MODE" +125: printf '%s\0' "$COMPONENT" +126- printf '%s\0' "$@" +127- } | +128- sha256sum | +129- awk '{print $1}' +130-)" +131- +132-PREVIOUS_FAILED_RC="$( +133- awk -F '\t' -v hash="$COMMAND_SHA256" ' +134- $3 == hash && $4 != "0" { +135- value=$4 +136- } +137- +138- END { +139- print value +140- } +141- ' "$LEDGER" +142-)" +143- +144-RUN_STARTED=true +145-REPEAT_BLOCKED=false +146-BLOCK_REASON='' +147:CHANGES_MADE=false +148- +149-case "$MODE" in +150: change) +151: CHANGES_MADE=true +152- ;; +153-esac +154- +155-if [ -n "$PREVIOUS_FAILED_RC" ] && +156- [ "${HOMELAB_ALLOW_REPEAT:-0}" != 1 ]; then +157- +158- RUN_STARTED=false +159- REPEAT_BLOCKED=true +160- BLOCK_REASON='EXACT_FAILED_COMMAND_REPEAT' +161- RC=73 +162- STATUS='BLOCKED' +163- +164- { +165- echo 'COMMAND_EXECUTION_BLOCKED=yes' +166- echo 'REASON=EXACT_FAILED_COMMAND_REPEAT' +167- echo "PREVIOUS_RC=$PREVIOUS_FAILED_RC" +168- echo "COMMAND_SHA256=$COMMAND_SHA256" +169- echo +170- echo 'Set HOMELAB_ALLOW_REPEAT=1 only after reviewing the previous failure.' +171- } >"$RAW_FILE" +172-else +173- set +e +174- +175- if [ -n "${HOMELAB_TIMEOUT:-}" ]; then +176- case "$HOMELAB_TIMEOUT" in +177- *[!0-9]*|'') +178- echo 'RESULT=FAIL' +179- echo 'REASON=INVALID_HOMELAB_TIMEOUT' +180- exit 74 +181- ;; +182- esac +183- +184- timeout \ +185- --foreground \ +186- --preserve-status \ +-- +310- sha256sum "$SANITIZED_FILE" | +311- awk '{print $1}' +312-)" +313- +314-python3 - \ +315- "$JSON_FILE" \ +316- "$SANITIZED_FILE" \ +317- "$COMMAND_ID" \ +318- "$STATUS" \ +319- "$RC" \ +320- "$HOST_SHORT" \ +321- "$MODE" \ +322: "$COMPONENT" \ +323- "$STARTED_AT" \ +324- "$FINISHED_AT" \ +325- "$REFERENCE_SHA256" \ +326- "$ERROR_REGISTER_SHA256" \ +327- "$COMMAND_SHA256" \ +328- "$RAW_SHA256" \ +329- "$SANITIZED_SHA256" \ +330- "$RUN_STARTED" \ +331- "$REPEAT_BLOCKED" \ +332- "$BLOCK_REASON" \ +333: "$CHANGES_MADE" <<'PY' +334-import json +335-import sys +336-from pathlib import Path +337- +338-( +339- json_path, +340- output_path, +341- command_id, +342- status, +343- rc, +344- host, +345- mode, +346: component, +347- started, +348- finished, +349- reference_sha, +350- error_sha, +351- command_sha, +352- raw_sha, +353- sanitized_sha, +354- run_started, +355- repeat_blocked, +356- block_reason, +357: changes_made, +358-) = sys.argv[1:] +359- +360-output = Path(output_path).read_text( +361- encoding="utf-8", +362- errors="replace", +363-) +364- +365-encoded = output.encode("utf-8") +366-json_limit = 300_000 +367-output_truncated = len(encoded) > json_limit +368- +369-if output_truncated: +370- output_preview = encoded[:json_limit].decode( +371- "utf-8", +372- errors="ignore", +373- ) +374-else: +375- output_preview = output +376- +377-document = { +378- "schema_version": 1, +379- "channel": "homelab-runtime", +380- "command_id": command_id, +381- "status": status, +382- "rc": int(rc), +383- "host": host, +384- "mode": mode, +385: "component": component, +386- "started_at_utc": started, +387- "finished_at_utc": finished, +388- "reference_register_checked": True, +389- "reference_sha256": reference_sha, +390- "error_register_checked": True, +391- "error_register_sha256": error_sha, +392- "command_sha256": command_sha, +393- "duplicate_failed_command_blocked": repeat_blocked == "true", +394- "block_reason": block_reason or None, +395- "execution_started": run_started == "true", +396: "changes_made": changes_made == "true", +397- "sanitized": True, +398- "secrets_included": False, +399- "private_addresses_included": False, +400- "raw_evidence_retained_locally": True, +401- "raw_evidence_sha256": raw_sha, +402- "sanitized_output_sha256": sanitized_sha, +403- "output_truncated_in_json": output_truncated, +404- "full_sanitized_output_url": ( +405- "https://git.gram1.ru/.well-known/" +406- "homelab-runtime/latest.txt" +407- ), +408- "output": output_preview, +409-} +410- +411-Path(json_path).write_text( +412- json.dumps( +413- document, +414- ensure_ascii=False, +415- indent=2, +416- ) +417- + "\n", +418- encoding="utf-8", +419-) +420-PY +421- +422-{ +423- echo 'CHAT_OUTPUT_BEGIN' +424- echo "COMMAND_ID=$COMMAND_ID" +425- echo "STATUS=$STATUS" +426- echo "RC=$RC" +427- echo "HOST=$HOST_SHORT" +428- echo "MODE=$MODE" +429: echo "COMPONENT=$COMPONENT" +430- echo 'REFERENCE_REGISTER_CHECK=OK' +431- echo "REFERENCE_SHA256=$REFERENCE_SHA256" +432- echo 'ERROR_REGISTER_CHECK=OK' +433- echo "ERROR_REGISTER_SHA256=$ERROR_REGISTER_SHA256" +434- echo "COMMAND_SHA256=$COMMAND_SHA256" +435- echo "DUPLICATE_FAILED_COMMAND_BLOCKED=$REPEAT_BLOCKED" +436- echo "EXECUTION_STARTED=$RUN_STARTED" +437: echo "CHANGES_MADE=$CHANGES_MADE" +438- echo 'SANITIZED=yes' +439- echo 'SECRETS_INCLUDED=no' +440- echo 'PRIVATE_ADDRESSES_INCLUDED=no' +441- echo "RAW_EVIDENCE_SHA256=$RAW_SHA256" +442- echo "SANITIZED_OUTPUT_SHA256=$SANITIZED_SHA256" +443- echo 'OUTPUT_BEGIN' +444- cat "$SANITIZED_FILE" +445- echo +446- echo 'OUTPUT_END' +447- echo 'CHAT_OUTPUT_END' +448-} >"$TXT_FILE" +449- +450-chmod 0600 \ +451- "$RAW_FILE" \ +452- "$SANITIZED_FILE" \ +453- "$JSON_FILE" \ +454- "$TXT_FILE" +455- +456-TARGET=debian@[PRIVATE_IP] +457-REMOTE_PREFIX="/tmp/homelab-runtime-${COMMAND_ID}-$$" +458- +459-SSH_OPTS=( +460- -o BatchMode=yes +461- -o ConnectTimeout=10 +462- -o ConnectionAttempts=1 +463- -o StrictHostKeyChecking=yes +464- -o UserKnownHostsFile=[SENSITIVE_PATH] +465-) +466- +467-scp \ +468- -q \ +469- "${SSH_OPTS[@]}" \ +470- "$JSON_FILE" \ +471- "${TARGET}:${REMOTE_PREFIX}.json" +472- +-- +518- echo 'REASON=PUBLIC_RUNTIME_RESULT_NOT_VISIBLE' +519- echo "COMMAND_ID=$COMMAND_ID" +520- echo "LOCAL_RAW_FILE=$RAW_FILE" +521- exit 75 +522-fi +523- +524-printf '%s\t%s\t%s\t%s\t%s\t%s\n' \ +525- "$FINISHED_AT" \ +526- "$COMMAND_ID" \ +527- "$COMMAND_SHA256" \ +528- "$RC" \ +529- "$STATUS" \ +530: "$COMPONENT" >>"$LEDGER" +531- +532-echo +533-echo '===== HOMELAB CHAT RUN COMPLETE =====' +534-echo "COMMAND_ID=$COMMAND_ID" +535-echo "STATUS=$STATUS" +536-echo "RC=$RC" +537-echo 'REFERENCE_REGISTER_CHECK=OK' +538-echo 'ERROR_REGISTER_CHECK=OK' +539-echo "DUPLICATE_FAILED_COMMAND_BLOCKED=$REPEAT_BLOCKED" +540-echo 'GITEA_PUBLICATION=OK' +541-echo "PUBLIC_URL=$PUBLIC_URL" +542-echo "LOCAL_RAW_FILE=$RAW_FILE" +543-echo 'SHELL_SESSION_PRESERVED=yes' +544- +545-exit "$RC" +=== PARTIAL_REFERENCE_STATE === +d 750 4096 2026-07-22T10:34:58.6964026990 /etc/homelab/reference/public +d 750 4096 2026-07-22T10:34:58.6964026990 /etc/homelab/reference/runbooks +d 750 4096 2026-07-22T10:34:58.6964026990 /etc/homelab/reference/snapshots +d 750 4096 2026-07-22T10:34:58.6970459770 /etc/homelab/reference/declared +d 750 4096 2026-07-22T10:34:58.6974681490 /etc/homelab/reference/observed +d 750 4096 2026-07-22T10:34:58.6978512600 /etc/homelab/reference/effective +d 750 4096 2026-07-22T10:34:58.6978691140 /etc/homelab/reference +d 750 4096 2026-07-22T10:34:58.6978834310 /etc/homelab/reference/state +d 750 4096 2026-07-22T11:03:47.1132880790 /etc/homelab/reference/schema +f 640 153 2026-07-22T10:34:58.6968014830 /etc/homelab/reference/declared/dns.json +f 640 153 2026-07-22T10:34:58.6976137370 /etc/homelab/reference/effective/dns.json +f 640 155 2026-07-22T10:34:58.6966969670 /etc/homelab/reference/declared/nodes.json +f 640 155 2026-07-22T10:34:58.6975198720 /etc/homelab/reference/effective/nodes.json +f 640 156 2026-07-22T10:34:58.6967158380 /etc/homelab/reference/declared/guests.json +f 640 156 2026-07-22T10:34:58.6969484530 /etc/homelab/reference/declared/access.json +f 640 156 2026-07-22T10:34:58.6969644820 /etc/homelab/reference/declared/owners.json +f 640 156 2026-07-22T10:34:58.6971898020 /etc/homelab/reference/observed/dns.json +f 640 156 2026-07-22T10:34:58.6975362380 /etc/homelab/reference/effective/guests.json +f 640 156 2026-07-22T10:34:58.6977531080 /etc/homelab/reference/effective/access.json +f 640 156 2026-07-22T10:34:58.6977693510 /etc/homelab/reference/effective/owners.json +f 640 157 2026-07-22T10:34:58.6966738300 /etc/homelab/reference/declared/cluster.json +f 640 157 2026-07-22T10:34:58.6968176220 /etc/homelab/reference/declared/domains.json +f 640 157 2026-07-22T10:34:58.6968507170 /etc/homelab/reference/declared/ingress.json +f 640 157 2026-07-22T10:34:58.6968665330 /etc/homelab/reference/declared/storage.json +f 640 157 2026-07-22T10:34:58.6968815380 /etc/homelab/reference/declared/backups.json +f 640 157 2026-07-22T10:34:58.6969804230 /etc/homelab/reference/declared/vendors.json +f 640 157 2026-07-22T10:34:58.6975026350 /etc/homelab/reference/effective/cluster.json +f 640 157 2026-07-22T10:34:58.6976296950 /etc/homelab/reference/effective/domains.json +f 640 157 2026-07-22T10:34:58.6976621460 /etc/homelab/reference/effective/ingress.json +f 640 157 2026-07-22T10:34:58.6976775090 /etc/homelab/reference/effective/storage.json +f 640 157 2026-07-22T10:34:58.6976922400 /etc/homelab/reference/effective/backups.json +f 640 157 2026-07-22T10:34:58.6977857550 /etc/homelab/reference/effective/vendors.json +f 640 158 2026-07-22T10:34:58.6967496500 /etc/homelab/reference/declared/services.json +f 640 158 2026-07-22T10:34:58.6967835450 /etc/homelab/reference/declared/networks.json +f 640 158 2026-07-22T10:34:58.6970122440 /etc/homelab/reference/declared/security.json +f 640 158 2026-07-22T10:34:58.6970626700 /etc/homelab/reference/declared/runbooks.json +f 640 158 2026-07-22T10:34:58.6970953450 /etc/homelab/reference/observed/nodes.json +f 640 158 2026-07-22T10:34:58.6975661010 /etc/homelab/reference/effective/services.json +f 640 158 2026-07-22T10:34:58.6975973330 /etc/homelab/reference/effective/networks.json +f 640 158 2026-07-22T10:34:58.6978179760 /etc/homelab/reference/effective/security.json +f 640 158 2026-07-22T10:34:58.6978691140 /etc/homelab/reference/effective/runbooks.json +f 640 159 2026-07-22T10:34:58.6969966040 /etc/homelab/reference/declared/contracts.json +f 640 159 2026-07-22T10:34:58.6971108280 /etc/homelab/reference/observed/guests.json +f 640 159 2026-07-22T10:34:58.6973313430 /etc/homelab/reference/observed/access.json +f 640 159 2026-07-22T10:34:58.6973523380 /etc/homelab/reference/observed/owners.json +f 640 159 2026-07-22T10:34:58.6978023050 /etc/homelab/reference/effective/contracts.json +f 640 160 2026-07-22T10:34:58.6967330550 /etc/homelab/reference/declared/containers.json +f 640 160 2026-07-22T10:34:58.6968977560 /etc/homelab/reference/declared/monitoring.json +f 640 160 2026-07-22T10:34:58.6970793370 /etc/homelab/reference/observed/cluster.json +f 640 160 2026-07-22T10:34:58.6972051460 /etc/homelab/reference/observed/domains.json +f 640 160 2026-07-22T10:34:58.6972368890 /etc/homelab/reference/observed/ingress.json +f 640 160 2026-07-22T10:34:58.6972525750 /etc/homelab/reference/observed/storage.json +f 640 160 2026-07-22T10:34:58.6972679110 /etc/homelab/reference/observed/backups.json +f 640 160 2026-07-22T10:34:58.6973681500 /etc/homelab/reference/observed/vendors.json +f 640 160 2026-07-22T10:34:58.6975511580 /etc/homelab/reference/effective/containers.json +f 640 160 2026-07-22T10:34:58.6977071740 /etc/homelab/reference/effective/monitoring.json +f 640 161 2026-07-22T10:34:58.6971427070 /etc/homelab/reference/observed/services.json +f 640 161 2026-07-22T10:34:58.6971740390 /etc/homelab/reference/observed/networks.json +f 640 161 2026-07-22T10:34:58.6974304020 /etc/homelab/reference/observed/security.json +f 640 161 2026-07-22T10:34:58.6974849460 /etc/homelab/reference/observed/runbooks.json +f 640 162 2026-07-22T10:34:58.6967668110 /etc/homelab/reference/declared/applications.json +f 640 162 2026-07-22T10:34:58.6968346450 /etc/homelab/reference/declared/certificates.json +f 640 162 2026-07-22T10:34:58.6969325220 /etc/homelab/reference/declared/dependencies.json +f 640 162 2026-07-22T10:34:58.6974142710 /etc/homelab/reference/observed/contracts.json +f 640 162 2026-07-22T10:34:58.6975824200 /etc/homelab/reference/effective/applications.json +f 640 162 2026-07-22T10:34:58.6976463310 /etc/homelab/reference/effective/certificates.json +f 640 162 2026-07-22T10:34:58.6977380340 /etc/homelab/reference/effective/dependencies.json +f 640 163 2026-07-22T10:34:58.6971273900 /etc/homelab/reference/observed/containers.json +f 640 163 2026-07-22T10:34:58.6972830790 /etc/homelab/reference/observed/monitoring.json +f 640 165 2026-07-22T10:34:58.6971583070 /etc/homelab/reference/observed/applications.json +f 640 165 2026-07-22T10:34:58.6972204840 /etc/homelab/reference/observed/certificates.json +f 640 165 2026-07-22T10:34:58.6973154820 /etc/homelab/reference/observed/dependencies.json +f 640 167 2026-07-22T10:34:58.6969164820 /etc/homelab/reference/declared/external_services.json +f 640 167 2026-07-22T10:34:58.6970278950 /etc/homelab/reference/declared/disaster_recovery.json +f 640 167 2026-07-22T10:34:58.6970459770 /etc/homelab/reference/declared/change_management.json +f 640 167 2026-07-22T10:34:58.6977227780 /etc/homelab/reference/effective/external_services.json +f 640 167 2026-07-22T10:34:58.6978337330 /etc/homelab/reference/effective/disaster_recovery.json +f 640 167 2026-07-22T10:34:58.6978512600 /etc/homelab/reference/effective/change_management.json +f 640 170 2026-07-22T10:34:58.6972985790 /etc/homelab/reference/observed/external_services.json +f 640 170 2026-07-22T10:34:58.6974490020 /etc/homelab/reference/observed/disaster_recovery.json +f 640 170 2026-07-22T10:34:58.6974681490 /etc/homelab/reference/observed/change_management.json +f 640 1780 2026-07-22T11:12:10.9514643580 /etc/homelab/reference/manifest.json +f 640 199 2026-07-22T11:12:10.9518855430 /etc/homelab/reference/state/current.json +f 640 262 2026-07-22T11:12:10.9515215670 /etc/homelab/reference/schema/record.schema.json +f 640 440 2026-07-22T11:03:47.1133590400 /etc/homelab/reference/schema/edit-policy.json +f 640 593 2026-07-22T11:12:10.9518574460 /etc/homelab/reference/README.md +CLUSTER_REFERENCE_PUBLICATION_ROOTCAUSE=COMPLETE + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 729b5d6..1c91cab 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,30 +1,30 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "CLUSTER-REFERENCE-CORE-INVENTORY-156", - "status": "FAIL", - "rc": 1, + "command_id": "CLUSTER-REFERENCE-PUBLICATION-ROOTCAUSE-157", + "status": "OK", + "rc": 0, "host": "pve01", - "mode": "change", - "component": "reference-update", - "started_at_utc": "2026-07-22T08:20:23Z", - "finished_at_utc": "2026-07-22T08:20:25Z", + "mode": "read-only", + "component": "cluster-audit", + "started_at_utc": "2026-07-22T08:27:15Z", + "finished_at_utc": "2026-07-22T08:27:15Z", "reference_register_checked": true, "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", "error_register_checked": true, "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", - "command_sha256": "4004a5fa6311925347ca61391364d80f4a990a2b0cdd56de0810325d41c4851c", + "command_sha256": "4aab341221543b56b703380a36a56615e9c1b8083637dd486284e765228f35cc", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, - "changes_made": true, + "changes_made": false, "sanitized": true, "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "028bdeec2160ad20c2f0234e6be183a522174af5aac4c345b2be6df78c312e7e", - "sanitized_output_sha256": "028bdeec2160ad20c2f0234e6be183a522174af5aac4c345b2be6df78c312e7e", + "raw_evidence_sha256": "d7f4d6f0c5fbb56a2635094046fe7bc28593aa8c1a32a616dba0306229034bfb", + "sanitized_output_sha256": "40566782503f51e3d02dabefddb0d16e849da8324d97462c4230bdd109a92e41", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": " File \"\", line 1\n import json,sys,pathlib,os,socket; r=pathlib.Path(sys.argv[1]); now=sys.argv[2]; t=pathlib.Path(sys.argv[3]); load=lambda n: json.loads((t/n).read_text(encoding=utf-8) or []); nodes=load(nodes.json); guests=load(guests.json); storage=load(storage.json); records={cluster:[{id:cluster-homelab,hostname:socket.gethostname(),kernel:os.uname().release,pve_version:(t/pveversion.txt).read_text(errors=replace),quorum_status:(t/quorum.txt).read_text(errors=replace)}],nodes:[dict(x,id=node-+str(x.get(node) or x.get(id) or unknown)) for x in nodes],guests:[{id:guest-+str(x.get(vmid)),vmid:x.get(vmid),name:x.get(name),type:x.get(type),node:x.get(node),status:x.get(status),template:x.get(template),uptime:x.get(uptime),cpus:x.get(maxcpu),memory_bytes:x.get(maxmem),disk_bytes:x.get(maxdisk),tags:x.get(tags)} for x in guests],containers:[{id:container-+str(x.get(vmid)),vmid:x.get(vmid),name:x.get(name),node:x.get(node),status:x.get(status),uptime:x.get(uptime),cpus:x.get(maxcpu),memory_bytes:x.get(maxmem),disk_bytes:x.get(maxdisk),tags:x.get(tags)} for x in guests if x.get(type)==lxc],storage:[dict(x,id=storage-+str(x.get(storage) or x.get(id) or unknown)) for x in storage],networks:[{id:network-state,interfaces:load(address.json),routes:load(routes.json)}],dns:[{id:resolver-state,hostname:socket.gethostname(),resolv_conf:(t/resolv.txt).read_text(errors=replace).splitlines()}]}; [(r/observed/(s+.json)).write_text(json.dumps({schema_version:2,section:s,layer:observed,updated_at:now,source:core-internal-collector,records:v},ensure_ascii=False,indent=2)+n,encoding=utf-8) for s,v in records.items()]; state=json.loads((r/state/current.json).read_text()); state.update({status:CORE_INTERNAL_INVENTORY_READY,updated_at:now,core_internal_counts:{s:len(v) for s,v in records.items()}}); (r/state/current.json).write_text(json.dumps(state,ensure_ascii=False,indent=2)+n); [json.loads(p.read_text()) for p in r.rglob(*.json)]; print(CORE_INTERNAL_SECTIONS=+str(len(records))); print(CORE_INTERNAL_RECORDS=+str(sum(len(v) for v in records.values())))\n ^\nSyntaxError: invalid syntax\n" + "output": "=== LOCAL_COMMAND_TRACES ===\n--- COMMAND_ID=CLUSTER-REFERENCE-FOUNDATION-151 ---\nFILE=/var/lib/homelab-chat-run/raw/CLUSTER-REFERENCE-FOUNDATION-151-20260722T081210Z.log SIZE=226 MTIME=2026-07-22 11:12:10.952034889 +0300\n1:REFERENCE_ROOT=/etc/homelab/reference\n2:REFERENCE_SECTIONS=24\n3:REFERENCE_JSON_FILES=76\n4:REFERENCE_BACKUP=/var/backups/homelab-reference/20260722T081210949547Z\n5:REFERENCE_SECRET_POLICY=ENFORCED\n6:CLUSTER_REFERENCE_FOUNDATION=COMPLETE\n--- COMMAND_ID=CLUSTER-REFERENCE-MODE-SEMANTICS-153 ---\nFILE=/var/lib/homelab-chat-run/raw/CLUSTER-REFERENCE-MODE-SEMANTICS-153-20260722T075606Z.log SIZE=8757 MTIME=2026-07-22 10:56:06.971072094 +0300\n1:PREVIOUS_RESULT=VERIFIED\n52:36- echo 'RESULT=FAIL'\n53:37- echo 'REASON=INVALID_COMMAND_ID'\n60:44- echo 'RESULT=FAIL'\n61:45- echo 'REASON=INVALID_COMPONENT'\n70:54- echo 'RESULT=FAIL'\n71:55: echo 'REASON=INVALID_MODE'\n86:70- echo 'RESULT=FAIL'\n87:71- echo \"REASON=REQUIRED_TOOL_MISSING:$TOOL\"\n96:80- echo 'RESULT=FAIL'\n97:81- echo 'REASON=REFERENCE_NOT_READABLE'\n102:86- echo 'RESULT=FAIL'\n103:87- echo 'REASON=ERROR_REGISTER_NOT_READABLE'\n114:132-PREVIOUS_FAILED_RC=\"$(\n128:146:BLOCK_REASON=''\n142:160: BLOCK_REASON='EXACT_FAILED_COMMAND_REPEAT'\n143:161- RC=73\n144:162- STATUS='BLOCKED'\n148:166- echo 'REASON=EXACT_FAILED_COMMAND_REPEAT'\n149:167- echo \"PREVIOUS_RC=$PREVIOUS_FAILED_RC\"\n160:178- echo 'RESULT=FAIL'\n161:179- echo 'REASON=INVALID_HOMELAB_TIMEOUT'\n171:189- RC=$?\n174:192- RC=$?\n180:198- STATUS='OK'\n182:200- STATUS='FAIL'\n192:325- \"$REFERENCE_SHA256\" \\\n292:425- echo \"STATUS=$STATUS\"\n293:426- echo \"RC=$RC\"\n297:430- echo 'REFERENCE_REGISTER_CHECK=OK'\n298:431- echo \"REFERENCE_SHA256=$REFERENCE_SHA256\"\n354:CLUSTER_REFERENCE_MODE_SEMANTICS=COMPLETE\n--- COMMAND_ID=CLUSTER-REFERENCE-FOUNDATION-154 ---\nFILE=/var/lib/homelab-chat-run/raw/CLUSTER-REFERENCE-FOUNDATION-154-20260722T080015Z.log SIZE=382 MTIME=2026-07-22 11:00:15.407910031 +0300\n4:REFERENCE_ROOT=/etc/homelab/reference\n5:REFERENCE_SECTIONS=24\n6:REFERENCE_BACKUP=/var/backups/homelab-reference/20260722T080015404968Z\n7:REFERENCE_SECRET_POLICY=ENFORCED\n8:CLUSTER_REFERENCE_FOUNDATION=COMPLETE\n--- COMMAND_ID=CLUSTER-REFERENCE-FOUNDATION-155 ---\nFILE=/var/lib/homelab-chat-run/raw/CLUSTER-REFERENCE-FOUNDATION-155-20260722T080346Z.log SIZE=454 MTIME=2026-07-22 11:03:47.117217539 +0300\n3:PREVIOUS_STATUS=OK\n4:PREVIOUS_RC=0\n5:PREVIOUS_REASON=None\n7:REFERENCE_ROOT=/etc/homelab/reference\n8:REFERENCE_SECTIONS=24\n9:REFERENCE_JSON_FILES=76\n10:REFERENCE_VALIDATION=PASS\n11:REFERENCE_SECRET_POLICY=ENFORCED\n12:REFERENCE_BACKUP=/var/backups/homelab-reference/20260722T080346Z\n13:CLUSTER_REFERENCE_FOUNDATION=COMPLETE\n--- COMMAND_ID=CLUSTER-REFERENCE-CORE-INVENTORY-156 ---\nFILE=/var/lib/homelab-chat-run/raw/CLUSTER-REFERENCE-CORE-INVENTORY-156-20260722T082023Z.log SIZE=3515 MTIME=2026-07-22 11:20:25.544704562 +0300\n4:SyntaxError: invalid syntax\n=== RUNNER_HELP ===\nUsage:\n homelab-chat-run COMMAND_ID MODE COMPONENT -- command [arguments...]\n\nMODE:\n read-only\n verify\n change\n transport-test\n\nEnvironment:\n HOMELAB_ALLOW_REPEAT=1 Allow exact repeat of a previously failed command.\n HOMELAB_TIMEOUT=SECONDS Optional command timeout.\n=== RUNNER_VALIDATION_RULES ===\n1-#!/usr/bin/env bash\n2-set -Eeuo pipefail\n3-umask 077\n4-\n5-usage() {\n6- cat >&2 <<'USAGE'\n7-Usage:\n8: homelab-chat-run COMMAND_ID MODE COMPONENT -- command [arguments...]\n9-\n10-MODE:\n11: read-only\n12: verify\n13: change\n14: transport-test\n15-\n16-Environment:\n17- HOMELAB_ALLOW_REPEAT=1 Allow exact repeat of a previously failed command.\n18- HOMELAB_TIMEOUT=SECONDS Optional command timeout.\n19-USAGE\n20- exit 64\n21-}\n22-\n23-[ \"$#\" -ge 5 ] || usage\n24-\n25-COMMAND_ID=\"$1\"\n26-MODE=\"$2\"\n27:COMPONENT=\"$3\"\n28-shift 3\n29-\n30-[ \"${1:-}\" = '--' ] || usage\n31-shift\n32-[ \"$#\" -ge 1 ] || usage\n33-\n34-case \"$COMMAND_ID\" in\n35- ''|*[!A-Za-z0-9._-]*)\n36- echo 'RESULT=FAIL'\n37- echo 'REASON=INVALID_COMMAND_ID'\n38- exit 65\n39- ;;\n40-esac\n41-\n42:case \"$COMPONENT\" in\n43- ''|*[!A-Za-z0-9._-]*)\n44- echo 'RESULT=FAIL'\n45: echo 'REASON=INVALID_COMPONENT'\n46- exit 66\n47- ;;\n48-esac\n49-\n50-case \"$MODE\" in\n51: read-only|verify|change|transport-test)\n52- ;;\n53- *)\n54- echo 'RESULT=FAIL'\n55: echo 'REASON=INVALID_MODE'\n56- exit 67\n57- ;;\n58-esac\n59-\n60-for TOOL in \\\n61- awk \\\n62- curl \\\n63- flock \\\n64- python3 \\\n65- scp \\\n66- sha256sum \\\n67- ssh\n68-do\n69- command -v \"$TOOL\" >/dev/null 2>&1 || {\n70- echo 'RESULT=FAIL'\n71- echo \"REASON=REQUIRED_TOOL_MISSING:$TOOL\"\n72- exit 68\n73- }\n74-done\n75-\n76-REFERENCE=/etc/pve/31_HOMELAB_REFERENCE.md\n77-ERROR_REGISTER=/etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md\n78-\n79-[ -r \"$REFERENCE\" ] || {\n80- echo 'RESULT=FAIL'\n81- echo 'REASON=REFERENCE_NOT_READABLE'\n82- exit 69\n83-}\n84-\n85-[ -r \"$ERROR_REGISTER\" ] || {\n86- echo 'RESULT=FAIL'\n87- echo 'REASON=ERROR_REGISTER_NOT_READABLE'\n88- exit 70\n89-}\n90-\n--\n113-\n114-RAW_FILE=\"$RAW_DIR/${COMMAND_ID}-${STAMP}.log\"\n115-SANITIZED_FILE=\"$PUBLIC_DIR/${COMMAND_ID}-${STAMP}.txt\"\n116-JSON_FILE=\"$PUBLIC_DIR/${COMMAND_ID}-${STAMP}.json\"\n117-TXT_FILE=\"$PUBLIC_DIR/${COMMAND_ID}-${STAMP}-latest.txt\"\n118-\n119-REFERENCE_SHA256=\"$(sha256sum \"$REFERENCE\" | awk '{print $1}')\"\n120-ERROR_REGISTER_SHA256=\"$(sha256sum \"$ERROR_REGISTER\" | awk '{print $1}')\"\n121-\n122-COMMAND_SHA256=\"$(\n123- {\n124- printf '%s\\0' \"$MODE\"\n125: printf '%s\\0' \"$COMPONENT\"\n126- printf '%s\\0' \"$@\"\n127- } |\n128- sha256sum |\n129- awk '{print $1}'\n130-)\"\n131-\n132-PREVIOUS_FAILED_RC=\"$(\n133- awk -F '\\t' -v hash=\"$COMMAND_SHA256\" '\n134- $3 == hash && $4 != \"0\" {\n135- value=$4\n136- }\n137-\n138- END {\n139- print value\n140- }\n141- ' \"$LEDGER\"\n142-)\"\n143-\n144-RUN_STARTED=true\n145-REPEAT_BLOCKED=false\n146-BLOCK_REASON=''\n147:CHANGES_MADE=false\n148-\n149-case \"$MODE\" in\n150: change)\n151: CHANGES_MADE=true\n152- ;;\n153-esac\n154-\n155-if [ -n \"$PREVIOUS_FAILED_RC\" ] &&\n156- [ \"${HOMELAB_ALLOW_REPEAT:-0}\" != 1 ]; then\n157-\n158- RUN_STARTED=false\n159- REPEAT_BLOCKED=true\n160- BLOCK_REASON='EXACT_FAILED_COMMAND_REPEAT'\n161- RC=73\n162- STATUS='BLOCKED'\n163-\n164- {\n165- echo 'COMMAND_EXECUTION_BLOCKED=yes'\n166- echo 'REASON=EXACT_FAILED_COMMAND_REPEAT'\n167- echo \"PREVIOUS_RC=$PREVIOUS_FAILED_RC\"\n168- echo \"COMMAND_SHA256=$COMMAND_SHA256\"\n169- echo\n170- echo 'Set HOMELAB_ALLOW_REPEAT=1 only after reviewing the previous failure.'\n171- } >\"$RAW_FILE\"\n172-else\n173- set +e\n174-\n175- if [ -n \"${HOMELAB_TIMEOUT:-}\" ]; then\n176- case \"$HOMELAB_TIMEOUT\" in\n177- *[!0-9]*|'')\n178- echo 'RESULT=FAIL'\n179- echo 'REASON=INVALID_HOMELAB_TIMEOUT'\n180- exit 74\n181- ;;\n182- esac\n183-\n184- timeout \\\n185- --foreground \\\n186- --preserve-status \\\n--\n310- sha256sum \"$SANITIZED_FILE\" |\n311- awk '{print $1}'\n312-)\"\n313-\n314-python3 - \\\n315- \"$JSON_FILE\" \\\n316- \"$SANITIZED_FILE\" \\\n317- \"$COMMAND_ID\" \\\n318- \"$STATUS\" \\\n319- \"$RC\" \\\n320- \"$HOST_SHORT\" \\\n321- \"$MODE\" \\\n322: \"$COMPONENT\" \\\n323- \"$STARTED_AT\" \\\n324- \"$FINISHED_AT\" \\\n325- \"$REFERENCE_SHA256\" \\\n326- \"$ERROR_REGISTER_SHA256\" \\\n327- \"$COMMAND_SHA256\" \\\n328- \"$RAW_SHA256\" \\\n329- \"$SANITIZED_SHA256\" \\\n330- \"$RUN_STARTED\" \\\n331- \"$REPEAT_BLOCKED\" \\\n332- \"$BLOCK_REASON\" \\\n333: \"$CHANGES_MADE\" <<'PY'\n334-import json\n335-import sys\n336-from pathlib import Path\n337-\n338-(\n339- json_path,\n340- output_path,\n341- command_id,\n342- status,\n343- rc,\n344- host,\n345- mode,\n346: component,\n347- started,\n348- finished,\n349- reference_sha,\n350- error_sha,\n351- command_sha,\n352- raw_sha,\n353- sanitized_sha,\n354- run_started,\n355- repeat_blocked,\n356- block_reason,\n357: changes_made,\n358-) = sys.argv[1:]\n359-\n360-output = Path(output_path).read_text(\n361- encoding=\"utf-8\",\n362- errors=\"replace\",\n363-)\n364-\n365-encoded = output.encode(\"utf-8\")\n366-json_limit = 300_000\n367-output_truncated = len(encoded) > json_limit\n368-\n369-if output_truncated:\n370- output_preview = encoded[:json_limit].decode(\n371- \"utf-8\",\n372- errors=\"ignore\",\n373- )\n374-else:\n375- output_preview = output\n376-\n377-document = {\n378- \"schema_version\": 1,\n379- \"channel\": \"homelab-runtime\",\n380- \"command_id\": command_id,\n381- \"status\": status,\n382- \"rc\": int(rc),\n383- \"host\": host,\n384- \"mode\": mode,\n385: \"component\": component,\n386- \"started_at_utc\": started,\n387- \"finished_at_utc\": finished,\n388- \"reference_register_checked\": True,\n389- \"reference_sha256\": reference_sha,\n390- \"error_register_checked\": True,\n391- \"error_register_sha256\": error_sha,\n392- \"command_sha256\": command_sha,\n393- \"duplicate_failed_command_blocked\": repeat_blocked == \"true\",\n394- \"block_reason\": block_reason or None,\n395- \"execution_started\": run_started == \"true\",\n396: \"changes_made\": changes_made == \"true\",\n397- \"sanitized\": True,\n398- \"secrets_included\": False,\n399- \"private_addresses_included\": False,\n400- \"raw_evidence_retained_locally\": True,\n401- \"raw_evidence_sha256\": raw_sha,\n402- \"sanitized_output_sha256\": sanitized_sha,\n403- \"output_truncated_in_json\": output_truncated,\n404- \"full_sanitized_output_url\": (\n405- \"https://git.gram1.ru/.well-known/\"\n406- \"homelab-runtime/latest.txt\"\n407- ),\n408- \"output\": output_preview,\n409-}\n410-\n411-Path(json_path).write_text(\n412- json.dumps(\n413- document,\n414- ensure_ascii=False,\n415- indent=2,\n416- )\n417- + \"\\n\",\n418- encoding=\"utf-8\",\n419-)\n420-PY\n421-\n422-{\n423- echo 'CHAT_OUTPUT_BEGIN'\n424- echo \"COMMAND_ID=$COMMAND_ID\"\n425- echo \"STATUS=$STATUS\"\n426- echo \"RC=$RC\"\n427- echo \"HOST=$HOST_SHORT\"\n428- echo \"MODE=$MODE\"\n429: echo \"COMPONENT=$COMPONENT\"\n430- echo 'REFERENCE_REGISTER_CHECK=OK'\n431- echo \"REFERENCE_SHA256=$REFERENCE_SHA256\"\n432- echo 'ERROR_REGISTER_CHECK=OK'\n433- echo \"ERROR_REGISTER_SHA256=$ERROR_REGISTER_SHA256\"\n434- echo \"COMMAND_SHA256=$COMMAND_SHA256\"\n435- echo \"DUPLICATE_FAILED_COMMAND_BLOCKED=$REPEAT_BLOCKED\"\n436- echo \"EXECUTION_STARTED=$RUN_STARTED\"\n437: echo \"CHANGES_MADE=$CHANGES_MADE\"\n438- echo 'SANITIZED=yes'\n439- echo 'SECRETS_INCLUDED=no'\n440- echo 'PRIVATE_ADDRESSES_INCLUDED=no'\n441- echo \"RAW_EVIDENCE_SHA256=$RAW_SHA256\"\n442- echo \"SANITIZED_OUTPUT_SHA256=$SANITIZED_SHA256\"\n443- echo 'OUTPUT_BEGIN'\n444- cat \"$SANITIZED_FILE\"\n445- echo\n446- echo 'OUTPUT_END'\n447- echo 'CHAT_OUTPUT_END'\n448-} >\"$TXT_FILE\"\n449-\n450-chmod 0600 \\\n451- \"$RAW_FILE\" \\\n452- \"$SANITIZED_FILE\" \\\n453- \"$JSON_FILE\" \\\n454- \"$TXT_FILE\"\n455-\n456-TARGET=debian@[PRIVATE_IP]\n457-REMOTE_PREFIX=\"/tmp/homelab-runtime-${COMMAND_ID}-$$\"\n458-\n459-SSH_OPTS=(\n460- -o BatchMode=yes\n461- -o ConnectTimeout=10\n462- -o ConnectionAttempts=1\n463- -o StrictHostKeyChecking=yes\n464- -o UserKnownHostsFile=[SENSITIVE_PATH]\n465-)\n466-\n467-scp \\\n468- -q \\\n469- \"${SSH_OPTS[@]}\" \\\n470- \"$JSON_FILE\" \\\n471- \"${TARGET}:${REMOTE_PREFIX}.json\"\n472-\n--\n518- echo 'REASON=PUBLIC_RUNTIME_RESULT_NOT_VISIBLE'\n519- echo \"COMMAND_ID=$COMMAND_ID\"\n520- echo \"LOCAL_RAW_FILE=$RAW_FILE\"\n521- exit 75\n522-fi\n523-\n524-printf '%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n' \\\n525- \"$FINISHED_AT\" \\\n526- \"$COMMAND_ID\" \\\n527- \"$COMMAND_SHA256\" \\\n528- \"$RC\" \\\n529- \"$STATUS\" \\\n530: \"$COMPONENT\" >>\"$LEDGER\"\n531-\n532-echo\n533-echo '===== HOMELAB CHAT RUN COMPLETE ====='\n534-echo \"COMMAND_ID=$COMMAND_ID\"\n535-echo \"STATUS=$STATUS\"\n536-echo \"RC=$RC\"\n537-echo 'REFERENCE_REGISTER_CHECK=OK'\n538-echo 'ERROR_REGISTER_CHECK=OK'\n539-echo \"DUPLICATE_FAILED_COMMAND_BLOCKED=$REPEAT_BLOCKED\"\n540-echo 'GITEA_PUBLICATION=OK'\n541-echo \"PUBLIC_URL=$PUBLIC_URL\"\n542-echo \"LOCAL_RAW_FILE=$RAW_FILE\"\n543-echo 'SHELL_SESSION_PRESERVED=yes'\n544-\n545-exit \"$RC\"\n=== PARTIAL_REFERENCE_STATE ===\nd 750 4096 2026-07-22T10:34:58.6964026990 /etc/homelab/reference/public\nd 750 4096 2026-07-22T10:34:58.6964026990 /etc/homelab/reference/runbooks\nd 750 4096 2026-07-22T10:34:58.6964026990 /etc/homelab/reference/snapshots\nd 750 4096 2026-07-22T10:34:58.6970459770 /etc/homelab/reference/declared\nd 750 4096 2026-07-22T10:34:58.6974681490 /etc/homelab/reference/observed\nd 750 4096 2026-07-22T10:34:58.6978512600 /etc/homelab/reference/effective\nd 750 4096 2026-07-22T10:34:58.6978691140 /etc/homelab/reference\nd 750 4096 2026-07-22T10:34:58.6978834310 /etc/homelab/reference/state\nd 750 4096 2026-07-22T11:03:47.1132880790 /etc/homelab/reference/schema\nf 640 153 2026-07-22T10:34:58.6968014830 /etc/homelab/reference/declared/dns.json\nf 640 153 2026-07-22T10:34:58.6976137370 /etc/homelab/reference/effective/dns.json\nf 640 155 2026-07-22T10:34:58.6966969670 /etc/homelab/reference/declared/nodes.json\nf 640 155 2026-07-22T10:34:58.6975198720 /etc/homelab/reference/effective/nodes.json\nf 640 156 2026-07-22T10:34:58.6967158380 /etc/homelab/reference/declared/guests.json\nf 640 156 2026-07-22T10:34:58.6969484530 /etc/homelab/reference/declared/access.json\nf 640 156 2026-07-22T10:34:58.6969644820 /etc/homelab/reference/declared/owners.json\nf 640 156 2026-07-22T10:34:58.6971898020 /etc/homelab/reference/observed/dns.json\nf 640 156 2026-07-22T10:34:58.6975362380 /etc/homelab/reference/effective/guests.json\nf 640 156 2026-07-22T10:34:58.6977531080 /etc/homelab/reference/effective/access.json\nf 640 156 2026-07-22T10:34:58.6977693510 /etc/homelab/reference/effective/owners.json\nf 640 157 2026-07-22T10:34:58.6966738300 /etc/homelab/reference/declared/cluster.json\nf 640 157 2026-07-22T10:34:58.6968176220 /etc/homelab/reference/declared/domains.json\nf 640 157 2026-07-22T10:34:58.6968507170 /etc/homelab/reference/declared/ingress.json\nf 640 157 2026-07-22T10:34:58.6968665330 /etc/homelab/reference/declared/storage.json\nf 640 157 2026-07-22T10:34:58.6968815380 /etc/homelab/reference/declared/backups.json\nf 640 157 2026-07-22T10:34:58.6969804230 /etc/homelab/reference/declared/vendors.json\nf 640 157 2026-07-22T10:34:58.6975026350 /etc/homelab/reference/effective/cluster.json\nf 640 157 2026-07-22T10:34:58.6976296950 /etc/homelab/reference/effective/domains.json\nf 640 157 2026-07-22T10:34:58.6976621460 /etc/homelab/reference/effective/ingress.json\nf 640 157 2026-07-22T10:34:58.6976775090 /etc/homelab/reference/effective/storage.json\nf 640 157 2026-07-22T10:34:58.6976922400 /etc/homelab/reference/effective/backups.json\nf 640 157 2026-07-22T10:34:58.6977857550 /etc/homelab/reference/effective/vendors.json\nf 640 158 2026-07-22T10:34:58.6967496500 /etc/homelab/reference/declared/services.json\nf 640 158 2026-07-22T10:34:58.6967835450 /etc/homelab/reference/declared/networks.json\nf 640 158 2026-07-22T10:34:58.6970122440 /etc/homelab/reference/declared/security.json\nf 640 158 2026-07-22T10:34:58.6970626700 /etc/homelab/reference/declared/runbooks.json\nf 640 158 2026-07-22T10:34:58.6970953450 /etc/homelab/reference/observed/nodes.json\nf 640 158 2026-07-22T10:34:58.6975661010 /etc/homelab/reference/effective/services.json\nf 640 158 2026-07-22T10:34:58.6975973330 /etc/homelab/reference/effective/networks.json\nf 640 158 2026-07-22T10:34:58.6978179760 /etc/homelab/reference/effective/security.json\nf 640 158 2026-07-22T10:34:58.6978691140 /etc/homelab/reference/effective/runbooks.json\nf 640 159 2026-07-22T10:34:58.6969966040 /etc/homelab/reference/declared/contracts.json\nf 640 159 2026-07-22T10:34:58.6971108280 /etc/homelab/reference/observed/guests.json\nf 640 159 2026-07-22T10:34:58.6973313430 /etc/homelab/reference/observed/access.json\nf 640 159 2026-07-22T10:34:58.6973523380 /etc/homelab/reference/observed/owners.json\nf 640 159 2026-07-22T10:34:58.6978023050 /etc/homelab/reference/effective/contracts.json\nf 640 160 2026-07-22T10:34:58.6967330550 /etc/homelab/reference/declared/containers.json\nf 640 160 2026-07-22T10:34:58.6968977560 /etc/homelab/reference/declared/monitoring.json\nf 640 160 2026-07-22T10:34:58.6970793370 /etc/homelab/reference/observed/cluster.json\nf 640 160 2026-07-22T10:34:58.6972051460 /etc/homelab/reference/observed/domains.json\nf 640 160 2026-07-22T10:34:58.6972368890 /etc/homelab/reference/observed/ingress.json\nf 640 160 2026-07-22T10:34:58.6972525750 /etc/homelab/reference/observed/storage.json\nf 640 160 2026-07-22T10:34:58.6972679110 /etc/homelab/reference/observed/backups.json\nf 640 160 2026-07-22T10:34:58.6973681500 /etc/homelab/reference/observed/vendors.json\nf 640 160 2026-07-22T10:34:58.6975511580 /etc/homelab/reference/effective/containers.json\nf 640 160 2026-07-22T10:34:58.6977071740 /etc/homelab/reference/effective/monitoring.json\nf 640 161 2026-07-22T10:34:58.6971427070 /etc/homelab/reference/observed/services.json\nf 640 161 2026-07-22T10:34:58.6971740390 /etc/homelab/reference/observed/networks.json\nf 640 161 2026-07-22T10:34:58.6974304020 /etc/homelab/reference/observed/security.json\nf 640 161 2026-07-22T10:34:58.6974849460 /etc/homelab/reference/observed/runbooks.json\nf 640 162 2026-07-22T10:34:58.6967668110 /etc/homelab/reference/declared/applications.json\nf 640 162 2026-07-22T10:34:58.6968346450 /etc/homelab/reference/declared/certificates.json\nf 640 162 2026-07-22T10:34:58.6969325220 /etc/homelab/reference/declared/dependencies.json\nf 640 162 2026-07-22T10:34:58.6974142710 /etc/homelab/reference/observed/contracts.json\nf 640 162 2026-07-22T10:34:58.6975824200 /etc/homelab/reference/effective/applications.json\nf 640 162 2026-07-22T10:34:58.6976463310 /etc/homelab/reference/effective/certificates.json\nf 640 162 2026-07-22T10:34:58.6977380340 /etc/homelab/reference/effective/dependencies.json\nf 640 163 2026-07-22T10:34:58.6971273900 /etc/homelab/reference/observed/containers.json\nf 640 163 2026-07-22T10:34:58.6972830790 /etc/homelab/reference/observed/monitoring.json\nf 640 165 2026-07-22T10:34:58.6971583070 /etc/homelab/reference/observed/applications.json\nf 640 165 2026-07-22T10:34:58.6972204840 /etc/homelab/reference/observed/certificates.json\nf 640 165 2026-07-22T10:34:58.6973154820 /etc/homelab/reference/observed/dependencies.json\nf 640 167 2026-07-22T10:34:58.6969164820 /etc/homelab/reference/declared/external_services.json\nf 640 167 2026-07-22T10:34:58.6970278950 /etc/homelab/reference/declared/disaster_recovery.json\nf 640 167 2026-07-22T10:34:58.6970459770 /etc/homelab/reference/declared/change_management.json\nf 640 167 2026-07-22T10:34:58.6977227780 /etc/homelab/reference/effective/external_services.json\nf 640 167 2026-07-22T10:34:58.6978337330 /etc/homelab/reference/effective/disaster_recovery.json\nf 640 167 2026-07-22T10:34:58.6978512600 /etc/homelab/reference/effective/change_management.json\nf 640 170 2026-07-22T10:34:58.6972985790 /etc/homelab/reference/observed/external_services.json\nf 640 170 2026-07-22T10:34:58.6974490020 /etc/homelab/reference/observed/disaster_recovery.json\nf 640 170 2026-07-22T10:34:58.6974681490 /etc/homelab/reference/observed/change_management.json\nf 640 1780 2026-07-22T11:12:10.9514643580 /etc/homelab/reference/manifest.json\nf 640 199 2026-07-22T11:12:10.9518855430 /etc/homelab/reference/state/current.json\nf 640 262 2026-07-22T11:12:10.9515215670 /etc/homelab/reference/schema/record.schema.json\nf 640 440 2026-07-22T11:03:47.1133590400 /etc/homelab/reference/schema/edit-policy.json\nf 640 593 2026-07-22T11:12:10.9518574460 /etc/homelab/reference/README.md\nCLUSTER_REFERENCE_PUBLICATION_ROOTCAUSE=COMPLETE\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 93da46f..6cb20c1 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,28 +1,549 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=CLUSTER-REFERENCE-CORE-INVENTORY-156 -STATUS=FAIL -RC=1 +COMMAND_ID=CLUSTER-REFERENCE-PUBLICATION-ROOTCAUSE-157 +STATUS=OK +RC=0 HOST=pve01 -MODE=change -COMPONENT=reference-update +MODE=read-only +COMPONENT=cluster-audit REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 -COMMAND_SHA256=4004a5fa6311925347ca61391364d80f4a990a2b0cdd56de0810325d41c4851c +COMMAND_SHA256=4aab341221543b56b703380a36a56615e9c1b8083637dd486284e765228f35cc DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true -CHANGES_MADE=true +CHANGES_MADE=false SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=028bdeec2160ad20c2f0234e6be183a522174af5aac4c345b2be6df78c312e7e -SANITIZED_OUTPUT_SHA256=028bdeec2160ad20c2f0234e6be183a522174af5aac4c345b2be6df78c312e7e +RAW_EVIDENCE_SHA256=d7f4d6f0c5fbb56a2635094046fe7bc28593aa8c1a32a616dba0306229034bfb +SANITIZED_OUTPUT_SHA256=40566782503f51e3d02dabefddb0d16e849da8324d97462c4230bdd109a92e41 OUTPUT_BEGIN - File "", line 1 - import json,sys,pathlib,os,socket; r=pathlib.Path(sys.argv[1]); now=sys.argv[2]; t=pathlib.Path(sys.argv[3]); load=lambda n: json.loads((t/n).read_text(encoding=utf-8) or []); nodes=load(nodes.json); guests=load(guests.json); storage=load(storage.json); records={cluster:[{id:cluster-homelab,hostname:socket.gethostname(),kernel:os.uname().release,pve_version:(t/pveversion.txt).read_text(errors=replace),quorum_status:(t/quorum.txt).read_text(errors=replace)}],nodes:[dict(x,id=node-+str(x.get(node) or x.get(id) or unknown)) for x in nodes],guests:[{id:guest-+str(x.get(vmid)),vmid:x.get(vmid),name:x.get(name),type:x.get(type),node:x.get(node),status:x.get(status),template:x.get(template),uptime:x.get(uptime),cpus:x.get(maxcpu),memory_bytes:x.get(maxmem),disk_bytes:x.get(maxdisk),tags:x.get(tags)} for x in guests],containers:[{id:container-+str(x.get(vmid)),vmid:x.get(vmid),name:x.get(name),node:x.get(node),status:x.get(status),uptime:x.get(uptime),cpus:x.get(maxcpu),memory_bytes:x.get(maxmem),disk_bytes:x.get(maxdisk),tags:x.get(tags)} for x in guests if x.get(type)==lxc],storage:[dict(x,id=storage-+str(x.get(storage) or x.get(id) or unknown)) for x in storage],networks:[{id:network-state,interfaces:load(address.json),routes:load(routes.json)}],dns:[{id:resolver-state,hostname:socket.gethostname(),resolv_conf:(t/resolv.txt).read_text(errors=replace).splitlines()}]}; [(r/observed/(s+.json)).write_text(json.dumps({schema_version:2,section:s,layer:observed,updated_at:now,source:core-internal-collector,records:v},ensure_ascii=False,indent=2)+n,encoding=utf-8) for s,v in records.items()]; state=json.loads((r/state/current.json).read_text()); state.update({status:CORE_INTERNAL_INVENTORY_READY,updated_at:now,core_internal_counts:{s:len(v) for s,v in records.items()}}); (r/state/current.json).write_text(json.dumps(state,ensure_ascii=False,indent=2)+n); [json.loads(p.read_text()) for p in r.rglob(*.json)]; print(CORE_INTERNAL_SECTIONS=+str(len(records))); print(CORE_INTERNAL_RECORDS=+str(sum(len(v) for v in records.values()))) - ^ -SyntaxError: invalid syntax +=== LOCAL_COMMAND_TRACES === +--- COMMAND_ID=CLUSTER-REFERENCE-FOUNDATION-151 --- +FILE=/var/lib/homelab-chat-run/raw/CLUSTER-REFERENCE-FOUNDATION-151-20260722T081210Z.log SIZE=226 MTIME=2026-07-22 11:12:10.952034889 +0300 +1:REFERENCE_ROOT=/etc/homelab/reference +2:REFERENCE_SECTIONS=24 +3:REFERENCE_JSON_FILES=76 +4:REFERENCE_BACKUP=/var/backups/homelab-reference/20260722T081210949547Z +5:REFERENCE_SECRET_POLICY=ENFORCED +6:CLUSTER_REFERENCE_FOUNDATION=COMPLETE +--- COMMAND_ID=CLUSTER-REFERENCE-MODE-SEMANTICS-153 --- +FILE=/var/lib/homelab-chat-run/raw/CLUSTER-REFERENCE-MODE-SEMANTICS-153-20260722T075606Z.log SIZE=8757 MTIME=2026-07-22 10:56:06.971072094 +0300 +1:PREVIOUS_RESULT=VERIFIED +52:36- echo 'RESULT=FAIL' +53:37- echo 'REASON=INVALID_COMMAND_ID' +60:44- echo 'RESULT=FAIL' +61:45- echo 'REASON=INVALID_COMPONENT' +70:54- echo 'RESULT=FAIL' +71:55: echo 'REASON=INVALID_MODE' +86:70- echo 'RESULT=FAIL' +87:71- echo "REASON=REQUIRED_TOOL_MISSING:$TOOL" +96:80- echo 'RESULT=FAIL' +97:81- echo 'REASON=REFERENCE_NOT_READABLE' +102:86- echo 'RESULT=FAIL' +103:87- echo 'REASON=ERROR_REGISTER_NOT_READABLE' +114:132-PREVIOUS_FAILED_RC="$( +128:146:BLOCK_REASON='' +142:160: BLOCK_REASON='EXACT_FAILED_COMMAND_REPEAT' +143:161- RC=73 +144:162- STATUS='BLOCKED' +148:166- echo 'REASON=EXACT_FAILED_COMMAND_REPEAT' +149:167- echo "PREVIOUS_RC=$PREVIOUS_FAILED_RC" +160:178- echo 'RESULT=FAIL' +161:179- echo 'REASON=INVALID_HOMELAB_TIMEOUT' +171:189- RC=$? +174:192- RC=$? +180:198- STATUS='OK' +182:200- STATUS='FAIL' +192:325- "$REFERENCE_SHA256" \ +292:425- echo "STATUS=$STATUS" +293:426- echo "RC=$RC" +297:430- echo 'REFERENCE_REGISTER_CHECK=OK' +298:431- echo "REFERENCE_SHA256=$REFERENCE_SHA256" +354:CLUSTER_REFERENCE_MODE_SEMANTICS=COMPLETE +--- COMMAND_ID=CLUSTER-REFERENCE-FOUNDATION-154 --- +FILE=/var/lib/homelab-chat-run/raw/CLUSTER-REFERENCE-FOUNDATION-154-20260722T080015Z.log SIZE=382 MTIME=2026-07-22 11:00:15.407910031 +0300 +4:REFERENCE_ROOT=/etc/homelab/reference +5:REFERENCE_SECTIONS=24 +6:REFERENCE_BACKUP=/var/backups/homelab-reference/20260722T080015404968Z +7:REFERENCE_SECRET_POLICY=ENFORCED +8:CLUSTER_REFERENCE_FOUNDATION=COMPLETE +--- COMMAND_ID=CLUSTER-REFERENCE-FOUNDATION-155 --- +FILE=/var/lib/homelab-chat-run/raw/CLUSTER-REFERENCE-FOUNDATION-155-20260722T080346Z.log SIZE=454 MTIME=2026-07-22 11:03:47.117217539 +0300 +3:PREVIOUS_STATUS=OK +4:PREVIOUS_RC=0 +5:PREVIOUS_REASON=None +7:REFERENCE_ROOT=/etc/homelab/reference +8:REFERENCE_SECTIONS=24 +9:REFERENCE_JSON_FILES=76 +10:REFERENCE_VALIDATION=PASS +11:REFERENCE_SECRET_POLICY=ENFORCED +12:REFERENCE_BACKUP=/var/backups/homelab-reference/20260722T080346Z +13:CLUSTER_REFERENCE_FOUNDATION=COMPLETE +--- COMMAND_ID=CLUSTER-REFERENCE-CORE-INVENTORY-156 --- +FILE=/var/lib/homelab-chat-run/raw/CLUSTER-REFERENCE-CORE-INVENTORY-156-20260722T082023Z.log SIZE=3515 MTIME=2026-07-22 11:20:25.544704562 +0300 +4:SyntaxError: invalid syntax +=== RUNNER_HELP === +Usage: + homelab-chat-run COMMAND_ID MODE COMPONENT -- command [arguments...] + +MODE: + read-only + verify + change + transport-test + +Environment: + HOMELAB_ALLOW_REPEAT=1 Allow exact repeat of a previously failed command. + HOMELAB_TIMEOUT=SECONDS Optional command timeout. +=== RUNNER_VALIDATION_RULES === +1-#!/usr/bin/env bash +2-set -Eeuo pipefail +3-umask 077 +4- +5-usage() { +6- cat >&2 <<'USAGE' +7-Usage: +8: homelab-chat-run COMMAND_ID MODE COMPONENT -- command [arguments...] +9- +10-MODE: +11: read-only +12: verify +13: change +14: transport-test +15- +16-Environment: +17- HOMELAB_ALLOW_REPEAT=1 Allow exact repeat of a previously failed command. +18- HOMELAB_TIMEOUT=SECONDS Optional command timeout. +19-USAGE +20- exit 64 +21-} +22- +23-[ "$#" -ge 5 ] || usage +24- +25-COMMAND_ID="$1" +26-MODE="$2" +27:COMPONENT="$3" +28-shift 3 +29- +30-[ "${1:-}" = '--' ] || usage +31-shift +32-[ "$#" -ge 1 ] || usage +33- +34-case "$COMMAND_ID" in +35- ''|*[!A-Za-z0-9._-]*) +36- echo 'RESULT=FAIL' +37- echo 'REASON=INVALID_COMMAND_ID' +38- exit 65 +39- ;; +40-esac +41- +42:case "$COMPONENT" in +43- ''|*[!A-Za-z0-9._-]*) +44- echo 'RESULT=FAIL' +45: echo 'REASON=INVALID_COMPONENT' +46- exit 66 +47- ;; +48-esac +49- +50-case "$MODE" in +51: read-only|verify|change|transport-test) +52- ;; +53- *) +54- echo 'RESULT=FAIL' +55: echo 'REASON=INVALID_MODE' +56- exit 67 +57- ;; +58-esac +59- +60-for TOOL in \ +61- awk \ +62- curl \ +63- flock \ +64- python3 \ +65- scp \ +66- sha256sum \ +67- ssh +68-do +69- command -v "$TOOL" >/dev/null 2>&1 || { +70- echo 'RESULT=FAIL' +71- echo "REASON=REQUIRED_TOOL_MISSING:$TOOL" +72- exit 68 +73- } +74-done +75- +76-REFERENCE=/etc/pve/31_HOMELAB_REFERENCE.md +77-ERROR_REGISTER=/etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md +78- +79-[ -r "$REFERENCE" ] || { +80- echo 'RESULT=FAIL' +81- echo 'REASON=REFERENCE_NOT_READABLE' +82- exit 69 +83-} +84- +85-[ -r "$ERROR_REGISTER" ] || { +86- echo 'RESULT=FAIL' +87- echo 'REASON=ERROR_REGISTER_NOT_READABLE' +88- exit 70 +89-} +90- +-- +113- +114-RAW_FILE="$RAW_DIR/${COMMAND_ID}-${STAMP}.log" +115-SANITIZED_FILE="$PUBLIC_DIR/${COMMAND_ID}-${STAMP}.txt" +116-JSON_FILE="$PUBLIC_DIR/${COMMAND_ID}-${STAMP}.json" +117-TXT_FILE="$PUBLIC_DIR/${COMMAND_ID}-${STAMP}-latest.txt" +118- +119-REFERENCE_SHA256="$(sha256sum "$REFERENCE" | awk '{print $1}')" +120-ERROR_REGISTER_SHA256="$(sha256sum "$ERROR_REGISTER" | awk '{print $1}')" +121- +122-COMMAND_SHA256="$( +123- { +124- printf '%s\0' "$MODE" +125: printf '%s\0' "$COMPONENT" +126- printf '%s\0' "$@" +127- } | +128- sha256sum | +129- awk '{print $1}' +130-)" +131- +132-PREVIOUS_FAILED_RC="$( +133- awk -F '\t' -v hash="$COMMAND_SHA256" ' +134- $3 == hash && $4 != "0" { +135- value=$4 +136- } +137- +138- END { +139- print value +140- } +141- ' "$LEDGER" +142-)" +143- +144-RUN_STARTED=true +145-REPEAT_BLOCKED=false +146-BLOCK_REASON='' +147:CHANGES_MADE=false +148- +149-case "$MODE" in +150: change) +151: CHANGES_MADE=true +152- ;; +153-esac +154- +155-if [ -n "$PREVIOUS_FAILED_RC" ] && +156- [ "${HOMELAB_ALLOW_REPEAT:-0}" != 1 ]; then +157- +158- RUN_STARTED=false +159- REPEAT_BLOCKED=true +160- BLOCK_REASON='EXACT_FAILED_COMMAND_REPEAT' +161- RC=73 +162- STATUS='BLOCKED' +163- +164- { +165- echo 'COMMAND_EXECUTION_BLOCKED=yes' +166- echo 'REASON=EXACT_FAILED_COMMAND_REPEAT' +167- echo "PREVIOUS_RC=$PREVIOUS_FAILED_RC" +168- echo "COMMAND_SHA256=$COMMAND_SHA256" +169- echo +170- echo 'Set HOMELAB_ALLOW_REPEAT=1 only after reviewing the previous failure.' +171- } >"$RAW_FILE" +172-else +173- set +e +174- +175- if [ -n "${HOMELAB_TIMEOUT:-}" ]; then +176- case "$HOMELAB_TIMEOUT" in +177- *[!0-9]*|'') +178- echo 'RESULT=FAIL' +179- echo 'REASON=INVALID_HOMELAB_TIMEOUT' +180- exit 74 +181- ;; +182- esac +183- +184- timeout \ +185- --foreground \ +186- --preserve-status \ +-- +310- sha256sum "$SANITIZED_FILE" | +311- awk '{print $1}' +312-)" +313- +314-python3 - \ +315- "$JSON_FILE" \ +316- "$SANITIZED_FILE" \ +317- "$COMMAND_ID" \ +318- "$STATUS" \ +319- "$RC" \ +320- "$HOST_SHORT" \ +321- "$MODE" \ +322: "$COMPONENT" \ +323- "$STARTED_AT" \ +324- "$FINISHED_AT" \ +325- "$REFERENCE_SHA256" \ +326- "$ERROR_REGISTER_SHA256" \ +327- "$COMMAND_SHA256" \ +328- "$RAW_SHA256" \ +329- "$SANITIZED_SHA256" \ +330- "$RUN_STARTED" \ +331- "$REPEAT_BLOCKED" \ +332- "$BLOCK_REASON" \ +333: "$CHANGES_MADE" <<'PY' +334-import json +335-import sys +336-from pathlib import Path +337- +338-( +339- json_path, +340- output_path, +341- command_id, +342- status, +343- rc, +344- host, +345- mode, +346: component, +347- started, +348- finished, +349- reference_sha, +350- error_sha, +351- command_sha, +352- raw_sha, +353- sanitized_sha, +354- run_started, +355- repeat_blocked, +356- block_reason, +357: changes_made, +358-) = sys.argv[1:] +359- +360-output = Path(output_path).read_text( +361- encoding="utf-8", +362- errors="replace", +363-) +364- +365-encoded = output.encode("utf-8") +366-json_limit = 300_000 +367-output_truncated = len(encoded) > json_limit +368- +369-if output_truncated: +370- output_preview = encoded[:json_limit].decode( +371- "utf-8", +372- errors="ignore", +373- ) +374-else: +375- output_preview = output +376- +377-document = { +378- "schema_version": 1, +379- "channel": "homelab-runtime", +380- "command_id": command_id, +381- "status": status, +382- "rc": int(rc), +383- "host": host, +384- "mode": mode, +385: "component": component, +386- "started_at_utc": started, +387- "finished_at_utc": finished, +388- "reference_register_checked": True, +389- "reference_sha256": reference_sha, +390- "error_register_checked": True, +391- "error_register_sha256": error_sha, +392- "command_sha256": command_sha, +393- "duplicate_failed_command_blocked": repeat_blocked == "true", +394- "block_reason": block_reason or None, +395- "execution_started": run_started == "true", +396: "changes_made": changes_made == "true", +397- "sanitized": True, +398- "secrets_included": False, +399- "private_addresses_included": False, +400- "raw_evidence_retained_locally": True, +401- "raw_evidence_sha256": raw_sha, +402- "sanitized_output_sha256": sanitized_sha, +403- "output_truncated_in_json": output_truncated, +404- "full_sanitized_output_url": ( +405- "https://git.gram1.ru/.well-known/" +406- "homelab-runtime/latest.txt" +407- ), +408- "output": output_preview, +409-} +410- +411-Path(json_path).write_text( +412- json.dumps( +413- document, +414- ensure_ascii=False, +415- indent=2, +416- ) +417- + "\n", +418- encoding="utf-8", +419-) +420-PY +421- +422-{ +423- echo 'CHAT_OUTPUT_BEGIN' +424- echo "COMMAND_ID=$COMMAND_ID" +425- echo "STATUS=$STATUS" +426- echo "RC=$RC" +427- echo "HOST=$HOST_SHORT" +428- echo "MODE=$MODE" +429: echo "COMPONENT=$COMPONENT" +430- echo 'REFERENCE_REGISTER_CHECK=OK' +431- echo "REFERENCE_SHA256=$REFERENCE_SHA256" +432- echo 'ERROR_REGISTER_CHECK=OK' +433- echo "ERROR_REGISTER_SHA256=$ERROR_REGISTER_SHA256" +434- echo "COMMAND_SHA256=$COMMAND_SHA256" +435- echo "DUPLICATE_FAILED_COMMAND_BLOCKED=$REPEAT_BLOCKED" +436- echo "EXECUTION_STARTED=$RUN_STARTED" +437: echo "CHANGES_MADE=$CHANGES_MADE" +438- echo 'SANITIZED=yes' +439- echo 'SECRETS_INCLUDED=no' +440- echo 'PRIVATE_ADDRESSES_INCLUDED=no' +441- echo "RAW_EVIDENCE_SHA256=$RAW_SHA256" +442- echo "SANITIZED_OUTPUT_SHA256=$SANITIZED_SHA256" +443- echo 'OUTPUT_BEGIN' +444- cat "$SANITIZED_FILE" +445- echo +446- echo 'OUTPUT_END' +447- echo 'CHAT_OUTPUT_END' +448-} >"$TXT_FILE" +449- +450-chmod 0600 \ +451- "$RAW_FILE" \ +452- "$SANITIZED_FILE" \ +453- "$JSON_FILE" \ +454- "$TXT_FILE" +455- +456-TARGET=debian@[PRIVATE_IP] +457-REMOTE_PREFIX="/tmp/homelab-runtime-${COMMAND_ID}-$$" +458- +459-SSH_OPTS=( +460- -o BatchMode=yes +461- -o ConnectTimeout=10 +462- -o ConnectionAttempts=1 +463- -o StrictHostKeyChecking=yes +464- -o UserKnownHostsFile=[SENSITIVE_PATH] +465-) +466- +467-scp \ +468- -q \ +469- "${SSH_OPTS[@]}" \ +470- "$JSON_FILE" \ +471- "${TARGET}:${REMOTE_PREFIX}.json" +472- +-- +518- echo 'REASON=PUBLIC_RUNTIME_RESULT_NOT_VISIBLE' +519- echo "COMMAND_ID=$COMMAND_ID" +520- echo "LOCAL_RAW_FILE=$RAW_FILE" +521- exit 75 +522-fi +523- +524-printf '%s\t%s\t%s\t%s\t%s\t%s\n' \ +525- "$FINISHED_AT" \ +526- "$COMMAND_ID" \ +527- "$COMMAND_SHA256" \ +528- "$RC" \ +529- "$STATUS" \ +530: "$COMPONENT" >>"$LEDGER" +531- +532-echo +533-echo '===== HOMELAB CHAT RUN COMPLETE =====' +534-echo "COMMAND_ID=$COMMAND_ID" +535-echo "STATUS=$STATUS" +536-echo "RC=$RC" +537-echo 'REFERENCE_REGISTER_CHECK=OK' +538-echo 'ERROR_REGISTER_CHECK=OK' +539-echo "DUPLICATE_FAILED_COMMAND_BLOCKED=$REPEAT_BLOCKED" +540-echo 'GITEA_PUBLICATION=OK' +541-echo "PUBLIC_URL=$PUBLIC_URL" +542-echo "LOCAL_RAW_FILE=$RAW_FILE" +543-echo 'SHELL_SESSION_PRESERVED=yes' +544- +545-exit "$RC" +=== PARTIAL_REFERENCE_STATE === +d 750 4096 2026-07-22T10:34:58.6964026990 /etc/homelab/reference/public +d 750 4096 2026-07-22T10:34:58.6964026990 /etc/homelab/reference/runbooks +d 750 4096 2026-07-22T10:34:58.6964026990 /etc/homelab/reference/snapshots +d 750 4096 2026-07-22T10:34:58.6970459770 /etc/homelab/reference/declared +d 750 4096 2026-07-22T10:34:58.6974681490 /etc/homelab/reference/observed +d 750 4096 2026-07-22T10:34:58.6978512600 /etc/homelab/reference/effective +d 750 4096 2026-07-22T10:34:58.6978691140 /etc/homelab/reference +d 750 4096 2026-07-22T10:34:58.6978834310 /etc/homelab/reference/state +d 750 4096 2026-07-22T11:03:47.1132880790 /etc/homelab/reference/schema +f 640 153 2026-07-22T10:34:58.6968014830 /etc/homelab/reference/declared/dns.json +f 640 153 2026-07-22T10:34:58.6976137370 /etc/homelab/reference/effective/dns.json +f 640 155 2026-07-22T10:34:58.6966969670 /etc/homelab/reference/declared/nodes.json +f 640 155 2026-07-22T10:34:58.6975198720 /etc/homelab/reference/effective/nodes.json +f 640 156 2026-07-22T10:34:58.6967158380 /etc/homelab/reference/declared/guests.json +f 640 156 2026-07-22T10:34:58.6969484530 /etc/homelab/reference/declared/access.json +f 640 156 2026-07-22T10:34:58.6969644820 /etc/homelab/reference/declared/owners.json +f 640 156 2026-07-22T10:34:58.6971898020 /etc/homelab/reference/observed/dns.json +f 640 156 2026-07-22T10:34:58.6975362380 /etc/homelab/reference/effective/guests.json +f 640 156 2026-07-22T10:34:58.6977531080 /etc/homelab/reference/effective/access.json +f 640 156 2026-07-22T10:34:58.6977693510 /etc/homelab/reference/effective/owners.json +f 640 157 2026-07-22T10:34:58.6966738300 /etc/homelab/reference/declared/cluster.json +f 640 157 2026-07-22T10:34:58.6968176220 /etc/homelab/reference/declared/domains.json +f 640 157 2026-07-22T10:34:58.6968507170 /etc/homelab/reference/declared/ingress.json +f 640 157 2026-07-22T10:34:58.6968665330 /etc/homelab/reference/declared/storage.json +f 640 157 2026-07-22T10:34:58.6968815380 /etc/homelab/reference/declared/backups.json +f 640 157 2026-07-22T10:34:58.6969804230 /etc/homelab/reference/declared/vendors.json +f 640 157 2026-07-22T10:34:58.6975026350 /etc/homelab/reference/effective/cluster.json +f 640 157 2026-07-22T10:34:58.6976296950 /etc/homelab/reference/effective/domains.json +f 640 157 2026-07-22T10:34:58.6976621460 /etc/homelab/reference/effective/ingress.json +f 640 157 2026-07-22T10:34:58.6976775090 /etc/homelab/reference/effective/storage.json +f 640 157 2026-07-22T10:34:58.6976922400 /etc/homelab/reference/effective/backups.json +f 640 157 2026-07-22T10:34:58.6977857550 /etc/homelab/reference/effective/vendors.json +f 640 158 2026-07-22T10:34:58.6967496500 /etc/homelab/reference/declared/services.json +f 640 158 2026-07-22T10:34:58.6967835450 /etc/homelab/reference/declared/networks.json +f 640 158 2026-07-22T10:34:58.6970122440 /etc/homelab/reference/declared/security.json +f 640 158 2026-07-22T10:34:58.6970626700 /etc/homelab/reference/declared/runbooks.json +f 640 158 2026-07-22T10:34:58.6970953450 /etc/homelab/reference/observed/nodes.json +f 640 158 2026-07-22T10:34:58.6975661010 /etc/homelab/reference/effective/services.json +f 640 158 2026-07-22T10:34:58.6975973330 /etc/homelab/reference/effective/networks.json +f 640 158 2026-07-22T10:34:58.6978179760 /etc/homelab/reference/effective/security.json +f 640 158 2026-07-22T10:34:58.6978691140 /etc/homelab/reference/effective/runbooks.json +f 640 159 2026-07-22T10:34:58.6969966040 /etc/homelab/reference/declared/contracts.json +f 640 159 2026-07-22T10:34:58.6971108280 /etc/homelab/reference/observed/guests.json +f 640 159 2026-07-22T10:34:58.6973313430 /etc/homelab/reference/observed/access.json +f 640 159 2026-07-22T10:34:58.6973523380 /etc/homelab/reference/observed/owners.json +f 640 159 2026-07-22T10:34:58.6978023050 /etc/homelab/reference/effective/contracts.json +f 640 160 2026-07-22T10:34:58.6967330550 /etc/homelab/reference/declared/containers.json +f 640 160 2026-07-22T10:34:58.6968977560 /etc/homelab/reference/declared/monitoring.json +f 640 160 2026-07-22T10:34:58.6970793370 /etc/homelab/reference/observed/cluster.json +f 640 160 2026-07-22T10:34:58.6972051460 /etc/homelab/reference/observed/domains.json +f 640 160 2026-07-22T10:34:58.6972368890 /etc/homelab/reference/observed/ingress.json +f 640 160 2026-07-22T10:34:58.6972525750 /etc/homelab/reference/observed/storage.json +f 640 160 2026-07-22T10:34:58.6972679110 /etc/homelab/reference/observed/backups.json +f 640 160 2026-07-22T10:34:58.6973681500 /etc/homelab/reference/observed/vendors.json +f 640 160 2026-07-22T10:34:58.6975511580 /etc/homelab/reference/effective/containers.json +f 640 160 2026-07-22T10:34:58.6977071740 /etc/homelab/reference/effective/monitoring.json +f 640 161 2026-07-22T10:34:58.6971427070 /etc/homelab/reference/observed/services.json +f 640 161 2026-07-22T10:34:58.6971740390 /etc/homelab/reference/observed/networks.json +f 640 161 2026-07-22T10:34:58.6974304020 /etc/homelab/reference/observed/security.json +f 640 161 2026-07-22T10:34:58.6974849460 /etc/homelab/reference/observed/runbooks.json +f 640 162 2026-07-22T10:34:58.6967668110 /etc/homelab/reference/declared/applications.json +f 640 162 2026-07-22T10:34:58.6968346450 /etc/homelab/reference/declared/certificates.json +f 640 162 2026-07-22T10:34:58.6969325220 /etc/homelab/reference/declared/dependencies.json +f 640 162 2026-07-22T10:34:58.6974142710 /etc/homelab/reference/observed/contracts.json +f 640 162 2026-07-22T10:34:58.6975824200 /etc/homelab/reference/effective/applications.json +f 640 162 2026-07-22T10:34:58.6976463310 /etc/homelab/reference/effective/certificates.json +f 640 162 2026-07-22T10:34:58.6977380340 /etc/homelab/reference/effective/dependencies.json +f 640 163 2026-07-22T10:34:58.6971273900 /etc/homelab/reference/observed/containers.json +f 640 163 2026-07-22T10:34:58.6972830790 /etc/homelab/reference/observed/monitoring.json +f 640 165 2026-07-22T10:34:58.6971583070 /etc/homelab/reference/observed/applications.json +f 640 165 2026-07-22T10:34:58.6972204840 /etc/homelab/reference/observed/certificates.json +f 640 165 2026-07-22T10:34:58.6973154820 /etc/homelab/reference/observed/dependencies.json +f 640 167 2026-07-22T10:34:58.6969164820 /etc/homelab/reference/declared/external_services.json +f 640 167 2026-07-22T10:34:58.6970278950 /etc/homelab/reference/declared/disaster_recovery.json +f 640 167 2026-07-22T10:34:58.6970459770 /etc/homelab/reference/declared/change_management.json +f 640 167 2026-07-22T10:34:58.6977227780 /etc/homelab/reference/effective/external_services.json +f 640 167 2026-07-22T10:34:58.6978337330 /etc/homelab/reference/effective/disaster_recovery.json +f 640 167 2026-07-22T10:34:58.6978512600 /etc/homelab/reference/effective/change_management.json +f 640 170 2026-07-22T10:34:58.6972985790 /etc/homelab/reference/observed/external_services.json +f 640 170 2026-07-22T10:34:58.6974490020 /etc/homelab/reference/observed/disaster_recovery.json +f 640 170 2026-07-22T10:34:58.6974681490 /etc/homelab/reference/observed/change_management.json +f 640 1780 2026-07-22T11:12:10.9514643580 /etc/homelab/reference/manifest.json +f 640 199 2026-07-22T11:12:10.9518855430 /etc/homelab/reference/state/current.json +f 640 262 2026-07-22T11:12:10.9515215670 /etc/homelab/reference/schema/record.schema.json +f 640 440 2026-07-22T11:03:47.1133590400 /etc/homelab/reference/schema/edit-policy.json +f 640 593 2026-07-22T11:12:10.9518574460 /etc/homelab/reference/README.md +CLUSTER_REFERENCE_PUBLICATION_ROOTCAUSE=COMPLETE OUTPUT_END CHAT_OUTPUT_END