homelab-admin/homelab-public-context
1
0
Fork 0
Code Activity
Files
038e8d256185e7e49146f5a14d2b39a1fa55585c
homelab-public-context/runtime/latest.json
T

39 lines
60 KiB
JSON
Raw Blame History

{
"schema_version": 1,
"channel": "homelab-runtime",
"command_id": "HOMELAB-CLUSTER-ADMIN-UPDATES-V12-EXACT-SOURCES-READ-ONLY-20260805T112600Z",
"status": "OK",
"rc": 0,
"host": "pve01",
"mode": "verify",
"component": "homelab-cluster-admin-update-visibility",
"started_at_utc": "2026-08-05T11:29:26Z",
"finished_at_utc": "2026-08-05T11:29:27Z",
"reference_register_checked": true,
"reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
"error_register_checked": true,
"error_register_sha256": "24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83",
"command_sha256": "4d9db09bb3348dda0505b55ec6f7557acc4d713b63fd53723330fc518f9d5b2d",
"duplicate_failed_command_blocked": false,
"block_reason": null,
"execution_started": true,
"change_declared": false,
"result_contract_valid": true,
"result_contract_status": null,
"result_contract_error": null,
"command_rc": 0,
"changes_made": false,
"rollback_started": false,
"rollback_restored": null,
"mutation_outcome": "NO_MUTATION",
"sanitized": true,
"secrets_included": false,
"private_addresses_included": false,
"raw_evidence_retained_locally": true,
"raw_evidence_sha256": "fd1c004e8835b870b2232c2e20a1735688d171426ed97840dfbfd6d86f9c3852",
"sanitized_output_sha256": "fd1c004e8835b870b2232c2e20a1735688d171426ed97840dfbfd6d86f9c3852",
"output_truncated_in_json": false,
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
"output": "HOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V12-EXACT-SOURCES-READ-ONLY-20260805T112600Z\",\"command_rc\":0,\"control_plane_error\":null,\"mutation_outcome\":\"NO_MUTATION\",\"primary_script_count\":4,\"primary_script_sha256\":{\"homelab-cluster-admin-incident-engine-backup-run\":\"a79c2a72b1fbdc13506b6db8b0ea790376dc20db8003961c5b44935d28f0c5ae\",\"homelab-cluster-admin-inventory-sync\":\"00814787de40176e025097b353a67c26edcd298a4126d2375333e83df4f37170\",\"homelab-cluster-admin-pull\":\"00913dbf4ef82cddd82850be7f2445924b7e75e537e7e5f6cef5785b05307d23\",\"homelab-cluster-admin-webpanel-sync\":\"2c57cca503cb0734e5114c2ab46b892a810bdc40c2baa7387b4094373db19276\"},\"publication_anchor_count\":48,\"rc\":0,\"ready_for_atomic_transaction\":true,\"report_bytes\":53302,\"report_sha256\":\"aa1895efb8a2578f48be1d3bba2776aca56aacaa737a58b0deac35bc5f45d807\",\"rollback_restored\":false,\"rollback_started\":false,\"status\":\"OK\",\"unit_count\":8,\"unit_sha256\":{\"homelab-cluster-admin-incident-engine-backup.service\":\"bc7f9010fd6e97dff3bbedd2fd2088d65836a1a842cbd6d465163251e2a7db31\",\"homelab-cluster-admin-incident-engine-backup.timer\":\"7b34396b19b8d9139e3f59d7a03a5bbfe08440c95d6a1d85031057665a724d54\",\"homelab-cluster-admin-inventory-sync.service\":\"5237b68c736e77ff0108c77321bbe9b9cb7285e3a23d135361a43cd2fcb74264\",\"homelab-cluster-admin-inventory-sync.timer\":\"4c9f51447b5d8f75fa05b494390e74089b6a2880ce899325643d47be4ce2180e\",\"homelab-cluster-admin-pull.service\":\"4a30f00113b7c68478f1cd2c8be959aed1931c55a0c7f7c99fda06cb742b914e\",\"homelab-cluster-admin-pull.timer\":\"5e4936d1379c96088b17816a7aba42d77051009bd0938a253b3487bcb3b658a6\",\"homelab-cluster-admin-webpanel-sync.service\":\"d8091d4bb52513929cd7df742532977f70e08ee60ab8762a8f027e5863612c3b\",\"homelab-cluster-admin-webpanel-sync.timer\":\"ef420cea1b506ff7ea6dbd02c1b19167e5771030f4f52eb850aa05b7e3eed094\"},\"v11_exact_result_identity_confirmed\":true,\"v11_output_sha256\":\"5392567d03d9794b8d677b56843b620f0f15f5515e1b6d7707ea5ec3b602fda5\",\"v11_runner_wrapper_matches_document\":false,\"version\":1}\n{\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V12-EXACT-SOURCES-READ-ONLY-20260805T112600Z\",\"mutation_outcome\":\"NO_MUTATION\",\"primary_scripts\":[{\"assignments\":{\"APP\":\"\\\"cluster-admin-incident-engine\\\"\",\"ARCHIVE\":\"\\\"${JOB_ID}.tar.gz\\\"\",\"BACKUP_PATH_REL\":\"\\\"$(homelab-backup-path job a \\\"$APP\\\" \\\"$SHORT_TS\\\")\\\"\",\"DATE\":\"\\\"$(date -u +%Y%m%dT%H%M%SZ)\\\"\",\"DUMP_SIZE\":\"\\\"$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"sudo -n rm -f \\\\\\\"$REMOTE_DUMP\\\\\\\"; sudo -n -u clusteradmin pg_dump --format=custom --no-owner --no-privileges --file=\\\\\\\"$REMOTE_DUMP\\\\\\\" clusteradmin; sudo -n test -s \\\\\\\"$REMOTE_DUMP\\\\\\\"; sudo -n -u clusteradmin pg_restore --list \\\\\\\"$REMOTE_DUMP\\\\\\\" >/dev/null; sudo -n chown debian:debian \\\\\\\"$REMOTE_DUMP\\\\\\\"; sudo -n chmod 600 \\\\\\\"$REMOTE_DUMP\\\\\\\"; stat -c %s \\\\\\\"$REMOTE_DUMP\\\\\\\"\\\")\\\"\",\"FILES\":\"\\\"$(find \\\"$STAGE/payload\\\" -type f | wc -l | tr -d \\\" \\\")\\\"\",\"JOB_ID\":\"\\\"homelab-${NODE}-app-${APP}-${DATE}\\\"\",\"NODE\":\"\\\"$(hostname -s)\\\"\",\"REMOTE\":\"\\\"debian@<PRIVATE_IPV4>\\\"\",\"REMOTE_DUMP\":\"\\\"/tmp/${JOB_ID}.pgdump\\\"\",\"SHORT_TS\":\"\\\"${DATE:2:6}-${DATE:9:6}\\\"\",\"STAGE\":\"\\\"${STAGING_ROOT}/${JOB_ID}\\\"\"},\"bytes\":2640,\"commands\":[{\"line\":15,\"text\":\"DUMP_SIZE=\\\"$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"sudo -n rm -f \\\\\\\"$REMOTE_DUMP\\\\\\\"; sudo -n -u clusteradmin pg_dump --format=custom --no-owner --no-privileges --file=\\\\\\\"$REMOTE_DUMP\\\\\\\" clusteradmin; sudo -n test -s \\\\\\\"$REMOTE_DUMP\\\\\\\"; sudo -n -u clusteradmin pg_restore --list \\\\\\\"$REMOTE_DUMP\\\\\\\" >/dev/null; sudo -n chown debian:debian \\\\\\\"$REMOTE_DUMP\\\\\\\"; sudo -n chmod 600 \\\\\\\"$REMOTE_DUMP\\\\\\\"; stat -c %s \\\\\\\"$REMOTE_DUMP\\\\\\\"\\\")\\\"\"},{\"line\":17,\"text\":\"scp -q -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE:$REMOTE_DUMP\\\" \\\"$STAGE/payload/clusteradmin.pgdump\\\"\"},{\"line\":18,\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"rm -f \\\\\\\"$REMOTE_DUMP\\\\\\\"\\\"\"},{\"line\":20,\"text\":\"python3 -c \\\"import sys; assert open(sys.argv[1],\\\\\\\"rb\\\\\\\").read(5)==b\\\\\\\"PGDMP\\\\\\\"\\\" \\\"$STAGE/payload/clusteradmin.pgdump\\\"\"},{\"line\":21,\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"sudo -n tar -C / -czf - opt/cluster-admin-incident-engine etc/cluster-admin-incident-engine etc/systemd/system/cluster-admin-incident-engine.service etc/systemd/system/cluster-admin-incident-engine-collector.service etc/systemd/system/cluster-admin-incident-engine-collector.timer var/lib/cluster-admin-incident-engine\\\" > \\\"$STAGE/runtime.tgz\\\"\"},{\"line\":25,\"text\":\"<REDACTED_SECRET_LINE>\"}],\"gid\":0,\"line_count\":28,\"mode\":\"0o750\",\"path\":\"/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run\",\"path_literals\":[\"/${JOB_ID\",\"/dev/null\",\"/etc/cluster-admin-incident-engine\",\"/etc/homelab-backup.env\",\"/opt/cluster-admin-incident-engine\",\"/tmp/${JOB_ID}.pgdump\",\"/usr/bin/env\",\"/var/lib/cluster-admin-incident-engine\"],\"publication_lines\":[{\"line\":15,\"text\":\"DUMP_SIZE=\\\"$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"sudo -n rm -f \\\\\\\"$REMOTE_DUMP\\\\\\\"; sudo -n -u clusteradmin pg_dump --format=custom --no-owner --no-privileges --file=\\\\\\\"$REMOTE_DUMP\\\\\\\" clusteradmin; sudo -n test -s \\\\\\\"$REMOTE_DUMP\\\\\\\"; sudo -n -u clusteradmin pg_restore --list \\\\\\\"$REMOTE_DUMP\\\\\\\" >/dev/null; sudo -n chown debian:debian \\\\\\\"$REMOTE_DUMP\\\\\\\"; sudo -n chmod 600 \\\\\\\"$REMOTE_DUMP\\\\\\\"; stat -c %s \\\\\\\"$REMOTE_DUMP\\\\\\\"\\\")\\\"\"},{\"line\":17,\"text\":\"scp -q -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE:$REMOTE_DUMP\\\" \\\"$STAGE/payload/clusteradmin.pgdump\\\"\"},{\"line\":18,\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"rm -f \\\\\\\"$REMOTE_DUMP\\\\\\\"\\\"\"},{\"line\":21,\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"sudo -n tar -C / -czf - opt/cluster-admin-incident-engine etc/cluster-admin-incident-engine etc/systemd/system/cluster-admin-incident-engine.service etc/systemd/system/cluster-admin-incident-engine-collector.service etc/systemd/system/cluster-admin-incident-engine-collector.timer var/lib/cluster-admin-incident-engine\\\" > \\\"$STAGE/runtime.tgz\\\"\"},{\"line\":25,\"text\":\"<REDACTED_SECRET_LINE>\"}],\"referenced_files\":[],\"sha256\":\"a79c2a72b1fbdc13506b6db8b0ea790376dc20db8003961c5b44935d28f0c5ae\",\"source\":[{\"line\":1,\"text\":\"#!/usr/bin/env bash\"},{\"line\":2,\"text\":\"set -Eeuo pipefail\"},{\"line\":3,\"text\":\"source /etc/homelab-backup.env\"},{\"line\":4,\"text\":\"APP=\\\"cluster-admin-incident-engine\\\"\"},{\"line\":5,\"text\":\"REMOTE=\\\"debian@<PRIVATE_IPV4>\\\"\"},{\"line\":6,\"text\":\"NODE=\\\"$(hostname -s)\\\"\"},{\"line\":7,\"text\":\"DATE=\\\"$(date -u +%Y%m%dT%H%M%SZ)\\\"\"},{\"line\":8,\"text\":\"export JOB_ID=\\\"homelab-${NODE}-app-${APP}-${DATE}\\\"\"},{\"line\":9,\"text\":\"SHORT_TS=\\\"${DATE:2:6}-${DATE:9:6}\\\"\"},{\"line\":10,\"text\":\"export BACKUP_PATH_REL=\\\"$(homelab-backup-path job a \\\"$APP\\\" \\\"$SHORT_TS\\\")\\\"\"},{\"line\":11,\"text\":\"STAGE=\\\"${STAGING_ROOT}/${JOB_ID}\\\"\"},{\"line\":12,\"text\":\"ARCHIVE=\\\"${JOB_ID}.tar.gz\\\"\"},{\"line\":13,\"text\":\"REMOTE_DUMP=\\\"/tmp/${JOB_ID}.pgdump\\\"\"},{\"line\":14,\"text\":\"mkdir -p \\\"$STAGE/payload\\\"\"},{\"line\":15,\"text\":\"DUMP_SIZE=\\\"$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"sudo -n rm -f \\\\\\\"$REMOTE_DUMP\\\\\\\"; sudo -n -u clusteradmin pg_dump --format=custom --no-owner --no-privileges --file=\\\\\\\"$REMOTE_DUMP\\\\\\\" clusteradmin; sudo -n test -s \\\\\\\"$REMOTE_DUMP\\\\\\\"; sudo -n -u clusteradmin pg_restore --list \\\\\\\"$REMOTE_DUMP\\\\\\\" >/dev/null; sudo -n chown debian:debian \\\\\\\"$REMOTE_DUMP\\\\\\\"; sudo -n chmod 600 \\\\\\\"$REMOTE_DUMP\\\\\\\"; stat -c %s \\\\\\\"$REMOTE_DUMP\\\\\\\"\\\")\\\"\"},{\"line\":16,\"text\":\"[ \\\"$DUMP_SIZE\\\" -gt 128 ]\"},{\"line\":17,\"text\":\"scp -q -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE:$REMOTE_DUMP\\\" \\\"$STAGE/payload/clusteradmin.pgdump\\\"\"},{\"line\":18,\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"rm -f \\\\\\\"$REMOTE_DUMP\\\\\\\"\\\"\"},{\"line\":19,\"text\":\"test -s \\\"$STAGE/payload/clusteradmin.pgdump\\\"\"},{\"line\":20,\"text\":\"python3 -c \\\"import sys; assert open(sys.argv[1],\\\\\\\"rb\\\\\\\").read(5)==b\\\\\\\"PGDMP\\\\\\\"\\\" \\\"$STAGE/payload/clusteradmin.pgdump\\\"\"},{\"line\":21,\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"sudo -n tar -C / -czf - opt/cluster-admin-incident-engine etc/cluster-admin-incident-engine etc/systemd/system/cluster-admin-incident-engine.service etc/systemd/system/cluster-admin-incident-engine-collector.service etc/systemd/system/cluster-admin-incident-engine-collector.timer var/lib/cluster-admin-incident-engine\\\" > \\\"$STAGE/runtime.tgz\\\"\"},{\"line\":22,\"text\":\"tar -C \\\"$STAGE/payload\\\" -xzf \\\"$STAGE/runtime.tgz\\\"\"},{\"line\":23,\"text\":\"rm -f \\\"$STAGE/runtime.tgz\\\"\"},{\"line\":24,\"text\":\"FILES=\\\"$(find \\\"$STAGE/payload\\\" -type f | wc -l | tr -d \\\" \\\")\\\"\"},{\"line\":25,\"text\":\"<REDACTED_SECRET_LINE>\"},{\"line\":26,\"text\":\"tar -C \\\"$STAGE/payload\\\" -czf \\\"$STAGE/$ARCHIVE\\\" .\"},{\"line\":27,\"text\":\"homelab-stage-2cloud-flat \\\"$APP\\\" \\\"$STAGE\\\" \\\"$ARCHIVE\\\"\"},{\"line\":28,\"text\":\"echo CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK\"}],\"uid\":0},{\"assignments\":{\"LOCAL\":\"/var/lib/homelab-cluster-admin-export/inventory.json\",\"LOCAL_SHA\":\"\\\"$(sha256sum \\\"$LOCAL\\\" | cut -d\\\" \\\" -f1)\\\"\",\"LOCK\":\"/run/homelab-cluster-admin-inventory-sync.lock\",\"REMOTE\":\"debian@<PRIVATE_IPV4>\",\"REMOTE_FILE\":\"/var/lib/cluster-admin-incident-engine/incoming/inventory.json\",\"REMOTE_SHA\":\"\\\"$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"sudo -n sha256sum $REMOTE_FILE\\\" | cut -d\\\" \\\" -f1)\\\"\",\"REMOTE_TMP\":\"\\\"/tmp/cluster-admin-inventory-${RANDOM}-$$.json\\\"\"},\"bytes\":1301,\"commands\":[{\"line\":10,\"text\":\"/usr/bin/python3 /usr/local/sbin/homelab-cluster-admin-inventory-export\"},{\"line\":11,\"text\":\"/usr/bin/python3 -m json.tool \\\"$LOCAL\\\" >/dev/null\"},{\"line\":13,\"text\":\"scp -q -o BatchMode=yes -o ConnectTimeout=8 \\\"$LOCAL\\\" \\\"$REMOTE:$REMOTE_TMP\\\"\"},{\"line\":14,\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"python3 -m json.tool $REMOTE_TMP >/dev/null\\\"\"},{\"line\":15,\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"sudo -n install -d -o clusteradmin -g clusteradmin -m 750 /var/lib/cluster-admin-incident-engine/incoming && sudo -n install -o clusteradmin -g clusteradmin -m 640 $REMOTE_TMP $REMOTE_FILE\\\"\"},{\"line\":16,\"text\":\"REMOTE_SHA=\\\"$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"sudo -n sha256sum $REMOTE_FILE\\\" | cut -d\\\" \\\" -f1)\\\"\"},{\"line\":19,\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"rm -f $REMOTE_TMP\\\"\"}],\"gid\":0,\"line_count\":22,\"mode\":\"0o750\",\"path\":\"/usr/local/sbin/homelab-cluster-admin-inventory-sync\",\"path_literals\":[\"/dev/null\",\"/run/homelab-cluster-admin-inventory-sync.lock\",\"/tmp/cluster-admin-inventory-${RANDOM}-$$.json\",\"/usr/bin/env\",\"/usr/bin/python3\",\"/usr/local/sbin/homelab-cluster-admin-inventory-export\",\"/var/lib/cluster-admin-incident-engine/incoming\",\"/var/lib/cluster-admin-incident-engine/incoming/inventory.json\",\"/var/lib/homelab-cluster-admin-export/inventory.json\"],\"publication_lines\":[{\"line\":3,\"text\":\"LOCK=/run/homelab-cluster-admin-inventory-sync.lock\"},{\"line\":6,\"text\":\"LOCAL=/var/lib/homelab-cluster-admin-export/inventory.json\"},{\"line\":8,\"text\":\"REMOTE_FILE=/var/lib/cluster-admin-incident-engine/incoming/inventory.json\"},{\"line\":9,\"text\":\"REMOTE_TMP=\\\"/tmp/cluster-admin-inventory-${RANDOM}-$$.json\\\"\"},{\"line\":10,\"text\":\"/usr/bin/python3 /usr/local/sbin/homelab-cluster-admin-inventory-export\"},{\"line\":11,\"text\":\"/usr/bin/python3 -m json.tool \\\"$LOCAL\\\" >/dev/null\"},{\"line\":13,\"text\":\"scp -q -o BatchMode=yes -o ConnectTimeout=8 \\\"$LOCAL\\\" \\\"$REMOTE:$REMOTE_TMP\\\"\"},{\"line\":14,\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"python3 -m json.tool $REMOTE_TMP >/dev/null\\\"\"},{\"line\":15,\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"sudo -n install -d -o clusteradmin -g clusteradmin -m 750 /var/lib/cluster-admin-incident-engine/incoming && sudo -n install -o clusteradmin -g clusteradmin -m 640 $REMOTE_TMP $REMOTE_FILE\\\"\"},{\"line\":16,\"text\":\"REMOTE_SHA=\\\"$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"sudo -n sha256sum $REMOTE_FILE\\\" | cut -d\\\" \\\" -f1)\\\"\"},{\"line\":19,\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"rm -f $REMOTE_TMP\\\"\"},{\"line\":20,\"text\":\"echo LOCAL_INVENTORY_SHA256=\\\"$LOCAL_SHA\\\"\"},{\"line\":21,\"text\":\"echo REMOTE_INVENTORY_SHA256=\\\"$REMOTE_SHA\\\"\"},{\"line\":22,\"text\":\"echo CLUSTER_ADMIN_INVENTORY_SYNC=OK\"}],\"referenced_files\":[{\"bytes\":5290,\"is_dir\":false,\"is_file\":true,\"literal\":\"/usr/local/sbin/homelab-cluster-admin-inventory-export\",\"mode\":\"0o750\",\"resolved\":\"/usr/local/sbin/homelab-cluster-admin-inventory-export\",\"sha256\":\"725b7ad80f6a0ce11096bc4d3f8ae615865f6031a70123dd1890b0a3e114c04d\"}],\"sha256\":\"00814787de40176e025097b353a67c26edcd298a4126d2375333e83df4f37170\",\"source\":[{\"line\":1,\"text\":\"#!/usr/bin/env bash\"},{\"line\":2,\"text\":\"set -Eeuo pipefail\"},{\"line\":3,\"text\":\"LOCK=/run/homelab-cluster-admin-inventory-sync.lock\"},{\"line\":4,\"text\":\"exec 9>\\\"$LOCK\\\"\"},{\"line\":5,\"text\":\"flock -w 30 9\"},{\"line\":6,\"text\":\"LOCAL=/var/lib/homelab-cluster-admin-export/inventory.json\"},{\"line\":7,\"text\":\"REMOTE=debian@<PRIVATE_IPV4>\"},{\"line\":8,\"text\":\"REMOTE_FILE=/var/lib/cluster-admin-incident-engine/incoming/inventory.json\"},{\"line\":9,\"text\":\"REMOTE_TMP=\\\"/tmp/cluster-admin-inventory-${RANDOM}-$$.json\\\"\"},{\"line\":10,\"text\":\"/usr/bin/python3 /usr/local/sbin/homelab-cluster-admin-inventory-export\"},{\"line\":11,\"text\":\"/usr/bin/python3 -m json.tool \\\"$LOCAL\\\" >/dev/null\"},{\"line\":12,\"text\":\"LOCAL_SHA=\\\"$(sha256sum \\\"$LOCAL\\\" | cut -d\\\" \\\" -f1)\\\"\"},{\"line\":13,\"text\":\"scp -q -o BatchMode=yes -o ConnectTimeout=8 \\\"$LOCAL\\\" \\\"$REMOTE:$REMOTE_TMP\\\"\"},{\"line\":14,\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"python3 -m json.tool $REMOTE_TMP >/dev/null\\\"\"},{\"line\":15,\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"sudo -n install -d -o clusteradmin -g clusteradmin -m 750 /var/lib/cluster-admin-incident-engine/incoming && sudo -n install -o clusteradmin -g clusteradmin -m 640 $REMOTE_TMP $REMOTE_FILE\\\"\"},{\"line\":16,\"text\":\"REMOTE_SHA=\\\"$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"sudo -n sha256sum $REMOTE_FILE\\\" | cut -d\\\" \\\" -f1)\\\"\"},{\"line\":17,\"text\":\"[ -n \\\"$LOCAL_SHA\\\" ]\"},{\"line\":18,\"text\":\"[ \\\"$LOCAL_SHA\\\" = \\\"$REMOTE_SHA\\\" ]\"},{\"line\":19,\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"rm -f $REMOTE_TMP\\\"\"},{\"line\":20,\"text\":\"echo LOCAL_INVENTORY_SHA256=\\\"$LOCAL_SHA\\\"\"},{\"line\":21,\"text\":\"echo REMOTE_INVENTORY_SHA256=\\\"$REMOTE_SHA\\\"\"},{\"line\":22,\"text\":\"echo CLUSTER_ADMIN_INVENTORY_SYNC=OK\"}],\"uid\":0},{\"assignments\":{\"BAD\":\"$((OBS_BAD+FULL_BAD)); [ \\\"$RC\\\" -eq 0 ] || BAD=$((BAD+1))\",\"FULL\":\"$(printf \\\"%s\\\\n\\\" \\\"$REMOTE\\\" | awk '/^__FULL__$/{f=1;next}f')\",\"FULL_STATUS\":\"$([ \\\"$FULL_BAD\\\" -eq 0 ] && [ \\\"$RC\\\" -eq 0 ] && echo OK || echo WARN)\",\"HEALTH_ROOT\":\"/var/lib/homelab-health\",\"OBS\":\"$(printf \\\"%s\\\\n\\\" \\\"$REMOTE\\\" | awk '/^__OBS__$/{f=1;next}/^__FULL__$/{f=0}f')\",\"OBS_BAD\":\"0; FULL_BAD=0\",\"OBS_STATUS\":\"$([ \\\"$OBS_BAD\\\" -eq 0 ] && [ \\\"$RC\\\" -eq 0 ] && echo OK || echo WARN)\",\"PROBE_BAD\":\"$(printf \\\"%s\\\\n\\\" \\\"$FULL\\\" | sed -n 's/.*PROBE_BAD=\\\\([0-9][0-9]*\\\\).*/\\\\1/p' | head -n1)\",\"PROBE_OK\":\"$(printf \\\"%s\\\\n\\\" \\\"$FULL\\\" | sed -n 's/.*PROBE_OK=\\\\([0-9][0-9]*\\\\).*/\\\\1/p' | head -n1)\",\"RC\":\"$?\",\"REMOTE\":\"$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@<PRIVATE_IPV4> 'sudo /opt/homelab-cluster-admin/cluster-observer.sh >/dev/null 2>&1 || true; sudo /opt/homelab-cluster-admin/full-observer.sh >/dev/null 2>&1 || true; printf \\\"__OBS__\\\\\\\\n\\\"; cat /var/lib/homelab-health/cluster-admin-observer.txt; printf \\\"__FULL__\\\\\\\\n\\\"; cat /var/lib/homelab-health/cluster-admin-full-observer.txt' 2>/tmp/homelab-cluster-admin-pull.err)\",\"ROOT\":\"/var/lib/homelab-cluster-admin\",\"TS\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"},\"bytes\":2501,\"commands\":[{\"line\":7,\"text\":\"REMOTE=$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@<PRIVATE_IPV4> 'sudo /opt/homelab-cluster-admin/cluster-observer.sh >/dev/null 2>&1 || true; sudo /opt/homelab-cluster-admin/full-observer.sh >/dev/null 2>&1 || true; printf \\\"__OBS__\\\\\\\\n\\\"; cat /var/lib/homelab-health/cluster-admin-observer.txt; printf \\\"__FULL__\\\\\\\\n\\\"; cat /var/lib/homelab-health/cluster-admin-full-observer.txt' 2>/tmp/homelab-cluster-admin-pull.err)\"},{\"line\":26,\"text\":\"cat \\\"$HEALTH_ROOT/cluster-admin-observer.txt\\\"\"},{\"line\":27,\"text\":\"cat \\\"$HEALTH_ROOT/cluster-admin-full-observer.txt\\\"\"},{\"line\":28,\"text\":\"cat \\\"$HEALTH_ROOT/cluster-admin-restricted-probes.txt\\\"\"}],\"gid\":0,\"line_count\":29,\"mode\":\"0o755\",\"path\":\"/usr/local/sbin/homelab-cluster-admin-pull\",\"path_literals\":[\"/dev/null\",\"/opt/homelab-cluster-admin/cluster-observer.sh\",\"/opt/homelab-cluster-admin/full-observer.sh\",\"/tmp/homelab-cluster-admin-pull.err\",\"/usr/bin/env\",\"/var/lib/homelab-cluster-admin\",\"/var/lib/homelab-health\",\"/var/lib/homelab-health/cluster-admin-full-observer.txt\",\"/var/lib/homelab-health/cluster-admin-observer.txt\",\"/{f=0}f\",\"/{f=1\"],\"publication_lines\":[{\"line\":5,\"text\":\"HEALTH_ROOT=/var/lib/homelab-health\"},{\"line\":6,\"text\":\"mkdir -p \\\"$ROOT\\\" \\\"$HEALTH_ROOT\\\"\"},{\"line\":7,\"text\":\"REMOTE=$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@<PRIVATE_IPV4> 'sudo /opt/homelab-cluster-admin/cluster-observer.sh >/dev/null 2>&1 || true; sudo /opt/homelab-cluster-admin/full-observer.sh >/dev/null 2>&1 || true; printf \\\"__OBS__\\\\\\\\n\\\"; cat /var/lib/homelab-health/cluster-admin-observer.txt; printf \\\"__FULL__\\\\\\\\n\\\"; cat /var/lib/homelab-health/cluster-admin-full-observer.txt' 2>/tmp/homelab-cluster-admin-pull.err)\"},{\"line\":12,\"text\":\"printf \\\"%s\\\\n\\\" \\\"$OBS\\\" | grep -q \\\"^STATUS=OK\\\" || OBS_BAD=1\"},{\"line\":13,\"text\":\"printf \\\"%s\\\\n\\\" \\\"$FULL\\\" | grep -q \\\"^STATUS=OK\\\" || FULL_BAD=1\"},{\"line\":21,\"text\":\"OBS_STATUS=$([ \\\"$OBS_BAD\\\" -eq 0 ] && [ \\\"$RC\\\" -eq 0 ] && echo OK || echo WARN)\"},{\"line\":22,\"text\":\"FULL_STATUS=$([ \\\"$FULL_BAD\\\" -eq 0 ] && [ \\\"$RC\\\" -eq 0 ] && echo OK || echo WARN)\"},{\"line\":23,\"text\":\"echo \\\"STATUS=$OBS_STATUS TS=$TS TYPE=cluster-admin-central-pull BAD=$OBS_BAD VMID=180 IP=<PRIVATE_IPV4> SSH_RC=$RC OBSERVER_STATUS=$OBS_STATUS REPORT=$ROOT/observer-remote-latest.txt\\\" > \\\"$HEALTH_ROOT/cluster-admin-observer.txt\\\"\"},{\"line\":24,\"text\":\"echo \\\"STATUS=$FULL_STATUS TS=$TS TYPE=cluster-admin-full-observer-central BAD=$FULL_BAD WARN=0 VMID=180 PULL_RC=$RC FULL_STATUS=$FULL_STATUS PROBE_OK=${PROBE_OK:-0} PROBE_BAD=${PROBE_BAD:-1} REPORT=$ROOT/full-observer-central-latest.md\\\" > \\\"$HEALTH_ROOT/cluster-admin-full-observer.txt\\\"\"},{\"line\":25,\"text\":\"echo \\\"STATUS=$FULL_STATUS TS=$TS TYPE=cluster-admin-restricted-probes BAD=$FULL_BAD VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=${PROBE_OK:-0} PROBE_BAD=${PROBE_BAD:-1} TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\\\" > \\\"$HEALTH_ROOT/cluster-admin-restricted-probes.txt\\\"\"},{\"line\":26,\"text\":\"cat \\\"$HEALTH_ROOT/cluster-admin-observer.txt\\\"\"},{\"line\":27,\"text\":\"cat \\\"$HEALTH_ROOT/cluster-admin-full-observer.txt\\\"\"},{\"line\":28,\"text\":\"cat \\\"$HEALTH_ROOT/cluster-admin-restricted-probes.txt\\\"\"}],\"referenced_files\":[],\"sha256\":\"00913dbf4ef82cddd82850be7f2445924b7e75e537e7e5f6cef5785b05307d23\",\"source\":[{\"line\":1,\"text\":\"#!/usr/bin/env bash\"},{\"line\":2,\"text\":\"set +e +u +o pipefail 2>/dev/null || true\"},{\"line\":3,\"text\":\"TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)\"},{\"line\":4,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":5,\"text\":\"HEALTH_ROOT=/var/lib/homelab-health\"},{\"line\":6,\"text\":\"mkdir -p \\\"$ROOT\\\" \\\"$HEALTH_ROOT\\\"\"},{\"line\":7,\"text\":\"REMOTE=$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@<PRIVATE_IPV4> 'sudo /opt/homelab-cluster-admin/cluster-observer.sh >/dev/null 2>&1 || true; sudo /opt/homelab-cluster-admin/full-observer.sh >/dev/null 2>&1 || true; printf \\\"__OBS__\\\\\\\\n\\\"; cat /var/lib/homelab-health/cluster-admin-observer.txt; printf \\\"__FULL__\\\\\\\\n\\\"; cat /var/lib/homelab-health/cluster-admin-full-observer.txt' 2>/tmp/homelab-cluster-admin-pull.err)\"},{\"line\":8,\"text\":\"RC=$?\"},{\"line\":9,\"text\":\"OBS=$(printf \\\"%s\\\\n\\\" \\\"$REMOTE\\\" | awk '/^__OBS__$/{f=1;next}/^__FULL__$/{f=0}f')\"},{\"line\":10,\"text\":\"FULL=$(printf \\\"%s\\\\n\\\" \\\"$REMOTE\\\" | awk '/^__FULL__$/{f=1;next}f')\"},{\"line\":11,\"text\":\"OBS_BAD=0; FULL_BAD=0\"},{\"line\":12,\"text\":\"printf \\\"%s\\\\n\\\" \\\"$OBS\\\" | grep -q \\\"^STATUS=OK\\\" || OBS_BAD=1\"},{\"line\":13,\"text\":\"printf \\\"%s\\\\n\\\" \\\"$FULL\\\" | grep -q \\\"^STATUS=OK\\\" || FULL_BAD=1\"},{\"line\":14,\"text\":\"printf \\\"%s\\\\n\\\" \\\"$FULL\\\" | grep -q \\\"BAD=0\\\" || FULL_BAD=1\"},{\"line\":15,\"text\":\"PROBE_OK=$(printf \\\"%s\\\\n\\\" \\\"$FULL\\\" | sed -n 's/.*PROBE_OK=\\\\([0-9][0-9]*\\\\).*/\\\\1/p' | head -n1)\"},{\"line\":16,\"text\":\"PROBE_BAD=$(printf \\\"%s\\\\n\\\" \\\"$FULL\\\" | sed -n 's/.*PROBE_BAD=\\\\([0-9][0-9]*\\\\).*/\\\\1/p' | head -n1)\"},{\"line\":17,\"text\":\"[ \\\"${PROBE_BAD:-1}\\\" = 0 ] || FULL_BAD=1\"},{\"line\":18,\"text\":\"BAD=$((OBS_BAD+FULL_BAD)); [ \\\"$RC\\\" -eq 0 ] || BAD=$((BAD+1))\"},{\"line\":19,\"text\":\"printf \\\"%s\\\\n\\\" \\\"$OBS\\\" > \\\"$ROOT/observer-remote-latest.txt\\\"\"},{\"line\":20,\"text\":\"printf \\\"%s\\\\n\\\" \\\"$FULL\\\" > \\\"$ROOT/full-observer-central-latest.md\\\"\"},{\"line\":21,\"text\":\"OBS_STATUS=$([ \\\"$OBS_BAD\\\" -eq 0 ] && [ \\\"$RC\\\" -eq 0 ] && echo OK || echo WARN)\"},{\"line\":22,\"text\":\"FULL_STATUS=$([ \\\"$FULL_BAD\\\" -eq 0 ] && [ \\\"$RC\\\" -eq 0 ] && echo OK || echo WARN)\"},{\"line\":23,\"text\":\"echo \\\"STATUS=$OBS_STATUS TS=$TS TYPE=cluster-admin-central-pull BAD=$OBS_BAD VMID=180 IP=<PRIVATE_IPV4> SSH_RC=$RC OBSERVER_STATUS=$OBS_STATUS REPORT=$ROOT/observer-remote-latest.txt\\\" > \\\"$HEALTH_ROOT/cluster-admin-observer.txt\\\"\"},{\"line\":24,\"text\":\"echo \\\"STATUS=$FULL_STATUS TS=$TS TYPE=cluster-admin-full-observer-central BAD=$FULL_BAD WARN=0 VMID=180 PULL_RC=$RC FULL_STATUS=$FULL_STATUS PROBE_OK=${PROBE_OK:-0} PROBE_BAD=${PROBE_BAD:-1} REPORT=$ROOT/full-observer-central-latest.md\\\" > \\\"$HEALTH_ROOT/cluster-admin-full-observer.txt\\\"\"},{\"line\":25,\"text\":\"echo \\\"STATUS=$FULL_STATUS TS=$TS TYPE=cluster-admin-restricted-probes BAD=$FULL_BAD VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=${PROBE_OK:-0} PROBE_BAD=${PROBE_BAD:-1} TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\\\" > \\\"$HEALTH_ROOT/cluster-admin-restricted-probes.txt\\\"\"},{\"line\":26,\"text\":\"cat \\\"$HEALTH_ROOT/cluster-admin-observer.txt\\\"\"},{\"line\":27,\"text\":\"cat \\\"$HEALTH_ROOT/cluster-admin-full-observer.txt\\\"\"},{\"line\":28,\"text\":\"cat \\\"$HEALTH_ROOT/cluster-admin-restricted-probes.txt\\\"\"},{\"line\":29,\"text\":\"[ \\\"$BAD\\\" -eq 0 ]\"}],\"uid\":0},{\"assignments\":{\"BAD\":\"$((BAD+1))\",\"SCP_RC\":\"$?\",\"SSH_RC\":\"$?\",\"TMP\":\"/tmp/cluster-admin-webpanel-sync\",\"TS\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\",\"dst\":\"\\\"$2\\\"\",\"src\":\"\\\"$1\\\"\"},\"bytes\":1609,\"commands\":[{\"line\":11,\"text\":\" cp \\\"$src\\\" \\\"$TMP/$dst\\\"\"},{\"line\":21,\"text\":\"scp -q -o BatchMode=yes -o ConnectTimeout=8 \\\"$TMP\\\"/*.txt debian@<PRIVATE_IPV4>:/tmp/\"},{\"line\":24,\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@<PRIVATE_IPV4> 'sudo install -d -m 0755 /var/lib/homelab-health/pve01-pushed; for f in /tmp/cluster-admin-vm-mail-cloud-backup.txt /tmp/overall.txt /tmp/final-readiness.txt /tmp/cluster-admin-strict-final-seal.txt; do test -s \\\"$f\\\" && sudo install -m 0644 \\\"$f\\\" /var/lib/homelab-health/pve01-pushed/; done'\"},{\"line\":29,\"text\":\"cat /var/lib/homelab-health/cluster-admin-webpanel-sync.txt\"}],\"gid\":0,\"line_count\":29,\"mode\":\"0o755\",\"path\":\"/usr/local/sbin/homelab-cluster-admin-webpanel-sync\",\"path_literals\":[\"/tmp/\",\"/tmp/cluster-admin-strict-final-seal.txt\",\"/tmp/cluster-admin-vm-mail-cloud-backup.txt\",\"/tmp/cluster-admin-webpanel-sync\",\"/tmp/final-readiness.txt\",\"/tmp/overall.txt\",\"/usr/bin/env\",\"/var/lib/homelab-health/cluster-admin-strict-final-seal.txt\",\"/var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt\",\"/var/lib/homelab-health/cluster-admin-webpanel-sync.txt\",\"/var/lib/homelab-health/final-readiness.txt\",\"/var/lib/homelab-health/overall.txt\",\"/var/lib/homelab-health/pve01-pushed\",\"/var/lib/homelab-health/pve01-pushed/\"],\"publication_lines\":[{\"line\":4,\"text\":\"TMP=/tmp/cluster-admin-webpanel-sync\"},{\"line\":7,\"text\":\"copy_health() {\"},{\"line\":13,\"text\":\" echo \\\"STATUS=WARN TS=$TS TYPE=missing-health BAD=1 SRC=$src\\\" > \\\"$TMP/$dst\\\"\"},{\"line\":17,\"text\":\"copy_health /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt cluster-admin-vm-mail-cloud-backup.txt\"},{\"line\":18,\"text\":\"copy_health /var/lib/homelab-health/overall.txt overall.txt\"},{\"line\":19,\"text\":\"copy_health /var/lib/homelab-health/final-readiness.txt final-readiness.txt\"},{\"line\":20,\"text\":\"copy_health /var/lib/homelab-health/cluster-admin-strict-final-seal.txt cluster-admin-strict-final-seal.txt\"},{\"line\":21,\"text\":\"scp -q -o BatchMode=yes -o ConnectTimeout=8 \\\"$TMP\\\"/*.txt debian@<PRIVATE_IPV4>:/tmp/\"},{\"line\":22,\"text\":\"SCP_RC=$?\"},{\"line\":23,\"text\":\"[ \\\"$SCP_RC\\\" = 0 ] || BAD=$((BAD+1))\"},{\"line\":24,\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@<PRIVATE_IPV4> 'sudo install -d -m 0755 /var/lib/homelab-health/pve01-pushed; for f in /tmp/cluster-admin-vm-mail-cloud-backup.txt /tmp/overall.txt /tmp/final-readiness.txt /tmp/cluster-admin-strict-final-seal.txt; do test -s \\\"$f\\\" && sudo install -m 0644 \\\"$f\\\" /var/lib/homelab-health/pve01-pushed/; done'\"},{\"line\":25,\"text\":\"SSH_RC=$?\"},{\"line\":26,\"text\":\"[ \\\"$SSH_RC\\\" = 0 ] || BAD=$((BAD+1))\"},{\"line\":27,\"text\":\"if [ \\\"$BAD\\\" = 0 ]; then STATUS=OK; else STATUS=WARN; fi\"},{\"line\":28,\"text\":\"echo \\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel-sync BAD=$BAD VMID=180 IP=<PRIVATE_IPV4> SCP_RC=$SCP_RC SSH_RC=$SSH_RC PUSHED=cluster-admin-vm-mail-cloud-backup,overall,final-readiness,cluster-admin-strict-final-seal\\\" > /var/lib/homelab-health/cluster-admin-webpanel-sync.txt\"},{\"line\":29,\"text\":\"cat /var/lib/homelab-health/cluster-admin-webpanel-sync.txt\"}],\"referenced_files\":[],\"sha256\":\"2c57cca503cb0734e5114c2ab46b892a810bdc40c2baa7387b4094373db19276\",\"source\":[{\"line\":1,\"text\":\"#!/usr/bin/env bash\"},{\"line\":2,\"text\":\"set +e\"},{\"line\":3,\"text\":\"TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)\"},{\"line\":4,\"text\":\"TMP=/tmp/cluster-admin-webpanel-sync\"},{\"line\":5,\"text\":\"mkdir -p \\\"$TMP\\\"\"},{\"line\":6,\"text\":\"BAD=0\"},{\"line\":7,\"text\":\"copy_health() {\"},{\"line\":8,\"text\":\" src=\\\"$1\\\"\"},{\"line\":9,\"text\":\" dst=\\\"$2\\\"\"},{\"line\":10,\"text\":\" if [ -s \\\"$src\\\" ]; then\"},{\"line\":11,\"text\":\" cp \\\"$src\\\" \\\"$TMP/$dst\\\"\"},{\"line\":12,\"text\":\" else\"},{\"line\":13,\"text\":\" echo \\\"STATUS=WARN TS=$TS TYPE=missing-health BAD=1 SRC=$src\\\" > \\\"$TMP/$dst\\\"\"},{\"line\":14,\"text\":\" BAD=$((BAD+1))\"},{\"line\":15,\"text\":\" fi\"},{\"line\":16,\"text\":\"}\"},{\"line\":17,\"text\":\"copy_health /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt cluster-admin-vm-mail-cloud-backup.txt\"},{\"line\":18,\"text\":\"copy_health /var/lib/homelab-health/overall.txt overall.txt\"},{\"line\":19,\"text\":\"copy_health /var/lib/homelab-health/final-readiness.txt final-readiness.txt\"},{\"line\":20,\"text\":\"copy_health /var/lib/homelab-health/cluster-admin-strict-final-seal.txt cluster-admin-strict-final-seal.txt\"},{\"line\":21,\"text\":\"scp -q -o BatchMode=yes -o ConnectTimeout=8 \\\"$TMP\\\"/*.txt debian@<PRIVATE_IPV4>:/tmp/\"},{\"line\":22,\"text\":\"SCP_RC=$?\"},{\"line\":23,\"text\":\"[ \\\"$SCP_RC\\\" = 0 ] || BAD=$((BAD+1))\"},{\"line\":24,\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@<PRIVATE_IPV4> 'sudo install -d -m 0755 /var/lib/homelab-health/pve01-pushed; for f in /tmp/cluster-admin-vm-mail-cloud-backup.txt /tmp/overall.txt /tmp/final-readiness.txt /tmp/cluster-admin-strict-final-seal.txt; do test -s \\\"$f\\\" && sudo install -m 0644 \\\"$f\\\" /var/lib/homelab-health/pve01-pushed/; done'\"},{\"line\":25,\"text\":\"SSH_RC=$?\"},{\"line\":26,\"text\":\"[ \\\"$SSH_RC\\\" = 0 ] || BAD=$((BAD+1))\"},{\"line\":27,\"text\":\"if [ \\\"$BAD\\\" = 0 ]; then STATUS=OK; else STATUS=WARN; fi\"},{\"line\":28,\"text\":\"echo \\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel-sync BAD=$BAD VMID=180 IP=<PRIVATE_IPV4> SCP_RC=$SCP_RC SSH_RC=$SSH_RC PUSHED=cluster-admin-vm-mail-cloud-backup,overall,final-readiness,cluster-admin-strict-final-seal\\\" > /var/lib/homelab-health/cluster-admin-webpanel-sync.txt\"},{\"line\":29,\"text\":\"cat /var/lib/homelab-health/cluster-admin-webpanel-sync.txt\"}],\"uid\":0}],\"read_only\":true,\"rollback_restored\":false,\"rollback_started\":false,\"schema\":12,\"status\":\"OK\",\"systemd_units\":[{\"bytes\":235,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-inventory-sync.service\",\"sha256\":\"5237b68c736e77ff0108c77321bbe9b9cb7285e3a23d135361a43cd2fcb74264\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Export and securely sync nonsensitive cluster inventory to Cluster Admin\"},{\"line\":3,\"text\":\"After=network-online.target\"},{\"line\":4,\"text\":\"Wants=network-online.target\"},{\"line\":5,\"text\":\"\"},{\"line\":6,\"text\":\"[Service]\"},{\"line\":7,\"text\":\"Type=oneshot\"},{\"line\":8,\"text\":\"ExecStart=/usr/local/sbin/homelab-cluster-admin-inventory-sync\"}],\"systemctl_show\":\"ExecStart={ path=/usr/local/sbin/homelab-cluster-admin-inventory-sync ; argv[]=/usr/local/sbin/homelab-cluster-admin-inventory-sync ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:27:22 MSK] ; stop_time=[Wed 2026-08-05 14:27:24 MSK] ; pid=2627139 ; code=exited ; status=0 }\\nWorkingDirectory=\\nTriggers=\\nTriggeredBy=homelab-cluster-admin-inventory-sync.timer\\nLoadState=loaded\\nActiveState=inactive\\nSubState=dead\\nFragmentPath=/etc/systemd/system/homelab-cluster-admin-inventory-sync.service\\nDropInPaths=\\nUnitFileState=static\\n\",\"systemctl_show_rc\":0,\"systemctl_show_stderr\":\"\",\"systemd_verify_rc\":0,\"systemd_verify_stderr\":\"/etc/systemd/system/homelab-storage-capacity.service:3: Invalid URL, ignoring: internal-homelab-evidence\\n/etc/systemd/system/homelab-evidence-catalog.service:3: Invalid URL, ignoring: internal-homelab-evidence\\n/etc/systemd/system/homelab-docker-health.service:3: Invalid URL, ignoring: internal-homelab-evidence\\n\",\"systemd_verify_stdout\":\"\",\"unit\":\"homelab-cluster-admin-inventory-sync.service\"},{\"bytes\":235,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-inventory-sync.timer\",\"sha256\":\"4c9f51447b5d8f75fa05b494390e74089b6a2880ce899325643d47be4ce2180e\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Periodic Cluster Admin inventory synchronization\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Timer]\"},{\"line\":5,\"text\":\"OnBootSec=45s\"},{\"line\":6,\"text\":\"OnUnitActiveSec=2min\"},{\"line\":7,\"text\":\"RandomizedDelaySec=15s\"},{\"line\":8,\"text\":\"Persistent=true\"},{\"line\":9,\"text\":\"Unit=homelab-cluster-admin-inventory-sync.service\"},{\"line\":10,\"text\":\"\"},{\"line\":11,\"text\":\"[Install]\"},{\"line\":12,\"text\":\"WantedBy=timers.target\"}],\"systemctl_show\":\"Triggers=homelab-cluster-admin-inventory-sync.service\\nTriggeredBy=\\nLoadState=loaded\\nActiveState=active\\nSubState=waiting\\nFragmentPath=/etc/systemd/system/homelab-cluster-admin-inventory-sync.timer\\nDropInPaths=\\nUnitFileState=enabled\\n\",\"systemctl_show_rc\":0,\"systemctl_show_stderr\":\"\",\"systemd_verify_rc\":0,\"systemd_verify_stderr\":\"/etc/systemd/system/homelab-storage-capacity.service:3: Invalid URL, ignoring: internal-homelab-evidence\\n/etc/systemd/system/homelab-evidence-catalog.service:3: Invalid URL, ignoring: internal-homelab-evidence\\n/etc/systemd/system/homelab-docker-health.service:3: Invalid URL, ignoring: internal-homelab-evidence\\n\",\"systemd_verify_stdout\":\"\",\"unit\":\"homelab-cluster-admin-inventory-sync.timer\"},{\"bytes\":162,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-sync.service\",\"sha256\":\"d8091d4bb52513929cd7df742532977f70e08ee60ab8762a8f027e5863612c3b\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Push public pve01 health files to cluster-admin webpanel\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Service]\"},{\"line\":5,\"text\":\"Type=oneshot\"},{\"line\":6,\"text\":\"ExecStart=/usr/local/sbin/homelab-cluster-admin-webpanel-sync\"}],\"systemctl_show\":\"ExecStart={ path=/usr/local/sbin/homelab-cluster-admin-webpanel-sync ; argv[]=/usr/local/sbin/homelab-cluster-admin-webpanel-sync ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:29:26 MSK] ; stop_time=[n/a] ; pid=2628024 ; code=(null) ; status=0/0 }\\nWorkingDirectory=\\nTriggers=\\nTriggeredBy=homelab-cluster-admin-webpanel-sync.timer\\nLoadState=loaded\\nActiveState=activating\\nSubState=start\\nFragmentPath=/etc/systemd/system/homelab-cluster-admin-webpanel-sync.service\\nDropInPaths=\\nUnitFileState=static\\n\",\"systemctl_show_rc\":0,\"systemctl_show_stderr\":\"\",\"systemd_verify_rc\":0,\"systemd_verify_stderr\":\"/etc/systemd/system/homelab-storage-capacity.service:3: Invalid URL, ignoring: internal-homelab-evidence\\n/etc/systemd/system/homelab-evidence-catalog.service:3: Invalid URL, ignoring: internal-homelab-evidence\\n/etc/systemd/system/homelab-docker-health.service:3: Invalid URL, ignoring: internal-homelab-evidence\\n\",\"systemd_verify_stdout\":\"\",\"unit\":\"homelab-cluster-admin-webpanel-sync.service\"},{\"bytes\":169,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-sync.timer\",\"sha256\":\"ef420cea1b506ff7ea6dbd02c1b19167e5771030f4f52eb850aa05b7e3eed094\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Refresh pushed health files for cluster-admin webpanel\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Timer]\"},{\"line\":5,\"text\":\"OnBootSec=2min\"},{\"line\":6,\"text\":\"OnUnitActiveSec=1min\"},{\"line\":7,\"text\":\"Persistent=true\"},{\"line\":8,\"text\":\"\"},{\"line\":9,\"text\":\"[Install]\"},{\"line\":10,\"text\":\"WantedBy=timers.target\"}],\"systemctl_show\":\"Triggers=homelab-cluster-admin-webpanel-sync.service\\nTriggeredBy=\\nLoadState=loaded\\nActiveState=active\\nSubState=running\\nFragmentPath=/etc/systemd/system/homelab-cluster-admin-webpanel-sync.timer\\nDropInPaths=\\nUnitFileState=enabled\\n\",\"systemctl_show_rc\":0,\"systemctl_show_stderr\":\"\",\"systemd_verify_rc\":0,\"systemd_verify_stderr\":\"/etc/systemd/system/homelab-storage-capacity.service:3: Invalid URL, ignoring: internal-homelab-evidence\\n/etc/systemd/system/homelab-evidence-catalog.service:3: Invalid URL, ignoring: internal-homelab-evidence\\n/etc/systemd/system/homelab-docker-health.service:3: Invalid URL, ignoring: internal-homelab-evidence\\n\",\"systemd_verify_stdout\":\"\",\"unit\":\"homelab-cluster-admin-webpanel-sync.timer\"},{\"bytes\":142,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-pull.service\",\"sha256\":\"4a30f00113b7c68478f1cd2c8be959aed1931c55a0c7f7c99fda06cb742b914e\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Pull cluster-admin observer health into pve01\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Service]\"},{\"line\":5,\"text\":\"Type=oneshot\"},{\"line\":6,\"text\":\"ExecStart=/usr/local/sbin/homelab-cluster-admin-pull\"}],\"systemctl_show\":\"ExecStart={ path=/usr/local/sbin/homelab-cluster-admin-pull ; argv[]=/usr/local/sbin/homelab-cluster-admin-pull ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\\nWorkingDirectory=\\nTriggers=\\nTriggeredBy=\\nLoadState=loaded\\nActiveState=inactive\\nSubState=dead\\nFragmentPath=/etc/systemd/system/homelab-cluster-admin-pull.service\\nDropInPaths=\\nUnitFileState=static\\n\",\"systemctl_show_rc\":0,\"systemctl_show_stderr\":\"\",\"systemd_verify_rc\":0,\"systemd_verify_stderr\":\"/etc/systemd/system/homelab-storage-capacity.service:3: Invalid URL, ignoring: internal-homelab-evidence\\n/etc/systemd/system/homelab-evidence-catalog.service:3: Invalid URL, ignoring: internal-homelab-evidence\\n/etc/systemd/system/homelab-docker-health.service:3: Invalid URL, ignoring: internal-homelab-evidence\\n\",\"systemd_verify_stdout\":\"\",\"unit\":\"homelab-cluster-admin-pull.service\"},{\"bytes\":145,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-pull.timer\",\"sha256\":\"5e4936d1379c96088b17816a7aba42d77051009bd0938a253b3487bcb3b658a6\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Run cluster-admin central pull\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Timer]\"},{\"line\":5,\"text\":\"OnBootSec=4min\"},{\"line\":6,\"text\":\"OnUnitActiveSec=5min\"},{\"line\":7,\"text\":\"Persistent=true\"},{\"line\":8,\"text\":\"\"},{\"line\":9,\"text\":\"[Install]\"},{\"line\":10,\"text\":\"WantedBy=timers.target\"}],\"systemctl_show\":\"Triggers=homelab-cluster-admin-pull.service\\nTriggeredBy=\\nLoadState=loaded\\nActiveState=inactive\\nSubState=dead\\nFragmentPath=/etc/systemd/system/homelab-cluster-admin-pull.timer\\nDropInPaths=\\nUnitFileState=disabled\\n\",\"systemctl_show_rc\":0,\"systemctl_show_stderr\":\"\",\"systemd_verify_rc\":0,\"systemd_verify_stderr\":\"/etc/systemd/system/homelab-storage-capacity.service:3: Invalid URL, ignoring: internal-homelab-evidence\\n/etc/systemd/system/homelab-evidence-catalog.service:3: Invalid URL, ignoring: internal-homelab-evidence\\n/etc/systemd/system/homelab-docker-health.service:3: Invalid URL, ignoring: internal-homelab-evidence\\n\",\"systemd_verify_stdout\":\"\",\"unit\":\"homelab-cluster-admin-pull.timer\"},{\"bytes\":231,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-incident-engine-backup.service\",\"sha256\":\"bc7f9010fd6e97dff3bbedd2fd2088d65836a1a842cbd6d465163251e2a7db31\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Cluster Admin Incident Engine encrypted two-cloud backup\"},{\"line\":3,\"text\":\"After=network-online.target\"},{\"line\":4,\"text\":\"Wants=network-online.target\"},{\"line\":5,\"text\":\"\"},{\"line\":6,\"text\":\"[Service]\"},{\"line\":7,\"text\":\"Type=oneshot\"},{\"line\":8,\"text\":\"ExecStart=/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run\"}],\"systemctl_show\":\"ExecStart={ path=/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run ; argv[]=/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\\nWorkingDirectory=\\nTriggers=\\nTriggeredBy=\\nLoadState=loaded\\nActiveState=inactive\\nSubState=dead\\nFragmentPath=/etc/systemd/system/homelab-cluster-admin-incident-engine-backup.service\\nDropInPaths=\\nUnitFileState=static\\n\",\"systemctl_show_rc\":0,\"systemctl_show_stderr\":\"\",\"systemd_verify_rc\":0,\"systemd_verify_stderr\":\"/etc/systemd/system/homelab-storage-capacity.service:3: Invalid URL, ignoring: internal-homelab-evidence\\n/etc/systemd/system/homelab-evidence-catalog.service:3: Invalid URL, ignoring: internal-homelab-evidence\\n/etc/systemd/system/homelab-docker-health.service:3: Invalid URL, ignoring: internal-homelab-evidence\\n\",\"systemd_verify_stdout\":\"\",\"unit\":\"homelab-cluster-admin-incident-engine-backup.service\"},{\"bytes\":238,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-incident-engine-backup.timer\",\"sha256\":\"7b34396b19b8d9139e3f59d7a03a5bbfe08440c95d6a1d85031057665a724d54\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Daily Cluster Admin Incident Engine two-cloud backup\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Timer]\"},{\"line\":5,\"text\":\"OnCalendar=*-*-* 07:25:00\"},{\"line\":6,\"text\":\"RandomizedDelaySec=15m\"},{\"line\":7,\"text\":\"Persistent=true\"},{\"line\":8,\"text\":\"Unit=homelab-cluster-admin-incident-engine-backup.service\"},{\"line\":9,\"text\":\"\"},{\"line\":10,\"text\":\"[Install]\"},{\"line\":11,\"text\":\"WantedBy=timers.target\"}],\"systemctl_show\":\"Triggers=homelab-cluster-admin-incident-engine-backup.service\\nTriggeredBy=\\nLoadState=loaded\\nActiveState=inactive\\nSubState=dead\\nFragmentPath=/etc/systemd/system/homelab-cluster-admin-incident-engine-backup.timer\\nDropInPaths=\\nUnitFileState=disabled\\n\",\"systemctl_show_rc\":0,\"systemctl_show_stderr\":\"\",\"systemd_verify_rc\":0,\"systemd_verify_stderr\":\"/etc/systemd/system/homelab-storage-capacity.service:3: Invalid URL, ignoring: internal-homelab-evidence\\n/etc/systemd/system/homelab-evidence-catalog.service:3: Invalid URL, ignoring: internal-homelab-evidence\\n/etc/systemd/system/homelab-docker-health.service:3: Invalid URL, ignoring: internal-homelab-evidence\\n\",\"systemd_verify_stdout\":\"\",\"unit\":\"homelab-cluster-admin-incident-engine-backup.timer\"}],\"transaction_blueprint\":{\"candidate_components\":[\"/usr/local/sbin/homelab-cluster-admin-update-visibility\",\"/etc/systemd/system/homelab-cluster-admin-update-visibility.service\",\"/etc/systemd/system/homelab-cluster-admin-update-visibility.timer\",\"atomic patch to proven Cluster Admin publication/inventory anchor\"],\"exact_source_preconditions\":{\"/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run\":\"a79c2a72b1fbdc13506b6db8b0ea790376dc20db8003961c5b44935d28f0c5ae\",\"/usr/local/sbin/homelab-cluster-admin-inventory-sync\":\"00814787de40176e025097b353a67c26edcd298a4126d2375333e83df4f37170\",\"/usr/local/sbin/homelab-cluster-admin-pull\":\"00913dbf4ef82cddd82850be7f2445924b7e75e537e7e5f6cef5785b05307d23\",\"/usr/local/sbin/homelab-cluster-admin-webpanel-sync\":\"2c57cca503cb0734e5114c2ab46b892a810bdc40c2baa7387b4094373db19276\"},\"publication_anchor_count\":48,\"publication_anchors\":[{\"line\":15,\"script\":\"/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run\",\"text\":\"DUMP_SIZE=\\\"$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"sudo -n rm -f \\\\\\\"$REMOTE_DUMP\\\\\\\"; sudo -n -u clusteradmin pg_dump --format=custom --no-owner --no-privileges --file=\\\\\\\"$REMOTE_DUMP\\\\\\\" clusteradmin; sudo -n test -s \\\\\\\"$REMOTE_DUMP\\\\\\\"; sudo -n -u clusteradmin pg_restore --list \\\\\\\"$REMOTE_DUMP\\\\\\\" >/dev/null; sudo -n chown debian:debian \\\\\\\"$REMOTE_DUMP\\\\\\\"; sudo -n chmod 600 \\\\\\\"$REMOTE_DUMP\\\\\\\"; stat -c %s \\\\\\\"$REMOTE_DUMP\\\\\\\"\\\")\\\"\"},{\"line\":17,\"script\":\"/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run\",\"text\":\"scp -q -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE:$REMOTE_DUMP\\\" \\\"$STAGE/payload/clusteradmin.pgdump\\\"\"},{\"line\":18,\"script\":\"/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run\",\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"rm -f \\\\\\\"$REMOTE_DUMP\\\\\\\"\\\"\"},{\"line\":21,\"script\":\"/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run\",\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"sudo -n tar -C / -czf - opt/cluster-admin-incident-engine etc/cluster-admin-incident-engine etc/systemd/system/cluster-admin-incident-engine.service etc/systemd/system/cluster-admin-incident-engine-collector.service etc/systemd/system/cluster-admin-incident-engine-collector.timer var/lib/cluster-admin-incident-engine\\\" > \\\"$STAGE/runtime.tgz\\\"\"},{\"line\":25,\"script\":\"/usr/local/sbin/homelab-cluster-admin-incident-engine-backup-run\",\"text\":\"<REDACTED_SECRET_LINE>\"},{\"line\":3,\"script\":\"/usr/local/sbin/homelab-cluster-admin-inventory-sync\",\"text\":\"LOCK=/run/homelab-cluster-admin-inventory-sync.lock\"},{\"line\":6,\"script\":\"/usr/local/sbin/homelab-cluster-admin-inventory-sync\",\"text\":\"LOCAL=/var/lib/homelab-cluster-admin-export/inventory.json\"},{\"line\":8,\"script\":\"/usr/local/sbin/homelab-cluster-admin-inventory-sync\",\"text\":\"REMOTE_FILE=/var/lib/cluster-admin-incident-engine/incoming/inventory.json\"},{\"line\":9,\"script\":\"/usr/local/sbin/homelab-cluster-admin-inventory-sync\",\"text\":\"REMOTE_TMP=\\\"/tmp/cluster-admin-inventory-${RANDOM}-$$.json\\\"\"},{\"line\":10,\"script\":\"/usr/local/sbin/homelab-cluster-admin-inventory-sync\",\"text\":\"/usr/bin/python3 /usr/local/sbin/homelab-cluster-admin-inventory-export\"},{\"line\":11,\"script\":\"/usr/local/sbin/homelab-cluster-admin-inventory-sync\",\"text\":\"/usr/bin/python3 -m json.tool \\\"$LOCAL\\\" >/dev/null\"},{\"line\":13,\"script\":\"/usr/local/sbin/homelab-cluster-admin-inventory-sync\",\"text\":\"scp -q -o BatchMode=yes -o ConnectTimeout=8 \\\"$LOCAL\\\" \\\"$REMOTE:$REMOTE_TMP\\\"\"},{\"line\":14,\"script\":\"/usr/local/sbin/homelab-cluster-admin-inventory-sync\",\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"python3 -m json.tool $REMOTE_TMP >/dev/null\\\"\"},{\"line\":15,\"script\":\"/usr/local/sbin/homelab-cluster-admin-inventory-sync\",\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"sudo -n install -d -o clusteradmin -g clusteradmin -m 750 /var/lib/cluster-admin-incident-engine/incoming && sudo -n install -o clusteradmin -g clusteradmin -m 640 $REMOTE_TMP $REMOTE_FILE\\\"\"},{\"line\":16,\"script\":\"/usr/local/sbin/homelab-cluster-admin-inventory-sync\",\"text\":\"REMOTE_SHA=\\\"$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"sudo -n sha256sum $REMOTE_FILE\\\" | cut -d\\\" \\\" -f1)\\\"\"},{\"line\":19,\"script\":\"/usr/local/sbin/homelab-cluster-admin-inventory-sync\",\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 \\\"$REMOTE\\\" \\\"rm -f $REMOTE_TMP\\\"\"},{\"line\":20,\"script\":\"/usr/local/sbin/homelab-cluster-admin-inventory-sync\",\"text\":\"echo LOCAL_INVENTORY_SHA256=\\\"$LOCAL_SHA\\\"\"},{\"line\":21,\"script\":\"/usr/local/sbin/homelab-cluster-admin-inventory-sync\",\"text\":\"echo REMOTE_INVENTORY_SHA256=\\\"$REMOTE_SHA\\\"\"},{\"line\":22,\"script\":\"/usr/local/sbin/homelab-cluster-admin-inventory-sync\",\"text\":\"echo CLUSTER_ADMIN_INVENTORY_SYNC=OK\"},{\"line\":5,\"script\":\"/usr/local/sbin/homelab-cluster-admin-pull\",\"text\":\"HEALTH_ROOT=/var/lib/homelab-health\"},{\"line\":6,\"script\":\"/usr/local/sbin/homelab-cluster-admin-pull\",\"text\":\"mkdir -p \\\"$ROOT\\\" \\\"$HEALTH_ROOT\\\"\"},{\"line\":7,\"script\":\"/usr/local/sbin/homelab-cluster-admin-pull\",\"text\":\"REMOTE=$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@<PRIVATE_IPV4> 'sudo /opt/homelab-cluster-admin/cluster-observer.sh >/dev/null 2>&1 || true; sudo /opt/homelab-cluster-admin/full-observer.sh >/dev/null 2>&1 || true; printf \\\"__OBS__\\\\\\\\n\\\"; cat /var/lib/homelab-health/cluster-admin-observer.txt; printf \\\"__FULL__\\\\\\\\n\\\"; cat /var/lib/homelab-health/cluster-admin-full-observer.txt' 2>/tmp/homelab-cluster-admin-pull.err)\"},{\"line\":12,\"script\":\"/usr/local/sbin/homelab-cluster-admin-pull\",\"text\":\"printf \\\"%s\\\\n\\\" \\\"$OBS\\\" | grep -q \\\"^STATUS=OK\\\" || OBS_BAD=1\"},{\"line\":13,\"script\":\"/usr/local/sbin/homelab-cluster-admin-pull\",\"text\":\"printf \\\"%s\\\\n\\\" \\\"$FULL\\\" | grep -q \\\"^STATUS=OK\\\" || FULL_BAD=1\"},{\"line\":21,\"script\":\"/usr/local/sbin/homelab-cluster-admin-pull\",\"text\":\"OBS_STATUS=$([ \\\"$OBS_BAD\\\" -eq 0 ] && [ \\\"$RC\\\" -eq 0 ] && echo OK || echo WARN)\"},{\"line\":22,\"script\":\"/usr/local/sbin/homelab-cluster-admin-pull\",\"text\":\"FULL_STATUS=$([ \\\"$FULL_BAD\\\" -eq 0 ] && [ \\\"$RC\\\" -eq 0 ] && echo OK || echo WARN)\"},{\"line\":23,\"script\":\"/usr/local/sbin/homelab-cluster-admin-pull\",\"text\":\"echo \\\"STATUS=$OBS_STATUS TS=$TS TYPE=cluster-admin-central-pull BAD=$OBS_BAD VMID=180 IP=<PRIVATE_IPV4> SSH_RC=$RC OBSERVER_STATUS=$OBS_STATUS REPORT=$ROOT/observer-remote-latest.txt\\\" > \\\"$HEALTH_ROOT/cluster-admin-observer.txt\\\"\"},{\"line\":24,\"script\":\"/usr/local/sbin/homelab-cluster-admin-pull\",\"text\":\"echo \\\"STATUS=$FULL_STATUS TS=$TS TYPE=cluster-admin-full-observer-central BAD=$FULL_BAD WARN=0 VMID=180 PULL_RC=$RC FULL_STATUS=$FULL_STATUS PROBE_OK=${PROBE_OK:-0} PROBE_BAD=${PROBE_BAD:-1} REPORT=$ROOT/full-observer-central-latest.md\\\" > \\\"$HEALTH_ROOT/cluster-admin-full-observer.txt\\\"\"},{\"line\":25,\"script\":\"/usr/local/sbin/homelab-cluster-admin-pull\",\"text\":\"echo \\\"STATUS=$FULL_STATUS TS=$TS TYPE=cluster-admin-restricted-probes BAD=$FULL_BAD VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=${PROBE_OK:-0} PROBE_BAD=${PROBE_BAD:-1} TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\\\" > \\\"$HEALTH_ROOT/cluster-admin-restricted-probes.txt\\\"\"},{\"line\":26,\"script\":\"/usr/local/sbin/homelab-cluster-admin-pull\",\"text\":\"cat \\\"$HEALTH_ROOT/cluster-admin-observer.txt\\\"\"},{\"line\":27,\"script\":\"/usr/local/sbin/homelab-cluster-admin-pull\",\"text\":\"cat \\\"$HEALTH_ROOT/cluster-admin-full-observer.txt\\\"\"},{\"line\":28,\"script\":\"/usr/local/sbin/homelab-cluster-admin-pull\",\"text\":\"cat \\\"$HEALTH_ROOT/cluster-admin-restricted-probes.txt\\\"\"},{\"line\":4,\"script\":\"/usr/local/sbin/homelab-cluster-admin-webpanel-sync\",\"text\":\"TMP=/tmp/cluster-admin-webpanel-sync\"},{\"line\":7,\"script\":\"/usr/local/sbin/homelab-cluster-admin-webpanel-sync\",\"text\":\"copy_health() {\"},{\"line\":13,\"script\":\"/usr/local/sbin/homelab-cluster-admin-webpanel-sync\",\"text\":\" echo \\\"STATUS=WARN TS=$TS TYPE=missing-health BAD=1 SRC=$src\\\" > \\\"$TMP/$dst\\\"\"},{\"line\":17,\"script\":\"/usr/local/sbin/homelab-cluster-admin-webpanel-sync\",\"text\":\"copy_health /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt cluster-admin-vm-mail-cloud-backup.txt\"},{\"line\":18,\"script\":\"/usr/local/sbin/homelab-cluster-admin-webpanel-sync\",\"text\":\"copy_health /var/lib/homelab-health/overall.txt overall.txt\"},{\"line\":19,\"script\":\"/usr/local/sbin/homelab-cluster-admin-webpanel-sync\",\"text\":\"copy_health /var/lib/homelab-health/final-readiness.txt final-readiness.txt\"},{\"line\":20,\"script\":\"/usr/local/sbin/homelab-cluster-admin-webpanel-sync\",\"text\":\"copy_health /var/lib/homelab-health/cluster-admin-strict-final-seal.txt cluster-admin-strict-final-seal.txt\"},{\"line\":21,\"script\":\"/usr/local/sbin/homelab-cluster-admin-webpanel-sync\",\"text\":\"scp -q -o BatchMode=yes -o ConnectTimeout=8 \\\"$TMP\\\"/*.txt debian@<PRIVATE_IPV4>:/tmp/\"},{\"line\":22,\"script\":\"/usr/local/sbin/homelab-cluster-admin-webpanel-sync\",\"text\":\"SCP_RC=$?\"},{\"line\":23,\"script\":\"/usr/local/sbin/homelab-cluster-admin-webpanel-sync\",\"text\":\"[ \\\"$SCP_RC\\\" = 0 ] || BAD=$((BAD+1))\"},{\"line\":24,\"script\":\"/usr/local/sbin/homelab-cluster-admin-webpanel-sync\",\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@<PRIVATE_IPV4> 'sudo install -d -m 0755 /var/lib/homelab-health/pve01-pushed; for f in /tmp/cluster-admin-vm-mail-cloud-backup.txt /tmp/overall.txt /tmp/final-readiness.txt /tmp/cluster-admin-strict-final-seal.txt; do test -s \\\"$f\\\" && sudo install -m 0644 \\\"$f\\\" /var/lib/homelab-health/pve01-pushed/; done'\"},{\"line\":25,\"script\":\"/usr/local/sbin/homelab-cluster-admin-webpanel-sync\",\"text\":\"SSH_RC=$?\"},{\"line\":26,\"script\":\"/usr/local/sbin/homelab-cluster-admin-webpanel-sync\",\"text\":\"[ \\\"$SSH_RC\\\" = 0 ] || BAD=$((BAD+1))\"},{\"line\":27,\"script\":\"/usr/local/sbin/homelab-cluster-admin-webpanel-sync\",\"text\":\"if [ \\\"$BAD\\\" = 0 ]; then STATUS=OK; else STATUS=WARN; fi\"},{\"line\":28,\"script\":\"/usr/local/sbin/homelab-cluster-admin-webpanel-sync\",\"text\":\"echo \\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel-sync BAD=$BAD VMID=180 IP=<PRIVATE_IPV4> SCP_RC=$SCP_RC SSH_RC=$SSH_RC PUSHED=cluster-admin-vm-mail-cloud-backup,overall,final-readiness,cluster-admin-strict-final-seal\\\" > /var/lib/homelab-health/cluster-admin-webpanel-sync.txt\"},{\"line\":29,\"script\":\"/usr/local/sbin/homelab-cluster-admin-webpanel-sync\",\"text\":\"cat /var/lib/homelab-health/cluster-admin-webpanel-sync.txt\"}],\"publication_anchors_by_script\":{\"homelab-cluster-admin-incident-engine-backup-run\":5,\"homelab-cluster-admin-inventory-sync\":14,\"homelab-cluster-admin-pull\":13,\"homelab-cluster-admin-webpanel-sync\":16},\"ready_for_atomic_transaction\":true,\"referenced_existing_targets\":[{\"bytes\":5290,\"is_dir\":false,\"is_file\":true,\"literal\":\"/usr/local/sbin/homelab-cluster-admin-inventory-export\",\"mode\":\"0o750\",\"resolved\":\"/usr/local/sbin/homelab-cluster-admin-inventory-export\",\"sha256\":\"725b7ad80f6a0ce11096bc4d3f8ae615865f6031a70123dd1890b0a3e114c04d\"}],\"required_transaction_properties\":[\"re-read and re-hash all exact preconditions\",\"build all candidates under one temporary directory\",\"compile/check scripts and systemd-analyze verify units\",\"run candidate collector in isolated temporary output directory\",\"byte-for-byte backup every existing file before replacement\",\"atomic os.replace for files and atomic symlink/unit enablement changes\",\"daemon-reload only after all file replacements succeed\",\"verify timer, service, generated JSON/HTML and Cluster Admin publication\",\"restore every original byte and prior unit state on any failure\"],\"unit_preconditions\":{\"/etc/systemd/system/homelab-cluster-admin-incident-engine-backup.service\":\"bc7f9010fd6e97dff3bbedd2fd2088d65836a1a842cbd6d465163251e2a7db31\",\"/etc/systemd/system/homelab-cluster-admin-incident-engine-backup.timer\":\"7b34396b19b8d9139e3f59d7a03a5bbfe08440c95d6a1d85031057665a724d54\",\"/etc/systemd/system/homelab-cluster-admin-inventory-sync.service\":\"5237b68c736e77ff0108c77321bbe9b9cb7285e3a23d135361a43cd2fcb74264\",\"/etc/systemd/system/homelab-cluster-admin-inventory-sync.timer\":\"4c9f51447b5d8f75fa05b494390e74089b6a2880ce899325643d47be4ce2180e\",\"/etc/systemd/system/homelab-cluster-admin-pull.service\":\"4a30f00113b7c68478f1cd2c8be959aed1931c55a0c7f7c99fda06cb742b914e\",\"/etc/systemd/system/homelab-cluster-admin-pull.timer\":\"5e4936d1379c96088b17816a7aba42d77051009bd0938a253b3487bcb3b658a6\",\"/etc/systemd/system/homelab-cluster-admin-webpanel-sync.service\":\"d8091d4bb52513929cd7df742532977f70e08ee60ab8762a8f027e5863612c3b\",\"/etc/systemd/system/homelab-cluster-admin-webpanel-sync.timer\":\"ef420cea1b506ff7ea6dbd02c1b19167e5771030f4f52eb850aa05b7e3eed094\"}},\"v11_exact_result\":{\"capsule_sha256\":\"c32730db30f2b2c16368ef4e4d873f75cf5ee3cf8a5b66aabf4636bf4173c706\",\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V11-SOURCE-MAP-READ-ONLY-20260805T111000Z\",\"control_plane_error\":{\"message\":\"existing runner returned a different rc after writing its public artifact; runtime publication or wrapper completion failed\",\"type\":\"RunnerWrapperMismatch\"},\"identity_confirmed\":true,\"mutation_outcome\":\"NO_MUTATION\",\"output_bytes\":159415,\"output_sha256\":\"5392567d03d9794b8d677b56843b620f0f15f5515e1b6d7707ea5ec3b602fda5\",\"output_source\":\"runner_json\",\"output_truncated_in_json\":false,\"result_contract\":{\"changes_made\":false,\"cluster_nodes_reachable\":3,\"cluster_nodes_total\":3,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V11-SOURCE-MAP-READ-ONLY-20260805T111000Z\",\"command_rc\":0,\"control_plane_error\":null,\"coverage_complete\":true,\"detected_sync_tools\":[\"scp\",\"ssh\"],\"existing_update_logic_detected\":true,\"mutation_outcome\":\"NO_MUTATION\",\"primary_script_bytes\":{\"homelab-cluster-admin-incident-engine-backup-run\":2640,\"homelab-cluster-admin-inventory-sync\":1301,\"homelab-cluster-admin-pull\":2501,\"homelab-cluster-admin-webpanel-sync\":1609},\"primary_script_count\":4,\"primary_script_manifest_sha256\":\"9783967a04ca6f561d361c3b6a4a7ea45a32961f1e232303e279bfcae6c38921\",\"primary_script_sha256\":{\"homelab-cluster-admin-incident-engine-backup-run\":\"a79c2a72b1fbdc13506b6db8b0ea790376dc20db8003961c5b44935d28f0c5ae\",\"homelab-cluster-admin-inventory-sync\":\"00814787de40176e025097b353a67c26edcd298a4126d2375333e83df4f37170\",\"homelab-cluster-admin-pull\":\"00913dbf4ef82cddd82850be7f2445924b7e75e537e7e5f6cef5785b05307d23\",\"homelab-cluster-admin-webpanel-sync\":\"2c57cca503cb0734e5114c2ab46b892a810bdc40c2baa7387b4094373db19276\"},\"publication_write_path_count\":16,\"rc\":0,\"ready_for_atomic_candidate_build\":true,\"related_file_count\":180,\"related_file_manifest_sha256\":\"f05ca871862daed14a4d453e98bad2f8988eb9f59b48a1350189dc31e27db5e6\",\"report_bytes\":157189,\"report_sha256\":\"2b6daa2459cfaabd9e5b24dd0386b096829cdb712a997670416fe4c42fbacfe0\",\"rollback_restored\":false,\"rollback_started\":false,\"status\":\"OK\",\"v10_exact_error_type\":\"JSONDecodeError\",\"v10_exact_result_identity_confirmed\":true,\"v10_output_sha256\":\"8c93138303597fbbb8a579d7db343b32cfe3e7edf074d0425bd14844eb9967df\",\"v10_runner_json_sha256\":\"7673962c52d217c1ee7b4173c07defbd43ab29a74ebc455d18b490fe386ca1c2\",\"v10_runner_text_sha256\":\"b17921aa090892f1785523f8afd1796390d7bdc9521fab1ca9517d430d19c944\",\"v9_exact_result_identity_confirmed\":true,\"v9_output_sha256\":\"4af4ea8924aa73eb833e6bca6bb6592947c9d182473c80829bcb0ab496cacf98\",\"v9_runner_json_sha256\":\"ebb1f5b5f0ceec5e68b9c05ca0d86b6605a12fa2a5e4f1bd9322d00802290269\",\"v9_runner_text_sha256\":\"71d9fcf01ced23dc2ad486e32d726e0dba1abf6038fec21793bed8282484e31d\",\"version\":1,\"webpanel_source_candidate_count\":0},\"rollback_restored\":null,\"rollback_started\":false,\"runner_document_rc\":0,\"runner_json_sha256\":\"638b92d7dc399d0c274c36f0b386cfff227e31c8b248b758c784a7b1a694fd45\",\"runner_rc\":75,\"runner_status\":\"OK\",\"runner_text_sha256\":\"212049d3b95be8d32bc7a027be42632d3fef6c61f83882b917096221dd1a27e6\",\"runner_wrapper_matches_document\":false,\"state_document_sha256\":\"5c48e6467b3483104b5c92368cba957269d67115acab9b053c56158dba4ab716\",\"state_path_sha256\":\"0d1aa6df80ef33edf38f5267e9673940c5ea161e3802217236bd9130ea26c2ac\",\"state_status\":\"FAIL\"}}\n"
}
View Git Blame Copy Permalink