From 60b04df7d42953f4ab374df89ee2e7d34075c07d Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Fri, 24 Jul 2026 09:43:33 +0000 Subject: [PATCH] runtime: publish HOMELAB-CLUSTER-CRITICAL-SNAPSHOT-428 --- ...HOMELAB-CLUSTER-CRITICAL-SNAPSHOT-428.json | 30 ++ .../HOMELAB-CLUSTER-CRITICAL-SNAPSHOT-428.txt | 175 ++++++ runtime/latest.json | 18 +- runtime/latest.txt | 506 ++++++------------ 4 files changed, 371 insertions(+), 358 deletions(-) create mode 100644 runtime/history/HOMELAB-CLUSTER-CRITICAL-SNAPSHOT-428.json create mode 100644 runtime/history/HOMELAB-CLUSTER-CRITICAL-SNAPSHOT-428.txt diff --git a/runtime/history/HOMELAB-CLUSTER-CRITICAL-SNAPSHOT-428.json b/runtime/history/HOMELAB-CLUSTER-CRITICAL-SNAPSHOT-428.json new file mode 100644 index 0000000..d52e43c --- /dev/null +++ b/runtime/history/HOMELAB-CLUSTER-CRITICAL-SNAPSHOT-428.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "HOMELAB-CLUSTER-CRITICAL-SNAPSHOT-428", + "status": "FAIL", + "rc": 127, + "host": "pve01", + "mode": "read-only", + "component": "cluster-audit", + "started_at_utc": "2026-07-24T09:43:29Z", + "finished_at_utc": "2026-07-24T09:43:33Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", + "command_sha256": "acb8cfc8df7523b005c7cc8d9dcc5e8d6955edc9869f35192b9aab7770149bb5", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "983c89419e9432c65180a176c36039953a5d07a157e9509f7eaa97016d1f65b6", + "sanitized_output_sha256": "50e542092e058ed4496ae5f17ec2fa7a38e02c39668e58030948a9916e980eb2", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "AUDIT_ID=HOMELAB-CLUSTER-CRITICAL-SNAPSHOT-428\nAUDIT_TIME_UTC=2026-07-24T09:43:29Z\nCHANGES_PLANNED=NO\n\n===== PROXMOX_QUORUM =====\nCluster information\n-------------------\nName: homelab\nConfig Version: 3\nTransport: knet\nSecure auth: on\n\nQuorum information\n------------------\nDate: Fri Jul 24 12:43:29 2026\nQuorum provider: corosync_votequorum\nNodes: 3\nNode ID: 0x00000001\nRing ID: 1.ba\nQuorate: Yes\n\nVotequorum information\n----------------------\nExpected votes: 3\nHighest expected: 3\nTotal votes: 3\nQuorum: 2 \nFlags: Quorate \n\nMembership information\n----------------------\n Nodeid Votes Name\n0x00000001 1 [PRIVATE_IP] (local)\n0x00000002 1 [PRIVATE_IP]\n0x00000003 1 [PRIVATE_IP]\n\n===== PROXMOX_NODES =====\n+-------+--------+-------+-------+--------+-----------+-----------+-------------------------------------------------------------------------------------------------+-------------+\n| node | status | cpu | level | maxcpu | maxmem | mem | ssl_fingerprint | uptime |\n+=======+========+=======+=======+========+===========+===========+=================================================================================================+=============+\n| pve01 | online | 1.98% | | 24 | 31.09 GiB | 22.07 GiB | EB:7A:01:C5:02:1B:B7:F9:54:F4:38:8B:7F:03:1F:87:62:D0:6E:5F:C0:39:FB:B5:42:AE:58:6F:58:9A:9C:C1 | 20h 22m 46s |\n+-------+--------+-------+-------+--------+-----------+-----------+-------------------------------------------------------------------------------------------------+-------------+\n| pve02 | online | 8.49% | | 4 | 15.40 GiB | 11.36 GiB | C0:88:6E:89:CD:BF:0C:82:54:20:E7:11:DE:CA:ED:09:12:C6:9C:24:2A:1E:5C:33:49:EC:4E:E0:67:BE:79:05 | 20h 20m 42s |\n+-------+--------+-------+-------+--------+-----------+-----------+-------------------------------------------------------------------------------------------------+-------------+\n| pve03 | online | 4.43% | | 8 | 31.25 GiB | 11.24 GiB | 7E:FE:4A:DC:32:3C:02:84:A2:DD:23:84:FD:19:1E:8A:1B:B1:60:F6:F4:16:D0:28:3B:13:AE:65:7D:1F:38:32 | 20h 21m 35s |\n+-------+--------+-------+-------+--------+-----------+-----------+-------------------------------------------------------------------------------------------------+-------------+\n\n===== PROXMOX_STORAGE =====\n+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+\n| id | type | cgroup-mode | content | cpu | disk | diskread | diskwrite | hastate | host-arch | level | lock | maxcpu | maxdisk | maxmem | mem | memhost | name | netin | netout | network | network-type | node | plugintype | pool | protocol | sdn | shared | status | storage | tags | template | uptime | vmid | zone-type |\n+=========================+=========+=============+==========================+=====+============+==========+===========+=========+===========+=======+======+========+============+========+=====+=========+======+=======+========+=========+==============+=======+============+======+==========+=====+========+===========+===========+======+==========+========+======+===========+\n| storage/pve01/local | storage | | iso,backup,vztmpl,import | | 40.39 GiB | | | | | | | | 97.87 GiB | | | | | | | | | pve01 | dir | | | | 0 | available | local | | | | | |\n+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+\n| storage/pve01/local-lvm | storage | | images,rootdir | | 310.84 GiB | | | | | | | | 809.48 GiB | | | | | | | | | pve01 | lvmthin | | | | 0 | available | local-lvm | | | | | |\n+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+\n| storage/pve02/local | storage | | iso,backup,vztmpl,import | | 26.14 GiB | | | | | | | | 97.87 GiB | | | | | | | | | pve02 | dir | | | | 0 | available | local | | | | | |\n+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+\n| storage/pve02/local-lvm | storage | | images,rootdir | | 290.27 GiB | | | | | | | | 787.91 GiB | | | | | | | | | pve02 | lvmthin | | | | 0 | available | local-lvm | | | | | |\n+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+\n| storage/pve03/local | storage | | iso,backup,vztmpl,import | | 53.72 GiB | | | | | | | | 97.87 GiB | | | | | | | | | pve03 | dir | | | | 0 | available | local | | | | | |\n+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+\n| storage/pve03/local-lvm | storage | | images,rootdir | | 145.74 GiB | | | | | | | | 331.23 GiB | | | | | | | | | pve03 | lvmthin | | | | 0 | available | local-lvm | | | | | |\n+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+\n\n===== LOCAL_FILESYSTEMS =====\nFilesystem Size Used Avail Use% Mounted on\n/dev/mapper/pve-root 98G 41G 53G 44% /\nefivarfs 192K 135K 53K 72% /sys/firmware/efi/efivars\n/dev/nvme0n1p2 1022M 9.1M 1013M 1% /boot/efi\n/dev/mapper/pve-homelab--staging 295G 148G 132G 53% /mnt/staging\n/dev/fuse 128M 1.4M 127M 2% /etc/pve\noverlay 98G 41G 53G 44% /var/lib/containers/storage/overlay/4639d2b7096bff2bf1482d2b2b867a1a21a3e9eba88986eb78f9daa256eb5a77/merged\n\n===== LOCAL_INODES =====\nFilesystem Inodes IUsed IFree IUse% Mounted on\n/dev/mapper/pve-root 6553600 139099 6414501 3% /\nefivarfs 0 0 0 - /sys/firmware/efi/efivars\n/dev/nvme0n1p2 0 0 0 - /boot/efi\n/dev/mapper/pve-homelab--staging 19660800 4475 19656325 1% /mnt/staging\n/dev/fuse 262144 89 262055 1% /etc/pve\noverlay 6553600 139099 6414501 3% /var/lib/containers/storage/overlay/4639d2b7096bff2bf1482d2b2b867a1a21a3e9eba88986eb78f9daa256eb5a77/merged\n\n===== ZFS =====\nno pools available\n\n===== CEPH =====\nSTATE=NOT_CONFIGURED_OR_TOOL_ABSENT\n\n===== PVE01_FAILED_UNITS =====\nhomelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state\n\n===== PVE01_KERNEL_CRITICAL_24H =====\n\n===== PROXMOX_FAILED_TASKS =====\nUnknown option: errors\nUnknown option: limit\n400 unable to parse option\npvesh get [OPTIONS] [FORMAT_OPTIONS]\n\n===== TIME_SYNCHRONIZATION =====\nTimezone=Europe/Moscow\nLocalRTC=no\nNTPSynchronized=yes\n\n===== SKLADCHIK_SUPERVISOR =====\ncut: '\"': No such file or directory\nSCRIPT_SHA256=\nEXPECTED_SHA256=bd40ee19ea8b277d52f4240b97d3c79b825c90154272bac8a43004591bd0ddc3\nResult=success\nExecMainStatus=0\nActiveState=inactive\nSubState=dead\nSUPERVISOR_TIMER=active\nSTATUS=OK\nTS=2026-07-24T09:38:15Z\nproblems=\nbackup_status=OK\ncookie_expiry_status=OK\ncookie_jar=yes\nmonitor_status=OK\nrestore_status=OK\nPOST_FIX_COOKIE_FAILURES=0\n\n===== EDGE_VM =====\nedge-vm.gram1.ru\nactive\nactive\nbash: line 1: /var/lib/skladchik-reports-monitor/cookies.jar: Permission denied\nstat: missing operand\nTry 'stat --help' for more information.\nbash: -c: line 1: syntax error near unexpected token `('\nbash: -c: line 1: `sudo -n grep -E ^(STATUS|TS|reports|open|resolved|cookie_jar|cookie_expiry_status|problems)= /var/lib/skladchik-reports-monitor/health.txt'\nFilesystem Size Used Avail Use% Mounted on\n/dev/sdb1 95G 33G 58G 36% /\n\n===== CORE_APPS =====\ncore-apps.gram1.ru\nbash: line 1: {{.Status}}: command not found\n7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f /opt/stacks/gitea/data/gitea/public/.well-known/homelab-bootstrap.json\n7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f /opt/stacks/gitea/data/gitea/public/.well-known/homelab-bootstrap/7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f.json\nFilesystem Size Used Avail Use% Mounted on\n/dev/sda1 118G 42G 72G 37% /\n\n===== PUBLIC_BOOTSTRAP =====\nEXPECTED_BOOTSTRAP_SHA256=7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f\n7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f -\n7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f -\n\n===== READER_CAPABILITY_EXPOSURE =====\n/usr/bin/bash: -c: line 207: syntax error near unexpected token `('\n/usr/bin/bash: -c: line 207: ` \\\"/homelab-reader/[0-9a-fA-F]{64}([/\\\\\\\"]|$)\\\" \\'\n" +} diff --git a/runtime/history/HOMELAB-CLUSTER-CRITICAL-SNAPSHOT-428.txt b/runtime/history/HOMELAB-CLUSTER-CRITICAL-SNAPSHOT-428.txt new file mode 100644 index 0000000..4670fd5 --- /dev/null +++ b/runtime/history/HOMELAB-CLUSTER-CRITICAL-SNAPSHOT-428.txt @@ -0,0 +1,175 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=HOMELAB-CLUSTER-CRITICAL-SNAPSHOT-428 +STATUS=FAIL +RC=127 +HOST=pve01 +MODE=read-only +COMPONENT=cluster-audit +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 +COMMAND_SHA256=acb8cfc8df7523b005c7cc8d9dcc5e8d6955edc9869f35192b9aab7770149bb5 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=false +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=983c89419e9432c65180a176c36039953a5d07a157e9509f7eaa97016d1f65b6 +SANITIZED_OUTPUT_SHA256=50e542092e058ed4496ae5f17ec2fa7a38e02c39668e58030948a9916e980eb2 +OUTPUT_BEGIN +AUDIT_ID=HOMELAB-CLUSTER-CRITICAL-SNAPSHOT-428 +AUDIT_TIME_UTC=2026-07-24T09:43:29Z +CHANGES_PLANNED=NO + +===== PROXMOX_QUORUM ===== +Cluster information +------------------- +Name: homelab +Config Version: 3 +Transport: knet +Secure auth: on + +Quorum information +------------------ +Date: Fri Jul 24 12:43:29 2026 +Quorum provider: corosync_votequorum +Nodes: 3 +Node ID: 0x00000001 +Ring ID: 1.ba +Quorate: Yes + +Votequorum information +---------------------- +Expected votes: 3 +Highest expected: 3 +Total votes: 3 +Quorum: 2 +Flags: Quorate + +Membership information +---------------------- + Nodeid Votes Name +0x00000001 1 [PRIVATE_IP] (local) +0x00000002 1 [PRIVATE_IP] +0x00000003 1 [PRIVATE_IP] + +===== PROXMOX_NODES ===== ++-------+--------+-------+-------+--------+-----------+-----------+-------------------------------------------------------------------------------------------------+-------------+ +| node | status | cpu | level | maxcpu | maxmem | mem | ssl_fingerprint | uptime | ++=======+========+=======+=======+========+===========+===========+=================================================================================================+=============+ +| pve01 | online | 1.98% | | 24 | 31.09 GiB | 22.07 GiB | EB:7A:01:C5:02:1B:B7:F9:54:F4:38:8B:7F:03:1F:87:62:D0:6E:5F:C0:39:FB:B5:42:AE:58:6F:58:9A:9C:C1 | 20h 22m 46s | ++-------+--------+-------+-------+--------+-----------+-----------+-------------------------------------------------------------------------------------------------+-------------+ +| pve02 | online | 8.49% | | 4 | 15.40 GiB | 11.36 GiB | C0:88:6E:89:CD:BF:0C:82:54:20:E7:11:DE:CA:ED:09:12:C6:9C:24:2A:1E:5C:33:49:EC:4E:E0:67:BE:79:05 | 20h 20m 42s | ++-------+--------+-------+-------+--------+-----------+-----------+-------------------------------------------------------------------------------------------------+-------------+ +| pve03 | online | 4.43% | | 8 | 31.25 GiB | 11.24 GiB | 7E:FE:4A:DC:32:3C:02:84:A2:DD:23:84:FD:19:1E:8A:1B:B1:60:F6:F4:16:D0:28:3B:13:AE:65:7D:1F:38:32 | 20h 21m 35s | ++-------+--------+-------+-------+--------+-----------+-----------+-------------------------------------------------------------------------------------------------+-------------+ + +===== PROXMOX_STORAGE ===== ++-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+ +| id | type | cgroup-mode | content | cpu | disk | diskread | diskwrite | hastate | host-arch | level | lock | maxcpu | maxdisk | maxmem | mem | memhost | name | netin | netout | network | network-type | node | plugintype | pool | protocol | sdn | shared | status | storage | tags | template | uptime | vmid | zone-type | ++=========================+=========+=============+==========================+=====+============+==========+===========+=========+===========+=======+======+========+============+========+=====+=========+======+=======+========+=========+==============+=======+============+======+==========+=====+========+===========+===========+======+==========+========+======+===========+ +| storage/pve01/local | storage | | iso,backup,vztmpl,import | | 40.39 GiB | | | | | | | | 97.87 GiB | | | | | | | | | pve01 | dir | | | | 0 | available | local | | | | | | ++-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+ +| storage/pve01/local-lvm | storage | | images,rootdir | | 310.84 GiB | | | | | | | | 809.48 GiB | | | | | | | | | pve01 | lvmthin | | | | 0 | available | local-lvm | | | | | | ++-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+ +| storage/pve02/local | storage | | iso,backup,vztmpl,import | | 26.14 GiB | | | | | | | | 97.87 GiB | | | | | | | | | pve02 | dir | | | | 0 | available | local | | | | | | ++-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+ +| storage/pve02/local-lvm | storage | | images,rootdir | | 290.27 GiB | | | | | | | | 787.91 GiB | | | | | | | | | pve02 | lvmthin | | | | 0 | available | local-lvm | | | | | | ++-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+ +| storage/pve03/local | storage | | iso,backup,vztmpl,import | | 53.72 GiB | | | | | | | | 97.87 GiB | | | | | | | | | pve03 | dir | | | | 0 | available | local | | | | | | ++-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+ +| storage/pve03/local-lvm | storage | | images,rootdir | | 145.74 GiB | | | | | | | | 331.23 GiB | | | | | | | | | pve03 | lvmthin | | | | 0 | available | local-lvm | | | | | | ++-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+ + +===== LOCAL_FILESYSTEMS ===== +Filesystem Size Used Avail Use% Mounted on +/dev/mapper/pve-root 98G 41G 53G 44% / +efivarfs 192K 135K 53K 72% /sys/firmware/efi/efivars +/dev/nvme0n1p2 1022M 9.1M 1013M 1% /boot/efi +/dev/mapper/pve-homelab--staging 295G 148G 132G 53% /mnt/staging +/dev/fuse 128M 1.4M 127M 2% /etc/pve +overlay 98G 41G 53G 44% /var/lib/containers/storage/overlay/4639d2b7096bff2bf1482d2b2b867a1a21a3e9eba88986eb78f9daa256eb5a77/merged + +===== LOCAL_INODES ===== +Filesystem Inodes IUsed IFree IUse% Mounted on +/dev/mapper/pve-root 6553600 139099 6414501 3% / +efivarfs 0 0 0 - /sys/firmware/efi/efivars +/dev/nvme0n1p2 0 0 0 - /boot/efi +/dev/mapper/pve-homelab--staging 19660800 4475 19656325 1% /mnt/staging +/dev/fuse 262144 89 262055 1% /etc/pve +overlay 6553600 139099 6414501 3% /var/lib/containers/storage/overlay/4639d2b7096bff2bf1482d2b2b867a1a21a3e9eba88986eb78f9daa256eb5a77/merged + +===== ZFS ===== +no pools available + +===== CEPH ===== +STATE=NOT_CONFIGURED_OR_TOOL_ABSENT + +===== PVE01_FAILED_UNITS ===== +homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state + +===== PVE01_KERNEL_CRITICAL_24H ===== + +===== PROXMOX_FAILED_TASKS ===== +Unknown option: errors +Unknown option: limit +400 unable to parse option +pvesh get [OPTIONS] [FORMAT_OPTIONS] + +===== TIME_SYNCHRONIZATION ===== +Timezone=Europe/Moscow +LocalRTC=no +NTPSynchronized=yes + +===== SKLADCHIK_SUPERVISOR ===== +cut: '"': No such file or directory +SCRIPT_SHA256= +EXPECTED_SHA256=bd40ee19ea8b277d52f4240b97d3c79b825c90154272bac8a43004591bd0ddc3 +Result=success +ExecMainStatus=0 +ActiveState=inactive +SubState=dead +SUPERVISOR_TIMER=active +STATUS=OK +TS=2026-07-24T09:38:15Z +problems= +backup_status=OK +cookie_expiry_status=OK +cookie_jar=yes +monitor_status=OK +restore_status=OK +POST_FIX_COOKIE_FAILURES=0 + +===== EDGE_VM ===== +edge-vm.gram1.ru +active +active +bash: line 1: /var/lib/skladchik-reports-monitor/cookies.jar: Permission denied +stat: missing operand +Try 'stat --help' for more information. +bash: -c: line 1: syntax error near unexpected token `(' +bash: -c: line 1: `sudo -n grep -E ^(STATUS|TS|reports|open|resolved|cookie_jar|cookie_expiry_status|problems)= /var/lib/skladchik-reports-monitor/health.txt' +Filesystem Size Used Avail Use% Mounted on +/dev/sdb1 95G 33G 58G 36% / + +===== CORE_APPS ===== +core-apps.gram1.ru +bash: line 1: {{.Status}}: command not found +7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f /opt/stacks/gitea/data/gitea/public/.well-known/homelab-bootstrap.json +7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f /opt/stacks/gitea/data/gitea/public/.well-known/homelab-bootstrap/7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f.json +Filesystem Size Used Avail Use% Mounted on +/dev/sda1 118G 42G 72G 37% / + +===== PUBLIC_BOOTSTRAP ===== +EXPECTED_BOOTSTRAP_SHA256=7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f +7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f - +7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f - + +===== READER_CAPABILITY_EXPOSURE ===== +/usr/bin/bash: -c: line 207: syntax error near unexpected token `(' +/usr/bin/bash: -c: line 207: ` \"/homelab-reader/[0-9a-fA-F]{64}([/\\\"]|$)\" \' + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index c25e287..d52e43c 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "HOMELAB-DUPLICATE-EXACT-LOGIC-427", - "status": "OK", - "rc": 0, + "command_id": "HOMELAB-CLUSTER-CRITICAL-SNAPSHOT-428", + "status": "FAIL", + "rc": 127, "host": "pve01", "mode": "read-only", "component": "cluster-audit", - "started_at_utc": "2026-07-24T09:37:54Z", - "finished_at_utc": "2026-07-24T09:37:54Z", + "started_at_utc": "2026-07-24T09:43:29Z", + "finished_at_utc": "2026-07-24T09:43:33Z", "reference_register_checked": true, "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", "error_register_checked": true, "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", - "command_sha256": "7e4f72d9877110d6d6d4ef926a611b1bd8ad15fa4b2ad3ef17e9027ba49bc111", + "command_sha256": "acb8cfc8df7523b005c7cc8d9dcc5e8d6955edc9869f35192b9aab7770149bb5", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -22,9 +22,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "8480bb49baedd9fdebe212ee204d07d772e707d3c09045de0c3334fde723f6e9", - "sanitized_output_sha256": "5e8f3e2de086ae3c1d1bf84dcffbcb8dc8770ebefb00aa959db1ff15db3468b9", + "raw_evidence_sha256": "983c89419e9432c65180a176c36039953a5d07a157e9509f7eaa97016d1f65b6", + "sanitized_output_sha256": "50e542092e058ed4496ae5f17ec2fa7a38e02c39668e58030948a9916e980eb2", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "DETAIL=FILE=/usr/local/sbin/homelab-chat-run-linked\nDETAIL=SHA256=5e8725e129fd2055b36fe2fae13fc95fc4b3271074a0e0d4bb7347e7b2f270a1\nDETAIL=EXACT_LOGIC_BEGIN\nDETAIL=TOTAL_LINES=39\nDETAIL=EXACT_LOGIC_END\nDETAIL=FILE=/usr/local/sbin/homelab-chat-run\nDETAIL=SHA256=2a88defef9517f89bc60934dce82097f8e316b08af9417e95167ea5d759a10d9\nDETAIL=EXACT_LOGIC_BEGIN\nDETAIL=SOURCE_LINE=0048:esac\nDETAIL=SOURCE_LINE=0049:\nDETAIL=SOURCE_LINE=0050:case \"$MODE\" in\nDETAIL=SOURCE_LINE=0051: read-only|verify|change|transport-test)\nDETAIL=SOURCE_LINE=0052: ;;\nDETAIL=SOURCE_LINE=0053: *)\nDETAIL=SOURCE_LINE=0054: echo 'RESULT=FAIL'\nDETAIL=SOURCE_LINE=0055: echo 'REASON=INVALID_MODE'\nDETAIL=SOURCE_LINE=0056: exit 67\nDETAIL=SOURCE_LINE=0057: ;;\nDETAIL=SOURCE_LINE=0058:esac\nDETAIL=SOURCE_LINE=0059:\nDETAIL=SOURCE_LINE=0060:for TOOL in \\\nDETAIL=SOURCE_LINE=0061: awk \\\nDETAIL=SOURCE_LINE=0062: curl \\\nDETAIL=SOURCE_LINE=0063: flock \\\nDETAIL=SOURCE_LINE=0064: python3 \\\nDETAIL=SOURCE_LINE=0065: scp \\\nDETAIL=SOURCE_LINE=0066: sha256sum \\\nDETAIL=SOURCE_LINE=0067: ssh\nDETAIL=SOURCE_LINE=0068:do\nDETAIL=SOURCE_LINE=0069: command -v \"$TOOL\" >/dev/null 2>&1 || {\nDETAIL=SOURCE_LINE=0070: echo 'RESULT=FAIL'\nDETAIL=SOURCE_LINE=0071: echo \"REASON=REQUIRED_TOOL_MISSING:$TOOL\"\nDETAIL=SOURCE_LINE=0072: exit 68\nDETAIL=SOURCE_LINE=0073: }\nDETAIL=SOURCE_LINE=0074:done\nDETAIL=SOURCE_LINE=0075:\nDETAIL=SOURCE_LINE=0076:REFERENCE=/etc/pve/31_HOMELAB_REFERENCE.md\nDETAIL=SOURCE_LINE=0077:ERROR_REGISTER=/etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md\nDETAIL=SOURCE_LINE=0078:\nDETAIL=SOURCE_LINE=0079:[ -r \"$REFERENCE\" ] || {\nDETAIL=SOURCE_LINE=0080: echo 'RESULT=FAIL'\nDETAIL=SOURCE_LINE=0081: echo 'REASON=REFERENCE_NOT_READABLE'\nDETAIL=SOURCE_LINE=0082: exit 69\nDETAIL=SOURCE_LINE=0083:}\nDETAIL=SOURCE_LINE=0084:\nDETAIL=SOURCE_LINE=0085:[ -r \"$ERROR_REGISTER\" ] || {\nDETAIL=SOURCE_LINE=0086: echo 'RESULT=FAIL'\nDETAIL=SOURCE_LINE=0087: echo 'REASON=ERROR_REGISTER_NOT_READABLE'\nDETAIL=SOURCE_LINE=0088: exit 70\nDETAIL=SOURCE_LINE=0089:}\nDETAIL=SOURCE_LINE=0090:\nDETAIL=SOURCE_LINE=0091:STATE_DIR=/var/lib/homelab-chat-run\nDETAIL=SOURCE_LINE=0092:RAW_DIR=\"$STATE_DIR/raw\"\nDETAIL=SOURCE_LINE=0093:PUBLIC_DIR=\"$STATE_DIR/public\"\nDETAIL=SOURCE_LINE=0094:LEDGER=\"$STATE_DIR/state/command-ledger.tsv\"\nDETAIL=SOURCE_LINE=0095:LOCK=\"$STATE_DIR/state/runner.lock\"\nDETAIL=SOURCE_LINE=0096:\nDETAIL=SOURCE_LINE=0097:mkdir -p \"$RAW_DIR\" \"$PUBLIC_DIR\" \"$(dirname \"$LEDGER\")\"\nDETAIL=SOURCE_LINE=0098:chmod 0700 \"$STATE_DIR\" \"$RAW_DIR\" \"$PUBLIC_DIR\" \"$(dirname \"$LEDGER\")\"\nDETAIL=SOURCE_LINE=0099:touch \"$LEDGER\"\nDETAIL=SOURCE_LINE=0100:chmod 0600 \"$LEDGER\"\nDETAIL=SOURCE_LINE=0101:\nDETAIL=SOURCE_LINE=0102:exec 9>\"$LOCK\"\nDETAIL=SOURCE_LINE=0103:\nDETAIL=SOURCE_LINE=0104:flock -w 60 9 || {\nDETAIL=SOURCE_LINE=0105: echo 'RESULT=FAIL'\nDETAIL=SOURCE_LINE=0106: echo 'REASON=RUNNER_LOCK_TIMEOUT'\nDETAIL=SOURCE_LINE=0107: exit 71\nDETAIL=SOURCE_LINE=0108:}\nDETAIL=SOURCE_LINE=0109:\nDETAIL=SOURCE_LINE=0110:STARTED_AT=\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"\nDETAIL=SOURCE_LINE=0111:STAMP=\"$(date -u +%Y%m%dT%H%M%SZ)\"\nDETAIL=SOURCE_LINE=0112:HOST_SHORT=\"$(hostname -s 2>/dev/null || hostname)\"\nDETAIL=SOURCE_LINE=0113:\nDETAIL=SOURCE_LINE=0114:RAW_FILE=\"$RAW_DIR/${COMMAND_ID}-${STAMP}.log\"\nDETAIL=SOURCE_LINE=0115:SANITIZED_FILE=\"$PUBLIC_DIR/${COMMAND_ID}-${STAMP}.txt\"\nDETAIL=SOURCE_LINE=0116:JSON_FILE=\"$PUBLIC_DIR/${COMMAND_ID}-${STAMP}.json\"\nDETAIL=SOURCE_LINE=0117:TXT_FILE=\"$PUBLIC_DIR/${COMMAND_ID}-${STAMP}-latest.txt\"\nDETAIL=SOURCE_LINE=0118:\nDETAIL=SOURCE_LINE=0119:REFERENCE_SHA256=\"$(sha256sum \"$REFERENCE\" | awk '{print $1}')\"\nDETAIL=SOURCE_LINE=0120:ERROR_REGISTER_SHA256=\"$(sha256sum \"$ERROR_REGISTER\" | awk '{print $1}')\"\nDETAIL=SOURCE_LINE=0121:\nDETAIL=SOURCE_LINE=0122:COMMAND_SHA256=\"$(\nDETAIL=SOURCE_LINE=0123: {\nDETAIL=SOURCE_LINE=0124: printf '%s\\0' \"$MODE\"\nDETAIL=SOURCE_LINE=0125: printf '%s\\0' \"$COMPONENT\"\nDETAIL=SOURCE_LINE=0126: printf '%s\\0' \"$@\"\nDETAIL=SOURCE_LINE=0127: } |\nDETAIL=SOURCE_LINE=0128: sha256sum |\nDETAIL=SOURCE_LINE=0129: awk '{print $1}'\nDETAIL=SOURCE_LINE=0130:)\"\nDETAIL=SOURCE_LINE=0131:\nDETAIL=SOURCE_LINE=0132:PREVIOUS_FAILED_RC=\"$(\nDETAIL=SOURCE_LINE=0133: awk -F '\\t' -v hash=\"$COMMAND_SHA256\" '\nDETAIL=SOURCE_LINE=0134: $3 == hash && $4 != \"0\" {\nDETAIL=SOURCE_LINE=0135: value=$4\nDETAIL=SOURCE_LINE=0136: }\nDETAIL=SOURCE_LINE=0137:\nDETAIL=SOURCE_LINE=0138: END {\nDETAIL=SOURCE_LINE=0139: print value\nDETAIL=SOURCE_LINE=0140: }\nDETAIL=SOURCE_LINE=0141: ' \"$LEDGER\"\nDETAIL=SOURCE_LINE=0142:)\"\nDETAIL=SOURCE_LINE=0143:\nDETAIL=SOURCE_LINE=0144:RUN_STARTED=true\nDETAIL=SOURCE_LINE=0145:REPEAT_BLOCKED=false\nDETAIL=SOURCE_LINE=0146:BLOCK_REASON=''\nDETAIL=SOURCE_LINE=0147:CHANGES_MADE=false\nDETAIL=SOURCE_LINE=0148:\nDETAIL=SOURCE_LINE=0149:case \"$MODE\" in\nDETAIL=SOURCE_LINE=0150: change)\nDETAIL=SOURCE_LINE=0151: CHANGES_MADE=true\nDETAIL=SOURCE_LINE=0152: ;;\nDETAIL=SOURCE_LINE=0153:esac\nDETAIL=SOURCE_LINE=0154:\nDETAIL=SOURCE_LINE=0155:if [ -n \"$PREVIOUS_FAILED_RC\" ] &&\nDETAIL=SOURCE_LINE=0156: [ \"${HOMELAB_ALLOW_REPEAT:-0}\" != 1 ]; then\nDETAIL=SOURCE_LINE=0157:\nDETAIL=SOURCE_LINE=0158: RUN_STARTED=false\nDETAIL=SOURCE_LINE=0159: REPEAT_BLOCKED=true\nDETAIL=SOURCE_LINE=0160: BLOCK_REASON='EXACT_FAILED_COMMAND_REPEAT'\nDETAIL=SOURCE_LINE=0161: RC=73\nDETAIL=SOURCE_LINE=0162: STATUS='BLOCKED'\nDETAIL=SOURCE_LINE=0163:\nDETAIL=SOURCE_LINE=0164: {\nDETAIL=SOURCE_LINE=0165: echo 'COMMAND_EXECUTION_BLOCKED=yes'\nDETAIL=SOURCE_LINE=0166: echo 'REASON=EXACT_FAILED_COMMAND_REPEAT'\nDETAIL=SOURCE_LINE=0167: echo \"PREVIOUS_RC=$PREVIOUS_FAILED_RC\"\nDETAIL=SOURCE_LINE=0168: echo \"COMMAND_SHA256=$COMMAND_SHA256\"\nDETAIL=SOURCE_LINE=0169: echo\nDETAIL=SOURCE_LINE=0170: echo 'Set HOMELAB_ALLOW_REPEAT=1 only after reviewing the previous failure.'\nDETAIL=SOURCE_LINE=0171: } >\"$RAW_FILE\"\nDETAIL=SOURCE_LINE=0172:else\nDETAIL=SOURCE_LINE=0173: set +e\nDETAIL=SOURCE_LINE=0174:\nDETAIL=SOURCE_LINE=0175: if [ -n \"${HOMELAB_TIMEOUT:-}\" ]; then\nDETAIL=SOURCE_LINE=0176: case \"$HOMELAB_TIMEOUT\" in\nDETAIL=SOURCE_LINE=0177: *[!0-9]*|'')\nDETAIL=SOURCE_LINE=0178: echo 'RESULT=FAIL'\nDETAIL=SOURCE_LINE=0179: echo 'REASON=INVALID_HOMELAB_TIMEOUT'\nDETAIL=SOURCE_LINE=0180: exit 74\nDETAIL=SOURCE_LINE=0181: ;;\nDETAIL=SOURCE_LINE=0182: esac\nDETAIL=SOURCE_LINE=0183:\nDETAIL=SOURCE_LINE=0184: timeout \\\nDETAIL=SOURCE_LINE=0185: --foreground \\\nDETAIL=SOURCE_LINE=0186: --preserve-status \\\nDETAIL=SOURCE_LINE=0187: \"$HOMELAB_TIMEOUT\" \\\nDETAIL=SOURCE_LINE=0188: \"$@\" >\"$RAW_FILE\" 2>&1\nDETAIL=SOURCE_LINE=0189: RC=$?\nDETAIL=SOURCE_LINE=0190: else\nDETAIL=SOURCE_LINE=0191: \"$@\" >\"$RAW_FILE\" 2>&1\nDETAIL=SOURCE_LINE=0192: RC=$?\nDETAIL=SOURCE_LINE=0193: fi\nDETAIL=SOURCE_LINE=0194:\nDETAIL=SOURCE_LINE=0195: set -e\nDETAIL=SOURCE_LINE=0196:\nDETAIL=SOURCE_LINE=0197: if [ \"$RC\" -eq 0 ]; then\nDETAIL=SOURCE_LINE=0198: STATUS='OK'\nDETAIL=SOURCE_LINE=0199: else\nDETAIL=SOURCE_LINE=0200: STATUS='FAIL'\nDETAIL=SOURCE_LINE=0201: fi\nDETAIL=SOURCE_LINE=0202:fi\nDETAIL=SOURCE_LINE=0203:\nDETAIL=SOURCE_LINE=0204:FINISHED_AT=\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"\nDETAIL=SOURCE_LINE=0205:RAW_SHA256=\"$(sha256sum \"$RAW_FILE\" | awk '{print $1}')\"\nDETAIL=SOURCE_LINE=0206:\nDETAIL=SOURCE_LINE=0207:python3 - \"$RAW_FILE\" \"$SANITIZED_FILE\" <<'PY'\nDETAIL=SOURCE_LINE=0208:import re\nDETAIL=SOURCE_LINE=0209:import sys\nDETAIL=SOURCE_LINE=0210:from pathlib import Path\nDETAIL=SOURCE_LINE=0211:\nDETAIL=SOURCE_LINE=0212:source = Path(sys.argv[1])\nDETAIL=SOURCE_LINE=0213:target = Path(sys.argv[2])\nDETAIL=SOURCE_LINE=0214:\nDETAIL=SOURCE_LINE=0215:text = source.read_bytes().decode(\"utf-8\", errors=\"replace\")\nDETAIL=SOURCE_LINE=0216:\nDETAIL=SOURCE_LINE=0217:patterns = [\nDETAIL=SOURCE_LINE=0218: (\nDETAIL=SOURCE_LINE=0219: re.compile(\nDETAIL=SOURCE_LINE=0220: r\"-----BEGIN (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----\"\nDETAIL=SOURCE_LINE=0221: r\".*?\"\nDETAIL=SOURCE_LINE=0222: r\"-----END (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----\",\nDETAIL=SOURCE_LINE=0223: re.IGNORECASE | re.DOTALL,\nDETAIL=SOURCE_LINE=0224: ),\nDETAIL=SOURCE_LINE=0225: \"[PRIVATE_KEY_REDACTED]\",\nDETAIL=SOURCE_LINE=0226: ),\nDETAIL=SOURCE_LINE=0227: (\nDETAIL=SOURCE_LINE=0228: re.compile(\nDETAIL=SOURCE_LINE=0229: r\"(?im)^(\\s*(?:password|passwd|passphrase|authorization|\"\nDETAIL=SOURCE_LINE=...\nDETAIL=SOURCE_LINE=0292:maximum = 900_000\nDETAIL=SOURCE_LINE=0293:\nDETAIL=SOURCE_LINE=0294:if len(encoded) > maximum:\nDETAIL=SOURCE_LINE=0295: first = encoded[:440_000].decode(\"utf-8\", errors=\"ignore\")\nDETAIL=SOURCE_LINE=0296: last = encoded[-440_000:].decode(\"utf-8\", errors=\"ignore\")\nDETAIL=SOURCE_LINE=0297:\nDETAIL=SOURCE_LINE=0298: text = (\nDETAIL=SOURCE_LINE=0299: first\nDETAIL=SOURCE_LINE=0300: + \"\\n\\n\"\nDETAIL=SOURCE_LINE=0301: + \"===== PUBLIC OUTPUT TRUNCATED; FULL RAW LOG RETAINED LOCALLY =====\"\nDETAIL=SOURCE_LINE=0302: + \"\\n\\n\"\nDETAIL=SOURCE_LINE=0303: + last\nDETAIL=SOURCE_LINE=0304: )\nDETAIL=SOURCE_LINE=0305:\nDETAIL=SOURCE_LINE=0306:target.write_text(text, encoding=\"utf-8\")\nDETAIL=SOURCE_LINE=0307:PY\nDETAIL=SOURCE_LINE=0308:\nDETAIL=SOURCE_LINE=0309:SANITIZED_SHA256=\"$(\nDETAIL=SOURCE_LINE=0310: sha256sum \"$SANITIZED_FILE\" |\nDETAIL=SOURCE_LINE=0311: awk '{print $1}'\nDETAIL=SOURCE_LINE=0312:)\"\nDETAIL=SOURCE_LINE=0313:\nDETAIL=SOURCE_LINE=0314:python3 - \\\nDETAIL=SOURCE_LINE=0315: \"$JSON_FILE\" \\\nDETAIL=SOURCE_LINE=0316: \"$SANITIZED_FILE\" \\\nDETAIL=SOURCE_LINE=0317: \"$COMMAND_ID\" \\\nDETAIL=SOURCE_LINE=0318: \"$STATUS\" \\\nDETAIL=SOURCE_LINE=0319: \"$RC\" \\\nDETAIL=SOURCE_LINE=0320: \"$HOST_SHORT\" \\\nDETAIL=SOURCE_LINE=0321: \"$MODE\" \\\nDETAIL=SOURCE_LINE=0322: \"$COMPONENT\" \\\nDETAIL=SOURCE_LINE=0323: \"$STARTED_AT\" \\\nDETAIL=SOURCE_LINE=0324: \"$FINISHED_AT\" \\\nDETAIL=SOURCE_LINE=0325: \"$REFERENCE_SHA256\" \\\nDETAIL=SOURCE_LINE=0326: \"$ERROR_REGISTER_SHA256\" \\\nDETAIL=SOURCE_LINE=0327: \"$COMMAND_SHA256\" \\\nDETAIL=SOURCE_LINE=0328: \"$RAW_SHA256\" \\\nDETAIL=SOURCE_LINE=0329: \"$SANITIZED_SHA256\" \\\nDETAIL=SOURCE_LINE=0330: \"$RUN_STARTED\" \\\nDETAIL=SOURCE_LINE=0331: \"$REPEAT_BLOCKED\" \\\nDETAIL=SOURCE_LINE=0332: \"$BLOCK_REASON\" \\\nDETAIL=SOURCE_LINE=0333: \"$CHANGES_MADE\" <<'PY'\nDETAIL=SOURCE_LINE=0334:import json\nDETAIL=SOURCE_LINE=0335:import sys\nDETAIL=SOURCE_LINE=0336:from pathlib import Path\nDETAIL=SOURCE_LINE=0337:\nDETAIL=SOURCE_LINE=0338:(\nDETAIL=SOURCE_LINE=0339: json_path,\nDETAIL=SOURCE_LINE=0340: output_path,\nDETAIL=SOURCE_LINE=0341: command_id,\nDETAIL=SOURCE_LINE=0342: status,\nDETAIL=SOURCE_LINE=0343: rc,\nDETAIL=SOURCE_LINE=0344: host,\nDETAIL=SOURCE_LINE=0345: mode,\nDETAIL=SOURCE_LINE=0346: component,\nDETAIL=SOURCE_LINE=0347: started,\nDETAIL=SOURCE_LINE=0348: finished,\nDETAIL=SOURCE_LINE=0349: reference_sha,\nDETAIL=SOURCE_LINE=0350: error_sha,\nDETAIL=SOURCE_LINE=0351: command_sha,\nDETAIL=SOURCE_LINE=0352: raw_sha,\nDETAIL=SOURCE_LINE=0353: sanitized_sha,\nDETAIL=SOURCE_LINE=0354: run_started,\nDETAIL=SOURCE_LINE=0355: repeat_blocked,\nDETAIL=SOURCE_LINE=...\nDETAIL=SOURCE_LINE=0416: )\nDETAIL=SOURCE_LINE=0417: + \"\\n\",\nDETAIL=SOURCE_LINE=0418: encoding=\"utf-8\",\nDETAIL=SOURCE_LINE=0419:)\nDETAIL=SOURCE_LINE=0420:PY\nDETAIL=SOURCE_LINE=0421:\nDETAIL=SOURCE_LINE=0422:{\nDETAIL=SOURCE_LINE=0423: echo 'CHAT_OUTPUT_BEGIN'\nDETAIL=SOURCE_LINE=0424: echo \"COMMAND_ID=$COMMAND_ID\"\nDETAIL=SOURCE_LINE=0425: echo \"STATUS=$STATUS\"\nDETAIL=SOURCE_LINE=0426: echo \"RC=$RC\"\nDETAIL=SOURCE_LINE=0427: echo \"HOST=$HOST_SHORT\"\nDETAIL=SOURCE_LINE=0428: echo \"MODE=$MODE\"\nDETAIL=SOURCE_LINE=0429: echo \"COMPONENT=$COMPONENT\"\nDETAIL=SOURCE_LINE=0430: echo 'REFERENCE_REGISTER_CHECK=OK'\nDETAIL=SOURCE_LINE=0431: echo \"REFERENCE_SHA256=$REFERENCE_SHA256\"\nDETAIL=SOURCE_LINE=0432: echo 'ERROR_REGISTER_CHECK=OK'\nDETAIL=SOURCE_LINE=0433: echo \"ERROR_REGISTER_SHA256=$ERROR_REGISTER_SHA256\"\nDETAIL=SOURCE_LINE=0434: echo \"COMMAND_SHA256=$COMMAND_SHA256\"\nDETAIL=SOURCE_LINE=0435: echo \"DUPLICATE_FAILED_COMMAND_BLOCKED=$REPEAT_BLOCKED\"\nDETAIL=SOURCE_LINE=0436: echo \"EXECUTION_STARTED=$RUN_STARTED\"\nDETAIL=SOURCE_LINE=0437: echo \"CHANGES_MADE=$CHANGES_MADE\"\nDETAIL=SOURCE_LINE=0438: echo 'SANITIZED=yes'\nDETAIL=SOURCE_LINE=0439: echo 'SECRETS_INCLUDED=no'\nDETAIL=SOURCE_LINE=0440: echo 'PRIVATE_ADDRESSES_INCLUDED=no'\nDETAIL=SOURCE_LINE=0441: echo \"RAW_EVIDENCE_SHA256=$RAW_SHA256\"\nDETAIL=SOURCE_LINE=0442: echo \"SANITIZED_OUTPUT_SHA256=$SANITIZED_SHA256\"\nDETAIL=SOURCE_LINE=0443: echo 'OUTPUT_BEGIN'\nDETAIL=SOURCE_LINE=0444: cat \"$SANITIZED_FILE\"\nDETAIL=SOURCE_LINE=0445: echo\nDETAIL=SOURCE_LINE=0446: echo 'OUTPUT_END'\nDETAIL=SOURCE_LINE=0447: echo 'CHAT_OUTPUT_END'\nDETAIL=SOURCE_LINE=0448:} >\"$TXT_FILE\"\nDETAIL=SOURCE_LINE=0449:\nDETAIL=SOURCE_LINE=0450:chmod 0600 \\\nDETAIL=SOURCE_LINE=0451: \"$RAW_FILE\" \\\nDETAIL=SOURCE_LINE=0452: \"$SANITIZED_FILE\" \\\nDETAIL=SOURCE_LINE=0453: \"$JSON_FILE\" \\\nDETAIL=SOURCE_LINE=0454: \"$TXT_FILE\"\nDETAIL=SOURCE_LINE=0455:\nDETAIL=SOURCE_LINE=0456:TARGET=debian@[PRIVATE_IP]\nDETAIL=SOURCE_LINE=0457:REMOTE_PREFIX=\"/tmp/homelab-runtime-${COMMAND_ID}-$$\"\nDETAIL=SOURCE_LINE=0458:\nDETAIL=SOURCE_LINE=0459:SSH_OPTS=(\nDETAIL=SOURCE_LINE=...\nDETAIL=SOURCE_LINE=0509: PUBLIC_VISIBLE=true\nDETAIL=SOURCE_LINE=0510: break\nDETAIL=SOURCE_LINE=0511: fi\nDETAIL=SOURCE_LINE=0512:\nDETAIL=SOURCE_LINE=0513: sleep 2\nDETAIL=SOURCE_LINE=0514:done\nDETAIL=SOURCE_LINE=0515:\nDETAIL=SOURCE_LINE=0516:if [ \"$PUBLIC_VISIBLE\" != true ]; then\nDETAIL=SOURCE_LINE=0517: echo 'RESULT=FAIL'\nDETAIL=SOURCE_LINE=0518: echo 'REASON=PUBLIC_RUNTIME_RESULT_NOT_VISIBLE'\nDETAIL=SOURCE_LINE=0519: echo \"COMMAND_ID=$COMMAND_ID\"\nDETAIL=SOURCE_LINE=0520: echo \"LOCAL_RAW_FILE=$RAW_FILE\"\nDETAIL=SOURCE_LINE=0521: exit 75\nDETAIL=SOURCE_LINE=0522:fi\nDETAIL=SOURCE_LINE=0523:\nDETAIL=SOURCE_LINE=0524:printf '%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n' \\\nDETAIL=SOURCE_LINE=0525: \"$FINISHED_AT\" \\\nDETAIL=SOURCE_LINE=0526: \"$COMMAND_ID\" \\\nDETAIL=SOURCE_LINE=0527: \"$COMMAND_SHA256\" \\\nDETAIL=SOURCE_LINE=0528: \"$RC\" \\\nDETAIL=SOURCE_LINE=0529: \"$STATUS\" \\\nDETAIL=SOURCE_LINE=0530: \"$COMPONENT\" >>\"$LEDGER\"\nDETAIL=SOURCE_LINE=0531:\nDETAIL=SOURCE_LINE=0532:echo\nDETAIL=SOURCE_LINE=0533:echo '===== HOMELAB CHAT RUN COMPLETE ====='\nDETAIL=SOURCE_LINE=0534:echo \"COMMAND_ID=$COMMAND_ID\"\nDETAIL=SOURCE_LINE=0535:echo \"STATUS=$STATUS\"\nDETAIL=SOURCE_LINE=0536:echo \"RC=$RC\"\nDETAIL=SOURCE_LINE=0537:echo 'REFERENCE_REGISTER_CHECK=OK'\nDETAIL=SOURCE_LINE=0538:echo 'ERROR_REGISTER_CHECK=OK'\nDETAIL=SOURCE_LINE=0539:echo \"DUPLICATE_FAILED_COMMAND_BLOCKED=$REPEAT_BLOCKED\"\nDETAIL=SOURCE_LINE=0540:echo 'GITEA_PUBLICATION=OK'\nDETAIL=SOURCE_LINE=0541:echo \"PUBLIC_URL=$PUBLIC_URL\"\nDETAIL=SOURCE_LINE=0542:echo \"LOCAL_RAW_FILE=$RAW_FILE\"\nDETAIL=SOURCE_LINE=0543:echo 'SHELL_SESSION_PRESERVED=yes'\nDETAIL=SOURCE_LINE=0544:\nDETAIL=SOURCE_LINE=0545:exit \"$RC\"\nDETAIL=TOTAL_LINES=545\nDETAIL=EXACT_LOGIC_END\nDETAIL=DUPLICATE_EXACT_LOGIC_AUDIT=PASS\nDETAIL=SECRET_VALUES_OUTPUT=NONE\nCHANGES_MADE=NO\n" + "output": "AUDIT_ID=HOMELAB-CLUSTER-CRITICAL-SNAPSHOT-428\nAUDIT_TIME_UTC=2026-07-24T09:43:29Z\nCHANGES_PLANNED=NO\n\n===== PROXMOX_QUORUM =====\nCluster information\n-------------------\nName: homelab\nConfig Version: 3\nTransport: knet\nSecure auth: on\n\nQuorum information\n------------------\nDate: Fri Jul 24 12:43:29 2026\nQuorum provider: corosync_votequorum\nNodes: 3\nNode ID: 0x00000001\nRing ID: 1.ba\nQuorate: Yes\n\nVotequorum information\n----------------------\nExpected votes: 3\nHighest expected: 3\nTotal votes: 3\nQuorum: 2 \nFlags: Quorate \n\nMembership information\n----------------------\n Nodeid Votes Name\n0x00000001 1 [PRIVATE_IP] (local)\n0x00000002 1 [PRIVATE_IP]\n0x00000003 1 [PRIVATE_IP]\n\n===== PROXMOX_NODES =====\n+-------+--------+-------+-------+--------+-----------+-----------+-------------------------------------------------------------------------------------------------+-------------+\n| node | status | cpu | level | maxcpu | maxmem | mem | ssl_fingerprint | uptime |\n+=======+========+=======+=======+========+===========+===========+=================================================================================================+=============+\n| pve01 | online | 1.98% | | 24 | 31.09 GiB | 22.07 GiB | EB:7A:01:C5:02:1B:B7:F9:54:F4:38:8B:7F:03:1F:87:62:D0:6E:5F:C0:39:FB:B5:42:AE:58:6F:58:9A:9C:C1 | 20h 22m 46s |\n+-------+--------+-------+-------+--------+-----------+-----------+-------------------------------------------------------------------------------------------------+-------------+\n| pve02 | online | 8.49% | | 4 | 15.40 GiB | 11.36 GiB | C0:88:6E:89:CD:BF:0C:82:54:20:E7:11:DE:CA:ED:09:12:C6:9C:24:2A:1E:5C:33:49:EC:4E:E0:67:BE:79:05 | 20h 20m 42s |\n+-------+--------+-------+-------+--------+-----------+-----------+-------------------------------------------------------------------------------------------------+-------------+\n| pve03 | online | 4.43% | | 8 | 31.25 GiB | 11.24 GiB | 7E:FE:4A:DC:32:3C:02:84:A2:DD:23:84:FD:19:1E:8A:1B:B1:60:F6:F4:16:D0:28:3B:13:AE:65:7D:1F:38:32 | 20h 21m 35s |\n+-------+--------+-------+-------+--------+-----------+-----------+-------------------------------------------------------------------------------------------------+-------------+\n\n===== PROXMOX_STORAGE =====\n+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+\n| id | type | cgroup-mode | content | cpu | disk | diskread | diskwrite | hastate | host-arch | level | lock | maxcpu | maxdisk | maxmem | mem | memhost | name | netin | netout | network | network-type | node | plugintype | pool | protocol | sdn | shared | status | storage | tags | template | uptime | vmid | zone-type |\n+=========================+=========+=============+==========================+=====+============+==========+===========+=========+===========+=======+======+========+============+========+=====+=========+======+=======+========+=========+==============+=======+============+======+==========+=====+========+===========+===========+======+==========+========+======+===========+\n| storage/pve01/local | storage | | iso,backup,vztmpl,import | | 40.39 GiB | | | | | | | | 97.87 GiB | | | | | | | | | pve01 | dir | | | | 0 | available | local | | | | | |\n+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+\n| storage/pve01/local-lvm | storage | | images,rootdir | | 310.84 GiB | | | | | | | | 809.48 GiB | | | | | | | | | pve01 | lvmthin | | | | 0 | available | local-lvm | | | | | |\n+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+\n| storage/pve02/local | storage | | iso,backup,vztmpl,import | | 26.14 GiB | | | | | | | | 97.87 GiB | | | | | | | | | pve02 | dir | | | | 0 | available | local | | | | | |\n+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+\n| storage/pve02/local-lvm | storage | | images,rootdir | | 290.27 GiB | | | | | | | | 787.91 GiB | | | | | | | | | pve02 | lvmthin | | | | 0 | available | local-lvm | | | | | |\n+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+\n| storage/pve03/local | storage | | iso,backup,vztmpl,import | | 53.72 GiB | | | | | | | | 97.87 GiB | | | | | | | | | pve03 | dir | | | | 0 | available | local | | | | | |\n+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+\n| storage/pve03/local-lvm | storage | | images,rootdir | | 145.74 GiB | | | | | | | | 331.23 GiB | | | | | | | | | pve03 | lvmthin | | | | 0 | available | local-lvm | | | | | |\n+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+\n\n===== LOCAL_FILESYSTEMS =====\nFilesystem Size Used Avail Use% Mounted on\n/dev/mapper/pve-root 98G 41G 53G 44% /\nefivarfs 192K 135K 53K 72% /sys/firmware/efi/efivars\n/dev/nvme0n1p2 1022M 9.1M 1013M 1% /boot/efi\n/dev/mapper/pve-homelab--staging 295G 148G 132G 53% /mnt/staging\n/dev/fuse 128M 1.4M 127M 2% /etc/pve\noverlay 98G 41G 53G 44% /var/lib/containers/storage/overlay/4639d2b7096bff2bf1482d2b2b867a1a21a3e9eba88986eb78f9daa256eb5a77/merged\n\n===== LOCAL_INODES =====\nFilesystem Inodes IUsed IFree IUse% Mounted on\n/dev/mapper/pve-root 6553600 139099 6414501 3% /\nefivarfs 0 0 0 - /sys/firmware/efi/efivars\n/dev/nvme0n1p2 0 0 0 - /boot/efi\n/dev/mapper/pve-homelab--staging 19660800 4475 19656325 1% /mnt/staging\n/dev/fuse 262144 89 262055 1% /etc/pve\noverlay 6553600 139099 6414501 3% /var/lib/containers/storage/overlay/4639d2b7096bff2bf1482d2b2b867a1a21a3e9eba88986eb78f9daa256eb5a77/merged\n\n===== ZFS =====\nno pools available\n\n===== CEPH =====\nSTATE=NOT_CONFIGURED_OR_TOOL_ABSENT\n\n===== PVE01_FAILED_UNITS =====\nhomelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state\n\n===== PVE01_KERNEL_CRITICAL_24H =====\n\n===== PROXMOX_FAILED_TASKS =====\nUnknown option: errors\nUnknown option: limit\n400 unable to parse option\npvesh get [OPTIONS] [FORMAT_OPTIONS]\n\n===== TIME_SYNCHRONIZATION =====\nTimezone=Europe/Moscow\nLocalRTC=no\nNTPSynchronized=yes\n\n===== SKLADCHIK_SUPERVISOR =====\ncut: '\"': No such file or directory\nSCRIPT_SHA256=\nEXPECTED_SHA256=bd40ee19ea8b277d52f4240b97d3c79b825c90154272bac8a43004591bd0ddc3\nResult=success\nExecMainStatus=0\nActiveState=inactive\nSubState=dead\nSUPERVISOR_TIMER=active\nSTATUS=OK\nTS=2026-07-24T09:38:15Z\nproblems=\nbackup_status=OK\ncookie_expiry_status=OK\ncookie_jar=yes\nmonitor_status=OK\nrestore_status=OK\nPOST_FIX_COOKIE_FAILURES=0\n\n===== EDGE_VM =====\nedge-vm.gram1.ru\nactive\nactive\nbash: line 1: /var/lib/skladchik-reports-monitor/cookies.jar: Permission denied\nstat: missing operand\nTry 'stat --help' for more information.\nbash: -c: line 1: syntax error near unexpected token `('\nbash: -c: line 1: `sudo -n grep -E ^(STATUS|TS|reports|open|resolved|cookie_jar|cookie_expiry_status|problems)= /var/lib/skladchik-reports-monitor/health.txt'\nFilesystem Size Used Avail Use% Mounted on\n/dev/sdb1 95G 33G 58G 36% /\n\n===== CORE_APPS =====\ncore-apps.gram1.ru\nbash: line 1: {{.Status}}: command not found\n7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f /opt/stacks/gitea/data/gitea/public/.well-known/homelab-bootstrap.json\n7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f /opt/stacks/gitea/data/gitea/public/.well-known/homelab-bootstrap/7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f.json\nFilesystem Size Used Avail Use% Mounted on\n/dev/sda1 118G 42G 72G 37% /\n\n===== PUBLIC_BOOTSTRAP =====\nEXPECTED_BOOTSTRAP_SHA256=7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f\n7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f -\n7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f -\n\n===== READER_CAPABILITY_EXPOSURE =====\n/usr/bin/bash: -c: line 207: syntax error near unexpected token `('\n/usr/bin/bash: -c: line 207: ` \\\"/homelab-reader/[0-9a-fA-F]{64}([/\\\\\\\"]|$)\\\" \\'\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index ee624d2..4670fd5 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,7 +1,7 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=HOMELAB-DUPLICATE-EXACT-LOGIC-427 -STATUS=OK -RC=0 +COMMAND_ID=HOMELAB-CLUSTER-CRITICAL-SNAPSHOT-428 +STATUS=FAIL +RC=127 HOST=pve01 MODE=read-only COMPONENT=cluster-audit @@ -9,359 +9,167 @@ REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 -COMMAND_SHA256=7e4f72d9877110d6d6d4ef926a611b1bd8ad15fa4b2ad3ef17e9027ba49bc111 +COMMAND_SHA256=acb8cfc8df7523b005c7cc8d9dcc5e8d6955edc9869f35192b9aab7770149bb5 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGES_MADE=false SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=8480bb49baedd9fdebe212ee204d07d772e707d3c09045de0c3334fde723f6e9 -SANITIZED_OUTPUT_SHA256=5e8f3e2de086ae3c1d1bf84dcffbcb8dc8770ebefb00aa959db1ff15db3468b9 +RAW_EVIDENCE_SHA256=983c89419e9432c65180a176c36039953a5d07a157e9509f7eaa97016d1f65b6 +SANITIZED_OUTPUT_SHA256=50e542092e058ed4496ae5f17ec2fa7a38e02c39668e58030948a9916e980eb2 OUTPUT_BEGIN -DETAIL=FILE=/usr/local/sbin/homelab-chat-run-linked -DETAIL=SHA256=5e8725e129fd2055b36fe2fae13fc95fc4b3271074a0e0d4bb7347e7b2f270a1 -DETAIL=EXACT_LOGIC_BEGIN -DETAIL=TOTAL_LINES=39 -DETAIL=EXACT_LOGIC_END -DETAIL=FILE=/usr/local/sbin/homelab-chat-run -DETAIL=SHA256=2a88defef9517f89bc60934dce82097f8e316b08af9417e95167ea5d759a10d9 -DETAIL=EXACT_LOGIC_BEGIN -DETAIL=SOURCE_LINE=0048:esac -DETAIL=SOURCE_LINE=0049: -DETAIL=SOURCE_LINE=0050:case "$MODE" in -DETAIL=SOURCE_LINE=0051: read-only|verify|change|transport-test) -DETAIL=SOURCE_LINE=0052: ;; -DETAIL=SOURCE_LINE=0053: *) -DETAIL=SOURCE_LINE=0054: echo 'RESULT=FAIL' -DETAIL=SOURCE_LINE=0055: echo 'REASON=INVALID_MODE' -DETAIL=SOURCE_LINE=0056: exit 67 -DETAIL=SOURCE_LINE=0057: ;; -DETAIL=SOURCE_LINE=0058:esac -DETAIL=SOURCE_LINE=0059: -DETAIL=SOURCE_LINE=0060:for TOOL in \ -DETAIL=SOURCE_LINE=0061: awk \ -DETAIL=SOURCE_LINE=0062: curl \ -DETAIL=SOURCE_LINE=0063: flock \ -DETAIL=SOURCE_LINE=0064: python3 \ -DETAIL=SOURCE_LINE=0065: scp \ -DETAIL=SOURCE_LINE=0066: sha256sum \ -DETAIL=SOURCE_LINE=0067: ssh -DETAIL=SOURCE_LINE=0068:do -DETAIL=SOURCE_LINE=0069: command -v "$TOOL" >/dev/null 2>&1 || { -DETAIL=SOURCE_LINE=0070: echo 'RESULT=FAIL' -DETAIL=SOURCE_LINE=0071: echo "REASON=REQUIRED_TOOL_MISSING:$TOOL" -DETAIL=SOURCE_LINE=0072: exit 68 -DETAIL=SOURCE_LINE=0073: } -DETAIL=SOURCE_LINE=0074:done -DETAIL=SOURCE_LINE=0075: -DETAIL=SOURCE_LINE=0076:REFERENCE=/etc/pve/31_HOMELAB_REFERENCE.md -DETAIL=SOURCE_LINE=0077:ERROR_REGISTER=/etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md -DETAIL=SOURCE_LINE=0078: -DETAIL=SOURCE_LINE=0079:[ -r "$REFERENCE" ] || { -DETAIL=SOURCE_LINE=0080: echo 'RESULT=FAIL' -DETAIL=SOURCE_LINE=0081: echo 'REASON=REFERENCE_NOT_READABLE' -DETAIL=SOURCE_LINE=0082: exit 69 -DETAIL=SOURCE_LINE=0083:} -DETAIL=SOURCE_LINE=0084: -DETAIL=SOURCE_LINE=0085:[ -r "$ERROR_REGISTER" ] || { -DETAIL=SOURCE_LINE=0086: echo 'RESULT=FAIL' -DETAIL=SOURCE_LINE=0087: echo 'REASON=ERROR_REGISTER_NOT_READABLE' -DETAIL=SOURCE_LINE=0088: exit 70 -DETAIL=SOURCE_LINE=0089:} -DETAIL=SOURCE_LINE=0090: -DETAIL=SOURCE_LINE=0091:STATE_DIR=/var/lib/homelab-chat-run -DETAIL=SOURCE_LINE=0092:RAW_DIR="$STATE_DIR/raw" -DETAIL=SOURCE_LINE=0093:PUBLIC_DIR="$STATE_DIR/public" -DETAIL=SOURCE_LINE=0094:LEDGER="$STATE_DIR/state/command-ledger.tsv" -DETAIL=SOURCE_LINE=0095:LOCK="$STATE_DIR/state/runner.lock" -DETAIL=SOURCE_LINE=0096: -DETAIL=SOURCE_LINE=0097:mkdir -p "$RAW_DIR" "$PUBLIC_DIR" "$(dirname "$LEDGER")" -DETAIL=SOURCE_LINE=0098:chmod 0700 "$STATE_DIR" "$RAW_DIR" "$PUBLIC_DIR" "$(dirname "$LEDGER")" -DETAIL=SOURCE_LINE=0099:touch "$LEDGER" -DETAIL=SOURCE_LINE=0100:chmod 0600 "$LEDGER" -DETAIL=SOURCE_LINE=0101: -DETAIL=SOURCE_LINE=0102:exec 9>"$LOCK" -DETAIL=SOURCE_LINE=0103: -DETAIL=SOURCE_LINE=0104:flock -w 60 9 || { -DETAIL=SOURCE_LINE=0105: echo 'RESULT=FAIL' -DETAIL=SOURCE_LINE=0106: echo 'REASON=RUNNER_LOCK_TIMEOUT' -DETAIL=SOURCE_LINE=0107: exit 71 -DETAIL=SOURCE_LINE=0108:} -DETAIL=SOURCE_LINE=0109: -DETAIL=SOURCE_LINE=0110:STARTED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)" -DETAIL=SOURCE_LINE=0111:STAMP="$(date -u +%Y%m%dT%H%M%SZ)" -DETAIL=SOURCE_LINE=0112:HOST_SHORT="$(hostname -s 2>/dev/null || hostname)" -DETAIL=SOURCE_LINE=0113: -DETAIL=SOURCE_LINE=0114:RAW_FILE="$RAW_DIR/${COMMAND_ID}-${STAMP}.log" -DETAIL=SOURCE_LINE=0115:SANITIZED_FILE="$PUBLIC_DIR/${COMMAND_ID}-${STAMP}.txt" -DETAIL=SOURCE_LINE=0116:JSON_FILE="$PUBLIC_DIR/${COMMAND_ID}-${STAMP}.json" -DETAIL=SOURCE_LINE=0117:TXT_FILE="$PUBLIC_DIR/${COMMAND_ID}-${STAMP}-latest.txt" -DETAIL=SOURCE_LINE=0118: -DETAIL=SOURCE_LINE=0119:REFERENCE_SHA256="$(sha256sum "$REFERENCE" | awk '{print $1}')" -DETAIL=SOURCE_LINE=0120:ERROR_REGISTER_SHA256="$(sha256sum "$ERROR_REGISTER" | awk '{print $1}')" -DETAIL=SOURCE_LINE=0121: -DETAIL=SOURCE_LINE=0122:COMMAND_SHA256="$( -DETAIL=SOURCE_LINE=0123: { -DETAIL=SOURCE_LINE=0124: printf '%s\0' "$MODE" -DETAIL=SOURCE_LINE=0125: printf '%s\0' "$COMPONENT" -DETAIL=SOURCE_LINE=0126: printf '%s\0' "$@" -DETAIL=SOURCE_LINE=0127: } | -DETAIL=SOURCE_LINE=0128: sha256sum | -DETAIL=SOURCE_LINE=0129: awk '{print $1}' -DETAIL=SOURCE_LINE=0130:)" -DETAIL=SOURCE_LINE=0131: -DETAIL=SOURCE_LINE=0132:PREVIOUS_FAILED_RC="$( -DETAIL=SOURCE_LINE=0133: awk -F '\t' -v hash="$COMMAND_SHA256" ' -DETAIL=SOURCE_LINE=0134: $3 == hash && $4 != "0" { -DETAIL=SOURCE_LINE=0135: value=$4 -DETAIL=SOURCE_LINE=0136: } -DETAIL=SOURCE_LINE=0137: -DETAIL=SOURCE_LINE=0138: END { -DETAIL=SOURCE_LINE=0139: print value -DETAIL=SOURCE_LINE=0140: } -DETAIL=SOURCE_LINE=0141: ' "$LEDGER" -DETAIL=SOURCE_LINE=0142:)" -DETAIL=SOURCE_LINE=0143: -DETAIL=SOURCE_LINE=0144:RUN_STARTED=true -DETAIL=SOURCE_LINE=0145:REPEAT_BLOCKED=false -DETAIL=SOURCE_LINE=0146:BLOCK_REASON='' -DETAIL=SOURCE_LINE=0147:CHANGES_MADE=false -DETAIL=SOURCE_LINE=0148: -DETAIL=SOURCE_LINE=0149:case "$MODE" in -DETAIL=SOURCE_LINE=0150: change) -DETAIL=SOURCE_LINE=0151: CHANGES_MADE=true -DETAIL=SOURCE_LINE=0152: ;; -DETAIL=SOURCE_LINE=0153:esac -DETAIL=SOURCE_LINE=0154: -DETAIL=SOURCE_LINE=0155:if [ -n "$PREVIOUS_FAILED_RC" ] && -DETAIL=SOURCE_LINE=0156: [ "${HOMELAB_ALLOW_REPEAT:-0}" != 1 ]; then -DETAIL=SOURCE_LINE=0157: -DETAIL=SOURCE_LINE=0158: RUN_STARTED=false -DETAIL=SOURCE_LINE=0159: REPEAT_BLOCKED=true -DETAIL=SOURCE_LINE=0160: BLOCK_REASON='EXACT_FAILED_COMMAND_REPEAT' -DETAIL=SOURCE_LINE=0161: RC=73 -DETAIL=SOURCE_LINE=0162: STATUS='BLOCKED' -DETAIL=SOURCE_LINE=0163: -DETAIL=SOURCE_LINE=0164: { -DETAIL=SOURCE_LINE=0165: echo 'COMMAND_EXECUTION_BLOCKED=yes' -DETAIL=SOURCE_LINE=0166: echo 'REASON=EXACT_FAILED_COMMAND_REPEAT' -DETAIL=SOURCE_LINE=0167: echo "PREVIOUS_RC=$PREVIOUS_FAILED_RC" -DETAIL=SOURCE_LINE=0168: echo "COMMAND_SHA256=$COMMAND_SHA256" -DETAIL=SOURCE_LINE=0169: echo -DETAIL=SOURCE_LINE=0170: echo 'Set HOMELAB_ALLOW_REPEAT=1 only after reviewing the previous failure.' -DETAIL=SOURCE_LINE=0171: } >"$RAW_FILE" -DETAIL=SOURCE_LINE=0172:else -DETAIL=SOURCE_LINE=0173: set +e -DETAIL=SOURCE_LINE=0174: -DETAIL=SOURCE_LINE=0175: if [ -n "${HOMELAB_TIMEOUT:-}" ]; then -DETAIL=SOURCE_LINE=0176: case "$HOMELAB_TIMEOUT" in -DETAIL=SOURCE_LINE=0177: *[!0-9]*|'') -DETAIL=SOURCE_LINE=0178: echo 'RESULT=FAIL' -DETAIL=SOURCE_LINE=0179: echo 'REASON=INVALID_HOMELAB_TIMEOUT' -DETAIL=SOURCE_LINE=0180: exit 74 -DETAIL=SOURCE_LINE=0181: ;; -DETAIL=SOURCE_LINE=0182: esac -DETAIL=SOURCE_LINE=0183: -DETAIL=SOURCE_LINE=0184: timeout \ -DETAIL=SOURCE_LINE=0185: --foreground \ -DETAIL=SOURCE_LINE=0186: --preserve-status \ -DETAIL=SOURCE_LINE=0187: "$HOMELAB_TIMEOUT" \ -DETAIL=SOURCE_LINE=0188: "$@" >"$RAW_FILE" 2>&1 -DETAIL=SOURCE_LINE=0189: RC=$? -DETAIL=SOURCE_LINE=0190: else -DETAIL=SOURCE_LINE=0191: "$@" >"$RAW_FILE" 2>&1 -DETAIL=SOURCE_LINE=0192: RC=$? -DETAIL=SOURCE_LINE=0193: fi -DETAIL=SOURCE_LINE=0194: -DETAIL=SOURCE_LINE=0195: set -e -DETAIL=SOURCE_LINE=0196: -DETAIL=SOURCE_LINE=0197: if [ "$RC" -eq 0 ]; then -DETAIL=SOURCE_LINE=0198: STATUS='OK' -DETAIL=SOURCE_LINE=0199: else -DETAIL=SOURCE_LINE=0200: STATUS='FAIL' -DETAIL=SOURCE_LINE=0201: fi -DETAIL=SOURCE_LINE=0202:fi -DETAIL=SOURCE_LINE=0203: -DETAIL=SOURCE_LINE=0204:FINISHED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)" -DETAIL=SOURCE_LINE=0205:RAW_SHA256="$(sha256sum "$RAW_FILE" | awk '{print $1}')" -DETAIL=SOURCE_LINE=0206: -DETAIL=SOURCE_LINE=0207:python3 - "$RAW_FILE" "$SANITIZED_FILE" <<'PY' -DETAIL=SOURCE_LINE=0208:import re -DETAIL=SOURCE_LINE=0209:import sys -DETAIL=SOURCE_LINE=0210:from pathlib import Path -DETAIL=SOURCE_LINE=0211: -DETAIL=SOURCE_LINE=0212:source = Path(sys.argv[1]) -DETAIL=SOURCE_LINE=0213:target = Path(sys.argv[2]) -DETAIL=SOURCE_LINE=0214: -DETAIL=SOURCE_LINE=0215:text = source.read_bytes().decode("utf-8", errors="replace") -DETAIL=SOURCE_LINE=0216: -DETAIL=SOURCE_LINE=0217:patterns = [ -DETAIL=SOURCE_LINE=0218: ( -DETAIL=SOURCE_LINE=0219: re.compile( -DETAIL=SOURCE_LINE=0220: r"-----BEGIN (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----" -DETAIL=SOURCE_LINE=0221: r".*?" -DETAIL=SOURCE_LINE=0222: r"-----END (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----", -DETAIL=SOURCE_LINE=0223: re.IGNORECASE | re.DOTALL, -DETAIL=SOURCE_LINE=0224: ), -DETAIL=SOURCE_LINE=0225: "[PRIVATE_KEY_REDACTED]", -DETAIL=SOURCE_LINE=0226: ), -DETAIL=SOURCE_LINE=0227: ( -DETAIL=SOURCE_LINE=0228: re.compile( -DETAIL=SOURCE_LINE=0229: r"(?im)^(\s*(?:password|passwd|passphrase|authorization|" -DETAIL=SOURCE_LINE=... -DETAIL=SOURCE_LINE=0292:maximum = 900_000 -DETAIL=SOURCE_LINE=0293: -DETAIL=SOURCE_LINE=0294:if len(encoded) > maximum: -DETAIL=SOURCE_LINE=0295: first = encoded[:440_000].decode("utf-8", errors="ignore") -DETAIL=SOURCE_LINE=0296: last = encoded[-440_000:].decode("utf-8", errors="ignore") -DETAIL=SOURCE_LINE=0297: -DETAIL=SOURCE_LINE=0298: text = ( -DETAIL=SOURCE_LINE=0299: first -DETAIL=SOURCE_LINE=0300: + "\n\n" -DETAIL=SOURCE_LINE=0301: + "===== PUBLIC OUTPUT TRUNCATED; FULL RAW LOG RETAINED LOCALLY =====" -DETAIL=SOURCE_LINE=0302: + "\n\n" -DETAIL=SOURCE_LINE=0303: + last -DETAIL=SOURCE_LINE=0304: ) -DETAIL=SOURCE_LINE=0305: -DETAIL=SOURCE_LINE=0306:target.write_text(text, encoding="utf-8") -DETAIL=SOURCE_LINE=0307:PY -DETAIL=SOURCE_LINE=0308: -DETAIL=SOURCE_LINE=0309:SANITIZED_SHA256="$( -DETAIL=SOURCE_LINE=0310: sha256sum "$SANITIZED_FILE" | -DETAIL=SOURCE_LINE=0311: awk '{print $1}' -DETAIL=SOURCE_LINE=0312:)" -DETAIL=SOURCE_LINE=0313: -DETAIL=SOURCE_LINE=0314:python3 - \ -DETAIL=SOURCE_LINE=0315: "$JSON_FILE" \ -DETAIL=SOURCE_LINE=0316: "$SANITIZED_FILE" \ -DETAIL=SOURCE_LINE=0317: "$COMMAND_ID" \ -DETAIL=SOURCE_LINE=0318: "$STATUS" \ -DETAIL=SOURCE_LINE=0319: "$RC" \ -DETAIL=SOURCE_LINE=0320: "$HOST_SHORT" \ -DETAIL=SOURCE_LINE=0321: "$MODE" \ -DETAIL=SOURCE_LINE=0322: "$COMPONENT" \ -DETAIL=SOURCE_LINE=0323: "$STARTED_AT" \ -DETAIL=SOURCE_LINE=0324: "$FINISHED_AT" \ -DETAIL=SOURCE_LINE=0325: "$REFERENCE_SHA256" \ -DETAIL=SOURCE_LINE=0326: "$ERROR_REGISTER_SHA256" \ -DETAIL=SOURCE_LINE=0327: "$COMMAND_SHA256" \ -DETAIL=SOURCE_LINE=0328: "$RAW_SHA256" \ -DETAIL=SOURCE_LINE=0329: "$SANITIZED_SHA256" \ -DETAIL=SOURCE_LINE=0330: "$RUN_STARTED" \ -DETAIL=SOURCE_LINE=0331: "$REPEAT_BLOCKED" \ -DETAIL=SOURCE_LINE=0332: "$BLOCK_REASON" \ -DETAIL=SOURCE_LINE=0333: "$CHANGES_MADE" <<'PY' -DETAIL=SOURCE_LINE=0334:import json -DETAIL=SOURCE_LINE=0335:import sys -DETAIL=SOURCE_LINE=0336:from pathlib import Path -DETAIL=SOURCE_LINE=0337: -DETAIL=SOURCE_LINE=0338:( -DETAIL=SOURCE_LINE=0339: json_path, -DETAIL=SOURCE_LINE=0340: output_path, -DETAIL=SOURCE_LINE=0341: command_id, -DETAIL=SOURCE_LINE=0342: status, -DETAIL=SOURCE_LINE=0343: rc, -DETAIL=SOURCE_LINE=0344: host, -DETAIL=SOURCE_LINE=0345: mode, -DETAIL=SOURCE_LINE=0346: component, -DETAIL=SOURCE_LINE=0347: started, -DETAIL=SOURCE_LINE=0348: finished, -DETAIL=SOURCE_LINE=0349: reference_sha, -DETAIL=SOURCE_LINE=0350: error_sha, -DETAIL=SOURCE_LINE=0351: command_sha, -DETAIL=SOURCE_LINE=0352: raw_sha, -DETAIL=SOURCE_LINE=0353: sanitized_sha, -DETAIL=SOURCE_LINE=0354: run_started, -DETAIL=SOURCE_LINE=0355: repeat_blocked, -DETAIL=SOURCE_LINE=... -DETAIL=SOURCE_LINE=0416: ) -DETAIL=SOURCE_LINE=0417: + "\n", -DETAIL=SOURCE_LINE=0418: encoding="utf-8", -DETAIL=SOURCE_LINE=0419:) -DETAIL=SOURCE_LINE=0420:PY -DETAIL=SOURCE_LINE=0421: -DETAIL=SOURCE_LINE=0422:{ -DETAIL=SOURCE_LINE=0423: echo 'CHAT_OUTPUT_BEGIN' -DETAIL=SOURCE_LINE=0424: echo "COMMAND_ID=$COMMAND_ID" -DETAIL=SOURCE_LINE=0425: echo "STATUS=$STATUS" -DETAIL=SOURCE_LINE=0426: echo "RC=$RC" -DETAIL=SOURCE_LINE=0427: echo "HOST=$HOST_SHORT" -DETAIL=SOURCE_LINE=0428: echo "MODE=$MODE" -DETAIL=SOURCE_LINE=0429: echo "COMPONENT=$COMPONENT" -DETAIL=SOURCE_LINE=0430: echo 'REFERENCE_REGISTER_CHECK=OK' -DETAIL=SOURCE_LINE=0431: echo "REFERENCE_SHA256=$REFERENCE_SHA256" -DETAIL=SOURCE_LINE=0432: echo 'ERROR_REGISTER_CHECK=OK' -DETAIL=SOURCE_LINE=0433: echo "ERROR_REGISTER_SHA256=$ERROR_REGISTER_SHA256" -DETAIL=SOURCE_LINE=0434: echo "COMMAND_SHA256=$COMMAND_SHA256" -DETAIL=SOURCE_LINE=0435: echo "DUPLICATE_FAILED_COMMAND_BLOCKED=$REPEAT_BLOCKED" -DETAIL=SOURCE_LINE=0436: echo "EXECUTION_STARTED=$RUN_STARTED" -DETAIL=SOURCE_LINE=0437: echo "CHANGES_MADE=$CHANGES_MADE" -DETAIL=SOURCE_LINE=0438: echo 'SANITIZED=yes' -DETAIL=SOURCE_LINE=0439: echo 'SECRETS_INCLUDED=no' -DETAIL=SOURCE_LINE=0440: echo 'PRIVATE_ADDRESSES_INCLUDED=no' -DETAIL=SOURCE_LINE=0441: echo "RAW_EVIDENCE_SHA256=$RAW_SHA256" -DETAIL=SOURCE_LINE=0442: echo "SANITIZED_OUTPUT_SHA256=$SANITIZED_SHA256" -DETAIL=SOURCE_LINE=0443: echo 'OUTPUT_BEGIN' -DETAIL=SOURCE_LINE=0444: cat "$SANITIZED_FILE" -DETAIL=SOURCE_LINE=0445: echo -DETAIL=SOURCE_LINE=0446: echo 'OUTPUT_END' -DETAIL=SOURCE_LINE=0447: echo 'CHAT_OUTPUT_END' -DETAIL=SOURCE_LINE=0448:} >"$TXT_FILE" -DETAIL=SOURCE_LINE=0449: -DETAIL=SOURCE_LINE=0450:chmod 0600 \ -DETAIL=SOURCE_LINE=0451: "$RAW_FILE" \ -DETAIL=SOURCE_LINE=0452: "$SANITIZED_FILE" \ -DETAIL=SOURCE_LINE=0453: "$JSON_FILE" \ -DETAIL=SOURCE_LINE=0454: "$TXT_FILE" -DETAIL=SOURCE_LINE=0455: -DETAIL=SOURCE_LINE=0456:TARGET=debian@[PRIVATE_IP] -DETAIL=SOURCE_LINE=0457:REMOTE_PREFIX="/tmp/homelab-runtime-${COMMAND_ID}-$$" -DETAIL=SOURCE_LINE=0458: -DETAIL=SOURCE_LINE=0459:SSH_OPTS=( -DETAIL=SOURCE_LINE=... -DETAIL=SOURCE_LINE=0509: PUBLIC_VISIBLE=true -DETAIL=SOURCE_LINE=0510: break -DETAIL=SOURCE_LINE=0511: fi -DETAIL=SOURCE_LINE=0512: -DETAIL=SOURCE_LINE=0513: sleep 2 -DETAIL=SOURCE_LINE=0514:done -DETAIL=SOURCE_LINE=0515: -DETAIL=SOURCE_LINE=0516:if [ "$PUBLIC_VISIBLE" != true ]; then -DETAIL=SOURCE_LINE=0517: echo 'RESULT=FAIL' -DETAIL=SOURCE_LINE=0518: echo 'REASON=PUBLIC_RUNTIME_RESULT_NOT_VISIBLE' -DETAIL=SOURCE_LINE=0519: echo "COMMAND_ID=$COMMAND_ID" -DETAIL=SOURCE_LINE=0520: echo "LOCAL_RAW_FILE=$RAW_FILE" -DETAIL=SOURCE_LINE=0521: exit 75 -DETAIL=SOURCE_LINE=0522:fi -DETAIL=SOURCE_LINE=0523: -DETAIL=SOURCE_LINE=0524:printf '%s\t%s\t%s\t%s\t%s\t%s\n' \ -DETAIL=SOURCE_LINE=0525: "$FINISHED_AT" \ -DETAIL=SOURCE_LINE=0526: "$COMMAND_ID" \ -DETAIL=SOURCE_LINE=0527: "$COMMAND_SHA256" \ -DETAIL=SOURCE_LINE=0528: "$RC" \ -DETAIL=SOURCE_LINE=0529: "$STATUS" \ -DETAIL=SOURCE_LINE=0530: "$COMPONENT" >>"$LEDGER" -DETAIL=SOURCE_LINE=0531: -DETAIL=SOURCE_LINE=0532:echo -DETAIL=SOURCE_LINE=0533:echo '===== HOMELAB CHAT RUN COMPLETE =====' -DETAIL=SOURCE_LINE=0534:echo "COMMAND_ID=$COMMAND_ID" -DETAIL=SOURCE_LINE=0535:echo "STATUS=$STATUS" -DETAIL=SOURCE_LINE=0536:echo "RC=$RC" -DETAIL=SOURCE_LINE=0537:echo 'REFERENCE_REGISTER_CHECK=OK' -DETAIL=SOURCE_LINE=0538:echo 'ERROR_REGISTER_CHECK=OK' -DETAIL=SOURCE_LINE=0539:echo "DUPLICATE_FAILED_COMMAND_BLOCKED=$REPEAT_BLOCKED" -DETAIL=SOURCE_LINE=0540:echo 'GITEA_PUBLICATION=OK' -DETAIL=SOURCE_LINE=0541:echo "PUBLIC_URL=$PUBLIC_URL" -DETAIL=SOURCE_LINE=0542:echo "LOCAL_RAW_FILE=$RAW_FILE" -DETAIL=SOURCE_LINE=0543:echo 'SHELL_SESSION_PRESERVED=yes' -DETAIL=SOURCE_LINE=0544: -DETAIL=SOURCE_LINE=0545:exit "$RC" -DETAIL=TOTAL_LINES=545 -DETAIL=EXACT_LOGIC_END -DETAIL=DUPLICATE_EXACT_LOGIC_AUDIT=PASS -DETAIL=SECRET_VALUES_OUTPUT=NONE -CHANGES_MADE=NO +AUDIT_ID=HOMELAB-CLUSTER-CRITICAL-SNAPSHOT-428 +AUDIT_TIME_UTC=2026-07-24T09:43:29Z +CHANGES_PLANNED=NO + +===== PROXMOX_QUORUM ===== +Cluster information +------------------- +Name: homelab +Config Version: 3 +Transport: knet +Secure auth: on + +Quorum information +------------------ +Date: Fri Jul 24 12:43:29 2026 +Quorum provider: corosync_votequorum +Nodes: 3 +Node ID: 0x00000001 +Ring ID: 1.ba +Quorate: Yes + +Votequorum information +---------------------- +Expected votes: 3 +Highest expected: 3 +Total votes: 3 +Quorum: 2 +Flags: Quorate + +Membership information +---------------------- + Nodeid Votes Name +0x00000001 1 [PRIVATE_IP] (local) +0x00000002 1 [PRIVATE_IP] +0x00000003 1 [PRIVATE_IP] + +===== PROXMOX_NODES ===== ++-------+--------+-------+-------+--------+-----------+-----------+-------------------------------------------------------------------------------------------------+-------------+ +| node | status | cpu | level | maxcpu | maxmem | mem | ssl_fingerprint | uptime | ++=======+========+=======+=======+========+===========+===========+=================================================================================================+=============+ +| pve01 | online | 1.98% | | 24 | 31.09 GiB | 22.07 GiB | EB:7A:01:C5:02:1B:B7:F9:54:F4:38:8B:7F:03:1F:87:62:D0:6E:5F:C0:39:FB:B5:42:AE:58:6F:58:9A:9C:C1 | 20h 22m 46s | ++-------+--------+-------+-------+--------+-----------+-----------+-------------------------------------------------------------------------------------------------+-------------+ +| pve02 | online | 8.49% | | 4 | 15.40 GiB | 11.36 GiB | C0:88:6E:89:CD:BF:0C:82:54:20:E7:11:DE:CA:ED:09:12:C6:9C:24:2A:1E:5C:33:49:EC:4E:E0:67:BE:79:05 | 20h 20m 42s | ++-------+--------+-------+-------+--------+-----------+-----------+-------------------------------------------------------------------------------------------------+-------------+ +| pve03 | online | 4.43% | | 8 | 31.25 GiB | 11.24 GiB | 7E:FE:4A:DC:32:3C:02:84:A2:DD:23:84:FD:19:1E:8A:1B:B1:60:F6:F4:16:D0:28:3B:13:AE:65:7D:1F:38:32 | 20h 21m 35s | ++-------+--------+-------+-------+--------+-----------+-----------+-------------------------------------------------------------------------------------------------+-------------+ + +===== PROXMOX_STORAGE ===== ++-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+ +| id | type | cgroup-mode | content | cpu | disk | diskread | diskwrite | hastate | host-arch | level | lock | maxcpu | maxdisk | maxmem | mem | memhost | name | netin | netout | network | network-type | node | plugintype | pool | protocol | sdn | shared | status | storage | tags | template | uptime | vmid | zone-type | ++=========================+=========+=============+==========================+=====+============+==========+===========+=========+===========+=======+======+========+============+========+=====+=========+======+=======+========+=========+==============+=======+============+======+==========+=====+========+===========+===========+======+==========+========+======+===========+ +| storage/pve01/local | storage | | iso,backup,vztmpl,import | | 40.39 GiB | | | | | | | | 97.87 GiB | | | | | | | | | pve01 | dir | | | | 0 | available | local | | | | | | ++-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+ +| storage/pve01/local-lvm | storage | | images,rootdir | | 310.84 GiB | | | | | | | | 809.48 GiB | | | | | | | | | pve01 | lvmthin | | | | 0 | available | local-lvm | | | | | | ++-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+ +| storage/pve02/local | storage | | iso,backup,vztmpl,import | | 26.14 GiB | | | | | | | | 97.87 GiB | | | | | | | | | pve02 | dir | | | | 0 | available | local | | | | | | ++-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+ +| storage/pve02/local-lvm | storage | | images,rootdir | | 290.27 GiB | | | | | | | | 787.91 GiB | | | | | | | | | pve02 | lvmthin | | | | 0 | available | local-lvm | | | | | | ++-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+ +| storage/pve03/local | storage | | iso,backup,vztmpl,import | | 53.72 GiB | | | | | | | | 97.87 GiB | | | | | | | | | pve03 | dir | | | | 0 | available | local | | | | | | ++-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+ +| storage/pve03/local-lvm | storage | | images,rootdir | | 145.74 GiB | | | | | | | | 331.23 GiB | | | | | | | | | pve03 | lvmthin | | | | 0 | available | local-lvm | | | | | | ++-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+ + +===== LOCAL_FILESYSTEMS ===== +Filesystem Size Used Avail Use% Mounted on +/dev/mapper/pve-root 98G 41G 53G 44% / +efivarfs 192K 135K 53K 72% /sys/firmware/efi/efivars +/dev/nvme0n1p2 1022M 9.1M 1013M 1% /boot/efi +/dev/mapper/pve-homelab--staging 295G 148G 132G 53% /mnt/staging +/dev/fuse 128M 1.4M 127M 2% /etc/pve +overlay 98G 41G 53G 44% /var/lib/containers/storage/overlay/4639d2b7096bff2bf1482d2b2b867a1a21a3e9eba88986eb78f9daa256eb5a77/merged + +===== LOCAL_INODES ===== +Filesystem Inodes IUsed IFree IUse% Mounted on +/dev/mapper/pve-root 6553600 139099 6414501 3% / +efivarfs 0 0 0 - /sys/firmware/efi/efivars +/dev/nvme0n1p2 0 0 0 - /boot/efi +/dev/mapper/pve-homelab--staging 19660800 4475 19656325 1% /mnt/staging +/dev/fuse 262144 89 262055 1% /etc/pve +overlay 6553600 139099 6414501 3% /var/lib/containers/storage/overlay/4639d2b7096bff2bf1482d2b2b867a1a21a3e9eba88986eb78f9daa256eb5a77/merged + +===== ZFS ===== +no pools available + +===== CEPH ===== +STATE=NOT_CONFIGURED_OR_TOOL_ABSENT + +===== PVE01_FAILED_UNITS ===== +homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state + +===== PVE01_KERNEL_CRITICAL_24H ===== + +===== PROXMOX_FAILED_TASKS ===== +Unknown option: errors +Unknown option: limit +400 unable to parse option +pvesh get [OPTIONS] [FORMAT_OPTIONS] + +===== TIME_SYNCHRONIZATION ===== +Timezone=Europe/Moscow +LocalRTC=no +NTPSynchronized=yes + +===== SKLADCHIK_SUPERVISOR ===== +cut: '"': No such file or directory +SCRIPT_SHA256= +EXPECTED_SHA256=bd40ee19ea8b277d52f4240b97d3c79b825c90154272bac8a43004591bd0ddc3 +Result=success +ExecMainStatus=0 +ActiveState=inactive +SubState=dead +SUPERVISOR_TIMER=active +STATUS=OK +TS=2026-07-24T09:38:15Z +problems= +backup_status=OK +cookie_expiry_status=OK +cookie_jar=yes +monitor_status=OK +restore_status=OK +POST_FIX_COOKIE_FAILURES=0 + +===== EDGE_VM ===== +edge-vm.gram1.ru +active +active +bash: line 1: /var/lib/skladchik-reports-monitor/cookies.jar: Permission denied +stat: missing operand +Try 'stat --help' for more information. +bash: -c: line 1: syntax error near unexpected token `(' +bash: -c: line 1: `sudo -n grep -E ^(STATUS|TS|reports|open|resolved|cookie_jar|cookie_expiry_status|problems)= /var/lib/skladchik-reports-monitor/health.txt' +Filesystem Size Used Avail Use% Mounted on +/dev/sdb1 95G 33G 58G 36% / + +===== CORE_APPS ===== +core-apps.gram1.ru +bash: line 1: {{.Status}}: command not found +7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f /opt/stacks/gitea/data/gitea/public/.well-known/homelab-bootstrap.json +7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f /opt/stacks/gitea/data/gitea/public/.well-known/homelab-bootstrap/7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f.json +Filesystem Size Used Avail Use% Mounted on +/dev/sda1 118G 42G 72G 37% / + +===== PUBLIC_BOOTSTRAP ===== +EXPECTED_BOOTSTRAP_SHA256=7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f +7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f - +7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f - + +===== READER_CAPABILITY_EXPOSURE ===== +/usr/bin/bash: -c: line 207: syntax error near unexpected token `(' +/usr/bin/bash: -c: line 207: ` \"/homelab-reader/[0-9a-fA-F]{64}([/\\\"]|$)\" \' OUTPUT_END CHAT_OUTPUT_END