From 0707577c8f6ab27c0f2e9b1909e0f524b0787d02 Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Fri, 24 Jul 2026 09:47:51 +0000 Subject: [PATCH] runtime: publish HOMELAB-BACKUP-MATRIX-FAILURE-AUDIT-429 --- ...MELAB-BACKUP-MATRIX-FAILURE-AUDIT-429.json | 30 ++ ...OMELAB-BACKUP-MATRIX-FAILURE-AUDIT-429.txt | 211 +++++++++++ runtime/latest.json | 18 +- runtime/latest.txt | 350 ++++++++++-------- 4 files changed, 443 insertions(+), 166 deletions(-) create mode 100644 runtime/history/HOMELAB-BACKUP-MATRIX-FAILURE-AUDIT-429.json create mode 100644 runtime/history/HOMELAB-BACKUP-MATRIX-FAILURE-AUDIT-429.txt diff --git a/runtime/history/HOMELAB-BACKUP-MATRIX-FAILURE-AUDIT-429.json b/runtime/history/HOMELAB-BACKUP-MATRIX-FAILURE-AUDIT-429.json new file mode 100644 index 0000000..01c951c --- /dev/null +++ b/runtime/history/HOMELAB-BACKUP-MATRIX-FAILURE-AUDIT-429.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "HOMELAB-BACKUP-MATRIX-FAILURE-AUDIT-429", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "cluster-audit", + "started_at_utc": "2026-07-24T09:47:50Z", + "finished_at_utc": "2026-07-24T09:47:50Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", + "command_sha256": "d361e4c1100f3da6fa9e708a966bc5ce11d491b55027d4766e5dbc29f3699816", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "b507fad0e10f74eee68181be851d7b53e3cf8f906f9815d2c42534be5d0cac55", + "sanitized_output_sha256": "b507fad0e10f74eee68181be851d7b53e3cf8f906f9815d2c42534be5d0cac55", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "DETAIL=SERVICE_STATE_BEGIN\nDETAIL=SERVICE_PROPERTY=Result=exit-code\nDETAIL=SERVICE_PROPERTY=ExecMainCode=1\nDETAIL=SERVICE_PROPERTY=ExecMainStatus=1\nDETAIL=SERVICE_PROPERTY=ExecStart={ path=/usr/local/sbin/homelab-backup-matrix-refresh ; argv[]=/usr/local/sbin/homelab-backup-matrix-refresh ; ignore_errors=no ; start_time=[Fri 2026-07-24 10:06:04 MSK] ; stop_time=[Fri 2026-07-24 10:06:05 MSK] ; pid=3581151 ; code=exited ; status=1 }\nDETAIL=SERVICE_PROPERTY=WorkingDirectory=\nDETAIL=SERVICE_PROPERTY=User=\nDETAIL=SERVICE_PROPERTY=Group=\nDETAIL=SERVICE_PROPERTY=LoadState=loaded\nDETAIL=SERVICE_PROPERTY=ActiveState=failed\nDETAIL=SERVICE_PROPERTY=SubState=failed\nDETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/homelab-backup-matrix-refresh.service\nDETAIL=SERVICE_PROPERTY=DropInPaths=\nDETAIL=SERVICE_STATE_END\nDETAIL=TIMER_STATE_BEGIN\nDETAIL=TIMER_PROPERTY=Unit=homelab-backup-matrix-refresh.service\nDETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=\nDETAIL=TIMER_PROPERTY=LastTriggerUSec=\nDETAIL=TIMER_PROPERTY=LoadState=loaded\nDETAIL=TIMER_PROPERTY=ActiveState=inactive\nDETAIL=TIMER_PROPERTY=SubState=dead\nDETAIL=TIMER_STATE_END\nDETAIL=LAST_FAILURE_BEGIN\nDETAIL=JOURNAL=2026-07-24T04:18:06+03:00 pve01 homelab-backup-matrix-refresh[2451935]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T04:18:06+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T04:18:06+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T04:18:06+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=JOURNAL=2026-07-24T04:28:41+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nDETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_JOBS_TOTAL=41\nDETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_JOBS_OK=30\nDETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_JOBS_BAD=6\nDETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_JOBS_UNMAPPED=0\nDETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_JOBS_DISABLED=5\nDETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nDETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=JOURNAL=2026-07-24T04:50:27+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nDETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_JOBS_TOTAL=41\nDETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_JOBS_OK=30\nDETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_JOBS_BAD=6\nDETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_JOBS_UNMAPPED=0\nDETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_JOBS_DISABLED=5\nDETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nDETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=JOURNAL=2026-07-24T05:21:32+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nDETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_JOBS_TOTAL=41\nDETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_JOBS_OK=30\nDETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_JOBS_BAD=6\nDETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_JOBS_UNMAPPED=0\nDETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_JOBS_DISABLED=5\nDETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nDETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=JOURNAL=2026-07-24T05:54:45+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nDETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_JOBS_TOTAL=41\nDETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_JOBS_OK=30\nDETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_JOBS_BAD=6\nDETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_JOBS_UNMAPPED=0\nDETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_JOBS_DISABLED=5\nDETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nDETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=JOURNAL=2026-07-24T06:08:12+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nDETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_JOBS_TOTAL=41\nDETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_JOBS_OK=30\nDETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_JOBS_BAD=6\nDETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_JOBS_UNMAPPED=0\nDETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_JOBS_DISABLED=5\nDETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nDETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=JOURNAL=2026-07-24T06:46:41+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nDETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_JOBS_TOTAL=41\nDETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_JOBS_OK=30\nDETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_JOBS_BAD=6\nDETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_JOBS_UNMAPPED=0\nDETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_JOBS_DISABLED=5\nDETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nDETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=JOURNAL=2026-07-24T07:04:41+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nDETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_JOBS_TOTAL=41\nDETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_JOBS_OK=30\nDETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_JOBS_BAD=6\nDETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_JOBS_UNMAPPED=0\nDETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_JOBS_DISABLED=5\nDETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nDETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=JOURNAL=2026-07-24T07:43:04+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nDETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_JOBS_TOTAL=41\nDETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_JOBS_OK=30\nDETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_JOBS_BAD=6\nDETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_JOBS_UNMAPPED=0\nDETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_JOBS_DISABLED=5\nDETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nDETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=JOURNAL=2026-07-24T08:20:04+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nDETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_JOBS_TOTAL=41\nDETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_JOBS_OK=30\nDETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_JOBS_BAD=6\nDETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_JOBS_UNMAPPED=0\nDETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_JOBS_DISABLED=5\nDETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nDETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=JOURNAL=2026-07-24T08:26:24+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nDETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_JOBS_TOTAL=41\nDETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_JOBS_OK=30\nDETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_JOBS_BAD=6\nDETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_JOBS_UNMAPPED=0\nDETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_JOBS_DISABLED=5\nDETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nDETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=JOURNAL=2026-07-24T08:35:15+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nDETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_JOBS_TOTAL=41\nDETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_JOBS_OK=30\nDETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_JOBS_BAD=6\nDETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_JOBS_UNMAPPED=0\nDETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_JOBS_DISABLED=5\nDETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nDETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=JOURNAL=2026-07-24T08:37:50+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nDETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_JOBS_TOTAL=41\nDETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_JOBS_OK=30\nDETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_JOBS_BAD=6\nDETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_JOBS_UNMAPPED=0\nDETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_JOBS_DISABLED=5\nDETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nDETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=JOURNAL=2026-07-24T10:06:04+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nDETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_JOBS_TOTAL=41\nDETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_JOBS_OK=30\nDETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_JOBS_BAD=6\nDETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_JOBS_UNMAPPED=0\nDETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_JOBS_DISABLED=5\nDETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nDETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=LAST_FAILURE_END\nDETAIL=SECRET_VALUES_OUTPUT=NONE\nDETAIL=BACKUP_MATRIX_FAILURE_AUDIT=PASS\nCHANGES_MADE=NO\n" +} diff --git a/runtime/history/HOMELAB-BACKUP-MATRIX-FAILURE-AUDIT-429.txt b/runtime/history/HOMELAB-BACKUP-MATRIX-FAILURE-AUDIT-429.txt new file mode 100644 index 0000000..6e427f6 --- /dev/null +++ b/runtime/history/HOMELAB-BACKUP-MATRIX-FAILURE-AUDIT-429.txt @@ -0,0 +1,211 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=HOMELAB-BACKUP-MATRIX-FAILURE-AUDIT-429 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=cluster-audit +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 +COMMAND_SHA256=d361e4c1100f3da6fa9e708a966bc5ce11d491b55027d4766e5dbc29f3699816 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=false +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=b507fad0e10f74eee68181be851d7b53e3cf8f906f9815d2c42534be5d0cac55 +SANITIZED_OUTPUT_SHA256=b507fad0e10f74eee68181be851d7b53e3cf8f906f9815d2c42534be5d0cac55 +OUTPUT_BEGIN +DETAIL=SERVICE_STATE_BEGIN +DETAIL=SERVICE_PROPERTY=Result=exit-code +DETAIL=SERVICE_PROPERTY=ExecMainCode=1 +DETAIL=SERVICE_PROPERTY=ExecMainStatus=1 +DETAIL=SERVICE_PROPERTY=ExecStart={ path=/usr/local/sbin/homelab-backup-matrix-refresh ; argv[]=/usr/local/sbin/homelab-backup-matrix-refresh ; ignore_errors=no ; start_time=[Fri 2026-07-24 10:06:04 MSK] ; stop_time=[Fri 2026-07-24 10:06:05 MSK] ; pid=3581151 ; code=exited ; status=1 } +DETAIL=SERVICE_PROPERTY=WorkingDirectory= +DETAIL=SERVICE_PROPERTY=User= +DETAIL=SERVICE_PROPERTY=Group= +DETAIL=SERVICE_PROPERTY=LoadState=loaded +DETAIL=SERVICE_PROPERTY=ActiveState=failed +DETAIL=SERVICE_PROPERTY=SubState=failed +DETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/homelab-backup-matrix-refresh.service +DETAIL=SERVICE_PROPERTY=DropInPaths= +DETAIL=SERVICE_STATE_END +DETAIL=TIMER_STATE_BEGIN +DETAIL=TIMER_PROPERTY=Unit=homelab-backup-matrix-refresh.service +DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime= +DETAIL=TIMER_PROPERTY=LastTriggerUSec= +DETAIL=TIMER_PROPERTY=LoadState=loaded +DETAIL=TIMER_PROPERTY=ActiveState=inactive +DETAIL=TIMER_PROPERTY=SubState=dead +DETAIL=TIMER_STATE_END +DETAIL=LAST_FAILURE_BEGIN +DETAIL=JOURNAL=2026-07-24T04:18:06+03:00 pve01 homelab-backup-matrix-refresh[2451935]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T04:18:06+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T04:18:06+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T04:18:06+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=JOURNAL=2026-07-24T04:28:41+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +DETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_STATUS=FAIL +DETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_JOBS_TOTAL=41 +DETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_JOBS_OK=30 +DETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_JOBS_BAD=6 +DETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_JOBS_UNMAPPED=0 +DETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_JOBS_DISABLED=5 +DETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +DETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=JOURNAL=2026-07-24T04:50:27+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +DETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_STATUS=FAIL +DETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_JOBS_TOTAL=41 +DETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_JOBS_OK=30 +DETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_JOBS_BAD=6 +DETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_JOBS_UNMAPPED=0 +DETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_JOBS_DISABLED=5 +DETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +DETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=JOURNAL=2026-07-24T05:21:32+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +DETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_STATUS=FAIL +DETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_JOBS_TOTAL=41 +DETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_JOBS_OK=30 +DETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_JOBS_BAD=6 +DETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_JOBS_UNMAPPED=0 +DETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_JOBS_DISABLED=5 +DETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +DETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=JOURNAL=2026-07-24T05:54:45+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +DETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_STATUS=FAIL +DETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_JOBS_TOTAL=41 +DETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_JOBS_OK=30 +DETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_JOBS_BAD=6 +DETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_JOBS_UNMAPPED=0 +DETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_JOBS_DISABLED=5 +DETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +DETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=JOURNAL=2026-07-24T06:08:12+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +DETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_STATUS=FAIL +DETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_JOBS_TOTAL=41 +DETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_JOBS_OK=30 +DETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_JOBS_BAD=6 +DETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_JOBS_UNMAPPED=0 +DETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_JOBS_DISABLED=5 +DETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +DETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=JOURNAL=2026-07-24T06:46:41+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +DETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_STATUS=FAIL +DETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_JOBS_TOTAL=41 +DETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_JOBS_OK=30 +DETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_JOBS_BAD=6 +DETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_JOBS_UNMAPPED=0 +DETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_JOBS_DISABLED=5 +DETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +DETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=JOURNAL=2026-07-24T07:04:41+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +DETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_STATUS=FAIL +DETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_JOBS_TOTAL=41 +DETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_JOBS_OK=30 +DETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_JOBS_BAD=6 +DETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_JOBS_UNMAPPED=0 +DETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_JOBS_DISABLED=5 +DETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +DETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=JOURNAL=2026-07-24T07:43:04+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +DETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_STATUS=FAIL +DETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_JOBS_TOTAL=41 +DETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_JOBS_OK=30 +DETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_JOBS_BAD=6 +DETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_JOBS_UNMAPPED=0 +DETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_JOBS_DISABLED=5 +DETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +DETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=JOURNAL=2026-07-24T08:20:04+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +DETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_STATUS=FAIL +DETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_JOBS_TOTAL=41 +DETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_JOBS_OK=30 +DETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_JOBS_BAD=6 +DETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_JOBS_UNMAPPED=0 +DETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_JOBS_DISABLED=5 +DETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +DETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=JOURNAL=2026-07-24T08:26:24+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +DETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_STATUS=FAIL +DETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_JOBS_TOTAL=41 +DETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_JOBS_OK=30 +DETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_JOBS_BAD=6 +DETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_JOBS_UNMAPPED=0 +DETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_JOBS_DISABLED=5 +DETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +DETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=JOURNAL=2026-07-24T08:35:15+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +DETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_STATUS=FAIL +DETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_JOBS_TOTAL=41 +DETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_JOBS_OK=30 +DETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_JOBS_BAD=6 +DETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_JOBS_UNMAPPED=0 +DETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_JOBS_DISABLED=5 +DETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +DETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=JOURNAL=2026-07-24T08:37:50+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +DETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_STATUS=FAIL +DETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_JOBS_TOTAL=41 +DETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_JOBS_OK=30 +DETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_JOBS_BAD=6 +DETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_JOBS_UNMAPPED=0 +DETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_JOBS_DISABLED=5 +DETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +DETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=JOURNAL=2026-07-24T10:06:04+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +DETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_STATUS=FAIL +DETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_JOBS_TOTAL=41 +DETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_JOBS_OK=30 +DETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_JOBS_BAD=6 +DETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_JOBS_UNMAPPED=0 +DETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_JOBS_DISABLED=5 +DETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +DETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=LAST_FAILURE_END +DETAIL=SECRET_VALUES_OUTPUT=NONE +DETAIL=BACKUP_MATRIX_FAILURE_AUDIT=PASS +CHANGES_MADE=NO + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index d52e43c..01c951c 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "HOMELAB-CLUSTER-CRITICAL-SNAPSHOT-428", - "status": "FAIL", - "rc": 127, + "command_id": "HOMELAB-BACKUP-MATRIX-FAILURE-AUDIT-429", + "status": "OK", + "rc": 0, "host": "pve01", "mode": "read-only", "component": "cluster-audit", - "started_at_utc": "2026-07-24T09:43:29Z", - "finished_at_utc": "2026-07-24T09:43:33Z", + "started_at_utc": "2026-07-24T09:47:50Z", + "finished_at_utc": "2026-07-24T09:47:50Z", "reference_register_checked": true, "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", "error_register_checked": true, "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", - "command_sha256": "acb8cfc8df7523b005c7cc8d9dcc5e8d6955edc9869f35192b9aab7770149bb5", + "command_sha256": "d361e4c1100f3da6fa9e708a966bc5ce11d491b55027d4766e5dbc29f3699816", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -22,9 +22,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "983c89419e9432c65180a176c36039953a5d07a157e9509f7eaa97016d1f65b6", - "sanitized_output_sha256": "50e542092e058ed4496ae5f17ec2fa7a38e02c39668e58030948a9916e980eb2", + "raw_evidence_sha256": "b507fad0e10f74eee68181be851d7b53e3cf8f906f9815d2c42534be5d0cac55", + "sanitized_output_sha256": "b507fad0e10f74eee68181be851d7b53e3cf8f906f9815d2c42534be5d0cac55", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "AUDIT_ID=HOMELAB-CLUSTER-CRITICAL-SNAPSHOT-428\nAUDIT_TIME_UTC=2026-07-24T09:43:29Z\nCHANGES_PLANNED=NO\n\n===== PROXMOX_QUORUM =====\nCluster information\n-------------------\nName: homelab\nConfig Version: 3\nTransport: knet\nSecure auth: on\n\nQuorum information\n------------------\nDate: Fri Jul 24 12:43:29 2026\nQuorum provider: corosync_votequorum\nNodes: 3\nNode ID: 0x00000001\nRing ID: 1.ba\nQuorate: Yes\n\nVotequorum information\n----------------------\nExpected votes: 3\nHighest expected: 3\nTotal votes: 3\nQuorum: 2 \nFlags: Quorate \n\nMembership information\n----------------------\n Nodeid Votes Name\n0x00000001 1 [PRIVATE_IP] (local)\n0x00000002 1 [PRIVATE_IP]\n0x00000003 1 [PRIVATE_IP]\n\n===== PROXMOX_NODES =====\n+-------+--------+-------+-------+--------+-----------+-----------+-------------------------------------------------------------------------------------------------+-------------+\n| node | status | cpu | level | maxcpu | maxmem | mem | ssl_fingerprint | uptime |\n+=======+========+=======+=======+========+===========+===========+=================================================================================================+=============+\n| pve01 | online | 1.98% | | 24 | 31.09 GiB | 22.07 GiB | EB:7A:01:C5:02:1B:B7:F9:54:F4:38:8B:7F:03:1F:87:62:D0:6E:5F:C0:39:FB:B5:42:AE:58:6F:58:9A:9C:C1 | 20h 22m 46s |\n+-------+--------+-------+-------+--------+-----------+-----------+-------------------------------------------------------------------------------------------------+-------------+\n| pve02 | online | 8.49% | | 4 | 15.40 GiB | 11.36 GiB | C0:88:6E:89:CD:BF:0C:82:54:20:E7:11:DE:CA:ED:09:12:C6:9C:24:2A:1E:5C:33:49:EC:4E:E0:67:BE:79:05 | 20h 20m 42s |\n+-------+--------+-------+-------+--------+-----------+-----------+-------------------------------------------------------------------------------------------------+-------------+\n| pve03 | online | 4.43% | | 8 | 31.25 GiB | 11.24 GiB | 7E:FE:4A:DC:32:3C:02:84:A2:DD:23:84:FD:19:1E:8A:1B:B1:60:F6:F4:16:D0:28:3B:13:AE:65:7D:1F:38:32 | 20h 21m 35s |\n+-------+--------+-------+-------+--------+-----------+-----------+-------------------------------------------------------------------------------------------------+-------------+\n\n===== PROXMOX_STORAGE =====\n+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+\n| id | type | cgroup-mode | content | cpu | disk | diskread | diskwrite | hastate | host-arch | level | lock | maxcpu | maxdisk | maxmem | mem | memhost | name | netin | netout | network | network-type | node | plugintype | pool | protocol | sdn | shared | status | storage | tags | template | uptime | vmid | zone-type |\n+=========================+=========+=============+==========================+=====+============+==========+===========+=========+===========+=======+======+========+============+========+=====+=========+======+=======+========+=========+==============+=======+============+======+==========+=====+========+===========+===========+======+==========+========+======+===========+\n| storage/pve01/local | storage | | iso,backup,vztmpl,import | | 40.39 GiB | | | | | | | | 97.87 GiB | | | | | | | | | pve01 | dir | | | | 0 | available | local | | | | | |\n+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+\n| storage/pve01/local-lvm | storage | | images,rootdir | | 310.84 GiB | | | | | | | | 809.48 GiB | | | | | | | | | pve01 | lvmthin | | | | 0 | available | local-lvm | | | | | |\n+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+\n| storage/pve02/local | storage | | iso,backup,vztmpl,import | | 26.14 GiB | | | | | | | | 97.87 GiB | | | | | | | | | pve02 | dir | | | | 0 | available | local | | | | | |\n+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+\n| storage/pve02/local-lvm | storage | | images,rootdir | | 290.27 GiB | | | | | | | | 787.91 GiB | | | | | | | | | pve02 | lvmthin | | | | 0 | available | local-lvm | | | | | |\n+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+\n| storage/pve03/local | storage | | iso,backup,vztmpl,import | | 53.72 GiB | | | | | | | | 97.87 GiB | | | | | | | | | pve03 | dir | | | | 0 | available | local | | | | | |\n+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+\n| storage/pve03/local-lvm | storage | | images,rootdir | | 145.74 GiB | | | | | | | | 331.23 GiB | | | | | | | | | pve03 | lvmthin | | | | 0 | available | local-lvm | | | | | |\n+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+\n\n===== LOCAL_FILESYSTEMS =====\nFilesystem Size Used Avail Use% Mounted on\n/dev/mapper/pve-root 98G 41G 53G 44% /\nefivarfs 192K 135K 53K 72% /sys/firmware/efi/efivars\n/dev/nvme0n1p2 1022M 9.1M 1013M 1% /boot/efi\n/dev/mapper/pve-homelab--staging 295G 148G 132G 53% /mnt/staging\n/dev/fuse 128M 1.4M 127M 2% /etc/pve\noverlay 98G 41G 53G 44% /var/lib/containers/storage/overlay/4639d2b7096bff2bf1482d2b2b867a1a21a3e9eba88986eb78f9daa256eb5a77/merged\n\n===== LOCAL_INODES =====\nFilesystem Inodes IUsed IFree IUse% Mounted on\n/dev/mapper/pve-root 6553600 139099 6414501 3% /\nefivarfs 0 0 0 - /sys/firmware/efi/efivars\n/dev/nvme0n1p2 0 0 0 - /boot/efi\n/dev/mapper/pve-homelab--staging 19660800 4475 19656325 1% /mnt/staging\n/dev/fuse 262144 89 262055 1% /etc/pve\noverlay 6553600 139099 6414501 3% /var/lib/containers/storage/overlay/4639d2b7096bff2bf1482d2b2b867a1a21a3e9eba88986eb78f9daa256eb5a77/merged\n\n===== ZFS =====\nno pools available\n\n===== CEPH =====\nSTATE=NOT_CONFIGURED_OR_TOOL_ABSENT\n\n===== PVE01_FAILED_UNITS =====\nhomelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state\n\n===== PVE01_KERNEL_CRITICAL_24H =====\n\n===== PROXMOX_FAILED_TASKS =====\nUnknown option: errors\nUnknown option: limit\n400 unable to parse option\npvesh get [OPTIONS] [FORMAT_OPTIONS]\n\n===== TIME_SYNCHRONIZATION =====\nTimezone=Europe/Moscow\nLocalRTC=no\nNTPSynchronized=yes\n\n===== SKLADCHIK_SUPERVISOR =====\ncut: '\"': No such file or directory\nSCRIPT_SHA256=\nEXPECTED_SHA256=bd40ee19ea8b277d52f4240b97d3c79b825c90154272bac8a43004591bd0ddc3\nResult=success\nExecMainStatus=0\nActiveState=inactive\nSubState=dead\nSUPERVISOR_TIMER=active\nSTATUS=OK\nTS=2026-07-24T09:38:15Z\nproblems=\nbackup_status=OK\ncookie_expiry_status=OK\ncookie_jar=yes\nmonitor_status=OK\nrestore_status=OK\nPOST_FIX_COOKIE_FAILURES=0\n\n===== EDGE_VM =====\nedge-vm.gram1.ru\nactive\nactive\nbash: line 1: /var/lib/skladchik-reports-monitor/cookies.jar: Permission denied\nstat: missing operand\nTry 'stat --help' for more information.\nbash: -c: line 1: syntax error near unexpected token `('\nbash: -c: line 1: `sudo -n grep -E ^(STATUS|TS|reports|open|resolved|cookie_jar|cookie_expiry_status|problems)= /var/lib/skladchik-reports-monitor/health.txt'\nFilesystem Size Used Avail Use% Mounted on\n/dev/sdb1 95G 33G 58G 36% /\n\n===== CORE_APPS =====\ncore-apps.gram1.ru\nbash: line 1: {{.Status}}: command not found\n7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f /opt/stacks/gitea/data/gitea/public/.well-known/homelab-bootstrap.json\n7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f /opt/stacks/gitea/data/gitea/public/.well-known/homelab-bootstrap/7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f.json\nFilesystem Size Used Avail Use% Mounted on\n/dev/sda1 118G 42G 72G 37% /\n\n===== PUBLIC_BOOTSTRAP =====\nEXPECTED_BOOTSTRAP_SHA256=7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f\n7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f -\n7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f -\n\n===== READER_CAPABILITY_EXPOSURE =====\n/usr/bin/bash: -c: line 207: syntax error near unexpected token `('\n/usr/bin/bash: -c: line 207: ` \\\"/homelab-reader/[0-9a-fA-F]{64}([/\\\\\\\"]|$)\\\" \\'\n" + "output": "DETAIL=SERVICE_STATE_BEGIN\nDETAIL=SERVICE_PROPERTY=Result=exit-code\nDETAIL=SERVICE_PROPERTY=ExecMainCode=1\nDETAIL=SERVICE_PROPERTY=ExecMainStatus=1\nDETAIL=SERVICE_PROPERTY=ExecStart={ path=/usr/local/sbin/homelab-backup-matrix-refresh ; argv[]=/usr/local/sbin/homelab-backup-matrix-refresh ; ignore_errors=no ; start_time=[Fri 2026-07-24 10:06:04 MSK] ; stop_time=[Fri 2026-07-24 10:06:05 MSK] ; pid=3581151 ; code=exited ; status=1 }\nDETAIL=SERVICE_PROPERTY=WorkingDirectory=\nDETAIL=SERVICE_PROPERTY=User=\nDETAIL=SERVICE_PROPERTY=Group=\nDETAIL=SERVICE_PROPERTY=LoadState=loaded\nDETAIL=SERVICE_PROPERTY=ActiveState=failed\nDETAIL=SERVICE_PROPERTY=SubState=failed\nDETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/homelab-backup-matrix-refresh.service\nDETAIL=SERVICE_PROPERTY=DropInPaths=\nDETAIL=SERVICE_STATE_END\nDETAIL=TIMER_STATE_BEGIN\nDETAIL=TIMER_PROPERTY=Unit=homelab-backup-matrix-refresh.service\nDETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=\nDETAIL=TIMER_PROPERTY=LastTriggerUSec=\nDETAIL=TIMER_PROPERTY=LoadState=loaded\nDETAIL=TIMER_PROPERTY=ActiveState=inactive\nDETAIL=TIMER_PROPERTY=SubState=dead\nDETAIL=TIMER_STATE_END\nDETAIL=LAST_FAILURE_BEGIN\nDETAIL=JOURNAL=2026-07-24T04:18:06+03:00 pve01 homelab-backup-matrix-refresh[2451935]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T04:18:06+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T04:18:06+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T04:18:06+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=JOURNAL=2026-07-24T04:28:41+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nDETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_JOBS_TOTAL=41\nDETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_JOBS_OK=30\nDETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_JOBS_BAD=6\nDETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_JOBS_UNMAPPED=0\nDETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_JOBS_DISABLED=5\nDETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nDETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=JOURNAL=2026-07-24T04:50:27+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nDETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_JOBS_TOTAL=41\nDETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_JOBS_OK=30\nDETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_JOBS_BAD=6\nDETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_JOBS_UNMAPPED=0\nDETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_JOBS_DISABLED=5\nDETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nDETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=JOURNAL=2026-07-24T05:21:32+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nDETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_JOBS_TOTAL=41\nDETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_JOBS_OK=30\nDETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_JOBS_BAD=6\nDETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_JOBS_UNMAPPED=0\nDETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_JOBS_DISABLED=5\nDETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nDETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=JOURNAL=2026-07-24T05:54:45+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nDETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_JOBS_TOTAL=41\nDETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_JOBS_OK=30\nDETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_JOBS_BAD=6\nDETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_JOBS_UNMAPPED=0\nDETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_JOBS_DISABLED=5\nDETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nDETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=JOURNAL=2026-07-24T06:08:12+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nDETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_JOBS_TOTAL=41\nDETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_JOBS_OK=30\nDETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_JOBS_BAD=6\nDETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_JOBS_UNMAPPED=0\nDETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_JOBS_DISABLED=5\nDETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nDETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=JOURNAL=2026-07-24T06:46:41+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nDETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_JOBS_TOTAL=41\nDETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_JOBS_OK=30\nDETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_JOBS_BAD=6\nDETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_JOBS_UNMAPPED=0\nDETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_JOBS_DISABLED=5\nDETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nDETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=JOURNAL=2026-07-24T07:04:41+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nDETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_JOBS_TOTAL=41\nDETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_JOBS_OK=30\nDETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_JOBS_BAD=6\nDETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_JOBS_UNMAPPED=0\nDETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_JOBS_DISABLED=5\nDETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nDETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=JOURNAL=2026-07-24T07:43:04+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nDETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_JOBS_TOTAL=41\nDETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_JOBS_OK=30\nDETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_JOBS_BAD=6\nDETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_JOBS_UNMAPPED=0\nDETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_JOBS_DISABLED=5\nDETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nDETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=JOURNAL=2026-07-24T08:20:04+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nDETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_JOBS_TOTAL=41\nDETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_JOBS_OK=30\nDETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_JOBS_BAD=6\nDETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_JOBS_UNMAPPED=0\nDETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_JOBS_DISABLED=5\nDETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nDETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=JOURNAL=2026-07-24T08:26:24+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nDETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_JOBS_TOTAL=41\nDETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_JOBS_OK=30\nDETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_JOBS_BAD=6\nDETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_JOBS_UNMAPPED=0\nDETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_JOBS_DISABLED=5\nDETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nDETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=JOURNAL=2026-07-24T08:35:15+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nDETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_JOBS_TOTAL=41\nDETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_JOBS_OK=30\nDETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_JOBS_BAD=6\nDETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_JOBS_UNMAPPED=0\nDETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_JOBS_DISABLED=5\nDETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nDETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=JOURNAL=2026-07-24T08:37:50+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nDETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_JOBS_TOTAL=41\nDETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_JOBS_OK=30\nDETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_JOBS_BAD=6\nDETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_JOBS_UNMAPPED=0\nDETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_JOBS_DISABLED=5\nDETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nDETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=JOURNAL=2026-07-24T10:06:04+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nDETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_JOBS_TOTAL=41\nDETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_JOBS_OK=30\nDETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_JOBS_BAD=6\nDETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_JOBS_UNMAPPED=0\nDETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_JOBS_DISABLED=5\nDETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nDETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nDETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nDETAIL=LAST_FAILURE_END\nDETAIL=SECRET_VALUES_OUTPUT=NONE\nDETAIL=BACKUP_MATRIX_FAILURE_AUDIT=PASS\nCHANGES_MADE=NO\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 4670fd5..6e427f6 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,7 +1,7 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=HOMELAB-CLUSTER-CRITICAL-SNAPSHOT-428 -STATUS=FAIL -RC=127 +COMMAND_ID=HOMELAB-BACKUP-MATRIX-FAILURE-AUDIT-429 +STATUS=OK +RC=0 HOST=pve01 MODE=read-only COMPONENT=cluster-audit @@ -9,167 +9,203 @@ REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 -COMMAND_SHA256=acb8cfc8df7523b005c7cc8d9dcc5e8d6955edc9869f35192b9aab7770149bb5 +COMMAND_SHA256=d361e4c1100f3da6fa9e708a966bc5ce11d491b55027d4766e5dbc29f3699816 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGES_MADE=false SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=983c89419e9432c65180a176c36039953a5d07a157e9509f7eaa97016d1f65b6 -SANITIZED_OUTPUT_SHA256=50e542092e058ed4496ae5f17ec2fa7a38e02c39668e58030948a9916e980eb2 +RAW_EVIDENCE_SHA256=b507fad0e10f74eee68181be851d7b53e3cf8f906f9815d2c42534be5d0cac55 +SANITIZED_OUTPUT_SHA256=b507fad0e10f74eee68181be851d7b53e3cf8f906f9815d2c42534be5d0cac55 OUTPUT_BEGIN -AUDIT_ID=HOMELAB-CLUSTER-CRITICAL-SNAPSHOT-428 -AUDIT_TIME_UTC=2026-07-24T09:43:29Z -CHANGES_PLANNED=NO - -===== PROXMOX_QUORUM ===== -Cluster information -------------------- -Name: homelab -Config Version: 3 -Transport: knet -Secure auth: on - -Quorum information ------------------- -Date: Fri Jul 24 12:43:29 2026 -Quorum provider: corosync_votequorum -Nodes: 3 -Node ID: 0x00000001 -Ring ID: 1.ba -Quorate: Yes - -Votequorum information ----------------------- -Expected votes: 3 -Highest expected: 3 -Total votes: 3 -Quorum: 2 -Flags: Quorate - -Membership information ----------------------- - Nodeid Votes Name -0x00000001 1 [PRIVATE_IP] (local) -0x00000002 1 [PRIVATE_IP] -0x00000003 1 [PRIVATE_IP] - -===== PROXMOX_NODES ===== -+-------+--------+-------+-------+--------+-----------+-----------+-------------------------------------------------------------------------------------------------+-------------+ -| node | status | cpu | level | maxcpu | maxmem | mem | ssl_fingerprint | uptime | -+=======+========+=======+=======+========+===========+===========+=================================================================================================+=============+ -| pve01 | online | 1.98% | | 24 | 31.09 GiB | 22.07 GiB | EB:7A:01:C5:02:1B:B7:F9:54:F4:38:8B:7F:03:1F:87:62:D0:6E:5F:C0:39:FB:B5:42:AE:58:6F:58:9A:9C:C1 | 20h 22m 46s | -+-------+--------+-------+-------+--------+-----------+-----------+-------------------------------------------------------------------------------------------------+-------------+ -| pve02 | online | 8.49% | | 4 | 15.40 GiB | 11.36 GiB | C0:88:6E:89:CD:BF:0C:82:54:20:E7:11:DE:CA:ED:09:12:C6:9C:24:2A:1E:5C:33:49:EC:4E:E0:67:BE:79:05 | 20h 20m 42s | -+-------+--------+-------+-------+--------+-----------+-----------+-------------------------------------------------------------------------------------------------+-------------+ -| pve03 | online | 4.43% | | 8 | 31.25 GiB | 11.24 GiB | 7E:FE:4A:DC:32:3C:02:84:A2:DD:23:84:FD:19:1E:8A:1B:B1:60:F6:F4:16:D0:28:3B:13:AE:65:7D:1F:38:32 | 20h 21m 35s | -+-------+--------+-------+-------+--------+-----------+-----------+-------------------------------------------------------------------------------------------------+-------------+ - -===== PROXMOX_STORAGE ===== -+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+ -| id | type | cgroup-mode | content | cpu | disk | diskread | diskwrite | hastate | host-arch | level | lock | maxcpu | maxdisk | maxmem | mem | memhost | name | netin | netout | network | network-type | node | plugintype | pool | protocol | sdn | shared | status | storage | tags | template | uptime | vmid | zone-type | -+=========================+=========+=============+==========================+=====+============+==========+===========+=========+===========+=======+======+========+============+========+=====+=========+======+=======+========+=========+==============+=======+============+======+==========+=====+========+===========+===========+======+==========+========+======+===========+ -| storage/pve01/local | storage | | iso,backup,vztmpl,import | | 40.39 GiB | | | | | | | | 97.87 GiB | | | | | | | | | pve01 | dir | | | | 0 | available | local | | | | | | -+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+ -| storage/pve01/local-lvm | storage | | images,rootdir | | 310.84 GiB | | | | | | | | 809.48 GiB | | | | | | | | | pve01 | lvmthin | | | | 0 | available | local-lvm | | | | | | -+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+ -| storage/pve02/local | storage | | iso,backup,vztmpl,import | | 26.14 GiB | | | | | | | | 97.87 GiB | | | | | | | | | pve02 | dir | | | | 0 | available | local | | | | | | -+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+ -| storage/pve02/local-lvm | storage | | images,rootdir | | 290.27 GiB | | | | | | | | 787.91 GiB | | | | | | | | | pve02 | lvmthin | | | | 0 | available | local-lvm | | | | | | -+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+ -| storage/pve03/local | storage | | iso,backup,vztmpl,import | | 53.72 GiB | | | | | | | | 97.87 GiB | | | | | | | | | pve03 | dir | | | | 0 | available | local | | | | | | -+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+ -| storage/pve03/local-lvm | storage | | images,rootdir | | 145.74 GiB | | | | | | | | 331.23 GiB | | | | | | | | | pve03 | lvmthin | | | | 0 | available | local-lvm | | | | | | -+-------------------------+---------+-------------+--------------------------+-----+------------+----------+-----------+---------+-----------+-------+------+--------+------------+--------+-----+---------+------+-------+--------+---------+--------------+-------+------------+------+----------+-----+--------+-----------+-----------+------+----------+--------+------+-----------+ - -===== LOCAL_FILESYSTEMS ===== -Filesystem Size Used Avail Use% Mounted on -/dev/mapper/pve-root 98G 41G 53G 44% / -efivarfs 192K 135K 53K 72% /sys/firmware/efi/efivars -/dev/nvme0n1p2 1022M 9.1M 1013M 1% /boot/efi -/dev/mapper/pve-homelab--staging 295G 148G 132G 53% /mnt/staging -/dev/fuse 128M 1.4M 127M 2% /etc/pve -overlay 98G 41G 53G 44% /var/lib/containers/storage/overlay/4639d2b7096bff2bf1482d2b2b867a1a21a3e9eba88986eb78f9daa256eb5a77/merged - -===== LOCAL_INODES ===== -Filesystem Inodes IUsed IFree IUse% Mounted on -/dev/mapper/pve-root 6553600 139099 6414501 3% / -efivarfs 0 0 0 - /sys/firmware/efi/efivars -/dev/nvme0n1p2 0 0 0 - /boot/efi -/dev/mapper/pve-homelab--staging 19660800 4475 19656325 1% /mnt/staging -/dev/fuse 262144 89 262055 1% /etc/pve -overlay 6553600 139099 6414501 3% /var/lib/containers/storage/overlay/4639d2b7096bff2bf1482d2b2b867a1a21a3e9eba88986eb78f9daa256eb5a77/merged - -===== ZFS ===== -no pools available - -===== CEPH ===== -STATE=NOT_CONFIGURED_OR_TOOL_ABSENT - -===== PVE01_FAILED_UNITS ===== -homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state - -===== PVE01_KERNEL_CRITICAL_24H ===== - -===== PROXMOX_FAILED_TASKS ===== -Unknown option: errors -Unknown option: limit -400 unable to parse option -pvesh get [OPTIONS] [FORMAT_OPTIONS] - -===== TIME_SYNCHRONIZATION ===== -Timezone=Europe/Moscow -LocalRTC=no -NTPSynchronized=yes - -===== SKLADCHIK_SUPERVISOR ===== -cut: '"': No such file or directory -SCRIPT_SHA256= -EXPECTED_SHA256=bd40ee19ea8b277d52f4240b97d3c79b825c90154272bac8a43004591bd0ddc3 -Result=success -ExecMainStatus=0 -ActiveState=inactive -SubState=dead -SUPERVISOR_TIMER=active -STATUS=OK -TS=2026-07-24T09:38:15Z -problems= -backup_status=OK -cookie_expiry_status=OK -cookie_jar=yes -monitor_status=OK -restore_status=OK -POST_FIX_COOKIE_FAILURES=0 - -===== EDGE_VM ===== -edge-vm.gram1.ru -active -active -bash: line 1: /var/lib/skladchik-reports-monitor/cookies.jar: Permission denied -stat: missing operand -Try 'stat --help' for more information. -bash: -c: line 1: syntax error near unexpected token `(' -bash: -c: line 1: `sudo -n grep -E ^(STATUS|TS|reports|open|resolved|cookie_jar|cookie_expiry_status|problems)= /var/lib/skladchik-reports-monitor/health.txt' -Filesystem Size Used Avail Use% Mounted on -/dev/sdb1 95G 33G 58G 36% / - -===== CORE_APPS ===== -core-apps.gram1.ru -bash: line 1: {{.Status}}: command not found -7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f /opt/stacks/gitea/data/gitea/public/.well-known/homelab-bootstrap.json -7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f /opt/stacks/gitea/data/gitea/public/.well-known/homelab-bootstrap/7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f.json -Filesystem Size Used Avail Use% Mounted on -/dev/sda1 118G 42G 72G 37% / - -===== PUBLIC_BOOTSTRAP ===== -EXPECTED_BOOTSTRAP_SHA256=7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f -7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f - -7be8012c619c01d5770417d3a26ca82c7cd14935ed5cf10fd7cb6e2fa687c82f - - -===== READER_CAPABILITY_EXPOSURE ===== -/usr/bin/bash: -c: line 207: syntax error near unexpected token `(' -/usr/bin/bash: -c: line 207: ` \"/homelab-reader/[0-9a-fA-F]{64}([/\\\"]|$)\" \' +DETAIL=SERVICE_STATE_BEGIN +DETAIL=SERVICE_PROPERTY=Result=exit-code +DETAIL=SERVICE_PROPERTY=ExecMainCode=1 +DETAIL=SERVICE_PROPERTY=ExecMainStatus=1 +DETAIL=SERVICE_PROPERTY=ExecStart={ path=/usr/local/sbin/homelab-backup-matrix-refresh ; argv[]=/usr/local/sbin/homelab-backup-matrix-refresh ; ignore_errors=no ; start_time=[Fri 2026-07-24 10:06:04 MSK] ; stop_time=[Fri 2026-07-24 10:06:05 MSK] ; pid=3581151 ; code=exited ; status=1 } +DETAIL=SERVICE_PROPERTY=WorkingDirectory= +DETAIL=SERVICE_PROPERTY=User= +DETAIL=SERVICE_PROPERTY=Group= +DETAIL=SERVICE_PROPERTY=LoadState=loaded +DETAIL=SERVICE_PROPERTY=ActiveState=failed +DETAIL=SERVICE_PROPERTY=SubState=failed +DETAIL=SERVICE_PROPERTY=FragmentPath=/etc/systemd/system/homelab-backup-matrix-refresh.service +DETAIL=SERVICE_PROPERTY=DropInPaths= +DETAIL=SERVICE_STATE_END +DETAIL=TIMER_STATE_BEGIN +DETAIL=TIMER_PROPERTY=Unit=homelab-backup-matrix-refresh.service +DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime= +DETAIL=TIMER_PROPERTY=LastTriggerUSec= +DETAIL=TIMER_PROPERTY=LoadState=loaded +DETAIL=TIMER_PROPERTY=ActiveState=inactive +DETAIL=TIMER_PROPERTY=SubState=dead +DETAIL=TIMER_STATE_END +DETAIL=LAST_FAILURE_BEGIN +DETAIL=JOURNAL=2026-07-24T04:18:06+03:00 pve01 homelab-backup-matrix-refresh[2451935]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T04:18:06+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T04:18:06+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T04:18:06+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=JOURNAL=2026-07-24T04:28:41+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +DETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_STATUS=FAIL +DETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_JOBS_TOTAL=41 +DETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_JOBS_OK=30 +DETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_JOBS_BAD=6 +DETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_JOBS_UNMAPPED=0 +DETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_JOBS_DISABLED=5 +DETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +DETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 homelab-backup-matrix-refresh[2481004]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T04:28:42+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=JOURNAL=2026-07-24T04:50:27+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +DETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_STATUS=FAIL +DETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_JOBS_TOTAL=41 +DETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_JOBS_OK=30 +DETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_JOBS_BAD=6 +DETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_JOBS_UNMAPPED=0 +DETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_JOBS_DISABLED=5 +DETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +DETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 homelab-backup-matrix-refresh[2557614]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T04:50:28+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=JOURNAL=2026-07-24T05:21:32+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +DETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_STATUS=FAIL +DETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_JOBS_TOTAL=41 +DETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_JOBS_OK=30 +DETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_JOBS_BAD=6 +DETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_JOBS_UNMAPPED=0 +DETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_JOBS_DISABLED=5 +DETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +DETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 homelab-backup-matrix-refresh[2650449]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T05:21:33+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=JOURNAL=2026-07-24T05:54:45+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +DETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_STATUS=FAIL +DETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_JOBS_TOTAL=41 +DETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_JOBS_OK=30 +DETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_JOBS_BAD=6 +DETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_JOBS_UNMAPPED=0 +DETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_JOBS_DISABLED=5 +DETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +DETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 homelab-backup-matrix-refresh[2763918]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T05:54:46+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=JOURNAL=2026-07-24T06:08:12+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +DETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_STATUS=FAIL +DETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_JOBS_TOTAL=41 +DETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_JOBS_OK=30 +DETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_JOBS_BAD=6 +DETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_JOBS_UNMAPPED=0 +DETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_JOBS_DISABLED=5 +DETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +DETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 homelab-backup-matrix-refresh[2814238]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T06:08:13+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=JOURNAL=2026-07-24T06:46:41+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +DETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_STATUS=FAIL +DETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_JOBS_TOTAL=41 +DETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_JOBS_OK=30 +DETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_JOBS_BAD=6 +DETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_JOBS_UNMAPPED=0 +DETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_JOBS_DISABLED=5 +DETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +DETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 homelab-backup-matrix-refresh[2942590]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T06:46:42+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=JOURNAL=2026-07-24T07:04:41+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +DETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_STATUS=FAIL +DETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_JOBS_TOTAL=41 +DETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_JOBS_OK=30 +DETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_JOBS_BAD=6 +DETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_JOBS_UNMAPPED=0 +DETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_JOBS_DISABLED=5 +DETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +DETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 homelab-backup-matrix-refresh[3006174]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T07:04:42+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=JOURNAL=2026-07-24T07:43:04+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +DETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_STATUS=FAIL +DETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_JOBS_TOTAL=41 +DETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_JOBS_OK=30 +DETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_JOBS_BAD=6 +DETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_JOBS_UNMAPPED=0 +DETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_JOBS_DISABLED=5 +DETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +DETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 homelab-backup-matrix-refresh[3125407]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T07:43:05+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=JOURNAL=2026-07-24T08:20:04+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +DETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_STATUS=FAIL +DETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_JOBS_TOTAL=41 +DETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_JOBS_OK=30 +DETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_JOBS_BAD=6 +DETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_JOBS_UNMAPPED=0 +DETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_JOBS_DISABLED=5 +DETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +DETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 homelab-backup-matrix-refresh[3253902]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T08:20:05+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=JOURNAL=2026-07-24T08:26:24+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +DETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_STATUS=FAIL +DETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_JOBS_TOTAL=41 +DETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_JOBS_OK=30 +DETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_JOBS_BAD=6 +DETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_JOBS_UNMAPPED=0 +DETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_JOBS_DISABLED=5 +DETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +DETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 homelab-backup-matrix-refresh[3268883]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T08:26:25+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=JOURNAL=2026-07-24T08:35:15+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +DETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_STATUS=FAIL +DETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_JOBS_TOTAL=41 +DETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_JOBS_OK=30 +DETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_JOBS_BAD=6 +DETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_JOBS_UNMAPPED=0 +DETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_JOBS_DISABLED=5 +DETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +DETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 homelab-backup-matrix-refresh[3293028]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T08:35:16+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=JOURNAL=2026-07-24T08:37:50+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +DETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_STATUS=FAIL +DETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_JOBS_TOTAL=41 +DETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_JOBS_OK=30 +DETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_JOBS_BAD=6 +DETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_JOBS_UNMAPPED=0 +DETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_JOBS_DISABLED=5 +DETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +DETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 homelab-backup-matrix-refresh[3305815]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T08:37:51+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=JOURNAL=2026-07-24T10:06:04+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +DETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_STATUS=FAIL +DETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_JOBS_TOTAL=41 +DETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_JOBS_OK=30 +DETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_JOBS_BAD=6 +DETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_JOBS_UNMAPPED=0 +DETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_JOBS_DISABLED=5 +DETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +DETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 homelab-backup-matrix-refresh[3581151]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +DETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +DETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +DETAIL=JOURNAL=2026-07-24T10:06:05+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +DETAIL=LAST_FAILURE_END +DETAIL=SECRET_VALUES_OUTPUT=NONE +DETAIL=BACKUP_MATRIX_FAILURE_AUDIT=PASS +CHANGES_MADE=NO OUTPUT_END CHAT_OUTPUT_END