homelab-runtime-publisher
31 lines
16 KiB
JSON
31 lines
16 KiB
JSON
{
|
|
"schema_version": 1,
|
|
"channel": "homelab-runtime",
|
|
"command_id": "HOMELAB-SKLADCHIK-DYNAMIC-PROBLEM-TRACE-421",
|
|
"status": "OK",
|
|
"rc": 0,
|
|
"host": "pve01",
|
|
"mode": "read-only",
|
|
"component": "cluster-audit",
|
|
"started_at_utc": "2026-07-24T09:11:43Z",
|
|
"finished_at_utc": "2026-07-24T09:11:43Z",
|
|
"reference_register_checked": true,
|
|
"reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
|
|
"error_register_checked": true,
|
|
"error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752",
|
|
"command_sha256": "e1c9ec96c23c56ee20087fbae165b41c11fcde575aa8c3b0f03db028063d0099",
|
|
"duplicate_failed_command_blocked": false,
|
|
"block_reason": null,
|
|
"execution_started": true,
|
|
"changes_made": false,
|
|
"sanitized": true,
|
|
"secrets_included": false,
|
|
"private_addresses_included": false,
|
|
"raw_evidence_retained_locally": true,
|
|
"raw_evidence_sha256": "55c3331e8fb9e7193d1641f54eba5bb0a57e3958042ac712a7e50ae3f58f1c2e",
|
|
"sanitized_output_sha256": "55c3331e8fb9e7193d1641f54eba5bb0a57e3958042ac712a7e50ae3f58f1c2e",
|
|
"output_truncated_in_json": false,
|
|
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
|
|
"output": "DETAIL=SCRIPT_PATH=/usr/local/sbin/skladchik-reports-monitor-supervisor\nDETAIL=GATHER_LINES=105-271\nDETAIL=PROBLEM_WRITE_COUNT=26\nDETAIL=PROBLEM_WRITES_BEGIN\nDETAIL=PROBLEM_WRITE=LINE:141|METHOD:append|DYNAMIC:NO|EXPRESSION:\"edge_ssh_failed\"\nDETAIL=ENCLOSING_IF=LINE:140|TEST:rc != 0\nDETAIL=PROBLEM_WRITE=LINE:171|METHOD:append|DYNAMIC:NO|EXPRESSION:\"monitor_cookie_jar_bad\"\nDETAIL=ENCLOSING_IF=LINE:170|TEST:mon.get(\"cookie_jar\") not in (\"yes\", \"empty\")\nDETAIL=PROBLEM_WRITE=LINE:178|METHOD:append|DYNAMIC:NO|EXPRESSION:\"cookie_expiry_missing\"\nDETAIL=ENCLOSING_IF=LINE:177|TEST:expiry_status in (\"\", \"NO_COOKIE_JAR\", \"NO_PERSISTENT_COOKIE\", \"ERROR\")\nDETAIL=PROBLEM_WRITE=LINE:180|METHOD:append|DYNAMIC:NO|EXPRESSION:\"cookie_expiry_expired\"\nDETAIL=ENCLOSING_IF=LINE:179|TEST:expiry_status == \"EXPIRED\" or (expires_in is not None and expires_in <= 0)\nDETAIL=PROBLEM_WRITE=LINE:182|METHOD:append|DYNAMIC:NO|EXPRESSION:\"cookie_expiry_warn\"\nDETAIL=ENCLOSING_IF=LINE:181|TEST:expiry_status == \"WARN\" or (expires_in is not None and expires_in < COOKIE_EXPIRY_WARN_SECONDS)\nDETAIL=PROBLEM_WRITE=LINE:193|METHOD:append|DYNAMIC:YES|EXPRESSION:f\"edge_{safe}_timer_bad\"\nDETAIL=ENCLOSING_IF=LINE:192|TEST:enabled != \"enabled\" or active != \"active\"\nDETAIL=ENCLOSING_FOR=LINE:185|TARGET:unit|ITER:[\"skladchik-reports-monitor.timer\", \"skladchik-reports-monitor-watchdog.timer\", \"skladchik-reports-monitor-heartbeat.timer\", \"skladchik-reports-monitor-weekly-summary.timer\", \"skladchik-reports-monitor-age-alert.timer\", \"skladchik-reports-monitor-event-selftest.timer\"]\nDETAIL=RELATED_ASSIGNMENT=LINE:186|SOURCE:safe = unit.replace(\".timer\", \"\").replace(\"_\", \"-\")\nDETAIL=RELATED_ASSIGNMENT=LINE:190|SOURCE:facts[f\"edge_{safe}_timer_enabled\"] = enabled\nDETAIL=RELATED_ASSIGNMENT=LINE:191|SOURCE:facts[f\"edge_{safe}_timer_active\"] = active\nDETAIL=PROBLEM_WRITE=LINE:205|METHOD:append|DYNAMIC:YES|EXPRESSION:f\"edge_{k}_bad\"\nDETAIL=ENCLOSING_IF=LINE:204|TEST:facts[k] != \"yes\"\nDETAIL=ENCLOSING_FOR=LINE:202|TARGET:k|ITER:[\"monitor_py\", \"config_ini\", \"secrets_env\", \"cookies_jar\", \"operational_digest_py\", \"event_selftest_runner_py\", \"event_selftest_core\", \"weekly_summary_health\", \"age_alert_health\", \"event_selftest_health\", \"py_compile\"]\nDETAIL=RELATED_ASSIGNMENT=LINE:203|SOURCE:facts[k] = files.get(k, \"no\")\nDETAIL=PROBLEM_WRITE=LINE:220|METHOD:append|DYNAMIC:NO|EXPRESSION:\"backup_cookie_jar_not_marked\"\nDETAIL=ENCLOSING_IF=LINE:219|TEST:backup.get(\"COOKIE_JAR_BACKED_UP\") != \"yes\"\nDETAIL=PROBLEM_WRITE=LINE:231|METHOD:append|DYNAMIC:NO|EXPRESSION:\"backup_size_bad\"\nDETAIL=ENCLOSING_IF=LINE:230|TEST:int(backup.get(\"SIZE_BYTES\", \"0\")) <= 0\nDETAIL=PROBLEM_WRITE=LINE:233|METHOD:append|DYNAMIC:NO|EXPRESSION:\"backup_size_bad\"\nDETAIL=PROBLEM_WRITE=LINE:247|METHOD:append|DYNAMIC:YES|EXPRESSION:key.lower() + \"_bad\"\nDETAIL=ENCLOSING_IF=LINE:246|TEST:restore.get(key) != \"OK\"\nDETAIL=ENCLOSING_FOR=LINE:245|TARGET:key|ITER:[\"CLOUD_1_STATUS\", \"CLOUD_2_STATUS\"]\nDETAIL=RELATED_ASSIGNMENT=LINE:240|SOURCE:facts[key.lower()] = restore.get(key, \"\")\nDETAIL=PROBLEM_WRITE=LINE:250|METHOD:append|DYNAMIC:YES|EXPRESSION:key.lower() + \"_bad\"\nDETAIL=ENCLOSING_IF=LINE:249|TEST:restore.get(key) != \"yes\"\nDETAIL=ENCLOSING_FOR=LINE:248|TARGET:key|ITER:[\"SHA256_OK\", \"TAR_OK\", \"REQUIRED_FILES_OK\", \"COOKIE_JAR_OK\", \"EXTENSION_HEALTH_FILES_OK\", \"EXTENSION_SCRIPTS_OK\", \"EXTENSION_UNITS_OK\", \"SQLITE_OK\", \"PY_COMPILE_OK\"]\nDETAIL=RELATED_ASSIGNMENT=LINE:240|SOURCE:facts[key.lower()] = restore.get(key, \"\")\nDETAIL=PROBLEM_WRITE=LINE:265|METHOD:append|DYNAMIC:YES|EXPRESSION:base + \"_timer_bad\"\nDETAIL=ENCLOSING_IF=LINE:264|TEST:p[\"enabled\"] != \"enabled\" or p[\"active\"] != \"active\"\nDETAIL=ENCLOSING_FOR=LINE:259|TARGET:u, p|ITER:props.items()\nDETAIL=RELATED_ASSIGNMENT=LINE:260|SOURCE:base = u.replace(\".timer\", \"\").replace(\"-\", \"_\")\nDETAIL=RELATED_ASSIGNMENT=LINE:261|SOURCE:facts[base + \"_timer_enabled\"] = p[\"enabled\"]\nDETAIL=RELATED_ASSIGNMENT=LINE:262|SOURCE:facts[base + \"_timer_active\"] = p[\"active\"]\nDETAIL=PROBLEM_WRITE=LINE:269|METHOD:append|DYNAMIC:YES|EXPRESSION:base + \"_timer_unknown\"\nDETAIL=ENCLOSING_IF=LINE:268|TEST:p[\"enabled\"] not in (\"enabled\", \"disabled\") or p[\"active\"] not in (\"active\", \"inactive\")\nDETAIL=ENCLOSING_FOR=LINE:259|TARGET:u, p|ITER:props.items()\nDETAIL=RELATED_ASSIGNMENT=LINE:260|SOURCE:base = u.replace(\".timer\", \"\").replace(\"-\", \"_\")\nDETAIL=RELATED_ASSIGNMENT=LINE:261|SOURCE:facts[base + \"_timer_enabled\"] = p[\"enabled\"]\nDETAIL=RELATED_ASSIGNMENT=LINE:262|SOURCE:facts[base + \"_timer_active\"] = p[\"active\"]\nDETAIL=PROBLEM_WRITES_END\nDETAIL=EDGE_COOKIE_SOURCE_BEGIN\nDETAIL=RELEVANT_NODE=LINE:109|TYPE:Assign|SOURCE:edge_cmd = r''' set +e echo __MONITOR_HEALTH_BEGIN__ cat /var/lib/skladchik-reports-monitor/health.txt 2>/dev/null || true echo __MONITOR_HEALTH_END__ echo __ALERT_HEALTH_BEGIN__ cat /var/lib/skladchik-reports-monitor/alert-health.txt 2>/dev/null || true echo __ALERT_HEALTH_END__ echo __TIMERS_BEGIN__ for u in skladchik-reports-monitor.timer skladchik-reports-monitor-watchdog.timer skladchik-reports-monitor-heartbeat.timer skladchik-reports-monitor-weekly-summary.timer skladchik-reports-monitor-age-alert.timer skladchik-reports-monitor-event-selftest.timer; do echo \"$u enabled=$(systemctl is-enabled \"$u\" 2>/dev/null) active=$(systemctl is-active \"$u\" 2>/dev/null)\"; done echo __TIMERS_END__ echo __ROUTE_BEGIN__ ip route get 1.1.1.1 2>&1 | head -n 1 echo __ROUTE_END__ echo __FILES_BEGIN__ test -s /opt/skladchik-reports-monitor/monitor.py && echo monitor_py=yes || echo monitor_py=no test -s /etc/skladchik-reports-monitor/config.ini && echo config_ini=yes || echo config_ini=no test -s /etc/skladchik-reports-monitor/secrets.env && echo secrets_env=yes || echo secrets_env=no test -s /var/lib/skladchik-reports-monitor/cookies.jar && echo cookies_jar=yes || echo cookies_jar=no [ -s /var/lib/skladchik-reports-monitor/cookies.jar ] && echo cookies_jar_bytes=$(stat -c %s /var/lib/skladchik-reports-monitor/cookies.jar 2>/dev/null) || echo cookies_jar_bytes=0 test -s /opt/skladchik-reports-monitor/operational_digest.py && echo operational_digest_py=yes || echo operational_digest_py=no test -s /opt/skladchik-reports-monitor/event-selftest-runner.py && echo event_selftest_runner_py=yes || echo event_selftest_runner_py=no test -s /opt/skladchik-reports-monitor/event-selftest-core && echo event_selftest_core=yes || echo event_selftest_core=no test -s /var/lib/skladchik-reports-monitor/weekly-summary-health.txt && echo weekly_summary_health=yes || echo weekly_summary_health=no test -s /var/lib/skladchik-reports-monitor/age-alert-health.txt && echo age_alert_health=yes || echo age_alert_health=no test -s /var/lib/skladchik-reports-monitor/event-selftest-health.txt && echo event_selftest_health=yes || echo event_selftest_health=no python3 -m py_compile /opt/skladchik-reports-monitor/monitor.py /opt/skladchik-reports-monitor/operational_digest.py /opt/skladchik-reports-monitor/event-selftest-runner.py /opt/skladchik-reports-monitor/event-selftest-core >/dev/null 2>&1 && echo py_compile=yes || echo py_compile=no echo __FILES_END__ '''\nDETAIL=RELEVANT_NODE=LINE:138|TYPE:Assign|SOURCE:rc, out = ssh_edge(edge_cmd, timeout=45)\nDETAIL=RELEVANT_NODE=LINE:138|TYPE:Call|SOURCE:ssh_edge(edge_cmd, timeout=45)\nDETAIL=RELEVANT_NODE=LINE:139|TYPE:Assign|SOURCE:facts[\"edge_ssh_rc\"] = str(rc)\nDETAIL=RELEVANT_NODE=LINE:140|TYPE:If|SOURCE:if rc != 0: problems.append(\"edge_ssh_failed\")\nDETAIL=RELEVANT_NODE=LINE:141|TYPE:Call|SOURCE:problems.append(\"edge_ssh_failed\")\nDETAIL=RELEVANT_NODE=LINE:154|TYPE:Assign|SOURCE:facts[\"cookie_jar\"] = mon.get(\"cookie_jar\", \"\")\nDETAIL=RELEVANT_NODE=LINE:154|TYPE:Call|SOURCE:mon.get(\"cookie_jar\", \"\")\nDETAIL=RELEVANT_NODE=LINE:155|TYPE:Assign|SOURCE:facts[\"cookie_jar_bytes\"] = mon.get(\"cookie_jar_bytes\", \"\")\nDETAIL=RELEVANT_NODE=LINE:155|TYPE:Call|SOURCE:mon.get(\"cookie_jar_bytes\", \"\")\nDETAIL=RELEVANT_NODE=LINE:156|TYPE:Assign|SOURCE:facts[\"cookie_expiry_status\"] = mon.get(\"cookie_expiry_status\", \"\")\nDETAIL=RELEVANT_NODE=LINE:156|TYPE:Call|SOURCE:mon.get(\"cookie_expiry_status\", \"\")\nDETAIL=RELEVANT_NODE=LINE:157|TYPE:Assign|SOURCE:facts[\"cookie_expiry_utc\"] = mon.get(\"cookie_expiry_utc\", \"\")\nDETAIL=RELEVANT_NODE=LINE:157|TYPE:Call|SOURCE:mon.get(\"cookie_expiry_utc\", \"\")\nDETAIL=RELEVANT_NODE=LINE:158|TYPE:Assign|SOURCE:facts[\"cookie_expires_in_seconds\"] = mon.get(\"cookie_expires_in_seconds\", \"\")\nDETAIL=RELEVANT_NODE=LINE:158|TYPE:Call|SOURCE:mon.get(\"cookie_expires_in_seconds\", \"\")\nDETAIL=RELEVANT_NODE=LINE:159|TYPE:Assign|SOURCE:facts[\"cookie_expires_in_days\"] = mon.get(\"cookie_expires_in_days\", \"\")\nDETAIL=RELEVANT_NODE=LINE:159|TYPE:Call|SOURCE:mon.get(\"cookie_expires_in_days\", \"\")\nDETAIL=RELEVANT_NODE=LINE:160|TYPE:Assign|SOURCE:facts[\"cookie_persistent_count\"] = mon.get(\"cookie_persistent_count\", \"\")\nDETAIL=RELEVANT_NODE=LINE:160|TYPE:Call|SOURCE:mon.get(\"cookie_persistent_count\", \"\")\nDETAIL=RELEVANT_NODE=LINE:168|TYPE:If|SOURCE:if mon.get(\"session_keepalive\") != \"persistent_cookie_jar\": problems.append(\"session_keepalive_not_persistent\")\nDETAIL=RELEVANT_NODE=LINE:170|TYPE:If|SOURCE:if mon.get(\"cookie_jar\") not in (\"yes\", \"empty\"): problems.append(\"monitor_cookie_jar_bad\")\nDETAIL=RELEVANT_NODE=LINE:170|TYPE:Call|SOURCE:mon.get(\"cookie_jar\")\nDETAIL=RELEVANT_NODE=LINE:171|TYPE:Call|SOURCE:problems.append(\"monitor_cookie_jar_bad\")\nDETAIL=RELEVANT_NODE=LINE:172|TYPE:Assign|SOURCE:expiry_status = mon.get(\"cookie_expiry_status\", \"\")\nDETAIL=RELEVANT_NODE=LINE:172|TYPE:Call|SOURCE:mon.get(\"cookie_expiry_status\", \"\")\nDETAIL=RELEVANT_NODE=LINE:174|TYPE:Assign|SOURCE:expires_in = int(mon.get(\"cookie_expires_in_seconds\", \"\"))\nDETAIL=RELEVANT_NODE=LINE:174|TYPE:Call|SOURCE:int(mon.get(\"cookie_expires_in_seconds\", \"\"))\nDETAIL=RELEVANT_NODE=LINE:174|TYPE:Call|SOURCE:mon.get(\"cookie_expires_in_seconds\", \"\")\nDETAIL=RELEVANT_NODE=LINE:177|TYPE:If|SOURCE:if expiry_status in (\"\", \"NO_COOKIE_JAR\", \"NO_PERSISTENT_COOKIE\", \"ERROR\"): problems.append(\"cookie_expiry_missing\") elif expiry_status == \"EXPIRED\" or (expires_in is not None and expires_in <= 0): problems.append(\"cookie_expiry_expired\") elif expiry_status == \"WARN\" or (expires_in is not None and expires_in < COOKIE_EXPIRY_WARN_SECONDS): problems.append(\"cookie_expiry_warn\")\nDETAIL=RELEVANT_NODE=LINE:178|TYPE:Call|SOURCE:problems.append(\"cookie_expiry_missing\")\nDETAIL=RELEVANT_NODE=LINE:179|TYPE:If|SOURCE:elif expiry_status == \"EXPIRED\" or (expires_in is not None and expires_in <= 0): problems.append(\"cookie_expiry_expired\") elif expiry_status == \"WARN\" or (expires_in is not None and expires_in < COOKIE_EXPIRY_WARN_SECONDS): problems.append(\"cookie_expiry_warn\")\nDETAIL=RELEVANT_NODE=LINE:180|TYPE:Call|SOURCE:problems.append(\"cookie_expiry_expired\")\nDETAIL=RELEVANT_NODE=LINE:181|TYPE:If|SOURCE:elif expiry_status == \"WARN\" or (expires_in is not None and expires_in < COOKIE_EXPIRY_WARN_SECONDS): problems.append(\"cookie_expiry_warn\")\nDETAIL=RELEVANT_NODE=LINE:182|TYPE:Call|SOURCE:problems.append(\"cookie_expiry_warn\")\nDETAIL=RELEVANT_NODE=LINE:185|TYPE:For|SOURCE:for unit in [\"skladchik-reports-monitor.timer\", \"skladchik-reports-monitor-watchdog.timer\", \"skladchik-reports-monitor-heartbeat.timer\", \"skladchik-reports-monitor-weekly-summary.timer\", \"skladchik-reports-monitor-age-alert.timer\", \"skladchik-reports-monitor-event-selftest.timer\"]: safe = unit.replace(\".timer\", \"\").replace(\"_\", \"-\") m = re.search(re.escape(unit) + r\"\\s+enabled=([^\\s]+)\\s+active=([^\\s]+)\", timers_text) enabled = m.group(1) if m else \"missing\" active = m.group(2) if m else \"missing\" facts[f\"edge_{safe}_timer_enabled\"] = enabled facts[f\"edge_{safe}_timer_active\"] = active if enabled != \"enabled\" or active != \"active\": problems.append(f\"edge_{safe}_timer_bad\")\nDETAIL=RELEVANT_NODE=LINE:190|TYPE:Assign|SOURCE:facts[f\"edge_{safe}_timer_enabled\"] = enabled\nDETAIL=RELEVANT_NODE=LINE:191|TYPE:Assign|SOURCE:facts[f\"edge_{safe}_timer_active\"] = active\nDETAIL=RELEVANT_NODE=LINE:192|TYPE:If|SOURCE:if enabled != \"enabled\" or active != \"active\": problems.append(f\"edge_{safe}_timer_bad\")\nDETAIL=RELEVANT_NODE=LINE:193|TYPE:Call|SOURCE:problems.append(f\"edge_{safe}_timer_bad\")\nDETAIL=RELEVANT_NODE=LINE:202|TYPE:For|SOURCE:for k in [\"monitor_py\", \"config_ini\", \"secrets_env\", \"cookies_jar\", \"operational_digest_py\", \"event_selftest_runner_py\", \"event_selftest_core\", \"weekly_summary_health\", \"age_alert_health\", \"event_selftest_health\", \"py_compile\"]: facts[k] = files.get(k, \"no\") if facts[k] != \"yes\": problems.append(f\"edge_{k}_bad\")\nDETAIL=RELEVANT_NODE=LINE:204|TYPE:If|SOURCE:if facts[k] != \"yes\": problems.append(f\"edge_{k}_bad\")\nDETAIL=RELEVANT_NODE=LINE:205|TYPE:Call|SOURCE:problems.append(f\"edge_{k}_bad\")\nDETAIL=RELEVANT_NODE=LINE:206|TYPE:Assign|SOURCE:facts[\"cookies_jar_bytes_file\"] = files.get(\"cookies_jar_bytes\", \"0\")\nDETAIL=RELEVANT_NODE=LINE:206|TYPE:Call|SOURCE:files.get(\"cookies_jar_bytes\", \"0\")\nDETAIL=RELEVANT_NODE=LINE:213|TYPE:Assign|SOURCE:facts[\"backup_cookie_jar_backed_up\"] = backup.get(\"COOKIE_JAR_BACKED_UP\", \"\")\nDETAIL=RELEVANT_NODE=LINE:213|TYPE:Call|SOURCE:backup.get(\"COOKIE_JAR_BACKED_UP\", \"\")\nDETAIL=RELEVANT_NODE=LINE:219|TYPE:If|SOURCE:if backup.get(\"COOKIE_JAR_BACKED_UP\") != \"yes\": problems.append(\"backup_cookie_jar_not_marked\")\nDETAIL=RELEVANT_NODE=LINE:219|TYPE:Call|SOURCE:backup.get(\"COOKIE_JAR_BACKED_UP\")\nDETAIL=RELEVANT_NODE=LINE:220|TYPE:Call|SOURCE:problems.append(\"backup_cookie_jar_not_marked\")\nDETAIL=RELEVANT_NODE=LINE:239|TYPE:For|SOURCE:for key in [\"CLOUD_1_STATUS\", \"CLOUD_2_STATUS\", \"SHA256_OK\", \"TAR_OK\", \"REQUIRED_FILES_OK\", \"COOKIE_JAR_OK\", \"EXTENSION_HEALTH_FILES_OK\", \"EXTENSION_SCRIPTS_OK\", \"EXTENSION_UNITS_OK\", \"SQLITE_OK\", \"PY_COMPILE_OK\"]: facts[key.lower()] = restore.get(key, \"\")\nDETAIL=RELEVANT_NODE=LINE:248|TYPE:For|SOURCE:for key in [\"SHA256_OK\", \"TAR_OK\", \"REQUIRED_FILES_OK\", \"COOKIE_JAR_OK\", \"EXTENSION_HEALTH_FILES_OK\", \"EXTENSION_SCRIPTS_OK\", \"EXTENSION_UNITS_OK\", \"SQLITE_OK\", \"PY_COMPILE_OK\"]: if restore.get(key) != \"yes\": problems.append(key.lower() + \"_bad\")\nDETAIL=EDGE_COOKIE_SOURCE_END\nDETAIL=SECRET_VALUES_OUTPUT=NONE\nDETAIL=COOKIE_CONTENT_OUTPUT=NONE\nDETAIL=DYNAMIC_PROBLEM_TRACE=PASS\nCHANGES_MADE=NO\n"
|
|
}
|