CHAT_OUTPUT_BEGIN
COMMAND_ID=HOMELAB-SKLADCHIK-DYNAMIC-PROBLEM-TRACE-421
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=cluster-audit
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
COMMAND_SHA256=e1c9ec96c23c56ee20087fbae165b41c11fcde575aa8c3b0f03db028063d0099
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGES_MADE=false
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=55c3331e8fb9e7193d1641f54eba5bb0a57e3958042ac712a7e50ae3f58f1c2e
SANITIZED_OUTPUT_SHA256=55c3331e8fb9e7193d1641f54eba5bb0a57e3958042ac712a7e50ae3f58f1c2e
OUTPUT_BEGIN
DETAIL=SCRIPT_PATH=/usr/local/sbin/skladchik-reports-monitor-supervisor
DETAIL=GATHER_LINES=105-271
DETAIL=PROBLEM_WRITE_COUNT=26
DETAIL=PROBLEM_WRITES_BEGIN
DETAIL=PROBLEM_WRITE=LINE:141|METHOD:append|DYNAMIC:NO|EXPRESSION:"edge_ssh_failed"
DETAIL=ENCLOSING_IF=LINE:140|TEST:rc != 0
DETAIL=PROBLEM_WRITE=LINE:171|METHOD:append|DYNAMIC:NO|EXPRESSION:"monitor_cookie_jar_bad"
DETAIL=ENCLOSING_IF=LINE:170|TEST:mon.get("cookie_jar") not in ("yes", "empty")
DETAIL=PROBLEM_WRITE=LINE:178|METHOD:append|DYNAMIC:NO|EXPRESSION:"cookie_expiry_missing"
DETAIL=ENCLOSING_IF=LINE:177|TEST:expiry_status in ("", "NO_COOKIE_JAR", "NO_PERSISTENT_COOKIE", "ERROR")
DETAIL=PROBLEM_WRITE=LINE:180|METHOD:append|DYNAMIC:NO|EXPRESSION:"cookie_expiry_expired"
DETAIL=ENCLOSING_IF=LINE:179|TEST:expiry_status == "EXPIRED" or (expires_in is not None and expires_in <= 0)
DETAIL=PROBLEM_WRITE=LINE:182|METHOD:append|DYNAMIC:NO|EXPRESSION:"cookie_expiry_warn"
DETAIL=ENCLOSING_IF=LINE:181|TEST:expiry_status == "WARN" or (expires_in is not None and expires_in < COOKIE_EXPIRY_WARN_SECONDS)
DETAIL=PROBLEM_WRITE=LINE:193|METHOD:append|DYNAMIC:YES|EXPRESSION:f"edge_{safe}_timer_bad"
DETAIL=ENCLOSING_IF=LINE:192|TEST:enabled != "enabled" or active != "active"
DETAIL=ENCLOSING_FOR=LINE:185|TARGET:unit|ITER:["skladchik-reports-monitor.timer", "skladchik-reports-monitor-watchdog.timer", "skladchik-reports-monitor-heartbeat.timer", "skladchik-reports-monitor-weekly-summary.timer", "skladchik-reports-monitor-age-alert.timer", "skladchik-reports-monitor-event-selftest.timer"]
DETAIL=RELATED_ASSIGNMENT=LINE:186|SOURCE:safe = unit.replace(".timer", "").replace("_", "-")
DETAIL=RELATED_ASSIGNMENT=LINE:190|SOURCE:facts[f"edge_{safe}_timer_enabled"] = enabled
DETAIL=RELATED_ASSIGNMENT=LINE:191|SOURCE:facts[f"edge_{safe}_timer_active"] = active
DETAIL=PROBLEM_WRITE=LINE:205|METHOD:append|DYNAMIC:YES|EXPRESSION:f"edge_{k}_bad"
DETAIL=ENCLOSING_IF=LINE:204|TEST:facts[k] != "yes"
DETAIL=ENCLOSING_FOR=LINE:202|TARGET:k|ITER:["monitor_py", "config_ini", "secrets_env", "cookies_jar", "operational_digest_py", "event_selftest_runner_py", "event_selftest_core", "weekly_summary_health", "age_alert_health", "event_selftest_health", "py_compile"]
DETAIL=RELATED_ASSIGNMENT=LINE:203|SOURCE:facts[k] = files.get(k, "no")
DETAIL=PROBLEM_WRITE=LINE:220|METHOD:append|DYNAMIC:NO|EXPRESSION:"backup_cookie_jar_not_marked"
DETAIL=ENCLOSING_IF=LINE:219|TEST:backup.get("COOKIE_JAR_BACKED_UP") != "yes"
DETAIL=PROBLEM_WRITE=LINE:231|METHOD:append|DYNAMIC:NO|EXPRESSION:"backup_size_bad"
DETAIL=ENCLOSING_IF=LINE:230|TEST:int(backup.get("SIZE_BYTES", "0")) <= 0
DETAIL=PROBLEM_WRITE=LINE:233|METHOD:append|DYNAMIC:NO|EXPRESSION:"backup_size_bad"
DETAIL=PROBLEM_WRITE=LINE:247|METHOD:append|DYNAMIC:YES|EXPRESSION:key.lower() + "_bad"
DETAIL=ENCLOSING_IF=LINE:246|TEST:restore.get(key) != "OK"
DETAIL=ENCLOSING_FOR=LINE:245|TARGET:key|ITER:["CLOUD_1_STATUS", "CLOUD_2_STATUS"]
DETAIL=RELATED_ASSIGNMENT=LINE:240|SOURCE:facts[key.lower()] = restore.get(key, "")
DETAIL=PROBLEM_WRITE=LINE:250|METHOD:append|DYNAMIC:YES|EXPRESSION:key.lower() + "_bad"
DETAIL=ENCLOSING_IF=LINE:249|TEST:restore.get(key) != "yes"
DETAIL=ENCLOSING_FOR=LINE:248|TARGET:key|ITER:["SHA256_OK", "TAR_OK", "REQUIRED_FILES_OK", "COOKIE_JAR_OK", "EXTENSION_HEALTH_FILES_OK", "EXTENSION_SCRIPTS_OK", "EXTENSION_UNITS_OK", "SQLITE_OK", "PY_COMPILE_OK"]
DETAIL=RELATED_ASSIGNMENT=LINE:240|SOURCE:facts[key.lower()] = restore.get(key, "")
DETAIL=PROBLEM_WRITE=LINE:265|METHOD:append|DYNAMIC:YES|EXPRESSION:base + "_timer_bad"
DETAIL=ENCLOSING_IF=LINE:264|TEST:p["enabled"] != "enabled" or p["active"] != "active"
DETAIL=ENCLOSING_FOR=LINE:259|TARGET:u, p|ITER:props.items()
DETAIL=RELATED_ASSIGNMENT=LINE:260|SOURCE:base = u.replace(".timer", "").replace("-", "_")
DETAIL=RELATED_ASSIGNMENT=LINE:261|SOURCE:facts[base + "_timer_enabled"] = p["enabled"]
DETAIL=RELATED_ASSIGNMENT=LINE:262|SOURCE:facts[base + "_timer_active"] = p["active"]
DETAIL=PROBLEM_WRITE=LINE:269|METHOD:append|DYNAMIC:YES|EXPRESSION:base + "_timer_unknown"
DETAIL=ENCLOSING_IF=LINE:268|TEST:p["enabled"] not in ("enabled", "disabled") or p["active"] not in ("active", "inactive")
DETAIL=ENCLOSING_FOR=LINE:259|TARGET:u, p|ITER:props.items()
DETAIL=RELATED_ASSIGNMENT=LINE:260|SOURCE:base = u.replace(".timer", "").replace("-", "_")
DETAIL=RELATED_ASSIGNMENT=LINE:261|SOURCE:facts[base + "_timer_enabled"] = p["enabled"]
DETAIL=RELATED_ASSIGNMENT=LINE:262|SOURCE:facts[base + "_timer_active"] = p["active"]
DETAIL=PROBLEM_WRITES_END
DETAIL=EDGE_COOKIE_SOURCE_BEGIN
DETAIL=RELEVANT_NODE=LINE:109|TYPE:Assign|SOURCE:edge_cmd = r''' set +e echo __MONITOR_HEALTH_BEGIN__ cat /var/lib/skladchik-reports-monitor/health.txt 2>/dev/null || true echo __MONITOR_HEALTH_END__ echo __ALERT_HEALTH_BEGIN__ cat /var/lib/skladchik-reports-monitor/alert-health.txt 2>/dev/null || true echo __ALERT_HEALTH_END__ echo __TIMERS_BEGIN__ for u in skladchik-reports-monitor.timer skladchik-reports-monitor-watchdog.timer skladchik-reports-monitor-heartbeat.timer skladchik-reports-monitor-weekly-summary.timer skladchik-reports-monitor-age-alert.timer skladchik-reports-monitor-event-selftest.timer; do echo "$u enabled=$(systemctl is-enabled "$u" 2>/dev/null) active=$(systemctl is-active "$u" 2>/dev/null)"; done echo __TIMERS_END__ echo __ROUTE_BEGIN__ ip route get 1.1.1.1 2>&1 | head -n 1 echo __ROUTE_END__ echo __FILES_BEGIN__ test -s /opt/skladchik-reports-monitor/monitor.py && echo monitor_py=yes || echo monitor_py=no test -s /etc/skladchik-reports-monitor/config.ini && echo config_ini=yes || echo config_ini=no test -s /etc/skladchik-reports-monitor/secrets.env && echo secrets_env=yes || echo secrets_env=no test -s /var/lib/skladchik-reports-monitor/cookies.jar && echo cookies_jar=yes || echo cookies_jar=no [ -s /var/lib/skladchik-reports-monitor/cookies.jar ] && echo cookies_jar_bytes=$(stat -c %s /var/lib/skladchik-reports-monitor/cookies.jar 2>/dev/null) || echo cookies_jar_bytes=0 test -s /opt/skladchik-reports-monitor/operational_digest.py && echo operational_digest_py=yes || echo operational_digest_py=no test -s /opt/skladchik-reports-monitor/event-selftest-runner.py && echo event_selftest_runner_py=yes || echo event_selftest_runner_py=no test -s /opt/skladchik-reports-monitor/event-selftest-core && echo event_selftest_core=yes || echo event_selftest_core=no test -s /var/lib/skladchik-reports-monitor/weekly-summary-health.txt && echo weekly_summary_health=yes || echo weekly_summary_health=no test -s /var/lib/skladchik-reports-monitor/age-alert-health.txt && echo age_alert_health=yes || echo age_alert_health=no test -s /var/lib/skladchik-reports-monitor/event-selftest-health.txt && echo event_selftest_health=yes || echo event_selftest_health=no python3 -m py_compile /opt/skladchik-reports-monitor/monitor.py /opt/skladchik-reports-monitor/operational_digest.py /opt/skladchik-reports-monitor/event-selftest-runner.py /opt/skladchik-reports-monitor/event-selftest-core >/dev/null 2>&1 && echo py_compile=yes || echo py_compile=no echo __FILES_END__ '''
DETAIL=RELEVANT_NODE=LINE:138|TYPE:Assign|SOURCE:rc, out = ssh_edge(edge_cmd, timeout=45)
DETAIL=RELEVANT_NODE=LINE:138|TYPE:Call|SOURCE:ssh_edge(edge_cmd, timeout=45)
DETAIL=RELEVANT_NODE=LINE:139|TYPE:Assign|SOURCE:facts["edge_ssh_rc"] = str(rc)
DETAIL=RELEVANT_NODE=LINE:140|TYPE:If|SOURCE:if rc != 0: problems.append("edge_ssh_failed")
DETAIL=RELEVANT_NODE=LINE:141|TYPE:Call|SOURCE:problems.append("edge_ssh_failed")
DETAIL=RELEVANT_NODE=LINE:154|TYPE:Assign|SOURCE:facts["cookie_jar"] = mon.get("cookie_jar", "")
DETAIL=RELEVANT_NODE=LINE:154|TYPE:Call|SOURCE:mon.get("cookie_jar", "")
DETAIL=RELEVANT_NODE=LINE:155|TYPE:Assign|SOURCE:facts["cookie_jar_bytes"] = mon.get("cookie_jar_bytes", "")
DETAIL=RELEVANT_NODE=LINE:155|TYPE:Call|SOURCE:mon.get("cookie_jar_bytes", "")
DETAIL=RELEVANT_NODE=LINE:156|TYPE:Assign|SOURCE:facts["cookie_expiry_status"] = mon.get("cookie_expiry_status", "")
DETAIL=RELEVANT_NODE=LINE:156|TYPE:Call|SOURCE:mon.get("cookie_expiry_status", "")
DETAIL=RELEVANT_NODE=LINE:157|TYPE:Assign|SOURCE:facts["cookie_expiry_utc"] = mon.get("cookie_expiry_utc", "")
DETAIL=RELEVANT_NODE=LINE:157|TYPE:Call|SOURCE:mon.get("cookie_expiry_utc", "")
DETAIL=RELEVANT_NODE=LINE:158|TYPE:Assign|SOURCE:facts["cookie_expires_in_seconds"] = mon.get("cookie_expires_in_seconds", "")
DETAIL=RELEVANT_NODE=LINE:158|TYPE:Call|SOURCE:mon.get("cookie_expires_in_seconds", "")
DETAIL=RELEVANT_NODE=LINE:159|TYPE:Assign|SOURCE:facts["cookie_expires_in_days"] = mon.get("cookie_expires_in_days", "")
DETAIL=RELEVANT_NODE=LINE:159|TYPE:Call|SOURCE:mon.get("cookie_expires_in_days", "")
DETAIL=RELEVANT_NODE=LINE:160|TYPE:Assign|SOURCE:facts["cookie_persistent_count"] = mon.get("cookie_persistent_count", "")
DETAIL=RELEVANT_NODE=LINE:160|TYPE:Call|SOURCE:mon.get("cookie_persistent_count", "")
DETAIL=RELEVANT_NODE=LINE:168|TYPE:If|SOURCE:if mon.get("session_keepalive") != "persistent_cookie_jar": problems.append("session_keepalive_not_persistent")
DETAIL=RELEVANT_NODE=LINE:170|TYPE:If|SOURCE:if mon.get("cookie_jar") not in ("yes", "empty"): problems.append("monitor_cookie_jar_bad")
DETAIL=RELEVANT_NODE=LINE:170|TYPE:Call|SOURCE:mon.get("cookie_jar")
DETAIL=RELEVANT_NODE=LINE:171|TYPE:Call|SOURCE:problems.append("monitor_cookie_jar_bad")
DETAIL=RELEVANT_NODE=LINE:172|TYPE:Assign|SOURCE:expiry_status = mon.get("cookie_expiry_status", "")
DETAIL=RELEVANT_NODE=LINE:172|TYPE:Call|SOURCE:mon.get("cookie_expiry_status", "")
DETAIL=RELEVANT_NODE=LINE:174|TYPE:Assign|SOURCE:expires_in = int(mon.get("cookie_expires_in_seconds", ""))
DETAIL=RELEVANT_NODE=LINE:174|TYPE:Call|SOURCE:int(mon.get("cookie_expires_in_seconds", ""))
DETAIL=RELEVANT_NODE=LINE:174|TYPE:Call|SOURCE:mon.get("cookie_expires_in_seconds", "")
DETAIL=RELEVANT_NODE=LINE:177|TYPE:If|SOURCE:if expiry_status in ("", "NO_COOKIE_JAR", "NO_PERSISTENT_COOKIE", "ERROR"): problems.append("cookie_expiry_missing") elif expiry_status == "EXPIRED" or (expires_in is not None and expires_in <= 0): problems.append("cookie_expiry_expired") elif expiry_status == "WARN" or (expires_in is not None and expires_in < COOKIE_EXPIRY_WARN_SECONDS): problems.append("cookie_expiry_warn")
DETAIL=RELEVANT_NODE=LINE:178|TYPE:Call|SOURCE:problems.append("cookie_expiry_missing")
DETAIL=RELEVANT_NODE=LINE:179|TYPE:If|SOURCE:elif expiry_status == "EXPIRED" or (expires_in is not None and expires_in <= 0): problems.append("cookie_expiry_expired") elif expiry_status == "WARN" or (expires_in is not None and expires_in < COOKIE_EXPIRY_WARN_SECONDS): problems.append("cookie_expiry_warn")
DETAIL=RELEVANT_NODE=LINE:180|TYPE:Call|SOURCE:problems.append("cookie_expiry_expired")
DETAIL=RELEVANT_NODE=LINE:181|TYPE:If|SOURCE:elif expiry_status == "WARN" or (expires_in is not None and expires_in < COOKIE_EXPIRY_WARN_SECONDS): problems.append("cookie_expiry_warn")
DETAIL=RELEVANT_NODE=LINE:182|TYPE:Call|SOURCE:problems.append("cookie_expiry_warn")
DETAIL=RELEVANT_NODE=LINE:185|TYPE:For|SOURCE:for unit in ["skladchik-reports-monitor.timer", "skladchik-reports-monitor-watchdog.timer", "skladchik-reports-monitor-heartbeat.timer", "skladchik-reports-monitor-weekly-summary.timer", "skladchik-reports-monitor-age-alert.timer", "skladchik-reports-monitor-event-selftest.timer"]: safe = unit.replace(".timer", "").replace("_", "-") m = re.search(re.escape(unit) + r"\s+enabled=([^\s]+)\s+active=([^\s]+)", timers_text) enabled = m.group(1) if m else "missing" active = m.group(2) if m else "missing" facts[f"edge_{safe}_timer_enabled"] = enabled facts[f"edge_{safe}_timer_active"] = active if enabled != "enabled" or active != "active": problems.append(f"edge_{safe}_timer_bad")
DETAIL=RELEVANT_NODE=LINE:190|TYPE:Assign|SOURCE:facts[f"edge_{safe}_timer_enabled"] = enabled
DETAIL=RELEVANT_NODE=LINE:191|TYPE:Assign|SOURCE:facts[f"edge_{safe}_timer_active"] = active
DETAIL=RELEVANT_NODE=LINE:192|TYPE:If|SOURCE:if enabled != "enabled" or active != "active": problems.append(f"edge_{safe}_timer_bad")
DETAIL=RELEVANT_NODE=LINE:193|TYPE:Call|SOURCE:problems.append(f"edge_{safe}_timer_bad")
DETAIL=RELEVANT_NODE=LINE:202|TYPE:For|SOURCE:for k in ["monitor_py", "config_ini", "secrets_env", "cookies_jar", "operational_digest_py", "event_selftest_runner_py", "event_selftest_core", "weekly_summary_health", "age_alert_health", "event_selftest_health", "py_compile"]: facts[k] = files.get(k, "no") if facts[k] != "yes": problems.append(f"edge_{k}_bad")
DETAIL=RELEVANT_NODE=LINE:204|TYPE:If|SOURCE:if facts[k] != "yes": problems.append(f"edge_{k}_bad")
DETAIL=RELEVANT_NODE=LINE:205|TYPE:Call|SOURCE:problems.append(f"edge_{k}_bad")
DETAIL=RELEVANT_NODE=LINE:206|TYPE:Assign|SOURCE:facts["cookies_jar_bytes_file"] = files.get("cookies_jar_bytes", "0")
DETAIL=RELEVANT_NODE=LINE:206|TYPE:Call|SOURCE:files.get("cookies_jar_bytes", "0")
DETAIL=RELEVANT_NODE=LINE:213|TYPE:Assign|SOURCE:facts["backup_cookie_jar_backed_up"] = backup.get("COOKIE_JAR_BACKED_UP", "")
DETAIL=RELEVANT_NODE=LINE:213|TYPE:Call|SOURCE:backup.get("COOKIE_JAR_BACKED_UP", "")
DETAIL=RELEVANT_NODE=LINE:219|TYPE:If|SOURCE:if backup.get("COOKIE_JAR_BACKED_UP") != "yes": problems.append("backup_cookie_jar_not_marked")
DETAIL=RELEVANT_NODE=LINE:219|TYPE:Call|SOURCE:backup.get("COOKIE_JAR_BACKED_UP")
DETAIL=RELEVANT_NODE=LINE:220|TYPE:Call|SOURCE:problems.append("backup_cookie_jar_not_marked")
DETAIL=RELEVANT_NODE=LINE:239|TYPE:For|SOURCE:for key in ["CLOUD_1_STATUS", "CLOUD_2_STATUS", "SHA256_OK", "TAR_OK", "REQUIRED_FILES_OK", "COOKIE_JAR_OK", "EXTENSION_HEALTH_FILES_OK", "EXTENSION_SCRIPTS_OK", "EXTENSION_UNITS_OK", "SQLITE_OK", "PY_COMPILE_OK"]: facts[key.lower()] = restore.get(key, "")
DETAIL=RELEVANT_NODE=LINE:248|TYPE:For|SOURCE:for key in ["SHA256_OK", "TAR_OK", "REQUIRED_FILES_OK", "COOKIE_JAR_OK", "EXTENSION_HEALTH_FILES_OK", "EXTENSION_SCRIPTS_OK", "EXTENSION_UNITS_OK", "SQLITE_OK", "PY_COMPILE_OK"]: if restore.get(key) != "yes": problems.append(key.lower() + "_bad")
DETAIL=EDGE_COOKIE_SOURCE_END
DETAIL=SECRET_VALUES_OUTPUT=NONE
DETAIL=COOKIE_CONTENT_OUTPUT=NONE
DETAIL=DYNAMIC_PROBLEM_TRACE=PASS
CHANGES_MADE=NO

OUTPUT_END
CHAT_OUTPUT_END
