CHAT_OUTPUT_BEGIN
COMMAND_ID=HOMELAB-SKLADCHIK-COOKIE-CHECK-SOURCE-AUDIT-418
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=cluster-audit
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
COMMAND_SHA256=77896618b46e2f97362240deb743fb8e625075381e57377ee0314130be2b2026
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGES_MADE=false
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=1c26f0b751c5e55421c706f56f4be25d92c17849df8a385fb66ce01144801022
SANITIZED_OUTPUT_SHA256=8e5a54db35050a5c0010241fb0bad91fc5966530dbac76bb07426b09af51a6a1
OUTPUT_BEGIN
DETAIL=SCRIPT_PATH=/usr/local/sbin/skladchik-reports-monitor-supervisor
DETAIL=SCRIPT_SHA256=35c68ae0929ff3795be2bde40de334fc8e26413a3d8a620a137be90d1f2c164e
DETAIL=SCRIPT_METADATA=BYTES:20463|MODE:755|OWNER:root:root|MTIME:2026-07-18 13:52:30.415780066 +0300
DETAIL=RELEVANT_SOURCE_BEGIN
DETAIL=SOURCE_LINE=0005:import os
DETAIL=SOURCE_LINE=0006:import re
DETAIL=SOURCE_LINE=0007:import shlex
DETAIL=SOURCE_LINE=0008:import subprocess
DETAIL=SOURCE_LINE=0009:import sys
DETAIL=SOURCE_LINE=0010:from pathlib import Path
DETAIL=SOURCE_LINE=0011:
DETAIL=SOURCE_LINE=0012:EDGE = os.environ.get("SKLADCHIK_EDGE", "debian@[PRIVATE_IP]")
DETAIL=SOURCE_LINE=0013:STATE_DIR = Path("/var/lib/skladchik-reports-monitor-supervisor")
DETAIL=SOURCE_LINE=0014:LOG_DIR = Path("/var/log/skladchik-reports-monitor-supervisor")
DETAIL=SOURCE_LINE=0015:HEALTH = STATE_DIR / "health.txt"
DETAIL=SOURCE_LINE=0016:STATE = STATE_DIR / "state.env"
DETAIL=SOURCE_LINE=0017:LOG = LOG_DIR / "supervisor.log"
DETAIL=SOURCE_LINE=0018:
DETAIL=SOURCE_LINE=0019:MONITOR_HEALTH_AGE_MAX = int(os.environ.get("SKLADCHIK_MONITOR_HEALTH_MAX_SECONDS", "600"))
DETAIL=SOURCE_LINE=0020:BACKUP_HEALTH_AGE_MAX = int(os.environ.get("SKLADCHIK_BACKUP_HEALTH_MAX_SECONDS", "129600"))
DETAIL=SOURCE_LINE=0021:RESTORE_HEALTH_AGE_MAX = int(os.environ.get("SKLADCHIK_RESTORE_HEALTH_MAX_SECONDS", "691200"))
DETAIL=SOURCE_LINE=0022:COOKIE_EXPIRY_WARN_SECONDS = int(os.environ.get("SKLADCHIK_COOKIE_EXPIRY_WARN_SECONDS", "604800"))
DETAIL=SOURCE_LINE=0023:
DETAIL=SOURCE_LINE=0024:PVE_BACKUP_HEALTH = Path("/var/lib/skladchik-reports-monitor-backup/health.txt")
DETAIL=SOURCE_LINE=0025:PVE_RESTORE_HEALTH = Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt")
DETAIL=SOURCE_LINE=0026:
DETAIL=SOURCE_LINE=0027:def now():
DETAIL=SOURCE_LINE=0028:    return _dt.datetime.now(_dt.UTC).replace(microsecond=0)
DETAIL=SOURCE_LINE=0029:
DETAIL=SOURCE_LINE=0030:def iso():
DETAIL=SOURCE_LINE=0031:    return now().isoformat().replace("+00:00", "Z")
DETAIL=SOURCE_LINE=0032:
DETAIL=SOURCE_LINE=0033:def log(msg):
DETAIL=SOURCE_LINE=0034:    LOG_DIR.mkdir(parents=True, exist_ok=True)
DETAIL=SOURCE_LINE=0035:    with LOG.open("a", encoding="utf-8") as f:
DETAIL=SOURCE_LINE=0036:        f.write(f"{iso()} {msg}\n")
DETAIL=SOURCE_LINE=0037:
DETAIL=SOURCE_LINE=0038:def run(cmd, timeout=45, input_text=None):
DETAIL=SOURCE_LINE=0039:    try:
DETAIL=SOURCE_LINE=0040:        p = subprocess.run(cmd, shell=True, text=True, input=input_text, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout)
DETAIL=SOURCE_LINE=0041:        return p.returncode, p.stdout.strip()
DETAIL=SOURCE_LINE=0042:    except subprocess.TimeoutExpired as e:
DETAIL=SOURCE_LINE=0043:        out = e.stdout if isinstance(e.stdout, str) else ""
DETAIL=SOURCE_LINE=0044:        return 124, (out or "").strip() + "\nTIMEOUT"
DETAIL=SOURCE_LINE=0045:
DETAIL=SOURCE_LINE=0046:def parse_env_text(text):
DETAIL=SOURCE_LINE=0047:    d = {}
DETAIL=SOURCE_LINE=0048:    for line in (text or "").splitlines():
DETAIL=SOURCE_LINE=0049:        if not line or line.startswith("#") or "=" not in line:
DETAIL=SOURCE_LINE=0050:            continue
DETAIL=SOURCE_LINE=0051:        k, v = line.split("=", 1)
DETAIL=SOURCE_LINE=...
DETAIL=SOURCE_LINE=0066:        dt = _dt.datetime.fromisoformat(s)
DETAIL=SOURCE_LINE=0067:        if dt.tzinfo is None:
DETAIL=SOURCE_LINE=0068:            dt = dt.replace(tzinfo=_dt.UTC)
DETAIL=SOURCE_LINE=0069:        return int((now() - dt.astimezone(_dt.UTC)).total_seconds())
DETAIL=SOURCE_LINE=0070:    except Exception:
DETAIL=SOURCE_LINE=0071:        return 999999999
DETAIL=SOURCE_LINE=0072:
DETAIL=SOURCE_LINE=0073:def ssh_edge(cmd, timeout=45):
DETAIL=SOURCE_LINE=0074:    q = "ssh -o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] " + shlex.quote(EDGE) + " " + shlex.quote("sudo bash -lc " + shlex.quote(cmd))
DETAIL=SOURCE_LINE=0075:    return run(q, timeout=timeout)
DETAIL=SOURCE_LINE=0076:
DETAIL=SOURCE_LINE=0077:def edge_mail(subject, body):
DETAIL=SOURCE_LINE=0078:    subj_b64 = base64.b64encode(subject.encode("utf-8")).decode("ascii")
DETAIL=SOURCE_LINE=0079:    body_b64 = base64.b64encode(body.encode("utf-8")).decode("ascii")
DETAIL=SOURCE_LINE=0080:    py = (
DETAIL=SOURCE_LINE=0081:        "import base64,importlib.util;"
DETAIL=SOURCE_LINE=0082:        "spec=importlib.util.spec_from_file_location('m','/opt/skladchik-reports-monitor/monitor.py');"
DETAIL=SOURCE_LINE=0083:        "m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m);"
DETAIL=SOURCE_LINE=0084:        "cfg,secrets=<REDACTED>
DETAIL=SOURCE_LINE=0085:        f"subject=base64.b64decode('{subj_b64}').decode('utf-8');"
DETAIL=SOURCE_LINE=0086:        f"body=base64.b64decode('{body_b64}').decode('utf-8');"
DETAIL=SOURCE_LINE=0087:        "ok=m.send_mail(cfg,secrets,subject,body);"
DETAIL=SOURCE_LINE=0088:        "print('MAIL_OK' if ok else 'MAIL_FALSE')"
DETAIL=SOURCE_LINE=0089:    )
DETAIL=SOURCE_LINE=0090:    rc, out = ssh_edge("python3 -c " + shlex.quote(py), timeout=60)
DETAIL=SOURCE_LINE=0091:    log(f"MAIL subject={subject!r} rc={rc} out={out!r}")
DETAIL=SOURCE_LINE=0092:    return rc == 0 and "MAIL_OK" in out
DETAIL=SOURCE_LINE=0093:
DETAIL=SOURCE_LINE=0094:def systemctl_props(units):
DETAIL=SOURCE_LINE=0095:    cmd = "systemctl is-enabled " + " ".join(map(shlex.quote, units)) + " 2>/dev/null || true; echo __ACTIVE__; systemctl is-active " + " ".join(map(shlex.quote, units)) + " 2>/dev/null || true"
DETAIL=SOURCE_LINE=0096:    rc, out = run(cmd, timeout=20)
DETAIL=SOURCE_LINE=0097:    parts = out.split("__ACTIVE__", 1)
DETAIL=SOURCE_LINE=0098:    enabled_lines = [x.strip() for x in parts[0].splitlines() if x.strip()]
DETAIL=SOURCE_LINE=0099:    active_lines = [x.strip() for x in parts[1].splitlines() if x.strip()] if len(parts) > 1 else []
DETAIL=SOURCE_LINE=0100:    res = {}
DETAIL=SOURCE_LINE=0101:    for i, u in enumerate(units):
DETAIL=SOURCE_LINE=0102:        res[u] = {"enabled": enabled_lines[i] if i < len(enabled_lines) else "unknown", "active": active_lines[i] if i < len(active_lines) else "unknown"}
DETAIL=SOURCE_LINE=0103:    return res
DETAIL=SOURCE_LINE=0104:
DETAIL=SOURCE_LINE=0105:def gather():
DETAIL=SOURCE_LINE=0106:    facts = {}
DETAIL=SOURCE_LINE=0107:    problems = []
DETAIL=SOURCE_LINE=0108:
DETAIL=SOURCE_LINE=0109:    edge_cmd = r'''
DETAIL=SOURCE_LINE=0110:set +e
DETAIL=SOURCE_LINE=0111:echo __MONITOR_HEALTH_BEGIN__
DETAIL=SOURCE_LINE=0112:cat /var/lib/skladchik-reports-monitor/health.txt 2>/dev/null || true
DETAIL=SOURCE_LINE=0113:echo __MONITOR_HEALTH_END__
DETAIL=SOURCE_LINE=0114:echo __ALERT_HEALTH_BEGIN__
DETAIL=SOURCE_LINE=0115:cat /var/lib/skladchik-reports-monitor/alert-health.txt 2>/dev/null || true
DETAIL=SOURCE_LINE=0116:echo __ALERT_HEALTH_END__
DETAIL=SOURCE_LINE=0117:echo __TIMERS_BEGIN__
DETAIL=SOURCE_LINE=0118:for u in skladchik-reports-monitor.timer skladchik-reports-monitor-watchdog.timer skladchik-reports-monitor-heartbeat.timer skladchik-reports-monitor-weekly-summary.timer skladchik-reports-monitor-age-alert.timer skladchik-reports-monitor-event-selftest.timer; do echo "$u enabled=$(systemctl is-enabled "$u" 2>/dev/null) active=$(systemctl is-active "$u" 2>/dev/null)"; done
DETAIL=SOURCE_LINE=0119:echo __TIMERS_END__
DETAIL=SOURCE_LINE=0120:echo __ROUTE_BEGIN__
DETAIL=SOURCE_LINE=0121:ip route get 1.1.1.1 2>&1 | head -n 1
DETAIL=SOURCE_LINE=0122:echo __ROUTE_END__
DETAIL=SOURCE_LINE=0123:echo __FILES_BEGIN__
DETAIL=SOURCE_LINE=0124:test -s /opt/skladchik-reports-monitor/monitor.py && echo monitor_py=yes || echo monitor_py=no
DETAIL=SOURCE_LINE=0125:test -s /etc/skladchik-reports-monitor/config.ini && echo config_ini=yes || echo config_ini=no
DETAIL=SOURCE_LINE=0126:test -s /etc/skladchik-reports-monitor/secrets.env && echo secrets_env=<REDACTED>
DETAIL=SOURCE_LINE=0127:test -s /var/lib/skladchik-reports-monitor/cookies.jar && echo cookies_jar=yes || echo cookies_jar=no
DETAIL=SOURCE_LINE=0128:[ -s /var/lib/skladchik-reports-monitor/cookies.jar ] && echo cookies_jar_bytes=$(stat -c %s /var/lib/skladchik-reports-monitor/cookies.jar 2>/dev/null) || echo cookies_jar_bytes=0
DETAIL=SOURCE_LINE=0129:test -s /opt/skladchik-reports-monitor/operational_digest.py && echo operational_digest_py=yes || echo operational_digest_py=no
DETAIL=SOURCE_LINE=0130:test -s /opt/skladchik-reports-monitor/event-selftest-runner.py && echo event_selftest_runner_py=yes || echo event_selftest_runner_py=no
DETAIL=SOURCE_LINE=0131:test -s /opt/skladchik-reports-monitor/event-selftest-core && echo event_selftest_core=yes || echo event_selftest_core=no
DETAIL=SOURCE_LINE=0132:test -s /var/lib/skladchik-reports-monitor/weekly-summary-health.txt && echo weekly_summary_health=yes || echo weekly_summary_health=no
DETAIL=SOURCE_LINE=0133:test -s /var/lib/skladchik-reports-monitor/age-alert-health.txt && echo age_alert_health=yes || echo age_alert_health=no
DETAIL=SOURCE_LINE=0134:test -s /var/lib/skladchik-reports-monitor/event-selftest-health.txt && echo event_selftest_health=yes || echo event_selftest_health=no
DETAIL=SOURCE_LINE=0135:python3 -m py_compile /opt/skladchik-reports-monitor/monitor.py /opt/skladchik-reports-monitor/operational_digest.py /opt/skladchik-reports-monitor/event-selftest-runner.py /opt/skladchik-reports-monitor/event-selftest-core >/dev/null 2>&1 && echo py_compile=yes || echo py_compile=no
DETAIL=SOURCE_LINE=0136:echo __FILES_END__
DETAIL=SOURCE_LINE=0137:'''
DETAIL=SOURCE_LINE=0138:    rc, out = ssh_edge(edge_cmd, timeout=45)
DETAIL=SOURCE_LINE=0139:    facts["edge_ssh_rc"] = str(rc)
DETAIL=SOURCE_LINE=0140:    if rc != 0:
DETAIL=SOURCE_LINE=0141:        problems.append("edge_ssh_failed")
DETAIL=SOURCE_LINE=0142:
DETAIL=SOURCE_LINE=0143:    def section(name):
DETAIL=SOURCE_LINE=0144:        m = re.search(rf"__{name}_BEGIN__\n(.*?)\n__{name}_END__", out, re.S)
DETAIL=SOURCE_LINE=0145:        return m.group(1).strip() if m else ""
DETAIL=SOURCE_LINE=0146:
DETAIL=SOURCE_LINE=0147:    mon = parse_env_text(section("MONITOR_HEALTH"))
DETAIL=SOURCE_LINE=0148:    facts["monitor_status"] = mon.get("STATUS", "MISSING")
DETAIL=SOURCE_LINE=0149:    facts["monitor_reports"] = mon.get("reports", "")
DETAIL=SOURCE_LINE=0150:    facts["monitor_open"] = mon.get("open", "")
DETAIL=SOURCE_LINE=0151:    facts["monitor_resolved"] = mon.get("resolved", "")
DETAIL=SOURCE_LINE=0152:    facts["monitor_smtp_configured"] = mon.get("smtp_configured", "")
DETAIL=SOURCE_LINE=0153:    facts["session_keepalive"] = mon.get("session_keepalive", "")
DETAIL=SOURCE_LINE=0154:    facts["cookie_jar"] = mon.get("cookie_jar", "")
DETAIL=SOURCE_LINE=0155:    facts["cookie_jar_bytes"] = mon.get("cookie_jar_bytes", "")
DETAIL=SOURCE_LINE=0156:    facts["cookie_expiry_status"] = mon.get("cookie_expiry_status", "")
DETAIL=SOURCE_LINE=0157:    facts["cookie_expiry_utc"] = mon.get("cookie_expiry_utc", "")
DETAIL=SOURCE_LINE=0158:    facts["cookie_expires_in_seconds"] = mon.get("cookie_expires_in_seconds", "")
DETAIL=SOURCE_LINE=0159:    facts["cookie_expires_in_days"] = mon.get("cookie_expires_in_days", "")
DETAIL=SOURCE_LINE=0160:    facts["cookie_persistent_count"] = mon.get("cookie_persistent_count", "")
DETAIL=SOURCE_LINE=0161:    facts["monitor_health_age_seconds"] = str(age_seconds_from_ts(mon.get("TS")))
DETAIL=SOURCE_LINE=0162:    if mon.get("STATUS") != "OK":
DETAIL=SOURCE_LINE=0163:        problems.append("monitor_status_not_ok")
DETAIL=SOURCE_LINE=0164:    if int(facts["monitor_health_age_seconds"]) > MONITOR_HEALTH_AGE_MAX:
DETAIL=SOURCE_LINE=0165:        problems.append("monitor_health_stale")
DETAIL=SOURCE_LINE=0166:    if mon.get("reports") in ("", "0"):
DETAIL=SOURCE_LINE=0167:        problems.append("monitor_reports_zero_or_missing")
DETAIL=SOURCE_LINE=0168:    if mon.get("session_keepalive") != "persistent_cookie_jar":
DETAIL=SOURCE_LINE=0169:        problems.append("session_keepalive_not_persistent")
DETAIL=SOURCE_LINE=0170:    if mon.get("cookie_jar") not in ("yes", "empty"):
DETAIL=SOURCE_LINE=0171:        problems.append("monitor_cookie_jar_bad")
DETAIL=SOURCE_LINE=0172:    expiry_status = mon.get("cookie_expiry_status", "")
DETAIL=SOURCE_LINE=0173:    try:
DETAIL=SOURCE_LINE=0174:        expires_in = int(mon.get("cookie_expires_in_seconds", ""))
DETAIL=SOURCE_LINE=0175:    except Exception:
DETAIL=SOURCE_LINE=0176:        expires_in = None
DETAIL=SOURCE_LINE=0177:    if expiry_status in ("", "NO_COOKIE_JAR", "NO_PERSISTENT_COOKIE", "ERROR"):
DETAIL=SOURCE_LINE=0178:        problems.append("cookie_expiry_missing")
DETAIL=SOURCE_LINE=0179:    elif expiry_status == "EXPIRED" or (expires_in is not None and expires_in <= 0):
DETAIL=SOURCE_LINE=0180:        problems.append("cookie_expiry_expired")
DETAIL=SOURCE_LINE=0181:    elif expiry_status == "WARN" or (expires_in is not None and expires_in < COOKIE_EXPIRY_WARN_SECONDS):
DETAIL=SOURCE_LINE=0182:        problems.append("cookie_expiry_warn")
DETAIL=SOURCE_LINE=0183:
DETAIL=SOURCE_LINE=0184:    timers_text = section("TIMERS")
DETAIL=SOURCE_LINE=0185:    for unit in ["skladchik-reports-monitor.timer", "skladchik-reports-monitor-watchdog.timer", "skladchik-reports-monitor-heartbeat.timer", "skladchik-reports-monitor-weekly-summary.timer", "skladchik-reports-monitor-age-alert.timer", "skladchik-reports-monitor-event-selftest.timer"]:
DETAIL=SOURCE_LINE=0186:        safe = unit.replace(".timer", "").replace("_", "-")
DETAIL=SOURCE_LINE=0187:        m = re.search(re.escape(unit) + r"\s+enabled=([^\s]+)\s+active=([^\s]+)", timers_text)
DETAIL=SOURCE_LINE=0188:        enabled = m.group(1) if m else "missing"
DETAIL=SOURCE_LINE=0189:        active = m.group(2) if m else "missing"
DETAIL=SOURCE_LINE=0190:        facts[f"edge_{safe}_timer_enabled"] = enabled
DETAIL=SOURCE_LINE=0191:        facts[f"edge_{safe}_timer_active"] = active
DETAIL=SOURCE_LINE=0192:        if enabled != "enabled" or active != "active":
DETAIL=SOURCE_LINE=0193:            problems.append(f"edge_{safe}_timer_bad")
DETAIL=SOURCE_LINE=0194:
DETAIL=SOURCE_LINE=0195:    route_lines = section("ROUTE").splitlines()
DETAIL=SOURCE_LINE=0196:    route = route_lines[0] if route_lines else ""
DETAIL=SOURCE_LINE=0197:    facts["netbird_route"] = route
DETAIL=SOURCE_LINE=0198:    if " wt0 " not in (" " + route + " "):
DETAIL=SOURCE_LINE=0199:        problems.append("netbird_route_not_wt0")
DETAIL=SOURCE_LINE=0200:
DETAIL=SOURCE_LINE=0201:    files = parse_env_text(section("FILES"))
DETAIL=SOURCE_LINE=0202:    for k in ["monitor_py", "config_ini", "secrets_env", "cookies_jar", "operational_digest_py", "event_selftest_runner_py", "event_selftest_core", "weekly_summary_health", "age_alert_health", "event_selftest_health", "py_compile"]:
DETAIL=SOURCE_LINE=0203:        facts[k] = files.get(k, "no")
DETAIL=SOURCE_LINE=0204:        if facts[k] != "yes":
DETAIL=SOURCE_LINE=0205:            problems.append(f"edge_{k}_bad")
DETAIL=SOURCE_LINE=0206:    facts["cookies_jar_bytes_file"] = files.get("cookies_jar_bytes", "0")
DETAIL=SOURCE_LINE=0207:
DETAIL=SOURCE_LINE=0208:    backup = read_env_file(PVE_BACKUP_HEALTH)
DETAIL=SOURCE_LINE=0209:    facts["backup_status"] = backup.get("STATUS", "MISSING")
DETAIL=SOURCE_LINE=0210:    facts["backup_run_id"] = backup.get("RUN_ID", "")
DETAIL=SOURCE_LINE=0211:    facts["backup_size_bytes"] = backup.get("SIZE_BYTES", "")
DETAIL=SOURCE_LINE=0212:    facts["backup_version"] = backup.get("BACKUP_VERSION", "")
DETAIL=SOURCE_LINE=0213:    facts["backup_cookie_jar_backed_up"] = backup.get("COOKIE_JAR_BACKED_UP", "")
DETAIL=SOURCE_LINE=0214:    facts["backup_extension_health_files_backed_up"] = backup.get("EXTENSION_HEALTH_FILES_BACKED_UP", "")
DETAIL=SOURCE_LINE=0215:    facts["backup_title_in_email_subject"] = backup.get("TITLE_IN_EMAIL_SUBJECT", "")
DETAIL=SOURCE_LINE=0216:    facts["backup_health_age_seconds"] = str(age_seconds_from_ts(backup.get("TS")))
DETAIL=SOURCE_LINE=0217:    if backup.get("STATUS") != "OK":
DETAIL=SOURCE_LINE=0218:        problems.append("backup_status_not_ok")
DETAIL=SOURCE_LINE=0219:    if backup.get("COOKIE_JAR_BACKED_UP") != "yes":
DETAIL=SOURCE_LINE=0220:        problems.append("backup_cookie_jar_not_marked")
DETAIL=SOURCE_LINE=0221:    if backup.get("BACKUP_VERSION") != "4":
DETAIL=SOURCE_LINE=0222:        problems.append("backup_version_not_4")
DETAIL=SOURCE_LINE=0223:    if backup.get("EXTENSION_HEALTH_FILES_BACKED_UP") != "yes":
DETAIL=SOURCE_LINE=0224:        problems.append("backup_extension_health_files_not_marked")
DETAIL=SOURCE_LINE=0225:    if backup.get("TITLE_IN_EMAIL_SUBJECT") != "yes":
DETAIL=SOURCE_LINE=0226:        problems.append("email_title_subject_not_marked")
DETAIL=SOURCE_LINE=0227:    if int(facts["backup_health_age_seconds"]) > BACKUP_HEALTH_AGE_MAX:
DETAIL=SOURCE_LINE=0228:        problems.append("backup_health_stale")
DETAIL=SOURCE_LINE=0229:    try:
DETAIL=SOURCE_LINE=0230:        if int(backup.get("SIZE_BYTES", "0")) <= 0:
DETAIL=SOURCE_LINE=0231:            problems.append("backup_size_bad")
DETAIL=SOURCE_LINE=0232:    except Exception:
DETAIL=SOURCE_LINE=0233:        problems.append("backup_size_bad")
DETAIL=SOURCE_LINE=0234:
DETAIL=SOURCE_LINE=0235:    restore = read_env_file(PVE_RESTORE_HEALTH)
DETAIL=SOURCE_LINE=0236:    facts["restore_status"] = restore.get("STATUS", "MISSING")
DETAIL=SOURCE_LINE=0237:    facts["restore_run_id"] = restore.get("RUN_ID", "")
DETAIL=SOURCE_LINE=0238:    facts["restore_health_age_seconds"] = str(age_seconds_from_ts(restore.get("TS")))
DETAIL=SOURCE_LINE=0239:    for key in ["CLOUD_1_STATUS", "CLOUD_2_STATUS", "SHA256_OK", "TAR_OK", "REQUIRED_FILES_OK", "COOKIE_JAR_OK", "EXTENSION_HEALTH_FILES_OK", "EXTENSION_SCRIPTS_OK", "EXTENSION_UNITS_OK", "SQLITE_OK", "PY_COMPILE_OK"]:
DETAIL=SOURCE_LINE=0240:        facts[key.lower()] = restore.get(key, "")
DETAIL=SOURCE_LINE=0241:    if restore.get("STATUS") != "OK":
DETAIL=SOURCE_LINE=0242:        problems.append("restore_status_not_ok")
DETAIL=SOURCE_LINE=0243:    if int(facts["restore_health_age_seconds"]) > RESTORE_HEALTH_AGE_MAX:
DETAIL=SOURCE_LINE=0244:        problems.append("restore_health_stale")
DETAIL=SOURCE_LINE=0245:    for key in ["CLOUD_1_STATUS", "CLOUD_2_STATUS"]:
DETAIL=SOURCE_LINE=0246:        if restore.get(key) != "OK":
DETAIL=SOURCE_LINE=0247:            problems.append(key.lower() + "_bad")
DETAIL=SOURCE_LINE=0248:    for key in ["SHA256_OK", "TAR_OK", "REQUIRED_FILES_OK", "COOKIE_JAR_OK", "EXTENSION_HEALTH_FILES_OK", "EXTENSION_SCRIPTS_OK", "EXTENSION_UNITS_OK", "SQLITE_OK", "PY_COMPILE_OK"]:
DETAIL=SOURCE_LINE=0249:        if restore.get(key) != "yes":
DETAIL=SOURCE_LINE=0250:            problems.append(key.lower() + "_bad")
DETAIL=SOURCE_LINE=0251:
DETAIL=SOURCE_LINE=0252:    local_units = [
DETAIL=SOURCE_LINE=0253:        "skladchik-reports-monitor-backup.timer",
DETAIL=SOURCE_LINE=0254:        "skladchik-reports-monitor-backup-restore-check.timer",
DETAIL=SOURCE_LINE=0255:        "skladchik-reports-monitor-supervisor.timer",
DETAIL=SOURCE_LINE=0256:        "skladchik-reports-monitor-monthly-selftest-watch.timer",
DETAIL=SOURCE_LINE=0257:    ]
DETAIL=SOURCE_LINE=0258:    props = systemctl_props(local_units)
DETAIL=SOURCE_LINE=0259:    for u, p in props.items():
DETAIL=SOURCE_LINE=0260:        base = u.replace(".timer", "").replace("-", "_")
DETAIL=SOURCE_LINE=0261:        facts[base + "_timer_enabled"] = p["enabled"]
DETAIL=SOURCE_LINE=0262:        facts[base + "_timer_active"] = p["active"]
DETAIL=SOURCE_LINE=0263:        if u != "skladchik-reports-monitor-supervisor.timer":
DETAIL=SOURCE_LINE=0264:            if p["enabled"] != "enabled" or p["active"] != "active":
DETAIL=SOURCE_LINE=0265:                problems.append(base + "_timer_bad")
DETAIL=SOURCE_LINE=0266:        else:
DETAIL=SOURCE_LINE=0267:            # Before first enable it can be disabled/inactive during test. After enable next runs will see enabled/active.
DETAIL=SOURCE_LINE=0268:            if p["enabled"] not in ("enabled", "disabled") or p["active"] not in ("active", "inactive"):
DETAIL=SOURCE_LINE=0269:                problems.append(base + "_timer_unknown")
DETAIL=SOURCE_LINE=0270:
DETAIL=SOURCE_LINE=0271:    return facts, problems
DETAIL=SOURCE_LINE=0272:
DETAIL=SOURCE_LINE=0273:def render_body(status, facts, problems, mode):
DETAIL=SOURCE_LINE=0274:    ru_status = "OK" if status == "OK" else "АВАРИЯ"
DETAIL=SOURCE_LINE=0275:    lines = [
DETAIL=SOURCE_LINE=0276:        f"Статус надзора: {ru_status}",
DETAIL=SOURCE_LINE=0277:        f"Время UTC: {iso()}",
DETAIL=SOURCE_LINE=0278:        "",
DETAIL=SOURCE_LINE=0279:        "Монитор жалоб:",
DETAIL=SOURCE_LINE=0280:        f"- статус: {facts.get('monitor_status')}",
DETAIL=SOURCE_LINE=0281:        f"- всего жалоб на странице: {facts.get('monitor_reports')}",
DETAIL=SOURCE_LINE=0282:        f"- открытые/назначенные: {facts.get('monitor_open')}",
DETAIL=SOURCE_LINE=0283:        f"- закрытые: {facts.get('monitor_resolved')}",
DETAIL=SOURCE_LINE=0284:        f"- возраст health, сек: {facts.get('monitor_health_age_seconds')}",
DETAIL=SOURCE_LINE=0285:        f"- автопродление авторизации: {facts.get('session_keepalive')}",
DETAIL=SOURCE_LINE=0286:        f"- cookie jar: {facts.get('cookie_jar')} / {facts.get('cookie_jar_bytes')} байт",
DETAIL=SOURCE_LINE=0287:        f"- постоянная cookie действует до UTC: {facts.get('cookie_expiry_utc') or 'не определено'}",
DETAIL=SOURCE_LINE=0288:        f"- осталось дней по cookie: <REDACTED>
DETAIL=SOURCE_LINE=0289:        f"- статус срока cookie: <REDACTED>
DETAIL=SOURCE_LINE=0290:        f"- NetBird route: {facts.get('netbird_route')}",
DETAIL=SOURCE_LINE=0291:        "",
DETAIL=SOURCE_LINE=0292:        "Бэкап:",
DETAIL=SOURCE_LINE=0293:        f"- статус: {facts.get('backup_status')}",
DETAIL=SOURCE_LINE=0294:        f"- последний RUN_ID: {facts.get('backup_run_id')}",
DETAIL=SOURCE_LINE=0295:        f"- размер архива, байт: {facts.get('backup_size_bytes')}",
DETAIL=SOURCE_LINE=0296:        f"- cookie jar в бэкапе: {facts.get('backup_cookie_jar_backed_up')}",
DETAIL=SOURCE_LINE=0297:        f"- health-файлы расширений в бэкапе: {facts.get('backup_extension_health_files_backed_up')}",
DETAIL=SOURCE_LINE=0298:        f"- название жалобы в теме письма: {facts.get('backup_title_in_email_subject')}",
DETAIL=SOURCE_LINE=0299:        f"- возраст health, сек: {facts.get('backup_health_age_seconds')}",
DETAIL=SOURCE_LINE=0300:        "",
DETAIL=SOURCE_LINE=0301:        "Restore-check:",
DETAIL=SOURCE_LINE=0302:        f"- статус: {facts.get('restore_status')}",
DETAIL=SOURCE_LINE=0303:        f"- последний RUN_ID: {facts.get('restore_run_id')}",
DETAIL=SOURCE_LINE=0304:        f"- cloud1/cloud2: {facts.get('cloud_1_status')}/{facts.get('cloud_2_status')}",
DETAIL=SOURCE_LINE=0305:        f"- sha256/tar/cookie/sqlite/python: {facts.get('sha256_ok')}/{facts.get('tar_ok')}/{facts.get('cookie_jar_ok')}/{facts.get('sqlite_ok')}/{facts.get('py_compile_ok')}",
DETAIL=SOURCE_LINE=0306:        f"- health/scripts/units расширений: {facts.get('extension_health_files_ok')}/{facts.get('extension_scripts_ok')}/{facts.get('extension_units_ok')}",
DETAIL=SOURCE_LINE=0307:        "",
DETAIL=SOURCE_LINE=0308:        "Проблемы:",
DETAIL=SOURCE_LINE=0309:    ]
DETAIL=SOURCE_LINE=0310:    if problems:
DETAIL=SOURCE_LINE=0311:        lines.extend([f"- {p}" for p in problems])
DETAIL=SOURCE_LINE=0312:    else:
DETAIL=SOURCE_LINE=0313:        lines.append("- нет")
DETAIL=SOURCE_LINE=0314:    lines.extend([
DETAIL=SOURCE_LINE=0315:        "",
DETAIL=SOURCE_LINE=0316:        "Авторизация:",
DETAIL=SOURCE_LINE=0317:        "- ручной ежемесячный перелогин обычно не нужен; монитор сам продлевает сессию постоянным cookie jar при каждом опросе;",
DETAIL=SOURCE_LINE=0318:        "- supervisor теперь отдельно следит за сроком постоянной cookie и заранее предупредит, если до истечения останется меньше 7 дней;",
DETAIL=SOURCE_LINE=0319:        "- если форум принудительно сбросит сессию, придёт отдельная тревога AUTH_REQUIRED; тогда нужно будет разово обновить вход, но это аварийный сценарий, а не ежемесячная рутина.",
DETAIL=SOURCE_LINE=0320:    ])
DETAIL=SOURCE_LINE=0321:    return "\n".join(lines)
DETAIL=SOURCE_LINE=0322:
DETAIL=SOURCE_LINE=0323:def write_health(status, facts, problems, mail_sent):
DETAIL=SOURCE_LINE=0324:    STATE_DIR.mkdir(parents=True, exist_ok=True)
DETAIL=SOURCE_LINE=0325:    lines = [f"STATUS={status}", f"TS={iso()}", "problems=" + ",".join(problems), f"mail_sent={'yes' if mail_sent else 'no'}"]
DETAIL=SOURCE_LINE=0326:    for k in sorted(facts):
DETAIL=SOURCE_LINE=0327:        v = str(facts[k]).replace("\n", " ")[:500]
DETAIL=SOURCE_LINE=0328:        lines.append(f"{k}={v}")
DETAIL=SOURCE_LINE=0329:    HEALTH.write_text("\n".join(lines) + "\n", encoding="utf-8")
DETAIL=SOURCE_LINE=0330:
DETAIL=SOURCE_LINE=0331:def read_state():
DETAIL=SOURCE_LINE=0332:    return read_env_file(STATE)
DETAIL=SOURCE_LINE=0333:
DETAIL=SOURCE_LINE=0334:def write_state(status, problems):
DETAIL=SOURCE_LINE=0335:    STATE_DIR.mkdir(parents=True, exist_ok=True)
DETAIL=SOURCE_LINE=0336:    STATE.write_text(f"LAST_STATUS={status}\nLAST_PROBLEMS={','.join(problems)}\nTS={iso()}\n", encoding="utf-8")
DETAIL=SOURCE_LINE=0337:
DETAIL=SOURCE_LINE=0338:def main():
DETAIL=SOURCE_LINE=0339:    mode = sys.argv[1] if len(sys.argv) > 1 else "check"
DETAIL=SOURCE_LINE=0340:    if mode not in ("check", "test"):
DETAIL=SOURCE_LINE=0341:        print("usage: skladchik-reports-monitor-supervisor [check|test]", file=sys.stderr)
DETAIL=SOURCE_LINE=0342:        return 2
DETAIL=SOURCE_LINE=0343:
DETAIL=SOURCE_LINE=0344:    facts, problems = gather()
DETAIL=SOURCE_LINE=0345:    status = "OK" if not problems else "FAIL"
DETAIL=SOURCE_LINE=0346:    old = read_state()
DETAIL=SOURCE_LINE=0347:    old_status = old.get("LAST_STATUS", "")
DETAIL=SOURCE_LINE=0348:    mail_sent = False
DETAIL=SOURCE_LINE=0349:
DETAIL=SOURCE_LINE=0350:    if mode == "test":
DETAIL=SOURCE_LINE=0351:        mail_sent = edge_mail("Тест: надзор монитора жалоб и бэкапов", render_body(status, facts, problems, mode))
DETAIL=SOURCE_LINE=0352:        if not mail_sent:
DETAIL=SOURCE_LINE=0353:            problems.append("supervisor_mail_send_failed")
DETAIL=SOURCE_LINE=0354:            status = "FAIL"
DETAIL=SOURCE_LINE=0355:    elif status != "OK":
DETAIL=SOURCE_LINE=0356:        if old_status != "FAIL":
DETAIL=SOURCE_LINE=0357:            mail_sent = edge_mail("Авария: монитор жалоб или его бэкапы", render_body(status, facts, problems, mode))
DETAIL=SOURCE_LINE=0358:            if not mail_sent:
DETAIL=SOURCE_LINE=0359:                problems.append("supervisor_mail_send_failed")
DETAIL=SOURCE_LINE=0360:                status = "FAIL"
DETAIL=SOURCE_LINE=0361:    elif old_status == "FAIL":
DETAIL=SOURCE_LINE=0362:        mail_sent = edge_mail("Восстановлено: монитор жалоб и бэкапы снова OK", render_body(status, facts, problems, mode))
DETAIL=SOURCE_LINE=0363:
DETAIL=SOURCE_LINE=0364:    write_health(status, facts, problems, mail_sent)
DETAIL=SOURCE_LINE=0365:    write_state(status, problems)
DETAIL=SOURCE_LINE=0366:
DETAIL=SOURCE_LINE=0367:    print(f"SUPERVISOR_STATUS={status}")
DETAIL=SOURCE_LINE=0368:    print(f"SUPERVISOR_MAIL_SENT={'yes' if mail_sent else 'no'}")
DETAIL=SOURCE_LINE=0369:    print(f"SUPERVISOR_PROBLEMS={','.join(problems)}")
DETAIL=SOURCE_LINE=0370:    print(f"MONITOR={facts.get('monitor_status')} reports={facts.get('monitor_reports')} open={facts.get('monitor_open')} resolved={facts.get('monitor_resolved')}")
DETAIL=SOURCE_LINE=0371:    print(f"BACKUP={facts.get('backup_status')} run={facts.get('backup_run_id')}")
DETAIL=SOURCE_LINE=0372:    print(f"RESTORE={facts.get('restore_status')} run={facts.get('restore_run_id')}")
DETAIL=SOURCE_LINE=0373:    log(f"CHECK mode={mode} status={status} mail_sent={'yes' if mail_sent else 'no'} problems={problems}")
DETAIL=SOURCE_LINE=0374:    return 0 if status == "OK" else 1
DETAIL=SOURCE_LINE=0375:
DETAIL=SOURCE_LINE=0376:if __name__ == "__main__":
DETAIL=SOURCE_LINE=0377:    sys.exit(main())
DETAIL=SCRIPT_TOTAL_LINES=377
DETAIL=RELEVANT_SOURCE_END
DETAIL=UNIT_CONTRACT_BEGIN
DETAIL=UNIT_NAME=skladchik-reports-monitor-supervisor.service
DETAIL=UNIT_LINE=# /etc/systemd/system/skladchik-reports-monitor-supervisor.service
DETAIL=UNIT_LINE=[Unit]
DETAIL=UNIT_LINE=Description=Skladchik reports monitor full supervisor
DETAIL=UNIT_LINE=After=network-online.target
DETAIL=UNIT_LINE=
DETAIL=UNIT_LINE=[Service]
DETAIL=UNIT_LINE=Type=oneshot
DETAIL=UNIT_LINE=ExecStart=/usr/local/sbin/skladchik-reports-monitor-supervisor check
DETAIL=UNIT_NAME=skladchik-reports-monitor-supervisor.timer
DETAIL=UNIT_LINE=# /etc/systemd/system/skladchik-reports-monitor-supervisor.timer
DETAIL=UNIT_LINE=[Unit]
DETAIL=UNIT_LINE=Description=Run Skladchik reports monitor full supervisor every 10 minutes
DETAIL=UNIT_LINE=
DETAIL=UNIT_LINE=[Timer]
DETAIL=UNIT_LINE=OnBootSec=5min
DETAIL=UNIT_LINE=OnUnitActiveSec=10min
DETAIL=UNIT_LINE=AccuracySec=30s
DETAIL=UNIT_LINE=Persistent=false
DETAIL=UNIT_LINE=Unit=skladchik-reports-monitor-supervisor.service
DETAIL=UNIT_LINE=
DETAIL=UNIT_LINE=[Install]
DETAIL=UNIT_LINE=WantedBy=timers.target
DETAIL=UNIT_CONTRACT_END
DETAIL=TIMER_STATE_BEGIN
DETAIL=TIMER_PROPERTY=Unit=skladchik-reports-monitor-supervisor.service
DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=
DETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 11:58:38 MSK
DETAIL=TIMER_PROPERTY=LoadState=loaded
DETAIL=TIMER_PROPERTY=ActiveState=active
DETAIL=TIMER_PROPERTY=SubState=waiting
DETAIL=TIMER_ROW=Fri 2026-07-24 12:08:38 MSK         3min 34s Fri 2026-07-24 11:58:38 MSK     6min ago skladchik-reports-monitor-supervisor.timer             skladchik-reports-monitor-supervisor.service
DETAIL=TIMER_ROW=Sat 2026-07-25 04:39:05 MSK              16h Fri 2026-07-24 04:28:08 MSK       7h ago skladchik-reports-monitor-backup.timer                 skladchik-reports-monitor-backup.service
DETAIL=TIMER_ROW=Sat 2026-07-25 10:15:00 MSK              22h Fri 2026-07-24 10:15:04 MSK 1h 49min ago skladchik-reports-monitor-monthly-selftest-watch.timer skladchik-reports-monitor-monthly-selftest-watch.service
DETAIL=TIMER_ROW=Sun 2026-07-26 05:15:13 MSK        1 day 17h Sun 2026-07-19 05:11:25 MSK            - skladchik-reports-monitor-backup-restore-check.timer   skladchik-reports-monitor-backup-restore-check.service
DETAIL=TIMER_STATE_END
DETAIL=SECRET_VALUES_OUTPUT=NONE
DETAIL=CHANGES_MADE=NO
DETAIL=COOKIE_CHECK_SOURCE_AUDIT=PASS

OUTPUT_END
CHAT_OUTPUT_END
