{
  "schema_version": 1,
  "channel": "homelab-runtime",
  "command_id": "HOMELAB-SKLADCHIK-COOKIE-CHECK-SOURCE-AUDIT-418",
  "status": "OK",
  "rc": 0,
  "host": "pve01",
  "mode": "read-only",
  "component": "cluster-audit",
  "started_at_utc": "2026-07-24T09:05:03Z",
  "finished_at_utc": "2026-07-24T09:05:03Z",
  "reference_register_checked": true,
  "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
  "error_register_checked": true,
  "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752",
  "command_sha256": "77896618b46e2f97362240deb743fb8e625075381e57377ee0314130be2b2026",
  "duplicate_failed_command_blocked": false,
  "block_reason": null,
  "execution_started": true,
  "changes_made": false,
  "sanitized": true,
  "secrets_included": false,
  "private_addresses_included": false,
  "raw_evidence_retained_locally": true,
  "raw_evidence_sha256": "1c26f0b751c5e55421c706f56f4be25d92c17849df8a385fb66ce01144801022",
  "sanitized_output_sha256": "8e5a54db35050a5c0010241fb0bad91fc5966530dbac76bb07426b09af51a6a1",
  "output_truncated_in_json": false,
  "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
  "output": "DETAIL=SCRIPT_PATH=/usr/local/sbin/skladchik-reports-monitor-supervisor\nDETAIL=SCRIPT_SHA256=35c68ae0929ff3795be2bde40de334fc8e26413a3d8a620a137be90d1f2c164e\nDETAIL=SCRIPT_METADATA=BYTES:20463|MODE:755|OWNER:root:root|MTIME:2026-07-18 13:52:30.415780066 +0300\nDETAIL=RELEVANT_SOURCE_BEGIN\nDETAIL=SOURCE_LINE=0005:import os\nDETAIL=SOURCE_LINE=0006:import re\nDETAIL=SOURCE_LINE=0007:import shlex\nDETAIL=SOURCE_LINE=0008:import subprocess\nDETAIL=SOURCE_LINE=0009:import sys\nDETAIL=SOURCE_LINE=0010:from pathlib import Path\nDETAIL=SOURCE_LINE=0011:\nDETAIL=SOURCE_LINE=0012:EDGE = os.environ.get(\"SKLADCHIK_EDGE\", \"debian@[PRIVATE_IP]\")\nDETAIL=SOURCE_LINE=0013:STATE_DIR = Path(\"/var/lib/skladchik-reports-monitor-supervisor\")\nDETAIL=SOURCE_LINE=0014:LOG_DIR = Path(\"/var/log/skladchik-reports-monitor-supervisor\")\nDETAIL=SOURCE_LINE=0015:HEALTH = STATE_DIR / \"health.txt\"\nDETAIL=SOURCE_LINE=0016:STATE = STATE_DIR / \"state.env\"\nDETAIL=SOURCE_LINE=0017:LOG = LOG_DIR / \"supervisor.log\"\nDETAIL=SOURCE_LINE=0018:\nDETAIL=SOURCE_LINE=0019:MONITOR_HEALTH_AGE_MAX = int(os.environ.get(\"SKLADCHIK_MONITOR_HEALTH_MAX_SECONDS\", \"600\"))\nDETAIL=SOURCE_LINE=0020:BACKUP_HEALTH_AGE_MAX = int(os.environ.get(\"SKLADCHIK_BACKUP_HEALTH_MAX_SECONDS\", \"129600\"))\nDETAIL=SOURCE_LINE=0021:RESTORE_HEALTH_AGE_MAX = int(os.environ.get(\"SKLADCHIK_RESTORE_HEALTH_MAX_SECONDS\", \"691200\"))\nDETAIL=SOURCE_LINE=0022:COOKIE_EXPIRY_WARN_SECONDS = int(os.environ.get(\"SKLADCHIK_COOKIE_EXPIRY_WARN_SECONDS\", \"604800\"))\nDETAIL=SOURCE_LINE=0023:\nDETAIL=SOURCE_LINE=0024:PVE_BACKUP_HEALTH = Path(\"/var/lib/skladchik-reports-monitor-backup/health.txt\")\nDETAIL=SOURCE_LINE=0025:PVE_RESTORE_HEALTH = Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\nDETAIL=SOURCE_LINE=0026:\nDETAIL=SOURCE_LINE=0027:def now():\nDETAIL=SOURCE_LINE=0028:    return _dt.datetime.now(_dt.UTC).replace(microsecond=0)\nDETAIL=SOURCE_LINE=0029:\nDETAIL=SOURCE_LINE=0030:def iso():\nDETAIL=SOURCE_LINE=0031:    return now().isoformat().replace(\"+00:00\", \"Z\")\nDETAIL=SOURCE_LINE=0032:\nDETAIL=SOURCE_LINE=0033:def log(msg):\nDETAIL=SOURCE_LINE=0034:    LOG_DIR.mkdir(parents=True, exist_ok=True)\nDETAIL=SOURCE_LINE=0035:    with LOG.open(\"a\", encoding=\"utf-8\") as f:\nDETAIL=SOURCE_LINE=0036:        f.write(f\"{iso()} {msg}\\n\")\nDETAIL=SOURCE_LINE=0037:\nDETAIL=SOURCE_LINE=0038:def run(cmd, timeout=45, input_text=None):\nDETAIL=SOURCE_LINE=0039:    try:\nDETAIL=SOURCE_LINE=0040:        p = subprocess.run(cmd, shell=True, text=True, input=input_text, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout)\nDETAIL=SOURCE_LINE=0041:        return p.returncode, p.stdout.strip()\nDETAIL=SOURCE_LINE=0042:    except subprocess.TimeoutExpired as e:\nDETAIL=SOURCE_LINE=0043:        out = e.stdout if isinstance(e.stdout, str) else \"\"\nDETAIL=SOURCE_LINE=0044:        return 124, (out or \"\").strip() + \"\\nTIMEOUT\"\nDETAIL=SOURCE_LINE=0045:\nDETAIL=SOURCE_LINE=0046:def parse_env_text(text):\nDETAIL=SOURCE_LINE=0047:    d = {}\nDETAIL=SOURCE_LINE=0048:    for line in (text or \"\").splitlines():\nDETAIL=SOURCE_LINE=0049:        if not line or line.startswith(\"#\") or \"=\" not in line:\nDETAIL=SOURCE_LINE=0050:            continue\nDETAIL=SOURCE_LINE=0051:        k, v = line.split(\"=\", 1)\nDETAIL=SOURCE_LINE=...\nDETAIL=SOURCE_LINE=0066:        dt = _dt.datetime.fromisoformat(s)\nDETAIL=SOURCE_LINE=0067:        if dt.tzinfo is None:\nDETAIL=SOURCE_LINE=0068:            dt = dt.replace(tzinfo=_dt.UTC)\nDETAIL=SOURCE_LINE=0069:        return int((now() - dt.astimezone(_dt.UTC)).total_seconds())\nDETAIL=SOURCE_LINE=0070:    except Exception:\nDETAIL=SOURCE_LINE=0071:        return 999999999\nDETAIL=SOURCE_LINE=0072:\nDETAIL=SOURCE_LINE=0073:def ssh_edge(cmd, timeout=45):\nDETAIL=SOURCE_LINE=0074:    q = \"ssh -o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] \" + shlex.quote(EDGE) + \" \" + shlex.quote(\"sudo bash -lc \" + shlex.quote(cmd))\nDETAIL=SOURCE_LINE=0075:    return run(q, timeout=timeout)\nDETAIL=SOURCE_LINE=0076:\nDETAIL=SOURCE_LINE=0077:def edge_mail(subject, body):\nDETAIL=SOURCE_LINE=0078:    subj_b64 = base64.b64encode(subject.encode(\"utf-8\")).decode(\"ascii\")\nDETAIL=SOURCE_LINE=0079:    body_b64 = base64.b64encode(body.encode(\"utf-8\")).decode(\"ascii\")\nDETAIL=SOURCE_LINE=0080:    py = (\nDETAIL=SOURCE_LINE=0081:        \"import base64,importlib.util;\"\nDETAIL=SOURCE_LINE=0082:        \"spec=importlib.util.spec_from_file_location('m','/opt/skladchik-reports-monitor/monitor.py');\"\nDETAIL=SOURCE_LINE=0083:        \"m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m);\"\nDETAIL=SOURCE_LINE=0084:        \"cfg,secrets=<REDACTED>\nDETAIL=SOURCE_LINE=0085:        f\"subject=base64.b64decode('{subj_b64}').decode('utf-8');\"\nDETAIL=SOURCE_LINE=0086:        f\"body=base64.b64decode('{body_b64}').decode('utf-8');\"\nDETAIL=SOURCE_LINE=0087:        \"ok=m.send_mail(cfg,secrets,subject,body);\"\nDETAIL=SOURCE_LINE=0088:        \"print('MAIL_OK' if ok else 'MAIL_FALSE')\"\nDETAIL=SOURCE_LINE=0089:    )\nDETAIL=SOURCE_LINE=0090:    rc, out = ssh_edge(\"python3 -c \" + shlex.quote(py), timeout=60)\nDETAIL=SOURCE_LINE=0091:    log(f\"MAIL subject={subject!r} rc={rc} out={out!r}\")\nDETAIL=SOURCE_LINE=0092:    return rc == 0 and \"MAIL_OK\" in out\nDETAIL=SOURCE_LINE=0093:\nDETAIL=SOURCE_LINE=0094:def systemctl_props(units):\nDETAIL=SOURCE_LINE=0095:    cmd = \"systemctl is-enabled \" + \" \".join(map(shlex.quote, units)) + \" 2>/dev/null || true; echo __ACTIVE__; systemctl is-active \" + \" \".join(map(shlex.quote, units)) + \" 2>/dev/null || true\"\nDETAIL=SOURCE_LINE=0096:    rc, out = run(cmd, timeout=20)\nDETAIL=SOURCE_LINE=0097:    parts = out.split(\"__ACTIVE__\", 1)\nDETAIL=SOURCE_LINE=0098:    enabled_lines = [x.strip() for x in parts[0].splitlines() if x.strip()]\nDETAIL=SOURCE_LINE=0099:    active_lines = [x.strip() for x in parts[1].splitlines() if x.strip()] if len(parts) > 1 else []\nDETAIL=SOURCE_LINE=0100:    res = {}\nDETAIL=SOURCE_LINE=0101:    for i, u in enumerate(units):\nDETAIL=SOURCE_LINE=0102:        res[u] = {\"enabled\": enabled_lines[i] if i < len(enabled_lines) else \"unknown\", \"active\": active_lines[i] if i < len(active_lines) else \"unknown\"}\nDETAIL=SOURCE_LINE=0103:    return res\nDETAIL=SOURCE_LINE=0104:\nDETAIL=SOURCE_LINE=0105:def gather():\nDETAIL=SOURCE_LINE=0106:    facts = {}\nDETAIL=SOURCE_LINE=0107:    problems = []\nDETAIL=SOURCE_LINE=0108:\nDETAIL=SOURCE_LINE=0109:    edge_cmd = r'''\nDETAIL=SOURCE_LINE=0110:set +e\nDETAIL=SOURCE_LINE=0111:echo __MONITOR_HEALTH_BEGIN__\nDETAIL=SOURCE_LINE=0112:cat /var/lib/skladchik-reports-monitor/health.txt 2>/dev/null || true\nDETAIL=SOURCE_LINE=0113:echo __MONITOR_HEALTH_END__\nDETAIL=SOURCE_LINE=0114:echo __ALERT_HEALTH_BEGIN__\nDETAIL=SOURCE_LINE=0115:cat /var/lib/skladchik-reports-monitor/alert-health.txt 2>/dev/null || true\nDETAIL=SOURCE_LINE=0116:echo __ALERT_HEALTH_END__\nDETAIL=SOURCE_LINE=0117:echo __TIMERS_BEGIN__\nDETAIL=SOURCE_LINE=0118:for u in skladchik-reports-monitor.timer skladchik-reports-monitor-watchdog.timer skladchik-reports-monitor-heartbeat.timer skladchik-reports-monitor-weekly-summary.timer skladchik-reports-monitor-age-alert.timer skladchik-reports-monitor-event-selftest.timer; do echo \"$u enabled=$(systemctl is-enabled \"$u\" 2>/dev/null) active=$(systemctl is-active \"$u\" 2>/dev/null)\"; done\nDETAIL=SOURCE_LINE=0119:echo __TIMERS_END__\nDETAIL=SOURCE_LINE=0120:echo __ROUTE_BEGIN__\nDETAIL=SOURCE_LINE=0121:ip route get 1.1.1.1 2>&1 | head -n 1\nDETAIL=SOURCE_LINE=0122:echo __ROUTE_END__\nDETAIL=SOURCE_LINE=0123:echo __FILES_BEGIN__\nDETAIL=SOURCE_LINE=0124:test -s /opt/skladchik-reports-monitor/monitor.py && echo monitor_py=yes || echo monitor_py=no\nDETAIL=SOURCE_LINE=0125:test -s /etc/skladchik-reports-monitor/config.ini && echo config_ini=yes || echo config_ini=no\nDETAIL=SOURCE_LINE=0126:test -s /etc/skladchik-reports-monitor/secrets.env && echo secrets_env=<REDACTED>\nDETAIL=SOURCE_LINE=0127:test -s /var/lib/skladchik-reports-monitor/cookies.jar && echo cookies_jar=yes || echo cookies_jar=no\nDETAIL=SOURCE_LINE=0128:[ -s /var/lib/skladchik-reports-monitor/cookies.jar ] && echo cookies_jar_bytes=$(stat -c %s /var/lib/skladchik-reports-monitor/cookies.jar 2>/dev/null) || echo cookies_jar_bytes=0\nDETAIL=SOURCE_LINE=0129:test -s /opt/skladchik-reports-monitor/operational_digest.py && echo operational_digest_py=yes || echo operational_digest_py=no\nDETAIL=SOURCE_LINE=0130:test -s /opt/skladchik-reports-monitor/event-selftest-runner.py && echo event_selftest_runner_py=yes || echo event_selftest_runner_py=no\nDETAIL=SOURCE_LINE=0131:test -s /opt/skladchik-reports-monitor/event-selftest-core && echo event_selftest_core=yes || echo event_selftest_core=no\nDETAIL=SOURCE_LINE=0132:test -s /var/lib/skladchik-reports-monitor/weekly-summary-health.txt && echo weekly_summary_health=yes || echo weekly_summary_health=no\nDETAIL=SOURCE_LINE=0133:test -s /var/lib/skladchik-reports-monitor/age-alert-health.txt && echo age_alert_health=yes || echo age_alert_health=no\nDETAIL=SOURCE_LINE=0134:test -s /var/lib/skladchik-reports-monitor/event-selftest-health.txt && echo event_selftest_health=yes || echo event_selftest_health=no\nDETAIL=SOURCE_LINE=0135:python3 -m py_compile /opt/skladchik-reports-monitor/monitor.py /opt/skladchik-reports-monitor/operational_digest.py /opt/skladchik-reports-monitor/event-selftest-runner.py /opt/skladchik-reports-monitor/event-selftest-core >/dev/null 2>&1 && echo py_compile=yes || echo py_compile=no\nDETAIL=SOURCE_LINE=0136:echo __FILES_END__\nDETAIL=SOURCE_LINE=0137:'''\nDETAIL=SOURCE_LINE=0138:    rc, out = ssh_edge(edge_cmd, timeout=45)\nDETAIL=SOURCE_LINE=0139:    facts[\"edge_ssh_rc\"] = str(rc)\nDETAIL=SOURCE_LINE=0140:    if rc != 0:\nDETAIL=SOURCE_LINE=0141:        problems.append(\"edge_ssh_failed\")\nDETAIL=SOURCE_LINE=0142:\nDETAIL=SOURCE_LINE=0143:    def section(name):\nDETAIL=SOURCE_LINE=0144:        m = re.search(rf\"__{name}_BEGIN__\\n(.*?)\\n__{name}_END__\", out, re.S)\nDETAIL=SOURCE_LINE=0145:        return m.group(1).strip() if m else \"\"\nDETAIL=SOURCE_LINE=0146:\nDETAIL=SOURCE_LINE=0147:    mon = parse_env_text(section(\"MONITOR_HEALTH\"))\nDETAIL=SOURCE_LINE=0148:    facts[\"monitor_status\"] = mon.get(\"STATUS\", \"MISSING\")\nDETAIL=SOURCE_LINE=0149:    facts[\"monitor_reports\"] = mon.get(\"reports\", \"\")\nDETAIL=SOURCE_LINE=0150:    facts[\"monitor_open\"] = mon.get(\"open\", \"\")\nDETAIL=SOURCE_LINE=0151:    facts[\"monitor_resolved\"] = mon.get(\"resolved\", \"\")\nDETAIL=SOURCE_LINE=0152:    facts[\"monitor_smtp_configured\"] = mon.get(\"smtp_configured\", \"\")\nDETAIL=SOURCE_LINE=0153:    facts[\"session_keepalive\"] = mon.get(\"session_keepalive\", \"\")\nDETAIL=SOURCE_LINE=0154:    facts[\"cookie_jar\"] = mon.get(\"cookie_jar\", \"\")\nDETAIL=SOURCE_LINE=0155:    facts[\"cookie_jar_bytes\"] = mon.get(\"cookie_jar_bytes\", \"\")\nDETAIL=SOURCE_LINE=0156:    facts[\"cookie_expiry_status\"] = mon.get(\"cookie_expiry_status\", \"\")\nDETAIL=SOURCE_LINE=0157:    facts[\"cookie_expiry_utc\"] = mon.get(\"cookie_expiry_utc\", \"\")\nDETAIL=SOURCE_LINE=0158:    facts[\"cookie_expires_in_seconds\"] = mon.get(\"cookie_expires_in_seconds\", \"\")\nDETAIL=SOURCE_LINE=0159:    facts[\"cookie_expires_in_days\"] = mon.get(\"cookie_expires_in_days\", \"\")\nDETAIL=SOURCE_LINE=0160:    facts[\"cookie_persistent_count\"] = mon.get(\"cookie_persistent_count\", \"\")\nDETAIL=SOURCE_LINE=0161:    facts[\"monitor_health_age_seconds\"] = str(age_seconds_from_ts(mon.get(\"TS\")))\nDETAIL=SOURCE_LINE=0162:    if mon.get(\"STATUS\") != \"OK\":\nDETAIL=SOURCE_LINE=0163:        problems.append(\"monitor_status_not_ok\")\nDETAIL=SOURCE_LINE=0164:    if int(facts[\"monitor_health_age_seconds\"]) > MONITOR_HEALTH_AGE_MAX:\nDETAIL=SOURCE_LINE=0165:        problems.append(\"monitor_health_stale\")\nDETAIL=SOURCE_LINE=0166:    if mon.get(\"reports\") in (\"\", \"0\"):\nDETAIL=SOURCE_LINE=0167:        problems.append(\"monitor_reports_zero_or_missing\")\nDETAIL=SOURCE_LINE=0168:    if mon.get(\"session_keepalive\") != \"persistent_cookie_jar\":\nDETAIL=SOURCE_LINE=0169:        problems.append(\"session_keepalive_not_persistent\")\nDETAIL=SOURCE_LINE=0170:    if mon.get(\"cookie_jar\") not in (\"yes\", \"empty\"):\nDETAIL=SOURCE_LINE=0171:        problems.append(\"monitor_cookie_jar_bad\")\nDETAIL=SOURCE_LINE=0172:    expiry_status = mon.get(\"cookie_expiry_status\", \"\")\nDETAIL=SOURCE_LINE=0173:    try:\nDETAIL=SOURCE_LINE=0174:        expires_in = int(mon.get(\"cookie_expires_in_seconds\", \"\"))\nDETAIL=SOURCE_LINE=0175:    except Exception:\nDETAIL=SOURCE_LINE=0176:        expires_in = None\nDETAIL=SOURCE_LINE=0177:    if expiry_status in (\"\", \"NO_COOKIE_JAR\", \"NO_PERSISTENT_COOKIE\", \"ERROR\"):\nDETAIL=SOURCE_LINE=0178:        problems.append(\"cookie_expiry_missing\")\nDETAIL=SOURCE_LINE=0179:    elif expiry_status == \"EXPIRED\" or (expires_in is not None and expires_in <= 0):\nDETAIL=SOURCE_LINE=0180:        problems.append(\"cookie_expiry_expired\")\nDETAIL=SOURCE_LINE=0181:    elif expiry_status == \"WARN\" or (expires_in is not None and expires_in < COOKIE_EXPIRY_WARN_SECONDS):\nDETAIL=SOURCE_LINE=0182:        problems.append(\"cookie_expiry_warn\")\nDETAIL=SOURCE_LINE=0183:\nDETAIL=SOURCE_LINE=0184:    timers_text = section(\"TIMERS\")\nDETAIL=SOURCE_LINE=0185:    for unit in [\"skladchik-reports-monitor.timer\", \"skladchik-reports-monitor-watchdog.timer\", \"skladchik-reports-monitor-heartbeat.timer\", \"skladchik-reports-monitor-weekly-summary.timer\", \"skladchik-reports-monitor-age-alert.timer\", \"skladchik-reports-monitor-event-selftest.timer\"]:\nDETAIL=SOURCE_LINE=0186:        safe = unit.replace(\".timer\", \"\").replace(\"_\", \"-\")\nDETAIL=SOURCE_LINE=0187:        m = re.search(re.escape(unit) + r\"\\s+enabled=([^\\s]+)\\s+active=([^\\s]+)\", timers_text)\nDETAIL=SOURCE_LINE=0188:        enabled = m.group(1) if m else \"missing\"\nDETAIL=SOURCE_LINE=0189:        active = m.group(2) if m else \"missing\"\nDETAIL=SOURCE_LINE=0190:        facts[f\"edge_{safe}_timer_enabled\"] = enabled\nDETAIL=SOURCE_LINE=0191:        facts[f\"edge_{safe}_timer_active\"] = active\nDETAIL=SOURCE_LINE=0192:        if enabled != \"enabled\" or active != \"active\":\nDETAIL=SOURCE_LINE=0193:            problems.append(f\"edge_{safe}_timer_bad\")\nDETAIL=SOURCE_LINE=0194:\nDETAIL=SOURCE_LINE=0195:    route_lines = section(\"ROUTE\").splitlines()\nDETAIL=SOURCE_LINE=0196:    route = route_lines[0] if route_lines else \"\"\nDETAIL=SOURCE_LINE=0197:    facts[\"netbird_route\"] = route\nDETAIL=SOURCE_LINE=0198:    if \" wt0 \" not in (\" \" + route + \" \"):\nDETAIL=SOURCE_LINE=0199:        problems.append(\"netbird_route_not_wt0\")\nDETAIL=SOURCE_LINE=0200:\nDETAIL=SOURCE_LINE=0201:    files = parse_env_text(section(\"FILES\"))\nDETAIL=SOURCE_LINE=0202:    for k in [\"monitor_py\", \"config_ini\", \"secrets_env\", \"cookies_jar\", \"operational_digest_py\", \"event_selftest_runner_py\", \"event_selftest_core\", \"weekly_summary_health\", \"age_alert_health\", \"event_selftest_health\", \"py_compile\"]:\nDETAIL=SOURCE_LINE=0203:        facts[k] = files.get(k, \"no\")\nDETAIL=SOURCE_LINE=0204:        if facts[k] != \"yes\":\nDETAIL=SOURCE_LINE=0205:            problems.append(f\"edge_{k}_bad\")\nDETAIL=SOURCE_LINE=0206:    facts[\"cookies_jar_bytes_file\"] = files.get(\"cookies_jar_bytes\", \"0\")\nDETAIL=SOURCE_LINE=0207:\nDETAIL=SOURCE_LINE=0208:    backup = read_env_file(PVE_BACKUP_HEALTH)\nDETAIL=SOURCE_LINE=0209:    facts[\"backup_status\"] = backup.get(\"STATUS\", \"MISSING\")\nDETAIL=SOURCE_LINE=0210:    facts[\"backup_run_id\"] = backup.get(\"RUN_ID\", \"\")\nDETAIL=SOURCE_LINE=0211:    facts[\"backup_size_bytes\"] = backup.get(\"SIZE_BYTES\", \"\")\nDETAIL=SOURCE_LINE=0212:    facts[\"backup_version\"] = backup.get(\"BACKUP_VERSION\", \"\")\nDETAIL=SOURCE_LINE=0213:    facts[\"backup_cookie_jar_backed_up\"] = backup.get(\"COOKIE_JAR_BACKED_UP\", \"\")\nDETAIL=SOURCE_LINE=0214:    facts[\"backup_extension_health_files_backed_up\"] = backup.get(\"EXTENSION_HEALTH_FILES_BACKED_UP\", \"\")\nDETAIL=SOURCE_LINE=0215:    facts[\"backup_title_in_email_subject\"] = backup.get(\"TITLE_IN_EMAIL_SUBJECT\", \"\")\nDETAIL=SOURCE_LINE=0216:    facts[\"backup_health_age_seconds\"] = str(age_seconds_from_ts(backup.get(\"TS\")))\nDETAIL=SOURCE_LINE=0217:    if backup.get(\"STATUS\") != \"OK\":\nDETAIL=SOURCE_LINE=0218:        problems.append(\"backup_status_not_ok\")\nDETAIL=SOURCE_LINE=0219:    if backup.get(\"COOKIE_JAR_BACKED_UP\") != \"yes\":\nDETAIL=SOURCE_LINE=0220:        problems.append(\"backup_cookie_jar_not_marked\")\nDETAIL=SOURCE_LINE=0221:    if backup.get(\"BACKUP_VERSION\") != \"4\":\nDETAIL=SOURCE_LINE=0222:        problems.append(\"backup_version_not_4\")\nDETAIL=SOURCE_LINE=0223:    if backup.get(\"EXTENSION_HEALTH_FILES_BACKED_UP\") != \"yes\":\nDETAIL=SOURCE_LINE=0224:        problems.append(\"backup_extension_health_files_not_marked\")\nDETAIL=SOURCE_LINE=0225:    if backup.get(\"TITLE_IN_EMAIL_SUBJECT\") != \"yes\":\nDETAIL=SOURCE_LINE=0226:        problems.append(\"email_title_subject_not_marked\")\nDETAIL=SOURCE_LINE=0227:    if int(facts[\"backup_health_age_seconds\"]) > BACKUP_HEALTH_AGE_MAX:\nDETAIL=SOURCE_LINE=0228:        problems.append(\"backup_health_stale\")\nDETAIL=SOURCE_LINE=0229:    try:\nDETAIL=SOURCE_LINE=0230:        if int(backup.get(\"SIZE_BYTES\", \"0\")) <= 0:\nDETAIL=SOURCE_LINE=0231:            problems.append(\"backup_size_bad\")\nDETAIL=SOURCE_LINE=0232:    except Exception:\nDETAIL=SOURCE_LINE=0233:        problems.append(\"backup_size_bad\")\nDETAIL=SOURCE_LINE=0234:\nDETAIL=SOURCE_LINE=0235:    restore = read_env_file(PVE_RESTORE_HEALTH)\nDETAIL=SOURCE_LINE=0236:    facts[\"restore_status\"] = restore.get(\"STATUS\", \"MISSING\")\nDETAIL=SOURCE_LINE=0237:    facts[\"restore_run_id\"] = restore.get(\"RUN_ID\", \"\")\nDETAIL=SOURCE_LINE=0238:    facts[\"restore_health_age_seconds\"] = str(age_seconds_from_ts(restore.get(\"TS\")))\nDETAIL=SOURCE_LINE=0239:    for key in [\"CLOUD_1_STATUS\", \"CLOUD_2_STATUS\", \"SHA256_OK\", \"TAR_OK\", \"REQUIRED_FILES_OK\", \"COOKIE_JAR_OK\", \"EXTENSION_HEALTH_FILES_OK\", \"EXTENSION_SCRIPTS_OK\", \"EXTENSION_UNITS_OK\", \"SQLITE_OK\", \"PY_COMPILE_OK\"]:\nDETAIL=SOURCE_LINE=0240:        facts[key.lower()] = restore.get(key, \"\")\nDETAIL=SOURCE_LINE=0241:    if restore.get(\"STATUS\") != \"OK\":\nDETAIL=SOURCE_LINE=0242:        problems.append(\"restore_status_not_ok\")\nDETAIL=SOURCE_LINE=0243:    if int(facts[\"restore_health_age_seconds\"]) > RESTORE_HEALTH_AGE_MAX:\nDETAIL=SOURCE_LINE=0244:        problems.append(\"restore_health_stale\")\nDETAIL=SOURCE_LINE=0245:    for key in [\"CLOUD_1_STATUS\", \"CLOUD_2_STATUS\"]:\nDETAIL=SOURCE_LINE=0246:        if restore.get(key) != \"OK\":\nDETAIL=SOURCE_LINE=0247:            problems.append(key.lower() + \"_bad\")\nDETAIL=SOURCE_LINE=0248:    for key in [\"SHA256_OK\", \"TAR_OK\", \"REQUIRED_FILES_OK\", \"COOKIE_JAR_OK\", \"EXTENSION_HEALTH_FILES_OK\", \"EXTENSION_SCRIPTS_OK\", \"EXTENSION_UNITS_OK\", \"SQLITE_OK\", \"PY_COMPILE_OK\"]:\nDETAIL=SOURCE_LINE=0249:        if restore.get(key) != \"yes\":\nDETAIL=SOURCE_LINE=0250:            problems.append(key.lower() + \"_bad\")\nDETAIL=SOURCE_LINE=0251:\nDETAIL=SOURCE_LINE=0252:    local_units = [\nDETAIL=SOURCE_LINE=0253:        \"skladchik-reports-monitor-backup.timer\",\nDETAIL=SOURCE_LINE=0254:        \"skladchik-reports-monitor-backup-restore-check.timer\",\nDETAIL=SOURCE_LINE=0255:        \"skladchik-reports-monitor-supervisor.timer\",\nDETAIL=SOURCE_LINE=0256:        \"skladchik-reports-monitor-monthly-selftest-watch.timer\",\nDETAIL=SOURCE_LINE=0257:    ]\nDETAIL=SOURCE_LINE=0258:    props = systemctl_props(local_units)\nDETAIL=SOURCE_LINE=0259:    for u, p in props.items():\nDETAIL=SOURCE_LINE=0260:        base = u.replace(\".timer\", \"\").replace(\"-\", \"_\")\nDETAIL=SOURCE_LINE=0261:        facts[base + \"_timer_enabled\"] = p[\"enabled\"]\nDETAIL=SOURCE_LINE=0262:        facts[base + \"_timer_active\"] = p[\"active\"]\nDETAIL=SOURCE_LINE=0263:        if u != \"skladchik-reports-monitor-supervisor.timer\":\nDETAIL=SOURCE_LINE=0264:            if p[\"enabled\"] != \"enabled\" or p[\"active\"] != \"active\":\nDETAIL=SOURCE_LINE=0265:                problems.append(base + \"_timer_bad\")\nDETAIL=SOURCE_LINE=0266:        else:\nDETAIL=SOURCE_LINE=0267:            # Before first enable it can be disabled/inactive during test. After enable next runs will see enabled/active.\nDETAIL=SOURCE_LINE=0268:            if p[\"enabled\"] not in (\"enabled\", \"disabled\") or p[\"active\"] not in (\"active\", \"inactive\"):\nDETAIL=SOURCE_LINE=0269:                problems.append(base + \"_timer_unknown\")\nDETAIL=SOURCE_LINE=0270:\nDETAIL=SOURCE_LINE=0271:    return facts, problems\nDETAIL=SOURCE_LINE=0272:\nDETAIL=SOURCE_LINE=0273:def render_body(status, facts, problems, mode):\nDETAIL=SOURCE_LINE=0274:    ru_status = \"OK\" if status == \"OK\" else \"АВАРИЯ\"\nDETAIL=SOURCE_LINE=0275:    lines = [\nDETAIL=SOURCE_LINE=0276:        f\"Статус надзора: {ru_status}\",\nDETAIL=SOURCE_LINE=0277:        f\"Время UTC: {iso()}\",\nDETAIL=SOURCE_LINE=0278:        \"\",\nDETAIL=SOURCE_LINE=0279:        \"Монитор жалоб:\",\nDETAIL=SOURCE_LINE=0280:        f\"- статус: {facts.get('monitor_status')}\",\nDETAIL=SOURCE_LINE=0281:        f\"- всего жалоб на странице: {facts.get('monitor_reports')}\",\nDETAIL=SOURCE_LINE=0282:        f\"- открытые/назначенные: {facts.get('monitor_open')}\",\nDETAIL=SOURCE_LINE=0283:        f\"- закрытые: {facts.get('monitor_resolved')}\",\nDETAIL=SOURCE_LINE=0284:        f\"- возраст health, сек: {facts.get('monitor_health_age_seconds')}\",\nDETAIL=SOURCE_LINE=0285:        f\"- автопродление авторизации: {facts.get('session_keepalive')}\",\nDETAIL=SOURCE_LINE=0286:        f\"- cookie jar: {facts.get('cookie_jar')} / {facts.get('cookie_jar_bytes')} байт\",\nDETAIL=SOURCE_LINE=0287:        f\"- постоянная cookie действует до UTC: {facts.get('cookie_expiry_utc') or 'не определено'}\",\nDETAIL=SOURCE_LINE=0288:        f\"- осталось дней по cookie: <REDACTED>\nDETAIL=SOURCE_LINE=0289:        f\"- статус срока cookie: <REDACTED>\nDETAIL=SOURCE_LINE=0290:        f\"- NetBird route: {facts.get('netbird_route')}\",\nDETAIL=SOURCE_LINE=0291:        \"\",\nDETAIL=SOURCE_LINE=0292:        \"Бэкап:\",\nDETAIL=SOURCE_LINE=0293:        f\"- статус: {facts.get('backup_status')}\",\nDETAIL=SOURCE_LINE=0294:        f\"- последний RUN_ID: {facts.get('backup_run_id')}\",\nDETAIL=SOURCE_LINE=0295:        f\"- размер архива, байт: {facts.get('backup_size_bytes')}\",\nDETAIL=SOURCE_LINE=0296:        f\"- cookie jar в бэкапе: {facts.get('backup_cookie_jar_backed_up')}\",\nDETAIL=SOURCE_LINE=0297:        f\"- health-файлы расширений в бэкапе: {facts.get('backup_extension_health_files_backed_up')}\",\nDETAIL=SOURCE_LINE=0298:        f\"- название жалобы в теме письма: {facts.get('backup_title_in_email_subject')}\",\nDETAIL=SOURCE_LINE=0299:        f\"- возраст health, сек: {facts.get('backup_health_age_seconds')}\",\nDETAIL=SOURCE_LINE=0300:        \"\",\nDETAIL=SOURCE_LINE=0301:        \"Restore-check:\",\nDETAIL=SOURCE_LINE=0302:        f\"- статус: {facts.get('restore_status')}\",\nDETAIL=SOURCE_LINE=0303:        f\"- последний RUN_ID: {facts.get('restore_run_id')}\",\nDETAIL=SOURCE_LINE=0304:        f\"- cloud1/cloud2: {facts.get('cloud_1_status')}/{facts.get('cloud_2_status')}\",\nDETAIL=SOURCE_LINE=0305:        f\"- sha256/tar/cookie/sqlite/python: {facts.get('sha256_ok')}/{facts.get('tar_ok')}/{facts.get('cookie_jar_ok')}/{facts.get('sqlite_ok')}/{facts.get('py_compile_ok')}\",\nDETAIL=SOURCE_LINE=0306:        f\"- health/scripts/units расширений: {facts.get('extension_health_files_ok')}/{facts.get('extension_scripts_ok')}/{facts.get('extension_units_ok')}\",\nDETAIL=SOURCE_LINE=0307:        \"\",\nDETAIL=SOURCE_LINE=0308:        \"Проблемы:\",\nDETAIL=SOURCE_LINE=0309:    ]\nDETAIL=SOURCE_LINE=0310:    if problems:\nDETAIL=SOURCE_LINE=0311:        lines.extend([f\"- {p}\" for p in problems])\nDETAIL=SOURCE_LINE=0312:    else:\nDETAIL=SOURCE_LINE=0313:        lines.append(\"- нет\")\nDETAIL=SOURCE_LINE=0314:    lines.extend([\nDETAIL=SOURCE_LINE=0315:        \"\",\nDETAIL=SOURCE_LINE=0316:        \"Авторизация:\",\nDETAIL=SOURCE_LINE=0317:        \"- ручной ежемесячный перелогин обычно не нужен; монитор сам продлевает сессию постоянным cookie jar при каждом опросе;\",\nDETAIL=SOURCE_LINE=0318:        \"- supervisor теперь отдельно следит за сроком постоянной cookie и заранее предупредит, если до истечения останется меньше 7 дней;\",\nDETAIL=SOURCE_LINE=0319:        \"- если форум принудительно сбросит сессию, придёт отдельная тревога AUTH_REQUIRED; тогда нужно будет разово обновить вход, но это аварийный сценарий, а не ежемесячная рутина.\",\nDETAIL=SOURCE_LINE=0320:    ])\nDETAIL=SOURCE_LINE=0321:    return \"\\n\".join(lines)\nDETAIL=SOURCE_LINE=0322:\nDETAIL=SOURCE_LINE=0323:def write_health(status, facts, problems, mail_sent):\nDETAIL=SOURCE_LINE=0324:    STATE_DIR.mkdir(parents=True, exist_ok=True)\nDETAIL=SOURCE_LINE=0325:    lines = [f\"STATUS={status}\", f\"TS={iso()}\", \"problems=\" + \",\".join(problems), f\"mail_sent={'yes' if mail_sent else 'no'}\"]\nDETAIL=SOURCE_LINE=0326:    for k in sorted(facts):\nDETAIL=SOURCE_LINE=0327:        v = str(facts[k]).replace(\"\\n\", \" \")[:500]\nDETAIL=SOURCE_LINE=0328:        lines.append(f\"{k}={v}\")\nDETAIL=SOURCE_LINE=0329:    HEALTH.write_text(\"\\n\".join(lines) + \"\\n\", encoding=\"utf-8\")\nDETAIL=SOURCE_LINE=0330:\nDETAIL=SOURCE_LINE=0331:def read_state():\nDETAIL=SOURCE_LINE=0332:    return read_env_file(STATE)\nDETAIL=SOURCE_LINE=0333:\nDETAIL=SOURCE_LINE=0334:def write_state(status, problems):\nDETAIL=SOURCE_LINE=0335:    STATE_DIR.mkdir(parents=True, exist_ok=True)\nDETAIL=SOURCE_LINE=0336:    STATE.write_text(f\"LAST_STATUS={status}\\nLAST_PROBLEMS={','.join(problems)}\\nTS={iso()}\\n\", encoding=\"utf-8\")\nDETAIL=SOURCE_LINE=0337:\nDETAIL=SOURCE_LINE=0338:def main():\nDETAIL=SOURCE_LINE=0339:    mode = sys.argv[1] if len(sys.argv) > 1 else \"check\"\nDETAIL=SOURCE_LINE=0340:    if mode not in (\"check\", \"test\"):\nDETAIL=SOURCE_LINE=0341:        print(\"usage: skladchik-reports-monitor-supervisor [check|test]\", file=sys.stderr)\nDETAIL=SOURCE_LINE=0342:        return 2\nDETAIL=SOURCE_LINE=0343:\nDETAIL=SOURCE_LINE=0344:    facts, problems = gather()\nDETAIL=SOURCE_LINE=0345:    status = \"OK\" if not problems else \"FAIL\"\nDETAIL=SOURCE_LINE=0346:    old = read_state()\nDETAIL=SOURCE_LINE=0347:    old_status = old.get(\"LAST_STATUS\", \"\")\nDETAIL=SOURCE_LINE=0348:    mail_sent = False\nDETAIL=SOURCE_LINE=0349:\nDETAIL=SOURCE_LINE=0350:    if mode == \"test\":\nDETAIL=SOURCE_LINE=0351:        mail_sent = edge_mail(\"Тест: надзор монитора жалоб и бэкапов\", render_body(status, facts, problems, mode))\nDETAIL=SOURCE_LINE=0352:        if not mail_sent:\nDETAIL=SOURCE_LINE=0353:            problems.append(\"supervisor_mail_send_failed\")\nDETAIL=SOURCE_LINE=0354:            status = \"FAIL\"\nDETAIL=SOURCE_LINE=0355:    elif status != \"OK\":\nDETAIL=SOURCE_LINE=0356:        if old_status != \"FAIL\":\nDETAIL=SOURCE_LINE=0357:            mail_sent = edge_mail(\"Авария: монитор жалоб или его бэкапы\", render_body(status, facts, problems, mode))\nDETAIL=SOURCE_LINE=0358:            if not mail_sent:\nDETAIL=SOURCE_LINE=0359:                problems.append(\"supervisor_mail_send_failed\")\nDETAIL=SOURCE_LINE=0360:                status = \"FAIL\"\nDETAIL=SOURCE_LINE=0361:    elif old_status == \"FAIL\":\nDETAIL=SOURCE_LINE=0362:        mail_sent = edge_mail(\"Восстановлено: монитор жалоб и бэкапы снова OK\", render_body(status, facts, problems, mode))\nDETAIL=SOURCE_LINE=0363:\nDETAIL=SOURCE_LINE=0364:    write_health(status, facts, problems, mail_sent)\nDETAIL=SOURCE_LINE=0365:    write_state(status, problems)\nDETAIL=SOURCE_LINE=0366:\nDETAIL=SOURCE_LINE=0367:    print(f\"SUPERVISOR_STATUS={status}\")\nDETAIL=SOURCE_LINE=0368:    print(f\"SUPERVISOR_MAIL_SENT={'yes' if mail_sent else 'no'}\")\nDETAIL=SOURCE_LINE=0369:    print(f\"SUPERVISOR_PROBLEMS={','.join(problems)}\")\nDETAIL=SOURCE_LINE=0370:    print(f\"MONITOR={facts.get('monitor_status')} reports={facts.get('monitor_reports')} open={facts.get('monitor_open')} resolved={facts.get('monitor_resolved')}\")\nDETAIL=SOURCE_LINE=0371:    print(f\"BACKUP={facts.get('backup_status')} run={facts.get('backup_run_id')}\")\nDETAIL=SOURCE_LINE=0372:    print(f\"RESTORE={facts.get('restore_status')} run={facts.get('restore_run_id')}\")\nDETAIL=SOURCE_LINE=0373:    log(f\"CHECK mode={mode} status={status} mail_sent={'yes' if mail_sent else 'no'} problems={problems}\")\nDETAIL=SOURCE_LINE=0374:    return 0 if status == \"OK\" else 1\nDETAIL=SOURCE_LINE=0375:\nDETAIL=SOURCE_LINE=0376:if __name__ == \"__main__\":\nDETAIL=SOURCE_LINE=0377:    sys.exit(main())\nDETAIL=SCRIPT_TOTAL_LINES=377\nDETAIL=RELEVANT_SOURCE_END\nDETAIL=UNIT_CONTRACT_BEGIN\nDETAIL=UNIT_NAME=skladchik-reports-monitor-supervisor.service\nDETAIL=UNIT_LINE=# /etc/systemd/system/skladchik-reports-monitor-supervisor.service\nDETAIL=UNIT_LINE=[Unit]\nDETAIL=UNIT_LINE=Description=Skladchik reports monitor full supervisor\nDETAIL=UNIT_LINE=After=network-online.target\nDETAIL=UNIT_LINE=\nDETAIL=UNIT_LINE=[Service]\nDETAIL=UNIT_LINE=Type=oneshot\nDETAIL=UNIT_LINE=ExecStart=/usr/local/sbin/skladchik-reports-monitor-supervisor check\nDETAIL=UNIT_NAME=skladchik-reports-monitor-supervisor.timer\nDETAIL=UNIT_LINE=# /etc/systemd/system/skladchik-reports-monitor-supervisor.timer\nDETAIL=UNIT_LINE=[Unit]\nDETAIL=UNIT_LINE=Description=Run Skladchik reports monitor full supervisor every 10 minutes\nDETAIL=UNIT_LINE=\nDETAIL=UNIT_LINE=[Timer]\nDETAIL=UNIT_LINE=OnBootSec=5min\nDETAIL=UNIT_LINE=OnUnitActiveSec=10min\nDETAIL=UNIT_LINE=AccuracySec=30s\nDETAIL=UNIT_LINE=Persistent=false\nDETAIL=UNIT_LINE=Unit=skladchik-reports-monitor-supervisor.service\nDETAIL=UNIT_LINE=\nDETAIL=UNIT_LINE=[Install]\nDETAIL=UNIT_LINE=WantedBy=timers.target\nDETAIL=UNIT_CONTRACT_END\nDETAIL=TIMER_STATE_BEGIN\nDETAIL=TIMER_PROPERTY=Unit=skladchik-reports-monitor-supervisor.service\nDETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=\nDETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 11:58:38 MSK\nDETAIL=TIMER_PROPERTY=LoadState=loaded\nDETAIL=TIMER_PROPERTY=ActiveState=active\nDETAIL=TIMER_PROPERTY=SubState=waiting\nDETAIL=TIMER_ROW=Fri 2026-07-24 12:08:38 MSK         3min 34s Fri 2026-07-24 11:58:38 MSK     6min ago skladchik-reports-monitor-supervisor.timer             skladchik-reports-monitor-supervisor.service\nDETAIL=TIMER_ROW=Sat 2026-07-25 04:39:05 MSK              16h Fri 2026-07-24 04:28:08 MSK       7h ago skladchik-reports-monitor-backup.timer                 skladchik-reports-monitor-backup.service\nDETAIL=TIMER_ROW=Sat 2026-07-25 10:15:00 MSK              22h Fri 2026-07-24 10:15:04 MSK 1h 49min ago skladchik-reports-monitor-monthly-selftest-watch.timer skladchik-reports-monitor-monthly-selftest-watch.service\nDETAIL=TIMER_ROW=Sun 2026-07-26 05:15:13 MSK        1 day 17h Sun 2026-07-19 05:11:25 MSK            - skladchik-reports-monitor-backup-restore-check.timer   skladchik-reports-monitor-backup-restore-check.service\nDETAIL=TIMER_STATE_END\nDETAIL=SECRET_VALUES_OUTPUT=NONE\nDETAIL=CHANGES_MADE=NO\nDETAIL=COOKIE_CHECK_SOURCE_AUDIT=PASS\n"
}