diff --git a/runtime/history/HOMELAB-SKLADCHIK-COOKIE-CHECK-SOURCE-AUDIT-418.json b/runtime/history/HOMELAB-SKLADCHIK-COOKIE-CHECK-SOURCE-AUDIT-418.json new file mode 100644 index 0000000..c5b7657 --- /dev/null +++ b/runtime/history/HOMELAB-SKLADCHIK-COOKIE-CHECK-SOURCE-AUDIT-418.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "HOMELAB-SKLADCHIK-COOKIE-CHECK-SOURCE-AUDIT-418", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "cluster-audit", + "started_at_utc": "2026-07-24T09:05:03Z", + "finished_at_utc": "2026-07-24T09:05:03Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", + "command_sha256": "77896618b46e2f97362240deb743fb8e625075381e57377ee0314130be2b2026", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "1c26f0b751c5e55421c706f56f4be25d92c17849df8a385fb66ce01144801022", + "sanitized_output_sha256": "8e5a54db35050a5c0010241fb0bad91fc5966530dbac76bb07426b09af51a6a1", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "DETAIL=SCRIPT_PATH=/usr/local/sbin/skladchik-reports-monitor-supervisor\nDETAIL=SCRIPT_SHA256=35c68ae0929ff3795be2bde40de334fc8e26413a3d8a620a137be90d1f2c164e\nDETAIL=SCRIPT_METADATA=BYTES:20463|MODE:755|OWNER:root:root|MTIME:2026-07-18 13:52:30.415780066 +0300\nDETAIL=RELEVANT_SOURCE_BEGIN\nDETAIL=SOURCE_LINE=0005:import os\nDETAIL=SOURCE_LINE=0006:import re\nDETAIL=SOURCE_LINE=0007:import shlex\nDETAIL=SOURCE_LINE=0008:import subprocess\nDETAIL=SOURCE_LINE=0009:import sys\nDETAIL=SOURCE_LINE=0010:from pathlib import Path\nDETAIL=SOURCE_LINE=0011:\nDETAIL=SOURCE_LINE=0012:EDGE = os.environ.get(\"SKLADCHIK_EDGE\", \"debian@[PRIVATE_IP]\")\nDETAIL=SOURCE_LINE=0013:STATE_DIR = Path(\"/var/lib/skladchik-reports-monitor-supervisor\")\nDETAIL=SOURCE_LINE=0014:LOG_DIR = Path(\"/var/log/skladchik-reports-monitor-supervisor\")\nDETAIL=SOURCE_LINE=0015:HEALTH = STATE_DIR / \"health.txt\"\nDETAIL=SOURCE_LINE=0016:STATE = STATE_DIR / \"state.env\"\nDETAIL=SOURCE_LINE=0017:LOG = LOG_DIR / \"supervisor.log\"\nDETAIL=SOURCE_LINE=0018:\nDETAIL=SOURCE_LINE=0019:MONITOR_HEALTH_AGE_MAX = int(os.environ.get(\"SKLADCHIK_MONITOR_HEALTH_MAX_SECONDS\", \"600\"))\nDETAIL=SOURCE_LINE=0020:BACKUP_HEALTH_AGE_MAX = int(os.environ.get(\"SKLADCHIK_BACKUP_HEALTH_MAX_SECONDS\", \"129600\"))\nDETAIL=SOURCE_LINE=0021:RESTORE_HEALTH_AGE_MAX = int(os.environ.get(\"SKLADCHIK_RESTORE_HEALTH_MAX_SECONDS\", \"691200\"))\nDETAIL=SOURCE_LINE=0022:COOKIE_EXPIRY_WARN_SECONDS = int(os.environ.get(\"SKLADCHIK_COOKIE_EXPIRY_WARN_SECONDS\", \"604800\"))\nDETAIL=SOURCE_LINE=0023:\nDETAIL=SOURCE_LINE=0024:PVE_BACKUP_HEALTH = Path(\"/var/lib/skladchik-reports-monitor-backup/health.txt\")\nDETAIL=SOURCE_LINE=0025:PVE_RESTORE_HEALTH = Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\nDETAIL=SOURCE_LINE=0026:\nDETAIL=SOURCE_LINE=0027:def now():\nDETAIL=SOURCE_LINE=0028: return _dt.datetime.now(_dt.UTC).replace(microsecond=0)\nDETAIL=SOURCE_LINE=0029:\nDETAIL=SOURCE_LINE=0030:def iso():\nDETAIL=SOURCE_LINE=0031: return now().isoformat().replace(\"+00:00\", \"Z\")\nDETAIL=SOURCE_LINE=0032:\nDETAIL=SOURCE_LINE=0033:def log(msg):\nDETAIL=SOURCE_LINE=0034: LOG_DIR.mkdir(parents=True, exist_ok=True)\nDETAIL=SOURCE_LINE=0035: with LOG.open(\"a\", encoding=\"utf-8\") as f:\nDETAIL=SOURCE_LINE=0036: f.write(f\"{iso()} {msg}\\n\")\nDETAIL=SOURCE_LINE=0037:\nDETAIL=SOURCE_LINE=0038:def run(cmd, timeout=45, input_text=None):\nDETAIL=SOURCE_LINE=0039: try:\nDETAIL=SOURCE_LINE=0040: p = subprocess.run(cmd, shell=True, text=True, input=input_text, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout)\nDETAIL=SOURCE_LINE=0041: return p.returncode, p.stdout.strip()\nDETAIL=SOURCE_LINE=0042: except subprocess.TimeoutExpired as e:\nDETAIL=SOURCE_LINE=0043: out = e.stdout if isinstance(e.stdout, str) else \"\"\nDETAIL=SOURCE_LINE=0044: return 124, (out or \"\").strip() + \"\\nTIMEOUT\"\nDETAIL=SOURCE_LINE=0045:\nDETAIL=SOURCE_LINE=0046:def parse_env_text(text):\nDETAIL=SOURCE_LINE=0047: d = {}\nDETAIL=SOURCE_LINE=0048: for line in (text or \"\").splitlines():\nDETAIL=SOURCE_LINE=0049: if not line or line.startswith(\"#\") or \"=\" not in line:\nDETAIL=SOURCE_LINE=0050: continue\nDETAIL=SOURCE_LINE=0051: k, v = line.split(\"=\", 1)\nDETAIL=SOURCE_LINE=...\nDETAIL=SOURCE_LINE=0066: dt = _dt.datetime.fromisoformat(s)\nDETAIL=SOURCE_LINE=0067: if dt.tzinfo is None:\nDETAIL=SOURCE_LINE=0068: dt = dt.replace(tzinfo=_dt.UTC)\nDETAIL=SOURCE_LINE=0069: return int((now() - dt.astimezone(_dt.UTC)).total_seconds())\nDETAIL=SOURCE_LINE=0070: except Exception:\nDETAIL=SOURCE_LINE=0071: return 999999999\nDETAIL=SOURCE_LINE=0072:\nDETAIL=SOURCE_LINE=0073:def ssh_edge(cmd, timeout=45):\nDETAIL=SOURCE_LINE=0074: q = \"ssh -o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] \" + shlex.quote(EDGE) + \" \" + shlex.quote(\"sudo bash -lc \" + shlex.quote(cmd))\nDETAIL=SOURCE_LINE=0075: return run(q, timeout=timeout)\nDETAIL=SOURCE_LINE=0076:\nDETAIL=SOURCE_LINE=0077:def edge_mail(subject, body):\nDETAIL=SOURCE_LINE=0078: subj_b64 = base64.b64encode(subject.encode(\"utf-8\")).decode(\"ascii\")\nDETAIL=SOURCE_LINE=0079: body_b64 = base64.b64encode(body.encode(\"utf-8\")).decode(\"ascii\")\nDETAIL=SOURCE_LINE=0080: py = (\nDETAIL=SOURCE_LINE=0081: \"import base64,importlib.util;\"\nDETAIL=SOURCE_LINE=0082: \"spec=importlib.util.spec_from_file_location('m','/opt/skladchik-reports-monitor/monitor.py');\"\nDETAIL=SOURCE_LINE=0083: \"m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m);\"\nDETAIL=SOURCE_LINE=0084: \"cfg,secrets=\nDETAIL=SOURCE_LINE=0085: f\"subject=base64.b64decode('{subj_b64}').decode('utf-8');\"\nDETAIL=SOURCE_LINE=0086: f\"body=base64.b64decode('{body_b64}').decode('utf-8');\"\nDETAIL=SOURCE_LINE=0087: \"ok=m.send_mail(cfg,secrets,subject,body);\"\nDETAIL=SOURCE_LINE=0088: \"print('MAIL_OK' if ok else 'MAIL_FALSE')\"\nDETAIL=SOURCE_LINE=0089: )\nDETAIL=SOURCE_LINE=0090: rc, out = ssh_edge(\"python3 -c \" + shlex.quote(py), timeout=60)\nDETAIL=SOURCE_LINE=0091: log(f\"MAIL subject={subject!r} rc={rc} out={out!r}\")\nDETAIL=SOURCE_LINE=0092: return rc == 0 and \"MAIL_OK\" in out\nDETAIL=SOURCE_LINE=0093:\nDETAIL=SOURCE_LINE=0094:def systemctl_props(units):\nDETAIL=SOURCE_LINE=0095: cmd = \"systemctl is-enabled \" + \" \".join(map(shlex.quote, units)) + \" 2>/dev/null || true; echo __ACTIVE__; systemctl is-active \" + \" \".join(map(shlex.quote, units)) + \" 2>/dev/null || true\"\nDETAIL=SOURCE_LINE=0096: rc, out = run(cmd, timeout=20)\nDETAIL=SOURCE_LINE=0097: parts = out.split(\"__ACTIVE__\", 1)\nDETAIL=SOURCE_LINE=0098: enabled_lines = [x.strip() for x in parts[0].splitlines() if x.strip()]\nDETAIL=SOURCE_LINE=0099: active_lines = [x.strip() for x in parts[1].splitlines() if x.strip()] if len(parts) > 1 else []\nDETAIL=SOURCE_LINE=0100: res = {}\nDETAIL=SOURCE_LINE=0101: for i, u in enumerate(units):\nDETAIL=SOURCE_LINE=0102: res[u] = {\"enabled\": enabled_lines[i] if i < len(enabled_lines) else \"unknown\", \"active\": active_lines[i] if i < len(active_lines) else \"unknown\"}\nDETAIL=SOURCE_LINE=0103: return res\nDETAIL=SOURCE_LINE=0104:\nDETAIL=SOURCE_LINE=0105:def gather():\nDETAIL=SOURCE_LINE=0106: facts = {}\nDETAIL=SOURCE_LINE=0107: problems = []\nDETAIL=SOURCE_LINE=0108:\nDETAIL=SOURCE_LINE=0109: edge_cmd = r'''\nDETAIL=SOURCE_LINE=0110:set +e\nDETAIL=SOURCE_LINE=0111:echo __MONITOR_HEALTH_BEGIN__\nDETAIL=SOURCE_LINE=0112:cat /var/lib/skladchik-reports-monitor/health.txt 2>/dev/null || true\nDETAIL=SOURCE_LINE=0113:echo __MONITOR_HEALTH_END__\nDETAIL=SOURCE_LINE=0114:echo __ALERT_HEALTH_BEGIN__\nDETAIL=SOURCE_LINE=0115:cat /var/lib/skladchik-reports-monitor/alert-health.txt 2>/dev/null || true\nDETAIL=SOURCE_LINE=0116:echo __ALERT_HEALTH_END__\nDETAIL=SOURCE_LINE=0117:echo __TIMERS_BEGIN__\nDETAIL=SOURCE_LINE=0118:for u in skladchik-reports-monitor.timer skladchik-reports-monitor-watchdog.timer skladchik-reports-monitor-heartbeat.timer skladchik-reports-monitor-weekly-summary.timer skladchik-reports-monitor-age-alert.timer skladchik-reports-monitor-event-selftest.timer; do echo \"$u enabled=$(systemctl is-enabled \"$u\" 2>/dev/null) active=$(systemctl is-active \"$u\" 2>/dev/null)\"; done\nDETAIL=SOURCE_LINE=0119:echo __TIMERS_END__\nDETAIL=SOURCE_LINE=0120:echo __ROUTE_BEGIN__\nDETAIL=SOURCE_LINE=0121:ip route get 1.1.1.1 2>&1 | head -n 1\nDETAIL=SOURCE_LINE=0122:echo __ROUTE_END__\nDETAIL=SOURCE_LINE=0123:echo __FILES_BEGIN__\nDETAIL=SOURCE_LINE=0124:test -s /opt/skladchik-reports-monitor/monitor.py && echo monitor_py=yes || echo monitor_py=no\nDETAIL=SOURCE_LINE=0125:test -s /etc/skladchik-reports-monitor/config.ini && echo config_ini=yes || echo config_ini=no\nDETAIL=SOURCE_LINE=0126:test -s /etc/skladchik-reports-monitor/secrets.env && echo secrets_env=\nDETAIL=SOURCE_LINE=0127:test -s /var/lib/skladchik-reports-monitor/cookies.jar && echo cookies_jar=yes || echo cookies_jar=no\nDETAIL=SOURCE_LINE=0128:[ -s /var/lib/skladchik-reports-monitor/cookies.jar ] && echo cookies_jar_bytes=$(stat -c %s /var/lib/skladchik-reports-monitor/cookies.jar 2>/dev/null) || echo cookies_jar_bytes=0\nDETAIL=SOURCE_LINE=0129:test -s /opt/skladchik-reports-monitor/operational_digest.py && echo operational_digest_py=yes || echo operational_digest_py=no\nDETAIL=SOURCE_LINE=0130:test -s /opt/skladchik-reports-monitor/event-selftest-runner.py && echo event_selftest_runner_py=yes || echo event_selftest_runner_py=no\nDETAIL=SOURCE_LINE=0131:test -s /opt/skladchik-reports-monitor/event-selftest-core && echo event_selftest_core=yes || echo event_selftest_core=no\nDETAIL=SOURCE_LINE=0132:test -s /var/lib/skladchik-reports-monitor/weekly-summary-health.txt && echo weekly_summary_health=yes || echo weekly_summary_health=no\nDETAIL=SOURCE_LINE=0133:test -s /var/lib/skladchik-reports-monitor/age-alert-health.txt && echo age_alert_health=yes || echo age_alert_health=no\nDETAIL=SOURCE_LINE=0134:test -s /var/lib/skladchik-reports-monitor/event-selftest-health.txt && echo event_selftest_health=yes || echo event_selftest_health=no\nDETAIL=SOURCE_LINE=0135:python3 -m py_compile /opt/skladchik-reports-monitor/monitor.py /opt/skladchik-reports-monitor/operational_digest.py /opt/skladchik-reports-monitor/event-selftest-runner.py /opt/skladchik-reports-monitor/event-selftest-core >/dev/null 2>&1 && echo py_compile=yes || echo py_compile=no\nDETAIL=SOURCE_LINE=0136:echo __FILES_END__\nDETAIL=SOURCE_LINE=0137:'''\nDETAIL=SOURCE_LINE=0138: rc, out = ssh_edge(edge_cmd, timeout=45)\nDETAIL=SOURCE_LINE=0139: facts[\"edge_ssh_rc\"] = str(rc)\nDETAIL=SOURCE_LINE=0140: if rc != 0:\nDETAIL=SOURCE_LINE=0141: problems.append(\"edge_ssh_failed\")\nDETAIL=SOURCE_LINE=0142:\nDETAIL=SOURCE_LINE=0143: def section(name):\nDETAIL=SOURCE_LINE=0144: m = re.search(rf\"__{name}_BEGIN__\\n(.*?)\\n__{name}_END__\", out, re.S)\nDETAIL=SOURCE_LINE=0145: return m.group(1).strip() if m else \"\"\nDETAIL=SOURCE_LINE=0146:\nDETAIL=SOURCE_LINE=0147: mon = parse_env_text(section(\"MONITOR_HEALTH\"))\nDETAIL=SOURCE_LINE=0148: facts[\"monitor_status\"] = mon.get(\"STATUS\", \"MISSING\")\nDETAIL=SOURCE_LINE=0149: facts[\"monitor_reports\"] = mon.get(\"reports\", \"\")\nDETAIL=SOURCE_LINE=0150: facts[\"monitor_open\"] = mon.get(\"open\", \"\")\nDETAIL=SOURCE_LINE=0151: facts[\"monitor_resolved\"] = mon.get(\"resolved\", \"\")\nDETAIL=SOURCE_LINE=0152: facts[\"monitor_smtp_configured\"] = mon.get(\"smtp_configured\", \"\")\nDETAIL=SOURCE_LINE=0153: facts[\"session_keepalive\"] = mon.get(\"session_keepalive\", \"\")\nDETAIL=SOURCE_LINE=0154: facts[\"cookie_jar\"] = mon.get(\"cookie_jar\", \"\")\nDETAIL=SOURCE_LINE=0155: facts[\"cookie_jar_bytes\"] = mon.get(\"cookie_jar_bytes\", \"\")\nDETAIL=SOURCE_LINE=0156: facts[\"cookie_expiry_status\"] = mon.get(\"cookie_expiry_status\", \"\")\nDETAIL=SOURCE_LINE=0157: facts[\"cookie_expiry_utc\"] = mon.get(\"cookie_expiry_utc\", \"\")\nDETAIL=SOURCE_LINE=0158: facts[\"cookie_expires_in_seconds\"] = mon.get(\"cookie_expires_in_seconds\", \"\")\nDETAIL=SOURCE_LINE=0159: facts[\"cookie_expires_in_days\"] = mon.get(\"cookie_expires_in_days\", \"\")\nDETAIL=SOURCE_LINE=0160: facts[\"cookie_persistent_count\"] = mon.get(\"cookie_persistent_count\", \"\")\nDETAIL=SOURCE_LINE=0161: facts[\"monitor_health_age_seconds\"] = str(age_seconds_from_ts(mon.get(\"TS\")))\nDETAIL=SOURCE_LINE=0162: if mon.get(\"STATUS\") != \"OK\":\nDETAIL=SOURCE_LINE=0163: problems.append(\"monitor_status_not_ok\")\nDETAIL=SOURCE_LINE=0164: if int(facts[\"monitor_health_age_seconds\"]) > MONITOR_HEALTH_AGE_MAX:\nDETAIL=SOURCE_LINE=0165: problems.append(\"monitor_health_stale\")\nDETAIL=SOURCE_LINE=0166: if mon.get(\"reports\") in (\"\", \"0\"):\nDETAIL=SOURCE_LINE=0167: problems.append(\"monitor_reports_zero_or_missing\")\nDETAIL=SOURCE_LINE=0168: if mon.get(\"session_keepalive\") != \"persistent_cookie_jar\":\nDETAIL=SOURCE_LINE=0169: problems.append(\"session_keepalive_not_persistent\")\nDETAIL=SOURCE_LINE=0170: if mon.get(\"cookie_jar\") not in (\"yes\", \"empty\"):\nDETAIL=SOURCE_LINE=0171: problems.append(\"monitor_cookie_jar_bad\")\nDETAIL=SOURCE_LINE=0172: expiry_status = mon.get(\"cookie_expiry_status\", \"\")\nDETAIL=SOURCE_LINE=0173: try:\nDETAIL=SOURCE_LINE=0174: expires_in = int(mon.get(\"cookie_expires_in_seconds\", \"\"))\nDETAIL=SOURCE_LINE=0175: except Exception:\nDETAIL=SOURCE_LINE=0176: expires_in = None\nDETAIL=SOURCE_LINE=0177: if expiry_status in (\"\", \"NO_COOKIE_JAR\", \"NO_PERSISTENT_COOKIE\", \"ERROR\"):\nDETAIL=SOURCE_LINE=0178: problems.append(\"cookie_expiry_missing\")\nDETAIL=SOURCE_LINE=0179: elif expiry_status == \"EXPIRED\" or (expires_in is not None and expires_in <= 0):\nDETAIL=SOURCE_LINE=0180: problems.append(\"cookie_expiry_expired\")\nDETAIL=SOURCE_LINE=0181: elif expiry_status == \"WARN\" or (expires_in is not None and expires_in < COOKIE_EXPIRY_WARN_SECONDS):\nDETAIL=SOURCE_LINE=0182: problems.append(\"cookie_expiry_warn\")\nDETAIL=SOURCE_LINE=0183:\nDETAIL=SOURCE_LINE=0184: timers_text = section(\"TIMERS\")\nDETAIL=SOURCE_LINE=0185: for unit in [\"skladchik-reports-monitor.timer\", \"skladchik-reports-monitor-watchdog.timer\", \"skladchik-reports-monitor-heartbeat.timer\", \"skladchik-reports-monitor-weekly-summary.timer\", \"skladchik-reports-monitor-age-alert.timer\", \"skladchik-reports-monitor-event-selftest.timer\"]:\nDETAIL=SOURCE_LINE=0186: safe = unit.replace(\".timer\", \"\").replace(\"_\", \"-\")\nDETAIL=SOURCE_LINE=0187: m = re.search(re.escape(unit) + r\"\\s+enabled=([^\\s]+)\\s+active=([^\\s]+)\", timers_text)\nDETAIL=SOURCE_LINE=0188: enabled = m.group(1) if m else \"missing\"\nDETAIL=SOURCE_LINE=0189: active = m.group(2) if m else \"missing\"\nDETAIL=SOURCE_LINE=0190: facts[f\"edge_{safe}_timer_enabled\"] = enabled\nDETAIL=SOURCE_LINE=0191: facts[f\"edge_{safe}_timer_active\"] = active\nDETAIL=SOURCE_LINE=0192: if enabled != \"enabled\" or active != \"active\":\nDETAIL=SOURCE_LINE=0193: problems.append(f\"edge_{safe}_timer_bad\")\nDETAIL=SOURCE_LINE=0194:\nDETAIL=SOURCE_LINE=0195: route_lines = section(\"ROUTE\").splitlines()\nDETAIL=SOURCE_LINE=0196: route = route_lines[0] if route_lines else \"\"\nDETAIL=SOURCE_LINE=0197: facts[\"netbird_route\"] = route\nDETAIL=SOURCE_LINE=0198: if \" wt0 \" not in (\" \" + route + \" \"):\nDETAIL=SOURCE_LINE=0199: problems.append(\"netbird_route_not_wt0\")\nDETAIL=SOURCE_LINE=0200:\nDETAIL=SOURCE_LINE=0201: files = parse_env_text(section(\"FILES\"))\nDETAIL=SOURCE_LINE=0202: for k in [\"monitor_py\", \"config_ini\", \"secrets_env\", \"cookies_jar\", \"operational_digest_py\", \"event_selftest_runner_py\", \"event_selftest_core\", \"weekly_summary_health\", \"age_alert_health\", \"event_selftest_health\", \"py_compile\"]:\nDETAIL=SOURCE_LINE=0203: facts[k] = files.get(k, \"no\")\nDETAIL=SOURCE_LINE=0204: if facts[k] != \"yes\":\nDETAIL=SOURCE_LINE=0205: problems.append(f\"edge_{k}_bad\")\nDETAIL=SOURCE_LINE=0206: facts[\"cookies_jar_bytes_file\"] = files.get(\"cookies_jar_bytes\", \"0\")\nDETAIL=SOURCE_LINE=0207:\nDETAIL=SOURCE_LINE=0208: backup = read_env_file(PVE_BACKUP_HEALTH)\nDETAIL=SOURCE_LINE=0209: facts[\"backup_status\"] = backup.get(\"STATUS\", \"MISSING\")\nDETAIL=SOURCE_LINE=0210: facts[\"backup_run_id\"] = backup.get(\"RUN_ID\", \"\")\nDETAIL=SOURCE_LINE=0211: facts[\"backup_size_bytes\"] = backup.get(\"SIZE_BYTES\", \"\")\nDETAIL=SOURCE_LINE=0212: facts[\"backup_version\"] = backup.get(\"BACKUP_VERSION\", \"\")\nDETAIL=SOURCE_LINE=0213: facts[\"backup_cookie_jar_backed_up\"] = backup.get(\"COOKIE_JAR_BACKED_UP\", \"\")\nDETAIL=SOURCE_LINE=0214: facts[\"backup_extension_health_files_backed_up\"] = backup.get(\"EXTENSION_HEALTH_FILES_BACKED_UP\", \"\")\nDETAIL=SOURCE_LINE=0215: facts[\"backup_title_in_email_subject\"] = backup.get(\"TITLE_IN_EMAIL_SUBJECT\", \"\")\nDETAIL=SOURCE_LINE=0216: facts[\"backup_health_age_seconds\"] = str(age_seconds_from_ts(backup.get(\"TS\")))\nDETAIL=SOURCE_LINE=0217: if backup.get(\"STATUS\") != \"OK\":\nDETAIL=SOURCE_LINE=0218: problems.append(\"backup_status_not_ok\")\nDETAIL=SOURCE_LINE=0219: if backup.get(\"COOKIE_JAR_BACKED_UP\") != \"yes\":\nDETAIL=SOURCE_LINE=0220: problems.append(\"backup_cookie_jar_not_marked\")\nDETAIL=SOURCE_LINE=0221: if backup.get(\"BACKUP_VERSION\") != \"4\":\nDETAIL=SOURCE_LINE=0222: problems.append(\"backup_version_not_4\")\nDETAIL=SOURCE_LINE=0223: if backup.get(\"EXTENSION_HEALTH_FILES_BACKED_UP\") != \"yes\":\nDETAIL=SOURCE_LINE=0224: problems.append(\"backup_extension_health_files_not_marked\")\nDETAIL=SOURCE_LINE=0225: if backup.get(\"TITLE_IN_EMAIL_SUBJECT\") != \"yes\":\nDETAIL=SOURCE_LINE=0226: problems.append(\"email_title_subject_not_marked\")\nDETAIL=SOURCE_LINE=0227: if int(facts[\"backup_health_age_seconds\"]) > BACKUP_HEALTH_AGE_MAX:\nDETAIL=SOURCE_LINE=0228: problems.append(\"backup_health_stale\")\nDETAIL=SOURCE_LINE=0229: try:\nDETAIL=SOURCE_LINE=0230: if int(backup.get(\"SIZE_BYTES\", \"0\")) <= 0:\nDETAIL=SOURCE_LINE=0231: problems.append(\"backup_size_bad\")\nDETAIL=SOURCE_LINE=0232: except Exception:\nDETAIL=SOURCE_LINE=0233: problems.append(\"backup_size_bad\")\nDETAIL=SOURCE_LINE=0234:\nDETAIL=SOURCE_LINE=0235: restore = read_env_file(PVE_RESTORE_HEALTH)\nDETAIL=SOURCE_LINE=0236: facts[\"restore_status\"] = restore.get(\"STATUS\", \"MISSING\")\nDETAIL=SOURCE_LINE=0237: facts[\"restore_run_id\"] = restore.get(\"RUN_ID\", \"\")\nDETAIL=SOURCE_LINE=0238: facts[\"restore_health_age_seconds\"] = str(age_seconds_from_ts(restore.get(\"TS\")))\nDETAIL=SOURCE_LINE=0239: for key in [\"CLOUD_1_STATUS\", \"CLOUD_2_STATUS\", \"SHA256_OK\", \"TAR_OK\", \"REQUIRED_FILES_OK\", \"COOKIE_JAR_OK\", \"EXTENSION_HEALTH_FILES_OK\", \"EXTENSION_SCRIPTS_OK\", \"EXTENSION_UNITS_OK\", \"SQLITE_OK\", \"PY_COMPILE_OK\"]:\nDETAIL=SOURCE_LINE=0240: facts[key.lower()] = restore.get(key, \"\")\nDETAIL=SOURCE_LINE=0241: if restore.get(\"STATUS\") != \"OK\":\nDETAIL=SOURCE_LINE=0242: problems.append(\"restore_status_not_ok\")\nDETAIL=SOURCE_LINE=0243: if int(facts[\"restore_health_age_seconds\"]) > RESTORE_HEALTH_AGE_MAX:\nDETAIL=SOURCE_LINE=0244: problems.append(\"restore_health_stale\")\nDETAIL=SOURCE_LINE=0245: for key in [\"CLOUD_1_STATUS\", \"CLOUD_2_STATUS\"]:\nDETAIL=SOURCE_LINE=0246: if restore.get(key) != \"OK\":\nDETAIL=SOURCE_LINE=0247: problems.append(key.lower() + \"_bad\")\nDETAIL=SOURCE_LINE=0248: for key in [\"SHA256_OK\", \"TAR_OK\", \"REQUIRED_FILES_OK\", \"COOKIE_JAR_OK\", \"EXTENSION_HEALTH_FILES_OK\", \"EXTENSION_SCRIPTS_OK\", \"EXTENSION_UNITS_OK\", \"SQLITE_OK\", \"PY_COMPILE_OK\"]:\nDETAIL=SOURCE_LINE=0249: if restore.get(key) != \"yes\":\nDETAIL=SOURCE_LINE=0250: problems.append(key.lower() + \"_bad\")\nDETAIL=SOURCE_LINE=0251:\nDETAIL=SOURCE_LINE=0252: local_units = [\nDETAIL=SOURCE_LINE=0253: \"skladchik-reports-monitor-backup.timer\",\nDETAIL=SOURCE_LINE=0254: \"skladchik-reports-monitor-backup-restore-check.timer\",\nDETAIL=SOURCE_LINE=0255: \"skladchik-reports-monitor-supervisor.timer\",\nDETAIL=SOURCE_LINE=0256: \"skladchik-reports-monitor-monthly-selftest-watch.timer\",\nDETAIL=SOURCE_LINE=0257: ]\nDETAIL=SOURCE_LINE=0258: props = systemctl_props(local_units)\nDETAIL=SOURCE_LINE=0259: for u, p in props.items():\nDETAIL=SOURCE_LINE=0260: base = u.replace(\".timer\", \"\").replace(\"-\", \"_\")\nDETAIL=SOURCE_LINE=0261: facts[base + \"_timer_enabled\"] = p[\"enabled\"]\nDETAIL=SOURCE_LINE=0262: facts[base + \"_timer_active\"] = p[\"active\"]\nDETAIL=SOURCE_LINE=0263: if u != \"skladchik-reports-monitor-supervisor.timer\":\nDETAIL=SOURCE_LINE=0264: if p[\"enabled\"] != \"enabled\" or p[\"active\"] != \"active\":\nDETAIL=SOURCE_LINE=0265: problems.append(base + \"_timer_bad\")\nDETAIL=SOURCE_LINE=0266: else:\nDETAIL=SOURCE_LINE=0267: # Before first enable it can be disabled/inactive during test. After enable next runs will see enabled/active.\nDETAIL=SOURCE_LINE=0268: if p[\"enabled\"] not in (\"enabled\", \"disabled\") or p[\"active\"] not in (\"active\", \"inactive\"):\nDETAIL=SOURCE_LINE=0269: problems.append(base + \"_timer_unknown\")\nDETAIL=SOURCE_LINE=0270:\nDETAIL=SOURCE_LINE=0271: return facts, problems\nDETAIL=SOURCE_LINE=0272:\nDETAIL=SOURCE_LINE=0273:def render_body(status, facts, problems, mode):\nDETAIL=SOURCE_LINE=0274: ru_status = \"OK\" if status == \"OK\" else \"АВАРИЯ\"\nDETAIL=SOURCE_LINE=0275: lines = [\nDETAIL=SOURCE_LINE=0276: f\"Статус надзора: {ru_status}\",\nDETAIL=SOURCE_LINE=0277: f\"Время UTC: {iso()}\",\nDETAIL=SOURCE_LINE=0278: \"\",\nDETAIL=SOURCE_LINE=0279: \"Монитор жалоб:\",\nDETAIL=SOURCE_LINE=0280: f\"- статус: {facts.get('monitor_status')}\",\nDETAIL=SOURCE_LINE=0281: f\"- всего жалоб на странице: {facts.get('monitor_reports')}\",\nDETAIL=SOURCE_LINE=0282: f\"- открытые/назначенные: {facts.get('monitor_open')}\",\nDETAIL=SOURCE_LINE=0283: f\"- закрытые: {facts.get('monitor_resolved')}\",\nDETAIL=SOURCE_LINE=0284: f\"- возраст health, сек: {facts.get('monitor_health_age_seconds')}\",\nDETAIL=SOURCE_LINE=0285: f\"- автопродление авторизации: {facts.get('session_keepalive')}\",\nDETAIL=SOURCE_LINE=0286: f\"- cookie jar: {facts.get('cookie_jar')} / {facts.get('cookie_jar_bytes')} байт\",\nDETAIL=SOURCE_LINE=0287: f\"- постоянная cookie действует до UTC: {facts.get('cookie_expiry_utc') or 'не определено'}\",\nDETAIL=SOURCE_LINE=0288: f\"- осталось дней по cookie: \nDETAIL=SOURCE_LINE=0289: f\"- статус срока cookie: \nDETAIL=SOURCE_LINE=0290: f\"- NetBird route: {facts.get('netbird_route')}\",\nDETAIL=SOURCE_LINE=0291: \"\",\nDETAIL=SOURCE_LINE=0292: \"Бэкап:\",\nDETAIL=SOURCE_LINE=0293: f\"- статус: {facts.get('backup_status')}\",\nDETAIL=SOURCE_LINE=0294: f\"- последний RUN_ID: {facts.get('backup_run_id')}\",\nDETAIL=SOURCE_LINE=0295: f\"- размер архива, байт: {facts.get('backup_size_bytes')}\",\nDETAIL=SOURCE_LINE=0296: f\"- cookie jar в бэкапе: {facts.get('backup_cookie_jar_backed_up')}\",\nDETAIL=SOURCE_LINE=0297: f\"- health-файлы расширений в бэкапе: {facts.get('backup_extension_health_files_backed_up')}\",\nDETAIL=SOURCE_LINE=0298: f\"- название жалобы в теме письма: {facts.get('backup_title_in_email_subject')}\",\nDETAIL=SOURCE_LINE=0299: f\"- возраст health, сек: {facts.get('backup_health_age_seconds')}\",\nDETAIL=SOURCE_LINE=0300: \"\",\nDETAIL=SOURCE_LINE=0301: \"Restore-check:\",\nDETAIL=SOURCE_LINE=0302: f\"- статус: {facts.get('restore_status')}\",\nDETAIL=SOURCE_LINE=0303: f\"- последний RUN_ID: {facts.get('restore_run_id')}\",\nDETAIL=SOURCE_LINE=0304: f\"- cloud1/cloud2: {facts.get('cloud_1_status')}/{facts.get('cloud_2_status')}\",\nDETAIL=SOURCE_LINE=0305: f\"- sha256/tar/cookie/sqlite/python: {facts.get('sha256_ok')}/{facts.get('tar_ok')}/{facts.get('cookie_jar_ok')}/{facts.get('sqlite_ok')}/{facts.get('py_compile_ok')}\",\nDETAIL=SOURCE_LINE=0306: f\"- health/scripts/units расширений: {facts.get('extension_health_files_ok')}/{facts.get('extension_scripts_ok')}/{facts.get('extension_units_ok')}\",\nDETAIL=SOURCE_LINE=0307: \"\",\nDETAIL=SOURCE_LINE=0308: \"Проблемы:\",\nDETAIL=SOURCE_LINE=0309: ]\nDETAIL=SOURCE_LINE=0310: if problems:\nDETAIL=SOURCE_LINE=0311: lines.extend([f\"- {p}\" for p in problems])\nDETAIL=SOURCE_LINE=0312: else:\nDETAIL=SOURCE_LINE=0313: lines.append(\"- нет\")\nDETAIL=SOURCE_LINE=0314: lines.extend([\nDETAIL=SOURCE_LINE=0315: \"\",\nDETAIL=SOURCE_LINE=0316: \"Авторизация:\",\nDETAIL=SOURCE_LINE=0317: \"- ручной ежемесячный перелогин обычно не нужен; монитор сам продлевает сессию постоянным cookie jar при каждом опросе;\",\nDETAIL=SOURCE_LINE=0318: \"- supervisor теперь отдельно следит за сроком постоянной cookie и заранее предупредит, если до истечения останется меньше 7 дней;\",\nDETAIL=SOURCE_LINE=0319: \"- если форум принудительно сбросит сессию, придёт отдельная тревога AUTH_REQUIRED; тогда нужно будет разово обновить вход, но это аварийный сценарий, а не ежемесячная рутина.\",\nDETAIL=SOURCE_LINE=0320: ])\nDETAIL=SOURCE_LINE=0321: return \"\\n\".join(lines)\nDETAIL=SOURCE_LINE=0322:\nDETAIL=SOURCE_LINE=0323:def write_health(status, facts, problems, mail_sent):\nDETAIL=SOURCE_LINE=0324: STATE_DIR.mkdir(parents=True, exist_ok=True)\nDETAIL=SOURCE_LINE=0325: lines = [f\"STATUS={status}\", f\"TS={iso()}\", \"problems=\" + \",\".join(problems), f\"mail_sent={'yes' if mail_sent else 'no'}\"]\nDETAIL=SOURCE_LINE=0326: for k in sorted(facts):\nDETAIL=SOURCE_LINE=0327: v = str(facts[k]).replace(\"\\n\", \" \")[:500]\nDETAIL=SOURCE_LINE=0328: lines.append(f\"{k}={v}\")\nDETAIL=SOURCE_LINE=0329: HEALTH.write_text(\"\\n\".join(lines) + \"\\n\", encoding=\"utf-8\")\nDETAIL=SOURCE_LINE=0330:\nDETAIL=SOURCE_LINE=0331:def read_state():\nDETAIL=SOURCE_LINE=0332: return read_env_file(STATE)\nDETAIL=SOURCE_LINE=0333:\nDETAIL=SOURCE_LINE=0334:def write_state(status, problems):\nDETAIL=SOURCE_LINE=0335: STATE_DIR.mkdir(parents=True, exist_ok=True)\nDETAIL=SOURCE_LINE=0336: STATE.write_text(f\"LAST_STATUS={status}\\nLAST_PROBLEMS={','.join(problems)}\\nTS={iso()}\\n\", encoding=\"utf-8\")\nDETAIL=SOURCE_LINE=0337:\nDETAIL=SOURCE_LINE=0338:def main():\nDETAIL=SOURCE_LINE=0339: mode = sys.argv[1] if len(sys.argv) > 1 else \"check\"\nDETAIL=SOURCE_LINE=0340: if mode not in (\"check\", \"test\"):\nDETAIL=SOURCE_LINE=0341: print(\"usage: skladchik-reports-monitor-supervisor [check|test]\", file=sys.stderr)\nDETAIL=SOURCE_LINE=0342: return 2\nDETAIL=SOURCE_LINE=0343:\nDETAIL=SOURCE_LINE=0344: facts, problems = gather()\nDETAIL=SOURCE_LINE=0345: status = \"OK\" if not problems else \"FAIL\"\nDETAIL=SOURCE_LINE=0346: old = read_state()\nDETAIL=SOURCE_LINE=0347: old_status = old.get(\"LAST_STATUS\", \"\")\nDETAIL=SOURCE_LINE=0348: mail_sent = False\nDETAIL=SOURCE_LINE=0349:\nDETAIL=SOURCE_LINE=0350: if mode == \"test\":\nDETAIL=SOURCE_LINE=0351: mail_sent = edge_mail(\"Тест: надзор монитора жалоб и бэкапов\", render_body(status, facts, problems, mode))\nDETAIL=SOURCE_LINE=0352: if not mail_sent:\nDETAIL=SOURCE_LINE=0353: problems.append(\"supervisor_mail_send_failed\")\nDETAIL=SOURCE_LINE=0354: status = \"FAIL\"\nDETAIL=SOURCE_LINE=0355: elif status != \"OK\":\nDETAIL=SOURCE_LINE=0356: if old_status != \"FAIL\":\nDETAIL=SOURCE_LINE=0357: mail_sent = edge_mail(\"Авария: монитор жалоб или его бэкапы\", render_body(status, facts, problems, mode))\nDETAIL=SOURCE_LINE=0358: if not mail_sent:\nDETAIL=SOURCE_LINE=0359: problems.append(\"supervisor_mail_send_failed\")\nDETAIL=SOURCE_LINE=0360: status = \"FAIL\"\nDETAIL=SOURCE_LINE=0361: elif old_status == \"FAIL\":\nDETAIL=SOURCE_LINE=0362: mail_sent = edge_mail(\"Восстановлено: монитор жалоб и бэкапы снова OK\", render_body(status, facts, problems, mode))\nDETAIL=SOURCE_LINE=0363:\nDETAIL=SOURCE_LINE=0364: write_health(status, facts, problems, mail_sent)\nDETAIL=SOURCE_LINE=0365: write_state(status, problems)\nDETAIL=SOURCE_LINE=0366:\nDETAIL=SOURCE_LINE=0367: print(f\"SUPERVISOR_STATUS={status}\")\nDETAIL=SOURCE_LINE=0368: print(f\"SUPERVISOR_MAIL_SENT={'yes' if mail_sent else 'no'}\")\nDETAIL=SOURCE_LINE=0369: print(f\"SUPERVISOR_PROBLEMS={','.join(problems)}\")\nDETAIL=SOURCE_LINE=0370: print(f\"MONITOR={facts.get('monitor_status')} reports={facts.get('monitor_reports')} open={facts.get('monitor_open')} resolved={facts.get('monitor_resolved')}\")\nDETAIL=SOURCE_LINE=0371: print(f\"BACKUP={facts.get('backup_status')} run={facts.get('backup_run_id')}\")\nDETAIL=SOURCE_LINE=0372: print(f\"RESTORE={facts.get('restore_status')} run={facts.get('restore_run_id')}\")\nDETAIL=SOURCE_LINE=0373: log(f\"CHECK mode={mode} status={status} mail_sent={'yes' if mail_sent else 'no'} problems={problems}\")\nDETAIL=SOURCE_LINE=0374: return 0 if status == \"OK\" else 1\nDETAIL=SOURCE_LINE=0375:\nDETAIL=SOURCE_LINE=0376:if __name__ == \"__main__\":\nDETAIL=SOURCE_LINE=0377: sys.exit(main())\nDETAIL=SCRIPT_TOTAL_LINES=377\nDETAIL=RELEVANT_SOURCE_END\nDETAIL=UNIT_CONTRACT_BEGIN\nDETAIL=UNIT_NAME=skladchik-reports-monitor-supervisor.service\nDETAIL=UNIT_LINE=# /etc/systemd/system/skladchik-reports-monitor-supervisor.service\nDETAIL=UNIT_LINE=[Unit]\nDETAIL=UNIT_LINE=Description=Skladchik reports monitor full supervisor\nDETAIL=UNIT_LINE=After=network-online.target\nDETAIL=UNIT_LINE=\nDETAIL=UNIT_LINE=[Service]\nDETAIL=UNIT_LINE=Type=oneshot\nDETAIL=UNIT_LINE=ExecStart=/usr/local/sbin/skladchik-reports-monitor-supervisor check\nDETAIL=UNIT_NAME=skladchik-reports-monitor-supervisor.timer\nDETAIL=UNIT_LINE=# /etc/systemd/system/skladchik-reports-monitor-supervisor.timer\nDETAIL=UNIT_LINE=[Unit]\nDETAIL=UNIT_LINE=Description=Run Skladchik reports monitor full supervisor every 10 minutes\nDETAIL=UNIT_LINE=\nDETAIL=UNIT_LINE=[Timer]\nDETAIL=UNIT_LINE=OnBootSec=5min\nDETAIL=UNIT_LINE=OnUnitActiveSec=10min\nDETAIL=UNIT_LINE=AccuracySec=30s\nDETAIL=UNIT_LINE=Persistent=false\nDETAIL=UNIT_LINE=Unit=skladchik-reports-monitor-supervisor.service\nDETAIL=UNIT_LINE=\nDETAIL=UNIT_LINE=[Install]\nDETAIL=UNIT_LINE=WantedBy=timers.target\nDETAIL=UNIT_CONTRACT_END\nDETAIL=TIMER_STATE_BEGIN\nDETAIL=TIMER_PROPERTY=Unit=skladchik-reports-monitor-supervisor.service\nDETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=\nDETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 11:58:38 MSK\nDETAIL=TIMER_PROPERTY=LoadState=loaded\nDETAIL=TIMER_PROPERTY=ActiveState=active\nDETAIL=TIMER_PROPERTY=SubState=waiting\nDETAIL=TIMER_ROW=Fri 2026-07-24 12:08:38 MSK 3min 34s Fri 2026-07-24 11:58:38 MSK 6min ago skladchik-reports-monitor-supervisor.timer skladchik-reports-monitor-supervisor.service\nDETAIL=TIMER_ROW=Sat 2026-07-25 04:39:05 MSK 16h Fri 2026-07-24 04:28:08 MSK 7h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service\nDETAIL=TIMER_ROW=Sat 2026-07-25 10:15:00 MSK 22h Fri 2026-07-24 10:15:04 MSK 1h 49min ago skladchik-reports-monitor-monthly-selftest-watch.timer skladchik-reports-monitor-monthly-selftest-watch.service\nDETAIL=TIMER_ROW=Sun 2026-07-26 05:15:13 MSK 1 day 17h Sun 2026-07-19 05:11:25 MSK - skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service\nDETAIL=TIMER_STATE_END\nDETAIL=SECRET_VALUES_OUTPUT=NONE\nDETAIL=CHANGES_MADE=NO\nDETAIL=COOKIE_CHECK_SOURCE_AUDIT=PASS\n" +} diff --git a/runtime/history/HOMELAB-SKLADCHIK-COOKIE-CHECK-SOURCE-AUDIT-418.txt b/runtime/history/HOMELAB-SKLADCHIK-COOKIE-CHECK-SOURCE-AUDIT-418.txt new file mode 100644 index 0000000..a7af1d9 --- /dev/null +++ b/runtime/history/HOMELAB-SKLADCHIK-COOKIE-CHECK-SOURCE-AUDIT-418.txt @@ -0,0 +1,430 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=HOMELAB-SKLADCHIK-COOKIE-CHECK-SOURCE-AUDIT-418 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=cluster-audit +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 +COMMAND_SHA256=77896618b46e2f97362240deb743fb8e625075381e57377ee0314130be2b2026 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=false +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=1c26f0b751c5e55421c706f56f4be25d92c17849df8a385fb66ce01144801022 +SANITIZED_OUTPUT_SHA256=8e5a54db35050a5c0010241fb0bad91fc5966530dbac76bb07426b09af51a6a1 +OUTPUT_BEGIN +DETAIL=SCRIPT_PATH=/usr/local/sbin/skladchik-reports-monitor-supervisor +DETAIL=SCRIPT_SHA256=35c68ae0929ff3795be2bde40de334fc8e26413a3d8a620a137be90d1f2c164e +DETAIL=SCRIPT_METADATA=BYTES:20463|MODE:755|OWNER:root:root|MTIME:2026-07-18 13:52:30.415780066 +0300 +DETAIL=RELEVANT_SOURCE_BEGIN +DETAIL=SOURCE_LINE=0005:import os +DETAIL=SOURCE_LINE=0006:import re +DETAIL=SOURCE_LINE=0007:import shlex +DETAIL=SOURCE_LINE=0008:import subprocess +DETAIL=SOURCE_LINE=0009:import sys +DETAIL=SOURCE_LINE=0010:from pathlib import Path +DETAIL=SOURCE_LINE=0011: +DETAIL=SOURCE_LINE=0012:EDGE = os.environ.get("SKLADCHIK_EDGE", "debian@[PRIVATE_IP]") +DETAIL=SOURCE_LINE=0013:STATE_DIR = Path("/var/lib/skladchik-reports-monitor-supervisor") +DETAIL=SOURCE_LINE=0014:LOG_DIR = Path("/var/log/skladchik-reports-monitor-supervisor") +DETAIL=SOURCE_LINE=0015:HEALTH = STATE_DIR / "health.txt" +DETAIL=SOURCE_LINE=0016:STATE = STATE_DIR / "state.env" +DETAIL=SOURCE_LINE=0017:LOG = LOG_DIR / "supervisor.log" +DETAIL=SOURCE_LINE=0018: +DETAIL=SOURCE_LINE=0019:MONITOR_HEALTH_AGE_MAX = int(os.environ.get("SKLADCHIK_MONITOR_HEALTH_MAX_SECONDS", "600")) +DETAIL=SOURCE_LINE=0020:BACKUP_HEALTH_AGE_MAX = int(os.environ.get("SKLADCHIK_BACKUP_HEALTH_MAX_SECONDS", "129600")) +DETAIL=SOURCE_LINE=0021:RESTORE_HEALTH_AGE_MAX = int(os.environ.get("SKLADCHIK_RESTORE_HEALTH_MAX_SECONDS", "691200")) +DETAIL=SOURCE_LINE=0022:COOKIE_EXPIRY_WARN_SECONDS = int(os.environ.get("SKLADCHIK_COOKIE_EXPIRY_WARN_SECONDS", "604800")) +DETAIL=SOURCE_LINE=0023: +DETAIL=SOURCE_LINE=0024:PVE_BACKUP_HEALTH = Path("/var/lib/skladchik-reports-monitor-backup/health.txt") +DETAIL=SOURCE_LINE=0025:PVE_RESTORE_HEALTH = Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt") +DETAIL=SOURCE_LINE=0026: +DETAIL=SOURCE_LINE=0027:def now(): +DETAIL=SOURCE_LINE=0028: return _dt.datetime.now(_dt.UTC).replace(microsecond=0) +DETAIL=SOURCE_LINE=0029: +DETAIL=SOURCE_LINE=0030:def iso(): +DETAIL=SOURCE_LINE=0031: return now().isoformat().replace("+00:00", "Z") +DETAIL=SOURCE_LINE=0032: +DETAIL=SOURCE_LINE=0033:def log(msg): +DETAIL=SOURCE_LINE=0034: LOG_DIR.mkdir(parents=True, exist_ok=True) +DETAIL=SOURCE_LINE=0035: with LOG.open("a", encoding="utf-8") as f: +DETAIL=SOURCE_LINE=0036: f.write(f"{iso()} {msg}\n") +DETAIL=SOURCE_LINE=0037: +DETAIL=SOURCE_LINE=0038:def run(cmd, timeout=45, input_text=None): +DETAIL=SOURCE_LINE=0039: try: +DETAIL=SOURCE_LINE=0040: p = subprocess.run(cmd, shell=True, text=True, input=input_text, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout) +DETAIL=SOURCE_LINE=0041: return p.returncode, p.stdout.strip() +DETAIL=SOURCE_LINE=0042: except subprocess.TimeoutExpired as e: +DETAIL=SOURCE_LINE=0043: out = e.stdout if isinstance(e.stdout, str) else "" +DETAIL=SOURCE_LINE=0044: return 124, (out or "").strip() + "\nTIMEOUT" +DETAIL=SOURCE_LINE=0045: +DETAIL=SOURCE_LINE=0046:def parse_env_text(text): +DETAIL=SOURCE_LINE=0047: d = {} +DETAIL=SOURCE_LINE=0048: for line in (text or "").splitlines(): +DETAIL=SOURCE_LINE=0049: if not line or line.startswith("#") or "=" not in line: +DETAIL=SOURCE_LINE=0050: continue +DETAIL=SOURCE_LINE=0051: k, v = line.split("=", 1) +DETAIL=SOURCE_LINE=... +DETAIL=SOURCE_LINE=0066: dt = _dt.datetime.fromisoformat(s) +DETAIL=SOURCE_LINE=0067: if dt.tzinfo is None: +DETAIL=SOURCE_LINE=0068: dt = dt.replace(tzinfo=_dt.UTC) +DETAIL=SOURCE_LINE=0069: return int((now() - dt.astimezone(_dt.UTC)).total_seconds()) +DETAIL=SOURCE_LINE=0070: except Exception: +DETAIL=SOURCE_LINE=0071: return 999999999 +DETAIL=SOURCE_LINE=0072: +DETAIL=SOURCE_LINE=0073:def ssh_edge(cmd, timeout=45): +DETAIL=SOURCE_LINE=0074: q = "ssh -o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] " + shlex.quote(EDGE) + " " + shlex.quote("sudo bash -lc " + shlex.quote(cmd)) +DETAIL=SOURCE_LINE=0075: return run(q, timeout=timeout) +DETAIL=SOURCE_LINE=0076: +DETAIL=SOURCE_LINE=0077:def edge_mail(subject, body): +DETAIL=SOURCE_LINE=0078: subj_b64 = base64.b64encode(subject.encode("utf-8")).decode("ascii") +DETAIL=SOURCE_LINE=0079: body_b64 = base64.b64encode(body.encode("utf-8")).decode("ascii") +DETAIL=SOURCE_LINE=0080: py = ( +DETAIL=SOURCE_LINE=0081: "import base64,importlib.util;" +DETAIL=SOURCE_LINE=0082: "spec=importlib.util.spec_from_file_location('m','/opt/skladchik-reports-monitor/monitor.py');" +DETAIL=SOURCE_LINE=0083: "m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m);" +DETAIL=SOURCE_LINE=0084: "cfg,secrets= +DETAIL=SOURCE_LINE=0085: f"subject=base64.b64decode('{subj_b64}').decode('utf-8');" +DETAIL=SOURCE_LINE=0086: f"body=base64.b64decode('{body_b64}').decode('utf-8');" +DETAIL=SOURCE_LINE=0087: "ok=m.send_mail(cfg,secrets,subject,body);" +DETAIL=SOURCE_LINE=0088: "print('MAIL_OK' if ok else 'MAIL_FALSE')" +DETAIL=SOURCE_LINE=0089: ) +DETAIL=SOURCE_LINE=0090: rc, out = ssh_edge("python3 -c " + shlex.quote(py), timeout=60) +DETAIL=SOURCE_LINE=0091: log(f"MAIL subject={subject!r} rc={rc} out={out!r}") +DETAIL=SOURCE_LINE=0092: return rc == 0 and "MAIL_OK" in out +DETAIL=SOURCE_LINE=0093: +DETAIL=SOURCE_LINE=0094:def systemctl_props(units): +DETAIL=SOURCE_LINE=0095: cmd = "systemctl is-enabled " + " ".join(map(shlex.quote, units)) + " 2>/dev/null || true; echo __ACTIVE__; systemctl is-active " + " ".join(map(shlex.quote, units)) + " 2>/dev/null || true" +DETAIL=SOURCE_LINE=0096: rc, out = run(cmd, timeout=20) +DETAIL=SOURCE_LINE=0097: parts = out.split("__ACTIVE__", 1) +DETAIL=SOURCE_LINE=0098: enabled_lines = [x.strip() for x in parts[0].splitlines() if x.strip()] +DETAIL=SOURCE_LINE=0099: active_lines = [x.strip() for x in parts[1].splitlines() if x.strip()] if len(parts) > 1 else [] +DETAIL=SOURCE_LINE=0100: res = {} +DETAIL=SOURCE_LINE=0101: for i, u in enumerate(units): +DETAIL=SOURCE_LINE=0102: res[u] = {"enabled": enabled_lines[i] if i < len(enabled_lines) else "unknown", "active": active_lines[i] if i < len(active_lines) else "unknown"} +DETAIL=SOURCE_LINE=0103: return res +DETAIL=SOURCE_LINE=0104: +DETAIL=SOURCE_LINE=0105:def gather(): +DETAIL=SOURCE_LINE=0106: facts = {} +DETAIL=SOURCE_LINE=0107: problems = [] +DETAIL=SOURCE_LINE=0108: +DETAIL=SOURCE_LINE=0109: edge_cmd = r''' +DETAIL=SOURCE_LINE=0110:set +e +DETAIL=SOURCE_LINE=0111:echo __MONITOR_HEALTH_BEGIN__ +DETAIL=SOURCE_LINE=0112:cat /var/lib/skladchik-reports-monitor/health.txt 2>/dev/null || true +DETAIL=SOURCE_LINE=0113:echo __MONITOR_HEALTH_END__ +DETAIL=SOURCE_LINE=0114:echo __ALERT_HEALTH_BEGIN__ +DETAIL=SOURCE_LINE=0115:cat /var/lib/skladchik-reports-monitor/alert-health.txt 2>/dev/null || true +DETAIL=SOURCE_LINE=0116:echo __ALERT_HEALTH_END__ +DETAIL=SOURCE_LINE=0117:echo __TIMERS_BEGIN__ +DETAIL=SOURCE_LINE=0118:for u in skladchik-reports-monitor.timer skladchik-reports-monitor-watchdog.timer skladchik-reports-monitor-heartbeat.timer skladchik-reports-monitor-weekly-summary.timer skladchik-reports-monitor-age-alert.timer skladchik-reports-monitor-event-selftest.timer; do echo "$u enabled=$(systemctl is-enabled "$u" 2>/dev/null) active=$(systemctl is-active "$u" 2>/dev/null)"; done +DETAIL=SOURCE_LINE=0119:echo __TIMERS_END__ +DETAIL=SOURCE_LINE=0120:echo __ROUTE_BEGIN__ +DETAIL=SOURCE_LINE=0121:ip route get 1.1.1.1 2>&1 | head -n 1 +DETAIL=SOURCE_LINE=0122:echo __ROUTE_END__ +DETAIL=SOURCE_LINE=0123:echo __FILES_BEGIN__ +DETAIL=SOURCE_LINE=0124:test -s /opt/skladchik-reports-monitor/monitor.py && echo monitor_py=yes || echo monitor_py=no +DETAIL=SOURCE_LINE=0125:test -s /etc/skladchik-reports-monitor/config.ini && echo config_ini=yes || echo config_ini=no +DETAIL=SOURCE_LINE=0126:test -s /etc/skladchik-reports-monitor/secrets.env && echo secrets_env= +DETAIL=SOURCE_LINE=0127:test -s /var/lib/skladchik-reports-monitor/cookies.jar && echo cookies_jar=yes || echo cookies_jar=no +DETAIL=SOURCE_LINE=0128:[ -s /var/lib/skladchik-reports-monitor/cookies.jar ] && echo cookies_jar_bytes=$(stat -c %s /var/lib/skladchik-reports-monitor/cookies.jar 2>/dev/null) || echo cookies_jar_bytes=0 +DETAIL=SOURCE_LINE=0129:test -s /opt/skladchik-reports-monitor/operational_digest.py && echo operational_digest_py=yes || echo operational_digest_py=no +DETAIL=SOURCE_LINE=0130:test -s /opt/skladchik-reports-monitor/event-selftest-runner.py && echo event_selftest_runner_py=yes || echo event_selftest_runner_py=no +DETAIL=SOURCE_LINE=0131:test -s /opt/skladchik-reports-monitor/event-selftest-core && echo event_selftest_core=yes || echo event_selftest_core=no +DETAIL=SOURCE_LINE=0132:test -s /var/lib/skladchik-reports-monitor/weekly-summary-health.txt && echo weekly_summary_health=yes || echo weekly_summary_health=no +DETAIL=SOURCE_LINE=0133:test -s /var/lib/skladchik-reports-monitor/age-alert-health.txt && echo age_alert_health=yes || echo age_alert_health=no +DETAIL=SOURCE_LINE=0134:test -s /var/lib/skladchik-reports-monitor/event-selftest-health.txt && echo event_selftest_health=yes || echo event_selftest_health=no +DETAIL=SOURCE_LINE=0135:python3 -m py_compile /opt/skladchik-reports-monitor/monitor.py /opt/skladchik-reports-monitor/operational_digest.py /opt/skladchik-reports-monitor/event-selftest-runner.py /opt/skladchik-reports-monitor/event-selftest-core >/dev/null 2>&1 && echo py_compile=yes || echo py_compile=no +DETAIL=SOURCE_LINE=0136:echo __FILES_END__ +DETAIL=SOURCE_LINE=0137:''' +DETAIL=SOURCE_LINE=0138: rc, out = ssh_edge(edge_cmd, timeout=45) +DETAIL=SOURCE_LINE=0139: facts["edge_ssh_rc"] = str(rc) +DETAIL=SOURCE_LINE=0140: if rc != 0: +DETAIL=SOURCE_LINE=0141: problems.append("edge_ssh_failed") +DETAIL=SOURCE_LINE=0142: +DETAIL=SOURCE_LINE=0143: def section(name): +DETAIL=SOURCE_LINE=0144: m = re.search(rf"__{name}_BEGIN__\n(.*?)\n__{name}_END__", out, re.S) +DETAIL=SOURCE_LINE=0145: return m.group(1).strip() if m else "" +DETAIL=SOURCE_LINE=0146: +DETAIL=SOURCE_LINE=0147: mon = parse_env_text(section("MONITOR_HEALTH")) +DETAIL=SOURCE_LINE=0148: facts["monitor_status"] = mon.get("STATUS", "MISSING") +DETAIL=SOURCE_LINE=0149: facts["monitor_reports"] = mon.get("reports", "") +DETAIL=SOURCE_LINE=0150: facts["monitor_open"] = mon.get("open", "") +DETAIL=SOURCE_LINE=0151: facts["monitor_resolved"] = mon.get("resolved", "") +DETAIL=SOURCE_LINE=0152: facts["monitor_smtp_configured"] = mon.get("smtp_configured", "") +DETAIL=SOURCE_LINE=0153: facts["session_keepalive"] = mon.get("session_keepalive", "") +DETAIL=SOURCE_LINE=0154: facts["cookie_jar"] = mon.get("cookie_jar", "") +DETAIL=SOURCE_LINE=0155: facts["cookie_jar_bytes"] = mon.get("cookie_jar_bytes", "") +DETAIL=SOURCE_LINE=0156: facts["cookie_expiry_status"] = mon.get("cookie_expiry_status", "") +DETAIL=SOURCE_LINE=0157: facts["cookie_expiry_utc"] = mon.get("cookie_expiry_utc", "") +DETAIL=SOURCE_LINE=0158: facts["cookie_expires_in_seconds"] = mon.get("cookie_expires_in_seconds", "") +DETAIL=SOURCE_LINE=0159: facts["cookie_expires_in_days"] = mon.get("cookie_expires_in_days", "") +DETAIL=SOURCE_LINE=0160: facts["cookie_persistent_count"] = mon.get("cookie_persistent_count", "") +DETAIL=SOURCE_LINE=0161: facts["monitor_health_age_seconds"] = str(age_seconds_from_ts(mon.get("TS"))) +DETAIL=SOURCE_LINE=0162: if mon.get("STATUS") != "OK": +DETAIL=SOURCE_LINE=0163: problems.append("monitor_status_not_ok") +DETAIL=SOURCE_LINE=0164: if int(facts["monitor_health_age_seconds"]) > MONITOR_HEALTH_AGE_MAX: +DETAIL=SOURCE_LINE=0165: problems.append("monitor_health_stale") +DETAIL=SOURCE_LINE=0166: if mon.get("reports") in ("", "0"): +DETAIL=SOURCE_LINE=0167: problems.append("monitor_reports_zero_or_missing") +DETAIL=SOURCE_LINE=0168: if mon.get("session_keepalive") != "persistent_cookie_jar": +DETAIL=SOURCE_LINE=0169: problems.append("session_keepalive_not_persistent") +DETAIL=SOURCE_LINE=0170: if mon.get("cookie_jar") not in ("yes", "empty"): +DETAIL=SOURCE_LINE=0171: problems.append("monitor_cookie_jar_bad") +DETAIL=SOURCE_LINE=0172: expiry_status = mon.get("cookie_expiry_status", "") +DETAIL=SOURCE_LINE=0173: try: +DETAIL=SOURCE_LINE=0174: expires_in = int(mon.get("cookie_expires_in_seconds", "")) +DETAIL=SOURCE_LINE=0175: except Exception: +DETAIL=SOURCE_LINE=0176: expires_in = None +DETAIL=SOURCE_LINE=0177: if expiry_status in ("", "NO_COOKIE_JAR", "NO_PERSISTENT_COOKIE", "ERROR"): +DETAIL=SOURCE_LINE=0178: problems.append("cookie_expiry_missing") +DETAIL=SOURCE_LINE=0179: elif expiry_status == "EXPIRED" or (expires_in is not None and expires_in <= 0): +DETAIL=SOURCE_LINE=0180: problems.append("cookie_expiry_expired") +DETAIL=SOURCE_LINE=0181: elif expiry_status == "WARN" or (expires_in is not None and expires_in < COOKIE_EXPIRY_WARN_SECONDS): +DETAIL=SOURCE_LINE=0182: problems.append("cookie_expiry_warn") +DETAIL=SOURCE_LINE=0183: +DETAIL=SOURCE_LINE=0184: timers_text = section("TIMERS") +DETAIL=SOURCE_LINE=0185: for unit in ["skladchik-reports-monitor.timer", "skladchik-reports-monitor-watchdog.timer", "skladchik-reports-monitor-heartbeat.timer", "skladchik-reports-monitor-weekly-summary.timer", "skladchik-reports-monitor-age-alert.timer", "skladchik-reports-monitor-event-selftest.timer"]: +DETAIL=SOURCE_LINE=0186: safe = unit.replace(".timer", "").replace("_", "-") +DETAIL=SOURCE_LINE=0187: m = re.search(re.escape(unit) + r"\s+enabled=([^\s]+)\s+active=([^\s]+)", timers_text) +DETAIL=SOURCE_LINE=0188: enabled = m.group(1) if m else "missing" +DETAIL=SOURCE_LINE=0189: active = m.group(2) if m else "missing" +DETAIL=SOURCE_LINE=0190: facts[f"edge_{safe}_timer_enabled"] = enabled +DETAIL=SOURCE_LINE=0191: facts[f"edge_{safe}_timer_active"] = active +DETAIL=SOURCE_LINE=0192: if enabled != "enabled" or active != "active": +DETAIL=SOURCE_LINE=0193: problems.append(f"edge_{safe}_timer_bad") +DETAIL=SOURCE_LINE=0194: +DETAIL=SOURCE_LINE=0195: route_lines = section("ROUTE").splitlines() +DETAIL=SOURCE_LINE=0196: route = route_lines[0] if route_lines else "" +DETAIL=SOURCE_LINE=0197: facts["netbird_route"] = route +DETAIL=SOURCE_LINE=0198: if " wt0 " not in (" " + route + " "): +DETAIL=SOURCE_LINE=0199: problems.append("netbird_route_not_wt0") +DETAIL=SOURCE_LINE=0200: +DETAIL=SOURCE_LINE=0201: files = parse_env_text(section("FILES")) +DETAIL=SOURCE_LINE=0202: for k in ["monitor_py", "config_ini", "secrets_env", "cookies_jar", "operational_digest_py", "event_selftest_runner_py", "event_selftest_core", "weekly_summary_health", "age_alert_health", "event_selftest_health", "py_compile"]: +DETAIL=SOURCE_LINE=0203: facts[k] = files.get(k, "no") +DETAIL=SOURCE_LINE=0204: if facts[k] != "yes": +DETAIL=SOURCE_LINE=0205: problems.append(f"edge_{k}_bad") +DETAIL=SOURCE_LINE=0206: facts["cookies_jar_bytes_file"] = files.get("cookies_jar_bytes", "0") +DETAIL=SOURCE_LINE=0207: +DETAIL=SOURCE_LINE=0208: backup = read_env_file(PVE_BACKUP_HEALTH) +DETAIL=SOURCE_LINE=0209: facts["backup_status"] = backup.get("STATUS", "MISSING") +DETAIL=SOURCE_LINE=0210: facts["backup_run_id"] = backup.get("RUN_ID", "") +DETAIL=SOURCE_LINE=0211: facts["backup_size_bytes"] = backup.get("SIZE_BYTES", "") +DETAIL=SOURCE_LINE=0212: facts["backup_version"] = backup.get("BACKUP_VERSION", "") +DETAIL=SOURCE_LINE=0213: facts["backup_cookie_jar_backed_up"] = backup.get("COOKIE_JAR_BACKED_UP", "") +DETAIL=SOURCE_LINE=0214: facts["backup_extension_health_files_backed_up"] = backup.get("EXTENSION_HEALTH_FILES_BACKED_UP", "") +DETAIL=SOURCE_LINE=0215: facts["backup_title_in_email_subject"] = backup.get("TITLE_IN_EMAIL_SUBJECT", "") +DETAIL=SOURCE_LINE=0216: facts["backup_health_age_seconds"] = str(age_seconds_from_ts(backup.get("TS"))) +DETAIL=SOURCE_LINE=0217: if backup.get("STATUS") != "OK": +DETAIL=SOURCE_LINE=0218: problems.append("backup_status_not_ok") +DETAIL=SOURCE_LINE=0219: if backup.get("COOKIE_JAR_BACKED_UP") != "yes": +DETAIL=SOURCE_LINE=0220: problems.append("backup_cookie_jar_not_marked") +DETAIL=SOURCE_LINE=0221: if backup.get("BACKUP_VERSION") != "4": +DETAIL=SOURCE_LINE=0222: problems.append("backup_version_not_4") +DETAIL=SOURCE_LINE=0223: if backup.get("EXTENSION_HEALTH_FILES_BACKED_UP") != "yes": +DETAIL=SOURCE_LINE=0224: problems.append("backup_extension_health_files_not_marked") +DETAIL=SOURCE_LINE=0225: if backup.get("TITLE_IN_EMAIL_SUBJECT") != "yes": +DETAIL=SOURCE_LINE=0226: problems.append("email_title_subject_not_marked") +DETAIL=SOURCE_LINE=0227: if int(facts["backup_health_age_seconds"]) > BACKUP_HEALTH_AGE_MAX: +DETAIL=SOURCE_LINE=0228: problems.append("backup_health_stale") +DETAIL=SOURCE_LINE=0229: try: +DETAIL=SOURCE_LINE=0230: if int(backup.get("SIZE_BYTES", "0")) <= 0: +DETAIL=SOURCE_LINE=0231: problems.append("backup_size_bad") +DETAIL=SOURCE_LINE=0232: except Exception: +DETAIL=SOURCE_LINE=0233: problems.append("backup_size_bad") +DETAIL=SOURCE_LINE=0234: +DETAIL=SOURCE_LINE=0235: restore = read_env_file(PVE_RESTORE_HEALTH) +DETAIL=SOURCE_LINE=0236: facts["restore_status"] = restore.get("STATUS", "MISSING") +DETAIL=SOURCE_LINE=0237: facts["restore_run_id"] = restore.get("RUN_ID", "") +DETAIL=SOURCE_LINE=0238: facts["restore_health_age_seconds"] = str(age_seconds_from_ts(restore.get("TS"))) +DETAIL=SOURCE_LINE=0239: for key in ["CLOUD_1_STATUS", "CLOUD_2_STATUS", "SHA256_OK", "TAR_OK", "REQUIRED_FILES_OK", "COOKIE_JAR_OK", "EXTENSION_HEALTH_FILES_OK", "EXTENSION_SCRIPTS_OK", "EXTENSION_UNITS_OK", "SQLITE_OK", "PY_COMPILE_OK"]: +DETAIL=SOURCE_LINE=0240: facts[key.lower()] = restore.get(key, "") +DETAIL=SOURCE_LINE=0241: if restore.get("STATUS") != "OK": +DETAIL=SOURCE_LINE=0242: problems.append("restore_status_not_ok") +DETAIL=SOURCE_LINE=0243: if int(facts["restore_health_age_seconds"]) > RESTORE_HEALTH_AGE_MAX: +DETAIL=SOURCE_LINE=0244: problems.append("restore_health_stale") +DETAIL=SOURCE_LINE=0245: for key in ["CLOUD_1_STATUS", "CLOUD_2_STATUS"]: +DETAIL=SOURCE_LINE=0246: if restore.get(key) != "OK": +DETAIL=SOURCE_LINE=0247: problems.append(key.lower() + "_bad") +DETAIL=SOURCE_LINE=0248: for key in ["SHA256_OK", "TAR_OK", "REQUIRED_FILES_OK", "COOKIE_JAR_OK", "EXTENSION_HEALTH_FILES_OK", "EXTENSION_SCRIPTS_OK", "EXTENSION_UNITS_OK", "SQLITE_OK", "PY_COMPILE_OK"]: +DETAIL=SOURCE_LINE=0249: if restore.get(key) != "yes": +DETAIL=SOURCE_LINE=0250: problems.append(key.lower() + "_bad") +DETAIL=SOURCE_LINE=0251: +DETAIL=SOURCE_LINE=0252: local_units = [ +DETAIL=SOURCE_LINE=0253: "skladchik-reports-monitor-backup.timer", +DETAIL=SOURCE_LINE=0254: "skladchik-reports-monitor-backup-restore-check.timer", +DETAIL=SOURCE_LINE=0255: "skladchik-reports-monitor-supervisor.timer", +DETAIL=SOURCE_LINE=0256: "skladchik-reports-monitor-monthly-selftest-watch.timer", +DETAIL=SOURCE_LINE=0257: ] +DETAIL=SOURCE_LINE=0258: props = systemctl_props(local_units) +DETAIL=SOURCE_LINE=0259: for u, p in props.items(): +DETAIL=SOURCE_LINE=0260: base = u.replace(".timer", "").replace("-", "_") +DETAIL=SOURCE_LINE=0261: facts[base + "_timer_enabled"] = p["enabled"] +DETAIL=SOURCE_LINE=0262: facts[base + "_timer_active"] = p["active"] +DETAIL=SOURCE_LINE=0263: if u != "skladchik-reports-monitor-supervisor.timer": +DETAIL=SOURCE_LINE=0264: if p["enabled"] != "enabled" or p["active"] != "active": +DETAIL=SOURCE_LINE=0265: problems.append(base + "_timer_bad") +DETAIL=SOURCE_LINE=0266: else: +DETAIL=SOURCE_LINE=0267: # Before first enable it can be disabled/inactive during test. After enable next runs will see enabled/active. +DETAIL=SOURCE_LINE=0268: if p["enabled"] not in ("enabled", "disabled") or p["active"] not in ("active", "inactive"): +DETAIL=SOURCE_LINE=0269: problems.append(base + "_timer_unknown") +DETAIL=SOURCE_LINE=0270: +DETAIL=SOURCE_LINE=0271: return facts, problems +DETAIL=SOURCE_LINE=0272: +DETAIL=SOURCE_LINE=0273:def render_body(status, facts, problems, mode): +DETAIL=SOURCE_LINE=0274: ru_status = "OK" if status == "OK" else "АВАРИЯ" +DETAIL=SOURCE_LINE=0275: lines = [ +DETAIL=SOURCE_LINE=0276: f"Статус надзора: {ru_status}", +DETAIL=SOURCE_LINE=0277: f"Время UTC: {iso()}", +DETAIL=SOURCE_LINE=0278: "", +DETAIL=SOURCE_LINE=0279: "Монитор жалоб:", +DETAIL=SOURCE_LINE=0280: f"- статус: {facts.get('monitor_status')}", +DETAIL=SOURCE_LINE=0281: f"- всего жалоб на странице: {facts.get('monitor_reports')}", +DETAIL=SOURCE_LINE=0282: f"- открытые/назначенные: {facts.get('monitor_open')}", +DETAIL=SOURCE_LINE=0283: f"- закрытые: {facts.get('monitor_resolved')}", +DETAIL=SOURCE_LINE=0284: f"- возраст health, сек: {facts.get('monitor_health_age_seconds')}", +DETAIL=SOURCE_LINE=0285: f"- автопродление авторизации: {facts.get('session_keepalive')}", +DETAIL=SOURCE_LINE=0286: f"- cookie jar: {facts.get('cookie_jar')} / {facts.get('cookie_jar_bytes')} байт", +DETAIL=SOURCE_LINE=0287: f"- постоянная cookie действует до UTC: {facts.get('cookie_expiry_utc') or 'не определено'}", +DETAIL=SOURCE_LINE=0288: f"- осталось дней по cookie: +DETAIL=SOURCE_LINE=0289: f"- статус срока cookie: +DETAIL=SOURCE_LINE=0290: f"- NetBird route: {facts.get('netbird_route')}", +DETAIL=SOURCE_LINE=0291: "", +DETAIL=SOURCE_LINE=0292: "Бэкап:", +DETAIL=SOURCE_LINE=0293: f"- статус: {facts.get('backup_status')}", +DETAIL=SOURCE_LINE=0294: f"- последний RUN_ID: {facts.get('backup_run_id')}", +DETAIL=SOURCE_LINE=0295: f"- размер архива, байт: {facts.get('backup_size_bytes')}", +DETAIL=SOURCE_LINE=0296: f"- cookie jar в бэкапе: {facts.get('backup_cookie_jar_backed_up')}", +DETAIL=SOURCE_LINE=0297: f"- health-файлы расширений в бэкапе: {facts.get('backup_extension_health_files_backed_up')}", +DETAIL=SOURCE_LINE=0298: f"- название жалобы в теме письма: {facts.get('backup_title_in_email_subject')}", +DETAIL=SOURCE_LINE=0299: f"- возраст health, сек: {facts.get('backup_health_age_seconds')}", +DETAIL=SOURCE_LINE=0300: "", +DETAIL=SOURCE_LINE=0301: "Restore-check:", +DETAIL=SOURCE_LINE=0302: f"- статус: {facts.get('restore_status')}", +DETAIL=SOURCE_LINE=0303: f"- последний RUN_ID: {facts.get('restore_run_id')}", +DETAIL=SOURCE_LINE=0304: f"- cloud1/cloud2: {facts.get('cloud_1_status')}/{facts.get('cloud_2_status')}", +DETAIL=SOURCE_LINE=0305: f"- sha256/tar/cookie/sqlite/python: {facts.get('sha256_ok')}/{facts.get('tar_ok')}/{facts.get('cookie_jar_ok')}/{facts.get('sqlite_ok')}/{facts.get('py_compile_ok')}", +DETAIL=SOURCE_LINE=0306: f"- health/scripts/units расширений: {facts.get('extension_health_files_ok')}/{facts.get('extension_scripts_ok')}/{facts.get('extension_units_ok')}", +DETAIL=SOURCE_LINE=0307: "", +DETAIL=SOURCE_LINE=0308: "Проблемы:", +DETAIL=SOURCE_LINE=0309: ] +DETAIL=SOURCE_LINE=0310: if problems: +DETAIL=SOURCE_LINE=0311: lines.extend([f"- {p}" for p in problems]) +DETAIL=SOURCE_LINE=0312: else: +DETAIL=SOURCE_LINE=0313: lines.append("- нет") +DETAIL=SOURCE_LINE=0314: lines.extend([ +DETAIL=SOURCE_LINE=0315: "", +DETAIL=SOURCE_LINE=0316: "Авторизация:", +DETAIL=SOURCE_LINE=0317: "- ручной ежемесячный перелогин обычно не нужен; монитор сам продлевает сессию постоянным cookie jar при каждом опросе;", +DETAIL=SOURCE_LINE=0318: "- supervisor теперь отдельно следит за сроком постоянной cookie и заранее предупредит, если до истечения останется меньше 7 дней;", +DETAIL=SOURCE_LINE=0319: "- если форум принудительно сбросит сессию, придёт отдельная тревога AUTH_REQUIRED; тогда нужно будет разово обновить вход, но это аварийный сценарий, а не ежемесячная рутина.", +DETAIL=SOURCE_LINE=0320: ]) +DETAIL=SOURCE_LINE=0321: return "\n".join(lines) +DETAIL=SOURCE_LINE=0322: +DETAIL=SOURCE_LINE=0323:def write_health(status, facts, problems, mail_sent): +DETAIL=SOURCE_LINE=0324: STATE_DIR.mkdir(parents=True, exist_ok=True) +DETAIL=SOURCE_LINE=0325: lines = [f"STATUS={status}", f"TS={iso()}", "problems=" + ",".join(problems), f"mail_sent={'yes' if mail_sent else 'no'}"] +DETAIL=SOURCE_LINE=0326: for k in sorted(facts): +DETAIL=SOURCE_LINE=0327: v = str(facts[k]).replace("\n", " ")[:500] +DETAIL=SOURCE_LINE=0328: lines.append(f"{k}={v}") +DETAIL=SOURCE_LINE=0329: HEALTH.write_text("\n".join(lines) + "\n", encoding="utf-8") +DETAIL=SOURCE_LINE=0330: +DETAIL=SOURCE_LINE=0331:def read_state(): +DETAIL=SOURCE_LINE=0332: return read_env_file(STATE) +DETAIL=SOURCE_LINE=0333: +DETAIL=SOURCE_LINE=0334:def write_state(status, problems): +DETAIL=SOURCE_LINE=0335: STATE_DIR.mkdir(parents=True, exist_ok=True) +DETAIL=SOURCE_LINE=0336: STATE.write_text(f"LAST_STATUS={status}\nLAST_PROBLEMS={','.join(problems)}\nTS={iso()}\n", encoding="utf-8") +DETAIL=SOURCE_LINE=0337: +DETAIL=SOURCE_LINE=0338:def main(): +DETAIL=SOURCE_LINE=0339: mode = sys.argv[1] if len(sys.argv) > 1 else "check" +DETAIL=SOURCE_LINE=0340: if mode not in ("check", "test"): +DETAIL=SOURCE_LINE=0341: print("usage: skladchik-reports-monitor-supervisor [check|test]", file=sys.stderr) +DETAIL=SOURCE_LINE=0342: return 2 +DETAIL=SOURCE_LINE=0343: +DETAIL=SOURCE_LINE=0344: facts, problems = gather() +DETAIL=SOURCE_LINE=0345: status = "OK" if not problems else "FAIL" +DETAIL=SOURCE_LINE=0346: old = read_state() +DETAIL=SOURCE_LINE=0347: old_status = old.get("LAST_STATUS", "") +DETAIL=SOURCE_LINE=0348: mail_sent = False +DETAIL=SOURCE_LINE=0349: +DETAIL=SOURCE_LINE=0350: if mode == "test": +DETAIL=SOURCE_LINE=0351: mail_sent = edge_mail("Тест: надзор монитора жалоб и бэкапов", render_body(status, facts, problems, mode)) +DETAIL=SOURCE_LINE=0352: if not mail_sent: +DETAIL=SOURCE_LINE=0353: problems.append("supervisor_mail_send_failed") +DETAIL=SOURCE_LINE=0354: status = "FAIL" +DETAIL=SOURCE_LINE=0355: elif status != "OK": +DETAIL=SOURCE_LINE=0356: if old_status != "FAIL": +DETAIL=SOURCE_LINE=0357: mail_sent = edge_mail("Авария: монитор жалоб или его бэкапы", render_body(status, facts, problems, mode)) +DETAIL=SOURCE_LINE=0358: if not mail_sent: +DETAIL=SOURCE_LINE=0359: problems.append("supervisor_mail_send_failed") +DETAIL=SOURCE_LINE=0360: status = "FAIL" +DETAIL=SOURCE_LINE=0361: elif old_status == "FAIL": +DETAIL=SOURCE_LINE=0362: mail_sent = edge_mail("Восстановлено: монитор жалоб и бэкапы снова OK", render_body(status, facts, problems, mode)) +DETAIL=SOURCE_LINE=0363: +DETAIL=SOURCE_LINE=0364: write_health(status, facts, problems, mail_sent) +DETAIL=SOURCE_LINE=0365: write_state(status, problems) +DETAIL=SOURCE_LINE=0366: +DETAIL=SOURCE_LINE=0367: print(f"SUPERVISOR_STATUS={status}") +DETAIL=SOURCE_LINE=0368: print(f"SUPERVISOR_MAIL_SENT={'yes' if mail_sent else 'no'}") +DETAIL=SOURCE_LINE=0369: print(f"SUPERVISOR_PROBLEMS={','.join(problems)}") +DETAIL=SOURCE_LINE=0370: print(f"MONITOR={facts.get('monitor_status')} reports={facts.get('monitor_reports')} open={facts.get('monitor_open')} resolved={facts.get('monitor_resolved')}") +DETAIL=SOURCE_LINE=0371: print(f"BACKUP={facts.get('backup_status')} run={facts.get('backup_run_id')}") +DETAIL=SOURCE_LINE=0372: print(f"RESTORE={facts.get('restore_status')} run={facts.get('restore_run_id')}") +DETAIL=SOURCE_LINE=0373: log(f"CHECK mode={mode} status={status} mail_sent={'yes' if mail_sent else 'no'} problems={problems}") +DETAIL=SOURCE_LINE=0374: return 0 if status == "OK" else 1 +DETAIL=SOURCE_LINE=0375: +DETAIL=SOURCE_LINE=0376:if __name__ == "__main__": +DETAIL=SOURCE_LINE=0377: sys.exit(main()) +DETAIL=SCRIPT_TOTAL_LINES=377 +DETAIL=RELEVANT_SOURCE_END +DETAIL=UNIT_CONTRACT_BEGIN +DETAIL=UNIT_NAME=skladchik-reports-monitor-supervisor.service +DETAIL=UNIT_LINE=# /etc/systemd/system/skladchik-reports-monitor-supervisor.service +DETAIL=UNIT_LINE=[Unit] +DETAIL=UNIT_LINE=Description=Skladchik reports monitor full supervisor +DETAIL=UNIT_LINE=After=network-online.target +DETAIL=UNIT_LINE= +DETAIL=UNIT_LINE=[Service] +DETAIL=UNIT_LINE=Type=oneshot +DETAIL=UNIT_LINE=ExecStart=/usr/local/sbin/skladchik-reports-monitor-supervisor check +DETAIL=UNIT_NAME=skladchik-reports-monitor-supervisor.timer +DETAIL=UNIT_LINE=# /etc/systemd/system/skladchik-reports-monitor-supervisor.timer +DETAIL=UNIT_LINE=[Unit] +DETAIL=UNIT_LINE=Description=Run Skladchik reports monitor full supervisor every 10 minutes +DETAIL=UNIT_LINE= +DETAIL=UNIT_LINE=[Timer] +DETAIL=UNIT_LINE=OnBootSec=5min +DETAIL=UNIT_LINE=OnUnitActiveSec=10min +DETAIL=UNIT_LINE=AccuracySec=30s +DETAIL=UNIT_LINE=Persistent=false +DETAIL=UNIT_LINE=Unit=skladchik-reports-monitor-supervisor.service +DETAIL=UNIT_LINE= +DETAIL=UNIT_LINE=[Install] +DETAIL=UNIT_LINE=WantedBy=timers.target +DETAIL=UNIT_CONTRACT_END +DETAIL=TIMER_STATE_BEGIN +DETAIL=TIMER_PROPERTY=Unit=skladchik-reports-monitor-supervisor.service +DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime= +DETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 11:58:38 MSK +DETAIL=TIMER_PROPERTY=LoadState=loaded +DETAIL=TIMER_PROPERTY=ActiveState=active +DETAIL=TIMER_PROPERTY=SubState=waiting +DETAIL=TIMER_ROW=Fri 2026-07-24 12:08:38 MSK 3min 34s Fri 2026-07-24 11:58:38 MSK 6min ago skladchik-reports-monitor-supervisor.timer skladchik-reports-monitor-supervisor.service +DETAIL=TIMER_ROW=Sat 2026-07-25 04:39:05 MSK 16h Fri 2026-07-24 04:28:08 MSK 7h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service +DETAIL=TIMER_ROW=Sat 2026-07-25 10:15:00 MSK 22h Fri 2026-07-24 10:15:04 MSK 1h 49min ago skladchik-reports-monitor-monthly-selftest-watch.timer skladchik-reports-monitor-monthly-selftest-watch.service +DETAIL=TIMER_ROW=Sun 2026-07-26 05:15:13 MSK 1 day 17h Sun 2026-07-19 05:11:25 MSK - skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service +DETAIL=TIMER_STATE_END +DETAIL=SECRET_VALUES_OUTPUT=NONE +DETAIL=CHANGES_MADE=NO +DETAIL=COOKIE_CHECK_SOURCE_AUDIT=PASS + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 4cc863d..c5b7657 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "HOMELAB-SKLADCHIK-SUPERVISOR-FAILURE-TRACE-417", + "command_id": "HOMELAB-SKLADCHIK-COOKIE-CHECK-SOURCE-AUDIT-418", "status": "OK", "rc": 0, "host": "pve01", "mode": "read-only", "component": "cluster-audit", - "started_at_utc": "2026-07-24T09:02:24Z", - "finished_at_utc": "2026-07-24T09:02:25Z", + "started_at_utc": "2026-07-24T09:05:03Z", + "finished_at_utc": "2026-07-24T09:05:03Z", "reference_register_checked": true, "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", "error_register_checked": true, "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", - "command_sha256": "9f6e05b9df78cf255d0b700cf196693048a5d3f95183c3f4aa1fd2e48188ab45", + "command_sha256": "77896618b46e2f97362240deb743fb8e625075381e57377ee0314130be2b2026", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -22,9 +22,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "4a1b1b699e9b8b13e867485522410253fe5bf1a821101e64dcf14e5186201798", - "sanitized_output_sha256": "4a1b1b699e9b8b13e867485522410253fe5bf1a821101e64dcf14e5186201798", + "raw_evidence_sha256": "1c26f0b751c5e55421c706f56f4be25d92c17849df8a385fb66ce01144801022", + "sanitized_output_sha256": "8e5a54db35050a5c0010241fb0bad91fc5966530dbac76bb07426b09af51a6a1", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "DETAIL=SUPERVISOR_STATE_BEGIN\nDETAIL=SYSTEMD_PROPERTY=Result=exit-code\nDETAIL=SYSTEMD_PROPERTY=ExecMainCode=1\nDETAIL=SYSTEMD_PROPERTY=ExecMainStatus=1\nDETAIL=SYSTEMD_PROPERTY=WorkingDirectory=\nDETAIL=SYSTEMD_PROPERTY=User=\nDETAIL=SYSTEMD_PROPERTY=Group=\nDETAIL=SYSTEMD_PROPERTY=LoadState=loaded\nDETAIL=SYSTEMD_PROPERTY=ActiveState=failed\nDETAIL=SYSTEMD_PROPERTY=SubState=failed\nDETAIL=SYSTEMD_PROPERTY=FragmentPath=/etc/systemd/system/skladchik-reports-monitor-supervisor.service\nDETAIL=SYSTEMD_PROPERTY=DropInPaths=\nDETAIL=SUPERVISOR_STATE_END\nDETAIL=SUPERVISOR_UNIT_CONTRACT_BEGIN\nDETAIL=UNIT_CONTRACT=# /etc/systemd/system/skladchik-reports-monitor-supervisor.service\nDETAIL=UNIT_CONTRACT=[Unit]\nDETAIL=UNIT_CONTRACT=After=network-online.target\nDETAIL=UNIT_CONTRACT=[Service]\nDETAIL=UNIT_CONTRACT=ExecStart=/usr/local/sbin/skladchik-reports-monitor-supervisor check\nDETAIL=SUPERVISOR_UNIT_CONTRACT_END\nDETAIL=SUPERVISOR_LAST_FAILURE_BEGIN\nDETAIL=JOURNAL=2026-07-24T06:38:28+03:00 pve01 skladchik-reports-monitor-supervisor[2914895]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T06:38:28+03:00 pve01 skladchik-reports-monitor-supervisor[2914895]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T06:48:29+03:00 pve01 skladchik-reports-monitor-supervisor[2943586]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T06:48:29+03:00 pve01 skladchik-reports-monitor-supervisor[2943586]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T06:58:29+03:00 pve01 skladchik-reports-monitor-supervisor[2971845]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T06:58:29+03:00 pve01 skladchik-reports-monitor-supervisor[2971845]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T07:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3008729]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T07:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3008729]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T07:18:28+03:00 pve01 skladchik-reports-monitor-supervisor[3048052]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T07:18:28+03:00 pve01 skladchik-reports-monitor-supervisor[3048052]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T07:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3075804]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T07:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3075804]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T07:38:28+03:00 pve01 skladchik-reports-monitor-supervisor[3111924]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T07:38:28+03:00 pve01 skladchik-reports-monitor-supervisor[3111924]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T07:48:29+03:00 pve01 skladchik-reports-monitor-supervisor[3148073]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T07:48:29+03:00 pve01 skladchik-reports-monitor-supervisor[3148073]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T07:58:28+03:00 pve01 skladchik-reports-monitor-supervisor[3175426]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T07:58:28+03:00 pve01 skladchik-reports-monitor-supervisor[3175426]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T08:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3205006]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T08:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3205006]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T08:18:29+03:00 pve01 skladchik-reports-monitor-supervisor[3241119]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T08:18:29+03:00 pve01 skladchik-reports-monitor-supervisor[3241119]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T08:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3269919]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T08:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3269919]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T08:38:29+03:00 pve01 skladchik-reports-monitor-supervisor[3306255]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T08:38:29+03:00 pve01 skladchik-reports-monitor-supervisor[3306255]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T08:48:29+03:00 pve01 skladchik-reports-monitor-supervisor[3342134]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T08:48:29+03:00 pve01 skladchik-reports-monitor-supervisor[3342134]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T08:58:29+03:00 pve01 skladchik-reports-monitor-supervisor[3369587]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T08:58:29+03:00 pve01 skladchik-reports-monitor-supervisor[3369587]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T09:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3405539]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T09:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3405539]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T09:18:29+03:00 pve01 skladchik-reports-monitor-supervisor[3430044]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T09:18:29+03:00 pve01 skladchik-reports-monitor-supervisor[3430044]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T09:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3457493]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T09:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3457493]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T09:38:29+03:00 pve01 skladchik-reports-monitor-supervisor[3493130]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T09:38:29+03:00 pve01 skladchik-reports-monitor-supervisor[3493130]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T09:48:29+03:00 pve01 skladchik-reports-monitor-supervisor[3528983]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T09:48:29+03:00 pve01 skladchik-reports-monitor-supervisor[3528983]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T09:58:29+03:00 pve01 skladchik-reports-monitor-supervisor[3556369]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T09:58:29+03:00 pve01 skladchik-reports-monitor-supervisor[3556369]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T10:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3594595]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T10:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3594595]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T10:18:29+03:00 pve01 skladchik-reports-monitor-supervisor[3633670]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T10:18:29+03:00 pve01 skladchik-reports-monitor-supervisor[3633670]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T10:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3652898]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T10:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3652898]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T10:38:31+03:00 pve01 skladchik-reports-monitor-supervisor[3691967]: SUPERVISOR_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T10:38:31+03:00 pve01 skladchik-reports-monitor-supervisor[3691967]: SUPERVISOR_PROBLEMS=edge_cookies_jar_bad\nDETAIL=JOURNAL=2026-07-24T10:38:31+03:00 pve01 systemd[1]: skladchik-reports-monitor-supervisor.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T10:38:31+03:00 pve01 systemd[1]: skladchik-reports-monitor-supervisor.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T10:38:31+03:00 pve01 systemd[1]: Failed to start skladchik-reports-monitor-supervisor.service - Skladchik reports monitor full supervisor.\nDETAIL=JOURNAL=2026-07-24T10:48:31+03:00 pve01 skladchik-reports-monitor-supervisor[3730900]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T10:48:31+03:00 pve01 skladchik-reports-monitor-supervisor[3730900]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T10:58:29+03:00 pve01 skladchik-reports-monitor-supervisor[3761552]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T10:58:29+03:00 pve01 skladchik-reports-monitor-supervisor[3761552]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T11:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3800692]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T11:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3800692]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T11:18:29+03:00 pve01 skladchik-reports-monitor-supervisor[3839724]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T11:18:29+03:00 pve01 skladchik-reports-monitor-supervisor[3839724]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T11:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3858965]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T11:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3858965]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T11:38:29+03:00 pve01 skladchik-reports-monitor-supervisor[3897854]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T11:38:29+03:00 pve01 skladchik-reports-monitor-supervisor[3897854]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T11:48:35+03:00 pve01 skladchik-reports-monitor-supervisor[3937127]: SUPERVISOR_STATUS=OK\nDETAIL=JOURNAL=2026-07-24T11:48:35+03:00 pve01 skladchik-reports-monitor-supervisor[3937127]: SUPERVISOR_PROBLEMS=\nDETAIL=JOURNAL=2026-07-24T11:58:40+03:00 pve01 skladchik-reports-monitor-supervisor[3967718]: SUPERVISOR_STATUS=FAIL\nDETAIL=JOURNAL=2026-07-24T11:58:40+03:00 pve01 skladchik-reports-monitor-supervisor[3967718]: SUPERVISOR_PROBLEMS=edge_cookies_jar_bad\nDETAIL=JOURNAL=2026-07-24T11:58:40+03:00 pve01 systemd[1]: skladchik-reports-monitor-supervisor.service: Main process exited, code=exited, status=1/FAILURE\nDETAIL=JOURNAL=2026-07-24T11:58:40+03:00 pve01 systemd[1]: skladchik-reports-monitor-supervisor.service: Failed with result 'exit-code'.\nDETAIL=JOURNAL=2026-07-24T11:58:40+03:00 pve01 systemd[1]: Failed to start skladchik-reports-monitor-supervisor.service - Skladchik reports monitor full supervisor.\nDETAIL=SUPERVISOR_LAST_FAILURE_END\nDETAIL=SECRET_VALUES_OUTPUT=NONE\nDETAIL=SUPERVISOR_FAILURE_TRACE=PASS\nCHANGES_MADE=NO\n" + "output": "DETAIL=SCRIPT_PATH=/usr/local/sbin/skladchik-reports-monitor-supervisor\nDETAIL=SCRIPT_SHA256=35c68ae0929ff3795be2bde40de334fc8e26413a3d8a620a137be90d1f2c164e\nDETAIL=SCRIPT_METADATA=BYTES:20463|MODE:755|OWNER:root:root|MTIME:2026-07-18 13:52:30.415780066 +0300\nDETAIL=RELEVANT_SOURCE_BEGIN\nDETAIL=SOURCE_LINE=0005:import os\nDETAIL=SOURCE_LINE=0006:import re\nDETAIL=SOURCE_LINE=0007:import shlex\nDETAIL=SOURCE_LINE=0008:import subprocess\nDETAIL=SOURCE_LINE=0009:import sys\nDETAIL=SOURCE_LINE=0010:from pathlib import Path\nDETAIL=SOURCE_LINE=0011:\nDETAIL=SOURCE_LINE=0012:EDGE = os.environ.get(\"SKLADCHIK_EDGE\", \"debian@[PRIVATE_IP]\")\nDETAIL=SOURCE_LINE=0013:STATE_DIR = Path(\"/var/lib/skladchik-reports-monitor-supervisor\")\nDETAIL=SOURCE_LINE=0014:LOG_DIR = Path(\"/var/log/skladchik-reports-monitor-supervisor\")\nDETAIL=SOURCE_LINE=0015:HEALTH = STATE_DIR / \"health.txt\"\nDETAIL=SOURCE_LINE=0016:STATE = STATE_DIR / \"state.env\"\nDETAIL=SOURCE_LINE=0017:LOG = LOG_DIR / \"supervisor.log\"\nDETAIL=SOURCE_LINE=0018:\nDETAIL=SOURCE_LINE=0019:MONITOR_HEALTH_AGE_MAX = int(os.environ.get(\"SKLADCHIK_MONITOR_HEALTH_MAX_SECONDS\", \"600\"))\nDETAIL=SOURCE_LINE=0020:BACKUP_HEALTH_AGE_MAX = int(os.environ.get(\"SKLADCHIK_BACKUP_HEALTH_MAX_SECONDS\", \"129600\"))\nDETAIL=SOURCE_LINE=0021:RESTORE_HEALTH_AGE_MAX = int(os.environ.get(\"SKLADCHIK_RESTORE_HEALTH_MAX_SECONDS\", \"691200\"))\nDETAIL=SOURCE_LINE=0022:COOKIE_EXPIRY_WARN_SECONDS = int(os.environ.get(\"SKLADCHIK_COOKIE_EXPIRY_WARN_SECONDS\", \"604800\"))\nDETAIL=SOURCE_LINE=0023:\nDETAIL=SOURCE_LINE=0024:PVE_BACKUP_HEALTH = Path(\"/var/lib/skladchik-reports-monitor-backup/health.txt\")\nDETAIL=SOURCE_LINE=0025:PVE_RESTORE_HEALTH = Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\nDETAIL=SOURCE_LINE=0026:\nDETAIL=SOURCE_LINE=0027:def now():\nDETAIL=SOURCE_LINE=0028: return _dt.datetime.now(_dt.UTC).replace(microsecond=0)\nDETAIL=SOURCE_LINE=0029:\nDETAIL=SOURCE_LINE=0030:def iso():\nDETAIL=SOURCE_LINE=0031: return now().isoformat().replace(\"+00:00\", \"Z\")\nDETAIL=SOURCE_LINE=0032:\nDETAIL=SOURCE_LINE=0033:def log(msg):\nDETAIL=SOURCE_LINE=0034: LOG_DIR.mkdir(parents=True, exist_ok=True)\nDETAIL=SOURCE_LINE=0035: with LOG.open(\"a\", encoding=\"utf-8\") as f:\nDETAIL=SOURCE_LINE=0036: f.write(f\"{iso()} {msg}\\n\")\nDETAIL=SOURCE_LINE=0037:\nDETAIL=SOURCE_LINE=0038:def run(cmd, timeout=45, input_text=None):\nDETAIL=SOURCE_LINE=0039: try:\nDETAIL=SOURCE_LINE=0040: p = subprocess.run(cmd, shell=True, text=True, input=input_text, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout)\nDETAIL=SOURCE_LINE=0041: return p.returncode, p.stdout.strip()\nDETAIL=SOURCE_LINE=0042: except subprocess.TimeoutExpired as e:\nDETAIL=SOURCE_LINE=0043: out = e.stdout if isinstance(e.stdout, str) else \"\"\nDETAIL=SOURCE_LINE=0044: return 124, (out or \"\").strip() + \"\\nTIMEOUT\"\nDETAIL=SOURCE_LINE=0045:\nDETAIL=SOURCE_LINE=0046:def parse_env_text(text):\nDETAIL=SOURCE_LINE=0047: d = {}\nDETAIL=SOURCE_LINE=0048: for line in (text or \"\").splitlines():\nDETAIL=SOURCE_LINE=0049: if not line or line.startswith(\"#\") or \"=\" not in line:\nDETAIL=SOURCE_LINE=0050: continue\nDETAIL=SOURCE_LINE=0051: k, v = line.split(\"=\", 1)\nDETAIL=SOURCE_LINE=...\nDETAIL=SOURCE_LINE=0066: dt = _dt.datetime.fromisoformat(s)\nDETAIL=SOURCE_LINE=0067: if dt.tzinfo is None:\nDETAIL=SOURCE_LINE=0068: dt = dt.replace(tzinfo=_dt.UTC)\nDETAIL=SOURCE_LINE=0069: return int((now() - dt.astimezone(_dt.UTC)).total_seconds())\nDETAIL=SOURCE_LINE=0070: except Exception:\nDETAIL=SOURCE_LINE=0071: return 999999999\nDETAIL=SOURCE_LINE=0072:\nDETAIL=SOURCE_LINE=0073:def ssh_edge(cmd, timeout=45):\nDETAIL=SOURCE_LINE=0074: q = \"ssh -o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] \" + shlex.quote(EDGE) + \" \" + shlex.quote(\"sudo bash -lc \" + shlex.quote(cmd))\nDETAIL=SOURCE_LINE=0075: return run(q, timeout=timeout)\nDETAIL=SOURCE_LINE=0076:\nDETAIL=SOURCE_LINE=0077:def edge_mail(subject, body):\nDETAIL=SOURCE_LINE=0078: subj_b64 = base64.b64encode(subject.encode(\"utf-8\")).decode(\"ascii\")\nDETAIL=SOURCE_LINE=0079: body_b64 = base64.b64encode(body.encode(\"utf-8\")).decode(\"ascii\")\nDETAIL=SOURCE_LINE=0080: py = (\nDETAIL=SOURCE_LINE=0081: \"import base64,importlib.util;\"\nDETAIL=SOURCE_LINE=0082: \"spec=importlib.util.spec_from_file_location('m','/opt/skladchik-reports-monitor/monitor.py');\"\nDETAIL=SOURCE_LINE=0083: \"m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m);\"\nDETAIL=SOURCE_LINE=0084: \"cfg,secrets=\nDETAIL=SOURCE_LINE=0085: f\"subject=base64.b64decode('{subj_b64}').decode('utf-8');\"\nDETAIL=SOURCE_LINE=0086: f\"body=base64.b64decode('{body_b64}').decode('utf-8');\"\nDETAIL=SOURCE_LINE=0087: \"ok=m.send_mail(cfg,secrets,subject,body);\"\nDETAIL=SOURCE_LINE=0088: \"print('MAIL_OK' if ok else 'MAIL_FALSE')\"\nDETAIL=SOURCE_LINE=0089: )\nDETAIL=SOURCE_LINE=0090: rc, out = ssh_edge(\"python3 -c \" + shlex.quote(py), timeout=60)\nDETAIL=SOURCE_LINE=0091: log(f\"MAIL subject={subject!r} rc={rc} out={out!r}\")\nDETAIL=SOURCE_LINE=0092: return rc == 0 and \"MAIL_OK\" in out\nDETAIL=SOURCE_LINE=0093:\nDETAIL=SOURCE_LINE=0094:def systemctl_props(units):\nDETAIL=SOURCE_LINE=0095: cmd = \"systemctl is-enabled \" + \" \".join(map(shlex.quote, units)) + \" 2>/dev/null || true; echo __ACTIVE__; systemctl is-active \" + \" \".join(map(shlex.quote, units)) + \" 2>/dev/null || true\"\nDETAIL=SOURCE_LINE=0096: rc, out = run(cmd, timeout=20)\nDETAIL=SOURCE_LINE=0097: parts = out.split(\"__ACTIVE__\", 1)\nDETAIL=SOURCE_LINE=0098: enabled_lines = [x.strip() for x in parts[0].splitlines() if x.strip()]\nDETAIL=SOURCE_LINE=0099: active_lines = [x.strip() for x in parts[1].splitlines() if x.strip()] if len(parts) > 1 else []\nDETAIL=SOURCE_LINE=0100: res = {}\nDETAIL=SOURCE_LINE=0101: for i, u in enumerate(units):\nDETAIL=SOURCE_LINE=0102: res[u] = {\"enabled\": enabled_lines[i] if i < len(enabled_lines) else \"unknown\", \"active\": active_lines[i] if i < len(active_lines) else \"unknown\"}\nDETAIL=SOURCE_LINE=0103: return res\nDETAIL=SOURCE_LINE=0104:\nDETAIL=SOURCE_LINE=0105:def gather():\nDETAIL=SOURCE_LINE=0106: facts = {}\nDETAIL=SOURCE_LINE=0107: problems = []\nDETAIL=SOURCE_LINE=0108:\nDETAIL=SOURCE_LINE=0109: edge_cmd = r'''\nDETAIL=SOURCE_LINE=0110:set +e\nDETAIL=SOURCE_LINE=0111:echo __MONITOR_HEALTH_BEGIN__\nDETAIL=SOURCE_LINE=0112:cat /var/lib/skladchik-reports-monitor/health.txt 2>/dev/null || true\nDETAIL=SOURCE_LINE=0113:echo __MONITOR_HEALTH_END__\nDETAIL=SOURCE_LINE=0114:echo __ALERT_HEALTH_BEGIN__\nDETAIL=SOURCE_LINE=0115:cat /var/lib/skladchik-reports-monitor/alert-health.txt 2>/dev/null || true\nDETAIL=SOURCE_LINE=0116:echo __ALERT_HEALTH_END__\nDETAIL=SOURCE_LINE=0117:echo __TIMERS_BEGIN__\nDETAIL=SOURCE_LINE=0118:for u in skladchik-reports-monitor.timer skladchik-reports-monitor-watchdog.timer skladchik-reports-monitor-heartbeat.timer skladchik-reports-monitor-weekly-summary.timer skladchik-reports-monitor-age-alert.timer skladchik-reports-monitor-event-selftest.timer; do echo \"$u enabled=$(systemctl is-enabled \"$u\" 2>/dev/null) active=$(systemctl is-active \"$u\" 2>/dev/null)\"; done\nDETAIL=SOURCE_LINE=0119:echo __TIMERS_END__\nDETAIL=SOURCE_LINE=0120:echo __ROUTE_BEGIN__\nDETAIL=SOURCE_LINE=0121:ip route get 1.1.1.1 2>&1 | head -n 1\nDETAIL=SOURCE_LINE=0122:echo __ROUTE_END__\nDETAIL=SOURCE_LINE=0123:echo __FILES_BEGIN__\nDETAIL=SOURCE_LINE=0124:test -s /opt/skladchik-reports-monitor/monitor.py && echo monitor_py=yes || echo monitor_py=no\nDETAIL=SOURCE_LINE=0125:test -s /etc/skladchik-reports-monitor/config.ini && echo config_ini=yes || echo config_ini=no\nDETAIL=SOURCE_LINE=0126:test -s /etc/skladchik-reports-monitor/secrets.env && echo secrets_env=\nDETAIL=SOURCE_LINE=0127:test -s /var/lib/skladchik-reports-monitor/cookies.jar && echo cookies_jar=yes || echo cookies_jar=no\nDETAIL=SOURCE_LINE=0128:[ -s /var/lib/skladchik-reports-monitor/cookies.jar ] && echo cookies_jar_bytes=$(stat -c %s /var/lib/skladchik-reports-monitor/cookies.jar 2>/dev/null) || echo cookies_jar_bytes=0\nDETAIL=SOURCE_LINE=0129:test -s /opt/skladchik-reports-monitor/operational_digest.py && echo operational_digest_py=yes || echo operational_digest_py=no\nDETAIL=SOURCE_LINE=0130:test -s /opt/skladchik-reports-monitor/event-selftest-runner.py && echo event_selftest_runner_py=yes || echo event_selftest_runner_py=no\nDETAIL=SOURCE_LINE=0131:test -s /opt/skladchik-reports-monitor/event-selftest-core && echo event_selftest_core=yes || echo event_selftest_core=no\nDETAIL=SOURCE_LINE=0132:test -s /var/lib/skladchik-reports-monitor/weekly-summary-health.txt && echo weekly_summary_health=yes || echo weekly_summary_health=no\nDETAIL=SOURCE_LINE=0133:test -s /var/lib/skladchik-reports-monitor/age-alert-health.txt && echo age_alert_health=yes || echo age_alert_health=no\nDETAIL=SOURCE_LINE=0134:test -s /var/lib/skladchik-reports-monitor/event-selftest-health.txt && echo event_selftest_health=yes || echo event_selftest_health=no\nDETAIL=SOURCE_LINE=0135:python3 -m py_compile /opt/skladchik-reports-monitor/monitor.py /opt/skladchik-reports-monitor/operational_digest.py /opt/skladchik-reports-monitor/event-selftest-runner.py /opt/skladchik-reports-monitor/event-selftest-core >/dev/null 2>&1 && echo py_compile=yes || echo py_compile=no\nDETAIL=SOURCE_LINE=0136:echo __FILES_END__\nDETAIL=SOURCE_LINE=0137:'''\nDETAIL=SOURCE_LINE=0138: rc, out = ssh_edge(edge_cmd, timeout=45)\nDETAIL=SOURCE_LINE=0139: facts[\"edge_ssh_rc\"] = str(rc)\nDETAIL=SOURCE_LINE=0140: if rc != 0:\nDETAIL=SOURCE_LINE=0141: problems.append(\"edge_ssh_failed\")\nDETAIL=SOURCE_LINE=0142:\nDETAIL=SOURCE_LINE=0143: def section(name):\nDETAIL=SOURCE_LINE=0144: m = re.search(rf\"__{name}_BEGIN__\\n(.*?)\\n__{name}_END__\", out, re.S)\nDETAIL=SOURCE_LINE=0145: return m.group(1).strip() if m else \"\"\nDETAIL=SOURCE_LINE=0146:\nDETAIL=SOURCE_LINE=0147: mon = parse_env_text(section(\"MONITOR_HEALTH\"))\nDETAIL=SOURCE_LINE=0148: facts[\"monitor_status\"] = mon.get(\"STATUS\", \"MISSING\")\nDETAIL=SOURCE_LINE=0149: facts[\"monitor_reports\"] = mon.get(\"reports\", \"\")\nDETAIL=SOURCE_LINE=0150: facts[\"monitor_open\"] = mon.get(\"open\", \"\")\nDETAIL=SOURCE_LINE=0151: facts[\"monitor_resolved\"] = mon.get(\"resolved\", \"\")\nDETAIL=SOURCE_LINE=0152: facts[\"monitor_smtp_configured\"] = mon.get(\"smtp_configured\", \"\")\nDETAIL=SOURCE_LINE=0153: facts[\"session_keepalive\"] = mon.get(\"session_keepalive\", \"\")\nDETAIL=SOURCE_LINE=0154: facts[\"cookie_jar\"] = mon.get(\"cookie_jar\", \"\")\nDETAIL=SOURCE_LINE=0155: facts[\"cookie_jar_bytes\"] = mon.get(\"cookie_jar_bytes\", \"\")\nDETAIL=SOURCE_LINE=0156: facts[\"cookie_expiry_status\"] = mon.get(\"cookie_expiry_status\", \"\")\nDETAIL=SOURCE_LINE=0157: facts[\"cookie_expiry_utc\"] = mon.get(\"cookie_expiry_utc\", \"\")\nDETAIL=SOURCE_LINE=0158: facts[\"cookie_expires_in_seconds\"] = mon.get(\"cookie_expires_in_seconds\", \"\")\nDETAIL=SOURCE_LINE=0159: facts[\"cookie_expires_in_days\"] = mon.get(\"cookie_expires_in_days\", \"\")\nDETAIL=SOURCE_LINE=0160: facts[\"cookie_persistent_count\"] = mon.get(\"cookie_persistent_count\", \"\")\nDETAIL=SOURCE_LINE=0161: facts[\"monitor_health_age_seconds\"] = str(age_seconds_from_ts(mon.get(\"TS\")))\nDETAIL=SOURCE_LINE=0162: if mon.get(\"STATUS\") != \"OK\":\nDETAIL=SOURCE_LINE=0163: problems.append(\"monitor_status_not_ok\")\nDETAIL=SOURCE_LINE=0164: if int(facts[\"monitor_health_age_seconds\"]) > MONITOR_HEALTH_AGE_MAX:\nDETAIL=SOURCE_LINE=0165: problems.append(\"monitor_health_stale\")\nDETAIL=SOURCE_LINE=0166: if mon.get(\"reports\") in (\"\", \"0\"):\nDETAIL=SOURCE_LINE=0167: problems.append(\"monitor_reports_zero_or_missing\")\nDETAIL=SOURCE_LINE=0168: if mon.get(\"session_keepalive\") != \"persistent_cookie_jar\":\nDETAIL=SOURCE_LINE=0169: problems.append(\"session_keepalive_not_persistent\")\nDETAIL=SOURCE_LINE=0170: if mon.get(\"cookie_jar\") not in (\"yes\", \"empty\"):\nDETAIL=SOURCE_LINE=0171: problems.append(\"monitor_cookie_jar_bad\")\nDETAIL=SOURCE_LINE=0172: expiry_status = mon.get(\"cookie_expiry_status\", \"\")\nDETAIL=SOURCE_LINE=0173: try:\nDETAIL=SOURCE_LINE=0174: expires_in = int(mon.get(\"cookie_expires_in_seconds\", \"\"))\nDETAIL=SOURCE_LINE=0175: except Exception:\nDETAIL=SOURCE_LINE=0176: expires_in = None\nDETAIL=SOURCE_LINE=0177: if expiry_status in (\"\", \"NO_COOKIE_JAR\", \"NO_PERSISTENT_COOKIE\", \"ERROR\"):\nDETAIL=SOURCE_LINE=0178: problems.append(\"cookie_expiry_missing\")\nDETAIL=SOURCE_LINE=0179: elif expiry_status == \"EXPIRED\" or (expires_in is not None and expires_in <= 0):\nDETAIL=SOURCE_LINE=0180: problems.append(\"cookie_expiry_expired\")\nDETAIL=SOURCE_LINE=0181: elif expiry_status == \"WARN\" or (expires_in is not None and expires_in < COOKIE_EXPIRY_WARN_SECONDS):\nDETAIL=SOURCE_LINE=0182: problems.append(\"cookie_expiry_warn\")\nDETAIL=SOURCE_LINE=0183:\nDETAIL=SOURCE_LINE=0184: timers_text = section(\"TIMERS\")\nDETAIL=SOURCE_LINE=0185: for unit in [\"skladchik-reports-monitor.timer\", \"skladchik-reports-monitor-watchdog.timer\", \"skladchik-reports-monitor-heartbeat.timer\", \"skladchik-reports-monitor-weekly-summary.timer\", \"skladchik-reports-monitor-age-alert.timer\", \"skladchik-reports-monitor-event-selftest.timer\"]:\nDETAIL=SOURCE_LINE=0186: safe = unit.replace(\".timer\", \"\").replace(\"_\", \"-\")\nDETAIL=SOURCE_LINE=0187: m = re.search(re.escape(unit) + r\"\\s+enabled=([^\\s]+)\\s+active=([^\\s]+)\", timers_text)\nDETAIL=SOURCE_LINE=0188: enabled = m.group(1) if m else \"missing\"\nDETAIL=SOURCE_LINE=0189: active = m.group(2) if m else \"missing\"\nDETAIL=SOURCE_LINE=0190: facts[f\"edge_{safe}_timer_enabled\"] = enabled\nDETAIL=SOURCE_LINE=0191: facts[f\"edge_{safe}_timer_active\"] = active\nDETAIL=SOURCE_LINE=0192: if enabled != \"enabled\" or active != \"active\":\nDETAIL=SOURCE_LINE=0193: problems.append(f\"edge_{safe}_timer_bad\")\nDETAIL=SOURCE_LINE=0194:\nDETAIL=SOURCE_LINE=0195: route_lines = section(\"ROUTE\").splitlines()\nDETAIL=SOURCE_LINE=0196: route = route_lines[0] if route_lines else \"\"\nDETAIL=SOURCE_LINE=0197: facts[\"netbird_route\"] = route\nDETAIL=SOURCE_LINE=0198: if \" wt0 \" not in (\" \" + route + \" \"):\nDETAIL=SOURCE_LINE=0199: problems.append(\"netbird_route_not_wt0\")\nDETAIL=SOURCE_LINE=0200:\nDETAIL=SOURCE_LINE=0201: files = parse_env_text(section(\"FILES\"))\nDETAIL=SOURCE_LINE=0202: for k in [\"monitor_py\", \"config_ini\", \"secrets_env\", \"cookies_jar\", \"operational_digest_py\", \"event_selftest_runner_py\", \"event_selftest_core\", \"weekly_summary_health\", \"age_alert_health\", \"event_selftest_health\", \"py_compile\"]:\nDETAIL=SOURCE_LINE=0203: facts[k] = files.get(k, \"no\")\nDETAIL=SOURCE_LINE=0204: if facts[k] != \"yes\":\nDETAIL=SOURCE_LINE=0205: problems.append(f\"edge_{k}_bad\")\nDETAIL=SOURCE_LINE=0206: facts[\"cookies_jar_bytes_file\"] = files.get(\"cookies_jar_bytes\", \"0\")\nDETAIL=SOURCE_LINE=0207:\nDETAIL=SOURCE_LINE=0208: backup = read_env_file(PVE_BACKUP_HEALTH)\nDETAIL=SOURCE_LINE=0209: facts[\"backup_status\"] = backup.get(\"STATUS\", \"MISSING\")\nDETAIL=SOURCE_LINE=0210: facts[\"backup_run_id\"] = backup.get(\"RUN_ID\", \"\")\nDETAIL=SOURCE_LINE=0211: facts[\"backup_size_bytes\"] = backup.get(\"SIZE_BYTES\", \"\")\nDETAIL=SOURCE_LINE=0212: facts[\"backup_version\"] = backup.get(\"BACKUP_VERSION\", \"\")\nDETAIL=SOURCE_LINE=0213: facts[\"backup_cookie_jar_backed_up\"] = backup.get(\"COOKIE_JAR_BACKED_UP\", \"\")\nDETAIL=SOURCE_LINE=0214: facts[\"backup_extension_health_files_backed_up\"] = backup.get(\"EXTENSION_HEALTH_FILES_BACKED_UP\", \"\")\nDETAIL=SOURCE_LINE=0215: facts[\"backup_title_in_email_subject\"] = backup.get(\"TITLE_IN_EMAIL_SUBJECT\", \"\")\nDETAIL=SOURCE_LINE=0216: facts[\"backup_health_age_seconds\"] = str(age_seconds_from_ts(backup.get(\"TS\")))\nDETAIL=SOURCE_LINE=0217: if backup.get(\"STATUS\") != \"OK\":\nDETAIL=SOURCE_LINE=0218: problems.append(\"backup_status_not_ok\")\nDETAIL=SOURCE_LINE=0219: if backup.get(\"COOKIE_JAR_BACKED_UP\") != \"yes\":\nDETAIL=SOURCE_LINE=0220: problems.append(\"backup_cookie_jar_not_marked\")\nDETAIL=SOURCE_LINE=0221: if backup.get(\"BACKUP_VERSION\") != \"4\":\nDETAIL=SOURCE_LINE=0222: problems.append(\"backup_version_not_4\")\nDETAIL=SOURCE_LINE=0223: if backup.get(\"EXTENSION_HEALTH_FILES_BACKED_UP\") != \"yes\":\nDETAIL=SOURCE_LINE=0224: problems.append(\"backup_extension_health_files_not_marked\")\nDETAIL=SOURCE_LINE=0225: if backup.get(\"TITLE_IN_EMAIL_SUBJECT\") != \"yes\":\nDETAIL=SOURCE_LINE=0226: problems.append(\"email_title_subject_not_marked\")\nDETAIL=SOURCE_LINE=0227: if int(facts[\"backup_health_age_seconds\"]) > BACKUP_HEALTH_AGE_MAX:\nDETAIL=SOURCE_LINE=0228: problems.append(\"backup_health_stale\")\nDETAIL=SOURCE_LINE=0229: try:\nDETAIL=SOURCE_LINE=0230: if int(backup.get(\"SIZE_BYTES\", \"0\")) <= 0:\nDETAIL=SOURCE_LINE=0231: problems.append(\"backup_size_bad\")\nDETAIL=SOURCE_LINE=0232: except Exception:\nDETAIL=SOURCE_LINE=0233: problems.append(\"backup_size_bad\")\nDETAIL=SOURCE_LINE=0234:\nDETAIL=SOURCE_LINE=0235: restore = read_env_file(PVE_RESTORE_HEALTH)\nDETAIL=SOURCE_LINE=0236: facts[\"restore_status\"] = restore.get(\"STATUS\", \"MISSING\")\nDETAIL=SOURCE_LINE=0237: facts[\"restore_run_id\"] = restore.get(\"RUN_ID\", \"\")\nDETAIL=SOURCE_LINE=0238: facts[\"restore_health_age_seconds\"] = str(age_seconds_from_ts(restore.get(\"TS\")))\nDETAIL=SOURCE_LINE=0239: for key in [\"CLOUD_1_STATUS\", \"CLOUD_2_STATUS\", \"SHA256_OK\", \"TAR_OK\", \"REQUIRED_FILES_OK\", \"COOKIE_JAR_OK\", \"EXTENSION_HEALTH_FILES_OK\", \"EXTENSION_SCRIPTS_OK\", \"EXTENSION_UNITS_OK\", \"SQLITE_OK\", \"PY_COMPILE_OK\"]:\nDETAIL=SOURCE_LINE=0240: facts[key.lower()] = restore.get(key, \"\")\nDETAIL=SOURCE_LINE=0241: if restore.get(\"STATUS\") != \"OK\":\nDETAIL=SOURCE_LINE=0242: problems.append(\"restore_status_not_ok\")\nDETAIL=SOURCE_LINE=0243: if int(facts[\"restore_health_age_seconds\"]) > RESTORE_HEALTH_AGE_MAX:\nDETAIL=SOURCE_LINE=0244: problems.append(\"restore_health_stale\")\nDETAIL=SOURCE_LINE=0245: for key in [\"CLOUD_1_STATUS\", \"CLOUD_2_STATUS\"]:\nDETAIL=SOURCE_LINE=0246: if restore.get(key) != \"OK\":\nDETAIL=SOURCE_LINE=0247: problems.append(key.lower() + \"_bad\")\nDETAIL=SOURCE_LINE=0248: for key in [\"SHA256_OK\", \"TAR_OK\", \"REQUIRED_FILES_OK\", \"COOKIE_JAR_OK\", \"EXTENSION_HEALTH_FILES_OK\", \"EXTENSION_SCRIPTS_OK\", \"EXTENSION_UNITS_OK\", \"SQLITE_OK\", \"PY_COMPILE_OK\"]:\nDETAIL=SOURCE_LINE=0249: if restore.get(key) != \"yes\":\nDETAIL=SOURCE_LINE=0250: problems.append(key.lower() + \"_bad\")\nDETAIL=SOURCE_LINE=0251:\nDETAIL=SOURCE_LINE=0252: local_units = [\nDETAIL=SOURCE_LINE=0253: \"skladchik-reports-monitor-backup.timer\",\nDETAIL=SOURCE_LINE=0254: \"skladchik-reports-monitor-backup-restore-check.timer\",\nDETAIL=SOURCE_LINE=0255: \"skladchik-reports-monitor-supervisor.timer\",\nDETAIL=SOURCE_LINE=0256: \"skladchik-reports-monitor-monthly-selftest-watch.timer\",\nDETAIL=SOURCE_LINE=0257: ]\nDETAIL=SOURCE_LINE=0258: props = systemctl_props(local_units)\nDETAIL=SOURCE_LINE=0259: for u, p in props.items():\nDETAIL=SOURCE_LINE=0260: base = u.replace(\".timer\", \"\").replace(\"-\", \"_\")\nDETAIL=SOURCE_LINE=0261: facts[base + \"_timer_enabled\"] = p[\"enabled\"]\nDETAIL=SOURCE_LINE=0262: facts[base + \"_timer_active\"] = p[\"active\"]\nDETAIL=SOURCE_LINE=0263: if u != \"skladchik-reports-monitor-supervisor.timer\":\nDETAIL=SOURCE_LINE=0264: if p[\"enabled\"] != \"enabled\" or p[\"active\"] != \"active\":\nDETAIL=SOURCE_LINE=0265: problems.append(base + \"_timer_bad\")\nDETAIL=SOURCE_LINE=0266: else:\nDETAIL=SOURCE_LINE=0267: # Before first enable it can be disabled/inactive during test. After enable next runs will see enabled/active.\nDETAIL=SOURCE_LINE=0268: if p[\"enabled\"] not in (\"enabled\", \"disabled\") or p[\"active\"] not in (\"active\", \"inactive\"):\nDETAIL=SOURCE_LINE=0269: problems.append(base + \"_timer_unknown\")\nDETAIL=SOURCE_LINE=0270:\nDETAIL=SOURCE_LINE=0271: return facts, problems\nDETAIL=SOURCE_LINE=0272:\nDETAIL=SOURCE_LINE=0273:def render_body(status, facts, problems, mode):\nDETAIL=SOURCE_LINE=0274: ru_status = \"OK\" if status == \"OK\" else \"АВАРИЯ\"\nDETAIL=SOURCE_LINE=0275: lines = [\nDETAIL=SOURCE_LINE=0276: f\"Статус надзора: {ru_status}\",\nDETAIL=SOURCE_LINE=0277: f\"Время UTC: {iso()}\",\nDETAIL=SOURCE_LINE=0278: \"\",\nDETAIL=SOURCE_LINE=0279: \"Монитор жалоб:\",\nDETAIL=SOURCE_LINE=0280: f\"- статус: {facts.get('monitor_status')}\",\nDETAIL=SOURCE_LINE=0281: f\"- всего жалоб на странице: {facts.get('monitor_reports')}\",\nDETAIL=SOURCE_LINE=0282: f\"- открытые/назначенные: {facts.get('monitor_open')}\",\nDETAIL=SOURCE_LINE=0283: f\"- закрытые: {facts.get('monitor_resolved')}\",\nDETAIL=SOURCE_LINE=0284: f\"- возраст health, сек: {facts.get('monitor_health_age_seconds')}\",\nDETAIL=SOURCE_LINE=0285: f\"- автопродление авторизации: {facts.get('session_keepalive')}\",\nDETAIL=SOURCE_LINE=0286: f\"- cookie jar: {facts.get('cookie_jar')} / {facts.get('cookie_jar_bytes')} байт\",\nDETAIL=SOURCE_LINE=0287: f\"- постоянная cookie действует до UTC: {facts.get('cookie_expiry_utc') or 'не определено'}\",\nDETAIL=SOURCE_LINE=0288: f\"- осталось дней по cookie: \nDETAIL=SOURCE_LINE=0289: f\"- статус срока cookie: \nDETAIL=SOURCE_LINE=0290: f\"- NetBird route: {facts.get('netbird_route')}\",\nDETAIL=SOURCE_LINE=0291: \"\",\nDETAIL=SOURCE_LINE=0292: \"Бэкап:\",\nDETAIL=SOURCE_LINE=0293: f\"- статус: {facts.get('backup_status')}\",\nDETAIL=SOURCE_LINE=0294: f\"- последний RUN_ID: {facts.get('backup_run_id')}\",\nDETAIL=SOURCE_LINE=0295: f\"- размер архива, байт: {facts.get('backup_size_bytes')}\",\nDETAIL=SOURCE_LINE=0296: f\"- cookie jar в бэкапе: {facts.get('backup_cookie_jar_backed_up')}\",\nDETAIL=SOURCE_LINE=0297: f\"- health-файлы расширений в бэкапе: {facts.get('backup_extension_health_files_backed_up')}\",\nDETAIL=SOURCE_LINE=0298: f\"- название жалобы в теме письма: {facts.get('backup_title_in_email_subject')}\",\nDETAIL=SOURCE_LINE=0299: f\"- возраст health, сек: {facts.get('backup_health_age_seconds')}\",\nDETAIL=SOURCE_LINE=0300: \"\",\nDETAIL=SOURCE_LINE=0301: \"Restore-check:\",\nDETAIL=SOURCE_LINE=0302: f\"- статус: {facts.get('restore_status')}\",\nDETAIL=SOURCE_LINE=0303: f\"- последний RUN_ID: {facts.get('restore_run_id')}\",\nDETAIL=SOURCE_LINE=0304: f\"- cloud1/cloud2: {facts.get('cloud_1_status')}/{facts.get('cloud_2_status')}\",\nDETAIL=SOURCE_LINE=0305: f\"- sha256/tar/cookie/sqlite/python: {facts.get('sha256_ok')}/{facts.get('tar_ok')}/{facts.get('cookie_jar_ok')}/{facts.get('sqlite_ok')}/{facts.get('py_compile_ok')}\",\nDETAIL=SOURCE_LINE=0306: f\"- health/scripts/units расширений: {facts.get('extension_health_files_ok')}/{facts.get('extension_scripts_ok')}/{facts.get('extension_units_ok')}\",\nDETAIL=SOURCE_LINE=0307: \"\",\nDETAIL=SOURCE_LINE=0308: \"Проблемы:\",\nDETAIL=SOURCE_LINE=0309: ]\nDETAIL=SOURCE_LINE=0310: if problems:\nDETAIL=SOURCE_LINE=0311: lines.extend([f\"- {p}\" for p in problems])\nDETAIL=SOURCE_LINE=0312: else:\nDETAIL=SOURCE_LINE=0313: lines.append(\"- нет\")\nDETAIL=SOURCE_LINE=0314: lines.extend([\nDETAIL=SOURCE_LINE=0315: \"\",\nDETAIL=SOURCE_LINE=0316: \"Авторизация:\",\nDETAIL=SOURCE_LINE=0317: \"- ручной ежемесячный перелогин обычно не нужен; монитор сам продлевает сессию постоянным cookie jar при каждом опросе;\",\nDETAIL=SOURCE_LINE=0318: \"- supervisor теперь отдельно следит за сроком постоянной cookie и заранее предупредит, если до истечения останется меньше 7 дней;\",\nDETAIL=SOURCE_LINE=0319: \"- если форум принудительно сбросит сессию, придёт отдельная тревога AUTH_REQUIRED; тогда нужно будет разово обновить вход, но это аварийный сценарий, а не ежемесячная рутина.\",\nDETAIL=SOURCE_LINE=0320: ])\nDETAIL=SOURCE_LINE=0321: return \"\\n\".join(lines)\nDETAIL=SOURCE_LINE=0322:\nDETAIL=SOURCE_LINE=0323:def write_health(status, facts, problems, mail_sent):\nDETAIL=SOURCE_LINE=0324: STATE_DIR.mkdir(parents=True, exist_ok=True)\nDETAIL=SOURCE_LINE=0325: lines = [f\"STATUS={status}\", f\"TS={iso()}\", \"problems=\" + \",\".join(problems), f\"mail_sent={'yes' if mail_sent else 'no'}\"]\nDETAIL=SOURCE_LINE=0326: for k in sorted(facts):\nDETAIL=SOURCE_LINE=0327: v = str(facts[k]).replace(\"\\n\", \" \")[:500]\nDETAIL=SOURCE_LINE=0328: lines.append(f\"{k}={v}\")\nDETAIL=SOURCE_LINE=0329: HEALTH.write_text(\"\\n\".join(lines) + \"\\n\", encoding=\"utf-8\")\nDETAIL=SOURCE_LINE=0330:\nDETAIL=SOURCE_LINE=0331:def read_state():\nDETAIL=SOURCE_LINE=0332: return read_env_file(STATE)\nDETAIL=SOURCE_LINE=0333:\nDETAIL=SOURCE_LINE=0334:def write_state(status, problems):\nDETAIL=SOURCE_LINE=0335: STATE_DIR.mkdir(parents=True, exist_ok=True)\nDETAIL=SOURCE_LINE=0336: STATE.write_text(f\"LAST_STATUS={status}\\nLAST_PROBLEMS={','.join(problems)}\\nTS={iso()}\\n\", encoding=\"utf-8\")\nDETAIL=SOURCE_LINE=0337:\nDETAIL=SOURCE_LINE=0338:def main():\nDETAIL=SOURCE_LINE=0339: mode = sys.argv[1] if len(sys.argv) > 1 else \"check\"\nDETAIL=SOURCE_LINE=0340: if mode not in (\"check\", \"test\"):\nDETAIL=SOURCE_LINE=0341: print(\"usage: skladchik-reports-monitor-supervisor [check|test]\", file=sys.stderr)\nDETAIL=SOURCE_LINE=0342: return 2\nDETAIL=SOURCE_LINE=0343:\nDETAIL=SOURCE_LINE=0344: facts, problems = gather()\nDETAIL=SOURCE_LINE=0345: status = \"OK\" if not problems else \"FAIL\"\nDETAIL=SOURCE_LINE=0346: old = read_state()\nDETAIL=SOURCE_LINE=0347: old_status = old.get(\"LAST_STATUS\", \"\")\nDETAIL=SOURCE_LINE=0348: mail_sent = False\nDETAIL=SOURCE_LINE=0349:\nDETAIL=SOURCE_LINE=0350: if mode == \"test\":\nDETAIL=SOURCE_LINE=0351: mail_sent = edge_mail(\"Тест: надзор монитора жалоб и бэкапов\", render_body(status, facts, problems, mode))\nDETAIL=SOURCE_LINE=0352: if not mail_sent:\nDETAIL=SOURCE_LINE=0353: problems.append(\"supervisor_mail_send_failed\")\nDETAIL=SOURCE_LINE=0354: status = \"FAIL\"\nDETAIL=SOURCE_LINE=0355: elif status != \"OK\":\nDETAIL=SOURCE_LINE=0356: if old_status != \"FAIL\":\nDETAIL=SOURCE_LINE=0357: mail_sent = edge_mail(\"Авария: монитор жалоб или его бэкапы\", render_body(status, facts, problems, mode))\nDETAIL=SOURCE_LINE=0358: if not mail_sent:\nDETAIL=SOURCE_LINE=0359: problems.append(\"supervisor_mail_send_failed\")\nDETAIL=SOURCE_LINE=0360: status = \"FAIL\"\nDETAIL=SOURCE_LINE=0361: elif old_status == \"FAIL\":\nDETAIL=SOURCE_LINE=0362: mail_sent = edge_mail(\"Восстановлено: монитор жалоб и бэкапы снова OK\", render_body(status, facts, problems, mode))\nDETAIL=SOURCE_LINE=0363:\nDETAIL=SOURCE_LINE=0364: write_health(status, facts, problems, mail_sent)\nDETAIL=SOURCE_LINE=0365: write_state(status, problems)\nDETAIL=SOURCE_LINE=0366:\nDETAIL=SOURCE_LINE=0367: print(f\"SUPERVISOR_STATUS={status}\")\nDETAIL=SOURCE_LINE=0368: print(f\"SUPERVISOR_MAIL_SENT={'yes' if mail_sent else 'no'}\")\nDETAIL=SOURCE_LINE=0369: print(f\"SUPERVISOR_PROBLEMS={','.join(problems)}\")\nDETAIL=SOURCE_LINE=0370: print(f\"MONITOR={facts.get('monitor_status')} reports={facts.get('monitor_reports')} open={facts.get('monitor_open')} resolved={facts.get('monitor_resolved')}\")\nDETAIL=SOURCE_LINE=0371: print(f\"BACKUP={facts.get('backup_status')} run={facts.get('backup_run_id')}\")\nDETAIL=SOURCE_LINE=0372: print(f\"RESTORE={facts.get('restore_status')} run={facts.get('restore_run_id')}\")\nDETAIL=SOURCE_LINE=0373: log(f\"CHECK mode={mode} status={status} mail_sent={'yes' if mail_sent else 'no'} problems={problems}\")\nDETAIL=SOURCE_LINE=0374: return 0 if status == \"OK\" else 1\nDETAIL=SOURCE_LINE=0375:\nDETAIL=SOURCE_LINE=0376:if __name__ == \"__main__\":\nDETAIL=SOURCE_LINE=0377: sys.exit(main())\nDETAIL=SCRIPT_TOTAL_LINES=377\nDETAIL=RELEVANT_SOURCE_END\nDETAIL=UNIT_CONTRACT_BEGIN\nDETAIL=UNIT_NAME=skladchik-reports-monitor-supervisor.service\nDETAIL=UNIT_LINE=# /etc/systemd/system/skladchik-reports-monitor-supervisor.service\nDETAIL=UNIT_LINE=[Unit]\nDETAIL=UNIT_LINE=Description=Skladchik reports monitor full supervisor\nDETAIL=UNIT_LINE=After=network-online.target\nDETAIL=UNIT_LINE=\nDETAIL=UNIT_LINE=[Service]\nDETAIL=UNIT_LINE=Type=oneshot\nDETAIL=UNIT_LINE=ExecStart=/usr/local/sbin/skladchik-reports-monitor-supervisor check\nDETAIL=UNIT_NAME=skladchik-reports-monitor-supervisor.timer\nDETAIL=UNIT_LINE=# /etc/systemd/system/skladchik-reports-monitor-supervisor.timer\nDETAIL=UNIT_LINE=[Unit]\nDETAIL=UNIT_LINE=Description=Run Skladchik reports monitor full supervisor every 10 minutes\nDETAIL=UNIT_LINE=\nDETAIL=UNIT_LINE=[Timer]\nDETAIL=UNIT_LINE=OnBootSec=5min\nDETAIL=UNIT_LINE=OnUnitActiveSec=10min\nDETAIL=UNIT_LINE=AccuracySec=30s\nDETAIL=UNIT_LINE=Persistent=false\nDETAIL=UNIT_LINE=Unit=skladchik-reports-monitor-supervisor.service\nDETAIL=UNIT_LINE=\nDETAIL=UNIT_LINE=[Install]\nDETAIL=UNIT_LINE=WantedBy=timers.target\nDETAIL=UNIT_CONTRACT_END\nDETAIL=TIMER_STATE_BEGIN\nDETAIL=TIMER_PROPERTY=Unit=skladchik-reports-monitor-supervisor.service\nDETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=\nDETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 11:58:38 MSK\nDETAIL=TIMER_PROPERTY=LoadState=loaded\nDETAIL=TIMER_PROPERTY=ActiveState=active\nDETAIL=TIMER_PROPERTY=SubState=waiting\nDETAIL=TIMER_ROW=Fri 2026-07-24 12:08:38 MSK 3min 34s Fri 2026-07-24 11:58:38 MSK 6min ago skladchik-reports-monitor-supervisor.timer skladchik-reports-monitor-supervisor.service\nDETAIL=TIMER_ROW=Sat 2026-07-25 04:39:05 MSK 16h Fri 2026-07-24 04:28:08 MSK 7h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service\nDETAIL=TIMER_ROW=Sat 2026-07-25 10:15:00 MSK 22h Fri 2026-07-24 10:15:04 MSK 1h 49min ago skladchik-reports-monitor-monthly-selftest-watch.timer skladchik-reports-monitor-monthly-selftest-watch.service\nDETAIL=TIMER_ROW=Sun 2026-07-26 05:15:13 MSK 1 day 17h Sun 2026-07-19 05:11:25 MSK - skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service\nDETAIL=TIMER_STATE_END\nDETAIL=SECRET_VALUES_OUTPUT=NONE\nDETAIL=CHANGES_MADE=NO\nDETAIL=COOKIE_CHECK_SOURCE_AUDIT=PASS\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 69011f3..a7af1d9 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,5 +1,5 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=HOMELAB-SKLADCHIK-SUPERVISOR-FAILURE-TRACE-417 +COMMAND_ID=HOMELAB-SKLADCHIK-COOKIE-CHECK-SOURCE-AUDIT-418 STATUS=OK RC=0 HOST=pve01 @@ -9,113 +9,422 @@ REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 -COMMAND_SHA256=9f6e05b9df78cf255d0b700cf196693048a5d3f95183c3f4aa1fd2e48188ab45 +COMMAND_SHA256=77896618b46e2f97362240deb743fb8e625075381e57377ee0314130be2b2026 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGES_MADE=false SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=4a1b1b699e9b8b13e867485522410253fe5bf1a821101e64dcf14e5186201798 -SANITIZED_OUTPUT_SHA256=4a1b1b699e9b8b13e867485522410253fe5bf1a821101e64dcf14e5186201798 +RAW_EVIDENCE_SHA256=1c26f0b751c5e55421c706f56f4be25d92c17849df8a385fb66ce01144801022 +SANITIZED_OUTPUT_SHA256=8e5a54db35050a5c0010241fb0bad91fc5966530dbac76bb07426b09af51a6a1 OUTPUT_BEGIN -DETAIL=SUPERVISOR_STATE_BEGIN -DETAIL=SYSTEMD_PROPERTY=Result=exit-code -DETAIL=SYSTEMD_PROPERTY=ExecMainCode=1 -DETAIL=SYSTEMD_PROPERTY=ExecMainStatus=1 -DETAIL=SYSTEMD_PROPERTY=WorkingDirectory= -DETAIL=SYSTEMD_PROPERTY=User= -DETAIL=SYSTEMD_PROPERTY=Group= -DETAIL=SYSTEMD_PROPERTY=LoadState=loaded -DETAIL=SYSTEMD_PROPERTY=ActiveState=failed -DETAIL=SYSTEMD_PROPERTY=SubState=failed -DETAIL=SYSTEMD_PROPERTY=FragmentPath=/etc/systemd/system/skladchik-reports-monitor-supervisor.service -DETAIL=SYSTEMD_PROPERTY=DropInPaths= -DETAIL=SUPERVISOR_STATE_END -DETAIL=SUPERVISOR_UNIT_CONTRACT_BEGIN -DETAIL=UNIT_CONTRACT=# /etc/systemd/system/skladchik-reports-monitor-supervisor.service -DETAIL=UNIT_CONTRACT=[Unit] -DETAIL=UNIT_CONTRACT=After=network-online.target -DETAIL=UNIT_CONTRACT=[Service] -DETAIL=UNIT_CONTRACT=ExecStart=/usr/local/sbin/skladchik-reports-monitor-supervisor check -DETAIL=SUPERVISOR_UNIT_CONTRACT_END -DETAIL=SUPERVISOR_LAST_FAILURE_BEGIN -DETAIL=JOURNAL=2026-07-24T06:38:28+03:00 pve01 skladchik-reports-monitor-supervisor[2914895]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T06:38:28+03:00 pve01 skladchik-reports-monitor-supervisor[2914895]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T06:48:29+03:00 pve01 skladchik-reports-monitor-supervisor[2943586]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T06:48:29+03:00 pve01 skladchik-reports-monitor-supervisor[2943586]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T06:58:29+03:00 pve01 skladchik-reports-monitor-supervisor[2971845]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T06:58:29+03:00 pve01 skladchik-reports-monitor-supervisor[2971845]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T07:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3008729]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T07:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3008729]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T07:18:28+03:00 pve01 skladchik-reports-monitor-supervisor[3048052]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T07:18:28+03:00 pve01 skladchik-reports-monitor-supervisor[3048052]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T07:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3075804]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T07:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3075804]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T07:38:28+03:00 pve01 skladchik-reports-monitor-supervisor[3111924]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T07:38:28+03:00 pve01 skladchik-reports-monitor-supervisor[3111924]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T07:48:29+03:00 pve01 skladchik-reports-monitor-supervisor[3148073]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T07:48:29+03:00 pve01 skladchik-reports-monitor-supervisor[3148073]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T07:58:28+03:00 pve01 skladchik-reports-monitor-supervisor[3175426]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T07:58:28+03:00 pve01 skladchik-reports-monitor-supervisor[3175426]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T08:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3205006]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T08:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3205006]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T08:18:29+03:00 pve01 skladchik-reports-monitor-supervisor[3241119]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T08:18:29+03:00 pve01 skladchik-reports-monitor-supervisor[3241119]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T08:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3269919]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T08:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3269919]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T08:38:29+03:00 pve01 skladchik-reports-monitor-supervisor[3306255]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T08:38:29+03:00 pve01 skladchik-reports-monitor-supervisor[3306255]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T08:48:29+03:00 pve01 skladchik-reports-monitor-supervisor[3342134]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T08:48:29+03:00 pve01 skladchik-reports-monitor-supervisor[3342134]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T08:58:29+03:00 pve01 skladchik-reports-monitor-supervisor[3369587]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T08:58:29+03:00 pve01 skladchik-reports-monitor-supervisor[3369587]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T09:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3405539]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T09:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3405539]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T09:18:29+03:00 pve01 skladchik-reports-monitor-supervisor[3430044]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T09:18:29+03:00 pve01 skladchik-reports-monitor-supervisor[3430044]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T09:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3457493]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T09:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3457493]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T09:38:29+03:00 pve01 skladchik-reports-monitor-supervisor[3493130]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T09:38:29+03:00 pve01 skladchik-reports-monitor-supervisor[3493130]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T09:48:29+03:00 pve01 skladchik-reports-monitor-supervisor[3528983]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T09:48:29+03:00 pve01 skladchik-reports-monitor-supervisor[3528983]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T09:58:29+03:00 pve01 skladchik-reports-monitor-supervisor[3556369]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T09:58:29+03:00 pve01 skladchik-reports-monitor-supervisor[3556369]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T10:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3594595]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T10:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3594595]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T10:18:29+03:00 pve01 skladchik-reports-monitor-supervisor[3633670]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T10:18:29+03:00 pve01 skladchik-reports-monitor-supervisor[3633670]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T10:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3652898]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T10:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3652898]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T10:38:31+03:00 pve01 skladchik-reports-monitor-supervisor[3691967]: SUPERVISOR_STATUS=FAIL -DETAIL=JOURNAL=2026-07-24T10:38:31+03:00 pve01 skladchik-reports-monitor-supervisor[3691967]: SUPERVISOR_PROBLEMS=edge_cookies_jar_bad -DETAIL=JOURNAL=2026-07-24T10:38:31+03:00 pve01 systemd[1]: skladchik-reports-monitor-supervisor.service: Main process exited, code=exited, status=1/FAILURE -DETAIL=JOURNAL=2026-07-24T10:38:31+03:00 pve01 systemd[1]: skladchik-reports-monitor-supervisor.service: Failed with result 'exit-code'. -DETAIL=JOURNAL=2026-07-24T10:38:31+03:00 pve01 systemd[1]: Failed to start skladchik-reports-monitor-supervisor.service - Skladchik reports monitor full supervisor. -DETAIL=JOURNAL=2026-07-24T10:48:31+03:00 pve01 skladchik-reports-monitor-supervisor[3730900]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T10:48:31+03:00 pve01 skladchik-reports-monitor-supervisor[3730900]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T10:58:29+03:00 pve01 skladchik-reports-monitor-supervisor[3761552]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T10:58:29+03:00 pve01 skladchik-reports-monitor-supervisor[3761552]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T11:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3800692]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T11:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3800692]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T11:18:29+03:00 pve01 skladchik-reports-monitor-supervisor[3839724]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T11:18:29+03:00 pve01 skladchik-reports-monitor-supervisor[3839724]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T11:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3858965]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T11:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3858965]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T11:38:29+03:00 pve01 skladchik-reports-monitor-supervisor[3897854]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T11:38:29+03:00 pve01 skladchik-reports-monitor-supervisor[3897854]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T11:48:35+03:00 pve01 skladchik-reports-monitor-supervisor[3937127]: SUPERVISOR_STATUS=OK -DETAIL=JOURNAL=2026-07-24T11:48:35+03:00 pve01 skladchik-reports-monitor-supervisor[3937127]: SUPERVISOR_PROBLEMS= -DETAIL=JOURNAL=2026-07-24T11:58:40+03:00 pve01 skladchik-reports-monitor-supervisor[3967718]: SUPERVISOR_STATUS=FAIL -DETAIL=JOURNAL=2026-07-24T11:58:40+03:00 pve01 skladchik-reports-monitor-supervisor[3967718]: SUPERVISOR_PROBLEMS=edge_cookies_jar_bad -DETAIL=JOURNAL=2026-07-24T11:58:40+03:00 pve01 systemd[1]: skladchik-reports-monitor-supervisor.service: Main process exited, code=exited, status=1/FAILURE -DETAIL=JOURNAL=2026-07-24T11:58:40+03:00 pve01 systemd[1]: skladchik-reports-monitor-supervisor.service: Failed with result 'exit-code'. -DETAIL=JOURNAL=2026-07-24T11:58:40+03:00 pve01 systemd[1]: Failed to start skladchik-reports-monitor-supervisor.service - Skladchik reports monitor full supervisor. -DETAIL=SUPERVISOR_LAST_FAILURE_END +DETAIL=SCRIPT_PATH=/usr/local/sbin/skladchik-reports-monitor-supervisor +DETAIL=SCRIPT_SHA256=35c68ae0929ff3795be2bde40de334fc8e26413a3d8a620a137be90d1f2c164e +DETAIL=SCRIPT_METADATA=BYTES:20463|MODE:755|OWNER:root:root|MTIME:2026-07-18 13:52:30.415780066 +0300 +DETAIL=RELEVANT_SOURCE_BEGIN +DETAIL=SOURCE_LINE=0005:import os +DETAIL=SOURCE_LINE=0006:import re +DETAIL=SOURCE_LINE=0007:import shlex +DETAIL=SOURCE_LINE=0008:import subprocess +DETAIL=SOURCE_LINE=0009:import sys +DETAIL=SOURCE_LINE=0010:from pathlib import Path +DETAIL=SOURCE_LINE=0011: +DETAIL=SOURCE_LINE=0012:EDGE = os.environ.get("SKLADCHIK_EDGE", "debian@[PRIVATE_IP]") +DETAIL=SOURCE_LINE=0013:STATE_DIR = Path("/var/lib/skladchik-reports-monitor-supervisor") +DETAIL=SOURCE_LINE=0014:LOG_DIR = Path("/var/log/skladchik-reports-monitor-supervisor") +DETAIL=SOURCE_LINE=0015:HEALTH = STATE_DIR / "health.txt" +DETAIL=SOURCE_LINE=0016:STATE = STATE_DIR / "state.env" +DETAIL=SOURCE_LINE=0017:LOG = LOG_DIR / "supervisor.log" +DETAIL=SOURCE_LINE=0018: +DETAIL=SOURCE_LINE=0019:MONITOR_HEALTH_AGE_MAX = int(os.environ.get("SKLADCHIK_MONITOR_HEALTH_MAX_SECONDS", "600")) +DETAIL=SOURCE_LINE=0020:BACKUP_HEALTH_AGE_MAX = int(os.environ.get("SKLADCHIK_BACKUP_HEALTH_MAX_SECONDS", "129600")) +DETAIL=SOURCE_LINE=0021:RESTORE_HEALTH_AGE_MAX = int(os.environ.get("SKLADCHIK_RESTORE_HEALTH_MAX_SECONDS", "691200")) +DETAIL=SOURCE_LINE=0022:COOKIE_EXPIRY_WARN_SECONDS = int(os.environ.get("SKLADCHIK_COOKIE_EXPIRY_WARN_SECONDS", "604800")) +DETAIL=SOURCE_LINE=0023: +DETAIL=SOURCE_LINE=0024:PVE_BACKUP_HEALTH = Path("/var/lib/skladchik-reports-monitor-backup/health.txt") +DETAIL=SOURCE_LINE=0025:PVE_RESTORE_HEALTH = Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt") +DETAIL=SOURCE_LINE=0026: +DETAIL=SOURCE_LINE=0027:def now(): +DETAIL=SOURCE_LINE=0028: return _dt.datetime.now(_dt.UTC).replace(microsecond=0) +DETAIL=SOURCE_LINE=0029: +DETAIL=SOURCE_LINE=0030:def iso(): +DETAIL=SOURCE_LINE=0031: return now().isoformat().replace("+00:00", "Z") +DETAIL=SOURCE_LINE=0032: +DETAIL=SOURCE_LINE=0033:def log(msg): +DETAIL=SOURCE_LINE=0034: LOG_DIR.mkdir(parents=True, exist_ok=True) +DETAIL=SOURCE_LINE=0035: with LOG.open("a", encoding="utf-8") as f: +DETAIL=SOURCE_LINE=0036: f.write(f"{iso()} {msg}\n") +DETAIL=SOURCE_LINE=0037: +DETAIL=SOURCE_LINE=0038:def run(cmd, timeout=45, input_text=None): +DETAIL=SOURCE_LINE=0039: try: +DETAIL=SOURCE_LINE=0040: p = subprocess.run(cmd, shell=True, text=True, input=input_text, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout) +DETAIL=SOURCE_LINE=0041: return p.returncode, p.stdout.strip() +DETAIL=SOURCE_LINE=0042: except subprocess.TimeoutExpired as e: +DETAIL=SOURCE_LINE=0043: out = e.stdout if isinstance(e.stdout, str) else "" +DETAIL=SOURCE_LINE=0044: return 124, (out or "").strip() + "\nTIMEOUT" +DETAIL=SOURCE_LINE=0045: +DETAIL=SOURCE_LINE=0046:def parse_env_text(text): +DETAIL=SOURCE_LINE=0047: d = {} +DETAIL=SOURCE_LINE=0048: for line in (text or "").splitlines(): +DETAIL=SOURCE_LINE=0049: if not line or line.startswith("#") or "=" not in line: +DETAIL=SOURCE_LINE=0050: continue +DETAIL=SOURCE_LINE=0051: k, v = line.split("=", 1) +DETAIL=SOURCE_LINE=... +DETAIL=SOURCE_LINE=0066: dt = _dt.datetime.fromisoformat(s) +DETAIL=SOURCE_LINE=0067: if dt.tzinfo is None: +DETAIL=SOURCE_LINE=0068: dt = dt.replace(tzinfo=_dt.UTC) +DETAIL=SOURCE_LINE=0069: return int((now() - dt.astimezone(_dt.UTC)).total_seconds()) +DETAIL=SOURCE_LINE=0070: except Exception: +DETAIL=SOURCE_LINE=0071: return 999999999 +DETAIL=SOURCE_LINE=0072: +DETAIL=SOURCE_LINE=0073:def ssh_edge(cmd, timeout=45): +DETAIL=SOURCE_LINE=0074: q = "ssh -o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] " + shlex.quote(EDGE) + " " + shlex.quote("sudo bash -lc " + shlex.quote(cmd)) +DETAIL=SOURCE_LINE=0075: return run(q, timeout=timeout) +DETAIL=SOURCE_LINE=0076: +DETAIL=SOURCE_LINE=0077:def edge_mail(subject, body): +DETAIL=SOURCE_LINE=0078: subj_b64 = base64.b64encode(subject.encode("utf-8")).decode("ascii") +DETAIL=SOURCE_LINE=0079: body_b64 = base64.b64encode(body.encode("utf-8")).decode("ascii") +DETAIL=SOURCE_LINE=0080: py = ( +DETAIL=SOURCE_LINE=0081: "import base64,importlib.util;" +DETAIL=SOURCE_LINE=0082: "spec=importlib.util.spec_from_file_location('m','/opt/skladchik-reports-monitor/monitor.py');" +DETAIL=SOURCE_LINE=0083: "m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m);" +DETAIL=SOURCE_LINE=0084: "cfg,secrets= +DETAIL=SOURCE_LINE=0085: f"subject=base64.b64decode('{subj_b64}').decode('utf-8');" +DETAIL=SOURCE_LINE=0086: f"body=base64.b64decode('{body_b64}').decode('utf-8');" +DETAIL=SOURCE_LINE=0087: "ok=m.send_mail(cfg,secrets,subject,body);" +DETAIL=SOURCE_LINE=0088: "print('MAIL_OK' if ok else 'MAIL_FALSE')" +DETAIL=SOURCE_LINE=0089: ) +DETAIL=SOURCE_LINE=0090: rc, out = ssh_edge("python3 -c " + shlex.quote(py), timeout=60) +DETAIL=SOURCE_LINE=0091: log(f"MAIL subject={subject!r} rc={rc} out={out!r}") +DETAIL=SOURCE_LINE=0092: return rc == 0 and "MAIL_OK" in out +DETAIL=SOURCE_LINE=0093: +DETAIL=SOURCE_LINE=0094:def systemctl_props(units): +DETAIL=SOURCE_LINE=0095: cmd = "systemctl is-enabled " + " ".join(map(shlex.quote, units)) + " 2>/dev/null || true; echo __ACTIVE__; systemctl is-active " + " ".join(map(shlex.quote, units)) + " 2>/dev/null || true" +DETAIL=SOURCE_LINE=0096: rc, out = run(cmd, timeout=20) +DETAIL=SOURCE_LINE=0097: parts = out.split("__ACTIVE__", 1) +DETAIL=SOURCE_LINE=0098: enabled_lines = [x.strip() for x in parts[0].splitlines() if x.strip()] +DETAIL=SOURCE_LINE=0099: active_lines = [x.strip() for x in parts[1].splitlines() if x.strip()] if len(parts) > 1 else [] +DETAIL=SOURCE_LINE=0100: res = {} +DETAIL=SOURCE_LINE=0101: for i, u in enumerate(units): +DETAIL=SOURCE_LINE=0102: res[u] = {"enabled": enabled_lines[i] if i < len(enabled_lines) else "unknown", "active": active_lines[i] if i < len(active_lines) else "unknown"} +DETAIL=SOURCE_LINE=0103: return res +DETAIL=SOURCE_LINE=0104: +DETAIL=SOURCE_LINE=0105:def gather(): +DETAIL=SOURCE_LINE=0106: facts = {} +DETAIL=SOURCE_LINE=0107: problems = [] +DETAIL=SOURCE_LINE=0108: +DETAIL=SOURCE_LINE=0109: edge_cmd = r''' +DETAIL=SOURCE_LINE=0110:set +e +DETAIL=SOURCE_LINE=0111:echo __MONITOR_HEALTH_BEGIN__ +DETAIL=SOURCE_LINE=0112:cat /var/lib/skladchik-reports-monitor/health.txt 2>/dev/null || true +DETAIL=SOURCE_LINE=0113:echo __MONITOR_HEALTH_END__ +DETAIL=SOURCE_LINE=0114:echo __ALERT_HEALTH_BEGIN__ +DETAIL=SOURCE_LINE=0115:cat /var/lib/skladchik-reports-monitor/alert-health.txt 2>/dev/null || true +DETAIL=SOURCE_LINE=0116:echo __ALERT_HEALTH_END__ +DETAIL=SOURCE_LINE=0117:echo __TIMERS_BEGIN__ +DETAIL=SOURCE_LINE=0118:for u in skladchik-reports-monitor.timer skladchik-reports-monitor-watchdog.timer skladchik-reports-monitor-heartbeat.timer skladchik-reports-monitor-weekly-summary.timer skladchik-reports-monitor-age-alert.timer skladchik-reports-monitor-event-selftest.timer; do echo "$u enabled=$(systemctl is-enabled "$u" 2>/dev/null) active=$(systemctl is-active "$u" 2>/dev/null)"; done +DETAIL=SOURCE_LINE=0119:echo __TIMERS_END__ +DETAIL=SOURCE_LINE=0120:echo __ROUTE_BEGIN__ +DETAIL=SOURCE_LINE=0121:ip route get 1.1.1.1 2>&1 | head -n 1 +DETAIL=SOURCE_LINE=0122:echo __ROUTE_END__ +DETAIL=SOURCE_LINE=0123:echo __FILES_BEGIN__ +DETAIL=SOURCE_LINE=0124:test -s /opt/skladchik-reports-monitor/monitor.py && echo monitor_py=yes || echo monitor_py=no +DETAIL=SOURCE_LINE=0125:test -s /etc/skladchik-reports-monitor/config.ini && echo config_ini=yes || echo config_ini=no +DETAIL=SOURCE_LINE=0126:test -s /etc/skladchik-reports-monitor/secrets.env && echo secrets_env= +DETAIL=SOURCE_LINE=0127:test -s /var/lib/skladchik-reports-monitor/cookies.jar && echo cookies_jar=yes || echo cookies_jar=no +DETAIL=SOURCE_LINE=0128:[ -s /var/lib/skladchik-reports-monitor/cookies.jar ] && echo cookies_jar_bytes=$(stat -c %s /var/lib/skladchik-reports-monitor/cookies.jar 2>/dev/null) || echo cookies_jar_bytes=0 +DETAIL=SOURCE_LINE=0129:test -s /opt/skladchik-reports-monitor/operational_digest.py && echo operational_digest_py=yes || echo operational_digest_py=no +DETAIL=SOURCE_LINE=0130:test -s /opt/skladchik-reports-monitor/event-selftest-runner.py && echo event_selftest_runner_py=yes || echo event_selftest_runner_py=no +DETAIL=SOURCE_LINE=0131:test -s /opt/skladchik-reports-monitor/event-selftest-core && echo event_selftest_core=yes || echo event_selftest_core=no +DETAIL=SOURCE_LINE=0132:test -s /var/lib/skladchik-reports-monitor/weekly-summary-health.txt && echo weekly_summary_health=yes || echo weekly_summary_health=no +DETAIL=SOURCE_LINE=0133:test -s /var/lib/skladchik-reports-monitor/age-alert-health.txt && echo age_alert_health=yes || echo age_alert_health=no +DETAIL=SOURCE_LINE=0134:test -s /var/lib/skladchik-reports-monitor/event-selftest-health.txt && echo event_selftest_health=yes || echo event_selftest_health=no +DETAIL=SOURCE_LINE=0135:python3 -m py_compile /opt/skladchik-reports-monitor/monitor.py /opt/skladchik-reports-monitor/operational_digest.py /opt/skladchik-reports-monitor/event-selftest-runner.py /opt/skladchik-reports-monitor/event-selftest-core >/dev/null 2>&1 && echo py_compile=yes || echo py_compile=no +DETAIL=SOURCE_LINE=0136:echo __FILES_END__ +DETAIL=SOURCE_LINE=0137:''' +DETAIL=SOURCE_LINE=0138: rc, out = ssh_edge(edge_cmd, timeout=45) +DETAIL=SOURCE_LINE=0139: facts["edge_ssh_rc"] = str(rc) +DETAIL=SOURCE_LINE=0140: if rc != 0: +DETAIL=SOURCE_LINE=0141: problems.append("edge_ssh_failed") +DETAIL=SOURCE_LINE=0142: +DETAIL=SOURCE_LINE=0143: def section(name): +DETAIL=SOURCE_LINE=0144: m = re.search(rf"__{name}_BEGIN__\n(.*?)\n__{name}_END__", out, re.S) +DETAIL=SOURCE_LINE=0145: return m.group(1).strip() if m else "" +DETAIL=SOURCE_LINE=0146: +DETAIL=SOURCE_LINE=0147: mon = parse_env_text(section("MONITOR_HEALTH")) +DETAIL=SOURCE_LINE=0148: facts["monitor_status"] = mon.get("STATUS", "MISSING") +DETAIL=SOURCE_LINE=0149: facts["monitor_reports"] = mon.get("reports", "") +DETAIL=SOURCE_LINE=0150: facts["monitor_open"] = mon.get("open", "") +DETAIL=SOURCE_LINE=0151: facts["monitor_resolved"] = mon.get("resolved", "") +DETAIL=SOURCE_LINE=0152: facts["monitor_smtp_configured"] = mon.get("smtp_configured", "") +DETAIL=SOURCE_LINE=0153: facts["session_keepalive"] = mon.get("session_keepalive", "") +DETAIL=SOURCE_LINE=0154: facts["cookie_jar"] = mon.get("cookie_jar", "") +DETAIL=SOURCE_LINE=0155: facts["cookie_jar_bytes"] = mon.get("cookie_jar_bytes", "") +DETAIL=SOURCE_LINE=0156: facts["cookie_expiry_status"] = mon.get("cookie_expiry_status", "") +DETAIL=SOURCE_LINE=0157: facts["cookie_expiry_utc"] = mon.get("cookie_expiry_utc", "") +DETAIL=SOURCE_LINE=0158: facts["cookie_expires_in_seconds"] = mon.get("cookie_expires_in_seconds", "") +DETAIL=SOURCE_LINE=0159: facts["cookie_expires_in_days"] = mon.get("cookie_expires_in_days", "") +DETAIL=SOURCE_LINE=0160: facts["cookie_persistent_count"] = mon.get("cookie_persistent_count", "") +DETAIL=SOURCE_LINE=0161: facts["monitor_health_age_seconds"] = str(age_seconds_from_ts(mon.get("TS"))) +DETAIL=SOURCE_LINE=0162: if mon.get("STATUS") != "OK": +DETAIL=SOURCE_LINE=0163: problems.append("monitor_status_not_ok") +DETAIL=SOURCE_LINE=0164: if int(facts["monitor_health_age_seconds"]) > MONITOR_HEALTH_AGE_MAX: +DETAIL=SOURCE_LINE=0165: problems.append("monitor_health_stale") +DETAIL=SOURCE_LINE=0166: if mon.get("reports") in ("", "0"): +DETAIL=SOURCE_LINE=0167: problems.append("monitor_reports_zero_or_missing") +DETAIL=SOURCE_LINE=0168: if mon.get("session_keepalive") != "persistent_cookie_jar": +DETAIL=SOURCE_LINE=0169: problems.append("session_keepalive_not_persistent") +DETAIL=SOURCE_LINE=0170: if mon.get("cookie_jar") not in ("yes", "empty"): +DETAIL=SOURCE_LINE=0171: problems.append("monitor_cookie_jar_bad") +DETAIL=SOURCE_LINE=0172: expiry_status = mon.get("cookie_expiry_status", "") +DETAIL=SOURCE_LINE=0173: try: +DETAIL=SOURCE_LINE=0174: expires_in = int(mon.get("cookie_expires_in_seconds", "")) +DETAIL=SOURCE_LINE=0175: except Exception: +DETAIL=SOURCE_LINE=0176: expires_in = None +DETAIL=SOURCE_LINE=0177: if expiry_status in ("", "NO_COOKIE_JAR", "NO_PERSISTENT_COOKIE", "ERROR"): +DETAIL=SOURCE_LINE=0178: problems.append("cookie_expiry_missing") +DETAIL=SOURCE_LINE=0179: elif expiry_status == "EXPIRED" or (expires_in is not None and expires_in <= 0): +DETAIL=SOURCE_LINE=0180: problems.append("cookie_expiry_expired") +DETAIL=SOURCE_LINE=0181: elif expiry_status == "WARN" or (expires_in is not None and expires_in < COOKIE_EXPIRY_WARN_SECONDS): +DETAIL=SOURCE_LINE=0182: problems.append("cookie_expiry_warn") +DETAIL=SOURCE_LINE=0183: +DETAIL=SOURCE_LINE=0184: timers_text = section("TIMERS") +DETAIL=SOURCE_LINE=0185: for unit in ["skladchik-reports-monitor.timer", "skladchik-reports-monitor-watchdog.timer", "skladchik-reports-monitor-heartbeat.timer", "skladchik-reports-monitor-weekly-summary.timer", "skladchik-reports-monitor-age-alert.timer", "skladchik-reports-monitor-event-selftest.timer"]: +DETAIL=SOURCE_LINE=0186: safe = unit.replace(".timer", "").replace("_", "-") +DETAIL=SOURCE_LINE=0187: m = re.search(re.escape(unit) + r"\s+enabled=([^\s]+)\s+active=([^\s]+)", timers_text) +DETAIL=SOURCE_LINE=0188: enabled = m.group(1) if m else "missing" +DETAIL=SOURCE_LINE=0189: active = m.group(2) if m else "missing" +DETAIL=SOURCE_LINE=0190: facts[f"edge_{safe}_timer_enabled"] = enabled +DETAIL=SOURCE_LINE=0191: facts[f"edge_{safe}_timer_active"] = active +DETAIL=SOURCE_LINE=0192: if enabled != "enabled" or active != "active": +DETAIL=SOURCE_LINE=0193: problems.append(f"edge_{safe}_timer_bad") +DETAIL=SOURCE_LINE=0194: +DETAIL=SOURCE_LINE=0195: route_lines = section("ROUTE").splitlines() +DETAIL=SOURCE_LINE=0196: route = route_lines[0] if route_lines else "" +DETAIL=SOURCE_LINE=0197: facts["netbird_route"] = route +DETAIL=SOURCE_LINE=0198: if " wt0 " not in (" " + route + " "): +DETAIL=SOURCE_LINE=0199: problems.append("netbird_route_not_wt0") +DETAIL=SOURCE_LINE=0200: +DETAIL=SOURCE_LINE=0201: files = parse_env_text(section("FILES")) +DETAIL=SOURCE_LINE=0202: for k in ["monitor_py", "config_ini", "secrets_env", "cookies_jar", "operational_digest_py", "event_selftest_runner_py", "event_selftest_core", "weekly_summary_health", "age_alert_health", "event_selftest_health", "py_compile"]: +DETAIL=SOURCE_LINE=0203: facts[k] = files.get(k, "no") +DETAIL=SOURCE_LINE=0204: if facts[k] != "yes": +DETAIL=SOURCE_LINE=0205: problems.append(f"edge_{k}_bad") +DETAIL=SOURCE_LINE=0206: facts["cookies_jar_bytes_file"] = files.get("cookies_jar_bytes", "0") +DETAIL=SOURCE_LINE=0207: +DETAIL=SOURCE_LINE=0208: backup = read_env_file(PVE_BACKUP_HEALTH) +DETAIL=SOURCE_LINE=0209: facts["backup_status"] = backup.get("STATUS", "MISSING") +DETAIL=SOURCE_LINE=0210: facts["backup_run_id"] = backup.get("RUN_ID", "") +DETAIL=SOURCE_LINE=0211: facts["backup_size_bytes"] = backup.get("SIZE_BYTES", "") +DETAIL=SOURCE_LINE=0212: facts["backup_version"] = backup.get("BACKUP_VERSION", "") +DETAIL=SOURCE_LINE=0213: facts["backup_cookie_jar_backed_up"] = backup.get("COOKIE_JAR_BACKED_UP", "") +DETAIL=SOURCE_LINE=0214: facts["backup_extension_health_files_backed_up"] = backup.get("EXTENSION_HEALTH_FILES_BACKED_UP", "") +DETAIL=SOURCE_LINE=0215: facts["backup_title_in_email_subject"] = backup.get("TITLE_IN_EMAIL_SUBJECT", "") +DETAIL=SOURCE_LINE=0216: facts["backup_health_age_seconds"] = str(age_seconds_from_ts(backup.get("TS"))) +DETAIL=SOURCE_LINE=0217: if backup.get("STATUS") != "OK": +DETAIL=SOURCE_LINE=0218: problems.append("backup_status_not_ok") +DETAIL=SOURCE_LINE=0219: if backup.get("COOKIE_JAR_BACKED_UP") != "yes": +DETAIL=SOURCE_LINE=0220: problems.append("backup_cookie_jar_not_marked") +DETAIL=SOURCE_LINE=0221: if backup.get("BACKUP_VERSION") != "4": +DETAIL=SOURCE_LINE=0222: problems.append("backup_version_not_4") +DETAIL=SOURCE_LINE=0223: if backup.get("EXTENSION_HEALTH_FILES_BACKED_UP") != "yes": +DETAIL=SOURCE_LINE=0224: problems.append("backup_extension_health_files_not_marked") +DETAIL=SOURCE_LINE=0225: if backup.get("TITLE_IN_EMAIL_SUBJECT") != "yes": +DETAIL=SOURCE_LINE=0226: problems.append("email_title_subject_not_marked") +DETAIL=SOURCE_LINE=0227: if int(facts["backup_health_age_seconds"]) > BACKUP_HEALTH_AGE_MAX: +DETAIL=SOURCE_LINE=0228: problems.append("backup_health_stale") +DETAIL=SOURCE_LINE=0229: try: +DETAIL=SOURCE_LINE=0230: if int(backup.get("SIZE_BYTES", "0")) <= 0: +DETAIL=SOURCE_LINE=0231: problems.append("backup_size_bad") +DETAIL=SOURCE_LINE=0232: except Exception: +DETAIL=SOURCE_LINE=0233: problems.append("backup_size_bad") +DETAIL=SOURCE_LINE=0234: +DETAIL=SOURCE_LINE=0235: restore = read_env_file(PVE_RESTORE_HEALTH) +DETAIL=SOURCE_LINE=0236: facts["restore_status"] = restore.get("STATUS", "MISSING") +DETAIL=SOURCE_LINE=0237: facts["restore_run_id"] = restore.get("RUN_ID", "") +DETAIL=SOURCE_LINE=0238: facts["restore_health_age_seconds"] = str(age_seconds_from_ts(restore.get("TS"))) +DETAIL=SOURCE_LINE=0239: for key in ["CLOUD_1_STATUS", "CLOUD_2_STATUS", "SHA256_OK", "TAR_OK", "REQUIRED_FILES_OK", "COOKIE_JAR_OK", "EXTENSION_HEALTH_FILES_OK", "EXTENSION_SCRIPTS_OK", "EXTENSION_UNITS_OK", "SQLITE_OK", "PY_COMPILE_OK"]: +DETAIL=SOURCE_LINE=0240: facts[key.lower()] = restore.get(key, "") +DETAIL=SOURCE_LINE=0241: if restore.get("STATUS") != "OK": +DETAIL=SOURCE_LINE=0242: problems.append("restore_status_not_ok") +DETAIL=SOURCE_LINE=0243: if int(facts["restore_health_age_seconds"]) > RESTORE_HEALTH_AGE_MAX: +DETAIL=SOURCE_LINE=0244: problems.append("restore_health_stale") +DETAIL=SOURCE_LINE=0245: for key in ["CLOUD_1_STATUS", "CLOUD_2_STATUS"]: +DETAIL=SOURCE_LINE=0246: if restore.get(key) != "OK": +DETAIL=SOURCE_LINE=0247: problems.append(key.lower() + "_bad") +DETAIL=SOURCE_LINE=0248: for key in ["SHA256_OK", "TAR_OK", "REQUIRED_FILES_OK", "COOKIE_JAR_OK", "EXTENSION_HEALTH_FILES_OK", "EXTENSION_SCRIPTS_OK", "EXTENSION_UNITS_OK", "SQLITE_OK", "PY_COMPILE_OK"]: +DETAIL=SOURCE_LINE=0249: if restore.get(key) != "yes": +DETAIL=SOURCE_LINE=0250: problems.append(key.lower() + "_bad") +DETAIL=SOURCE_LINE=0251: +DETAIL=SOURCE_LINE=0252: local_units = [ +DETAIL=SOURCE_LINE=0253: "skladchik-reports-monitor-backup.timer", +DETAIL=SOURCE_LINE=0254: "skladchik-reports-monitor-backup-restore-check.timer", +DETAIL=SOURCE_LINE=0255: "skladchik-reports-monitor-supervisor.timer", +DETAIL=SOURCE_LINE=0256: "skladchik-reports-monitor-monthly-selftest-watch.timer", +DETAIL=SOURCE_LINE=0257: ] +DETAIL=SOURCE_LINE=0258: props = systemctl_props(local_units) +DETAIL=SOURCE_LINE=0259: for u, p in props.items(): +DETAIL=SOURCE_LINE=0260: base = u.replace(".timer", "").replace("-", "_") +DETAIL=SOURCE_LINE=0261: facts[base + "_timer_enabled"] = p["enabled"] +DETAIL=SOURCE_LINE=0262: facts[base + "_timer_active"] = p["active"] +DETAIL=SOURCE_LINE=0263: if u != "skladchik-reports-monitor-supervisor.timer": +DETAIL=SOURCE_LINE=0264: if p["enabled"] != "enabled" or p["active"] != "active": +DETAIL=SOURCE_LINE=0265: problems.append(base + "_timer_bad") +DETAIL=SOURCE_LINE=0266: else: +DETAIL=SOURCE_LINE=0267: # Before first enable it can be disabled/inactive during test. After enable next runs will see enabled/active. +DETAIL=SOURCE_LINE=0268: if p["enabled"] not in ("enabled", "disabled") or p["active"] not in ("active", "inactive"): +DETAIL=SOURCE_LINE=0269: problems.append(base + "_timer_unknown") +DETAIL=SOURCE_LINE=0270: +DETAIL=SOURCE_LINE=0271: return facts, problems +DETAIL=SOURCE_LINE=0272: +DETAIL=SOURCE_LINE=0273:def render_body(status, facts, problems, mode): +DETAIL=SOURCE_LINE=0274: ru_status = "OK" if status == "OK" else "АВАРИЯ" +DETAIL=SOURCE_LINE=0275: lines = [ +DETAIL=SOURCE_LINE=0276: f"Статус надзора: {ru_status}", +DETAIL=SOURCE_LINE=0277: f"Время UTC: {iso()}", +DETAIL=SOURCE_LINE=0278: "", +DETAIL=SOURCE_LINE=0279: "Монитор жалоб:", +DETAIL=SOURCE_LINE=0280: f"- статус: {facts.get('monitor_status')}", +DETAIL=SOURCE_LINE=0281: f"- всего жалоб на странице: {facts.get('monitor_reports')}", +DETAIL=SOURCE_LINE=0282: f"- открытые/назначенные: {facts.get('monitor_open')}", +DETAIL=SOURCE_LINE=0283: f"- закрытые: {facts.get('monitor_resolved')}", +DETAIL=SOURCE_LINE=0284: f"- возраст health, сек: {facts.get('monitor_health_age_seconds')}", +DETAIL=SOURCE_LINE=0285: f"- автопродление авторизации: {facts.get('session_keepalive')}", +DETAIL=SOURCE_LINE=0286: f"- cookie jar: {facts.get('cookie_jar')} / {facts.get('cookie_jar_bytes')} байт", +DETAIL=SOURCE_LINE=0287: f"- постоянная cookie действует до UTC: {facts.get('cookie_expiry_utc') or 'не определено'}", +DETAIL=SOURCE_LINE=0288: f"- осталось дней по cookie: +DETAIL=SOURCE_LINE=0289: f"- статус срока cookie: +DETAIL=SOURCE_LINE=0290: f"- NetBird route: {facts.get('netbird_route')}", +DETAIL=SOURCE_LINE=0291: "", +DETAIL=SOURCE_LINE=0292: "Бэкап:", +DETAIL=SOURCE_LINE=0293: f"- статус: {facts.get('backup_status')}", +DETAIL=SOURCE_LINE=0294: f"- последний RUN_ID: {facts.get('backup_run_id')}", +DETAIL=SOURCE_LINE=0295: f"- размер архива, байт: {facts.get('backup_size_bytes')}", +DETAIL=SOURCE_LINE=0296: f"- cookie jar в бэкапе: {facts.get('backup_cookie_jar_backed_up')}", +DETAIL=SOURCE_LINE=0297: f"- health-файлы расширений в бэкапе: {facts.get('backup_extension_health_files_backed_up')}", +DETAIL=SOURCE_LINE=0298: f"- название жалобы в теме письма: {facts.get('backup_title_in_email_subject')}", +DETAIL=SOURCE_LINE=0299: f"- возраст health, сек: {facts.get('backup_health_age_seconds')}", +DETAIL=SOURCE_LINE=0300: "", +DETAIL=SOURCE_LINE=0301: "Restore-check:", +DETAIL=SOURCE_LINE=0302: f"- статус: {facts.get('restore_status')}", +DETAIL=SOURCE_LINE=0303: f"- последний RUN_ID: {facts.get('restore_run_id')}", +DETAIL=SOURCE_LINE=0304: f"- cloud1/cloud2: {facts.get('cloud_1_status')}/{facts.get('cloud_2_status')}", +DETAIL=SOURCE_LINE=0305: f"- sha256/tar/cookie/sqlite/python: {facts.get('sha256_ok')}/{facts.get('tar_ok')}/{facts.get('cookie_jar_ok')}/{facts.get('sqlite_ok')}/{facts.get('py_compile_ok')}", +DETAIL=SOURCE_LINE=0306: f"- health/scripts/units расширений: {facts.get('extension_health_files_ok')}/{facts.get('extension_scripts_ok')}/{facts.get('extension_units_ok')}", +DETAIL=SOURCE_LINE=0307: "", +DETAIL=SOURCE_LINE=0308: "Проблемы:", +DETAIL=SOURCE_LINE=0309: ] +DETAIL=SOURCE_LINE=0310: if problems: +DETAIL=SOURCE_LINE=0311: lines.extend([f"- {p}" for p in problems]) +DETAIL=SOURCE_LINE=0312: else: +DETAIL=SOURCE_LINE=0313: lines.append("- нет") +DETAIL=SOURCE_LINE=0314: lines.extend([ +DETAIL=SOURCE_LINE=0315: "", +DETAIL=SOURCE_LINE=0316: "Авторизация:", +DETAIL=SOURCE_LINE=0317: "- ручной ежемесячный перелогин обычно не нужен; монитор сам продлевает сессию постоянным cookie jar при каждом опросе;", +DETAIL=SOURCE_LINE=0318: "- supervisor теперь отдельно следит за сроком постоянной cookie и заранее предупредит, если до истечения останется меньше 7 дней;", +DETAIL=SOURCE_LINE=0319: "- если форум принудительно сбросит сессию, придёт отдельная тревога AUTH_REQUIRED; тогда нужно будет разово обновить вход, но это аварийный сценарий, а не ежемесячная рутина.", +DETAIL=SOURCE_LINE=0320: ]) +DETAIL=SOURCE_LINE=0321: return "\n".join(lines) +DETAIL=SOURCE_LINE=0322: +DETAIL=SOURCE_LINE=0323:def write_health(status, facts, problems, mail_sent): +DETAIL=SOURCE_LINE=0324: STATE_DIR.mkdir(parents=True, exist_ok=True) +DETAIL=SOURCE_LINE=0325: lines = [f"STATUS={status}", f"TS={iso()}", "problems=" + ",".join(problems), f"mail_sent={'yes' if mail_sent else 'no'}"] +DETAIL=SOURCE_LINE=0326: for k in sorted(facts): +DETAIL=SOURCE_LINE=0327: v = str(facts[k]).replace("\n", " ")[:500] +DETAIL=SOURCE_LINE=0328: lines.append(f"{k}={v}") +DETAIL=SOURCE_LINE=0329: HEALTH.write_text("\n".join(lines) + "\n", encoding="utf-8") +DETAIL=SOURCE_LINE=0330: +DETAIL=SOURCE_LINE=0331:def read_state(): +DETAIL=SOURCE_LINE=0332: return read_env_file(STATE) +DETAIL=SOURCE_LINE=0333: +DETAIL=SOURCE_LINE=0334:def write_state(status, problems): +DETAIL=SOURCE_LINE=0335: STATE_DIR.mkdir(parents=True, exist_ok=True) +DETAIL=SOURCE_LINE=0336: STATE.write_text(f"LAST_STATUS={status}\nLAST_PROBLEMS={','.join(problems)}\nTS={iso()}\n", encoding="utf-8") +DETAIL=SOURCE_LINE=0337: +DETAIL=SOURCE_LINE=0338:def main(): +DETAIL=SOURCE_LINE=0339: mode = sys.argv[1] if len(sys.argv) > 1 else "check" +DETAIL=SOURCE_LINE=0340: if mode not in ("check", "test"): +DETAIL=SOURCE_LINE=0341: print("usage: skladchik-reports-monitor-supervisor [check|test]", file=sys.stderr) +DETAIL=SOURCE_LINE=0342: return 2 +DETAIL=SOURCE_LINE=0343: +DETAIL=SOURCE_LINE=0344: facts, problems = gather() +DETAIL=SOURCE_LINE=0345: status = "OK" if not problems else "FAIL" +DETAIL=SOURCE_LINE=0346: old = read_state() +DETAIL=SOURCE_LINE=0347: old_status = old.get("LAST_STATUS", "") +DETAIL=SOURCE_LINE=0348: mail_sent = False +DETAIL=SOURCE_LINE=0349: +DETAIL=SOURCE_LINE=0350: if mode == "test": +DETAIL=SOURCE_LINE=0351: mail_sent = edge_mail("Тест: надзор монитора жалоб и бэкапов", render_body(status, facts, problems, mode)) +DETAIL=SOURCE_LINE=0352: if not mail_sent: +DETAIL=SOURCE_LINE=0353: problems.append("supervisor_mail_send_failed") +DETAIL=SOURCE_LINE=0354: status = "FAIL" +DETAIL=SOURCE_LINE=0355: elif status != "OK": +DETAIL=SOURCE_LINE=0356: if old_status != "FAIL": +DETAIL=SOURCE_LINE=0357: mail_sent = edge_mail("Авария: монитор жалоб или его бэкапы", render_body(status, facts, problems, mode)) +DETAIL=SOURCE_LINE=0358: if not mail_sent: +DETAIL=SOURCE_LINE=0359: problems.append("supervisor_mail_send_failed") +DETAIL=SOURCE_LINE=0360: status = "FAIL" +DETAIL=SOURCE_LINE=0361: elif old_status == "FAIL": +DETAIL=SOURCE_LINE=0362: mail_sent = edge_mail("Восстановлено: монитор жалоб и бэкапы снова OK", render_body(status, facts, problems, mode)) +DETAIL=SOURCE_LINE=0363: +DETAIL=SOURCE_LINE=0364: write_health(status, facts, problems, mail_sent) +DETAIL=SOURCE_LINE=0365: write_state(status, problems) +DETAIL=SOURCE_LINE=0366: +DETAIL=SOURCE_LINE=0367: print(f"SUPERVISOR_STATUS={status}") +DETAIL=SOURCE_LINE=0368: print(f"SUPERVISOR_MAIL_SENT={'yes' if mail_sent else 'no'}") +DETAIL=SOURCE_LINE=0369: print(f"SUPERVISOR_PROBLEMS={','.join(problems)}") +DETAIL=SOURCE_LINE=0370: print(f"MONITOR={facts.get('monitor_status')} reports={facts.get('monitor_reports')} open={facts.get('monitor_open')} resolved={facts.get('monitor_resolved')}") +DETAIL=SOURCE_LINE=0371: print(f"BACKUP={facts.get('backup_status')} run={facts.get('backup_run_id')}") +DETAIL=SOURCE_LINE=0372: print(f"RESTORE={facts.get('restore_status')} run={facts.get('restore_run_id')}") +DETAIL=SOURCE_LINE=0373: log(f"CHECK mode={mode} status={status} mail_sent={'yes' if mail_sent else 'no'} problems={problems}") +DETAIL=SOURCE_LINE=0374: return 0 if status == "OK" else 1 +DETAIL=SOURCE_LINE=0375: +DETAIL=SOURCE_LINE=0376:if __name__ == "__main__": +DETAIL=SOURCE_LINE=0377: sys.exit(main()) +DETAIL=SCRIPT_TOTAL_LINES=377 +DETAIL=RELEVANT_SOURCE_END +DETAIL=UNIT_CONTRACT_BEGIN +DETAIL=UNIT_NAME=skladchik-reports-monitor-supervisor.service +DETAIL=UNIT_LINE=# /etc/systemd/system/skladchik-reports-monitor-supervisor.service +DETAIL=UNIT_LINE=[Unit] +DETAIL=UNIT_LINE=Description=Skladchik reports monitor full supervisor +DETAIL=UNIT_LINE=After=network-online.target +DETAIL=UNIT_LINE= +DETAIL=UNIT_LINE=[Service] +DETAIL=UNIT_LINE=Type=oneshot +DETAIL=UNIT_LINE=ExecStart=/usr/local/sbin/skladchik-reports-monitor-supervisor check +DETAIL=UNIT_NAME=skladchik-reports-monitor-supervisor.timer +DETAIL=UNIT_LINE=# /etc/systemd/system/skladchik-reports-monitor-supervisor.timer +DETAIL=UNIT_LINE=[Unit] +DETAIL=UNIT_LINE=Description=Run Skladchik reports monitor full supervisor every 10 minutes +DETAIL=UNIT_LINE= +DETAIL=UNIT_LINE=[Timer] +DETAIL=UNIT_LINE=OnBootSec=5min +DETAIL=UNIT_LINE=OnUnitActiveSec=10min +DETAIL=UNIT_LINE=AccuracySec=30s +DETAIL=UNIT_LINE=Persistent=false +DETAIL=UNIT_LINE=Unit=skladchik-reports-monitor-supervisor.service +DETAIL=UNIT_LINE= +DETAIL=UNIT_LINE=[Install] +DETAIL=UNIT_LINE=WantedBy=timers.target +DETAIL=UNIT_CONTRACT_END +DETAIL=TIMER_STATE_BEGIN +DETAIL=TIMER_PROPERTY=Unit=skladchik-reports-monitor-supervisor.service +DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime= +DETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 11:58:38 MSK +DETAIL=TIMER_PROPERTY=LoadState=loaded +DETAIL=TIMER_PROPERTY=ActiveState=active +DETAIL=TIMER_PROPERTY=SubState=waiting +DETAIL=TIMER_ROW=Fri 2026-07-24 12:08:38 MSK 3min 34s Fri 2026-07-24 11:58:38 MSK 6min ago skladchik-reports-monitor-supervisor.timer skladchik-reports-monitor-supervisor.service +DETAIL=TIMER_ROW=Sat 2026-07-25 04:39:05 MSK 16h Fri 2026-07-24 04:28:08 MSK 7h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service +DETAIL=TIMER_ROW=Sat 2026-07-25 10:15:00 MSK 22h Fri 2026-07-24 10:15:04 MSK 1h 49min ago skladchik-reports-monitor-monthly-selftest-watch.timer skladchik-reports-monitor-monthly-selftest-watch.service +DETAIL=TIMER_ROW=Sun 2026-07-26 05:15:13 MSK 1 day 17h Sun 2026-07-19 05:11:25 MSK - skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service +DETAIL=TIMER_STATE_END DETAIL=SECRET_VALUES_OUTPUT=NONE -DETAIL=SUPERVISOR_FAILURE_TRACE=PASS -CHANGES_MADE=NO +DETAIL=CHANGES_MADE=NO +DETAIL=COOKIE_CHECK_SOURCE_AUDIT=PASS OUTPUT_END CHAT_OUTPUT_END