From f6568215f4185dc6acdad6304a041c0358e4af35 Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Fri, 24 Jul 2026 09:06:50 +0000 Subject: [PATCH] runtime: publish HOMELAB-SKLADCHIK-COOKIE-EXACT-LOGIC-419 --- ...ELAB-SKLADCHIK-COOKIE-EXACT-LOGIC-419.json | 30 ++ ...MELAB-SKLADCHIK-COOKIE-EXACT-LOGIC-419.txt | 267 ++++++++++++++++++ runtime/latest.json | 12 +- runtime/latest.txt | 219 ++------------ 4 files changed, 331 insertions(+), 197 deletions(-) create mode 100644 runtime/history/HOMELAB-SKLADCHIK-COOKIE-EXACT-LOGIC-419.json create mode 100644 runtime/history/HOMELAB-SKLADCHIK-COOKIE-EXACT-LOGIC-419.txt diff --git a/runtime/history/HOMELAB-SKLADCHIK-COOKIE-EXACT-LOGIC-419.json b/runtime/history/HOMELAB-SKLADCHIK-COOKIE-EXACT-LOGIC-419.json new file mode 100644 index 0000000..ec64093 --- /dev/null +++ b/runtime/history/HOMELAB-SKLADCHIK-COOKIE-EXACT-LOGIC-419.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "HOMELAB-SKLADCHIK-COOKIE-EXACT-LOGIC-419", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "cluster-audit", + "started_at_utc": "2026-07-24T09:06:49Z", + "finished_at_utc": "2026-07-24T09:06:49Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", + "command_sha256": "77896618b46e2f97362240deb743fb8e625075381e57377ee0314130be2b2026", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "63d14ea58a82bd34d78884b4db6d047e8fa64822200c7bc20a8587057f3f9e25", + "sanitized_output_sha256": "702451cb7bfbc7d831b17b7e062a00f5215e5dc3729bd8e3331dfe4c3dc28851", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "DETAIL=SCRIPT_SHA256=35c68ae0929ff3795be2bde40de334fc8e26413a3d8a620a137be90d1f2c164e\nDETAIL=EXACT_COOKIE_LOGIC_BEGIN\nDETAIL=SOURCE_BLOCK=TOP_LEVEL_CONSTANT|LINES=12-12\nDETAIL=SOURCE_LINE=0012:EDGE = os.environ.get(\"SKLADCHIK_EDGE\", \"debian@[PRIVATE_IP]\")\nDETAIL=SOURCE_BLOCK=TOP_LEVEL_CONSTANT|LINES=22-22\nDETAIL=SOURCE_LINE=0022:COOKIE_EXPIRY_WARN_SECONDS = int(os.environ.get(\"SKLADCHIK_COOKIE_EXPIRY_WARN_SECONDS\", \"604800\"))\nDETAIL=SOURCE_BLOCK=FUNCTION:gather|LINES=105-271\nDETAIL=SOURCE_LINE=0105:def gather():\nDETAIL=SOURCE_LINE=0106: facts = {}\nDETAIL=SOURCE_LINE=0107: problems = []\nDETAIL=SOURCE_LINE=0108:\nDETAIL=SOURCE_LINE=0109: edge_cmd = r'''\nDETAIL=SOURCE_LINE=0110:set +e\nDETAIL=SOURCE_LINE=0111:echo __MONITOR_HEALTH_BEGIN__\nDETAIL=SOURCE_LINE=0112:cat /var/lib/skladchik-reports-monitor/health.txt 2>/dev/null || true\nDETAIL=SOURCE_LINE=0113:echo __MONITOR_HEALTH_END__\nDETAIL=SOURCE_LINE=0114:echo __ALERT_HEALTH_BEGIN__\nDETAIL=SOURCE_LINE=0115:cat /var/lib/skladchik-reports-monitor/alert-health.txt 2>/dev/null || true\nDETAIL=SOURCE_LINE=0116:echo __ALERT_HEALTH_END__\nDETAIL=SOURCE_LINE=0117:echo __TIMERS_BEGIN__\nDETAIL=SOURCE_LINE=0118:for u in skladchik-reports-monitor.timer skladchik-reports-monitor-watchdog.timer skladchik-reports-monitor-heartbeat.timer skladchik-reports-monitor-weekly-summary.timer skladchik-reports-monitor-age-alert.timer skladchik-reports-monitor-event-selftest.timer; do echo \"$u enabled=$(systemctl is-enabled \"$u\" 2>/dev/null) active=$(systemctl is-active \"$u\" 2>/dev/null)\"; done\nDETAIL=SOURCE_LINE=0119:echo __TIMERS_END__\nDETAIL=SOURCE_LINE=0120:echo __ROUTE_BEGIN__\nDETAIL=SOURCE_LINE=0121:ip route get 1.1.1.1 2>&1 | head -n 1\nDETAIL=SOURCE_LINE=0122:echo __ROUTE_END__\nDETAIL=SOURCE_LINE=0123:echo __FILES_BEGIN__\nDETAIL=SOURCE_LINE=0124:test -s /opt/skladchik-reports-monitor/monitor.py && echo monitor_py=yes || echo monitor_py=no\nDETAIL=SOURCE_LINE=0125:test -s /etc/skladchik-reports-monitor/config.ini && echo config_ini=yes || echo config_ini=no\nDETAIL=SOURCE_LINE=0126:test -s /etc/skladchik-reports-monitor/secrets.env && echo secrets_env=\nDETAIL=SOURCE_LINE=0127:test -s /var/lib/skladchik-reports-monitor/cookies.jar && echo cookies_jar=yes || echo cookies_jar=no\nDETAIL=SOURCE_LINE=0128:[ -s /var/lib/skladchik-reports-monitor/cookies.jar ] && echo cookies_jar_bytes=$(stat -c %s /var/lib/skladchik-reports-monitor/cookies.jar 2>/dev/null) || echo cookies_jar_bytes=0\nDETAIL=SOURCE_LINE=0129:test -s /opt/skladchik-reports-monitor/operational_digest.py && echo operational_digest_py=yes || echo operational_digest_py=no\nDETAIL=SOURCE_LINE=0130:test -s /opt/skladchik-reports-monitor/event-selftest-runner.py && echo event_selftest_runner_py=yes || echo event_selftest_runner_py=no\nDETAIL=SOURCE_LINE=0131:test -s /opt/skladchik-reports-monitor/event-selftest-core && echo event_selftest_core=yes || echo event_selftest_core=no\nDETAIL=SOURCE_LINE=0132:test -s /var/lib/skladchik-reports-monitor/weekly-summary-health.txt && echo weekly_summary_health=yes || echo weekly_summary_health=no\nDETAIL=SOURCE_LINE=0133:test -s /var/lib/skladchik-reports-monitor/age-alert-health.txt && echo age_alert_health=yes || echo age_alert_health=no\nDETAIL=SOURCE_LINE=0134:test -s /var/lib/skladchik-reports-monitor/event-selftest-health.txt && echo event_selftest_health=yes || echo event_selftest_health=no\nDETAIL=SOURCE_LINE=0135:python3 -m py_compile /opt/skladchik-reports-monitor/monitor.py /opt/skladchik-reports-monitor/operational_digest.py /opt/skladchik-reports-monitor/event-selftest-runner.py /opt/skladchik-reports-monitor/event-selftest-core >/dev/null 2>&1 && echo py_compile=yes || echo py_compile=no\nDETAIL=SOURCE_LINE=0136:echo __FILES_END__\nDETAIL=SOURCE_LINE=0137:'''\nDETAIL=SOURCE_LINE=0138: rc, out = ssh_edge(edge_cmd, timeout=45)\nDETAIL=SOURCE_LINE=0139: facts[\"edge_ssh_rc\"] = str(rc)\nDETAIL=SOURCE_LINE=0140: if rc != 0:\nDETAIL=SOURCE_LINE=0141: problems.append(\"edge_ssh_failed\")\nDETAIL=SOURCE_LINE=0142:\nDETAIL=SOURCE_LINE=0143: def section(name):\nDETAIL=SOURCE_LINE=0144: m = re.search(rf\"__{name}_BEGIN__\\n(.*?)\\n__{name}_END__\", out, re.S)\nDETAIL=SOURCE_LINE=0145: return m.group(1).strip() if m else \"\"\nDETAIL=SOURCE_LINE=0146:\nDETAIL=SOURCE_LINE=0147: mon = parse_env_text(section(\"MONITOR_HEALTH\"))\nDETAIL=SOURCE_LINE=0148: facts[\"monitor_status\"] = mon.get(\"STATUS\", \"MISSING\")\nDETAIL=SOURCE_LINE=0149: facts[\"monitor_reports\"] = mon.get(\"reports\", \"\")\nDETAIL=SOURCE_LINE=0150: facts[\"monitor_open\"] = mon.get(\"open\", \"\")\nDETAIL=SOURCE_LINE=0151: facts[\"monitor_resolved\"] = mon.get(\"resolved\", \"\")\nDETAIL=SOURCE_LINE=0152: facts[\"monitor_smtp_configured\"] = mon.get(\"smtp_configured\", \"\")\nDETAIL=SOURCE_LINE=0153: facts[\"session_keepalive\"] = mon.get(\"session_keepalive\", \"\")\nDETAIL=SOURCE_LINE=0154: facts[\"cookie_jar\"] = mon.get(\"cookie_jar\", \"\")\nDETAIL=SOURCE_LINE=0155: facts[\"cookie_jar_bytes\"] = mon.get(\"cookie_jar_bytes\", \"\")\nDETAIL=SOURCE_LINE=0156: facts[\"cookie_expiry_status\"] = mon.get(\"cookie_expiry_status\", \"\")\nDETAIL=SOURCE_LINE=0157: facts[\"cookie_expiry_utc\"] = mon.get(\"cookie_expiry_utc\", \"\")\nDETAIL=SOURCE_LINE=0158: facts[\"cookie_expires_in_seconds\"] = mon.get(\"cookie_expires_in_seconds\", \"\")\nDETAIL=SOURCE_LINE=0159: facts[\"cookie_expires_in_days\"] = mon.get(\"cookie_expires_in_days\", \"\")\nDETAIL=SOURCE_LINE=0160: facts[\"cookie_persistent_count\"] = mon.get(\"cookie_persistent_count\", \"\")\nDETAIL=SOURCE_LINE=0161: facts[\"monitor_health_age_seconds\"] = str(age_seconds_from_ts(mon.get(\"TS\")))\nDETAIL=SOURCE_LINE=0162: if mon.get(\"STATUS\") != \"OK\":\nDETAIL=SOURCE_LINE=0163: problems.append(\"monitor_status_not_ok\")\nDETAIL=SOURCE_LINE=0164: if int(facts[\"monitor_health_age_seconds\"]) > MONITOR_HEALTH_AGE_MAX:\nDETAIL=SOURCE_LINE=0165: problems.append(\"monitor_health_stale\")\nDETAIL=SOURCE_LINE=0166: if mon.get(\"reports\") in (\"\", \"0\"):\nDETAIL=SOURCE_LINE=0167: problems.append(\"monitor_reports_zero_or_missing\")\nDETAIL=SOURCE_LINE=0168: if mon.get(\"session_keepalive\") != \"persistent_cookie_jar\":\nDETAIL=SOURCE_LINE=0169: problems.append(\"session_keepalive_not_persistent\")\nDETAIL=SOURCE_LINE=0170: if mon.get(\"cookie_jar\") not in (\"yes\", \"empty\"):\nDETAIL=SOURCE_LINE=0171: problems.append(\"monitor_cookie_jar_bad\")\nDETAIL=SOURCE_LINE=0172: expiry_status = mon.get(\"cookie_expiry_status\", \"\")\nDETAIL=SOURCE_LINE=0173: try:\nDETAIL=SOURCE_LINE=0174: expires_in = int(mon.get(\"cookie_expires_in_seconds\", \"\"))\nDETAIL=SOURCE_LINE=0175: except Exception:\nDETAIL=SOURCE_LINE=0176: expires_in = None\nDETAIL=SOURCE_LINE=0177: if expiry_status in (\"\", \"NO_COOKIE_JAR\", \"NO_PERSISTENT_COOKIE\", \"ERROR\"):\nDETAIL=SOURCE_LINE=0178: problems.append(\"cookie_expiry_missing\")\nDETAIL=SOURCE_LINE=0179: elif expiry_status == \"EXPIRED\" or (expires_in is not None and expires_in <= 0):\nDETAIL=SOURCE_LINE=0180: problems.append(\"cookie_expiry_expired\")\nDETAIL=SOURCE_LINE=0181: elif expiry_status == \"WARN\" or (expires_in is not None and expires_in < COOKIE_EXPIRY_WARN_SECONDS):\nDETAIL=SOURCE_LINE=0182: problems.append(\"cookie_expiry_warn\")\nDETAIL=SOURCE_LINE=0183:\nDETAIL=SOURCE_LINE=0184: timers_text = section(\"TIMERS\")\nDETAIL=SOURCE_LINE=0185: for unit in [\"skladchik-reports-monitor.timer\", \"skladchik-reports-monitor-watchdog.timer\", \"skladchik-reports-monitor-heartbeat.timer\", \"skladchik-reports-monitor-weekly-summary.timer\", \"skladchik-reports-monitor-age-alert.timer\", \"skladchik-reports-monitor-event-selftest.timer\"]:\nDETAIL=SOURCE_LINE=0186: safe = unit.replace(\".timer\", \"\").replace(\"_\", \"-\")\nDETAIL=SOURCE_LINE=0187: m = re.search(re.escape(unit) + r\"\\s+enabled=([^\\s]+)\\s+active=([^\\s]+)\", timers_text)\nDETAIL=SOURCE_LINE=0188: enabled = m.group(1) if m else \"missing\"\nDETAIL=SOURCE_LINE=0189: active = m.group(2) if m else \"missing\"\nDETAIL=SOURCE_LINE=0190: facts[f\"edge_{safe}_timer_enabled\"] = enabled\nDETAIL=SOURCE_LINE=0191: facts[f\"edge_{safe}_timer_active\"] = active\nDETAIL=SOURCE_LINE=0192: if enabled != \"enabled\" or active != \"active\":\nDETAIL=SOURCE_LINE=0193: problems.append(f\"edge_{safe}_timer_bad\")\nDETAIL=SOURCE_LINE=0194:\nDETAIL=SOURCE_LINE=0195: route_lines = section(\"ROUTE\").splitlines()\nDETAIL=SOURCE_LINE=0196: route = route_lines[0] if route_lines else \"\"\nDETAIL=SOURCE_LINE=0197: facts[\"netbird_route\"] = route\nDETAIL=SOURCE_LINE=0198: if \" wt0 \" not in (\" \" + route + \" \"):\nDETAIL=SOURCE_LINE=0199: problems.append(\"netbird_route_not_wt0\")\nDETAIL=SOURCE_LINE=0200:\nDETAIL=SOURCE_LINE=0201: files = parse_env_text(section(\"FILES\"))\nDETAIL=SOURCE_LINE=0202: for k in [\"monitor_py\", \"config_ini\", \"secrets_env\", \"cookies_jar\", \"operational_digest_py\", \"event_selftest_runner_py\", \"event_selftest_core\", \"weekly_summary_health\", \"age_alert_health\", \"event_selftest_health\", \"py_compile\"]:\nDETAIL=SOURCE_LINE=0203: facts[k] = files.get(k, \"no\")\nDETAIL=SOURCE_LINE=0204: if facts[k] != \"yes\":\nDETAIL=SOURCE_LINE=0205: problems.append(f\"edge_{k}_bad\")\nDETAIL=SOURCE_LINE=0206: facts[\"cookies_jar_bytes_file\"] = files.get(\"cookies_jar_bytes\", \"0\")\nDETAIL=SOURCE_LINE=0207:\nDETAIL=SOURCE_LINE=0208: backup = read_env_file(PVE_BACKUP_HEALTH)\nDETAIL=SOURCE_LINE=0209: facts[\"backup_status\"] = backup.get(\"STATUS\", \"MISSING\")\nDETAIL=SOURCE_LINE=0210: facts[\"backup_run_id\"] = backup.get(\"RUN_ID\", \"\")\nDETAIL=SOURCE_LINE=0211: facts[\"backup_size_bytes\"] = backup.get(\"SIZE_BYTES\", \"\")\nDETAIL=SOURCE_LINE=0212: facts[\"backup_version\"] = backup.get(\"BACKUP_VERSION\", \"\")\nDETAIL=SOURCE_LINE=0213: facts[\"backup_cookie_jar_backed_up\"] = backup.get(\"COOKIE_JAR_BACKED_UP\", \"\")\nDETAIL=SOURCE_LINE=0214: facts[\"backup_extension_health_files_backed_up\"] = backup.get(\"EXTENSION_HEALTH_FILES_BACKED_UP\", \"\")\nDETAIL=SOURCE_LINE=0215: facts[\"backup_title_in_email_subject\"] = backup.get(\"TITLE_IN_EMAIL_SUBJECT\", \"\")\nDETAIL=SOURCE_LINE=0216: facts[\"backup_health_age_seconds\"] = str(age_seconds_from_ts(backup.get(\"TS\")))\nDETAIL=SOURCE_LINE=0217: if backup.get(\"STATUS\") != \"OK\":\nDETAIL=SOURCE_LINE=0218: problems.append(\"backup_status_not_ok\")\nDETAIL=SOURCE_LINE=0219: if backup.get(\"COOKIE_JAR_BACKED_UP\") != \"yes\":\nDETAIL=SOURCE_LINE=0220: problems.append(\"backup_cookie_jar_not_marked\")\nDETAIL=SOURCE_LINE=0221: if backup.get(\"BACKUP_VERSION\") != \"4\":\nDETAIL=SOURCE_LINE=0222: problems.append(\"backup_version_not_4\")\nDETAIL=SOURCE_LINE=0223: if backup.get(\"EXTENSION_HEALTH_FILES_BACKED_UP\") != \"yes\":\nDETAIL=SOURCE_LINE=0224: problems.append(\"backup_extension_health_files_not_marked\")\nDETAIL=SOURCE_LINE=0225: if backup.get(\"TITLE_IN_EMAIL_SUBJECT\") != \"yes\":\nDETAIL=SOURCE_LINE=0226: problems.append(\"email_title_subject_not_marked\")\nDETAIL=SOURCE_LINE=0227: if int(facts[\"backup_health_age_seconds\"]) > BACKUP_HEALTH_AGE_MAX:\nDETAIL=SOURCE_LINE=0228: problems.append(\"backup_health_stale\")\nDETAIL=SOURCE_LINE=0229: try:\nDETAIL=SOURCE_LINE=0230: if int(backup.get(\"SIZE_BYTES\", \"0\")) <= 0:\nDETAIL=SOURCE_LINE=0231: problems.append(\"backup_size_bad\")\nDETAIL=SOURCE_LINE=0232: except Exception:\nDETAIL=SOURCE_LINE=0233: problems.append(\"backup_size_bad\")\nDETAIL=SOURCE_LINE=0234:\nDETAIL=SOURCE_LINE=0235: restore = read_env_file(PVE_RESTORE_HEALTH)\nDETAIL=SOURCE_LINE=0236: facts[\"restore_status\"] = restore.get(\"STATUS\", \"MISSING\")\nDETAIL=SOURCE_LINE=0237: facts[\"restore_run_id\"] = restore.get(\"RUN_ID\", \"\")\nDETAIL=SOURCE_LINE=0238: facts[\"restore_health_age_seconds\"] = str(age_seconds_from_ts(restore.get(\"TS\")))\nDETAIL=SOURCE_LINE=0239: for key in [\"CLOUD_1_STATUS\", \"CLOUD_2_STATUS\", \"SHA256_OK\", \"TAR_OK\", \"REQUIRED_FILES_OK\", \"COOKIE_JAR_OK\", \"EXTENSION_HEALTH_FILES_OK\", \"EXTENSION_SCRIPTS_OK\", \"EXTENSION_UNITS_OK\", \"SQLITE_OK\", \"PY_COMPILE_OK\"]:\nDETAIL=SOURCE_LINE=0240: facts[key.lower()] = restore.get(key, \"\")\nDETAIL=SOURCE_LINE=0241: if restore.get(\"STATUS\") != \"OK\":\nDETAIL=SOURCE_LINE=0242: problems.append(\"restore_status_not_ok\")\nDETAIL=SOURCE_LINE=0243: if int(facts[\"restore_health_age_seconds\"]) > RESTORE_HEALTH_AGE_MAX:\nDETAIL=SOURCE_LINE=0244: problems.append(\"restore_health_stale\")\nDETAIL=SOURCE_LINE=0245: for key in [\"CLOUD_1_STATUS\", \"CLOUD_2_STATUS\"]:\nDETAIL=SOURCE_LINE=0246: if restore.get(key) != \"OK\":\nDETAIL=SOURCE_LINE=0247: problems.append(key.lower() + \"_bad\")\nDETAIL=SOURCE_LINE=0248: for key in [\"SHA256_OK\", \"TAR_OK\", \"REQUIRED_FILES_OK\", \"COOKIE_JAR_OK\", \"EXTENSION_HEALTH_FILES_OK\", \"EXTENSION_SCRIPTS_OK\", \"EXTENSION_UNITS_OK\", \"SQLITE_OK\", \"PY_COMPILE_OK\"]:\nDETAIL=SOURCE_LINE=0249: if restore.get(key) != \"yes\":\nDETAIL=SOURCE_LINE=0250: problems.append(key.lower() + \"_bad\")\nDETAIL=SOURCE_LINE=0251:\nDETAIL=SOURCE_LINE=0252: local_units = [\nDETAIL=SOURCE_LINE=0253: \"skladchik-reports-monitor-backup.timer\",\nDETAIL=SOURCE_LINE=0254: \"skladchik-reports-monitor-backup-restore-check.timer\",\nDETAIL=SOURCE_LINE=0255: \"skladchik-reports-monitor-supervisor.timer\",\nDETAIL=SOURCE_LINE=0256: \"skladchik-reports-monitor-monthly-selftest-watch.timer\",\nDETAIL=SOURCE_LINE=0257: ]\nDETAIL=SOURCE_LINE=0258: props = systemctl_props(local_units)\nDETAIL=SOURCE_LINE=0259: for u, p in props.items():\nDETAIL=SOURCE_LINE=0260: base = u.replace(\".timer\", \"\").replace(\"-\", \"_\")\nDETAIL=SOURCE_LINE=0261: facts[base + \"_timer_enabled\"] = p[\"enabled\"]\nDETAIL=SOURCE_LINE=0262: facts[base + \"_timer_active\"] = p[\"active\"]\nDETAIL=SOURCE_LINE=0263: if u != \"skladchik-reports-monitor-supervisor.timer\":\nDETAIL=SOURCE_LINE=0264: if p[\"enabled\"] != \"enabled\" or p[\"active\"] != \"active\":\nDETAIL=SOURCE_LINE=0265: problems.append(base + \"_timer_bad\")\nDETAIL=SOURCE_LINE=0266: else:\nDETAIL=SOURCE_LINE=0267: # Before first enable it can be disabled/inactive during test. After enable next runs will see enabled/active.\nDETAIL=SOURCE_LINE=0268: if p[\"enabled\"] not in (\"enabled\", \"disabled\") or p[\"active\"] not in (\"active\", \"inactive\"):\nDETAIL=SOURCE_LINE=0269: problems.append(base + \"_timer_unknown\")\nDETAIL=SOURCE_LINE=0270:\nDETAIL=SOURCE_LINE=0271: return facts, problems\nDETAIL=SOURCE_BLOCK=FUNCTION:render_body|LINES=273-321\nDETAIL=SOURCE_LINE=0273:def render_body(status, facts, problems, mode):\nDETAIL=SOURCE_LINE=0274: ru_status = \"OK\" if status == \"OK\" else \"АВАРИЯ\"\nDETAIL=SOURCE_LINE=0275: lines = [\nDETAIL=SOURCE_LINE=0276: f\"Статус надзора: {ru_status}\",\nDETAIL=SOURCE_LINE=0277: f\"Время UTC: {iso()}\",\nDETAIL=SOURCE_LINE=0278: \"\",\nDETAIL=SOURCE_LINE=0279: \"Монитор жалоб:\",\nDETAIL=SOURCE_LINE=0280: f\"- статус: {facts.get('monitor_status')}\",\nDETAIL=SOURCE_LINE=0281: f\"- всего жалоб на странице: {facts.get('monitor_reports')}\",\nDETAIL=SOURCE_LINE=0282: f\"- открытые/назначенные: {facts.get('monitor_open')}\",\nDETAIL=SOURCE_LINE=0283: f\"- закрытые: {facts.get('monitor_resolved')}\",\nDETAIL=SOURCE_LINE=0284: f\"- возраст health, сек: {facts.get('monitor_health_age_seconds')}\",\nDETAIL=SOURCE_LINE=0285: f\"- автопродление авторизации: {facts.get('session_keepalive')}\",\nDETAIL=SOURCE_LINE=0286: f\"- cookie jar: {facts.get('cookie_jar')} / {facts.get('cookie_jar_bytes')} байт\",\nDETAIL=SOURCE_LINE=0287: f\"- постоянная cookie действует до UTC: {facts.get('cookie_expiry_utc') or 'не определено'}\",\nDETAIL=SOURCE_LINE=0288: f\"- осталось дней по cookie: {facts.get('cookie_expires_in_days') or 'не определено'}\",\nDETAIL=SOURCE_LINE=0289: f\"- статус срока cookie: {facts.get('cookie_expiry_status') or 'не определено'}\",\nDETAIL=SOURCE_LINE=0290: f\"- NetBird route: {facts.get('netbird_route')}\",\nDETAIL=SOURCE_LINE=0291: \"\",\nDETAIL=SOURCE_LINE=0292: \"Бэкап:\",\nDETAIL=SOURCE_LINE=0293: f\"- статус: {facts.get('backup_status')}\",\nDETAIL=SOURCE_LINE=0294: f\"- последний RUN_ID: {facts.get('backup_run_id')}\",\nDETAIL=SOURCE_LINE=0295: f\"- размер архива, байт: {facts.get('backup_size_bytes')}\",\nDETAIL=SOURCE_LINE=0296: f\"- cookie jar в бэкапе: {facts.get('backup_cookie_jar_backed_up')}\",\nDETAIL=SOURCE_LINE=0297: f\"- health-файлы расширений в бэкапе: {facts.get('backup_extension_health_files_backed_up')}\",\nDETAIL=SOURCE_LINE=0298: f\"- название жалобы в теме письма: {facts.get('backup_title_in_email_subject')}\",\nDETAIL=SOURCE_LINE=0299: f\"- возраст health, сек: {facts.get('backup_health_age_seconds')}\",\nDETAIL=SOURCE_LINE=0300: \"\",\nDETAIL=SOURCE_LINE=0301: \"Restore-check:\",\nDETAIL=SOURCE_LINE=0302: f\"- статус: {facts.get('restore_status')}\",\nDETAIL=SOURCE_LINE=0303: f\"- последний RUN_ID: {facts.get('restore_run_id')}\",\nDETAIL=SOURCE_LINE=0304: f\"- cloud1/cloud2: {facts.get('cloud_1_status')}/{facts.get('cloud_2_status')}\",\nDETAIL=SOURCE_LINE=0305: f\"- sha256/tar/cookie/sqlite/python: {facts.get('sha256_ok')}/{facts.get('tar_ok')}/{facts.get('cookie_jar_ok')}/{facts.get('sqlite_ok')}/{facts.get('py_compile_ok')}\",\nDETAIL=SOURCE_LINE=0306: f\"- health/scripts/units расширений: {facts.get('extension_health_files_ok')}/{facts.get('extension_scripts_ok')}/{facts.get('extension_units_ok')}\",\nDETAIL=SOURCE_LINE=0307: \"\",\nDETAIL=SOURCE_LINE=0308: \"Проблемы:\",\nDETAIL=SOURCE_LINE=0309: ]\nDETAIL=SOURCE_LINE=0310: if problems:\nDETAIL=SOURCE_LINE=0311: lines.extend([f\"- {p}\" for p in problems])\nDETAIL=SOURCE_LINE=0312: else:\nDETAIL=SOURCE_LINE=0313: lines.append(\"- нет\")\nDETAIL=SOURCE_LINE=0314: lines.extend([\nDETAIL=SOURCE_LINE=0315: \"\",\nDETAIL=SOURCE_LINE=0316: \"Авторизация:\",\nDETAIL=SOURCE_LINE=0317: \"- ручной ежемесячный перелогин обычно не нужен; монитор сам продлевает сессию постоянным cookie jar при каждом опросе;\",\nDETAIL=SOURCE_LINE=0318: \"- supervisor теперь отдельно следит за сроком постоянной cookie и заранее предупредит, если до истечения останется меньше 7 дней;\",\nDETAIL=SOURCE_LINE=0319: \"- если форум принудительно сбросит сессию, придёт отдельная тревога AUTH_REQUIRED; тогда нужно будет разово обновить вход, но это аварийный сценарий, а не ежемесячная рутина.\",\nDETAIL=SOURCE_LINE=0320: ])\nDETAIL=SOURCE_LINE=0321: return \"\\n\".join(lines)\nDETAIL=EXACT_COOKIE_LOGIC_END\nDETAIL=PVE01_TIMER_CORRELATION_BEGIN\nDETAIL=PVE01_TIMER=Fri 2026-07-24 12:08:38 MSK 1min 48s Fri 2026-07-24 11:58:38 MSK 8min ago skladchik-reports-monitor-supervisor.timer skladchik-reports-monitor-supervisor.service\nDETAIL=PVE01_TIMER=Sat 2026-07-25 04:39:05 MSK 16h Fri 2026-07-24 04:28:08 MSK 7h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service\nDETAIL=PVE01_TIMER=Sat 2026-07-25 10:15:00 MSK 22h Fri 2026-07-24 10:15:04 MSK 1h 51min ago skladchik-reports-monitor-monthly-selftest-watch.timer skladchik-reports-monitor-monthly-selftest-watch.service\nDETAIL=PVE01_TIMER=Sun 2026-07-26 05:15:13 MSK 1 day 17h Sun 2026-07-19 05:11:25 MSK - skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service\nDETAIL=PVE01_TIMER_CORRELATION_END\nDETAIL=EDGE_TIMER_CORRELATION_BEGIN\nDETAIL=EDGE_TIMER=Fri 2026-07-24 12:06:51 MSK 1s Fri 2026-07-24 12:06:21 MSK 28s ago skladchik-reports-monitor.timer skladchik-reports-monitor.service\nDETAIL=EDGE_TIMER=Fri 2026-07-24 12:08:35 MSK 1min 45s Fri 2026-07-24 12:03:35 MSK 3min 14s ago skladchik-reports-monitor-watchdog.timer skladchik-reports-monitor-watchdog.service\nDETAIL=EDGE_TIMER=Sat 2026-07-25 09:16:28 MSK 21h Fri 2026-07-24 09:18:35 MSK 2h 48min ago skladchik-reports-monitor-heartbeat.timer skladchik-reports-monitor-heartbeat.service\nDETAIL=EDGE_TIMER=Sat 2026-07-25 09:45:00 MSK 21h Fri 2026-07-24 09:45:03 MSK 2h 21min ago skladchik-reports-monitor-age-alert.timer skladchik-reports-monitor-age-alert.service\nDETAIL=EDGE_TIMER=Mon 2026-07-27 09:35:00 MSK 2 days Mon 2026-07-20 09:35:06 MSK - skladchik-reports-monitor-weekly-summary.timer skladchik-reports-monitor-weekly-summary.service\nDETAIL=EDGE_TIMER=Sat 2026-08-01 09:25:00 MSK 1 week 0 days Fri 2026-07-10 10:28:28 MSK - skladchik-reports-monitor-event-selftest.timer skladchik-reports-monitor-event-selftest.service\nDETAIL=EDGE_TIMER_CORRELATION_END\nDETAIL=SECRET_VALUES_OUTPUT=NONE\nDETAIL=COOKIE_CONTENT_OUTPUT=NONE\nDETAIL=COOKIE_EXACT_LOGIC_AUDIT=PASS\nCHANGES_MADE=NO\n" +} diff --git a/runtime/history/HOMELAB-SKLADCHIK-COOKIE-EXACT-LOGIC-419.txt b/runtime/history/HOMELAB-SKLADCHIK-COOKIE-EXACT-LOGIC-419.txt new file mode 100644 index 0000000..2936ff3 --- /dev/null +++ b/runtime/history/HOMELAB-SKLADCHIK-COOKIE-EXACT-LOGIC-419.txt @@ -0,0 +1,267 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=HOMELAB-SKLADCHIK-COOKIE-EXACT-LOGIC-419 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=cluster-audit +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 +COMMAND_SHA256=77896618b46e2f97362240deb743fb8e625075381e57377ee0314130be2b2026 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=false +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=63d14ea58a82bd34d78884b4db6d047e8fa64822200c7bc20a8587057f3f9e25 +SANITIZED_OUTPUT_SHA256=702451cb7bfbc7d831b17b7e062a00f5215e5dc3729bd8e3331dfe4c3dc28851 +OUTPUT_BEGIN +DETAIL=SCRIPT_SHA256=35c68ae0929ff3795be2bde40de334fc8e26413a3d8a620a137be90d1f2c164e +DETAIL=EXACT_COOKIE_LOGIC_BEGIN +DETAIL=SOURCE_BLOCK=TOP_LEVEL_CONSTANT|LINES=12-12 +DETAIL=SOURCE_LINE=0012:EDGE = os.environ.get("SKLADCHIK_EDGE", "debian@[PRIVATE_IP]") +DETAIL=SOURCE_BLOCK=TOP_LEVEL_CONSTANT|LINES=22-22 +DETAIL=SOURCE_LINE=0022:COOKIE_EXPIRY_WARN_SECONDS = int(os.environ.get("SKLADCHIK_COOKIE_EXPIRY_WARN_SECONDS", "604800")) +DETAIL=SOURCE_BLOCK=FUNCTION:gather|LINES=105-271 +DETAIL=SOURCE_LINE=0105:def gather(): +DETAIL=SOURCE_LINE=0106: facts = {} +DETAIL=SOURCE_LINE=0107: problems = [] +DETAIL=SOURCE_LINE=0108: +DETAIL=SOURCE_LINE=0109: edge_cmd = r''' +DETAIL=SOURCE_LINE=0110:set +e +DETAIL=SOURCE_LINE=0111:echo __MONITOR_HEALTH_BEGIN__ +DETAIL=SOURCE_LINE=0112:cat /var/lib/skladchik-reports-monitor/health.txt 2>/dev/null || true +DETAIL=SOURCE_LINE=0113:echo __MONITOR_HEALTH_END__ +DETAIL=SOURCE_LINE=0114:echo __ALERT_HEALTH_BEGIN__ +DETAIL=SOURCE_LINE=0115:cat /var/lib/skladchik-reports-monitor/alert-health.txt 2>/dev/null || true +DETAIL=SOURCE_LINE=0116:echo __ALERT_HEALTH_END__ +DETAIL=SOURCE_LINE=0117:echo __TIMERS_BEGIN__ +DETAIL=SOURCE_LINE=0118:for u in skladchik-reports-monitor.timer skladchik-reports-monitor-watchdog.timer skladchik-reports-monitor-heartbeat.timer skladchik-reports-monitor-weekly-summary.timer skladchik-reports-monitor-age-alert.timer skladchik-reports-monitor-event-selftest.timer; do echo "$u enabled=$(systemctl is-enabled "$u" 2>/dev/null) active=$(systemctl is-active "$u" 2>/dev/null)"; done +DETAIL=SOURCE_LINE=0119:echo __TIMERS_END__ +DETAIL=SOURCE_LINE=0120:echo __ROUTE_BEGIN__ +DETAIL=SOURCE_LINE=0121:ip route get 1.1.1.1 2>&1 | head -n 1 +DETAIL=SOURCE_LINE=0122:echo __ROUTE_END__ +DETAIL=SOURCE_LINE=0123:echo __FILES_BEGIN__ +DETAIL=SOURCE_LINE=0124:test -s /opt/skladchik-reports-monitor/monitor.py && echo monitor_py=yes || echo monitor_py=no +DETAIL=SOURCE_LINE=0125:test -s /etc/skladchik-reports-monitor/config.ini && echo config_ini=yes || echo config_ini=no +DETAIL=SOURCE_LINE=0126:test -s /etc/skladchik-reports-monitor/secrets.env && echo secrets_env= +DETAIL=SOURCE_LINE=0127:test -s /var/lib/skladchik-reports-monitor/cookies.jar && echo cookies_jar=yes || echo cookies_jar=no +DETAIL=SOURCE_LINE=0128:[ -s /var/lib/skladchik-reports-monitor/cookies.jar ] && echo cookies_jar_bytes=$(stat -c %s /var/lib/skladchik-reports-monitor/cookies.jar 2>/dev/null) || echo cookies_jar_bytes=0 +DETAIL=SOURCE_LINE=0129:test -s /opt/skladchik-reports-monitor/operational_digest.py && echo operational_digest_py=yes || echo operational_digest_py=no +DETAIL=SOURCE_LINE=0130:test -s /opt/skladchik-reports-monitor/event-selftest-runner.py && echo event_selftest_runner_py=yes || echo event_selftest_runner_py=no +DETAIL=SOURCE_LINE=0131:test -s /opt/skladchik-reports-monitor/event-selftest-core && echo event_selftest_core=yes || echo event_selftest_core=no +DETAIL=SOURCE_LINE=0132:test -s /var/lib/skladchik-reports-monitor/weekly-summary-health.txt && echo weekly_summary_health=yes || echo weekly_summary_health=no +DETAIL=SOURCE_LINE=0133:test -s /var/lib/skladchik-reports-monitor/age-alert-health.txt && echo age_alert_health=yes || echo age_alert_health=no +DETAIL=SOURCE_LINE=0134:test -s /var/lib/skladchik-reports-monitor/event-selftest-health.txt && echo event_selftest_health=yes || echo event_selftest_health=no +DETAIL=SOURCE_LINE=0135:python3 -m py_compile /opt/skladchik-reports-monitor/monitor.py /opt/skladchik-reports-monitor/operational_digest.py /opt/skladchik-reports-monitor/event-selftest-runner.py /opt/skladchik-reports-monitor/event-selftest-core >/dev/null 2>&1 && echo py_compile=yes || echo py_compile=no +DETAIL=SOURCE_LINE=0136:echo __FILES_END__ +DETAIL=SOURCE_LINE=0137:''' +DETAIL=SOURCE_LINE=0138: rc, out = ssh_edge(edge_cmd, timeout=45) +DETAIL=SOURCE_LINE=0139: facts["edge_ssh_rc"] = str(rc) +DETAIL=SOURCE_LINE=0140: if rc != 0: +DETAIL=SOURCE_LINE=0141: problems.append("edge_ssh_failed") +DETAIL=SOURCE_LINE=0142: +DETAIL=SOURCE_LINE=0143: def section(name): +DETAIL=SOURCE_LINE=0144: m = re.search(rf"__{name}_BEGIN__\n(.*?)\n__{name}_END__", out, re.S) +DETAIL=SOURCE_LINE=0145: return m.group(1).strip() if m else "" +DETAIL=SOURCE_LINE=0146: +DETAIL=SOURCE_LINE=0147: mon = parse_env_text(section("MONITOR_HEALTH")) +DETAIL=SOURCE_LINE=0148: facts["monitor_status"] = mon.get("STATUS", "MISSING") +DETAIL=SOURCE_LINE=0149: facts["monitor_reports"] = mon.get("reports", "") +DETAIL=SOURCE_LINE=0150: facts["monitor_open"] = mon.get("open", "") +DETAIL=SOURCE_LINE=0151: facts["monitor_resolved"] = mon.get("resolved", "") +DETAIL=SOURCE_LINE=0152: facts["monitor_smtp_configured"] = mon.get("smtp_configured", "") +DETAIL=SOURCE_LINE=0153: facts["session_keepalive"] = mon.get("session_keepalive", "") +DETAIL=SOURCE_LINE=0154: facts["cookie_jar"] = mon.get("cookie_jar", "") +DETAIL=SOURCE_LINE=0155: facts["cookie_jar_bytes"] = mon.get("cookie_jar_bytes", "") +DETAIL=SOURCE_LINE=0156: facts["cookie_expiry_status"] = mon.get("cookie_expiry_status", "") +DETAIL=SOURCE_LINE=0157: facts["cookie_expiry_utc"] = mon.get("cookie_expiry_utc", "") +DETAIL=SOURCE_LINE=0158: facts["cookie_expires_in_seconds"] = mon.get("cookie_expires_in_seconds", "") +DETAIL=SOURCE_LINE=0159: facts["cookie_expires_in_days"] = mon.get("cookie_expires_in_days", "") +DETAIL=SOURCE_LINE=0160: facts["cookie_persistent_count"] = mon.get("cookie_persistent_count", "") +DETAIL=SOURCE_LINE=0161: facts["monitor_health_age_seconds"] = str(age_seconds_from_ts(mon.get("TS"))) +DETAIL=SOURCE_LINE=0162: if mon.get("STATUS") != "OK": +DETAIL=SOURCE_LINE=0163: problems.append("monitor_status_not_ok") +DETAIL=SOURCE_LINE=0164: if int(facts["monitor_health_age_seconds"]) > MONITOR_HEALTH_AGE_MAX: +DETAIL=SOURCE_LINE=0165: problems.append("monitor_health_stale") +DETAIL=SOURCE_LINE=0166: if mon.get("reports") in ("", "0"): +DETAIL=SOURCE_LINE=0167: problems.append("monitor_reports_zero_or_missing") +DETAIL=SOURCE_LINE=0168: if mon.get("session_keepalive") != "persistent_cookie_jar": +DETAIL=SOURCE_LINE=0169: problems.append("session_keepalive_not_persistent") +DETAIL=SOURCE_LINE=0170: if mon.get("cookie_jar") not in ("yes", "empty"): +DETAIL=SOURCE_LINE=0171: problems.append("monitor_cookie_jar_bad") +DETAIL=SOURCE_LINE=0172: expiry_status = mon.get("cookie_expiry_status", "") +DETAIL=SOURCE_LINE=0173: try: +DETAIL=SOURCE_LINE=0174: expires_in = int(mon.get("cookie_expires_in_seconds", "")) +DETAIL=SOURCE_LINE=0175: except Exception: +DETAIL=SOURCE_LINE=0176: expires_in = None +DETAIL=SOURCE_LINE=0177: if expiry_status in ("", "NO_COOKIE_JAR", "NO_PERSISTENT_COOKIE", "ERROR"): +DETAIL=SOURCE_LINE=0178: problems.append("cookie_expiry_missing") +DETAIL=SOURCE_LINE=0179: elif expiry_status == "EXPIRED" or (expires_in is not None and expires_in <= 0): +DETAIL=SOURCE_LINE=0180: problems.append("cookie_expiry_expired") +DETAIL=SOURCE_LINE=0181: elif expiry_status == "WARN" or (expires_in is not None and expires_in < COOKIE_EXPIRY_WARN_SECONDS): +DETAIL=SOURCE_LINE=0182: problems.append("cookie_expiry_warn") +DETAIL=SOURCE_LINE=0183: +DETAIL=SOURCE_LINE=0184: timers_text = section("TIMERS") +DETAIL=SOURCE_LINE=0185: for unit in ["skladchik-reports-monitor.timer", "skladchik-reports-monitor-watchdog.timer", "skladchik-reports-monitor-heartbeat.timer", "skladchik-reports-monitor-weekly-summary.timer", "skladchik-reports-monitor-age-alert.timer", "skladchik-reports-monitor-event-selftest.timer"]: +DETAIL=SOURCE_LINE=0186: safe = unit.replace(".timer", "").replace("_", "-") +DETAIL=SOURCE_LINE=0187: m = re.search(re.escape(unit) + r"\s+enabled=([^\s]+)\s+active=([^\s]+)", timers_text) +DETAIL=SOURCE_LINE=0188: enabled = m.group(1) if m else "missing" +DETAIL=SOURCE_LINE=0189: active = m.group(2) if m else "missing" +DETAIL=SOURCE_LINE=0190: facts[f"edge_{safe}_timer_enabled"] = enabled +DETAIL=SOURCE_LINE=0191: facts[f"edge_{safe}_timer_active"] = active +DETAIL=SOURCE_LINE=0192: if enabled != "enabled" or active != "active": +DETAIL=SOURCE_LINE=0193: problems.append(f"edge_{safe}_timer_bad") +DETAIL=SOURCE_LINE=0194: +DETAIL=SOURCE_LINE=0195: route_lines = section("ROUTE").splitlines() +DETAIL=SOURCE_LINE=0196: route = route_lines[0] if route_lines else "" +DETAIL=SOURCE_LINE=0197: facts["netbird_route"] = route +DETAIL=SOURCE_LINE=0198: if " wt0 " not in (" " + route + " "): +DETAIL=SOURCE_LINE=0199: problems.append("netbird_route_not_wt0") +DETAIL=SOURCE_LINE=0200: +DETAIL=SOURCE_LINE=0201: files = parse_env_text(section("FILES")) +DETAIL=SOURCE_LINE=0202: for k in ["monitor_py", "config_ini", "secrets_env", "cookies_jar", "operational_digest_py", "event_selftest_runner_py", "event_selftest_core", "weekly_summary_health", "age_alert_health", "event_selftest_health", "py_compile"]: +DETAIL=SOURCE_LINE=0203: facts[k] = files.get(k, "no") +DETAIL=SOURCE_LINE=0204: if facts[k] != "yes": +DETAIL=SOURCE_LINE=0205: problems.append(f"edge_{k}_bad") +DETAIL=SOURCE_LINE=0206: facts["cookies_jar_bytes_file"] = files.get("cookies_jar_bytes", "0") +DETAIL=SOURCE_LINE=0207: +DETAIL=SOURCE_LINE=0208: backup = read_env_file(PVE_BACKUP_HEALTH) +DETAIL=SOURCE_LINE=0209: facts["backup_status"] = backup.get("STATUS", "MISSING") +DETAIL=SOURCE_LINE=0210: facts["backup_run_id"] = backup.get("RUN_ID", "") +DETAIL=SOURCE_LINE=0211: facts["backup_size_bytes"] = backup.get("SIZE_BYTES", "") +DETAIL=SOURCE_LINE=0212: facts["backup_version"] = backup.get("BACKUP_VERSION", "") +DETAIL=SOURCE_LINE=0213: facts["backup_cookie_jar_backed_up"] = backup.get("COOKIE_JAR_BACKED_UP", "") +DETAIL=SOURCE_LINE=0214: facts["backup_extension_health_files_backed_up"] = backup.get("EXTENSION_HEALTH_FILES_BACKED_UP", "") +DETAIL=SOURCE_LINE=0215: facts["backup_title_in_email_subject"] = backup.get("TITLE_IN_EMAIL_SUBJECT", "") +DETAIL=SOURCE_LINE=0216: facts["backup_health_age_seconds"] = str(age_seconds_from_ts(backup.get("TS"))) +DETAIL=SOURCE_LINE=0217: if backup.get("STATUS") != "OK": +DETAIL=SOURCE_LINE=0218: problems.append("backup_status_not_ok") +DETAIL=SOURCE_LINE=0219: if backup.get("COOKIE_JAR_BACKED_UP") != "yes": +DETAIL=SOURCE_LINE=0220: problems.append("backup_cookie_jar_not_marked") +DETAIL=SOURCE_LINE=0221: if backup.get("BACKUP_VERSION") != "4": +DETAIL=SOURCE_LINE=0222: problems.append("backup_version_not_4") +DETAIL=SOURCE_LINE=0223: if backup.get("EXTENSION_HEALTH_FILES_BACKED_UP") != "yes": +DETAIL=SOURCE_LINE=0224: problems.append("backup_extension_health_files_not_marked") +DETAIL=SOURCE_LINE=0225: if backup.get("TITLE_IN_EMAIL_SUBJECT") != "yes": +DETAIL=SOURCE_LINE=0226: problems.append("email_title_subject_not_marked") +DETAIL=SOURCE_LINE=0227: if int(facts["backup_health_age_seconds"]) > BACKUP_HEALTH_AGE_MAX: +DETAIL=SOURCE_LINE=0228: problems.append("backup_health_stale") +DETAIL=SOURCE_LINE=0229: try: +DETAIL=SOURCE_LINE=0230: if int(backup.get("SIZE_BYTES", "0")) <= 0: +DETAIL=SOURCE_LINE=0231: problems.append("backup_size_bad") +DETAIL=SOURCE_LINE=0232: except Exception: +DETAIL=SOURCE_LINE=0233: problems.append("backup_size_bad") +DETAIL=SOURCE_LINE=0234: +DETAIL=SOURCE_LINE=0235: restore = read_env_file(PVE_RESTORE_HEALTH) +DETAIL=SOURCE_LINE=0236: facts["restore_status"] = restore.get("STATUS", "MISSING") +DETAIL=SOURCE_LINE=0237: facts["restore_run_id"] = restore.get("RUN_ID", "") +DETAIL=SOURCE_LINE=0238: facts["restore_health_age_seconds"] = str(age_seconds_from_ts(restore.get("TS"))) +DETAIL=SOURCE_LINE=0239: for key in ["CLOUD_1_STATUS", "CLOUD_2_STATUS", "SHA256_OK", "TAR_OK", "REQUIRED_FILES_OK", "COOKIE_JAR_OK", "EXTENSION_HEALTH_FILES_OK", "EXTENSION_SCRIPTS_OK", "EXTENSION_UNITS_OK", "SQLITE_OK", "PY_COMPILE_OK"]: +DETAIL=SOURCE_LINE=0240: facts[key.lower()] = restore.get(key, "") +DETAIL=SOURCE_LINE=0241: if restore.get("STATUS") != "OK": +DETAIL=SOURCE_LINE=0242: problems.append("restore_status_not_ok") +DETAIL=SOURCE_LINE=0243: if int(facts["restore_health_age_seconds"]) > RESTORE_HEALTH_AGE_MAX: +DETAIL=SOURCE_LINE=0244: problems.append("restore_health_stale") +DETAIL=SOURCE_LINE=0245: for key in ["CLOUD_1_STATUS", "CLOUD_2_STATUS"]: +DETAIL=SOURCE_LINE=0246: if restore.get(key) != "OK": +DETAIL=SOURCE_LINE=0247: problems.append(key.lower() + "_bad") +DETAIL=SOURCE_LINE=0248: for key in ["SHA256_OK", "TAR_OK", "REQUIRED_FILES_OK", "COOKIE_JAR_OK", "EXTENSION_HEALTH_FILES_OK", "EXTENSION_SCRIPTS_OK", "EXTENSION_UNITS_OK", "SQLITE_OK", "PY_COMPILE_OK"]: +DETAIL=SOURCE_LINE=0249: if restore.get(key) != "yes": +DETAIL=SOURCE_LINE=0250: problems.append(key.lower() + "_bad") +DETAIL=SOURCE_LINE=0251: +DETAIL=SOURCE_LINE=0252: local_units = [ +DETAIL=SOURCE_LINE=0253: "skladchik-reports-monitor-backup.timer", +DETAIL=SOURCE_LINE=0254: "skladchik-reports-monitor-backup-restore-check.timer", +DETAIL=SOURCE_LINE=0255: "skladchik-reports-monitor-supervisor.timer", +DETAIL=SOURCE_LINE=0256: "skladchik-reports-monitor-monthly-selftest-watch.timer", +DETAIL=SOURCE_LINE=0257: ] +DETAIL=SOURCE_LINE=0258: props = systemctl_props(local_units) +DETAIL=SOURCE_LINE=0259: for u, p in props.items(): +DETAIL=SOURCE_LINE=0260: base = u.replace(".timer", "").replace("-", "_") +DETAIL=SOURCE_LINE=0261: facts[base + "_timer_enabled"] = p["enabled"] +DETAIL=SOURCE_LINE=0262: facts[base + "_timer_active"] = p["active"] +DETAIL=SOURCE_LINE=0263: if u != "skladchik-reports-monitor-supervisor.timer": +DETAIL=SOURCE_LINE=0264: if p["enabled"] != "enabled" or p["active"] != "active": +DETAIL=SOURCE_LINE=0265: problems.append(base + "_timer_bad") +DETAIL=SOURCE_LINE=0266: else: +DETAIL=SOURCE_LINE=0267: # Before first enable it can be disabled/inactive during test. After enable next runs will see enabled/active. +DETAIL=SOURCE_LINE=0268: if p["enabled"] not in ("enabled", "disabled") or p["active"] not in ("active", "inactive"): +DETAIL=SOURCE_LINE=0269: problems.append(base + "_timer_unknown") +DETAIL=SOURCE_LINE=0270: +DETAIL=SOURCE_LINE=0271: return facts, problems +DETAIL=SOURCE_BLOCK=FUNCTION:render_body|LINES=273-321 +DETAIL=SOURCE_LINE=0273:def render_body(status, facts, problems, mode): +DETAIL=SOURCE_LINE=0274: ru_status = "OK" if status == "OK" else "АВАРИЯ" +DETAIL=SOURCE_LINE=0275: lines = [ +DETAIL=SOURCE_LINE=0276: f"Статус надзора: {ru_status}", +DETAIL=SOURCE_LINE=0277: f"Время UTC: {iso()}", +DETAIL=SOURCE_LINE=0278: "", +DETAIL=SOURCE_LINE=0279: "Монитор жалоб:", +DETAIL=SOURCE_LINE=0280: f"- статус: {facts.get('monitor_status')}", +DETAIL=SOURCE_LINE=0281: f"- всего жалоб на странице: {facts.get('monitor_reports')}", +DETAIL=SOURCE_LINE=0282: f"- открытые/назначенные: {facts.get('monitor_open')}", +DETAIL=SOURCE_LINE=0283: f"- закрытые: {facts.get('monitor_resolved')}", +DETAIL=SOURCE_LINE=0284: f"- возраст health, сек: {facts.get('monitor_health_age_seconds')}", +DETAIL=SOURCE_LINE=0285: f"- автопродление авторизации: {facts.get('session_keepalive')}", +DETAIL=SOURCE_LINE=0286: f"- cookie jar: {facts.get('cookie_jar')} / {facts.get('cookie_jar_bytes')} байт", +DETAIL=SOURCE_LINE=0287: f"- постоянная cookie действует до UTC: {facts.get('cookie_expiry_utc') or 'не определено'}", +DETAIL=SOURCE_LINE=0288: f"- осталось дней по cookie: {facts.get('cookie_expires_in_days') or 'не определено'}", +DETAIL=SOURCE_LINE=0289: f"- статус срока cookie: {facts.get('cookie_expiry_status') or 'не определено'}", +DETAIL=SOURCE_LINE=0290: f"- NetBird route: {facts.get('netbird_route')}", +DETAIL=SOURCE_LINE=0291: "", +DETAIL=SOURCE_LINE=0292: "Бэкап:", +DETAIL=SOURCE_LINE=0293: f"- статус: {facts.get('backup_status')}", +DETAIL=SOURCE_LINE=0294: f"- последний RUN_ID: {facts.get('backup_run_id')}", +DETAIL=SOURCE_LINE=0295: f"- размер архива, байт: {facts.get('backup_size_bytes')}", +DETAIL=SOURCE_LINE=0296: f"- cookie jar в бэкапе: {facts.get('backup_cookie_jar_backed_up')}", +DETAIL=SOURCE_LINE=0297: f"- health-файлы расширений в бэкапе: {facts.get('backup_extension_health_files_backed_up')}", +DETAIL=SOURCE_LINE=0298: f"- название жалобы в теме письма: {facts.get('backup_title_in_email_subject')}", +DETAIL=SOURCE_LINE=0299: f"- возраст health, сек: {facts.get('backup_health_age_seconds')}", +DETAIL=SOURCE_LINE=0300: "", +DETAIL=SOURCE_LINE=0301: "Restore-check:", +DETAIL=SOURCE_LINE=0302: f"- статус: {facts.get('restore_status')}", +DETAIL=SOURCE_LINE=0303: f"- последний RUN_ID: {facts.get('restore_run_id')}", +DETAIL=SOURCE_LINE=0304: f"- cloud1/cloud2: {facts.get('cloud_1_status')}/{facts.get('cloud_2_status')}", +DETAIL=SOURCE_LINE=0305: f"- sha256/tar/cookie/sqlite/python: {facts.get('sha256_ok')}/{facts.get('tar_ok')}/{facts.get('cookie_jar_ok')}/{facts.get('sqlite_ok')}/{facts.get('py_compile_ok')}", +DETAIL=SOURCE_LINE=0306: f"- health/scripts/units расширений: {facts.get('extension_health_files_ok')}/{facts.get('extension_scripts_ok')}/{facts.get('extension_units_ok')}", +DETAIL=SOURCE_LINE=0307: "", +DETAIL=SOURCE_LINE=0308: "Проблемы:", +DETAIL=SOURCE_LINE=0309: ] +DETAIL=SOURCE_LINE=0310: if problems: +DETAIL=SOURCE_LINE=0311: lines.extend([f"- {p}" for p in problems]) +DETAIL=SOURCE_LINE=0312: else: +DETAIL=SOURCE_LINE=0313: lines.append("- нет") +DETAIL=SOURCE_LINE=0314: lines.extend([ +DETAIL=SOURCE_LINE=0315: "", +DETAIL=SOURCE_LINE=0316: "Авторизация:", +DETAIL=SOURCE_LINE=0317: "- ручной ежемесячный перелогин обычно не нужен; монитор сам продлевает сессию постоянным cookie jar при каждом опросе;", +DETAIL=SOURCE_LINE=0318: "- supervisor теперь отдельно следит за сроком постоянной cookie и заранее предупредит, если до истечения останется меньше 7 дней;", +DETAIL=SOURCE_LINE=0319: "- если форум принудительно сбросит сессию, придёт отдельная тревога AUTH_REQUIRED; тогда нужно будет разово обновить вход, но это аварийный сценарий, а не ежемесячная рутина.", +DETAIL=SOURCE_LINE=0320: ]) +DETAIL=SOURCE_LINE=0321: return "\n".join(lines) +DETAIL=EXACT_COOKIE_LOGIC_END +DETAIL=PVE01_TIMER_CORRELATION_BEGIN +DETAIL=PVE01_TIMER=Fri 2026-07-24 12:08:38 MSK 1min 48s Fri 2026-07-24 11:58:38 MSK 8min ago skladchik-reports-monitor-supervisor.timer skladchik-reports-monitor-supervisor.service +DETAIL=PVE01_TIMER=Sat 2026-07-25 04:39:05 MSK 16h Fri 2026-07-24 04:28:08 MSK 7h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service +DETAIL=PVE01_TIMER=Sat 2026-07-25 10:15:00 MSK 22h Fri 2026-07-24 10:15:04 MSK 1h 51min ago skladchik-reports-monitor-monthly-selftest-watch.timer skladchik-reports-monitor-monthly-selftest-watch.service +DETAIL=PVE01_TIMER=Sun 2026-07-26 05:15:13 MSK 1 day 17h Sun 2026-07-19 05:11:25 MSK - skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service +DETAIL=PVE01_TIMER_CORRELATION_END +DETAIL=EDGE_TIMER_CORRELATION_BEGIN +DETAIL=EDGE_TIMER=Fri 2026-07-24 12:06:51 MSK 1s Fri 2026-07-24 12:06:21 MSK 28s ago skladchik-reports-monitor.timer skladchik-reports-monitor.service +DETAIL=EDGE_TIMER=Fri 2026-07-24 12:08:35 MSK 1min 45s Fri 2026-07-24 12:03:35 MSK 3min 14s ago skladchik-reports-monitor-watchdog.timer skladchik-reports-monitor-watchdog.service +DETAIL=EDGE_TIMER=Sat 2026-07-25 09:16:28 MSK 21h Fri 2026-07-24 09:18:35 MSK 2h 48min ago skladchik-reports-monitor-heartbeat.timer skladchik-reports-monitor-heartbeat.service +DETAIL=EDGE_TIMER=Sat 2026-07-25 09:45:00 MSK 21h Fri 2026-07-24 09:45:03 MSK 2h 21min ago skladchik-reports-monitor-age-alert.timer skladchik-reports-monitor-age-alert.service +DETAIL=EDGE_TIMER=Mon 2026-07-27 09:35:00 MSK 2 days Mon 2026-07-20 09:35:06 MSK - skladchik-reports-monitor-weekly-summary.timer skladchik-reports-monitor-weekly-summary.service +DETAIL=EDGE_TIMER=Sat 2026-08-01 09:25:00 MSK 1 week 0 days Fri 2026-07-10 10:28:28 MSK - skladchik-reports-monitor-event-selftest.timer skladchik-reports-monitor-event-selftest.service +DETAIL=EDGE_TIMER_CORRELATION_END +DETAIL=SECRET_VALUES_OUTPUT=NONE +DETAIL=COOKIE_CONTENT_OUTPUT=NONE +DETAIL=COOKIE_EXACT_LOGIC_AUDIT=PASS +CHANGES_MADE=NO + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index c5b7657..ec64093 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,14 +1,14 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "HOMELAB-SKLADCHIK-COOKIE-CHECK-SOURCE-AUDIT-418", + "command_id": "HOMELAB-SKLADCHIK-COOKIE-EXACT-LOGIC-419", "status": "OK", "rc": 0, "host": "pve01", "mode": "read-only", "component": "cluster-audit", - "started_at_utc": "2026-07-24T09:05:03Z", - "finished_at_utc": "2026-07-24T09:05:03Z", + "started_at_utc": "2026-07-24T09:06:49Z", + "finished_at_utc": "2026-07-24T09:06:49Z", "reference_register_checked": true, "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", "error_register_checked": true, @@ -22,9 +22,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "1c26f0b751c5e55421c706f56f4be25d92c17849df8a385fb66ce01144801022", - "sanitized_output_sha256": "8e5a54db35050a5c0010241fb0bad91fc5966530dbac76bb07426b09af51a6a1", + "raw_evidence_sha256": "63d14ea58a82bd34d78884b4db6d047e8fa64822200c7bc20a8587057f3f9e25", + "sanitized_output_sha256": "702451cb7bfbc7d831b17b7e062a00f5215e5dc3729bd8e3331dfe4c3dc28851", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "DETAIL=SCRIPT_PATH=/usr/local/sbin/skladchik-reports-monitor-supervisor\nDETAIL=SCRIPT_SHA256=35c68ae0929ff3795be2bde40de334fc8e26413a3d8a620a137be90d1f2c164e\nDETAIL=SCRIPT_METADATA=BYTES:20463|MODE:755|OWNER:root:root|MTIME:2026-07-18 13:52:30.415780066 +0300\nDETAIL=RELEVANT_SOURCE_BEGIN\nDETAIL=SOURCE_LINE=0005:import os\nDETAIL=SOURCE_LINE=0006:import re\nDETAIL=SOURCE_LINE=0007:import shlex\nDETAIL=SOURCE_LINE=0008:import subprocess\nDETAIL=SOURCE_LINE=0009:import sys\nDETAIL=SOURCE_LINE=0010:from pathlib import Path\nDETAIL=SOURCE_LINE=0011:\nDETAIL=SOURCE_LINE=0012:EDGE = os.environ.get(\"SKLADCHIK_EDGE\", \"debian@[PRIVATE_IP]\")\nDETAIL=SOURCE_LINE=0013:STATE_DIR = Path(\"/var/lib/skladchik-reports-monitor-supervisor\")\nDETAIL=SOURCE_LINE=0014:LOG_DIR = Path(\"/var/log/skladchik-reports-monitor-supervisor\")\nDETAIL=SOURCE_LINE=0015:HEALTH = STATE_DIR / \"health.txt\"\nDETAIL=SOURCE_LINE=0016:STATE = STATE_DIR / \"state.env\"\nDETAIL=SOURCE_LINE=0017:LOG = LOG_DIR / \"supervisor.log\"\nDETAIL=SOURCE_LINE=0018:\nDETAIL=SOURCE_LINE=0019:MONITOR_HEALTH_AGE_MAX = int(os.environ.get(\"SKLADCHIK_MONITOR_HEALTH_MAX_SECONDS\", \"600\"))\nDETAIL=SOURCE_LINE=0020:BACKUP_HEALTH_AGE_MAX = int(os.environ.get(\"SKLADCHIK_BACKUP_HEALTH_MAX_SECONDS\", \"129600\"))\nDETAIL=SOURCE_LINE=0021:RESTORE_HEALTH_AGE_MAX = int(os.environ.get(\"SKLADCHIK_RESTORE_HEALTH_MAX_SECONDS\", \"691200\"))\nDETAIL=SOURCE_LINE=0022:COOKIE_EXPIRY_WARN_SECONDS = int(os.environ.get(\"SKLADCHIK_COOKIE_EXPIRY_WARN_SECONDS\", \"604800\"))\nDETAIL=SOURCE_LINE=0023:\nDETAIL=SOURCE_LINE=0024:PVE_BACKUP_HEALTH = Path(\"/var/lib/skladchik-reports-monitor-backup/health.txt\")\nDETAIL=SOURCE_LINE=0025:PVE_RESTORE_HEALTH = Path(\"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\")\nDETAIL=SOURCE_LINE=0026:\nDETAIL=SOURCE_LINE=0027:def now():\nDETAIL=SOURCE_LINE=0028: return _dt.datetime.now(_dt.UTC).replace(microsecond=0)\nDETAIL=SOURCE_LINE=0029:\nDETAIL=SOURCE_LINE=0030:def iso():\nDETAIL=SOURCE_LINE=0031: return now().isoformat().replace(\"+00:00\", \"Z\")\nDETAIL=SOURCE_LINE=0032:\nDETAIL=SOURCE_LINE=0033:def log(msg):\nDETAIL=SOURCE_LINE=0034: LOG_DIR.mkdir(parents=True, exist_ok=True)\nDETAIL=SOURCE_LINE=0035: with LOG.open(\"a\", encoding=\"utf-8\") as f:\nDETAIL=SOURCE_LINE=0036: f.write(f\"{iso()} {msg}\\n\")\nDETAIL=SOURCE_LINE=0037:\nDETAIL=SOURCE_LINE=0038:def run(cmd, timeout=45, input_text=None):\nDETAIL=SOURCE_LINE=0039: try:\nDETAIL=SOURCE_LINE=0040: p = subprocess.run(cmd, shell=True, text=True, input=input_text, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout)\nDETAIL=SOURCE_LINE=0041: return p.returncode, p.stdout.strip()\nDETAIL=SOURCE_LINE=0042: except subprocess.TimeoutExpired as e:\nDETAIL=SOURCE_LINE=0043: out = e.stdout if isinstance(e.stdout, str) else \"\"\nDETAIL=SOURCE_LINE=0044: return 124, (out or \"\").strip() + \"\\nTIMEOUT\"\nDETAIL=SOURCE_LINE=0045:\nDETAIL=SOURCE_LINE=0046:def parse_env_text(text):\nDETAIL=SOURCE_LINE=0047: d = {}\nDETAIL=SOURCE_LINE=0048: for line in (text or \"\").splitlines():\nDETAIL=SOURCE_LINE=0049: if not line or line.startswith(\"#\") or \"=\" not in line:\nDETAIL=SOURCE_LINE=0050: continue\nDETAIL=SOURCE_LINE=0051: k, v = line.split(\"=\", 1)\nDETAIL=SOURCE_LINE=...\nDETAIL=SOURCE_LINE=0066: dt = _dt.datetime.fromisoformat(s)\nDETAIL=SOURCE_LINE=0067: if dt.tzinfo is None:\nDETAIL=SOURCE_LINE=0068: dt = dt.replace(tzinfo=_dt.UTC)\nDETAIL=SOURCE_LINE=0069: return int((now() - dt.astimezone(_dt.UTC)).total_seconds())\nDETAIL=SOURCE_LINE=0070: except Exception:\nDETAIL=SOURCE_LINE=0071: return 999999999\nDETAIL=SOURCE_LINE=0072:\nDETAIL=SOURCE_LINE=0073:def ssh_edge(cmd, timeout=45):\nDETAIL=SOURCE_LINE=0074: q = \"ssh -o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] \" + shlex.quote(EDGE) + \" \" + shlex.quote(\"sudo bash -lc \" + shlex.quote(cmd))\nDETAIL=SOURCE_LINE=0075: return run(q, timeout=timeout)\nDETAIL=SOURCE_LINE=0076:\nDETAIL=SOURCE_LINE=0077:def edge_mail(subject, body):\nDETAIL=SOURCE_LINE=0078: subj_b64 = base64.b64encode(subject.encode(\"utf-8\")).decode(\"ascii\")\nDETAIL=SOURCE_LINE=0079: body_b64 = base64.b64encode(body.encode(\"utf-8\")).decode(\"ascii\")\nDETAIL=SOURCE_LINE=0080: py = (\nDETAIL=SOURCE_LINE=0081: \"import base64,importlib.util;\"\nDETAIL=SOURCE_LINE=0082: \"spec=importlib.util.spec_from_file_location('m','/opt/skladchik-reports-monitor/monitor.py');\"\nDETAIL=SOURCE_LINE=0083: \"m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m);\"\nDETAIL=SOURCE_LINE=0084: \"cfg,secrets=\nDETAIL=SOURCE_LINE=0085: f\"subject=base64.b64decode('{subj_b64}').decode('utf-8');\"\nDETAIL=SOURCE_LINE=0086: f\"body=base64.b64decode('{body_b64}').decode('utf-8');\"\nDETAIL=SOURCE_LINE=0087: \"ok=m.send_mail(cfg,secrets,subject,body);\"\nDETAIL=SOURCE_LINE=0088: \"print('MAIL_OK' if ok else 'MAIL_FALSE')\"\nDETAIL=SOURCE_LINE=0089: )\nDETAIL=SOURCE_LINE=0090: rc, out = ssh_edge(\"python3 -c \" + shlex.quote(py), timeout=60)\nDETAIL=SOURCE_LINE=0091: log(f\"MAIL subject={subject!r} rc={rc} out={out!r}\")\nDETAIL=SOURCE_LINE=0092: return rc == 0 and \"MAIL_OK\" in out\nDETAIL=SOURCE_LINE=0093:\nDETAIL=SOURCE_LINE=0094:def systemctl_props(units):\nDETAIL=SOURCE_LINE=0095: cmd = \"systemctl is-enabled \" + \" \".join(map(shlex.quote, units)) + \" 2>/dev/null || true; echo __ACTIVE__; systemctl is-active \" + \" \".join(map(shlex.quote, units)) + \" 2>/dev/null || true\"\nDETAIL=SOURCE_LINE=0096: rc, out = run(cmd, timeout=20)\nDETAIL=SOURCE_LINE=0097: parts = out.split(\"__ACTIVE__\", 1)\nDETAIL=SOURCE_LINE=0098: enabled_lines = [x.strip() for x in parts[0].splitlines() if x.strip()]\nDETAIL=SOURCE_LINE=0099: active_lines = [x.strip() for x in parts[1].splitlines() if x.strip()] if len(parts) > 1 else []\nDETAIL=SOURCE_LINE=0100: res = {}\nDETAIL=SOURCE_LINE=0101: for i, u in enumerate(units):\nDETAIL=SOURCE_LINE=0102: res[u] = {\"enabled\": enabled_lines[i] if i < len(enabled_lines) else \"unknown\", \"active\": active_lines[i] if i < len(active_lines) else \"unknown\"}\nDETAIL=SOURCE_LINE=0103: return res\nDETAIL=SOURCE_LINE=0104:\nDETAIL=SOURCE_LINE=0105:def gather():\nDETAIL=SOURCE_LINE=0106: facts = {}\nDETAIL=SOURCE_LINE=0107: problems = []\nDETAIL=SOURCE_LINE=0108:\nDETAIL=SOURCE_LINE=0109: edge_cmd = r'''\nDETAIL=SOURCE_LINE=0110:set +e\nDETAIL=SOURCE_LINE=0111:echo __MONITOR_HEALTH_BEGIN__\nDETAIL=SOURCE_LINE=0112:cat /var/lib/skladchik-reports-monitor/health.txt 2>/dev/null || true\nDETAIL=SOURCE_LINE=0113:echo __MONITOR_HEALTH_END__\nDETAIL=SOURCE_LINE=0114:echo __ALERT_HEALTH_BEGIN__\nDETAIL=SOURCE_LINE=0115:cat /var/lib/skladchik-reports-monitor/alert-health.txt 2>/dev/null || true\nDETAIL=SOURCE_LINE=0116:echo __ALERT_HEALTH_END__\nDETAIL=SOURCE_LINE=0117:echo __TIMERS_BEGIN__\nDETAIL=SOURCE_LINE=0118:for u in skladchik-reports-monitor.timer skladchik-reports-monitor-watchdog.timer skladchik-reports-monitor-heartbeat.timer skladchik-reports-monitor-weekly-summary.timer skladchik-reports-monitor-age-alert.timer skladchik-reports-monitor-event-selftest.timer; do echo \"$u enabled=$(systemctl is-enabled \"$u\" 2>/dev/null) active=$(systemctl is-active \"$u\" 2>/dev/null)\"; done\nDETAIL=SOURCE_LINE=0119:echo __TIMERS_END__\nDETAIL=SOURCE_LINE=0120:echo __ROUTE_BEGIN__\nDETAIL=SOURCE_LINE=0121:ip route get 1.1.1.1 2>&1 | head -n 1\nDETAIL=SOURCE_LINE=0122:echo __ROUTE_END__\nDETAIL=SOURCE_LINE=0123:echo __FILES_BEGIN__\nDETAIL=SOURCE_LINE=0124:test -s /opt/skladchik-reports-monitor/monitor.py && echo monitor_py=yes || echo monitor_py=no\nDETAIL=SOURCE_LINE=0125:test -s /etc/skladchik-reports-monitor/config.ini && echo config_ini=yes || echo config_ini=no\nDETAIL=SOURCE_LINE=0126:test -s /etc/skladchik-reports-monitor/secrets.env && echo secrets_env=\nDETAIL=SOURCE_LINE=0127:test -s /var/lib/skladchik-reports-monitor/cookies.jar && echo cookies_jar=yes || echo cookies_jar=no\nDETAIL=SOURCE_LINE=0128:[ -s /var/lib/skladchik-reports-monitor/cookies.jar ] && echo cookies_jar_bytes=$(stat -c %s /var/lib/skladchik-reports-monitor/cookies.jar 2>/dev/null) || echo cookies_jar_bytes=0\nDETAIL=SOURCE_LINE=0129:test -s /opt/skladchik-reports-monitor/operational_digest.py && echo operational_digest_py=yes || echo operational_digest_py=no\nDETAIL=SOURCE_LINE=0130:test -s /opt/skladchik-reports-monitor/event-selftest-runner.py && echo event_selftest_runner_py=yes || echo event_selftest_runner_py=no\nDETAIL=SOURCE_LINE=0131:test -s /opt/skladchik-reports-monitor/event-selftest-core && echo event_selftest_core=yes || echo event_selftest_core=no\nDETAIL=SOURCE_LINE=0132:test -s /var/lib/skladchik-reports-monitor/weekly-summary-health.txt && echo weekly_summary_health=yes || echo weekly_summary_health=no\nDETAIL=SOURCE_LINE=0133:test -s /var/lib/skladchik-reports-monitor/age-alert-health.txt && echo age_alert_health=yes || echo age_alert_health=no\nDETAIL=SOURCE_LINE=0134:test -s /var/lib/skladchik-reports-monitor/event-selftest-health.txt && echo event_selftest_health=yes || echo event_selftest_health=no\nDETAIL=SOURCE_LINE=0135:python3 -m py_compile /opt/skladchik-reports-monitor/monitor.py /opt/skladchik-reports-monitor/operational_digest.py /opt/skladchik-reports-monitor/event-selftest-runner.py /opt/skladchik-reports-monitor/event-selftest-core >/dev/null 2>&1 && echo py_compile=yes || echo py_compile=no\nDETAIL=SOURCE_LINE=0136:echo __FILES_END__\nDETAIL=SOURCE_LINE=0137:'''\nDETAIL=SOURCE_LINE=0138: rc, out = ssh_edge(edge_cmd, timeout=45)\nDETAIL=SOURCE_LINE=0139: facts[\"edge_ssh_rc\"] = str(rc)\nDETAIL=SOURCE_LINE=0140: if rc != 0:\nDETAIL=SOURCE_LINE=0141: problems.append(\"edge_ssh_failed\")\nDETAIL=SOURCE_LINE=0142:\nDETAIL=SOURCE_LINE=0143: def section(name):\nDETAIL=SOURCE_LINE=0144: m = re.search(rf\"__{name}_BEGIN__\\n(.*?)\\n__{name}_END__\", out, re.S)\nDETAIL=SOURCE_LINE=0145: return m.group(1).strip() if m else \"\"\nDETAIL=SOURCE_LINE=0146:\nDETAIL=SOURCE_LINE=0147: mon = parse_env_text(section(\"MONITOR_HEALTH\"))\nDETAIL=SOURCE_LINE=0148: facts[\"monitor_status\"] = mon.get(\"STATUS\", \"MISSING\")\nDETAIL=SOURCE_LINE=0149: facts[\"monitor_reports\"] = mon.get(\"reports\", \"\")\nDETAIL=SOURCE_LINE=0150: facts[\"monitor_open\"] = mon.get(\"open\", \"\")\nDETAIL=SOURCE_LINE=0151: facts[\"monitor_resolved\"] = mon.get(\"resolved\", \"\")\nDETAIL=SOURCE_LINE=0152: facts[\"monitor_smtp_configured\"] = mon.get(\"smtp_configured\", \"\")\nDETAIL=SOURCE_LINE=0153: facts[\"session_keepalive\"] = mon.get(\"session_keepalive\", \"\")\nDETAIL=SOURCE_LINE=0154: facts[\"cookie_jar\"] = mon.get(\"cookie_jar\", \"\")\nDETAIL=SOURCE_LINE=0155: facts[\"cookie_jar_bytes\"] = mon.get(\"cookie_jar_bytes\", \"\")\nDETAIL=SOURCE_LINE=0156: facts[\"cookie_expiry_status\"] = mon.get(\"cookie_expiry_status\", \"\")\nDETAIL=SOURCE_LINE=0157: facts[\"cookie_expiry_utc\"] = mon.get(\"cookie_expiry_utc\", \"\")\nDETAIL=SOURCE_LINE=0158: facts[\"cookie_expires_in_seconds\"] = mon.get(\"cookie_expires_in_seconds\", \"\")\nDETAIL=SOURCE_LINE=0159: facts[\"cookie_expires_in_days\"] = mon.get(\"cookie_expires_in_days\", \"\")\nDETAIL=SOURCE_LINE=0160: facts[\"cookie_persistent_count\"] = mon.get(\"cookie_persistent_count\", \"\")\nDETAIL=SOURCE_LINE=0161: facts[\"monitor_health_age_seconds\"] = str(age_seconds_from_ts(mon.get(\"TS\")))\nDETAIL=SOURCE_LINE=0162: if mon.get(\"STATUS\") != \"OK\":\nDETAIL=SOURCE_LINE=0163: problems.append(\"monitor_status_not_ok\")\nDETAIL=SOURCE_LINE=0164: if int(facts[\"monitor_health_age_seconds\"]) > MONITOR_HEALTH_AGE_MAX:\nDETAIL=SOURCE_LINE=0165: problems.append(\"monitor_health_stale\")\nDETAIL=SOURCE_LINE=0166: if mon.get(\"reports\") in (\"\", \"0\"):\nDETAIL=SOURCE_LINE=0167: problems.append(\"monitor_reports_zero_or_missing\")\nDETAIL=SOURCE_LINE=0168: if mon.get(\"session_keepalive\") != \"persistent_cookie_jar\":\nDETAIL=SOURCE_LINE=0169: problems.append(\"session_keepalive_not_persistent\")\nDETAIL=SOURCE_LINE=0170: if mon.get(\"cookie_jar\") not in (\"yes\", \"empty\"):\nDETAIL=SOURCE_LINE=0171: problems.append(\"monitor_cookie_jar_bad\")\nDETAIL=SOURCE_LINE=0172: expiry_status = mon.get(\"cookie_expiry_status\", \"\")\nDETAIL=SOURCE_LINE=0173: try:\nDETAIL=SOURCE_LINE=0174: expires_in = int(mon.get(\"cookie_expires_in_seconds\", \"\"))\nDETAIL=SOURCE_LINE=0175: except Exception:\nDETAIL=SOURCE_LINE=0176: expires_in = None\nDETAIL=SOURCE_LINE=0177: if expiry_status in (\"\", \"NO_COOKIE_JAR\", \"NO_PERSISTENT_COOKIE\", \"ERROR\"):\nDETAIL=SOURCE_LINE=0178: problems.append(\"cookie_expiry_missing\")\nDETAIL=SOURCE_LINE=0179: elif expiry_status == \"EXPIRED\" or (expires_in is not None and expires_in <= 0):\nDETAIL=SOURCE_LINE=0180: problems.append(\"cookie_expiry_expired\")\nDETAIL=SOURCE_LINE=0181: elif expiry_status == \"WARN\" or (expires_in is not None and expires_in < COOKIE_EXPIRY_WARN_SECONDS):\nDETAIL=SOURCE_LINE=0182: problems.append(\"cookie_expiry_warn\")\nDETAIL=SOURCE_LINE=0183:\nDETAIL=SOURCE_LINE=0184: timers_text = section(\"TIMERS\")\nDETAIL=SOURCE_LINE=0185: for unit in [\"skladchik-reports-monitor.timer\", \"skladchik-reports-monitor-watchdog.timer\", \"skladchik-reports-monitor-heartbeat.timer\", \"skladchik-reports-monitor-weekly-summary.timer\", \"skladchik-reports-monitor-age-alert.timer\", \"skladchik-reports-monitor-event-selftest.timer\"]:\nDETAIL=SOURCE_LINE=0186: safe = unit.replace(\".timer\", \"\").replace(\"_\", \"-\")\nDETAIL=SOURCE_LINE=0187: m = re.search(re.escape(unit) + r\"\\s+enabled=([^\\s]+)\\s+active=([^\\s]+)\", timers_text)\nDETAIL=SOURCE_LINE=0188: enabled = m.group(1) if m else \"missing\"\nDETAIL=SOURCE_LINE=0189: active = m.group(2) if m else \"missing\"\nDETAIL=SOURCE_LINE=0190: facts[f\"edge_{safe}_timer_enabled\"] = enabled\nDETAIL=SOURCE_LINE=0191: facts[f\"edge_{safe}_timer_active\"] = active\nDETAIL=SOURCE_LINE=0192: if enabled != \"enabled\" or active != \"active\":\nDETAIL=SOURCE_LINE=0193: problems.append(f\"edge_{safe}_timer_bad\")\nDETAIL=SOURCE_LINE=0194:\nDETAIL=SOURCE_LINE=0195: route_lines = section(\"ROUTE\").splitlines()\nDETAIL=SOURCE_LINE=0196: route = route_lines[0] if route_lines else \"\"\nDETAIL=SOURCE_LINE=0197: facts[\"netbird_route\"] = route\nDETAIL=SOURCE_LINE=0198: if \" wt0 \" not in (\" \" + route + \" \"):\nDETAIL=SOURCE_LINE=0199: problems.append(\"netbird_route_not_wt0\")\nDETAIL=SOURCE_LINE=0200:\nDETAIL=SOURCE_LINE=0201: files = parse_env_text(section(\"FILES\"))\nDETAIL=SOURCE_LINE=0202: for k in [\"monitor_py\", \"config_ini\", \"secrets_env\", \"cookies_jar\", \"operational_digest_py\", \"event_selftest_runner_py\", \"event_selftest_core\", \"weekly_summary_health\", \"age_alert_health\", \"event_selftest_health\", \"py_compile\"]:\nDETAIL=SOURCE_LINE=0203: facts[k] = files.get(k, \"no\")\nDETAIL=SOURCE_LINE=0204: if facts[k] != \"yes\":\nDETAIL=SOURCE_LINE=0205: problems.append(f\"edge_{k}_bad\")\nDETAIL=SOURCE_LINE=0206: facts[\"cookies_jar_bytes_file\"] = files.get(\"cookies_jar_bytes\", \"0\")\nDETAIL=SOURCE_LINE=0207:\nDETAIL=SOURCE_LINE=0208: backup = read_env_file(PVE_BACKUP_HEALTH)\nDETAIL=SOURCE_LINE=0209: facts[\"backup_status\"] = backup.get(\"STATUS\", \"MISSING\")\nDETAIL=SOURCE_LINE=0210: facts[\"backup_run_id\"] = backup.get(\"RUN_ID\", \"\")\nDETAIL=SOURCE_LINE=0211: facts[\"backup_size_bytes\"] = backup.get(\"SIZE_BYTES\", \"\")\nDETAIL=SOURCE_LINE=0212: facts[\"backup_version\"] = backup.get(\"BACKUP_VERSION\", \"\")\nDETAIL=SOURCE_LINE=0213: facts[\"backup_cookie_jar_backed_up\"] = backup.get(\"COOKIE_JAR_BACKED_UP\", \"\")\nDETAIL=SOURCE_LINE=0214: facts[\"backup_extension_health_files_backed_up\"] = backup.get(\"EXTENSION_HEALTH_FILES_BACKED_UP\", \"\")\nDETAIL=SOURCE_LINE=0215: facts[\"backup_title_in_email_subject\"] = backup.get(\"TITLE_IN_EMAIL_SUBJECT\", \"\")\nDETAIL=SOURCE_LINE=0216: facts[\"backup_health_age_seconds\"] = str(age_seconds_from_ts(backup.get(\"TS\")))\nDETAIL=SOURCE_LINE=0217: if backup.get(\"STATUS\") != \"OK\":\nDETAIL=SOURCE_LINE=0218: problems.append(\"backup_status_not_ok\")\nDETAIL=SOURCE_LINE=0219: if backup.get(\"COOKIE_JAR_BACKED_UP\") != \"yes\":\nDETAIL=SOURCE_LINE=0220: problems.append(\"backup_cookie_jar_not_marked\")\nDETAIL=SOURCE_LINE=0221: if backup.get(\"BACKUP_VERSION\") != \"4\":\nDETAIL=SOURCE_LINE=0222: problems.append(\"backup_version_not_4\")\nDETAIL=SOURCE_LINE=0223: if backup.get(\"EXTENSION_HEALTH_FILES_BACKED_UP\") != \"yes\":\nDETAIL=SOURCE_LINE=0224: problems.append(\"backup_extension_health_files_not_marked\")\nDETAIL=SOURCE_LINE=0225: if backup.get(\"TITLE_IN_EMAIL_SUBJECT\") != \"yes\":\nDETAIL=SOURCE_LINE=0226: problems.append(\"email_title_subject_not_marked\")\nDETAIL=SOURCE_LINE=0227: if int(facts[\"backup_health_age_seconds\"]) > BACKUP_HEALTH_AGE_MAX:\nDETAIL=SOURCE_LINE=0228: problems.append(\"backup_health_stale\")\nDETAIL=SOURCE_LINE=0229: try:\nDETAIL=SOURCE_LINE=0230: if int(backup.get(\"SIZE_BYTES\", \"0\")) <= 0:\nDETAIL=SOURCE_LINE=0231: problems.append(\"backup_size_bad\")\nDETAIL=SOURCE_LINE=0232: except Exception:\nDETAIL=SOURCE_LINE=0233: problems.append(\"backup_size_bad\")\nDETAIL=SOURCE_LINE=0234:\nDETAIL=SOURCE_LINE=0235: restore = read_env_file(PVE_RESTORE_HEALTH)\nDETAIL=SOURCE_LINE=0236: facts[\"restore_status\"] = restore.get(\"STATUS\", \"MISSING\")\nDETAIL=SOURCE_LINE=0237: facts[\"restore_run_id\"] = restore.get(\"RUN_ID\", \"\")\nDETAIL=SOURCE_LINE=0238: facts[\"restore_health_age_seconds\"] = str(age_seconds_from_ts(restore.get(\"TS\")))\nDETAIL=SOURCE_LINE=0239: for key in [\"CLOUD_1_STATUS\", \"CLOUD_2_STATUS\", \"SHA256_OK\", \"TAR_OK\", \"REQUIRED_FILES_OK\", \"COOKIE_JAR_OK\", \"EXTENSION_HEALTH_FILES_OK\", \"EXTENSION_SCRIPTS_OK\", \"EXTENSION_UNITS_OK\", \"SQLITE_OK\", \"PY_COMPILE_OK\"]:\nDETAIL=SOURCE_LINE=0240: facts[key.lower()] = restore.get(key, \"\")\nDETAIL=SOURCE_LINE=0241: if restore.get(\"STATUS\") != \"OK\":\nDETAIL=SOURCE_LINE=0242: problems.append(\"restore_status_not_ok\")\nDETAIL=SOURCE_LINE=0243: if int(facts[\"restore_health_age_seconds\"]) > RESTORE_HEALTH_AGE_MAX:\nDETAIL=SOURCE_LINE=0244: problems.append(\"restore_health_stale\")\nDETAIL=SOURCE_LINE=0245: for key in [\"CLOUD_1_STATUS\", \"CLOUD_2_STATUS\"]:\nDETAIL=SOURCE_LINE=0246: if restore.get(key) != \"OK\":\nDETAIL=SOURCE_LINE=0247: problems.append(key.lower() + \"_bad\")\nDETAIL=SOURCE_LINE=0248: for key in [\"SHA256_OK\", \"TAR_OK\", \"REQUIRED_FILES_OK\", \"COOKIE_JAR_OK\", \"EXTENSION_HEALTH_FILES_OK\", \"EXTENSION_SCRIPTS_OK\", \"EXTENSION_UNITS_OK\", \"SQLITE_OK\", \"PY_COMPILE_OK\"]:\nDETAIL=SOURCE_LINE=0249: if restore.get(key) != \"yes\":\nDETAIL=SOURCE_LINE=0250: problems.append(key.lower() + \"_bad\")\nDETAIL=SOURCE_LINE=0251:\nDETAIL=SOURCE_LINE=0252: local_units = [\nDETAIL=SOURCE_LINE=0253: \"skladchik-reports-monitor-backup.timer\",\nDETAIL=SOURCE_LINE=0254: \"skladchik-reports-monitor-backup-restore-check.timer\",\nDETAIL=SOURCE_LINE=0255: \"skladchik-reports-monitor-supervisor.timer\",\nDETAIL=SOURCE_LINE=0256: \"skladchik-reports-monitor-monthly-selftest-watch.timer\",\nDETAIL=SOURCE_LINE=0257: ]\nDETAIL=SOURCE_LINE=0258: props = systemctl_props(local_units)\nDETAIL=SOURCE_LINE=0259: for u, p in props.items():\nDETAIL=SOURCE_LINE=0260: base = u.replace(\".timer\", \"\").replace(\"-\", \"_\")\nDETAIL=SOURCE_LINE=0261: facts[base + \"_timer_enabled\"] = p[\"enabled\"]\nDETAIL=SOURCE_LINE=0262: facts[base + \"_timer_active\"] = p[\"active\"]\nDETAIL=SOURCE_LINE=0263: if u != \"skladchik-reports-monitor-supervisor.timer\":\nDETAIL=SOURCE_LINE=0264: if p[\"enabled\"] != \"enabled\" or p[\"active\"] != \"active\":\nDETAIL=SOURCE_LINE=0265: problems.append(base + \"_timer_bad\")\nDETAIL=SOURCE_LINE=0266: else:\nDETAIL=SOURCE_LINE=0267: # Before first enable it can be disabled/inactive during test. After enable next runs will see enabled/active.\nDETAIL=SOURCE_LINE=0268: if p[\"enabled\"] not in (\"enabled\", \"disabled\") or p[\"active\"] not in (\"active\", \"inactive\"):\nDETAIL=SOURCE_LINE=0269: problems.append(base + \"_timer_unknown\")\nDETAIL=SOURCE_LINE=0270:\nDETAIL=SOURCE_LINE=0271: return facts, problems\nDETAIL=SOURCE_LINE=0272:\nDETAIL=SOURCE_LINE=0273:def render_body(status, facts, problems, mode):\nDETAIL=SOURCE_LINE=0274: ru_status = \"OK\" if status == \"OK\" else \"АВАРИЯ\"\nDETAIL=SOURCE_LINE=0275: lines = [\nDETAIL=SOURCE_LINE=0276: f\"Статус надзора: {ru_status}\",\nDETAIL=SOURCE_LINE=0277: f\"Время UTC: {iso()}\",\nDETAIL=SOURCE_LINE=0278: \"\",\nDETAIL=SOURCE_LINE=0279: \"Монитор жалоб:\",\nDETAIL=SOURCE_LINE=0280: f\"- статус: {facts.get('monitor_status')}\",\nDETAIL=SOURCE_LINE=0281: f\"- всего жалоб на странице: {facts.get('monitor_reports')}\",\nDETAIL=SOURCE_LINE=0282: f\"- открытые/назначенные: {facts.get('monitor_open')}\",\nDETAIL=SOURCE_LINE=0283: f\"- закрытые: {facts.get('monitor_resolved')}\",\nDETAIL=SOURCE_LINE=0284: f\"- возраст health, сек: {facts.get('monitor_health_age_seconds')}\",\nDETAIL=SOURCE_LINE=0285: f\"- автопродление авторизации: {facts.get('session_keepalive')}\",\nDETAIL=SOURCE_LINE=0286: f\"- cookie jar: {facts.get('cookie_jar')} / {facts.get('cookie_jar_bytes')} байт\",\nDETAIL=SOURCE_LINE=0287: f\"- постоянная cookie действует до UTC: {facts.get('cookie_expiry_utc') or 'не определено'}\",\nDETAIL=SOURCE_LINE=0288: f\"- осталось дней по cookie: \nDETAIL=SOURCE_LINE=0289: f\"- статус срока cookie: \nDETAIL=SOURCE_LINE=0290: f\"- NetBird route: {facts.get('netbird_route')}\",\nDETAIL=SOURCE_LINE=0291: \"\",\nDETAIL=SOURCE_LINE=0292: \"Бэкап:\",\nDETAIL=SOURCE_LINE=0293: f\"- статус: {facts.get('backup_status')}\",\nDETAIL=SOURCE_LINE=0294: f\"- последний RUN_ID: {facts.get('backup_run_id')}\",\nDETAIL=SOURCE_LINE=0295: f\"- размер архива, байт: {facts.get('backup_size_bytes')}\",\nDETAIL=SOURCE_LINE=0296: f\"- cookie jar в бэкапе: {facts.get('backup_cookie_jar_backed_up')}\",\nDETAIL=SOURCE_LINE=0297: f\"- health-файлы расширений в бэкапе: {facts.get('backup_extension_health_files_backed_up')}\",\nDETAIL=SOURCE_LINE=0298: f\"- название жалобы в теме письма: {facts.get('backup_title_in_email_subject')}\",\nDETAIL=SOURCE_LINE=0299: f\"- возраст health, сек: {facts.get('backup_health_age_seconds')}\",\nDETAIL=SOURCE_LINE=0300: \"\",\nDETAIL=SOURCE_LINE=0301: \"Restore-check:\",\nDETAIL=SOURCE_LINE=0302: f\"- статус: {facts.get('restore_status')}\",\nDETAIL=SOURCE_LINE=0303: f\"- последний RUN_ID: {facts.get('restore_run_id')}\",\nDETAIL=SOURCE_LINE=0304: f\"- cloud1/cloud2: {facts.get('cloud_1_status')}/{facts.get('cloud_2_status')}\",\nDETAIL=SOURCE_LINE=0305: f\"- sha256/tar/cookie/sqlite/python: {facts.get('sha256_ok')}/{facts.get('tar_ok')}/{facts.get('cookie_jar_ok')}/{facts.get('sqlite_ok')}/{facts.get('py_compile_ok')}\",\nDETAIL=SOURCE_LINE=0306: f\"- health/scripts/units расширений: {facts.get('extension_health_files_ok')}/{facts.get('extension_scripts_ok')}/{facts.get('extension_units_ok')}\",\nDETAIL=SOURCE_LINE=0307: \"\",\nDETAIL=SOURCE_LINE=0308: \"Проблемы:\",\nDETAIL=SOURCE_LINE=0309: ]\nDETAIL=SOURCE_LINE=0310: if problems:\nDETAIL=SOURCE_LINE=0311: lines.extend([f\"- {p}\" for p in problems])\nDETAIL=SOURCE_LINE=0312: else:\nDETAIL=SOURCE_LINE=0313: lines.append(\"- нет\")\nDETAIL=SOURCE_LINE=0314: lines.extend([\nDETAIL=SOURCE_LINE=0315: \"\",\nDETAIL=SOURCE_LINE=0316: \"Авторизация:\",\nDETAIL=SOURCE_LINE=0317: \"- ручной ежемесячный перелогин обычно не нужен; монитор сам продлевает сессию постоянным cookie jar при каждом опросе;\",\nDETAIL=SOURCE_LINE=0318: \"- supervisor теперь отдельно следит за сроком постоянной cookie и заранее предупредит, если до истечения останется меньше 7 дней;\",\nDETAIL=SOURCE_LINE=0319: \"- если форум принудительно сбросит сессию, придёт отдельная тревога AUTH_REQUIRED; тогда нужно будет разово обновить вход, но это аварийный сценарий, а не ежемесячная рутина.\",\nDETAIL=SOURCE_LINE=0320: ])\nDETAIL=SOURCE_LINE=0321: return \"\\n\".join(lines)\nDETAIL=SOURCE_LINE=0322:\nDETAIL=SOURCE_LINE=0323:def write_health(status, facts, problems, mail_sent):\nDETAIL=SOURCE_LINE=0324: STATE_DIR.mkdir(parents=True, exist_ok=True)\nDETAIL=SOURCE_LINE=0325: lines = [f\"STATUS={status}\", f\"TS={iso()}\", \"problems=\" + \",\".join(problems), f\"mail_sent={'yes' if mail_sent else 'no'}\"]\nDETAIL=SOURCE_LINE=0326: for k in sorted(facts):\nDETAIL=SOURCE_LINE=0327: v = str(facts[k]).replace(\"\\n\", \" \")[:500]\nDETAIL=SOURCE_LINE=0328: lines.append(f\"{k}={v}\")\nDETAIL=SOURCE_LINE=0329: HEALTH.write_text(\"\\n\".join(lines) + \"\\n\", encoding=\"utf-8\")\nDETAIL=SOURCE_LINE=0330:\nDETAIL=SOURCE_LINE=0331:def read_state():\nDETAIL=SOURCE_LINE=0332: return read_env_file(STATE)\nDETAIL=SOURCE_LINE=0333:\nDETAIL=SOURCE_LINE=0334:def write_state(status, problems):\nDETAIL=SOURCE_LINE=0335: STATE_DIR.mkdir(parents=True, exist_ok=True)\nDETAIL=SOURCE_LINE=0336: STATE.write_text(f\"LAST_STATUS={status}\\nLAST_PROBLEMS={','.join(problems)}\\nTS={iso()}\\n\", encoding=\"utf-8\")\nDETAIL=SOURCE_LINE=0337:\nDETAIL=SOURCE_LINE=0338:def main():\nDETAIL=SOURCE_LINE=0339: mode = sys.argv[1] if len(sys.argv) > 1 else \"check\"\nDETAIL=SOURCE_LINE=0340: if mode not in (\"check\", \"test\"):\nDETAIL=SOURCE_LINE=0341: print(\"usage: skladchik-reports-monitor-supervisor [check|test]\", file=sys.stderr)\nDETAIL=SOURCE_LINE=0342: return 2\nDETAIL=SOURCE_LINE=0343:\nDETAIL=SOURCE_LINE=0344: facts, problems = gather()\nDETAIL=SOURCE_LINE=0345: status = \"OK\" if not problems else \"FAIL\"\nDETAIL=SOURCE_LINE=0346: old = read_state()\nDETAIL=SOURCE_LINE=0347: old_status = old.get(\"LAST_STATUS\", \"\")\nDETAIL=SOURCE_LINE=0348: mail_sent = False\nDETAIL=SOURCE_LINE=0349:\nDETAIL=SOURCE_LINE=0350: if mode == \"test\":\nDETAIL=SOURCE_LINE=0351: mail_sent = edge_mail(\"Тест: надзор монитора жалоб и бэкапов\", render_body(status, facts, problems, mode))\nDETAIL=SOURCE_LINE=0352: if not mail_sent:\nDETAIL=SOURCE_LINE=0353: problems.append(\"supervisor_mail_send_failed\")\nDETAIL=SOURCE_LINE=0354: status = \"FAIL\"\nDETAIL=SOURCE_LINE=0355: elif status != \"OK\":\nDETAIL=SOURCE_LINE=0356: if old_status != \"FAIL\":\nDETAIL=SOURCE_LINE=0357: mail_sent = edge_mail(\"Авария: монитор жалоб или его бэкапы\", render_body(status, facts, problems, mode))\nDETAIL=SOURCE_LINE=0358: if not mail_sent:\nDETAIL=SOURCE_LINE=0359: problems.append(\"supervisor_mail_send_failed\")\nDETAIL=SOURCE_LINE=0360: status = \"FAIL\"\nDETAIL=SOURCE_LINE=0361: elif old_status == \"FAIL\":\nDETAIL=SOURCE_LINE=0362: mail_sent = edge_mail(\"Восстановлено: монитор жалоб и бэкапы снова OK\", render_body(status, facts, problems, mode))\nDETAIL=SOURCE_LINE=0363:\nDETAIL=SOURCE_LINE=0364: write_health(status, facts, problems, mail_sent)\nDETAIL=SOURCE_LINE=0365: write_state(status, problems)\nDETAIL=SOURCE_LINE=0366:\nDETAIL=SOURCE_LINE=0367: print(f\"SUPERVISOR_STATUS={status}\")\nDETAIL=SOURCE_LINE=0368: print(f\"SUPERVISOR_MAIL_SENT={'yes' if mail_sent else 'no'}\")\nDETAIL=SOURCE_LINE=0369: print(f\"SUPERVISOR_PROBLEMS={','.join(problems)}\")\nDETAIL=SOURCE_LINE=0370: print(f\"MONITOR={facts.get('monitor_status')} reports={facts.get('monitor_reports')} open={facts.get('monitor_open')} resolved={facts.get('monitor_resolved')}\")\nDETAIL=SOURCE_LINE=0371: print(f\"BACKUP={facts.get('backup_status')} run={facts.get('backup_run_id')}\")\nDETAIL=SOURCE_LINE=0372: print(f\"RESTORE={facts.get('restore_status')} run={facts.get('restore_run_id')}\")\nDETAIL=SOURCE_LINE=0373: log(f\"CHECK mode={mode} status={status} mail_sent={'yes' if mail_sent else 'no'} problems={problems}\")\nDETAIL=SOURCE_LINE=0374: return 0 if status == \"OK\" else 1\nDETAIL=SOURCE_LINE=0375:\nDETAIL=SOURCE_LINE=0376:if __name__ == \"__main__\":\nDETAIL=SOURCE_LINE=0377: sys.exit(main())\nDETAIL=SCRIPT_TOTAL_LINES=377\nDETAIL=RELEVANT_SOURCE_END\nDETAIL=UNIT_CONTRACT_BEGIN\nDETAIL=UNIT_NAME=skladchik-reports-monitor-supervisor.service\nDETAIL=UNIT_LINE=# /etc/systemd/system/skladchik-reports-monitor-supervisor.service\nDETAIL=UNIT_LINE=[Unit]\nDETAIL=UNIT_LINE=Description=Skladchik reports monitor full supervisor\nDETAIL=UNIT_LINE=After=network-online.target\nDETAIL=UNIT_LINE=\nDETAIL=UNIT_LINE=[Service]\nDETAIL=UNIT_LINE=Type=oneshot\nDETAIL=UNIT_LINE=ExecStart=/usr/local/sbin/skladchik-reports-monitor-supervisor check\nDETAIL=UNIT_NAME=skladchik-reports-monitor-supervisor.timer\nDETAIL=UNIT_LINE=# /etc/systemd/system/skladchik-reports-monitor-supervisor.timer\nDETAIL=UNIT_LINE=[Unit]\nDETAIL=UNIT_LINE=Description=Run Skladchik reports monitor full supervisor every 10 minutes\nDETAIL=UNIT_LINE=\nDETAIL=UNIT_LINE=[Timer]\nDETAIL=UNIT_LINE=OnBootSec=5min\nDETAIL=UNIT_LINE=OnUnitActiveSec=10min\nDETAIL=UNIT_LINE=AccuracySec=30s\nDETAIL=UNIT_LINE=Persistent=false\nDETAIL=UNIT_LINE=Unit=skladchik-reports-monitor-supervisor.service\nDETAIL=UNIT_LINE=\nDETAIL=UNIT_LINE=[Install]\nDETAIL=UNIT_LINE=WantedBy=timers.target\nDETAIL=UNIT_CONTRACT_END\nDETAIL=TIMER_STATE_BEGIN\nDETAIL=TIMER_PROPERTY=Unit=skladchik-reports-monitor-supervisor.service\nDETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=\nDETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 11:58:38 MSK\nDETAIL=TIMER_PROPERTY=LoadState=loaded\nDETAIL=TIMER_PROPERTY=ActiveState=active\nDETAIL=TIMER_PROPERTY=SubState=waiting\nDETAIL=TIMER_ROW=Fri 2026-07-24 12:08:38 MSK 3min 34s Fri 2026-07-24 11:58:38 MSK 6min ago skladchik-reports-monitor-supervisor.timer skladchik-reports-monitor-supervisor.service\nDETAIL=TIMER_ROW=Sat 2026-07-25 04:39:05 MSK 16h Fri 2026-07-24 04:28:08 MSK 7h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service\nDETAIL=TIMER_ROW=Sat 2026-07-25 10:15:00 MSK 22h Fri 2026-07-24 10:15:04 MSK 1h 49min ago skladchik-reports-monitor-monthly-selftest-watch.timer skladchik-reports-monitor-monthly-selftest-watch.service\nDETAIL=TIMER_ROW=Sun 2026-07-26 05:15:13 MSK 1 day 17h Sun 2026-07-19 05:11:25 MSK - skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service\nDETAIL=TIMER_STATE_END\nDETAIL=SECRET_VALUES_OUTPUT=NONE\nDETAIL=CHANGES_MADE=NO\nDETAIL=COOKIE_CHECK_SOURCE_AUDIT=PASS\n" + "output": "DETAIL=SCRIPT_SHA256=35c68ae0929ff3795be2bde40de334fc8e26413a3d8a620a137be90d1f2c164e\nDETAIL=EXACT_COOKIE_LOGIC_BEGIN\nDETAIL=SOURCE_BLOCK=TOP_LEVEL_CONSTANT|LINES=12-12\nDETAIL=SOURCE_LINE=0012:EDGE = os.environ.get(\"SKLADCHIK_EDGE\", \"debian@[PRIVATE_IP]\")\nDETAIL=SOURCE_BLOCK=TOP_LEVEL_CONSTANT|LINES=22-22\nDETAIL=SOURCE_LINE=0022:COOKIE_EXPIRY_WARN_SECONDS = int(os.environ.get(\"SKLADCHIK_COOKIE_EXPIRY_WARN_SECONDS\", \"604800\"))\nDETAIL=SOURCE_BLOCK=FUNCTION:gather|LINES=105-271\nDETAIL=SOURCE_LINE=0105:def gather():\nDETAIL=SOURCE_LINE=0106: facts = {}\nDETAIL=SOURCE_LINE=0107: problems = []\nDETAIL=SOURCE_LINE=0108:\nDETAIL=SOURCE_LINE=0109: edge_cmd = r'''\nDETAIL=SOURCE_LINE=0110:set +e\nDETAIL=SOURCE_LINE=0111:echo __MONITOR_HEALTH_BEGIN__\nDETAIL=SOURCE_LINE=0112:cat /var/lib/skladchik-reports-monitor/health.txt 2>/dev/null || true\nDETAIL=SOURCE_LINE=0113:echo __MONITOR_HEALTH_END__\nDETAIL=SOURCE_LINE=0114:echo __ALERT_HEALTH_BEGIN__\nDETAIL=SOURCE_LINE=0115:cat /var/lib/skladchik-reports-monitor/alert-health.txt 2>/dev/null || true\nDETAIL=SOURCE_LINE=0116:echo __ALERT_HEALTH_END__\nDETAIL=SOURCE_LINE=0117:echo __TIMERS_BEGIN__\nDETAIL=SOURCE_LINE=0118:for u in skladchik-reports-monitor.timer skladchik-reports-monitor-watchdog.timer skladchik-reports-monitor-heartbeat.timer skladchik-reports-monitor-weekly-summary.timer skladchik-reports-monitor-age-alert.timer skladchik-reports-monitor-event-selftest.timer; do echo \"$u enabled=$(systemctl is-enabled \"$u\" 2>/dev/null) active=$(systemctl is-active \"$u\" 2>/dev/null)\"; done\nDETAIL=SOURCE_LINE=0119:echo __TIMERS_END__\nDETAIL=SOURCE_LINE=0120:echo __ROUTE_BEGIN__\nDETAIL=SOURCE_LINE=0121:ip route get 1.1.1.1 2>&1 | head -n 1\nDETAIL=SOURCE_LINE=0122:echo __ROUTE_END__\nDETAIL=SOURCE_LINE=0123:echo __FILES_BEGIN__\nDETAIL=SOURCE_LINE=0124:test -s /opt/skladchik-reports-monitor/monitor.py && echo monitor_py=yes || echo monitor_py=no\nDETAIL=SOURCE_LINE=0125:test -s /etc/skladchik-reports-monitor/config.ini && echo config_ini=yes || echo config_ini=no\nDETAIL=SOURCE_LINE=0126:test -s /etc/skladchik-reports-monitor/secrets.env && echo secrets_env=\nDETAIL=SOURCE_LINE=0127:test -s /var/lib/skladchik-reports-monitor/cookies.jar && echo cookies_jar=yes || echo cookies_jar=no\nDETAIL=SOURCE_LINE=0128:[ -s /var/lib/skladchik-reports-monitor/cookies.jar ] && echo cookies_jar_bytes=$(stat -c %s /var/lib/skladchik-reports-monitor/cookies.jar 2>/dev/null) || echo cookies_jar_bytes=0\nDETAIL=SOURCE_LINE=0129:test -s /opt/skladchik-reports-monitor/operational_digest.py && echo operational_digest_py=yes || echo operational_digest_py=no\nDETAIL=SOURCE_LINE=0130:test -s /opt/skladchik-reports-monitor/event-selftest-runner.py && echo event_selftest_runner_py=yes || echo event_selftest_runner_py=no\nDETAIL=SOURCE_LINE=0131:test -s /opt/skladchik-reports-monitor/event-selftest-core && echo event_selftest_core=yes || echo event_selftest_core=no\nDETAIL=SOURCE_LINE=0132:test -s /var/lib/skladchik-reports-monitor/weekly-summary-health.txt && echo weekly_summary_health=yes || echo weekly_summary_health=no\nDETAIL=SOURCE_LINE=0133:test -s /var/lib/skladchik-reports-monitor/age-alert-health.txt && echo age_alert_health=yes || echo age_alert_health=no\nDETAIL=SOURCE_LINE=0134:test -s /var/lib/skladchik-reports-monitor/event-selftest-health.txt && echo event_selftest_health=yes || echo event_selftest_health=no\nDETAIL=SOURCE_LINE=0135:python3 -m py_compile /opt/skladchik-reports-monitor/monitor.py /opt/skladchik-reports-monitor/operational_digest.py /opt/skladchik-reports-monitor/event-selftest-runner.py /opt/skladchik-reports-monitor/event-selftest-core >/dev/null 2>&1 && echo py_compile=yes || echo py_compile=no\nDETAIL=SOURCE_LINE=0136:echo __FILES_END__\nDETAIL=SOURCE_LINE=0137:'''\nDETAIL=SOURCE_LINE=0138: rc, out = ssh_edge(edge_cmd, timeout=45)\nDETAIL=SOURCE_LINE=0139: facts[\"edge_ssh_rc\"] = str(rc)\nDETAIL=SOURCE_LINE=0140: if rc != 0:\nDETAIL=SOURCE_LINE=0141: problems.append(\"edge_ssh_failed\")\nDETAIL=SOURCE_LINE=0142:\nDETAIL=SOURCE_LINE=0143: def section(name):\nDETAIL=SOURCE_LINE=0144: m = re.search(rf\"__{name}_BEGIN__\\n(.*?)\\n__{name}_END__\", out, re.S)\nDETAIL=SOURCE_LINE=0145: return m.group(1).strip() if m else \"\"\nDETAIL=SOURCE_LINE=0146:\nDETAIL=SOURCE_LINE=0147: mon = parse_env_text(section(\"MONITOR_HEALTH\"))\nDETAIL=SOURCE_LINE=0148: facts[\"monitor_status\"] = mon.get(\"STATUS\", \"MISSING\")\nDETAIL=SOURCE_LINE=0149: facts[\"monitor_reports\"] = mon.get(\"reports\", \"\")\nDETAIL=SOURCE_LINE=0150: facts[\"monitor_open\"] = mon.get(\"open\", \"\")\nDETAIL=SOURCE_LINE=0151: facts[\"monitor_resolved\"] = mon.get(\"resolved\", \"\")\nDETAIL=SOURCE_LINE=0152: facts[\"monitor_smtp_configured\"] = mon.get(\"smtp_configured\", \"\")\nDETAIL=SOURCE_LINE=0153: facts[\"session_keepalive\"] = mon.get(\"session_keepalive\", \"\")\nDETAIL=SOURCE_LINE=0154: facts[\"cookie_jar\"] = mon.get(\"cookie_jar\", \"\")\nDETAIL=SOURCE_LINE=0155: facts[\"cookie_jar_bytes\"] = mon.get(\"cookie_jar_bytes\", \"\")\nDETAIL=SOURCE_LINE=0156: facts[\"cookie_expiry_status\"] = mon.get(\"cookie_expiry_status\", \"\")\nDETAIL=SOURCE_LINE=0157: facts[\"cookie_expiry_utc\"] = mon.get(\"cookie_expiry_utc\", \"\")\nDETAIL=SOURCE_LINE=0158: facts[\"cookie_expires_in_seconds\"] = mon.get(\"cookie_expires_in_seconds\", \"\")\nDETAIL=SOURCE_LINE=0159: facts[\"cookie_expires_in_days\"] = mon.get(\"cookie_expires_in_days\", \"\")\nDETAIL=SOURCE_LINE=0160: facts[\"cookie_persistent_count\"] = mon.get(\"cookie_persistent_count\", \"\")\nDETAIL=SOURCE_LINE=0161: facts[\"monitor_health_age_seconds\"] = str(age_seconds_from_ts(mon.get(\"TS\")))\nDETAIL=SOURCE_LINE=0162: if mon.get(\"STATUS\") != \"OK\":\nDETAIL=SOURCE_LINE=0163: problems.append(\"monitor_status_not_ok\")\nDETAIL=SOURCE_LINE=0164: if int(facts[\"monitor_health_age_seconds\"]) > MONITOR_HEALTH_AGE_MAX:\nDETAIL=SOURCE_LINE=0165: problems.append(\"monitor_health_stale\")\nDETAIL=SOURCE_LINE=0166: if mon.get(\"reports\") in (\"\", \"0\"):\nDETAIL=SOURCE_LINE=0167: problems.append(\"monitor_reports_zero_or_missing\")\nDETAIL=SOURCE_LINE=0168: if mon.get(\"session_keepalive\") != \"persistent_cookie_jar\":\nDETAIL=SOURCE_LINE=0169: problems.append(\"session_keepalive_not_persistent\")\nDETAIL=SOURCE_LINE=0170: if mon.get(\"cookie_jar\") not in (\"yes\", \"empty\"):\nDETAIL=SOURCE_LINE=0171: problems.append(\"monitor_cookie_jar_bad\")\nDETAIL=SOURCE_LINE=0172: expiry_status = mon.get(\"cookie_expiry_status\", \"\")\nDETAIL=SOURCE_LINE=0173: try:\nDETAIL=SOURCE_LINE=0174: expires_in = int(mon.get(\"cookie_expires_in_seconds\", \"\"))\nDETAIL=SOURCE_LINE=0175: except Exception:\nDETAIL=SOURCE_LINE=0176: expires_in = None\nDETAIL=SOURCE_LINE=0177: if expiry_status in (\"\", \"NO_COOKIE_JAR\", \"NO_PERSISTENT_COOKIE\", \"ERROR\"):\nDETAIL=SOURCE_LINE=0178: problems.append(\"cookie_expiry_missing\")\nDETAIL=SOURCE_LINE=0179: elif expiry_status == \"EXPIRED\" or (expires_in is not None and expires_in <= 0):\nDETAIL=SOURCE_LINE=0180: problems.append(\"cookie_expiry_expired\")\nDETAIL=SOURCE_LINE=0181: elif expiry_status == \"WARN\" or (expires_in is not None and expires_in < COOKIE_EXPIRY_WARN_SECONDS):\nDETAIL=SOURCE_LINE=0182: problems.append(\"cookie_expiry_warn\")\nDETAIL=SOURCE_LINE=0183:\nDETAIL=SOURCE_LINE=0184: timers_text = section(\"TIMERS\")\nDETAIL=SOURCE_LINE=0185: for unit in [\"skladchik-reports-monitor.timer\", \"skladchik-reports-monitor-watchdog.timer\", \"skladchik-reports-monitor-heartbeat.timer\", \"skladchik-reports-monitor-weekly-summary.timer\", \"skladchik-reports-monitor-age-alert.timer\", \"skladchik-reports-monitor-event-selftest.timer\"]:\nDETAIL=SOURCE_LINE=0186: safe = unit.replace(\".timer\", \"\").replace(\"_\", \"-\")\nDETAIL=SOURCE_LINE=0187: m = re.search(re.escape(unit) + r\"\\s+enabled=([^\\s]+)\\s+active=([^\\s]+)\", timers_text)\nDETAIL=SOURCE_LINE=0188: enabled = m.group(1) if m else \"missing\"\nDETAIL=SOURCE_LINE=0189: active = m.group(2) if m else \"missing\"\nDETAIL=SOURCE_LINE=0190: facts[f\"edge_{safe}_timer_enabled\"] = enabled\nDETAIL=SOURCE_LINE=0191: facts[f\"edge_{safe}_timer_active\"] = active\nDETAIL=SOURCE_LINE=0192: if enabled != \"enabled\" or active != \"active\":\nDETAIL=SOURCE_LINE=0193: problems.append(f\"edge_{safe}_timer_bad\")\nDETAIL=SOURCE_LINE=0194:\nDETAIL=SOURCE_LINE=0195: route_lines = section(\"ROUTE\").splitlines()\nDETAIL=SOURCE_LINE=0196: route = route_lines[0] if route_lines else \"\"\nDETAIL=SOURCE_LINE=0197: facts[\"netbird_route\"] = route\nDETAIL=SOURCE_LINE=0198: if \" wt0 \" not in (\" \" + route + \" \"):\nDETAIL=SOURCE_LINE=0199: problems.append(\"netbird_route_not_wt0\")\nDETAIL=SOURCE_LINE=0200:\nDETAIL=SOURCE_LINE=0201: files = parse_env_text(section(\"FILES\"))\nDETAIL=SOURCE_LINE=0202: for k in [\"monitor_py\", \"config_ini\", \"secrets_env\", \"cookies_jar\", \"operational_digest_py\", \"event_selftest_runner_py\", \"event_selftest_core\", \"weekly_summary_health\", \"age_alert_health\", \"event_selftest_health\", \"py_compile\"]:\nDETAIL=SOURCE_LINE=0203: facts[k] = files.get(k, \"no\")\nDETAIL=SOURCE_LINE=0204: if facts[k] != \"yes\":\nDETAIL=SOURCE_LINE=0205: problems.append(f\"edge_{k}_bad\")\nDETAIL=SOURCE_LINE=0206: facts[\"cookies_jar_bytes_file\"] = files.get(\"cookies_jar_bytes\", \"0\")\nDETAIL=SOURCE_LINE=0207:\nDETAIL=SOURCE_LINE=0208: backup = read_env_file(PVE_BACKUP_HEALTH)\nDETAIL=SOURCE_LINE=0209: facts[\"backup_status\"] = backup.get(\"STATUS\", \"MISSING\")\nDETAIL=SOURCE_LINE=0210: facts[\"backup_run_id\"] = backup.get(\"RUN_ID\", \"\")\nDETAIL=SOURCE_LINE=0211: facts[\"backup_size_bytes\"] = backup.get(\"SIZE_BYTES\", \"\")\nDETAIL=SOURCE_LINE=0212: facts[\"backup_version\"] = backup.get(\"BACKUP_VERSION\", \"\")\nDETAIL=SOURCE_LINE=0213: facts[\"backup_cookie_jar_backed_up\"] = backup.get(\"COOKIE_JAR_BACKED_UP\", \"\")\nDETAIL=SOURCE_LINE=0214: facts[\"backup_extension_health_files_backed_up\"] = backup.get(\"EXTENSION_HEALTH_FILES_BACKED_UP\", \"\")\nDETAIL=SOURCE_LINE=0215: facts[\"backup_title_in_email_subject\"] = backup.get(\"TITLE_IN_EMAIL_SUBJECT\", \"\")\nDETAIL=SOURCE_LINE=0216: facts[\"backup_health_age_seconds\"] = str(age_seconds_from_ts(backup.get(\"TS\")))\nDETAIL=SOURCE_LINE=0217: if backup.get(\"STATUS\") != \"OK\":\nDETAIL=SOURCE_LINE=0218: problems.append(\"backup_status_not_ok\")\nDETAIL=SOURCE_LINE=0219: if backup.get(\"COOKIE_JAR_BACKED_UP\") != \"yes\":\nDETAIL=SOURCE_LINE=0220: problems.append(\"backup_cookie_jar_not_marked\")\nDETAIL=SOURCE_LINE=0221: if backup.get(\"BACKUP_VERSION\") != \"4\":\nDETAIL=SOURCE_LINE=0222: problems.append(\"backup_version_not_4\")\nDETAIL=SOURCE_LINE=0223: if backup.get(\"EXTENSION_HEALTH_FILES_BACKED_UP\") != \"yes\":\nDETAIL=SOURCE_LINE=0224: problems.append(\"backup_extension_health_files_not_marked\")\nDETAIL=SOURCE_LINE=0225: if backup.get(\"TITLE_IN_EMAIL_SUBJECT\") != \"yes\":\nDETAIL=SOURCE_LINE=0226: problems.append(\"email_title_subject_not_marked\")\nDETAIL=SOURCE_LINE=0227: if int(facts[\"backup_health_age_seconds\"]) > BACKUP_HEALTH_AGE_MAX:\nDETAIL=SOURCE_LINE=0228: problems.append(\"backup_health_stale\")\nDETAIL=SOURCE_LINE=0229: try:\nDETAIL=SOURCE_LINE=0230: if int(backup.get(\"SIZE_BYTES\", \"0\")) <= 0:\nDETAIL=SOURCE_LINE=0231: problems.append(\"backup_size_bad\")\nDETAIL=SOURCE_LINE=0232: except Exception:\nDETAIL=SOURCE_LINE=0233: problems.append(\"backup_size_bad\")\nDETAIL=SOURCE_LINE=0234:\nDETAIL=SOURCE_LINE=0235: restore = read_env_file(PVE_RESTORE_HEALTH)\nDETAIL=SOURCE_LINE=0236: facts[\"restore_status\"] = restore.get(\"STATUS\", \"MISSING\")\nDETAIL=SOURCE_LINE=0237: facts[\"restore_run_id\"] = restore.get(\"RUN_ID\", \"\")\nDETAIL=SOURCE_LINE=0238: facts[\"restore_health_age_seconds\"] = str(age_seconds_from_ts(restore.get(\"TS\")))\nDETAIL=SOURCE_LINE=0239: for key in [\"CLOUD_1_STATUS\", \"CLOUD_2_STATUS\", \"SHA256_OK\", \"TAR_OK\", \"REQUIRED_FILES_OK\", \"COOKIE_JAR_OK\", \"EXTENSION_HEALTH_FILES_OK\", \"EXTENSION_SCRIPTS_OK\", \"EXTENSION_UNITS_OK\", \"SQLITE_OK\", \"PY_COMPILE_OK\"]:\nDETAIL=SOURCE_LINE=0240: facts[key.lower()] = restore.get(key, \"\")\nDETAIL=SOURCE_LINE=0241: if restore.get(\"STATUS\") != \"OK\":\nDETAIL=SOURCE_LINE=0242: problems.append(\"restore_status_not_ok\")\nDETAIL=SOURCE_LINE=0243: if int(facts[\"restore_health_age_seconds\"]) > RESTORE_HEALTH_AGE_MAX:\nDETAIL=SOURCE_LINE=0244: problems.append(\"restore_health_stale\")\nDETAIL=SOURCE_LINE=0245: for key in [\"CLOUD_1_STATUS\", \"CLOUD_2_STATUS\"]:\nDETAIL=SOURCE_LINE=0246: if restore.get(key) != \"OK\":\nDETAIL=SOURCE_LINE=0247: problems.append(key.lower() + \"_bad\")\nDETAIL=SOURCE_LINE=0248: for key in [\"SHA256_OK\", \"TAR_OK\", \"REQUIRED_FILES_OK\", \"COOKIE_JAR_OK\", \"EXTENSION_HEALTH_FILES_OK\", \"EXTENSION_SCRIPTS_OK\", \"EXTENSION_UNITS_OK\", \"SQLITE_OK\", \"PY_COMPILE_OK\"]:\nDETAIL=SOURCE_LINE=0249: if restore.get(key) != \"yes\":\nDETAIL=SOURCE_LINE=0250: problems.append(key.lower() + \"_bad\")\nDETAIL=SOURCE_LINE=0251:\nDETAIL=SOURCE_LINE=0252: local_units = [\nDETAIL=SOURCE_LINE=0253: \"skladchik-reports-monitor-backup.timer\",\nDETAIL=SOURCE_LINE=0254: \"skladchik-reports-monitor-backup-restore-check.timer\",\nDETAIL=SOURCE_LINE=0255: \"skladchik-reports-monitor-supervisor.timer\",\nDETAIL=SOURCE_LINE=0256: \"skladchik-reports-monitor-monthly-selftest-watch.timer\",\nDETAIL=SOURCE_LINE=0257: ]\nDETAIL=SOURCE_LINE=0258: props = systemctl_props(local_units)\nDETAIL=SOURCE_LINE=0259: for u, p in props.items():\nDETAIL=SOURCE_LINE=0260: base = u.replace(\".timer\", \"\").replace(\"-\", \"_\")\nDETAIL=SOURCE_LINE=0261: facts[base + \"_timer_enabled\"] = p[\"enabled\"]\nDETAIL=SOURCE_LINE=0262: facts[base + \"_timer_active\"] = p[\"active\"]\nDETAIL=SOURCE_LINE=0263: if u != \"skladchik-reports-monitor-supervisor.timer\":\nDETAIL=SOURCE_LINE=0264: if p[\"enabled\"] != \"enabled\" or p[\"active\"] != \"active\":\nDETAIL=SOURCE_LINE=0265: problems.append(base + \"_timer_bad\")\nDETAIL=SOURCE_LINE=0266: else:\nDETAIL=SOURCE_LINE=0267: # Before first enable it can be disabled/inactive during test. After enable next runs will see enabled/active.\nDETAIL=SOURCE_LINE=0268: if p[\"enabled\"] not in (\"enabled\", \"disabled\") or p[\"active\"] not in (\"active\", \"inactive\"):\nDETAIL=SOURCE_LINE=0269: problems.append(base + \"_timer_unknown\")\nDETAIL=SOURCE_LINE=0270:\nDETAIL=SOURCE_LINE=0271: return facts, problems\nDETAIL=SOURCE_BLOCK=FUNCTION:render_body|LINES=273-321\nDETAIL=SOURCE_LINE=0273:def render_body(status, facts, problems, mode):\nDETAIL=SOURCE_LINE=0274: ru_status = \"OK\" if status == \"OK\" else \"АВАРИЯ\"\nDETAIL=SOURCE_LINE=0275: lines = [\nDETAIL=SOURCE_LINE=0276: f\"Статус надзора: {ru_status}\",\nDETAIL=SOURCE_LINE=0277: f\"Время UTC: {iso()}\",\nDETAIL=SOURCE_LINE=0278: \"\",\nDETAIL=SOURCE_LINE=0279: \"Монитор жалоб:\",\nDETAIL=SOURCE_LINE=0280: f\"- статус: {facts.get('monitor_status')}\",\nDETAIL=SOURCE_LINE=0281: f\"- всего жалоб на странице: {facts.get('monitor_reports')}\",\nDETAIL=SOURCE_LINE=0282: f\"- открытые/назначенные: {facts.get('monitor_open')}\",\nDETAIL=SOURCE_LINE=0283: f\"- закрытые: {facts.get('monitor_resolved')}\",\nDETAIL=SOURCE_LINE=0284: f\"- возраст health, сек: {facts.get('monitor_health_age_seconds')}\",\nDETAIL=SOURCE_LINE=0285: f\"- автопродление авторизации: {facts.get('session_keepalive')}\",\nDETAIL=SOURCE_LINE=0286: f\"- cookie jar: {facts.get('cookie_jar')} / {facts.get('cookie_jar_bytes')} байт\",\nDETAIL=SOURCE_LINE=0287: f\"- постоянная cookie действует до UTC: {facts.get('cookie_expiry_utc') or 'не определено'}\",\nDETAIL=SOURCE_LINE=0288: f\"- осталось дней по cookie: {facts.get('cookie_expires_in_days') or 'не определено'}\",\nDETAIL=SOURCE_LINE=0289: f\"- статус срока cookie: {facts.get('cookie_expiry_status') or 'не определено'}\",\nDETAIL=SOURCE_LINE=0290: f\"- NetBird route: {facts.get('netbird_route')}\",\nDETAIL=SOURCE_LINE=0291: \"\",\nDETAIL=SOURCE_LINE=0292: \"Бэкап:\",\nDETAIL=SOURCE_LINE=0293: f\"- статус: {facts.get('backup_status')}\",\nDETAIL=SOURCE_LINE=0294: f\"- последний RUN_ID: {facts.get('backup_run_id')}\",\nDETAIL=SOURCE_LINE=0295: f\"- размер архива, байт: {facts.get('backup_size_bytes')}\",\nDETAIL=SOURCE_LINE=0296: f\"- cookie jar в бэкапе: {facts.get('backup_cookie_jar_backed_up')}\",\nDETAIL=SOURCE_LINE=0297: f\"- health-файлы расширений в бэкапе: {facts.get('backup_extension_health_files_backed_up')}\",\nDETAIL=SOURCE_LINE=0298: f\"- название жалобы в теме письма: {facts.get('backup_title_in_email_subject')}\",\nDETAIL=SOURCE_LINE=0299: f\"- возраст health, сек: {facts.get('backup_health_age_seconds')}\",\nDETAIL=SOURCE_LINE=0300: \"\",\nDETAIL=SOURCE_LINE=0301: \"Restore-check:\",\nDETAIL=SOURCE_LINE=0302: f\"- статус: {facts.get('restore_status')}\",\nDETAIL=SOURCE_LINE=0303: f\"- последний RUN_ID: {facts.get('restore_run_id')}\",\nDETAIL=SOURCE_LINE=0304: f\"- cloud1/cloud2: {facts.get('cloud_1_status')}/{facts.get('cloud_2_status')}\",\nDETAIL=SOURCE_LINE=0305: f\"- sha256/tar/cookie/sqlite/python: {facts.get('sha256_ok')}/{facts.get('tar_ok')}/{facts.get('cookie_jar_ok')}/{facts.get('sqlite_ok')}/{facts.get('py_compile_ok')}\",\nDETAIL=SOURCE_LINE=0306: f\"- health/scripts/units расширений: {facts.get('extension_health_files_ok')}/{facts.get('extension_scripts_ok')}/{facts.get('extension_units_ok')}\",\nDETAIL=SOURCE_LINE=0307: \"\",\nDETAIL=SOURCE_LINE=0308: \"Проблемы:\",\nDETAIL=SOURCE_LINE=0309: ]\nDETAIL=SOURCE_LINE=0310: if problems:\nDETAIL=SOURCE_LINE=0311: lines.extend([f\"- {p}\" for p in problems])\nDETAIL=SOURCE_LINE=0312: else:\nDETAIL=SOURCE_LINE=0313: lines.append(\"- нет\")\nDETAIL=SOURCE_LINE=0314: lines.extend([\nDETAIL=SOURCE_LINE=0315: \"\",\nDETAIL=SOURCE_LINE=0316: \"Авторизация:\",\nDETAIL=SOURCE_LINE=0317: \"- ручной ежемесячный перелогин обычно не нужен; монитор сам продлевает сессию постоянным cookie jar при каждом опросе;\",\nDETAIL=SOURCE_LINE=0318: \"- supervisor теперь отдельно следит за сроком постоянной cookie и заранее предупредит, если до истечения останется меньше 7 дней;\",\nDETAIL=SOURCE_LINE=0319: \"- если форум принудительно сбросит сессию, придёт отдельная тревога AUTH_REQUIRED; тогда нужно будет разово обновить вход, но это аварийный сценарий, а не ежемесячная рутина.\",\nDETAIL=SOURCE_LINE=0320: ])\nDETAIL=SOURCE_LINE=0321: return \"\\n\".join(lines)\nDETAIL=EXACT_COOKIE_LOGIC_END\nDETAIL=PVE01_TIMER_CORRELATION_BEGIN\nDETAIL=PVE01_TIMER=Fri 2026-07-24 12:08:38 MSK 1min 48s Fri 2026-07-24 11:58:38 MSK 8min ago skladchik-reports-monitor-supervisor.timer skladchik-reports-monitor-supervisor.service\nDETAIL=PVE01_TIMER=Sat 2026-07-25 04:39:05 MSK 16h Fri 2026-07-24 04:28:08 MSK 7h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service\nDETAIL=PVE01_TIMER=Sat 2026-07-25 10:15:00 MSK 22h Fri 2026-07-24 10:15:04 MSK 1h 51min ago skladchik-reports-monitor-monthly-selftest-watch.timer skladchik-reports-monitor-monthly-selftest-watch.service\nDETAIL=PVE01_TIMER=Sun 2026-07-26 05:15:13 MSK 1 day 17h Sun 2026-07-19 05:11:25 MSK - skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service\nDETAIL=PVE01_TIMER_CORRELATION_END\nDETAIL=EDGE_TIMER_CORRELATION_BEGIN\nDETAIL=EDGE_TIMER=Fri 2026-07-24 12:06:51 MSK 1s Fri 2026-07-24 12:06:21 MSK 28s ago skladchik-reports-monitor.timer skladchik-reports-monitor.service\nDETAIL=EDGE_TIMER=Fri 2026-07-24 12:08:35 MSK 1min 45s Fri 2026-07-24 12:03:35 MSK 3min 14s ago skladchik-reports-monitor-watchdog.timer skladchik-reports-monitor-watchdog.service\nDETAIL=EDGE_TIMER=Sat 2026-07-25 09:16:28 MSK 21h Fri 2026-07-24 09:18:35 MSK 2h 48min ago skladchik-reports-monitor-heartbeat.timer skladchik-reports-monitor-heartbeat.service\nDETAIL=EDGE_TIMER=Sat 2026-07-25 09:45:00 MSK 21h Fri 2026-07-24 09:45:03 MSK 2h 21min ago skladchik-reports-monitor-age-alert.timer skladchik-reports-monitor-age-alert.service\nDETAIL=EDGE_TIMER=Mon 2026-07-27 09:35:00 MSK 2 days Mon 2026-07-20 09:35:06 MSK - skladchik-reports-monitor-weekly-summary.timer skladchik-reports-monitor-weekly-summary.service\nDETAIL=EDGE_TIMER=Sat 2026-08-01 09:25:00 MSK 1 week 0 days Fri 2026-07-10 10:28:28 MSK - skladchik-reports-monitor-event-selftest.timer skladchik-reports-monitor-event-selftest.service\nDETAIL=EDGE_TIMER_CORRELATION_END\nDETAIL=SECRET_VALUES_OUTPUT=NONE\nDETAIL=COOKIE_CONTENT_OUTPUT=NONE\nDETAIL=COOKIE_EXACT_LOGIC_AUDIT=PASS\nCHANGES_MADE=NO\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index a7af1d9..2936ff3 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,5 +1,5 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=HOMELAB-SKLADCHIK-COOKIE-CHECK-SOURCE-AUDIT-418 +COMMAND_ID=HOMELAB-SKLADCHIK-COOKIE-EXACT-LOGIC-419 STATUS=OK RC=0 HOST=pve01 @@ -16,100 +16,16 @@ CHANGES_MADE=false SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=1c26f0b751c5e55421c706f56f4be25d92c17849df8a385fb66ce01144801022 -SANITIZED_OUTPUT_SHA256=8e5a54db35050a5c0010241fb0bad91fc5966530dbac76bb07426b09af51a6a1 +RAW_EVIDENCE_SHA256=63d14ea58a82bd34d78884b4db6d047e8fa64822200c7bc20a8587057f3f9e25 +SANITIZED_OUTPUT_SHA256=702451cb7bfbc7d831b17b7e062a00f5215e5dc3729bd8e3331dfe4c3dc28851 OUTPUT_BEGIN -DETAIL=SCRIPT_PATH=/usr/local/sbin/skladchik-reports-monitor-supervisor DETAIL=SCRIPT_SHA256=35c68ae0929ff3795be2bde40de334fc8e26413a3d8a620a137be90d1f2c164e -DETAIL=SCRIPT_METADATA=BYTES:20463|MODE:755|OWNER:root:root|MTIME:2026-07-18 13:52:30.415780066 +0300 -DETAIL=RELEVANT_SOURCE_BEGIN -DETAIL=SOURCE_LINE=0005:import os -DETAIL=SOURCE_LINE=0006:import re -DETAIL=SOURCE_LINE=0007:import shlex -DETAIL=SOURCE_LINE=0008:import subprocess -DETAIL=SOURCE_LINE=0009:import sys -DETAIL=SOURCE_LINE=0010:from pathlib import Path -DETAIL=SOURCE_LINE=0011: +DETAIL=EXACT_COOKIE_LOGIC_BEGIN +DETAIL=SOURCE_BLOCK=TOP_LEVEL_CONSTANT|LINES=12-12 DETAIL=SOURCE_LINE=0012:EDGE = os.environ.get("SKLADCHIK_EDGE", "debian@[PRIVATE_IP]") -DETAIL=SOURCE_LINE=0013:STATE_DIR = Path("/var/lib/skladchik-reports-monitor-supervisor") -DETAIL=SOURCE_LINE=0014:LOG_DIR = Path("/var/log/skladchik-reports-monitor-supervisor") -DETAIL=SOURCE_LINE=0015:HEALTH = STATE_DIR / "health.txt" -DETAIL=SOURCE_LINE=0016:STATE = STATE_DIR / "state.env" -DETAIL=SOURCE_LINE=0017:LOG = LOG_DIR / "supervisor.log" -DETAIL=SOURCE_LINE=0018: -DETAIL=SOURCE_LINE=0019:MONITOR_HEALTH_AGE_MAX = int(os.environ.get("SKLADCHIK_MONITOR_HEALTH_MAX_SECONDS", "600")) -DETAIL=SOURCE_LINE=0020:BACKUP_HEALTH_AGE_MAX = int(os.environ.get("SKLADCHIK_BACKUP_HEALTH_MAX_SECONDS", "129600")) -DETAIL=SOURCE_LINE=0021:RESTORE_HEALTH_AGE_MAX = int(os.environ.get("SKLADCHIK_RESTORE_HEALTH_MAX_SECONDS", "691200")) +DETAIL=SOURCE_BLOCK=TOP_LEVEL_CONSTANT|LINES=22-22 DETAIL=SOURCE_LINE=0022:COOKIE_EXPIRY_WARN_SECONDS = int(os.environ.get("SKLADCHIK_COOKIE_EXPIRY_WARN_SECONDS", "604800")) -DETAIL=SOURCE_LINE=0023: -DETAIL=SOURCE_LINE=0024:PVE_BACKUP_HEALTH = Path("/var/lib/skladchik-reports-monitor-backup/health.txt") -DETAIL=SOURCE_LINE=0025:PVE_RESTORE_HEALTH = Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt") -DETAIL=SOURCE_LINE=0026: -DETAIL=SOURCE_LINE=0027:def now(): -DETAIL=SOURCE_LINE=0028: return _dt.datetime.now(_dt.UTC).replace(microsecond=0) -DETAIL=SOURCE_LINE=0029: -DETAIL=SOURCE_LINE=0030:def iso(): -DETAIL=SOURCE_LINE=0031: return now().isoformat().replace("+00:00", "Z") -DETAIL=SOURCE_LINE=0032: -DETAIL=SOURCE_LINE=0033:def log(msg): -DETAIL=SOURCE_LINE=0034: LOG_DIR.mkdir(parents=True, exist_ok=True) -DETAIL=SOURCE_LINE=0035: with LOG.open("a", encoding="utf-8") as f: -DETAIL=SOURCE_LINE=0036: f.write(f"{iso()} {msg}\n") -DETAIL=SOURCE_LINE=0037: -DETAIL=SOURCE_LINE=0038:def run(cmd, timeout=45, input_text=None): -DETAIL=SOURCE_LINE=0039: try: -DETAIL=SOURCE_LINE=0040: p = subprocess.run(cmd, shell=True, text=True, input=input_text, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout) -DETAIL=SOURCE_LINE=0041: return p.returncode, p.stdout.strip() -DETAIL=SOURCE_LINE=0042: except subprocess.TimeoutExpired as e: -DETAIL=SOURCE_LINE=0043: out = e.stdout if isinstance(e.stdout, str) else "" -DETAIL=SOURCE_LINE=0044: return 124, (out or "").strip() + "\nTIMEOUT" -DETAIL=SOURCE_LINE=0045: -DETAIL=SOURCE_LINE=0046:def parse_env_text(text): -DETAIL=SOURCE_LINE=0047: d = {} -DETAIL=SOURCE_LINE=0048: for line in (text or "").splitlines(): -DETAIL=SOURCE_LINE=0049: if not line or line.startswith("#") or "=" not in line: -DETAIL=SOURCE_LINE=0050: continue -DETAIL=SOURCE_LINE=0051: k, v = line.split("=", 1) -DETAIL=SOURCE_LINE=... -DETAIL=SOURCE_LINE=0066: dt = _dt.datetime.fromisoformat(s) -DETAIL=SOURCE_LINE=0067: if dt.tzinfo is None: -DETAIL=SOURCE_LINE=0068: dt = dt.replace(tzinfo=_dt.UTC) -DETAIL=SOURCE_LINE=0069: return int((now() - dt.astimezone(_dt.UTC)).total_seconds()) -DETAIL=SOURCE_LINE=0070: except Exception: -DETAIL=SOURCE_LINE=0071: return 999999999 -DETAIL=SOURCE_LINE=0072: -DETAIL=SOURCE_LINE=0073:def ssh_edge(cmd, timeout=45): -DETAIL=SOURCE_LINE=0074: q = "ssh -o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] " + shlex.quote(EDGE) + " " + shlex.quote("sudo bash -lc " + shlex.quote(cmd)) -DETAIL=SOURCE_LINE=0075: return run(q, timeout=timeout) -DETAIL=SOURCE_LINE=0076: -DETAIL=SOURCE_LINE=0077:def edge_mail(subject, body): -DETAIL=SOURCE_LINE=0078: subj_b64 = base64.b64encode(subject.encode("utf-8")).decode("ascii") -DETAIL=SOURCE_LINE=0079: body_b64 = base64.b64encode(body.encode("utf-8")).decode("ascii") -DETAIL=SOURCE_LINE=0080: py = ( -DETAIL=SOURCE_LINE=0081: "import base64,importlib.util;" -DETAIL=SOURCE_LINE=0082: "spec=importlib.util.spec_from_file_location('m','/opt/skladchik-reports-monitor/monitor.py');" -DETAIL=SOURCE_LINE=0083: "m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m);" -DETAIL=SOURCE_LINE=0084: "cfg,secrets= -DETAIL=SOURCE_LINE=0085: f"subject=base64.b64decode('{subj_b64}').decode('utf-8');" -DETAIL=SOURCE_LINE=0086: f"body=base64.b64decode('{body_b64}').decode('utf-8');" -DETAIL=SOURCE_LINE=0087: "ok=m.send_mail(cfg,secrets,subject,body);" -DETAIL=SOURCE_LINE=0088: "print('MAIL_OK' if ok else 'MAIL_FALSE')" -DETAIL=SOURCE_LINE=0089: ) -DETAIL=SOURCE_LINE=0090: rc, out = ssh_edge("python3 -c " + shlex.quote(py), timeout=60) -DETAIL=SOURCE_LINE=0091: log(f"MAIL subject={subject!r} rc={rc} out={out!r}") -DETAIL=SOURCE_LINE=0092: return rc == 0 and "MAIL_OK" in out -DETAIL=SOURCE_LINE=0093: -DETAIL=SOURCE_LINE=0094:def systemctl_props(units): -DETAIL=SOURCE_LINE=0095: cmd = "systemctl is-enabled " + " ".join(map(shlex.quote, units)) + " 2>/dev/null || true; echo __ACTIVE__; systemctl is-active " + " ".join(map(shlex.quote, units)) + " 2>/dev/null || true" -DETAIL=SOURCE_LINE=0096: rc, out = run(cmd, timeout=20) -DETAIL=SOURCE_LINE=0097: parts = out.split("__ACTIVE__", 1) -DETAIL=SOURCE_LINE=0098: enabled_lines = [x.strip() for x in parts[0].splitlines() if x.strip()] -DETAIL=SOURCE_LINE=0099: active_lines = [x.strip() for x in parts[1].splitlines() if x.strip()] if len(parts) > 1 else [] -DETAIL=SOURCE_LINE=0100: res = {} -DETAIL=SOURCE_LINE=0101: for i, u in enumerate(units): -DETAIL=SOURCE_LINE=0102: res[u] = {"enabled": enabled_lines[i] if i < len(enabled_lines) else "unknown", "active": active_lines[i] if i < len(active_lines) else "unknown"} -DETAIL=SOURCE_LINE=0103: return res -DETAIL=SOURCE_LINE=0104: +DETAIL=SOURCE_BLOCK=FUNCTION:gather|LINES=105-271 DETAIL=SOURCE_LINE=0105:def gather(): DETAIL=SOURCE_LINE=0106: facts = {} DETAIL=SOURCE_LINE=0107: problems = [] @@ -277,7 +193,7 @@ DETAIL=SOURCE_LINE=0268: if p["enabled"] not in ("enabled", "disabled DETAIL=SOURCE_LINE=0269: problems.append(base + "_timer_unknown") DETAIL=SOURCE_LINE=0270: DETAIL=SOURCE_LINE=0271: return facts, problems -DETAIL=SOURCE_LINE=0272: +DETAIL=SOURCE_BLOCK=FUNCTION:render_body|LINES=273-321 DETAIL=SOURCE_LINE=0273:def render_body(status, facts, problems, mode): DETAIL=SOURCE_LINE=0274: ru_status = "OK" if status == "OK" else "АВАРИЯ" DETAIL=SOURCE_LINE=0275: lines = [ @@ -293,8 +209,8 @@ DETAIL=SOURCE_LINE=0284: f"- возраст health, сек: {facts.get('m DETAIL=SOURCE_LINE=0285: f"- автопродление авторизации: {facts.get('session_keepalive')}", DETAIL=SOURCE_LINE=0286: f"- cookie jar: {facts.get('cookie_jar')} / {facts.get('cookie_jar_bytes')} байт", DETAIL=SOURCE_LINE=0287: f"- постоянная cookie действует до UTC: {facts.get('cookie_expiry_utc') or 'не определено'}", -DETAIL=SOURCE_LINE=0288: f"- осталось дней по cookie: -DETAIL=SOURCE_LINE=0289: f"- статус срока cookie: +DETAIL=SOURCE_LINE=0288: f"- осталось дней по cookie: {facts.get('cookie_expires_in_days') or 'не определено'}", +DETAIL=SOURCE_LINE=0289: f"- статус срока cookie: {facts.get('cookie_expiry_status') or 'не определено'}", DETAIL=SOURCE_LINE=0290: f"- NetBird route: {facts.get('netbird_route')}", DETAIL=SOURCE_LINE=0291: "", DETAIL=SOURCE_LINE=0292: "Бэкап:", @@ -327,104 +243,25 @@ DETAIL=SOURCE_LINE=0318: "- supervisor теперь отдельно сл DETAIL=SOURCE_LINE=0319: "- если форум принудительно сбросит сессию, придёт отдельная тревога AUTH_REQUIRED; тогда нужно будет разово обновить вход, но это аварийный сценарий, а не ежемесячная рутина.", DETAIL=SOURCE_LINE=0320: ]) DETAIL=SOURCE_LINE=0321: return "\n".join(lines) -DETAIL=SOURCE_LINE=0322: -DETAIL=SOURCE_LINE=0323:def write_health(status, facts, problems, mail_sent): -DETAIL=SOURCE_LINE=0324: STATE_DIR.mkdir(parents=True, exist_ok=True) -DETAIL=SOURCE_LINE=0325: lines = [f"STATUS={status}", f"TS={iso()}", "problems=" + ",".join(problems), f"mail_sent={'yes' if mail_sent else 'no'}"] -DETAIL=SOURCE_LINE=0326: for k in sorted(facts): -DETAIL=SOURCE_LINE=0327: v = str(facts[k]).replace("\n", " ")[:500] -DETAIL=SOURCE_LINE=0328: lines.append(f"{k}={v}") -DETAIL=SOURCE_LINE=0329: HEALTH.write_text("\n".join(lines) + "\n", encoding="utf-8") -DETAIL=SOURCE_LINE=0330: -DETAIL=SOURCE_LINE=0331:def read_state(): -DETAIL=SOURCE_LINE=0332: return read_env_file(STATE) -DETAIL=SOURCE_LINE=0333: -DETAIL=SOURCE_LINE=0334:def write_state(status, problems): -DETAIL=SOURCE_LINE=0335: STATE_DIR.mkdir(parents=True, exist_ok=True) -DETAIL=SOURCE_LINE=0336: STATE.write_text(f"LAST_STATUS={status}\nLAST_PROBLEMS={','.join(problems)}\nTS={iso()}\n", encoding="utf-8") -DETAIL=SOURCE_LINE=0337: -DETAIL=SOURCE_LINE=0338:def main(): -DETAIL=SOURCE_LINE=0339: mode = sys.argv[1] if len(sys.argv) > 1 else "check" -DETAIL=SOURCE_LINE=0340: if mode not in ("check", "test"): -DETAIL=SOURCE_LINE=0341: print("usage: skladchik-reports-monitor-supervisor [check|test]", file=sys.stderr) -DETAIL=SOURCE_LINE=0342: return 2 -DETAIL=SOURCE_LINE=0343: -DETAIL=SOURCE_LINE=0344: facts, problems = gather() -DETAIL=SOURCE_LINE=0345: status = "OK" if not problems else "FAIL" -DETAIL=SOURCE_LINE=0346: old = read_state() -DETAIL=SOURCE_LINE=0347: old_status = old.get("LAST_STATUS", "") -DETAIL=SOURCE_LINE=0348: mail_sent = False -DETAIL=SOURCE_LINE=0349: -DETAIL=SOURCE_LINE=0350: if mode == "test": -DETAIL=SOURCE_LINE=0351: mail_sent = edge_mail("Тест: надзор монитора жалоб и бэкапов", render_body(status, facts, problems, mode)) -DETAIL=SOURCE_LINE=0352: if not mail_sent: -DETAIL=SOURCE_LINE=0353: problems.append("supervisor_mail_send_failed") -DETAIL=SOURCE_LINE=0354: status = "FAIL" -DETAIL=SOURCE_LINE=0355: elif status != "OK": -DETAIL=SOURCE_LINE=0356: if old_status != "FAIL": -DETAIL=SOURCE_LINE=0357: mail_sent = edge_mail("Авария: монитор жалоб или его бэкапы", render_body(status, facts, problems, mode)) -DETAIL=SOURCE_LINE=0358: if not mail_sent: -DETAIL=SOURCE_LINE=0359: problems.append("supervisor_mail_send_failed") -DETAIL=SOURCE_LINE=0360: status = "FAIL" -DETAIL=SOURCE_LINE=0361: elif old_status == "FAIL": -DETAIL=SOURCE_LINE=0362: mail_sent = edge_mail("Восстановлено: монитор жалоб и бэкапы снова OK", render_body(status, facts, problems, mode)) -DETAIL=SOURCE_LINE=0363: -DETAIL=SOURCE_LINE=0364: write_health(status, facts, problems, mail_sent) -DETAIL=SOURCE_LINE=0365: write_state(status, problems) -DETAIL=SOURCE_LINE=0366: -DETAIL=SOURCE_LINE=0367: print(f"SUPERVISOR_STATUS={status}") -DETAIL=SOURCE_LINE=0368: print(f"SUPERVISOR_MAIL_SENT={'yes' if mail_sent else 'no'}") -DETAIL=SOURCE_LINE=0369: print(f"SUPERVISOR_PROBLEMS={','.join(problems)}") -DETAIL=SOURCE_LINE=0370: print(f"MONITOR={facts.get('monitor_status')} reports={facts.get('monitor_reports')} open={facts.get('monitor_open')} resolved={facts.get('monitor_resolved')}") -DETAIL=SOURCE_LINE=0371: print(f"BACKUP={facts.get('backup_status')} run={facts.get('backup_run_id')}") -DETAIL=SOURCE_LINE=0372: print(f"RESTORE={facts.get('restore_status')} run={facts.get('restore_run_id')}") -DETAIL=SOURCE_LINE=0373: log(f"CHECK mode={mode} status={status} mail_sent={'yes' if mail_sent else 'no'} problems={problems}") -DETAIL=SOURCE_LINE=0374: return 0 if status == "OK" else 1 -DETAIL=SOURCE_LINE=0375: -DETAIL=SOURCE_LINE=0376:if __name__ == "__main__": -DETAIL=SOURCE_LINE=0377: sys.exit(main()) -DETAIL=SCRIPT_TOTAL_LINES=377 -DETAIL=RELEVANT_SOURCE_END -DETAIL=UNIT_CONTRACT_BEGIN -DETAIL=UNIT_NAME=skladchik-reports-monitor-supervisor.service -DETAIL=UNIT_LINE=# /etc/systemd/system/skladchik-reports-monitor-supervisor.service -DETAIL=UNIT_LINE=[Unit] -DETAIL=UNIT_LINE=Description=Skladchik reports monitor full supervisor -DETAIL=UNIT_LINE=After=network-online.target -DETAIL=UNIT_LINE= -DETAIL=UNIT_LINE=[Service] -DETAIL=UNIT_LINE=Type=oneshot -DETAIL=UNIT_LINE=ExecStart=/usr/local/sbin/skladchik-reports-monitor-supervisor check -DETAIL=UNIT_NAME=skladchik-reports-monitor-supervisor.timer -DETAIL=UNIT_LINE=# /etc/systemd/system/skladchik-reports-monitor-supervisor.timer -DETAIL=UNIT_LINE=[Unit] -DETAIL=UNIT_LINE=Description=Run Skladchik reports monitor full supervisor every 10 minutes -DETAIL=UNIT_LINE= -DETAIL=UNIT_LINE=[Timer] -DETAIL=UNIT_LINE=OnBootSec=5min -DETAIL=UNIT_LINE=OnUnitActiveSec=10min -DETAIL=UNIT_LINE=AccuracySec=30s -DETAIL=UNIT_LINE=Persistent=false -DETAIL=UNIT_LINE=Unit=skladchik-reports-monitor-supervisor.service -DETAIL=UNIT_LINE= -DETAIL=UNIT_LINE=[Install] -DETAIL=UNIT_LINE=WantedBy=timers.target -DETAIL=UNIT_CONTRACT_END -DETAIL=TIMER_STATE_BEGIN -DETAIL=TIMER_PROPERTY=Unit=skladchik-reports-monitor-supervisor.service -DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime= -DETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 11:58:38 MSK -DETAIL=TIMER_PROPERTY=LoadState=loaded -DETAIL=TIMER_PROPERTY=ActiveState=active -DETAIL=TIMER_PROPERTY=SubState=waiting -DETAIL=TIMER_ROW=Fri 2026-07-24 12:08:38 MSK 3min 34s Fri 2026-07-24 11:58:38 MSK 6min ago skladchik-reports-monitor-supervisor.timer skladchik-reports-monitor-supervisor.service -DETAIL=TIMER_ROW=Sat 2026-07-25 04:39:05 MSK 16h Fri 2026-07-24 04:28:08 MSK 7h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service -DETAIL=TIMER_ROW=Sat 2026-07-25 10:15:00 MSK 22h Fri 2026-07-24 10:15:04 MSK 1h 49min ago skladchik-reports-monitor-monthly-selftest-watch.timer skladchik-reports-monitor-monthly-selftest-watch.service -DETAIL=TIMER_ROW=Sun 2026-07-26 05:15:13 MSK 1 day 17h Sun 2026-07-19 05:11:25 MSK - skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service -DETAIL=TIMER_STATE_END +DETAIL=EXACT_COOKIE_LOGIC_END +DETAIL=PVE01_TIMER_CORRELATION_BEGIN +DETAIL=PVE01_TIMER=Fri 2026-07-24 12:08:38 MSK 1min 48s Fri 2026-07-24 11:58:38 MSK 8min ago skladchik-reports-monitor-supervisor.timer skladchik-reports-monitor-supervisor.service +DETAIL=PVE01_TIMER=Sat 2026-07-25 04:39:05 MSK 16h Fri 2026-07-24 04:28:08 MSK 7h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service +DETAIL=PVE01_TIMER=Sat 2026-07-25 10:15:00 MSK 22h Fri 2026-07-24 10:15:04 MSK 1h 51min ago skladchik-reports-monitor-monthly-selftest-watch.timer skladchik-reports-monitor-monthly-selftest-watch.service +DETAIL=PVE01_TIMER=Sun 2026-07-26 05:15:13 MSK 1 day 17h Sun 2026-07-19 05:11:25 MSK - skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service +DETAIL=PVE01_TIMER_CORRELATION_END +DETAIL=EDGE_TIMER_CORRELATION_BEGIN +DETAIL=EDGE_TIMER=Fri 2026-07-24 12:06:51 MSK 1s Fri 2026-07-24 12:06:21 MSK 28s ago skladchik-reports-monitor.timer skladchik-reports-monitor.service +DETAIL=EDGE_TIMER=Fri 2026-07-24 12:08:35 MSK 1min 45s Fri 2026-07-24 12:03:35 MSK 3min 14s ago skladchik-reports-monitor-watchdog.timer skladchik-reports-monitor-watchdog.service +DETAIL=EDGE_TIMER=Sat 2026-07-25 09:16:28 MSK 21h Fri 2026-07-24 09:18:35 MSK 2h 48min ago skladchik-reports-monitor-heartbeat.timer skladchik-reports-monitor-heartbeat.service +DETAIL=EDGE_TIMER=Sat 2026-07-25 09:45:00 MSK 21h Fri 2026-07-24 09:45:03 MSK 2h 21min ago skladchik-reports-monitor-age-alert.timer skladchik-reports-monitor-age-alert.service +DETAIL=EDGE_TIMER=Mon 2026-07-27 09:35:00 MSK 2 days Mon 2026-07-20 09:35:06 MSK - skladchik-reports-monitor-weekly-summary.timer skladchik-reports-monitor-weekly-summary.service +DETAIL=EDGE_TIMER=Sat 2026-08-01 09:25:00 MSK 1 week 0 days Fri 2026-07-10 10:28:28 MSK - skladchik-reports-monitor-event-selftest.timer skladchik-reports-monitor-event-selftest.service +DETAIL=EDGE_TIMER_CORRELATION_END DETAIL=SECRET_VALUES_OUTPUT=NONE -DETAIL=CHANGES_MADE=NO -DETAIL=COOKIE_CHECK_SOURCE_AUDIT=PASS +DETAIL=COOKIE_CONTENT_OUTPUT=NONE +DETAIL=COOKIE_EXACT_LOGIC_AUDIT=PASS +CHANGES_MADE=NO OUTPUT_END CHAT_OUTPUT_END