runtime: publish HOMELAB-SKLADCHIK-COOKIE-CHECK-SOURCE-AUDIT-418
This commit is contained in:
homelab-runtime-publisher
File diff suppressed because one or more lines are too long
@@ -0,0 +1,430 @@
|
|||||||
|
CHAT_OUTPUT_BEGIN
|
||||||
|
COMMAND_ID=HOMELAB-SKLADCHIK-COOKIE-CHECK-SOURCE-AUDIT-418
|
||||||
|
STATUS=OK
|
||||||
|
RC=0
|
||||||
|
HOST=pve01
|
||||||
|
MODE=read-only
|
||||||
|
COMPONENT=cluster-audit
|
||||||
|
REFERENCE_REGISTER_CHECK=OK
|
||||||
|
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
|
||||||
|
ERROR_REGISTER_CHECK=OK
|
||||||
|
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
|
||||||
|
COMMAND_SHA256=77896618b46e2f97362240deb743fb8e625075381e57377ee0314130be2b2026
|
||||||
|
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||||
|
EXECUTION_STARTED=true
|
||||||
|
CHANGES_MADE=false
|
||||||
|
SANITIZED=yes
|
||||||
|
SECRETS_INCLUDED=no
|
||||||
|
PRIVATE_ADDRESSES_INCLUDED=no
|
||||||
|
RAW_EVIDENCE_SHA256=1c26f0b751c5e55421c706f56f4be25d92c17849df8a385fb66ce01144801022
|
||||||
|
SANITIZED_OUTPUT_SHA256=8e5a54db35050a5c0010241fb0bad91fc5966530dbac76bb07426b09af51a6a1
|
||||||
|
OUTPUT_BEGIN
|
||||||
|
DETAIL=SCRIPT_PATH=/usr/local/sbin/skladchik-reports-monitor-supervisor
|
||||||
|
DETAIL=SCRIPT_SHA256=35c68ae0929ff3795be2bde40de334fc8e26413a3d8a620a137be90d1f2c164e
|
||||||
|
DETAIL=SCRIPT_METADATA=BYTES:20463|MODE:755|OWNER:root:root|MTIME:2026-07-18 13:52:30.415780066 +0300
|
||||||
|
DETAIL=RELEVANT_SOURCE_BEGIN
|
||||||
|
DETAIL=SOURCE_LINE=0005:import os
|
||||||
|
DETAIL=SOURCE_LINE=0006:import re
|
||||||
|
DETAIL=SOURCE_LINE=0007:import shlex
|
||||||
|
DETAIL=SOURCE_LINE=0008:import subprocess
|
||||||
|
DETAIL=SOURCE_LINE=0009:import sys
|
||||||
|
DETAIL=SOURCE_LINE=0010:from pathlib import Path
|
||||||
|
DETAIL=SOURCE_LINE=0011:
|
||||||
|
DETAIL=SOURCE_LINE=0012:EDGE = os.environ.get("SKLADCHIK_EDGE", "debian@[PRIVATE_IP]")
|
||||||
|
DETAIL=SOURCE_LINE=0013:STATE_DIR = Path("/var/lib/skladchik-reports-monitor-supervisor")
|
||||||
|
DETAIL=SOURCE_LINE=0014:LOG_DIR = Path("/var/log/skladchik-reports-monitor-supervisor")
|
||||||
|
DETAIL=SOURCE_LINE=0015:HEALTH = STATE_DIR / "health.txt"
|
||||||
|
DETAIL=SOURCE_LINE=0016:STATE = STATE_DIR / "state.env"
|
||||||
|
DETAIL=SOURCE_LINE=0017:LOG = LOG_DIR / "supervisor.log"
|
||||||
|
DETAIL=SOURCE_LINE=0018:
|
||||||
|
DETAIL=SOURCE_LINE=0019:MONITOR_HEALTH_AGE_MAX = int(os.environ.get("SKLADCHIK_MONITOR_HEALTH_MAX_SECONDS", "600"))
|
||||||
|
DETAIL=SOURCE_LINE=0020:BACKUP_HEALTH_AGE_MAX = int(os.environ.get("SKLADCHIK_BACKUP_HEALTH_MAX_SECONDS", "129600"))
|
||||||
|
DETAIL=SOURCE_LINE=0021:RESTORE_HEALTH_AGE_MAX = int(os.environ.get("SKLADCHIK_RESTORE_HEALTH_MAX_SECONDS", "691200"))
|
||||||
|
DETAIL=SOURCE_LINE=0022:COOKIE_EXPIRY_WARN_SECONDS = int(os.environ.get("SKLADCHIK_COOKIE_EXPIRY_WARN_SECONDS", "604800"))
|
||||||
|
DETAIL=SOURCE_LINE=0023:
|
||||||
|
DETAIL=SOURCE_LINE=0024:PVE_BACKUP_HEALTH = Path("/var/lib/skladchik-reports-monitor-backup/health.txt")
|
||||||
|
DETAIL=SOURCE_LINE=0025:PVE_RESTORE_HEALTH = Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt")
|
||||||
|
DETAIL=SOURCE_LINE=0026:
|
||||||
|
DETAIL=SOURCE_LINE=0027:def now():
|
||||||
|
DETAIL=SOURCE_LINE=0028: return _dt.datetime.now(_dt.UTC).replace(microsecond=0)
|
||||||
|
DETAIL=SOURCE_LINE=0029:
|
||||||
|
DETAIL=SOURCE_LINE=0030:def iso():
|
||||||
|
DETAIL=SOURCE_LINE=0031: return now().isoformat().replace("+00:00", "Z")
|
||||||
|
DETAIL=SOURCE_LINE=0032:
|
||||||
|
DETAIL=SOURCE_LINE=0033:def log(msg):
|
||||||
|
DETAIL=SOURCE_LINE=0034: LOG_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
DETAIL=SOURCE_LINE=0035: with LOG.open("a", encoding="utf-8") as f:
|
||||||
|
DETAIL=SOURCE_LINE=0036: f.write(f"{iso()} {msg}\n")
|
||||||
|
DETAIL=SOURCE_LINE=0037:
|
||||||
|
DETAIL=SOURCE_LINE=0038:def run(cmd, timeout=45, input_text=None):
|
||||||
|
DETAIL=SOURCE_LINE=0039: try:
|
||||||
|
DETAIL=SOURCE_LINE=0040: p = subprocess.run(cmd, shell=True, text=True, input=input_text, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout)
|
||||||
|
DETAIL=SOURCE_LINE=0041: return p.returncode, p.stdout.strip()
|
||||||
|
DETAIL=SOURCE_LINE=0042: except subprocess.TimeoutExpired as e:
|
||||||
|
DETAIL=SOURCE_LINE=0043: out = e.stdout if isinstance(e.stdout, str) else ""
|
||||||
|
DETAIL=SOURCE_LINE=0044: return 124, (out or "").strip() + "\nTIMEOUT"
|
||||||
|
DETAIL=SOURCE_LINE=0045:
|
||||||
|
DETAIL=SOURCE_LINE=0046:def parse_env_text(text):
|
||||||
|
DETAIL=SOURCE_LINE=0047: d = {}
|
||||||
|
DETAIL=SOURCE_LINE=0048: for line in (text or "").splitlines():
|
||||||
|
DETAIL=SOURCE_LINE=0049: if not line or line.startswith("#") or "=" not in line:
|
||||||
|
DETAIL=SOURCE_LINE=0050: continue
|
||||||
|
DETAIL=SOURCE_LINE=0051: k, v = line.split("=", 1)
|
||||||
|
DETAIL=SOURCE_LINE=...
|
||||||
|
DETAIL=SOURCE_LINE=0066: dt = _dt.datetime.fromisoformat(s)
|
||||||
|
DETAIL=SOURCE_LINE=0067: if dt.tzinfo is None:
|
||||||
|
DETAIL=SOURCE_LINE=0068: dt = dt.replace(tzinfo=_dt.UTC)
|
||||||
|
DETAIL=SOURCE_LINE=0069: return int((now() - dt.astimezone(_dt.UTC)).total_seconds())
|
||||||
|
DETAIL=SOURCE_LINE=0070: except Exception:
|
||||||
|
DETAIL=SOURCE_LINE=0071: return 999999999
|
||||||
|
DETAIL=SOURCE_LINE=0072:
|
||||||
|
DETAIL=SOURCE_LINE=0073:def ssh_edge(cmd, timeout=45):
|
||||||
|
DETAIL=SOURCE_LINE=0074: q = "ssh -o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] " + shlex.quote(EDGE) + " " + shlex.quote("sudo bash -lc " + shlex.quote(cmd))
|
||||||
|
DETAIL=SOURCE_LINE=0075: return run(q, timeout=timeout)
|
||||||
|
DETAIL=SOURCE_LINE=0076:
|
||||||
|
DETAIL=SOURCE_LINE=0077:def edge_mail(subject, body):
|
||||||
|
DETAIL=SOURCE_LINE=0078: subj_b64 = base64.b64encode(subject.encode("utf-8")).decode("ascii")
|
||||||
|
DETAIL=SOURCE_LINE=0079: body_b64 = base64.b64encode(body.encode("utf-8")).decode("ascii")
|
||||||
|
DETAIL=SOURCE_LINE=0080: py = (
|
||||||
|
DETAIL=SOURCE_LINE=0081: "import base64,importlib.util;"
|
||||||
|
DETAIL=SOURCE_LINE=0082: "spec=importlib.util.spec_from_file_location('m','/opt/skladchik-reports-monitor/monitor.py');"
|
||||||
|
DETAIL=SOURCE_LINE=0083: "m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m);"
|
||||||
|
DETAIL=SOURCE_LINE=0084: "cfg,secrets=<REDACTED>
|
||||||
|
DETAIL=SOURCE_LINE=0085: f"subject=base64.b64decode('{subj_b64}').decode('utf-8');"
|
||||||
|
DETAIL=SOURCE_LINE=0086: f"body=base64.b64decode('{body_b64}').decode('utf-8');"
|
||||||
|
DETAIL=SOURCE_LINE=0087: "ok=m.send_mail(cfg,secrets,subject,body);"
|
||||||
|
DETAIL=SOURCE_LINE=0088: "print('MAIL_OK' if ok else 'MAIL_FALSE')"
|
||||||
|
DETAIL=SOURCE_LINE=0089: )
|
||||||
|
DETAIL=SOURCE_LINE=0090: rc, out = ssh_edge("python3 -c " + shlex.quote(py), timeout=60)
|
||||||
|
DETAIL=SOURCE_LINE=0091: log(f"MAIL subject={subject!r} rc={rc} out={out!r}")
|
||||||
|
DETAIL=SOURCE_LINE=0092: return rc == 0 and "MAIL_OK" in out
|
||||||
|
DETAIL=SOURCE_LINE=0093:
|
||||||
|
DETAIL=SOURCE_LINE=0094:def systemctl_props(units):
|
||||||
|
DETAIL=SOURCE_LINE=0095: cmd = "systemctl is-enabled " + " ".join(map(shlex.quote, units)) + " 2>/dev/null || true; echo __ACTIVE__; systemctl is-active " + " ".join(map(shlex.quote, units)) + " 2>/dev/null || true"
|
||||||
|
DETAIL=SOURCE_LINE=0096: rc, out = run(cmd, timeout=20)
|
||||||
|
DETAIL=SOURCE_LINE=0097: parts = out.split("__ACTIVE__", 1)
|
||||||
|
DETAIL=SOURCE_LINE=0098: enabled_lines = [x.strip() for x in parts[0].splitlines() if x.strip()]
|
||||||
|
DETAIL=SOURCE_LINE=0099: active_lines = [x.strip() for x in parts[1].splitlines() if x.strip()] if len(parts) > 1 else []
|
||||||
|
DETAIL=SOURCE_LINE=0100: res = {}
|
||||||
|
DETAIL=SOURCE_LINE=0101: for i, u in enumerate(units):
|
||||||
|
DETAIL=SOURCE_LINE=0102: res[u] = {"enabled": enabled_lines[i] if i < len(enabled_lines) else "unknown", "active": active_lines[i] if i < len(active_lines) else "unknown"}
|
||||||
|
DETAIL=SOURCE_LINE=0103: return res
|
||||||
|
DETAIL=SOURCE_LINE=0104:
|
||||||
|
DETAIL=SOURCE_LINE=0105:def gather():
|
||||||
|
DETAIL=SOURCE_LINE=0106: facts = {}
|
||||||
|
DETAIL=SOURCE_LINE=0107: problems = []
|
||||||
|
DETAIL=SOURCE_LINE=0108:
|
||||||
|
DETAIL=SOURCE_LINE=0109: edge_cmd = r'''
|
||||||
|
DETAIL=SOURCE_LINE=0110:set +e
|
||||||
|
DETAIL=SOURCE_LINE=0111:echo __MONITOR_HEALTH_BEGIN__
|
||||||
|
DETAIL=SOURCE_LINE=0112:cat /var/lib/skladchik-reports-monitor/health.txt 2>/dev/null || true
|
||||||
|
DETAIL=SOURCE_LINE=0113:echo __MONITOR_HEALTH_END__
|
||||||
|
DETAIL=SOURCE_LINE=0114:echo __ALERT_HEALTH_BEGIN__
|
||||||
|
DETAIL=SOURCE_LINE=0115:cat /var/lib/skladchik-reports-monitor/alert-health.txt 2>/dev/null || true
|
||||||
|
DETAIL=SOURCE_LINE=0116:echo __ALERT_HEALTH_END__
|
||||||
|
DETAIL=SOURCE_LINE=0117:echo __TIMERS_BEGIN__
|
||||||
|
DETAIL=SOURCE_LINE=0118:for u in skladchik-reports-monitor.timer skladchik-reports-monitor-watchdog.timer skladchik-reports-monitor-heartbeat.timer skladchik-reports-monitor-weekly-summary.timer skladchik-reports-monitor-age-alert.timer skladchik-reports-monitor-event-selftest.timer; do echo "$u enabled=$(systemctl is-enabled "$u" 2>/dev/null) active=$(systemctl is-active "$u" 2>/dev/null)"; done
|
||||||
|
DETAIL=SOURCE_LINE=0119:echo __TIMERS_END__
|
||||||
|
DETAIL=SOURCE_LINE=0120:echo __ROUTE_BEGIN__
|
||||||
|
DETAIL=SOURCE_LINE=0121:ip route get 1.1.1.1 2>&1 | head -n 1
|
||||||
|
DETAIL=SOURCE_LINE=0122:echo __ROUTE_END__
|
||||||
|
DETAIL=SOURCE_LINE=0123:echo __FILES_BEGIN__
|
||||||
|
DETAIL=SOURCE_LINE=0124:test -s /opt/skladchik-reports-monitor/monitor.py && echo monitor_py=yes || echo monitor_py=no
|
||||||
|
DETAIL=SOURCE_LINE=0125:test -s /etc/skladchik-reports-monitor/config.ini && echo config_ini=yes || echo config_ini=no
|
||||||
|
DETAIL=SOURCE_LINE=0126:test -s /etc/skladchik-reports-monitor/secrets.env && echo secrets_env=<REDACTED>
|
||||||
|
DETAIL=SOURCE_LINE=0127:test -s /var/lib/skladchik-reports-monitor/cookies.jar && echo cookies_jar=yes || echo cookies_jar=no
|
||||||
|
DETAIL=SOURCE_LINE=0128:[ -s /var/lib/skladchik-reports-monitor/cookies.jar ] && echo cookies_jar_bytes=$(stat -c %s /var/lib/skladchik-reports-monitor/cookies.jar 2>/dev/null) || echo cookies_jar_bytes=0
|
||||||
|
DETAIL=SOURCE_LINE=0129:test -s /opt/skladchik-reports-monitor/operational_digest.py && echo operational_digest_py=yes || echo operational_digest_py=no
|
||||||
|
DETAIL=SOURCE_LINE=0130:test -s /opt/skladchik-reports-monitor/event-selftest-runner.py && echo event_selftest_runner_py=yes || echo event_selftest_runner_py=no
|
||||||
|
DETAIL=SOURCE_LINE=0131:test -s /opt/skladchik-reports-monitor/event-selftest-core && echo event_selftest_core=yes || echo event_selftest_core=no
|
||||||
|
DETAIL=SOURCE_LINE=0132:test -s /var/lib/skladchik-reports-monitor/weekly-summary-health.txt && echo weekly_summary_health=yes || echo weekly_summary_health=no
|
||||||
|
DETAIL=SOURCE_LINE=0133:test -s /var/lib/skladchik-reports-monitor/age-alert-health.txt && echo age_alert_health=yes || echo age_alert_health=no
|
||||||
|
DETAIL=SOURCE_LINE=0134:test -s /var/lib/skladchik-reports-monitor/event-selftest-health.txt && echo event_selftest_health=yes || echo event_selftest_health=no
|
||||||
|
DETAIL=SOURCE_LINE=0135:python3 -m py_compile /opt/skladchik-reports-monitor/monitor.py /opt/skladchik-reports-monitor/operational_digest.py /opt/skladchik-reports-monitor/event-selftest-runner.py /opt/skladchik-reports-monitor/event-selftest-core >/dev/null 2>&1 && echo py_compile=yes || echo py_compile=no
|
||||||
|
DETAIL=SOURCE_LINE=0136:echo __FILES_END__
|
||||||
|
DETAIL=SOURCE_LINE=0137:'''
|
||||||
|
DETAIL=SOURCE_LINE=0138: rc, out = ssh_edge(edge_cmd, timeout=45)
|
||||||
|
DETAIL=SOURCE_LINE=0139: facts["edge_ssh_rc"] = str(rc)
|
||||||
|
DETAIL=SOURCE_LINE=0140: if rc != 0:
|
||||||
|
DETAIL=SOURCE_LINE=0141: problems.append("edge_ssh_failed")
|
||||||
|
DETAIL=SOURCE_LINE=0142:
|
||||||
|
DETAIL=SOURCE_LINE=0143: def section(name):
|
||||||
|
DETAIL=SOURCE_LINE=0144: m = re.search(rf"__{name}_BEGIN__\n(.*?)\n__{name}_END__", out, re.S)
|
||||||
|
DETAIL=SOURCE_LINE=0145: return m.group(1).strip() if m else ""
|
||||||
|
DETAIL=SOURCE_LINE=0146:
|
||||||
|
DETAIL=SOURCE_LINE=0147: mon = parse_env_text(section("MONITOR_HEALTH"))
|
||||||
|
DETAIL=SOURCE_LINE=0148: facts["monitor_status"] = mon.get("STATUS", "MISSING")
|
||||||
|
DETAIL=SOURCE_LINE=0149: facts["monitor_reports"] = mon.get("reports", "")
|
||||||
|
DETAIL=SOURCE_LINE=0150: facts["monitor_open"] = mon.get("open", "")
|
||||||
|
DETAIL=SOURCE_LINE=0151: facts["monitor_resolved"] = mon.get("resolved", "")
|
||||||
|
DETAIL=SOURCE_LINE=0152: facts["monitor_smtp_configured"] = mon.get("smtp_configured", "")
|
||||||
|
DETAIL=SOURCE_LINE=0153: facts["session_keepalive"] = mon.get("session_keepalive", "")
|
||||||
|
DETAIL=SOURCE_LINE=0154: facts["cookie_jar"] = mon.get("cookie_jar", "")
|
||||||
|
DETAIL=SOURCE_LINE=0155: facts["cookie_jar_bytes"] = mon.get("cookie_jar_bytes", "")
|
||||||
|
DETAIL=SOURCE_LINE=0156: facts["cookie_expiry_status"] = mon.get("cookie_expiry_status", "")
|
||||||
|
DETAIL=SOURCE_LINE=0157: facts["cookie_expiry_utc"] = mon.get("cookie_expiry_utc", "")
|
||||||
|
DETAIL=SOURCE_LINE=0158: facts["cookie_expires_in_seconds"] = mon.get("cookie_expires_in_seconds", "")
|
||||||
|
DETAIL=SOURCE_LINE=0159: facts["cookie_expires_in_days"] = mon.get("cookie_expires_in_days", "")
|
||||||
|
DETAIL=SOURCE_LINE=0160: facts["cookie_persistent_count"] = mon.get("cookie_persistent_count", "")
|
||||||
|
DETAIL=SOURCE_LINE=0161: facts["monitor_health_age_seconds"] = str(age_seconds_from_ts(mon.get("TS")))
|
||||||
|
DETAIL=SOURCE_LINE=0162: if mon.get("STATUS") != "OK":
|
||||||
|
DETAIL=SOURCE_LINE=0163: problems.append("monitor_status_not_ok")
|
||||||
|
DETAIL=SOURCE_LINE=0164: if int(facts["monitor_health_age_seconds"]) > MONITOR_HEALTH_AGE_MAX:
|
||||||
|
DETAIL=SOURCE_LINE=0165: problems.append("monitor_health_stale")
|
||||||
|
DETAIL=SOURCE_LINE=0166: if mon.get("reports") in ("", "0"):
|
||||||
|
DETAIL=SOURCE_LINE=0167: problems.append("monitor_reports_zero_or_missing")
|
||||||
|
DETAIL=SOURCE_LINE=0168: if mon.get("session_keepalive") != "persistent_cookie_jar":
|
||||||
|
DETAIL=SOURCE_LINE=0169: problems.append("session_keepalive_not_persistent")
|
||||||
|
DETAIL=SOURCE_LINE=0170: if mon.get("cookie_jar") not in ("yes", "empty"):
|
||||||
|
DETAIL=SOURCE_LINE=0171: problems.append("monitor_cookie_jar_bad")
|
||||||
|
DETAIL=SOURCE_LINE=0172: expiry_status = mon.get("cookie_expiry_status", "")
|
||||||
|
DETAIL=SOURCE_LINE=0173: try:
|
||||||
|
DETAIL=SOURCE_LINE=0174: expires_in = int(mon.get("cookie_expires_in_seconds", ""))
|
||||||
|
DETAIL=SOURCE_LINE=0175: except Exception:
|
||||||
|
DETAIL=SOURCE_LINE=0176: expires_in = None
|
||||||
|
DETAIL=SOURCE_LINE=0177: if expiry_status in ("", "NO_COOKIE_JAR", "NO_PERSISTENT_COOKIE", "ERROR"):
|
||||||
|
DETAIL=SOURCE_LINE=0178: problems.append("cookie_expiry_missing")
|
||||||
|
DETAIL=SOURCE_LINE=0179: elif expiry_status == "EXPIRED" or (expires_in is not None and expires_in <= 0):
|
||||||
|
DETAIL=SOURCE_LINE=0180: problems.append("cookie_expiry_expired")
|
||||||
|
DETAIL=SOURCE_LINE=0181: elif expiry_status == "WARN" or (expires_in is not None and expires_in < COOKIE_EXPIRY_WARN_SECONDS):
|
||||||
|
DETAIL=SOURCE_LINE=0182: problems.append("cookie_expiry_warn")
|
||||||
|
DETAIL=SOURCE_LINE=0183:
|
||||||
|
DETAIL=SOURCE_LINE=0184: timers_text = section("TIMERS")
|
||||||
|
DETAIL=SOURCE_LINE=0185: for unit in ["skladchik-reports-monitor.timer", "skladchik-reports-monitor-watchdog.timer", "skladchik-reports-monitor-heartbeat.timer", "skladchik-reports-monitor-weekly-summary.timer", "skladchik-reports-monitor-age-alert.timer", "skladchik-reports-monitor-event-selftest.timer"]:
|
||||||
|
DETAIL=SOURCE_LINE=0186: safe = unit.replace(".timer", "").replace("_", "-")
|
||||||
|
DETAIL=SOURCE_LINE=0187: m = re.search(re.escape(unit) + r"\s+enabled=([^\s]+)\s+active=([^\s]+)", timers_text)
|
||||||
|
DETAIL=SOURCE_LINE=0188: enabled = m.group(1) if m else "missing"
|
||||||
|
DETAIL=SOURCE_LINE=0189: active = m.group(2) if m else "missing"
|
||||||
|
DETAIL=SOURCE_LINE=0190: facts[f"edge_{safe}_timer_enabled"] = enabled
|
||||||
|
DETAIL=SOURCE_LINE=0191: facts[f"edge_{safe}_timer_active"] = active
|
||||||
|
DETAIL=SOURCE_LINE=0192: if enabled != "enabled" or active != "active":
|
||||||
|
DETAIL=SOURCE_LINE=0193: problems.append(f"edge_{safe}_timer_bad")
|
||||||
|
DETAIL=SOURCE_LINE=0194:
|
||||||
|
DETAIL=SOURCE_LINE=0195: route_lines = section("ROUTE").splitlines()
|
||||||
|
DETAIL=SOURCE_LINE=0196: route = route_lines[0] if route_lines else ""
|
||||||
|
DETAIL=SOURCE_LINE=0197: facts["netbird_route"] = route
|
||||||
|
DETAIL=SOURCE_LINE=0198: if " wt0 " not in (" " + route + " "):
|
||||||
|
DETAIL=SOURCE_LINE=0199: problems.append("netbird_route_not_wt0")
|
||||||
|
DETAIL=SOURCE_LINE=0200:
|
||||||
|
DETAIL=SOURCE_LINE=0201: files = parse_env_text(section("FILES"))
|
||||||
|
DETAIL=SOURCE_LINE=0202: for k in ["monitor_py", "config_ini", "secrets_env", "cookies_jar", "operational_digest_py", "event_selftest_runner_py", "event_selftest_core", "weekly_summary_health", "age_alert_health", "event_selftest_health", "py_compile"]:
|
||||||
|
DETAIL=SOURCE_LINE=0203: facts[k] = files.get(k, "no")
|
||||||
|
DETAIL=SOURCE_LINE=0204: if facts[k] != "yes":
|
||||||
|
DETAIL=SOURCE_LINE=0205: problems.append(f"edge_{k}_bad")
|
||||||
|
DETAIL=SOURCE_LINE=0206: facts["cookies_jar_bytes_file"] = files.get("cookies_jar_bytes", "0")
|
||||||
|
DETAIL=SOURCE_LINE=0207:
|
||||||
|
DETAIL=SOURCE_LINE=0208: backup = read_env_file(PVE_BACKUP_HEALTH)
|
||||||
|
DETAIL=SOURCE_LINE=0209: facts["backup_status"] = backup.get("STATUS", "MISSING")
|
||||||
|
DETAIL=SOURCE_LINE=0210: facts["backup_run_id"] = backup.get("RUN_ID", "")
|
||||||
|
DETAIL=SOURCE_LINE=0211: facts["backup_size_bytes"] = backup.get("SIZE_BYTES", "")
|
||||||
|
DETAIL=SOURCE_LINE=0212: facts["backup_version"] = backup.get("BACKUP_VERSION", "")
|
||||||
|
DETAIL=SOURCE_LINE=0213: facts["backup_cookie_jar_backed_up"] = backup.get("COOKIE_JAR_BACKED_UP", "")
|
||||||
|
DETAIL=SOURCE_LINE=0214: facts["backup_extension_health_files_backed_up"] = backup.get("EXTENSION_HEALTH_FILES_BACKED_UP", "")
|
||||||
|
DETAIL=SOURCE_LINE=0215: facts["backup_title_in_email_subject"] = backup.get("TITLE_IN_EMAIL_SUBJECT", "")
|
||||||
|
DETAIL=SOURCE_LINE=0216: facts["backup_health_age_seconds"] = str(age_seconds_from_ts(backup.get("TS")))
|
||||||
|
DETAIL=SOURCE_LINE=0217: if backup.get("STATUS") != "OK":
|
||||||
|
DETAIL=SOURCE_LINE=0218: problems.append("backup_status_not_ok")
|
||||||
|
DETAIL=SOURCE_LINE=0219: if backup.get("COOKIE_JAR_BACKED_UP") != "yes":
|
||||||
|
DETAIL=SOURCE_LINE=0220: problems.append("backup_cookie_jar_not_marked")
|
||||||
|
DETAIL=SOURCE_LINE=0221: if backup.get("BACKUP_VERSION") != "4":
|
||||||
|
DETAIL=SOURCE_LINE=0222: problems.append("backup_version_not_4")
|
||||||
|
DETAIL=SOURCE_LINE=0223: if backup.get("EXTENSION_HEALTH_FILES_BACKED_UP") != "yes":
|
||||||
|
DETAIL=SOURCE_LINE=0224: problems.append("backup_extension_health_files_not_marked")
|
||||||
|
DETAIL=SOURCE_LINE=0225: if backup.get("TITLE_IN_EMAIL_SUBJECT") != "yes":
|
||||||
|
DETAIL=SOURCE_LINE=0226: problems.append("email_title_subject_not_marked")
|
||||||
|
DETAIL=SOURCE_LINE=0227: if int(facts["backup_health_age_seconds"]) > BACKUP_HEALTH_AGE_MAX:
|
||||||
|
DETAIL=SOURCE_LINE=0228: problems.append("backup_health_stale")
|
||||||
|
DETAIL=SOURCE_LINE=0229: try:
|
||||||
|
DETAIL=SOURCE_LINE=0230: if int(backup.get("SIZE_BYTES", "0")) <= 0:
|
||||||
|
DETAIL=SOURCE_LINE=0231: problems.append("backup_size_bad")
|
||||||
|
DETAIL=SOURCE_LINE=0232: except Exception:
|
||||||
|
DETAIL=SOURCE_LINE=0233: problems.append("backup_size_bad")
|
||||||
|
DETAIL=SOURCE_LINE=0234:
|
||||||
|
DETAIL=SOURCE_LINE=0235: restore = read_env_file(PVE_RESTORE_HEALTH)
|
||||||
|
DETAIL=SOURCE_LINE=0236: facts["restore_status"] = restore.get("STATUS", "MISSING")
|
||||||
|
DETAIL=SOURCE_LINE=0237: facts["restore_run_id"] = restore.get("RUN_ID", "")
|
||||||
|
DETAIL=SOURCE_LINE=0238: facts["restore_health_age_seconds"] = str(age_seconds_from_ts(restore.get("TS")))
|
||||||
|
DETAIL=SOURCE_LINE=0239: for key in ["CLOUD_1_STATUS", "CLOUD_2_STATUS", "SHA256_OK", "TAR_OK", "REQUIRED_FILES_OK", "COOKIE_JAR_OK", "EXTENSION_HEALTH_FILES_OK", "EXTENSION_SCRIPTS_OK", "EXTENSION_UNITS_OK", "SQLITE_OK", "PY_COMPILE_OK"]:
|
||||||
|
DETAIL=SOURCE_LINE=0240: facts[key.lower()] = restore.get(key, "")
|
||||||
|
DETAIL=SOURCE_LINE=0241: if restore.get("STATUS") != "OK":
|
||||||
|
DETAIL=SOURCE_LINE=0242: problems.append("restore_status_not_ok")
|
||||||
|
DETAIL=SOURCE_LINE=0243: if int(facts["restore_health_age_seconds"]) > RESTORE_HEALTH_AGE_MAX:
|
||||||
|
DETAIL=SOURCE_LINE=0244: problems.append("restore_health_stale")
|
||||||
|
DETAIL=SOURCE_LINE=0245: for key in ["CLOUD_1_STATUS", "CLOUD_2_STATUS"]:
|
||||||
|
DETAIL=SOURCE_LINE=0246: if restore.get(key) != "OK":
|
||||||
|
DETAIL=SOURCE_LINE=0247: problems.append(key.lower() + "_bad")
|
||||||
|
DETAIL=SOURCE_LINE=0248: for key in ["SHA256_OK", "TAR_OK", "REQUIRED_FILES_OK", "COOKIE_JAR_OK", "EXTENSION_HEALTH_FILES_OK", "EXTENSION_SCRIPTS_OK", "EXTENSION_UNITS_OK", "SQLITE_OK", "PY_COMPILE_OK"]:
|
||||||
|
DETAIL=SOURCE_LINE=0249: if restore.get(key) != "yes":
|
||||||
|
DETAIL=SOURCE_LINE=0250: problems.append(key.lower() + "_bad")
|
||||||
|
DETAIL=SOURCE_LINE=0251:
|
||||||
|
DETAIL=SOURCE_LINE=0252: local_units = [
|
||||||
|
DETAIL=SOURCE_LINE=0253: "skladchik-reports-monitor-backup.timer",
|
||||||
|
DETAIL=SOURCE_LINE=0254: "skladchik-reports-monitor-backup-restore-check.timer",
|
||||||
|
DETAIL=SOURCE_LINE=0255: "skladchik-reports-monitor-supervisor.timer",
|
||||||
|
DETAIL=SOURCE_LINE=0256: "skladchik-reports-monitor-monthly-selftest-watch.timer",
|
||||||
|
DETAIL=SOURCE_LINE=0257: ]
|
||||||
|
DETAIL=SOURCE_LINE=0258: props = systemctl_props(local_units)
|
||||||
|
DETAIL=SOURCE_LINE=0259: for u, p in props.items():
|
||||||
|
DETAIL=SOURCE_LINE=0260: base = u.replace(".timer", "").replace("-", "_")
|
||||||
|
DETAIL=SOURCE_LINE=0261: facts[base + "_timer_enabled"] = p["enabled"]
|
||||||
|
DETAIL=SOURCE_LINE=0262: facts[base + "_timer_active"] = p["active"]
|
||||||
|
DETAIL=SOURCE_LINE=0263: if u != "skladchik-reports-monitor-supervisor.timer":
|
||||||
|
DETAIL=SOURCE_LINE=0264: if p["enabled"] != "enabled" or p["active"] != "active":
|
||||||
|
DETAIL=SOURCE_LINE=0265: problems.append(base + "_timer_bad")
|
||||||
|
DETAIL=SOURCE_LINE=0266: else:
|
||||||
|
DETAIL=SOURCE_LINE=0267: # Before first enable it can be disabled/inactive during test. After enable next runs will see enabled/active.
|
||||||
|
DETAIL=SOURCE_LINE=0268: if p["enabled"] not in ("enabled", "disabled") or p["active"] not in ("active", "inactive"):
|
||||||
|
DETAIL=SOURCE_LINE=0269: problems.append(base + "_timer_unknown")
|
||||||
|
DETAIL=SOURCE_LINE=0270:
|
||||||
|
DETAIL=SOURCE_LINE=0271: return facts, problems
|
||||||
|
DETAIL=SOURCE_LINE=0272:
|
||||||
|
DETAIL=SOURCE_LINE=0273:def render_body(status, facts, problems, mode):
|
||||||
|
DETAIL=SOURCE_LINE=0274: ru_status = "OK" if status == "OK" else "АВАРИЯ"
|
||||||
|
DETAIL=SOURCE_LINE=0275: lines = [
|
||||||
|
DETAIL=SOURCE_LINE=0276: f"Статус надзора: {ru_status}",
|
||||||
|
DETAIL=SOURCE_LINE=0277: f"Время UTC: {iso()}",
|
||||||
|
DETAIL=SOURCE_LINE=0278: "",
|
||||||
|
DETAIL=SOURCE_LINE=0279: "Монитор жалоб:",
|
||||||
|
DETAIL=SOURCE_LINE=0280: f"- статус: {facts.get('monitor_status')}",
|
||||||
|
DETAIL=SOURCE_LINE=0281: f"- всего жалоб на странице: {facts.get('monitor_reports')}",
|
||||||
|
DETAIL=SOURCE_LINE=0282: f"- открытые/назначенные: {facts.get('monitor_open')}",
|
||||||
|
DETAIL=SOURCE_LINE=0283: f"- закрытые: {facts.get('monitor_resolved')}",
|
||||||
|
DETAIL=SOURCE_LINE=0284: f"- возраст health, сек: {facts.get('monitor_health_age_seconds')}",
|
||||||
|
DETAIL=SOURCE_LINE=0285: f"- автопродление авторизации: {facts.get('session_keepalive')}",
|
||||||
|
DETAIL=SOURCE_LINE=0286: f"- cookie jar: {facts.get('cookie_jar')} / {facts.get('cookie_jar_bytes')} байт",
|
||||||
|
DETAIL=SOURCE_LINE=0287: f"- постоянная cookie действует до UTC: {facts.get('cookie_expiry_utc') or 'не определено'}",
|
||||||
|
DETAIL=SOURCE_LINE=0288: f"- осталось дней по cookie: <REDACTED>
|
||||||
|
DETAIL=SOURCE_LINE=0289: f"- статус срока cookie: <REDACTED>
|
||||||
|
DETAIL=SOURCE_LINE=0290: f"- NetBird route: {facts.get('netbird_route')}",
|
||||||
|
DETAIL=SOURCE_LINE=0291: "",
|
||||||
|
DETAIL=SOURCE_LINE=0292: "Бэкап:",
|
||||||
|
DETAIL=SOURCE_LINE=0293: f"- статус: {facts.get('backup_status')}",
|
||||||
|
DETAIL=SOURCE_LINE=0294: f"- последний RUN_ID: {facts.get('backup_run_id')}",
|
||||||
|
DETAIL=SOURCE_LINE=0295: f"- размер архива, байт: {facts.get('backup_size_bytes')}",
|
||||||
|
DETAIL=SOURCE_LINE=0296: f"- cookie jar в бэкапе: {facts.get('backup_cookie_jar_backed_up')}",
|
||||||
|
DETAIL=SOURCE_LINE=0297: f"- health-файлы расширений в бэкапе: {facts.get('backup_extension_health_files_backed_up')}",
|
||||||
|
DETAIL=SOURCE_LINE=0298: f"- название жалобы в теме письма: {facts.get('backup_title_in_email_subject')}",
|
||||||
|
DETAIL=SOURCE_LINE=0299: f"- возраст health, сек: {facts.get('backup_health_age_seconds')}",
|
||||||
|
DETAIL=SOURCE_LINE=0300: "",
|
||||||
|
DETAIL=SOURCE_LINE=0301: "Restore-check:",
|
||||||
|
DETAIL=SOURCE_LINE=0302: f"- статус: {facts.get('restore_status')}",
|
||||||
|
DETAIL=SOURCE_LINE=0303: f"- последний RUN_ID: {facts.get('restore_run_id')}",
|
||||||
|
DETAIL=SOURCE_LINE=0304: f"- cloud1/cloud2: {facts.get('cloud_1_status')}/{facts.get('cloud_2_status')}",
|
||||||
|
DETAIL=SOURCE_LINE=0305: f"- sha256/tar/cookie/sqlite/python: {facts.get('sha256_ok')}/{facts.get('tar_ok')}/{facts.get('cookie_jar_ok')}/{facts.get('sqlite_ok')}/{facts.get('py_compile_ok')}",
|
||||||
|
DETAIL=SOURCE_LINE=0306: f"- health/scripts/units расширений: {facts.get('extension_health_files_ok')}/{facts.get('extension_scripts_ok')}/{facts.get('extension_units_ok')}",
|
||||||
|
DETAIL=SOURCE_LINE=0307: "",
|
||||||
|
DETAIL=SOURCE_LINE=0308: "Проблемы:",
|
||||||
|
DETAIL=SOURCE_LINE=0309: ]
|
||||||
|
DETAIL=SOURCE_LINE=0310: if problems:
|
||||||
|
DETAIL=SOURCE_LINE=0311: lines.extend([f"- {p}" for p in problems])
|
||||||
|
DETAIL=SOURCE_LINE=0312: else:
|
||||||
|
DETAIL=SOURCE_LINE=0313: lines.append("- нет")
|
||||||
|
DETAIL=SOURCE_LINE=0314: lines.extend([
|
||||||
|
DETAIL=SOURCE_LINE=0315: "",
|
||||||
|
DETAIL=SOURCE_LINE=0316: "Авторизация:",
|
||||||
|
DETAIL=SOURCE_LINE=0317: "- ручной ежемесячный перелогин обычно не нужен; монитор сам продлевает сессию постоянным cookie jar при каждом опросе;",
|
||||||
|
DETAIL=SOURCE_LINE=0318: "- supervisor теперь отдельно следит за сроком постоянной cookie и заранее предупредит, если до истечения останется меньше 7 дней;",
|
||||||
|
DETAIL=SOURCE_LINE=0319: "- если форум принудительно сбросит сессию, придёт отдельная тревога AUTH_REQUIRED; тогда нужно будет разово обновить вход, но это аварийный сценарий, а не ежемесячная рутина.",
|
||||||
|
DETAIL=SOURCE_LINE=0320: ])
|
||||||
|
DETAIL=SOURCE_LINE=0321: return "\n".join(lines)
|
||||||
|
DETAIL=SOURCE_LINE=0322:
|
||||||
|
DETAIL=SOURCE_LINE=0323:def write_health(status, facts, problems, mail_sent):
|
||||||
|
DETAIL=SOURCE_LINE=0324: STATE_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
DETAIL=SOURCE_LINE=0325: lines = [f"STATUS={status}", f"TS={iso()}", "problems=" + ",".join(problems), f"mail_sent={'yes' if mail_sent else 'no'}"]
|
||||||
|
DETAIL=SOURCE_LINE=0326: for k in sorted(facts):
|
||||||
|
DETAIL=SOURCE_LINE=0327: v = str(facts[k]).replace("\n", " ")[:500]
|
||||||
|
DETAIL=SOURCE_LINE=0328: lines.append(f"{k}={v}")
|
||||||
|
DETAIL=SOURCE_LINE=0329: HEALTH.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||||
|
DETAIL=SOURCE_LINE=0330:
|
||||||
|
DETAIL=SOURCE_LINE=0331:def read_state():
|
||||||
|
DETAIL=SOURCE_LINE=0332: return read_env_file(STATE)
|
||||||
|
DETAIL=SOURCE_LINE=0333:
|
||||||
|
DETAIL=SOURCE_LINE=0334:def write_state(status, problems):
|
||||||
|
DETAIL=SOURCE_LINE=0335: STATE_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
DETAIL=SOURCE_LINE=0336: STATE.write_text(f"LAST_STATUS={status}\nLAST_PROBLEMS={','.join(problems)}\nTS={iso()}\n", encoding="utf-8")
|
||||||
|
DETAIL=SOURCE_LINE=0337:
|
||||||
|
DETAIL=SOURCE_LINE=0338:def main():
|
||||||
|
DETAIL=SOURCE_LINE=0339: mode = sys.argv[1] if len(sys.argv) > 1 else "check"
|
||||||
|
DETAIL=SOURCE_LINE=0340: if mode not in ("check", "test"):
|
||||||
|
DETAIL=SOURCE_LINE=0341: print("usage: skladchik-reports-monitor-supervisor [check|test]", file=sys.stderr)
|
||||||
|
DETAIL=SOURCE_LINE=0342: return 2
|
||||||
|
DETAIL=SOURCE_LINE=0343:
|
||||||
|
DETAIL=SOURCE_LINE=0344: facts, problems = gather()
|
||||||
|
DETAIL=SOURCE_LINE=0345: status = "OK" if not problems else "FAIL"
|
||||||
|
DETAIL=SOURCE_LINE=0346: old = read_state()
|
||||||
|
DETAIL=SOURCE_LINE=0347: old_status = old.get("LAST_STATUS", "")
|
||||||
|
DETAIL=SOURCE_LINE=0348: mail_sent = False
|
||||||
|
DETAIL=SOURCE_LINE=0349:
|
||||||
|
DETAIL=SOURCE_LINE=0350: if mode == "test":
|
||||||
|
DETAIL=SOURCE_LINE=0351: mail_sent = edge_mail("Тест: надзор монитора жалоб и бэкапов", render_body(status, facts, problems, mode))
|
||||||
|
DETAIL=SOURCE_LINE=0352: if not mail_sent:
|
||||||
|
DETAIL=SOURCE_LINE=0353: problems.append("supervisor_mail_send_failed")
|
||||||
|
DETAIL=SOURCE_LINE=0354: status = "FAIL"
|
||||||
|
DETAIL=SOURCE_LINE=0355: elif status != "OK":
|
||||||
|
DETAIL=SOURCE_LINE=0356: if old_status != "FAIL":
|
||||||
|
DETAIL=SOURCE_LINE=0357: mail_sent = edge_mail("Авария: монитор жалоб или его бэкапы", render_body(status, facts, problems, mode))
|
||||||
|
DETAIL=SOURCE_LINE=0358: if not mail_sent:
|
||||||
|
DETAIL=SOURCE_LINE=0359: problems.append("supervisor_mail_send_failed")
|
||||||
|
DETAIL=SOURCE_LINE=0360: status = "FAIL"
|
||||||
|
DETAIL=SOURCE_LINE=0361: elif old_status == "FAIL":
|
||||||
|
DETAIL=SOURCE_LINE=0362: mail_sent = edge_mail("Восстановлено: монитор жалоб и бэкапы снова OK", render_body(status, facts, problems, mode))
|
||||||
|
DETAIL=SOURCE_LINE=0363:
|
||||||
|
DETAIL=SOURCE_LINE=0364: write_health(status, facts, problems, mail_sent)
|
||||||
|
DETAIL=SOURCE_LINE=0365: write_state(status, problems)
|
||||||
|
DETAIL=SOURCE_LINE=0366:
|
||||||
|
DETAIL=SOURCE_LINE=0367: print(f"SUPERVISOR_STATUS={status}")
|
||||||
|
DETAIL=SOURCE_LINE=0368: print(f"SUPERVISOR_MAIL_SENT={'yes' if mail_sent else 'no'}")
|
||||||
|
DETAIL=SOURCE_LINE=0369: print(f"SUPERVISOR_PROBLEMS={','.join(problems)}")
|
||||||
|
DETAIL=SOURCE_LINE=0370: print(f"MONITOR={facts.get('monitor_status')} reports={facts.get('monitor_reports')} open={facts.get('monitor_open')} resolved={facts.get('monitor_resolved')}")
|
||||||
|
DETAIL=SOURCE_LINE=0371: print(f"BACKUP={facts.get('backup_status')} run={facts.get('backup_run_id')}")
|
||||||
|
DETAIL=SOURCE_LINE=0372: print(f"RESTORE={facts.get('restore_status')} run={facts.get('restore_run_id')}")
|
||||||
|
DETAIL=SOURCE_LINE=0373: log(f"CHECK mode={mode} status={status} mail_sent={'yes' if mail_sent else 'no'} problems={problems}")
|
||||||
|
DETAIL=SOURCE_LINE=0374: return 0 if status == "OK" else 1
|
||||||
|
DETAIL=SOURCE_LINE=0375:
|
||||||
|
DETAIL=SOURCE_LINE=0376:if __name__ == "__main__":
|
||||||
|
DETAIL=SOURCE_LINE=0377: sys.exit(main())
|
||||||
|
DETAIL=SCRIPT_TOTAL_LINES=377
|
||||||
|
DETAIL=RELEVANT_SOURCE_END
|
||||||
|
DETAIL=UNIT_CONTRACT_BEGIN
|
||||||
|
DETAIL=UNIT_NAME=skladchik-reports-monitor-supervisor.service
|
||||||
|
DETAIL=UNIT_LINE=# /etc/systemd/system/skladchik-reports-monitor-supervisor.service
|
||||||
|
DETAIL=UNIT_LINE=[Unit]
|
||||||
|
DETAIL=UNIT_LINE=Description=Skladchik reports monitor full supervisor
|
||||||
|
DETAIL=UNIT_LINE=After=network-online.target
|
||||||
|
DETAIL=UNIT_LINE=
|
||||||
|
DETAIL=UNIT_LINE=[Service]
|
||||||
|
DETAIL=UNIT_LINE=Type=oneshot
|
||||||
|
DETAIL=UNIT_LINE=ExecStart=/usr/local/sbin/skladchik-reports-monitor-supervisor check
|
||||||
|
DETAIL=UNIT_NAME=skladchik-reports-monitor-supervisor.timer
|
||||||
|
DETAIL=UNIT_LINE=# /etc/systemd/system/skladchik-reports-monitor-supervisor.timer
|
||||||
|
DETAIL=UNIT_LINE=[Unit]
|
||||||
|
DETAIL=UNIT_LINE=Description=Run Skladchik reports monitor full supervisor every 10 minutes
|
||||||
|
DETAIL=UNIT_LINE=
|
||||||
|
DETAIL=UNIT_LINE=[Timer]
|
||||||
|
DETAIL=UNIT_LINE=OnBootSec=5min
|
||||||
|
DETAIL=UNIT_LINE=OnUnitActiveSec=10min
|
||||||
|
DETAIL=UNIT_LINE=AccuracySec=30s
|
||||||
|
DETAIL=UNIT_LINE=Persistent=false
|
||||||
|
DETAIL=UNIT_LINE=Unit=skladchik-reports-monitor-supervisor.service
|
||||||
|
DETAIL=UNIT_LINE=
|
||||||
|
DETAIL=UNIT_LINE=[Install]
|
||||||
|
DETAIL=UNIT_LINE=WantedBy=timers.target
|
||||||
|
DETAIL=UNIT_CONTRACT_END
|
||||||
|
DETAIL=TIMER_STATE_BEGIN
|
||||||
|
DETAIL=TIMER_PROPERTY=Unit=skladchik-reports-monitor-supervisor.service
|
||||||
|
DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=
|
||||||
|
DETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 11:58:38 MSK
|
||||||
|
DETAIL=TIMER_PROPERTY=LoadState=loaded
|
||||||
|
DETAIL=TIMER_PROPERTY=ActiveState=active
|
||||||
|
DETAIL=TIMER_PROPERTY=SubState=waiting
|
||||||
|
DETAIL=TIMER_ROW=Fri 2026-07-24 12:08:38 MSK 3min 34s Fri 2026-07-24 11:58:38 MSK 6min ago skladchik-reports-monitor-supervisor.timer skladchik-reports-monitor-supervisor.service
|
||||||
|
DETAIL=TIMER_ROW=Sat 2026-07-25 04:39:05 MSK 16h Fri 2026-07-24 04:28:08 MSK 7h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service
|
||||||
|
DETAIL=TIMER_ROW=Sat 2026-07-25 10:15:00 MSK 22h Fri 2026-07-24 10:15:04 MSK 1h 49min ago skladchik-reports-monitor-monthly-selftest-watch.timer skladchik-reports-monitor-monthly-selftest-watch.service
|
||||||
|
DETAIL=TIMER_ROW=Sun 2026-07-26 05:15:13 MSK 1 day 17h Sun 2026-07-19 05:11:25 MSK - skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service
|
||||||
|
DETAIL=TIMER_STATE_END
|
||||||
|
DETAIL=SECRET_VALUES_OUTPUT=NONE
|
||||||
|
DETAIL=CHANGES_MADE=NO
|
||||||
|
DETAIL=COOKIE_CHECK_SOURCE_AUDIT=PASS
|
||||||
|
|
||||||
|
OUTPUT_END
|
||||||
|
CHAT_OUTPUT_END
|
||||||
+7
-7
File diff suppressed because one or more lines are too long
+409
-100
@@ -1,5 +1,5 @@
|
|||||||
CHAT_OUTPUT_BEGIN
|
CHAT_OUTPUT_BEGIN
|
||||||
COMMAND_ID=HOMELAB-SKLADCHIK-SUPERVISOR-FAILURE-TRACE-417
|
COMMAND_ID=HOMELAB-SKLADCHIK-COOKIE-CHECK-SOURCE-AUDIT-418
|
||||||
STATUS=OK
|
STATUS=OK
|
||||||
RC=0
|
RC=0
|
||||||
HOST=pve01
|
HOST=pve01
|
||||||
@@ -9,113 +9,422 @@ REFERENCE_REGISTER_CHECK=OK
|
|||||||
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
|
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
|
||||||
ERROR_REGISTER_CHECK=OK
|
ERROR_REGISTER_CHECK=OK
|
||||||
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
|
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
|
||||||
COMMAND_SHA256=9f6e05b9df78cf255d0b700cf196693048a5d3f95183c3f4aa1fd2e48188ab45
|
COMMAND_SHA256=77896618b46e2f97362240deb743fb8e625075381e57377ee0314130be2b2026
|
||||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||||
EXECUTION_STARTED=true
|
EXECUTION_STARTED=true
|
||||||
CHANGES_MADE=false
|
CHANGES_MADE=false
|
||||||
SANITIZED=yes
|
SANITIZED=yes
|
||||||
SECRETS_INCLUDED=no
|
SECRETS_INCLUDED=no
|
||||||
PRIVATE_ADDRESSES_INCLUDED=no
|
PRIVATE_ADDRESSES_INCLUDED=no
|
||||||
RAW_EVIDENCE_SHA256=4a1b1b699e9b8b13e867485522410253fe5bf1a821101e64dcf14e5186201798
|
RAW_EVIDENCE_SHA256=1c26f0b751c5e55421c706f56f4be25d92c17849df8a385fb66ce01144801022
|
||||||
SANITIZED_OUTPUT_SHA256=4a1b1b699e9b8b13e867485522410253fe5bf1a821101e64dcf14e5186201798
|
SANITIZED_OUTPUT_SHA256=8e5a54db35050a5c0010241fb0bad91fc5966530dbac76bb07426b09af51a6a1
|
||||||
OUTPUT_BEGIN
|
OUTPUT_BEGIN
|
||||||
DETAIL=SUPERVISOR_STATE_BEGIN
|
DETAIL=SCRIPT_PATH=/usr/local/sbin/skladchik-reports-monitor-supervisor
|
||||||
DETAIL=SYSTEMD_PROPERTY=Result=exit-code
|
DETAIL=SCRIPT_SHA256=35c68ae0929ff3795be2bde40de334fc8e26413a3d8a620a137be90d1f2c164e
|
||||||
DETAIL=SYSTEMD_PROPERTY=ExecMainCode=1
|
DETAIL=SCRIPT_METADATA=BYTES:20463|MODE:755|OWNER:root:root|MTIME:2026-07-18 13:52:30.415780066 +0300
|
||||||
DETAIL=SYSTEMD_PROPERTY=ExecMainStatus=1
|
DETAIL=RELEVANT_SOURCE_BEGIN
|
||||||
DETAIL=SYSTEMD_PROPERTY=WorkingDirectory=
|
DETAIL=SOURCE_LINE=0005:import os
|
||||||
DETAIL=SYSTEMD_PROPERTY=User=
|
DETAIL=SOURCE_LINE=0006:import re
|
||||||
DETAIL=SYSTEMD_PROPERTY=Group=
|
DETAIL=SOURCE_LINE=0007:import shlex
|
||||||
DETAIL=SYSTEMD_PROPERTY=LoadState=loaded
|
DETAIL=SOURCE_LINE=0008:import subprocess
|
||||||
DETAIL=SYSTEMD_PROPERTY=ActiveState=failed
|
DETAIL=SOURCE_LINE=0009:import sys
|
||||||
DETAIL=SYSTEMD_PROPERTY=SubState=failed
|
DETAIL=SOURCE_LINE=0010:from pathlib import Path
|
||||||
DETAIL=SYSTEMD_PROPERTY=FragmentPath=/etc/systemd/system/skladchik-reports-monitor-supervisor.service
|
DETAIL=SOURCE_LINE=0011:
|
||||||
DETAIL=SYSTEMD_PROPERTY=DropInPaths=
|
DETAIL=SOURCE_LINE=0012:EDGE = os.environ.get("SKLADCHIK_EDGE", "debian@[PRIVATE_IP]")
|
||||||
DETAIL=SUPERVISOR_STATE_END
|
DETAIL=SOURCE_LINE=0013:STATE_DIR = Path("/var/lib/skladchik-reports-monitor-supervisor")
|
||||||
DETAIL=SUPERVISOR_UNIT_CONTRACT_BEGIN
|
DETAIL=SOURCE_LINE=0014:LOG_DIR = Path("/var/log/skladchik-reports-monitor-supervisor")
|
||||||
DETAIL=UNIT_CONTRACT=# /etc/systemd/system/skladchik-reports-monitor-supervisor.service
|
DETAIL=SOURCE_LINE=0015:HEALTH = STATE_DIR / "health.txt"
|
||||||
DETAIL=UNIT_CONTRACT=[Unit]
|
DETAIL=SOURCE_LINE=0016:STATE = STATE_DIR / "state.env"
|
||||||
DETAIL=UNIT_CONTRACT=After=network-online.target
|
DETAIL=SOURCE_LINE=0017:LOG = LOG_DIR / "supervisor.log"
|
||||||
DETAIL=UNIT_CONTRACT=[Service]
|
DETAIL=SOURCE_LINE=0018:
|
||||||
DETAIL=UNIT_CONTRACT=ExecStart=/usr/local/sbin/skladchik-reports-monitor-supervisor check
|
DETAIL=SOURCE_LINE=0019:MONITOR_HEALTH_AGE_MAX = int(os.environ.get("SKLADCHIK_MONITOR_HEALTH_MAX_SECONDS", "600"))
|
||||||
DETAIL=SUPERVISOR_UNIT_CONTRACT_END
|
DETAIL=SOURCE_LINE=0020:BACKUP_HEALTH_AGE_MAX = int(os.environ.get("SKLADCHIK_BACKUP_HEALTH_MAX_SECONDS", "129600"))
|
||||||
DETAIL=SUPERVISOR_LAST_FAILURE_BEGIN
|
DETAIL=SOURCE_LINE=0021:RESTORE_HEALTH_AGE_MAX = int(os.environ.get("SKLADCHIK_RESTORE_HEALTH_MAX_SECONDS", "691200"))
|
||||||
DETAIL=JOURNAL=2026-07-24T06:38:28+03:00 pve01 skladchik-reports-monitor-supervisor[2914895]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0022:COOKIE_EXPIRY_WARN_SECONDS = int(os.environ.get("SKLADCHIK_COOKIE_EXPIRY_WARN_SECONDS", "604800"))
|
||||||
DETAIL=JOURNAL=2026-07-24T06:38:28+03:00 pve01 skladchik-reports-monitor-supervisor[2914895]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0023:
|
||||||
DETAIL=JOURNAL=2026-07-24T06:48:29+03:00 pve01 skladchik-reports-monitor-supervisor[2943586]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0024:PVE_BACKUP_HEALTH = Path("/var/lib/skladchik-reports-monitor-backup/health.txt")
|
||||||
DETAIL=JOURNAL=2026-07-24T06:48:29+03:00 pve01 skladchik-reports-monitor-supervisor[2943586]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0025:PVE_RESTORE_HEALTH = Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt")
|
||||||
DETAIL=JOURNAL=2026-07-24T06:58:29+03:00 pve01 skladchik-reports-monitor-supervisor[2971845]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0026:
|
||||||
DETAIL=JOURNAL=2026-07-24T06:58:29+03:00 pve01 skladchik-reports-monitor-supervisor[2971845]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0027:def now():
|
||||||
DETAIL=JOURNAL=2026-07-24T07:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3008729]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0028: return _dt.datetime.now(_dt.UTC).replace(microsecond=0)
|
||||||
DETAIL=JOURNAL=2026-07-24T07:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3008729]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0029:
|
||||||
DETAIL=JOURNAL=2026-07-24T07:18:28+03:00 pve01 skladchik-reports-monitor-supervisor[3048052]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0030:def iso():
|
||||||
DETAIL=JOURNAL=2026-07-24T07:18:28+03:00 pve01 skladchik-reports-monitor-supervisor[3048052]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0031: return now().isoformat().replace("+00:00", "Z")
|
||||||
DETAIL=JOURNAL=2026-07-24T07:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3075804]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0032:
|
||||||
DETAIL=JOURNAL=2026-07-24T07:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3075804]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0033:def log(msg):
|
||||||
DETAIL=JOURNAL=2026-07-24T07:38:28+03:00 pve01 skladchik-reports-monitor-supervisor[3111924]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0034: LOG_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
DETAIL=JOURNAL=2026-07-24T07:38:28+03:00 pve01 skladchik-reports-monitor-supervisor[3111924]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0035: with LOG.open("a", encoding="utf-8") as f:
|
||||||
DETAIL=JOURNAL=2026-07-24T07:48:29+03:00 pve01 skladchik-reports-monitor-supervisor[3148073]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0036: f.write(f"{iso()} {msg}\n")
|
||||||
DETAIL=JOURNAL=2026-07-24T07:48:29+03:00 pve01 skladchik-reports-monitor-supervisor[3148073]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0037:
|
||||||
DETAIL=JOURNAL=2026-07-24T07:58:28+03:00 pve01 skladchik-reports-monitor-supervisor[3175426]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0038:def run(cmd, timeout=45, input_text=None):
|
||||||
DETAIL=JOURNAL=2026-07-24T07:58:28+03:00 pve01 skladchik-reports-monitor-supervisor[3175426]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0039: try:
|
||||||
DETAIL=JOURNAL=2026-07-24T08:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3205006]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0040: p = subprocess.run(cmd, shell=True, text=True, input=input_text, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout)
|
||||||
DETAIL=JOURNAL=2026-07-24T08:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3205006]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0041: return p.returncode, p.stdout.strip()
|
||||||
DETAIL=JOURNAL=2026-07-24T08:18:29+03:00 pve01 skladchik-reports-monitor-supervisor[3241119]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0042: except subprocess.TimeoutExpired as e:
|
||||||
DETAIL=JOURNAL=2026-07-24T08:18:29+03:00 pve01 skladchik-reports-monitor-supervisor[3241119]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0043: out = e.stdout if isinstance(e.stdout, str) else ""
|
||||||
DETAIL=JOURNAL=2026-07-24T08:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3269919]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0044: return 124, (out or "").strip() + "\nTIMEOUT"
|
||||||
DETAIL=JOURNAL=2026-07-24T08:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3269919]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0045:
|
||||||
DETAIL=JOURNAL=2026-07-24T08:38:29+03:00 pve01 skladchik-reports-monitor-supervisor[3306255]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0046:def parse_env_text(text):
|
||||||
DETAIL=JOURNAL=2026-07-24T08:38:29+03:00 pve01 skladchik-reports-monitor-supervisor[3306255]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0047: d = {}
|
||||||
DETAIL=JOURNAL=2026-07-24T08:48:29+03:00 pve01 skladchik-reports-monitor-supervisor[3342134]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0048: for line in (text or "").splitlines():
|
||||||
DETAIL=JOURNAL=2026-07-24T08:48:29+03:00 pve01 skladchik-reports-monitor-supervisor[3342134]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0049: if not line or line.startswith("#") or "=" not in line:
|
||||||
DETAIL=JOURNAL=2026-07-24T08:58:29+03:00 pve01 skladchik-reports-monitor-supervisor[3369587]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0050: continue
|
||||||
DETAIL=JOURNAL=2026-07-24T08:58:29+03:00 pve01 skladchik-reports-monitor-supervisor[3369587]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0051: k, v = line.split("=", 1)
|
||||||
DETAIL=JOURNAL=2026-07-24T09:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3405539]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=...
|
||||||
DETAIL=JOURNAL=2026-07-24T09:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3405539]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0066: dt = _dt.datetime.fromisoformat(s)
|
||||||
DETAIL=JOURNAL=2026-07-24T09:18:29+03:00 pve01 skladchik-reports-monitor-supervisor[3430044]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0067: if dt.tzinfo is None:
|
||||||
DETAIL=JOURNAL=2026-07-24T09:18:29+03:00 pve01 skladchik-reports-monitor-supervisor[3430044]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0068: dt = dt.replace(tzinfo=_dt.UTC)
|
||||||
DETAIL=JOURNAL=2026-07-24T09:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3457493]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0069: return int((now() - dt.astimezone(_dt.UTC)).total_seconds())
|
||||||
DETAIL=JOURNAL=2026-07-24T09:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3457493]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0070: except Exception:
|
||||||
DETAIL=JOURNAL=2026-07-24T09:38:29+03:00 pve01 skladchik-reports-monitor-supervisor[3493130]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0071: return 999999999
|
||||||
DETAIL=JOURNAL=2026-07-24T09:38:29+03:00 pve01 skladchik-reports-monitor-supervisor[3493130]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0072:
|
||||||
DETAIL=JOURNAL=2026-07-24T09:48:29+03:00 pve01 skladchik-reports-monitor-supervisor[3528983]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0073:def ssh_edge(cmd, timeout=45):
|
||||||
DETAIL=JOURNAL=2026-07-24T09:48:29+03:00 pve01 skladchik-reports-monitor-supervisor[3528983]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0074: q = "ssh -o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] " + shlex.quote(EDGE) + " " + shlex.quote("sudo bash -lc " + shlex.quote(cmd))
|
||||||
DETAIL=JOURNAL=2026-07-24T09:58:29+03:00 pve01 skladchik-reports-monitor-supervisor[3556369]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0075: return run(q, timeout=timeout)
|
||||||
DETAIL=JOURNAL=2026-07-24T09:58:29+03:00 pve01 skladchik-reports-monitor-supervisor[3556369]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0076:
|
||||||
DETAIL=JOURNAL=2026-07-24T10:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3594595]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0077:def edge_mail(subject, body):
|
||||||
DETAIL=JOURNAL=2026-07-24T10:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3594595]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0078: subj_b64 = base64.b64encode(subject.encode("utf-8")).decode("ascii")
|
||||||
DETAIL=JOURNAL=2026-07-24T10:18:29+03:00 pve01 skladchik-reports-monitor-supervisor[3633670]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0079: body_b64 = base64.b64encode(body.encode("utf-8")).decode("ascii")
|
||||||
DETAIL=JOURNAL=2026-07-24T10:18:29+03:00 pve01 skladchik-reports-monitor-supervisor[3633670]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0080: py = (
|
||||||
DETAIL=JOURNAL=2026-07-24T10:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3652898]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0081: "import base64,importlib.util;"
|
||||||
DETAIL=JOURNAL=2026-07-24T10:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3652898]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0082: "spec=importlib.util.spec_from_file_location('m','/opt/skladchik-reports-monitor/monitor.py');"
|
||||||
DETAIL=JOURNAL=2026-07-24T10:38:31+03:00 pve01 skladchik-reports-monitor-supervisor[3691967]: SUPERVISOR_STATUS=FAIL
|
DETAIL=SOURCE_LINE=0083: "m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m);"
|
||||||
DETAIL=JOURNAL=2026-07-24T10:38:31+03:00 pve01 skladchik-reports-monitor-supervisor[3691967]: SUPERVISOR_PROBLEMS=edge_cookies_jar_bad
|
DETAIL=SOURCE_LINE=0084: "cfg,secrets=<REDACTED>
|
||||||
DETAIL=JOURNAL=2026-07-24T10:38:31+03:00 pve01 systemd[1]: skladchik-reports-monitor-supervisor.service: Main process exited, code=exited, status=1/FAILURE
|
DETAIL=SOURCE_LINE=0085: f"subject=base64.b64decode('{subj_b64}').decode('utf-8');"
|
||||||
DETAIL=JOURNAL=2026-07-24T10:38:31+03:00 pve01 systemd[1]: skladchik-reports-monitor-supervisor.service: Failed with result 'exit-code'.
|
DETAIL=SOURCE_LINE=0086: f"body=base64.b64decode('{body_b64}').decode('utf-8');"
|
||||||
DETAIL=JOURNAL=2026-07-24T10:38:31+03:00 pve01 systemd[1]: Failed to start skladchik-reports-monitor-supervisor.service - Skladchik reports monitor full supervisor.
|
DETAIL=SOURCE_LINE=0087: "ok=m.send_mail(cfg,secrets,subject,body);"
|
||||||
DETAIL=JOURNAL=2026-07-24T10:48:31+03:00 pve01 skladchik-reports-monitor-supervisor[3730900]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0088: "print('MAIL_OK' if ok else 'MAIL_FALSE')"
|
||||||
DETAIL=JOURNAL=2026-07-24T10:48:31+03:00 pve01 skladchik-reports-monitor-supervisor[3730900]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0089: )
|
||||||
DETAIL=JOURNAL=2026-07-24T10:58:29+03:00 pve01 skladchik-reports-monitor-supervisor[3761552]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0090: rc, out = ssh_edge("python3 -c " + shlex.quote(py), timeout=60)
|
||||||
DETAIL=JOURNAL=2026-07-24T10:58:29+03:00 pve01 skladchik-reports-monitor-supervisor[3761552]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0091: log(f"MAIL subject={subject!r} rc={rc} out={out!r}")
|
||||||
DETAIL=JOURNAL=2026-07-24T11:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3800692]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0092: return rc == 0 and "MAIL_OK" in out
|
||||||
DETAIL=JOURNAL=2026-07-24T11:08:29+03:00 pve01 skladchik-reports-monitor-supervisor[3800692]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0093:
|
||||||
DETAIL=JOURNAL=2026-07-24T11:18:29+03:00 pve01 skladchik-reports-monitor-supervisor[3839724]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0094:def systemctl_props(units):
|
||||||
DETAIL=JOURNAL=2026-07-24T11:18:29+03:00 pve01 skladchik-reports-monitor-supervisor[3839724]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0095: cmd = "systemctl is-enabled " + " ".join(map(shlex.quote, units)) + " 2>/dev/null || true; echo __ACTIVE__; systemctl is-active " + " ".join(map(shlex.quote, units)) + " 2>/dev/null || true"
|
||||||
DETAIL=JOURNAL=2026-07-24T11:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3858965]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0096: rc, out = run(cmd, timeout=20)
|
||||||
DETAIL=JOURNAL=2026-07-24T11:28:29+03:00 pve01 skladchik-reports-monitor-supervisor[3858965]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0097: parts = out.split("__ACTIVE__", 1)
|
||||||
DETAIL=JOURNAL=2026-07-24T11:38:29+03:00 pve01 skladchik-reports-monitor-supervisor[3897854]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0098: enabled_lines = [x.strip() for x in parts[0].splitlines() if x.strip()]
|
||||||
DETAIL=JOURNAL=2026-07-24T11:38:29+03:00 pve01 skladchik-reports-monitor-supervisor[3897854]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0099: active_lines = [x.strip() for x in parts[1].splitlines() if x.strip()] if len(parts) > 1 else []
|
||||||
DETAIL=JOURNAL=2026-07-24T11:48:35+03:00 pve01 skladchik-reports-monitor-supervisor[3937127]: SUPERVISOR_STATUS=OK
|
DETAIL=SOURCE_LINE=0100: res = {}
|
||||||
DETAIL=JOURNAL=2026-07-24T11:48:35+03:00 pve01 skladchik-reports-monitor-supervisor[3937127]: SUPERVISOR_PROBLEMS=
|
DETAIL=SOURCE_LINE=0101: for i, u in enumerate(units):
|
||||||
DETAIL=JOURNAL=2026-07-24T11:58:40+03:00 pve01 skladchik-reports-monitor-supervisor[3967718]: SUPERVISOR_STATUS=FAIL
|
DETAIL=SOURCE_LINE=0102: res[u] = {"enabled": enabled_lines[i] if i < len(enabled_lines) else "unknown", "active": active_lines[i] if i < len(active_lines) else "unknown"}
|
||||||
DETAIL=JOURNAL=2026-07-24T11:58:40+03:00 pve01 skladchik-reports-monitor-supervisor[3967718]: SUPERVISOR_PROBLEMS=edge_cookies_jar_bad
|
DETAIL=SOURCE_LINE=0103: return res
|
||||||
DETAIL=JOURNAL=2026-07-24T11:58:40+03:00 pve01 systemd[1]: skladchik-reports-monitor-supervisor.service: Main process exited, code=exited, status=1/FAILURE
|
DETAIL=SOURCE_LINE=0104:
|
||||||
DETAIL=JOURNAL=2026-07-24T11:58:40+03:00 pve01 systemd[1]: skladchik-reports-monitor-supervisor.service: Failed with result 'exit-code'.
|
DETAIL=SOURCE_LINE=0105:def gather():
|
||||||
DETAIL=JOURNAL=2026-07-24T11:58:40+03:00 pve01 systemd[1]: Failed to start skladchik-reports-monitor-supervisor.service - Skladchik reports monitor full supervisor.
|
DETAIL=SOURCE_LINE=0106: facts = {}
|
||||||
DETAIL=SUPERVISOR_LAST_FAILURE_END
|
DETAIL=SOURCE_LINE=0107: problems = []
|
||||||
|
DETAIL=SOURCE_LINE=0108:
|
||||||
|
DETAIL=SOURCE_LINE=0109: edge_cmd = r'''
|
||||||
|
DETAIL=SOURCE_LINE=0110:set +e
|
||||||
|
DETAIL=SOURCE_LINE=0111:echo __MONITOR_HEALTH_BEGIN__
|
||||||
|
DETAIL=SOURCE_LINE=0112:cat /var/lib/skladchik-reports-monitor/health.txt 2>/dev/null || true
|
||||||
|
DETAIL=SOURCE_LINE=0113:echo __MONITOR_HEALTH_END__
|
||||||
|
DETAIL=SOURCE_LINE=0114:echo __ALERT_HEALTH_BEGIN__
|
||||||
|
DETAIL=SOURCE_LINE=0115:cat /var/lib/skladchik-reports-monitor/alert-health.txt 2>/dev/null || true
|
||||||
|
DETAIL=SOURCE_LINE=0116:echo __ALERT_HEALTH_END__
|
||||||
|
DETAIL=SOURCE_LINE=0117:echo __TIMERS_BEGIN__
|
||||||
|
DETAIL=SOURCE_LINE=0118:for u in skladchik-reports-monitor.timer skladchik-reports-monitor-watchdog.timer skladchik-reports-monitor-heartbeat.timer skladchik-reports-monitor-weekly-summary.timer skladchik-reports-monitor-age-alert.timer skladchik-reports-monitor-event-selftest.timer; do echo "$u enabled=$(systemctl is-enabled "$u" 2>/dev/null) active=$(systemctl is-active "$u" 2>/dev/null)"; done
|
||||||
|
DETAIL=SOURCE_LINE=0119:echo __TIMERS_END__
|
||||||
|
DETAIL=SOURCE_LINE=0120:echo __ROUTE_BEGIN__
|
||||||
|
DETAIL=SOURCE_LINE=0121:ip route get 1.1.1.1 2>&1 | head -n 1
|
||||||
|
DETAIL=SOURCE_LINE=0122:echo __ROUTE_END__
|
||||||
|
DETAIL=SOURCE_LINE=0123:echo __FILES_BEGIN__
|
||||||
|
DETAIL=SOURCE_LINE=0124:test -s /opt/skladchik-reports-monitor/monitor.py && echo monitor_py=yes || echo monitor_py=no
|
||||||
|
DETAIL=SOURCE_LINE=0125:test -s /etc/skladchik-reports-monitor/config.ini && echo config_ini=yes || echo config_ini=no
|
||||||
|
DETAIL=SOURCE_LINE=0126:test -s /etc/skladchik-reports-monitor/secrets.env && echo secrets_env=<REDACTED>
|
||||||
|
DETAIL=SOURCE_LINE=0127:test -s /var/lib/skladchik-reports-monitor/cookies.jar && echo cookies_jar=yes || echo cookies_jar=no
|
||||||
|
DETAIL=SOURCE_LINE=0128:[ -s /var/lib/skladchik-reports-monitor/cookies.jar ] && echo cookies_jar_bytes=$(stat -c %s /var/lib/skladchik-reports-monitor/cookies.jar 2>/dev/null) || echo cookies_jar_bytes=0
|
||||||
|
DETAIL=SOURCE_LINE=0129:test -s /opt/skladchik-reports-monitor/operational_digest.py && echo operational_digest_py=yes || echo operational_digest_py=no
|
||||||
|
DETAIL=SOURCE_LINE=0130:test -s /opt/skladchik-reports-monitor/event-selftest-runner.py && echo event_selftest_runner_py=yes || echo event_selftest_runner_py=no
|
||||||
|
DETAIL=SOURCE_LINE=0131:test -s /opt/skladchik-reports-monitor/event-selftest-core && echo event_selftest_core=yes || echo event_selftest_core=no
|
||||||
|
DETAIL=SOURCE_LINE=0132:test -s /var/lib/skladchik-reports-monitor/weekly-summary-health.txt && echo weekly_summary_health=yes || echo weekly_summary_health=no
|
||||||
|
DETAIL=SOURCE_LINE=0133:test -s /var/lib/skladchik-reports-monitor/age-alert-health.txt && echo age_alert_health=yes || echo age_alert_health=no
|
||||||
|
DETAIL=SOURCE_LINE=0134:test -s /var/lib/skladchik-reports-monitor/event-selftest-health.txt && echo event_selftest_health=yes || echo event_selftest_health=no
|
||||||
|
DETAIL=SOURCE_LINE=0135:python3 -m py_compile /opt/skladchik-reports-monitor/monitor.py /opt/skladchik-reports-monitor/operational_digest.py /opt/skladchik-reports-monitor/event-selftest-runner.py /opt/skladchik-reports-monitor/event-selftest-core >/dev/null 2>&1 && echo py_compile=yes || echo py_compile=no
|
||||||
|
DETAIL=SOURCE_LINE=0136:echo __FILES_END__
|
||||||
|
DETAIL=SOURCE_LINE=0137:'''
|
||||||
|
DETAIL=SOURCE_LINE=0138: rc, out = ssh_edge(edge_cmd, timeout=45)
|
||||||
|
DETAIL=SOURCE_LINE=0139: facts["edge_ssh_rc"] = str(rc)
|
||||||
|
DETAIL=SOURCE_LINE=0140: if rc != 0:
|
||||||
|
DETAIL=SOURCE_LINE=0141: problems.append("edge_ssh_failed")
|
||||||
|
DETAIL=SOURCE_LINE=0142:
|
||||||
|
DETAIL=SOURCE_LINE=0143: def section(name):
|
||||||
|
DETAIL=SOURCE_LINE=0144: m = re.search(rf"__{name}_BEGIN__\n(.*?)\n__{name}_END__", out, re.S)
|
||||||
|
DETAIL=SOURCE_LINE=0145: return m.group(1).strip() if m else ""
|
||||||
|
DETAIL=SOURCE_LINE=0146:
|
||||||
|
DETAIL=SOURCE_LINE=0147: mon = parse_env_text(section("MONITOR_HEALTH"))
|
||||||
|
DETAIL=SOURCE_LINE=0148: facts["monitor_status"] = mon.get("STATUS", "MISSING")
|
||||||
|
DETAIL=SOURCE_LINE=0149: facts["monitor_reports"] = mon.get("reports", "")
|
||||||
|
DETAIL=SOURCE_LINE=0150: facts["monitor_open"] = mon.get("open", "")
|
||||||
|
DETAIL=SOURCE_LINE=0151: facts["monitor_resolved"] = mon.get("resolved", "")
|
||||||
|
DETAIL=SOURCE_LINE=0152: facts["monitor_smtp_configured"] = mon.get("smtp_configured", "")
|
||||||
|
DETAIL=SOURCE_LINE=0153: facts["session_keepalive"] = mon.get("session_keepalive", "")
|
||||||
|
DETAIL=SOURCE_LINE=0154: facts["cookie_jar"] = mon.get("cookie_jar", "")
|
||||||
|
DETAIL=SOURCE_LINE=0155: facts["cookie_jar_bytes"] = mon.get("cookie_jar_bytes", "")
|
||||||
|
DETAIL=SOURCE_LINE=0156: facts["cookie_expiry_status"] = mon.get("cookie_expiry_status", "")
|
||||||
|
DETAIL=SOURCE_LINE=0157: facts["cookie_expiry_utc"] = mon.get("cookie_expiry_utc", "")
|
||||||
|
DETAIL=SOURCE_LINE=0158: facts["cookie_expires_in_seconds"] = mon.get("cookie_expires_in_seconds", "")
|
||||||
|
DETAIL=SOURCE_LINE=0159: facts["cookie_expires_in_days"] = mon.get("cookie_expires_in_days", "")
|
||||||
|
DETAIL=SOURCE_LINE=0160: facts["cookie_persistent_count"] = mon.get("cookie_persistent_count", "")
|
||||||
|
DETAIL=SOURCE_LINE=0161: facts["monitor_health_age_seconds"] = str(age_seconds_from_ts(mon.get("TS")))
|
||||||
|
DETAIL=SOURCE_LINE=0162: if mon.get("STATUS") != "OK":
|
||||||
|
DETAIL=SOURCE_LINE=0163: problems.append("monitor_status_not_ok")
|
||||||
|
DETAIL=SOURCE_LINE=0164: if int(facts["monitor_health_age_seconds"]) > MONITOR_HEALTH_AGE_MAX:
|
||||||
|
DETAIL=SOURCE_LINE=0165: problems.append("monitor_health_stale")
|
||||||
|
DETAIL=SOURCE_LINE=0166: if mon.get("reports") in ("", "0"):
|
||||||
|
DETAIL=SOURCE_LINE=0167: problems.append("monitor_reports_zero_or_missing")
|
||||||
|
DETAIL=SOURCE_LINE=0168: if mon.get("session_keepalive") != "persistent_cookie_jar":
|
||||||
|
DETAIL=SOURCE_LINE=0169: problems.append("session_keepalive_not_persistent")
|
||||||
|
DETAIL=SOURCE_LINE=0170: if mon.get("cookie_jar") not in ("yes", "empty"):
|
||||||
|
DETAIL=SOURCE_LINE=0171: problems.append("monitor_cookie_jar_bad")
|
||||||
|
DETAIL=SOURCE_LINE=0172: expiry_status = mon.get("cookie_expiry_status", "")
|
||||||
|
DETAIL=SOURCE_LINE=0173: try:
|
||||||
|
DETAIL=SOURCE_LINE=0174: expires_in = int(mon.get("cookie_expires_in_seconds", ""))
|
||||||
|
DETAIL=SOURCE_LINE=0175: except Exception:
|
||||||
|
DETAIL=SOURCE_LINE=0176: expires_in = None
|
||||||
|
DETAIL=SOURCE_LINE=0177: if expiry_status in ("", "NO_COOKIE_JAR", "NO_PERSISTENT_COOKIE", "ERROR"):
|
||||||
|
DETAIL=SOURCE_LINE=0178: problems.append("cookie_expiry_missing")
|
||||||
|
DETAIL=SOURCE_LINE=0179: elif expiry_status == "EXPIRED" or (expires_in is not None and expires_in <= 0):
|
||||||
|
DETAIL=SOURCE_LINE=0180: problems.append("cookie_expiry_expired")
|
||||||
|
DETAIL=SOURCE_LINE=0181: elif expiry_status == "WARN" or (expires_in is not None and expires_in < COOKIE_EXPIRY_WARN_SECONDS):
|
||||||
|
DETAIL=SOURCE_LINE=0182: problems.append("cookie_expiry_warn")
|
||||||
|
DETAIL=SOURCE_LINE=0183:
|
||||||
|
DETAIL=SOURCE_LINE=0184: timers_text = section("TIMERS")
|
||||||
|
DETAIL=SOURCE_LINE=0185: for unit in ["skladchik-reports-monitor.timer", "skladchik-reports-monitor-watchdog.timer", "skladchik-reports-monitor-heartbeat.timer", "skladchik-reports-monitor-weekly-summary.timer", "skladchik-reports-monitor-age-alert.timer", "skladchik-reports-monitor-event-selftest.timer"]:
|
||||||
|
DETAIL=SOURCE_LINE=0186: safe = unit.replace(".timer", "").replace("_", "-")
|
||||||
|
DETAIL=SOURCE_LINE=0187: m = re.search(re.escape(unit) + r"\s+enabled=([^\s]+)\s+active=([^\s]+)", timers_text)
|
||||||
|
DETAIL=SOURCE_LINE=0188: enabled = m.group(1) if m else "missing"
|
||||||
|
DETAIL=SOURCE_LINE=0189: active = m.group(2) if m else "missing"
|
||||||
|
DETAIL=SOURCE_LINE=0190: facts[f"edge_{safe}_timer_enabled"] = enabled
|
||||||
|
DETAIL=SOURCE_LINE=0191: facts[f"edge_{safe}_timer_active"] = active
|
||||||
|
DETAIL=SOURCE_LINE=0192: if enabled != "enabled" or active != "active":
|
||||||
|
DETAIL=SOURCE_LINE=0193: problems.append(f"edge_{safe}_timer_bad")
|
||||||
|
DETAIL=SOURCE_LINE=0194:
|
||||||
|
DETAIL=SOURCE_LINE=0195: route_lines = section("ROUTE").splitlines()
|
||||||
|
DETAIL=SOURCE_LINE=0196: route = route_lines[0] if route_lines else ""
|
||||||
|
DETAIL=SOURCE_LINE=0197: facts["netbird_route"] = route
|
||||||
|
DETAIL=SOURCE_LINE=0198: if " wt0 " not in (" " + route + " "):
|
||||||
|
DETAIL=SOURCE_LINE=0199: problems.append("netbird_route_not_wt0")
|
||||||
|
DETAIL=SOURCE_LINE=0200:
|
||||||
|
DETAIL=SOURCE_LINE=0201: files = parse_env_text(section("FILES"))
|
||||||
|
DETAIL=SOURCE_LINE=0202: for k in ["monitor_py", "config_ini", "secrets_env", "cookies_jar", "operational_digest_py", "event_selftest_runner_py", "event_selftest_core", "weekly_summary_health", "age_alert_health", "event_selftest_health", "py_compile"]:
|
||||||
|
DETAIL=SOURCE_LINE=0203: facts[k] = files.get(k, "no")
|
||||||
|
DETAIL=SOURCE_LINE=0204: if facts[k] != "yes":
|
||||||
|
DETAIL=SOURCE_LINE=0205: problems.append(f"edge_{k}_bad")
|
||||||
|
DETAIL=SOURCE_LINE=0206: facts["cookies_jar_bytes_file"] = files.get("cookies_jar_bytes", "0")
|
||||||
|
DETAIL=SOURCE_LINE=0207:
|
||||||
|
DETAIL=SOURCE_LINE=0208: backup = read_env_file(PVE_BACKUP_HEALTH)
|
||||||
|
DETAIL=SOURCE_LINE=0209: facts["backup_status"] = backup.get("STATUS", "MISSING")
|
||||||
|
DETAIL=SOURCE_LINE=0210: facts["backup_run_id"] = backup.get("RUN_ID", "")
|
||||||
|
DETAIL=SOURCE_LINE=0211: facts["backup_size_bytes"] = backup.get("SIZE_BYTES", "")
|
||||||
|
DETAIL=SOURCE_LINE=0212: facts["backup_version"] = backup.get("BACKUP_VERSION", "")
|
||||||
|
DETAIL=SOURCE_LINE=0213: facts["backup_cookie_jar_backed_up"] = backup.get("COOKIE_JAR_BACKED_UP", "")
|
||||||
|
DETAIL=SOURCE_LINE=0214: facts["backup_extension_health_files_backed_up"] = backup.get("EXTENSION_HEALTH_FILES_BACKED_UP", "")
|
||||||
|
DETAIL=SOURCE_LINE=0215: facts["backup_title_in_email_subject"] = backup.get("TITLE_IN_EMAIL_SUBJECT", "")
|
||||||
|
DETAIL=SOURCE_LINE=0216: facts["backup_health_age_seconds"] = str(age_seconds_from_ts(backup.get("TS")))
|
||||||
|
DETAIL=SOURCE_LINE=0217: if backup.get("STATUS") != "OK":
|
||||||
|
DETAIL=SOURCE_LINE=0218: problems.append("backup_status_not_ok")
|
||||||
|
DETAIL=SOURCE_LINE=0219: if backup.get("COOKIE_JAR_BACKED_UP") != "yes":
|
||||||
|
DETAIL=SOURCE_LINE=0220: problems.append("backup_cookie_jar_not_marked")
|
||||||
|
DETAIL=SOURCE_LINE=0221: if backup.get("BACKUP_VERSION") != "4":
|
||||||
|
DETAIL=SOURCE_LINE=0222: problems.append("backup_version_not_4")
|
||||||
|
DETAIL=SOURCE_LINE=0223: if backup.get("EXTENSION_HEALTH_FILES_BACKED_UP") != "yes":
|
||||||
|
DETAIL=SOURCE_LINE=0224: problems.append("backup_extension_health_files_not_marked")
|
||||||
|
DETAIL=SOURCE_LINE=0225: if backup.get("TITLE_IN_EMAIL_SUBJECT") != "yes":
|
||||||
|
DETAIL=SOURCE_LINE=0226: problems.append("email_title_subject_not_marked")
|
||||||
|
DETAIL=SOURCE_LINE=0227: if int(facts["backup_health_age_seconds"]) > BACKUP_HEALTH_AGE_MAX:
|
||||||
|
DETAIL=SOURCE_LINE=0228: problems.append("backup_health_stale")
|
||||||
|
DETAIL=SOURCE_LINE=0229: try:
|
||||||
|
DETAIL=SOURCE_LINE=0230: if int(backup.get("SIZE_BYTES", "0")) <= 0:
|
||||||
|
DETAIL=SOURCE_LINE=0231: problems.append("backup_size_bad")
|
||||||
|
DETAIL=SOURCE_LINE=0232: except Exception:
|
||||||
|
DETAIL=SOURCE_LINE=0233: problems.append("backup_size_bad")
|
||||||
|
DETAIL=SOURCE_LINE=0234:
|
||||||
|
DETAIL=SOURCE_LINE=0235: restore = read_env_file(PVE_RESTORE_HEALTH)
|
||||||
|
DETAIL=SOURCE_LINE=0236: facts["restore_status"] = restore.get("STATUS", "MISSING")
|
||||||
|
DETAIL=SOURCE_LINE=0237: facts["restore_run_id"] = restore.get("RUN_ID", "")
|
||||||
|
DETAIL=SOURCE_LINE=0238: facts["restore_health_age_seconds"] = str(age_seconds_from_ts(restore.get("TS")))
|
||||||
|
DETAIL=SOURCE_LINE=0239: for key in ["CLOUD_1_STATUS", "CLOUD_2_STATUS", "SHA256_OK", "TAR_OK", "REQUIRED_FILES_OK", "COOKIE_JAR_OK", "EXTENSION_HEALTH_FILES_OK", "EXTENSION_SCRIPTS_OK", "EXTENSION_UNITS_OK", "SQLITE_OK", "PY_COMPILE_OK"]:
|
||||||
|
DETAIL=SOURCE_LINE=0240: facts[key.lower()] = restore.get(key, "")
|
||||||
|
DETAIL=SOURCE_LINE=0241: if restore.get("STATUS") != "OK":
|
||||||
|
DETAIL=SOURCE_LINE=0242: problems.append("restore_status_not_ok")
|
||||||
|
DETAIL=SOURCE_LINE=0243: if int(facts["restore_health_age_seconds"]) > RESTORE_HEALTH_AGE_MAX:
|
||||||
|
DETAIL=SOURCE_LINE=0244: problems.append("restore_health_stale")
|
||||||
|
DETAIL=SOURCE_LINE=0245: for key in ["CLOUD_1_STATUS", "CLOUD_2_STATUS"]:
|
||||||
|
DETAIL=SOURCE_LINE=0246: if restore.get(key) != "OK":
|
||||||
|
DETAIL=SOURCE_LINE=0247: problems.append(key.lower() + "_bad")
|
||||||
|
DETAIL=SOURCE_LINE=0248: for key in ["SHA256_OK", "TAR_OK", "REQUIRED_FILES_OK", "COOKIE_JAR_OK", "EXTENSION_HEALTH_FILES_OK", "EXTENSION_SCRIPTS_OK", "EXTENSION_UNITS_OK", "SQLITE_OK", "PY_COMPILE_OK"]:
|
||||||
|
DETAIL=SOURCE_LINE=0249: if restore.get(key) != "yes":
|
||||||
|
DETAIL=SOURCE_LINE=0250: problems.append(key.lower() + "_bad")
|
||||||
|
DETAIL=SOURCE_LINE=0251:
|
||||||
|
DETAIL=SOURCE_LINE=0252: local_units = [
|
||||||
|
DETAIL=SOURCE_LINE=0253: "skladchik-reports-monitor-backup.timer",
|
||||||
|
DETAIL=SOURCE_LINE=0254: "skladchik-reports-monitor-backup-restore-check.timer",
|
||||||
|
DETAIL=SOURCE_LINE=0255: "skladchik-reports-monitor-supervisor.timer",
|
||||||
|
DETAIL=SOURCE_LINE=0256: "skladchik-reports-monitor-monthly-selftest-watch.timer",
|
||||||
|
DETAIL=SOURCE_LINE=0257: ]
|
||||||
|
DETAIL=SOURCE_LINE=0258: props = systemctl_props(local_units)
|
||||||
|
DETAIL=SOURCE_LINE=0259: for u, p in props.items():
|
||||||
|
DETAIL=SOURCE_LINE=0260: base = u.replace(".timer", "").replace("-", "_")
|
||||||
|
DETAIL=SOURCE_LINE=0261: facts[base + "_timer_enabled"] = p["enabled"]
|
||||||
|
DETAIL=SOURCE_LINE=0262: facts[base + "_timer_active"] = p["active"]
|
||||||
|
DETAIL=SOURCE_LINE=0263: if u != "skladchik-reports-monitor-supervisor.timer":
|
||||||
|
DETAIL=SOURCE_LINE=0264: if p["enabled"] != "enabled" or p["active"] != "active":
|
||||||
|
DETAIL=SOURCE_LINE=0265: problems.append(base + "_timer_bad")
|
||||||
|
DETAIL=SOURCE_LINE=0266: else:
|
||||||
|
DETAIL=SOURCE_LINE=0267: # Before first enable it can be disabled/inactive during test. After enable next runs will see enabled/active.
|
||||||
|
DETAIL=SOURCE_LINE=0268: if p["enabled"] not in ("enabled", "disabled") or p["active"] not in ("active", "inactive"):
|
||||||
|
DETAIL=SOURCE_LINE=0269: problems.append(base + "_timer_unknown")
|
||||||
|
DETAIL=SOURCE_LINE=0270:
|
||||||
|
DETAIL=SOURCE_LINE=0271: return facts, problems
|
||||||
|
DETAIL=SOURCE_LINE=0272:
|
||||||
|
DETAIL=SOURCE_LINE=0273:def render_body(status, facts, problems, mode):
|
||||||
|
DETAIL=SOURCE_LINE=0274: ru_status = "OK" if status == "OK" else "АВАРИЯ"
|
||||||
|
DETAIL=SOURCE_LINE=0275: lines = [
|
||||||
|
DETAIL=SOURCE_LINE=0276: f"Статус надзора: {ru_status}",
|
||||||
|
DETAIL=SOURCE_LINE=0277: f"Время UTC: {iso()}",
|
||||||
|
DETAIL=SOURCE_LINE=0278: "",
|
||||||
|
DETAIL=SOURCE_LINE=0279: "Монитор жалоб:",
|
||||||
|
DETAIL=SOURCE_LINE=0280: f"- статус: {facts.get('monitor_status')}",
|
||||||
|
DETAIL=SOURCE_LINE=0281: f"- всего жалоб на странице: {facts.get('monitor_reports')}",
|
||||||
|
DETAIL=SOURCE_LINE=0282: f"- открытые/назначенные: {facts.get('monitor_open')}",
|
||||||
|
DETAIL=SOURCE_LINE=0283: f"- закрытые: {facts.get('monitor_resolved')}",
|
||||||
|
DETAIL=SOURCE_LINE=0284: f"- возраст health, сек: {facts.get('monitor_health_age_seconds')}",
|
||||||
|
DETAIL=SOURCE_LINE=0285: f"- автопродление авторизации: {facts.get('session_keepalive')}",
|
||||||
|
DETAIL=SOURCE_LINE=0286: f"- cookie jar: {facts.get('cookie_jar')} / {facts.get('cookie_jar_bytes')} байт",
|
||||||
|
DETAIL=SOURCE_LINE=0287: f"- постоянная cookie действует до UTC: {facts.get('cookie_expiry_utc') or 'не определено'}",
|
||||||
|
DETAIL=SOURCE_LINE=0288: f"- осталось дней по cookie: <REDACTED>
|
||||||
|
DETAIL=SOURCE_LINE=0289: f"- статус срока cookie: <REDACTED>
|
||||||
|
DETAIL=SOURCE_LINE=0290: f"- NetBird route: {facts.get('netbird_route')}",
|
||||||
|
DETAIL=SOURCE_LINE=0291: "",
|
||||||
|
DETAIL=SOURCE_LINE=0292: "Бэкап:",
|
||||||
|
DETAIL=SOURCE_LINE=0293: f"- статус: {facts.get('backup_status')}",
|
||||||
|
DETAIL=SOURCE_LINE=0294: f"- последний RUN_ID: {facts.get('backup_run_id')}",
|
||||||
|
DETAIL=SOURCE_LINE=0295: f"- размер архива, байт: {facts.get('backup_size_bytes')}",
|
||||||
|
DETAIL=SOURCE_LINE=0296: f"- cookie jar в бэкапе: {facts.get('backup_cookie_jar_backed_up')}",
|
||||||
|
DETAIL=SOURCE_LINE=0297: f"- health-файлы расширений в бэкапе: {facts.get('backup_extension_health_files_backed_up')}",
|
||||||
|
DETAIL=SOURCE_LINE=0298: f"- название жалобы в теме письма: {facts.get('backup_title_in_email_subject')}",
|
||||||
|
DETAIL=SOURCE_LINE=0299: f"- возраст health, сек: {facts.get('backup_health_age_seconds')}",
|
||||||
|
DETAIL=SOURCE_LINE=0300: "",
|
||||||
|
DETAIL=SOURCE_LINE=0301: "Restore-check:",
|
||||||
|
DETAIL=SOURCE_LINE=0302: f"- статус: {facts.get('restore_status')}",
|
||||||
|
DETAIL=SOURCE_LINE=0303: f"- последний RUN_ID: {facts.get('restore_run_id')}",
|
||||||
|
DETAIL=SOURCE_LINE=0304: f"- cloud1/cloud2: {facts.get('cloud_1_status')}/{facts.get('cloud_2_status')}",
|
||||||
|
DETAIL=SOURCE_LINE=0305: f"- sha256/tar/cookie/sqlite/python: {facts.get('sha256_ok')}/{facts.get('tar_ok')}/{facts.get('cookie_jar_ok')}/{facts.get('sqlite_ok')}/{facts.get('py_compile_ok')}",
|
||||||
|
DETAIL=SOURCE_LINE=0306: f"- health/scripts/units расширений: {facts.get('extension_health_files_ok')}/{facts.get('extension_scripts_ok')}/{facts.get('extension_units_ok')}",
|
||||||
|
DETAIL=SOURCE_LINE=0307: "",
|
||||||
|
DETAIL=SOURCE_LINE=0308: "Проблемы:",
|
||||||
|
DETAIL=SOURCE_LINE=0309: ]
|
||||||
|
DETAIL=SOURCE_LINE=0310: if problems:
|
||||||
|
DETAIL=SOURCE_LINE=0311: lines.extend([f"- {p}" for p in problems])
|
||||||
|
DETAIL=SOURCE_LINE=0312: else:
|
||||||
|
DETAIL=SOURCE_LINE=0313: lines.append("- нет")
|
||||||
|
DETAIL=SOURCE_LINE=0314: lines.extend([
|
||||||
|
DETAIL=SOURCE_LINE=0315: "",
|
||||||
|
DETAIL=SOURCE_LINE=0316: "Авторизация:",
|
||||||
|
DETAIL=SOURCE_LINE=0317: "- ручной ежемесячный перелогин обычно не нужен; монитор сам продлевает сессию постоянным cookie jar при каждом опросе;",
|
||||||
|
DETAIL=SOURCE_LINE=0318: "- supervisor теперь отдельно следит за сроком постоянной cookie и заранее предупредит, если до истечения останется меньше 7 дней;",
|
||||||
|
DETAIL=SOURCE_LINE=0319: "- если форум принудительно сбросит сессию, придёт отдельная тревога AUTH_REQUIRED; тогда нужно будет разово обновить вход, но это аварийный сценарий, а не ежемесячная рутина.",
|
||||||
|
DETAIL=SOURCE_LINE=0320: ])
|
||||||
|
DETAIL=SOURCE_LINE=0321: return "\n".join(lines)
|
||||||
|
DETAIL=SOURCE_LINE=0322:
|
||||||
|
DETAIL=SOURCE_LINE=0323:def write_health(status, facts, problems, mail_sent):
|
||||||
|
DETAIL=SOURCE_LINE=0324: STATE_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
DETAIL=SOURCE_LINE=0325: lines = [f"STATUS={status}", f"TS={iso()}", "problems=" + ",".join(problems), f"mail_sent={'yes' if mail_sent else 'no'}"]
|
||||||
|
DETAIL=SOURCE_LINE=0326: for k in sorted(facts):
|
||||||
|
DETAIL=SOURCE_LINE=0327: v = str(facts[k]).replace("\n", " ")[:500]
|
||||||
|
DETAIL=SOURCE_LINE=0328: lines.append(f"{k}={v}")
|
||||||
|
DETAIL=SOURCE_LINE=0329: HEALTH.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||||
|
DETAIL=SOURCE_LINE=0330:
|
||||||
|
DETAIL=SOURCE_LINE=0331:def read_state():
|
||||||
|
DETAIL=SOURCE_LINE=0332: return read_env_file(STATE)
|
||||||
|
DETAIL=SOURCE_LINE=0333:
|
||||||
|
DETAIL=SOURCE_LINE=0334:def write_state(status, problems):
|
||||||
|
DETAIL=SOURCE_LINE=0335: STATE_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
DETAIL=SOURCE_LINE=0336: STATE.write_text(f"LAST_STATUS={status}\nLAST_PROBLEMS={','.join(problems)}\nTS={iso()}\n", encoding="utf-8")
|
||||||
|
DETAIL=SOURCE_LINE=0337:
|
||||||
|
DETAIL=SOURCE_LINE=0338:def main():
|
||||||
|
DETAIL=SOURCE_LINE=0339: mode = sys.argv[1] if len(sys.argv) > 1 else "check"
|
||||||
|
DETAIL=SOURCE_LINE=0340: if mode not in ("check", "test"):
|
||||||
|
DETAIL=SOURCE_LINE=0341: print("usage: skladchik-reports-monitor-supervisor [check|test]", file=sys.stderr)
|
||||||
|
DETAIL=SOURCE_LINE=0342: return 2
|
||||||
|
DETAIL=SOURCE_LINE=0343:
|
||||||
|
DETAIL=SOURCE_LINE=0344: facts, problems = gather()
|
||||||
|
DETAIL=SOURCE_LINE=0345: status = "OK" if not problems else "FAIL"
|
||||||
|
DETAIL=SOURCE_LINE=0346: old = read_state()
|
||||||
|
DETAIL=SOURCE_LINE=0347: old_status = old.get("LAST_STATUS", "")
|
||||||
|
DETAIL=SOURCE_LINE=0348: mail_sent = False
|
||||||
|
DETAIL=SOURCE_LINE=0349:
|
||||||
|
DETAIL=SOURCE_LINE=0350: if mode == "test":
|
||||||
|
DETAIL=SOURCE_LINE=0351: mail_sent = edge_mail("Тест: надзор монитора жалоб и бэкапов", render_body(status, facts, problems, mode))
|
||||||
|
DETAIL=SOURCE_LINE=0352: if not mail_sent:
|
||||||
|
DETAIL=SOURCE_LINE=0353: problems.append("supervisor_mail_send_failed")
|
||||||
|
DETAIL=SOURCE_LINE=0354: status = "FAIL"
|
||||||
|
DETAIL=SOURCE_LINE=0355: elif status != "OK":
|
||||||
|
DETAIL=SOURCE_LINE=0356: if old_status != "FAIL":
|
||||||
|
DETAIL=SOURCE_LINE=0357: mail_sent = edge_mail("Авария: монитор жалоб или его бэкапы", render_body(status, facts, problems, mode))
|
||||||
|
DETAIL=SOURCE_LINE=0358: if not mail_sent:
|
||||||
|
DETAIL=SOURCE_LINE=0359: problems.append("supervisor_mail_send_failed")
|
||||||
|
DETAIL=SOURCE_LINE=0360: status = "FAIL"
|
||||||
|
DETAIL=SOURCE_LINE=0361: elif old_status == "FAIL":
|
||||||
|
DETAIL=SOURCE_LINE=0362: mail_sent = edge_mail("Восстановлено: монитор жалоб и бэкапы снова OK", render_body(status, facts, problems, mode))
|
||||||
|
DETAIL=SOURCE_LINE=0363:
|
||||||
|
DETAIL=SOURCE_LINE=0364: write_health(status, facts, problems, mail_sent)
|
||||||
|
DETAIL=SOURCE_LINE=0365: write_state(status, problems)
|
||||||
|
DETAIL=SOURCE_LINE=0366:
|
||||||
|
DETAIL=SOURCE_LINE=0367: print(f"SUPERVISOR_STATUS={status}")
|
||||||
|
DETAIL=SOURCE_LINE=0368: print(f"SUPERVISOR_MAIL_SENT={'yes' if mail_sent else 'no'}")
|
||||||
|
DETAIL=SOURCE_LINE=0369: print(f"SUPERVISOR_PROBLEMS={','.join(problems)}")
|
||||||
|
DETAIL=SOURCE_LINE=0370: print(f"MONITOR={facts.get('monitor_status')} reports={facts.get('monitor_reports')} open={facts.get('monitor_open')} resolved={facts.get('monitor_resolved')}")
|
||||||
|
DETAIL=SOURCE_LINE=0371: print(f"BACKUP={facts.get('backup_status')} run={facts.get('backup_run_id')}")
|
||||||
|
DETAIL=SOURCE_LINE=0372: print(f"RESTORE={facts.get('restore_status')} run={facts.get('restore_run_id')}")
|
||||||
|
DETAIL=SOURCE_LINE=0373: log(f"CHECK mode={mode} status={status} mail_sent={'yes' if mail_sent else 'no'} problems={problems}")
|
||||||
|
DETAIL=SOURCE_LINE=0374: return 0 if status == "OK" else 1
|
||||||
|
DETAIL=SOURCE_LINE=0375:
|
||||||
|
DETAIL=SOURCE_LINE=0376:if __name__ == "__main__":
|
||||||
|
DETAIL=SOURCE_LINE=0377: sys.exit(main())
|
||||||
|
DETAIL=SCRIPT_TOTAL_LINES=377
|
||||||
|
DETAIL=RELEVANT_SOURCE_END
|
||||||
|
DETAIL=UNIT_CONTRACT_BEGIN
|
||||||
|
DETAIL=UNIT_NAME=skladchik-reports-monitor-supervisor.service
|
||||||
|
DETAIL=UNIT_LINE=# /etc/systemd/system/skladchik-reports-monitor-supervisor.service
|
||||||
|
DETAIL=UNIT_LINE=[Unit]
|
||||||
|
DETAIL=UNIT_LINE=Description=Skladchik reports monitor full supervisor
|
||||||
|
DETAIL=UNIT_LINE=After=network-online.target
|
||||||
|
DETAIL=UNIT_LINE=
|
||||||
|
DETAIL=UNIT_LINE=[Service]
|
||||||
|
DETAIL=UNIT_LINE=Type=oneshot
|
||||||
|
DETAIL=UNIT_LINE=ExecStart=/usr/local/sbin/skladchik-reports-monitor-supervisor check
|
||||||
|
DETAIL=UNIT_NAME=skladchik-reports-monitor-supervisor.timer
|
||||||
|
DETAIL=UNIT_LINE=# /etc/systemd/system/skladchik-reports-monitor-supervisor.timer
|
||||||
|
DETAIL=UNIT_LINE=[Unit]
|
||||||
|
DETAIL=UNIT_LINE=Description=Run Skladchik reports monitor full supervisor every 10 minutes
|
||||||
|
DETAIL=UNIT_LINE=
|
||||||
|
DETAIL=UNIT_LINE=[Timer]
|
||||||
|
DETAIL=UNIT_LINE=OnBootSec=5min
|
||||||
|
DETAIL=UNIT_LINE=OnUnitActiveSec=10min
|
||||||
|
DETAIL=UNIT_LINE=AccuracySec=30s
|
||||||
|
DETAIL=UNIT_LINE=Persistent=false
|
||||||
|
DETAIL=UNIT_LINE=Unit=skladchik-reports-monitor-supervisor.service
|
||||||
|
DETAIL=UNIT_LINE=
|
||||||
|
DETAIL=UNIT_LINE=[Install]
|
||||||
|
DETAIL=UNIT_LINE=WantedBy=timers.target
|
||||||
|
DETAIL=UNIT_CONTRACT_END
|
||||||
|
DETAIL=TIMER_STATE_BEGIN
|
||||||
|
DETAIL=TIMER_PROPERTY=Unit=skladchik-reports-monitor-supervisor.service
|
||||||
|
DETAIL=TIMER_PROPERTY=NextElapseUSecRealtime=
|
||||||
|
DETAIL=TIMER_PROPERTY=LastTriggerUSec=Fri 2026-07-24 11:58:38 MSK
|
||||||
|
DETAIL=TIMER_PROPERTY=LoadState=loaded
|
||||||
|
DETAIL=TIMER_PROPERTY=ActiveState=active
|
||||||
|
DETAIL=TIMER_PROPERTY=SubState=waiting
|
||||||
|
DETAIL=TIMER_ROW=Fri 2026-07-24 12:08:38 MSK 3min 34s Fri 2026-07-24 11:58:38 MSK 6min ago skladchik-reports-monitor-supervisor.timer skladchik-reports-monitor-supervisor.service
|
||||||
|
DETAIL=TIMER_ROW=Sat 2026-07-25 04:39:05 MSK 16h Fri 2026-07-24 04:28:08 MSK 7h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service
|
||||||
|
DETAIL=TIMER_ROW=Sat 2026-07-25 10:15:00 MSK 22h Fri 2026-07-24 10:15:04 MSK 1h 49min ago skladchik-reports-monitor-monthly-selftest-watch.timer skladchik-reports-monitor-monthly-selftest-watch.service
|
||||||
|
DETAIL=TIMER_ROW=Sun 2026-07-26 05:15:13 MSK 1 day 17h Sun 2026-07-19 05:11:25 MSK - skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service
|
||||||
|
DETAIL=TIMER_STATE_END
|
||||||
DETAIL=SECRET_VALUES_OUTPUT=NONE
|
DETAIL=SECRET_VALUES_OUTPUT=NONE
|
||||||
DETAIL=SUPERVISOR_FAILURE_TRACE=PASS
|
DETAIL=CHANGES_MADE=NO
|
||||||
CHANGES_MADE=NO
|
DETAIL=COOKIE_CHECK_SOURCE_AUDIT=PASS
|
||||||
|
|
||||||
OUTPUT_END
|
OUTPUT_END
|
||||||
CHAT_OUTPUT_END
|
CHAT_OUTPUT_END
|
||||||
|
|||||||
Reference in New Issue
Block a user