1
0
Files
homelab-public-context/runtime/latest.txt
T
2026-07-23 15:28:49 +00:00

209 lines
20 KiB
Plaintext

CHAT_OUTPUT_BEGIN
COMMAND_ID=HOMELAB-SKLADCHIK-COOKIE-IMPORT-CONTRACT-AUDIT-371
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=cluster-audit
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
COMMAND_SHA256=e05e1429084dee5c0e0748f3b91c4e68d16934c6e5a32f2a25b1332611dfe14b
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGES_MADE=false
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=05d9b9a806fa45499b8800218e4ec5c6af5ae34a60a9085db4a4360bd3d4132c
SANITIZED_OUTPUT_SHA256=47b740df38ba8b8afe4df80ad5e67206017b5aa2f70d1d3bad26bcf7eda2c5a3
OUTPUT_BEGIN
DETAIL=STDIN_IS_TTY=YES
DETAIL=DEV_TTY_READ_WRITE=YES
DETAIL=COOKIE_IMPORT_CONTRACT_BEGIN
/usr/local/sbin/skladchik-reports-monitor-supervisor-1-#!/usr/bin/env python3
/usr/local/sbin/skladchik-reports-monitor-supervisor-2-# -*- coding: utf-8 -*-
/usr/local/sbin/skladchik-reports-monitor-supervisor:3:import base64
/usr/local/sbin/skladchik-reports-monitor-supervisor:4:import datetime as _dt
/usr/local/sbin/skladchik-reports-monitor-supervisor:5:import os
/usr/local/sbin/skladchik-reports-monitor-supervisor:6:import re
/usr/local/sbin/skladchik-reports-monitor-supervisor:7:import shlex
/usr/local/sbin/skladchik-reports-monitor-supervisor:8:import subprocess
/usr/local/sbin/skladchik-reports-monitor-supervisor:9:import sys
/usr/local/sbin/skladchik-reports-monitor-supervisor:10:from pathlib import Path
/usr/local/sbin/skladchik-reports-monitor-supervisor-11-
/usr/local/sbin/skladchik-reports-monitor-supervisor:12:EDGE = os.environ.get("SKLADCHIK_EDGE", "debian@[PRIVATE_IP]")
/usr/local/sbin/skladchik-reports-monitor-supervisor-13-STATE_DIR = Path("/var/lib/skladchik-reports-monitor-supervisor")
/usr/local/sbin/skladchik-reports-monitor-supervisor-14-LOG_DIR = Path("/var/log/skladchik-reports-monitor-supervisor")
--
/usr/local/sbin/skladchik-reports-monitor-supervisor-19-MONITOR_HEALTH_AGE_MAX = int(os.environ.get("SKLADCHIK_MONITOR_HEALTH_MAX_SECONDS", "600"))
/usr/local/sbin/skladchik-reports-monitor-supervisor-20-BACKUP_HEALTH_AGE_MAX = int(os.environ.get("SKLADCHIK_BACKUP_HEALTH_MAX_SECONDS", "129600"))
/usr/local/sbin/skladchik-reports-monitor-supervisor:21:RESTORE_HEALTH_AGE_MAX = int(os.environ.get("SKLADCHIK_RESTORE_HEALTH_MAX_SECONDS", "691200"))
/usr/local/sbin/skladchik-reports-monitor-supervisor:22:COOKIE_EXPIRY_WARN_SECONDS = int(os.environ.get("SKLADCHIK_COOKIE_EXPIRY_WARN_SECONDS", "604800"))
/usr/local/sbin/skladchik-reports-monitor-supervisor-23-
/usr/local/sbin/skladchik-reports-monitor-supervisor-24-PVE_BACKUP_HEALTH = Path("/var/lib/skladchik-reports-monitor-backup/health.txt")
/usr/local/sbin/skladchik-reports-monitor-supervisor:25:PVE_RESTORE_HEALTH = Path("/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt")
/usr/local/sbin/skladchik-reports-monitor-supervisor-26-
/usr/local/sbin/skladchik-reports-monitor-supervisor-27-def now():
--
/usr/local/sbin/skladchik-reports-monitor-supervisor-79- body_b64 = base64.b64encode(body.encode("utf-8")).decode("ascii")
/usr/local/sbin/skladchik-reports-monitor-supervisor-80- py = (
/usr/local/sbin/skladchik-reports-monitor-supervisor:81: "import base64,importlib.util;"
/usr/local/sbin/skladchik-reports-monitor-supervisor:82: "spec=importlib.util.spec_from_file_location('m','/opt/skladchik-reports-monitor/monitor.py');"
/usr/local/sbin/skladchik-reports-monitor-supervisor:83: "m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m);"
/usr/local/sbin/skladchik-reports-monitor-supervisor-84- "cfg,secrets=m.load_config();"
/usr/local/sbin/skladchik-reports-monitor-supervisor-85- f"subject=base64.b64decode('{subj_b64}').decode('utf-8');"
--
/usr/local/sbin/skladchik-reports-monitor-supervisor-125-test -s /etc/skladchik-reports-monitor/config.ini && echo config_ini=yes || echo config_ini=no
/usr/local/sbin/skladchik-reports-monitor-supervisor-126-test -s /etc/skladchik-reports-monitor/secrets.env && echo secrets_env=yes || echo secrets_env=no
/usr/local/sbin/skladchik-reports-monitor-supervisor:127:test -s /var/lib/skladchik-reports-monitor/cookies.jar && echo cookies_jar=yes || echo cookies_jar=no
/usr/local/sbin/skladchik-reports-monitor-supervisor:128:[ -s /var/lib/skladchik-reports-monitor/cookies.jar ] && echo cookies_jar_bytes=$(stat -c %s /var/lib/skladchik-reports-monitor/cookies.jar 2>/dev/null) || echo cookies_jar_bytes=0
/usr/local/sbin/skladchik-reports-monitor-supervisor-129-test -s /opt/skladchik-reports-monitor/operational_digest.py && echo operational_digest_py=yes || echo operational_digest_py=no
/usr/local/sbin/skladchik-reports-monitor-supervisor-130-test -s /opt/skladchik-reports-monitor/event-selftest-runner.py && echo event_selftest_runner_py=yes || echo event_selftest_runner_py=no
--
/usr/local/sbin/skladchik-reports-monitor-supervisor-152- facts["monitor_smtp_configured"] = mon.get("smtp_configured", "")
/usr/local/sbin/skladchik-reports-monitor-supervisor-153- facts["session_keepalive"] = mon.get("session_keepalive", "")
/usr/local/sbin/skladchik-reports-monitor-supervisor:154: facts["cookie_jar"] = mon.get("cookie_jar", "")
/usr/local/sbin/skladchik-reports-monitor-supervisor:155: facts["cookie_jar_bytes"] = mon.get("cookie_jar_bytes", "")
/usr/local/sbin/skladchik-reports-monitor-supervisor:156: facts["cookie_expiry_status"] = mon.get("cookie_expiry_status", "")
/usr/local/sbin/skladchik-reports-monitor-supervisor:157: facts["cookie_expiry_utc"] = mon.get("cookie_expiry_utc", "")
/usr/local/sbin/skladchik-reports-monitor-supervisor:158: facts["cookie_expires_in_seconds"] = mon.get("cookie_expires_in_seconds", "")
/usr/local/sbin/skladchik-reports-monitor-supervisor:159: facts["cookie_expires_in_days"] = mon.get("cookie_expires_in_days", "")
/usr/local/sbin/skladchik-reports-monitor-supervisor:160: facts["cookie_persistent_count"] = mon.get("cookie_persistent_count", "")
/usr/local/sbin/skladchik-reports-monitor-supervisor-161- facts["monitor_health_age_seconds"] = str(age_seconds_from_ts(mon.get("TS")))
/usr/local/sbin/skladchik-reports-monitor-supervisor-162- if mon.get("STATUS") != "OK":
--
/usr/local/sbin/skladchik-reports-monitor-supervisor-166- if mon.get("reports") in ("", "0"):
/usr/local/sbin/skladchik-reports-monitor-supervisor-167- problems.append("monitor_reports_zero_or_missing")
/usr/local/sbin/skladchik-reports-monitor-supervisor:168: if mon.get("session_keepalive") != "persistent_cookie_jar":
/usr/local/sbin/skladchik-reports-monitor-supervisor-169- problems.append("session_keepalive_not_persistent")
/usr/local/sbin/skladchik-reports-monitor-supervisor:170: if mon.get("cookie_jar") not in ("yes", "empty"):
/usr/local/sbin/skladchik-reports-monitor-supervisor:171: problems.append("monitor_cookie_jar_bad")
/usr/local/sbin/skladchik-reports-monitor-supervisor:172: expiry_status = mon.get("cookie_expiry_status", "")
/usr/local/sbin/skladchik-reports-monitor-supervisor-173- try:
/usr/local/sbin/skladchik-reports-monitor-supervisor:174: expires_in = int(mon.get("cookie_expires_in_seconds", ""))
/usr/local/sbin/skladchik-reports-monitor-supervisor-175- except Exception:
/usr/local/sbin/skladchik-reports-monitor-supervisor-176- expires_in = None
/usr/local/sbin/skladchik-reports-monitor-supervisor:177: if expiry_status in ("", "NO_COOKIE_JAR", "NO_PERSISTENT_COOKIE", "ERROR"):
/usr/local/sbin/skladchik-reports-monitor-supervisor:178: problems.append("cookie_expiry_missing")
/usr/local/sbin/skladchik-reports-monitor-supervisor-179- elif expiry_status == "EXPIRED" or (expires_in is not None and expires_in <= 0):
/usr/local/sbin/skladchik-reports-monitor-supervisor:180: problems.append("cookie_expiry_expired")
/usr/local/sbin/skladchik-reports-monitor-supervisor:181: elif expiry_status == "WARN" or (expires_in is not None and expires_in < COOKIE_EXPIRY_WARN_SECONDS):
/usr/local/sbin/skladchik-reports-monitor-supervisor:182: problems.append("cookie_expiry_warn")
/usr/local/sbin/skladchik-reports-monitor-supervisor-183-
/usr/local/sbin/skladchik-reports-monitor-supervisor-184- timers_text = section("TIMERS")
--
/usr/local/sbin/skladchik-reports-monitor-supervisor-200-
/usr/local/sbin/skladchik-reports-monitor-supervisor-201- files = parse_env_text(section("FILES"))
/usr/local/sbin/skladchik-reports-monitor-supervisor:202: for k in ["monitor_py", "config_ini", "secrets_env", "cookies_jar", "operational_digest_py", "event_selftest_runner_py", "event_selftest_core", "weekly_summary_health", "age_alert_health", "event_selftest_health", "py_compile"]:
/usr/local/sbin/skladchik-reports-monitor-supervisor-203- facts[k] = files.get(k, "no")
/usr/local/sbin/skladchik-reports-monitor-supervisor-204- if facts[k] != "yes":
/usr/local/sbin/skladchik-reports-monitor-supervisor-205- problems.append(f"edge_{k}_bad")
/usr/local/sbin/skladchik-reports-monitor-supervisor:206: facts["cookies_jar_bytes_file"] = files.get("cookies_jar_bytes", "0")
/usr/local/sbin/skladchik-reports-monitor-supervisor-207-
/usr/local/sbin/skladchik-reports-monitor-supervisor-208- backup = read_env_file(PVE_BACKUP_HEALTH)
--
/usr/local/sbin/skladchik-reports-monitor-supervisor-211- facts["backup_size_bytes"] = backup.get("SIZE_BYTES", "")
/usr/local/sbin/skladchik-reports-monitor-supervisor-212- facts["backup_version"] = backup.get("BACKUP_VERSION", "")
/usr/local/sbin/skladchik-reports-monitor-supervisor:213: facts["backup_cookie_jar_backed_up"] = backup.get("COOKIE_JAR_BACKED_UP", "")
/usr/local/sbin/skladchik-reports-monitor-supervisor-214- facts["backup_extension_health_files_backed_up"] = backup.get("EXTENSION_HEALTH_FILES_BACKED_UP", "")
/usr/local/sbin/skladchik-reports-monitor-supervisor-215- facts["backup_title_in_email_subject"] = backup.get("TITLE_IN_EMAIL_SUBJECT", "")
--
/usr/local/sbin/skladchik-reports-monitor-supervisor-217- if backup.get("STATUS") != "OK":
/usr/local/sbin/skladchik-reports-monitor-supervisor-218- problems.append("backup_status_not_ok")
/usr/local/sbin/skladchik-reports-monitor-supervisor:219: if backup.get("COOKIE_JAR_BACKED_UP") != "yes":
/usr/local/sbin/skladchik-reports-monitor-supervisor:220: problems.append("backup_cookie_jar_not_marked")
/usr/local/sbin/skladchik-reports-monitor-supervisor-221- if backup.get("BACKUP_VERSION") != "4":
/usr/local/sbin/skladchik-reports-monitor-supervisor-222- problems.append("backup_version_not_4")
--
/usr/local/sbin/skladchik-reports-monitor-supervisor-233- problems.append("backup_size_bad")
/usr/local/sbin/skladchik-reports-monitor-supervisor-234-
/usr/local/sbin/skladchik-reports-monitor-supervisor:235: restore = read_env_file(PVE_RESTORE_HEALTH)
/usr/local/sbin/skladchik-reports-monitor-supervisor:236: facts["restore_status"] = restore.get("STATUS", "MISSING")
/usr/local/sbin/skladchik-reports-monitor-supervisor:237: facts["restore_run_id"] = restore.get("RUN_ID", "")
/usr/local/sbin/skladchik-reports-monitor-supervisor:238: facts["restore_health_age_seconds"] = str(age_seconds_from_ts(restore.get("TS")))
/usr/local/sbin/skladchik-reports-monitor-supervisor:239: for key in ["CLOUD_1_STATUS", "CLOUD_2_STATUS", "SHA256_OK", "TAR_OK", "REQUIRED_FILES_OK", "COOKIE_JAR_OK", "EXTENSION_HEALTH_FILES_OK", "EXTENSION_SCRIPTS_OK", "EXTENSION_UNITS_OK", "SQLITE_OK", "PY_COMPILE_OK"]:
/usr/local/sbin/skladchik-reports-monitor-supervisor:240: facts[key.lower()] = restore.get(key, "")
/usr/local/sbin/skladchik-reports-monitor-supervisor:241: if restore.get("STATUS") != "OK":
/usr/local/sbin/skladchik-reports-monitor-supervisor:242: problems.append("restore_status_not_ok")
/usr/local/sbin/skladchik-reports-monitor-supervisor:243: if int(facts["restore_health_age_seconds"]) > RESTORE_HEALTH_AGE_MAX:
/usr/local/sbin/skladchik-reports-monitor-supervisor:244: problems.append("restore_health_stale")
/usr/local/sbin/skladchik-reports-monitor-supervisor-245- for key in ["CLOUD_1_STATUS", "CLOUD_2_STATUS"]:
/usr/local/sbin/skladchik-reports-monitor-supervisor:246: if restore.get(key) != "OK":
/usr/local/sbin/skladchik-reports-monitor-supervisor-247- problems.append(key.lower() + "_bad")
/usr/local/sbin/skladchik-reports-monitor-supervisor:248: for key in ["SHA256_OK", "TAR_OK", "REQUIRED_FILES_OK", "COOKIE_JAR_OK", "EXTENSION_HEALTH_FILES_OK", "EXTENSION_SCRIPTS_OK", "EXTENSION_UNITS_OK", "SQLITE_OK", "PY_COMPILE_OK"]:
/usr/local/sbin/skladchik-reports-monitor-supervisor:249: if restore.get(key) != "yes":
/usr/local/sbin/skladchik-reports-monitor-supervisor-250- problems.append(key.lower() + "_bad")
/usr/local/sbin/skladchik-reports-monitor-supervisor-251-
/usr/local/sbin/skladchik-reports-monitor-supervisor-252- local_units = [
/usr/local/sbin/skladchik-reports-monitor-supervisor-253- "skladchik-reports-monitor-backup.timer",
/usr/local/sbin/skladchik-reports-monitor-supervisor:254: "skladchik-reports-monitor-backup-restore-check.timer",
/usr/local/sbin/skladchik-reports-monitor-supervisor-255- "skladchik-reports-monitor-supervisor.timer",
/usr/local/sbin/skladchik-reports-monitor-supervisor-256- "skladchik-reports-monitor-monthly-selftest-watch.timer",
--
/usr/local/sbin/skladchik-reports-monitor-supervisor-284- f"- возраст health, сек: {facts.get('monitor_health_age_seconds')}",
/usr/local/sbin/skladchik-reports-monitor-supervisor-285- f"- автопродление авторизации: {facts.get('session_keepalive')}",
/usr/local/sbin/skladchik-reports-monitor-supervisor:286: f"- cookie jar: {facts.get('cookie_jar')} / {facts.get('cookie_jar_bytes')} байт",
/usr/local/sbin/skladchik-reports-monitor-supervisor:287: f"- постоянная cookie действует до UTC: {facts.get('cookie_expiry_utc') or 'не определено'}",
/usr/local/sbin/skladchik-reports-monitor-supervisor:288: f"- осталось дней по cookie: {facts.get('cookie_expires_in_days') or 'не определено'}",
/usr/local/sbin/skladchik-reports-monitor-supervisor:289: f"- статус срока cookie: {facts.get('cookie_expiry_status') or 'не определено'}",
/usr/local/sbin/skladchik-reports-monitor-supervisor-290- f"- NetBird route: {facts.get('netbird_route')}",
/usr/local/sbin/skladchik-reports-monitor-supervisor-291- "",
--
/usr/local/sbin/skladchik-reports-monitor-supervisor-294- f"- последний RUN_ID: {facts.get('backup_run_id')}",
/usr/local/sbin/skladchik-reports-monitor-supervisor-295- f"- размер архива, байт: {facts.get('backup_size_bytes')}",
/usr/local/sbin/skladchik-reports-monitor-supervisor:296: f"- cookie jar в бэкапе: {facts.get('backup_cookie_jar_backed_up')}",
/usr/local/sbin/skladchik-reports-monitor-supervisor-297- f"- health-файлы расширений в бэкапе: {facts.get('backup_extension_health_files_backed_up')}",
/usr/local/sbin/skladchik-reports-monitor-supervisor-298- f"- название жалобы в теме письма: {facts.get('backup_title_in_email_subject')}",
/usr/local/sbin/skladchik-reports-monitor-supervisor-299- f"- возраст health, сек: {facts.get('backup_health_age_seconds')}",
/usr/local/sbin/skladchik-reports-monitor-supervisor-300- "",
/usr/local/sbin/skladchik-reports-monitor-supervisor:301: "Restore-check:",
/usr/local/sbin/skladchik-reports-monitor-supervisor:302: f"- статус: {facts.get('restore_status')}",
/usr/local/sbin/skladchik-reports-monitor-supervisor:303: f"- последний RUN_ID: {facts.get('restore_run_id')}",
/usr/local/sbin/skladchik-reports-monitor-supervisor-304- f"- cloud1/cloud2: {facts.get('cloud_1_status')}/{facts.get('cloud_2_status')}",
/usr/local/sbin/skladchik-reports-monitor-supervisor:305: f"- sha256/tar/cookie/sqlite/python: {facts.get('sha256_ok')}/{facts.get('tar_ok')}/{facts.get('cookie_jar_ok')}/{facts.get('sqlite_ok')}/{facts.get('py_compile_ok')}",
/usr/local/sbin/skladchik-reports-monitor-supervisor-306- f"- health/scripts/units расширений: {facts.get('extension_health_files_ok')}/{facts.get('extension_scripts_ok')}/{facts.get('extension_units_ok')}",
/usr/local/sbin/skladchik-reports-monitor-supervisor-307- "",
--
/usr/local/sbin/skladchik-reports-monitor-supervisor-315- "",
/usr/local/sbin/skladchik-reports-monitor-supervisor-316- "Авторизация:",
/usr/local/sbin/skladchik-reports-monitor-supervisor:317: "- ручной ежемесячный перелогин обычно не нужен; монитор сам продлевает сессию постоянным cookie jar при каждом опросе;",
/usr/local/sbin/skladchik-reports-monitor-supervisor:318: "- supervisor теперь отдельно следит за сроком постоянной cookie и заранее предупредит, если до истечения останется меньше 7 дней;",
/usr/local/sbin/skladchik-reports-monitor-supervisor-319- "- если форум принудительно сбросит сессию, придёт отдельная тревога AUTH_REQUIRED; тогда нужно будет разово обновить вход, но это аварийный сценарий, а не ежемесячная рутина.",
/usr/local/sbin/skladchik-reports-monitor-supervisor-320- ])
--
/usr/local/sbin/skladchik-reports-monitor-supervisor-337-
/usr/local/sbin/skladchik-reports-monitor-supervisor-338-def main():
/usr/local/sbin/skladchik-reports-monitor-supervisor:339: mode = sys.argv[1] if len(sys.argv) > 1 else "check"
/usr/local/sbin/skladchik-reports-monitor-supervisor-340- if mode not in ("check", "test"):
/usr/local/sbin/skladchik-reports-monitor-supervisor-341- print("usage: skladchik-reports-monitor-supervisor [check|test]", file=sys.stderr)
--
/usr/local/sbin/skladchik-reports-monitor-supervisor-370- print(f"MONITOR={facts.get('monitor_status')} reports={facts.get('monitor_reports')} open={facts.get('monitor_open')} resolved={facts.get('monitor_resolved')}")
/usr/local/sbin/skladchik-reports-monitor-supervisor-371- print(f"BACKUP={facts.get('backup_status')} run={facts.get('backup_run_id')}")
/usr/local/sbin/skladchik-reports-monitor-supervisor:372: print(f"RESTORE={facts.get('restore_status')} run={facts.get('restore_run_id')}")
/usr/local/sbin/skladchik-reports-monitor-supervisor-373- log(f"CHECK mode={mode} status={status} mail_sent={'yes' if mail_sent else 'no'} problems={problems}")
/usr/local/sbin/skladchik-reports-monitor-supervisor-374- return 0 if status == "OK" else 1
--
/usr/local/sbin/skladchik-reports-monitor-status-37-section "PVE01: таймеры"
/usr/local/sbin/skladchik-reports-monitor-status-38-check_timer skladchik-reports-monitor-backup.timer
/usr/local/sbin/skladchik-reports-monitor-status:39:check_timer skladchik-reports-monitor-backup-restore-check.timer
/usr/local/sbin/skladchik-reports-monitor-status-40-check_timer skladchik-reports-monitor-supervisor.timer
/usr/local/sbin/skladchik-reports-monitor-status-41-check_timer skladchik-reports-monitor-monthly-selftest-watch.timer
/usr/local/sbin/skladchik-reports-monitor-status-42-
/usr/local/sbin/skladchik-reports-monitor-status-43-check_health "PVE01: ежедневный бэкап" "/var/lib/skladchik-reports-monitor-backup/health.txt"
/usr/local/sbin/skladchik-reports-monitor-status:44:check_health "PVE01: проверка восстановления" "/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt"
/usr/local/sbin/skladchik-reports-monitor-status-45-check_health "PVE01: общий supervisor" "/var/lib/skladchik-reports-monitor-supervisor/health.txt"
/usr/local/sbin/skladchik-reports-monitor-status-46-check_health "PVE01: контроль ежемесячного самотеста" "/var/lib/skladchik-reports-monitor-selftest-watch/health.txt"
DETAIL=COOKIE_IMPORT_CONTRACT_END
DETAIL=LOCAL_COOKIE_STATE=ABSENT
Result=exit-code
ExecMainStatus=1
ActiveState=failed
SubState=failed
DETAIL=COOKIE_IMPORT_CONTRACT_AUDIT=PASS
CHANGES_MADE=NO
OUTPUT_END
CHAT_OUTPUT_END