447 lines
14 KiB
Python
447 lines
14 KiB
Python
"""Журнал событий периферии (HX711, GPIO) и снимок hardware-status для админки."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import logging
|
||
import os
|
||
import threading
|
||
import time
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
from typing import Any, Dict, List, Optional, Tuple
|
||
|
||
_LOG = logging.getLogger(__name__)
|
||
_LOCK = threading.Lock()
|
||
# (component, code) -> monotonic time of last append
|
||
_DEBOUNCE_LAST: Dict[Tuple[str, str], float] = {}
|
||
_HOST_ALERT_LAST: Dict[str, float] = {}
|
||
|
||
|
||
def _peripheral_log_path(app) -> str:
|
||
return (app.config.get("WESP_ADMIN_PERIPHERAL_LOG_PATH") or "").strip()
|
||
|
||
|
||
def _resolve_path(app, path_str: str) -> Path:
|
||
path = Path(path_str)
|
||
if not path.is_absolute():
|
||
path = Path(app.config.get("BASE_DIR", ".")) / path
|
||
return path
|
||
|
||
|
||
def _line_unix_ts(obj: Dict[str, Any]) -> Optional[float]:
|
||
u = obj.get("u")
|
||
if isinstance(u, (int, float)):
|
||
return float(u)
|
||
return None
|
||
|
||
|
||
def _debounce_sec(app) -> float:
|
||
return float(app.config.get("WESP_PERIPHERAL_EVENT_DEBOUNCE_SEC", 30) or 30)
|
||
|
||
|
||
def append_peripheral_event(
|
||
app,
|
||
*,
|
||
component: str,
|
||
code: str,
|
||
level: str = "warn",
|
||
message: str,
|
||
details: Optional[Dict[str, Any]] = None,
|
||
force: bool = False,
|
||
) -> None:
|
||
"""Дописывает событие в JSONL; одинаковые (component, code) не чаще debounce."""
|
||
try:
|
||
_append_peripheral_event_impl(
|
||
app,
|
||
component=component,
|
||
code=code,
|
||
level=level,
|
||
message=message,
|
||
details=details,
|
||
force=force,
|
||
)
|
||
except Exception:
|
||
_LOG.exception("Сбой журнала периферии (WESP_ADMIN_PERIPHERAL_LOG_PATH).")
|
||
|
||
|
||
def _append_peripheral_event_impl(
|
||
app,
|
||
*,
|
||
component: str,
|
||
code: str,
|
||
level: str,
|
||
message: str,
|
||
details: Optional[Dict[str, Any]],
|
||
force: bool,
|
||
) -> None:
|
||
path_str = _peripheral_log_path(app)
|
||
if not path_str:
|
||
return
|
||
|
||
comp = (component or "unknown")[:64]
|
||
cod = (code or "event")[:64]
|
||
lvl = level if level in ("ok", "warn", "err", "info") else "warn"
|
||
|
||
key = (comp, cod)
|
||
now = time.monotonic()
|
||
debounce = _debounce_sec(app)
|
||
with _LOCK:
|
||
prev = _DEBOUNCE_LAST.get(key)
|
||
if (
|
||
not force
|
||
and isinstance(prev, (int, float))
|
||
and (now - float(prev)) < debounce
|
||
):
|
||
return
|
||
_DEBOUNCE_LAST[key] = now
|
||
|
||
path = _resolve_path(app, path_str)
|
||
row: Dict[str, Any] = {
|
||
"u": int(time.time()),
|
||
"component": comp,
|
||
"code": cod,
|
||
"level": lvl,
|
||
"message": (message or "")[:2000],
|
||
}
|
||
if details:
|
||
row["details"] = details
|
||
|
||
line = json.dumps(row, ensure_ascii=False, separators=(",", ":")) + "\n"
|
||
days = int(app.config.get("WESP_ADMIN_PERIPHERAL_LOG_RETENTION_DAYS", 7) or 7)
|
||
days = max(1, min(days, 30))
|
||
cutoff = time.time() - float(days * 86400)
|
||
|
||
with _LOCK:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
_rewrite_prune_and_append(path, cutoff, line)
|
||
|
||
|
||
def _rewrite_prune_and_append(path: Path, cutoff_ts: float, new_line: str) -> None:
|
||
lines_kept: List[str] = []
|
||
if path.is_file():
|
||
try:
|
||
with path.open("r", encoding="utf-8") as f:
|
||
for ln in f:
|
||
raw = ln.strip()
|
||
if not raw:
|
||
continue
|
||
try:
|
||
obj = json.loads(raw)
|
||
except json.JSONDecodeError:
|
||
continue
|
||
ts = _line_unix_ts(obj)
|
||
if ts is not None and ts >= cutoff_ts:
|
||
lines_kept.append(raw + "\n")
|
||
except OSError:
|
||
lines_kept = []
|
||
|
||
tmp = path.with_name(f"{path.name}.tmp.{os.getpid()}")
|
||
try:
|
||
with tmp.open("w", encoding="utf-8") as out:
|
||
for ln in lines_kept:
|
||
out.write(ln)
|
||
out.write(new_line)
|
||
os.replace(str(tmp), str(path))
|
||
except OSError:
|
||
try:
|
||
if tmp.is_file():
|
||
tmp.unlink(missing_ok=True)
|
||
except OSError:
|
||
pass
|
||
|
||
|
||
def read_peripheral_events(
|
||
app,
|
||
limit: int = 80,
|
||
component: Optional[str] = None,
|
||
exclude_component: Optional[str] = None,
|
||
) -> Tuple[List[Dict[str, Any]], str]:
|
||
path_str = _peripheral_log_path(app)
|
||
if not path_str:
|
||
return [], ""
|
||
|
||
path = _resolve_path(app, path_str)
|
||
if not path.is_file():
|
||
return [], str(path)
|
||
|
||
days = int(app.config.get("WESP_ADMIN_PERIPHERAL_LOG_RETENTION_DAYS", 7) or 7)
|
||
days = max(1, min(days, 30))
|
||
cutoff = time.time() - float(days * 86400)
|
||
comp_filter = (component or "").strip().lower() or None
|
||
comp_exclude = (exclude_component or "").strip().lower() or None
|
||
cap = max(1, min(int(limit), 500))
|
||
|
||
out: List[Dict[str, Any]] = []
|
||
try:
|
||
with path.open("r", encoding="utf-8") as f:
|
||
for ln in f:
|
||
raw = ln.strip()
|
||
if not raw:
|
||
continue
|
||
try:
|
||
obj = json.loads(raw)
|
||
except json.JSONDecodeError:
|
||
continue
|
||
if not isinstance(obj, dict):
|
||
continue
|
||
ts = _line_unix_ts(obj)
|
||
if ts is None or ts < cutoff:
|
||
continue
|
||
comp = str(obj.get("component") or "").lower()
|
||
if comp_filter and comp != comp_filter:
|
||
continue
|
||
if comp_exclude and comp == comp_exclude:
|
||
continue
|
||
out.append(obj)
|
||
except OSError:
|
||
return [], str(path)
|
||
|
||
out.sort(key=lambda x: int(x.get("u") or 0))
|
||
return out[-cap:], str(path)
|
||
|
||
|
||
def clear_peripheral_events(
|
||
app,
|
||
*,
|
||
component: Optional[str] = None,
|
||
exclude_component: Optional[str] = None,
|
||
) -> Tuple[bool, str]:
|
||
"""Очищает JSONL-журнал: целиком, по component или всё кроме exclude_component."""
|
||
path_str = _peripheral_log_path(app)
|
||
if not path_str:
|
||
return False, ""
|
||
|
||
path = _resolve_path(app, path_str)
|
||
comp_filter = (component or "").strip().lower() or None
|
||
comp_exclude = (exclude_component or "").strip().lower() or None
|
||
with _LOCK:
|
||
_DEBOUNCE_LAST.clear()
|
||
if not comp_filter and not comp_exclude:
|
||
_HOST_ALERT_LAST.clear()
|
||
try:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
if not comp_filter and not comp_exclude:
|
||
path.write_text("", encoding="utf-8")
|
||
elif not path.is_file():
|
||
path.write_text("", encoding="utf-8")
|
||
else:
|
||
kept: List[str] = []
|
||
with path.open("r", encoding="utf-8") as f:
|
||
for ln in f:
|
||
raw = ln.strip()
|
||
if not raw:
|
||
continue
|
||
line = ln if ln.endswith("\n") else ln + "\n"
|
||
try:
|
||
obj = json.loads(raw)
|
||
except json.JSONDecodeError:
|
||
if comp_exclude:
|
||
kept.append(line)
|
||
continue
|
||
if not isinstance(obj, dict):
|
||
if comp_exclude:
|
||
kept.append(line)
|
||
continue
|
||
comp = str(obj.get("component") or "").lower()
|
||
if comp_filter:
|
||
if comp != comp_filter:
|
||
kept.append(line)
|
||
elif comp_exclude:
|
||
if comp == comp_exclude:
|
||
kept.append(line)
|
||
else:
|
||
kept.append(line)
|
||
path.write_text("".join(kept), encoding="utf-8")
|
||
except OSError as exc:
|
||
raise OSError(f"Не удалось очистить журнал: {exc}") from exc
|
||
return True, str(path)
|
||
|
||
|
||
def _collect_host_alerts(app, metrics: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||
"""Пороги CPU/RAM/диск; при срабатывании пишет в журнал (с debounce)."""
|
||
if not metrics.get("available"):
|
||
return []
|
||
|
||
alerts: List[Dict[str, Any]] = []
|
||
cpu_thr = float(app.config.get("WESP_HOST_ALERT_CPU_PCT", 90) or 90)
|
||
ram_thr = float(app.config.get("WESP_HOST_ALERT_RAM_PCT", 90) or 90)
|
||
disk_thr = float(app.config.get("WESP_HOST_ALERT_DISK_PCT", 90) or 90)
|
||
|
||
cpu = metrics.get("cpu") if isinstance(metrics.get("cpu"), dict) else {}
|
||
mem = metrics.get("memory") if isinstance(metrics.get("memory"), dict) else {}
|
||
disk = metrics.get("disk") if isinstance(metrics.get("disk"), dict) else {}
|
||
|
||
checks = [
|
||
("host_cpu_high", "cpu", float(cpu.get("percent") or 0), cpu_thr, "CPU"),
|
||
("host_ram_high", "memory", float(mem.get("percent") or 0), ram_thr, "RAM"),
|
||
("host_disk_high", "disk", float(disk.get("percent") or 0), disk_thr, "Диск /"),
|
||
]
|
||
|
||
now = time.monotonic()
|
||
debounce = _debounce_sec(app)
|
||
|
||
for code, component, pct, thr, label in checks:
|
||
ok = pct < thr
|
||
alerts.append(
|
||
{
|
||
"code": code,
|
||
"component": "host",
|
||
"ok": ok,
|
||
"percent": round(pct, 1),
|
||
"threshold": thr,
|
||
"label": label,
|
||
}
|
||
)
|
||
if ok:
|
||
continue
|
||
with _LOCK:
|
||
prev = _HOST_ALERT_LAST.get(code)
|
||
if isinstance(prev, (int, float)) and (now - float(prev)) < debounce:
|
||
continue
|
||
_HOST_ALERT_LAST[code] = now
|
||
append_peripheral_event(
|
||
app,
|
||
component="host",
|
||
code=code,
|
||
level="warn",
|
||
message=f"{label}: загрузка {pct:.1f}% (порог {thr:.0f}%)",
|
||
details={"percent": pct, "threshold": thr},
|
||
force=True,
|
||
)
|
||
|
||
return alerts
|
||
|
||
|
||
def _scales_snapshot_from_reader(reader) -> Dict[str, Any]:
|
||
health = reader.get_scale_health()
|
||
debug = reader.get_scale_debug_snapshot()
|
||
sim = reader.get_simulation_state()
|
||
payload = {
|
||
"weight_kg": reader.get_current_weight(),
|
||
**health,
|
||
**debug,
|
||
**sim,
|
||
}
|
||
return payload
|
||
|
||
|
||
def _effective_simulation_mode(app) -> bool:
|
||
try:
|
||
from app.routes.scales import _get_reader
|
||
|
||
return bool(_get_reader().get_simulation_state().get("simulation_mode"))
|
||
except Exception:
|
||
return bool(app.config.get("SIMULATION_MODE", False))
|
||
|
||
|
||
def diagnostics_hardware_block(app) -> Dict[str, Any]:
|
||
"""Текущее состояние драйверов GPIO/HX711 (без опроса весов)."""
|
||
from app.services.hardware import GPIOController, HX711Wrapper
|
||
from app.services.hardware_settings_service import is_scale_hardware_platform
|
||
|
||
cfg = app.config
|
||
simulation = _effective_simulation_mode(app)
|
||
scale_hw = is_scale_hardware_platform()
|
||
pin = int(cfg.get("GPIO_LED_PIN", 18))
|
||
dout = int(cfg.get("HX711_DOUT_PIN", 2))
|
||
pd_sck = int(cfg.get("HX711_PD_SCK_PIN", 3))
|
||
ctrl = GPIOController.get_instance(pin=pin, simulation_mode=simulation or not scale_hw)
|
||
wrapper = HX711Wrapper(simulation_mode=simulation or not scale_hw, dout_pin=dout, pd_sck_pin=pd_sck)
|
||
rpi = bool(ctrl.rpi_gpio_available)
|
||
hx = bool(getattr(wrapper, "_device", None) is not None)
|
||
drivers_absent = (not simulation) and scale_hw and (not rpi) and (not hx)
|
||
return {
|
||
"simulation_mode": simulation,
|
||
"scale_hardware_platform": scale_hw,
|
||
"gpio_led_pin": pin,
|
||
"hx711_dout_pin": dout,
|
||
"hx711_pd_sck_pin": pd_sck,
|
||
"rpi_gpio_available": rpi,
|
||
"hx711_driver_ready": hx,
|
||
"drivers_absent": drivers_absent,
|
||
}
|
||
|
||
|
||
def _peripherals_block(app, scales_error: Optional[str] = None) -> Dict[str, Any]:
|
||
block = diagnostics_hardware_block(app)
|
||
if scales_error:
|
||
block["scales_reader_error"] = scales_error
|
||
block["last_hx711_error"] = None
|
||
try:
|
||
from app.routes.scales import _get_reader
|
||
|
||
reader = _get_reader()
|
||
block["last_hx711_error"] = reader.get_last_hx711_error()
|
||
except Exception as exc:
|
||
block["scales_reader_error"] = str(exc)
|
||
return block
|
||
|
||
|
||
def build_hardware_status(app) -> Dict[str, Any]:
|
||
from app.services.admin_system_metrics import (
|
||
collect_host_hardware_overview,
|
||
collect_system_metrics,
|
||
)
|
||
|
||
metrics = collect_system_metrics()
|
||
host_alerts = _collect_host_alerts(app, metrics)
|
||
|
||
scales: Dict[str, Any] = {
|
||
"available": False,
|
||
"error": None,
|
||
}
|
||
try:
|
||
from app.routes.scales import _get_reader
|
||
|
||
reader = _get_reader()
|
||
scales = {"available": True, **_scales_snapshot_from_reader(reader)}
|
||
except Exception as exc:
|
||
scales = {"available": False, "error": str(exc)}
|
||
|
||
peripherals = _peripherals_block(app, scales.get("error"))
|
||
|
||
return {
|
||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||
"machine": collect_host_hardware_overview(),
|
||
"metrics_available": bool(metrics.get("available")),
|
||
"host_alerts": host_alerts,
|
||
"peripherals": peripherals,
|
||
"scales": scales,
|
||
"config": {
|
||
"simulation_mode": _effective_simulation_mode(app),
|
||
"read_interval": float(app.config.get("READ_INTERVAL", 0.05)),
|
||
"samples_per_read": int(app.config.get("SAMPLES_PER_READ", 3)),
|
||
"hx711_dout_pin": int(app.config.get("HX711_DOUT_PIN", 2)),
|
||
"hx711_pd_sck_pin": int(app.config.get("HX711_PD_SCK_PIN", 3)),
|
||
"gpio_led_pin": int(app.config.get("GPIO_LED_PIN", 18)),
|
||
},
|
||
}
|
||
|
||
|
||
def notify_hx711_error(app, message: str) -> None:
|
||
append_peripheral_event(
|
||
app,
|
||
component="hx711",
|
||
code="hx711_read_error",
|
||
level="err",
|
||
message=message,
|
||
)
|
||
|
||
|
||
def notify_hx711_recovered(app) -> None:
|
||
append_peripheral_event(
|
||
app,
|
||
component="hx711",
|
||
code="hx711_recovered",
|
||
level="ok",
|
||
message="Чтение HX711 восстановлено",
|
||
force=True,
|
||
)
|
||
|
||
|
||
def reset_debounce_state_for_tests() -> None:
|
||
with _LOCK:
|
||
_DEBOUNCE_LAST.clear()
|
||
_HOST_ALERT_LAST.clear()
|