543 lines
17 KiB
Python
543 lines
17 KiB
Python
"""Агрегат observability для админки: GET /api/admin/sync-diagnostics."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
import os
|
||
from datetime import datetime, timedelta
|
||
from typing import Any, Dict, List, Optional
|
||
|
||
from flask import Flask
|
||
from sqlalchemy import func, select
|
||
|
||
from app import db
|
||
from app.models import (
|
||
SyncClient,
|
||
SyncConflict,
|
||
SyncDelivery,
|
||
SyncEngineState,
|
||
SyncMetadata,
|
||
SyncQueue,
|
||
)
|
||
from app.services.sync_error_display import sync_error_text_for_display
|
||
from app.services.sync_manager import build_consistency_snapshot
|
||
from app.services.sync_runtime import sync_runtime
|
||
from config import Config, read_sync_client_state, resolve_effective_sync_role, resolve_sync_server_url
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
def _effective_sync_connection(app: Flask) -> dict:
|
||
state = read_sync_client_state()
|
||
role_env = os.getenv("WESP_SYNC_ROLE", "").strip()
|
||
url_env = os.getenv("WESP_SYNC_SERVER_URL", "").strip()
|
||
role, role_locked = resolve_effective_sync_role(state=state, config_defaults=app.config)
|
||
server_url, explicit = resolve_sync_server_url(state=state, config_defaults=app.config)
|
||
return {
|
||
"role": role,
|
||
"server_url": server_url,
|
||
"sync_server_url_explicit": bool(explicit),
|
||
"role_locked_by_env": bool(role_locked),
|
||
"server_url_locked_by_env": bool(url_env),
|
||
"env_role": role_env or None,
|
||
"env_server_url": url_env or None,
|
||
"sync_client_autostart": bool(app.config.get("SYNC_CLIENT_AUTOSTART", True)),
|
||
}
|
||
|
||
|
||
def _sanitize_client_state(state: dict) -> dict:
|
||
return {
|
||
k: v
|
||
for k, v in (state or {}).items()
|
||
if k not in {"token", "auth_token", "password", "secret"}
|
||
}
|
||
|
||
|
||
def _recent_queue_errors(limit: int = 25) -> list[dict]:
|
||
order_col = func.coalesce(
|
||
SyncQueue.processed_at,
|
||
SyncQueue.updated_at,
|
||
SyncQueue.created_at,
|
||
)
|
||
rows = db.session.execute(
|
||
select(SyncQueue)
|
||
.where(SyncQueue.is_deleted.is_(False))
|
||
.where(SyncQueue.status == "failed")
|
||
.order_by(order_col.desc())
|
||
.limit(400)
|
||
).scalars().all()
|
||
out: list[dict] = []
|
||
for r in rows:
|
||
disp = sync_error_text_for_display(r.error_message)
|
||
if not disp:
|
||
continue
|
||
ts = r.processed_at or r.updated_at or r.created_at
|
||
err = disp if len(disp) <= 4000 else disp[:4000] + "…"
|
||
out.append(
|
||
{
|
||
"task_id": r.id,
|
||
"table_name": r.table_name,
|
||
"record_id": r.record_id,
|
||
"action": r.action,
|
||
"error": err,
|
||
"at": ts.isoformat() if ts else None,
|
||
}
|
||
)
|
||
if len(out) >= limit:
|
||
break
|
||
return out
|
||
|
||
|
||
def _queue_stats_full() -> dict:
|
||
base = select(SyncQueue).where(SyncQueue.is_deleted.is_(False)).subquery()
|
||
stats = {
|
||
"total": int(db.session.scalar(select(func.count()).select_from(base)) or 0),
|
||
"pending": int(
|
||
db.session.scalar(
|
||
select(func.count()).select_from(base).where(base.c.status == "pending")
|
||
)
|
||
or 0
|
||
),
|
||
"processing": int(
|
||
db.session.scalar(
|
||
select(func.count()).select_from(base).where(base.c.status == "processing")
|
||
)
|
||
or 0
|
||
),
|
||
"completed": int(
|
||
db.session.scalar(
|
||
select(func.count()).select_from(base).where(base.c.status == "completed")
|
||
)
|
||
or 0
|
||
),
|
||
"failed": int(
|
||
db.session.scalar(
|
||
select(func.count()).select_from(base).where(base.c.status == "failed")
|
||
)
|
||
or 0
|
||
),
|
||
"by_priority": {},
|
||
}
|
||
for p in range(1, 6):
|
||
stats["by_priority"][f"priority_{p}"] = int(
|
||
db.session.scalar(
|
||
select(func.count()).select_from(base).where(base.c.priority == p)
|
||
)
|
||
or 0
|
||
)
|
||
return stats
|
||
|
||
|
||
def _queue_by_table_and_status() -> Dict[str, Dict[str, int]]:
|
||
rows = db.session.execute(
|
||
select(SyncQueue.table_name, SyncQueue.status, func.count())
|
||
.where(SyncQueue.is_deleted.is_(False))
|
||
.group_by(SyncQueue.table_name, SyncQueue.status)
|
||
).all()
|
||
out: Dict[str, Dict[str, int]] = {}
|
||
for table_name, status, cnt in rows:
|
||
t = str(table_name or "")
|
||
s = str(status or "")
|
||
out.setdefault(t, {})[s] = int(cnt or 0)
|
||
return out
|
||
|
||
|
||
def _serialize_task(r: SyncQueue) -> dict:
|
||
return {
|
||
"id": r.id,
|
||
"table_name": r.table_name,
|
||
"record_id": r.record_id,
|
||
"action": r.action,
|
||
"status": r.status,
|
||
"target_node_id": r.target_node_id,
|
||
"source_node_id": r.source_node_id,
|
||
"priority": r.priority,
|
||
"retry_count": r.retry_count,
|
||
"max_retries": r.max_retries,
|
||
"error_message": (r.error_message or "")[:2000],
|
||
"created_at": r.created_at.isoformat() if r.created_at else None,
|
||
"processed_at": r.processed_at.isoformat() if r.processed_at else None,
|
||
"completed_at": r.completed_at.isoformat() if r.completed_at else None,
|
||
}
|
||
|
||
|
||
def _stuck_processing_tasks(timeout_minutes: int) -> List[dict]:
|
||
cutoff = datetime.now() - timedelta(minutes=max(1, int(timeout_minutes)))
|
||
rows = (
|
||
db.session.execute(
|
||
select(SyncQueue)
|
||
.where(
|
||
SyncQueue.is_deleted.is_(False),
|
||
SyncQueue.status == "processing",
|
||
SyncQueue.processed_at.is_not(None),
|
||
SyncQueue.processed_at < cutoff,
|
||
)
|
||
.order_by(SyncQueue.processed_at.asc())
|
||
.limit(80)
|
||
)
|
||
.scalars()
|
||
.all()
|
||
)
|
||
return [_serialize_task(r) | {"stuck_reason": "processing_timeout"} for r in rows]
|
||
|
||
|
||
def _processing_without_timestamp() -> List[dict]:
|
||
rows = (
|
||
db.session.execute(
|
||
select(SyncQueue)
|
||
.where(
|
||
SyncQueue.is_deleted.is_(False),
|
||
SyncQueue.status == "processing",
|
||
SyncQueue.processed_at.is_(None),
|
||
)
|
||
.order_by(SyncQueue.created_at.asc())
|
||
.limit(40)
|
||
)
|
||
.scalars()
|
||
.all()
|
||
)
|
||
return [_serialize_task(r) | {"stuck_reason": "processing_missing_processed_at"} for r in rows]
|
||
|
||
|
||
def _failed_recent(limit: int = 40) -> List[dict]:
|
||
order_col = func.coalesce(SyncQueue.updated_at, SyncQueue.processed_at, SyncQueue.created_at)
|
||
rows = (
|
||
db.session.execute(
|
||
select(SyncQueue)
|
||
.where(SyncQueue.is_deleted.is_(False), SyncQueue.status == "failed")
|
||
.order_by(order_col.desc())
|
||
.limit(limit)
|
||
)
|
||
.scalars()
|
||
.all()
|
||
)
|
||
return [_serialize_task(r) for r in rows]
|
||
|
||
|
||
def _high_retry_pending(limit: int = 40) -> List[dict]:
|
||
rows = (
|
||
db.session.execute(
|
||
select(SyncQueue)
|
||
.where(
|
||
SyncQueue.is_deleted.is_(False),
|
||
SyncQueue.status == "pending",
|
||
SyncQueue.retry_count > 0,
|
||
)
|
||
.order_by(SyncQueue.retry_count.desc(), SyncQueue.updated_at.desc())
|
||
.limit(limit)
|
||
)
|
||
.scalars()
|
||
.all()
|
||
)
|
||
return [_serialize_task(r) for r in rows]
|
||
|
||
|
||
def _conflicts_summary() -> dict:
|
||
conflicts_base = select(SyncConflict).where(SyncConflict.is_deleted.is_(False)).subquery()
|
||
return {
|
||
"total": int(db.session.scalar(select(func.count()).select_from(conflicts_base)) or 0),
|
||
"pending": int(
|
||
db.session.scalar(
|
||
select(func.count())
|
||
.select_from(conflicts_base)
|
||
.where(
|
||
(conflicts_base.c.resolution == "pending")
|
||
| (conflicts_base.c.resolution.is_(None))
|
||
)
|
||
)
|
||
or 0
|
||
),
|
||
"resolved": int(
|
||
db.session.scalar(
|
||
select(func.count())
|
||
.select_from(conflicts_base)
|
||
.where(
|
||
(conflicts_base.c.resolution != "pending")
|
||
& (conflicts_base.c.resolution.is_not(None))
|
||
)
|
||
)
|
||
or 0
|
||
),
|
||
}
|
||
|
||
|
||
def _recent_conflicts(limit: int = 15) -> List[dict]:
|
||
rows = (
|
||
db.session.execute(
|
||
select(SyncConflict)
|
||
.where(SyncConflict.is_deleted.is_(False))
|
||
.order_by(SyncConflict.created_at.desc())
|
||
.limit(limit)
|
||
)
|
||
.scalars()
|
||
.all()
|
||
)
|
||
out = []
|
||
for c in rows:
|
||
out.append(
|
||
{
|
||
"id": c.id,
|
||
"table_name": c.table_name,
|
||
"record_id": c.record_id,
|
||
"conflict_type": c.conflict_type,
|
||
"resolution": c.resolution,
|
||
"created_at": c.created_at.isoformat() if c.created_at else None,
|
||
}
|
||
)
|
||
return out
|
||
|
||
|
||
def _clients_registry(limit: int = 25) -> List[dict]:
|
||
rows = (
|
||
db.session.execute(
|
||
select(SyncClient)
|
||
.where(SyncClient.is_deleted.is_(False))
|
||
.order_by(SyncClient.last_seen.desc().nullslast(), SyncClient.updated_at.desc())
|
||
.limit(limit)
|
||
)
|
||
.scalars()
|
||
.all()
|
||
)
|
||
out = []
|
||
for c in rows:
|
||
out.append(
|
||
{
|
||
"node_id": c.node_id,
|
||
"client_name": c.client_name,
|
||
"status": c.status,
|
||
"ip_address": c.ip_address,
|
||
"port": c.port,
|
||
"last_seen": c.last_seen.isoformat() if c.last_seen else None,
|
||
"total_syncs": c.total_syncs,
|
||
"last_error": sync_error_text_for_display(c.last_error),
|
||
"is_enabled": c.is_enabled,
|
||
"personal_snapshot_completed_at": c.personal_snapshot_completed_at.isoformat()
|
||
if c.personal_snapshot_completed_at
|
||
else None,
|
||
"personal_snapshot_cursor": c.personal_snapshot_cursor,
|
||
}
|
||
)
|
||
return out
|
||
|
||
|
||
def _recent_deliveries(limit: int = 20) -> List[dict]:
|
||
rows = (
|
||
db.session.execute(
|
||
select(SyncDelivery)
|
||
.where(SyncDelivery.is_deleted.is_(False))
|
||
.order_by(SyncDelivery.delivered_at.desc())
|
||
.limit(limit)
|
||
)
|
||
.scalars()
|
||
.all()
|
||
)
|
||
return [
|
||
{
|
||
"id": d.id,
|
||
"client_id": d.client_id,
|
||
"task_id": d.task_id,
|
||
"delivered_at": d.delivered_at.isoformat() if d.delivered_at else None,
|
||
}
|
||
for d in rows
|
||
]
|
||
|
||
|
||
def _engine_state_row() -> Optional[dict]:
|
||
row = db.session.get(SyncEngineState, 1)
|
||
if row is None:
|
||
return None
|
||
return {
|
||
"universal_bootstrap_completed_at": row.universal_bootstrap_completed_at.isoformat()
|
||
if row.universal_bootstrap_completed_at
|
||
else None,
|
||
"universal_bootstrap_cursor": row.universal_bootstrap_cursor,
|
||
"universal_bootstrap_last_error": (row.universal_bootstrap_last_error or "")[:2000],
|
||
}
|
||
|
||
|
||
def _sync_metadata_row() -> Optional[dict]:
|
||
row = db.session.execute(
|
||
select(SyncMetadata).where(SyncMetadata.is_deleted.is_(False))
|
||
).scalar_one_or_none()
|
||
if row is None:
|
||
return None
|
||
return {
|
||
"node_id": row.node_id,
|
||
"node_type": row.node_type,
|
||
"sync_status": row.sync_status,
|
||
"last_sync": row.last_sync.isoformat() if row.last_sync else None,
|
||
"last_heartbeat": row.last_heartbeat.isoformat() if row.last_heartbeat else None,
|
||
}
|
||
|
||
|
||
def _report_push_local_snapshot(app: Flask) -> dict:
|
||
"""Локальная очередь push (справочники + отчёты на узле role=client)."""
|
||
conn = _effective_sync_connection(app)
|
||
role = str(conn.get("role") or "server").strip().lower()
|
||
if role != "client":
|
||
return {"scope": "server", "note": "Очередь push живёт на узлах с role=client."}
|
||
|
||
try:
|
||
from sync_client import REPORT_PUSH_TABLES, client_push_tables
|
||
|
||
tables = tuple(sorted(client_push_tables()))
|
||
report_only = tuple(sorted(REPORT_PUSH_TABLES))
|
||
except Exception:
|
||
tables = ()
|
||
report_only = ()
|
||
|
||
if not tables:
|
||
return {"scope": "client", "error": "client_push_tables недоступен"}
|
||
|
||
try:
|
||
rows = db.session.execute(
|
||
select(SyncQueue.status, func.count())
|
||
.where(
|
||
SyncQueue.is_deleted.is_(False),
|
||
SyncQueue.table_name.in_(tables),
|
||
)
|
||
.group_by(SyncQueue.status)
|
||
).all()
|
||
except Exception as exc:
|
||
logger.debug("report_push snapshot: %s", exc, exc_info=True)
|
||
return {"scope": "client", "error": str(exc)}
|
||
|
||
by_status = {str(s or ""): int(n or 0) for s, n in rows}
|
||
total = sum(by_status.values())
|
||
return {
|
||
"scope": "client",
|
||
"tables": list(tables),
|
||
"report_tables": list(report_only),
|
||
"by_status": by_status,
|
||
"total": total,
|
||
}
|
||
|
||
|
||
def _initial_sync_progress_safe() -> dict:
|
||
try:
|
||
from sync_client import get_initial_sync_progress_snapshot
|
||
|
||
return get_initial_sync_progress_snapshot()
|
||
except Exception:
|
||
logger.debug("initial sync progress недоступен", exc_info=True)
|
||
return {"active": False}
|
||
|
||
|
||
def _sync_client_runtime_flags() -> dict:
|
||
try:
|
||
import sync_client as sc
|
||
|
||
c = getattr(sc, "sync_client", None)
|
||
if c is None:
|
||
return {"initialized": False}
|
||
th = getattr(c, "sync_thread", None)
|
||
alive = bool(th and getattr(th, "is_alive", lambda: False)())
|
||
return {
|
||
"initialized": True,
|
||
"client_thread_alive": alive,
|
||
}
|
||
except Exception:
|
||
return {"initialized": False, "error": "sync_client import failed"}
|
||
|
||
|
||
def build_sync_diagnostics_payload(app: Flask) -> dict:
|
||
generated_at = datetime.utcnow().isoformat()
|
||
conn = _effective_sync_connection(app)
|
||
effective_role = str(conn.get("role") or "server").strip() or "server"
|
||
state_safe = _sanitize_client_state(read_sync_client_state())
|
||
timeout_min = int(getattr(Config, "SYNC_REQUEUE_TIMEOUT_MINUTES", 15))
|
||
|
||
overview = {
|
||
"effective_role": effective_role,
|
||
"connection": conn,
|
||
"client_state": state_safe,
|
||
"first_sync_banner": bool(
|
||
effective_role == "client" and not state_safe.get("first_bootstrap_done", True)
|
||
),
|
||
"config": {
|
||
"sync_max_concurrent": int(app.config.get("SYNC_MAX_CONCURRENT", Config.SYNC_MAX_CONCURRENT)),
|
||
"sync_requeue_timeout_minutes": timeout_min,
|
||
"sync_requeue_interval_sec": int(
|
||
getattr(Config, "SYNC_REQUEUE_INTERVAL_SEC", 30)
|
||
),
|
||
"sync_background_requeue": bool(getattr(Config, "SYNC_BACKGROUND_REQUEUE", True)),
|
||
"client_poll_interval_sec": int(
|
||
app.config.get("SYNC_CLIENT_POLL_INTERVAL_SEC", 14)
|
||
),
|
||
"client_poll_interval_initial_sec": int(
|
||
app.config.get("SYNC_CLIENT_POLL_INTERVAL_INITIAL_SEC", 14)
|
||
),
|
||
"client_pull_limit_steady": int(
|
||
app.config.get("SYNC_CLIENT_DEFAULT_PULL_LIMIT", 100)
|
||
),
|
||
"client_pull_limit_initial": int(
|
||
app.config.get("SYNC_CLIENT_PULL_LIMIT_INITIAL", 50)
|
||
),
|
||
},
|
||
"metadata": _sync_metadata_row(),
|
||
}
|
||
|
||
runtime = sync_runtime.diagnostics_snapshot()
|
||
|
||
queues = {
|
||
"stats": _queue_stats_full(),
|
||
"by_table_status": _queue_by_table_and_status(),
|
||
"stuck_processing": _stuck_processing_tasks(timeout_min),
|
||
"processing_missing_processed_at": _processing_without_timestamp(),
|
||
"failed_recent": _failed_recent(),
|
||
"high_retry_pending": _high_retry_pending(),
|
||
"recent_errors": _recent_queue_errors(25),
|
||
"requeue_policy": {
|
||
"timeout_minutes": timeout_min,
|
||
"matches_requeue_stuck_processing": True,
|
||
},
|
||
}
|
||
|
||
bootstrap = {
|
||
"engine_state": _engine_state_row(),
|
||
"initial_sync_progress": _initial_sync_progress_safe(),
|
||
}
|
||
|
||
report_push = _report_push_local_snapshot(app)
|
||
report_push["client_runtime"] = _sync_client_runtime_flags()
|
||
|
||
clients = {
|
||
"registry": _clients_registry(30),
|
||
"recent_deliveries": _recent_deliveries(25),
|
||
}
|
||
|
||
conflicts = {
|
||
"summary": _conflicts_summary(),
|
||
"recent": _recent_conflicts(20),
|
||
}
|
||
|
||
consistency: dict = {"snapshot": None, "error": None}
|
||
try:
|
||
consistency["snapshot"] = build_consistency_snapshot()
|
||
except Exception as exc:
|
||
logger.warning("build_consistency_snapshot failed: %s", exc, exc_info=True)
|
||
consistency["error"] = str(exc)
|
||
|
||
testing = bool(app.config.get("TESTING"))
|
||
autostart = bool(app.config.get("SYNC_CLIENT_AUTOSTART", True))
|
||
can_restart = (not testing) and autostart and effective_role == "client"
|
||
actions_capabilities = {
|
||
"requeue_stuck": True,
|
||
"restart_local_sync": bool(can_restart),
|
||
"refresh_diagnostics": True,
|
||
"retry_report_push": bool(effective_role == "client" and not testing),
|
||
}
|
||
|
||
return {
|
||
"generated_at": generated_at,
|
||
"overview": overview,
|
||
"runtime": runtime,
|
||
"queues": queues,
|
||
"bootstrap": bootstrap,
|
||
"report_push": report_push,
|
||
"clients": clients,
|
||
"conflicts": conflicts,
|
||
"consistency": consistency,
|
||
"actions_capabilities": actions_capabilities,
|
||
}
|