Интегрирован wesp в сайт

2026-07-17 12:57:18 +03:00
parent 5dfa06ddbe
commit 355c0ef9f1
883 changed files with 194576 additions and 177 deletions
@@ -0,0 +1,542 @@
+"""Агрегат observability для админки: GET /api/admin/sync-diagnostics."""
+
+from __future__ import annotations
+
+import logging
+import os
+from datetime import datetime, timedelta
+from typing import Any, Dict, List, Optional
+
+from flask import Flask
+from sqlalchemy import func, select
+
+from app import db
+from app.models import (
+    SyncClient,
+    SyncConflict,
+    SyncDelivery,
+    SyncEngineState,
+    SyncMetadata,
+    SyncQueue,
+)
+from app.services.sync_error_display import sync_error_text_for_display
+from app.services.sync_manager import build_consistency_snapshot
+from app.services.sync_runtime import sync_runtime
+from config import Config, read_sync_client_state, resolve_effective_sync_role, resolve_sync_server_url
+
+logger = logging.getLogger(__name__)
+
+
+def _effective_sync_connection(app: Flask) -> dict:
+    state = read_sync_client_state()
+    role_env = os.getenv("WESP_SYNC_ROLE", "").strip()
+    url_env = os.getenv("WESP_SYNC_SERVER_URL", "").strip()
+    role, role_locked = resolve_effective_sync_role(state=state, config_defaults=app.config)
+    server_url, explicit = resolve_sync_server_url(state=state, config_defaults=app.config)
+    return {
+        "role": role,
+        "server_url": server_url,
+        "sync_server_url_explicit": bool(explicit),
+        "role_locked_by_env": bool(role_locked),
+        "server_url_locked_by_env": bool(url_env),
+        "env_role": role_env or None,
+        "env_server_url": url_env or None,
+        "sync_client_autostart": bool(app.config.get("SYNC_CLIENT_AUTOSTART", True)),
+    }
+
+
+def _sanitize_client_state(state: dict) -> dict:
+    return {
+        k: v
+        for k, v in (state or {}).items()
+        if k not in {"token", "auth_token", "password", "secret"}
+    }
+
+
+def _recent_queue_errors(limit: int = 25) -> list[dict]:
+    order_col = func.coalesce(
+        SyncQueue.processed_at,
+        SyncQueue.updated_at,
+        SyncQueue.created_at,
+    )
+    rows = db.session.execute(
+        select(SyncQueue)
+        .where(SyncQueue.is_deleted.is_(False))
+        .where(SyncQueue.status == "failed")
+        .order_by(order_col.desc())
+        .limit(400)
+    ).scalars().all()
+    out: list[dict] = []
+    for r in rows:
+        disp = sync_error_text_for_display(r.error_message)
+        if not disp:
+            continue
+        ts = r.processed_at or r.updated_at or r.created_at
+        err = disp if len(disp) <= 4000 else disp[:4000] + "…"
+        out.append(
+            {
+                "task_id": r.id,
+                "table_name": r.table_name,
+                "record_id": r.record_id,
+                "action": r.action,
+                "error": err,
+                "at": ts.isoformat() if ts else None,
+            }
+        )
+        if len(out) >= limit:
+            break
+    return out
+
+
+def _queue_stats_full() -> dict:
+    base = select(SyncQueue).where(SyncQueue.is_deleted.is_(False)).subquery()
+    stats = {
+        "total": int(db.session.scalar(select(func.count()).select_from(base)) or 0),
+        "pending": int(
+            db.session.scalar(
+                select(func.count()).select_from(base).where(base.c.status == "pending")
+            )
+            or 0
+        ),
+        "processing": int(
+            db.session.scalar(
+                select(func.count()).select_from(base).where(base.c.status == "processing")
+            )
+            or 0
+        ),
+        "completed": int(
+            db.session.scalar(
+                select(func.count()).select_from(base).where(base.c.status == "completed")
+            )
+            or 0
+        ),
+        "failed": int(
+            db.session.scalar(
+                select(func.count()).select_from(base).where(base.c.status == "failed")
+            )
+            or 0
+        ),
+        "by_priority": {},
+    }
+    for p in range(1, 6):
+        stats["by_priority"][f"priority_{p}"] = int(
+            db.session.scalar(
+                select(func.count()).select_from(base).where(base.c.priority == p)
+            )
+            or 0
+        )
+    return stats
+
+
+def _queue_by_table_and_status() -> Dict[str, Dict[str, int]]:
+    rows = db.session.execute(
+        select(SyncQueue.table_name, SyncQueue.status, func.count())
+        .where(SyncQueue.is_deleted.is_(False))
+        .group_by(SyncQueue.table_name, SyncQueue.status)
+    ).all()
+    out: Dict[str, Dict[str, int]] = {}
+    for table_name, status, cnt in rows:
+        t = str(table_name or "")
+        s = str(status or "")
+        out.setdefault(t, {})[s] = int(cnt or 0)
+    return out
+
+
+def _serialize_task(r: SyncQueue) -> dict:
+    return {
+        "id": r.id,
+        "table_name": r.table_name,
+        "record_id": r.record_id,
+        "action": r.action,
+        "status": r.status,
+        "target_node_id": r.target_node_id,
+        "source_node_id": r.source_node_id,
+        "priority": r.priority,
+        "retry_count": r.retry_count,
+        "max_retries": r.max_retries,
+        "error_message": (r.error_message or "")[:2000],
+        "created_at": r.created_at.isoformat() if r.created_at else None,
+        "processed_at": r.processed_at.isoformat() if r.processed_at else None,
+        "completed_at": r.completed_at.isoformat() if r.completed_at else None,
+    }
+
+
+def _stuck_processing_tasks(timeout_minutes: int) -> List[dict]:
+    cutoff = datetime.now() - timedelta(minutes=max(1, int(timeout_minutes)))
+    rows = (
+        db.session.execute(
+            select(SyncQueue)
+            .where(
+                SyncQueue.is_deleted.is_(False),
+                SyncQueue.status == "processing",
+                SyncQueue.processed_at.is_not(None),
+                SyncQueue.processed_at < cutoff,
+            )
+            .order_by(SyncQueue.processed_at.asc())
+            .limit(80)
+        )
+        .scalars()
+        .all()
+    )
+    return [_serialize_task(r) | {"stuck_reason": "processing_timeout"} for r in rows]
+
+
+def _processing_without_timestamp() -> List[dict]:
+    rows = (
+        db.session.execute(
+            select(SyncQueue)
+            .where(
+                SyncQueue.is_deleted.is_(False),
+                SyncQueue.status == "processing",
+                SyncQueue.processed_at.is_(None),
+            )
+            .order_by(SyncQueue.created_at.asc())
+            .limit(40)
+        )
+        .scalars()
+        .all()
+    )
+    return [_serialize_task(r) | {"stuck_reason": "processing_missing_processed_at"} for r in rows]
+
+
+def _failed_recent(limit: int = 40) -> List[dict]:
+    order_col = func.coalesce(SyncQueue.updated_at, SyncQueue.processed_at, SyncQueue.created_at)
+    rows = (
+        db.session.execute(
+            select(SyncQueue)
+            .where(SyncQueue.is_deleted.is_(False), SyncQueue.status == "failed")
+            .order_by(order_col.desc())
+            .limit(limit)
+        )
+        .scalars()
+        .all()
+    )
+    return [_serialize_task(r) for r in rows]
+
+
+def _high_retry_pending(limit: int = 40) -> List[dict]:
+    rows = (
+        db.session.execute(
+            select(SyncQueue)
+            .where(
+                SyncQueue.is_deleted.is_(False),
+                SyncQueue.status == "pending",
+                SyncQueue.retry_count > 0,
+            )
+            .order_by(SyncQueue.retry_count.desc(), SyncQueue.updated_at.desc())
+            .limit(limit)
+        )
+        .scalars()
+        .all()
+    )
+    return [_serialize_task(r) for r in rows]
+
+
+def _conflicts_summary() -> dict:
+    conflicts_base = select(SyncConflict).where(SyncConflict.is_deleted.is_(False)).subquery()
+    return {
+        "total": int(db.session.scalar(select(func.count()).select_from(conflicts_base)) or 0),
+        "pending": int(
+            db.session.scalar(
+                select(func.count())
+                .select_from(conflicts_base)
+                .where(
+                    (conflicts_base.c.resolution == "pending")
+                    | (conflicts_base.c.resolution.is_(None))
+                )
+            )
+            or 0
+        ),
+        "resolved": int(
+            db.session.scalar(
+                select(func.count())
+                .select_from(conflicts_base)
+                .where(
+                    (conflicts_base.c.resolution != "pending")
+                    & (conflicts_base.c.resolution.is_not(None))
+                )
+            )
+            or 0
+        ),
+    }
+
+
+def _recent_conflicts(limit: int = 15) -> List[dict]:
+    rows = (
+        db.session.execute(
+            select(SyncConflict)
+            .where(SyncConflict.is_deleted.is_(False))
+            .order_by(SyncConflict.created_at.desc())
+            .limit(limit)
+        )
+        .scalars()
+        .all()
+    )
+    out = []
+    for c in rows:
+        out.append(
+            {
+                "id": c.id,
+                "table_name": c.table_name,
+                "record_id": c.record_id,
+                "conflict_type": c.conflict_type,
+                "resolution": c.resolution,
+                "created_at": c.created_at.isoformat() if c.created_at else None,
+            }
+        )
+    return out
+
+
+def _clients_registry(limit: int = 25) -> List[dict]:
+    rows = (
+        db.session.execute(
+            select(SyncClient)
+            .where(SyncClient.is_deleted.is_(False))
+            .order_by(SyncClient.last_seen.desc().nullslast(), SyncClient.updated_at.desc())
+            .limit(limit)
+        )
+        .scalars()
+        .all()
+    )
+    out = []
+    for c in rows:
+        out.append(
+            {
+                "node_id": c.node_id,
+                "client_name": c.client_name,
+                "status": c.status,
+                "ip_address": c.ip_address,
+                "port": c.port,
+                "last_seen": c.last_seen.isoformat() if c.last_seen else None,
+                "total_syncs": c.total_syncs,
+                "last_error": sync_error_text_for_display(c.last_error),
+                "is_enabled": c.is_enabled,
+                "personal_snapshot_completed_at": c.personal_snapshot_completed_at.isoformat()
+                if c.personal_snapshot_completed_at
+                else None,
+                "personal_snapshot_cursor": c.personal_snapshot_cursor,
+            }
+        )
+    return out
+
+
+def _recent_deliveries(limit: int = 20) -> List[dict]:
+    rows = (
+        db.session.execute(
+            select(SyncDelivery)
+            .where(SyncDelivery.is_deleted.is_(False))
+            .order_by(SyncDelivery.delivered_at.desc())
+            .limit(limit)
+        )
+        .scalars()
+        .all()
+    )
+    return [
+        {
+            "id": d.id,
+            "client_id": d.client_id,
+            "task_id": d.task_id,
+            "delivered_at": d.delivered_at.isoformat() if d.delivered_at else None,
+        }
+        for d in rows
+    ]
+
+
+def _engine_state_row() -> Optional[dict]:
+    row = db.session.get(SyncEngineState, 1)
+    if row is None:
+        return None
+    return {
+        "universal_bootstrap_completed_at": row.universal_bootstrap_completed_at.isoformat()
+        if row.universal_bootstrap_completed_at
+        else None,
+        "universal_bootstrap_cursor": row.universal_bootstrap_cursor,
+        "universal_bootstrap_last_error": (row.universal_bootstrap_last_error or "")[:2000],
+    }
+
+
+def _sync_metadata_row() -> Optional[dict]:
+    row = db.session.execute(
+        select(SyncMetadata).where(SyncMetadata.is_deleted.is_(False))
+    ).scalar_one_or_none()
+    if row is None:
+        return None
+    return {
+        "node_id": row.node_id,
+        "node_type": row.node_type,
+        "sync_status": row.sync_status,
+        "last_sync": row.last_sync.isoformat() if row.last_sync else None,
+        "last_heartbeat": row.last_heartbeat.isoformat() if row.last_heartbeat else None,
+    }
+
+
+def _report_push_local_snapshot(app: Flask) -> dict:
+    """Локальная очередь push (справочники + отчёты на узле role=client)."""
+    conn = _effective_sync_connection(app)
+    role = str(conn.get("role") or "server").strip().lower()
+    if role != "client":
+        return {"scope": "server", "note": "Очередь push живёт на узлах с role=client."}
+
+    try:
+        from sync_client import REPORT_PUSH_TABLES, client_push_tables
+
+        tables = tuple(sorted(client_push_tables()))
+        report_only = tuple(sorted(REPORT_PUSH_TABLES))
+    except Exception:
+        tables = ()
+        report_only = ()
+
+    if not tables:
+        return {"scope": "client", "error": "client_push_tables недоступен"}
+
+    try:
+        rows = db.session.execute(
+            select(SyncQueue.status, func.count())
+            .where(
+                SyncQueue.is_deleted.is_(False),
+                SyncQueue.table_name.in_(tables),
+            )
+            .group_by(SyncQueue.status)
+        ).all()
+    except Exception as exc:
+        logger.debug("report_push snapshot: %s", exc, exc_info=True)
+        return {"scope": "client", "error": str(exc)}
+
+    by_status = {str(s or ""): int(n or 0) for s, n in rows}
+    total = sum(by_status.values())
+    return {
+        "scope": "client",
+        "tables": list(tables),
+        "report_tables": list(report_only),
+        "by_status": by_status,
+        "total": total,
+    }
+
+
+def _initial_sync_progress_safe() -> dict:
+    try:
+        from sync_client import get_initial_sync_progress_snapshot
+
+        return get_initial_sync_progress_snapshot()
+    except Exception:
+        logger.debug("initial sync progress недоступен", exc_info=True)
+        return {"active": False}
+
+
+def _sync_client_runtime_flags() -> dict:
+    try:
+        import sync_client as sc
+
+        c = getattr(sc, "sync_client", None)
+        if c is None:
+            return {"initialized": False}
+        th = getattr(c, "sync_thread", None)
+        alive = bool(th and getattr(th, "is_alive", lambda: False)())
+        return {
+            "initialized": True,
+            "client_thread_alive": alive,
+        }
+    except Exception:
+        return {"initialized": False, "error": "sync_client import failed"}
+
+
+def build_sync_diagnostics_payload(app: Flask) -> dict:
+    generated_at = datetime.utcnow().isoformat()
+    conn = _effective_sync_connection(app)
+    effective_role = str(conn.get("role") or "server").strip() or "server"
+    state_safe = _sanitize_client_state(read_sync_client_state())
+    timeout_min = int(getattr(Config, "SYNC_REQUEUE_TIMEOUT_MINUTES", 15))
+
+    overview = {
+        "effective_role": effective_role,
+        "connection": conn,
+        "client_state": state_safe,
+        "first_sync_banner": bool(
+            effective_role == "client" and not state_safe.get("first_bootstrap_done", True)
+        ),
+        "config": {
+            "sync_max_concurrent": int(app.config.get("SYNC_MAX_CONCURRENT", Config.SYNC_MAX_CONCURRENT)),
+            "sync_requeue_timeout_minutes": timeout_min,
+            "sync_requeue_interval_sec": int(
+                getattr(Config, "SYNC_REQUEUE_INTERVAL_SEC", 30)
+            ),
+            "sync_background_requeue": bool(getattr(Config, "SYNC_BACKGROUND_REQUEUE", True)),
+            "client_poll_interval_sec": int(
+                app.config.get("SYNC_CLIENT_POLL_INTERVAL_SEC", 14)
+            ),
+            "client_poll_interval_initial_sec": int(
+                app.config.get("SYNC_CLIENT_POLL_INTERVAL_INITIAL_SEC", 14)
+            ),
+            "client_pull_limit_steady": int(
+                app.config.get("SYNC_CLIENT_DEFAULT_PULL_LIMIT", 100)
+            ),
+            "client_pull_limit_initial": int(
+                app.config.get("SYNC_CLIENT_PULL_LIMIT_INITIAL", 50)
+            ),
+        },
+        "metadata": _sync_metadata_row(),
+    }
+
+    runtime = sync_runtime.diagnostics_snapshot()
+
+    queues = {
+        "stats": _queue_stats_full(),
+        "by_table_status": _queue_by_table_and_status(),
+        "stuck_processing": _stuck_processing_tasks(timeout_min),
+        "processing_missing_processed_at": _processing_without_timestamp(),
+        "failed_recent": _failed_recent(),
+        "high_retry_pending": _high_retry_pending(),
+        "recent_errors": _recent_queue_errors(25),
+        "requeue_policy": {
+            "timeout_minutes": timeout_min,
+            "matches_requeue_stuck_processing": True,
+        },
+    }
+
+    bootstrap = {
+        "engine_state": _engine_state_row(),
+        "initial_sync_progress": _initial_sync_progress_safe(),
+    }
+
+    report_push = _report_push_local_snapshot(app)
+    report_push["client_runtime"] = _sync_client_runtime_flags()
+
+    clients = {
+        "registry": _clients_registry(30),
+        "recent_deliveries": _recent_deliveries(25),
+    }
+
+    conflicts = {
+        "summary": _conflicts_summary(),
+        "recent": _recent_conflicts(20),
+    }
+
+    consistency: dict = {"snapshot": None, "error": None}
+    try:
+        consistency["snapshot"] = build_consistency_snapshot()
+    except Exception as exc:
+        logger.warning("build_consistency_snapshot failed: %s", exc, exc_info=True)
+        consistency["error"] = str(exc)
+
+    testing = bool(app.config.get("TESTING"))
+    autostart = bool(app.config.get("SYNC_CLIENT_AUTOSTART", True))
+    can_restart = (not testing) and autostart and effective_role == "client"
+    actions_capabilities = {
+        "requeue_stuck": True,
+        "restart_local_sync": bool(can_restart),
+        "refresh_diagnostics": True,
+        "retry_report_push": bool(effective_role == "client" and not testing),
+    }
+
+    return {
+        "generated_at": generated_at,
+        "overview": overview,
+        "runtime": runtime,
+        "queues": queues,
+        "bootstrap": bootstrap,
+        "report_push": report_push,
+        "clients": clients,
+        "conflicts": conflicts,
+        "consistency": consistency,
+        "actions_capabilities": actions_capabilities,
+    }