"""Агрегат observability для админки: GET /api/admin/sync-diagnostics.""" from __future__ import annotations import logging import os from datetime import datetime, timedelta from typing import Any, Dict, List, Optional from flask import Flask from sqlalchemy import func, select from app import db from app.models import ( SyncClient, SyncConflict, SyncDelivery, SyncEngineState, SyncMetadata, SyncQueue, ) from app.services.sync_error_display import sync_error_text_for_display from app.services.sync_manager import build_consistency_snapshot from app.services.sync_runtime import sync_runtime from config import Config, read_sync_client_state, resolve_effective_sync_role, resolve_sync_server_url logger = logging.getLogger(__name__) def _effective_sync_connection(app: Flask) -> dict: state = read_sync_client_state() role_env = os.getenv("WESP_SYNC_ROLE", "").strip() url_env = os.getenv("WESP_SYNC_SERVER_URL", "").strip() role, role_locked = resolve_effective_sync_role(state=state, config_defaults=app.config) server_url, explicit = resolve_sync_server_url(state=state, config_defaults=app.config) return { "role": role, "server_url": server_url, "sync_server_url_explicit": bool(explicit), "role_locked_by_env": bool(role_locked), "server_url_locked_by_env": bool(url_env), "env_role": role_env or None, "env_server_url": url_env or None, "sync_client_autostart": bool(app.config.get("SYNC_CLIENT_AUTOSTART", True)), } def _sanitize_client_state(state: dict) -> dict: return { k: v for k, v in (state or {}).items() if k not in {"token", "auth_token", "password", "secret"} } def _recent_queue_errors(limit: int = 25) -> list[dict]: order_col = func.coalesce( SyncQueue.processed_at, SyncQueue.updated_at, SyncQueue.created_at, ) rows = db.session.execute( select(SyncQueue) .where(SyncQueue.is_deleted.is_(False)) .where(SyncQueue.status == "failed") .order_by(order_col.desc()) .limit(400) ).scalars().all() out: list[dict] = [] for r in rows: disp = sync_error_text_for_display(r.error_message) if not disp: continue ts = r.processed_at or r.updated_at or r.created_at err = disp if len(disp) <= 4000 else disp[:4000] + "…" out.append( { "task_id": r.id, "table_name": r.table_name, "record_id": r.record_id, "action": r.action, "error": err, "at": ts.isoformat() if ts else None, } ) if len(out) >= limit: break return out def _queue_stats_full() -> dict: base = select(SyncQueue).where(SyncQueue.is_deleted.is_(False)).subquery() stats = { "total": int(db.session.scalar(select(func.count()).select_from(base)) or 0), "pending": int( db.session.scalar( select(func.count()).select_from(base).where(base.c.status == "pending") ) or 0 ), "processing": int( db.session.scalar( select(func.count()).select_from(base).where(base.c.status == "processing") ) or 0 ), "completed": int( db.session.scalar( select(func.count()).select_from(base).where(base.c.status == "completed") ) or 0 ), "failed": int( db.session.scalar( select(func.count()).select_from(base).where(base.c.status == "failed") ) or 0 ), "by_priority": {}, } for p in range(1, 6): stats["by_priority"][f"priority_{p}"] = int( db.session.scalar( select(func.count()).select_from(base).where(base.c.priority == p) ) or 0 ) return stats def _queue_by_table_and_status() -> Dict[str, Dict[str, int]]: rows = db.session.execute( select(SyncQueue.table_name, SyncQueue.status, func.count()) .where(SyncQueue.is_deleted.is_(False)) .group_by(SyncQueue.table_name, SyncQueue.status) ).all() out: Dict[str, Dict[str, int]] = {} for table_name, status, cnt in rows: t = str(table_name or "") s = str(status or "") out.setdefault(t, {})[s] = int(cnt or 0) return out def _serialize_task(r: SyncQueue) -> dict: return { "id": r.id, "table_name": r.table_name, "record_id": r.record_id, "action": r.action, "status": r.status, "target_node_id": r.target_node_id, "source_node_id": r.source_node_id, "priority": r.priority, "retry_count": r.retry_count, "max_retries": r.max_retries, "error_message": (r.error_message or "")[:2000], "created_at": r.created_at.isoformat() if r.created_at else None, "processed_at": r.processed_at.isoformat() if r.processed_at else None, "completed_at": r.completed_at.isoformat() if r.completed_at else None, } def _stuck_processing_tasks(timeout_minutes: int) -> List[dict]: cutoff = datetime.now() - timedelta(minutes=max(1, int(timeout_minutes))) rows = ( db.session.execute( select(SyncQueue) .where( SyncQueue.is_deleted.is_(False), SyncQueue.status == "processing", SyncQueue.processed_at.is_not(None), SyncQueue.processed_at < cutoff, ) .order_by(SyncQueue.processed_at.asc()) .limit(80) ) .scalars() .all() ) return [_serialize_task(r) | {"stuck_reason": "processing_timeout"} for r in rows] def _processing_without_timestamp() -> List[dict]: rows = ( db.session.execute( select(SyncQueue) .where( SyncQueue.is_deleted.is_(False), SyncQueue.status == "processing", SyncQueue.processed_at.is_(None), ) .order_by(SyncQueue.created_at.asc()) .limit(40) ) .scalars() .all() ) return [_serialize_task(r) | {"stuck_reason": "processing_missing_processed_at"} for r in rows] def _failed_recent(limit: int = 40) -> List[dict]: order_col = func.coalesce(SyncQueue.updated_at, SyncQueue.processed_at, SyncQueue.created_at) rows = ( db.session.execute( select(SyncQueue) .where(SyncQueue.is_deleted.is_(False), SyncQueue.status == "failed") .order_by(order_col.desc()) .limit(limit) ) .scalars() .all() ) return [_serialize_task(r) for r in rows] def _high_retry_pending(limit: int = 40) -> List[dict]: rows = ( db.session.execute( select(SyncQueue) .where( SyncQueue.is_deleted.is_(False), SyncQueue.status == "pending", SyncQueue.retry_count > 0, ) .order_by(SyncQueue.retry_count.desc(), SyncQueue.updated_at.desc()) .limit(limit) ) .scalars() .all() ) return [_serialize_task(r) for r in rows] def _conflicts_summary() -> dict: conflicts_base = select(SyncConflict).where(SyncConflict.is_deleted.is_(False)).subquery() return { "total": int(db.session.scalar(select(func.count()).select_from(conflicts_base)) or 0), "pending": int( db.session.scalar( select(func.count()) .select_from(conflicts_base) .where( (conflicts_base.c.resolution == "pending") | (conflicts_base.c.resolution.is_(None)) ) ) or 0 ), "resolved": int( db.session.scalar( select(func.count()) .select_from(conflicts_base) .where( (conflicts_base.c.resolution != "pending") & (conflicts_base.c.resolution.is_not(None)) ) ) or 0 ), } def _recent_conflicts(limit: int = 15) -> List[dict]: rows = ( db.session.execute( select(SyncConflict) .where(SyncConflict.is_deleted.is_(False)) .order_by(SyncConflict.created_at.desc()) .limit(limit) ) .scalars() .all() ) out = [] for c in rows: out.append( { "id": c.id, "table_name": c.table_name, "record_id": c.record_id, "conflict_type": c.conflict_type, "resolution": c.resolution, "created_at": c.created_at.isoformat() if c.created_at else None, } ) return out def _clients_registry(limit: int = 25) -> List[dict]: rows = ( db.session.execute( select(SyncClient) .where(SyncClient.is_deleted.is_(False)) .order_by(SyncClient.last_seen.desc().nullslast(), SyncClient.updated_at.desc()) .limit(limit) ) .scalars() .all() ) out = [] for c in rows: out.append( { "node_id": c.node_id, "client_name": c.client_name, "status": c.status, "ip_address": c.ip_address, "port": c.port, "last_seen": c.last_seen.isoformat() if c.last_seen else None, "total_syncs": c.total_syncs, "last_error": sync_error_text_for_display(c.last_error), "is_enabled": c.is_enabled, "personal_snapshot_completed_at": c.personal_snapshot_completed_at.isoformat() if c.personal_snapshot_completed_at else None, "personal_snapshot_cursor": c.personal_snapshot_cursor, } ) return out def _recent_deliveries(limit: int = 20) -> List[dict]: rows = ( db.session.execute( select(SyncDelivery) .where(SyncDelivery.is_deleted.is_(False)) .order_by(SyncDelivery.delivered_at.desc()) .limit(limit) ) .scalars() .all() ) return [ { "id": d.id, "client_id": d.client_id, "task_id": d.task_id, "delivered_at": d.delivered_at.isoformat() if d.delivered_at else None, } for d in rows ] def _engine_state_row() -> Optional[dict]: row = db.session.get(SyncEngineState, 1) if row is None: return None return { "universal_bootstrap_completed_at": row.universal_bootstrap_completed_at.isoformat() if row.universal_bootstrap_completed_at else None, "universal_bootstrap_cursor": row.universal_bootstrap_cursor, "universal_bootstrap_last_error": (row.universal_bootstrap_last_error or "")[:2000], } def _sync_metadata_row() -> Optional[dict]: row = db.session.execute( select(SyncMetadata).where(SyncMetadata.is_deleted.is_(False)) ).scalar_one_or_none() if row is None: return None return { "node_id": row.node_id, "node_type": row.node_type, "sync_status": row.sync_status, "last_sync": row.last_sync.isoformat() if row.last_sync else None, "last_heartbeat": row.last_heartbeat.isoformat() if row.last_heartbeat else None, } def _report_push_local_snapshot(app: Flask) -> dict: """Локальная очередь push (справочники + отчёты на узле role=client).""" conn = _effective_sync_connection(app) role = str(conn.get("role") or "server").strip().lower() if role != "client": return {"scope": "server", "note": "Очередь push живёт на узлах с role=client."} try: from sync_client import REPORT_PUSH_TABLES, client_push_tables tables = tuple(sorted(client_push_tables())) report_only = tuple(sorted(REPORT_PUSH_TABLES)) except Exception: tables = () report_only = () if not tables: return {"scope": "client", "error": "client_push_tables недоступен"} try: rows = db.session.execute( select(SyncQueue.status, func.count()) .where( SyncQueue.is_deleted.is_(False), SyncQueue.table_name.in_(tables), ) .group_by(SyncQueue.status) ).all() except Exception as exc: logger.debug("report_push snapshot: %s", exc, exc_info=True) return {"scope": "client", "error": str(exc)} by_status = {str(s or ""): int(n or 0) for s, n in rows} total = sum(by_status.values()) return { "scope": "client", "tables": list(tables), "report_tables": list(report_only), "by_status": by_status, "total": total, } def _initial_sync_progress_safe() -> dict: try: from sync_client import get_initial_sync_progress_snapshot return get_initial_sync_progress_snapshot() except Exception: logger.debug("initial sync progress недоступен", exc_info=True) return {"active": False} def _sync_client_runtime_flags() -> dict: try: import sync_client as sc c = getattr(sc, "sync_client", None) if c is None: return {"initialized": False} th = getattr(c, "sync_thread", None) alive = bool(th and getattr(th, "is_alive", lambda: False)()) return { "initialized": True, "client_thread_alive": alive, } except Exception: return {"initialized": False, "error": "sync_client import failed"} def build_sync_diagnostics_payload(app: Flask) -> dict: generated_at = datetime.utcnow().isoformat() conn = _effective_sync_connection(app) effective_role = str(conn.get("role") or "server").strip() or "server" state_safe = _sanitize_client_state(read_sync_client_state()) timeout_min = int(getattr(Config, "SYNC_REQUEUE_TIMEOUT_MINUTES", 15)) overview = { "effective_role": effective_role, "connection": conn, "client_state": state_safe, "first_sync_banner": bool( effective_role == "client" and not state_safe.get("first_bootstrap_done", True) ), "config": { "sync_max_concurrent": int(app.config.get("SYNC_MAX_CONCURRENT", Config.SYNC_MAX_CONCURRENT)), "sync_requeue_timeout_minutes": timeout_min, "sync_requeue_interval_sec": int( getattr(Config, "SYNC_REQUEUE_INTERVAL_SEC", 30) ), "sync_background_requeue": bool(getattr(Config, "SYNC_BACKGROUND_REQUEUE", True)), "client_poll_interval_sec": int( app.config.get("SYNC_CLIENT_POLL_INTERVAL_SEC", 14) ), "client_poll_interval_initial_sec": int( app.config.get("SYNC_CLIENT_POLL_INTERVAL_INITIAL_SEC", 14) ), "client_pull_limit_steady": int( app.config.get("SYNC_CLIENT_DEFAULT_PULL_LIMIT", 100) ), "client_pull_limit_initial": int( app.config.get("SYNC_CLIENT_PULL_LIMIT_INITIAL", 50) ), }, "metadata": _sync_metadata_row(), } runtime = sync_runtime.diagnostics_snapshot() queues = { "stats": _queue_stats_full(), "by_table_status": _queue_by_table_and_status(), "stuck_processing": _stuck_processing_tasks(timeout_min), "processing_missing_processed_at": _processing_without_timestamp(), "failed_recent": _failed_recent(), "high_retry_pending": _high_retry_pending(), "recent_errors": _recent_queue_errors(25), "requeue_policy": { "timeout_minutes": timeout_min, "matches_requeue_stuck_processing": True, }, } bootstrap = { "engine_state": _engine_state_row(), "initial_sync_progress": _initial_sync_progress_safe(), } report_push = _report_push_local_snapshot(app) report_push["client_runtime"] = _sync_client_runtime_flags() clients = { "registry": _clients_registry(30), "recent_deliveries": _recent_deliveries(25), } conflicts = { "summary": _conflicts_summary(), "recent": _recent_conflicts(20), } consistency: dict = {"snapshot": None, "error": None} try: consistency["snapshot"] = build_consistency_snapshot() except Exception as exc: logger.warning("build_consistency_snapshot failed: %s", exc, exc_info=True) consistency["error"] = str(exc) testing = bool(app.config.get("TESTING")) autostart = bool(app.config.get("SYNC_CLIENT_AUTOSTART", True)) can_restart = (not testing) and autostart and effective_role == "client" actions_capabilities = { "requeue_stuck": True, "restart_local_sync": bool(can_restart), "refresh_diagnostics": True, "retry_report_push": bool(effective_role == "client" and not testing), } return { "generated_at": generated_at, "overview": overview, "runtime": runtime, "queues": queues, "bootstrap": bootstrap, "report_push": report_push, "clients": clients, "conflicts": conflicts, "consistency": consistency, "actions_capabilities": actions_capabilities, }