влад
@@ -0,0 +1,265 @@
|
||||
"""Import tab reference PostgreSQL → WESP SQLite (offline admin ETL)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import psycopg2
|
||||
from psycopg2.extras import RealDictCursor
|
||||
|
||||
from app import db
|
||||
from app.lab.models import LabAnimalProfile
|
||||
from app.models import Component, Recipe, WESP_SUPPRESS_SYNC_ENQUEUE
|
||||
from app.lab.services.profile_norms import import_norms_from_legacy_text, save_norms_from_payload
|
||||
|
||||
|
||||
DEFAULT_PG_URL = "postgresql://neoton:neoton_secret@localhost:5432/neoton"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImportStats:
|
||||
components_enriched: int = 0
|
||||
components_unmatched: int = 0
|
||||
tab_components_purged: int = 0
|
||||
animal_profiles: int = 0
|
||||
recipe_links: int = 0
|
||||
errors: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def _pg_url() -> str:
|
||||
return os.environ.get("TAB_REFERENCE_DATABASE_URL", DEFAULT_PG_URL).strip()
|
||||
|
||||
|
||||
def _json_text(value: Any) -> str:
|
||||
if value is None:
|
||||
return "{}"
|
||||
if isinstance(value, str):
|
||||
return value if value else "{}"
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
|
||||
|
||||
def _dt(value: Any) -> datetime | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, datetime):
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _fetch_all(conn, sql: str) -> list[dict[str, Any]]:
|
||||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||||
cur.execute(sql)
|
||||
return list(cur.fetchall())
|
||||
|
||||
|
||||
def _normalize_name(name: str) -> str:
|
||||
return " ".join((name or "").lower().split())
|
||||
|
||||
|
||||
def _find_wesp_component(row: dict[str, Any]) -> Component | None:
|
||||
"""Match tab feed_ingredient → existing WESP component (never create)."""
|
||||
external_no = row.get("external_no")
|
||||
name = (row.get("name") or "").strip()
|
||||
norm = _normalize_name(name)
|
||||
|
||||
if external_no is not None:
|
||||
hit = Component.query.filter_by(external_no=external_no, is_deleted=False).first()
|
||||
if hit is not None:
|
||||
return hit
|
||||
|
||||
if name:
|
||||
hit = Component.query.filter(
|
||||
Component.name == name, Component.is_deleted.is_(False)
|
||||
).first()
|
||||
if hit is not None:
|
||||
return hit
|
||||
|
||||
if norm:
|
||||
for comp in Component.query.filter(Component.is_deleted.is_(False)).all():
|
||||
if _normalize_name(comp.name) == norm:
|
||||
return comp
|
||||
if len(name) >= 12:
|
||||
prefix = name[:20].lower()
|
||||
for comp in Component.query.filter(Component.is_deleted.is_(False)).all():
|
||||
if prefix in (comp.name or "").lower():
|
||||
return comp
|
||||
return None
|
||||
|
||||
|
||||
def _enrich_wesp_component(component: Component, row: dict[str, Any]) -> None:
|
||||
"""Copy zootech nutrients from tab; WESP id/name/dry_matter/price stay canonical."""
|
||||
from app.lab.services.component_nutrients import save_component_nutrients
|
||||
|
||||
raw = row.get("nutrients")
|
||||
if isinstance(raw, str):
|
||||
try:
|
||||
raw = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
raw = {}
|
||||
if not isinstance(raw, dict):
|
||||
raw = {}
|
||||
save_component_nutrients(component.id, raw, user_id="reference-db-import")
|
||||
if row.get("external_no") is not None:
|
||||
component.external_no = row.get("external_no")
|
||||
component.updated_by = "tab-enrich"
|
||||
|
||||
|
||||
def _purge_tab_imported_components(stats: ImportStats) -> None:
|
||||
"""Remove components created from tab feed_ingredients (not used in WESP calc)."""
|
||||
tab_ids = [
|
||||
c.id
|
||||
for c in Component.query.filter(
|
||||
Component.created_by == "tab-import", Component.is_deleted.is_(False)
|
||||
).all()
|
||||
]
|
||||
if not tab_ids:
|
||||
return
|
||||
for comp in Component.query.filter(Component.id.in_(tab_ids)).all():
|
||||
comp.soft_delete("tab-import-cleanup")
|
||||
stats.tab_components_purged += 1
|
||||
|
||||
|
||||
def _import_feed_ingredients(conn, stats: ImportStats) -> None:
|
||||
"""Map tab feed_ingredient → WESP component; enrich nutrients only."""
|
||||
rows = _fetch_all(
|
||||
conn,
|
||||
"""
|
||||
SELECT id, external_no, name, price_per_kg, dry_matter, nutrients, row_index
|
||||
FROM feed_ingredients
|
||||
ORDER BY row_index NULLS LAST, external_no NULLS LAST
|
||||
""",
|
||||
)
|
||||
for row in rows:
|
||||
component = _find_wesp_component(row)
|
||||
if component is None:
|
||||
stats.components_unmatched += 1
|
||||
continue
|
||||
_enrich_wesp_component(component, row)
|
||||
stats.components_enriched += 1
|
||||
|
||||
|
||||
def build_feed_ingredient_mapping(rows: list[dict[str, Any]], stats: ImportStats | None = None) -> dict[str, str | None]:
|
||||
st = stats or ImportStats()
|
||||
mapping: dict[str, str | None] = {}
|
||||
for row in rows:
|
||||
tab_id = str(row.get("id") or "")
|
||||
component = _find_wesp_component(row)
|
||||
if component is None:
|
||||
st.components_unmatched += 1
|
||||
mapping[tab_id] = None
|
||||
continue
|
||||
_enrich_wesp_component(component, row)
|
||||
mapping[tab_id] = component.id
|
||||
st.components_enriched += 1
|
||||
return mapping
|
||||
|
||||
|
||||
def apply_animal_profile_rows(rows: list[dict[str, Any]], stats: ImportStats | None = None) -> ImportStats:
|
||||
"""Импорт профилей из списка строк (PG-формат или fixtures JSON)."""
|
||||
st = stats or ImportStats()
|
||||
for row in rows:
|
||||
_upsert_animal_profile_row(row, st)
|
||||
return st
|
||||
|
||||
|
||||
def _upsert_animal_profile_row(row: dict[str, Any], stats: ImportStats) -> None:
|
||||
profile = LabAnimalProfile.query.get(row["id"])
|
||||
if profile is None:
|
||||
profile = LabAnimalProfile(id=row["id"])
|
||||
db.session.add(profile)
|
||||
profile.profile_key = row["key"]
|
||||
profile.label = row["label"]
|
||||
profile.ration_type = str(row["type"])
|
||||
norms_raw = row.get("norms_data")
|
||||
if isinstance(norms_raw, dict):
|
||||
save_norms_from_payload(profile, norms_raw)
|
||||
else:
|
||||
import_norms_from_legacy_text(profile, norms_raw)
|
||||
profile.created_at = _dt(row.get("created_at")) or profile.created_at
|
||||
profile.updated_at = _dt(row.get("updated_at")) or profile.updated_at
|
||||
profile.created_by = "tab-import"
|
||||
profile.updated_by = "tab-import"
|
||||
stats.animal_profiles += 1
|
||||
|
||||
|
||||
def apply_feed_ingredient_rows(rows: list[dict[str, Any]], stats: ImportStats | None = None) -> ImportStats:
|
||||
"""Обогащение WESP component из строк feed_ingredients (без PG)."""
|
||||
st = stats or ImportStats()
|
||||
for row in rows:
|
||||
component = _find_wesp_component(row)
|
||||
if component is None:
|
||||
st.components_unmatched += 1
|
||||
continue
|
||||
_enrich_wesp_component(component, row)
|
||||
st.components_enriched += 1
|
||||
return st
|
||||
|
||||
|
||||
def _import_animal_profiles(conn, stats: ImportStats) -> None:
|
||||
rows = _fetch_all(
|
||||
conn,
|
||||
"""
|
||||
SELECT id, key, label, type, norms_data, created_at, updated_at
|
||||
FROM animal_profiles
|
||||
ORDER BY key
|
||||
""",
|
||||
)
|
||||
apply_animal_profile_rows(rows, stats)
|
||||
|
||||
|
||||
def _recipe_by_name(name: str) -> Recipe | None:
|
||||
if not name:
|
||||
return None
|
||||
return Recipe.query.filter(Recipe.name == name, Recipe.is_deleted.is_(False)).first()
|
||||
|
||||
|
||||
def _link_recipes_from_tab_projects(conn, stats: ImportStats) -> None:
|
||||
"""Только ration_type на recipe по имени — без staging-таблиц."""
|
||||
rows = _fetch_all(
|
||||
conn,
|
||||
"""
|
||||
SELECT name, type
|
||||
FROM ration_projects
|
||||
ORDER BY updated_at DESC NULLS LAST
|
||||
""",
|
||||
)
|
||||
seen: set[str] = set()
|
||||
for row in rows:
|
||||
name = (row.get("name") or "").strip()
|
||||
if not name or name in seen:
|
||||
continue
|
||||
linked = _recipe_by_name(name)
|
||||
if linked is None:
|
||||
continue
|
||||
if row.get("type"):
|
||||
linked.ration_type = str(row["type"])
|
||||
seen.add(name)
|
||||
stats.recipe_links += 1
|
||||
|
||||
|
||||
def import_from_reference_db(pg_url: str | None = None) -> ImportStats:
|
||||
"""Full dump from tab PostgreSQL into WESP SQLite."""
|
||||
stats = ImportStats()
|
||||
url = pg_url or _pg_url()
|
||||
conn = psycopg2.connect(url)
|
||||
db.session.info[WESP_SUPPRESS_SYNC_ENQUEUE] = True
|
||||
try:
|
||||
_purge_tab_imported_components(stats)
|
||||
db.session.flush()
|
||||
_import_feed_ingredients(conn, stats)
|
||||
db.session.flush()
|
||||
_import_animal_profiles(conn, stats)
|
||||
_link_recipes_from_tab_projects(conn, stats)
|
||||
db.session.commit()
|
||||
except Exception as exc:
|
||||
db.session.rollback()
|
||||
stats.errors.append(str(exc))
|
||||
raise
|
||||
finally:
|
||||
db.session.info.pop(WESP_SUPPRESS_SYNC_ENQUEUE, None)
|
||||
conn.close()
|
||||
return stats
|
||||
Reference in New Issue
Block a user