300 lines
8.9 KiB
Python
300 lines
8.9 KiB
Python
"""Generic ETL: CSV из data/seed/ → lab profiles и component nutrients."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import csv
|
||
import logging
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path
|
||
|
||
from app import db
|
||
from app.lab.calc.ingredient_catalog import INGREDIENT_HEADERS
|
||
from app.lab.calc.ingredient_derive import derive_ingredient_nutrients
|
||
from app.lab.models import LabAnimalProfile
|
||
from app.lab.norm_catalog import norm_header, norm_title_to_key
|
||
from app.lab.seed_paths import norms_dir, nutrients_dir
|
||
from app.lab.services.component_nutrients import save_component_nutrients
|
||
from app.lab.services.profile_norms import save_norms_from_payload
|
||
from app.models.base import default_uuid
|
||
from app.models.component import Component
|
||
|
||
_log = logging.getLogger(__name__)
|
||
|
||
_SKIP_NUTRIENT_HEADERS = frozenset({"№", "Наименование", "Цена 1 кг"})
|
||
|
||
|
||
@dataclass
|
||
class NormsImportStats:
|
||
profiles_created: int = 0
|
||
profiles_updated: int = 0
|
||
norm_rows: int = 0
|
||
skipped: int = 0
|
||
unmapped_headers: list[str] = field(default_factory=list)
|
||
errors: list[str] = field(default_factory=list)
|
||
|
||
|
||
@dataclass
|
||
class NutrientsImportStats:
|
||
imported: int = 0
|
||
skipped: int = 0
|
||
unmatched: int = 0
|
||
errors: list[str] = field(default_factory=list)
|
||
|
||
|
||
def _parse_min_max_groups(h1: list[str], h2: list[str]) -> list[tuple[str, int, int | None]]:
|
||
groups: list[tuple[str, int, int | None]] = []
|
||
current_name: str | None = None
|
||
limit = min(len(h1), len(h2))
|
||
for i in range(limit):
|
||
b = (h2[i] or "").strip().lower().replace("і", "и")
|
||
n = (h1[i] or "").strip()
|
||
if n and n not in ("№", "Наименование", "Масса", "КРС"):
|
||
current_name = norm_header(n)
|
||
if b.startswith("мин") and current_name:
|
||
max_i = i + 1 if i + 1 < limit and "мах" in (h2[i + 1] or "").lower() else None
|
||
groups.append((current_name, i, max_i))
|
||
return groups
|
||
|
||
|
||
def norm_profile_key(ration_type: str, external_no: int) -> str:
|
||
return f"norm_{ration_type.lower()}_{external_no:04d}"
|
||
|
||
|
||
def parse_norm_profile_row(
|
||
row: list[str],
|
||
groups: list[tuple[str, int, int | None]],
|
||
ration_type: str,
|
||
*,
|
||
unmapped: set[str],
|
||
profile_key_fn=norm_profile_key,
|
||
) -> dict | None:
|
||
if len(row) < 2 or not (row[1] or "").strip():
|
||
return None
|
||
try:
|
||
external_no = int(float((row[0] or "").strip()))
|
||
except (ValueError, TypeError):
|
||
return None
|
||
if external_no <= 0:
|
||
return None
|
||
|
||
indicators: dict[str, dict[str, float | None]] = {}
|
||
for header, min_i, max_i in groups:
|
||
key = norm_title_to_key(header)
|
||
if not key:
|
||
unmapped.add(header)
|
||
continue
|
||
|
||
def _num(idx: int | None) -> float | None:
|
||
if idx is None or idx >= len(row):
|
||
return None
|
||
raw = (row[idx] or "").strip()
|
||
if not raw:
|
||
return None
|
||
try:
|
||
return float(raw)
|
||
except ValueError:
|
||
return None
|
||
|
||
min_v, max_v = _num(min_i), _num(max_i)
|
||
if min_v is None and max_v is None:
|
||
continue
|
||
indicators[key] = {"min": min_v, "max": max_v}
|
||
|
||
mass_kg = None
|
||
if len(row) > 2 and (row[2] or "").strip():
|
||
try:
|
||
mass_kg = float(row[2])
|
||
except ValueError:
|
||
pass
|
||
|
||
return {
|
||
"external_no": external_no,
|
||
"profile_key": profile_key_fn(ration_type, external_no),
|
||
"label": row[1].strip(),
|
||
"ration_type": ration_type,
|
||
"mass_kg": mass_kg,
|
||
"indicators": indicators,
|
||
}
|
||
|
||
|
||
_IMPORT_BATCH_SIZE = 50
|
||
|
||
|
||
def _lookup_reference_profile(profile_key: str) -> LabAnimalProfile | None:
|
||
"""Поиск по profile_key включая soft-deleted (UNIQUE на всю таблицу)."""
|
||
return LabAnimalProfile.query.filter_by(profile_key=profile_key).first()
|
||
|
||
|
||
def _restore_reference_profile(profile: LabAnimalProfile) -> None:
|
||
if not profile.is_deleted:
|
||
return
|
||
profile.is_deleted = False
|
||
profile.deleted_at = None
|
||
profile.deleted_by = None
|
||
|
||
|
||
def _import_norm_sheet(
|
||
csv_path: Path,
|
||
ration_type: str,
|
||
stats: NormsImportStats,
|
||
unmapped: set[str],
|
||
*,
|
||
replace_reference: bool = True,
|
||
) -> None:
|
||
if not csv_path.is_file():
|
||
stats.errors.append(f"Файл не найден: {csv_path}")
|
||
return
|
||
|
||
with csv_path.open(encoding="utf-8") as f:
|
||
rows = list(csv.reader(f))
|
||
if len(rows) < 8:
|
||
stats.errors.append(f"Слишком мало строк: {csv_path}")
|
||
return
|
||
|
||
groups = _parse_min_max_groups(rows[2], rows[3])
|
||
batch_count = 0
|
||
for row in rows[6:]:
|
||
parsed = parse_norm_profile_row(row, groups, ration_type, unmapped=unmapped)
|
||
if parsed is None:
|
||
stats.skipped += 1
|
||
continue
|
||
|
||
profile = _lookup_reference_profile(parsed["profile_key"])
|
||
if profile is not None:
|
||
if not replace_reference and not profile.is_deleted:
|
||
stats.skipped += 1
|
||
continue
|
||
if not replace_reference and profile.is_deleted:
|
||
stats.skipped += 1
|
||
continue
|
||
_restore_reference_profile(profile)
|
||
stats.profiles_updated += 1
|
||
else:
|
||
profile = LabAnimalProfile(
|
||
id=default_uuid(),
|
||
profile_key=parsed["profile_key"],
|
||
created_by="seed-import",
|
||
)
|
||
db.session.add(profile)
|
||
stats.profiles_created += 1
|
||
|
||
profile.label = parsed["label"]
|
||
profile.ration_type = ration_type
|
||
profile.external_no = parsed["external_no"]
|
||
save_norms_from_payload(
|
||
profile,
|
||
{
|
||
"massKg": parsed["mass_kg"],
|
||
"externalNo": parsed["external_no"],
|
||
"indicators": parsed["indicators"],
|
||
},
|
||
)
|
||
stats.norm_rows += len(parsed["indicators"])
|
||
batch_count += 1
|
||
if batch_count >= _IMPORT_BATCH_SIZE:
|
||
db.session.commit()
|
||
batch_count = 0
|
||
|
||
|
||
def import_norm_profiles(
|
||
csv_dir: Path | None = None,
|
||
*,
|
||
replace_reference: bool = True,
|
||
) -> NormsImportStats:
|
||
stats = NormsImportStats()
|
||
base = csv_dir or norms_dir()
|
||
unmapped: set[str] = set()
|
||
|
||
_import_norm_sheet(
|
||
base / "Нормы КРС.csv", "BEEF", stats, unmapped, replace_reference=replace_reference
|
||
)
|
||
_import_norm_sheet(
|
||
base / "Нормы Дойн.csv", "DAIRY", stats, unmapped, replace_reference=replace_reference
|
||
)
|
||
stats.unmapped_headers = sorted(unmapped)
|
||
db.session.commit()
|
||
|
||
_log.info(
|
||
"seed norms import: created=%s updated=%s norm_rows=%s",
|
||
stats.profiles_created,
|
||
stats.profiles_updated,
|
||
stats.norm_rows,
|
||
)
|
||
return stats
|
||
|
||
|
||
def _parse_nutrient_row(row: list[str]) -> dict[str, float]:
|
||
nutrients: dict[str, float] = {}
|
||
for i, header in enumerate(INGREDIENT_HEADERS):
|
||
if header in _SKIP_NUTRIENT_HEADERS or i >= len(row):
|
||
continue
|
||
raw = (row[i] or "").strip()
|
||
if not raw:
|
||
continue
|
||
try:
|
||
nutrients[header] = float(raw)
|
||
except ValueError:
|
||
continue
|
||
return nutrients
|
||
|
||
|
||
def _find_component(name: str, external_no: int | None) -> Component | None:
|
||
if external_no is not None:
|
||
hit = Component.query.filter_by(external_no=external_no, is_deleted=False).first()
|
||
if hit:
|
||
return hit
|
||
if name:
|
||
hit = Component.query.filter(Component.name == name, Component.is_deleted.is_(False)).first()
|
||
if hit:
|
||
return hit
|
||
return None
|
||
|
||
|
||
def import_component_nutrients(
|
||
csv_path: Path | None = None,
|
||
*,
|
||
derive: bool = True,
|
||
dry_run: bool = False,
|
||
) -> NutrientsImportStats:
|
||
path = csv_path or (nutrients_dir() / "База сырья.csv")
|
||
stats = NutrientsImportStats()
|
||
if not path.is_file():
|
||
stats.errors.append(f"CSV not found: {path}")
|
||
return stats
|
||
|
||
with path.open(encoding="utf-8") as f:
|
||
rows = list(csv.reader(f))
|
||
|
||
for row in rows[5:]:
|
||
name = row[1].strip() if len(row) > 1 else ""
|
||
if not name:
|
||
stats.skipped += 1
|
||
continue
|
||
try:
|
||
external_no = int(float(row[0])) if row[0].strip() else None
|
||
except (ValueError, IndexError):
|
||
external_no = None
|
||
|
||
comp = _find_component(name, external_no)
|
||
if comp is None:
|
||
stats.unmatched += 1
|
||
continue
|
||
|
||
nutrients = _parse_nutrient_row(row)
|
||
if derive:
|
||
nutrients = derive_ingredient_nutrients(nutrients)
|
||
|
||
if not dry_run:
|
||
save_component_nutrients(comp.id, nutrients, user_id="seed-import")
|
||
stats.imported += 1
|
||
|
||
if not dry_run:
|
||
db.session.commit()
|
||
_log.info(
|
||
"seed nutrients import: imported=%s skipped=%s unmatched=%s",
|
||
stats.imported,
|
||
stats.skipped,
|
||
stats.unmatched,
|
||
)
|
||
return stats
|