влад
@@ -0,0 +1,299 @@
|
||||
"""Generic ETL: CSV из data/seed/ → lab profiles и component nutrients."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from app import db
|
||||
from app.lab.calc.ingredient_catalog import INGREDIENT_HEADERS
|
||||
from app.lab.calc.ingredient_derive import derive_ingredient_nutrients
|
||||
from app.lab.models import LabAnimalProfile
|
||||
from app.lab.norm_catalog import norm_header, norm_title_to_key
|
||||
from app.lab.seed_paths import norms_dir, nutrients_dir
|
||||
from app.lab.services.component_nutrients import save_component_nutrients
|
||||
from app.lab.services.profile_norms import save_norms_from_payload
|
||||
from app.models.base import default_uuid
|
||||
from app.models.component import Component
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
|
||||
_SKIP_NUTRIENT_HEADERS = frozenset({"№", "Наименование", "Цена 1 кг"})
|
||||
|
||||
|
||||
@dataclass
|
||||
class NormsImportStats:
|
||||
profiles_created: int = 0
|
||||
profiles_updated: int = 0
|
||||
norm_rows: int = 0
|
||||
skipped: int = 0
|
||||
unmapped_headers: list[str] = field(default_factory=list)
|
||||
errors: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class NutrientsImportStats:
|
||||
imported: int = 0
|
||||
skipped: int = 0
|
||||
unmatched: int = 0
|
||||
errors: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def _parse_min_max_groups(h1: list[str], h2: list[str]) -> list[tuple[str, int, int | None]]:
|
||||
groups: list[tuple[str, int, int | None]] = []
|
||||
current_name: str | None = None
|
||||
limit = min(len(h1), len(h2))
|
||||
for i in range(limit):
|
||||
b = (h2[i] or "").strip().lower().replace("і", "и")
|
||||
n = (h1[i] or "").strip()
|
||||
if n and n not in ("№", "Наименование", "Масса", "КРС"):
|
||||
current_name = norm_header(n)
|
||||
if b.startswith("мин") and current_name:
|
||||
max_i = i + 1 if i + 1 < limit and "мах" in (h2[i + 1] or "").lower() else None
|
||||
groups.append((current_name, i, max_i))
|
||||
return groups
|
||||
|
||||
|
||||
def norm_profile_key(ration_type: str, external_no: int) -> str:
|
||||
return f"norm_{ration_type.lower()}_{external_no:04d}"
|
||||
|
||||
|
||||
def parse_norm_profile_row(
|
||||
row: list[str],
|
||||
groups: list[tuple[str, int, int | None]],
|
||||
ration_type: str,
|
||||
*,
|
||||
unmapped: set[str],
|
||||
profile_key_fn=norm_profile_key,
|
||||
) -> dict | None:
|
||||
if len(row) < 2 or not (row[1] or "").strip():
|
||||
return None
|
||||
try:
|
||||
external_no = int(float((row[0] or "").strip()))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
if external_no <= 0:
|
||||
return None
|
||||
|
||||
indicators: dict[str, dict[str, float | None]] = {}
|
||||
for header, min_i, max_i in groups:
|
||||
key = norm_title_to_key(header)
|
||||
if not key:
|
||||
unmapped.add(header)
|
||||
continue
|
||||
|
||||
def _num(idx: int | None) -> float | None:
|
||||
if idx is None or idx >= len(row):
|
||||
return None
|
||||
raw = (row[idx] or "").strip()
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
return float(raw)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
min_v, max_v = _num(min_i), _num(max_i)
|
||||
if min_v is None and max_v is None:
|
||||
continue
|
||||
indicators[key] = {"min": min_v, "max": max_v}
|
||||
|
||||
mass_kg = None
|
||||
if len(row) > 2 and (row[2] or "").strip():
|
||||
try:
|
||||
mass_kg = float(row[2])
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return {
|
||||
"external_no": external_no,
|
||||
"profile_key": profile_key_fn(ration_type, external_no),
|
||||
"label": row[1].strip(),
|
||||
"ration_type": ration_type,
|
||||
"mass_kg": mass_kg,
|
||||
"indicators": indicators,
|
||||
}
|
||||
|
||||
|
||||
_IMPORT_BATCH_SIZE = 50
|
||||
|
||||
|
||||
def _lookup_reference_profile(profile_key: str) -> LabAnimalProfile | None:
|
||||
"""Поиск по profile_key включая soft-deleted (UNIQUE на всю таблицу)."""
|
||||
return LabAnimalProfile.query.filter_by(profile_key=profile_key).first()
|
||||
|
||||
|
||||
def _restore_reference_profile(profile: LabAnimalProfile) -> None:
|
||||
if not profile.is_deleted:
|
||||
return
|
||||
profile.is_deleted = False
|
||||
profile.deleted_at = None
|
||||
profile.deleted_by = None
|
||||
|
||||
|
||||
def _import_norm_sheet(
|
||||
csv_path: Path,
|
||||
ration_type: str,
|
||||
stats: NormsImportStats,
|
||||
unmapped: set[str],
|
||||
*,
|
||||
replace_reference: bool = True,
|
||||
) -> None:
|
||||
if not csv_path.is_file():
|
||||
stats.errors.append(f"Файл не найден: {csv_path}")
|
||||
return
|
||||
|
||||
with csv_path.open(encoding="utf-8") as f:
|
||||
rows = list(csv.reader(f))
|
||||
if len(rows) < 8:
|
||||
stats.errors.append(f"Слишком мало строк: {csv_path}")
|
||||
return
|
||||
|
||||
groups = _parse_min_max_groups(rows[2], rows[3])
|
||||
batch_count = 0
|
||||
for row in rows[6:]:
|
||||
parsed = parse_norm_profile_row(row, groups, ration_type, unmapped=unmapped)
|
||||
if parsed is None:
|
||||
stats.skipped += 1
|
||||
continue
|
||||
|
||||
profile = _lookup_reference_profile(parsed["profile_key"])
|
||||
if profile is not None:
|
||||
if not replace_reference and not profile.is_deleted:
|
||||
stats.skipped += 1
|
||||
continue
|
||||
if not replace_reference and profile.is_deleted:
|
||||
stats.skipped += 1
|
||||
continue
|
||||
_restore_reference_profile(profile)
|
||||
stats.profiles_updated += 1
|
||||
else:
|
||||
profile = LabAnimalProfile(
|
||||
id=default_uuid(),
|
||||
profile_key=parsed["profile_key"],
|
||||
created_by="seed-import",
|
||||
)
|
||||
db.session.add(profile)
|
||||
stats.profiles_created += 1
|
||||
|
||||
profile.label = parsed["label"]
|
||||
profile.ration_type = ration_type
|
||||
profile.external_no = parsed["external_no"]
|
||||
save_norms_from_payload(
|
||||
profile,
|
||||
{
|
||||
"massKg": parsed["mass_kg"],
|
||||
"externalNo": parsed["external_no"],
|
||||
"indicators": parsed["indicators"],
|
||||
},
|
||||
)
|
||||
stats.norm_rows += len(parsed["indicators"])
|
||||
batch_count += 1
|
||||
if batch_count >= _IMPORT_BATCH_SIZE:
|
||||
db.session.commit()
|
||||
batch_count = 0
|
||||
|
||||
|
||||
def import_norm_profiles(
|
||||
csv_dir: Path | None = None,
|
||||
*,
|
||||
replace_reference: bool = True,
|
||||
) -> NormsImportStats:
|
||||
stats = NormsImportStats()
|
||||
base = csv_dir or norms_dir()
|
||||
unmapped: set[str] = set()
|
||||
|
||||
_import_norm_sheet(
|
||||
base / "Нормы КРС.csv", "BEEF", stats, unmapped, replace_reference=replace_reference
|
||||
)
|
||||
_import_norm_sheet(
|
||||
base / "Нормы Дойн.csv", "DAIRY", stats, unmapped, replace_reference=replace_reference
|
||||
)
|
||||
stats.unmapped_headers = sorted(unmapped)
|
||||
db.session.commit()
|
||||
|
||||
_log.info(
|
||||
"seed norms import: created=%s updated=%s norm_rows=%s",
|
||||
stats.profiles_created,
|
||||
stats.profiles_updated,
|
||||
stats.norm_rows,
|
||||
)
|
||||
return stats
|
||||
|
||||
|
||||
def _parse_nutrient_row(row: list[str]) -> dict[str, float]:
|
||||
nutrients: dict[str, float] = {}
|
||||
for i, header in enumerate(INGREDIENT_HEADERS):
|
||||
if header in _SKIP_NUTRIENT_HEADERS or i >= len(row):
|
||||
continue
|
||||
raw = (row[i] or "").strip()
|
||||
if not raw:
|
||||
continue
|
||||
try:
|
||||
nutrients[header] = float(raw)
|
||||
except ValueError:
|
||||
continue
|
||||
return nutrients
|
||||
|
||||
|
||||
def _find_component(name: str, external_no: int | None) -> Component | None:
|
||||
if external_no is not None:
|
||||
hit = Component.query.filter_by(external_no=external_no, is_deleted=False).first()
|
||||
if hit:
|
||||
return hit
|
||||
if name:
|
||||
hit = Component.query.filter(Component.name == name, Component.is_deleted.is_(False)).first()
|
||||
if hit:
|
||||
return hit
|
||||
return None
|
||||
|
||||
|
||||
def import_component_nutrients(
|
||||
csv_path: Path | None = None,
|
||||
*,
|
||||
derive: bool = True,
|
||||
dry_run: bool = False,
|
||||
) -> NutrientsImportStats:
|
||||
path = csv_path or (nutrients_dir() / "База сырья.csv")
|
||||
stats = NutrientsImportStats()
|
||||
if not path.is_file():
|
||||
stats.errors.append(f"CSV not found: {path}")
|
||||
return stats
|
||||
|
||||
with path.open(encoding="utf-8") as f:
|
||||
rows = list(csv.reader(f))
|
||||
|
||||
for row in rows[5:]:
|
||||
name = row[1].strip() if len(row) > 1 else ""
|
||||
if not name:
|
||||
stats.skipped += 1
|
||||
continue
|
||||
try:
|
||||
external_no = int(float(row[0])) if row[0].strip() else None
|
||||
except (ValueError, IndexError):
|
||||
external_no = None
|
||||
|
||||
comp = _find_component(name, external_no)
|
||||
if comp is None:
|
||||
stats.unmatched += 1
|
||||
continue
|
||||
|
||||
nutrients = _parse_nutrient_row(row)
|
||||
if derive:
|
||||
nutrients = derive_ingredient_nutrients(nutrients)
|
||||
|
||||
if not dry_run:
|
||||
save_component_nutrients(comp.id, nutrients, user_id="seed-import")
|
||||
stats.imported += 1
|
||||
|
||||
if not dry_run:
|
||||
db.session.commit()
|
||||
_log.info(
|
||||
"seed nutrients import: imported=%s skipped=%s unmatched=%s",
|
||||
stats.imported,
|
||||
stats.skipped,
|
||||
stats.unmatched,
|
||||
)
|
||||
return stats
|
||||
Reference in New Issue
Block a user