96 lines
3.5 KiB
Python
96 lines
3.5 KiB
Python
"""T+ 六类源表的只读读取器。"""
|
||
|
||
from collections.abc import Iterator
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
from sqlalchemy import BigInteger, Integer, bindparam, text
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app.services.tplus.mapping import TPLUS_MAPPINGS, get_mapping
|
||
|
||
_ENTITY_QUERY_FILES = {
|
||
"personnel": "personnel.sql",
|
||
"products": "products.sql",
|
||
"customers": "customers.sql",
|
||
"businesses": "businesses.sql",
|
||
"shipments": "shipments.sql",
|
||
"receipts": "receipts.sql",
|
||
"business_items": "business_items.sql",
|
||
"shipment_items": "shipment_items.sql",
|
||
"receipt_items": "receipt_items.sql",
|
||
}
|
||
|
||
|
||
class TPlusReader:
|
||
"""执行白名单 SQL 文件，禁止调用方传入任意表名或 SQL。"""
|
||
|
||
def __init__(self, session: Session, query_dir: Path | None = None) -> None:
|
||
self.session = session
|
||
self.query_dir = query_dir or self._default_query_dir()
|
||
|
||
@staticmethod
|
||
def _default_query_dir() -> Path:
|
||
return Path(__file__).resolve().parents[4] / "db" / "queries" / "tplus"
|
||
|
||
def iter_entity(
|
||
self,
|
||
entity_type: str,
|
||
*,
|
||
watermark: datetime | None = None,
|
||
batch_size: int = 500,
|
||
) -> Iterator[dict[str, Any]]:
|
||
"""按实体流式读取，水位只通过绑定参数传入。"""
|
||
if entity_type not in _ENTITY_QUERY_FILES:
|
||
raise KeyError(f"不支持的 T+ 实体类型：{entity_type}")
|
||
if batch_size < 1 or batch_size > 5000:
|
||
raise ValueError("batch_size 必须在 1 到 5000 之间")
|
||
if watermark is not None:
|
||
raise ValueError("增量水位语义尚未确认，当前只支持全量读取和哈希去重")
|
||
get_mapping(entity_type)
|
||
sql_path = self.query_dir / _ENTITY_QUERY_FILES[entity_type]
|
||
if not sql_path.is_file():
|
||
raise FileNotFoundError(f"缺少 T+ 查询文件：{sql_path}")
|
||
|
||
statement = text(sql_path.read_text(encoding="utf-8")).bindparams(
|
||
bindparam("batch_size", type_=Integer()), bindparam("after_id", type_=BigInteger())
|
||
)
|
||
after_id = -2147483649 # 所有源表主键均已核验为 SQL Server int，包含其最小值。
|
||
while True:
|
||
result = self.session.execute(
|
||
statement, {"batch_size": batch_size, "after_id": after_id}
|
||
)
|
||
try:
|
||
rows = [dict(row) for row in result.mappings().all()]
|
||
finally:
|
||
result.close()
|
||
if not rows:
|
||
return
|
||
next_id = int(rows[-1]["id"])
|
||
if next_id <= after_id:
|
||
raise ValueError("T+ 源主键未递增，停止读取以避免重复循环")
|
||
yield from rows
|
||
after_id = next_id
|
||
|
||
def fetch_entity(
|
||
self,
|
||
entity_type: str,
|
||
*,
|
||
watermark: datetime | None = None,
|
||
max_rows: int = 5000,
|
||
) -> list[dict[str, Any]]:
|
||
"""读取有限数量的实体行，供核验命令和单元测试使用。"""
|
||
if max_rows < 1:
|
||
raise ValueError("max_rows 必须大于 0")
|
||
rows: list[dict[str, Any]] = []
|
||
for row in self.iter_entity(entity_type, watermark=watermark):
|
||
rows.append(row)
|
||
if len(rows) >= max_rows:
|
||
break
|
||
return rows
|
||
|
||
def inventory(self) -> tuple[str, ...]:
|
||
"""返回当前适配器支持的实体类型。"""
|
||
return tuple(mapping.entity_type for mapping in TPLUS_MAPPINGS)
|