PayrollManagementSystem/backend/app/services/tplus/sync_service.py

"""T+ 只读快照同步：批量 staging、幂等去重和失败隔离。"""

from collections.abc import Iterable
from datetime import datetime
from itertools import batched
from pathlib import Path

from sqlalchemy import func, select
from sqlalchemy.orm import Session

from app.core.config import get_settings
from app.core.tplus_lock import sync_lock
from app.models import TPlusSyncBatch, TPlusSyncError
from app.services.tplus.mapping import TPlusMapping, get_mapping
from app.services.tplus.reader import TPlusReader
from app.services.tplus.staging import stage_rows
from app.services.tplus.task_service import fail_batch, new_batch_no, recover_running


class TPlusSyncService:
    """按分页提交原始快照，业务表始终不参与本服务。"""

    def __init__(self, source_session: Session, target_session: Session) -> None:
        self.source_session = source_session
        self.target_session = target_session
        configured = get_settings().t_plus_query_dir
        self.reader = TPlusReader(source_session, Path(configured) if configured else None)

    def run(
        self,
        *,
        entity_types: Iterable[str],
        watermark: datetime | None = None,
        batch_no: str | None = None,
        batch_size: int = 500,
    ) -> TPlusSyncBatch:
        """执行可重复的全量读取，提交每页进度；字段语义未定时拒绝时间增量。"""
        entities = tuple(dict.fromkeys(entity_types))
        if not entities or not 1 <= batch_size <= 5000:
            raise ValueError("至少指定一个实体，batch_size 必须在 1 到 5000 之间")
        if watermark is not None:
            raise ValueError("增量水位语义尚未确认，当前只支持全量读取和哈希去重")
        for entity in entities:
            get_mapping(entity)
        with sync_lock(self.target_session):
            recover_running(self.target_session)
            batch = self._start_direct(entities, batch_no)
            try:
                return self.execute_claimed(batch, batch_size)
            except BaseException:
                self.target_session.rollback()
                fail_batch(self.target_session, batch, "INTERRUPTED")
                raise

    def _start_direct(self, entities, batch_no):
        batch = TPlusSyncBatch(
            batch_no=batch_no or new_batch_no(),
            sync_mode="FULL",
            status="RUNNING",
            source_database=get_settings().t_plus_database or None,
            started_at=datetime.now(),
            source_record_count=0,
            staged_record_count=0,
            failed_record_count=0,
            skipped_record_count=0,
            entity_types_json=list(entities),
        )
        self.target_session.add(batch)
        self.target_session.commit()
        return batch

    def execute_claimed(self, batch: TPlusSyncBatch, batch_size: int = 500):
        """处理已持锁且标记 RUNNING 的任务；仅由直接入口或 worker 调用。"""
        if batch.status != "RUNNING" or not batch.entity_types_json:
            raise ValueError("任务尚未领取或读取范围不完整")
        mappings = [get_mapping(entity) for entity in batch.entity_types_json]
        for mapping in mappings:
            try:
                self._read_entity(batch, mapping, batch_size)
            except Exception as exc:
                self.target_session.rollback()
                self._record_error(batch, mapping, {}, "READ_ERROR", type(exc).__name__)
                self.target_session.commit()
        batch.status = "SUCCEEDED"
        if batch.failed_record_count:
            failed_rows = (
                self.target_session.scalar(
                    select(func.count())
                    .select_from(TPlusSyncError)
                    .where(
                        TPlusSyncError.batch_no == batch.batch_no,
                        TPlusSyncError.error_code == "STAGE_ERROR",
                    )
                )
                or 0
            )
            batch.status = "PARTIAL" if batch.source_record_count > failed_rows else "FAILED"
            batch.error_message = "部分数据未完成，请查询同步错误记录并重跑全量快照"
        batch.finished_at = datetime.now()
        batch.watermark_to = None
        self.target_session.commit()
        return batch

    def _read_entity(self, batch: TPlusSyncBatch, mapping: TPlusMapping, size: int) -> None:
        """批量写入一类数据；每页提交后即使进程中断也可幂等重跑。"""
        source = self.reader.iter_entity(mapping.entity_type, batch_size=size)
        for chunk in batched(source, size):
            inserted, errors = stage_rows(self.target_session, batch.batch_no, mapping, list(chunk))
            batch.source_record_count += len(chunk)
            batch.staged_record_count += inserted
            batch.skipped_record_count += len(chunk) - inserted - len(errors)
            for row, reason in errors:
                self._record_error(batch, mapping, row, "STAGE_ERROR", reason)
            self.target_session.commit()

    def _record_error(
        self, batch: TPlusSyncBatch, mapping: TPlusMapping, row: dict, code: str, reason: str
    ) -> None:
        """只记录错误类别和源 ID，避免数据库异常携带个人信息或凭据。"""
        self.target_session.add(
            TPlusSyncError(
                batch_no=batch.batch_no,
                entity_type=mapping.entity_type,
                source_table=mapping.source_table,
                source_id=str(row["id"]) if row.get("id") is not None else None,
                source_line_no=str(row["line_no"]) if row.get("line_no") is not None else None,
                error_code=code,
                error_message=f"源数据处理失败（{reason}），请核对连接和字段",
            )
        )
        batch.failed_record_count += 1