1461a8d113
按 2026-11-02 需求补齐计划交付后端能力： - 角色授权：权限目录化，接口统一准入校验，未登记默认拒绝； 管理员判断改为按数据库角色重新判定，防止权限码持有者绕过 - 系统监控：单进程采集资源、请求与异常指标，分钟级聚合， 阈值配置化；敏感配置增加 repr 保护，Redis 密码 URL 转义 - 试用期计提：按回款日判定并写入快照，年度结清与调整单联动； 人员模型补试用期、离职日期与工资版本历史字段 - 计提模板：公式骨架受控，页面维护名称与参数绑定 - T+ 采集：自动化配置表与进度查询 内置角色禁停用保护对齐模块注释；核算规则版本提为常量并 补充批量核算失败堆栈日志。 影响范围：后端全部模块、数据库迁移 V22-V28 验证：uv run ruff check 通过；922 个单元测试全部通过； 集成测试因本机 MySQL 数据卷被其他容器锁定暂未运行
298 lines
13 KiB
Python
298 lines
13 KiB
Python
"""使用隔离 SQLite 事务验证监控落库、恢复、权限及脱敏，不接触开发业务库。"""
|
|
|
|
import json
|
|
from datetime import timedelta
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
from fastapi import FastAPI
|
|
from fastapi.encoders import jsonable_encoder
|
|
from fastapi.testclient import TestClient
|
|
from sqlalchemy import Integer, MetaData, create_engine, func, select
|
|
from sqlalchemy.orm import sessionmaker
|
|
from sqlalchemy.pool import StaticPool
|
|
|
|
from app.api.deps import get_current_user, get_db
|
|
from app.api.v1.system.monitor import router
|
|
from app.core.config import Settings
|
|
from app.core.exception_handlers import register_exception_handlers
|
|
from app.core.monitoring import MonitorMiddleware, Recorder, minute_of, now_local
|
|
from app.models.monitor import MonitorEvent, MonitorRequestMinute, MonitorSnapshot
|
|
from app.schemas.monitor import MonitorWindow
|
|
from app.services import monitor_service as service
|
|
from app.services.monitor_collector import MonitorCollector, RuntimeErrorHandler
|
|
|
|
|
|
@pytest.fixture
|
|
def monitor_case(monkeypatch):
|
|
engine = create_engine(
|
|
"sqlite://", poolclass=StaticPool, connect_args={"check_same_thread": False}
|
|
)
|
|
metadata = MetaData()
|
|
for model in (MonitorSnapshot, MonitorRequestMinute, MonitorEvent):
|
|
table = model.__table__.to_metadata(metadata)
|
|
table.c.id.type = Integer()
|
|
metadata.create_all(engine)
|
|
factory = sessionmaker(engine, expire_on_commit=False)
|
|
settings = Settings(_env_file=None, app_secret_key="x" * 32, monitor_enabled=False)
|
|
collector = MonitorCollector(settings, Recorder())
|
|
monkeypatch.setattr(collector, "_factory", lambda: factory)
|
|
yield collector, factory
|
|
engine.dispose()
|
|
|
|
|
|
def seed(collector, at):
|
|
collector.recorder.record("GET", "/api/items/{id}", 500, 2200, "a" * 32, at=at)
|
|
collector.recorder.record("GET", "/api/items/{id}", 400, 10, "b" * 32, at=at)
|
|
collector.samples[minute_of(at)] = [
|
|
{"memory_percent": 40, "cpu_percent": 10, "disk_percent": 25}
|
|
]
|
|
|
|
|
|
def test_历史跨采集器重启保留且当前分钟不重复统计(monitor_case):
|
|
collector, factory = monitor_case
|
|
at = minute_of(now_local()) - timedelta(minutes=2)
|
|
seed(collector, at)
|
|
collector._persist(now_local())
|
|
collector._persist(now_local())
|
|
with factory() as session:
|
|
assert session.scalar(select(func.count()).select_from(MonitorSnapshot)) == 1
|
|
assert session.scalar(select(func.count()).select_from(MonitorRequestMinute)) == 1
|
|
assert service.overview(session, collector).requests["count"] == 2
|
|
restarted = MonitorCollector(collector.settings, Recorder())
|
|
trend = service.trends(session, restarted, MonitorWindow())
|
|
assert sum(point["request_count"] or 0 for point in trend["points"]) == 2
|
|
row = service.interfaces(session, restarted, MonitorWindow())["list"][0]
|
|
assert (row["count"], row["errors_5xx"], row["errors_4xx"]) == (2, 1, 1)
|
|
assert row["p95_ms"] == 2200
|
|
|
|
|
|
def test_持久化失败保留事件恢复后写入且不影响记录(monitor_case, monkeypatch):
|
|
collector, factory = monitor_case
|
|
at = minute_of(now_local()) - timedelta(minutes=1)
|
|
seed(collector, at)
|
|
|
|
def fail():
|
|
raise RuntimeError("synthetic-db-failure")
|
|
|
|
monkeypatch.setattr(collector, "_factory", fail)
|
|
collector._persist(now_local())
|
|
assert len(collector.recorder.events) == 1
|
|
assert collector.current()["collection_status"] == "warning"
|
|
monkeypatch.setattr(collector, "_factory", lambda: factory)
|
|
collector._persist(now_local())
|
|
with factory() as session:
|
|
assert session.scalar(select(func.count()).select_from(MonitorEvent)) == 1
|
|
assert not collector.recorder.events
|
|
|
|
|
|
def test_同类健康异常合并恢复并跨重启复用首次时间(monitor_case):
|
|
collector, factory = monitor_case
|
|
at = now_local() - timedelta(minutes=3)
|
|
bad = [{"name": "MySQL", "status": "error", "message": "连接检查失败。"}]
|
|
collector._evaluate(at, {}, bad, {})
|
|
collector._persist(at)
|
|
restarted = MonitorCollector(collector.settings, Recorder())
|
|
restarted._factory = lambda: factory
|
|
restarted._evaluate(at + timedelta(minutes=1), {}, bad, {})
|
|
restarted._persist(at + timedelta(minutes=1))
|
|
with factory() as session:
|
|
row = session.scalar(select(MonitorEvent))
|
|
assert row.first_seen == at and row.occurrences == 2 and row.resolved_at is None
|
|
assert session.scalar(select(func.count()).select_from(MonitorEvent)) == 1
|
|
restarted._evaluate(at + timedelta(minutes=2), {}, [], {})
|
|
restarted._persist(at + timedelta(minutes=2))
|
|
with factory() as session:
|
|
assert session.scalar(select(MonitorEvent)).resolved_at == at + timedelta(minutes=2)
|
|
|
|
|
|
def test_资源持续五分钟才告警且未知不算恢复(monitor_case):
|
|
collector, _ = monitor_case
|
|
at = now_local() - timedelta(minutes=6)
|
|
collector._evaluate(at, {"cpu_percent": 90}, [], {})
|
|
assert not collector.conditions
|
|
collector._evaluate(at + timedelta(minutes=5), {"cpu_percent": 90}, [], {})
|
|
assert "CPU_HIGH" in collector.conditions
|
|
collector._evaluate(at + timedelta(minutes=6), {}, [], {})
|
|
assert "CPU_HIGH" in collector.conditions
|
|
collector._evaluate(at + timedelta(minutes=7), {"cpu_percent": 30}, [], {})
|
|
assert "CPU_HIGH" not in collector.conditions
|
|
|
|
|
|
def test_低请求量不报错误率且四百不触发系统故障(monitor_case):
|
|
collector, _ = monitor_case
|
|
at = now_local()
|
|
for i in range(19):
|
|
collector.recorder.record("GET", "/items", 500, 1, str(i), at=at)
|
|
collector._evaluate(at, {}, [], {})
|
|
assert "HTTP_ERROR_RATE" not in collector.conditions
|
|
collector.recorder.record("GET", "/items", 400, 1, "x", at=at)
|
|
collector._evaluate(at, {}, [], {})
|
|
assert "HTTP_ERROR_RATE" in collector.conditions
|
|
|
|
|
|
def test_过期清理仅限监控表且保留活动异常(monitor_case):
|
|
collector, factory = monitor_case
|
|
old = now_local() - timedelta(days=8)
|
|
seed(collector, old)
|
|
collector._evaluate(old, {"disk_percent": 99}, [], {})
|
|
collector._persist(now_local())
|
|
with factory() as session:
|
|
assert session.scalar(select(func.count()).select_from(MonitorSnapshot)) == 0
|
|
assert session.scalar(select(func.count()).select_from(MonitorRequestMinute)) == 0
|
|
assert session.scalar(select(func.count()).select_from(MonitorEvent)) == 1
|
|
assert session.scalar(select(MonitorEvent)).code == "DISK_HIGH"
|
|
|
|
|
|
def make_api(collector, factory):
|
|
app = FastAPI()
|
|
app.state.monitor = collector
|
|
app.include_router(router, prefix="/api/v1/system")
|
|
register_exception_handlers(app)
|
|
app.add_middleware(MonitorMiddleware, recorder=collector.recorder)
|
|
|
|
def database():
|
|
with factory() as session:
|
|
yield session
|
|
|
|
app.dependency_overrides[get_db] = database
|
|
return app
|
|
|
|
|
|
def test_未登录与普通用户均不能读监控(monitor_case, monkeypatch):
|
|
collector, factory = monitor_case
|
|
app = make_api(collector, factory)
|
|
client = TestClient(app)
|
|
assert client.get("/api/v1/system/monitor/overview").status_code == 401
|
|
app.dependency_overrides[get_current_user] = lambda: SimpleNamespace(id=1)
|
|
monkeypatch.setattr("app.api.deps.user_repository.get_role_codes", lambda _: ["user"])
|
|
for path in ("overview", "trends", "interfaces", "events", "events/1", "diagnostic"):
|
|
assert client.get(f"/api/v1/system/monitor/{path}").status_code == 403
|
|
|
|
|
|
def test_管理员契约查询约束与脱敏导出(monitor_case, monkeypatch):
|
|
collector, factory = monitor_case
|
|
seed(collector, now_local() - timedelta(minutes=1))
|
|
collector._persist(now_local())
|
|
app = make_api(collector, factory)
|
|
app.dependency_overrides[get_current_user] = lambda: SimpleNamespace(id=1)
|
|
monkeypatch.setattr("app.api.deps.user_repository.get_role_codes", lambda _: ["admin"])
|
|
client = TestClient(app)
|
|
for path in ("overview", "trends", "interfaces", "events", "events/1"):
|
|
response = client.get(f"/api/v1/system/monitor/{path}")
|
|
assert response.status_code == 200, response.text
|
|
assert response.json()["code"] == 200
|
|
assert client.get("/api/v1/system/monitor/trends?hours=999").status_code == 400
|
|
for hours in (1, 24, 168):
|
|
for endpoint in ("trends", "interfaces", "events"):
|
|
assert client.get(f"/api/v1/system/monitor/{endpoint}?hours={hours}").status_code == 200
|
|
assert client.get("/api/v1/system/monitor/events?request_id=invalid").status_code == 400
|
|
assert client.get("/api/v1/system/monitor/events?pageSize=201").status_code == 400
|
|
result = client.get("/api/v1/system/monitor/events?request_id=" + "a" * 32).json()["data"]
|
|
assert result["total"] == 1
|
|
export = client.get("/api/v1/system/monitor/diagnostic")
|
|
assert "attachment" in export.headers["content-disposition"]
|
|
assert not any(
|
|
field in export.text
|
|
for field in ("password", "secret", "token", "traceback", "request_data", "payload_json")
|
|
)
|
|
|
|
|
|
def test_未处理异常编号匹配日志且只记录一次(monitor_case, caplog):
|
|
collector, factory = monitor_case
|
|
app = make_api(collector, factory)
|
|
|
|
@app.get("/explode")
|
|
def explode():
|
|
raise RuntimeError("synthetic-private-details")
|
|
|
|
response = TestClient(app, raise_server_exceptions=False).get("/explode")
|
|
request_id = response.headers["x-request-id"]
|
|
assert response.status_code == 500
|
|
assert "synthetic-private-details" not in response.text
|
|
assert len(collector.recorder.events) == 1
|
|
assert collector.recorder.events[0]["request_id"] == request_id
|
|
assert any(getattr(record, "request_id", None) == request_id for record in caplog.records)
|
|
assert "synthetic-private-details" not in json.dumps(
|
|
jsonable_encoder(list(collector.recorder.events))
|
|
)
|
|
|
|
|
|
def test_后台日志摘要不读取异常原文且请求日志不重复(monitor_case):
|
|
import logging
|
|
|
|
from app.core.monitoring import REQUEST_ID
|
|
|
|
collector, _ = monitor_case
|
|
handler = RuntimeErrorHandler(collector.recorder)
|
|
record = logging.LogRecord("app.jobs", logging.ERROR, "", 0, "password=synthetic", (), None)
|
|
handler.emit(record)
|
|
token = REQUEST_ID.set("a" * 32)
|
|
try:
|
|
handler.emit(record)
|
|
finally:
|
|
REQUEST_ID.reset(token)
|
|
assert len(collector.recorder.events) == 1
|
|
assert "synthetic" not in str(collector.recorder.events)
|
|
|
|
|
|
def test_过期采集不继续显示依赖正常(monitor_case):
|
|
collector, factory = monitor_case
|
|
collector.state.update(
|
|
collected_at=now_local() - timedelta(minutes=2),
|
|
dependencies=[
|
|
{"name": "MySQL", "status": "normal", "message": "正常", "checked_at": now_local()}
|
|
],
|
|
)
|
|
with factory() as session:
|
|
result = service.overview(session, collector)
|
|
assert result.status == "unknown"
|
|
assert result.dependencies[0].status == "unknown"
|
|
|
|
|
|
def test_重启首次CPU未知不恢复已有告警且健康后恢复(monitor_case):
|
|
collector, factory = monitor_case
|
|
at = now_local() - timedelta(minutes=7)
|
|
collector._evaluate(at, {"cpu_percent": 99}, [], {})
|
|
collector._evaluate(at + timedelta(minutes=5), {"cpu_percent": 99}, [], {})
|
|
collector._persist(at + timedelta(minutes=5))
|
|
restarted = MonitorCollector(collector.settings, Recorder())
|
|
restarted._factory = lambda: factory
|
|
restarted._evaluate(at + timedelta(minutes=6), {}, [], {})
|
|
restarted._persist(at + timedelta(minutes=6))
|
|
with factory() as session:
|
|
assert session.scalar(select(MonitorEvent)).resolved_at is None
|
|
restarted._evaluate(now_local(), {"cpu_percent": 30}, [], {})
|
|
restarted._persist(now_local())
|
|
with factory() as session:
|
|
assert session.scalar(select(MonitorEvent)).resolved_at is not None
|
|
|
|
|
|
def test_T加状态未知不误报恢复(monitor_case):
|
|
collector, factory = monitor_case
|
|
at = now_local() - timedelta(minutes=1)
|
|
collector._evaluate(
|
|
at,
|
|
{},
|
|
[],
|
|
{"enabled": True, "alerts": [{"code": "WORKER_OFFLINE", "message": "采集进程心跳中断"}]},
|
|
)
|
|
collector._persist(at)
|
|
collector._evaluate(now_local(), {}, [], {"status": "unknown", "alerts": []})
|
|
collector._persist(now_local())
|
|
with factory() as session:
|
|
assert session.scalar(select(MonitorEvent)).resolved_at is None
|
|
|
|
|
|
def test_监控权限登记且不能授予普通角色(monitor_case):
|
|
from app.services.authorization_policy import permission_for
|
|
from app.services.authorization_service import calculate
|
|
from app.services.permission_catalog import CATALOG
|
|
|
|
_, factory = monitor_case
|
|
assert CATALOG["system:monitor:view"]["admin_only"]
|
|
assert permission_for("system.monitor", "diagnostic", "GET") == "system:monitor:view"
|
|
with factory() as session:
|
|
user = SimpleNamespace(id=1, status=1)
|
|
assert "system:monitor:view" in calculate(session, user, roles=["admin"])["permissions"]
|