478 lines
20 KiB
Python
478 lines
20 KiB
Python
|
"""单应用进程的独立采集线程；探测有超时，写入异常不会进入业务请求。"""
|
|||
|
|
|
|||
|
|
import logging
|
|||
|
|
import threading
|
|||
|
|
from collections import OrderedDict
|
|||
|
|
from copy import deepcopy
|
|||
|
|
from datetime import timedelta
|
|||
|
|
from pathlib import Path
|
|||
|
|
from time import monotonic, perf_counter
|
|||
|
|
|
|||
|
|
import psutil
|
|||
|
|
from redis import Redis
|
|||
|
|
from sqlalchemy import create_engine, delete, select, text
|
|||
|
|
from sqlalchemy.orm import sessionmaker
|
|||
|
|
|
|||
|
|
from app.core.config import BACKEND_DIR
|
|||
|
|
from app.core.monitoring import (
|
|||
|
|
REQUEST_ID,
|
|||
|
|
Recorder,
|
|||
|
|
fingerprint,
|
|||
|
|
merge_metrics,
|
|||
|
|
minute_of,
|
|||
|
|
now_local,
|
|||
|
|
)
|
|||
|
|
from app.models.monitor import MonitorEvent, MonitorRequestMinute, MonitorSnapshot
|
|||
|
|
from app.repositories import monitor_repository as repo
|
|||
|
|
from app.services.tplus import automation
|
|||
|
|
|
|||
|
|
logger = logging.getLogger(__name__)
|
|||
|
|
|
|||
|
|
|
|||
|
|
class RuntimeErrorHandler(logging.Handler):
|
|||
|
|
"""后台错误只采集固定摘要；HTTP 错误由中间件记录，避免同一异常重复。"""
|
|||
|
|
|
|||
|
|
def __init__(self, recorder):
|
|||
|
|
super().__init__(logging.ERROR)
|
|||
|
|
self.recorder = recorder
|
|||
|
|
|
|||
|
|
def emit(self, record):
|
|||
|
|
if (
|
|||
|
|
record.name.startswith("app.")
|
|||
|
|
and ".monitor" not in record.name
|
|||
|
|
and REQUEST_ID.get() == "-"
|
|||
|
|
and getattr(record, "request_id", "-") == "-"
|
|||
|
|
):
|
|||
|
|
self.recorder.runtime_error(record.name)
|
|||
|
|
|
|||
|
|
|
|||
|
|
class MonitorCollector:
|
|||
|
|
def __init__(self, settings, recorder: Recorder):
|
|||
|
|
self.settings, self.recorder = settings, recorder
|
|||
|
|
self.recorder.slow_ms = settings.monitor_slow_request_ms
|
|||
|
|
self.lock = threading.Lock()
|
|||
|
|
self.stopped = threading.Event()
|
|||
|
|
self.thread = None
|
|||
|
|
self.engine = None
|
|||
|
|
self.redis = None
|
|||
|
|
self.process = None
|
|||
|
|
self.samples = OrderedDict()
|
|||
|
|
self.conditions = {}
|
|||
|
|
self.unknown_codes = set()
|
|||
|
|
self.pending_incidents = {}
|
|||
|
|
self.since = {}
|
|||
|
|
self.persisted = set()
|
|||
|
|
self.last_log = 0
|
|||
|
|
self.handler = RuntimeErrorHandler(recorder)
|
|||
|
|
self.state = {
|
|||
|
|
"collected_at": None,
|
|||
|
|
"last_persisted_at": None,
|
|||
|
|
"resources": {},
|
|||
|
|
"dependencies": [],
|
|||
|
|
"tasks": {},
|
|||
|
|
"alerts": [],
|
|||
|
|
"collection_status": "unknown",
|
|||
|
|
"collection_message": "正在等待首次采集。"
|
|||
|
|
if settings.monitor_enabled
|
|||
|
|
else "后台采集未启用。",
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
def current(self):
|
|||
|
|
with self.lock:
|
|||
|
|
return deepcopy(self.state)
|
|||
|
|
|
|||
|
|
def sample_history(self):
|
|||
|
|
with self.lock:
|
|||
|
|
return deepcopy(self.samples)
|
|||
|
|
|
|||
|
|
def start(self):
|
|||
|
|
if not self.settings.monitor_enabled or self.thread:
|
|||
|
|
return
|
|||
|
|
self.stopped.clear()
|
|||
|
|
logging.getLogger().addHandler(self.handler)
|
|||
|
|
self.thread = threading.Thread(target=self._run, name="system-monitor", daemon=True)
|
|||
|
|
self.thread.start()
|
|||
|
|
|
|||
|
|
def stop(self):
|
|||
|
|
self.stopped.set()
|
|||
|
|
if self.thread:
|
|||
|
|
self.thread.join(timeout=15)
|
|||
|
|
if self.thread.is_alive():
|
|||
|
|
logger.warning("监控采集线程未在关闭窗口内结束")
|
|||
|
|
return
|
|||
|
|
logging.getLogger().removeHandler(self.handler)
|
|||
|
|
if self.redis:
|
|||
|
|
self.redis.close()
|
|||
|
|
if self.engine:
|
|||
|
|
self.engine.dispose()
|
|||
|
|
self.thread = None
|
|||
|
|
|
|||
|
|
def _factory(self):
|
|||
|
|
if self.engine is None:
|
|||
|
|
self.engine = create_engine(
|
|||
|
|
self.settings.database_url,
|
|||
|
|
pool_size=1,
|
|||
|
|
max_overflow=0,
|
|||
|
|
pool_timeout=2,
|
|||
|
|
pool_pre_ping=True,
|
|||
|
|
connect_args={"connect_timeout": 2, "read_timeout": 2, "write_timeout": 2},
|
|||
|
|
)
|
|||
|
|
return sessionmaker(self.engine, expire_on_commit=False)
|
|||
|
|
|
|||
|
|
def _resources(self):
|
|||
|
|
first = self.process is None
|
|||
|
|
if first:
|
|||
|
|
self.process = psutil.Process()
|
|||
|
|
psutil.cpu_percent(None)
|
|||
|
|
self.process.cpu_percent(None)
|
|||
|
|
memory = psutil.virtual_memory()
|
|||
|
|
disk = psutil.disk_usage(str(BACKEND_DIR.anchor))
|
|||
|
|
return {
|
|||
|
|
"cpu_percent": None if first else psutil.cpu_percent(None),
|
|||
|
|
"memory_percent": memory.percent,
|
|||
|
|
"disk_percent": disk.percent,
|
|||
|
|
"memory_total_mb": round(memory.total / 1024**2),
|
|||
|
|
"disk_free_gb": round(disk.free / 1024**3, 2),
|
|||
|
|
"process_cpu_percent": None
|
|||
|
|
if first
|
|||
|
|
else round(self.process.cpu_percent(None) / (psutil.cpu_count() or 1), 2),
|
|||
|
|
"process_memory_mb": round(self.process.memory_info().rss / 1024**2, 2),
|
|||
|
|
"scope": "进程可见系统资源，未换算容器限额"
|
|||
|
|
if Path("/.dockerenv").exists()
|
|||
|
|
else "应用所在系统的可见资源",
|
|||
|
|
"process_cpu_scope": "占可见 CPU 总量的比例",
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
def _dependencies(self, at):
|
|||
|
|
dependencies = [
|
|||
|
|
{
|
|||
|
|
"name": "应用",
|
|||
|
|
"status": "normal",
|
|||
|
|
"message": "采集线程正在运行。",
|
|||
|
|
"checked_at": at,
|
|||
|
|
"duration_ms": None,
|
|||
|
|
}
|
|||
|
|
]
|
|||
|
|
tasks = {"status": "unknown", "message": "尚无采集记录。", "alerts": []}
|
|||
|
|
for name in ("MySQL", "Redis"):
|
|||
|
|
started = perf_counter()
|
|||
|
|
try:
|
|||
|
|
if name == "MySQL":
|
|||
|
|
with self._factory()() as session:
|
|||
|
|
session.execute(text("SELECT 1"))
|
|||
|
|
else:
|
|||
|
|
if self.redis is None:
|
|||
|
|
self.redis = Redis.from_url(
|
|||
|
|
self.settings.redis_url, socket_connect_timeout=2, socket_timeout=2
|
|||
|
|
)
|
|||
|
|
self.redis.ping()
|
|||
|
|
status, message = "normal", "连接正常。"
|
|||
|
|
except Exception:
|
|||
|
|
status, message = "error", "连接检查失败，请核对服务和网络。"
|
|||
|
|
dependencies.append(
|
|||
|
|
{
|
|||
|
|
"name": name,
|
|||
|
|
"status": status,
|
|||
|
|
"message": message,
|
|||
|
|
"checked_at": at,
|
|||
|
|
"duration_ms": round((perf_counter() - started) * 1000, 2),
|
|||
|
|
}
|
|||
|
|
)
|
|||
|
|
if dependencies[1]["status"] == "normal":
|
|||
|
|
try:
|
|||
|
|
with self._factory()() as session:
|
|||
|
|
tasks = automation.overview(session, settings=self.settings, now=at)
|
|||
|
|
source_configured = bool(
|
|||
|
|
self.settings.t_plus_enabled or self.settings.t_plus_connection_file
|
|||
|
|
)
|
|||
|
|
if not source_configured:
|
|||
|
|
status, message = "unknown", "T+ 未配置。"
|
|||
|
|
elif tasks.get("stop_requested"):
|
|||
|
|
status, message = "unknown", "已主动请求停止采集。"
|
|||
|
|
elif not tasks["enabled"]:
|
|||
|
|
status, message = "unknown", "自动采集未启用，可在原页面查看手动任务。"
|
|||
|
|
else:
|
|||
|
|
status = "warning" if tasks["alerts"] else "normal"
|
|||
|
|
message = (
|
|||
|
|
"存在采集异常，请查看任务与同步。"
|
|||
|
|
if tasks["alerts"]
|
|||
|
|
else "周期采集状态正常。"
|
|||
|
|
)
|
|||
|
|
tasks.update(status=status, message=message)
|
|||
|
|
except Exception:
|
|||
|
|
tasks = {
|
|||
|
|
"status": "unknown",
|
|||
|
|
"message": "任务状态读取失败，请检查迁移和服务日志。",
|
|||
|
|
"alerts": [],
|
|||
|
|
}
|
|||
|
|
dependencies.append(
|
|||
|
|
{
|
|||
|
|
"name": "T+ 采集进程",
|
|||
|
|
"status": tasks["status"],
|
|||
|
|
"message": tasks["message"],
|
|||
|
|
"checked_at": at,
|
|||
|
|
"duration_ms": None,
|
|||
|
|
}
|
|||
|
|
)
|
|||
|
|
return dependencies, tasks
|
|||
|
|
|
|||
|
|
def _evaluate(self, at, resources, dependencies, tasks):
|
|||
|
|
active = {}
|
|||
|
|
self.unknown_codes = set()
|
|||
|
|
for dependency in dependencies:
|
|||
|
|
if dependency["status"] == "error":
|
|||
|
|
code = "MYSQL_UNAVAILABLE" if dependency["name"] == "MySQL" else "REDIS_UNAVAILABLE"
|
|||
|
|
active[code] = ("error", dependency["name"], dependency["message"], None)
|
|||
|
|
for field, threshold, code, title in (
|
|||
|
|
("cpu_percent", self.settings.monitor_cpu_threshold, "CPU_HIGH", "CPU 占用持续偏高"),
|
|||
|
|
(
|
|||
|
|
"memory_percent",
|
|||
|
|
self.settings.monitor_memory_threshold,
|
|||
|
|
"MEMORY_HIGH",
|
|||
|
|
"内存占用持续偏高",
|
|||
|
|
),
|
|||
|
|
("disk_percent", self.settings.monitor_disk_threshold, "DISK_HIGH", "磁盘空间不足"),
|
|||
|
|
):
|
|||
|
|
value = resources.get(field)
|
|||
|
|
if value is None:
|
|||
|
|
self.unknown_codes.add(code)
|
|||
|
|
self.since.pop(code, None)
|
|||
|
|
# 采集未知时不将仍存在的问题标为恢复。
|
|||
|
|
if code in self.conditions:
|
|||
|
|
previous = self.conditions[code]
|
|||
|
|
active[code] = (
|
|||
|
|
previous["level"],
|
|||
|
|
previous["component"],
|
|||
|
|
previous["message"],
|
|||
|
|
None,
|
|||
|
|
)
|
|||
|
|
continue
|
|||
|
|
if value >= threshold:
|
|||
|
|
since = self.since.setdefault(code, at)
|
|||
|
|
# 重启后的持续时间尚未建立时，不关闭历史同类告警。
|
|||
|
|
self.unknown_codes.add(code)
|
|||
|
|
if (
|
|||
|
|
field == "disk_percent"
|
|||
|
|
or (at - since).total_seconds()
|
|||
|
|
>= self.settings.monitor_resource_duration_seconds
|
|||
|
|
):
|
|||
|
|
active[code] = ("warning", "resources", title + "，请检查资源趋势。", None)
|
|||
|
|
else:
|
|||
|
|
self.since.pop(code, None)
|
|||
|
|
recent = self.recorder.metrics(minute_of(at) - timedelta(minutes=4))
|
|||
|
|
metric = merge_metrics(m for endpoints in recent.values() for m in endpoints.values())
|
|||
|
|
if (
|
|||
|
|
metric["count"] >= self.settings.monitor_error_min_requests
|
|||
|
|
and metric["error_rate"] >= self.settings.monitor_error_rate_threshold
|
|||
|
|
):
|
|||
|
|
active["HTTP_ERROR_RATE"] = (
|
|||
|
|
"warning",
|
|||
|
|
"http",
|
|||
|
|
"近 5 分钟服务端错误率偏高，请查看失败接口。",
|
|||
|
|
None,
|
|||
|
|
)
|
|||
|
|
if tasks.get("status") == "unknown" and "任务状态读取失败" in tasks.get("message", ""):
|
|||
|
|
active["TASK_STATUS_UNKNOWN"] = ("warning", "tplus", tasks["message"], None)
|
|||
|
|
for alert in tasks.get("alerts", []):
|
|||
|
|
active[alert["code"]] = ("warning", "tplus", alert["message"], alert.get("batch_no"))
|
|||
|
|
if "enabled" not in tasks:
|
|||
|
|
self.unknown_codes.update(
|
|||
|
|
{"WORKER_OFFLINE", "QUEUE_OVERDUE", "SYNC_STALE", "SYNC_REQUIRES_ATTENTION"}
|
|||
|
|
)
|
|||
|
|
for code, previous in self.conditions.items():
|
|||
|
|
if code in self.unknown_codes:
|
|||
|
|
active[code] = (
|
|||
|
|
previous["level"],
|
|||
|
|
previous["component"],
|
|||
|
|
previous["message"],
|
|||
|
|
previous.get("batch_no"),
|
|||
|
|
)
|
|||
|
|
for code, (level, component, message, batch) in active.items():
|
|||
|
|
previous = self.conditions.get(code)
|
|||
|
|
incident = previous or {
|
|||
|
|
"id": 0,
|
|||
|
|
"instance_id": self.recorder.instance_id,
|
|||
|
|
"kind": "HEALTH",
|
|||
|
|
"first_seen": at,
|
|||
|
|
"occurrences": 0,
|
|||
|
|
"code": code,
|
|||
|
|
"fingerprint": fingerprint(code),
|
|||
|
|
"resolved_at": None,
|
|||
|
|
}
|
|||
|
|
incident.update(
|
|||
|
|
last_seen=at,
|
|||
|
|
occurrences=incident["occurrences"] + 1,
|
|||
|
|
level=level,
|
|||
|
|
component=component,
|
|||
|
|
message=message,
|
|||
|
|
batch_no=batch,
|
|||
|
|
)
|
|||
|
|
self.conditions[code] = incident
|
|||
|
|
self.pending_incidents[code] = deepcopy(incident)
|
|||
|
|
for code in set(self.conditions) - set(active):
|
|||
|
|
incident = self.conditions.pop(code)
|
|||
|
|
incident.update(resolved_at=at)
|
|||
|
|
self.pending_incidents[code] = deepcopy(incident)
|
|||
|
|
|
|||
|
|
def _persist(self, at, *, final=False):
|
|||
|
|
pending = self.recorder.take_events()
|
|||
|
|
try:
|
|||
|
|
with self._factory()() as session:
|
|||
|
|
# 首次采集复用历史未恢复事件，跨应用重启保留故障起点。
|
|||
|
|
historical = {item.code: item for item in repo.active_events(session)}
|
|||
|
|
for code, item in self.pending_incidents.items():
|
|||
|
|
row = historical.get(code)
|
|||
|
|
fields = {key: value for key, value in item.items() if key != "id"}
|
|||
|
|
if row is None:
|
|||
|
|
row = MonitorEvent(**fields)
|
|||
|
|
session.add(row)
|
|||
|
|
else:
|
|||
|
|
for key in (
|
|||
|
|
"last_seen",
|
|||
|
|
"resolved_at",
|
|||
|
|
"level",
|
|||
|
|
"message",
|
|||
|
|
"batch_no",
|
|||
|
|
"instance_id",
|
|||
|
|
):
|
|||
|
|
setattr(row, key, fields.get(key))
|
|||
|
|
row.occurrences = (
|
|||
|
|
max(row.occurrences, fields["occurrences"])
|
|||
|
|
if item.get("id") == row.id
|
|||
|
|
else row.occurrences + fields["occurrences"]
|
|||
|
|
)
|
|||
|
|
session.flush()
|
|||
|
|
if code in self.conditions:
|
|||
|
|
self.conditions[code].update(
|
|||
|
|
id=row.id, first_seen=row.first_seen, occurrences=row.occurrences
|
|||
|
|
)
|
|||
|
|
for code, row in historical.items():
|
|||
|
|
if (
|
|||
|
|
code not in self.conditions
|
|||
|
|
and code not in self.pending_incidents
|
|||
|
|
and code not in self.unknown_codes
|
|||
|
|
):
|
|||
|
|
# 只在本次完整采集后确认历史异常已恢复。
|
|||
|
|
row.resolved_at = at
|
|||
|
|
for item in pending:
|
|||
|
|
session.add(MonitorEvent(instance_id=self.recorder.instance_id, **item))
|
|||
|
|
completed = minute_of(at) + (timedelta(minutes=1) if final else timedelta())
|
|||
|
|
metrics = self.recorder.metrics(end=completed)
|
|||
|
|
written = []
|
|||
|
|
for minute in sorted(set(self.samples) | set(metrics)):
|
|||
|
|
if minute >= completed or minute in self.persisted:
|
|||
|
|
continue
|
|||
|
|
samples = self.samples.get(minute, [])
|
|||
|
|
endpoints = metrics.get(minute, {})
|
|||
|
|
resources = dict(samples[-1]) if samples else {}
|
|||
|
|
for key in (
|
|||
|
|
"cpu_percent",
|
|||
|
|
"memory_percent",
|
|||
|
|
"disk_percent",
|
|||
|
|
"process_cpu_percent",
|
|||
|
|
"process_memory_mb",
|
|||
|
|
):
|
|||
|
|
numbers = [sample[key] for sample in samples if sample.get(key) is not None]
|
|||
|
|
resources[key] = round(sum(numbers) / len(numbers), 2) if numbers else None
|
|||
|
|
session.add(
|
|||
|
|
MonitorSnapshot(
|
|||
|
|
instance_id=self.recorder.instance_id,
|
|||
|
|
minute=minute,
|
|||
|
|
sample_count=len(samples),
|
|||
|
|
resources=resources,
|
|||
|
|
requests=merge_metrics(endpoints.values()),
|
|||
|
|
)
|
|||
|
|
)
|
|||
|
|
for (method, route), values in endpoints.items():
|
|||
|
|
session.add(
|
|||
|
|
MonitorRequestMinute(
|
|||
|
|
instance_id=self.recorder.instance_id,
|
|||
|
|
minute=minute,
|
|||
|
|
method=method,
|
|||
|
|
route=route,
|
|||
|
|
metrics=values,
|
|||
|
|
)
|
|||
|
|
)
|
|||
|
|
written.append(minute)
|
|||
|
|
# 每次最多删除每表 1000 条，限定监控表及过期时间，避免长事务。
|
|||
|
|
cutoff = at - timedelta(days=7)
|
|||
|
|
for model, column in (
|
|||
|
|
(MonitorSnapshot, MonitorSnapshot.minute),
|
|||
|
|
(MonitorRequestMinute, MonitorRequestMinute.minute),
|
|||
|
|
(MonitorEvent, MonitorEvent.last_seen),
|
|||
|
|
):
|
|||
|
|
expired = select(model.id).where(column < cutoff)
|
|||
|
|
if model is MonitorEvent:
|
|||
|
|
expired = expired.where(
|
|||
|
|
(MonitorEvent.kind != "HEALTH") | MonitorEvent.resolved_at.is_not(None)
|
|||
|
|
)
|
|||
|
|
ids = session.scalars(expired.order_by(column).limit(1000)).all()
|
|||
|
|
if ids:
|
|||
|
|
session.execute(delete(model).where(model.id.in_(ids)))
|
|||
|
|
session.commit()
|
|||
|
|
self.persisted.update(written)
|
|||
|
|
self.persisted = {
|
|||
|
|
minute
|
|||
|
|
for minute in self.persisted
|
|||
|
|
if minute >= minute_of(at) - timedelta(minutes=10)
|
|||
|
|
}
|
|||
|
|
self.pending_incidents.clear()
|
|||
|
|
with self.lock:
|
|||
|
|
self.state.update(
|
|||
|
|
last_persisted_at=at,
|
|||
|
|
collection_status="normal",
|
|||
|
|
collection_message="采集正常，分钟历史保留 7 天。",
|
|||
|
|
)
|
|||
|
|
except Exception:
|
|||
|
|
self.recorder.restore_events(pending)
|
|||
|
|
with self.lock:
|
|||
|
|
self.state.update(
|
|||
|
|
collection_status="warning",
|
|||
|
|
collection_message="历史写入失败，部分数据可能缺失；请检查数据库迁移和服务日志。",
|
|||
|
|
)
|
|||
|
|
if monotonic() - self.last_log >= 60:
|
|||
|
|
logger.warning("监控历史写入失败，请检查迁移及数据库", exc_info=True)
|
|||
|
|
self.last_log = monotonic()
|
|||
|
|
|
|||
|
|
def tick(self):
|
|||
|
|
at = now_local()
|
|||
|
|
try:
|
|||
|
|
resources = self._resources()
|
|||
|
|
except Exception:
|
|||
|
|
resources = {"scope": "资源采集失败"}
|
|||
|
|
logger.warning("系统资源采集失败", exc_info=True)
|
|||
|
|
dependencies, tasks = self._dependencies(at)
|
|||
|
|
with self.lock:
|
|||
|
|
self.samples.setdefault(minute_of(at), []).append(resources)
|
|||
|
|
while len(self.samples) > 10:
|
|||
|
|
self.samples.popitem(last=False)
|
|||
|
|
self._evaluate(at, resources, dependencies, tasks)
|
|||
|
|
with self.lock:
|
|||
|
|
self.state.update(
|
|||
|
|
collected_at=at, resources=resources, dependencies=dependencies, tasks=tasks
|
|||
|
|
)
|
|||
|
|
self._persist(at)
|
|||
|
|
with self.lock:
|
|||
|
|
self.state["alerts"] = deepcopy(list(self.conditions.values()))
|
|||
|
|
if not resources.get("memory_percent"):
|
|||
|
|
self.state.update(
|
|||
|
|
collection_status="warning",
|
|||
|
|
collection_message="资源采集失败，当前资源指标未知。",
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def _run(self):
|
|||
|
|
while not self.stopped.is_set():
|
|||
|
|
started = monotonic()
|
|||
|
|
try:
|
|||
|
|
self.tick()
|
|||
|
|
except Exception:
|
|||
|
|
logger.warning("监控采集周期失败", exc_info=True)
|
|||
|
|
with self.lock:
|
|||
|
|
self.state.update(
|
|||
|
|
collection_status="warning",
|
|||
|
|
collection_message="采集周期失败，请检查服务日志。",
|
|||
|
|
)
|
|||
|
|
self.stopped.wait(max(0, 15 - (monotonic() - started)))
|
|||
|
|
if self.engine is not None:
|
|||
|
|
self._persist(now_local(), final=True)
|