PayrollManagementSystem/scripts/payroll_backup.py

490 lines
18 KiB
Python
Raw Normal View History

"""跨平台 MySQL 备份:一致位点、binlog 完整链、独立容器恢复、只读检查。"""
import argparse
import gzip
import hashlib
import json
import os
import re
import secrets
import shutil
import subprocess
import sys
import time
from datetime import UTC, datetime
from itertools import pairwise
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
IDENT = re.compile(r"^[A-Za-z][A-Za-z0-9_]{0,63}$")
LOG = re.compile(r"^binlog\.\d{6,}$")
class BackupError(RuntimeError):
pass
def run(args, *, data=None, env=None):
result = subprocess.run(args, input=data, capture_output=True, env=env, check=False)
if result.returncode:
# 原始数据库错误可能含业务内容;只报告操作及退出状态。
raise BackupError(f"{args[0]} 操作失败,退出码 {result.returncode}")
return result.stdout
def mysql(container, sql, database=None):
args = [
"docker",
"exec",
"-i",
container,
"sh",
"-c",
# 官方镜像初始化期间会启动禁用 TCP 的临时服务,不能把它当作就绪实例。
'export MYSQL_PWD="$MYSQL_ROOT_PASSWORD"; '
'exec mysql -uroot --host=127.0.0.1 --protocol=TCP -N -B "$@"',
"mysql",
]
if database:
args.append(database)
return run(args, data=sql.encode()).decode("utf-8").strip()
def json_write(path, value):
temporary = path.with_suffix(path.suffix + ".partial")
temporary.write_text(
json.dumps(value, ensure_ascii=False, indent=2), encoding="utf-8"
)
temporary.replace(path)
def checksum(path):
with path.open("rb") as stream:
return hashlib.file_digest(stream, "sha256").hexdigest()
def stream_to(args, source):
"""按块恢复大库和日志,避免将全部数据加载进内存。"""
with subprocess.Popen(
args,
stdin=subprocess.PIPE,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
) as process:
shutil.copyfileobj(source, process.stdin, 1024 * 1024)
process.stdin.close()
if process.wait() != 0:
raise BackupError("恢复数据流执行失败")
def logs(container):
rows = [r.split("\t")[0] for r in mysql(container, "SHOW BINARY LOGS").splitlines()]
if not rows or any(not LOG.fullmatch(r) for r in rows):
raise BackupError("binlog 未开启或文件名不受支持")
return rows
def position(container):
parts = mysql(container, "SHOW MASTER STATUS").split("\t")
if len(parts) < 2 or not LOG.fullmatch(parts[0]):
raise BackupError("无法获得 binlog 位点")
return {"file": parts[0], "position": int(parts[1])}
def stream_gzip(args, out):
temporary = out.with_suffix(out.suffix + ".partial")
with subprocess.Popen(
args, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL
) as process:
with gzip.open(temporary, "wb") as dest:
shutil.copyfileobj(process.stdout, dest, 1024 * 1024)
if process.wait() != 0:
raise BackupError("备份流失败,未发布成功清单,临时文件保留供核对")
temporary.replace(out)
return {"file": out.name, "sha256": checksum(out)}
def full(args):
stamp = (
datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
+ "_"
+ secrets.token_hex(3)
)
out = args.directory / f"full_{stamp}.sql.gz"
server = mysql(args.container, "SELECT @@server_uuid")
binary = logs(args.container) # 关闭 binlog 时不生成无法续接的全量。
command = [
"docker",
"exec",
args.container,
"sh",
"-c",
(
'export MYSQL_PWD="$MYSQL_ROOT_PASSWORD"; exec mysqldump -uroot '
"--single-transaction --source-data=2 --set-gtid-purged=OFF "
'--routines --events --triggers --hex-blob --no-tablespaces "$1"'
),
"backup",
args.database,
]
entry = stream_gzip(command, out)
point, completed, tables = None, False, []
with gzip.open(out, "rt", encoding="utf-8") as stream:
for line in stream:
match = re.search(
r"(?:SOURCE|MASTER)_LOG_FILE='([^']+)', (?:SOURCE|MASTER)_LOG_POS=(\d+)",
line,
)
if match:
point = {"file": match[1], "position": int(match[2])}
table = re.match(r"CREATE TABLE `([^`]+)`", line)
if table:
tables.append(table[1])
completed = completed or line.startswith("-- Dump completed on ")
if not completed or not point or not tables or not LOG.fullmatch(point["file"]):
raise BackupError("全量缺少完成标识、业务表或一致快照位点,不能作为恢复基线")
manifest = {
"kind": "FULL",
"database": args.database,
"server_uuid": server,
"created_at": datetime.now(UTC).isoformat(),
"image": args.image,
"point": point,
"entries": [entry],
"tables": sorted(tables),
"logs_seen": binary,
}
name = f"full_{stamp}.json"
json_write(args.directory / name, manifest)
json_write(
args.directory / "state.json",
{
"full": name,
"point": point,
"increments": [],
"checked_at": datetime.now(UTC).isoformat(),
},
)
return {"status": "SUCCEEDED", "manifest": name, "point": point}
def load_manifest(directory, name):
if Path(name).name != name:
raise BackupError("备份清单必须是备份目录内的文件名")
value = json.loads((directory / name).read_text(encoding="utf-8"))
for entry in value["entries"]:
filename = entry["file"]
if (
Path(filename).name != filename
or checksum(directory / filename) != entry["sha256"]
):
raise BackupError("备份文件缺失或 SHA-256 不符")
return value
def incremental(args):
state_path = args.directory / "state.json"
if not state_path.exists():
raise BackupError("请先完成全量备份")
state = json.loads(state_path.read_text(encoding="utf-8"))
baseline = load_manifest(args.directory, state["full"])
if baseline["database"] != args.database or baseline["server_uuid"] != mysql(
args.container, "SELECT @@server_uuid"
):
raise BackupError("数据库或服务器身份已变化,必须先建立新的全量基线")
current = position(args.container)
if state["point"] == current:
state["checked_at"] = datetime.now(UTC).isoformat()
json_write(state_path, state)
return {"status": "UNCHANGED", "point": current}
available = logs(args.container)
if state["point"]["file"] not in available:
raise BackupError("增量链起点已被清理,必须重新全量备份")
mysql(args.container, "FLUSH BINARY LOGS")
end = position(args.container)
available = logs(args.container)
names = available[
available.index(state["point"]["file"]) : available.index(end["file"]) + 1
]
numbers = [int(n.split(".")[1]) for n in names]
if any(b != a + 1 for a, b in pairwise(numbers)):
raise BackupError("binlog 序号存在缺口,拒绝发布增量成功")
stamp = secrets.token_hex(8)
entries = []
for name in names:
# 轮转后读取位点前仍可能有事务写入新文件;末文件也要归档到该位点,
# 否则下一次从 end 续接时会永久跳过这些事务。固定字节边界避免重复重放。
command = ["docker", "exec", args.container]
if name == end["file"]:
command.extend(["head", "-c", str(end["position"]), f"/var/lib/mysql/{name}"])
else:
command.extend(["cat", f"/var/lib/mysql/{name}"])
entry = stream_gzip(
command,
args.directory / f"{stamp}_{name}.gz",
)
entry["binlog"] = name
entries.append(entry)
manifest_name = f"increment_{stamp}.json"
manifest = {
"kind": "INCREMENTAL",
"full": state["full"],
"start": state["point"],
"end": end,
"entries": entries,
"created_at": datetime.now(UTC).isoformat(),
}
json_write(args.directory / manifest_name, manifest)
state["increments"].append(manifest_name)
state["point"] = end
state["checked_at"] = datetime.now(UTC).isoformat()
json_write(state_path, state)
return {"status": "SUCCEEDED", "manifest": manifest_name, "files": len(entries)}
def checked_chain(args):
state = json.loads((args.directory / "state.json").read_text(encoding="utf-8"))
baseline = load_manifest(args.directory, state["full"])
point, increments = baseline["point"], []
for name in state["increments"]:
change = load_manifest(args.directory, name)
if change["full"] != state["full"] or change["start"] != point:
raise BackupError("增量清单不连续或跨全量基线")
increments.append(change)
point = change["end"]
if point != state["point"]:
raise BackupError("增量末位点与状态不一致")
return baseline, increments
def restore(args):
"""只创建随机命名、独立可写层的临时 MySQL;不接触已有数据卷。"""
baseline, increments = checked_chain(args)
target = "payroll-drill-" + secrets.token_hex(8)
env = os.environ | {"MYSQL_ROOT_PASSWORD": secrets.token_urlsafe(32)}
volume = target + "-data"
run(["docker", "volume", "create", "--label", "payroll.restore-drill=true", volume])
run(
[
"docker",
"run",
"-d",
"--name",
target,
"--label",
"payroll.restore-drill=true",
"--mount",
f"type=volume,source={volume},target=/var/lib/mysql",
"--env",
"MYSQL_ROOT_PASSWORD",
args.image,
],
env=env,
)
try:
for _ in range(300):
try:
mysql(target, "SELECT 1")
break
except BackupError:
time.sleep(2)
else:
raise BackupError("隔离恢复容器未就绪")
database = baseline["database"]
if not IDENT.fullmatch(database):
raise BackupError("备份数据库名称无效")
mysql(target, f"CREATE DATABASE `{database}` CHARACTER SET utf8mb4")
command = [
"docker",
"exec",
"-i",
target,
"sh",
"-c",
'export MYSQL_PWD="$MYSQL_ROOT_PASSWORD"; exec mysql -uroot "$1"',
"restore",
database,
]
with subprocess.Popen(
command,
stdin=subprocess.PIPE,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
) as process:
with gzip.open(
args.directory / baseline["entries"][0]["file"], "rb"
) as source:
shutil.copyfileobj(source, process.stdin, 1024 * 1024)
process.stdin.close()
if process.wait() != 0:
raise BackupError("隔离全量恢复失败")
for change in increments:
paths = []
for entry in change["entries"]:
name = entry["binlog"]
if not LOG.fullmatch(name):
raise BackupError("增量文件名无效")
with gzip.open(args.directory / entry["file"], "rb") as source:
stream_to(
[
"docker",
"exec",
"-i",
target,
"sh",
"-c",
'cat > "/tmp/$1"',
"archive",
name,
],
source,
)
paths.append(f"/tmp/{name}")
if paths:
# mysqlbinlog 必须在同一次调用中按顺序消费多个文件,保留临时表上下文。
decoder_command = [
"docker",
"exec",
target,
"mysqlbinlog",
"--disable-log-bin",
f"--database={database}",
f"--start-position={change['start']['position']}",
*paths,
]
with subprocess.Popen(
decoder_command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL
) as decoder:
with subprocess.Popen(
command,
stdin=decoder.stdout,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
) as consumer:
decoder.stdout.close()
consumed = consumer.wait()
if decoder.wait() or consumed:
raise BackupError("增量重放失败,不能将恢复标为成功")
tables = mysql(target, "SHOW TABLES", database).splitlines()
if not increments and sorted(tables) != baseline["tables"]:
raise BackupError("恢复后表清单不符")
counts = {
table: int(mysql(target, f"SELECT COUNT(*) FROM `{table}`", database))
for table in tables
if IDENT.fullmatch(table)
}
result = {
"status": "SUCCEEDED",
"container": target,
"database": database,
"increments": len(increments),
"table_counts": counts,
"scope": "独立容器;生产库未覆盖",
}
json_write(args.directory / f"{target}.json", result)
return result
finally:
# 仅停止本次随机创建且带标签的容器,保留恢复内容供复核,不批量删除历史演练。
run(["docker", "stop", target])
def check(args):
baseline, increments = checked_chain(args)
if baseline["database"] != args.database:
raise BackupError("备份数据库与检查目标不符")
age = (
datetime.now(UTC) - datetime.fromisoformat(baseline["created_at"])
).total_seconds()
if age > 26 * 3600:
raise BackupError("最新全量备份超过 26 小时")
uuid = mysql(args.container, "SELECT @@server_uuid")
if uuid != baseline["server_uuid"]:
raise BackupError("运行服务器与备份身份不符")
if mysql(args.container, "SELECT @@innodb_force_recovery") != "0":
raise BackupError("数据库仍处于强制恢复模式")
state = json.loads((args.directory / "state.json").read_text(encoding="utf-8"))
checked_at = state.get("checked_at", baseline["created_at"])
if (
datetime.now(UTC) - datetime.fromisoformat(checked_at)
).total_seconds() > 30 * 60:
raise BackupError("增量采集检查超过 30 分钟未运行")
redis = getattr(args, "redis_container", None)
if redis:
pong = run(
[
"docker",
"exec",
redis,
"sh",
"-c",
'REDISCLI_AUTH="$REDIS_PASSWORD" redis-cli --no-auth-warning PING',
]
)
if pong.strip() != b"PONG":
raise BackupError("Redis 探活失败")
stat = shutil.disk_usage(args.directory)
if stat.free / stat.total < 0.1:
raise BackupError("备份盘可用空间低于 10%")
return {
"status": "SUCCEEDED",
"full_age_hours": round(age / 3600, 2),
"increment_count": len(increments),
"free_bytes": stat.free,
}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"action", choices=["full", "incremental", "check", "restore-drill"]
)
parser.add_argument("--container", default="ys-mysql-dev")
parser.add_argument("--database", default="payroll")
parser.add_argument("--directory", type=Path, default=ROOT / "db/backups")
parser.add_argument("--image", default="payroll-mysql-backup:8.0.46")
parser.add_argument("--redis-container", default="ys-redis-dev")
args = parser.parse_args()
if not IDENT.fullmatch(args.database) or not re.fullmatch(
r"[A-Za-z0-9_.-]+", args.container
):
parser.error("容器名或数据库名无效")
args.directory.mkdir(parents=True, exist_ok=True)
lock = args.directory / ".backup.lock"
try:
with lock.open("x", encoding="utf-8") as stream:
stream.write(str(os.getpid()))
except FileExistsError:
print("备份目录已有活动锁,请核实上次进程状态", file=sys.stderr)
return 1
try:
result = {
"full": full,
"incremental": incremental,
"check": check,
"restore-drill": restore,
}[args.action](args)
json_write(
args.directory / f"last-{args.action}-run.json",
{"finished_at": datetime.now(UTC).isoformat(), **result},
)
print(json.dumps(result, ensure_ascii=False))
return 0
except (BackupError, OSError, ValueError, KeyError) as error:
json_write(
args.directory / "last-failure.json",
{
"finished_at": datetime.now(UTC).isoformat(),
"action": args.action,
"status": "FAILED",
"reason": str(error),
},
)
print(f"操作失败:{error}", file=sys.stderr)
return 1
finally:
lock.unlink()
if __name__ == "__main__":
raise SystemExit(main())