ae12403c0b
统一跨平台备份入口，校验快照位点、完整性及增量链后隔离恢复。 末个 binlog 按读取位点归档，避免轮转后并发事务被续接跳过。 修复 Windows 安装器的编码、默认路径和花括号路径参数。
490 lines
18 KiB
Python
490 lines
18 KiB
Python
"""跨平台 MySQL 备份：一致位点、binlog 完整链、独立容器恢复、只读检查。"""
|
||
|
||
import argparse
|
||
import gzip
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import re
|
||
import secrets
|
||
import shutil
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
from datetime import UTC, datetime
|
||
from itertools import pairwise
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
IDENT = re.compile(r"^[A-Za-z][A-Za-z0-9_]{0,63}$")
|
||
LOG = re.compile(r"^binlog\.\d{6,}$")
|
||
|
||
|
||
class BackupError(RuntimeError):
|
||
pass
|
||
|
||
|
||
def run(args, *, data=None, env=None):
|
||
result = subprocess.run(args, input=data, capture_output=True, env=env, check=False)
|
||
if result.returncode:
|
||
# 原始数据库错误可能含业务内容；只报告操作及退出状态。
|
||
raise BackupError(f"{args[0]} 操作失败，退出码 {result.returncode}")
|
||
return result.stdout
|
||
|
||
|
||
def mysql(container, sql, database=None):
|
||
args = [
|
||
"docker",
|
||
"exec",
|
||
"-i",
|
||
container,
|
||
"sh",
|
||
"-c",
|
||
# 官方镜像初始化期间会启动禁用 TCP 的临时服务，不能把它当作就绪实例。
|
||
'export MYSQL_PWD="$MYSQL_ROOT_PASSWORD"; '
|
||
'exec mysql -uroot --host=127.0.0.1 --protocol=TCP -N -B "$@"',
|
||
"mysql",
|
||
]
|
||
if database:
|
||
args.append(database)
|
||
return run(args, data=sql.encode()).decode("utf-8").strip()
|
||
|
||
|
||
def json_write(path, value):
|
||
temporary = path.with_suffix(path.suffix + ".partial")
|
||
temporary.write_text(
|
||
json.dumps(value, ensure_ascii=False, indent=2), encoding="utf-8"
|
||
)
|
||
temporary.replace(path)
|
||
|
||
|
||
def checksum(path):
|
||
with path.open("rb") as stream:
|
||
return hashlib.file_digest(stream, "sha256").hexdigest()
|
||
|
||
|
||
def stream_to(args, source):
|
||
"""按块恢复大库和日志，避免将全部数据加载进内存。"""
|
||
with subprocess.Popen(
|
||
args,
|
||
stdin=subprocess.PIPE,
|
||
stdout=subprocess.DEVNULL,
|
||
stderr=subprocess.DEVNULL,
|
||
) as process:
|
||
shutil.copyfileobj(source, process.stdin, 1024 * 1024)
|
||
process.stdin.close()
|
||
if process.wait() != 0:
|
||
raise BackupError("恢复数据流执行失败")
|
||
|
||
|
||
def logs(container):
|
||
rows = [r.split("\t")[0] for r in mysql(container, "SHOW BINARY LOGS").splitlines()]
|
||
if not rows or any(not LOG.fullmatch(r) for r in rows):
|
||
raise BackupError("binlog 未开启或文件名不受支持")
|
||
return rows
|
||
|
||
|
||
def position(container):
|
||
parts = mysql(container, "SHOW MASTER STATUS").split("\t")
|
||
if len(parts) < 2 or not LOG.fullmatch(parts[0]):
|
||
raise BackupError("无法获得 binlog 位点")
|
||
return {"file": parts[0], "position": int(parts[1])}
|
||
|
||
|
||
def stream_gzip(args, out):
|
||
temporary = out.with_suffix(out.suffix + ".partial")
|
||
with subprocess.Popen(
|
||
args, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL
|
||
) as process:
|
||
with gzip.open(temporary, "wb") as dest:
|
||
shutil.copyfileobj(process.stdout, dest, 1024 * 1024)
|
||
if process.wait() != 0:
|
||
raise BackupError("备份流失败，未发布成功清单，临时文件保留供核对")
|
||
temporary.replace(out)
|
||
return {"file": out.name, "sha256": checksum(out)}
|
||
|
||
|
||
def full(args):
|
||
stamp = (
|
||
datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
||
+ "_"
|
||
+ secrets.token_hex(3)
|
||
)
|
||
out = args.directory / f"full_{stamp}.sql.gz"
|
||
server = mysql(args.container, "SELECT @@server_uuid")
|
||
binary = logs(args.container) # 关闭 binlog 时不生成无法续接的全量。
|
||
command = [
|
||
"docker",
|
||
"exec",
|
||
args.container,
|
||
"sh",
|
||
"-c",
|
||
(
|
||
'export MYSQL_PWD="$MYSQL_ROOT_PASSWORD"; exec mysqldump -uroot '
|
||
"--single-transaction --source-data=2 --set-gtid-purged=OFF "
|
||
'--routines --events --triggers --hex-blob --no-tablespaces "$1"'
|
||
),
|
||
"backup",
|
||
args.database,
|
||
]
|
||
entry = stream_gzip(command, out)
|
||
point, completed, tables = None, False, []
|
||
with gzip.open(out, "rt", encoding="utf-8") as stream:
|
||
for line in stream:
|
||
match = re.search(
|
||
r"(?:SOURCE|MASTER)_LOG_FILE='([^']+)', (?:SOURCE|MASTER)_LOG_POS=(\d+)",
|
||
line,
|
||
)
|
||
if match:
|
||
point = {"file": match[1], "position": int(match[2])}
|
||
table = re.match(r"CREATE TABLE `([^`]+)`", line)
|
||
if table:
|
||
tables.append(table[1])
|
||
completed = completed or line.startswith("-- Dump completed on ")
|
||
if not completed or not point or not tables or not LOG.fullmatch(point["file"]):
|
||
raise BackupError("全量缺少完成标识、业务表或一致快照位点，不能作为恢复基线")
|
||
manifest = {
|
||
"kind": "FULL",
|
||
"database": args.database,
|
||
"server_uuid": server,
|
||
"created_at": datetime.now(UTC).isoformat(),
|
||
"image": args.image,
|
||
"point": point,
|
||
"entries": [entry],
|
||
"tables": sorted(tables),
|
||
"logs_seen": binary,
|
||
}
|
||
name = f"full_{stamp}.json"
|
||
json_write(args.directory / name, manifest)
|
||
json_write(
|
||
args.directory / "state.json",
|
||
{
|
||
"full": name,
|
||
"point": point,
|
||
"increments": [],
|
||
"checked_at": datetime.now(UTC).isoformat(),
|
||
},
|
||
)
|
||
return {"status": "SUCCEEDED", "manifest": name, "point": point}
|
||
|
||
|
||
def load_manifest(directory, name):
|
||
if Path(name).name != name:
|
||
raise BackupError("备份清单必须是备份目录内的文件名")
|
||
value = json.loads((directory / name).read_text(encoding="utf-8"))
|
||
for entry in value["entries"]:
|
||
filename = entry["file"]
|
||
if (
|
||
Path(filename).name != filename
|
||
or checksum(directory / filename) != entry["sha256"]
|
||
):
|
||
raise BackupError("备份文件缺失或 SHA-256 不符")
|
||
return value
|
||
|
||
|
||
def incremental(args):
|
||
state_path = args.directory / "state.json"
|
||
if not state_path.exists():
|
||
raise BackupError("请先完成全量备份")
|
||
state = json.loads(state_path.read_text(encoding="utf-8"))
|
||
baseline = load_manifest(args.directory, state["full"])
|
||
if baseline["database"] != args.database or baseline["server_uuid"] != mysql(
|
||
args.container, "SELECT @@server_uuid"
|
||
):
|
||
raise BackupError("数据库或服务器身份已变化，必须先建立新的全量基线")
|
||
current = position(args.container)
|
||
if state["point"] == current:
|
||
state["checked_at"] = datetime.now(UTC).isoformat()
|
||
json_write(state_path, state)
|
||
return {"status": "UNCHANGED", "point": current}
|
||
available = logs(args.container)
|
||
if state["point"]["file"] not in available:
|
||
raise BackupError("增量链起点已被清理，必须重新全量备份")
|
||
mysql(args.container, "FLUSH BINARY LOGS")
|
||
end = position(args.container)
|
||
available = logs(args.container)
|
||
names = available[
|
||
available.index(state["point"]["file"]) : available.index(end["file"]) + 1
|
||
]
|
||
numbers = [int(n.split(".")[1]) for n in names]
|
||
if any(b != a + 1 for a, b in pairwise(numbers)):
|
||
raise BackupError("binlog 序号存在缺口，拒绝发布增量成功")
|
||
stamp = secrets.token_hex(8)
|
||
entries = []
|
||
for name in names:
|
||
# 轮转后读取位点前仍可能有事务写入新文件；末文件也要归档到该位点，
|
||
# 否则下一次从 end 续接时会永久跳过这些事务。固定字节边界避免重复重放。
|
||
command = ["docker", "exec", args.container]
|
||
if name == end["file"]:
|
||
command.extend(["head", "-c", str(end["position"]), f"/var/lib/mysql/{name}"])
|
||
else:
|
||
command.extend(["cat", f"/var/lib/mysql/{name}"])
|
||
entry = stream_gzip(
|
||
command,
|
||
args.directory / f"{stamp}_{name}.gz",
|
||
)
|
||
entry["binlog"] = name
|
||
entries.append(entry)
|
||
manifest_name = f"increment_{stamp}.json"
|
||
manifest = {
|
||
"kind": "INCREMENTAL",
|
||
"full": state["full"],
|
||
"start": state["point"],
|
||
"end": end,
|
||
"entries": entries,
|
||
"created_at": datetime.now(UTC).isoformat(),
|
||
}
|
||
json_write(args.directory / manifest_name, manifest)
|
||
state["increments"].append(manifest_name)
|
||
state["point"] = end
|
||
state["checked_at"] = datetime.now(UTC).isoformat()
|
||
json_write(state_path, state)
|
||
return {"status": "SUCCEEDED", "manifest": manifest_name, "files": len(entries)}
|
||
|
||
|
||
def checked_chain(args):
|
||
state = json.loads((args.directory / "state.json").read_text(encoding="utf-8"))
|
||
baseline = load_manifest(args.directory, state["full"])
|
||
point, increments = baseline["point"], []
|
||
for name in state["increments"]:
|
||
change = load_manifest(args.directory, name)
|
||
if change["full"] != state["full"] or change["start"] != point:
|
||
raise BackupError("增量清单不连续或跨全量基线")
|
||
increments.append(change)
|
||
point = change["end"]
|
||
if point != state["point"]:
|
||
raise BackupError("增量末位点与状态不一致")
|
||
return baseline, increments
|
||
|
||
|
||
def restore(args):
|
||
"""只创建随机命名、独立可写层的临时 MySQL；不接触已有数据卷。"""
|
||
baseline, increments = checked_chain(args)
|
||
target = "payroll-drill-" + secrets.token_hex(8)
|
||
env = os.environ | {"MYSQL_ROOT_PASSWORD": secrets.token_urlsafe(32)}
|
||
volume = target + "-data"
|
||
run(["docker", "volume", "create", "--label", "payroll.restore-drill=true", volume])
|
||
run(
|
||
[
|
||
"docker",
|
||
"run",
|
||
"-d",
|
||
"--name",
|
||
target,
|
||
"--label",
|
||
"payroll.restore-drill=true",
|
||
"--mount",
|
||
f"type=volume,source={volume},target=/var/lib/mysql",
|
||
"--env",
|
||
"MYSQL_ROOT_PASSWORD",
|
||
args.image,
|
||
],
|
||
env=env,
|
||
)
|
||
try:
|
||
for _ in range(300):
|
||
try:
|
||
mysql(target, "SELECT 1")
|
||
break
|
||
except BackupError:
|
||
time.sleep(2)
|
||
else:
|
||
raise BackupError("隔离恢复容器未就绪")
|
||
database = baseline["database"]
|
||
if not IDENT.fullmatch(database):
|
||
raise BackupError("备份数据库名称无效")
|
||
mysql(target, f"CREATE DATABASE `{database}` CHARACTER SET utf8mb4")
|
||
command = [
|
||
"docker",
|
||
"exec",
|
||
"-i",
|
||
target,
|
||
"sh",
|
||
"-c",
|
||
'export MYSQL_PWD="$MYSQL_ROOT_PASSWORD"; exec mysql -uroot "$1"',
|
||
"restore",
|
||
database,
|
||
]
|
||
with subprocess.Popen(
|
||
command,
|
||
stdin=subprocess.PIPE,
|
||
stdout=subprocess.DEVNULL,
|
||
stderr=subprocess.DEVNULL,
|
||
) as process:
|
||
with gzip.open(
|
||
args.directory / baseline["entries"][0]["file"], "rb"
|
||
) as source:
|
||
shutil.copyfileobj(source, process.stdin, 1024 * 1024)
|
||
process.stdin.close()
|
||
if process.wait() != 0:
|
||
raise BackupError("隔离全量恢复失败")
|
||
for change in increments:
|
||
paths = []
|
||
for entry in change["entries"]:
|
||
name = entry["binlog"]
|
||
if not LOG.fullmatch(name):
|
||
raise BackupError("增量文件名无效")
|
||
with gzip.open(args.directory / entry["file"], "rb") as source:
|
||
stream_to(
|
||
[
|
||
"docker",
|
||
"exec",
|
||
"-i",
|
||
target,
|
||
"sh",
|
||
"-c",
|
||
'cat > "/tmp/$1"',
|
||
"archive",
|
||
name,
|
||
],
|
||
source,
|
||
)
|
||
paths.append(f"/tmp/{name}")
|
||
if paths:
|
||
# mysqlbinlog 必须在同一次调用中按顺序消费多个文件，保留临时表上下文。
|
||
decoder_command = [
|
||
"docker",
|
||
"exec",
|
||
target,
|
||
"mysqlbinlog",
|
||
"--disable-log-bin",
|
||
f"--database={database}",
|
||
f"--start-position={change['start']['position']}",
|
||
*paths,
|
||
]
|
||
with subprocess.Popen(
|
||
decoder_command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL
|
||
) as decoder:
|
||
with subprocess.Popen(
|
||
command,
|
||
stdin=decoder.stdout,
|
||
stdout=subprocess.DEVNULL,
|
||
stderr=subprocess.DEVNULL,
|
||
) as consumer:
|
||
decoder.stdout.close()
|
||
consumed = consumer.wait()
|
||
if decoder.wait() or consumed:
|
||
raise BackupError("增量重放失败，不能将恢复标为成功")
|
||
tables = mysql(target, "SHOW TABLES", database).splitlines()
|
||
if not increments and sorted(tables) != baseline["tables"]:
|
||
raise BackupError("恢复后表清单不符")
|
||
counts = {
|
||
table: int(mysql(target, f"SELECT COUNT(*) FROM `{table}`", database))
|
||
for table in tables
|
||
if IDENT.fullmatch(table)
|
||
}
|
||
result = {
|
||
"status": "SUCCEEDED",
|
||
"container": target,
|
||
"database": database,
|
||
"increments": len(increments),
|
||
"table_counts": counts,
|
||
"scope": "独立容器；生产库未覆盖",
|
||
}
|
||
json_write(args.directory / f"{target}.json", result)
|
||
return result
|
||
finally:
|
||
# 仅停止本次随机创建且带标签的容器，保留恢复内容供复核，不批量删除历史演练。
|
||
run(["docker", "stop", target])
|
||
|
||
|
||
def check(args):
|
||
baseline, increments = checked_chain(args)
|
||
if baseline["database"] != args.database:
|
||
raise BackupError("备份数据库与检查目标不符")
|
||
age = (
|
||
datetime.now(UTC) - datetime.fromisoformat(baseline["created_at"])
|
||
).total_seconds()
|
||
if age > 26 * 3600:
|
||
raise BackupError("最新全量备份超过 26 小时")
|
||
uuid = mysql(args.container, "SELECT @@server_uuid")
|
||
if uuid != baseline["server_uuid"]:
|
||
raise BackupError("运行服务器与备份身份不符")
|
||
if mysql(args.container, "SELECT @@innodb_force_recovery") != "0":
|
||
raise BackupError("数据库仍处于强制恢复模式")
|
||
state = json.loads((args.directory / "state.json").read_text(encoding="utf-8"))
|
||
checked_at = state.get("checked_at", baseline["created_at"])
|
||
if (
|
||
datetime.now(UTC) - datetime.fromisoformat(checked_at)
|
||
).total_seconds() > 30 * 60:
|
||
raise BackupError("增量采集检查超过 30 分钟未运行")
|
||
redis = getattr(args, "redis_container", None)
|
||
if redis:
|
||
pong = run(
|
||
[
|
||
"docker",
|
||
"exec",
|
||
redis,
|
||
"sh",
|
||
"-c",
|
||
'REDISCLI_AUTH="$REDIS_PASSWORD" redis-cli --no-auth-warning PING',
|
||
]
|
||
)
|
||
if pong.strip() != b"PONG":
|
||
raise BackupError("Redis 探活失败")
|
||
stat = shutil.disk_usage(args.directory)
|
||
if stat.free / stat.total < 0.1:
|
||
raise BackupError("备份盘可用空间低于 10%")
|
||
return {
|
||
"status": "SUCCEEDED",
|
||
"full_age_hours": round(age / 3600, 2),
|
||
"increment_count": len(increments),
|
||
"free_bytes": stat.free,
|
||
}
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
parser.add_argument(
|
||
"action", choices=["full", "incremental", "check", "restore-drill"]
|
||
)
|
||
parser.add_argument("--container", default="ys-mysql-dev")
|
||
parser.add_argument("--database", default="payroll")
|
||
parser.add_argument("--directory", type=Path, default=ROOT / "db/backups")
|
||
parser.add_argument("--image", default="payroll-mysql-backup:8.0.46")
|
||
parser.add_argument("--redis-container", default="ys-redis-dev")
|
||
args = parser.parse_args()
|
||
if not IDENT.fullmatch(args.database) or not re.fullmatch(
|
||
r"[A-Za-z0-9_.-]+", args.container
|
||
):
|
||
parser.error("容器名或数据库名无效")
|
||
args.directory.mkdir(parents=True, exist_ok=True)
|
||
lock = args.directory / ".backup.lock"
|
||
try:
|
||
with lock.open("x", encoding="utf-8") as stream:
|
||
stream.write(str(os.getpid()))
|
||
except FileExistsError:
|
||
print("备份目录已有活动锁，请核实上次进程状态", file=sys.stderr)
|
||
return 1
|
||
try:
|
||
result = {
|
||
"full": full,
|
||
"incremental": incremental,
|
||
"check": check,
|
||
"restore-drill": restore,
|
||
}[args.action](args)
|
||
json_write(
|
||
args.directory / f"last-{args.action}-run.json",
|
||
{"finished_at": datetime.now(UTC).isoformat(), **result},
|
||
)
|
||
print(json.dumps(result, ensure_ascii=False))
|
||
return 0
|
||
except (BackupError, OSError, ValueError, KeyError) as error:
|
||
json_write(
|
||
args.directory / "last-failure.json",
|
||
{
|
||
"finished_at": datetime.now(UTC).isoformat(),
|
||
"action": args.action,
|
||
"status": "FAILED",
|
||
"reason": str(error),
|
||
},
|
||
)
|
||
print(f"操作失败：{error}", file=sys.stderr)
|
||
return 1
|
||
finally:
|
||
lock.unlink()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|