Add DB+.env auto backup with download and upload restore in Settings.
Backups land under /root/eth_hedge_backups; restore accepts zip body for new-server migration and restarts the process. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
dekun
@@ -0,0 +1,330 @@
|
||||
"""数据库 + .env 备份 / 恢复（自动落盘到 /root/eth_hedge_backups）。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import time
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .config import get_settings
|
||||
from .credentials import resolve_env_file_path
|
||||
from .models.db import Database, get_db
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MANIFEST_NAME = "manifest.json"
|
||||
DB_ARCNAME = "hedge.db"
|
||||
ENV_ARCNAME = ".env"
|
||||
BACKUP_PREFIX = "eth_hedge_backup_"
|
||||
DEFAULT_KEEP = 14
|
||||
DEFAULT_INTERVAL_HOURS = 24
|
||||
|
||||
|
||||
def resolve_backup_dir() -> Path:
|
||||
"""优先 BACKUP_DIR；Linux 默认 /root/eth_hedge_backups；否则用户目录。"""
|
||||
override = (os.environ.get("BACKUP_DIR") or "").strip()
|
||||
if override:
|
||||
p = Path(override)
|
||||
p.mkdir(parents=True, exist_ok=True)
|
||||
return p
|
||||
if os.name != "nt":
|
||||
root = Path("/root/eth_hedge_backups")
|
||||
try:
|
||||
if Path("/root").is_dir():
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
return root
|
||||
except OSError as e:
|
||||
logger.warning("cannot use /root/eth_hedge_backups: %s", e)
|
||||
p = Path.home() / "eth_hedge_backups"
|
||||
p.mkdir(parents=True, exist_ok=True)
|
||||
return p
|
||||
|
||||
|
||||
def resolve_env_path() -> Path | None:
|
||||
p = resolve_env_file_path()
|
||||
return p
|
||||
|
||||
|
||||
def _settings_int(db: Database, key: str, default: int) -> int:
|
||||
raw = db.get_setting(key, str(default))
|
||||
try:
|
||||
return int(float(raw or default))
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _settings_bool(db: Database, key: str, default: bool) -> bool:
|
||||
raw = db.get_setting(key, "1" if default else "0")
|
||||
if raw is None or raw == "":
|
||||
return default
|
||||
return str(raw).strip().lower() in ("1", "true", "yes", "on")
|
||||
|
||||
|
||||
def backup_settings(db: Database | None = None) -> dict[str, Any]:
|
||||
db = db or get_db()
|
||||
return {
|
||||
"auto_enabled": _settings_bool(db, "backup_auto_enabled", True),
|
||||
"interval_hours": max(1, _settings_int(db, "backup_interval_hours", DEFAULT_INTERVAL_HOURS)),
|
||||
"keep_count": max(1, min(90, _settings_int(db, "backup_keep_count", DEFAULT_KEEP))),
|
||||
"last_at_ms": _settings_int(db, "backup_last_at_ms", 0) or None,
|
||||
"backup_dir": str(resolve_backup_dir()),
|
||||
}
|
||||
|
||||
|
||||
def list_backups() -> list[dict[str, Any]]:
|
||||
d = resolve_backup_dir()
|
||||
items: list[dict[str, Any]] = []
|
||||
if not d.is_dir():
|
||||
return items
|
||||
for p in sorted(d.glob(f"{BACKUP_PREFIX}*.zip"), key=lambda x: x.stat().st_mtime, reverse=True):
|
||||
st = p.stat()
|
||||
items.append(
|
||||
{
|
||||
"name": p.name,
|
||||
"path": str(p),
|
||||
"size_bytes": st.st_size,
|
||||
"mtime_ms": int(st.st_mtime * 1000),
|
||||
}
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
def prune_backups(keep: int | None = None) -> int:
|
||||
keep_n = keep if keep is not None else backup_settings()["keep_count"]
|
||||
items = list_backups()
|
||||
removed = 0
|
||||
for old in items[int(keep_n) :]:
|
||||
try:
|
||||
Path(old["path"]).unlink(missing_ok=True)
|
||||
removed += 1
|
||||
except OSError as e:
|
||||
logger.warning("prune backup failed %s: %s", old.get("name"), e)
|
||||
return removed
|
||||
|
||||
|
||||
def _sqlite_snapshot(db: Database, dest: Path) -> None:
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
if dest.exists():
|
||||
dest.unlink()
|
||||
with db._lock:
|
||||
dst = sqlite3.connect(str(dest))
|
||||
try:
|
||||
db._conn.backup(dst)
|
||||
dst.commit()
|
||||
finally:
|
||||
dst.close()
|
||||
|
||||
|
||||
def create_backup(
|
||||
*,
|
||||
db: Database | None = None,
|
||||
reason: str = "manual",
|
||||
) -> dict[str, Any]:
|
||||
db = db or get_db()
|
||||
backup_dir = resolve_backup_dir()
|
||||
ts = time.strftime("%Y%m%d_%H%M%S")
|
||||
out = backup_dir / f"{BACKUP_PREFIX}{ts}.zip"
|
||||
env_path = resolve_env_path()
|
||||
db_path = Path(db.path)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="eth_hedge_bak_") as td:
|
||||
tmp_db = Path(td) / "hedge.db"
|
||||
_sqlite_snapshot(db, tmp_db)
|
||||
manifest = {
|
||||
"product": "比特骆驼自动化对冲系统",
|
||||
"version": 1,
|
||||
"created_at_ms": int(time.time() * 1000),
|
||||
"reason": reason,
|
||||
"db_source": str(db_path),
|
||||
"env_source": str(env_path) if env_path else None,
|
||||
"has_env": bool(env_path and env_path.is_file()),
|
||||
"mode": get_settings().mode,
|
||||
"env_name": get_settings().env_name,
|
||||
}
|
||||
with zipfile.ZipFile(out, "w", compression=zipfile.ZIP_DEFLATED) as zf:
|
||||
zf.write(tmp_db, DB_ARCNAME)
|
||||
if env_path and env_path.is_file():
|
||||
zf.write(env_path, ENV_ARCNAME)
|
||||
zf.writestr(
|
||||
MANIFEST_NAME,
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2),
|
||||
)
|
||||
|
||||
db.set_setting("backup_last_at_ms", str(int(time.time() * 1000)))
|
||||
prune_backups()
|
||||
st = out.stat()
|
||||
logger.info("backup created path=%s reason=%s size=%s", out, reason, st.st_size)
|
||||
return {
|
||||
"ok": True,
|
||||
"name": out.name,
|
||||
"path": str(out),
|
||||
"size_bytes": st.st_size,
|
||||
"mtime_ms": int(st.st_mtime * 1000),
|
||||
"backup_dir": str(backup_dir),
|
||||
"manifest": manifest,
|
||||
}
|
||||
|
||||
|
||||
def read_backup_file(name: str) -> Path:
|
||||
safe = Path(name).name
|
||||
if not safe.startswith(BACKUP_PREFIX) or not safe.endswith(".zip"):
|
||||
raise ValueError("非法备份文件名")
|
||||
path = resolve_backup_dir() / safe
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(f"备份不存在: {safe}")
|
||||
return path
|
||||
|
||||
|
||||
def _extract_backup_zip(zip_path: Path, dest_dir: Path) -> dict[str, Path]:
|
||||
with zipfile.ZipFile(zip_path, "r") as zf:
|
||||
names = set(zf.namelist())
|
||||
if DB_ARCNAME not in names:
|
||||
# 兼容偶发相对路径
|
||||
db_candidates = [n for n in names if n.endswith("hedge.db") or n.endswith(".db")]
|
||||
if not db_candidates:
|
||||
raise ValueError("备份包缺少 hedge.db")
|
||||
db_name = db_candidates[0]
|
||||
else:
|
||||
db_name = DB_ARCNAME
|
||||
env_name = ENV_ARCNAME if ENV_ARCNAME in names else (
|
||||
next((n for n in names if n.endswith(".env") or n == "env"), None)
|
||||
)
|
||||
zf.extract(db_name, dest_dir)
|
||||
db_out = dest_dir / db_name
|
||||
if not db_out.is_file():
|
||||
# zip 内可能带目录
|
||||
found = list(dest_dir.rglob("*.db"))
|
||||
if not found:
|
||||
raise ValueError("解压后未找到数据库文件")
|
||||
db_out = found[0]
|
||||
env_out: Path | None = None
|
||||
if env_name:
|
||||
zf.extract(env_name, dest_dir)
|
||||
env_out = dest_dir / env_name
|
||||
if not env_out.is_file():
|
||||
found_env = list(dest_dir.rglob(".env")) + list(dest_dir.rglob("*.env"))
|
||||
env_out = found_env[0] if found_env else None
|
||||
return {"db": db_out, "env": env_out} # type: ignore[dict-item]
|
||||
|
||||
|
||||
def validate_backup_zip(zip_path: Path) -> dict[str, Any]:
|
||||
with zipfile.ZipFile(zip_path, "r") as zf:
|
||||
names = zf.namelist()
|
||||
has_db = any(n.endswith("hedge.db") or n == DB_ARCNAME or n.endswith(".db") for n in names)
|
||||
has_env = any(n.endswith(".env") or n == ENV_ARCNAME for n in names)
|
||||
manifest = None
|
||||
if MANIFEST_NAME in names:
|
||||
try:
|
||||
manifest = json.loads(zf.read(MANIFEST_NAME).decode("utf-8"))
|
||||
except Exception:
|
||||
manifest = None
|
||||
if not has_db:
|
||||
raise ValueError("备份包无效：缺少数据库")
|
||||
return {"has_db": has_db, "has_env": has_env, "manifest": manifest, "files": names}
|
||||
|
||||
|
||||
def restore_from_zip(
|
||||
zip_path: Path,
|
||||
*,
|
||||
db: Database | None = None,
|
||||
make_safety_backup: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""用备份包覆盖当前 hedge.db 与 .env。调用方应在此前后暂停策略并准备重启进程。"""
|
||||
db = db or get_db()
|
||||
info = validate_backup_zip(zip_path)
|
||||
safety = None
|
||||
if make_safety_backup:
|
||||
try:
|
||||
safety = create_backup(db=db, reason="pre_restore")
|
||||
except Exception as e:
|
||||
logger.warning("pre_restore backup failed: %s", e)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="eth_hedge_restore_") as td:
|
||||
extracted = _extract_backup_zip(zip_path, Path(td))
|
||||
src_db = extracted["db"]
|
||||
src_env = extracted.get("env")
|
||||
|
||||
db_path = Path(db.path)
|
||||
bak_live: Path | None = None
|
||||
db.close()
|
||||
|
||||
if db_path.is_file():
|
||||
bak_live = db_path.with_suffix(db_path.suffix + f".pre_restore_{int(time.time())}")
|
||||
shutil.copy2(db_path, bak_live)
|
||||
shutil.copy2(src_db, db_path)
|
||||
|
||||
env_written = None
|
||||
if src_env and Path(src_env).is_file():
|
||||
env_target = resolve_env_file_path()
|
||||
env_target.parent.mkdir(parents=True, exist_ok=True)
|
||||
if env_target.is_file():
|
||||
shutil.copy2(env_target, env_target.with_suffix(".env.pre_restore"))
|
||||
shutil.copy2(src_env, env_target)
|
||||
try:
|
||||
env_target.chmod(0o600)
|
||||
except Exception:
|
||||
pass
|
||||
env_written = str(env_target)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"db_path": str(db_path),
|
||||
"env_path": env_written,
|
||||
"had_env_in_backup": bool(info.get("has_env")),
|
||||
"safety_backup": safety.get("name") if safety else None,
|
||||
"local_db_copy": str(bak_live) if bak_live and Path(bak_live).exists() else None,
|
||||
"restart_required": True,
|
||||
"manifest": info.get("manifest"),
|
||||
}
|
||||
|
||||
|
||||
_auto_task: asyncio.Task[None] | None = None
|
||||
|
||||
|
||||
async def auto_backup_loop() -> None:
|
||||
"""后台定时备份；默认开启。"""
|
||||
logger.info("auto backup loop started dir=%s", resolve_backup_dir())
|
||||
while True:
|
||||
try:
|
||||
await asyncio.sleep(60)
|
||||
db = get_db()
|
||||
cfg = backup_settings(db)
|
||||
if not cfg["auto_enabled"]:
|
||||
continue
|
||||
interval_ms = int(cfg["interval_hours"]) * 3600 * 1000
|
||||
last = int(cfg["last_at_ms"] or 0)
|
||||
now = int(time.time() * 1000)
|
||||
if last and now - last < interval_ms:
|
||||
continue
|
||||
await asyncio.to_thread(create_backup, db=db, reason="auto")
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("auto backup tick failed")
|
||||
await asyncio.sleep(30)
|
||||
|
||||
|
||||
def start_auto_backup_task() -> asyncio.Task[None]:
|
||||
global _auto_task
|
||||
if _auto_task is None or _auto_task.done():
|
||||
_auto_task = asyncio.create_task(auto_backup_loop(), name="auto-backup")
|
||||
return _auto_task
|
||||
|
||||
|
||||
async def stop_auto_backup_task() -> None:
|
||||
global _auto_task
|
||||
if _auto_task and not _auto_task.done():
|
||||
_auto_task.cancel()
|
||||
try:
|
||||
await _auto_task
|
||||
except Exception:
|
||||
pass
|
||||
_auto_task = None
|
||||
Reference in New Issue
Block a user