Add PostgreSQL production backend to eliminate SQLite lock contention.

Support DATABASE_URL with connection pooling, pg_dump backups, SQLite migration script, and deploy_postgres.sh with docs. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-01 08:11:42 +08:00
parent 39eac983ff
commit 52aca456e9
23 changed files with 1208 additions and 150 deletions
@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+# qihuo · PostgreSQL 一键部署 / 从 SQLite 迁移
+# 用法: sudo bash scripts/deploy_postgres.sh
+# 可选: MIGRATE_SQLITE=1 自动从 /opt/qihuo/futures.db 迁移
+
+set -euo pipefail
+
+APP_DIR="${APP_DIR:-/opt/qihuo}"
+PG_DB="${PG_DB:-qihuo}"
+PG_USER="${PG_USER:-qihuo}"
+PG_HOST="${PG_HOST:-127.0.0.1}"
+PG_PORT="${PG_PORT:-5432}"
+MIGRATE_SQLITE="${MIGRATE_SQLITE:-0}"
+
+if [ "$(id -u)" -ne 0 ]; then
+  echo "请使用 root: sudo bash scripts/deploy_postgres.sh"
+  exit 1
+fi
+
+if [ ! -d "$APP_DIR" ]; then
+  echo "错误: 应用目录不存在 $APP_DIR，请先 bash deploy.sh"
+  exit 1
+fi
+
+echo "==> 安装 PostgreSQL..."
+export DEBIAN_FRONTEND=noninteractive
+apt-get update -qq
+apt-get install -y postgresql postgresql-contrib
+
+echo "==> 创建数据库与用户..."
+if [ -z "${PG_PASSWORD:-}" ]; then
+  PG_PASSWORD="$(python3 -c 'import secrets; print(secrets.token_urlsafe(16))')"
+fi
+
+sudo -u postgres psql -v ON_ERROR_STOP=1 <<SQL
+DO \$\$
+BEGIN
+  IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '${PG_USER}') THEN
+    CREATE ROLE ${PG_USER} LOGIN PASSWORD '${PG_PASSWORD}';
+  ELSE
+    ALTER ROLE ${PG_USER} WITH PASSWORD '${PG_PASSWORD}';
+  END IF;
+END
+\$\$;
+SELECT 'CREATE DATABASE ${PG_DB} OWNER ${PG_USER}'
+WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = '${PG_DB}')\\gexec
+GRANT ALL PRIVILEGES ON DATABASE ${PG_DB} TO ${PG_USER};
+SQL
+
+DATABASE_URL="postgresql://${PG_USER}:${PG_PASSWORD}@${PG_HOST}:${PG_PORT}/${PG_DB}"
+
+echo "==> 写入 .env DATABASE_URL..."
+ENV_FILE="$APP_DIR/.env"
+if grep -q "^DATABASE_URL=" "$ENV_FILE" 2>/dev/null; then
+  sed -i "s|^DATABASE_URL=.*|DATABASE_URL=${DATABASE_URL}|" "$ENV_FILE"
+else
+  echo "" >>"$ENV_FILE"
+  echo "# PostgreSQL（生产推荐，消除 SQLite 并发锁）" >>"$ENV_FILE"
+  echo "DATABASE_URL=${DATABASE_URL}" >>"$ENV_FILE"
+  echo "PG_POOL_MIN=2" >>"$ENV_FILE"
+  echo "PG_POOL_MAX=20" >>"$ENV_FILE"
+fi
+
+echo "==> Python 依赖..."
+# shellcheck disable=SC1091
+source "$APP_DIR/venv/bin/activate"
+pip install -q -r "$APP_DIR/requirements.txt"
+
+echo "==> 初始化 PostgreSQL 表结构..."
+cd "$APP_DIR"
+export DATABASE_URL
+python3 -c "from app import init_db; init_db(); from db_conn import database_label; print('OK:', database_label())"
+
+if [ "$MIGRATE_SQLITE" = "1" ] && [ -f "$APP_DIR/futures.db" ]; then
+  echo "==> 从 SQLite 迁移数据..."
+  python3 "$APP_DIR/scripts/migrate_sqlite_to_postgres.py" --sqlite "$APP_DIR/futures.db"
+  if [ "${BACKUP_SQLITE:-1}" = "1" ]; then
+    BAK="$APP_DIR/futures.db.pre_pg.$(date +%Y%m%d_%H%M%S)"
+    cp -a "$APP_DIR/futures.db" "$BAK"
+    echo "  已备份旧库: $BAK"
+  fi
+elif [ -f "$APP_DIR/futures.db" ]; then
+  echo "提示: 检测到 futures.db，如需迁移请: MIGRATE_SQLITE=1 bash scripts/deploy_postgres.sh"
+fi
+
+echo "==> 重启 PM2..."
+if pm2 describe qihuo &>/dev/null; then
+  pm2 restart ecosystem.config.cjs --update-env
+else
+  pm2 start "$APP_DIR/ecosystem.config.cjs"
+fi
+pm2 save
+
+echo ""
+echo "=========================================="
+echo "  PostgreSQL 部署完成"
+echo "  DATABASE_URL=${DATABASE_URL}"
+echo "  请妥善保存数据库密码: ${PG_PASSWORD}"
+echo "  文档: docs/POSTGRES.md"
+echo "  备份: 系统设置页 或 pg_dump / 自动备份"
+echo "=========================================="
@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+# Copyright (c) 2025-2026 马建军. All rights reserved.
+"""将 SQLite futures.db 迁移到 PostgreSQL（需已配置 DATABASE_URL 并 init 空库）。"""
+from __future__ import annotations
+
+import argparse
+import os
+import sqlite3
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+from dotenv import load_dotenv
+
+load_dotenv(ROOT / ".env")
+
+from db_conn import DB_PATH, connect_db, db_backend, is_postgres  # noqa: E402
+
+
+def _sqlite_tables(conn: sqlite3.Connection) -> list[str]:
+    rows = conn.execute(
+        "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name"
+    ).fetchall()
+    return [r[0] for r in rows]
+
+
+def _pg_columns(pg_conn, table: str) -> list[str]:
+    rows = pg_conn.execute(
+        """SELECT column_name FROM information_schema.columns
+           WHERE table_schema='public' AND table_name=%s
+           ORDER BY ordinal_position""",
+        (table,),
+    ).fetchall()
+    return [r["column_name"] for r in rows]
+
+
+def _reset_sequences(pg_conn, table: str, pk: str = "id") -> None:
+    try:
+        pg_conn.execute(
+            f"""SELECT setval(
+                pg_get_serial_sequence('{table}', '{pk}'),
+                COALESCE((SELECT MAX({pk}) FROM {table}), 1),
+                true
+            )"""
+        )
+    except Exception:
+        pass
+
+
+def migrate(*, sqlite_path: str | None = None, dry_run: bool = False) -> dict:
+    if not is_postgres():
+        raise RuntimeError("请先配置 DATABASE_URL=postgresql://... 后再运行迁移")
+
+    src_path = sqlite_path or DB_PATH
+    if not os.path.isfile(src_path):
+        raise FileNotFoundError(f"SQLite 源库不存在: {src_path}")
+
+    print(f"==> 源库: {src_path}")
+    print(f"==> 目标: PostgreSQL ({os.getenv('DATABASE_URL', '').split('@')[-1]})")
+
+    if not dry_run:
+        print("==> 初始化 PostgreSQL 表结构...")
+        from app import init_db
+
+        init_db()
+
+    src = sqlite3.connect(src_path)
+    src.row_factory = sqlite3.Row
+    dst = connect_db()
+
+    stats: dict[str, int] = {}
+    tables = _sqlite_tables(src)
+    print(f"==> 共 {len(tables)} 张表: {', '.join(tables)}")
+
+    try:
+        for table in tables:
+            pg_cols = _pg_columns(dst, table)
+            if not pg_cols:
+                print(f"  跳过 {table}（PostgreSQL 无此表，请先 init_db）")
+                continue
+            src_cols = [c[1] for c in src.execute(f"PRAGMA table_info({table})").fetchall()]
+            cols = [c for c in src_cols if c in pg_cols]
+            if not cols:
+                print(f"  跳过 {table}（无共同列）")
+                continue
+            rows = src.execute(f"SELECT {', '.join(cols)} FROM {table}").fetchall()
+            if dry_run:
+                stats[table] = len(rows)
+                print(f"  [dry-run] {table}: {len(rows)} 行")
+                continue
+            if not rows:
+                stats[table] = 0
+                continue
+            dst.execute(f"DELETE FROM {table}")
+            placeholders = ", ".join(["?"] * len(cols))
+            col_sql = ", ".join(cols)
+            insert_sql = f"INSERT INTO {table} ({col_sql}) VALUES ({placeholders})"
+            for row in rows:
+                dst.execute(insert_sql, tuple(row[c] for c in cols))
+            stats[table] = len(rows)
+            if "id" in cols:
+                _reset_sequences(dst, table, "id")
+            print(f"  {table}: {len(rows)} 行")
+        if not dry_run:
+            dst.commit()
+    finally:
+        src.close()
+        dst.close()
+
+    total = sum(stats.values())
+    print(f"==> 完成，共迁移 {total} 行")
+    return stats
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="SQLite -> PostgreSQL 数据迁移")
+    parser.add_argument("--sqlite", default=DB_PATH, help=f"SQLite 路径，默认 {DB_PATH}")
+    parser.add_argument("--dry-run", action="store_true", help="仅统计行数，不写入")
+    args = parser.parse_args()
+
+    if db_backend() != "postgres":
+        print("错误: 未检测到 DATABASE_URL（postgresql://...）", file=sys.stderr)
+        return 1
+    try:
+        migrate(sqlite_path=args.sqlite, dry_run=args.dry_run)
+    except Exception as exc:
+        print(f"迁移失败: {exc}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())