Fix env save-and-restart by deferring PM2 restart until after HTTP response.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
dekun
2026-07-13 10:54:48 +08:00
parent c4753ffb89
commit 1bda78ceb2
5 changed files with 69 additions and 27 deletions
+5 -1
View File
@@ -335,7 +335,11 @@
}
async function restartInstance() {
await fetchJson("/api/admin/restart", { method: "POST" });
try {
await fetchJson("/api/admin/restart", { method: "POST" });
} catch (_) {
// 重启会中断当前 HTTP 连接;只要后续 health 恢复即视为成功.
}
const deadline = Date.now() + 90000;
while (Date.now() < deadline) {
await new Promise((r) => setTimeout(r, 2000));
+26 -1
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import os
import shlex
import subprocess
import sys
from typing import Any
@@ -23,10 +24,34 @@ def resolve_pm2_app_name(exchange_key: str) -> str:
return default_pm2_app_name(exchange_key)
def restart_instance_pm2(exchange_key: str) -> dict[str, Any]:
def schedule_pm2_restart(app_name: str, *, delay_seconds: float = 1.0) -> dict[str, Any]:
"""延迟触发 PM2 重启,便于 HTTP 响应先返回(避免重启当前进程导致请求中断)."""
if not sys.platform.startswith("linux"):
return {"ok": False, "msg": "仅 Linux 服务器支持 PM2 重启", "app": app_name}
if not (app_name or "").strip():
return {"ok": False, "msg": "未指定 PM2 应用名", "app": app_name}
app_name = app_name.strip()
try:
cmd = f"sleep {delay_seconds} && exec pm2 restart {shlex.quote(app_name)} --update-env"
subprocess.Popen(
["bash", "-c", cmd],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
start_new_session=True,
)
return {"ok": True, "app": app_name, "msg": "重启已触发", "deferred": True}
except FileNotFoundError:
return {"ok": False, "msg": "未找到 bash 或 pm2 命令", "app": app_name}
except Exception as e:
return {"ok": False, "msg": str(e), "app": app_name}
def restart_instance_pm2(exchange_key: str, *, defer: bool = False) -> dict[str, Any]:
if not sys.platform.startswith("linux"):
return {"ok": False, "msg": "仅 Linux 服务器支持 PM2 重启", "app": None}
app_name = resolve_pm2_app_name(exchange_key)
if defer:
return schedule_pm2_restart(app_name)
try:
proc = subprocess.run(
["pm2", "restart", app_name, "--update-env"],
+1 -1
View File
@@ -140,7 +140,7 @@ def register_instance_settings_routes(
@app.route("/api/admin/restart", methods=["POST"])
@api_auth
def api_admin_restart():
result = restart_instance_pm2(exchange_key)
result = restart_instance_pm2(exchange_key, defer=True)
code = 200 if result.get("ok") else 500
return jsonify({"ok": bool(result.get("ok")), **result}), code
+2 -24
View File
@@ -2,8 +2,6 @@
from __future__ import annotations
import os
import subprocess
import sys
from pathlib import Path
from typing import Any
@@ -13,6 +11,7 @@ from lib.env.shared_env_lib import (
build_ai_env_payload,
restart_instances_then_hub_pm2,
)
from lib.instance.instance_pm2_lib import schedule_pm2_restart
HUB_DIR = Path(__file__).resolve().parent
@@ -46,25 +45,4 @@ def restart_all_pm2() -> dict[str, Any]:
def restart_hub_pm2() -> dict[str, Any]:
app_name = (os.getenv("PM2_APP_NAME") or "").strip() or "manual-trading-hub"
if not sys.platform.startswith("linux"):
return {"ok": False, "msg": "仅 Linux 服务器支持 PM2 重启", "app": app_name}
try:
proc = subprocess.run(
["pm2", "restart", app_name, "--update-env"],
capture_output=True,
text=True,
timeout=120,
)
ok = proc.returncode == 0
return {
"ok": ok,
"app": app_name,
"msg": (proc.stdout or proc.stderr or "").strip()[:500],
"returncode": proc.returncode,
}
except FileNotFoundError:
return {"ok": False, "msg": "未找到 pm2 命令", "app": app_name}
except subprocess.TimeoutExpired:
return {"ok": False, "msg": "pm2 restart 超时", "app": app_name}
except Exception as e:
return {"ok": False, "msg": str(e), "app": app_name}
return schedule_pm2_restart(app_name)
+35
View File
@@ -0,0 +1,35 @@
from unittest.mock import MagicMock, patch
from lib.instance.instance_pm2_lib import restart_instance_pm2, schedule_pm2_restart
@patch("lib.instance.instance_pm2_lib.sys.platform", "linux")
@patch("lib.instance.instance_pm2_lib.subprocess.Popen")
def test_schedule_pm2_restart_returns_before_pm2(mock_popen):
result = schedule_pm2_restart("crypto_okx")
assert result["ok"] is True
assert result["deferred"] is True
mock_popen.assert_called_once()
@patch("lib.instance.instance_pm2_lib.sys.platform", "linux")
@patch("lib.instance.instance_pm2_lib.schedule_pm2_restart")
def test_restart_instance_pm2_defer_uses_schedule(mock_schedule):
mock_schedule.return_value = {"ok": True, "app": "crypto_okx", "deferred": True}
result = restart_instance_pm2("okx", defer=True)
mock_schedule.assert_called_once_with("crypto_okx")
assert result["deferred"] is True
@patch("lib.instance.instance_pm2_lib.sys.platform", "linux")
@patch("lib.instance.instance_pm2_lib.subprocess.run")
def test_restart_instance_pm2_sync_runs_pm2(mock_run):
mock_run.return_value = MagicMock(returncode=0, stdout="ok", stderr="")
result = restart_instance_pm2("okx", defer=False)
mock_run.assert_called_once()
assert result["ok"] is True