Fix env save-and-restart by deferring PM2 restart until after HTTP response.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -335,7 +335,11 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function restartInstance() {
|
async function restartInstance() {
|
||||||
|
try {
|
||||||
await fetchJson("/api/admin/restart", { method: "POST" });
|
await fetchJson("/api/admin/restart", { method: "POST" });
|
||||||
|
} catch (_) {
|
||||||
|
// 重启会中断当前 HTTP 连接;只要后续 health 恢复即视为成功.
|
||||||
|
}
|
||||||
const deadline = Date.now() + 90000;
|
const deadline = Date.now() + 90000;
|
||||||
while (Date.now() < deadline) {
|
while (Date.now() < deadline) {
|
||||||
await new Promise((r) => setTimeout(r, 2000));
|
await new Promise((r) => setTimeout(r, 2000));
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import shlex
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
from typing import Any
|
from typing import Any
|
||||||
@@ -23,10 +24,34 @@ def resolve_pm2_app_name(exchange_key: str) -> str:
|
|||||||
return default_pm2_app_name(exchange_key)
|
return default_pm2_app_name(exchange_key)
|
||||||
|
|
||||||
|
|
||||||
def restart_instance_pm2(exchange_key: str) -> dict[str, Any]:
|
def schedule_pm2_restart(app_name: str, *, delay_seconds: float = 1.0) -> dict[str, Any]:
|
||||||
|
"""延迟触发 PM2 重启,便于 HTTP 响应先返回(避免重启当前进程导致请求中断)."""
|
||||||
|
if not sys.platform.startswith("linux"):
|
||||||
|
return {"ok": False, "msg": "仅 Linux 服务器支持 PM2 重启", "app": app_name}
|
||||||
|
if not (app_name or "").strip():
|
||||||
|
return {"ok": False, "msg": "未指定 PM2 应用名", "app": app_name}
|
||||||
|
app_name = app_name.strip()
|
||||||
|
try:
|
||||||
|
cmd = f"sleep {delay_seconds} && exec pm2 restart {shlex.quote(app_name)} --update-env"
|
||||||
|
subprocess.Popen(
|
||||||
|
["bash", "-c", cmd],
|
||||||
|
stdout=subprocess.DEVNULL,
|
||||||
|
stderr=subprocess.DEVNULL,
|
||||||
|
start_new_session=True,
|
||||||
|
)
|
||||||
|
return {"ok": True, "app": app_name, "msg": "重启已触发", "deferred": True}
|
||||||
|
except FileNotFoundError:
|
||||||
|
return {"ok": False, "msg": "未找到 bash 或 pm2 命令", "app": app_name}
|
||||||
|
except Exception as e:
|
||||||
|
return {"ok": False, "msg": str(e), "app": app_name}
|
||||||
|
|
||||||
|
|
||||||
|
def restart_instance_pm2(exchange_key: str, *, defer: bool = False) -> dict[str, Any]:
|
||||||
if not sys.platform.startswith("linux"):
|
if not sys.platform.startswith("linux"):
|
||||||
return {"ok": False, "msg": "仅 Linux 服务器支持 PM2 重启", "app": None}
|
return {"ok": False, "msg": "仅 Linux 服务器支持 PM2 重启", "app": None}
|
||||||
app_name = resolve_pm2_app_name(exchange_key)
|
app_name = resolve_pm2_app_name(exchange_key)
|
||||||
|
if defer:
|
||||||
|
return schedule_pm2_restart(app_name)
|
||||||
try:
|
try:
|
||||||
proc = subprocess.run(
|
proc = subprocess.run(
|
||||||
["pm2", "restart", app_name, "--update-env"],
|
["pm2", "restart", app_name, "--update-env"],
|
||||||
|
|||||||
@@ -140,7 +140,7 @@ def register_instance_settings_routes(
|
|||||||
@app.route("/api/admin/restart", methods=["POST"])
|
@app.route("/api/admin/restart", methods=["POST"])
|
||||||
@api_auth
|
@api_auth
|
||||||
def api_admin_restart():
|
def api_admin_restart():
|
||||||
result = restart_instance_pm2(exchange_key)
|
result = restart_instance_pm2(exchange_key, defer=True)
|
||||||
code = 200 if result.get("ok") else 500
|
code = 200 if result.get("ok") else 500
|
||||||
return jsonify({"ok": bool(result.get("ok")), **result}), code
|
return jsonify({"ok": bool(result.get("ok")), **result}), code
|
||||||
|
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
@@ -13,6 +11,7 @@ from lib.env.shared_env_lib import (
|
|||||||
build_ai_env_payload,
|
build_ai_env_payload,
|
||||||
restart_instances_then_hub_pm2,
|
restart_instances_then_hub_pm2,
|
||||||
)
|
)
|
||||||
|
from lib.instance.instance_pm2_lib import schedule_pm2_restart
|
||||||
|
|
||||||
HUB_DIR = Path(__file__).resolve().parent
|
HUB_DIR = Path(__file__).resolve().parent
|
||||||
|
|
||||||
@@ -46,25 +45,4 @@ def restart_all_pm2() -> dict[str, Any]:
|
|||||||
|
|
||||||
def restart_hub_pm2() -> dict[str, Any]:
|
def restart_hub_pm2() -> dict[str, Any]:
|
||||||
app_name = (os.getenv("PM2_APP_NAME") or "").strip() or "manual-trading-hub"
|
app_name = (os.getenv("PM2_APP_NAME") or "").strip() or "manual-trading-hub"
|
||||||
if not sys.platform.startswith("linux"):
|
return schedule_pm2_restart(app_name)
|
||||||
return {"ok": False, "msg": "仅 Linux 服务器支持 PM2 重启", "app": app_name}
|
|
||||||
try:
|
|
||||||
proc = subprocess.run(
|
|
||||||
["pm2", "restart", app_name, "--update-env"],
|
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
timeout=120,
|
|
||||||
)
|
|
||||||
ok = proc.returncode == 0
|
|
||||||
return {
|
|
||||||
"ok": ok,
|
|
||||||
"app": app_name,
|
|
||||||
"msg": (proc.stdout or proc.stderr or "").strip()[:500],
|
|
||||||
"returncode": proc.returncode,
|
|
||||||
}
|
|
||||||
except FileNotFoundError:
|
|
||||||
return {"ok": False, "msg": "未找到 pm2 命令", "app": app_name}
|
|
||||||
except subprocess.TimeoutExpired:
|
|
||||||
return {"ok": False, "msg": "pm2 restart 超时", "app": app_name}
|
|
||||||
except Exception as e:
|
|
||||||
return {"ok": False, "msg": str(e), "app": app_name}
|
|
||||||
|
|||||||
@@ -0,0 +1,35 @@
|
|||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
from lib.instance.instance_pm2_lib import restart_instance_pm2, schedule_pm2_restart
|
||||||
|
|
||||||
|
|
||||||
|
@patch("lib.instance.instance_pm2_lib.sys.platform", "linux")
|
||||||
|
@patch("lib.instance.instance_pm2_lib.subprocess.Popen")
|
||||||
|
def test_schedule_pm2_restart_returns_before_pm2(mock_popen):
|
||||||
|
result = schedule_pm2_restart("crypto_okx")
|
||||||
|
|
||||||
|
assert result["ok"] is True
|
||||||
|
assert result["deferred"] is True
|
||||||
|
mock_popen.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
@patch("lib.instance.instance_pm2_lib.sys.platform", "linux")
|
||||||
|
@patch("lib.instance.instance_pm2_lib.schedule_pm2_restart")
|
||||||
|
def test_restart_instance_pm2_defer_uses_schedule(mock_schedule):
|
||||||
|
mock_schedule.return_value = {"ok": True, "app": "crypto_okx", "deferred": True}
|
||||||
|
|
||||||
|
result = restart_instance_pm2("okx", defer=True)
|
||||||
|
|
||||||
|
mock_schedule.assert_called_once_with("crypto_okx")
|
||||||
|
assert result["deferred"] is True
|
||||||
|
|
||||||
|
|
||||||
|
@patch("lib.instance.instance_pm2_lib.sys.platform", "linux")
|
||||||
|
@patch("lib.instance.instance_pm2_lib.subprocess.run")
|
||||||
|
def test_restart_instance_pm2_sync_runs_pm2(mock_run):
|
||||||
|
mock_run.return_value = MagicMock(returncode=0, stdout="ok", stderr="")
|
||||||
|
|
||||||
|
result = restart_instance_pm2("okx", defer=False)
|
||||||
|
|
||||||
|
mock_run.assert_called_once()
|
||||||
|
assert result["ok"] is True
|
||||||
Reference in New Issue
Block a user