系统设置增加中控企业微信告警，后台巡检离线/故障并冷却推送。

Co-authored-by: Cursor <cursoragent@cursor.com>

.env.example

@@ -7,3 +7,9 @@ FLASK_SECRET_KEY=change-me-to-long-random
 DB_PATH=hub.db
 HUB_PULL_INTERVAL_SEC=10
 HUB_PULL_TIMEOUT_SEC=5
+
+# 中控企业微信(离线/故障告警,与客户端通知分开)
+HUB_WECHAT_ENABLED=false
+HUB_WECHAT_WEBHOOK=
+HUB_ALERT_COOLDOWN_SEC=600
+HUB_HEALTH_FAIL_THRESHOLD=2

app.py

@@ -3,6 +3,8 @@ from __future__ import annotations
 
 import os
 import sys
+import threading
+import time
 from functools import wraps
 from pathlib import Path
 
@@ -34,7 +36,8 @@ from lib.bridge_client_lib import (
     pull_all_enabled,
 )
 from lib.db_lib import init_db
-from lib.env_lib import apply_env_updates
+from lib.env_lib import apply_env_updates, env_get
+from lib.health_watch_lib import inspect_overview_rows
 from lib.instances_lib import (
     create_instance,
     delete_instance,
@@ -43,6 +46,13 @@ from lib.instances_lib import (
     update_instance,
 )
 from lib.sso_lib import build_embed_open_url
+from lib.wechat_lib import (
+    hub_alert_cooldown_sec,
+    hub_health_fail_threshold,
+    hub_wechat_enabled,
+    hub_wechat_webhook,
+    send_hub_wechat,
+)
 
 APP_HOST = (os.getenv("APP_HOST") or "0.0.0.0").strip() or "0.0.0.0"
 APP_PORT = int(os.getenv("APP_PORT") or 5100)
@@ -155,10 +165,28 @@ def instances_page():
     return render_template("instances.html", instances=list_instances())
 
 
-@app.route("/settings", methods=["GET", "POST"])
+def _wechat_settings_view() -> dict:
-@login_required
+    enabled_raw = (env_get(ENV_PATH, "HUB_WECHAT_ENABLED") or os.getenv("HUB_WECHAT_ENABLED") or "").strip().lower()
-def settings_page():
+    if enabled_raw in ("0", "false", "no", "off"):
-    if request.method == "POST":
+        enabled = False
+    elif enabled_raw in ("1", "true", "yes", "on"):
+        enabled = True
+    else:
+        enabled = bool(hub_wechat_webhook())
+    return {
+        "enabled": enabled,
+        "webhook": hub_wechat_webhook() or env_get(ENV_PATH, "HUB_WECHAT_WEBHOOK", ""),
+        "cooldown_sec": hub_alert_cooldown_sec(),
+        "fail_threshold": hub_health_fail_threshold(),
+    }
+
+
+def _apply_runtime_env(updates: dict[str, str]) -> None:
+    for key, val in updates.items():
+        os.environ[key] = str(val)
+
+
+def _handle_account_settings() -> None:
     current_pw = request.form.get("current_password") or ""
     new_username = (request.form.get("new_username") or "").strip()
     new_password = request.form.get("new_password") or ""
@@ -166,23 +194,23 @@ def settings_page():
 
     if current_pw != hub_password():
         flash("当前密码错误")
-            return redirect(url_for("settings_page"))
+        return
     if len(new_username) < 2 or len(new_username) > 64:
         flash("用户名长度需为 2–64")
-            return redirect(url_for("settings_page"))
+        return
     if any(ch.isspace() for ch in new_username):
         flash("用户名不能包含空格")
-            return redirect(url_for("settings_page"))
+        return
 
     updates: dict[str, str] = {"HUB_USERNAME": new_username}
     final_password = hub_password()
     if new_password or confirm:
         if len(new_password) < 6:
             flash("新密码至少 6 位")
-                return redirect(url_for("settings_page"))
+            return
         if new_password != confirm:
             flash("两次输入的新密码不一致")
-                return redirect(url_for("settings_page"))
+            return
         updates["HUB_PASSWORD"] = new_password
         final_password = new_password
 
@@ -190,13 +218,67 @@ def settings_page():
         apply_env_updates(ENV_PATH, updates)
     except OSError as exc:
         flash(f"写入 .env 失败: {exc}")
-            return redirect(url_for("settings_page"))
+        return
 
     set_hub_credentials(new_username, final_password)
     flash("账号已更新,立即生效")
+
+
+def _handle_wechat_settings(*, send_test: bool = False) -> None:
+    enabled = "1" if (request.form.get("wechat_enabled") or "") in ("1", "on", "true") else "0"
+    webhook = (request.form.get("wechat_webhook") or "").strip()
+    try:
+        cooldown = max(60, int(float(request.form.get("alert_cooldown_sec") or 600)))
+    except (TypeError, ValueError):
+        flash("告警冷却须为数字(秒)")
+        return
+    try:
+        threshold = max(1, int(float(request.form.get("health_fail_threshold") or 2)))
+    except (TypeError, ValueError):
+        flash("连续失败阈值须为正整数")
+        return
+
+    updates = {
+        "HUB_WECHAT_ENABLED": "true" if enabled == "1" else "false",
+        "HUB_WECHAT_WEBHOOK": webhook,
+        "HUB_ALERT_COOLDOWN_SEC": str(cooldown),
+        "HUB_HEALTH_FAIL_THRESHOLD": str(threshold),
+    }
+    try:
+        apply_env_updates(ENV_PATH, updates)
+    except OSError as exc:
+        flash(f"写入 .env 失败: {exc}")
+        return
+    _apply_runtime_env(updates)
+
+    if send_test:
+        result = send_hub_wechat("中控企业微信测试消息\n配置已保存并立即生效")
+        if result.get("ok"):
+            flash("告警配置已保存,测试消息已发送")
+        else:
+            flash(f"告警配置已保存,但测试发送失败: {result.get('msg') or '未知错误'}")
+        return
+    flash("企业微信告警配置已保存,立即生效")
+
+
+@app.route("/settings", methods=["GET", "POST"])
+@login_required
+def settings_page():
+    if request.method == "POST":
+        action = (request.form.get("form_action") or "account").strip()
+        if action == "wechat":
+            _handle_wechat_settings(send_test=False)
+        elif action == "wechat_test":
+            _handle_wechat_settings(send_test=True)
+        else:
+            _handle_account_settings()
         return redirect(url_for("settings_page"))
 
-    return render_template("settings.html", current_username=hub_username())
+    return render_template(
+        "settings.html",
+        current_username=hub_username(),
+        wechat=_wechat_settings_view(),
+    )
 
 
 @app.route("/desk")
@@ -233,6 +315,32 @@ def api_overview():
     return jsonify({"ok": True, "instances": rows})
 
 
+def _health_pull_interval_sec() -> float:
+    try:
+        return max(5.0, float(os.getenv("HUB_PULL_INTERVAL_SEC") or 10))
+    except (TypeError, ValueError):
+        return 10.0
+
+
+def _health_watch_loop() -> None:
+    """后台巡检:即使无人打开总览也会告警."""
+    while True:
+        try:
+            rows = pull_all_enabled()
+            inspect_overview_rows(rows)
+        except Exception:
+            pass
+        time.sleep(_health_pull_interval_sec())
+
+
+def start_health_watch() -> None:
+    t = threading.Thread(target=_health_watch_loop, name="hub-health-watch", daemon=True)
+    t.start()
+
+
+start_health_watch()
+
+
 @app.route("/api/instances/<int:iid>/open_url")
 @login_required
 def api_open_url(iid: int):

lib/env_lib.py

@@ -53,3 +53,20 @@ def apply_env_updates(path: Path | str, updates: Dict[str, str]) -> List[str]:
     env_path.parent.mkdir(parents=True, exist_ok=True)
     env_path.write_text("".join(out), encoding="utf-8")
     return changed
+
+
+def env_get(path: Path | str, key: str, default: str = "") -> str:
+    env_path = Path(path)
+    if not env_path.is_file():
+        return default
+    want = (key or "").strip()
+    if not want:
+        return default
+    for line in env_path.read_text(encoding="utf-8").splitlines():
+        k, _ = _parse_line(line)
+        if k == want:
+            m = _KEY_RE.match(line.rstrip("\n\r"))
+            if m:
+                return m.group(2)
+            return default
+    return default

lib/health_watch_lib.py

@@ -0,0 +1,129 @@
+"""客户端在线健康巡检与企业微信告警(冷却去重)."""
+from __future__ import annotations
+
+import threading
+import time
+from typing import Any
+
+from lib.wechat_lib import (
+    hub_alert_cooldown_sec,
+    hub_health_fail_threshold,
+    hub_wechat_enabled,
+    send_hub_wechat,
+)
+
+_lock = threading.Lock()
+# instance_id -> {online, fail_count, last_alert_at, last_reason, name}
+_state: dict[Any, dict[str, Any]] = {}
+
+
+def _now() -> float:
+    return time.time()
+
+
+def reset_health_state() -> None:
+    with _lock:
+        _state.clear()
+
+
+def inspect_overview_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """根据总览拉取结果更新状态并按需告警.返回本次新发出的告警摘要."""
+    if not rows:
+        return []
+    if not hub_wechat_enabled():
+        # 仍更新在线状态,便于开关打开后立刻有基线
+        with _lock:
+            for row in rows:
+                iid = row.get("id")
+                if iid is None:
+                    continue
+                online = bool(row.get("online"))
+                st = _state.get(iid) or {}
+                st.update(
+                    {
+                        "online": online,
+                        "fail_count": 0 if online else int(st.get("fail_count") or 0) + 1,
+                        "name": row.get("name") or st.get("name") or str(iid),
+                        "last_reason": "" if online else (row.get("error") or "离线"),
+                    }
+                )
+                _state[iid] = st
+        return []
+
+    threshold = hub_health_fail_threshold()
+    cooldown = hub_alert_cooldown_sec()
+    sent: list[dict[str, Any]] = []
+    now = _now()
+
+    with _lock:
+        for row in rows:
+            iid = row.get("id")
+            if iid is None:
+                continue
+            name = str(row.get("name") or iid)
+            online = bool(row.get("online"))
+            err = str(row.get("error") or "").strip()
+            st = _state.get(iid) or {
+                "online": None,
+                "fail_count": 0,
+                "last_alert_at": 0.0,
+                "last_reason": "",
+                "name": name,
+            }
+            prev_online = st.get("online")
+            st["name"] = name
+
+            if online:
+                # 从离线恢复
+                if prev_online is False:
+                    if now - float(st.get("last_alert_at") or 0) >= cooldown:
+                        msg = f"客户端恢复在线\n名称:{name}\nID:{iid}"
+                        result = send_hub_wechat(msg)
+                        if result.get("ok"):
+                            st["last_alert_at"] = now
+                            sent.append({"instance_id": iid, "kind": "online", "msg": msg})
+                st["online"] = True
+                st["fail_count"] = 0
+                st["last_reason"] = ""
+            else:
+                reason = err or "离线"
+                st["fail_count"] = int(st.get("fail_count") or 0) + 1
+                st["online"] = False
+                st["last_reason"] = reason
+                # 首次已知状态若就是离线,也累计;达到阈值才告警
+                if st["fail_count"] >= threshold:
+                    if now - float(st.get("last_alert_at") or 0) >= cooldown:
+                        kind = "fault" if reason and reason not in ("离线", "超时") else "offline"
+                        title = "客户端故障" if kind == "fault" else "客户端离线"
+                        msg = "\n".join(
+                            [
+                                title,
+                                f"名称:{name}",
+                                f"ID:{iid}",
+                                f"原因:{reason}",
+                                f"连续失败:{st['fail_count']} 次",
+                            ]
+                        )
+                        result = send_hub_wechat(msg)
+                        if result.get("ok"):
+                            st["last_alert_at"] = now
+                            sent.append({"instance_id": iid, "kind": kind, "msg": msg})
+            _state[iid] = st
+    return sent
+
+
+def health_snapshot() -> list[dict[str, Any]]:
+    with _lock:
+        out = []
+        for iid, st in _state.items():
+            out.append(
+                {
+                    "id": iid,
+                    "name": st.get("name"),
+                    "online": st.get("online"),
+                    "fail_count": st.get("fail_count") or 0,
+                    "last_reason": st.get("last_reason") or "",
+                    "last_alert_at": st.get("last_alert_at") or 0,
+                }
+            )
+        return out

lib/wechat_lib.py

@@ -0,0 +1,68 @@
+"""中控企业微信 Webhook 推送."""
+from __future__ import annotations
+
+import os
+from typing import Any
+
+import requests
+
+
+def hub_wechat_enabled() -> bool:
+    raw = (os.getenv("HUB_WECHAT_ENABLED") or "").strip().lower()
+    if raw in ("0", "false", "no", "off"):
+        return False
+    if raw in ("1", "true", "yes", "on"):
+        return True
+    # 未显式开关时:有有效 webhook 即视为开启
+    return bool(hub_wechat_webhook())
+
+
+def hub_wechat_webhook() -> str:
+    return (os.getenv("HUB_WECHAT_WEBHOOK") or "").strip()
+
+
+def hub_alert_cooldown_sec() -> int:
+    try:
+        return max(60, int(float(os.getenv("HUB_ALERT_COOLDOWN_SEC") or 600)))
+    except (TypeError, ValueError):
+        return 600
+
+
+def hub_health_fail_threshold() -> int:
+    try:
+        return max(1, int(float(os.getenv("HUB_HEALTH_FAIL_THRESHOLD") or 2)))
+    except (TypeError, ValueError):
+        return 2
+
+
+def send_hub_wechat(content: str, *, timeout: float = 8.0) -> dict[str, Any]:
+    """发送文本消息;返回 {ok, msg}."""
+    if not hub_wechat_enabled():
+        return {"ok": False, "msg": "企业微信推送未启用"}
+    url = hub_wechat_webhook()
+    if not url or "replace-me" in url:
+        return {"ok": False, "msg": "未配置有效 Webhook"}
+    body = str(content or "").strip()
+    if not body:
+        return {"ok": False, "msg": "消息为空"}
+    full = f"【期权中控】\n{body}"
+    try:
+        resp = requests.post(
+            url,
+            json={"msgtype": "text", "text": {"content": full}},
+            timeout=timeout,
+        )
+        data = {}
+        try:
+            data = resp.json()
+        except Exception:
+            data = {}
+        if resp.status_code == 200 and int(data.get("errcode", -1)) == 0:
+            return {"ok": True, "msg": "已发送"}
+        return {
+            "ok": False,
+            "msg": (data.get("errmsg") if isinstance(data, dict) else None)
+            or f"HTTP {resp.status_code}",
+        }
+    except Exception as e:
+        return {"ok": False, "msg": str(e) or "发送失败"}

static/hub.css

@@ -399,8 +399,44 @@ button, .btn {
   box-shadow: var(--shadow);
 }
 .empty { padding: 40px; color: var(--muted); }
+.settings-grid {
+  display: grid;
+  grid-template-columns: repeat(auto-fit, minmax(320px, 480px));
+  gap: 20px;
+  align-items: start;
+}
 .settings-card { max-width: 480px; }
 .settings-form { max-width: 420px; }
+.settings-form .check-row {
+  display: flex;
+  align-items: center;
+  gap: 8px;
+  font-weight: 500;
+}
+.settings-form .check-row input { width: auto; margin: 0; }
+.settings-form .field-hint {
+  display: block;
+  margin-top: 4px;
+  font-size: 12px;
+  color: var(--muted);
+  font-weight: 400;
+}
+.settings-actions {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 10px;
+  margin-top: 4px;
+}
+.settings-actions .btn-secondary,
+button.btn-secondary {
+  background: transparent;
+  color: var(--text);
+  border: 1px solid var(--line);
+}
+.settings-actions .btn-secondary:hover,
+button.btn-secondary:hover {
+  border-color: var(--accent, #60a5fa);
+}
 input.readonly {
   opacity: 0.75;
   cursor: default;

templates/settings.html

@@ -3,10 +3,12 @@
 {% block body %}
 <section class="page-head"><h1>系统设置</h1></section>
 
+<div class="settings-grid">
 <section class="panel settings-card">
   <h2>登录账号</h2>
   <p class="muted">修改后立即生效,并写入服务器 <code>.env</code>(<code>HUB_USERNAME</code> / <code>HUB_PASSWORD</code>).</p>
   <form method="post" class="stack settings-form" autocomplete="off">
+    <input type="hidden" name="form_action" value="account">
     <label>当前用户名
       <input type="text" value="{{ current_username }}" readonly class="readonly">
     </label>
@@ -25,4 +27,30 @@
     <button type="submit">保存账号</button>
   </form>
 </section>
+
+<section class="panel settings-card">
+  <h2>企业微信告警</h2>
+  <p class="muted">中控专用:客户端离线/故障与恢复时推送。与各交易机自身通知分开配置。</p>
+  <form method="post" class="stack settings-form" autocomplete="off">
+    <label class="check-row">
+      <input type="checkbox" name="wechat_enabled" value="1"{% if wechat.enabled %} checked{% endif %}>
+      启用企业微信推送
+    </label>
+    <label>Webhook 地址
+      <input type="url" name="wechat_webhook" value="{{ wechat.webhook }}" placeholder="https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=..." autocomplete="off">
+    </label>
+    <label>告警冷却(秒)
+      <input type="number" name="alert_cooldown_sec" value="{{ wechat.cooldown_sec }}" min="60" max="86400" step="1">
+    </label>
+    <label>连续失败阈值
+      <input type="number" name="health_fail_threshold" value="{{ wechat.fail_threshold }}" min="1" max="20" step="1">
+      <span class="field-hint">连续拉失败达到次数后才告警,默认 2</span>
+    </label>
+    <div class="settings-actions">
+      <button type="submit" name="form_action" value="wechat">保存告警配置</button>
+      <button type="submit" name="form_action" value="wechat_test" class="btn-secondary">发送测试消息</button>
+    </div>
+  </form>
+</section>
+</div>
 {% endblock %}

使用说明.md

@@ -29,7 +29,7 @@ OKX **期权多机中控**：在一个入口查看多台期权客户端的持仓
 | 内嵌打开 | 在中控工作台 iframe 中打开该客户端 |
 | 新标签打开 | 浏览器新标签直达同一客户端公网地址 |
 | 实例管理 | 增删改：名称、公网 URL、Token、标签、是否启用 |
-| 系统设置 | 修改中控登录用户名 / 密码 |
+| 系统设置 | 修改中控登录账号；配置企业微信离线/故障告警 |
 
 **不做：** 中控里直接下单、改客户端 env、保管 OKX 密钥。
 
@@ -38,7 +38,7 @@ OKX **期权多机中控**：在一个入口查看多台期权客户端的持仓
 ## 3. 使用流程
 
 1. 部署并登录中控（默认 **admin / admin123**）。
-2. **系统设置** 中建议立刻修改密码。
+2. **系统设置** 中建议立刻修改密码；可配置中控企业微信 Webhook，用于客户端离线/故障推送。
 3. **实例** 中添加客户端：名称、公网根地址、Bridge Token（与客户端 `HUB_BRIDGE_TOKEN` 一致）。
 4. 打开 **总览**：确认各机在线。
 5. 点 **打开** → 内嵌工作台；需要独立窗口时点 **新标签打开**。

部署文档.md

@@ -56,8 +56,12 @@ bash /opt/crypto_okx_hub/deploy/manage.sh
 | `HUB_USERNAME` / `HUB_PASSWORD` | 默认 `admin` / `admin123`；也可在页面「系统设置」修改 |
 | `FLASK_SECRET_KEY` | Session；部署脚本可自动生成 |
 | `DB_PATH` | 注册表 SQLite，默认 `hub.db` |
-| `HUB_PULL_INTERVAL_SEC` | 总览拉取间隔，默认 `10` |
+| `HUB_PULL_INTERVAL_SEC` | 后台健康巡检/总览拉取间隔，默认 `10` |
 | `HUB_PULL_TIMEOUT_SEC` | 单机超时，默认 `5` |
+| `HUB_WECHAT_ENABLED` | 中控企业微信告警开关，默认 `false`；可在系统设置修改 |
+| `HUB_WECHAT_WEBHOOK` | 企业微信机器人 Webhook |
+| `HUB_ALERT_COOLDOWN_SEC` | 同一机器告警冷却秒数，默认 `600` |
+| `HUB_HEALTH_FAIL_THRESHOLD` | 连续失败几次后告警，默认 `2` |
 
 **不要**在中控配置任何 `OKX_API_*`。
 
@@ -79,7 +83,7 @@ bash /opt/crypto_okx_hub/deploy/manage.sh
 1. 中控添加 2 个客户端（不同公网域名 + Token）。
 2. 总览两卡均在线。
 3. 「打开」内嵌 / 「新标签」正常。
-4. 系统设置可改用户名密码，改后可用新账号登录。
+4. 系统设置可改用户名密码，改后可用新账号登录；可配置企业微信告警并发送测试消息。
 
 ---