Add SSE second-level status push for control monitor.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
dekun
2026-07-30 13:32:28 +08:00
parent 7136adcaa9
commit b3ba0fd431
10 changed files with 273 additions and 41 deletions
+107 -26
View File
@@ -1,9 +1,13 @@
from __future__ import annotations
import asyncio
import json
import secrets
from typing import Annotated, Any
import time
from typing import Annotated, Any, AsyncIterator
from fastapi import APIRouter, Depends, HTTPException
from fastapi import APIRouter, Depends, HTTPException, Request
from fastapi.responses import StreamingResponse
from pydantic import BaseModel, Field
from ..auth import require_control_user
@@ -47,6 +51,50 @@ def _raise_node_error(code: int, data: Any) -> None:
)
def _health_looks_like_fleet(health: Any) -> bool:
return isinstance(health, dict) and "strategy" in health and "mode" in health
async def _collect_one_status(
node: dict[str, Any],
*,
timeout: float | None = None,
) -> dict[str, Any]:
probe = await probe_health(node, timeout=timeout)
item: dict[str, Any] = {**_public_node(node), **probe}
if probe.get("online") and node.get("token_sealed"):
health = probe.get("health")
# probe_health 在 /health 失败时可能已用 fleet/status 回填到 health
if _health_looks_like_fleet(health):
code, data = 200, health
else:
code, data = await call_node(
node, "GET", "/api/fleet/status", timeout=timeout
)
if code == 200 and isinstance(data, dict):
item["fleet"] = data
item["fleet_ok"] = True
else:
item["fleet_ok"] = False
item["fleet_error"] = _http_detail(data) if data else f"HTTP {code}"
elif not node.get("token_sealed"):
item["fleet_ok"] = False
item["fleet_error"] = "未生成 Token"
return item
async def collect_all_status(*, timeout: float | None = None) -> list[dict[str, Any]]:
db = get_control_db()
nodes = db.list_nodes()
if not nodes:
return []
return list(
await asyncio.gather(
*[_collect_one_status(n, timeout=timeout) for n in nodes]
)
)
class NodeCreate(BaseModel):
name: str = Field(min_length=1, max_length=64)
base_url: str = Field(min_length=8, max_length=256)
@@ -79,26 +127,65 @@ async def create_node(
@router.get("/status/all")
async def status_all(_user: Annotated[str, Depends(require_control_user)]) -> dict:
db = get_control_db()
items = []
for node in db.list_nodes():
probe = await probe_health(node)
item = {**_public_node(node), **probe}
if probe.get("online") and node.get("token_sealed"):
code, data = await call_node(node, "GET", "/api/fleet/status")
if code == 200 and isinstance(data, dict):
item["fleet"] = data
item["fleet_ok"] = True
else:
item["fleet_ok"] = False
item["fleet_error"] = _http_detail(data) if data else f"HTTP {code}"
elif not node.get("token_sealed"):
item["fleet_ok"] = False
item["fleet_error"] = "未生成 Token"
items.append(item)
items = await collect_all_status()
return {"nodes": items}
@router.get("/status/stream")
async def status_stream(
request: Request,
_user: Annotated[str, Depends(require_control_user)],
) -> StreamingResponse:
"""SSE:中控约每秒并行拉取策略机状态并推送到浏览器。"""
settings = get_control_settings()
interval = max(0.5, float(settings.control_sse_interval_sec))
probe_timeout = max(0.5, float(settings.control_sse_probe_timeout_sec))
async def event_gen() -> AsyncIterator[str]:
last_payload = ""
last_heartbeat = 0.0
yield f": connected interval={interval}\n\n"
while True:
if await request.is_disconnected():
break
started = time.monotonic()
try:
items = await collect_all_status(timeout=probe_timeout)
payload = json.dumps(
{"nodes": items, "ts_ms": int(time.time() * 1000)},
ensure_ascii=False,
separators=(",", ":"),
default=str,
)
now = time.monotonic()
if payload != last_payload:
last_payload = payload
yield f"event: nodes\ndata: {payload}\n\n"
last_heartbeat = now
elif now - last_heartbeat >= 5.0:
yield f": heartbeat {int(time.time())}\n\n"
last_heartbeat = now
except Exception as e:
err = json.dumps(
{"detail": str(e)},
ensure_ascii=False,
separators=(",", ":"),
)
yield f"event: error\ndata: {err}\n\n"
elapsed = time.monotonic() - started
await asyncio.sleep(max(0.05, interval - elapsed))
return StreamingResponse(
event_gen(),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache",
"Connection": "keep-alive",
"X-Accel-Buffering": "no",
},
)
@router.post("/update-batch")
async def update_batch(
body: dict,
@@ -179,13 +266,7 @@ async def node_status(
node = db.get_node(node_id)
if not node:
raise HTTPException(status_code=404, detail="节点不存在")
probe = await probe_health(node)
out = {**_public_node(node), **probe}
if probe.get("online") and node.get("token_sealed"):
code, data = await call_node(node, "GET", "/api/fleet/status")
if code == 200 and isinstance(data, dict):
out["fleet"] = data
return out
return await _collect_one_status(node)
@router.post("/{node_id}/start")