Add SSE second-level status push for control monitor.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -92,6 +92,7 @@ async def me(
|
||||
return {
|
||||
"username": username,
|
||||
"poll_interval_sec": settings.control_poll_interval_sec,
|
||||
"sse_interval_sec": settings.control_sse_interval_sec,
|
||||
"show_default_hint": settings.is_default_credentials,
|
||||
"lan_bypass_enabled": settings.lan_auth_bypass,
|
||||
}
|
||||
|
||||
@@ -1,9 +1,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import secrets
|
||||
from typing import Annotated, Any
|
||||
import time
|
||||
from typing import Annotated, Any, AsyncIterator
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from ..auth import require_control_user
|
||||
@@ -47,6 +51,50 @@ def _raise_node_error(code: int, data: Any) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _health_looks_like_fleet(health: Any) -> bool:
|
||||
return isinstance(health, dict) and "strategy" in health and "mode" in health
|
||||
|
||||
|
||||
async def _collect_one_status(
|
||||
node: dict[str, Any],
|
||||
*,
|
||||
timeout: float | None = None,
|
||||
) -> dict[str, Any]:
|
||||
probe = await probe_health(node, timeout=timeout)
|
||||
item: dict[str, Any] = {**_public_node(node), **probe}
|
||||
if probe.get("online") and node.get("token_sealed"):
|
||||
health = probe.get("health")
|
||||
# probe_health 在 /health 失败时可能已用 fleet/status 回填到 health
|
||||
if _health_looks_like_fleet(health):
|
||||
code, data = 200, health
|
||||
else:
|
||||
code, data = await call_node(
|
||||
node, "GET", "/api/fleet/status", timeout=timeout
|
||||
)
|
||||
if code == 200 and isinstance(data, dict):
|
||||
item["fleet"] = data
|
||||
item["fleet_ok"] = True
|
||||
else:
|
||||
item["fleet_ok"] = False
|
||||
item["fleet_error"] = _http_detail(data) if data else f"HTTP {code}"
|
||||
elif not node.get("token_sealed"):
|
||||
item["fleet_ok"] = False
|
||||
item["fleet_error"] = "未生成 Token"
|
||||
return item
|
||||
|
||||
|
||||
async def collect_all_status(*, timeout: float | None = None) -> list[dict[str, Any]]:
|
||||
db = get_control_db()
|
||||
nodes = db.list_nodes()
|
||||
if not nodes:
|
||||
return []
|
||||
return list(
|
||||
await asyncio.gather(
|
||||
*[_collect_one_status(n, timeout=timeout) for n in nodes]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class NodeCreate(BaseModel):
|
||||
name: str = Field(min_length=1, max_length=64)
|
||||
base_url: str = Field(min_length=8, max_length=256)
|
||||
@@ -79,26 +127,65 @@ async def create_node(
|
||||
|
||||
@router.get("/status/all")
|
||||
async def status_all(_user: Annotated[str, Depends(require_control_user)]) -> dict:
|
||||
db = get_control_db()
|
||||
items = []
|
||||
for node in db.list_nodes():
|
||||
probe = await probe_health(node)
|
||||
item = {**_public_node(node), **probe}
|
||||
if probe.get("online") and node.get("token_sealed"):
|
||||
code, data = await call_node(node, "GET", "/api/fleet/status")
|
||||
if code == 200 and isinstance(data, dict):
|
||||
item["fleet"] = data
|
||||
item["fleet_ok"] = True
|
||||
else:
|
||||
item["fleet_ok"] = False
|
||||
item["fleet_error"] = _http_detail(data) if data else f"HTTP {code}"
|
||||
elif not node.get("token_sealed"):
|
||||
item["fleet_ok"] = False
|
||||
item["fleet_error"] = "未生成 Token"
|
||||
items.append(item)
|
||||
items = await collect_all_status()
|
||||
return {"nodes": items}
|
||||
|
||||
|
||||
@router.get("/status/stream")
|
||||
async def status_stream(
|
||||
request: Request,
|
||||
_user: Annotated[str, Depends(require_control_user)],
|
||||
) -> StreamingResponse:
|
||||
"""SSE:中控约每秒并行拉取策略机状态并推送到浏览器。"""
|
||||
settings = get_control_settings()
|
||||
interval = max(0.5, float(settings.control_sse_interval_sec))
|
||||
probe_timeout = max(0.5, float(settings.control_sse_probe_timeout_sec))
|
||||
|
||||
async def event_gen() -> AsyncIterator[str]:
|
||||
last_payload = ""
|
||||
last_heartbeat = 0.0
|
||||
yield f": connected interval={interval}\n\n"
|
||||
while True:
|
||||
if await request.is_disconnected():
|
||||
break
|
||||
started = time.monotonic()
|
||||
try:
|
||||
items = await collect_all_status(timeout=probe_timeout)
|
||||
payload = json.dumps(
|
||||
{"nodes": items, "ts_ms": int(time.time() * 1000)},
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
default=str,
|
||||
)
|
||||
now = time.monotonic()
|
||||
if payload != last_payload:
|
||||
last_payload = payload
|
||||
yield f"event: nodes\ndata: {payload}\n\n"
|
||||
last_heartbeat = now
|
||||
elif now - last_heartbeat >= 5.0:
|
||||
yield f": heartbeat {int(time.time())}\n\n"
|
||||
last_heartbeat = now
|
||||
except Exception as e:
|
||||
err = json.dumps(
|
||||
{"detail": str(e)},
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
yield f"event: error\ndata: {err}\n\n"
|
||||
elapsed = time.monotonic() - started
|
||||
await asyncio.sleep(max(0.05, interval - elapsed))
|
||||
|
||||
return StreamingResponse(
|
||||
event_gen(),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.post("/update-batch")
|
||||
async def update_batch(
|
||||
body: dict,
|
||||
@@ -179,13 +266,7 @@ async def node_status(
|
||||
node = db.get_node(node_id)
|
||||
if not node:
|
||||
raise HTTPException(status_code=404, detail="节点不存在")
|
||||
probe = await probe_health(node)
|
||||
out = {**_public_node(node), **probe}
|
||||
if probe.get("online") and node.get("token_sealed"):
|
||||
code, data = await call_node(node, "GET", "/api/fleet/status")
|
||||
if code == 200 and isinstance(data, dict):
|
||||
out["fleet"] = data
|
||||
return out
|
||||
return await _collect_one_status(node)
|
||||
|
||||
|
||||
@router.post("/{node_id}/start")
|
||||
|
||||
Reference in New Issue
Block a user