控制台改为选项卡,并支持封禁节点与下发计算任务。
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -40,6 +40,8 @@ def record_heartbeat(
|
||||
moment = now or utcnow()
|
||||
node = get_node_by_sn(session, sn.strip())
|
||||
assert_node_token(node, token)
|
||||
if node.is_banned:
|
||||
raise AppError("节点已封禁", 403)
|
||||
if node.license_expires_at <= moment:
|
||||
raise AppError("节点授权已到期", 403)
|
||||
bound_ip = (client_ip or "").strip() or None
|
||||
@@ -65,6 +67,18 @@ def record_heartbeat(
|
||||
return node
|
||||
|
||||
|
||||
def set_node_banned(session: Session, node: ComputeNode, banned: bool, now: datetime | None = None) -> ComputeNode:
|
||||
"""封禁立即结束在线区间。解除后要等下一次心跳才会重新上线。"""
|
||||
moment = now or utcnow()
|
||||
node.is_banned = banned
|
||||
node.updated_at = moment
|
||||
if banned and node.status == NodeStatus.ONLINE:
|
||||
_close_open_intervals(session, node.id, moment, "banned")
|
||||
node.status = NodeStatus.OFFLINE
|
||||
session.flush()
|
||||
return node
|
||||
|
||||
|
||||
def mark_stale_nodes(session: Session, now: datetime | None = None) -> list[str]:
|
||||
"""把超过心跳时限或授权已到期的在线节点置为 offline,并关闭在线区间。"""
|
||||
moment = now or utcnow()
|
||||
|
||||
@@ -82,6 +82,7 @@ def live_nodes(session: Session, user_id: int | None = None) -> list[dict]:
|
||||
"generation_locked": bool(node.generation_locked),
|
||||
"recoverable_stake_amount": node.recoverable_stake_amount,
|
||||
"is_whitelist": bool(node.is_whitelist),
|
||||
"is_banned": bool(node.is_banned),
|
||||
"client_ip": node.client_ip,
|
||||
"hardware_fingerprint": node.hardware_fingerprint,
|
||||
}
|
||||
|
||||
+99
-4
@@ -18,7 +18,7 @@ from urllib.error import URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.orm import Session, selectinload
|
||||
|
||||
from app.clock import utcnow
|
||||
from app.config import settings
|
||||
@@ -28,13 +28,105 @@ from app.services.common import get_node_by_sn
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ADMIN_TASK_TYPES = {
|
||||
"alpha_mining": "量化因子挖掘",
|
||||
"llm_inference": "大模型推理",
|
||||
"lora_training": "QLoRA 微调",
|
||||
"stress_test": "简易压测",
|
||||
}
|
||||
_BUSY = (TaskStatus.PENDING, TaskStatus.PROCESSING)
|
||||
|
||||
|
||||
def create_admin_tasks(
|
||||
session: Session,
|
||||
task_type: str,
|
||||
payload: dict | None,
|
||||
target_node_sn: str | None = None,
|
||||
now: datetime | None = None,
|
||||
) -> list[ComputeTask]:
|
||||
"""向指定节点或当前全部在线节点各放一条任务。payload 里补上一次性 nonce。"""
|
||||
if task_type not in ADMIN_TASK_TYPES:
|
||||
raise AppError("不支持的任务类型", 400)
|
||||
moment = now or utcnow()
|
||||
body = dict(payload or {})
|
||||
if len(json.dumps(body, ensure_ascii=False)) > 8000:
|
||||
raise AppError("任务参数过长", 400)
|
||||
targets = _dispatch_targets(session, target_node_sn, moment)
|
||||
deadline = moment + timedelta(minutes=settings.task_deadline_minutes)
|
||||
tasks: list[ComputeTask] = []
|
||||
for node in targets:
|
||||
task_payload = dict(body)
|
||||
task_payload["nonce"] = secrets.token_hex(16)
|
||||
task = ComputeTask(
|
||||
id=str(uuid.uuid4()),
|
||||
node_id=node.id,
|
||||
task_type=task_type,
|
||||
payload=task_payload,
|
||||
status=TaskStatus.PENDING,
|
||||
push_status=PushStatus.QUEUED,
|
||||
dispatched_at=moment,
|
||||
deadline_at=deadline,
|
||||
)
|
||||
session.add(task)
|
||||
tasks.append(task)
|
||||
session.flush()
|
||||
for node, task in zip(targets, tasks):
|
||||
_push_task(node, task)
|
||||
session.flush()
|
||||
return tasks
|
||||
|
||||
|
||||
def list_admin_tasks(session: Session) -> list[ComputeTask]:
|
||||
return list(
|
||||
session.scalars(
|
||||
select(ComputeTask)
|
||||
.options(selectinload(ComputeTask.node))
|
||||
.order_by(ComputeTask.dispatched_at.desc(), ComputeTask.id.desc())
|
||||
).all()
|
||||
)
|
||||
|
||||
|
||||
def result_preview(result: dict | None) -> str:
|
||||
if not result:
|
||||
return ""
|
||||
for key in ("formula", "alpha", "output", "summary"):
|
||||
value = result.get(key)
|
||||
if isinstance(value, str) and value.strip():
|
||||
return value.strip()[:120]
|
||||
for key in ("patch_bytes", "patch_size", "adapter_bytes"):
|
||||
if result.get(key) is not None:
|
||||
return f"补丁 {result[key]}"
|
||||
text = json.dumps(result, ensure_ascii=False)
|
||||
return text if len(text) <= 120 else text[:117] + "..."
|
||||
|
||||
|
||||
def _dispatch_targets(session: Session, target_node_sn: str | None, moment: datetime) -> list[ComputeNode]:
|
||||
sn = (target_node_sn or "").strip()
|
||||
if sn:
|
||||
node = get_node_by_sn(session, sn)
|
||||
if node.is_banned:
|
||||
raise AppError("节点已封禁,不能下发任务", 409)
|
||||
if node.license_expires_at <= moment:
|
||||
raise AppError("节点授权已到期,不能下发任务", 409)
|
||||
return [node]
|
||||
nodes = session.scalars(
|
||||
select(ComputeNode).where(
|
||||
ComputeNode.status == NodeStatus.ONLINE,
|
||||
ComputeNode.license_expires_at > moment,
|
||||
ComputeNode.is_banned.is_(False),
|
||||
)
|
||||
).all()
|
||||
if not nodes:
|
||||
raise AppError("当前没有可下发的在线节点", 409)
|
||||
return list(nodes)
|
||||
|
||||
|
||||
def dispatch_random_tasks(session: Session, now: datetime | None = None) -> list[ComputeTask]:
|
||||
"""向当前在线、授权仍有效、且没有未完成任务的节点随机下发一轮任务。"""
|
||||
moment = now or utcnow()
|
||||
expire_overdue_tasks(session, moment)
|
||||
pending_node_ids = set(
|
||||
session.scalars(select(ComputeTask.node_id).where(ComputeTask.status == TaskStatus.PENDING)).all()
|
||||
session.scalars(select(ComputeTask.node_id).where(ComputeTask.status.in_(_BUSY))).all()
|
||||
)
|
||||
candidates = [
|
||||
node
|
||||
@@ -90,6 +182,7 @@ def list_pending_tasks(session: Session, sn: str, now: datetime | None = None) -
|
||||
if task.deadline_at < moment:
|
||||
task.status = TaskStatus.EXPIRED
|
||||
continue
|
||||
task.status = TaskStatus.PROCESSING
|
||||
visible.append(task)
|
||||
session.flush()
|
||||
return visible
|
||||
@@ -102,6 +195,7 @@ def report_task(
|
||||
success: bool,
|
||||
nonce: str,
|
||||
now: datetime | None = None,
|
||||
result: dict | None = None,
|
||||
) -> ComputeTask:
|
||||
"""校验任务归属、时限和 nonce。只有校验通过且 success 为真才记为有效响应。"""
|
||||
moment = now or utcnow()
|
||||
@@ -111,7 +205,7 @@ def report_task(
|
||||
raise AppError("任务不存在", 404)
|
||||
if task.node_id != node.id:
|
||||
raise AppError("任务不属于该节点", 403)
|
||||
if task.status != TaskStatus.PENDING:
|
||||
if task.status not in _BUSY:
|
||||
raise AppError("任务已结束,不能重复回传", 409)
|
||||
if task.deadline_at < moment:
|
||||
task.status = TaskStatus.EXPIRED
|
||||
@@ -124,6 +218,7 @@ def report_task(
|
||||
|
||||
task.responded_at = moment
|
||||
task.success = success
|
||||
task.result = result or None
|
||||
task.status = TaskStatus.COMPLETED if success else TaskStatus.FAILED
|
||||
session.flush()
|
||||
return task
|
||||
@@ -133,7 +228,7 @@ def expire_overdue_tasks(session: Session, now: datetime | None = None) -> int:
|
||||
moment = now or utcnow()
|
||||
tasks = session.scalars(
|
||||
select(ComputeTask).where(
|
||||
ComputeTask.status == TaskStatus.PENDING,
|
||||
ComputeTask.status.in_(_BUSY),
|
||||
ComputeTask.deadline_at < moment,
|
||||
)
|
||||
).all()
|
||||
|
||||
Reference in New Issue
Block a user