自动排队生成量化因子挖掘任务，心跳派发后将高 IC 因子入库。

Co-authored-by: Cursor <cursoragent@cursor.com>

app/api.py

@@ -72,6 +72,7 @@ from app.services.common import get_node_by_sn, get_user_by_wallet
 from app.services.datacenter import approve_application, list_applications, reject_application, submit_application
 from app.services.hardware import HardwareReport
 from app.services.hardware import display_node_id, displayed_population
+from app.services.alpha_queue import assign_queued_alpha_task, maybe_generate_alpha_batch
 from app.services.heartbeat import assert_node_token, record_heartbeat, set_node_banned
 from app.services.ip_guard import add_whitelist_ip, client_ip, list_whitelist, remove_whitelist_ip
 from app.services.orders import approve_order, list_orders, place_order, reject_order, render_client
@@ -327,6 +328,8 @@ def heartbeat(body: HeartbeatIn, request: Request, db: Db) -> HeartbeatOut:
         client_ip=client_ip(request),
         hardware_fingerprint=body.hardware_fingerprint,
     )
+    maybe_generate_alpha_batch(db)
+    assigned = assign_queued_alpha_task(db, node)
     days = node.recovery_days or 540
     return HeartbeatOut(
         sn=node.sn,
@@ -340,6 +343,7 @@ def heartbeat(body: HeartbeatIn, request: Request, db: Db) -> HeartbeatOut:
         recoverable_stake_amount=node.recoverable_stake_amount,
         purchase_amount=node.purchase_amount,
         estimated_daily_points=money(node.purchase_amount / Decimal(days)),
+        assigned_task=_task_out(assigned) if assigned is not None else None,
     )
 
 
@@ -514,8 +518,8 @@ def _admin_task_out(task: ComputeTask) -> AdminTaskOut:
     return AdminTaskOut(
         task_id=task.id,
         task_type=task.task_type,
-        target_node_sn=node.sn,
-        display_node_id=node.display_node_id,
+        target_node_sn=node.sn if node is not None else "",
+        display_node_id=node.display_node_id if node is not None else 0,
         status=task.status,
         payload=task.payload or {},
         result=task.result,

app/models.py

@@ -249,7 +249,7 @@ class ComputeTask(Base):
     )
 
     id: Mapped[str] = mapped_column(String(36), primary_key=True, comment="task_id")
-    node_id: Mapped[int] = mapped_column(ForeignKey("compute_nodes.id"), index=True)
+    node_id: Mapped[int | None] = mapped_column(ForeignKey("compute_nodes.id"), nullable=True, index=True)
     task_type: Mapped[str] = mapped_column(String(32), default="simulated_compute")
     payload: Mapped[dict] = mapped_column(JSON)
     status: Mapped[TaskStatus] = mapped_column(_enum(TaskStatus), default=TaskStatus.PENDING, index=True)
@@ -261,7 +261,26 @@ class ComputeTask(Base):
     success: Mapped[bool | None] = mapped_column()
     result: Mapped[dict | None] = mapped_column(JSON, comment="节点回传的任务结果")
 
-    node: Mapped[ComputeNode] = relationship(back_populates="tasks")
+    node: Mapped[ComputeNode | None] = relationship(back_populates="tasks")
+
+
+class AlphaFactor(Base):
+    """节点回传且达到 IC 门槛的因子。"""
+
+    __tablename__ = "alpha_factors"
+    __table_args__ = (
+        Index("ix_alpha_factor_task", "task_id"),
+        Index("ix_alpha_factor_ic", "ic"),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    task_id: Mapped[str] = mapped_column(ForeignKey("compute_tasks.id"), index=True)
+    node_id: Mapped[int] = mapped_column(ForeignKey("compute_nodes.id"), index=True)
+    formula: Mapped[str] = mapped_column(Text)
+    ic: Mapped[float] = mapped_column(Float)
+    symbol: Mapped[str | None] = mapped_column(String(32))
+    seed: Mapped[str | None] = mapped_column(String(64))
+    created_at: Mapped[datetime] = mapped_column(UtcDateTime, default=utcnow)
 
 
 class NodeYield(Base):

app/scheduler.py

@@ -13,6 +13,7 @@ from zoneinfo import ZoneInfo
 from app.config import settings
 from app.database import session_scope
 from app.jobs.daily_yield import run_daily_yield_cron
+from app.services.alpha_queue import maybe_generate_alpha_batch
 from app.services.heartbeat import mark_stale_nodes
 from app.services.locks import acquire_lock
 from app.services.tasks import dispatch_random_tasks
@@ -45,6 +46,13 @@ def start_scheduler() -> None:
         replace_existing=True,
         misfire_grace_time=120,
     )
+    scheduler.add_job(
+        _guard(_alpha_job),
+        IntervalTrigger(hours=2, timezone=UTC),
+        id="alpha_batch",
+        replace_existing=True,
+        misfire_grace_time=600,
+    )
     yield_zone = ZoneInfo(settings.yield_timezone)
     scheduler.add_job(
         _guard(_yield_job),
@@ -98,6 +106,20 @@ def _dispatch_job() -> None:
         tasks = dispatch_random_tasks(session)
     if tasks:
         logger.info("dispatched %s tasks", len(tasks))
+    _refill_alpha_queue(force=False)
+
+
+def _alpha_job() -> None:
+    _refill_alpha_queue(force=True)
+
+
+def _refill_alpha_queue(force: bool) -> None:
+    if not acquire_lock("alpha_batch", timedelta(minutes=5)):
+        return
+    with session_scope() as session:
+        created = maybe_generate_alpha_batch(session, force=force)
+    if created:
+        logger.info("alpha batch slices=%s force=%s", len(created), force)
 
 
 def _yield_job() -> None:

app/schema_upgrade.py

@@ -73,15 +73,15 @@ def upgrade_schema() -> None:
                 "ON compute_nodes (hardware_fingerprint)"
             )
     index_ready = "compute_nodes" in tables and ("node_index" in node_columns or any("node_index" in item for item in statements))
-    if not statements and not need_username_index and not index_ready:
-        return
-    with engine.begin() as conn:
-        for statement in statements:
-            conn.execute(text(statement))
-        if need_username_index:
-            conn.execute(text("CREATE UNIQUE INDEX IF NOT EXISTS uq_users_username ON users (username)"))
-        if index_ready:
-            _backfill_node_index(conn, "recovery_days" in node_columns or any("recovery_days" in item for item in statements))
+    if statements or need_username_index or index_ready:
+        with engine.begin() as conn:
+            for statement in statements:
+                conn.execute(text(statement))
+            if need_username_index:
+                conn.execute(text("CREATE UNIQUE INDEX IF NOT EXISTS uq_users_username ON users (username)"))
+            if index_ready:
+                _backfill_node_index(conn, "recovery_days" in node_columns or any("recovery_days" in item for item in statements))
+    _relax_task_node_id()
 
 
 def _backfill_node_index(conn, has_recovery: bool) -> None:
@@ -97,6 +97,69 @@ def _backfill_node_index(conn, has_recovery: bool) -> None:
         conn.execute(text("UPDATE compute_nodes SET recovery_days = 30 WHERE node_index BETWEEN 1 AND 20"))
 
 
+def _relax_task_node_id(bind=None) -> None:
+    """旧库 compute_tasks.node_id 为 NOT NULL，排队任务需要改成可空。SQLite 不能直接改空约束，整表重建。"""
+    target = bind or engine
+    inspector = inspect(target)
+    if "compute_tasks" not in inspector.get_table_names():
+        return
+    columns = {column["name"]: column for column in inspector.get_columns("compute_tasks")}
+    node_column = columns.get("node_id")
+    if node_column is None or node_column.get("nullable", True):
+        return
+    names = [
+        name
+        for name in (
+            "id",
+            "node_id",
+            "task_type",
+            "payload",
+            "status",
+            "push_status",
+            "push_error",
+            "dispatched_at",
+            "deadline_at",
+            "responded_at",
+            "success",
+            "result",
+        )
+        if name in columns
+    ]
+    copied = ", ".join(names)
+    with target.connect().execution_options(isolation_level="AUTOCOMMIT") as conn:
+        conn.execute(text("PRAGMA foreign_keys=OFF"))
+        conn.execute(text("DROP TABLE IF EXISTS compute_tasks_queue"))
+        conn.execute(
+            text(
+                """
+                CREATE TABLE compute_tasks_queue (
+                    id VARCHAR(36) NOT NULL PRIMARY KEY,
+                    node_id INTEGER,
+                    task_type VARCHAR(32) NOT NULL,
+                    payload JSON NOT NULL,
+                    status VARCHAR(16) NOT NULL,
+                    push_status VARCHAR(16) NOT NULL,
+                    push_error VARCHAR(255),
+                    dispatched_at DATETIME NOT NULL,
+                    deadline_at DATETIME NOT NULL,
+                    responded_at DATETIME,
+                    success BOOLEAN,
+                    result JSON,
+                    FOREIGN KEY(node_id) REFERENCES compute_nodes (id)
+                )
+                """
+            )
+        )
+        conn.execute(text(f"INSERT INTO compute_tasks_queue ({copied}) SELECT {copied} FROM compute_tasks"))
+        conn.execute(text("DROP TABLE compute_tasks"))
+        conn.execute(text("ALTER TABLE compute_tasks_queue RENAME TO compute_tasks"))
+        conn.execute(text("CREATE INDEX IF NOT EXISTS ix_compute_tasks_node_id ON compute_tasks (node_id)"))
+        conn.execute(text("CREATE INDEX IF NOT EXISTS ix_compute_tasks_status ON compute_tasks (status)"))
+        conn.execute(text("CREATE INDEX IF NOT EXISTS ix_task_node_status ON compute_tasks (node_id, status)"))
+        conn.execute(text("CREATE INDEX IF NOT EXISTS ix_task_dispatched_at ON compute_tasks (dispatched_at)"))
+        conn.execute(text("PRAGMA foreign_keys=ON"))
+
+
 def _has_username_index(inspector) -> bool:
     if "users" not in inspector.get_table_names():
         return False

app/schemas.py

@@ -128,6 +128,7 @@ class HeartbeatOut(APIModel):
     recoverable_stake_amount: MoneyText | None = None
     purchase_amount: MoneyText | None = None
     estimated_daily_points: MoneyText | None = None
+    assigned_task: TaskOut | None = None
 
 
 class TaskOut(APIModel):
@@ -358,3 +359,4 @@ class DatacenterView(DatacenterOut):
 
 
 DashboardOut.model_rebuild()
+HeartbeatOut.model_rebuild()

app/services/alpha_queue.py

@@ -0,0 +1,200 @@
+"""量化因子挖掘的排队与自动调度。
+
+没有待领取的 alpha_mining 任务时补一批；距上一批已满 2 小时时再补一批。
+切片先不绑定节点，心跳到来时再改成 processing。高 IC 因子写入 alpha_factors。
+"""
+
+from __future__ import annotations
+
+import random
+import secrets
+import uuid
+from datetime import datetime, timedelta, timezone
+
+from sqlalchemy import func, select
+from sqlalchemy.orm import Session
+
+from app.clock import utcnow
+from app.config import settings
+from app.models import AlphaFactor, ComputeNode, ComputeTask, PushStatus, TaskStatus
+
+ALPHA_BATCH_HOURS = 2
+MIN_ABS_IC = 0.03
+SLICE_COUNT = 3
+QUEUE_HOLD_DAYS = 7
+OPERATORS = (
+    "ts_mean",
+    "ts_std",
+    "rank",
+    "delta",
+    "decay_linear",
+    "correlation",
+    "ts_rank",
+    "scale",
+)
+SYMBOLS = ("BTCUSDT", "ETHUSDT", "BNBUSDT", "SOLUSDT")
+
+
+def maybe_generate_alpha_batch(
+    session: Session,
+    now: datetime | None = None,
+    force: bool = False,
+) -> list[ComputeTask]:
+    """队列里没有 pending 的挖掘任务，或距上一批已满 2 小时，就生成新的一批切片。"""
+    moment = now or utcnow()
+    pending = session.scalar(
+        select(func.count())
+        .select_from(ComputeTask)
+        .where(
+            ComputeTask.task_type == "alpha_mining",
+            ComputeTask.status == TaskStatus.PENDING,
+        )
+    ) or 0
+    latest = _as_utc(
+        session.scalar(select(func.max(ComputeTask.dispatched_at)).where(ComputeTask.task_type == "alpha_mining"))
+    )
+    due = latest is None or moment - latest >= timedelta(hours=ALPHA_BATCH_HOURS)
+    if pending and not due and not force:
+        return []
+    return _create_alpha_batch(session, moment)
+
+
+def assign_queued_alpha_task(
+    session: Session,
+    node: ComputeNode,
+    now: datetime | None = None,
+) -> ComputeTask | None:
+    """把最早的一条未绑定切片派给当前节点。节点已有执行中的挖掘任务时不再加派。"""
+    moment = now or utcnow()
+    holding = session.scalar(
+        select(ComputeTask.id).where(
+            ComputeTask.node_id == node.id,
+            ComputeTask.task_type == "alpha_mining",
+            ComputeTask.status == TaskStatus.PROCESSING,
+        )
+    )
+    if holding is not None:
+        return None
+    task = session.scalar(
+        select(ComputeTask)
+        .where(
+            ComputeTask.task_type == "alpha_mining",
+            ComputeTask.status == TaskStatus.PENDING,
+            ComputeTask.node_id.is_(None),
+        )
+        .order_by(ComputeTask.dispatched_at, ComputeTask.id)
+        .limit(1)
+    )
+    if task is None:
+        return None
+    task.node_id = node.id
+    task.status = TaskStatus.PROCESSING
+    task.deadline_at = moment + timedelta(minutes=settings.task_deadline_minutes)
+    session.flush()
+    return task
+
+
+def ingest_alpha_factors(session: Session, task: ComputeTask) -> list[AlphaFactor]:
+    """成功回传的挖掘结果里，只保存公式非空且绝对 IC 达到门槛的因子。"""
+    if task.task_type != "alpha_mining" or task.status != TaskStatus.COMPLETED or not task.success:
+        return []
+    if task.node_id is None:
+        return []
+    payload = task.payload or {}
+    saved: list[AlphaFactor] = []
+    for item in _factor_items(task.result or {}):
+        formula = str(item.get("formula") or item.get("alpha") or "").strip()
+        ic = _as_float(item.get("ic"))
+        if not formula or ic is None or abs(ic) < MIN_ABS_IC:
+            continue
+        symbol = item.get("symbol") or payload.get("symbol")
+        seed = payload.get("seed")
+        row = AlphaFactor(
+            task_id=task.id,
+            node_id=task.node_id,
+            formula=formula,
+            ic=ic,
+            symbol=str(symbol) if symbol else None,
+            seed=None if seed is None else str(seed),
+            created_at=task.responded_at or utcnow(),
+        )
+        session.add(row)
+        saved.append(row)
+    if saved:
+        session.flush()
+    return saved
+
+
+def _create_alpha_batch(session: Session, moment: datetime) -> list[ComputeTask]:
+    seed = secrets.randbelow(2**31)
+    rng = random.Random(seed)
+    batch_id = uuid.uuid4().hex[:12]
+    combos = _operator_combos(rng, SLICE_COUNT)
+    symbols = rng.sample(SYMBOLS, k=SLICE_COUNT)
+    deadline = moment + timedelta(days=QUEUE_HOLD_DAYS)
+    tasks: list[ComputeTask] = []
+    for combo, symbol in zip(combos, symbols):
+        task = ComputeTask(
+            id=str(uuid.uuid4()),
+            node_id=None,
+            task_type="alpha_mining",
+            payload={
+                "batch_id": batch_id,
+                "seed": seed,
+                "operators": combo,
+                "symbol": symbol,
+                "nonce": secrets.token_hex(16),
+            },
+            status=TaskStatus.PENDING,
+            push_status=PushStatus.QUEUED,
+            dispatched_at=moment,
+            deadline_at=deadline,
+        )
+        session.add(task)
+        tasks.append(task)
+    session.flush()
+    return tasks
+
+
+def _operator_combos(rng: random.Random, count: int) -> list[list[str]]:
+    combos: list[list[str]] = []
+    seen: set[tuple[str, ...]] = set()
+    while len(combos) < count:
+        size = 2 if rng.random() < 0.7 else 3
+        combo = tuple(sorted(rng.sample(OPERATORS, k=size)))
+        if combo in seen:
+            continue
+        seen.add(combo)
+        combos.append(list(combo))
+    return combos
+
+
+def _factor_items(result: dict) -> list[dict]:
+    factors = result.get("factors")
+    if isinstance(factors, list):
+        return [item for item in factors if isinstance(item, dict)]
+    if any(key in result for key in ("formula", "alpha", "ic")):
+        return [result]
+    return []
+
+
+def _as_utc(value: datetime | str | None) -> datetime | None:
+    if value is None:
+        return None
+    parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) if isinstance(value, str) else value
+    if parsed.tzinfo is None:
+        return parsed.replace(tzinfo=timezone.utc)
+    return parsed
+
+
+def _as_float(value: object) -> float | None:
+    if isinstance(value, bool) or value is None:
+        return None
+    if isinstance(value, (int, float)):
+        return float(value)
+    if isinstance(value, str):
+        try:
+            return float(value)
+        except ValueError:
+            return None
+    return None

app/services/tasks.py

@@ -24,6 +24,7 @@ from app.clock import utcnow
 from app.config import settings
 from app.errors import AppError
 from app.models import ComputeNode, ComputeTask, NodeStatus, PushStatus, TaskStatus
+from app.services.alpha_queue import ingest_alpha_factors
 from app.services.common import get_node_by_sn
 
 logger = logging.getLogger(__name__)
@@ -125,8 +126,15 @@ def dispatch_random_tasks(session: Session, now: datetime | None = None) -> list
     """向当前在线、授权仍有效、且没有未完成任务的节点随机下发一轮任务。"""
     moment = now or utcnow()
     expire_overdue_tasks(session, moment)
+    # 未绑定的挖掘切片不算占用节点。已派发的挖掘任务单独排队，不挡住模拟任务。
     pending_node_ids = set(
-        session.scalars(select(ComputeTask.node_id).where(ComputeTask.status.in_(_BUSY))).all()
+        session.scalars(
+            select(ComputeTask.node_id).where(
+                ComputeTask.node_id.is_not(None),
+                ComputeTask.status.in_(_BUSY),
+                ComputeTask.task_type != "alpha_mining",
+            )
+        ).all()
     )
     candidates = [
         node
@@ -174,15 +182,16 @@ def list_pending_tasks(session: Session, sn: str, now: datetime | None = None) -
     node = get_node_by_sn(session, sn.strip())
     tasks = session.scalars(
         select(ComputeTask)
-        .where(ComputeTask.node_id == node.id, ComputeTask.status == TaskStatus.PENDING)
-        .order_by(ComputeTask.dispatched_at)
+        .where(ComputeTask.node_id == node.id, ComputeTask.status.in_(_BUSY))
+        .order_by(ComputeTask.dispatched_at, ComputeTask.id)
     ).all()
     visible: list[ComputeTask] = []
     for task in tasks:
         if task.deadline_at < moment:
             task.status = TaskStatus.EXPIRED
             continue
-        task.status = TaskStatus.PROCESSING
+        if task.status == TaskStatus.PENDING:
+            task.status = TaskStatus.PROCESSING
         visible.append(task)
     session.flush()
     return visible
@@ -221,6 +230,8 @@ def report_task(
     task.result = result or None
     task.status = TaskStatus.COMPLETED if success else TaskStatus.FAILED
     session.flush()
+    if task.status == TaskStatus.COMPLETED and task.task_type == "alpha_mining":
+        ingest_alpha_factors(session, task)
     return task
 
 
@@ -228,6 +239,7 @@ def expire_overdue_tasks(session: Session, now: datetime | None = None) -> int:
     moment = now or utcnow()
     tasks = session.scalars(
         select(ComputeTask).where(
+            ComputeTask.node_id.is_not(None),
             ComputeTask.status.in_(_BUSY),
             ComputeTask.deadline_at < moment,
         )