自动排队生成量化因子挖掘任务,心跳派发后将高 IC 因子入库。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
dekun
2026-10-11 01:49:16 +08:00
parent f0b35e6b34
commit 598cecad5f
11 changed files with 529 additions and 23 deletions
+6 -2
View File
@@ -72,6 +72,7 @@ from app.services.common import get_node_by_sn, get_user_by_wallet
from app.services.datacenter import approve_application, list_applications, reject_application, submit_application from app.services.datacenter import approve_application, list_applications, reject_application, submit_application
from app.services.hardware import HardwareReport from app.services.hardware import HardwareReport
from app.services.hardware import display_node_id, displayed_population from app.services.hardware import display_node_id, displayed_population
from app.services.alpha_queue import assign_queued_alpha_task, maybe_generate_alpha_batch
from app.services.heartbeat import assert_node_token, record_heartbeat, set_node_banned from app.services.heartbeat import assert_node_token, record_heartbeat, set_node_banned
from app.services.ip_guard import add_whitelist_ip, client_ip, list_whitelist, remove_whitelist_ip from app.services.ip_guard import add_whitelist_ip, client_ip, list_whitelist, remove_whitelist_ip
from app.services.orders import approve_order, list_orders, place_order, reject_order, render_client from app.services.orders import approve_order, list_orders, place_order, reject_order, render_client
@@ -327,6 +328,8 @@ def heartbeat(body: HeartbeatIn, request: Request, db: Db) -> HeartbeatOut:
client_ip=client_ip(request), client_ip=client_ip(request),
hardware_fingerprint=body.hardware_fingerprint, hardware_fingerprint=body.hardware_fingerprint,
) )
maybe_generate_alpha_batch(db)
assigned = assign_queued_alpha_task(db, node)
days = node.recovery_days or 540 days = node.recovery_days or 540
return HeartbeatOut( return HeartbeatOut(
sn=node.sn, sn=node.sn,
@@ -340,6 +343,7 @@ def heartbeat(body: HeartbeatIn, request: Request, db: Db) -> HeartbeatOut:
recoverable_stake_amount=node.recoverable_stake_amount, recoverable_stake_amount=node.recoverable_stake_amount,
purchase_amount=node.purchase_amount, purchase_amount=node.purchase_amount,
estimated_daily_points=money(node.purchase_amount / Decimal(days)), estimated_daily_points=money(node.purchase_amount / Decimal(days)),
assigned_task=_task_out(assigned) if assigned is not None else None,
) )
@@ -514,8 +518,8 @@ def _admin_task_out(task: ComputeTask) -> AdminTaskOut:
return AdminTaskOut( return AdminTaskOut(
task_id=task.id, task_id=task.id,
task_type=task.task_type, task_type=task.task_type,
target_node_sn=node.sn, target_node_sn=node.sn if node is not None else "",
display_node_id=node.display_node_id, display_node_id=node.display_node_id if node is not None else 0,
status=task.status, status=task.status,
payload=task.payload or {}, payload=task.payload or {},
result=task.result, result=task.result,
+21 -2
View File
@@ -249,7 +249,7 @@ class ComputeTask(Base):
) )
id: Mapped[str] = mapped_column(String(36), primary_key=True, comment="task_id") id: Mapped[str] = mapped_column(String(36), primary_key=True, comment="task_id")
node_id: Mapped[int] = mapped_column(ForeignKey("compute_nodes.id"), index=True) node_id: Mapped[int | None] = mapped_column(ForeignKey("compute_nodes.id"), nullable=True, index=True)
task_type: Mapped[str] = mapped_column(String(32), default="simulated_compute") task_type: Mapped[str] = mapped_column(String(32), default="simulated_compute")
payload: Mapped[dict] = mapped_column(JSON) payload: Mapped[dict] = mapped_column(JSON)
status: Mapped[TaskStatus] = mapped_column(_enum(TaskStatus), default=TaskStatus.PENDING, index=True) status: Mapped[TaskStatus] = mapped_column(_enum(TaskStatus), default=TaskStatus.PENDING, index=True)
@@ -261,7 +261,26 @@ class ComputeTask(Base):
success: Mapped[bool | None] = mapped_column() success: Mapped[bool | None] = mapped_column()
result: Mapped[dict | None] = mapped_column(JSON, comment="节点回传的任务结果") result: Mapped[dict | None] = mapped_column(JSON, comment="节点回传的任务结果")
node: Mapped[ComputeNode] = relationship(back_populates="tasks") node: Mapped[ComputeNode | None] = relationship(back_populates="tasks")
class AlphaFactor(Base):
"""节点回传且达到 IC 门槛的因子。"""
__tablename__ = "alpha_factors"
__table_args__ = (
Index("ix_alpha_factor_task", "task_id"),
Index("ix_alpha_factor_ic", "ic"),
)
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
task_id: Mapped[str] = mapped_column(ForeignKey("compute_tasks.id"), index=True)
node_id: Mapped[int] = mapped_column(ForeignKey("compute_nodes.id"), index=True)
formula: Mapped[str] = mapped_column(Text)
ic: Mapped[float] = mapped_column(Float)
symbol: Mapped[str | None] = mapped_column(String(32))
seed: Mapped[str | None] = mapped_column(String(64))
created_at: Mapped[datetime] = mapped_column(UtcDateTime, default=utcnow)
class NodeYield(Base): class NodeYield(Base):
+22
View File
@@ -13,6 +13,7 @@ from zoneinfo import ZoneInfo
from app.config import settings from app.config import settings
from app.database import session_scope from app.database import session_scope
from app.jobs.daily_yield import run_daily_yield_cron from app.jobs.daily_yield import run_daily_yield_cron
from app.services.alpha_queue import maybe_generate_alpha_batch
from app.services.heartbeat import mark_stale_nodes from app.services.heartbeat import mark_stale_nodes
from app.services.locks import acquire_lock from app.services.locks import acquire_lock
from app.services.tasks import dispatch_random_tasks from app.services.tasks import dispatch_random_tasks
@@ -45,6 +46,13 @@ def start_scheduler() -> None:
replace_existing=True, replace_existing=True,
misfire_grace_time=120, misfire_grace_time=120,
) )
scheduler.add_job(
_guard(_alpha_job),
IntervalTrigger(hours=2, timezone=UTC),
id="alpha_batch",
replace_existing=True,
misfire_grace_time=600,
)
yield_zone = ZoneInfo(settings.yield_timezone) yield_zone = ZoneInfo(settings.yield_timezone)
scheduler.add_job( scheduler.add_job(
_guard(_yield_job), _guard(_yield_job),
@@ -98,6 +106,20 @@ def _dispatch_job() -> None:
tasks = dispatch_random_tasks(session) tasks = dispatch_random_tasks(session)
if tasks: if tasks:
logger.info("dispatched %s tasks", len(tasks)) logger.info("dispatched %s tasks", len(tasks))
_refill_alpha_queue(force=False)
def _alpha_job() -> None:
_refill_alpha_queue(force=True)
def _refill_alpha_queue(force: bool) -> None:
if not acquire_lock("alpha_batch", timedelta(minutes=5)):
return
with session_scope() as session:
created = maybe_generate_alpha_batch(session, force=force)
if created:
logger.info("alpha batch slices=%s force=%s", len(created), force)
def _yield_job() -> None: def _yield_job() -> None:
+72 -9
View File
@@ -73,15 +73,15 @@ def upgrade_schema() -> None:
"ON compute_nodes (hardware_fingerprint)" "ON compute_nodes (hardware_fingerprint)"
) )
index_ready = "compute_nodes" in tables and ("node_index" in node_columns or any("node_index" in item for item in statements)) index_ready = "compute_nodes" in tables and ("node_index" in node_columns or any("node_index" in item for item in statements))
if not statements and not need_username_index and not index_ready: if statements or need_username_index or index_ready:
return with engine.begin() as conn:
with engine.begin() as conn: for statement in statements:
for statement in statements: conn.execute(text(statement))
conn.execute(text(statement)) if need_username_index:
if need_username_index: conn.execute(text("CREATE UNIQUE INDEX IF NOT EXISTS uq_users_username ON users (username)"))
conn.execute(text("CREATE UNIQUE INDEX IF NOT EXISTS uq_users_username ON users (username)")) if index_ready:
if index_ready: _backfill_node_index(conn, "recovery_days" in node_columns or any("recovery_days" in item for item in statements))
_backfill_node_index(conn, "recovery_days" in node_columns or any("recovery_days" in item for item in statements)) _relax_task_node_id()
def _backfill_node_index(conn, has_recovery: bool) -> None: def _backfill_node_index(conn, has_recovery: bool) -> None:
@@ -97,6 +97,69 @@ def _backfill_node_index(conn, has_recovery: bool) -> None:
conn.execute(text("UPDATE compute_nodes SET recovery_days = 30 WHERE node_index BETWEEN 1 AND 20")) conn.execute(text("UPDATE compute_nodes SET recovery_days = 30 WHERE node_index BETWEEN 1 AND 20"))
def _relax_task_node_id(bind=None) -> None:
"""旧库 compute_tasks.node_id 为 NOT NULL,排队任务需要改成可空。SQLite 不能直接改空约束,整表重建。"""
target = bind or engine
inspector = inspect(target)
if "compute_tasks" not in inspector.get_table_names():
return
columns = {column["name"]: column for column in inspector.get_columns("compute_tasks")}
node_column = columns.get("node_id")
if node_column is None or node_column.get("nullable", True):
return
names = [
name
for name in (
"id",
"node_id",
"task_type",
"payload",
"status",
"push_status",
"push_error",
"dispatched_at",
"deadline_at",
"responded_at",
"success",
"result",
)
if name in columns
]
copied = ", ".join(names)
with target.connect().execution_options(isolation_level="AUTOCOMMIT") as conn:
conn.execute(text("PRAGMA foreign_keys=OFF"))
conn.execute(text("DROP TABLE IF EXISTS compute_tasks_queue"))
conn.execute(
text(
"""
CREATE TABLE compute_tasks_queue (
id VARCHAR(36) NOT NULL PRIMARY KEY,
node_id INTEGER,
task_type VARCHAR(32) NOT NULL,
payload JSON NOT NULL,
status VARCHAR(16) NOT NULL,
push_status VARCHAR(16) NOT NULL,
push_error VARCHAR(255),
dispatched_at DATETIME NOT NULL,
deadline_at DATETIME NOT NULL,
responded_at DATETIME,
success BOOLEAN,
result JSON,
FOREIGN KEY(node_id) REFERENCES compute_nodes (id)
)
"""
)
)
conn.execute(text(f"INSERT INTO compute_tasks_queue ({copied}) SELECT {copied} FROM compute_tasks"))
conn.execute(text("DROP TABLE compute_tasks"))
conn.execute(text("ALTER TABLE compute_tasks_queue RENAME TO compute_tasks"))
conn.execute(text("CREATE INDEX IF NOT EXISTS ix_compute_tasks_node_id ON compute_tasks (node_id)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS ix_compute_tasks_status ON compute_tasks (status)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS ix_task_node_status ON compute_tasks (node_id, status)"))
conn.execute(text("CREATE INDEX IF NOT EXISTS ix_task_dispatched_at ON compute_tasks (dispatched_at)"))
conn.execute(text("PRAGMA foreign_keys=ON"))
def _has_username_index(inspector) -> bool: def _has_username_index(inspector) -> bool:
if "users" not in inspector.get_table_names(): if "users" not in inspector.get_table_names():
return False return False
+2
View File
@@ -128,6 +128,7 @@ class HeartbeatOut(APIModel):
recoverable_stake_amount: MoneyText | None = None recoverable_stake_amount: MoneyText | None = None
purchase_amount: MoneyText | None = None purchase_amount: MoneyText | None = None
estimated_daily_points: MoneyText | None = None estimated_daily_points: MoneyText | None = None
assigned_task: TaskOut | None = None
class TaskOut(APIModel): class TaskOut(APIModel):
@@ -358,3 +359,4 @@ class DatacenterView(DatacenterOut):
DashboardOut.model_rebuild() DashboardOut.model_rebuild()
HeartbeatOut.model_rebuild()
+200
View File
@@ -0,0 +1,200 @@
"""量化因子挖掘的排队与自动调度。
没有待领取的 alpha_mining 任务时补一批;距上一批已满 2 小时时再补一批。
切片先不绑定节点,心跳到来时再改成 processing。高 IC 因子写入 alpha_factors。
"""
from __future__ import annotations
import random
import secrets
import uuid
from datetime import datetime, timedelta, timezone
from sqlalchemy import func, select
from sqlalchemy.orm import Session
from app.clock import utcnow
from app.config import settings
from app.models import AlphaFactor, ComputeNode, ComputeTask, PushStatus, TaskStatus
ALPHA_BATCH_HOURS = 2
MIN_ABS_IC = 0.03
SLICE_COUNT = 3
QUEUE_HOLD_DAYS = 7
OPERATORS = (
"ts_mean",
"ts_std",
"rank",
"delta",
"decay_linear",
"correlation",
"ts_rank",
"scale",
)
SYMBOLS = ("BTCUSDT", "ETHUSDT", "BNBUSDT", "SOLUSDT")
def maybe_generate_alpha_batch(
session: Session,
now: datetime | None = None,
force: bool = False,
) -> list[ComputeTask]:
"""队列里没有 pending 的挖掘任务,或距上一批已满 2 小时,就生成新的一批切片。"""
moment = now or utcnow()
pending = session.scalar(
select(func.count())
.select_from(ComputeTask)
.where(
ComputeTask.task_type == "alpha_mining",
ComputeTask.status == TaskStatus.PENDING,
)
) or 0
latest = _as_utc(
session.scalar(select(func.max(ComputeTask.dispatched_at)).where(ComputeTask.task_type == "alpha_mining"))
)
due = latest is None or moment - latest >= timedelta(hours=ALPHA_BATCH_HOURS)
if pending and not due and not force:
return []
return _create_alpha_batch(session, moment)
def assign_queued_alpha_task(
session: Session,
node: ComputeNode,
now: datetime | None = None,
) -> ComputeTask | None:
"""把最早的一条未绑定切片派给当前节点。节点已有执行中的挖掘任务时不再加派。"""
moment = now or utcnow()
holding = session.scalar(
select(ComputeTask.id).where(
ComputeTask.node_id == node.id,
ComputeTask.task_type == "alpha_mining",
ComputeTask.status == TaskStatus.PROCESSING,
)
)
if holding is not None:
return None
task = session.scalar(
select(ComputeTask)
.where(
ComputeTask.task_type == "alpha_mining",
ComputeTask.status == TaskStatus.PENDING,
ComputeTask.node_id.is_(None),
)
.order_by(ComputeTask.dispatched_at, ComputeTask.id)
.limit(1)
)
if task is None:
return None
task.node_id = node.id
task.status = TaskStatus.PROCESSING
task.deadline_at = moment + timedelta(minutes=settings.task_deadline_minutes)
session.flush()
return task
def ingest_alpha_factors(session: Session, task: ComputeTask) -> list[AlphaFactor]:
"""成功回传的挖掘结果里,只保存公式非空且绝对 IC 达到门槛的因子。"""
if task.task_type != "alpha_mining" or task.status != TaskStatus.COMPLETED or not task.success:
return []
if task.node_id is None:
return []
payload = task.payload or {}
saved: list[AlphaFactor] = []
for item in _factor_items(task.result or {}):
formula = str(item.get("formula") or item.get("alpha") or "").strip()
ic = _as_float(item.get("ic"))
if not formula or ic is None or abs(ic) < MIN_ABS_IC:
continue
symbol = item.get("symbol") or payload.get("symbol")
seed = payload.get("seed")
row = AlphaFactor(
task_id=task.id,
node_id=task.node_id,
formula=formula,
ic=ic,
symbol=str(symbol) if symbol else None,
seed=None if seed is None else str(seed),
created_at=task.responded_at or utcnow(),
)
session.add(row)
saved.append(row)
if saved:
session.flush()
return saved
def _create_alpha_batch(session: Session, moment: datetime) -> list[ComputeTask]:
seed = secrets.randbelow(2**31)
rng = random.Random(seed)
batch_id = uuid.uuid4().hex[:12]
combos = _operator_combos(rng, SLICE_COUNT)
symbols = rng.sample(SYMBOLS, k=SLICE_COUNT)
deadline = moment + timedelta(days=QUEUE_HOLD_DAYS)
tasks: list[ComputeTask] = []
for combo, symbol in zip(combos, symbols):
task = ComputeTask(
id=str(uuid.uuid4()),
node_id=None,
task_type="alpha_mining",
payload={
"batch_id": batch_id,
"seed": seed,
"operators": combo,
"symbol": symbol,
"nonce": secrets.token_hex(16),
},
status=TaskStatus.PENDING,
push_status=PushStatus.QUEUED,
dispatched_at=moment,
deadline_at=deadline,
)
session.add(task)
tasks.append(task)
session.flush()
return tasks
def _operator_combos(rng: random.Random, count: int) -> list[list[str]]:
combos: list[list[str]] = []
seen: set[tuple[str, ...]] = set()
while len(combos) < count:
size = 2 if rng.random() < 0.7 else 3
combo = tuple(sorted(rng.sample(OPERATORS, k=size)))
if combo in seen:
continue
seen.add(combo)
combos.append(list(combo))
return combos
def _factor_items(result: dict) -> list[dict]:
factors = result.get("factors")
if isinstance(factors, list):
return [item for item in factors if isinstance(item, dict)]
if any(key in result for key in ("formula", "alpha", "ic")):
return [result]
return []
def _as_utc(value: datetime | str | None) -> datetime | None:
if value is None:
return None
parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) if isinstance(value, str) else value
if parsed.tzinfo is None:
return parsed.replace(tzinfo=timezone.utc)
return parsed
def _as_float(value: object) -> float | None:
if isinstance(value, bool) or value is None:
return None
if isinstance(value, (int, float)):
return float(value)
if isinstance(value, str):
try:
return float(value)
except ValueError:
return None
return None
+16 -4
View File
@@ -24,6 +24,7 @@ from app.clock import utcnow
from app.config import settings from app.config import settings
from app.errors import AppError from app.errors import AppError
from app.models import ComputeNode, ComputeTask, NodeStatus, PushStatus, TaskStatus from app.models import ComputeNode, ComputeTask, NodeStatus, PushStatus, TaskStatus
from app.services.alpha_queue import ingest_alpha_factors
from app.services.common import get_node_by_sn from app.services.common import get_node_by_sn
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -125,8 +126,15 @@ def dispatch_random_tasks(session: Session, now: datetime | None = None) -> list
"""向当前在线、授权仍有效、且没有未完成任务的节点随机下发一轮任务。""" """向当前在线、授权仍有效、且没有未完成任务的节点随机下发一轮任务。"""
moment = now or utcnow() moment = now or utcnow()
expire_overdue_tasks(session, moment) expire_overdue_tasks(session, moment)
# 未绑定的挖掘切片不算占用节点。已派发的挖掘任务单独排队,不挡住模拟任务。
pending_node_ids = set( pending_node_ids = set(
session.scalars(select(ComputeTask.node_id).where(ComputeTask.status.in_(_BUSY))).all() session.scalars(
select(ComputeTask.node_id).where(
ComputeTask.node_id.is_not(None),
ComputeTask.status.in_(_BUSY),
ComputeTask.task_type != "alpha_mining",
)
).all()
) )
candidates = [ candidates = [
node node
@@ -174,15 +182,16 @@ def list_pending_tasks(session: Session, sn: str, now: datetime | None = None) -
node = get_node_by_sn(session, sn.strip()) node = get_node_by_sn(session, sn.strip())
tasks = session.scalars( tasks = session.scalars(
select(ComputeTask) select(ComputeTask)
.where(ComputeTask.node_id == node.id, ComputeTask.status == TaskStatus.PENDING) .where(ComputeTask.node_id == node.id, ComputeTask.status.in_(_BUSY))
.order_by(ComputeTask.dispatched_at) .order_by(ComputeTask.dispatched_at, ComputeTask.id)
).all() ).all()
visible: list[ComputeTask] = [] visible: list[ComputeTask] = []
for task in tasks: for task in tasks:
if task.deadline_at < moment: if task.deadline_at < moment:
task.status = TaskStatus.EXPIRED task.status = TaskStatus.EXPIRED
continue continue
task.status = TaskStatus.PROCESSING if task.status == TaskStatus.PENDING:
task.status = TaskStatus.PROCESSING
visible.append(task) visible.append(task)
session.flush() session.flush()
return visible return visible
@@ -221,6 +230,8 @@ def report_task(
task.result = result or None task.result = result or None
task.status = TaskStatus.COMPLETED if success else TaskStatus.FAILED task.status = TaskStatus.COMPLETED if success else TaskStatus.FAILED
session.flush() session.flush()
if task.status == TaskStatus.COMPLETED and task.task_type == "alpha_mining":
ingest_alpha_factors(session, task)
return task return task
@@ -228,6 +239,7 @@ def expire_overdue_tasks(session: Session, now: datetime | None = None) -> int:
moment = now or utcnow() moment = now or utcnow()
tasks = session.scalars( tasks = session.scalars(
select(ComputeTask).where( select(ComputeTask).where(
ComputeTask.node_id.is_not(None),
ComputeTask.status.in_(_BUSY), ComputeTask.status.in_(_BUSY),
ComputeTask.deadline_at < moment, ComputeTask.deadline_at < moment,
) )
+1 -1
View File
@@ -483,7 +483,7 @@ onUnmounted(() => window.clearInterval(timer));
<tr v-for="row in dispatched" :key="row.task_id"> <tr v-for="row in dispatched" :key="row.task_id">
<td class="mono" :title="row.task_id">{{ row.task_id.slice(0, 8) }}</td> <td class="mono" :title="row.task_id">{{ row.task_id.slice(0, 8) }}</td>
<td>{{ taskTypeText(row.task_type) }}</td> <td>{{ taskTypeText(row.task_type) }}</td>
<td>{{ row.display_node_id || "—" }} · {{ row.target_node_sn }}</td> <td>{{ row.target_node_sn ? `${row.display_node_id || "—"} · ${row.target_node_sn}` : "排队中" }}</td>
<td><span class="pill" :class="row.status">{{ taskStatusText[row.status] || row.status }}</span></td> <td><span class="pill" :class="row.status">{{ taskStatusText[row.status] || row.status }}</span></td>
<td>{{ when(row.responded_at) }}</td> <td>{{ when(row.responded_at) }}</td>
<td class="mono">{{ row.result_preview || "—" }}</td> <td class="mono">{{ row.result_preview || "—" }}</td>
+187 -3
View File
@@ -21,7 +21,7 @@ os.environ["ADMIN_PASSWORD"] = "test-admin-pass"
os.environ["ADMIN_WALLET"] = "admin-root" os.environ["ADMIN_WALLET"] = "admin-root"
from fastapi.testclient import TestClient from fastapi.testclient import TestClient
from sqlalchemy import select from sqlalchemy import create_engine, select, text
from app.clock import utcnow from app.clock import utcnow
from app.database import Base, SessionLocal, engine from app.database import Base, SessionLocal, engine
@@ -29,6 +29,7 @@ from app.main import app
from app.jobs.daily_yield import run_daily_yield_cron from app.jobs.daily_yield import run_daily_yield_cron
from app.money import money from app.money import money
from app.models import ( from app.models import (
AlphaFactor,
ComputeNode, ComputeNode,
ComputeTask, ComputeTask,
IPWhitelist, IPWhitelist,
@@ -42,7 +43,9 @@ from app.models import (
from app.services.heartbeat import mark_stale_nodes from app.services.heartbeat import mark_stale_nodes
from app.services.datacenter import FINGERPRINT_CLASH from app.services.datacenter import FINGERPRINT_CLASH
from app.services.ip_guard import DATACENTER_BATCH_MESSAGE from app.services.ip_guard import DATACENTER_BATCH_MESSAGE
from app.services.tasks import dispatch_random_tasks from app.schema_upgrade import _relax_task_node_id
from app.services.alpha_queue import maybe_generate_alpha_batch
from app.services.tasks import dispatch_random_tasks, expire_overdue_tasks
class CoreFlowTest(unittest.TestCase): class CoreFlowTest(unittest.TestCase):
@@ -197,7 +200,7 @@ class CoreFlowTest(unittest.TestCase):
pending = self.client.get("/api/v1/node/NODE-T/tasks") pending = self.client.get("/api/v1/node/NODE-T/tasks")
self.assertEqual(pending.status_code, 200, pending.text) self.assertEqual(pending.status_code, 200, pending.text)
self.assertEqual(pending.json()[0]["task_id"], task_id) self.assertTrue(any(row["task_id"] == task_id for row in pending.json()))
rejected = self.client.post( rejected = self.client.post(
"/api/v1/node/task/report", "/api/v1/node/task/report",
@@ -763,6 +766,187 @@ class CoreFlowTest(unittest.TestCase):
self.assertEqual(logged.json()["role"], "admin") self.assertEqual(logged.json()["role"], "admin")
return {"Authorization": f"Bearer {logged.json()['access_token']}"} return {"Authorization": f"Bearer {logged.json()['access_token']}"}
def test_alpha_queue_generates_assigns_and_keeps_high_ic(self) -> None:
self._register("0xalpha", "NODE-ALPHA", "100")
start = utcnow()
with SessionLocal() as session:
first = maybe_generate_alpha_batch(session, now=start)
session.commit()
self.assertEqual(len(first), 3)
self.assertEqual(len({task.payload["batch_id"] for task in first}), 1)
for task in first:
self.assertIsNone(task.node_id)
self.assertEqual(task.status, TaskStatus.PENDING)
self.assertEqual(task.task_type, "alpha_mining")
self.assertIsInstance(task.payload["seed"], int)
self.assertTrue(task.payload["operators"])
self.assertTrue(task.payload["symbol"])
self.assertGreaterEqual(len(task.payload["nonce"]), 8)
quiet = maybe_generate_alpha_batch(session, now=start + timedelta(hours=1))
self.assertEqual(quiet, [])
later = maybe_generate_alpha_batch(session, now=start + timedelta(hours=2))
session.commit()
self.assertEqual(len(later), 3)
self.assertNotEqual(later[0].payload["batch_id"], first[0].payload["batch_id"])
queued = session.get(ComputeTask, first[0].id)
queued.deadline_at = start - timedelta(minutes=1)
session.commit()
self.assertEqual(expire_overdue_tasks(session, start), 0)
self.assertEqual(session.get(ComputeTask, first[0].id).status, TaskStatus.PENDING)
listed = self.client.get("/api/v1/admin/tasks/list", headers=self._admin_headers())
self.assertEqual(listed.status_code, 200, listed.text)
queued_row = next(row for row in listed.json() if row["task_id"] == first[0].id)
self.assertEqual(queued_row["target_node_sn"], "")
self.assertEqual(queued_row["display_node_id"], 0)
beat = self.client.post("/api/v1/node/heartbeat", json={"sn": "NODE-ALPHA"})
self.assertEqual(beat.status_code, 200, beat.text)
assigned = beat.json()["assigned_task"]
self.assertIsNotNone(assigned)
self.assertEqual(assigned["status"], "processing")
self.assertEqual(assigned["task_type"], "alpha_mining")
again = self.client.post("/api/v1/node/heartbeat", json={"sn": "NODE-ALPHA"})
self.assertIsNone(again.json()["assigned_task"])
with SessionLocal() as session:
node = session.scalar(select(ComputeNode).where(ComputeNode.sn == "NODE-ALPHA"))
holding = session.scalars(
select(ComputeTask).where(
ComputeTask.node_id == node.id,
ComputeTask.task_type == "alpha_mining",
ComputeTask.status == TaskStatus.PROCESSING,
)
).all()
self.assertEqual(len(holding), 1)
high = self.client.post(
"/api/v1/node/task/report",
json={
"sn": "NODE-ALPHA",
"task_id": assigned["task_id"],
"success": True,
"nonce": assigned["payload"]["nonce"],
"result": {
"factors": [
{"formula": "rank(delta(close, 1))", "ic": 0.08},
{"formula": "scale(ts_std(close, 5))", "ic": 0.01},
{"alpha": "", "ic": 0.2},
]
},
},
)
self.assertEqual(high.status_code, 200, high.text)
self.assertEqual(high.json()["status"], "completed")
with SessionLocal() as session:
rows = session.scalars(select(AlphaFactor)).all()
self.assertEqual(len(rows), 1)
self.assertEqual(rows[0].formula, "rank(delta(close, 1))")
self.assertAlmostEqual(rows[0].ic, 0.08)
self.assertEqual(rows[0].symbol, assigned["payload"]["symbol"])
follow = self.client.post("/api/v1/node/heartbeat", json={"sn": "NODE-ALPHA"})
second = follow.json()["assigned_task"]
self.assertIsNotNone(second)
self.assertNotEqual(second["task_id"], assigned["task_id"])
low = self.client.post(
"/api/v1/node/task/report",
json={
"sn": "NODE-ALPHA",
"task_id": second["task_id"],
"success": True,
"nonce": second["payload"]["nonce"],
"result": {"formula": "ts_mean(close, 3)", "ic": 0.01},
},
)
self.assertEqual(low.status_code, 200, low.text)
self.assertEqual(low.json()["status"], "completed")
with SessionLocal() as session:
self.assertEqual(len(session.scalars(select(AlphaFactor)).all()), 1)
self.assertEqual(session.get(ComputeTask, second["task_id"]).status, TaskStatus.COMPLETED)
follow_again = self.client.post("/api/v1/node/heartbeat", json={"sn": "NODE-ALPHA"})
third = follow_again.json()["assigned_task"]
self.assertIsNotNone(third)
visible = self.client.get("/api/v1/node/NODE-ALPHA/tasks")
self.assertEqual(
[row for row in visible.json() if row["task_id"] == third["task_id"]][0]["status"],
"processing",
)
failed = self.client.post(
"/api/v1/node/task/report",
json={
"sn": "NODE-ALPHA",
"task_id": third["task_id"],
"success": False,
"nonce": third["payload"]["nonce"],
"result": {"formula": "rank(close)", "ic": 0.2},
},
)
self.assertEqual(failed.status_code, 200, failed.text)
self.assertEqual(failed.json()["status"], "failed")
with SessionLocal() as session:
self.assertEqual(len(session.scalars(select(AlphaFactor)).all()), 1)
def test_old_task_table_accepts_unassigned_rows(self) -> None:
path = Path(tempfile.gettempdir()) / "hypernode_relax_node.db"
if path.exists():
path.unlink()
other = create_engine("sqlite:///" + path.as_posix())
with other.begin() as conn:
conn.execute(text("CREATE TABLE compute_nodes (id INTEGER PRIMARY KEY)"))
conn.execute(text("INSERT INTO compute_nodes (id) VALUES (1)"))
conn.execute(
text(
"""
CREATE TABLE compute_tasks (
id VARCHAR(36) NOT NULL PRIMARY KEY,
node_id INTEGER NOT NULL,
task_type VARCHAR(32) NOT NULL,
payload JSON NOT NULL,
status VARCHAR(16) NOT NULL,
push_status VARCHAR(16) NOT NULL,
push_error VARCHAR(255),
dispatched_at DATETIME NOT NULL,
deadline_at DATETIME NOT NULL,
responded_at DATETIME,
success BOOLEAN,
result JSON
)
"""
)
)
conn.execute(
text(
"""
INSERT INTO compute_tasks (
id, node_id, task_type, payload, status, push_status, dispatched_at, deadline_at
) VALUES (
'kept-task', 1, 'simulated_compute', '{}', 'pending', 'queued',
'2026-10-01 00:00:00', '2026-10-01 00:10:00'
)
"""
)
)
_relax_task_node_id(other)
with other.begin() as conn:
conn.execute(
text(
"""
INSERT INTO compute_tasks (
id, node_id, task_type, payload, status, push_status, dispatched_at, deadline_at
) VALUES (
'queued-task', NULL, 'alpha_mining', '{}', 'pending', 'queued',
'2026-10-01 00:00:00', '2026-10-08 00:00:00'
)
"""
)
)
kept = conn.execute(text("SELECT node_id FROM compute_tasks WHERE id = 'kept-task'")).scalar()
queued = conn.execute(text("SELECT node_id FROM compute_tasks WHERE id = 'queued-task'")).scalar()
other.dispose()
self.assertEqual(kept, 1)
self.assertIsNone(queued)
def _register(self, wallet: str, sn: str, amount: str, headers: dict[str, str] | None = None) -> None: def _register(self, wallet: str, sn: str, amount: str, headers: dict[str, str] | None = None) -> None:
user = self.client.post("/api/v1/users", json={"wallet_address": wallet}) user = self.client.post("/api/v1/users", json={"wallet_address": wallet})
self.assertEqual(user.status_code, 200, user.text) self.assertEqual(user.status_code, 200, user.text)
File diff suppressed because one or more lines are too long
+1 -1
View File
@@ -7,7 +7,7 @@
<link rel="preconnect" href="https://fonts.googleapis.com" /> <link rel="preconnect" href="https://fonts.googleapis.com" />
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin /> <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
<link href="https://fonts.googleapis.com/css2?family=Fraunces:opsz,wght@9..144,560;9..144,650&family=IBM+Plex+Mono:wght@400;500&family=Manrope:wght@400;500;600;700&display=swap" rel="stylesheet" /> <link href="https://fonts.googleapis.com/css2?family=Fraunces:opsz,wght@9..144,560;9..144,650&family=IBM+Plex+Mono:wght@400;500&family=Manrope:wght@400;500;600;700&display=swap" rel="stylesheet" />
<script type="module" crossorigin src="/assets/index-Lxi5Kb1N.js"></script> <script type="module" crossorigin src="/assets/index-De4QqK0A.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-BA5SUri0.css"> <link rel="stylesheet" crossorigin href="/assets/index-BA5SUri0.css">
</head> </head>
<body> <body>