commit d9a34d4f2064eb4398046617f322a6b1cedce63c Author: dekun Date: Sat Aug 1 10:33:19 2026 +0800 first commit diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..d437cb6 --- /dev/null +++ b/.env.example @@ -0,0 +1,38 @@ +# 比特骆驼行情采集分析 — 环境变量示例 +# 复制为 .env 后按需修改。一键部署时：已有非空值不覆盖。 + +# ---- 服务 ---- +MI_PORT=5170 +TZ=Asia/Shanghai +AUTH_SECRET=change-me +ADMIN_PASSWORD=admin123 + +# ---- 采集（OKX 只读；公开行情可留空 Key）---- +OKX_API_KEY= +OKX_API_SECRET= +OKX_API_PASSPHRASE= +OKX_BASE_URL=https://www.okx.com +OKX_PROXY= +INDEX_INST_ID=ETH-USD +OPTION_INST_FAMILY=ETH-USD_UM +UNDERLYING=ETH +SAMPLE_INTERVAL_SEC=30 +INDEX_SAMPLE_INTERVAL_SEC=60 +INSTRUMENTS_REFRESH_SEC=300 +MIN_OPTION_HOURS=12 +MIN_OPTION_LEVERAGE=100 + +# ---- 数据库 ---- +# 容器内默认 /app/data/market_intel.db（volume 持久化） +MI_DB_PATH=/app/data/market_intel.db + +# ---- 统计默认 ---- +SETTLE_BACKFILL_INTERVAL_SEC=300 +BUCKET_MINUTES=60 +MONTH_RANGE_MODE=rolling_30 + +# ---- 企微告警（可选）---- +WECOM_ENABLED=0 +WECOM_WEBHOOK_URL= +WECOM_MACHINE_NAME= +ALERT_FAIL_THRESHOLD=5 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dd18834 --- /dev/null +++ b/.gitignore @@ -0,0 +1,41 @@ +# Python +.venv/ +venv/ +__pycache__/ +*.py[cod] +*.egg-info/ +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.coverage +htmlcov/ + +# Env & secrets +.env +!.env.example + +# Data +data/*.db +data/*.db-* +data/*.sqlite +data/*.sqlite-* +!data/.gitkeep + +# Node / frontend +web/node_modules/ +web/.vite/ +# 保留 web/dist 静态看板以便无 npm 时也能 Docker 启动；Vite build 可覆盖 + +# IDE / OS +.idea/ +.vscode/ +*.swp +.DS_Store +Thumbs.db + +# Logs +logs/ +*.log + +# Docker +*.tar diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0da0aa7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +# syntax=docker/dockerfile:1 +FROM python:3.11-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=/app \ + TZ=Asia/Shanghai + +WORKDIR /app + +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl ca-certificates tzdata \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY apps ./apps +COPY packages ./packages +COPY scripts ./scripts +COPY web/dist ./web/dist +COPY data/.gitkeep ./data/.gitkeep + +RUN mkdir -p /app/data /app/logs + +EXPOSE 5170 + +# 默认 API；Compose 里 collector 覆盖 command +CMD ["uvicorn", "apps.api.main:app", "--host", "0.0.0.0", "--port", "5170"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..76c40b0 --- /dev/null +++ b/README.md @@ -0,0 +1,92 @@ +# 比特骆驼行情采集分析（market_intel） + +独立仓库：**只做行情采集 → 落库 → 分析统计 → 只读展示 / API**。 +与 `eth_hedge_sim`（策略 / 中控）无代码共用、无进程共用、无交易密钥共用。 + +- 产品名：比特骆驼行情采集分析 +- 安装目录：`/opt/market_intel` +- 默认端口：`5170` +- 时区：`Asia/Shanghai` +- 第一期：OKX ETH 只读行情 + +详细实现标准见 [开发方案.md](./开发方案.md)。 + +## 快速开始（Docker） + +```bash +cp .env.example .env +docker compose up -d --build +curl -fsS http://127.0.0.1:5170/health +``` + +## 一键部署（Ubuntu 22.04） + +```bash +curl -fsSL https://git.bz121.com/dekun/market_intel/raw/branch/main/deploy/manage.sh | bash +``` + +已安装： + +```bash +bash /opt/market_intel/deploy/manage.sh +``` + +## 本地开发 + +```bash +python -m venv .venv +# Windows: .venv\Scripts\activate +source .venv/bin/activate +pip install -r requirements.txt +cp .env.example .env +# 本机 SQLite 路径 +# MI_DB_PATH=./data/market_intel.db + +set PYTHONPATH=. +python -m apps.collector.main # 终端 1 +uvicorn apps.api.main:app --reload --port 5170 # 终端 2 +``` + +前端（可选）： + +```bash +cd web && npm i && npm run build +# 产物挂到 API 静态目录；开发可用 npm run dev（代理到 5170） +``` + +## 核心口径 + +1. **期权杠杆** = 标的指数 ÷ 期权卖一（`ask`） +2. **波动点数** = 到期指数 − 时段代表指数（未到期标记 `pending_expiry=true`） +3. 统计按上海自然日切分；桶默认 1 小时 + +## API（第一期） + +| 方法 | 路径 | 说明 | +|------|------|------| +| GET | `/health` | 存活 + 采集延迟（公开） | +| GET | `/api/auth/status` | 是否需要登录 | +| POST | `/api/auth/login` | 密码换 token（Cookie + JSON） | +| GET | `/api/meta/latest` | 最新 Call/Put 杠杆（需鉴权） | +| GET | `/api/stats/leverage` | 日/周/月时段杠杆聚合 | +| GET | `/api/stats/move_points` | 时段→到期波动（signed/abs；未到期 pending） | +| GET | `/api/stats/ops-map` | 作战地图主接口（杠杆 + 波动） | +| GET | `/api/notify/wecom/status` | 企微配置状态 | +| POST | `/api/notify/wecom/test` | 企微测试推送 | + +鉴权：`AUTH_SECRET=disabled` 关闭；否则 `Authorization: Bearer ` / Cookie `mi_token`。 +企微：`WECOM_ENABLED=1` + `WECOM_WEBHOOK_URL`；连续失败 ≥ `ALERT_FAIL_THRESHOLD`（默认 5）推送。 + +## 分期 + +| 阶段 | 交付 | +|------|------| +| P0 | 骨架、Compose、manage.sh、健康检查 | +| P1 | OKX 指数 + ATM Call/Put 采样落库 | +| P2 | 杠杆日周月统计 + Web 图 | +| P3 | 到期回填 + 波动点数 | +| P4 | 鉴权加固、企微告警 | + +## 许可 + +内部项目。 diff --git a/apps/__init__.py b/apps/__init__.py new file mode 100644 index 0000000..4764c3a --- /dev/null +++ b/apps/__init__.py @@ -0,0 +1 @@ +# Collector package diff --git a/apps/api/__init__.py b/apps/api/__init__.py new file mode 100644 index 0000000..b7640bb --- /dev/null +++ b/apps/api/__init__.py @@ -0,0 +1 @@ +# API package diff --git a/apps/api/auth.py b/apps/api/auth.py new file mode 100644 index 0000000..1133d94 --- /dev/null +++ b/apps/api/auth.py @@ -0,0 +1,87 @@ +"""简单 Token 鉴权（对齐策略仓：密码换 HMAC token）。""" + +from __future__ import annotations + +import hashlib +import hmac +import secrets +from typing import Annotated + +from fastapi import Depends, Header, HTTPException, Request, status +from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer + +from packages.config import get_settings + +_bearer = HTTPBearer(auto_error=False) +COOKIE_NAME = "mi_token" + + +def auth_disabled() -> bool: + s = get_settings() + return (s.auth_secret or "").strip().lower() in ("", "disabled", "off", "none") + + +def _token_for_password(password: str, secret: str) -> str: + return hmac.new( + secret.encode("utf-8"), + password.encode("utf-8"), + hashlib.sha256, + ).hexdigest() + + +def expected_token() -> str: + s = get_settings() + return _token_for_password(s.admin_password, s.auth_secret) + + +def issue_token(password: str) -> str | None: + s = get_settings() + if not secrets.compare_digest(password, s.admin_password): + return None + return _token_for_password(password, s.auth_secret) + + +def _extract_token( + request: Request, + authorization: str | None, + x_mi_token: str | None, + creds: HTTPAuthorizationCredentials | None, +) -> str | None: + if creds and creds.credentials: + return creds.credentials.strip() + if authorization and authorization.lower().startswith("bearer "): + return authorization[7:].strip() + if x_mi_token: + return x_mi_token.strip() + # 查询参数兜底（方便内网脚本；生产建议只用 Header） + q = request.query_params.get("token") + if q: + return q.strip() + cookie = request.cookies.get(COOKIE_NAME) + if cookie: + return cookie.strip() + return None + + +def require_auth( + request: Request, + authorization: Annotated[str | None, Header()] = None, + x_mi_token: Annotated[str | None, Header(alias="X-MI-Token")] = None, + creds: Annotated[HTTPAuthorizationCredentials | None, Depends(_bearer)] = None, +) -> None: + """ + AUTH_SECRET=disabled 时跳过。 + 否则需要 Bearer / X-MI-Token / Cookie / ?token=。 + """ + if auth_disabled(): + return + token = _extract_token(request, authorization, x_mi_token, creds) + if not token or not secrets.compare_digest(token, expected_token()): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="unauthorized", + headers={"WWW-Authenticate": "Bearer"}, + ) + + +AuthDep = Depends(require_auth) diff --git a/apps/api/main.py b/apps/api/main.py new file mode 100644 index 0000000..0a4f0f4 --- /dev/null +++ b/apps/api/main.py @@ -0,0 +1,70 @@ +"""FastAPI 入口：健康检查 + 只读 API + 静态看板。""" + +from __future__ import annotations + +from pathlib import Path + +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import FileResponse, HTMLResponse, Response +from fastapi.staticfiles import StaticFiles + +from apps.api.routes import auth, health, meta, notify, samples, stats + +app = FastAPI( + title="比特骆驼行情采集分析", + description="market_intel — 只读行情采集与统计", + version="0.1.0", +) + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +app.include_router(health.router) +app.include_router(auth.router, prefix="/api") +app.include_router(meta.router, prefix="/api") +app.include_router(samples.router, prefix="/api") +app.include_router(stats.router, prefix="/api") +app.include_router(notify.router, prefix="/api") + +_WEB_DIST = Path(__file__).resolve().parents[2] / "web" / "dist" + +_FALLBACK_HTML = """ +比特骆驼行情采集分析 + +

比特骆驼行情采集分析

+

API 已就绪。/health · /api/meta/latest

+

构建前端：cd web && npm i && npm run build

+""" + + +def _index_response() -> Response: + index_html = _WEB_DIST / "index.html" + if index_html.is_file(): + return FileResponse(index_html) + return HTMLResponse(_FALLBACK_HTML) + + +@app.get("/") +def index() -> Response: + return _index_response() + + +@app.get("/ops-map") +def ops_map_page() -> Response: + """SPA / 静态看板入口（hash 或 React Router）。""" + return _index_response() + + +if _WEB_DIST.is_dir(): + assets = _WEB_DIST / "assets" + if assets.is_dir(): + app.mount("/assets", StaticFiles(directory=str(assets)), name="assets") diff --git a/apps/api/routes/__init__.py b/apps/api/routes/__init__.py new file mode 100644 index 0000000..2acd8c9 --- /dev/null +++ b/apps/api/routes/__init__.py @@ -0,0 +1 @@ +# routes package diff --git a/apps/api/routes/auth.py b/apps/api/routes/auth.py new file mode 100644 index 0000000..f04ca0e --- /dev/null +++ b/apps/api/routes/auth.py @@ -0,0 +1,48 @@ +"""登录 / 鉴权状态。""" + +from __future__ import annotations + +from fastapi import APIRouter, Response +from pydantic import BaseModel, Field + +from apps.api.auth import COOKIE_NAME, auth_disabled, issue_token + +router = APIRouter(prefix="/auth", tags=["auth"]) + + +class LoginBody(BaseModel): + password: str = Field(min_length=1, max_length=256) + + +@router.get("/status") +def auth_status() -> dict: + return { + "auth_required": not auth_disabled(), + "product": "比特骆驼行情采集分析", + } + + +@router.post("/login") +def login(body: LoginBody, response: Response) -> dict: + if auth_disabled(): + return {"ok": True, "auth_required": False, "token": None} + token = issue_token(body.password) + if not token: + from fastapi import HTTPException, status + + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="invalid password") + response.set_cookie( + key=COOKIE_NAME, + value=token, + httponly=True, + samesite="lax", + max_age=7 * 24 * 3600, + path="/", + ) + return {"ok": True, "auth_required": True, "token": token} + + +@router.post("/logout") +def logout(response: Response) -> dict: + response.delete_cookie(COOKIE_NAME, path="/") + return {"ok": True} diff --git a/apps/api/routes/health.py b/apps/api/routes/health.py new file mode 100644 index 0000000..38ddddd --- /dev/null +++ b/apps/api/routes/health.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from fastapi import APIRouter + +from packages.config import get_settings +from packages.db import Repository + +router = APIRouter(tags=["health"]) + + +@router.get("/health") +def health() -> dict: + s = get_settings() + repo = Repository(s.db_path) + try: + hb = repo.get_heartbeat() + last_ok = hb.get("last_ok_ts_ms") + lag_ms = None + if last_ok: + import time + + lag_ms = max(0, int(time.time() * 1000) - int(last_ok)) + return { + "ok": True, + "service": "market_intel", + "product": "比特骆驼行情采集分析", + "collector_lag_ms": lag_ms, + "consecutive_failures": hb.get("consecutive_failures", 0), + "option_quotes": repo.count_option_quotes(), + "index_ticks": repo.count_index_ticks(), + } + finally: + repo.close() diff --git a/apps/api/routes/meta.py b/apps/api/routes/meta.py new file mode 100644 index 0000000..1b3edfd --- /dev/null +++ b/apps/api/routes/meta.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from fastapi import APIRouter, Depends + +from apps.api.auth import require_auth +from packages.config import get_settings +from packages.db import Repository +from packages.domain import LEVERAGE_FORMULA_VERSION + +router = APIRouter(tags=["meta"], dependencies=[Depends(require_auth)]) + + +@router.get("/meta/latest") +def latest() -> dict: + s = get_settings() + repo = Repository(s.db_path) + try: + hb = repo.get_heartbeat() + by_side = repo.latest_quotes_by_side() + return { + "formula_version": LEVERAGE_FORMULA_VERSION, + "leverage_def": "index_px / ask", + "timezone": s.tz, + "underlying": s.underlying, + "min_option_leverage": s.min_option_leverage, + "heartbeat": { + "last_ok_ts_ms": hb.get("last_ok_ts_ms"), + "last_error": hb.get("last_error"), + "consecutive_failures": hb.get("consecutive_failures"), + "meta": hb.get("meta"), + }, + "call": by_side.get("C"), + "put": by_side.get("P"), + } + finally: + repo.close() diff --git a/apps/api/routes/notify.py b/apps/api/routes/notify.py new file mode 100644 index 0000000..95accd1 --- /dev/null +++ b/apps/api/routes/notify.py @@ -0,0 +1,34 @@ +"""通知相关 API（需鉴权）。""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends + +from apps.api.auth import require_auth +from packages.notify import wecom + +router = APIRouter(prefix="/notify", tags=["notify"], dependencies=[Depends(require_auth)]) + + +@router.get("/wecom/status") +def wecom_status() -> dict: + url = wecom.wecom_webhook_url() + masked = None + if url: + if len(url) > 24: + masked = url[:18] + "…" + url[-6:] + else: + masked = "***" + return { + "enabled": wecom.wecom_enabled(), + "webhook_configured": bool(url), + "webhook_url_masked": masked, + "machine_name": wecom.wecom_machine_name(), + "alert_fail_threshold": wecom.alert_fail_threshold(), + } + + +@router.post("/wecom/test") +def wecom_test() -> dict: + ok, msg = wecom.notify_test() + return {"ok": ok, "message": msg} diff --git a/apps/api/routes/samples.py b/apps/api/routes/samples.py new file mode 100644 index 0000000..d168d6c --- /dev/null +++ b/apps/api/routes/samples.py @@ -0,0 +1,29 @@ +"""原始样本调试接口。""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends, Query + +from apps.api.auth import require_auth +from packages.config import get_settings +from packages.db import Repository + +router = APIRouter(tags=["samples"], dependencies=[Depends(require_auth)]) + + +@router.get("/samples/recent") +def recent_samples(limit: int = Query(default=20, ge=1, le=200)) -> dict: + s = get_settings() + repo = Repository(s.db_path) + try: + rows = repo.conn.execute( + """ + SELECT * FROM option_quotes + ORDER BY ts_ms DESC, id DESC + LIMIT ? + """, + (limit,), + ).fetchall() + return {"items": [dict(r) for r in rows]} + finally: + repo.close() diff --git a/apps/api/routes/stats.py b/apps/api/routes/stats.py new file mode 100644 index 0000000..2adf860 --- /dev/null +++ b/apps/api/routes/stats.py @@ -0,0 +1,113 @@ +"""日/周/月统计 API。""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends, HTTPException, Query + +from apps.api.auth import require_auth +from apps.worker.settle import ensure_settlements_for_ymds +from packages.config import get_settings +from packages.db import Repository +from packages.domain.aggregate import leverage_stats_payload, move_points_stats_payload +from packages.domain.range import resolve_range + +router = APIRouter(prefix="/stats", tags=["stats"], dependencies=[Depends(require_auth)]) + + +def _range_info(range_name: str, date: str | None) -> dict: + s = get_settings() + try: + return resolve_range( + range_name, + date, + month_mode=s.month_range_mode, + ) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) from e + + +@router.get("/leverage") +def leverage_stats( + range: str = Query(default="day", pattern="^(day|week|month)$"), + date: str | None = Query(default=None, description="锚点日 YYYY-MM-DD（上海）"), + side: str = Query(default="both", pattern="^(C|P|both)$"), + bucket_minutes: int = Query(default=60, ge=15, le=120), +) -> dict: + s = get_settings() + info = _range_info(range, date) + repo = Repository(s.db_path) + try: + rows = repo.fetch_option_quotes( + start_ms=info["start_ms"], + end_ms=info["end_ms"], + side=side, + underlying=s.underlying, + ) + return leverage_stats_payload( + rows, + range_info=info, + bucket_minutes=bucket_minutes, + min_leverage=float(s.min_option_leverage), + side=side, + ) + finally: + repo.close() + + +@router.get("/move_points") +def move_points_stats( + range: str = Query(default="day", pattern="^(day|week|month)$"), + date: str | None = Query(default=None, description="锚点日 YYYY-MM-DD"), + side: str = Query(default="both", pattern="^(C|P|both)$"), + bucket_minutes: int = Query(default=60, ge=15, le=120), +) -> dict: + s = get_settings() + info = _range_info(range, date) + repo = Repository(s.db_path) + try: + rows = repo.fetch_option_quotes( + start_ms=info["start_ms"], + end_ms=info["end_ms"], + side=side, + underlying=s.underlying, + ) + ymds = sorted({str(r.get("expiry_ymd")) for r in rows if r.get("expiry_ymd")}) + # 懒回填：已到期但缺锚点时尽量补齐（本地指数优先，失败则跳过） + try: + ensure_settlements_for_ymds( + repo, + ymds, + underlying=s.underlying, + index_inst_id=s.index_inst_id, + ) + except Exception: # noqa: BLE001 — 回填失败不阻断统计 + pass + settlements = repo.list_settlements(ymds) + return move_points_stats_payload( + rows, + settlements, + range_info=info, + bucket_minutes=bucket_minutes, + side=side, + ) + finally: + repo.close() + + +@router.get("/ops-map") +def ops_map( + range: str = Query(default="day", pattern="^(day|week|month)$"), + date: str | None = Query(default=None), + side: str = Query(default="both", pattern="^(C|P|both)$"), + bucket_minutes: int = Query(default=60, ge=15, le=120), +) -> dict: + lev = leverage_stats(range=range, date=date, side=side, bucket_minutes=bucket_minutes) + mov = move_points_stats(range=range, date=date, side=side, bucket_minutes=bucket_minutes) + return { + "range": range, + "date": lev.get("date"), + "side": side, + "bucket_minutes": bucket_minutes, + "leverage": lev, + "move_points": mov, + } diff --git a/apps/collector/__init__.py b/apps/collector/__init__.py new file mode 100644 index 0000000..41776c2 --- /dev/null +++ b/apps/collector/__init__.py @@ -0,0 +1 @@ +# OKX market collector diff --git a/apps/collector/main.py b/apps/collector/main.py new file mode 100644 index 0000000..deb5aba --- /dev/null +++ b/apps/collector/main.py @@ -0,0 +1,173 @@ +"""采集入口：OKX 指数 + ATM Call/Put 周期采样落库。""" + +from __future__ import annotations + +import logging +import signal +import sys +import time +from typing import Any + +from apps.collector.okx_rest import OkxRestClient +from apps.collector.selectors import rows_to_contracts, select_atm_pair +from packages.config import get_settings +from packages.db import Repository +from packages.db.repository import OptionQuoteRow +from packages.domain import option_leverage +from packages.notify import wecom + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s [collector] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", +) +log = logging.getLogger("collector") + +_STOP = False + + +def _handle_signal(signum: int, _frame: Any) -> None: + global _STOP + log.info("signal %s received, stopping…", signum) + _STOP = True + + +def _now_ms() -> int: + return int(time.time() * 1000) + + +def sample_once( + client: OkxRestClient, + repo: Repository, + *, + contracts_cache: list[dict[str, Any]], + settings: Any, +) -> dict[str, Any]: + ts_ms = _now_ms() + index_px = client.fetch_index_ticker(settings.index_inst_id) + if index_px is None or index_px <= 0: + raise RuntimeError(f"index unavailable: {settings.index_inst_id}") + + repo.insert_index_tick( + ts_ms=ts_ms, + exchange="okx", + underlying=settings.underlying, + index_px=float(index_px), + ) + + pair = select_atm_pair( + contracts_cache, + index_px=float(index_px), + min_hours=float(settings.min_option_hours), + ) + if pair is None: + raise RuntimeError("no eligible ATM option pair") + + meta: dict[str, Any] = { + "index_px": index_px, + "expiry_ymd": pair.expiry_ymd, + "strike": pair.strike, + "call_inst_id": pair.call_inst_id, + "put_inst_id": pair.put_inst_id, + } + + for side, inst_id in (("C", pair.call_inst_id), ("P", pair.put_inst_id)): + ask, bid, ask_sz, bid_sz, book_ts = client.fetch_books(inst_id) + lev = option_leverage(float(index_px), ask) + repo.insert_option_quote( + OptionQuoteRow( + ts_ms=book_ts or ts_ms, + exchange="okx", + underlying=settings.underlying, + inst_id=inst_id, + expiry_ymd=pair.expiry_ymd, + strike=pair.strike, + side=side, + index_px=float(index_px), + ask=ask, + bid=bid, + ask_sz=ask_sz, + bid_sz=bid_sz, + leverage=lev, + ) + ) + meta[f"{side}_ask"] = ask + meta[f"{side}_leverage"] = lev + + return meta + + +def run() -> int: + settings = get_settings() + log.info( + "start underlying=%s family=%s interval=%ss db=%s", + settings.underlying, + settings.option_inst_family, + settings.sample_interval_sec, + settings.db_path, + ) + + repo = Repository(settings.db_path) + client = OkxRestClient( + base_url=settings.okx_base_url, + proxy=settings.okx_proxy or None, + ) + + contracts: list[dict[str, Any]] = [] + last_instruments_at = 0.0 + + try: + while not _STOP: + t0 = time.monotonic() + try: + now = time.monotonic() + if ( + not contracts + or now - last_instruments_at >= float(settings.instruments_refresh_sec) + ): + raw = client.fetch_option_instruments(settings.option_inst_family) + contracts = rows_to_contracts(raw) + last_instruments_at = now + log.info("instruments refreshed: %d contracts", len(contracts)) + + meta = sample_once(client, repo, contracts_cache=contracts, settings=settings) + repo.upsert_heartbeat(ok=True, meta=meta) + wecom.notify_collector_recovered() + log.info( + "sampled index=%.2f expiry=%s strike=%.0f C_lev=%s P_lev=%s", + meta["index_px"], + meta["expiry_ymd"], + meta["strike"], + f"{meta.get('C_leverage'):.1f}" if meta.get("C_leverage") else "-", + f"{meta.get('P_leverage'):.1f}" if meta.get("P_leverage") else "-", + ) + except Exception as e: # noqa: BLE001 — 单次失败记日志并跳过 + log.exception("sample failed: %s", e) + repo.upsert_heartbeat(ok=False, error=str(e)) + hb = repo.get_heartbeat() + wecom.notify_collector_fault( + error=str(e), + consecutive_failures=int(hb.get("consecutive_failures") or 0), + ) + + elapsed = time.monotonic() - t0 + sleep_for = max(1.0, float(settings.sample_interval_sec) - elapsed) + # 可中断 sleep + end = time.monotonic() + sleep_for + while not _STOP and time.monotonic() < end: + time.sleep(min(0.5, end - time.monotonic())) + finally: + client.close() + repo.close() + log.info("stopped") + return 0 + + +def main() -> None: + signal.signal(signal.SIGINT, _handle_signal) + signal.signal(signal.SIGTERM, _handle_signal) + sys.exit(run()) + + +if __name__ == "__main__": + main() diff --git a/apps/collector/okx_rest.py b/apps/collector/okx_rest.py new file mode 100644 index 0000000..d5b6fd3 --- /dev/null +++ b/apps/collector/okx_rest.py @@ -0,0 +1,154 @@ +"""OKX REST 只读行情。禁止任何交易类接口。""" + +from __future__ import annotations + +from typing import Any + +import httpx + + +def safe_float(v: Any) -> float | None: + if v is None or v == "": + return None + try: + return float(v) + except (TypeError, ValueError): + return None + + +class OkxRestClient: + """仅调用公开行情 / 公共接口。""" + + # 硬黑名单：防止误用交易路径 + _FORBIDDEN_PREFIXES = ( + "/api/v5/trade", + "/api/v5/account", + "/api/v5/asset", + "/api/v5/users", + ) + + def __init__( + self, + base_url: str = "https://www.okx.com", + timeout: float = 15.0, + proxy: str | None = None, + ) -> None: + self.base_url = base_url.rstrip("/") + self.proxy = (proxy or "").strip() or None + self._client = httpx.Client( + base_url=self.base_url, + timeout=timeout, + proxy=self.proxy, + headers={"Accept": "application/json", "User-Agent": "market_intel/0.1"}, + ) + + def close(self) -> None: + self._client.close() + + def __enter__(self) -> OkxRestClient: + return self + + def __exit__(self, *args: object) -> None: + self.close() + + def _get(self, path: str, params: dict[str, Any] | None = None) -> list[dict[str, Any]]: + for bad in self._FORBIDDEN_PREFIXES: + if path.startswith(bad): + raise RuntimeError(f"forbidden trading path: {path}") + r = self._client.get(path, params=params or {}) + r.raise_for_status() + body = r.json() + if str(body.get("code")) != "0": + raise RuntimeError(f"OKX REST error code={body.get('code')} msg={body.get('msg')}") + data = body.get("data") or [] + return [x for x in data if isinstance(x, dict)] + + def _get_raw(self, path: str, params: dict[str, Any] | None = None) -> list[Any]: + for bad in self._FORBIDDEN_PREFIXES: + if path.startswith(bad): + raise RuntimeError(f"forbidden trading path: {path}") + r = self._client.get(path, params=params or {}) + r.raise_for_status() + body = r.json() + if str(body.get("code")) != "0": + raise RuntimeError(f"OKX REST error code={body.get('code')} msg={body.get('msg')}") + data = body.get("data") or [] + return data if isinstance(data, list) else [] + + def fetch_option_instruments(self, inst_family: str) -> list[dict[str, Any]]: + rows = self._get( + "/api/v5/public/instruments", + {"instType": "OPTION", "instFamily": inst_family}, + ) + return [r for r in rows if str(r.get("state") or "").lower() == "live"] + + def fetch_index_ticker(self, inst_id: str) -> float | None: + rows = self._get("/api/v5/market/index-tickers", {"instId": inst_id}) + if not rows: + return None + return safe_float(rows[0].get("idxPx")) + + def fetch_index_at( + self, inst_id: str, target_ts_ms: int + ) -> tuple[float | None, int | None]: + """ + 用 1m 历史指数 K 线取最接近 target 的收盘价。 + OKX: /api/v5/market/history-index-candles + candle: [ts, o, h, l, c, confirm, ...] + """ + # before = 请求此时间戳之前的数据；取到期前后窗口 + before = int(target_ts_ms) + 60_000 + after = int(target_ts_ms) - 10 * 60_000 + rows = self._get_raw( + "/api/v5/market/history-index-candles", + { + "instId": inst_id, + "bar": "1m", + "before": str(before), + "after": str(after), + "limit": "20", + }, + ) + best_px: float | None = None + best_ts: int | None = None + best_delta: int | None = None + for row in rows: + if not isinstance(row, (list, tuple)) or len(row) < 5: + continue + ts = safe_float(row[0]) + close = safe_float(row[4]) + if ts is None or close is None: + continue + ts_i = int(ts) + delta = abs(ts_i - int(target_ts_ms)) + if best_delta is None or delta < best_delta: + best_delta = delta + best_px = close + best_ts = ts_i + if best_delta is not None and best_delta > 5 * 60_000: + return None, None + return best_px, best_ts + + def fetch_books( + self, inst_id: str, sz: int = 5 + ) -> tuple[float | None, float | None, float | None, float | None, int | None]: + """返回 ask, bid, ask_sz, bid_sz, ts_ms。""" + rows = self._get( + "/api/v5/market/books", + {"instId": inst_id, "sz": str(max(1, min(int(sz), 400)))}, + ) + if not rows: + return None, None, None, None, None + row = rows[0] + ts = safe_float(row.get("ts")) + ts_ms = int(ts) if ts is not None else None + asks = row.get("asks") or [] + bids = row.get("bids") or [] + ask = ask_sz = bid = bid_sz = None + if asks and isinstance(asks[0], (list, tuple)) and len(asks[0]) >= 2: + ask = safe_float(asks[0][0]) + ask_sz = safe_float(asks[0][1]) + if bids and isinstance(bids[0], (list, tuple)) and len(bids[0]) >= 2: + bid = safe_float(bids[0][0]) + bid_sz = safe_float(bids[0][1]) + return ask, bid, ask_sz, bid_sz, ts_ms diff --git a/apps/collector/okx_ws.py b/apps/collector/okx_ws.py new file mode 100644 index 0000000..fb9140a --- /dev/null +++ b/apps/collector/okx_ws.py @@ -0,0 +1,5 @@ +"""WebSocket 占位（P1 用 REST；WS 后期可接）。""" + +from __future__ import annotations + +# 第一期采集走 REST 轮询；此模块预留多路订阅入口。 diff --git a/apps/collector/selectors.py b/apps/collector/selectors.py new file mode 100644 index 0000000..2fa9c8e --- /dev/null +++ b/apps/collector/selectors.py @@ -0,0 +1,157 @@ +"""ATM / 合资格到期选择。规则：最接近指数的行权价；最近剩余时长 ≥ min_hours 的到期。""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from datetime import datetime +from typing import Any +from zoneinfo import ZoneInfo + +from packages.domain.expiry import expiry_ms_from_ymd + +_SH = ZoneInfo("Asia/Shanghai") +_DATE_RE = re.compile(r"^\d{6}$") + + +@dataclass(frozen=True) +class OptionPair: + expiry_ymd: str + expiry_ms: int + strike: float + call_inst_id: str + put_inst_id: str + + +def safe_float(v: Any) -> float | None: + if v is None or v == "": + return None + try: + return float(v) + except (TypeError, ValueError): + return None + + +def parse_option_inst_id(inst_id: str) -> tuple[str | None, float | None, str | None]: + """ETH-USD_UM-YYMMDD-STRIKE-C → (YYMMDD, strike, C|P).""" + parts = (inst_id or "").strip().split("-") + if len(parts) < 5: + return None, None, None + ymd = parts[-3] + strike = safe_float(parts[-2]) + opt = parts[-1].upper() + if not _DATE_RE.fullmatch(ymd) or strike is None or opt not in ("C", "P"): + return None, None, None + return ymd, strike, opt + + +def rows_to_contracts(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + for row in rows: + if not isinstance(row, dict): + continue + state = str(row.get("state") or "live").lower() + if state and state != "live": + continue + inst_id = str(row.get("instId") or "") + y, stk, opt = parse_option_inst_id(inst_id) + exp_ms: int | None = None + if y is None or stk is None or opt is None: + from datetime import timezone + + exp = safe_float(row.get("expTime")) + if exp: + ms = int(exp) if exp > 10_000_000_000 else int(exp * 1000) + y = datetime.fromtimestamp(ms / 1000, tz=timezone.utc).strftime("%y%m%d") + exp_ms = ms + stk = safe_float(row.get("stk")) + opt_raw = str(row.get("optType") or "").upper() + opt = opt_raw if opt_raw in ("C", "P") else None + if not inst_id or not y or stk is None or opt not in ("C", "P"): + continue + if exp_ms is None: + exp_ms = expiry_ms_from_ymd(y) + out.append( + { + "inst_id": inst_id, + "expiry_ymd": y, + "expiry_ms": int(exp_ms), + "strike": float(stk), + "side": opt, + } + ) + return out + + +def hours_until_ms(expiry_ms: int, now: datetime | None = None) -> float: + n = (now or datetime.now(tz=_SH)).astimezone(_SH) + return (int(expiry_ms) - int(n.timestamp() * 1000)) / 3_600_000.0 + + +def pick_atm_strike(strikes: list[float], index_px: float) -> float | None: + """最接近指数的行权价（平值）。""" + if not strikes or index_px <= 0: + return None + return min(strikes, key=lambda s: (abs(s - index_px), s)) + + +def _complete_by_expiry( + contracts: list[dict[str, Any]], +) -> dict[str, tuple[int, dict[float, dict[str, str]]]]: + by_exp: dict[str, dict[float, dict[str, str]]] = {} + ms_map: dict[str, int] = {} + for c in contracts: + y = str(c.get("expiry_ymd") or "") + stk = c.get("strike") + opt = str(c.get("side") or "").upper() + inst_id = str(c.get("inst_id") or "") + if not y or stk is None or opt not in ("C", "P") or not inst_id: + continue + by_exp.setdefault(y, {}).setdefault(float(stk), {})[opt] = inst_id + if c.get("expiry_ms") is not None: + ms_map[y] = int(c["expiry_ms"]) + out: dict[str, tuple[int, dict[float, dict[str, str]]]] = {} + for ymd, strikes in by_exp.items(): + complete = {s: v for s, v in strikes.items() if "C" in v and "P" in v} + if not complete: + continue + ems = ms_map.get(ymd) or expiry_ms_from_ymd(ymd) + out[ymd] = (ems, complete) + return out + + +def select_atm_pair( + contracts: list[dict[str, Any]], + *, + index_px: float, + min_hours: float = 12.0, + now: datetime | None = None, +) -> OptionPair | None: + """ + 选最近合资格到期（剩余 ≥ min_hours）+ ATM Call/Put。 + ATM = 行权价最接近指数。 + """ + complete = _complete_by_expiry(contracts) + if not complete or index_px <= 0: + return None + eligible = [ + ymd + for ymd, (ems, _) in complete.items() + if hours_until_ms(ems, now) + 1e-9 >= float(min_hours) + ] + if not eligible: + return None + eligible.sort(key=lambda y: complete[y][0]) + ymd = eligible[0] + ems, strikes_map = complete[ymd] + strike = pick_atm_strike(list(strikes_map.keys()), index_px) + if strike is None: + return None + legs = strikes_map[strike] + return OptionPair( + expiry_ymd=ymd, + expiry_ms=ems, + strike=float(strike), + call_inst_id=legs["C"], + put_inst_id=legs["P"], + ) diff --git a/apps/worker/__init__.py b/apps/worker/__init__.py new file mode 100644 index 0000000..56ccfc5 --- /dev/null +++ b/apps/worker/__init__.py @@ -0,0 +1 @@ +# worker package — 到期回填 / 日终聚合 diff --git a/apps/worker/main.py b/apps/worker/main.py new file mode 100644 index 0000000..2e4bca9 --- /dev/null +++ b/apps/worker/main.py @@ -0,0 +1,82 @@ +"""Worker 入口：周期回填到期结算指数。""" + +from __future__ import annotations + +import logging +import signal +import sys +import time +from typing import Any + +from apps.collector.okx_rest import OkxRestClient +from apps.worker.settle import backfill_settlements +from packages.config import get_settings +from packages.db import Repository + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s [worker] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", +) +log = logging.getLogger("worker") + +_STOP = False + + +def _handle_signal(signum: int, _frame: Any) -> None: + global _STOP + log.info("signal %s received, stopping…", signum) + _STOP = True + + +def run() -> int: + settings = get_settings() + interval = max(60, int(settings.settle_backfill_interval_sec)) + log.info( + "start settle backfill interval=%ss db=%s", + interval, + settings.db_path, + ) + repo = Repository(settings.db_path) + client = OkxRestClient( + base_url=settings.okx_base_url, + proxy=settings.okx_proxy or None, + ) + try: + while not _STOP: + try: + result = backfill_settlements( + repo, + underlying=settings.underlying, + index_inst_id=settings.index_inst_id, + client=client, + ) + log.info( + "backfill filled=%s skipped=%s errors=%s", + len(result["filled"]), + len(result["skipped"]), + len(result["errors"]), + ) + for err in result["errors"][:5]: + log.warning(" %s", err) + except Exception as e: # noqa: BLE001 + log.exception("backfill loop failed: %s", e) + + end = time.monotonic() + interval + while not _STOP and time.monotonic() < end: + time.sleep(min(1.0, end - time.monotonic())) + finally: + client.close() + repo.close() + log.info("stopped") + return 0 + + +def main() -> None: + signal.signal(signal.SIGINT, _handle_signal) + signal.signal(signal.SIGTERM, _handle_signal) + sys.exit(run()) + + +if __name__ == "__main__": + main() diff --git a/apps/worker/settle.py b/apps/worker/settle.py new file mode 100644 index 0000000..dace237 --- /dev/null +++ b/apps/worker/settle.py @@ -0,0 +1,187 @@ +"""到期结算回填：从本地指数或 OKX 历史指数锚定 settle_index_px。""" + +from __future__ import annotations + +import logging +import time +from typing import Any + +from apps.collector.okx_rest import OkxRestClient, safe_float +from packages.db.repository import Repository +from packages.domain.expiry import expiry_ms_from_ymd + +log = logging.getLogger("worker.settle") + +# 本地 index_ticks 与到期时刻的最大偏离 +_LOCAL_MAX_DELTA_MS = 15 * 60 * 1000 + + +def list_expiry_ymds_needing_settle(repo: Repository, *, now_ms: int | None = None) -> list[str]: + """option_quotes 中已到期且尚未写入 settlements 的 expiry_ymd。""" + now = int(now_ms if now_ms is not None else time.time() * 1000) + rows = repo.conn.execute( + """ + SELECT DISTINCT expiry_ymd FROM option_quotes + WHERE expiry_ymd IS NOT NULL AND expiry_ymd != '' + ORDER BY expiry_ymd ASC + """ + ).fetchall() + out: list[str] = [] + for r in rows: + ymd = str(r["expiry_ymd"]) + try: + settle_ts = expiry_ms_from_ymd(ymd) + except ValueError: + continue + if settle_ts > now: + continue + if repo.get_settlement(ymd) is not None: + continue + out.append(ymd) + return out + + +def resolve_settle_index( + repo: Repository, + *, + expiry_ymd: str, + underlying: str, + index_inst_id: str, + exchange: str = "okx", + client: OkxRestClient | None = None, +) -> dict[str, Any] | None: + """ + 解析到期指数。优先本地 index_ticks 最近点；否则 OKX 历史指数 K 线。 + """ + settle_ts = expiry_ms_from_ymd(expiry_ymd) + local = repo.nearest_index_tick( + underlying=underlying, + target_ts_ms=settle_ts, + max_delta_ms=_LOCAL_MAX_DELTA_MS, + ) + if local is not None: + return { + "expiry_ymd": expiry_ymd, + "settle_ts_ms": settle_ts, + "settle_index_px": float(local["index_px"]), + "exchange": exchange, + "underlying": underlying, + "source": "index_ticks", + "source_ts_ms": int(local["ts_ms"]), + } + + own_client = client is None + cli = client or OkxRestClient() + try: + px, src_ts = cli.fetch_index_at(index_inst_id, settle_ts) + if px is None: + return None + return { + "expiry_ymd": expiry_ymd, + "settle_ts_ms": settle_ts, + "settle_index_px": float(px), + "exchange": exchange, + "underlying": underlying, + "source": "okx_history_index", + "source_ts_ms": src_ts, + } + finally: + if own_client: + cli.close() + + +def backfill_settlements( + repo: Repository, + *, + underlying: str, + index_inst_id: str, + client: OkxRestClient | None = None, + ymds: list[str] | None = None, + now_ms: int | None = None, +) -> dict[str, Any]: + """回填到期锚点；返回 {filled, skipped, pending, errors}。""" + targets = ymds if ymds is not None else list_expiry_ymds_needing_settle(repo, now_ms=now_ms) + filled: list[str] = [] + skipped: list[str] = [] + errors: list[str] = [] + + own_client = client is None + cli = client + try: + for ymd in targets: + if repo.get_settlement(ymd) is not None: + skipped.append(ymd) + continue + try: + if cli is None: + cli = OkxRestClient() + row = resolve_settle_index( + repo, + expiry_ymd=ymd, + underlying=underlying, + index_inst_id=index_inst_id, + client=cli, + ) + if row is None: + errors.append(f"{ymd}: settle index unavailable") + continue + repo.upsert_settlement( + expiry_ymd=row["expiry_ymd"], + settle_ts_ms=int(row["settle_ts_ms"]), + settle_index_px=float(row["settle_index_px"]), + exchange=str(row["exchange"]), + underlying=str(row["underlying"]), + ) + filled.append(ymd) + log.info( + "settled %s index=%.4f source=%s", + ymd, + row["settle_index_px"], + row.get("source"), + ) + except Exception as e: # noqa: BLE001 + errors.append(f"{ymd}: {e}") + log.exception("backfill %s failed", ymd) + finally: + if own_client and cli is not None: + cli.close() + + return {"filled": filled, "skipped": skipped, "errors": errors, "targets": targets} + + +def ensure_settlements_for_ymds( + repo: Repository, + ymds: list[str], + *, + underlying: str, + index_inst_id: str, + client: OkxRestClient | None = None, + now_ms: int | None = None, +) -> dict[str, Any]: + """对给定到期日尽量回填（未到期的跳过）。""" + now = int(now_ms if now_ms is not None else time.time() * 1000) + due = [] + for ymd in sorted(set(ymds)): + try: + if expiry_ms_from_ymd(ymd) <= now: + due.append(ymd) + except ValueError: + continue + return backfill_settlements( + repo, + underlying=underlying, + index_inst_id=index_inst_id, + client=client, + ymds=due, + now_ms=now, + ) + + +# re-export for typing clarity +__all__ = [ + "backfill_settlements", + "ensure_settlements_for_ymds", + "list_expiry_ymds_needing_settle", + "resolve_settle_index", + "safe_float", +] diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 0000000..d3f5a12 --- /dev/null +++ b/data/.gitkeep @@ -0,0 +1 @@ + diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh new file mode 100644 index 0000000..f2985d7 --- /dev/null +++ b/deploy/bootstrap.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# 兼容入口：转发到 manage.sh +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +exec bash "${ROOT}/deploy/manage.sh" "$@" diff --git a/deploy/lib/common.sh b/deploy/lib/common.sh new file mode 100644 index 0000000..01025f8 --- /dev/null +++ b/deploy/lib/common.sh @@ -0,0 +1,358 @@ +#!/usr/bin/env bash +# deploy/lib/common.sh — market_intel 部署公共函数（Docker Compose） +# 目标系统: Ubuntu 22.04 LTS +set -e +set -u +if [ -n "${BASH_VERSION:-}" ]; then + set -o pipefail +fi + +INSTALL_ROOT="${INSTALL_ROOT:-/opt/market_intel}" +GIT_URL="${GIT_URL:-https://git.bz121.com/dekun/market_intel.git}" +GIT_BRANCH="${GIT_BRANCH:-main}" +BACKUP_ROOT="${BACKUP_ROOT:-/root/backups/market_intel}" +TZ_NAME="${MI_TZ:-Asia/Shanghai}" +MI_PORT_DEFAULT="${MI_PORT_DEFAULT:-5170}" + +LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEPLOY_DIR="$(cd "${LIB_DIR}/.." && pwd)" +REPO_ROOT="$(cd "${DEPLOY_DIR}/.." && pwd)" + +log() { printf '[%s] %s\n' "$(TZ="${TZ_NAME}" date '+%Y-%m-%d %H:%M:%S')" "$*"; } +step() { echo ""; log "==> $*"; } + +die() { + echo "错误: $*" >&2 + exit 1 +} + +require_root() { + if [[ "$(id -u)" -ne 0 ]]; then + die "请使用 root 执行(推荐: sudo -i 后运行)" + fi +} + +_apt_lock_holders() { + ps -eo pid,cmd 2>/dev/null | grep -E '[u]nattended-upgr|[a]pt-get|[a]pt |[d]pkg ' | head -n 8 || true + if command -v fuser >/dev/null 2>&1; then + fuser -v /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/lib/apt/lists/lock 2>&1 | head -n 12 || true + fi +} + +_stop_auto_apt_for_deploy() { + if [[ "${APT_STOP_AUTO:-1}" != "1" ]]; then + return 0 + fi + if command -v systemctl >/dev/null 2>&1; then + log "临时停止 unattended-upgrades / apt-daily，避免占锁…" + systemctl stop unattended-upgrades.service 2>/dev/null || true + systemctl stop apt-daily.service apt-daily-upgrade.service 2>/dev/null || true + systemctl kill --kill-who=all unattended-upgrades.service 2>/dev/null || true + fi + if [[ "${APT_FORCE_UNLOCK:-0}" == "1" ]]; then + log "APT_FORCE_UNLOCK=1：结束残留 apt/dpkg 进程…" + pkill -9 -x unattended-upgr 2>/dev/null || true + pkill -9 -x apt-get 2>/dev/null || true + pkill -9 -x apt 2>/dev/null || true + pkill -9 -x dpkg 2>/dev/null || true + sleep 2 + dpkg --configure -a 2>/dev/null || true + fi +} + +wait_for_apt_lock() { + local max_wait="${1:-600}" + local waited=0 + local tried_stop=0 + if ! command -v apt-get >/dev/null 2>&1; then + return 0 + fi + while true; do + local busy=0 + if pgrep -x unattended-upgr >/dev/null 2>&1 \ + || pgrep -x apt-get >/dev/null 2>&1 \ + || pgrep -x apt >/dev/null 2>&1 \ + || pgrep -x dpkg >/dev/null 2>&1; then + busy=1 + fi + if command -v fuser >/dev/null 2>&1; then + if fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 \ + || fuser /var/lib/dpkg/lock >/dev/null 2>&1 \ + || fuser /var/lib/apt/lists/lock >/dev/null 2>&1; then + busy=1 + fi + fi + if [[ "${busy}" -eq 0 ]]; then + [[ "${waited}" -gt 0 ]] && log "apt 锁已释放，继续安装" + return 0 + fi + if [[ "${waited}" -eq 0 ]]; then + log "检测到 apt/dpkg 正被占用，等待释放…" + _apt_lock_holders | while IFS= read -r line; do log " ${line}"; done + elif [[ "${tried_stop}" -eq 0 && "${waited}" -ge 15 ]]; then + tried_stop=1 + _stop_auto_apt_for_deploy + elif [[ $((waited % 60)) -eq 0 ]]; then + log "仍在等待 apt 锁…已等 ${waited}s / ${max_wait}s" + fi + if [[ "${waited}" -ge "${max_wait}" ]]; then + die "等待 apt 锁超时(${max_wait}s)" + fi + sleep 5 + waited=$((waited + 5)) + done +} + +apt_update() { + wait_for_apt_lock + apt-get update -qq +} + +apt_install() { + wait_for_apt_lock + apt-get install -y "$@" +} + +detect_server_ip() { + local ip="" + if command -v hostname >/dev/null 2>&1; then + ip="$(hostname -I 2>/dev/null | awk '{print $1}')" + fi + [[ -z "${ip}" ]] && ip="127.0.0.1" + echo "${ip}" +} + +cm_read() { + local __var="$1" + local __prompt="${2:-}" + local __line="" + if [[ -n "${__prompt}" ]]; then + printf '%s' "${__prompt}" >/dev/tty 2>/dev/null || printf '%s' "${__prompt}" + fi + if [[ -r /dev/tty ]]; then + IFS= read -r __line /dev/null | tail -n1 | cut -d= -f2- || true)" + fi + printf '%s' "${val}" +} + +# 已有非空不覆盖；空或缺省则写入/询问 +ensure_env_key() { + local file="$1" + local key="$2" + local prompt="$3" + local default="$4" + local current + current="$(read_env_value "${file}" "${key}")" + if [[ -n "${current}" ]]; then + log "保留 ${key}=***（已有非空值）" + return 0 + fi + local input="" + if [[ -n "${prompt}" ]]; then + cm_read input "${prompt} [${default}]: " + fi + if [[ -z "${input}" ]]; then + input="${default}" + fi + if grep -qE "^${key}=" "${file}" 2>/dev/null; then + # 替换空值行 + local tmp + tmp="$(mktemp)" + awk -v k="${key}" -v v="${input}" ' + BEGIN{FS=OFS="="} + $1==k {$0=k"="v} + {print} + ' "${file}" >"${tmp}" && mv "${tmp}" "${file}" + else + printf '%s=%s\n' "${key}" "${input}" >>"${file}" + fi + log "已设置 ${key}" +} + +ensure_dotenv() { + local root="$1" + local envf="${root}/.env" + if [[ ! -f "${envf}" ]]; then + if [[ -f "${root}/.env.example" ]]; then + cp -a "${root}/.env.example" "${envf}" + log "已从 .env.example 生成 .env" + else + touch "${envf}" + fi + fi + echo "" + echo "配置 .env（回车采用默认；已有非空值不覆盖）" + ensure_env_key "${envf}" "MI_PORT" "HTTP 端口" "${MI_PORT_DEFAULT}" + ensure_env_key "${envf}" "SAMPLE_INTERVAL_SEC" "采样间隔秒" "30" + ensure_env_key "${envf}" "MIN_OPTION_LEVERAGE" "杠杆达标线" "100" + ensure_env_key "${envf}" "AUTH_SECRET" "鉴权密钥(disabled 关闭)" "change-me" + ensure_env_key "${envf}" "ADMIN_PASSWORD" "管理员密码" "admin123" + ensure_env_key "${envf}" "OKX_API_KEY" "OKX API Key(可空)" "" + ensure_env_key "${envf}" "OKX_API_SECRET" "OKX API Secret(可空)" "" + ensure_env_key "${envf}" "OKX_API_PASSPHRASE" "OKX Passphrase(可空)" "" + ensure_env_key "${envf}" "WECOM_ENABLED" "企微告警 1/0" "0" + ensure_env_key "${envf}" "WECOM_WEBHOOK_URL" "企微 Webhook(可空)" "" + ensure_env_key "${envf}" "WECOM_MACHINE_NAME" "机器名(可空)" "" + ensure_env_key "${envf}" "ALERT_FAIL_THRESHOLD" "连续失败告警阈值" "5" + # 固定库路径（容器内） + if [[ -z "$(read_env_value "${envf}" "MI_DB_PATH")" ]]; then + ensure_env_key "${envf}" "MI_DB_PATH" "" "/app/data/market_intel.db" + fi +} +docker_ok() { + command -v docker >/dev/null 2>&1 || return 1 + docker compose version >/dev/null 2>&1 || return 1 + return 0 +} + +ensure_docker() { + step "环境检测 (Docker / Compose)" + if docker_ok; then + log "Docker 已就绪: $(docker --version 2>&1)" + log "Compose: $(docker compose version 2>&1)" + return 0 + fi + if ! command -v apt-get >/dev/null 2>&1; then + die "未找到 Docker，且无 apt-get，请手动安装 Docker + Compose 插件" + fi + step "安装 Docker Engine + Compose 插件" + export DEBIAN_FRONTEND=noninteractive + apt_update + apt_install ca-certificates curl gnupg + install -m 0755 -d /etc/apt/keyrings + if [[ ! -f /etc/apt/keyrings/docker.gpg ]]; then + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg + chmod a+r /etc/apt/keyrings/docker.gpg + fi + local codename + codename="$(. /etc/os-release && echo "${VERSION_CODENAME}")" + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu ${codename} stable" \ + >/etc/apt/sources.list.d/docker.list + apt_update + apt_install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + systemctl enable --now docker 2>/dev/null || true + if ! docker_ok; then + die "Docker 安装后仍不可用" + fi + log "Docker 安装完成" +} + +compose() { + local root="${REPO_ROOT:-${INSTALL_ROOT}}" + (cd "${root}" && docker compose "$@") +} + +compose_up_build() { + step "docker compose up -d --build" + compose up -d --build +} + +compose_stop() { + require_root + if ! REPO_ROOT="$(resolve_repo_root)"; then + die "未找到安装目录 ${INSTALL_ROOT}" + fi + step "停止服务" + compose stop + log "已停止" +} + +compose_start() { + require_root + if ! REPO_ROOT="$(resolve_repo_root)"; then + die "未找到安装目录 ${INSTALL_ROOT}" + fi + step "启动服务" + compose up -d + verify_health || true +} + +read_mi_port() { + local root="${1:-${REPO_ROOT:-${INSTALL_ROOT}}}" + local p + p="$(read_env_value "${root}/.env" "MI_PORT")" + if [[ -z "${p}" ]]; then + p="${MI_PORT_DEFAULT}" + fi + echo "${p}" +} + +verify_health() { + local port + port="$(read_mi_port)" + local url="http://127.0.0.1:${port}/health" + step "健康检查 ${url}" + local i + for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do + if curl -fsS "${url}" >/dev/null 2>&1; then + log "health OK" + curl -fsS "${url}" || true + echo "" + return 0 + fi + sleep 2 + done + log "警告: health 暂未就绪，请检查: docker compose -f ${REPO_ROOT}/docker-compose.yml logs" + return 1 +} + +show_status() { + if ! REPO_ROOT="$(resolve_repo_root)"; then + die "未找到安装目录 ${INSTALL_ROOT}" + fi + step "容器状态" + compose ps || true + verify_health || true +} + +print_post_install_guide() { + local ip port + ip="$(detect_server_ip)" + port="$(read_mi_port)" + echo "" + echo "══════════════════════════════════════" + echo " 比特骆驼行情采集分析 部署完成" + echo " 目录: ${INSTALL_ROOT}" + echo " 本机: http://${ip}:${port}/health" + echo " 看板: http://${ip}:${port}/" + echo " 管理: bash ${INSTALL_ROOT}/deploy/manage.sh" + echo " 配置: ${INSTALL_ROOT}/.env" + echo "══════════════════════════════════════" + echo "" +} diff --git a/deploy/lib/install.sh b/deploy/lib/install.sh new file mode 100644 index 0000000..2c6a1d4 --- /dev/null +++ b/deploy/lib/install.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# deploy/lib/install.sh — 一键部署 market_intel（Docker Compose） +set -e +set -u +if [ -n "${BASH_VERSION:-}" ]; then + set -o pipefail +fi + +LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${LIB_DIR}/common.sh" + +run_pipeline() { + local root="$1" + REPO_ROOT="${root}" + ensure_dotenv "${root}" + compose_up_build + verify_health || true + print_post_install_guide +} + +install_fresh() { + require_root + step "一键部署 — Docker 环境检测" + ensure_docker + # 基础包 + if ! command -v git >/dev/null 2>&1 || ! command -v curl >/dev/null 2>&1; then + export DEBIAN_FRONTEND=noninteractive + apt_update + apt_install ca-certificates git curl + fi + step "克隆仓库 → ${INSTALL_ROOT}" + if [[ -d "${INSTALL_ROOT}" ]]; then + die "目录已存在: ${INSTALL_ROOT}，请先卸载或选修复" + fi + mkdir -p "$(dirname "${INSTALL_ROOT}")" + git clone -b "${GIT_BRANCH}" "${GIT_URL}" "${INSTALL_ROOT}" + run_pipeline "${INSTALL_ROOT}" +} + +install_repair() { + require_root + step "修复部署 — Docker 环境检测" + ensure_docker + if ! REPO_ROOT="$(resolve_repo_root)"; then + die "未找到安装目录 ${INSTALL_ROOT}" + fi + step "修复环境(保留 .env 与 data volume)" + if [[ -d "${REPO_ROOT}/.git" ]]; then + git -C "${REPO_ROOT}" pull --ff-only origin "${GIT_BRANCH}" 2>/dev/null \ + || git -C "${REPO_ROOT}" pull --ff-only 2>/dev/null \ + || true + fi + run_pipeline "${REPO_ROOT}" +} + +handle_existing() { + echo "" + echo "检测到已部署: ${INSTALL_ROOT}" + echo " a) 取消" + echo " b) 修复/重装环境(保留 .env 与数据 volume)" + local choice="" + cm_read choice "请选择 [a/b]: " + case "${choice}" in + b|B) install_repair ;; + *) log "已取消" ;; + esac +} + +main_install() { + require_root + if repo_ready "${INSTALL_ROOT}"; then + handle_existing + else + install_fresh + fi +} + +main_install "$@" diff --git a/deploy/lib/uninstall.sh b/deploy/lib/uninstall.sh new file mode 100644 index 0000000..dbaf3a6 --- /dev/null +++ b/deploy/lib/uninstall.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# deploy/lib/uninstall.sh — 停容器；可选删 volume；备份 .env；删除安装目录 +set -e +set -u +if [ -n "${BASH_VERSION:-}" ]; then + set -o pipefail +fi + +LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${LIB_DIR}/common.sh" + +main_uninstall() { + require_root + + local root="" + if ! root="$(resolve_repo_root)"; then + if [[ -d "${INSTALL_ROOT}" ]]; then + root="${INSTALL_ROOT}" + else + die "未找到安装目录 ${INSTALL_ROOT}" + fi + fi + REPO_ROOT="${root}" + + echo "" + echo "将卸载 比特骆驼行情采集分析 (market_intel):" + echo " - 停止并移除 Compose 容器" + echo " - 备份 .env 到 ${BACKUP_ROOT}" + echo " - 删除安装目录: ${INSTALL_ROOT}" + echo " - 可选：删除 data volume" + echo "" + if ! confirm_yes "确认卸载并删除 ${INSTALL_ROOT}?"; then + log "已取消卸载" + return 0 + fi + + local remove_volume=0 + if confirm_yes "是否同时删除 Docker data volume (mi_data)?"; then + remove_volume=1 + fi + + local stamp backup_dir + stamp="$(TZ="${TZ_NAME}" date +%Y%m%d-%H%M%S)" + backup_dir="${BACKUP_ROOT}/pre-uninstall-${stamp}" + mkdir -p "${backup_dir}" + + step "备份 .env" + if [[ -f "${REPO_ROOT}/.env" ]]; then + cp -a "${REPO_ROOT}/.env" "${backup_dir}/.env" + fi + { + echo "created_at=${stamp}" + echo "install_root=${INSTALL_ROOT}" + echo "remove_volume=${remove_volume}" + echo "action=rm_rf_install_root" + } >"${backup_dir}/uninstall.manifest" + + step "停止并移除容器" + if command -v docker >/dev/null 2>&1 && [[ -f "${REPO_ROOT}/docker-compose.yml" ]]; then + if [[ "${remove_volume}" -eq 1 ]]; then + (cd "${REPO_ROOT}" && docker compose down -v) || true + else + (cd "${REPO_ROOT}" && docker compose down) || true + fi + fi + + step "删除安装目录 ${INSTALL_ROOT}" + if [[ -d "${INSTALL_ROOT}" ]]; then + case "${INSTALL_ROOT}" in + /opt/market_intel|/opt/market_intel/) + rm -rf "${INSTALL_ROOT}" + log "已删除: ${INSTALL_ROOT}" + ;; + *) + if [[ "${ALLOW_UNSAFE_UNINSTALL:-}" == "1" ]]; then + rm -rf "${INSTALL_ROOT}" + log "已删除(ALLOW_UNSAFE_UNINSTALL=1): ${INSTALL_ROOT}" + else + die "拒绝删除非默认路径 ${INSTALL_ROOT}；若确认，设置 ALLOW_UNSAFE_UNINSTALL=1" + fi + ;; + esac + else + log "安装目录不存在,跳过删除" + fi + + local leftover + for leftover in /opt/market_intel.removed.* /opt/market_intel.old.*; do + if [[ -e "${leftover}" ]]; then + rm -rf "${leftover}" + log "已清理残留: ${leftover}" + fi + done + + echo "" + echo "卸载完成." + echo " 配置备份: ${backup_dir}" + echo " volume 已删除: $([[ ${remove_volume} -eq 1 ]] && echo yes || echo no)" + echo "" + echo "重新部署:" + echo " curl -fsSL https://git.bz121.com/dekun/market_intel/raw/branch/main/deploy/manage.sh | bash" +} + +main_uninstall "$@" diff --git a/deploy/lib/update.sh b/deploy/lib/update.sh new file mode 100644 index 0000000..7f0c224 --- /dev/null +++ b/deploy/lib/update.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# deploy/lib/update.sh — git pull + compose build/up；保留 .env 与 volume +set -e +set -u +if [ -n "${BASH_VERSION:-}" ]; then + set -o pipefail +fi + +LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${LIB_DIR}/common.sh" + +main_update() { + require_root + if ! REPO_ROOT="$(resolve_repo_root)"; then + die "未找到安装目录 ${INSTALL_ROOT}，请先执行「1) 一键部署」" + fi + if ! repo_ready "${REPO_ROOT}"; then + die "安装不完整，请先执行「1) 一键部署」" + fi + + step "更新 — Docker 环境检测" + ensure_docker + step "git pull" + if [[ -d "${REPO_ROOT}/.git" ]]; then + git -C "${REPO_ROOT}" fetch --all --prune + git -C "${REPO_ROOT}" checkout "${GIT_BRANCH}" 2>/dev/null || true + git -C "${REPO_ROOT}" pull --ff-only origin "${GIT_BRANCH}" \ + || git -C "${REPO_ROOT}" pull --ff-only + else + log "警告: 非 git 目录，跳过 pull" + fi + # 补全缺失 env key，不覆盖已有 + ensure_dotenv "${REPO_ROOT}" + compose_up_build + verify_health || true + echo "" + log "更新完成（.env 与 data volume 已保留）" +} + +main_update "$@" diff --git a/deploy/manage.sh b/deploy/manage.sh new file mode 100644 index 0000000..1eaed1a --- /dev/null +++ b/deploy/manage.sh @@ -0,0 +1,228 @@ +#!/usr/bin/env bash +# 比特骆驼行情采集分析 部署管理器（工程 market_intel） +# 部署环境: Ubuntu 22.04 LTS · Docker Compose +# +# 新服务器(免克隆): +# curl -fsSL https://git.bz121.com/dekun/market_intel/raw/branch/main/deploy/manage.sh | bash +# +# 已安装: +# bash /opt/market_intel/deploy/manage.sh +# +set -e +if [ -n "${BASH_VERSION:-}" ]; then + set -o pipefail +fi + +INSTALL_ROOT="${INSTALL_ROOT:-/opt/market_intel}" +GIT_URL="${GIT_URL:-https://git.bz121.com/dekun/market_intel.git}" +GIT_BRANCH="${GIT_BRANCH:-main}" + +_script_src="${BASH_SOURCE[0]:-}" +if [[ -n "${_script_src}" && -f "${_script_src}" ]]; then + DEPLOY_DIR="$(cd "$(dirname "${_script_src}")" && pwd)" + REPO_ROOT="$(cd "${DEPLOY_DIR}/.." && pwd)" + LIB_DIR="${DEPLOY_DIR}/lib" +else + DEPLOY_DIR="" + REPO_ROOT="" + LIB_DIR="" +fi +unset _script_src + +set -u + +repo_ready() { + [[ -f "${1}/deploy/manage.sh" && -f "${1}/docker-compose.yml" && -d "${1}/deploy/lib" ]] +} + +ensure_git_cli() { + if command -v git >/dev/null 2>&1; then + return 0 + fi + if command -v apt-get >/dev/null 2>&1; then + export DEBIAN_FRONTEND=noninteractive + local waited=0 + while pgrep -x unattended-upgr >/dev/null 2>&1 \ + || pgrep -x apt-get >/dev/null 2>&1 \ + || pgrep -x apt >/dev/null 2>&1 \ + || pgrep -x dpkg >/dev/null 2>&1 \ + || { command -v fuser >/dev/null 2>&1 && fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; }; do + if [[ "${waited}" -eq 0 ]]; then + echo "等待 apt 锁释放（unattended-upgrades）…" + fi + if [[ "${waited}" -ge 600 ]]; then + echo "错误: 等待 apt 锁超时，请稍后再试" >&2 + exit 1 + fi + sleep 5 + waited=$((waited + 5)) + done + apt-get update -qq + apt-get install -y git ca-certificates curl + else + echo "错误: 未找到 git" >&2 + exit 1 + fi +} + +sync_repo_if_present() { + local root="$1" + if [[ -d "${root}/.git" ]] && command -v git >/dev/null 2>&1; then + git -C "${root}" fetch --all --prune 2>/dev/null || true + git -C "${root}" checkout "${GIT_BRANCH}" 2>/dev/null || true + git -C "${root}" pull --ff-only origin "${GIT_BRANCH}" 2>/dev/null \ + || git -C "${root}" pull --ff-only 2>/dev/null \ + || true + fi +} + +heal_existing_install() { + local root="$1" + echo "检测到已有目录但缺少管理脚本: ${root}" + echo "尝试同步最新代码…" + ensure_git_cli + + if [[ -d "${root}/.git" ]]; then + sync_repo_if_present "${root}" + if repo_ready "${root}"; then + echo "同步成功，切换到仓库内 manage.sh" + exec bash "${root}/deploy/manage.sh" "$@" &2 + exit 1 + fi + heal_existing_install "${INSTALL_ROOT}" "$@" + fi + + echo "比特骆驼行情采集分析 部署管理器 — 首次自举" + echo "将克隆到: ${INSTALL_ROOT}" + if [[ "$(id -u)" -ne 0 ]]; then + echo "错误: 请使用 root 执行" >&2 + exit 1 + fi + ensure_git_cli + mkdir -p "$(dirname "${INSTALL_ROOT}")" + git clone -b "${GIT_BRANCH}" "${GIT_URL}" "${INSTALL_ROOT}" + exec bash "${INSTALL_ROOT}/deploy/manage.sh" "$@" /dev/tty 2>/dev/null || printf '%s' "${__prompt}" + fi + if [[ -r /dev/tty ]]; then + IFS= read -r __line Path: + return Path(self.mi_db_path) + + +@lru_cache +def get_settings() -> Settings: + return Settings() diff --git a/packages/db/__init__.py b/packages/db/__init__.py new file mode 100644 index 0000000..bd9fda5 --- /dev/null +++ b/packages/db/__init__.py @@ -0,0 +1,4 @@ +from packages.db.schema import init_db +from packages.db.repository import Repository + +__all__ = ["init_db", "Repository"] diff --git a/packages/db/repository.py b/packages/db/repository.py new file mode 100644 index 0000000..26bff4a --- /dev/null +++ b/packages/db/repository.py @@ -0,0 +1,275 @@ +"""数据访问。""" + +from __future__ import annotations + +import json +import sqlite3 +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from packages.db.schema import init_db + + +def _now_ms() -> int: + return int(time.time() * 1000) + + +@dataclass +class OptionQuoteRow: + ts_ms: int + exchange: str + underlying: str + inst_id: str + expiry_ymd: str + strike: float + side: str + index_px: float + ask: float | None + bid: float | None + ask_sz: float | None + bid_sz: float | None + leverage: float | None + + +class Repository: + def __init__(self, db_path: str | Path) -> None: + self.db_path = Path(db_path) + self.conn = init_db(self.db_path) + + def close(self) -> None: + self.conn.close() + + def insert_option_quote(self, row: OptionQuoteRow) -> int: + cur = self.conn.execute( + """ + INSERT INTO option_quotes ( + ts_ms, exchange, underlying, inst_id, expiry_ymd, strike, side, + index_px, ask, bid, ask_sz, bid_sz, leverage, created_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + row.ts_ms, + row.exchange, + row.underlying, + row.inst_id, + row.expiry_ymd, + row.strike, + row.side, + row.index_px, + row.ask, + row.bid, + row.ask_sz, + row.bid_sz, + row.leverage, + _now_ms(), + ), + ) + self.conn.commit() + return int(cur.lastrowid) + + def insert_index_tick( + self, + *, + ts_ms: int, + exchange: str, + underlying: str, + index_px: float, + ) -> int: + cur = self.conn.execute( + """ + INSERT INTO index_ticks (ts_ms, exchange, underlying, index_px, created_at_ms) + VALUES (?, ?, ?, ?, ?) + """, + (ts_ms, exchange, underlying, index_px, _now_ms()), + ) + self.conn.commit() + return int(cur.lastrowid) + + def upsert_heartbeat( + self, + *, + ok: bool, + error: str | None = None, + meta: dict[str, Any] | None = None, + ) -> None: + now = _now_ms() + row = self.conn.execute( + "SELECT consecutive_failures FROM collector_heartbeat WHERE id = 1" + ).fetchone() + fails = int(row["consecutive_failures"] if row else 0) + if ok: + fails = 0 + self.conn.execute( + """ + UPDATE collector_heartbeat + SET last_ok_ts_ms = ?, last_error = NULL, consecutive_failures = 0, + meta_json = COALESCE(?, meta_json) + WHERE id = 1 + """, + (now, json.dumps(meta, ensure_ascii=False) if meta else None), + ) + else: + fails += 1 + self.conn.execute( + """ + UPDATE collector_heartbeat + SET last_error = ?, last_error_ts_ms = ?, consecutive_failures = ?, + meta_json = COALESCE(?, meta_json) + WHERE id = 1 + """, + ( + (error or "unknown")[:2000], + now, + fails, + json.dumps(meta, ensure_ascii=False) if meta else None, + ), + ) + self.conn.commit() + + def get_heartbeat(self) -> dict[str, Any]: + row = self.conn.execute( + "SELECT * FROM collector_heartbeat WHERE id = 1" + ).fetchone() + if not row: + return {} + d = dict(row) + meta = d.get("meta_json") + if meta: + try: + d["meta"] = json.loads(meta) + except json.JSONDecodeError: + d["meta"] = None + else: + d["meta"] = None + return d + + def latest_quotes_by_side(self) -> dict[str, dict[str, Any]]: + """返回 side -> 最新一条。""" + out: dict[str, dict[str, Any]] = {} + for side in ("C", "P"): + row = self.conn.execute( + """ + SELECT * FROM option_quotes + WHERE side = ? + ORDER BY ts_ms DESC, id DESC + LIMIT 1 + """, + (side,), + ).fetchone() + if row: + out[side] = dict(row) + return out + + def count_option_quotes(self) -> int: + row = self.conn.execute("SELECT COUNT(*) AS n FROM option_quotes").fetchone() + return int(row["n"] if row else 0) + + def count_index_ticks(self) -> int: + row = self.conn.execute("SELECT COUNT(*) AS n FROM index_ticks").fetchone() + return int(row["n"] if row else 0) + + def fetch_option_quotes( + self, + *, + start_ms: int, + end_ms: int, + side: str = "both", + underlying: str | None = None, + ) -> list[dict[str, Any]]: + """[start_ms, end_ms) 半开区间。""" + clauses = ["ts_ms >= ?", "ts_ms < ?"] + params: list[Any] = [int(start_ms), int(end_ms)] + want = (side or "both").upper() + if want in ("C", "P"): + clauses.append("side = ?") + params.append(want) + if underlying: + clauses.append("underlying = ?") + params.append(underlying) + sql = f""" + SELECT ts_ms, exchange, underlying, inst_id, expiry_ymd, strike, side, + index_px, ask, bid, ask_sz, bid_sz, leverage + FROM option_quotes + WHERE {' AND '.join(clauses)} + ORDER BY ts_ms ASC, id ASC + """ + rows = self.conn.execute(sql, params).fetchall() + return [dict(r) for r in rows] + + def get_settlement(self, expiry_ymd: str) -> dict[str, Any] | None: + row = self.conn.execute( + "SELECT * FROM expiry_settlements WHERE expiry_ymd = ?", + (expiry_ymd,), + ).fetchone() + return dict(row) if row else None + + def list_settlements(self, ymds: list[str] | None = None) -> dict[str, dict[str, Any]]: + if ymds is not None and not ymds: + return {} + if ymds is None: + rows = self.conn.execute("SELECT * FROM expiry_settlements").fetchall() + else: + placeholders = ",".join("?" for _ in ymds) + rows = self.conn.execute( + f"SELECT * FROM expiry_settlements WHERE expiry_ymd IN ({placeholders})", + list(ymds), + ).fetchall() + return {str(r["expiry_ymd"]): dict(r) for r in rows} + + def upsert_settlement( + self, + *, + expiry_ymd: str, + settle_ts_ms: int, + settle_index_px: float, + exchange: str, + underlying: str, + ) -> None: + self.conn.execute( + """ + INSERT INTO expiry_settlements ( + expiry_ymd, settle_ts_ms, settle_index_px, exchange, underlying, created_at_ms + ) VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(expiry_ymd) DO UPDATE SET + settle_ts_ms = excluded.settle_ts_ms, + settle_index_px = excluded.settle_index_px, + exchange = excluded.exchange, + underlying = excluded.underlying + """, + ( + expiry_ymd, + int(settle_ts_ms), + float(settle_index_px), + exchange, + underlying, + _now_ms(), + ), + ) + self.conn.commit() + + def nearest_index_tick( + self, + *, + underlying: str, + target_ts_ms: int, + max_delta_ms: int, + ) -> dict[str, Any] | None: + row = self.conn.execute( + """ + SELECT ts_ms, index_px, ABS(ts_ms - ?) AS delta + FROM index_ticks + WHERE underlying = ? + AND ts_ms BETWEEN ? AND ? + ORDER BY delta ASC + LIMIT 1 + """, + ( + int(target_ts_ms), + underlying, + int(target_ts_ms) - int(max_delta_ms), + int(target_ts_ms) + int(max_delta_ms), + ), + ).fetchone() + return dict(row) if row else None diff --git a/packages/db/schema.py b/packages/db/schema.py new file mode 100644 index 0000000..9b4f19f --- /dev/null +++ b/packages/db/schema.py @@ -0,0 +1,75 @@ +"""SQLite schema（可迁移设计）。时间戳存 UTC ms。""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +SCHEMA_SQL = """ +CREATE TABLE IF NOT EXISTS option_quotes ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts_ms INTEGER NOT NULL, + exchange TEXT NOT NULL, + underlying TEXT NOT NULL, + inst_id TEXT NOT NULL, + expiry_ymd TEXT NOT NULL, + strike REAL NOT NULL, + side TEXT NOT NULL, + index_px REAL NOT NULL, + ask REAL, + bid REAL, + ask_sz REAL, + bid_sz REAL, + leverage REAL, + created_at_ms INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_oq_ts ON option_quotes(ts_ms); +CREATE INDEX IF NOT EXISTS idx_oq_side_ts ON option_quotes(side, ts_ms); +CREATE INDEX IF NOT EXISTS idx_oq_expiry ON option_quotes(expiry_ymd); + +CREATE TABLE IF NOT EXISTS index_ticks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts_ms INTEGER NOT NULL, + exchange TEXT NOT NULL, + underlying TEXT NOT NULL, + index_px REAL NOT NULL, + created_at_ms INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_it_ts ON index_ticks(ts_ms); +CREATE INDEX IF NOT EXISTS idx_it_u_ts ON index_ticks(underlying, ts_ms); + +CREATE TABLE IF NOT EXISTS expiry_settlements ( + expiry_ymd TEXT PRIMARY KEY, + settle_ts_ms INTEGER NOT NULL, + settle_index_px REAL NOT NULL, + exchange TEXT NOT NULL, + underlying TEXT NOT NULL, + created_at_ms INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS collector_heartbeat ( + id INTEGER PRIMARY KEY CHECK (id = 1), + last_ok_ts_ms INTEGER, + last_error TEXT, + last_error_ts_ms INTEGER, + consecutive_failures INTEGER NOT NULL DEFAULT 0, + meta_json TEXT +); +""" + + +def init_db(db_path: str | Path) -> sqlite3.Connection: + path = Path(db_path) + path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(str(path), check_same_thread=False) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA journal_mode=WAL;") + conn.execute("PRAGMA synchronous=NORMAL;") + conn.executescript(SCHEMA_SQL) + conn.execute( + "INSERT OR IGNORE INTO collector_heartbeat (id, consecutive_failures) VALUES (1, 0)" + ) + conn.commit() + return conn diff --git a/packages/domain/__init__.py b/packages/domain/__init__.py new file mode 100644 index 0000000..348fc83 --- /dev/null +++ b/packages/domain/__init__.py @@ -0,0 +1,29 @@ +"""领域口径：杠杆、时段桶、波动点数。变更需升版本。""" + +from __future__ import annotations + +from packages.domain.aggregate import ( + aggregate_leverage, + aggregate_move_points, + summarize_values, +) +from packages.domain.buckets import shanghai_bucket, shanghai_day +from packages.domain.expiry import expiry_ms_from_ymd +from packages.domain.leverage import LEVERAGE_FORMULA_VERSION, option_leverage +from packages.domain.move_points import MOVE_POINTS_FORMULA_VERSION, move_points +from packages.domain.range import resolve_range, today_shanghai + +__all__ = [ + "LEVERAGE_FORMULA_VERSION", + "MOVE_POINTS_FORMULA_VERSION", + "option_leverage", + "move_points", + "expiry_ms_from_ymd", + "shanghai_day", + "shanghai_bucket", + "resolve_range", + "today_shanghai", + "aggregate_leverage", + "aggregate_move_points", + "summarize_values", +] diff --git a/packages/domain/aggregate.py b/packages/domain/aggregate.py new file mode 100644 index 0000000..6a2e5d3 --- /dev/null +++ b/packages/domain/aggregate.py @@ -0,0 +1,342 @@ +"""杠杆按时段桶聚合。""" + +from __future__ import annotations + +import math +from collections import defaultdict +from typing import Any, Iterable, Sequence + +from packages.domain.buckets import shanghai_bucket +from packages.domain.leverage import LEVERAGE_FORMULA_VERSION + + +def _percentile(sorted_vals: Sequence[float], p: float) -> float | None: + """线性插值百分位；p in [0,100]。""" + if not sorted_vals: + return None + if len(sorted_vals) == 1: + return float(sorted_vals[0]) + p = max(0.0, min(100.0, float(p))) + k = (len(sorted_vals) - 1) * (p / 100.0) + f = math.floor(k) + c = math.ceil(k) + if f == c: + return float(sorted_vals[int(k)]) + d0 = sorted_vals[f] * (c - k) + d1 = sorted_vals[c] * (k - f) + return float(d0 + d1) + + +def summarize_values(values: Sequence[float], *, min_leverage: float) -> dict[str, Any]: + if not values: + return { + "n": 0, + "mean": None, + "median": None, + "p25": None, + "p75": None, + "min": None, + "max": None, + "pct_ge_min": None, + } + xs = sorted(float(v) for v in values) + n = len(xs) + ge = sum(1 for v in xs if v >= float(min_leverage)) + return { + "n": n, + "mean": sum(xs) / n, + "median": _percentile(xs, 50), + "p25": _percentile(xs, 25), + "p75": _percentile(xs, 75), + "min": xs[0], + "max": xs[-1], + "pct_ge_min": ge / n, + } + + +def bucket_label(bucket_start_min: int, bucket_minutes: int) -> str: + """如 14:00 或 14:00-14:30。""" + h, m = divmod(int(bucket_start_min), 60) + start = f"{h:02d}:{m:02d}" + if bucket_minutes >= 60 and bucket_minutes % 60 == 0 and m == 0: + return f"{h:02d}:00" + end_min = bucket_start_min + bucket_minutes + eh, em = divmod(end_min % (24 * 60), 60) + return f"{start}-{eh:02d}:{em:02d}" + + +def all_bucket_starts(bucket_minutes: int) -> list[int]: + if bucket_minutes <= 0 or 1440 % bucket_minutes != 0: + # 允许非整除：仍按步进生成到 <1440 + out = [] + t = 0 + while t < 1440: + out.append(t) + t += bucket_minutes + return out + return list(range(0, 1440, bucket_minutes)) + + +def aggregate_leverage( + rows: Iterable[dict[str, Any]], + *, + bucket_minutes: int = 60, + min_leverage: float = 100.0, + side: str = "both", +) -> list[dict[str, Any]]: + """ + rows: 需含 ts_ms, leverage, side。 + 返回按桶排序的聚合列表（含空桶）。 + """ + want = (side or "both").upper() + by_bucket: dict[int, list[float]] = defaultdict(list) + + for r in rows: + lev = r.get("leverage") + if lev is None: + continue + try: + lev_f = float(lev) + except (TypeError, ValueError): + continue + if not math.isfinite(lev_f) or lev_f <= 0: + continue + s = str(r.get("side") or "").upper() + if want in ("C", "P") and s != want: + continue + if want == "BOTH" and s not in ("C", "P"): + continue + b = shanghai_bucket(int(r["ts_ms"]), bucket_minutes) + by_bucket[b].append(lev_f) + + out: list[dict[str, Any]] = [] + for b in all_bucket_starts(bucket_minutes): + stats = summarize_values(by_bucket.get(b, []), min_leverage=min_leverage) + out.append( + { + "bucket_start_min": b, + "bucket_hour": b // 60 if bucket_minutes >= 60 else None, + "label": bucket_label(b, bucket_minutes), + **stats, + } + ) + return out + + +def summarize_distribution(values: Sequence[float]) -> dict[str, Any]: + """通用分布摘要（无达标线）。""" + if not values: + return { + "n": 0, + "mean": None, + "median": None, + "p25": None, + "p75": None, + "min": None, + "max": None, + } + xs = sorted(float(v) for v in values) + n = len(xs) + return { + "n": n, + "mean": sum(xs) / n, + "median": _percentile(xs, 50), + "p25": _percentile(xs, 25), + "p75": _percentile(xs, 75), + "min": xs[0], + "max": xs[-1], + } + + +def aggregate_move_points( + samples: Iterable[dict[str, Any]], + *, + bucket_minutes: int = 60, +) -> list[dict[str, Any]]: + """ + samples: {ts_ms, move_signed, move_abs} + 桶内同时给出 signed / abs 分布。 + """ + by_signed: dict[int, list[float]] = defaultdict(list) + by_abs: dict[int, list[float]] = defaultdict(list) + for s in samples: + ts = s.get("ts_ms") + signed = s.get("move_signed") + if ts is None or signed is None: + continue + try: + signed_f = float(signed) + abs_f = float(s.get("move_abs", abs(signed_f))) + except (TypeError, ValueError): + continue + if not math.isfinite(signed_f): + continue + b = shanghai_bucket(int(ts), bucket_minutes) + by_signed[b].append(signed_f) + by_abs[b].append(abs_f) + + out: list[dict[str, Any]] = [] + for b in all_bucket_starts(bucket_minutes): + signed_stats = summarize_distribution(by_signed.get(b, [])) + abs_stats = summarize_distribution(by_abs.get(b, [])) + out.append( + { + "bucket_start_min": b, + "bucket_hour": b // 60 if bucket_minutes >= 60 else None, + "label": bucket_label(b, bucket_minutes), + "n": signed_stats["n"], + "signed": signed_stats, + "abs": abs_stats, + # 便捷字段（看板默认用 abs 均值） + "mean_signed": signed_stats["mean"], + "median_signed": signed_stats["median"], + "mean_abs": abs_stats["mean"], + "median_abs": abs_stats["median"], + } + ) + return out + + +def build_move_samples( + rows: Iterable[dict[str, Any]], + settlements: dict[str, dict[str, Any]], + *, + side: str = "both", + now_ms: int | None = None, +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + """ + 对期权样本计算波动点数。 + 返回 (settled_samples, meta)。 + meta: pending_expiry, pending_count, settled_count, pending_ymds, settled_ymds + """ + import time + + from packages.domain.move_points import move_points as calc_move + + now = int(now_ms if now_ms is not None else time.time() * 1000) + want = (side or "both").upper() + settled: list[dict[str, Any]] = [] + pending_ymds: set[str] = set() + settled_ymds: set[str] = set() + pending_count = 0 + settled_count = 0 + + for r in rows: + s = str(r.get("side") or "").upper() + if want in ("C", "P") and s != want: + continue + if want == "BOTH" and s not in ("C", "P"): + continue + ymd = str(r.get("expiry_ymd") or "") + idx = r.get("index_px") + ts = r.get("ts_ms") + if not ymd or idx is None or ts is None: + continue + settle = settlements.get(ymd) + if settle is None or int(settle.get("settle_ts_ms") or 0) > now: + pending_ymds.add(ymd) + pending_count += 1 + continue + try: + signed = calc_move(float(settle["settle_index_px"]), float(idx)) + except (TypeError, ValueError): + pending_ymds.add(ymd) + pending_count += 1 + continue + settled_ymds.add(ymd) + settled_count += 1 + settled.append( + { + "ts_ms": int(ts), + "expiry_ymd": ymd, + "side": s, + "index_at_t": float(idx), + "settle_index_px": float(settle["settle_index_px"]), + "move_signed": signed, + "move_abs": abs(signed), + } + ) + + meta = { + "pending_expiry": pending_count > 0, + "pending_count": pending_count, + "settled_count": settled_count, + "pending_ymds": sorted(pending_ymds), + "settled_ymds": sorted(settled_ymds), + } + return settled, meta + + +def move_points_stats_payload( + rows: Iterable[dict[str, Any]], + settlements: dict[str, dict[str, Any]], + *, + range_info: dict[str, Any], + bucket_minutes: int, + side: str, + now_ms: int | None = None, +) -> dict[str, Any]: + from packages.domain.move_points import MOVE_POINTS_FORMULA_VERSION + + samples, meta = build_move_samples( + rows, settlements, side=side, now_ms=now_ms + ) + buckets = aggregate_move_points(samples, bucket_minutes=bucket_minutes) + return { + "status": "ok", + "formula_version": MOVE_POINTS_FORMULA_VERSION, + "move_def": "settle_index_px - index_at(t)", + "range": range_info["range"], + "date": range_info["anchor"], + "start_ymd": range_info["start_ymd"], + "end_ymd": range_info["end_ymd"], + "days": range_info["days"], + "month_mode": range_info.get("month_mode"), + "side": side, + "bucket_minutes": bucket_minutes, + "pending_expiry": meta["pending_expiry"], + "pending_count": meta["pending_count"], + "settled_count": meta["settled_count"], + "pending_ymds": meta["pending_ymds"], + "settled_ymds": meta["settled_ymds"], + "sample_count": meta["settled_count"], + "buckets": buckets, + "message": ( + "部分样本未到期或缺少结算锚点，已排除出分布" + if meta["pending_expiry"] + else None + ), + } + + +def leverage_stats_payload( + rows: Iterable[dict[str, Any]], + *, + range_info: dict[str, Any], + bucket_minutes: int, + min_leverage: float, + side: str, +) -> dict[str, Any]: + buckets = aggregate_leverage( + rows, + bucket_minutes=bucket_minutes, + min_leverage=min_leverage, + side=side, + ) + total_n = sum(int(b["n"]) for b in buckets) + return { + "status": "ok", + "formula_version": LEVERAGE_FORMULA_VERSION, + "leverage_def": "index_px / ask", + "range": range_info["range"], + "date": range_info["anchor"], + "start_ymd": range_info["start_ymd"], + "end_ymd": range_info["end_ymd"], + "days": range_info["days"], + "month_mode": range_info.get("month_mode"), + "side": side, + "bucket_minutes": bucket_minutes, + "min_leverage": min_leverage, + "sample_count": total_n, + "buckets": buckets, + } diff --git a/packages/domain/buckets.py b/packages/domain/buckets.py new file mode 100644 index 0000000..678b114 --- /dev/null +++ b/packages/domain/buckets.py @@ -0,0 +1,41 @@ +"""Asia/Shanghai 日切与时段桶。时间戳一律 UTC ms 入，上海出。""" + +from __future__ import annotations + +from datetime import datetime, timedelta +from zoneinfo import ZoneInfo + +_SH = ZoneInfo("Asia/Shanghai") + + +def _to_shanghai(ts_ms: int) -> datetime: + return datetime.fromtimestamp(int(ts_ms) / 1000.0, tz=_SH) + + +def shanghai_day(ts_ms: int) -> str: + """上海自然日 YYYY-MM-DD。""" + return _to_shanghai(ts_ms).strftime("%Y-%m-%d") + + +def shanghai_bucket(ts_ms: int, bucket_minutes: int = 60) -> int: + """ + 日内时段桶起点（分钟，0–1439）。 + 默认 60 → 0,60,120,...,1380（即小时 0–23）。 + """ + if bucket_minutes <= 0: + raise ValueError("bucket_minutes must be > 0") + dt = _to_shanghai(ts_ms) + minutes = dt.hour * 60 + dt.minute + return (minutes // bucket_minutes) * bucket_minutes + + +def shanghai_bucket_hour(ts_ms: int) -> int: + """0–23 小时桶。""" + return shanghai_bucket(ts_ms, 60) // 60 + + +def rolling_day_start(anchor_ymd: str, days: int) -> str: + """锚点日（含）往前 days-1 天的起始 YYYY-MM-DD。""" + d = datetime.strptime(anchor_ymd, "%Y-%m-%d").date() + start = d - timedelta(days=max(0, days - 1)) + return start.strftime("%Y-%m-%d") diff --git a/packages/domain/expiry.py b/packages/domain/expiry.py new file mode 100644 index 0000000..9031756 --- /dev/null +++ b/packages/domain/expiry.py @@ -0,0 +1,15 @@ +"""OKX / 欧式期权惯例：到期日当日 08:00 UTC（上海 16:00）。""" + +from __future__ import annotations + +from datetime import datetime, timezone + + +def expiry_ms_from_ymd(ymd: str) -> int: + """YYMMDD → 到期毫秒时间戳（UTC 08:00）。""" + ymd = (ymd or "").strip() + if len(ymd) != 6 or not ymd.isdigit(): + raise ValueError(f"invalid expiry ymd: {ymd!r}") + yy, mm, dd = int(ymd[0:2]), int(ymd[2:4]), int(ymd[4:6]) + dt = datetime(2000 + yy, mm, dd, 8, 0, 0, tzinfo=timezone.utc) + return int(dt.timestamp() * 1000) diff --git a/packages/domain/leverage.py b/packages/domain/leverage.py new file mode 100644 index 0000000..06a9522 --- /dev/null +++ b/packages/domain/leverage.py @@ -0,0 +1,14 @@ +"""杠杆口径 v1：杠杆 = 标的指数 ÷ 期权卖一（ask）。""" + +from __future__ import annotations + +LEVERAGE_FORMULA_VERSION = "1.0" + + +def option_leverage(index_px: float, ask: float | None) -> float | None: + """index_px / ask；ask 无效时返回 None。""" + if index_px is None or index_px <= 0: + return None + if ask is None or ask <= 0: + return None + return float(index_px) / float(ask) diff --git a/packages/domain/move_points.py b/packages/domain/move_points.py new file mode 100644 index 0000000..bd8ebd5 --- /dev/null +++ b/packages/domain/move_points.py @@ -0,0 +1,10 @@ +"""波动点数口径 v1：到期指数 − 时段代表指数。""" + +from __future__ import annotations + +MOVE_POINTS_FORMULA_VERSION = "1.0" + + +def move_points(settle_index_px: float, index_at_t: float) -> float: + """signed 点数；绝对值由调用方取 abs。""" + return float(settle_index_px) - float(index_at_t) diff --git a/packages/domain/range.py b/packages/domain/range.py new file mode 100644 index 0000000..18dadf9 --- /dev/null +++ b/packages/domain/range.py @@ -0,0 +1,77 @@ +"""日/周/月时间范围解析（Asia/Shanghai）。""" + +from __future__ import annotations + +from datetime import date, datetime, timedelta +from zoneinfo import ZoneInfo + +_SH = ZoneInfo("Asia/Shanghai") + + +def today_shanghai(now: datetime | None = None) -> str: + n = (now or datetime.now(tz=_SH)).astimezone(_SH) + return n.strftime("%Y-%m-%d") + + +def parse_ymd(ymd: str) -> date: + return datetime.strptime(ymd, "%Y-%m-%d").date() + + +def day_bounds_ms(ymd: str) -> tuple[int, int]: + """上海自然日 [start_ms, end_ms)，end 为次日 00:00。""" + d = parse_ymd(ymd) + start = datetime(d.year, d.month, d.day, 0, 0, 0, tzinfo=_SH) + end = start + timedelta(days=1) + return int(start.timestamp() * 1000), int(end.timestamp() * 1000) + + +def resolve_range( + range_name: str, + anchor_ymd: str | None = None, + *, + month_mode: str = "rolling_30", + now: datetime | None = None, +) -> dict: + """ + 返回: + anchor, start_ymd, end_ymd (含), start_ms, end_ms (半开区间), days + """ + anchor = anchor_ymd or today_shanghai(now) + parse_ymd(anchor) # validate + + name = (range_name or "day").strip().lower() + if name == "day": + start_ymd = end_ymd = anchor + days = 1 + elif name == "week": + end_ymd = anchor + start = parse_ymd(anchor) - timedelta(days=6) + start_ymd = start.strftime("%Y-%m-%d") + days = 7 + elif name == "month": + end_ymd = anchor + mode = (month_mode or "rolling_30").strip().lower() + if mode in ("calendar", "natural", "natural_month"): + d = parse_ymd(anchor) + start_ymd = d.replace(day=1).strftime("%Y-%m-%d") + days = (parse_ymd(end_ymd) - parse_ymd(start_ymd)).days + 1 + else: + # rolling_30 + start = parse_ymd(anchor) - timedelta(days=29) + start_ymd = start.strftime("%Y-%m-%d") + days = 30 + else: + raise ValueError(f"invalid range: {range_name!r}") + + start_ms, _ = day_bounds_ms(start_ymd) + _, end_ms = day_bounds_ms(end_ymd) + return { + "range": name, + "anchor": anchor, + "start_ymd": start_ymd, + "end_ymd": end_ymd, + "start_ms": start_ms, + "end_ms": end_ms, + "days": days, + "month_mode": month_mode if name == "month" else None, + } diff --git a/packages/notify/__init__.py b/packages/notify/__init__.py new file mode 100644 index 0000000..432d3f9 --- /dev/null +++ b/packages/notify/__init__.py @@ -0,0 +1,3 @@ +from packages.notify import wecom + +__all__ = ["wecom"] diff --git a/packages/notify/wecom.py b/packages/notify/wecom.py new file mode 100644 index 0000000..a82e97b --- /dev/null +++ b/packages/notify/wecom.py @@ -0,0 +1,163 @@ +"""企业微信群机器人通知（独立实现，不依赖策略仓）。""" + +from __future__ import annotations + +import logging +import time +from typing import Any + +import httpx + +from packages.config import get_settings + +log = logging.getLogger("notify.wecom") + +_last_fault_key: str | None = None +_last_fault_ms: float = 0.0 +_fault_active: bool = False + + +def _as_bool(raw: Any, default: bool = False) -> bool: + if raw is None or raw == "": + return default + return str(raw).strip().lower() in ("1", "true", "yes", "on") + + +def wecom_enabled() -> bool: + s = get_settings() + return _as_bool(getattr(s, "wecom_enabled", False)) + + +def wecom_webhook_url() -> str: + s = get_settings() + return (getattr(s, "wecom_webhook_url", "") or "").strip() + + +def wecom_machine_name() -> str: + s = get_settings() + return (getattr(s, "wecom_machine_name", "") or "").strip()[:64] + + +def alert_fail_threshold() -> int: + s = get_settings() + try: + return max(1, int(getattr(s, "alert_fail_threshold", 5) or 5)) + except (TypeError, ValueError): + return 5 + + +def build_markdown(*, tag: str, title: str, lines: list[str] | None = None) -> str: + machine = wecom_machine_name() + prefix = f"【{machine}】" if machine else "" + parts = [ + f"## {prefix}{title}", + f"> **标识**: `{tag}`", + f"> **系统**: 比特骆驼行情采集分析", + f"> **时间**: {time.strftime('%Y-%m-%d %H:%M:%S')}", + ] + if machine: + parts.append(f"> **机器**: {machine}") + if lines: + parts.append("") + for ln in lines: + parts.append(f"> {ln}" if not ln.startswith(">") else ln) + return "\n".join(parts) + + +def post_markdown(content: str) -> tuple[bool, str]: + if not wecom_enabled(): + return False, "未开启企业微信通知" + url = wecom_webhook_url() + if not url: + return False, "未配置 Webhook" + raw = content.encode("utf-8") + if len(raw) > 4000: + content = raw[:3900].decode("utf-8", errors="ignore") + "\n…" + payload = {"msgtype": "markdown", "markdown": {"content": content}} + try: + with httpx.Client(timeout=10.0) as client: + r = client.post(url, json=payload) + body = r.json() if r.content else {} + if r.status_code != 200 or str(body.get("errcode", 0)) not in ("0", "0.0"): + return False, f"webhook failed status={r.status_code} body={body}" + return True, "ok" + except Exception as e: # noqa: BLE001 + return False, str(e) + + +def notify_test() -> tuple[bool, str]: + md = build_markdown( + tag="TEST", + title="行情采集分析 · 测试推送", + lines=["这是一条测试消息，说明企微 Webhook 可用。"], + ) + return post_markdown(md) + + +def notify_collector_fault( + *, + error: str, + consecutive_failures: int, + dedup_sec: float = 300.0, +) -> tuple[bool, str]: + """连续失败告警；同错误键在 dedup_sec 内不重复推。""" + global _last_fault_key, _last_fault_ms, _fault_active + threshold = alert_fail_threshold() + if consecutive_failures < threshold: + return False, f"below threshold ({consecutive_failures}<{threshold})" + + key = f"{consecutive_failures // threshold}:{(error or '')[:120]}" + now = time.time() + if ( + _last_fault_key == key + and (now - _last_fault_ms) < dedup_sec + ): + return False, "dedup" + _last_fault_key = key + _last_fault_ms = now + _fault_active = True + + # 脱敏：避免日志/推送里出现完整密钥形态串 + err_show = (error or "unknown").replace("\n", " ")[:300] + md = build_markdown( + tag="FAULT", + title="行情采集异常", + lines=[ + f"**连续失败**: {consecutive_failures}（阈值 {threshold}）", + f"**错误**: {err_show}", + "请检查 OKX 连通性 / 代理 / 合约是否可交易。", + ], + ) + ok, msg = post_markdown(md) + if ok: + log.info("wecom fault notified failures=%s", consecutive_failures) + else: + log.warning("wecom fault notify failed: %s", msg) + return ok, msg + + +def notify_collector_recovered(*, consecutive_failures: int = 0) -> tuple[bool, str]: + global _fault_active, _last_fault_key + if not _fault_active: + return False, "no active fault" + _fault_active = False + _last_fault_key = None + md = build_markdown( + tag="RECOVER", + title="行情采集已恢复", + lines=["采样已恢复正常。"], + ) + ok, msg = post_markdown(md) + if ok: + log.info("wecom recovered notified") + else: + log.warning("wecom recover notify failed: %s", msg) + return ok, msg + + +def reset_alert_state() -> None: + """测试用。""" + global _last_fault_key, _last_fault_ms, _fault_active + _last_fault_key = None + _last_fault_ms = 0.0 + _fault_active = False diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..57c75d3 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +pythonpath = . +testpaths = tests diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a4b79c0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +fastapi>=0.111.0,<1.0 +uvicorn[standard]>=0.30.0,<1.0 +httpx>=0.27.0,<1.0 +pydantic>=2.7.0,<3.0 +pydantic-settings>=2.3.0,<3.0 +python-dotenv>=1.0.0,<2.0 +pytest>=8.0.0,<9.0 diff --git a/scripts/backfill_index.py b/scripts/backfill_index.py new file mode 100644 index 0000000..7a15db4 --- /dev/null +++ b/scripts/backfill_index.py @@ -0,0 +1,34 @@ +"""可选：手动回填到期结算 / 历史指数。""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +os.chdir(ROOT) + +from apps.worker.settle import backfill_settlements +from packages.config import get_settings +from packages.db import Repository + + +def main() -> int: + s = get_settings() + repo = Repository(s.db_path) + try: + result = backfill_settlements( + repo, + underlying=s.underlying, + index_inst_id=s.index_inst_id, + ) + print(result) + return 0 if not result["errors"] else 1 + finally: + repo.close() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/smoke_collect.py b/scripts/smoke_collect.py new file mode 100644 index 0000000..df1825e --- /dev/null +++ b/scripts/smoke_collect.py @@ -0,0 +1,43 @@ +"""冒烟：拉一次 OKX 指数 + 选 ATM 并打印（不强制写库）。""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +os.chdir(ROOT) + +from apps.collector.okx_rest import OkxRestClient +from apps.collector.selectors import rows_to_contracts, select_atm_pair +from packages.config import get_settings +from packages.domain import option_leverage + + +def main() -> int: + s = get_settings() + with OkxRestClient(base_url=s.okx_base_url, proxy=s.okx_proxy or None) as client: + idx = client.fetch_index_ticker(s.index_inst_id) + print(f"index {s.index_inst_id} = {idx}") + raw = client.fetch_option_instruments(s.option_inst_family) + contracts = rows_to_contracts(raw) + print(f"live contracts = {len(contracts)}") + pair = select_atm_pair(contracts, index_px=float(idx or 0), min_hours=s.min_option_hours) + if not pair: + print("no ATM pair") + return 1 + print( + f"ATM expiry={pair.expiry_ymd} strike={pair.strike} " + f"C={pair.call_inst_id} P={pair.put_inst_id}" + ) + for side, inst in (("C", pair.call_inst_id), ("P", pair.put_inst_id)): + ask, bid, *_ = client.fetch_books(inst) + lev = option_leverage(float(idx), ask) + print(f" {side}: ask={ask} bid={bid} leverage={lev}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_aggregate.py b/tests/test_aggregate.py new file mode 100644 index 0000000..01622c9 --- /dev/null +++ b/tests/test_aggregate.py @@ -0,0 +1,65 @@ +from packages.domain.aggregate import aggregate_leverage, summarize_values +from packages.domain.range import resolve_range + + +def test_summarize_values(): + s = summarize_values([100, 200, 300, 400], min_leverage=200) + assert s["n"] == 4 + assert s["mean"] == 250 + assert s["median"] == 250 + assert s["pct_ge_min"] == 0.75 + + +def test_summarize_empty(): + s = summarize_values([], min_leverage=100) + assert s["n"] == 0 + assert s["mean"] is None + + +def test_resolve_range_day(): + info = resolve_range("day", "2026-07-31") + assert info["start_ymd"] == "2026-07-31" + assert info["end_ymd"] == "2026-07-31" + assert info["days"] == 1 + assert info["end_ms"] > info["start_ms"] + + +def test_resolve_range_week(): + info = resolve_range("week", "2026-07-31") + assert info["start_ymd"] == "2026-07-25" + assert info["end_ymd"] == "2026-07-31" + assert info["days"] == 7 + + +def test_resolve_range_month_rolling(): + info = resolve_range("month", "2026-07-31", month_mode="rolling_30") + assert info["start_ymd"] == "2026-07-02" + assert info["days"] == 30 + + +def test_resolve_range_month_calendar(): + info = resolve_range("month", "2026-07-31", month_mode="calendar") + assert info["start_ymd"] == "2026-07-01" + assert info["end_ymd"] == "2026-07-31" + + +def test_aggregate_leverage_buckets(): + # 2026-07-31 14:30 Asia/Shanghai + from datetime import datetime + from zoneinfo import ZoneInfo + + sh = ZoneInfo("Asia/Shanghai") + ts = int(datetime(2026, 7, 31, 14, 30, tzinfo=sh).timestamp() * 1000) + rows = [ + {"ts_ms": ts, "side": "C", "leverage": 120}, + {"ts_ms": ts, "side": "P", "leverage": 80}, + {"ts_ms": ts, "side": "C", "leverage": 180}, + ] + buckets = aggregate_leverage(rows, bucket_minutes=60, min_leverage=100, side="C") + assert len(buckets) == 24 + b14 = next(b for b in buckets if b["bucket_start_min"] == 14 * 60) + assert b14["n"] == 2 + assert b14["mean"] == 150 + assert b14["label"] == "14:00" + empty = next(b for b in buckets if b["bucket_start_min"] == 0) + assert empty["n"] == 0 diff --git a/tests/test_auth.py b/tests/test_auth.py new file mode 100644 index 0000000..f3e6668 --- /dev/null +++ b/tests/test_auth.py @@ -0,0 +1,44 @@ +import os + +from packages.config.settings import get_settings + + +def _with_env(**kwargs): + old = {} + for k, v in kwargs.items(): + old[k] = os.environ.get(k) + os.environ[k] = v + get_settings.cache_clear() + return old + + +def _restore(old: dict): + for k, v in old.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + get_settings.cache_clear() + + +def test_issue_token_ok(): + from apps.api.auth import expected_token, issue_token + + old = _with_env(AUTH_SECRET="unit-secret", ADMIN_PASSWORD="pass123") + try: + tok = issue_token("pass123") + assert tok is not None + assert tok == expected_token() + assert issue_token("wrong") is None + finally: + _restore(old) + + +def test_auth_disabled(): + from apps.api.auth import auth_disabled + + old = _with_env(AUTH_SECRET="disabled") + try: + assert auth_disabled() is True + finally: + _restore(old) diff --git a/tests/test_buckets.py b/tests/test_buckets.py new file mode 100644 index 0000000..c34c0eb --- /dev/null +++ b/tests/test_buckets.py @@ -0,0 +1,27 @@ +from datetime import datetime +from zoneinfo import ZoneInfo + +from packages.domain.buckets import shanghai_bucket, shanghai_bucket_hour, shanghai_day + +_SH = ZoneInfo("Asia/Shanghai") + + +def _ms(y, m, d, hh, mm=0): + return int(datetime(y, m, d, hh, mm, tzinfo=_SH).timestamp() * 1000) + + +def test_shanghai_day(): + # UTC 2026-07-31 16:00 = 上海 2026-08-01 00:00 + ts = int(datetime(2026, 7, 31, 16, 0, tzinfo=ZoneInfo("UTC")).timestamp() * 1000) + assert shanghai_day(ts) == "2026-08-01" + + +def test_bucket_hour(): + ts = _ms(2026, 7, 31, 14, 35) + assert shanghai_bucket(ts, 60) == 14 * 60 + assert shanghai_bucket_hour(ts) == 14 + + +def test_bucket_30m(): + ts = _ms(2026, 7, 31, 14, 35) + assert shanghai_bucket(ts, 30) == 14 * 60 + 30 diff --git a/tests/test_leverage.py b/tests/test_leverage.py new file mode 100644 index 0000000..bf83ae7 --- /dev/null +++ b/tests/test_leverage.py @@ -0,0 +1,13 @@ +from packages.domain import option_leverage + + +def test_leverage_basic(): + assert option_leverage(2000.0, 20.0) == 100.0 + assert option_leverage(3500.0, 35.0) == 100.0 + + +def test_leverage_invalid(): + assert option_leverage(0, 10) is None + assert option_leverage(100, 0) is None + assert option_leverage(100, None) is None + assert option_leverage(-1, 1) is None diff --git a/tests/test_move_aggregate.py b/tests/test_move_aggregate.py new file mode 100644 index 0000000..a04bd65 --- /dev/null +++ b/tests/test_move_aggregate.py @@ -0,0 +1,57 @@ +from packages.domain.aggregate import aggregate_move_points, build_move_samples, move_points_stats_payload +from packages.domain.range import resolve_range + + +def test_build_move_samples_settled_and_pending(): + from datetime import datetime + from zoneinfo import ZoneInfo + + sh = ZoneInfo("Asia/Shanghai") + ts = int(datetime(2026, 7, 30, 10, 0, tzinfo=sh).timestamp() * 1000) + rows = [ + {"ts_ms": ts, "side": "C", "expiry_ymd": "260720", "index_px": 3400}, # settled + {"ts_ms": ts, "side": "P", "expiry_ymd": "260720", "index_px": 3450}, + {"ts_ms": ts, "side": "C", "expiry_ymd": "991231", "index_px": 3500}, # pending far + ] + settlements = { + "260720": {"settle_ts_ms": ts - 1000, "settle_index_px": 3500}, + } + samples, meta = build_move_samples(rows, settlements, side="both", now_ms=ts) + assert meta["settled_count"] == 2 + assert meta["pending_count"] == 1 + assert meta["pending_expiry"] is True + assert samples[0]["move_signed"] == 100.0 # 3500-3400 + assert samples[1]["move_abs"] == 50.0 + + +def test_aggregate_move_points(): + from datetime import datetime + from zoneinfo import ZoneInfo + + sh = ZoneInfo("Asia/Shanghai") + ts = int(datetime(2026, 7, 30, 14, 20, tzinfo=sh).timestamp() * 1000) + samples = [ + {"ts_ms": ts, "move_signed": 100, "move_abs": 100}, + {"ts_ms": ts, "move_signed": -40, "move_abs": 40}, + ] + buckets = aggregate_move_points(samples, bucket_minutes=60) + b14 = next(b for b in buckets if b["bucket_start_min"] == 14 * 60) + assert b14["n"] == 2 + assert b14["mean_signed"] == 30.0 + assert b14["mean_abs"] == 70.0 + + +def test_move_points_payload_pending_message(): + info = resolve_range("day", "2026-07-30") + payload = move_points_stats_payload( + [{"ts_ms": info["start_ms"] + 3600_000, "side": "C", "expiry_ymd": "991231", "index_px": 1}], + {}, + range_info=info, + bucket_minutes=60, + side="both", + now_ms=info["start_ms"], + ) + assert payload["status"] == "ok" + assert payload["pending_expiry"] is True + assert payload["settled_count"] == 0 + assert payload["sample_count"] == 0 diff --git a/tests/test_move_points.py b/tests/test_move_points.py new file mode 100644 index 0000000..076590d --- /dev/null +++ b/tests/test_move_points.py @@ -0,0 +1,18 @@ +from packages.domain import move_points +from packages.domain.expiry import expiry_ms_from_ymd + + +def test_move_points_signed(): + assert move_points(3600.0, 3500.0) == 100.0 + assert move_points(3400.0, 3500.0) == -100.0 + assert abs(move_points(3400.0, 3500.0)) == 100.0 + + +def test_expiry_ms_utc8(): + # 260731 → 2026-07-31 08:00 UTC + ms = expiry_ms_from_ymd("260731") + from datetime import datetime, timezone + + dt = datetime.fromtimestamp(ms / 1000, tz=timezone.utc) + assert dt.year == 2026 and dt.month == 7 and dt.day == 31 + assert dt.hour == 8 and dt.minute == 0 diff --git a/tests/test_selectors.py b/tests/test_selectors.py new file mode 100644 index 0000000..aae4666 --- /dev/null +++ b/tests/test_selectors.py @@ -0,0 +1,27 @@ +from apps.collector.selectors import pick_atm_strike, select_atm_pair + + +def test_pick_atm_strike(): + assert pick_atm_strike([3490, 3500, 3510], 3502) == 3500 + assert pick_atm_strike([3490, 3510], 3500) == 3490 # 等距取较小 + + +def test_select_atm_pair(): + # 构造远到期，避免 min_hours 过滤 + contracts = [] + for k in (3490.0, 3500.0, 3510.0): + for side in ("C", "P"): + contracts.append( + { + "inst_id": f"ETH-USD_UM-991231-{int(k)}-{side}", + "expiry_ymd": "991231", + "expiry_ms": 4102358400000, # 远未来 + "strike": k, + "side": side, + } + ) + pair = select_atm_pair(contracts, index_px=3501.0, min_hours=0) + assert pair is not None + assert pair.strike == 3500.0 + assert pair.call_inst_id.endswith("-C") + assert pair.put_inst_id.endswith("-P") diff --git a/tests/test_wecom.py b/tests/test_wecom.py new file mode 100644 index 0000000..65bd888 --- /dev/null +++ b/tests/test_wecom.py @@ -0,0 +1,56 @@ +import os + +from packages.config.settings import get_settings +from packages.notify import wecom + + +def _with_env(**kwargs): + old = {} + for k, v in kwargs.items(): + old[k] = os.environ.get(k) + os.environ[k] = v + get_settings.cache_clear() + return old + + +def _restore(old: dict): + for k, v in old.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + get_settings.cache_clear() + + +def test_build_markdown_contains_tag(): + md = wecom.build_markdown(tag="FAULT", title="测试", lines=["a", "b"]) + assert "`FAULT`" in md + assert "比特骆驼行情采集分析" in md + assert "测试" in md + + +def test_fault_below_threshold(): + wecom.reset_alert_state() + old = _with_env(WECOM_ENABLED="0", ALERT_FAIL_THRESHOLD="5") + try: + ok, msg = wecom.notify_collector_fault(error="x", consecutive_failures=2) + assert ok is False + assert "below threshold" in msg + finally: + _restore(old) + wecom.reset_alert_state() + + +def test_fault_when_disabled(): + wecom.reset_alert_state() + old = _with_env(WECOM_ENABLED="0", ALERT_FAIL_THRESHOLD="3") + try: + ok, msg = wecom.notify_collector_fault(error="boom", consecutive_failures=3) + assert ok is False + assert "未开启" in msg + ok2, msg2 = wecom.notify_collector_fault(error="boom", consecutive_failures=3) + assert ok2 is False + assert msg2 == "dedup" + finally: + _restore(old) + wecom.reset_alert_state() diff --git a/web/dist/index.html b/web/dist/index.html new file mode 100644 index 0000000..041cfd6 --- /dev/null +++ b/web/dist/index.html @@ -0,0 +1,343 @@ + + + + + + 比特骆驼行情采集分析 + + + +
+
比特骆驼行情采集分析
+
只读采集 · 杠杆 = 指数 ÷ 卖一 · Asia/Shanghai
+
+
+ 总览 + 作战地图 + +
+
+ +
+
+
采集状态
…
—
+
ATM Call 杠杆
—
—
+
ATM Put 杠杆
—
—
+
指数
—
—
+
+ 
加载中…
+
+ +
+ + + diff --git a/web/index.html b/web/index.html new file mode 100644 index 0000000..167a0af --- /dev/null +++ b/web/index.html @@ -0,0 +1,12 @@ + + + + + + 比特骆驼行情采集分析 + + +
+ + + diff --git a/web/package.json b/web/package.json new file mode 100644 index 0000000..6ebbc32 --- /dev/null +++ b/web/package.json @@ -0,0 +1,23 @@ +{ + "name": "market-intel-web", + "private": true, + "version": "0.1.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview" + }, + "dependencies": { + "react": "^18.3.1", + "react-dom": "^18.3.1", + "react-router-dom": "^6.26.0" + }, + "devDependencies": { + "@types/react": "^18.3.3", + "@types/react-dom": "^18.3.0", + "@vitejs/plugin-react": "^4.3.1", + "typescript": "^5.5.4", + "vite": "^5.4.0" + } +} diff --git a/web/src/api/client.ts b/web/src/api/client.ts new file mode 100644 index 0000000..d92dedd --- /dev/null +++ b/web/src/api/client.ts @@ -0,0 +1,150 @@ +const TOKEN_KEY = "mi_token"; + +export function getToken(): string | null { + try { + return localStorage.getItem(TOKEN_KEY); + } catch { + return null; + } +} + +export function setToken(token: string | null) { + try { + if (token) localStorage.setItem(TOKEN_KEY, token); + else localStorage.removeItem(TOKEN_KEY); + } catch { + /* ignore */ + } +} + +function authHeaders(): HeadersInit { + const t = getToken(); + return t ? { Authorization: `Bearer ${t}` } : {}; +} + +async function apiFetch(input: string, init?: RequestInit): Promise { + const headers = { + ...(init?.headers || {}), + ...authHeaders(), + }; + const r = await fetch(input, { ...init, headers, credentials: "include" }); + if (r.status === 401) { + setToken(null); + } + return r; +} + +export type AuthStatus = { auth_required: boolean }; + +export async function fetchAuthStatus(): Promise { + const r = await fetch("/api/auth/status", { credentials: "include" }); + if (!r.ok) throw new Error("auth status failed"); + return r.json(); +} + +export async function login(password: string): Promise<{ ok: boolean; token?: string | null }> { + const r = await fetch("/api/auth/login", { + method: "POST", + credentials: "include", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ password }), + }); + if (!r.ok) throw new Error("密码错误"); + const body = await r.json(); + if (body.token) setToken(body.token); + return body; +} + +export async function logout(): Promise { + setToken(null); + await fetch("/api/auth/logout", { method: "POST", credentials: "include" }); +} + +export type Health = { + ok: boolean; + collector_lag_ms: number | null; + consecutive_failures: number; + option_quotes: number; +}; + +export type Latest = { + call?: { leverage?: number; ask?: number; inst_id?: string; index_px?: number }; + put?: { leverage?: number; ask?: number; inst_id?: string; index_px?: number }; + heartbeat?: { meta?: { index_px?: number; expiry_ymd?: string; strike?: number } }; +}; + +export type LeverageBucket = { + bucket_start_min: number; + label: string; + n: number; + mean: number | null; + median: number | null; + p25: number | null; + p75: number | null; + pct_ge_min: number | null; +}; + +export type LeverageStats = { + status: string; + range: string; + date: string; + start_ymd: string; + end_ymd: string; + side: string; + sample_count: number; + min_leverage: number; + buckets: LeverageBucket[]; +}; + +export type MoveBucket = { + bucket_start_min: number; + label: string; + n: number; + mean_abs: number | null; + median_abs: number | null; + mean_signed: number | null; + median_signed: number | null; +}; + +export type MovePointsStats = { + status: string; + pending_expiry?: boolean; + pending_count?: number; + settled_count?: number; + sample_count?: number; + message?: string | null; + buckets: MoveBucket[]; +}; + +export type OpsMap = { + leverage: LeverageStats; + move_points: MovePointsStats; +}; + +export async function fetchHealth(): Promise { + const r = await fetch("/health"); + if (!r.ok) throw new Error("health failed"); + return r.json(); +} + +export async function fetchLatest(): Promise { + const r = await apiFetch("/api/meta/latest"); + if (!r.ok) throw new Error(r.status === 401 ? "unauthorized" : "latest failed"); + return r.json(); +} + +export async function fetchOpsMap(params: { + range: string; + date?: string; + side?: string; + bucket_minutes?: number; +}): Promise { + const q = new URLSearchParams(); + q.set("range", params.range); + if (params.date) q.set("date", params.date); + if (params.side) q.set("side", params.side); + if (params.bucket_minutes) q.set("bucket_minutes", String(params.bucket_minutes)); + const r = await apiFetch(`/api/stats/ops-map?${q}`); + if (!r.ok) throw new Error(r.status === 401 ? "unauthorized" : "ops-map failed"); + return r.json(); +} diff --git a/web/src/components/LeverageChart.tsx b/web/src/components/LeverageChart.tsx new file mode 100644 index 0000000..3a9bba2 --- /dev/null +++ b/web/src/components/LeverageChart.tsx @@ -0,0 +1,98 @@ +import { LeverageBucket } from "../api/client"; + +type Props = { + buckets: LeverageBucket[]; + minLeverage: number; + title: string; +}; + +export default function LeverageChart({ buckets, minLeverage, title }: Props) { + const width = 880; + const height = 260; + const padL = 44; + const padR = 12; + const padT = 24; + const padB = 36; + const innerW = width - padL - padR; + const innerH = height - padT - padB; + + const vals = buckets.map((b) => b.mean ?? 0); + const maxV = Math.max(minLeverage * 1.2, ...vals, 1); + const barW = innerW / Math.max(buckets.length, 1); + + return ( +
+
{title}
+ + + + min {minLeverage} + + {buckets.map((b, i) => { + const v = b.mean ?? 0; + const h = b.n > 0 ? (v / maxV) * innerH : 0; + const x = padL + i * barW + barW * 0.15; + const y = padT + innerH - h; + const w = barW * 0.7; + const fill = b.n === 0 ? "#243041" : v >= minLeverage ? "#3ecf8e" : "#3d8fd1"; + return ( + + 0 ? 2 : 0)} fill={fill} rx="2"> + + {b.label}: mean={b.mean?.toFixed(1) ?? "—"} n={b.n} median= + {b.median?.toFixed(1) ?? "—"} + + + {i % 2 === 0 && ( + + {b.label.replace(":00", "")} + + )} + + ); + })} + + + + {maxV.toFixed(0)} + + + 0 + + +
+ ≥达标线均值 + <达标线 + 无样本 + 达标线 +
+
+ ); +} diff --git a/web/src/components/LoginGate.tsx b/web/src/components/LoginGate.tsx new file mode 100644 index 0000000..1f0631c --- /dev/null +++ b/web/src/components/LoginGate.tsx @@ -0,0 +1,67 @@ +import { FormEvent, useState } from "react"; +import { login } from "../api/client"; + +type Props = { + onOk: () => void; +}; + +export default function LoginGate({ onOk }: Props) { + const [password, setPassword] = useState(""); + const [err, setErr] = useState(null); + const [busy, setBusy] = useState(false); + + const submit = async (e: FormEvent) => { + e.preventDefault(); + setBusy(true); + setErr(null); + try { + await login(password); + onOk(); + } catch (ex) { + setErr(String(ex)); + } finally { + setBusy(false); + } + }; + + return ( +
+
比特骆驼行情采集分析
+
需要登录后查看看板
+
+
管理员密码
+ setPassword(e.target.value)} + autoFocus + style={{ + width: "100%", + marginTop: "0.5rem", + padding: "0.55rem 0.65rem", + borderRadius: 6, + border: "1px solid #243041", + background: "#0c1117", + color: "#e8eef5", + }} + /> + {err &&

{err}

} + +
+
+ ); +} diff --git a/web/src/components/MovePointsChart.tsx b/web/src/components/MovePointsChart.tsx new file mode 100644 index 0000000..989cfad --- /dev/null +++ b/web/src/components/MovePointsChart.tsx @@ -0,0 +1,102 @@ +import { MoveBucket } from "../api/client"; + +type Props = { + buckets: MoveBucket[]; + title: string; + mode?: "abs" | "signed"; +}; + +export default function MovePointsChart({ buckets, title, mode = "abs" }: Props) { + const width = 880; + const height = 260; + const padL = 44; + const padR = 12; + const padT = 24; + const padB = 36; + const innerW = width - padL - padR; + const innerH = height - padT - padB; + + const vals = buckets.map((b) => + mode === "abs" ? b.mean_abs ?? 0 : b.mean_signed ?? 0 + ); + const maxAbs = Math.max(...vals.map((v) => Math.abs(v)), 1); + const y0 = mode === "signed" ? padT + innerH / 2 : padT + innerH; + const scale = mode === "signed" ? innerH / 2 / maxAbs : innerH / maxAbs; + const barW = innerW / Math.max(buckets.length, 1); + + return ( +
+
{title}
+ + {mode === "signed" && ( + + )} + {buckets.map((b, i) => { + const v = mode === "abs" ? b.mean_abs ?? 0 : b.mean_signed ?? 0; + const h = b.n > 0 ? Math.abs(v) * scale : 0; + const x = padL + i * barW + barW * 0.15; + const y = mode === "signed" ? (v >= 0 ? y0 - h : y0) : y0 - h; + const w = barW * 0.7; + const fill = + b.n === 0 ? "#243041" : mode === "abs" ? "#9b7bff" : v >= 0 ? "#3ecf8e" : "#e85d5d"; + return ( + + 0 ? 2 : 0)} fill={fill} rx="2"> + + {b.label}: abs={b.mean_abs?.toFixed(1) ?? "—"} signed= + {b.mean_signed?.toFixed(1) ?? "—"} n={b.n} + + + {i % 2 === 0 && ( + + {b.label.replace(":00", "")} + + )} + + ); + })} + + + + {mode === "signed" ? maxAbs.toFixed(0) : maxAbs.toFixed(0)} + + + {mode === "signed" ? `-${maxAbs.toFixed(0)}` : "0"} + + +
+ {mode === "abs" ? ( + <> + 绝对波动均值 + + ) : ( + <> + 上涨 + 下跌 + + )} + 无样本/未到期 +
+
+ ); +} diff --git a/web/src/main.tsx b/web/src/main.tsx new file mode 100644 index 0000000..86d97bf --- /dev/null +++ b/web/src/main.tsx @@ -0,0 +1,17 @@ +import React from "react"; +import ReactDOM from "react-dom/client"; +import { BrowserRouter, Route, Routes } from "react-router-dom"; +import Dashboard from "./pages/Dashboard"; +import OpsMap from "./pages/OpsMap"; +import "./styles.css"; + +ReactDOM.createRoot(document.getElementById("root")!).render( + + + + } /> + } /> + + + +); diff --git a/web/src/pages/Dashboard.tsx b/web/src/pages/Dashboard.tsx new file mode 100644 index 0000000..6c94d48 --- /dev/null +++ b/web/src/pages/Dashboard.tsx @@ -0,0 +1,143 @@ +import { useEffect, useState } from "react"; +import { Link } from "react-router-dom"; +import { + fetchAuthStatus, + fetchHealth, + fetchLatest, + Health, + Latest, + logout, +} from "../api/client"; +import LoginGate from "../components/LoginGate"; + +function fmt(n?: number | null, d = 1) { + if (n == null || Number.isNaN(n)) return "—"; + return n.toFixed(d); +} + +export default function Dashboard() { + const [needLogin, setNeedLogin] = useState(false); + const [ready, setReady] = useState(false); + const [health, setHealth] = useState(null); + const [latest, setLatest] = useState(null); + const [err, setErr] = useState(null); + + const bootstrap = async () => { + const st = await fetchAuthStatus(); + if (!st.auth_required) { + setNeedLogin(false); + setReady(true); + return; + } + try { + await fetchLatest(); + setNeedLogin(false); + } catch { + setNeedLogin(true); + } + setReady(true); + }; + + useEffect(() => { + bootstrap().catch((e) => setErr(String(e))); + }, []); + + useEffect(() => { + if (!ready || needLogin) return; + let alive = true; + const load = async () => { + try { + const [h, m] = await Promise.all([fetchHealth(), fetchLatest()]); + if (!alive) return; + setHealth(h); + setLatest(m); + setErr(null); + } catch (e) { + if (!alive) return; + const msg = String(e); + if (msg.includes("unauthorized")) setNeedLogin(true); + else setErr(msg); + } + }; + load(); + const t = setInterval(load, 10000); + return () => { + alive = false; + clearInterval(t); + }; + }, [ready, needLogin]); + + if (!ready) { + return ( +
+
加载中…
+
+ ); + } + if (needLogin) { + return ( + { + setNeedLogin(false); + setErr(null); + }} + /> + ); + } + + const lag = health?.collector_lag_ms; + const ok = + !!health?.ok && (lag == null || lag < 120_000) && (health.consecutive_failures || 0) < 5; + const meta = latest?.heartbeat?.meta; + + return ( +
+
比特骆驼行情采集分析
+
只读采集 · 杠杆 = 指数 ÷ 卖一 · Asia/Shanghai
+ + {err &&

{err}

} +
+
+
采集状态
+
+ {health ? (ok ? "正常" : "异常/等待") : "…"} +
+
+ {lag == null + ? "尚无采样" + : `延迟 ${Math.round(lag / 1000)}s · 样本 ${health?.option_quotes ?? 0}`} +
+
+
+
ATM Call 杠杆
+
{fmt(latest?.call?.leverage)}
+
{latest?.call?.inst_id ?? "—"}
+
+
+
ATM Put 杠杆
+
{fmt(latest?.put?.leverage)}
+
{latest?.put?.inst_id ?? "—"}
+
+
+
指数
+
{fmt(meta?.index_px ?? latest?.call?.index_px, 2)}
+
+ {meta?.expiry_ymd ? `到期 ${meta.expiry_ymd} · 行权 ${meta.strike ?? "—"}` : "—"} +
+
+
+
+ ); +} diff --git a/web/src/pages/OpsMap.tsx b/web/src/pages/OpsMap.tsx new file mode 100644 index 0000000..3ebde21 --- /dev/null +++ b/web/src/pages/OpsMap.tsx @@ -0,0 +1,219 @@ +import { useEffect, useState } from "react"; +import { Link } from "react-router-dom"; +import { + fetchAuthStatus, + fetchOpsMap, + LeverageStats, + logout, + MovePointsStats, + OpsMap, +} from "../api/client"; +import LeverageChart from "../components/LeverageChart"; +import LoginGate from "../components/LoginGate"; +import MovePointsChart from "../components/MovePointsChart"; + +type RangeKey = "day" | "week" | "month"; +type SideKey = "both" | "C" | "P"; + +function todayYmd() { + const d = new Date(); + const y = d.getFullYear(); + const m = String(d.getMonth() + 1).padStart(2, "0"); + const day = String(d.getDate()).padStart(2, "0"); + return `${y}-${m}-${day}`; +} + +export default function OpsMapPage() { + const [needLogin, setNeedLogin] = useState(false); + const [ready, setReady] = useState(false); + const [range, setRange] = useState("day"); + const [side, setSide] = useState("both"); + const [date, setDate] = useState(todayYmd()); + const [moveMode, setMoveMode] = useState<"abs" | "signed">("abs"); + const [data, setData] = useState(null); + const [err, setErr] = useState(null); + const [loading, setLoading] = useState(false); + + useEffect(() => { + fetchAuthStatus() + .then(async (st) => { + if (!st.auth_required) { + setNeedLogin(false); + setReady(true); + return; + } + try { + await fetchOpsMap({ range: "day", bucket_minutes: 60 }); + setNeedLogin(false); + } catch { + setNeedLogin(true); + } + setReady(true); + }) + .catch((e) => setErr(String(e))); + }, []); + + useEffect(() => { + if (!ready || needLogin) return; + let alive = true; + setLoading(true); + fetchOpsMap({ range, date, side, bucket_minutes: 60 }) + .then((d) => { + if (!alive) return; + setData(d); + setErr(null); + }) + .catch((e) => { + if (!alive) return; + const msg = String(e); + if (msg.includes("unauthorized")) setNeedLogin(true); + else setErr(msg); + }) + .finally(() => { + if (alive) setLoading(false); + }); + return () => { + alive = false; + }; + }, [range, side, date, ready, needLogin]); + + if (!ready) { + return ( +
+
加载中…
+
+ ); + } + if (needLogin) { + return setNeedLogin(false)} />; + } + + const lev: LeverageStats | undefined = data?.leverage; + const mov: MovePointsStats | undefined = data?.move_points; + const rangeLabel = range === "day" ? "日" : range === "week" ? "近7日" : "近30日"; + + return ( +
+
比特骆驼行情采集分析
+
作战地图 · 杠杆 × 时段 → 到期波动
+ + +
+
+ {(["day", "week", "month"] as RangeKey[]).map((k) => ( + + ))} +
+
+ {(["both", "C", "P"] as SideKey[]).map((k) => ( + + ))} +
+ +
+ + {err &&

{err}

} + {loading && !data &&

加载中…

} + + {lev && ( + <> +
+
+
范围
+
+ {lev.start_ymd} → {lev.end_ymd} +
+
杠杆样本 {lev.sample_count}
+
+
+
达标线
+
{lev.min_leverage}
+
杠杆 = 指数 ÷ 卖一
+
+
+
波动样本
+
{mov?.settled_count ?? 0}
+
+ {mov?.pending_expiry + ? `pending ${mov.pending_count ?? 0}（未到期已排除）` + : "全部已结算"} +
+
+
+ + + +
+
+ + +
+ {mov?.pending_expiry && ( + + pending_expiry=true + + )} +
+ + + {mov?.message && ( +

+ {mov.message} +

+ )} + + )} +
+ ); +} diff --git a/web/src/styles.css b/web/src/styles.css new file mode 100644 index 0000000..5d4e871 --- /dev/null +++ b/web/src/styles.css @@ -0,0 +1,108 @@ +:root { + --bg: #0c1117; + --panel: #151b24; + --text: #e8eef5; + --muted: #8b9aab; + --accent: #3d8fd1; + --ok: #3ecf8e; + --warn: #e6a23c; +} +* { box-sizing: border-box; } +body { + margin: 0; + font-family: "Segoe UI", "PingFang SC", "Microsoft YaHei", sans-serif; + color: var(--text); + background: + radial-gradient(1200px 600px at 10% -10%, #1a2a3d 0%, transparent 55%), + var(--bg); +} +a { color: var(--accent); text-decoration: none; } +.layout { max-width: 980px; margin: 0 auto; padding: 1.5rem; } +.brand { font-size: 1.35rem; font-weight: 700; } +.sub { color: var(--muted); margin: 0.35rem 0 1.25rem; } +.nav { display: flex; gap: 1rem; margin-bottom: 1.25rem; } +.row { display: grid; grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); gap: 1rem; } +.tile { + background: var(--panel); + border: 1px solid #243041; + border-radius: 10px; + padding: 1rem; +} +.label { color: var(--muted); font-size: 0.8rem; } +.value { margin-top: 0.35rem; font-size: 1.5rem; font-weight: 650; } +.hint { margin-top: 0.3rem; color: var(--muted); font-size: 0.78rem; } + +.toolbar { + display: flex; + flex-wrap: wrap; + gap: 0.75rem; + align-items: center; + margin-bottom: 1.25rem; +} +.seg { + display: inline-flex; + background: var(--panel); + border: 1px solid #243041; + border-radius: 8px; + overflow: hidden; +} +.seg button { + appearance: none; + border: 0; + background: transparent; + color: var(--muted); + padding: 0.45rem 0.85rem; + cursor: pointer; + font-size: 0.9rem; +} +.seg button.active { + background: #1e2a3a; + color: var(--text); +} +.date-field { + display: inline-flex; + align-items: center; + gap: 0.5rem; + color: var(--muted); + font-size: 0.85rem; +} +.date-field input { + background: var(--panel); + border: 1px solid #243041; + color: var(--text); + border-radius: 6px; + padding: 0.35rem 0.5rem; +} + +.chart-wrap { + background: var(--panel); + border: 1px solid #243041; + border-radius: 10px; + padding: 0.75rem 0.5rem 0.5rem; +} +.chart-title { + padding: 0 0.75rem 0.25rem; + color: var(--muted); + font-size: 0.85rem; +} +.chart-svg { width: 100%; height: auto; display: block; } +.chart-legend { + display: flex; + flex-wrap: wrap; + gap: 0.85rem; + padding: 0.25rem 0.75rem 0.5rem; + color: var(--muted); + font-size: 0.75rem; +} +.dot { + display: inline-block; + width: 8px; + height: 8px; + border-radius: 2px; + margin-right: 0.3rem; + vertical-align: middle; +} +.dot.ok { background: var(--ok); } +.dot.mid { background: var(--accent); } +.dot.empty { background: #243041; } +.dot.warn { background: var(--warn); } diff --git a/web/tsconfig.json b/web/tsconfig.json new file mode 100644 index 0000000..1bd23da --- /dev/null +++ b/web/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2020", + "useDefineForClassFields": true, + "lib": ["ES2020", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": true, + "jsx": "react-jsx", + "strict": true + }, + "include": ["src"] +} diff --git a/web/vite.config.ts b/web/vite.config.ts new file mode 100644 index 0000000..3ac8204 --- /dev/null +++ b/web/vite.config.ts @@ -0,0 +1,17 @@ +import { defineConfig } from "vite"; +import react from "@vitejs/plugin-react"; + +export default defineConfig({ + plugins: [react()], + server: { + port: 5173, + proxy: { + "/api": "http://127.0.0.1:5170", + "/health": "http://127.0.0.1:5170", + }, + }, + build: { + outDir: "dist", + emptyOutDir: true, + }, +}); diff --git a/开发方案.md b/开发方案.md new file mode 100644 index 0000000..b6dd406 --- /dev/null +++ b/开发方案.md @@ -0,0 +1,383 @@ +# 比特骆驼行情采集分析 — 开发方案 + +> **独立仓库**，与 `eth_hedge_sim`（策略 / 中控）**无代码共用、无进程共用、无交易密钥共用**。 +> 本系统只做：**行情采集 → 落库 → 分析统计 → 只读展示 / API**。 +> **不下单、不持仓、不替代策略选约。** +> Git 仓库由负责人在 `git.bz121.com` 创建；本目录为方案与后续工程落点。 + +--- + +## 1. 命名与定位 + +| 项 | 约定 | +|----|------| +| **产品名** | **比特骆驼行情采集分析** | +| **项目名称（对外）** | 比特骆驼行情采集分析系统 | +| **仓库名称 / 工程名** | `market_intel` | +| **Git 地址（拟）** | `https://git.bz121.com/dekun/market_intel.git` | +| **安装目录（生产）** | `/opt/market_intel` | +| **部署操作系统** | **Ubuntu 22.04 LTS**（与策略仓一键脚本一致） | +| **运行方式** | **Docker Compose**（采集 + API + Web 同栈编排） | +| **一键入口** | `deploy/manage.sh` 交互菜单（对齐 `eth_hedge_sim`） | + +### 1.1 硬边界 + +- **只读行情**：REST / WebSocket；禁止任何交易类 API。 +- **与策略解耦**：策略机挂了，本系统仍可继续采；本系统挂了，策略仍可独立交易。 +- **第一期标的**：ETH（OKX 指数 + ETH-USD_UM 期权盘口）；架构预留多币种 / 多所，但实现标准先钉死 OKX ETH。 +- **时区**：统计与「日 / 周 / 月」切分一律 **Asia/Shanghai**。 + +### 1.2 核心分析目标（作战地图数据底座） + +1. **期权杠杆 × 日内时段** + 口径：`杠杆 = 标的指数 ÷ 期权卖一`（与策略选约门限一致）。 +2. **该时段 → 期权到期 的波动点数** + 口径：到期时刻指数 − 该时段代表指数（可同时存带符号与绝对值）。 +3. **范围**：按 **日 / 周 / 月** 过滤样本，横轴均为 **日内时段桶**（默认 1 小时）。 + +--- + +## 2. 实现标准 + +### 2.1 技术栈 + +| 层 | 标准 | +|----|------| +| 语言 | Python 3.11+（采集 / API / 聚合） | +| Web API | FastAPI | +| 前端 | React + Vite（只读看板：采集状态、时段图、日周月切换） | +| 数据库 | 第一期 **SQLite**（volume 持久化）；上量后可换 Postgres，表结构先按可迁移设计 | +| 容器 | Docker + Docker Compose v2 | +| 反向代理（可选） | 同 Compose 内 Caddy/Nginx，或宿主机已有反代 | +| 配置 | 仓库根 `.env`（密钥、交易所、采样间隔、端口）；**已有非空值不覆盖** | + +### 2.2 代码与工程规范 + +- 仓库根英文名固定 `market_intel`；文档可用中文。 +- 配置项集中、可环境变量覆盖；禁止把 API Key 写进镜像层。 +- 采集与 API **可同容器或分服务**；Compose 内用服务名互访。 +- 所有时间戳存 **UTC ms**；展示与「自然日」按上海转换。 +- 杠杆 / 波动口径写进代码常量 + 本文档，变更需改版本号与迁移说明。 +- 日志：结构化或按日滚动；脱敏（不打完整密钥）。 +- 测试：采集解析、杠杆计算、时段聚合、到期回填 有单元测试。 + +### 2.3 采集标准（第一期） + +| 项 | 标准 | +|----|------| +| 交易所 | OKX（只读） | +| 指数 | ETH-USD 指数（或与策略一致的 index） | +| 期权 | 最近合资格到期的 ATM Call + ATM Put（规则文档化：最接近指数的行权价） | +| 杠杆采样间隔 | 默认 **30s**（可配 15–120s） | +| 指数采样 | 可与杠杆同频，或单独 **60s** | +| 字段最小集 | `ts_ms, exchange, underlying, expiry_ymd, strike, side(C/P), index_px, ask, bid, ask_sz, bid_sz, leverage, inst_id` | +| 失败策略 | 单次失败记日志并跳过；连续失败告警（可选企微，后期） | + +### 2.4 统计标准 + +| 项 | 标准 | +|----|------| +| 时段桶 | 默认 **1 小时**（0–23，上海）；可扩展 30 分钟 | +| 日 | 上海自然日 `YYYY-MM-DD` | +| 周 | **滚动近 7 个上海自然日**（第一期）；后期可加自然周 | +| 月 | **滚动近 30 日** 或自然月（设置可选，默认滚动 30 日） | +| 杠杆聚合 | 桶内：样本数、均值、中位数、P25/P75、≥`min_leverage` 占比 | +| 波动点数 | 到期后回填；桶内：均值/中位/分位；同时提供 **signed** 与 **abs** | +| 未到期 | API 标记 `pending_expiry=true`，不假装有完整「到到期」分布 | + +### 2.5 安全与权限 + +- 仅行情只读 Key（若需要）；无交易权限。 +- Web / API 默认需登录或 Token（对齐策略仓简单鉴权即可）。 +- 局域网部署时可绑 `127.0.0.1` / 内网 IP；公网必须 HTTPS + 强密码。 + +--- + +## 3. 系统架构 + +```text + ┌─────────────────────────────────────┐ + │ market_intel（本仓库 · Docker） │ + OKX 行情 ────────►│ collector → SQLite/DB │ + (REST/WS) │ analytics（日/周/月 · 时段聚合） │ + │ api + web（只读看板） │ + └─────────────────────────────────────┘ + │ + │ 可选：只读 API + ▼ + 人工浏览器 / 其它系统（后期） +``` + +- **不**嵌入 `eth_hedge_sim` 进程。 +- 后期若中控要展示，由中控 **HTTP 调用本系统 API**，本仓仍独立演进。 + +--- + +## 4. 代码结构（目标仓库） + +```text +market_intel/ +├── README.md +├── 开发方案.md # 可从本文件迁入 docs/ +├── .env.example +├── .gitignore +├── docker-compose.yml # 一键编排入口 +├── Dockerfile # API + 采集（或多阶段） +├── requirements.txt +├── deploy/ +│ ├── manage.sh # 交互式菜单（curl | bash） +│ ├── bootstrap.sh # 缺 git/docker 时引导 +│ └── lib/ +│ ├── common.sh # 日志、读入、路径、.env 合并 +│ ├── install.sh # 一键部署（clone + compose up） +│ ├── update.sh # git pull + compose build/up +│ └── uninstall.sh # 停容器；可选保留 data volume +├── apps/ +│ ├── collector/ # 行情采集进程 +│ │ ├── __init__.py +│ │ ├── main.py # 入口：循环 / WS +│ │ ├── okx_rest.py +│ │ ├── okx_ws.py +│ │ └── selectors.py # ATM / 到期选择 +│ ├── api/ # FastAPI +│ │ ├── main.py +│ │ ├── routes/ +│ │ │ ├── health.py +│ │ │ ├── samples.py # 原始/明细（调试） +│ │ │ └── stats.py # 日周月 · 杠杆 · 波动 +│ │ └── auth.py +│ └── worker/ # 可选：到期回填、日终聚合 +├── packages/ +│ ├── db/ # schema、迁移、repository +│ ├── domain/ # option_leverage、bucket、move_points +│ └── config/ # settings from env +├── web/ # React 看板 +│ ├── package.json +│ └── src/ +│ ├── pages/ +│ │ ├── Dashboard.tsx # 采集心跳、最新杠杆 +│ │ └── OpsMap.tsx # 作战地图：杠杆 + 波动 · 日/周/月 +│ └── api/ +├── data/ # 本地/挂载：SQLite（.gitignore 内容） +├── scripts/ +│ ├── smoke_collect.py +│ └── backfill_index.py # 可选：历史指数回填波动 +└── tests/ + ├── test_leverage.py + ├── test_buckets.py + └── test_move_points.py +``` + +> 实现时可把 `apps/` 收成单包 `src/market_intel/`，但 **deploy / docker / web / 采集与 API 分离** 的边界保持不变。 + +--- + +## 5. 数据模型（摘要） + +### 5.1 `option_quotes`（杠杆明细） + +| 字段 | 说明 | +|------|------| +| id | 自增 | +| ts_ms | UTC | +| exchange | `okx` | +| inst_id | 合约 ID | +| expiry_ymd | `YYMMDD` | +| strike | 行权价 | +| side | `C` / `P` | +| index_px | 指数 | +| ask / bid | 卖一 / 买一 | +| ask_sz / bid_sz | 可选 | +| leverage | `index_px / ask`（ask>0） | + +### 5.2 `index_ticks`（指数明细） + +| 字段 | 说明 | +|------|------| +| ts_ms | UTC | +| underlying | `ETH` | +| index_px | 指数 | + +### 5.3 `expiry_settlements`（到期锚点） + +| 字段 | 说明 | +|------|------| +| expiry_ymd | 到期日 | +| settle_ts_ms | 到期时刻（OKX：UTC 08:00） | +| settle_index_px | 结算/到期指数 | + +波动点数：对历史某桶代表时刻 \(t\)， +`move = settle_index_px - index_at(t)`（同 `expiry_ymd`）。 + +--- + +## 6. API 约定（第一期） + +| 方法 | 路径 | 说明 | +|------|------|------| +| GET | `/health` | 存活；可选返回采集延迟 | +| GET | `/api/stats/leverage` | `range=day\|week\|month` + 日期；返回各时段桶聚合 | +| GET | `/api/stats/move_points` | 同上；返回时段→到期波动 | +| GET | `/api/stats/ops-map` | 一次返回杠杆 + 波动（看板主接口） | +| GET | `/api/meta/latest` | 最新一条 Call/Put 杠杆、采集时间 | + +查询参数统一：`range`、`date`（锚点日）、`side=C|P|both`、`bucket_minutes=60`。 + +--- + +## 7. Docker 运行标准 + +### 7.1 服务划分（Compose） + +| 服务名 | 职责 | 说明 | +|--------|------|------| +| `collector` | 写库 | 重启策略 `unless-stopped` | +| `api` | FastAPI + 静态前端（或挂 `web` 构建产物） | 默认端口 **5170**（可配，避开策略 5155 / 中控 5160） | +| `db` | 第一期可省略（SQLite 挂 volume） | 后期 Postgres 再加 | + +```yaml +# docker-compose.yml 示意（实现时落地） +services: + collector: + build: . + command: ["python", "-m", "apps.collector.main"] + env_file: .env + volumes: + - mi_data:/app/data + restart: unless-stopped + api: + build: . + command: ["uvicorn", "apps.api.main:app", "--host", "0.0.0.0", "--port", "5170"] + env_file: .env + ports: + - "${MI_PORT:-5170}:5170" + volumes: + - mi_data:/app/data + depends_on: + - collector + restart: unless-stopped +volumes: + mi_data: +``` + +### 7.2 本地开发（非必须 Docker） + +```bash +python -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt +cp .env.example .env +python -m apps.collector.main # 终端 1 +uvicorn apps.api.main:app --reload # 终端 2 +cd web && npm i && npm run dev # 终端 3 +``` + +**生产标准路径以 Docker Compose 为准。** + +--- + +## 8. 一键部署（对齐 eth_hedge_sim） + +### 8.1 新机器（免克隆） + +```bash +curl -fsSL https://git.bz121.com/dekun/market_intel/raw/branch/main/deploy/manage.sh | bash +``` + +要求：Ubuntu 22.04；脚本内检测并安装 **Git、Docker、Docker Compose 插件**（已有则跳过）。 + +### 8.2 已安装 + +```bash +bash /opt/market_intel/deploy/manage.sh +``` + +### 8.3 交互菜单（实现标准） + +与策略仓 `deploy/manage.sh` 同级体验：**数字选项 + 读 `/dev/tty`**，支持 `curl | bash`。 + +| 选项 | 作用 | +|------|------| +| **1) 一键部署** | 检测 Docker → clone 到 `/opt/market_intel` → 生成/补全 `.env`（已有非空不覆盖）→ `docker compose up -d --build` → 健康检查 | +| **2) 更新** | `git pull` + `compose build/up`；保留 `.env` 与 data volume | +| **3) 停止** | `docker compose stop` | +| **4) 启动** | `docker compose start` / `up -d` | +| **5) 查看状态** | `compose ps` + `/health` | +| **6) 一键卸载** | 停容器；询问是否删除 data volume；备份 `.env` 到 `/root/backups/market_intel/`；按确认删除 `/opt/market_intel` | +| **0) 退出** | — | + +已存在安装目录时，选项 1 进入子菜单：**取消 / 修复（保留 .env 与数据）**，行为对齐策略仓 `install.sh`。 + +### 8.4 `.env` 交互补全（首次部署） + +脚本可交互询问（有默认值，回车采用默认）： + +- `OKX` API Key / Secret / Passphrase（只读；可留空若仅用公开行情） +- `MI_PORT`（默认 `5170`） +- `SAMPLE_INTERVAL_SEC`（默认 `30`） +- `MIN_OPTION_LEVERAGE`（统计达标线，默认 `100`） +- 管理员密码 / `AUTH_SECRET` + +原则：**文件中已有非空值不覆盖**（对齐中控 `.env.control` 行为）。 + +### 8.5 部署后验收 + +1. `curl -fsS http://127.0.0.1:5170/health` 返回 ok +2. 等待 ≥1 个采样周期后，库中有 `option_quotes` 行 +3. 打开 Web 看板能看到最新杠杆 +4. `manage.sh` → 更新 → 容器重建后数据 volume 仍在 + +--- + +## 9. Web 看板（第一期页面） + +| 页面 | 内容 | +|------|------| +| 总览 | 采集是否正常、延迟、当前 ATM Call/Put 杠杆 | +| 作战地图 | 切换 **日 / 周 / 月**；上图时段杠杆；下图时段→到期波动点数 | +| 设置（简） | 只读展示当前采样参数（改参走 `.env` + 更新重启） | + +UI 要求：暗色可与策略仓风格接近，但 **独立品牌标题「行情采集分析」**，避免与对冲策略页混淆。 + +--- + +## 10. 分期计划 + +| 阶段 | 交付 | +|------|------| +| **P0** | 仓库骨架、Docker Compose、manage.sh 菜单、健康检查 | +| **P1** | OKX 指数 + ATM Call/Put 采样落库 | +| **P2** | `/api/stats/leverage` 日周月；Web 作战地图杠杆图 | +| **P3** | 到期回填 + 波动点数统计与下图 | +| **P4** | 鉴权加固、企微采集异常推送、可选历史指数回填 | +| **P5** | （可选）中控只读嵌入；多 underlying | + +--- + +## 11. 与 `eth_hedge_sim` 的关系（再强调） + +| | `eth_hedge_sim` | `market_intel`（本仓） | +|--|-----------------|------------------------| +| 职责 | 对冲交易 / 中控运维 | 行情采集与统计分析 | +| 运行 | PM2（现状） | **Docker Compose** | +| 端口 | 5155 / 5160 | **5170**（默认） | +| 密钥 | 可含交易权限 | **仅只读行情** | +| 依赖 | 互不依赖 | 互不依赖 | + +--- + +## 12. 验收清单（方案级） + +- [ ] 仓库名 `market_intel`，产品名「比特骆驼行情采集分析」 +- [ ] `curl | bash` 出交互菜单，可一键部署 / 更新 / 卸载 +- [ ] 全程 Docker 运行，数据落 volume +- [ ] 杠杆口径 = 指数 ÷ 卖一；时段统计支持日 / 周 / 月 +- [ ] 波动点数 = 时段指数 → 到期指数；未到期显式标记 +- [ ] 零交易 API；与策略仓进程隔离 + +--- + +## 13. 文档修订 + +| 日期 | 说明 | +|------|------| +| 2026-07-31 | 初稿：独立仓、Docker、交互式一键部署、作战地图数据标准 |