dekun/crypto_monitor
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor market K-line storage with tiered retention and chunked loading.

Browse Source 
Store 1m/5m/1h/12h/1d/1w with per-timeframe policies, aggregate 15m and 2h/4h on read, and support left-pan history fetches via before_ms.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
dekun
2026-06-08 07:27:16 +08:00
parent 41bdee2416
commit 11cc482599
5 changed files with 762 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files
manual_trading_hub/hub.py
+30 -3
View File
@@ -15,7 +15,15 @@ if str(_REPO_ROOT) not in sys.path:
sys.path.insert(0, str(_REPO_ROOT))
from hub_kline_store import format_ohlcv_detail, resolve_chart_bars, retention_days
from hub_ohlcv_lib import CHART_TIMEFRAME_ORDER, CHART_TIMEFRAMES, bar_limit_for_timeframe
from hub_ohlcv_lib import (
CHART_TIMEFRAME_ORDER,
CHART_TIMEFRAMES,
bar_limit_for_timeframe,
chart_chunk_limit,
chart_initial_limit,
chart_memory_cap,
retention_policy_meta,
)
from hub_symbol_archive_lib import (
ARCHIVE_DEFAULT_TIMEFRAME,
ARCHIVE_SEED_LOOKBACK_DAYS,
@@ -629,7 +637,11 @@ def api_chart_meta():
"ok": True,
"timeframes": [tf for tf in tfs if tf in CHART_TIMEFRAMES],
"retention_days": retention_days(),
"retention_policy": retention_policy_meta(),
"limits": {tf: bar_limit_for_timeframe(tf) for tf in tfs if tf in CHART_TIMEFRAMES},
"initial_limits": {tf: chart_initial_limit(tf) for tf in tfs if tf in CHART_TIMEFRAMES},
"chunk_limits": {tf: chart_chunk_limit(tf) for tf in tfs if tf in CHART_TIMEFRAMES},
"memory_caps": {tf: chart_memory_cap(tf) for tf in tfs if tf in CHART_TIMEFRAMES},
"exchanges": exchanges,
}
@@ -640,6 +652,8 @@ def api_chart_ohlcv(
symbol: str = "",
timeframe: str = "1d",
refresh: str = "",
limit: int = 0,
before_ms: str = "",
):
ex = _find_exchange_by_key(exchange_key)
if not ex:
@@ -651,14 +665,23 @@ def api_chart_ohlcv(
raise HTTPException(status_code=400, detail="请输入币种")
ex_key = str(ex.get("key") or "").strip().lower()
force = (refresh or "").strip().lower() in ("1", "true", "yes", "on")
lim = int(limit) if int(limit or 0) > 0 else None
bms_raw = (before_ms or "").strip()
bms = None
if bms_raw:
try:
bms = int(bms_raw)
except ValueError:
raise HTTPException(status_code=400, detail="before_ms 无效")
def remote_fetch(**kwargs):
tf_use = kwargs.get("timeframe") or timeframe
return _fetch_instance_ohlcv_sync(
ex,
symbol=kwargs.get("symbol") or sym,
timeframe=kwargs.get("timeframe") or timeframe,
timeframe=tf_use,
since_ms=kwargs.get("since_ms"),
limit=int(kwargs.get("limit") or bar_limit_for_timeframe(timeframe)),
limit=int(kwargs.get("limit") or bar_limit_for_timeframe(tf_use)),
)
result = resolve_chart_bars(
@@ -667,9 +690,13 @@ def api_chart_ohlcv(
timeframe,
remote_fetch,
force_refresh=force,
limit=lim,
before_ms=bms,
)
if not result.get("ok"):
raise HTTPException(status_code=502, detail=result.get("msg") or "K线加载失败")
if not result.get("candles") and result.get("before_ms") is None:
raise HTTPException(status_code=502, detail=result.get("msg") or "无 K 线")
tick = result.get("price_tick")
last = result["candles"][-1] if result.get("candles") else None
result["ohlcv"] = format_ohlcv_detail(
manual_trading_hub/static/chart.js
+225 -43
View File
@@ -6,6 +6,40 @@
const CHART_WATCH_HEARTBEAT_MS = 25000;
const CHART_SSE_FALLBACK_MS = 30000;
const DEFAULT_VISIBLE_BARS = 200;
const CHART_LOAD_LEFT_THRESHOLD = 25;
const CHART_INITIAL_LIMITS = {
"1m": 300,
"5m": 300,
"15m": 300,
"1h": 200,
"2h": 200,
"4h": 200,
"12h": 200,
"1d": 200,
"1w": 150,
};
const CHART_CHUNK_LIMITS = {
"1m": 500,
"5m": 500,
"15m": 500,
"1h": 300,
"2h": 300,
"4h": 300,
"12h": 200,
"1d": 200,
"1w": 150,
};
const CHART_MEMORY_CAPS = {
"1m": 5000,
"5m": 5000,
"15m": 5000,
"1h": 1000,
"2h": 1000,
"4h": 1000,
"12h": 1000,
"1d": 1000,
"1w": 500,
};
const RIGHT_OFFSET_BARS = 10;
const CANDLE_SCALE_BOTTOM = 0.26;
const VOLUME_SCALE_TOP = 0.73;
@@ -141,6 +175,8 @@
let localSeriesVersion = 0;
let lastViewKey = "";
let currentTf = "1d";
let exhaustedLeft = false;
let loadingLeft = false;
let priceTagTimer = null;
let tfDigitBuf = "";
let tfDigitTimer = null;
@@ -1914,9 +1950,13 @@
paintOhlcv(bar);
});
chart.timeScale().subscribeVisibleLogicalRangeChange(function () {
chart.timeScale().subscribeVisibleLogicalRangeChange(function (range) {
updateVisibleRangeMarkers();
updatePriceTag();
if (!range || loadingLeft || exhaustedLeft || !lastCandles.length) return;
if (range.from < CHART_LOAD_LEFT_THRESHOLD) {
void loadOlderCandles();
}
});
window.addEventListener("resize", function () {
@@ -1939,6 +1979,169 @@
return (exKey || "") + "|" + (sym || "") + "|" + (tf || "");
}
function chartInitialLimit(tf) {
return CHART_INITIAL_LIMITS[tf] || 200;
}
function chartChunkLimit(tf) {
return CHART_CHUNK_LIMITS[tf] || 200;
}
function chartMemoryCap(tf) {
return CHART_MEMORY_CAPS[tf] || 1000;
}
function resetChartHistoryState() {
exhaustedLeft = false;
loadingLeft = false;
}
function mergeCandles(existing, incoming, opts) {
opts = opts || {};
const prepend = !!opts.prepend;
const byTime = {};
(existing || []).forEach(function (c) {
if (c && c.time != null) byTime[c.time] = c;
});
(incoming || []).forEach(function (c) {
if (c && c.time != null) byTime[c.time] = c;
});
let merged = Object.keys(byTime)
.map(function (t) {
return Number(t);
})
.sort(function (a, b) {
return a - b;
})
.map(function (t) {
return byTime[t];
});
const cap = chartMemoryCap(currentTf);
if (merged.length > cap) {
merged = prepend ? merged.slice(0, cap) : merged.slice(-cap);
}
return merged;
}
function applyCandlesToChart(candles, rangeShift) {
lastCandles = alignCandlesToTick(candles);
indexCandles(lastCandles);
candleSeries.setData(lastCandles);
volumeSeries.setData(buildVolumeData(lastCandles));
applyChartRightGap();
if (rangeShift && chart) {
const range = chart.timeScale().getVisibleLogicalRange();
if (range) {
chart.timeScale().setVisibleLogicalRange({
from: range.from + rangeShift,
to: range.to + rangeShift,
});
}
}
applyPriceAutoScale();
updateVisibleRangeMarkers();
try {
updateIndicators();
} catch (indErr) {}
showLatestOhlcv();
}
async function fetchChartChunk(params) {
const qs = new URLSearchParams({
exchange_key: params.exchange_key,
symbol: params.symbol,
timeframe: params.timeframe,
limit: String(params.limit),
});
if (params.before_ms) qs.set("before_ms", String(params.before_ms));
if (params.refresh) qs.set("refresh", "1");
const r = await fetch("/api/chart/ohlcv?" + qs.toString(), { credentials: "same-origin" });
const data = await r.json();
if (!r.ok) {
throw new Error(data.detail || data.msg || "请求失败");
}
return data;
}
async function loadOlderCandles() {
if (loadingLeft || exhaustedLeft || !lastCandles.length) return;
const exKey = (elExchange && elExchange.value) || "";
const sym = (elSymbol && elSymbol.value.trim().toUpperCase()) || "";
const tf = (elTf && elTf.value) || "1d";
if (!exKey || !sym) return;
loadingLeft = true;
const beforeMs = Number(lastCandles[0].time) * 1000;
try {
const data = await fetchChartChunk({
exchange_key: exKey,
symbol: sym,
timeframe: tf,
limit: chartChunkLimit(tf),
before_ms: beforeMs,
});
if (data.exhausted) exhaustedLeft = true;
const incoming = alignCandlesToTick(data.candles || []);
if (!incoming.length) return;
const shift = incoming.length;
applyCandlesToChart(mergeCandles(lastCandles, incoming, { prepend: true }), shift);
if (elStatus && !elStatus.classList.contains("err")) {
elStatus.textContent =
"已加载 " +
lastCandles.length +
" 根（向左 +" +
incoming.length +
(exhaustedLeft ? " · 已到最早" : "") +
"）";
}
} catch (e) {
if (elStatus) {
elStatus.className = "market-status warn";
elStatus.textContent = "加载更早 K 线失败：" + String(e.message || e);
}
} finally {
loadingLeft = false;
}
}
async function refreshChartTail() {
const exKey = (elExchange && elExchange.value) || "";
const sym = (elSymbol && elSymbol.value.trim().toUpperCase()) || "";
const tf = (elTf && elTf.value) || "1d";
if (!exKey || !sym || !lastCandles.length) return;
const myToken = loadToken;
let savedRange = null;
if (chart) savedRange = chart.timeScale().getVisibleLogicalRange();
try {
const data = await fetchChartChunk({
exchange_key: exKey,
symbol: sym,
timeframe: tf,
limit: chartChunkLimit(tf),
});
if (myToken !== loadToken) return;
if (!data.ok || !data.candles || !data.candles.length) return;
if (data.price_tick != null) {
priceTick = data.price_tick;
try {
applyChartPriceFormat();
} catch (fmtErr) {
priceTick = null;
applyChartPriceFormat();
}
}
applyCandlesToChart(mergeCandles(lastCandles, alignCandlesToTick(data.candles), { prepend: false }), 0);
if (savedRange) chart.timeScale().setVisibleLogicalRange(savedRange);
if (posContext) {
updateLivePosPnl();
refreshPosPnlFromBoard();
}
if (data.series_version != null) localSeriesVersion = Number(data.series_version) || localSeriesVersion;
if (data.chart_version != null) localChartVersion = Number(data.chart_version) || localChartVersion;
if (elUpdated) elUpdated.textContent = "数据 " + (data.updated_at || "--");
tickLiveClock();
} catch (_) {}
}
function applyChartRightGap() {
if (!chart) return;
chart.timeScale().applyOptions({
@@ -2073,7 +2276,7 @@
if (seriesChanged) {
localSeriesVersion = sVer;
localChartVersion = ver;
loadChart(false, { autoTick: true });
refreshChartTail();
} else if (posContext) {
updateLivePosPnl();
} else if (ver !== localChartVersion) {
@@ -2111,7 +2314,7 @@
refreshTimer = setInterval(function () {
const page = document.getElementById("page-market");
if (!page || page.classList.contains("hidden")) return;
loadChart(false, { autoTick: true });
refreshChartTail();
}, CHART_SSE_FALLBACK_MS);
}
@@ -2151,10 +2354,11 @@
async function loadChart(force, options) {
options = options || {};
const autoTick = !!options.autoTick;
if (!autoTick) {
localSeriesVersion = 0;
void postChartWatch();
if (autoTick) {
return refreshChartTail();
}
localSeriesVersion = 0;
void postChartWatch();
if (!ensureChart()) return;
const exKey = (elExchange && elExchange.value) || "";
const sym = (elSymbol && elSymbol.value.trim().toUpperCase()) || "";
@@ -2169,31 +2373,23 @@
}
const myToken = ++loadToken;
const vKey = viewKey(exKey, sym, tf);
const resetView = !!force || !autoTick || vKey !== lastViewKey;
let savedRange = null;
if (!resetView && chart) {
savedRange = chart.timeScale().getVisibleLogicalRange();
}
if (!autoTick && elStatus) {
const resetView = !!force || vKey !== lastViewKey;
if (resetView) resetChartHistoryState();
if (elStatus) {
elStatus.className = "market-status";
elStatus.textContent = "加载中…";
}
updateHeaderLabels(sym, tf);
const qs = new URLSearchParams({
exchange_key: exKey,
symbol: sym,
timeframe: tf,
});
if (force) qs.set("refresh", "1");
try {
const r = await fetch("/api/chart/ohlcv?" + qs.toString(), { credentials: "same-origin" });
const data = await r.json();
const data = await fetchChartChunk({
exchange_key: exKey,
symbol: sym,
timeframe: tf,
limit: chartInitialLimit(tf),
refresh: !!force,
});
if (myToken !== loadToken) return;
if (!r.ok) {
throw new Error(data.detail || data.msg || "请求失败");
}
if (!data.ok || !data.candles || !data.candles.length) {
throw new Error(data.msg || "无 K 线");
}
@@ -2205,45 +2401,31 @@
priceTick = null;
applyChartPriceFormat();
}
lastCandles = alignCandlesToTick(data.candles);
indexCandles(lastCandles);
candleSeries.setData(lastCandles);
volumeSeries.setData(buildVolumeData(lastCandles));
applyChartRightGap();
applyCandlesToChart(alignCandlesToTick(data.candles), 0);
if (resetView) {
lastViewKey = vKey;
applyDefaultVisibleRange();
} else if (savedRange) {
chart.timeScale().setVisibleLogicalRange(savedRange);
}
applyPriceAutoScale();
updateVisibleRangeMarkers();
syncPosContextForView(exKey, sym);
if (posContext) {
updateLivePosPnl();
refreshPosPnlFromBoard();
}
showLatestOhlcv();
try {
updateIndicators();
} catch (indErr) {
/* 指标序列 priceFormat 异常时不阻断主图 */
}
scheduleChartResize();
const limit = data.limit || lastCandles.length;
let hint =
"已加载 " +
data.candles.length +
" 根（目标 " +
lastCandles.length +
" 根（首屏 " +
limit +
"）· 库 " +
(data.from_cache || 0) +
" / 新拉 " +
(data.fetched || 0) +
"）· 后台 " +
" · 左拖加载更多 · 后台 " +
(data.chart_poll_interval_sec || 5) +
"s 轮询 · SSE";
"s";
if (data.stale && data.stale_message) {
hint += " · 缓存：" + data.stale_message;
}