Fix discontinuous hub chart candles from orphaned DB bars.
Keep only the latest contiguous K-line segment, purge isolated stale rows, and backfill when the tail is still shorter than the initial limit. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -345,6 +345,54 @@ def load_bars_before(
|
|||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def trim_contiguous_tail(
|
||||||
|
bars: list[dict[str, Any]],
|
||||||
|
period_ms: int,
|
||||||
|
*,
|
||||||
|
max_gap_factor: float = 1.5,
|
||||||
|
) -> tuple[list[dict[str, Any]], int]:
|
||||||
|
"""只保留最近一段连续 K 线,丢弃左侧与主段断开的孤立数据。"""
|
||||||
|
if len(bars) <= 1:
|
||||||
|
return list(bars), 0
|
||||||
|
try:
|
||||||
|
period = max(1, int(period_ms))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
period = 60_000
|
||||||
|
max_gap = int(period * max_gap_factor)
|
||||||
|
split = 0
|
||||||
|
for i in range(len(bars) - 1, 0, -1):
|
||||||
|
gap = int(bars[i]["open_time_ms"]) - int(bars[i - 1]["open_time_ms"])
|
||||||
|
if gap > max_gap:
|
||||||
|
split = i
|
||||||
|
break
|
||||||
|
return bars[split:], split
|
||||||
|
|
||||||
|
|
||||||
|
def purge_bars_open_before(
|
||||||
|
exchange_key: str,
|
||||||
|
symbol: str,
|
||||||
|
timeframe: str,
|
||||||
|
open_time_ms: int,
|
||||||
|
db_path: Path | None = None,
|
||||||
|
) -> int:
|
||||||
|
"""删除某品种周期下早于 open_time_ms 的 K 线(清理与主段断开的孤立历史)。"""
|
||||||
|
ex_k = (exchange_key or "").strip().lower()
|
||||||
|
sym = (symbol or "").strip().upper()
|
||||||
|
tf = normalize_chart_timeframe(timeframe)
|
||||||
|
conn = _connect(db_path)
|
||||||
|
try:
|
||||||
|
cur = conn.execute(
|
||||||
|
"""
|
||||||
|
DELETE FROM ohlcv_bars
|
||||||
|
WHERE exchange_key=? AND symbol=? AND timeframe=? AND open_time_ms < ?
|
||||||
|
""",
|
||||||
|
(ex_k, sym, tf, int(open_time_ms)),
|
||||||
|
)
|
||||||
|
return int(cur.rowcount or 0)
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
def _rows_to_bars(rows) -> list[dict[str, Any]]:
|
def _rows_to_bars(rows) -> list[dict[str, Any]]:
|
||||||
return [
|
return [
|
||||||
{
|
{
|
||||||
@@ -532,6 +580,43 @@ def resolve_chart_bars(
|
|||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
if not is_history and db_rows and len(db_rows) > 1:
|
||||||
|
trimmed, split_at = trim_contiguous_tail(db_rows, period_display)
|
||||||
|
if split_at > 0:
|
||||||
|
purge_bars_open_before(
|
||||||
|
ex_k, sym, storage_tf, int(trimmed[0]["open_time_ms"]), db_path
|
||||||
|
)
|
||||||
|
db_rows = trimmed
|
||||||
|
|
||||||
|
if (
|
||||||
|
not is_history
|
||||||
|
and db_rows
|
||||||
|
and len(db_rows) < need
|
||||||
|
and not force_refresh
|
||||||
|
):
|
||||||
|
oldest = int(db_rows[0]["open_time_ms"])
|
||||||
|
missing = need - len(db_rows)
|
||||||
|
backfill_since = max(cutoff, oldest - period_storage * (missing + 40))
|
||||||
|
try:
|
||||||
|
remote_back = remote_fetch(
|
||||||
|
symbol=sym,
|
||||||
|
timeframe=storage_tf,
|
||||||
|
since_ms=backfill_since,
|
||||||
|
limit=min(missing + 60, 1500),
|
||||||
|
)
|
||||||
|
if remote_back.get("ok") and remote_back.get("bars"):
|
||||||
|
fetched += upsert_bars(ex_k, sym, storage_tf, remote_back["bars"], db_path)
|
||||||
|
db_rows = load_display_rows()
|
||||||
|
if len(db_rows) > 1:
|
||||||
|
trimmed, split_at = trim_contiguous_tail(db_rows, period_display)
|
||||||
|
if split_at > 0:
|
||||||
|
purge_bars_open_before(
|
||||||
|
ex_k, sym, storage_tf, int(trimmed[0]["open_time_ms"]), db_path
|
||||||
|
)
|
||||||
|
db_rows = trimmed
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
price_tick = normalize_price_tick(price_tick)
|
price_tick = normalize_price_tick(price_tick)
|
||||||
if db_rows and price_tick is not None:
|
if db_rows and price_tick is not None:
|
||||||
round_ohlcv_bars_to_tick(db_rows, price_tick)
|
round_ohlcv_bars_to_tick(db_rows, price_tick)
|
||||||
|
|||||||
@@ -349,7 +349,7 @@
|
|||||||
|
|
||||||
<div id="toast"></div>
|
<div id="toast"></div>
|
||||||
<script src="https://unpkg.com/lightweight-charts@4.2.0/dist/lightweight-charts.standalone.production.js"></script>
|
<script src="https://unpkg.com/lightweight-charts@4.2.0/dist/lightweight-charts.standalone.production.js"></script>
|
||||||
<script src="/assets/chart.js?v=20260608-tail-patch"></script>
|
<script src="/assets/chart.js?v=20260608-contiguous"></script>
|
||||||
<script src="/assets/archive.js?v=20260607-hub-archive-v6"></script>
|
<script src="/assets/archive.js?v=20260607-hub-archive-v6"></script>
|
||||||
<script src="/assets/ai_review_render.js?v=2"></script>
|
<script src="/assets/ai_review_render.js?v=2"></script>
|
||||||
<script src="/assets/app.js?v=20260607-hub-archive-v1"></script>
|
<script src="/assets/app.js?v=20260607-hub-archive-v1"></script>
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ from hub_kline_store import (
|
|||||||
purge_timeframe_by_days,
|
purge_timeframe_by_days,
|
||||||
resolve_chart_bars,
|
resolve_chart_bars,
|
||||||
retention_days,
|
retention_days,
|
||||||
|
trim_contiguous_tail,
|
||||||
upsert_bars,
|
upsert_bars,
|
||||||
)
|
)
|
||||||
from hub_ohlcv_lib import (
|
from hub_ohlcv_lib import (
|
||||||
@@ -190,6 +191,92 @@ class TestHubKlineStore(unittest.TestCase):
|
|||||||
self.assertEqual(len(got), 2)
|
self.assertEqual(len(got), 2)
|
||||||
self.assertEqual(got[-1]["open_time_ms"], base + 2 * period)
|
self.assertEqual(got[-1]["open_time_ms"], base + 2 * period)
|
||||||
|
|
||||||
|
def test_trim_contiguous_tail_drops_orphan_prefix(self):
|
||||||
|
period = TIMEFRAME_MS["15m"]
|
||||||
|
base_old = 1_700_000_000_000
|
||||||
|
base_new = base_old + period * 500
|
||||||
|
bars = []
|
||||||
|
for i in range(3):
|
||||||
|
bars.append(
|
||||||
|
{
|
||||||
|
"open_time_ms": base_old + i * period,
|
||||||
|
"open": 1,
|
||||||
|
"high": 2,
|
||||||
|
"low": 0.5,
|
||||||
|
"close": 1.5,
|
||||||
|
"volume": 1,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
for i in range(5):
|
||||||
|
bars.append(
|
||||||
|
{
|
||||||
|
"open_time_ms": base_new + i * period,
|
||||||
|
"open": 2,
|
||||||
|
"high": 3,
|
||||||
|
"low": 1.5,
|
||||||
|
"close": 2.5,
|
||||||
|
"volume": 2,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
trimmed, split = trim_contiguous_tail(bars, period)
|
||||||
|
self.assertEqual(split, 3)
|
||||||
|
self.assertEqual(len(trimmed), 5)
|
||||||
|
self.assertEqual(trimmed[0]["open_time_ms"], base_new)
|
||||||
|
|
||||||
|
def test_resolve_drops_discontinuous_orphans(self):
|
||||||
|
init_db(self.db)
|
||||||
|
period = TIMEFRAME_MS["15m"]
|
||||||
|
now = int(time.time() * 1000)
|
||||||
|
old_ms = now - period * 800
|
||||||
|
upsert_bars(
|
||||||
|
"okx",
|
||||||
|
"ONDO/USDT",
|
||||||
|
"15m",
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"open_time_ms": old_ms,
|
||||||
|
"open": 0.33,
|
||||||
|
"high": 0.34,
|
||||||
|
"low": 0.32,
|
||||||
|
"close": 0.335,
|
||||||
|
"volume": 100,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
self.db,
|
||||||
|
)
|
||||||
|
recent = []
|
||||||
|
start = now - period * 20
|
||||||
|
for i in range(20):
|
||||||
|
recent.append(
|
||||||
|
{
|
||||||
|
"open_time_ms": start + i * period,
|
||||||
|
"open": 0.35,
|
||||||
|
"high": 0.36,
|
||||||
|
"low": 0.34,
|
||||||
|
"close": 0.355,
|
||||||
|
"volume": 50,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
def remote_fetch(**kwargs):
|
||||||
|
return {"ok": True, "bars": recent, "price_tick": 0.0001}
|
||||||
|
|
||||||
|
out = resolve_chart_bars(
|
||||||
|
"okx",
|
||||||
|
"ONDO/USDT",
|
||||||
|
"15m",
|
||||||
|
remote_fetch,
|
||||||
|
db_path=self.db,
|
||||||
|
limit=50,
|
||||||
|
)
|
||||||
|
self.assertTrue(out.get("ok"))
|
||||||
|
candles = out.get("candles") or []
|
||||||
|
self.assertGreaterEqual(len(candles), 19)
|
||||||
|
if len(candles) >= 2:
|
||||||
|
for i in range(1, len(candles)):
|
||||||
|
gap = candles[i]["time"] - candles[i - 1]["time"]
|
||||||
|
self.assertLessEqual(gap, int(period / 1000 * 1.5))
|
||||||
|
|
||||||
def test_resolve_before_ms_exhausted(self):
|
def test_resolve_before_ms_exhausted(self):
|
||||||
init_db(self.db)
|
init_db(self.db)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user