`;
+
+ window.renderIndicatorList?.();
+ }
+
+ async loadStats() {
+ try {
+ const response = await fetch('/api/v1/stats?symbol=BTC');
+ this.statsData = await response.json();
+ } catch (error) {
+ console.error('Error loading stats:', error);
+ }
}
updateStats(candle) {
const price = candle.close;
- const change = ((price - candle.open) / candle.open * 100);
const isUp = candle.close >= candle.open;
if (this.currentPriceLine) {
@@ -538,11 +549,15 @@ export class TradingDashboard {
}
document.getElementById('currentPrice').textContent = price.toFixed(2);
- document.getElementById('currentPrice').className = 'stat-value ' + (change >= 0 ? 'positive' : 'negative');
- document.getElementById('priceChange').textContent = (change >= 0 ? '+' : '') + change.toFixed(2) + '%';
- document.getElementById('priceChange').className = 'stat-value ' + (change >= 0 ? 'positive' : 'negative');
- document.getElementById('dailyHigh').textContent = candle.high.toFixed(2);
- document.getElementById('dailyLow').textContent = candle.low.toFixed(2);
+
+ if (this.statsData) {
+ const change = this.statsData.change_24h;
+ document.getElementById('currentPrice').className = 'stat-value ' + (change >= 0 ? 'positive' : 'negative');
+ document.getElementById('priceChange').textContent = (change >= 0 ? '+' : '') + change.toFixed(2) + '%';
+ document.getElementById('priceChange').className = 'stat-value ' + (change >= 0 ? 'positive' : 'negative');
+ document.getElementById('dailyHigh').textContent = this.statsData.high_24h.toFixed(2);
+ document.getElementById('dailyLow').textContent = this.statsData.low_24h.toFixed(2);
+ }
}
switchTimeframe(interval) {
diff --git a/src/api/dashboard/static/js/ui/indicators-panel.js b/src/api/dashboard/static/js/ui/indicators-panel.js
index 44167d8..b246745 100644
--- a/src/api/dashboard/static/js/ui/indicators-panel.js
+++ b/src/api/dashboard/static/js/ui/indicators-panel.js
@@ -2,7 +2,40 @@ import { AVAILABLE_INDICATORS } from '../strategies/config.js';
import { IndicatorRegistry as IR } from '../indicators/index.js';
let activeIndicators = [];
-let selectedIndicatorIndex = -1;
+let configuringIndex = -1;
+
+const DEFAULT_COLORS = ['#2962ff', '#26a69a', '#ef5350', '#ff9800', '#9c27b0', '#00bcd4', '#ffeb3b', '#e91e63'];
+const LINE_TYPES = ['solid', 'dotted', 'dashed'];
+
+function getDefaultColor(index) {
+ return DEFAULT_COLORS[index % DEFAULT_COLORS.length];
+}
+
+function getPlotGroupName(plotId) {
+ if (plotId.toLowerCase().includes('fast')) return 'Fast';
+ if (plotId.toLowerCase().includes('slow')) return 'Slow';
+ if (plotId.toLowerCase().includes('upper')) return 'Upper';
+ if (plotId.toLowerCase().includes('lower')) return 'Lower';
+ if (plotId.toLowerCase().includes('middle') || plotId.toLowerCase().includes('basis')) return 'Middle';
+ if (plotId.toLowerCase().includes('signal')) return 'Signal';
+ if (plotId.toLowerCase().includes('histogram')) return 'Histogram';
+ if (plotId.toLowerCase().includes('k')) return '%K';
+ if (plotId.toLowerCase().includes('d')) return '%D';
+ return plotId;
+}
+
+function groupPlotsByColor(plots) {
+ const groups = {};
+ plots.forEach((plot, idx) => {
+ const groupName = getPlotGroupName(plot.id);
+ if (!groups[groupName]) {
+ groups[groupName] = { name: groupName, indices: [], plots: [] };
+ }
+ groups[groupName].indices.push(idx);
+ groups[groupName].plots.push(plot);
+ });
+ return Object.values(groups);
+}
export function getActiveIndicators() {
return activeIndicators;
@@ -16,109 +49,221 @@ export function renderIndicatorList() {
const container = document.getElementById('indicatorList');
if (!container) return;
- if (activeIndicators.length === 0) {
- container.innerHTML = '
';
- return;
- }
-
- container.innerHTML = activeIndicators.map((ind, idx) => `
-
-
-
${ind.name}
-
${ind.params.short || ind.params.period || ind.params.fast || 'N/A'}
+ container.innerHTML = AVAILABLE_INDICATORS.map((ind, idx) => {
+ const activeIdx = activeIndicators.findIndex(a => a.type === ind.type);
+ const isActive = activeIdx >= 0;
+ const isConfiguring = activeIdx === configuringIndex;
+
+ let colorDots = '';
+ if (isActive) {
+ const indicator = activeIndicators[activeIdx];
+ const plotGroups = groupPlotsByColor(indicator.plots || []);
+ colorDots = plotGroups.map(group => {
+ const firstIdx = group.indices[0];
+ const color = indicator.params[`_color_${firstIdx}`] || '#2962ff';
+ return `
`;
+ }).join('');
+ }
+
+ return `
+
+
+
+ ${isActive ? `` : ''}
-
-
- `).join('');
+ `;
+ }).join('');
+ updateConfigPanel();
+}
+
+function updateConfigPanel() {
const configPanel = document.getElementById('indicatorConfigPanel');
- if (selectedIndicatorIndex >= 0) {
- configPanel.style.display = 'block';
- renderIndicatorConfig(selectedIndicatorIndex);
+ const configButtons = document.getElementById('configButtons');
+ if (!configPanel) return;
+
+ configPanel.style.display = 'block';
+
+ if (configuringIndex >= 0 && configuringIndex < activeIndicators.length) {
+ renderIndicatorConfig(configuringIndex);
+ if (configButtons) configButtons.style.display = 'flex';
} else {
- configPanel.style.display = 'none';
+ const container = document.getElementById('configForm');
+ if (container) {
+ container.innerHTML = '
Select an active indicator to configure its settings
';
+ }
+ if (configButtons) configButtons.style.display = 'none';
}
}
-export function addNewIndicator() {
- const type = prompt('Enter indicator type:\n' + AVAILABLE_INDICATORS.map(i => `${i.type}: ${i.name}`).join('\n'));
- if (!type) return;
+export function toggleIndicator(type) {
+ const existingIdx = activeIndicators.findIndex(a => a.type === type);
- const indicatorDef = AVAILABLE_INDICATORS.find(i => i.type === type.toLowerCase());
- if (!indicatorDef) {
- alert('Unknown indicator type');
- return;
+ if (existingIdx >= 0) {
+ activeIndicators[existingIdx].series?.forEach(s => {
+ try { window.dashboard?.chart?.removeSeries(s); } catch(e) {}
+ });
+ activeIndicators.splice(existingIdx, 1);
+ if (configuringIndex >= activeIndicators.length) {
+ configuringIndex = -1;
+ } else if (configuringIndex === existingIdx) {
+ configuringIndex = -1;
+ } else if (configuringIndex > existingIdx) {
+ configuringIndex--;
+ }
+ } else {
+ const indicatorDef = AVAILABLE_INDICATORS.find(i => i.type === type);
+ if (!indicatorDef) return;
+
+ const IndicatorClass = IR?.[type];
+ if (!IndicatorClass) return;
+
+ const instance = new IndicatorClass({ type, params: {}, name: indicatorDef.name });
+ const metadata = instance.getMetadata();
+
+ const params = {
+ _lineType: 'solid',
+ _lineWidth: 2
+ };
+ metadata.plots.forEach((plot, idx) => {
+ params[`_color_${idx}`] = plot.color || getDefaultColor(activeIndicators.length + idx);
+ });
+ metadata.inputs.forEach(input => {
+ params[input.name] = input.default;
+ });
+
+ activeIndicators.push({
+ type: type,
+ name: metadata.name,
+ params: params,
+ plots: metadata.plots,
+ series: []
+ });
+
+ configuringIndex = activeIndicators.length - 1;
}
- const IndicatorClass = IR?.[type.toLowerCase()];
- if (!IndicatorClass) {
- alert('Indicator class not found');
- return;
- }
-
- const instance = new IndicatorClass({ type: type.toLowerCase(), params: {}, name: indicatorDef.name });
- const metadata = instance.getMetadata();
-
- const params = {};
- metadata.inputs.forEach(input => {
- params[input.name] = input.default;
- });
-
- activeIndicators.push({
- type: type.toLowerCase(),
- name: metadata.name,
- params: params,
- plots: metadata.plots,
- series: []
- });
-
- selectedIndicatorIndex = activeIndicators.length - 1;
renderIndicatorList();
drawIndicatorsOnChart();
}
-export function selectIndicator(index) {
- selectedIndicatorIndex = index;
+export function showIndicatorConfig(index) {
+ if (configuringIndex === index) {
+ configuringIndex = -1;
+ } else {
+ configuringIndex = index;
+ }
renderIndicatorList();
}
+export function showIndicatorConfigByType(type) {
+ const idx = activeIndicators.findIndex(a => a.type === type);
+ if (idx >= 0) {
+ configuringIndex = idx;
+ renderIndicatorList();
+ }
+}
+
export function renderIndicatorConfig(index) {
const container = document.getElementById('configForm');
+ if (!container) return;
+
const indicator = activeIndicators[index];
- if (!indicator || !indicator.plots) {
+ if (!indicator) {
container.innerHTML = '';
return;
}
const IndicatorClass = IR?.[indicator.type];
- if (!IndicatorClass) return;
+ if (!IndicatorClass) {
+ container.innerHTML = '
Error loading indicator
';
+ return;
+ }
const instance = new IndicatorClass({ type: indicator.type, params: indicator.params, name: indicator.name });
const meta = instance.getMetadata();
- container.innerHTML = meta.inputs.map(input => `
+ const plotGroups = groupPlotsByColor(meta.plots);
+
+ const colorInputs = plotGroups.map(group => {
+ const firstIdx = group.indices[0];
+ const color = indicator.params[`_color_${firstIdx}`] || meta.plots[firstIdx].color || '#2962ff';
+ return `
+
+
+
+
+ `;
+ }).join('');
+
+ container.innerHTML = `
+
${indicator.name}
+
+ ${colorInputs}
+
-
- ${input.type === 'select' ?
- `` :
- ``
- }
+
+
- `).join('');
+
+
+
+
+
+
+ ${meta.inputs.map(input => `
+
+
+ ${input.type === 'select' ?
+ `` :
+ ``
+ }
+
+ `).join('')}
+ `;
}
export function applyIndicatorConfig() {
- if (selectedIndicatorIndex < 0) return;
+ if (configuringIndex < 0 || configuringIndex >= activeIndicators.length) return;
- const indicator = activeIndicators[selectedIndicatorIndex];
+ const indicator = activeIndicators[configuringIndex];
const IndicatorClass = IR?.[indicator.type];
if (!IndicatorClass) return;
const instance = new IndicatorClass({ type: indicator.type, params: {}, name: indicator.name });
const meta = instance.getMetadata();
+ const plotGroups = groupPlotsByColor(meta.plots);
+ plotGroups.forEach(group => {
+ const firstIdx = group.indices[0];
+ const colorEl = document.getElementById(`config__color_${firstIdx}`);
+ if (colorEl) {
+ const color = colorEl.value;
+ group.indices.forEach(idx => {
+ indicator.params[`_color_${idx}`] = color;
+ });
+ }
+ });
+
+ const lineTypeEl = document.getElementById('config__lineType');
+ const lineWidthEl = document.getElementById('config__lineWidth');
+
+ if (lineTypeEl) indicator.params._lineType = lineTypeEl.value;
+ if (lineWidthEl) indicator.params._lineWidth = parseInt(lineWidthEl.value);
+
meta.inputs.forEach(input => {
const el = document.getElementById(`config_${input.name}`);
if (el) {
@@ -131,18 +276,33 @@ export function applyIndicatorConfig() {
}
export function removeIndicator() {
- if (selectedIndicatorIndex < 0) return;
- activeIndicators.splice(selectedIndicatorIndex, 1);
- selectedIndicatorIndex = -1;
+ if (configuringIndex < 0 || configuringIndex >= activeIndicators.length) return;
+
+ activeIndicators[configuringIndex].series?.forEach(s => {
+ try { window.dashboard?.chart?.removeSeries(s); } catch(e) {}
+ });
+
+ activeIndicators.splice(configuringIndex, 1);
+ configuringIndex = -1;
+
renderIndicatorList();
drawIndicatorsOnChart();
}
export function removeIndicatorByIndex(index) {
+ if (index < 0 || index >= activeIndicators.length) return;
+
+ activeIndicators[index].series?.forEach(s => {
+ try { window.dashboard?.chart?.removeSeries(s); } catch(e) {}
+ });
+
activeIndicators.splice(index, 1);
- if (selectedIndicatorIndex >= activeIndicators.length) {
- selectedIndicatorIndex = activeIndicators.length - 1;
+ if (configuringIndex === index) {
+ configuringIndex = -1;
+ } else if (configuringIndex > index) {
+ configuringIndex--;
}
+
renderIndicatorList();
drawIndicatorsOnChart();
}
@@ -159,6 +319,8 @@ export function drawIndicatorsOnChart() {
const candles = window.dashboard.allData.get(window.dashboard.currentInterval);
if (!candles || candles.length === 0) return;
+ const lineStyleMap = { 'solid': 0, 'dotted': 1, 'dashed': 2 };
+
activeIndicators.forEach((indicator, idx) => {
const IndicatorClass = IR?.[indicator.type];
if (!IndicatorClass) return;
@@ -173,30 +335,53 @@ export function drawIndicatorsOnChart() {
const meta = instance.getMetadata();
indicator.series = [];
- meta.plots.forEach(plot => {
- if (results[0] && typeof results[0][plot.id] === 'undefined') return;
+ const lineStyle = lineStyleMap[indicator.params._lineType] || 0;
+ const lineWidth = indicator.params._lineWidth || 2;
+
+ const firstNonNull = results?.find(r => r !== null && r !== undefined);
+ const isObjectResult = firstNonNull && typeof firstNonNull === 'object';
+
+ meta.plots.forEach((plot, plotIdx) => {
+ if (isObjectResult && typeof firstNonNull[plot.id] === 'undefined') return;
+
+ const plotColor = indicator.params[`_color_${plotIdx}`] || plot.color || '#2962ff';
const lineSeries = window.dashboard.chart.addLineSeries({
- color: plot.color || '#2962ff',
- lineWidth: plot.width || 1,
+ color: plotColor,
+ lineWidth: plot.width || lineWidth,
+ lineStyle: lineStyle,
title: plot.title,
priceLineVisible: false,
lastValueVisible: true
});
- const data = candles.map((c, i) => ({
- time: c.time,
- value: results[i]?.[plot.id] ?? null
- })).filter(d => d.value !== null && d.value !== undefined);
+ const data = [];
+ for (let i = 0; i < candles.length; i++) {
+ let value;
+ if (isObjectResult) {
+ value = results[i]?.[plot.id];
+ } else {
+ value = results[i];
+ }
+
+ if (value !== null && value !== undefined) {
+ data.push({
+ time: candles[i].time,
+ value: value
+ });
+ }
+ }
- lineSeries.setData(data);
- indicator.series.push(lineSeries);
+ if (data.length > 0) {
+ lineSeries.setData(data);
+ indicator.series.push(lineSeries);
+ }
});
});
}
-window.addNewIndicator = addNewIndicator;
-window.selectIndicator = selectIndicator;
+window.toggleIndicator = toggleIndicator;
+window.showIndicatorConfig = showIndicatorConfig;
window.applyIndicatorConfig = applyIndicatorConfig;
window.removeIndicator = removeIndicator;
window.removeIndicatorByIndex = removeIndicatorByIndex;
diff --git a/src/data_collector/backfill_gap.py b/src/data_collector/backfill_gap.py
new file mode 100644
index 0000000..4ab0f61
--- /dev/null
+++ b/src/data_collector/backfill_gap.py
@@ -0,0 +1,154 @@
+"""
+One-time backfill script to fill gaps in data.
+Run with: python -m data_collector.backfill_gap --start "2024-01-01 09:34" --end "2024-01-01 19:39"
+"""
+
+import asyncio
+import logging
+import sys
+import os
+from datetime import datetime, timezone
+from typing import Optional
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+from .database import DatabaseManager
+from .backfill import HyperliquidBackfill
+
+logging.basicConfig(
+ level=logging.INFO,
+ format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+INTERVALS = ["1m", "3m", "5m", "15m", "30m", "1h", "2h", "4h", "8h", "12h", "1d", "3d", "1w"]
+
+
+async def backfill_gap(
+ start_time: datetime,
+ end_time: datetime,
+ symbol: str = "BTC",
+ intervals: Optional[list] = None
+) -> dict:
+ """
+ Backfill a specific time gap for all intervals.
+
+ Args:
+ start_time: Gap start time (UTC)
+ end_time: Gap end time (UTC)
+ symbol: Trading symbol
+ intervals: List of intervals to backfill (default: all standard)
+
+ Returns:
+ Dictionary with interval -> count mapping
+ """
+ intervals = intervals or INTERVALS
+ results = {}
+
+ db = DatabaseManager()
+ await db.connect()
+
+ logger.info(f"Backfilling gap: {start_time} to {end_time} for {symbol}")
+
+ try:
+ async with HyperliquidBackfill(db, symbol, intervals) as backfill:
+ for interval in intervals:
+ try:
+ logger.info(f"Backfilling {interval}...")
+ candles = await backfill.fetch_candles(interval, start_time, end_time)
+
+ if candles:
+ inserted = await db.insert_candles(candles)
+ results[interval] = inserted
+ logger.info(f" {interval}: {inserted} candles inserted")
+ else:
+ results[interval] = 0
+ logger.warning(f" {interval}: No candles returned")
+
+ await asyncio.sleep(0.3)
+
+ except Exception as e:
+ logger.error(f" {interval}: Error - {e}")
+ results[interval] = 0
+
+ finally:
+ await db.disconnect()
+
+ logger.info(f"Backfill complete. Total: {sum(results.values())} candles")
+ return results
+
+
+async def auto_detect_and_fill_gaps(symbol: str = "BTC") -> dict:
+ """
+ Detect and fill all gaps in the database for all intervals.
+ """
+ db = DatabaseManager()
+ await db.connect()
+
+ results = {}
+
+ try:
+ async with HyperliquidBackfill(db, symbol, INTERVALS) as backfill:
+ for interval in INTERVALS:
+ try:
+ # Detect gaps
+ gaps = await db.detect_gaps(symbol, interval)
+
+ if not gaps:
+ logger.info(f"{interval}: No gaps detected")
+ results[interval] = 0
+ continue
+
+ logger.info(f"{interval}: {len(gaps)} gaps detected")
+ total_filled = 0
+
+ for gap in gaps:
+ gap_start = datetime.fromisoformat(gap['gap_start'].replace('Z', '+00:00'))
+ gap_end = datetime.fromisoformat(gap['gap_end'].replace('Z', '+00:00'))
+
+ logger.info(f" Filling gap: {gap_start} to {gap_end}")
+
+ candles = await backfill.fetch_candles(interval, gap_start, gap_end)
+
+ if candles:
+ inserted = await db.insert_candles(candles)
+ total_filled += inserted
+ logger.info(f" Filled {inserted} candles")
+
+ await asyncio.sleep(0.2)
+
+ results[interval] = total_filled
+
+ except Exception as e:
+ logger.error(f"{interval}: Error - {e}")
+ results[interval] = 0
+
+ finally:
+ await db.disconnect()
+
+ return results
+
+
+async def main():
+ import argparse
+
+ parser = argparse.ArgumentParser(description="Backfill gaps in BTC data")
+ parser.add_argument("--start", help="Start time (YYYY-MM-DD HH:MM)", default=None)
+ parser.add_argument("--end", help="End time (YYYY-MM-DD HH:MM)", default=None)
+ parser.add_argument("--auto", action="store_true", help="Auto-detect and fill all gaps")
+ parser.add_argument("--symbol", default="BTC", help="Symbol to backfill")
+
+ args = parser.parse_args()
+
+ if args.auto:
+ await auto_detect_and_fill_gaps(args.symbol)
+ elif args.start and args.end:
+ start_time = datetime.strptime(args.start, "%Y-%m-%d %H:%M").replace(tzinfo=timezone.utc)
+ end_time = datetime.strptime(args.end, "%Y-%m-%d %H:%M").replace(tzinfo=timezone.utc)
+ await backfill_gap(start_time, end_time, args.symbol)
+ else:
+ parser.print_help()
+
+
+if __name__ == "__main__":
+ asyncio.run(main())
diff --git a/src/data_collector/custom_timeframe_generator.py b/src/data_collector/custom_timeframe_generator.py
index 55b690f..ce1aad4 100644
--- a/src/data_collector/custom_timeframe_generator.py
+++ b/src/data_collector/custom_timeframe_generator.py
@@ -269,7 +269,6 @@ class CustomTimeframeGenerator:
logger.info(f"Generating historical {interval} from {source_interval}...")
- # Get date range available in source data
async with self.db.acquire() as conn:
min_max = await conn.fetchrow("""
SELECT MIN(time), MAX(time) FROM candles
@@ -287,7 +286,6 @@ class CustomTimeframeGenerator:
await self.aggregate_and_upsert('BTC', interval, curr)
total_inserted += 1
- # Advance curr
if interval == '1M':
_, days = calendar.monthrange(curr.year, curr.month)
curr += timedelta(days=days)
@@ -297,7 +295,7 @@ class CustomTimeframeGenerator:
elif cfg['type'] == 'hour': curr += timedelta(hours=cfg['value'])
elif cfg['type'] == 'day': curr += timedelta(days=cfg['value'])
elif cfg['type'] == 'week': curr += timedelta(weeks=1)
- else: # Custom
+ else:
minutes = self.CUSTOM_INTERVALS[interval]['minutes']
curr += timedelta(minutes=minutes)
@@ -307,6 +305,87 @@ class CustomTimeframeGenerator:
return total_inserted
+ async def generate_from_gap(self, interval: str) -> int:
+ """
+ Generate candles only from where they're missing.
+ Compares source interval max time with target interval max time.
+ """
+ if not self.first_1m_time:
+ await self.initialize()
+
+ if not self.first_1m_time:
+ return 0
+
+ config = self.CUSTOM_INTERVALS.get(interval) or {'source': '1m'}
+ source_interval = config.get('source', '1m')
+
+ async with self.db.acquire() as conn:
+ # Get source range
+ source_min_max = await conn.fetchrow("""
+ SELECT MIN(time), MAX(time) FROM candles
+ WHERE symbol = 'BTC' AND interval = $1
+ """, source_interval)
+
+ if not source_min_max or not source_min_max[1]:
+ return 0
+
+ # Get target (this interval) max time
+ target_max = await conn.fetchval("""
+ SELECT MAX(time) FROM candles
+ WHERE symbol = 'BTC' AND interval = $1
+ """, interval)
+
+ source_max = source_min_max[1]
+
+ if target_max:
+ # Start from next bucket after target_max
+ curr = self.get_bucket_start(target_max, interval)
+ if interval in self.CUSTOM_INTERVALS:
+ minutes = self.CUSTOM_INTERVALS[interval]['minutes']
+ curr = curr + timedelta(minutes=minutes)
+ elif interval in self.STANDARD_INTERVALS:
+ cfg = self.STANDARD_INTERVALS[interval]
+ if cfg['type'] == 'min': curr = curr + timedelta(minutes=cfg['value'])
+ elif cfg['type'] == 'hour': curr = curr + timedelta(hours=cfg['value'])
+ elif cfg['type'] == 'day': curr = curr + timedelta(days=cfg['value'])
+ elif cfg['type'] == 'week': curr = curr + timedelta(weeks=1)
+ else:
+ # No target data, start from source min
+ curr = self.get_bucket_start(source_min_max[0], interval)
+
+ end = source_max
+
+ if curr > end:
+ logger.info(f"{interval}: Already up to date (target: {target_max}, source: {source_max})")
+ return 0
+
+ logger.info(f"Generating {interval} from {curr} to {end}...")
+
+ total_inserted = 0
+ while curr <= end:
+ await self.aggregate_and_upsert('BTC', interval, curr)
+ total_inserted += 1
+
+ if interval == '1M':
+ _, days = calendar.monthrange(curr.year, curr.month)
+ curr += timedelta(days=days)
+ elif interval in self.STANDARD_INTERVALS:
+ cfg = self.STANDARD_INTERVALS[interval]
+ if cfg['type'] == 'min': curr += timedelta(minutes=cfg['value'])
+ elif cfg['type'] == 'hour': curr += timedelta(hours=cfg['value'])
+ elif cfg['type'] == 'day': curr += timedelta(days=cfg['value'])
+ elif cfg['type'] == 'week': curr += timedelta(weeks=1)
+ else:
+ minutes = self.CUSTOM_INTERVALS[interval]['minutes']
+ curr += timedelta(minutes=minutes)
+
+ if total_inserted % 50 == 0:
+ logger.info(f"Generated {total_inserted} {interval} candles...")
+ await asyncio.sleep(0.01)
+
+ logger.info(f"{interval}: Generated {total_inserted} candles")
+ return total_inserted
+
async def verify_integrity(self, interval: str) -> Dict:
async with self.db.acquire() as conn:
stats = await conn.fetchrow("""
diff --git a/src/data_collector/database.py b/src/data_collector/database.py
index 9f704fe..57d7fc8 100644
--- a/src/data_collector/database.py
+++ b/src/data_collector/database.py
@@ -3,6 +3,7 @@ Database interface for TimescaleDB
Optimized for batch inserts and low resource usage
"""
+import asyncio
import logging
from contextlib import asynccontextmanager
from datetime import datetime
diff --git a/src/data_collector/main.py b/src/data_collector/main.py
index 5bf6410..e11a864 100644
--- a/src/data_collector/main.py
+++ b/src/data_collector/main.py
@@ -41,6 +41,8 @@ class DataCollector:
Manages WebSocket connection, buffering, and database writes
"""
+ STANDARD_INTERVALS = ["1m", "3m", "5m", "15m", "30m", "1h", "2h", "4h", "8h", "12h", "1d", "3d", "1w"]
+
def __init__(
self,
symbol: str = "BTC",
@@ -69,10 +71,16 @@ class DataCollector:
self.db = DatabaseManager()
await self.db.connect()
+ # Run startup backfill for all intervals
+ await self._startup_backfill()
+
# Initialize custom timeframe generator
self.custom_tf_generator = CustomTimeframeGenerator(self.db)
await self.custom_tf_generator.initialize()
+ # Regenerate custom timeframes after startup backfill
+ await self._regenerate_custom_timeframes()
+
# Initialize indicator engine
# Hardcoded config for now, eventually load from yaml
indicator_configs = [
@@ -126,6 +134,99 @@ class DataCollector:
finally:
await self.stop()
+ async def _startup_backfill(self) -> None:
+ """
+ Backfill missing data on startup for all standard intervals.
+ Uses both gap detection AND time-based backfill for robustness.
+ """
+ logger.info("Running startup backfill for all intervals...")
+
+ try:
+ async with HyperliquidBackfill(self.db, self.symbol, self.STANDARD_INTERVALS) as backfill:
+ for interval in self.STANDARD_INTERVALS:
+ try:
+ # First, use gap detection to find any holes
+ gaps = await self.db.detect_gaps(self.symbol, interval)
+
+ if gaps:
+ logger.info(f"{interval}: {len(gaps)} gaps detected")
+ for gap in gaps:
+ gap_start = datetime.fromisoformat(gap['gap_start'].replace('Z', '+00:00'))
+ gap_end = datetime.fromisoformat(gap['gap_end'].replace('Z', '+00:00'))
+
+ logger.info(f" Filling gap: {gap_start} to {gap_end}")
+ candles = await backfill.fetch_candles(interval, gap_start, gap_end)
+
+ if candles:
+ inserted = await self.db.insert_candles(candles)
+ logger.info(f" Inserted {inserted} candles for gap")
+
+ await asyncio.sleep(0.2)
+
+ # Second, check if we're behind current time
+ latest = await self.db.get_latest_candle(self.symbol, interval)
+ now = datetime.now(timezone.utc)
+
+ if latest:
+ last_time = latest['time']
+ gap_minutes = (now - last_time).total_seconds() / 60
+
+ if gap_minutes > 2:
+ logger.info(f"{interval}: {gap_minutes:.0f} min behind, backfilling to now...")
+ candles = await backfill.fetch_candles(interval, last_time, now)
+
+ if candles:
+ inserted = await self.db.insert_candles(candles)
+ logger.info(f" Inserted {inserted} candles")
+ else:
+ logger.info(f"{interval}: up to date")
+ else:
+ # No data exists, backfill last 7 days
+ logger.info(f"{interval}: No data, backfilling 7 days...")
+ count = await backfill.backfill_interval(interval, days_back=7)
+ logger.info(f" Inserted {count} candles")
+
+ await asyncio.sleep(0.2)
+
+ except Exception as e:
+ logger.error(f"Startup backfill failed for {interval}: {e}")
+ import traceback
+ logger.error(traceback.format_exc())
+ continue
+
+ except Exception as e:
+ logger.error(f"Startup backfill error: {e}")
+ import traceback
+ logger.error(traceback.format_exc())
+
+ logger.info("Startup backfill complete")
+
+ async def _regenerate_custom_timeframes(self) -> None:
+ """
+ Regenerate custom timeframes (37m, 148m) only from gaps.
+ Only generates candles that are missing, not all from beginning.
+ """
+ if not self.custom_tf_generator:
+ return
+
+ logger.info("Checking custom timeframes for gaps...")
+
+ try:
+ for interval in ['37m', '148m']:
+ try:
+ count = await self.custom_tf_generator.generate_from_gap(interval)
+ if count > 0:
+ logger.info(f"{interval}: Generated {count} candles")
+ else:
+ logger.info(f"{interval}: Up to date")
+ except Exception as e:
+ logger.error(f"Failed to regenerate {interval}: {e}")
+
+ except Exception as e:
+ logger.error(f"Custom timeframe regeneration error: {e}")
+
+ logger.info("Custom timeframe check complete")
+
async def stop(self) -> None:
"""Graceful shutdown"""
if not self.is_running: