Improve indicators panel and fix data collector startup

Dashboard:
- Add indicator checkboxes with multi-select support
- Show color dots for each plot group in indicator list
- Config panel always visible with helpful empty state message
- Support multi-color indicators (HTS shows Fast/Slow colors)
- Add line type selector (solid/dotted/dashed) and line width
- Combine SMA/EMA into unified MA indicator with type selector

Data Collector:
- Fix missing asyncio import in database.py (caused NameError on timeout)
- Add startup backfill for all standard intervals on collector restart
- Add gap detection + time-based backfill for robustness
- Add backfill_gap.py tool for manual gap filling
- Optimize custom timeframe generation to only generate missing candles (not all from beginning)

Files changed:
- src/api/dashboard/static/js/ui/indicators-panel.js
- src/api/dashboard/static/js/indicators/index.js, ma_indicator.js
- src/api/dashboard/static/js/strategies/config.js
- src/api/dashboard/static/js/ui/chart.js
- src/api/dashboard/static/js/app.js
- src/api/dashboard/static/index.html
- src/data_collector/database.py
- src/data_collector/main.py
- src/data_collector/custom_timeframe_generator.py
- src/data_collector/backfill_gap.py (new)
This commit is contained in:
BTC Bot
2026-02-24 10:21:41 +01:00
parent 09ec51c185
commit 610911bca0
11 changed files with 733 additions and 109 deletions

View File

@ -767,15 +767,84 @@
.ta-ma-change.positive { color: var(--tv-green); } .ta-ma-change.positive { color: var(--tv-green); }
.ta-ma-change.negative { color: var(--tv-red); } .ta-ma-change.negative { color: var(--tv-red); }
.indicator-item.selected { .indicator-list {
border-color: var(--tv-blue) !important; display: flex;
background: rgba(41, 98, 255, 0.1) !important; flex-direction: column;
gap: 4px;
margin-top: 8px;
} }
.indicator-item:hover { .indicator-checkbox-item {
display: flex;
align-items: center;
gap: 8px;
padding: 6px 8px;
border-radius: 4px;
cursor: pointer;
transition: background 0.2s;
border: 1px solid transparent;
}
.indicator-checkbox-item:hover {
background: var(--tv-hover); background: var(--tv-hover);
} }
.indicator-checkbox-item.configuring {
background: rgba(41, 98, 255, 0.1);
border-color: var(--tv-blue);
}
.indicator-checkbox {
width: 14px;
height: 14px;
accent-color: var(--tv-blue);
cursor: pointer;
}
.indicator-checkbox-item label {
flex: 1;
font-size: 12px;
cursor: pointer;
color: var(--tv-text);
}
.indicator-config-btn {
background: transparent;
border: 1px solid var(--tv-border);
color: var(--tv-text-secondary);
width: 22px;
height: 22px;
border-radius: 4px;
cursor: pointer;
font-size: 11px;
display: flex;
align-items: center;
justify-content: center;
transition: all 0.2s;
}
.indicator-config-btn:hover {
background: var(--tv-hover);
border-color: var(--tv-blue);
color: var(--tv-blue);
}
.indicator-config-btn.active {
background: var(--tv-blue);
border-color: var(--tv-blue);
color: white;
}
.indicator-color-dot {
display: inline-block;
width: 10px;
height: 10px;
border-radius: 50%;
margin-left: 8px;
vertical-align: middle;
border: 1px solid rgba(255,255,255,0.2);
}
.ta-level { .ta-level {
display: flex; display: flex;
justify-content: space-between; justify-content: space-between;

View File

@ -22,8 +22,8 @@ import {
} from './ui/strategies-panel.js'; } from './ui/strategies-panel.js';
import { import {
renderIndicatorList, renderIndicatorList,
addNewIndicator, toggleIndicator,
selectIndicator, showIndicatorConfig,
applyIndicatorConfig, applyIndicatorConfig,
removeIndicator, removeIndicator,
removeIndicatorByIndex, removeIndicatorByIndex,
@ -64,6 +64,9 @@ window.loadSavedSimulation = loadSavedSimulation;
window.deleteSavedSimulation = deleteSavedSimulation; window.deleteSavedSimulation = deleteSavedSimulation;
window.clearSimulationResults = clearSimulationResults; window.clearSimulationResults = clearSimulationResults;
window.updateTimeframeDisplay = updateTimeframeDisplay; window.updateTimeframeDisplay = updateTimeframeDisplay;
window.renderIndicatorList = renderIndicatorList;
window.toggleIndicator = toggleIndicator;
window.showIndicatorConfig = showIndicatorConfig;
window.StrategyParams = StrategyParams; window.StrategyParams = StrategyParams;
window.SimulationStorage = SimulationStorage; window.SimulationStorage = SimulationStorage;

View File

@ -1,8 +1,7 @@
export { MA } from './ma.js'; export { MA } from './ma.js';
export { BaseIndicator } from './base.js'; export { BaseIndicator } from './base.js';
export { HTSIndicator } from './hts.js'; export { HTSIndicator } from './hts.js';
export { SMAIndicator } from './sma.js'; export { MAIndicator } from './ma_indicator.js';
export { EMAIndicator } from './ema.js';
export { RSIIndicator } from './rsi.js'; export { RSIIndicator } from './rsi.js';
export { BollingerBandsIndicator } from './bb.js'; export { BollingerBandsIndicator } from './bb.js';
export { MACDIndicator } from './macd.js'; export { MACDIndicator } from './macd.js';
@ -10,8 +9,7 @@ export { StochasticIndicator } from './stoch.js';
export { ATRIndicator } from './atr.js'; export { ATRIndicator } from './atr.js';
import { HTSIndicator } from './hts.js'; import { HTSIndicator } from './hts.js';
import { SMAIndicator } from './sma.js'; import { MAIndicator } from './ma_indicator.js';
import { EMAIndicator } from './ema.js';
import { RSIIndicator } from './rsi.js'; import { RSIIndicator } from './rsi.js';
import { BollingerBandsIndicator } from './bb.js'; import { BollingerBandsIndicator } from './bb.js';
import { MACDIndicator } from './macd.js'; import { MACDIndicator } from './macd.js';
@ -20,8 +18,7 @@ import { ATRIndicator } from './atr.js';
export const IndicatorRegistry = { export const IndicatorRegistry = {
hts: HTSIndicator, hts: HTSIndicator,
sma: SMAIndicator, ma: MAIndicator,
ema: EMAIndicator,
rsi: RSIIndicator, rsi: RSIIndicator,
bb: BollingerBandsIndicator, bb: BollingerBandsIndicator,
macd: MACDIndicator, macd: MACDIndicator,

View File

@ -0,0 +1,21 @@
import { MA } from './ma.js';
import { BaseIndicator } from './base.js';
export class MAIndicator extends BaseIndicator {
calculate(candles) {
const period = this.params.period || 44;
const maType = this.params.maType || 'SMA';
return MA.get(maType, candles, period, 'close');
}
getMetadata() {
return {
name: 'MA',
inputs: [
{ name: 'period', label: 'Period', type: 'number', default: 44, min: 1, max: 500 },
{ name: 'maType', label: 'MA Type', type: 'select', options: ['SMA', 'EMA', 'RMA', 'WMA', 'VWMA'], default: 'SMA' }
],
plots: [{ id: 'value', color: '#2962ff', title: 'MA' }]
};
}
}

View File

@ -6,8 +6,7 @@ export const StrategyParams = {
export const AVAILABLE_INDICATORS = [ export const AVAILABLE_INDICATORS = [
{ type: 'hts', name: 'HTS Trend System', description: 'Fast/Slow MAs of High/Low prices' }, { type: 'hts', name: 'HTS Trend System', description: 'Fast/Slow MAs of High/Low prices' },
{ type: 'sma', name: 'SMA', description: 'Simple Moving Average' }, { type: 'ma', name: 'MA', description: 'Moving Average (SMA/EMA/RMA/WMA/VWMA)' },
{ type: 'ema', name: 'EMA', description: 'Exponential Moving Average' },
{ type: 'rsi', name: 'RSI', description: 'Relative Strength Index' }, { type: 'rsi', name: 'RSI', description: 'Relative Strength Index' },
{ type: 'bb', name: 'Bollinger Bands', description: 'Volatility bands' }, { type: 'bb', name: 'Bollinger Bands', description: 'Volatility bands' },
{ type: 'macd', name: 'MACD', description: 'Moving Average Convergence Divergence' }, { type: 'macd', name: 'MACD', description: 'Moving Average Convergence Divergence' },

View File

@ -22,6 +22,7 @@ export class TradingDashboard {
setInterval(() => { setInterval(() => {
this.loadNewData(); this.loadNewData();
this.loadStats();
if (new Date().getSeconds() < 15) this.loadTA(); if (new Date().getSeconds() < 15) this.loadTA();
}, 10000); }, 10000);
} }
@ -281,7 +282,10 @@ export class TradingDashboard {
} }
async loadInitialData() { async loadInitialData() {
await this.loadData(1000, true); await Promise.all([
this.loadData(1000, true),
this.loadStats()
]);
this.hasInitialLoad = true; this.hasInitialLoad = true;
} }
@ -508,26 +512,33 @@ export class TradingDashboard {
<div class="ta-section"> <div class="ta-section">
<div class="ta-section-title">Indicators</div> <div class="ta-section-title">Indicators</div>
<div id="indicatorList" style="display: flex; flex-direction: column; gap: 6px; margin-top: 8px;"></div> <div id="indicatorList" class="indicator-list"></div>
<button class="ta-btn" onclick="addNewIndicator()" style="width: 100%; margin-top: 12px; font-size: 11px;">
+ Add Indicator
</button>
</div> </div>
<div class="ta-section" id="indicatorConfigPanel"> <div class="ta-section" id="indicatorConfigPanel">
<div class="ta-section-title">Configuration</div> <div class="ta-section-title">Configuration</div>
<div id="configForm" style="margin-top: 8px;"></div> <div id="configForm" style="margin-top: 8px;"></div>
<div style="display: flex; gap: 8px; margin-top: 12px;"> <div style="display: flex; gap: 8px; margin-top: 12px;" id="configButtons">
<button class="ta-btn" onclick="applyIndicatorConfig()" style="flex: 1; font-size: 11px; background: var(--tv-blue); color: white; border: none;">Apply</button> <button class="ta-btn" onclick="applyIndicatorConfig()" style="flex: 1; font-size: 11px; background: var(--tv-blue); color: white; border: none;">Apply</button>
<button class="ta-btn" onclick="removeIndicator()" style="flex: 1; font-size: 11px; border-color: var(--tv-red); color: var(--tv-red);">Remove</button> <button class="ta-btn" onclick="removeIndicator()" style="flex: 1; font-size: 11px; border-color: var(--tv-red); color: var(--tv-red);">Remove</button>
</div> </div>
</div> </div>
`; `;
window.renderIndicatorList?.();
}
async loadStats() {
try {
const response = await fetch('/api/v1/stats?symbol=BTC');
this.statsData = await response.json();
} catch (error) {
console.error('Error loading stats:', error);
}
} }
updateStats(candle) { updateStats(candle) {
const price = candle.close; const price = candle.close;
const change = ((price - candle.open) / candle.open * 100);
const isUp = candle.close >= candle.open; const isUp = candle.close >= candle.open;
if (this.currentPriceLine) { if (this.currentPriceLine) {
@ -538,11 +549,15 @@ export class TradingDashboard {
} }
document.getElementById('currentPrice').textContent = price.toFixed(2); document.getElementById('currentPrice').textContent = price.toFixed(2);
document.getElementById('currentPrice').className = 'stat-value ' + (change >= 0 ? 'positive' : 'negative');
document.getElementById('priceChange').textContent = (change >= 0 ? '+' : '') + change.toFixed(2) + '%'; if (this.statsData) {
document.getElementById('priceChange').className = 'stat-value ' + (change >= 0 ? 'positive' : 'negative'); const change = this.statsData.change_24h;
document.getElementById('dailyHigh').textContent = candle.high.toFixed(2); document.getElementById('currentPrice').className = 'stat-value ' + (change >= 0 ? 'positive' : 'negative');
document.getElementById('dailyLow').textContent = candle.low.toFixed(2); document.getElementById('priceChange').textContent = (change >= 0 ? '+' : '') + change.toFixed(2) + '%';
document.getElementById('priceChange').className = 'stat-value ' + (change >= 0 ? 'positive' : 'negative');
document.getElementById('dailyHigh').textContent = this.statsData.high_24h.toFixed(2);
document.getElementById('dailyLow').textContent = this.statsData.low_24h.toFixed(2);
}
} }
switchTimeframe(interval) { switchTimeframe(interval) {

View File

@ -2,7 +2,40 @@ import { AVAILABLE_INDICATORS } from '../strategies/config.js';
import { IndicatorRegistry as IR } from '../indicators/index.js'; import { IndicatorRegistry as IR } from '../indicators/index.js';
let activeIndicators = []; let activeIndicators = [];
let selectedIndicatorIndex = -1; let configuringIndex = -1;
const DEFAULT_COLORS = ['#2962ff', '#26a69a', '#ef5350', '#ff9800', '#9c27b0', '#00bcd4', '#ffeb3b', '#e91e63'];
const LINE_TYPES = ['solid', 'dotted', 'dashed'];
function getDefaultColor(index) {
return DEFAULT_COLORS[index % DEFAULT_COLORS.length];
}
function getPlotGroupName(plotId) {
if (plotId.toLowerCase().includes('fast')) return 'Fast';
if (plotId.toLowerCase().includes('slow')) return 'Slow';
if (plotId.toLowerCase().includes('upper')) return 'Upper';
if (plotId.toLowerCase().includes('lower')) return 'Lower';
if (plotId.toLowerCase().includes('middle') || plotId.toLowerCase().includes('basis')) return 'Middle';
if (plotId.toLowerCase().includes('signal')) return 'Signal';
if (plotId.toLowerCase().includes('histogram')) return 'Histogram';
if (plotId.toLowerCase().includes('k')) return '%K';
if (plotId.toLowerCase().includes('d')) return '%D';
return plotId;
}
function groupPlotsByColor(plots) {
const groups = {};
plots.forEach((plot, idx) => {
const groupName = getPlotGroupName(plot.id);
if (!groups[groupName]) {
groups[groupName] = { name: groupName, indices: [], plots: [] };
}
groups[groupName].indices.push(idx);
groups[groupName].plots.push(plot);
});
return Object.values(groups);
}
export function getActiveIndicators() { export function getActiveIndicators() {
return activeIndicators; return activeIndicators;
@ -16,109 +49,221 @@ export function renderIndicatorList() {
const container = document.getElementById('indicatorList'); const container = document.getElementById('indicatorList');
if (!container) return; if (!container) return;
if (activeIndicators.length === 0) { container.innerHTML = AVAILABLE_INDICATORS.map((ind, idx) => {
container.innerHTML = '<div style="text-align: center; color: var(--tv-text-secondary); padding: 10px; font-size: 12px;">No indicators added</div>'; const activeIdx = activeIndicators.findIndex(a => a.type === ind.type);
return; const isActive = activeIdx >= 0;
} const isConfiguring = activeIdx === configuringIndex;
container.innerHTML = activeIndicators.map((ind, idx) => ` let colorDots = '';
<div class="indicator-item ${idx === selectedIndicatorIndex ? 'selected' : ''}" if (isActive) {
onclick="selectIndicator(${idx})" const indicator = activeIndicators[activeIdx];
style="display: flex; align-items: center; justify-content: space-between; padding: 8px; background: var(--tv-bg); border-radius: 4px; cursor: pointer; border: 1px solid ${idx === selectedIndicatorIndex ? 'var(--tv-blue)' : 'var(--tv-border)'}"> const plotGroups = groupPlotsByColor(indicator.plots || []);
<div> colorDots = plotGroups.map(group => {
<div style="font-size: 12px; font-weight: 600;">${ind.name}</div> const firstIdx = group.indices[0];
<div style="font-size: 10px; color: var(--tv-text-secondary);">${ind.params.short || ind.params.period || ind.params.fast || 'N/A'}</div> const color = indicator.params[`_color_${firstIdx}`] || '#2962ff';
return `<span class="indicator-color-dot" style="background: ${color};"></span>`;
}).join('');
}
return `
<div class="indicator-checkbox-item ${isConfiguring ? 'configuring' : ''}"
onclick="toggleIndicator('${ind.type}')"
title="${ind.description}">
<input type="checkbox"
id="ind_${ind.type}"
${isActive ? 'checked' : ''}
onclick="event.stopPropagation(); toggleIndicator('${ind.type}')"
class="indicator-checkbox">
<label for="ind_${ind.type}" onclick="event.stopPropagation(); toggleIndicator('${ind.type}')" style="flex: 1;">
${ind.name}
${colorDots}
</label>
${isActive ? `<button class="indicator-config-btn ${isConfiguring ? 'active' : ''}"
onclick="event.stopPropagation(); showIndicatorConfig(${activeIdx})"
title="Configure ${ind.name}">⚙</button>` : ''}
</div> </div>
<button onclick="event.stopPropagation(); removeIndicatorByIndex(${idx})" style="background: none; border: none; color: var(--tv-red); cursor: pointer; font-size: 14px;">×</button> `;
</div> }).join('');
`).join('');
updateConfigPanel();
}
function updateConfigPanel() {
const configPanel = document.getElementById('indicatorConfigPanel'); const configPanel = document.getElementById('indicatorConfigPanel');
if (selectedIndicatorIndex >= 0) { const configButtons = document.getElementById('configButtons');
configPanel.style.display = 'block'; if (!configPanel) return;
renderIndicatorConfig(selectedIndicatorIndex);
configPanel.style.display = 'block';
if (configuringIndex >= 0 && configuringIndex < activeIndicators.length) {
renderIndicatorConfig(configuringIndex);
if (configButtons) configButtons.style.display = 'flex';
} else { } else {
configPanel.style.display = 'none'; const container = document.getElementById('configForm');
if (container) {
container.innerHTML = '<div style="text-align: center; color: var(--tv-text-secondary); padding: 20px; font-size: 12px;">Select an active indicator to configure its settings</div>';
}
if (configButtons) configButtons.style.display = 'none';
} }
} }
export function addNewIndicator() { export function toggleIndicator(type) {
const type = prompt('Enter indicator type:\n' + AVAILABLE_INDICATORS.map(i => `${i.type}: ${i.name}`).join('\n')); const existingIdx = activeIndicators.findIndex(a => a.type === type);
if (!type) return;
const indicatorDef = AVAILABLE_INDICATORS.find(i => i.type === type.toLowerCase()); if (existingIdx >= 0) {
if (!indicatorDef) { activeIndicators[existingIdx].series?.forEach(s => {
alert('Unknown indicator type'); try { window.dashboard?.chart?.removeSeries(s); } catch(e) {}
return; });
activeIndicators.splice(existingIdx, 1);
if (configuringIndex >= activeIndicators.length) {
configuringIndex = -1;
} else if (configuringIndex === existingIdx) {
configuringIndex = -1;
} else if (configuringIndex > existingIdx) {
configuringIndex--;
}
} else {
const indicatorDef = AVAILABLE_INDICATORS.find(i => i.type === type);
if (!indicatorDef) return;
const IndicatorClass = IR?.[type];
if (!IndicatorClass) return;
const instance = new IndicatorClass({ type, params: {}, name: indicatorDef.name });
const metadata = instance.getMetadata();
const params = {
_lineType: 'solid',
_lineWidth: 2
};
metadata.plots.forEach((plot, idx) => {
params[`_color_${idx}`] = plot.color || getDefaultColor(activeIndicators.length + idx);
});
metadata.inputs.forEach(input => {
params[input.name] = input.default;
});
activeIndicators.push({
type: type,
name: metadata.name,
params: params,
plots: metadata.plots,
series: []
});
configuringIndex = activeIndicators.length - 1;
} }
const IndicatorClass = IR?.[type.toLowerCase()];
if (!IndicatorClass) {
alert('Indicator class not found');
return;
}
const instance = new IndicatorClass({ type: type.toLowerCase(), params: {}, name: indicatorDef.name });
const metadata = instance.getMetadata();
const params = {};
metadata.inputs.forEach(input => {
params[input.name] = input.default;
});
activeIndicators.push({
type: type.toLowerCase(),
name: metadata.name,
params: params,
plots: metadata.plots,
series: []
});
selectedIndicatorIndex = activeIndicators.length - 1;
renderIndicatorList(); renderIndicatorList();
drawIndicatorsOnChart(); drawIndicatorsOnChart();
} }
export function selectIndicator(index) { export function showIndicatorConfig(index) {
selectedIndicatorIndex = index; if (configuringIndex === index) {
configuringIndex = -1;
} else {
configuringIndex = index;
}
renderIndicatorList(); renderIndicatorList();
} }
export function showIndicatorConfigByType(type) {
const idx = activeIndicators.findIndex(a => a.type === type);
if (idx >= 0) {
configuringIndex = idx;
renderIndicatorList();
}
}
export function renderIndicatorConfig(index) { export function renderIndicatorConfig(index) {
const container = document.getElementById('configForm'); const container = document.getElementById('configForm');
if (!container) return;
const indicator = activeIndicators[index]; const indicator = activeIndicators[index];
if (!indicator || !indicator.plots) { if (!indicator) {
container.innerHTML = ''; container.innerHTML = '';
return; return;
} }
const IndicatorClass = IR?.[indicator.type]; const IndicatorClass = IR?.[indicator.type];
if (!IndicatorClass) return; if (!IndicatorClass) {
container.innerHTML = '<div style="color: var(--tv-red);">Error loading indicator</div>';
return;
}
const instance = new IndicatorClass({ type: indicator.type, params: indicator.params, name: indicator.name }); const instance = new IndicatorClass({ type: indicator.type, params: indicator.params, name: indicator.name });
const meta = instance.getMetadata(); const meta = instance.getMetadata();
container.innerHTML = meta.inputs.map(input => ` const plotGroups = groupPlotsByColor(meta.plots);
const colorInputs = plotGroups.map(group => {
const firstIdx = group.indices[0];
const color = indicator.params[`_color_${firstIdx}`] || meta.plots[firstIdx].color || '#2962ff';
return `
<div style="margin-bottom: 8px;">
<label style="font-size: 10px; color: var(--tv-text-secondary); text-transform: uppercase; display: block; margin-bottom: 4px;">${group.name} Color</label>
<input type="color" id="config__color_${firstIdx}" value="${color}" style="width: 100%; height: 28px; border: 1px solid var(--tv-border); border-radius: 4px; cursor: pointer; background: var(--tv-bg);">
</div>
`;
}).join('');
container.innerHTML = `
<div style="font-size: 11px; color: var(--tv-blue); margin-bottom: 8px; font-weight: 600;">${indicator.name}</div>
${colorInputs}
<div style="margin-bottom: 8px;"> <div style="margin-bottom: 8px;">
<label style="font-size: 10px; color: var(--tv-text-secondary); text-transform: uppercase; display: block; margin-bottom: 4px;">${input.label}</label> <label style="font-size: 10px; color: var(--tv-text-secondary); text-transform: uppercase; display: block; margin-bottom: 4px;">Line Type</label>
${input.type === 'select' ? <select id="config__lineType" class="sim-input" style="font-size: 12px; padding: 6px;">
`<select id="config_${input.name}" class="sim-input" style="font-size: 12px; padding: 6px;">${input.options.map(o => `<option value="${o}" ${indicator.params[input.name] === o ? 'selected' : ''}>${o}</option>`).join('')}</select>` : ${LINE_TYPES.map(lt => `<option value="${lt}" ${indicator.params._lineType === lt ? 'selected' : ''}>${lt.charAt(0).toUpperCase() + lt.slice(1)}</option>`).join('')}
`<input type="number" id="config_${input.name}" class="sim-input" value="${indicator.params[input.name]}" ${input.min !== undefined ? `min="${input.min}"` : ''} ${input.max !== undefined ? `max="${input.max}"` : ''} style="font-size: 12px; padding: 6px;">` </select>
}
</div> </div>
`).join('');
<div style="margin-bottom: 8px;">
<label style="font-size: 10px; color: var(--tv-text-secondary); text-transform: uppercase; display: block; margin-bottom: 4px;">Line Width</label>
<input type="number" id="config__lineWidth" class="sim-input" value="${indicator.params._lineWidth || 2}" min="1" max="5" style="font-size: 12px; padding: 6px;">
</div>
${meta.inputs.map(input => `
<div style="margin-bottom: 8px;">
<label style="font-size: 10px; color: var(--tv-text-secondary); text-transform: uppercase; display: block; margin-bottom: 4px;">${input.label}</label>
${input.type === 'select' ?
`<select id="config_${input.name}" class="sim-input" style="font-size: 12px; padding: 6px;">${input.options.map(o => `<option value="${o}" ${indicator.params[input.name] === o ? 'selected' : ''}>${o}</option>`).join('')}</select>` :
`<input type="number" id="config_${input.name}" class="sim-input" value="${indicator.params[input.name]}" ${input.min !== undefined ? `min="${input.min}"` : ''} ${input.max !== undefined ? `max="${input.max}"` : ''} style="font-size: 12px; padding: 6px;">`
}
</div>
`).join('')}
`;
} }
export function applyIndicatorConfig() { export function applyIndicatorConfig() {
if (selectedIndicatorIndex < 0) return; if (configuringIndex < 0 || configuringIndex >= activeIndicators.length) return;
const indicator = activeIndicators[selectedIndicatorIndex]; const indicator = activeIndicators[configuringIndex];
const IndicatorClass = IR?.[indicator.type]; const IndicatorClass = IR?.[indicator.type];
if (!IndicatorClass) return; if (!IndicatorClass) return;
const instance = new IndicatorClass({ type: indicator.type, params: {}, name: indicator.name }); const instance = new IndicatorClass({ type: indicator.type, params: {}, name: indicator.name });
const meta = instance.getMetadata(); const meta = instance.getMetadata();
const plotGroups = groupPlotsByColor(meta.plots);
plotGroups.forEach(group => {
const firstIdx = group.indices[0];
const colorEl = document.getElementById(`config__color_${firstIdx}`);
if (colorEl) {
const color = colorEl.value;
group.indices.forEach(idx => {
indicator.params[`_color_${idx}`] = color;
});
}
});
const lineTypeEl = document.getElementById('config__lineType');
const lineWidthEl = document.getElementById('config__lineWidth');
if (lineTypeEl) indicator.params._lineType = lineTypeEl.value;
if (lineWidthEl) indicator.params._lineWidth = parseInt(lineWidthEl.value);
meta.inputs.forEach(input => { meta.inputs.forEach(input => {
const el = document.getElementById(`config_${input.name}`); const el = document.getElementById(`config_${input.name}`);
if (el) { if (el) {
@ -131,18 +276,33 @@ export function applyIndicatorConfig() {
} }
export function removeIndicator() { export function removeIndicator() {
if (selectedIndicatorIndex < 0) return; if (configuringIndex < 0 || configuringIndex >= activeIndicators.length) return;
activeIndicators.splice(selectedIndicatorIndex, 1);
selectedIndicatorIndex = -1; activeIndicators[configuringIndex].series?.forEach(s => {
try { window.dashboard?.chart?.removeSeries(s); } catch(e) {}
});
activeIndicators.splice(configuringIndex, 1);
configuringIndex = -1;
renderIndicatorList(); renderIndicatorList();
drawIndicatorsOnChart(); drawIndicatorsOnChart();
} }
export function removeIndicatorByIndex(index) { export function removeIndicatorByIndex(index) {
if (index < 0 || index >= activeIndicators.length) return;
activeIndicators[index].series?.forEach(s => {
try { window.dashboard?.chart?.removeSeries(s); } catch(e) {}
});
activeIndicators.splice(index, 1); activeIndicators.splice(index, 1);
if (selectedIndicatorIndex >= activeIndicators.length) { if (configuringIndex === index) {
selectedIndicatorIndex = activeIndicators.length - 1; configuringIndex = -1;
} else if (configuringIndex > index) {
configuringIndex--;
} }
renderIndicatorList(); renderIndicatorList();
drawIndicatorsOnChart(); drawIndicatorsOnChart();
} }
@ -159,6 +319,8 @@ export function drawIndicatorsOnChart() {
const candles = window.dashboard.allData.get(window.dashboard.currentInterval); const candles = window.dashboard.allData.get(window.dashboard.currentInterval);
if (!candles || candles.length === 0) return; if (!candles || candles.length === 0) return;
const lineStyleMap = { 'solid': 0, 'dotted': 1, 'dashed': 2 };
activeIndicators.forEach((indicator, idx) => { activeIndicators.forEach((indicator, idx) => {
const IndicatorClass = IR?.[indicator.type]; const IndicatorClass = IR?.[indicator.type];
if (!IndicatorClass) return; if (!IndicatorClass) return;
@ -173,30 +335,53 @@ export function drawIndicatorsOnChart() {
const meta = instance.getMetadata(); const meta = instance.getMetadata();
indicator.series = []; indicator.series = [];
meta.plots.forEach(plot => { const lineStyle = lineStyleMap[indicator.params._lineType] || 0;
if (results[0] && typeof results[0][plot.id] === 'undefined') return; const lineWidth = indicator.params._lineWidth || 2;
const firstNonNull = results?.find(r => r !== null && r !== undefined);
const isObjectResult = firstNonNull && typeof firstNonNull === 'object';
meta.plots.forEach((plot, plotIdx) => {
if (isObjectResult && typeof firstNonNull[plot.id] === 'undefined') return;
const plotColor = indicator.params[`_color_${plotIdx}`] || plot.color || '#2962ff';
const lineSeries = window.dashboard.chart.addLineSeries({ const lineSeries = window.dashboard.chart.addLineSeries({
color: plot.color || '#2962ff', color: plotColor,
lineWidth: plot.width || 1, lineWidth: plot.width || lineWidth,
lineStyle: lineStyle,
title: plot.title, title: plot.title,
priceLineVisible: false, priceLineVisible: false,
lastValueVisible: true lastValueVisible: true
}); });
const data = candles.map((c, i) => ({ const data = [];
time: c.time, for (let i = 0; i < candles.length; i++) {
value: results[i]?.[plot.id] ?? null let value;
})).filter(d => d.value !== null && d.value !== undefined); if (isObjectResult) {
value = results[i]?.[plot.id];
} else {
value = results[i];
}
if (value !== null && value !== undefined) {
data.push({
time: candles[i].time,
value: value
});
}
}
lineSeries.setData(data); if (data.length > 0) {
indicator.series.push(lineSeries); lineSeries.setData(data);
indicator.series.push(lineSeries);
}
}); });
}); });
} }
window.addNewIndicator = addNewIndicator; window.toggleIndicator = toggleIndicator;
window.selectIndicator = selectIndicator; window.showIndicatorConfig = showIndicatorConfig;
window.applyIndicatorConfig = applyIndicatorConfig; window.applyIndicatorConfig = applyIndicatorConfig;
window.removeIndicator = removeIndicator; window.removeIndicator = removeIndicator;
window.removeIndicatorByIndex = removeIndicatorByIndex; window.removeIndicatorByIndex = removeIndicatorByIndex;

View File

@ -0,0 +1,154 @@
"""
One-time backfill script to fill gaps in data.
Run with: python -m data_collector.backfill_gap --start "2024-01-01 09:34" --end "2024-01-01 19:39"
"""
import asyncio
import logging
import sys
import os
from datetime import datetime, timezone
from typing import Optional
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
from .database import DatabaseManager
from .backfill import HyperliquidBackfill
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)
INTERVALS = ["1m", "3m", "5m", "15m", "30m", "1h", "2h", "4h", "8h", "12h", "1d", "3d", "1w"]
async def backfill_gap(
start_time: datetime,
end_time: datetime,
symbol: str = "BTC",
intervals: Optional[list] = None
) -> dict:
"""
Backfill a specific time gap for all intervals.
Args:
start_time: Gap start time (UTC)
end_time: Gap end time (UTC)
symbol: Trading symbol
intervals: List of intervals to backfill (default: all standard)
Returns:
Dictionary with interval -> count mapping
"""
intervals = intervals or INTERVALS
results = {}
db = DatabaseManager()
await db.connect()
logger.info(f"Backfilling gap: {start_time} to {end_time} for {symbol}")
try:
async with HyperliquidBackfill(db, symbol, intervals) as backfill:
for interval in intervals:
try:
logger.info(f"Backfilling {interval}...")
candles = await backfill.fetch_candles(interval, start_time, end_time)
if candles:
inserted = await db.insert_candles(candles)
results[interval] = inserted
logger.info(f" {interval}: {inserted} candles inserted")
else:
results[interval] = 0
logger.warning(f" {interval}: No candles returned")
await asyncio.sleep(0.3)
except Exception as e:
logger.error(f" {interval}: Error - {e}")
results[interval] = 0
finally:
await db.disconnect()
logger.info(f"Backfill complete. Total: {sum(results.values())} candles")
return results
async def auto_detect_and_fill_gaps(symbol: str = "BTC") -> dict:
"""
Detect and fill all gaps in the database for all intervals.
"""
db = DatabaseManager()
await db.connect()
results = {}
try:
async with HyperliquidBackfill(db, symbol, INTERVALS) as backfill:
for interval in INTERVALS:
try:
# Detect gaps
gaps = await db.detect_gaps(symbol, interval)
if not gaps:
logger.info(f"{interval}: No gaps detected")
results[interval] = 0
continue
logger.info(f"{interval}: {len(gaps)} gaps detected")
total_filled = 0
for gap in gaps:
gap_start = datetime.fromisoformat(gap['gap_start'].replace('Z', '+00:00'))
gap_end = datetime.fromisoformat(gap['gap_end'].replace('Z', '+00:00'))
logger.info(f" Filling gap: {gap_start} to {gap_end}")
candles = await backfill.fetch_candles(interval, gap_start, gap_end)
if candles:
inserted = await db.insert_candles(candles)
total_filled += inserted
logger.info(f" Filled {inserted} candles")
await asyncio.sleep(0.2)
results[interval] = total_filled
except Exception as e:
logger.error(f"{interval}: Error - {e}")
results[interval] = 0
finally:
await db.disconnect()
return results
async def main():
import argparse
parser = argparse.ArgumentParser(description="Backfill gaps in BTC data")
parser.add_argument("--start", help="Start time (YYYY-MM-DD HH:MM)", default=None)
parser.add_argument("--end", help="End time (YYYY-MM-DD HH:MM)", default=None)
parser.add_argument("--auto", action="store_true", help="Auto-detect and fill all gaps")
parser.add_argument("--symbol", default="BTC", help="Symbol to backfill")
args = parser.parse_args()
if args.auto:
await auto_detect_and_fill_gaps(args.symbol)
elif args.start and args.end:
start_time = datetime.strptime(args.start, "%Y-%m-%d %H:%M").replace(tzinfo=timezone.utc)
end_time = datetime.strptime(args.end, "%Y-%m-%d %H:%M").replace(tzinfo=timezone.utc)
await backfill_gap(start_time, end_time, args.symbol)
else:
parser.print_help()
if __name__ == "__main__":
asyncio.run(main())

View File

@ -269,7 +269,6 @@ class CustomTimeframeGenerator:
logger.info(f"Generating historical {interval} from {source_interval}...") logger.info(f"Generating historical {interval} from {source_interval}...")
# Get date range available in source data
async with self.db.acquire() as conn: async with self.db.acquire() as conn:
min_max = await conn.fetchrow(""" min_max = await conn.fetchrow("""
SELECT MIN(time), MAX(time) FROM candles SELECT MIN(time), MAX(time) FROM candles
@ -287,7 +286,6 @@ class CustomTimeframeGenerator:
await self.aggregate_and_upsert('BTC', interval, curr) await self.aggregate_and_upsert('BTC', interval, curr)
total_inserted += 1 total_inserted += 1
# Advance curr
if interval == '1M': if interval == '1M':
_, days = calendar.monthrange(curr.year, curr.month) _, days = calendar.monthrange(curr.year, curr.month)
curr += timedelta(days=days) curr += timedelta(days=days)
@ -297,7 +295,7 @@ class CustomTimeframeGenerator:
elif cfg['type'] == 'hour': curr += timedelta(hours=cfg['value']) elif cfg['type'] == 'hour': curr += timedelta(hours=cfg['value'])
elif cfg['type'] == 'day': curr += timedelta(days=cfg['value']) elif cfg['type'] == 'day': curr += timedelta(days=cfg['value'])
elif cfg['type'] == 'week': curr += timedelta(weeks=1) elif cfg['type'] == 'week': curr += timedelta(weeks=1)
else: # Custom else:
minutes = self.CUSTOM_INTERVALS[interval]['minutes'] minutes = self.CUSTOM_INTERVALS[interval]['minutes']
curr += timedelta(minutes=minutes) curr += timedelta(minutes=minutes)
@ -307,6 +305,87 @@ class CustomTimeframeGenerator:
return total_inserted return total_inserted
async def generate_from_gap(self, interval: str) -> int:
"""
Generate candles only from where they're missing.
Compares source interval max time with target interval max time.
"""
if not self.first_1m_time:
await self.initialize()
if not self.first_1m_time:
return 0
config = self.CUSTOM_INTERVALS.get(interval) or {'source': '1m'}
source_interval = config.get('source', '1m')
async with self.db.acquire() as conn:
# Get source range
source_min_max = await conn.fetchrow("""
SELECT MIN(time), MAX(time) FROM candles
WHERE symbol = 'BTC' AND interval = $1
""", source_interval)
if not source_min_max or not source_min_max[1]:
return 0
# Get target (this interval) max time
target_max = await conn.fetchval("""
SELECT MAX(time) FROM candles
WHERE symbol = 'BTC' AND interval = $1
""", interval)
source_max = source_min_max[1]
if target_max:
# Start from next bucket after target_max
curr = self.get_bucket_start(target_max, interval)
if interval in self.CUSTOM_INTERVALS:
minutes = self.CUSTOM_INTERVALS[interval]['minutes']
curr = curr + timedelta(minutes=minutes)
elif interval in self.STANDARD_INTERVALS:
cfg = self.STANDARD_INTERVALS[interval]
if cfg['type'] == 'min': curr = curr + timedelta(minutes=cfg['value'])
elif cfg['type'] == 'hour': curr = curr + timedelta(hours=cfg['value'])
elif cfg['type'] == 'day': curr = curr + timedelta(days=cfg['value'])
elif cfg['type'] == 'week': curr = curr + timedelta(weeks=1)
else:
# No target data, start from source min
curr = self.get_bucket_start(source_min_max[0], interval)
end = source_max
if curr > end:
logger.info(f"{interval}: Already up to date (target: {target_max}, source: {source_max})")
return 0
logger.info(f"Generating {interval} from {curr} to {end}...")
total_inserted = 0
while curr <= end:
await self.aggregate_and_upsert('BTC', interval, curr)
total_inserted += 1
if interval == '1M':
_, days = calendar.monthrange(curr.year, curr.month)
curr += timedelta(days=days)
elif interval in self.STANDARD_INTERVALS:
cfg = self.STANDARD_INTERVALS[interval]
if cfg['type'] == 'min': curr += timedelta(minutes=cfg['value'])
elif cfg['type'] == 'hour': curr += timedelta(hours=cfg['value'])
elif cfg['type'] == 'day': curr += timedelta(days=cfg['value'])
elif cfg['type'] == 'week': curr += timedelta(weeks=1)
else:
minutes = self.CUSTOM_INTERVALS[interval]['minutes']
curr += timedelta(minutes=minutes)
if total_inserted % 50 == 0:
logger.info(f"Generated {total_inserted} {interval} candles...")
await asyncio.sleep(0.01)
logger.info(f"{interval}: Generated {total_inserted} candles")
return total_inserted
async def verify_integrity(self, interval: str) -> Dict: async def verify_integrity(self, interval: str) -> Dict:
async with self.db.acquire() as conn: async with self.db.acquire() as conn:
stats = await conn.fetchrow(""" stats = await conn.fetchrow("""

View File

@ -3,6 +3,7 @@ Database interface for TimescaleDB
Optimized for batch inserts and low resource usage Optimized for batch inserts and low resource usage
""" """
import asyncio
import logging import logging
from contextlib import asynccontextmanager from contextlib import asynccontextmanager
from datetime import datetime from datetime import datetime

View File

@ -41,6 +41,8 @@ class DataCollector:
Manages WebSocket connection, buffering, and database writes Manages WebSocket connection, buffering, and database writes
""" """
STANDARD_INTERVALS = ["1m", "3m", "5m", "15m", "30m", "1h", "2h", "4h", "8h", "12h", "1d", "3d", "1w"]
def __init__( def __init__(
self, self,
symbol: str = "BTC", symbol: str = "BTC",
@ -69,10 +71,16 @@ class DataCollector:
self.db = DatabaseManager() self.db = DatabaseManager()
await self.db.connect() await self.db.connect()
# Run startup backfill for all intervals
await self._startup_backfill()
# Initialize custom timeframe generator # Initialize custom timeframe generator
self.custom_tf_generator = CustomTimeframeGenerator(self.db) self.custom_tf_generator = CustomTimeframeGenerator(self.db)
await self.custom_tf_generator.initialize() await self.custom_tf_generator.initialize()
# Regenerate custom timeframes after startup backfill
await self._regenerate_custom_timeframes()
# Initialize indicator engine # Initialize indicator engine
# Hardcoded config for now, eventually load from yaml # Hardcoded config for now, eventually load from yaml
indicator_configs = [ indicator_configs = [
@ -126,6 +134,99 @@ class DataCollector:
finally: finally:
await self.stop() await self.stop()
async def _startup_backfill(self) -> None:
"""
Backfill missing data on startup for all standard intervals.
Uses both gap detection AND time-based backfill for robustness.
"""
logger.info("Running startup backfill for all intervals...")
try:
async with HyperliquidBackfill(self.db, self.symbol, self.STANDARD_INTERVALS) as backfill:
for interval in self.STANDARD_INTERVALS:
try:
# First, use gap detection to find any holes
gaps = await self.db.detect_gaps(self.symbol, interval)
if gaps:
logger.info(f"{interval}: {len(gaps)} gaps detected")
for gap in gaps:
gap_start = datetime.fromisoformat(gap['gap_start'].replace('Z', '+00:00'))
gap_end = datetime.fromisoformat(gap['gap_end'].replace('Z', '+00:00'))
logger.info(f" Filling gap: {gap_start} to {gap_end}")
candles = await backfill.fetch_candles(interval, gap_start, gap_end)
if candles:
inserted = await self.db.insert_candles(candles)
logger.info(f" Inserted {inserted} candles for gap")
await asyncio.sleep(0.2)
# Second, check if we're behind current time
latest = await self.db.get_latest_candle(self.symbol, interval)
now = datetime.now(timezone.utc)
if latest:
last_time = latest['time']
gap_minutes = (now - last_time).total_seconds() / 60
if gap_minutes > 2:
logger.info(f"{interval}: {gap_minutes:.0f} min behind, backfilling to now...")
candles = await backfill.fetch_candles(interval, last_time, now)
if candles:
inserted = await self.db.insert_candles(candles)
logger.info(f" Inserted {inserted} candles")
else:
logger.info(f"{interval}: up to date")
else:
# No data exists, backfill last 7 days
logger.info(f"{interval}: No data, backfilling 7 days...")
count = await backfill.backfill_interval(interval, days_back=7)
logger.info(f" Inserted {count} candles")
await asyncio.sleep(0.2)
except Exception as e:
logger.error(f"Startup backfill failed for {interval}: {e}")
import traceback
logger.error(traceback.format_exc())
continue
except Exception as e:
logger.error(f"Startup backfill error: {e}")
import traceback
logger.error(traceback.format_exc())
logger.info("Startup backfill complete")
async def _regenerate_custom_timeframes(self) -> None:
"""
Regenerate custom timeframes (37m, 148m) only from gaps.
Only generates candles that are missing, not all from beginning.
"""
if not self.custom_tf_generator:
return
logger.info("Checking custom timeframes for gaps...")
try:
for interval in ['37m', '148m']:
try:
count = await self.custom_tf_generator.generate_from_gap(interval)
if count > 0:
logger.info(f"{interval}: Generated {count} candles")
else:
logger.info(f"{interval}: Up to date")
except Exception as e:
logger.error(f"Failed to regenerate {interval}: {e}")
except Exception as e:
logger.error(f"Custom timeframe regeneration error: {e}")
logger.info("Custom timeframe check complete")
async def stop(self) -> None: async def stop(self) -> None:
"""Graceful shutdown""" """Graceful shutdown"""
if not self.is_running: if not self.is_running: