ditus/personal_TV
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first hurst working, only for current TF

Browse Source
This commit is contained in:
DiTus
2025-07-14 21:45:48 +02:00
parent f8064f2f44
commit 0c3c9ecd81
4 changed files with 103 additions and 225 deletions
Download Patch File Download Diff File Expand all files Collapse all files

186
app.py
View File

@ -3,18 +3,16 @@ import logging
import asyncio
import os
import json
import csv
from flask import Flask, render_template, request
from flask_socketio import SocketIO
from binance import Client
import websockets
from threading import Lock
from datetime import datetime, timedelta, timezone
from datetime import datetime, timedelta
# --- Configuration ---
SYMBOL = 'ETHUSDT'
# The CSV file is now the primary source of historical data.
HISTORY_CSV_FILE = 'ETHUSDT_1m_Binance.csv'
HISTORY_FILE = 'historical_data_1m.json' # Used as a cache to prevent re-downloading
RESTART_TIMEOUT_S = 15
BINANCE_WS_URL = f"wss://stream.binance.com:9443/ws/{SYMBOL.lower()}@trade"
@ -29,136 +27,57 @@ socketio = SocketIO(app, async_mode='threading')
# --- Global State ---
app_initialized = False
app_init_lock = Lock()
# This cache will hold the filtered historical data to be sent to the frontend.
historical_data_cache = []
# --- Helper Function for Optimized Reading ---
def get_last_timestamp_from_csv(filepath):
# --- Historical Data Streaming ---
def stream_historical_data(sid):
"""
Efficiently reads the end of a CSV to get the timestamp from the last valid row.
This avoids reading the entire file into memory.
Returns a datetime object or None.
Fetches historical data in chunks and streams it to the client with progress updates.
"""
try:
with open(filepath, 'rb') as f:
# Seek to a position near the end of the file to read a chunk.
# 4096 bytes should be enough to contain several lines.
f.seek(0, os.SEEK_END)
filesize = f.tell()
if filesize == 0:
return None
f.seek(max(0, filesize - 4096), os.SEEK_SET)
# Read the last part of the file
lines = f.readlines()
if not lines:
return None
# Get the last non-empty line
last_line_str = ''
for line in reversed(lines):
decoded_line = line.decode('utf-8').strip()
if decoded_line:
last_line_str = decoded_line
break
if not last_line_str or 'Open time' in last_line_str:
return None
last_row = last_line_str.split(',')
dt_obj = datetime.strptime(last_row[0], '%Y-%m-%d %H:%M:%S')
return dt_obj.replace(tzinfo=timezone.utc)
except (IOError, IndexError, ValueError) as e:
logging.error(f"Could not get last timestamp from CSV: {e}")
return None
# --- Data Management ---
def load_and_update_data():
"""
Loads historical data from the CSV, updates it with the latest data from Binance,
and then filters it for the frontend.
"""
global historical_data_cache
client = Client()
# 1. Check if the primary CSV data source exists.
if not os.path.exists(HISTORY_CSV_FILE):
logging.critical(f"CRITICAL: History file '{HISTORY_CSV_FILE}' not found. Please provide the CSV file. Halting data load.")
historical_data_cache = []
return
# 2. OPTIMIZED: Efficiently get the last timestamp to determine where to start fetching.
last_dt_in_csv = get_last_timestamp_from_csv(HISTORY_CSV_FILE)
start_fetch_date = None
if last_dt_in_csv:
start_fetch_date = last_dt_in_csv + timedelta(minutes=1)
logging.info(f"Last record in CSV is from {last_dt_in_csv}. Checking for new data since {start_fetch_date}.")
else:
logging.warning("Could not determine last timestamp from CSV. Assuming file is new or empty. No new data will be fetched.")
# 3. Fetch new data from Binance.
new_klines = []
if start_fetch_date and start_fetch_date < datetime.now(timezone.utc):
while True:
logging.info(f"Fetching new klines from {start_fetch_date}...")
fetched = client.get_historical_klines(SYMBOL, Client.KLINE_INTERVAL_1MINUTE, start_fetch_date.strftime("%Y-%m-%d %H:%M:%S"))
if not fetched:
logging.info("No new klines to fetch.")
break
new_klines.extend(fetched)
last_fetched_dt = datetime.fromtimestamp(fetched[-1][0] / 1000, tz=timezone.utc)
start_fetch_date = last_fetched_dt + timedelta(minutes=1)
logging.info(f"Fetched {len(fetched)} new klines, up to {last_fetched_dt}.")
if len(fetched) < 1000:
break
time.sleep(0.1)
# 4. If new data was found, append it to the CSV file.
if new_klines:
logging.info(f"Appending {len(new_klines)} new candles to {HISTORY_CSV_FILE}.")
try:
with open(HISTORY_CSV_FILE, 'a', newline='') as f:
writer = csv.writer(f)
for kline in new_klines:
open_time_dt = datetime.fromtimestamp(kline[0] / 1000, tz=timezone.utc)
open_time_str = open_time_dt.strftime('%Y-%m-%d %H:%M:%S')
close_time_dt = datetime.fromtimestamp(kline[6] / 1000, tz=timezone.utc)
close_time_str = close_time_dt.strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]
writer.writerow([open_time_str] + kline[1:6] + [close_time_str] + kline[7:])
except Exception as e:
logging.error(f"Failed to append new data to {HISTORY_CSV_FILE}: {e}")
# 5. OPTIMIZED: Read the CSV and load only the necessary data (2025 onwards) for the frontend.
logging.info("Reading CSV to populate cache with data from 01.01.2025 onwards...")
frontend_klines = []
frontend_start_dt = datetime(2025, 1, 1, tzinfo=timezone.utc)
try:
with open(HISTORY_CSV_FILE, 'r', newline='') as f:
reader = csv.reader(f)
next(reader) # Skip header
for row in reader:
try:
dt_obj = datetime.strptime(row[0], '%Y-%m-%d %H:%M:%S').replace(tzinfo=timezone.utc)
if dt_obj >= frontend_start_dt:
timestamp_ms = int(dt_obj.timestamp() * 1000)
frontend_klines.append([
timestamp_ms, row[1], row[2], row[3], row[4],
"0", "0", "0", "0", "0", "0"
])
except (ValueError, IndexError):
continue
logging.info(f"Starting historical data stream for SID={sid}")
client = Client()
historical_data_cache = frontend_klines
logging.info(f"--- Data initialization complete. {len(historical_data_cache)} candles cached for frontend. ---")
# Fetch the last 90 days of data in 6 chunks of 15 days each.
num_chunks = 6
chunk_size_days = 15
end_date = datetime.utcnow()
all_klines = []
for i in range(num_chunks):
start_date = end_date - timedelta(days=chunk_size_days)
logging.info(f"Fetching chunk {i + 1}/{num_chunks} ({start_date} to {end_date}) for SID={sid}")
new_klines = client.get_historical_klines(SYMBOL, Client.KLINE_INTERVAL_1MINUTE, str(start_date), str(end_date))
if new_klines:
all_klines.extend(new_klines)
progress_payload = {
'progress': ((i + 1) / num_chunks) * 100
}
socketio.emit('history_progress', progress_payload, to=sid)
end_date = start_date
socketio.sleep(0.05)
seen = set()
unique_klines = []
for kline in sorted(all_klines, key=lambda x: x[0]):
kline_tuple = tuple(kline)
if kline_tuple not in seen:
unique_klines.append(kline)
seen.add(kline_tuple)
with open(HISTORY_FILE, 'w') as f:
json.dump(unique_klines, f)
logging.info(f"Finished data stream for SID={sid}. Sending final payload of {len(unique_klines)} klines.")
socketio.emit('history_finished', {'klines_1m': unique_klines}, to=sid)
except Exception as e:
logging.error(f"Failed to read CSV for frontend cache: {e}")
historical_data_cache = []
logging.error(f"Error in stream_historical_data for SID={sid}: {e}", exc_info=True)
socketio.emit('history_error', {'message': str(e)}, to=sid)
# --- Real-time Data Listener ---
@ -183,21 +102,12 @@ def binance_listener_thread():
def handle_connect():
global app_initialized
logging.info(f"Client connected: IP={request.remote_addr}, SID={request.sid}")
with app_init_lock:
if not app_initialized:
logging.info("--- First client connected, initializing application data ---")
socketio.start_background_task(load_and_update_data)
logging.info("--- Initializing Application ---")
socketio.start_background_task(binance_listener_thread)
app_initialized = True
# Wait until the cache is populated.
while not historical_data_cache:
logging.info(f"SID={request.sid} is waiting for historical data cache...")
socketio.sleep(1)
logging.info(f"Sending {len(historical_data_cache)} cached klines to SID={request.sid}")
socketio.emit('history_finished', {'klines_1m': historical_data_cache}, to=request.sid)
socketio.start_background_task(target=stream_historical_data, sid=request.sid)
@socketio.on('analyze_chart')

1
historical_data_1m.json Normal file
View File

File diff suppressed because one or more lines are too long

92
static/indicator-manager.js
View File

@ -2,13 +2,10 @@
* Creates and manages all indicator-related logic for the chart.
* @param {Object} chart - The Lightweight Charts instance.
* @param {Array<Object>} baseCandleData - A reference to the array holding the chart's BASE 1m candle data.
* @param {Array<Object>} displayedCandleData - A reference to the array with currently visible candles.
* @returns {Object} A manager object with public methods to control indicators.
*/
function createIndicatorManager(chart, baseCandleData) {
// This holds the candle data currently displayed on the chart (e.g., 5m, 10m)
let currentAggregatedData = [];
// Defines the 4 slots available in the UI for indicators.
function createIndicatorManager(chart, baseCandleData, displayedCandleData) {
const indicatorSlots = [
{ id: 1, cellId: 'indicator-cell-1', series: [], definition: null, params: {} },
{ id: 2, cellId: 'indicator-cell-2', series: [], definition: null, params: {} },
@ -16,17 +13,14 @@ function createIndicatorManager(chart, baseCandleData) {
{ id: 4, cellId: 'indicator-cell-4', series: [], definition: null, params: {} },
];
// Pre-defined colors for the indicator lines.
const colors = {
bb1: { upper: '#FF9800', lower: '#FF9800' }, // Orange
bb2: { upper: '#2196F3', lower: '#2196F3' }, // Blue
bb3: { upper: '#9C27B0', lower: '#9C27B0' }, // Purple
default: ['#FF5722', '#03A9F4', '#8BC34A', '#F44336'] // Fallback colors for other indicators
bb1: { upper: '#FF9800', lower: '#FF9800' },
bb2: { upper: '#2196F3', lower: '#2196F3' },
bb3: { upper: '#9C27B0', lower: '#9C27B0' },
hurst: { topBand: '#673ab7', bottomBand: '#673ab7' },
default: ['#FF5722', '#03A9F4', '#8BC34A', '#F44336']
};
/**
* Populates the dropdown menus in each indicator cell.
*/
function populateDropdowns() {
indicatorSlots.forEach(slot => {
const cell = document.getElementById(slot.cellId);
@ -41,30 +35,20 @@ function createIndicatorManager(chart, baseCandleData) {
const controlsContainer = document.createElement('div');
controlsContainer.className = 'indicator-controls';
cell.innerHTML = ''; // Clear previous content
cell.innerHTML = '';
cell.appendChild(select);
cell.appendChild(controlsContainer);
select.addEventListener('change', (e) => {
const indicatorName = e.target.value;
loadIndicator(slot.id, indicatorName);
});
select.addEventListener('change', (e) => loadIndicator(slot.id, e.target.value));
});
}
/**
* Loads a new indicator into a specified slot.
* @param {number} slotId - The ID of the slot (1-4).
* @param {string} indicatorName - The name of the indicator to load (e.g., 'SMA').
*/
function loadIndicator(slotId, indicatorName) {
const slot = indicatorSlots.find(s => s.id === slotId);
if (!slot) return;
// Clean up any previous indicator series in this slot
slot.series.forEach(s => chart.removeSeries(s));
slot.series = [];
slot.definition = null;
slot.params = {};
@ -78,7 +62,6 @@ function createIndicatorManager(chart, baseCandleData) {
slot.definition = definition;
// Create UI controls for the indicator's parameters
definition.params.forEach(param => {
const label = document.createElement('label');
label.textContent = param.label || param.name;
@ -90,7 +73,6 @@ function createIndicatorManager(chart, baseCandleData) {
if (param.min !== undefined) input.min = param.min;
if (param.step !== undefined) input.step = param.step;
input.className = 'input-field';
input.placeholder = param.name;
slot.params[param.name] = input.type === 'number' ? parseFloat(input.value) : input.value;
let debounceTimer;
@ -112,37 +94,25 @@ function createIndicatorManager(chart, baseCandleData) {
updateIndicator(slot.id);
}
/**
* Recalculates and redraws the lines for a specific indicator.
* @param {number} slotId - The ID of the slot to update.
*/
function updateIndicator(slotId) {
const slot = indicatorSlots.find(s => s.id === slotId);
const candleDataForCalc = (slot.definition.usesBaseData) ? baseCandleData : currentAggregatedData;
const candleDataForCalc = (slot.definition.usesBaseData) ? baseCandleData : displayedCandleData;
if (!slot || !slot.definition || candleDataForCalc.length === 0) {
return;
}
if (!slot || !slot.definition || candleDataForCalc.length === 0) return;
// Clean up previous series before creating new ones
slot.series.forEach(s => chart.removeSeries(s));
slot.series = [];
console.log(`Recalculating ${slot.definition.name} for slot ${slot.id} on ${candleDataForCalc.length} candles.`);
const indicatorResult = slot.definition.calculateFull(candleDataForCalc, slot.params);
// Handle multi-line indicators like Bollinger Bands
if (typeof indicatorResult === 'object' && !Array.isArray(indicatorResult)) {
Object.keys(indicatorResult).forEach(key => {
const seriesData = indicatorResult[key];
const bandName = key.split('_')[0];
const bandType = key.split('_')[1];
const indicatorNameLower = slot.definition.name.toLowerCase();
const series = chart.addLineSeries({
color: colors[bandName] ? colors[bandName][bandType] : colors.default[slot.id - 1],
lineWidth: 2,
color: (colors[indicatorNameLower] && colors[indicatorNameLower][key]) ? colors[indicatorNameLower][key] : colors.default[slot.id - 1],
lineWidth: indicatorNameLower === 'hurst' ? 1 : 2,
title: `${slot.definition.label} - ${key}`,
lastValueVisible: false,
priceLineVisible: false,
@ -150,7 +120,7 @@ function createIndicatorManager(chart, baseCandleData) {
series.setData(seriesData);
slot.series.push(series);
});
} else { // Handle single-line indicators like SMA/EMA
} else {
const series = chart.addLineSeries({
color: colors.default[slot.id - 1],
lineWidth: 2,
@ -161,36 +131,18 @@ function createIndicatorManager(chart, baseCandleData) {
}
}
/**
* Internal function to recalculate all active indicators.
*/
function recalculateAllIndicators() {
function recalculateAllAfterHistory(newDisplayedCandleData) {
displayedCandleData = newDisplayedCandleData;
indicatorSlots.forEach(slot => {
if (slot.definition) {
updateIndicator(slot.id);
}
if (slot.definition) updateIndicator(slot.id);
});
}
/**
* Sets the candle data for indicators and triggers a full recalculation.
* @param {Array<Object>} aggregatedCandleData - The candle data for the currently selected timeframe.
*/
function recalculateAllAfterHistory(aggregatedCandleData) {
currentAggregatedData = aggregatedCandleData;
recalculateAllIndicators();
// This function is not currently used with the new model but is kept for potential future use.
function updateIndicatorsOnNewCandle(newCandle) {
// Real-time updates would go here.
}
/**
* Updates all indicators in response to a new candle closing.
* @param {Array<Object>} aggregatedCandleData - The latest candle data for the currently selected timeframe.
*/
function updateIndicatorsOnNewCandle(aggregatedCandleData) {
currentAggregatedData = aggregatedCandleData;
recalculateAllIndicators();
}
// Public API for the manager
return {
populateDropdowns,
recalculateAllAfterHistory,

49
templates/index.html
View File

@ -9,7 +9,8 @@
<script src="{{ url_for('static', filename='candle-aggregator.js') }}"></script>
<script src="{{ url_for('static', filename='sma.js') }}"></script>
<script src="{{ url_for('static', filename='ema.js') }}"></script>
<script src="{{ url_for('static', filename='bb.js') }}"></script>
<script src="{{ url_for('static', filename='bb.js') }}"></script>
<script src="{{ url_for('static', filename='hurst.js') }}"></script>
<script src="{{ url_for('static', filename='indicators.js') }}"></script>
<script src="{{ url_for('static', filename='indicator-manager.js') }}"></script>
@ -61,6 +62,14 @@
}
#candle-timer { font-size: 2rem; font-weight: 500; color: var(--accent-orange); }
#timeframe-select { margin-top: 10px; }
.progress-bar-container {
width: 80%; height: 4px; background-color: var(--button-bg);
border-radius: 2px; margin-top: 10px; overflow: hidden;
}
.progress-bar {
width: 0%; height: 100%; background-color: var(--green);
transition: width 0.4s ease-out;
}
</style>
</head>
<body>
@ -86,6 +95,9 @@
<option value="9">9m</option>
<option value="10">10m</option>
</select>
<div id="progress-container" class="progress-bar-container">
<div class="progress-bar"></div>
</div>
</div>
<div class="control-cell" id="indicator-cell-1"></div>
<div class="control-cell" id="indicator-cell-2"></div>
@ -123,27 +135,32 @@
const chartTitle = document.getElementById('chart-title');
const analyzeButton = document.getElementById('analyzeButton');
const analysisResultDiv = document.getElementById('analysisResult');
const progressContainer = document.getElementById('progress-container');
const progressBar = document.querySelector('.progress-bar');
manager = createIndicatorManager(chart, baseCandleData1m);
manager = createIndicatorManager(chart, baseCandleData1m, displayedCandleData);
manager.populateDropdowns();
const socket = io();
socket.on('connect', () => console.log('Socket.IO connected.'));
socket.on('history_progress', (data) => {
if (data && data.progress) progressBar.style.width = `${data.progress}%`;
});
socket.on('history_finished', (data) => {
if (!data || !data.klines_1m) return;
const mappedKlines = data.klines_1m.map(k => ({
progressBar.style.width = '100%';
baseCandleData1m = data.klines_1m.map(k => ({
time: k[0] / 1000, open: parseFloat(k[1]), high: parseFloat(k[2]),
low: parseFloat(k[3]), close: parseFloat(k[4])
}));
baseCandleData1m.length = 0;
for (const kline of mappedKlines) {
baseCandleData1m.push(kline);
}
updateChartForTimeframe();
setTimeout(() => { progressContainer.style.display = 'none'; }, 500);
});
socket.on('trade', (trade) => {
@ -152,7 +169,10 @@
const candleTimestamp1m = tradeTime - (tradeTime % 60);
if (!currentCandle1m || candleTimestamp1m > currentCandle1m.time) {
if (currentCandle1m) baseCandleData1m.push(currentCandle1m);
if (currentCandle1m) {
baseCandleData1m.push(currentCandle1m);
manager.updateIndicatorsOnNewCandle(currentCandle1m);
}
currentCandle1m = { time: candleTimestamp1m, open: price, high: price, low: price, close: price };
} else {
currentCandle1m.high = Math.max(currentCandle1m.high, price);
@ -166,17 +186,11 @@
let candleForUpdate;
if (lastDisplayedCandle && displayedCandleTimestamp === lastDisplayedCandle.time) {
candleForUpdate = { ...lastDisplayedCandle };
candleForUpdate.high = Math.max(candleForUpdate.high, price);
candleForUpdate.low = Math.min(candleForUpdate.low, price);
candleForUpdate.close = price;
candleForUpdate = { ...lastDisplayedCandle, high: Math.max(lastDisplayedCandle.high, price), low: Math.min(lastDisplayedCandle.low, price), close: price };
displayedCandleData[displayedCandleData.length - 1] = candleForUpdate;
} else if (!lastDisplayedCandle || displayedCandleTimestamp > lastDisplayedCandle.time) {
} else {
candleForUpdate = { time: displayedCandleTimestamp, open: price, high: price, low: price, close: price };
displayedCandleData.push(candleForUpdate);
// A new candle has started, so update the indicators.
manager.updateIndicatorsOnNewCandle(displayedCandleData);
}
if (candleForUpdate) candlestickSeries.update(candleForUpdate);
@ -201,6 +215,7 @@
setInterval(() => {
const selectedIntervalSeconds = parseInt(timeframeSelect.value, 10) * 60;
// **FIX**: Corrected syntax
const now = new Date().getTime() / 1000;
const secondsRemaining = Math.floor(selectedIntervalSeconds - (now % selectedIntervalSeconds));
const minutes = Math.floor(secondsRemaining / 60);