Migrate data pipeline from SQLite to PostgreSQL + Docker setup
- Add db.py PostgreSQL abstraction layer (connection, upsert, table mgmt) - Replace sqlite3 with psycopg2 in: live_candle_fetcher, resampler, data_fetcher, fetch_history, import_csv, indicators, base_strategy - Sanitize table names (colons -> underscores) for PostgreSQL compat - Replace INSERT OR REPLACE with ON CONFLICT upserts - Replace pandas to_sql() with batch upsert_candles() - Add scripts: resampler_loop, gap_detector, backup_runner, cron_scheduler - Add migrate_sqlite_to_pg.py for one-time data migration - Add Dockerfile, docker-compose.yml, supervisord.conf - Add postgres/postgresql.conf tuned for 4GB RAM (Synology DS1513+) - Add .dockerignore, .env.docker.example, secrets template - Update requirements.txt (psycopg2-binary), .gitignore - Add MIGRATION_PLAN.md with full plan and todo list
This commit is contained in:
@ -4,7 +4,7 @@ import json
|
||||
import os
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
import sqlite3
|
||||
import psycopg2
|
||||
import multiprocessing
|
||||
import time
|
||||
|
||||
@ -27,7 +27,7 @@ class BaseStrategy(ABC):
|
||||
|
||||
self.coin = params.get("coin", "N/A")
|
||||
self.timeframe = params.get("timeframe", "N/A")
|
||||
self.db_path = os.path.join("_data", "market_data.db")
|
||||
self.db_path = os.environ.get("PG_CONN_STR", "postgresql://hyper:hyper@localhost:5432/hyper")
|
||||
self.status_file_path = os.path.join("_data", f"strategy_status_{self.strategy_name}.json")
|
||||
|
||||
self.current_signal = "INIT"
|
||||
@ -38,19 +38,23 @@ class BaseStrategy(ABC):
|
||||
|
||||
def load_data(self) -> pd.DataFrame:
|
||||
"""Loads historical data for the configured coin and timeframe."""
|
||||
table_name = f"{self.coin}_{self.timeframe}"
|
||||
table_name = f"{self.coin.replace(':', '_')}_{self.timeframe}"
|
||||
|
||||
periods = [v for k, v in self.params.items() if 'period' in k or '_ma' in k or 'slow' in k or 'fast' in k]
|
||||
limit = max(periods) + 50 if periods else 500
|
||||
|
||||
try:
|
||||
with sqlite3.connect(f"file:{self.db_path}?mode=ro", uri=True) as conn:
|
||||
conn = psycopg2.connect(self.db_path)
|
||||
conn.set_session(readonly=True)
|
||||
try:
|
||||
query = f'SELECT * FROM "{table_name}" ORDER BY datetime_utc DESC LIMIT {limit}'
|
||||
df = pd.read_sql(query, conn, parse_dates=['datetime_utc'])
|
||||
if df.empty: return pd.DataFrame()
|
||||
df.set_index('datetime_utc', inplace=True)
|
||||
df.sort_index(inplace=True)
|
||||
return df
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to load data from table '{table_name}': {e}")
|
||||
return pd.DataFrame()
|
||||
|
||||
Reference in New Issue
Block a user