Migrate data pipeline from SQLite to PostgreSQL + Docker setup
- Add db.py PostgreSQL abstraction layer (connection, upsert, table mgmt) - Replace sqlite3 with psycopg2 in: live_candle_fetcher, resampler, data_fetcher, fetch_history, import_csv, indicators, base_strategy - Sanitize table names (colons -> underscores) for PostgreSQL compat - Replace INSERT OR REPLACE with ON CONFLICT upserts - Replace pandas to_sql() with batch upsert_candles() - Add scripts: resampler_loop, gap_detector, backup_runner, cron_scheduler - Add migrate_sqlite_to_pg.py for one-time data migration - Add Dockerfile, docker-compose.yml, supervisord.conf - Add postgres/postgresql.conf tuned for 4GB RAM (Synology DS1513+) - Add .dockerignore, .env.docker.example, secrets template - Update requirements.txt (psycopg2-binary), .gitignore - Add MIGRATION_PLAN.md with full plan and todo list
This commit is contained in:
@ -7,7 +7,8 @@ and custom functions.
|
||||
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import psycopg2
|
||||
from contextlib import closing
|
||||
import importlib
|
||||
import logging
|
||||
import pandas as pd
|
||||
@ -36,9 +37,9 @@ class IndicatorCalculator:
|
||||
|
||||
def _get_latest_close(self, coin, timeframe="1m"):
|
||||
"""Get the latest close price from a candle table."""
|
||||
table = f"{coin}_{timeframe}"
|
||||
table = f"{coin.replace(':', '_')}_{timeframe}"
|
||||
try:
|
||||
with sqlite3.connect(self.db_path) as conn:
|
||||
with closing(psycopg2.connect(self.db_path)) as conn:
|
||||
result = conn.execute(
|
||||
f'SELECT close FROM "{table}" ORDER BY timestamp_ms DESC LIMIT 1'
|
||||
).fetchone()
|
||||
@ -49,9 +50,9 @@ class IndicatorCalculator:
|
||||
|
||||
def _get_close_n_candles_ago(self, coin, timeframe, n=1):
|
||||
"""Get the close price from n candles ago (n=1 = most recent completed candle)."""
|
||||
table = f"{coin}_{timeframe}"
|
||||
table = f"{coin.replace(':', '_')}_{timeframe}"
|
||||
try:
|
||||
with sqlite3.connect(self.db_path) as conn:
|
||||
with closing(psycopg2.connect(self.db_path)) as conn:
|
||||
result = conn.execute(
|
||||
f'SELECT close FROM "{table}" ORDER BY timestamp_ms DESC LIMIT 1 OFFSET {n}'
|
||||
).fetchone()
|
||||
@ -62,9 +63,9 @@ class IndicatorCalculator:
|
||||
|
||||
def _get_all_closes(self, coin, timeframe="1d"):
|
||||
"""Get all close prices from a candle table, ordered by time."""
|
||||
table = f"{coin}_{timeframe}"
|
||||
table = f"{coin.replace(':', '_')}_{timeframe}"
|
||||
try:
|
||||
with sqlite3.connect(self.db_path) as conn:
|
||||
with closing(psycopg2.connect(self.db_path)) as conn:
|
||||
result = conn.execute(
|
||||
f'SELECT close FROM "{table}" ORDER BY timestamp_ms'
|
||||
).fetchall()
|
||||
@ -75,10 +76,10 @@ class IndicatorCalculator:
|
||||
|
||||
def _get_all_ratio(self, num_coin, den_coin, timeframe="1d"):
|
||||
"""Get all ratio values (num/den) from candle tables, ordered by time."""
|
||||
num_table = f"{num_coin}_{timeframe}"
|
||||
den_table = f"{den_coin}_{timeframe}"
|
||||
num_table = f"{num_coin.replace(':', '_')}_{timeframe}"
|
||||
den_table = f"{den_coin.replace(':', '_')}_{timeframe}"
|
||||
try:
|
||||
with sqlite3.connect(self.db_path) as conn:
|
||||
with closing(psycopg2.connect(self.db_path)) as conn:
|
||||
result = conn.execute(
|
||||
f'SELECT n.close / d.close as ratio '
|
||||
f'FROM "{num_table}" n '
|
||||
@ -92,10 +93,10 @@ class IndicatorCalculator:
|
||||
|
||||
def _get_all_spread(self, num_coin, den_coin, timeframe="1d"):
|
||||
"""Get all spread values (num - den) from candle tables, ordered by time."""
|
||||
num_table = f"{num_coin}_{timeframe}"
|
||||
den_table = f"{den_coin}_{timeframe}"
|
||||
num_table = f"{num_coin.replace(':', '_')}_{timeframe}"
|
||||
den_table = f"{den_coin.replace(':', '_')}_{timeframe}"
|
||||
try:
|
||||
with sqlite3.connect(self.db_path) as conn:
|
||||
with closing(psycopg2.connect(self.db_path)) as conn:
|
||||
result = conn.execute(
|
||||
f'SELECT n.close - d.close as spread '
|
||||
f'FROM "{num_table}" n '
|
||||
@ -109,10 +110,10 @@ class IndicatorCalculator:
|
||||
|
||||
def _get_all_diff_pct(self, num_coin, den_coin, timeframe="1d"):
|
||||
"""Get all percentage difference values ((num-den)/den*100) from candle tables."""
|
||||
num_table = f"{num_coin}_{timeframe}"
|
||||
den_table = f"{den_coin}_{timeframe}"
|
||||
num_table = f"{num_coin.replace(':', '_')}_{timeframe}"
|
||||
den_table = f"{den_coin.replace(':', '_')}_{timeframe}"
|
||||
try:
|
||||
with sqlite3.connect(self.db_path) as conn:
|
||||
with closing(psycopg2.connect(self.db_path)) as conn:
|
||||
result = conn.execute(
|
||||
f'SELECT (n.close - d.close) / d.close * 100 as diff_pct '
|
||||
f'FROM "{num_table}" n '
|
||||
|
||||
Reference in New Issue
Block a user