Migrate data pipeline from SQLite to PostgreSQL + Docker setup

- Add db.py PostgreSQL abstraction layer (connection, upsert, table mgmt)
- Replace sqlite3 with psycopg2 in: live_candle_fetcher, resampler,
  data_fetcher, fetch_history, import_csv, indicators, base_strategy
- Sanitize table names (colons -> underscores) for PostgreSQL compat
- Replace INSERT OR REPLACE with ON CONFLICT upserts
- Replace pandas to_sql() with batch upsert_candles()
- Add scripts: resampler_loop, gap_detector, backup_runner, cron_scheduler
- Add migrate_sqlite_to_pg.py for one-time data migration
- Add Dockerfile, docker-compose.yml, supervisord.conf
- Add postgres/postgresql.conf tuned for 4GB RAM (Synology DS1513+)
- Add .dockerignore, .env.docker.example, secrets template
- Update requirements.txt (psycopg2-binary), .gitignore
- Add MIGRATION_PLAN.md with full plan and todo list
This commit is contained in:
DiTus
2026-07-30 22:14:31 +02:00
parent ade9b708a2
commit 7d702e9cbd
24 changed files with 1013 additions and 222 deletions

View File

@ -7,7 +7,8 @@ and custom functions.
import json
import os
import sqlite3
import psycopg2
from contextlib import closing
import importlib
import logging
import pandas as pd
@ -36,9 +37,9 @@ class IndicatorCalculator:
def _get_latest_close(self, coin, timeframe="1m"):
"""Get the latest close price from a candle table."""
table = f"{coin}_{timeframe}"
table = f"{coin.replace(':', '_')}_{timeframe}"
try:
with sqlite3.connect(self.db_path) as conn:
with closing(psycopg2.connect(self.db_path)) as conn:
result = conn.execute(
f'SELECT close FROM "{table}" ORDER BY timestamp_ms DESC LIMIT 1'
).fetchone()
@ -49,9 +50,9 @@ class IndicatorCalculator:
def _get_close_n_candles_ago(self, coin, timeframe, n=1):
"""Get the close price from n candles ago (n=1 = most recent completed candle)."""
table = f"{coin}_{timeframe}"
table = f"{coin.replace(':', '_')}_{timeframe}"
try:
with sqlite3.connect(self.db_path) as conn:
with closing(psycopg2.connect(self.db_path)) as conn:
result = conn.execute(
f'SELECT close FROM "{table}" ORDER BY timestamp_ms DESC LIMIT 1 OFFSET {n}'
).fetchone()
@ -62,9 +63,9 @@ class IndicatorCalculator:
def _get_all_closes(self, coin, timeframe="1d"):
"""Get all close prices from a candle table, ordered by time."""
table = f"{coin}_{timeframe}"
table = f"{coin.replace(':', '_')}_{timeframe}"
try:
with sqlite3.connect(self.db_path) as conn:
with closing(psycopg2.connect(self.db_path)) as conn:
result = conn.execute(
f'SELECT close FROM "{table}" ORDER BY timestamp_ms'
).fetchall()
@ -75,10 +76,10 @@ class IndicatorCalculator:
def _get_all_ratio(self, num_coin, den_coin, timeframe="1d"):
"""Get all ratio values (num/den) from candle tables, ordered by time."""
num_table = f"{num_coin}_{timeframe}"
den_table = f"{den_coin}_{timeframe}"
num_table = f"{num_coin.replace(':', '_')}_{timeframe}"
den_table = f"{den_coin.replace(':', '_')}_{timeframe}"
try:
with sqlite3.connect(self.db_path) as conn:
with closing(psycopg2.connect(self.db_path)) as conn:
result = conn.execute(
f'SELECT n.close / d.close as ratio '
f'FROM "{num_table}" n '
@ -92,10 +93,10 @@ class IndicatorCalculator:
def _get_all_spread(self, num_coin, den_coin, timeframe="1d"):
"""Get all spread values (num - den) from candle tables, ordered by time."""
num_table = f"{num_coin}_{timeframe}"
den_table = f"{den_coin}_{timeframe}"
num_table = f"{num_coin.replace(':', '_')}_{timeframe}"
den_table = f"{den_coin.replace(':', '_')}_{timeframe}"
try:
with sqlite3.connect(self.db_path) as conn:
with closing(psycopg2.connect(self.db_path)) as conn:
result = conn.execute(
f'SELECT n.close - d.close as spread '
f'FROM "{num_table}" n '
@ -109,10 +110,10 @@ class IndicatorCalculator:
def _get_all_diff_pct(self, num_coin, den_coin, timeframe="1d"):
"""Get all percentage difference values ((num-den)/den*100) from candle tables."""
num_table = f"{num_coin}_{timeframe}"
den_table = f"{den_coin}_{timeframe}"
num_table = f"{num_coin.replace(':', '_')}_{timeframe}"
den_table = f"{den_coin.replace(':', '_')}_{timeframe}"
try:
with sqlite3.connect(self.db_path) as conn:
with closing(psycopg2.connect(self.db_path)) as conn:
result = conn.execute(
f'SELECT (n.close - d.close) / d.close * 100 as diff_pct '
f'FROM "{num_table}" n '