Migrate data pipeline from SQLite to PostgreSQL + Docker setup

- Add db.py PostgreSQL abstraction layer (connection, upsert, table mgmt)
- Replace sqlite3 with psycopg2 in: live_candle_fetcher, resampler,
  data_fetcher, fetch_history, import_csv, indicators, base_strategy
- Sanitize table names (colons -> underscores) for PostgreSQL compat
- Replace INSERT OR REPLACE with ON CONFLICT upserts
- Replace pandas to_sql() with batch upsert_candles()
- Add scripts: resampler_loop, gap_detector, backup_runner, cron_scheduler
- Add migrate_sqlite_to_pg.py for one-time data migration
- Add Dockerfile, docker-compose.yml, supervisord.conf
- Add postgres/postgresql.conf tuned for 4GB RAM (Synology DS1513+)
- Add .dockerignore, .env.docker.example, secrets template
- Update requirements.txt (psycopg2-binary), .gitignore
- Add MIGRATION_PLAN.md with full plan and todo list

live_candle_fetcher.py

@@ -7,7 +7,7 @@ import time
 from datetime import datetime, timezone
 from hyperliquid.info import Info
 from hyperliquid.utils import constants
-import sqlite3
+import db
 from queue import Queue
 from threading import Thread
 
@@ -22,7 +22,7 @@ class LiveCandleFetcher:
 
     def __init__(self, log_level: str, coins: list):
         setup_logging(log_level, 'LiveCandleFetcher')
-        self.db_path = os.path.join("_data", "market_data.db")
+        self.db_path = os.environ.get("PG_CONN_STR", "postgresql://hyper:hyper@localhost:5432/hyper")
         self.coins_to_watch = set(coins)
         if not self.coins_to_watch:
             logging.error("No coins provided to watch. Exiting.")
@@ -34,65 +34,15 @@ class LiveCandleFetcher:
 
     def _ensure_tables_exist(self):
         """
-        Ensures that all necessary tables are created with the correct schema and PRIMARY KEY.
-        If a table exists with an incorrect schema, it attempts to migrate the data.
+        Ensures that all necessary tables are created with the correct schema.
+        Uses db.create_candle_table() which is idempotent (CREATE TABLE IF NOT EXISTS).
         """
-        with sqlite3.connect(self.db_path) as conn:
-            for coin in self.coins_to_watch:
-                table_name = f"{coin}_1m"
-                cursor = conn.cursor()
-                cursor.execute(f"PRAGMA table_info('{table_name}')")
-                columns = cursor.fetchall()
-                
-                if columns:
-                    pk_found = any(col[1] == 'timestamp_ms' and col[5] == 1 for col in columns)
-                    if not pk_found:
-                        logging.warning(f"Schema migration needed for table '{table_name}': 'timestamp_ms' is not the PRIMARY KEY.")
-                        logging.warning("Attempting to automatically rebuild the table...")
-                        try:
-                            # 1. Rename old table
-                            conn.execute(f'ALTER TABLE "{table_name}" RENAME TO "{table_name}_old"')
-                            logging.info(f"  -> Renamed existing table to '{table_name}_old'.")
-                            
-                            # 2. Create new table with correct schema
-                            self._create_candle_table(conn, table_name)
-                            logging.info(f"  -> Created new '{table_name}' table with correct schema.")
-                            
-                            # 3. Copy unique data from old table to new table
-                            conn.execute(f'''
-                                INSERT OR IGNORE INTO "{table_name}" (datetime_utc, timestamp_ms, open, high, low, close, volume, number_of_trades)
-                                SELECT datetime_utc, timestamp_ms, open, high, low, close, volume, number_of_trades
-                                FROM "{table_name}_old"
-                            ''')
-                            conn.commit()
-                            logging.info("  -> Copied data to new table.")
-
-                            # 4. Drop the old table
-                            conn.execute(f'DROP TABLE "{table_name}_old"')
-                            logging.info(f"  -> Removed old table. Migration for '{table_name}' complete.")
-                        except Exception as e:
-                            logging.error(f"FATAL: Automatic schema migration for '{table_name}' failed: {e}")
-                            logging.error("Please delete the database file '_data/market_data.db' manually and restart.")
-                            sys.exit(1)
-                else:
-                    # If table does not exist, create it
-                    self._create_candle_table(conn, table_name)
-            logging.info("Database tables verified.")
-
-    def _create_candle_table(self, conn, table_name: str):
-        """Creates a new candle table with the correct schema."""
-        conn.execute(f'''
-            CREATE TABLE "{table_name}" (
-                datetime_utc TEXT,
-                timestamp_ms INTEGER PRIMARY KEY,
-                open REAL,
-                high REAL,
-                low REAL,
-                close REAL,
-                volume REAL,
-                number_of_trades INTEGER
-            )
-        ''')
+        conn = db.get_connection()
+        for coin in self.coins_to_watch:
+            table_name = db.sanitize_table_name(coin, "1m")
+            db.create_candle_table(conn, table_name)
+        conn.close()
+        logging.info("Database tables verified.")
 
     def on_message(self, message):
         """
@@ -112,6 +62,7 @@ class LiveCandleFetcher:
         This is the "Consumer" thread. It runs forever, pulling candles from the
         queue and writing them to the database, ensuring all writes are serial.
         """
+        conn = db.get_connection()
         while True:
             try:
                 candle = self.candle_queue.get()
@@ -122,7 +73,7 @@ class LiveCandleFetcher:
                 if not coin:
                     continue
 
-                table_name = f"{coin}_1m"
+                table_name = db.sanitize_table_name(coin, "1m")
                 record = (
                     datetime.fromtimestamp(candle['t'] / 1000, tz=timezone.utc).strftime('%Y-%m-%d %H:%M:%S'),
                     candle['t'],
@@ -130,24 +81,21 @@ class LiveCandleFetcher:
                     candle.get('v'), candle.get('n')
                 )
                 
-                with sqlite3.connect(self.db_path) as conn:
-                    conn.execute(f'''
-                        INSERT OR REPLACE INTO "{table_name}" (datetime_utc, timestamp_ms, open, high, low, close, volume, number_of_trades)
-                        VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-                    ''', record)
-                    conn.commit()
+                db.upsert_candles(conn, table_name, [record])
                 logging.debug(f"Upserted candle for {coin} at {record[0]}")
 
             except Exception as e:
                 logging.error(f"Error in database writer thread: {e}")
+        conn.close()
 
     def _get_last_timestamp_from_db(self, coin: str) -> int:
         """Gets the most recent millisecond timestamp from a coin's 1m table."""
-        table_name = f"{coin}_1m"
+        table_name = db.sanitize_table_name(coin, "1m")
         try:
-            with sqlite3.connect(self.db_path) as conn:
-                result = conn.execute(f'SELECT MAX(timestamp_ms) FROM "{table_name}"').fetchone()
-                return int(result[0]) if result and result[0] is not None else None
+            conn = db.get_connection()
+            result = db.get_last_timestamp(conn, table_name)
+            conn.close()
+            return result
         except Exception as e:
             logging.error(f"Could not read last timestamp from table '{table_name}': {e}")
             return None