Migrate data pipeline from SQLite to PostgreSQL + Docker setup

- Add db.py PostgreSQL abstraction layer (connection, upsert, table mgmt)
- Replace sqlite3 with psycopg2 in: live_candle_fetcher, resampler,
  data_fetcher, fetch_history, import_csv, indicators, base_strategy
- Sanitize table names (colons -> underscores) for PostgreSQL compat
- Replace INSERT OR REPLACE with ON CONFLICT upserts
- Replace pandas to_sql() with batch upsert_candles()
- Add scripts: resampler_loop, gap_detector, backup_runner, cron_scheduler
- Add migrate_sqlite_to_pg.py for one-time data migration
- Add Dockerfile, docker-compose.yml, supervisord.conf
- Add postgres/postgresql.conf tuned for 4GB RAM (Synology DS1513+)
- Add .dockerignore, .env.docker.example, secrets template
- Update requirements.txt (psycopg2-binary), .gitignore
- Add MIGRATION_PLAN.md with full plan and todo list

data_fetcher.py

@@ -4,7 +4,7 @@ import logging
 import os
 import sys
 import time
-import sqlite3
+import db
 import pandas as pd
 from datetime import datetime, timedelta, timezone
 
@@ -26,7 +26,7 @@ class CandleFetcherDB:
         self.coins = self._resolve_coins(coins_to_fetch)
         self.interval = interval
         self.days_back = days_back
-        self.db_path = os.path.join("_data", "market_data.db")
+        self.db_path = os.environ.get("PG_CONN_STR", "postgresql://hyper:hyper@localhost:5432/hyper")
         self.column_rename_map = {
             't': 'timestamp_ms', 'o': 'open', 'h': 'high', 'l': 'low', 'c': 'close', 'v': 'volume', 'n': 'number_of_trades'
         }
@@ -47,13 +47,12 @@ class CandleFetcherDB:
 
     def run(self):
         """Starts the data fetching process and reports status after each coin."""
-        with sqlite3.connect(self.db_path, timeout=10) as self.conn:
-            self.conn.execute("PRAGMA journal_mode=WAL;")
-            for coin in self.coins:
-                logging.info(f"--- Starting process for {coin} ---")
-                num_updated = self._update_data_for_coin(coin)
-                self._report_status(coin, num_updated)
-                time.sleep(1)
+        self.conn = db.get_connection()
+        for coin in self.coins:
+            logging.info(f"--- Starting process for {coin} ---")
+            num_updated = self._update_data_for_coin(coin)
+            self._report_status(coin, num_updated)
+            time.sleep(1)
 
     def _report_status(self, last_coin: str, num_updated: int):
         """Saves the status of the fetcher run to a JSON file."""
@@ -73,11 +72,11 @@ class CandleFetcherDB:
 
     def _get_start_time(self, coin: str) -> (int, bool):
         """Checks the database for an existing table and returns the last timestamp."""
-        table_name = f"{coin}_{self.interval}"
+        table_name = db.sanitize_table_name(coin, self.interval)
         try:
             cursor = self.conn.cursor()
-            cursor.execute(f"SELECT name FROM sqlite_master WHERE type='table' AND name='{table_name}';")
-            if cursor.fetchone():
+            cursor.execute("SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = %s)", (table_name,))
+            if cursor.fetchone()[0]:
                 query = f'SELECT MAX(timestamp_ms) FROM "{table_name}"'
                 last_ts = pd.read_sql(query, self.conn).iloc[0, 0]
                 if pd.notna(last_ts):
@@ -150,23 +149,28 @@ class CandleFetcherDB:
         return None
 
     def _save_to_sqlite_with_pandas(self, df: pd.DataFrame, coin: str, is_append: bool) -> int:
-        """Saves a pandas DataFrame to an SQLite table and returns the number of saved rows."""
-        table_name = f"{coin}_{self.interval}"
+        """Saves a pandas DataFrame to a PostgreSQL table and returns the number of saved rows."""
+        table_name = db.sanitize_table_name(coin, self.interval)
         try:
             df.rename(columns=self.column_rename_map, inplace=True)
             df['datetime_utc'] = pd.to_datetime(df['timestamp_ms'], unit='ms')
             final_df = df[['datetime_utc', 'timestamp_ms', 'open', 'high', 'low', 'close', 'volume', 'number_of_trades']]
 
-            write_mode = 'append' if is_append else 'replace'
-            final_df.to_sql(table_name, self.conn, if_exists=write_mode, index=False)
-            
-            self.conn.execute(f'CREATE INDEX IF NOT EXISTS "idx_{table_name}_time" ON "{table_name}"(datetime_utc);')
+            if not is_append:
+                # Drop and recreate the table for 'replace' mode
+                with self.conn.cursor() as cur:
+                    cur.execute(f'DROP TABLE IF EXISTS "{table_name}"')
+                    self.conn.commit()
+                db.create_candle_table(self.conn, table_name)
+
+            records = list(final_df.itertuples(index=False, name=None))
+            db.upsert_candles(self.conn, table_name, records)
             
             num_saved = len(final_df)
             logging.info(f"Successfully saved {num_saved} candles to table '{table_name}'")
             return num_saved
         except Exception as e:
-            logging.error(f"Failed to write to SQLite table '{table_name}': {e}")
+            logging.error(f"Failed to write to table '{table_name}': {e}")
             return 0