Migrate data pipeline from SQLite to PostgreSQL + Docker setup
- Add db.py PostgreSQL abstraction layer (connection, upsert, table mgmt) - Replace sqlite3 with psycopg2 in: live_candle_fetcher, resampler, data_fetcher, fetch_history, import_csv, indicators, base_strategy - Sanitize table names (colons -> underscores) for PostgreSQL compat - Replace INSERT OR REPLACE with ON CONFLICT upserts - Replace pandas to_sql() with batch upsert_candles() - Add scripts: resampler_loop, gap_detector, backup_runner, cron_scheduler - Add migrate_sqlite_to_pg.py for one-time data migration - Add Dockerfile, docker-compose.yml, supervisord.conf - Add postgres/postgresql.conf tuned for 4GB RAM (Synology DS1513+) - Add .dockerignore, .env.docker.example, secrets template - Update requirements.txt (psycopg2-binary), .gitignore - Add MIGRATION_PLAN.md with full plan and todo list
This commit is contained in:
69
scripts/resampler_loop.py
Normal file
69
scripts/resampler_loop.py
Normal file
@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Resampler Loop Wrapper
|
||||
|
||||
Runs the Resampler in a continuous loop, executing it once per minute.
|
||||
This replaces the schedule-based approach used in main_app.py and is
|
||||
designed to run as a supervisord-managed process inside Docker.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import signal
|
||||
|
||||
from logging_utils import setup_logging
|
||||
from resampler import Resampler, parse_timeframes
|
||||
|
||||
shutdown_requested = False
|
||||
|
||||
|
||||
def handle_shutdown(signum, frame):
|
||||
global shutdown_requested
|
||||
shutdown_requested = True
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Run the resampler in a continuous loop.")
|
||||
parser.add_argument("--coins", nargs='+', required=True, help="List of coins to process.")
|
||||
parser.add_argument("--timeframes", nargs='+', required=True, help="List of timeframes to generate.")
|
||||
parser.add_argument("--log-level", default="normal", choices=['off', 'normal', 'debug'])
|
||||
args = parser.parse_args()
|
||||
|
||||
signal.signal(signal.SIGTERM, handle_shutdown)
|
||||
signal.signal(signal.SIGINT, handle_shutdown)
|
||||
|
||||
setup_logging(args.log_level, 'ResamplerLoop')
|
||||
|
||||
timeframes_dict = parse_timeframes(args.timeframes)
|
||||
logging.info(f"Resampler loop started. Coins: {args.coins}, Timeframes: {list(timeframes_dict.keys())}")
|
||||
|
||||
while not shutdown_requested:
|
||||
try:
|
||||
# Pass a copy because Resampler.run() deletes '1m' from the dict
|
||||
timeframes_copy = dict(timeframes_dict)
|
||||
resampler = Resampler(
|
||||
log_level=args.log_level,
|
||||
coins=args.coins,
|
||||
timeframes=timeframes_copy
|
||||
)
|
||||
resampler.run()
|
||||
except Exception as e:
|
||||
logging.error(f"Resampler run failed: {e}")
|
||||
|
||||
if shutdown_requested:
|
||||
break
|
||||
|
||||
# Sleep for 60 seconds, but check shutdown flag every second
|
||||
for _ in range(60):
|
||||
if shutdown_requested:
|
||||
break
|
||||
time.sleep(1)
|
||||
|
||||
logging.info("Resampler loop shutting down.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user