Migrate data pipeline from SQLite to PostgreSQL + Docker setup
- Add db.py PostgreSQL abstraction layer (connection, upsert, table mgmt) - Replace sqlite3 with psycopg2 in: live_candle_fetcher, resampler, data_fetcher, fetch_history, import_csv, indicators, base_strategy - Sanitize table names (colons -> underscores) for PostgreSQL compat - Replace INSERT OR REPLACE with ON CONFLICT upserts - Replace pandas to_sql() with batch upsert_candles() - Add scripts: resampler_loop, gap_detector, backup_runner, cron_scheduler - Add migrate_sqlite_to_pg.py for one-time data migration - Add Dockerfile, docker-compose.yml, supervisord.conf - Add postgres/postgresql.conf tuned for 4GB RAM (Synology DS1513+) - Add .dockerignore, .env.docker.example, secrets template - Update requirements.txt (psycopg2-binary), .gitignore - Add MIGRATION_PLAN.md with full plan and todo list
This commit is contained in:
DiTus
74
scripts/backup_runner.py
Normal file
74
scripts/backup_runner.py
Normal file
@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Backup Runner
|
||||
|
||||
Creates a daily pg_dump backup of the PostgreSQL database, compresses it,
|
||||
and retains only the last 7 days of backups.
|
||||
|
||||
Designed to run as a periodic cron job (daily) inside the Docker container.
|
||||
Backups are written to /backups which is mounted to a Synology shared folder.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from logging_utils import setup_logging
|
||||
|
||||
BACKUP_DIR = "/backups"
|
||||
RETENTION_DAYS = 7
|
||||
|
||||
|
||||
def run_backup():
|
||||
"""Run pg_dump and compress the output."""
|
||||
today = datetime.now().strftime("%Y%m%d")
|
||||
backup_file = os.path.join(BACKUP_DIR, f"hyper_{today}.sql.gz")
|
||||
|
||||
os.makedirs(BACKUP_DIR, exist_ok=True)
|
||||
|
||||
logging.info(f"Starting backup to {backup_file}")
|
||||
|
||||
cmd = f"pg_dump -h postgres -U hyper hyper | gzip > {backup_file}"
|
||||
result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
|
||||
|
||||
if result.returncode == 0:
|
||||
file_size = os.path.getsize(backup_file)
|
||||
logging.info(f"Backup completed: {backup_file} ({file_size:,} bytes)")
|
||||
else:
|
||||
logging.error(f"Backup failed: {result.stderr}")
|
||||
if os.path.exists(backup_file):
|
||||
os.remove(backup_file)
|
||||
|
||||
cleanup_old_backups()
|
||||
|
||||
|
||||
def cleanup_old_backups():
|
||||
"""Delete backup files older than RETENTION_DAYS."""
|
||||
cutoff = datetime.now() - timedelta(days=RETENTION_DAYS)
|
||||
|
||||
if not os.path.exists(BACKUP_DIR):
|
||||
return
|
||||
|
||||
for filename in os.listdir(BACKUP_DIR):
|
||||
if filename.startswith("hyper_") and filename.endswith(".sql.gz"):
|
||||
filepath = os.path.join(BACKUP_DIR, filename)
|
||||
mtime = datetime.fromtimestamp(os.path.getmtime(filepath))
|
||||
if mtime < cutoff:
|
||||
os.remove(filepath)
|
||||
logging.info(f"Deleted old backup: {filename}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Run PostgreSQL backup.")
|
||||
parser.add_argument("--log-level", default="normal", choices=['off', 'normal', 'debug'])
|
||||
args = parser.parse_args()
|
||||
|
||||
setup_logging(args.log_level, 'BackupRunner')
|
||||
|
||||
run_backup()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
99
scripts/cron_scheduler.py
Normal file
99
scripts/cron_scheduler.py
Normal file
@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Cron Scheduler
|
||||
|
||||
Runs periodic maintenance tasks inside the Docker container using the
|
||||
`schedule` library. This replaces a system cron daemon and keeps all
|
||||
scheduling logic in Python.
|
||||
|
||||
Scheduled tasks:
|
||||
- data_fetcher.py — daily at 02:00 UTC (full historical catch-up)
|
||||
- fetch_history.py — daily at 03:00 UTC (additional history fetch)
|
||||
- gap_detector.py — hourly at :15 (fill missing 1m candles)
|
||||
- backup_runner.py — daily at 04:00 UTC (pg_dump backup)
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import schedule
|
||||
import signal
|
||||
|
||||
from logging_utils import setup_logging
|
||||
|
||||
shutdown_requested = False
|
||||
|
||||
|
||||
def handle_shutdown(signum, frame):
|
||||
global shutdown_requested
|
||||
shutdown_requested = True
|
||||
|
||||
|
||||
def run_data_fetcher():
|
||||
try:
|
||||
logging.info("Running data_fetcher.py")
|
||||
subprocess.run([
|
||||
sys.executable, "data_fetcher.py",
|
||||
"--coins", "BTC", "ETH", "SOL", "BNB", "HYPE", "SUI",
|
||||
"xyz:BRENTOIL", "xyz:CL", "xyz:GOLD", "xyz:SILVER",
|
||||
"mkts:USTECH", "xyz:XYZ100",
|
||||
"--interval", "1m", "--days", "7", "--log-level", "normal"
|
||||
], check=True)
|
||||
except Exception as e:
|
||||
logging.error(f"Data fetcher failed: {e}")
|
||||
|
||||
|
||||
def run_fetch_history():
|
||||
try:
|
||||
logging.info("Running fetch_history.py")
|
||||
subprocess.run([sys.executable, "fetch_history.py", "--log-level", "normal"], check=True)
|
||||
except Exception as e:
|
||||
logging.error(f"Fetch history failed: {e}")
|
||||
|
||||
|
||||
def run_gap_detector():
|
||||
try:
|
||||
logging.info("Running gap_detector.py")
|
||||
subprocess.run([sys.executable, "scripts/gap_detector.py", "--log-level", "normal"], check=True)
|
||||
except Exception as e:
|
||||
logging.error(f"Gap detector failed: {e}")
|
||||
|
||||
|
||||
def run_backup():
|
||||
try:
|
||||
logging.info("Running backup_runner.py")
|
||||
subprocess.run([sys.executable, "scripts/backup_runner.py", "--log-level", "normal"], check=True)
|
||||
except Exception as e:
|
||||
logging.error(f"Backup failed: {e}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Run periodic maintenance tasks.")
|
||||
parser.add_argument("--log-level", default="normal", choices=['off', 'normal', 'debug'])
|
||||
args = parser.parse_args()
|
||||
|
||||
signal.signal(signal.SIGTERM, handle_shutdown)
|
||||
signal.signal(signal.SIGINT, handle_shutdown)
|
||||
|
||||
setup_logging(args.log_level, 'CronScheduler')
|
||||
|
||||
# Schedule jobs
|
||||
schedule.every().day.at("02:00").do(run_data_fetcher)
|
||||
schedule.every().day.at("03:00").do(run_fetch_history)
|
||||
schedule.every().hour.at(":15").do(run_gap_detector)
|
||||
schedule.every().day.at("04:00").do(run_backup)
|
||||
|
||||
logging.info("Cron scheduler started")
|
||||
|
||||
while not shutdown_requested:
|
||||
schedule.run_pending()
|
||||
time.sleep(1)
|
||||
|
||||
logging.info("Cron scheduler shutting down.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
137
scripts/gap_detector.py
Normal file
137
scripts/gap_detector.py
Normal file
@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Gap Detector
|
||||
|
||||
Detects missing 1-minute candle data in the PostgreSQL database and
|
||||
backfills gaps by fetching historical data from the Hyperliquid HTTP API.
|
||||
|
||||
Designed to run as a periodic cron job (hourly) inside the Docker container.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pandas as pd
|
||||
from hyperliquid.info import Info
|
||||
from hyperliquid.utils import constants
|
||||
|
||||
from logging_utils import setup_logging
|
||||
from db import get_connection, sanitize_table_name, upsert_candles
|
||||
|
||||
WATCHED_COINS = [
|
||||
"BTC", "ETH", "SOL", "BNB", "HYPE", "SUI",
|
||||
"xyz:BRENTOIL", "xyz:CL", "xyz:GOLD", "xyz:SILVER",
|
||||
"mkts:USTECH", "xyz:XYZ100"
|
||||
]
|
||||
|
||||
|
||||
def detect_and_fill_gaps(coin, conn):
|
||||
"""Detect gaps in the 1m data for a coin and backfill them."""
|
||||
table_name = sanitize_table_name(coin, "1m")
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
start = now - timedelta(hours=24)
|
||||
|
||||
query = f'SELECT timestamp_ms FROM "{table_name}" WHERE timestamp_ms >= %s ORDER BY timestamp_ms'
|
||||
df = pd.read_sql(query, conn, params=(int(start.timestamp() * 1000),))
|
||||
|
||||
if df.empty:
|
||||
logging.info(f"No data for {coin} in the last 24 hours, skipping gap detection")
|
||||
return
|
||||
|
||||
existing_timestamps = set(df['timestamp_ms'].tolist())
|
||||
|
||||
# Generate expected timestamps (every minute)
|
||||
expected_timestamps = set()
|
||||
current = start
|
||||
while current <= now:
|
||||
expected_timestamps.add(int(current.timestamp() * 1000))
|
||||
current += timedelta(minutes=1)
|
||||
|
||||
gaps = expected_timestamps - existing_timestamps
|
||||
|
||||
if not gaps:
|
||||
logging.info(f"No gaps found for {coin}")
|
||||
return
|
||||
|
||||
logging.info(f"Found {len(gaps)} gaps for {coin}, backfilling...")
|
||||
|
||||
# Find contiguous gap ranges
|
||||
sorted_gaps = sorted(gaps)
|
||||
gap_ranges = []
|
||||
gap_start = sorted_gaps[0]
|
||||
gap_end = sorted_gaps[0]
|
||||
|
||||
for ts in sorted_gaps[1:]:
|
||||
if ts == gap_end + 60000:
|
||||
gap_end = ts
|
||||
else:
|
||||
gap_ranges.append((gap_start, gap_end + 60000))
|
||||
gap_start = ts
|
||||
gap_end = ts
|
||||
gap_ranges.append((gap_start, gap_end + 60000))
|
||||
|
||||
info = Info(constants.MAINNET_API_URL, skip_ws=True)
|
||||
|
||||
for gap_start_ms, gap_end_ms in gap_ranges:
|
||||
logging.info(
|
||||
f"Backfilling gap for {coin}: "
|
||||
f"{datetime.fromtimestamp(gap_start_ms/1000, tz=timezone.utc)} "
|
||||
f"to {datetime.fromtimestamp(gap_end_ms/1000, tz=timezone.utc)}"
|
||||
)
|
||||
|
||||
current_start = gap_start_ms
|
||||
while current_start < gap_end_ms:
|
||||
try:
|
||||
batch = info.candles_snapshot(coin, "1m", current_start, gap_end_ms)
|
||||
if not batch:
|
||||
break
|
||||
|
||||
records = []
|
||||
for candle in batch:
|
||||
records.append((
|
||||
datetime.fromtimestamp(candle['t'] / 1000, tz=timezone.utc).strftime('%Y-%m-%d %H:%M:%S'),
|
||||
candle['t'],
|
||||
candle.get('o'), candle.get('h'), candle.get('l'), candle.get('c'),
|
||||
candle.get('v'), candle.get('n')
|
||||
))
|
||||
|
||||
upsert_candles(conn, table_name, records)
|
||||
|
||||
last_ts = batch[-1]['t']
|
||||
if last_ts < current_start:
|
||||
break
|
||||
current_start = last_ts + 1
|
||||
time.sleep(0.5)
|
||||
except Exception as e:
|
||||
logging.error(f"Error backfilling gap for {coin}: {e}")
|
||||
break
|
||||
|
||||
logging.info(f"Gap backfilling complete for {coin}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Detect and fill gaps in 1m candle data.")
|
||||
parser.add_argument("--log-level", default="normal", choices=['off', 'normal', 'debug'])
|
||||
args = parser.parse_args()
|
||||
|
||||
setup_logging(args.log_level, 'GapDetector')
|
||||
|
||||
conn = get_connection()
|
||||
|
||||
for coin in WATCHED_COINS:
|
||||
try:
|
||||
detect_and_fill_gaps(coin, conn)
|
||||
except Exception as e:
|
||||
logging.error(f"Error detecting gaps for {coin}: {e}")
|
||||
|
||||
conn.close()
|
||||
logging.info("Gap detection complete!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
69
scripts/resampler_loop.py
Normal file
69
scripts/resampler_loop.py
Normal file
@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Resampler Loop Wrapper
|
||||
|
||||
Runs the Resampler in a continuous loop, executing it once per minute.
|
||||
This replaces the schedule-based approach used in main_app.py and is
|
||||
designed to run as a supervisord-managed process inside Docker.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import signal
|
||||
|
||||
from logging_utils import setup_logging
|
||||
from resampler import Resampler, parse_timeframes
|
||||
|
||||
shutdown_requested = False
|
||||
|
||||
|
||||
def handle_shutdown(signum, frame):
|
||||
global shutdown_requested
|
||||
shutdown_requested = True
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Run the resampler in a continuous loop.")
|
||||
parser.add_argument("--coins", nargs='+', required=True, help="List of coins to process.")
|
||||
parser.add_argument("--timeframes", nargs='+', required=True, help="List of timeframes to generate.")
|
||||
parser.add_argument("--log-level", default="normal", choices=['off', 'normal', 'debug'])
|
||||
args = parser.parse_args()
|
||||
|
||||
signal.signal(signal.SIGTERM, handle_shutdown)
|
||||
signal.signal(signal.SIGINT, handle_shutdown)
|
||||
|
||||
setup_logging(args.log_level, 'ResamplerLoop')
|
||||
|
||||
timeframes_dict = parse_timeframes(args.timeframes)
|
||||
logging.info(f"Resampler loop started. Coins: {args.coins}, Timeframes: {list(timeframes_dict.keys())}")
|
||||
|
||||
while not shutdown_requested:
|
||||
try:
|
||||
# Pass a copy because Resampler.run() deletes '1m' from the dict
|
||||
timeframes_copy = dict(timeframes_dict)
|
||||
resampler = Resampler(
|
||||
log_level=args.log_level,
|
||||
coins=args.coins,
|
||||
timeframes=timeframes_copy
|
||||
)
|
||||
resampler.run()
|
||||
except Exception as e:
|
||||
logging.error(f"Resampler run failed: {e}")
|
||||
|
||||
if shutdown_requested:
|
||||
break
|
||||
|
||||
# Sleep for 60 seconds, but check shutdown flag every second
|
||||
for _ in range(60):
|
||||
if shutdown_requested:
|
||||
break
|
||||
time.sleep(1)
|
||||
|
||||
logging.info("Resampler loop shutting down.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user