7d702e9cbd
- Add db.py PostgreSQL abstraction layer (connection, upsert, table mgmt) - Replace sqlite3 with psycopg2 in: live_candle_fetcher, resampler, data_fetcher, fetch_history, import_csv, indicators, base_strategy - Sanitize table names (colons -> underscores) for PostgreSQL compat - Replace INSERT OR REPLACE with ON CONFLICT upserts - Replace pandas to_sql() with batch upsert_candles() - Add scripts: resampler_loop, gap_detector, backup_runner, cron_scheduler - Add migrate_sqlite_to_pg.py for one-time data migration - Add Dockerfile, docker-compose.yml, supervisord.conf - Add postgres/postgresql.conf tuned for 4GB RAM (Synology DS1513+) - Add .dockerignore, .env.docker.example, secrets template - Update requirements.txt (psycopg2-binary), .gitignore - Add MIGRATION_PLAN.md with full plan and todo list
100 lines
2.8 KiB
Python
100 lines
2.8 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Cron Scheduler
|
|
|
|
Runs periodic maintenance tasks inside the Docker container using the
|
|
`schedule` library. This replaces a system cron daemon and keeps all
|
|
scheduling logic in Python.
|
|
|
|
Scheduled tasks:
|
|
- data_fetcher.py — daily at 02:00 UTC (full historical catch-up)
|
|
- fetch_history.py — daily at 03:00 UTC (additional history fetch)
|
|
- gap_detector.py — hourly at :15 (fill missing 1m candles)
|
|
- backup_runner.py — daily at 04:00 UTC (pg_dump backup)
|
|
"""
|
|
|
|
import argparse
|
|
import logging
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import schedule
|
|
import signal
|
|
|
|
from logging_utils import setup_logging
|
|
|
|
shutdown_requested = False
|
|
|
|
|
|
def handle_shutdown(signum, frame):
|
|
global shutdown_requested
|
|
shutdown_requested = True
|
|
|
|
|
|
def run_data_fetcher():
|
|
try:
|
|
logging.info("Running data_fetcher.py")
|
|
subprocess.run([
|
|
sys.executable, "data_fetcher.py",
|
|
"--coins", "BTC", "ETH", "SOL", "BNB", "HYPE", "SUI",
|
|
"xyz:BRENTOIL", "xyz:CL", "xyz:GOLD", "xyz:SILVER",
|
|
"mkts:USTECH", "xyz:XYZ100",
|
|
"--interval", "1m", "--days", "7", "--log-level", "normal"
|
|
], check=True)
|
|
except Exception as e:
|
|
logging.error(f"Data fetcher failed: {e}")
|
|
|
|
|
|
def run_fetch_history():
|
|
try:
|
|
logging.info("Running fetch_history.py")
|
|
subprocess.run([sys.executable, "fetch_history.py", "--log-level", "normal"], check=True)
|
|
except Exception as e:
|
|
logging.error(f"Fetch history failed: {e}")
|
|
|
|
|
|
def run_gap_detector():
|
|
try:
|
|
logging.info("Running gap_detector.py")
|
|
subprocess.run([sys.executable, "scripts/gap_detector.py", "--log-level", "normal"], check=True)
|
|
except Exception as e:
|
|
logging.error(f"Gap detector failed: {e}")
|
|
|
|
|
|
def run_backup():
|
|
try:
|
|
logging.info("Running backup_runner.py")
|
|
subprocess.run([sys.executable, "scripts/backup_runner.py", "--log-level", "normal"], check=True)
|
|
except Exception as e:
|
|
logging.error(f"Backup failed: {e}")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Run periodic maintenance tasks.")
|
|
parser.add_argument("--log-level", default="normal", choices=['off', 'normal', 'debug'])
|
|
args = parser.parse_args()
|
|
|
|
signal.signal(signal.SIGTERM, handle_shutdown)
|
|
signal.signal(signal.SIGINT, handle_shutdown)
|
|
|
|
setup_logging(args.log_level, 'CronScheduler')
|
|
|
|
# Schedule jobs
|
|
schedule.every().day.at("02:00").do(run_data_fetcher)
|
|
schedule.every().day.at("03:00").do(run_fetch_history)
|
|
schedule.every().hour.at(":15").do(run_gap_detector)
|
|
schedule.every().day.at("04:00").do(run_backup)
|
|
|
|
logging.info("Cron scheduler started")
|
|
|
|
while not shutdown_requested:
|
|
schedule.run_pending()
|
|
time.sleep(1)
|
|
|
|
logging.info("Cron scheduler shutting down.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|