Migrate data pipeline from SQLite to PostgreSQL + Docker setup
- Add db.py PostgreSQL abstraction layer (connection, upsert, table mgmt) - Replace sqlite3 with psycopg2 in: live_candle_fetcher, resampler, data_fetcher, fetch_history, import_csv, indicators, base_strategy - Sanitize table names (colons -> underscores) for PostgreSQL compat - Replace INSERT OR REPLACE with ON CONFLICT upserts - Replace pandas to_sql() with batch upsert_candles() - Add scripts: resampler_loop, gap_detector, backup_runner, cron_scheduler - Add migrate_sqlite_to_pg.py for one-time data migration - Add Dockerfile, docker-compose.yml, supervisord.conf - Add postgres/postgresql.conf tuned for 4GB RAM (Synology DS1513+) - Add .dockerignore, .env.docker.example, secrets template - Update requirements.txt (psycopg2-binary), .gitignore - Add MIGRATION_PLAN.md with full plan and todo list
This commit is contained in:
74
scripts/backup_runner.py
Normal file
74
scripts/backup_runner.py
Normal file
@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Backup Runner
|
||||
|
||||
Creates a daily pg_dump backup of the PostgreSQL database, compresses it,
|
||||
and retains only the last 7 days of backups.
|
||||
|
||||
Designed to run as a periodic cron job (daily) inside the Docker container.
|
||||
Backups are written to /backups which is mounted to a Synology shared folder.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from logging_utils import setup_logging
|
||||
|
||||
BACKUP_DIR = "/backups"
|
||||
RETENTION_DAYS = 7
|
||||
|
||||
|
||||
def run_backup():
|
||||
"""Run pg_dump and compress the output."""
|
||||
today = datetime.now().strftime("%Y%m%d")
|
||||
backup_file = os.path.join(BACKUP_DIR, f"hyper_{today}.sql.gz")
|
||||
|
||||
os.makedirs(BACKUP_DIR, exist_ok=True)
|
||||
|
||||
logging.info(f"Starting backup to {backup_file}")
|
||||
|
||||
cmd = f"pg_dump -h postgres -U hyper hyper | gzip > {backup_file}"
|
||||
result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
|
||||
|
||||
if result.returncode == 0:
|
||||
file_size = os.path.getsize(backup_file)
|
||||
logging.info(f"Backup completed: {backup_file} ({file_size:,} bytes)")
|
||||
else:
|
||||
logging.error(f"Backup failed: {result.stderr}")
|
||||
if os.path.exists(backup_file):
|
||||
os.remove(backup_file)
|
||||
|
||||
cleanup_old_backups()
|
||||
|
||||
|
||||
def cleanup_old_backups():
|
||||
"""Delete backup files older than RETENTION_DAYS."""
|
||||
cutoff = datetime.now() - timedelta(days=RETENTION_DAYS)
|
||||
|
||||
if not os.path.exists(BACKUP_DIR):
|
||||
return
|
||||
|
||||
for filename in os.listdir(BACKUP_DIR):
|
||||
if filename.startswith("hyper_") and filename.endswith(".sql.gz"):
|
||||
filepath = os.path.join(BACKUP_DIR, filename)
|
||||
mtime = datetime.fromtimestamp(os.path.getmtime(filepath))
|
||||
if mtime < cutoff:
|
||||
os.remove(filepath)
|
||||
logging.info(f"Deleted old backup: {filename}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Run PostgreSQL backup.")
|
||||
parser.add_argument("--log-level", default="normal", choices=['off', 'normal', 'debug'])
|
||||
args = parser.parse_args()
|
||||
|
||||
setup_logging(args.log_level, 'BackupRunner')
|
||||
|
||||
run_backup()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user