Migrate data pipeline from SQLite to PostgreSQL + Docker setup
- Add db.py PostgreSQL abstraction layer (connection, upsert, table mgmt) - Replace sqlite3 with psycopg2 in: live_candle_fetcher, resampler, data_fetcher, fetch_history, import_csv, indicators, base_strategy - Sanitize table names (colons -> underscores) for PostgreSQL compat - Replace INSERT OR REPLACE with ON CONFLICT upserts - Replace pandas to_sql() with batch upsert_candles() - Add scripts: resampler_loop, gap_detector, backup_runner, cron_scheduler - Add migrate_sqlite_to_pg.py for one-time data migration - Add Dockerfile, docker-compose.yml, supervisord.conf - Add postgres/postgresql.conf tuned for 4GB RAM (Synology DS1513+) - Add .dockerignore, .env.docker.example, secrets template - Update requirements.txt (psycopg2-binary), .gitignore - Add MIGRATION_PLAN.md with full plan and todo list
This commit is contained in:
42
docker-compose.yml
Normal file
42
docker-compose.yml
Normal file
@ -0,0 +1,42 @@
|
||||
version: "3.8"
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:15-alpine
|
||||
container_name: hyper_pg
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_DB: hyper
|
||||
POSTGRES_USER: hyper
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
||||
volumes:
|
||||
- pg_data:/var/lib/postgresql/data
|
||||
- ./postgres/postgresql.conf:/etc/postgresql/postgresql.conf
|
||||
command: postgres -c config_file=/etc/postgresql/postgresql.conf
|
||||
ports:
|
||||
- "5432:5432"
|
||||
networks:
|
||||
- hyper_net
|
||||
|
||||
data-collector:
|
||||
build: .
|
||||
container_name: hyper_data
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
- postgres
|
||||
env_file:
|
||||
- .env.docker
|
||||
volumes:
|
||||
- ./_data:/app/_data
|
||||
- ./_logs:/app/_logs
|
||||
- ./secrets:/app/secrets
|
||||
- /volume1/docker/hyper/backups:/backups
|
||||
networks:
|
||||
- hyper_net
|
||||
|
||||
volumes:
|
||||
pg_data:
|
||||
|
||||
networks:
|
||||
hyper_net:
|
||||
driver: bridge
|
||||
Reference in New Issue
Block a user