docs: add DOCKER_GUIDE.md and fix .env parsing; chore: update docker and script configurations
This commit is contained in:
BTC Bot
199
docker/init-scripts/01-schema.sql
Normal file
199
docker/init-scripts/01-schema.sql
Normal file
@ -0,0 +1,199 @@
|
||||
-- 1. Enable TimescaleDB extension
|
||||
CREATE EXTENSION IF NOT EXISTS timescaledb;
|
||||
|
||||
-- 2. Create candles table (main data storage)
|
||||
CREATE TABLE IF NOT EXISTS candles (
|
||||
time TIMESTAMPTZ NOT NULL,
|
||||
symbol TEXT NOT NULL,
|
||||
interval TEXT NOT NULL,
|
||||
open DECIMAL(18,8) NOT NULL,
|
||||
high DECIMAL(18,8) NOT NULL,
|
||||
low DECIMAL(18,8) NOT NULL,
|
||||
close DECIMAL(18,8) NOT NULL,
|
||||
volume DECIMAL(18,8) NOT NULL,
|
||||
validated BOOLEAN DEFAULT FALSE,
|
||||
source TEXT DEFAULT 'hyperliquid',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- 3. Convert to hypertable (partitioned by time)
|
||||
SELECT create_hypertable('candles', 'time',
|
||||
chunk_time_interval => INTERVAL '7 days',
|
||||
if_not_exists => TRUE
|
||||
);
|
||||
|
||||
-- 4. Create unique constraint for upserts (required by ON CONFLICT)
|
||||
ALTER TABLE candles
|
||||
ADD CONSTRAINT candles_unique_candle
|
||||
UNIQUE (time, symbol, interval);
|
||||
|
||||
-- 5. Create indexes for efficient queries
|
||||
CREATE INDEX IF NOT EXISTS idx_candles_symbol_time
|
||||
ON candles (symbol, interval, time DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_candles_validated
|
||||
ON candles (validated) WHERE validated = FALSE;
|
||||
|
||||
-- 5. Create indicators table (computed values)
|
||||
CREATE TABLE IF NOT EXISTS indicators (
|
||||
time TIMESTAMPTZ NOT NULL,
|
||||
symbol TEXT NOT NULL,
|
||||
interval TEXT NOT NULL,
|
||||
indicator_name TEXT NOT NULL,
|
||||
value DECIMAL(18,8) NOT NULL,
|
||||
parameters JSONB,
|
||||
computed_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- 6. Convert indicators to hypertable
|
||||
SELECT create_hypertable('indicators', 'time',
|
||||
chunk_time_interval => INTERVAL '7 days',
|
||||
if_not_exists => TRUE
|
||||
);
|
||||
|
||||
-- 7. Create unique constraint + index for indicators (required for upserts)
|
||||
ALTER TABLE indicators
|
||||
ADD CONSTRAINT indicators_unique
|
||||
UNIQUE (time, symbol, interval, indicator_name);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_indicators_lookup
|
||||
ON indicators (symbol, interval, indicator_name, time DESC);
|
||||
|
||||
-- 8. Create data quality log table
|
||||
CREATE TABLE IF NOT EXISTS data_quality (
|
||||
time TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
check_type TEXT NOT NULL,
|
||||
severity TEXT NOT NULL,
|
||||
symbol TEXT,
|
||||
details JSONB,
|
||||
resolved BOOLEAN DEFAULT FALSE
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_quality_unresolved
|
||||
ON data_quality (resolved) WHERE resolved = FALSE;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_quality_time
|
||||
ON data_quality (time DESC);
|
||||
|
||||
-- 9. Create collector state tracking table
|
||||
CREATE TABLE IF NOT EXISTS collector_state (
|
||||
id SERIAL PRIMARY KEY,
|
||||
symbol TEXT NOT NULL UNIQUE,
|
||||
last_candle_time TIMESTAMPTZ,
|
||||
last_validation_time TIMESTAMPTZ,
|
||||
total_candles BIGINT DEFAULT 0,
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- 10. Insert initial state for cbBTC
|
||||
INSERT INTO collector_state (symbol, last_candle_time)
|
||||
VALUES ('cbBTC', NULL)
|
||||
ON CONFLICT (symbol) DO NOTHING;
|
||||
|
||||
-- 11. Enable compression for old data (after 7 days)
|
||||
ALTER TABLE candles SET (
|
||||
timescaledb.compress,
|
||||
timescaledb.compress_segmentby = 'symbol,interval'
|
||||
);
|
||||
|
||||
ALTER TABLE indicators SET (
|
||||
timescaledb.compress,
|
||||
timescaledb.compress_segmentby = 'symbol,interval,indicator_name'
|
||||
);
|
||||
|
||||
-- 12. Add compression policies
|
||||
SELECT add_compression_policy('candles', INTERVAL '7 days', if_not_exists => TRUE);
|
||||
SELECT add_compression_policy('indicators', INTERVAL '7 days', if_not_exists => TRUE);
|
||||
|
||||
-- 13. Create function to update collector state
|
||||
CREATE OR REPLACE FUNCTION update_collector_state()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
INSERT INTO collector_state (symbol, last_candle_time, total_candles)
|
||||
VALUES (NEW.symbol, NEW.time, 1)
|
||||
ON CONFLICT (symbol)
|
||||
DO UPDATE SET
|
||||
last_candle_time = NEW.time,
|
||||
total_candles = collector_state.total_candles + 1,
|
||||
updated_at = NOW();
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- 14. Create trigger to auto-update state
|
||||
DROP TRIGGER IF EXISTS trigger_update_state ON candles;
|
||||
CREATE TRIGGER trigger_update_state
|
||||
AFTER INSERT ON candles
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_collector_state();
|
||||
|
||||
-- 15. Create view for data health check
|
||||
CREATE OR REPLACE VIEW data_health AS
|
||||
SELECT
|
||||
symbol,
|
||||
COUNT(*) as total_candles,
|
||||
COUNT(*) FILTER (WHERE validated) as validated_candles,
|
||||
MAX(time) as latest_candle,
|
||||
MIN(time) as earliest_candle,
|
||||
NOW() - MAX(time) as time_since_last
|
||||
FROM candles
|
||||
GROUP BY symbol;
|
||||
|
||||
-- 16. Create decisions table (brain outputs - buy/sell/hold with full context)
|
||||
CREATE TABLE IF NOT EXISTS decisions (
|
||||
time TIMESTAMPTZ NOT NULL,
|
||||
symbol TEXT NOT NULL,
|
||||
interval TEXT NOT NULL,
|
||||
decision_type TEXT NOT NULL,
|
||||
strategy TEXT NOT NULL,
|
||||
confidence DECIMAL(5,4),
|
||||
price_at_decision DECIMAL(18,8),
|
||||
indicator_snapshot JSONB NOT NULL,
|
||||
candle_snapshot JSONB NOT NULL,
|
||||
reasoning TEXT,
|
||||
backtest_id TEXT,
|
||||
executed BOOLEAN DEFAULT FALSE,
|
||||
execution_price DECIMAL(18,8),
|
||||
execution_time TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- 17. Convert decisions to hypertable
|
||||
SELECT create_hypertable('decisions', 'time',
|
||||
chunk_time_interval => INTERVAL '7 days',
|
||||
if_not_exists => TRUE
|
||||
);
|
||||
|
||||
-- 18. Indexes for decisions - separate live from backtest queries
|
||||
CREATE INDEX IF NOT EXISTS idx_decisions_live
|
||||
ON decisions (symbol, interval, time DESC) WHERE backtest_id IS NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_decisions_backtest
|
||||
ON decisions (backtest_id, symbol, time DESC) WHERE backtest_id IS NOT NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_decisions_type
|
||||
ON decisions (symbol, decision_type, time DESC);
|
||||
|
||||
-- 19. Create backtest_runs metadata table
|
||||
CREATE TABLE IF NOT EXISTS backtest_runs (
|
||||
id TEXT PRIMARY KEY,
|
||||
strategy TEXT NOT NULL,
|
||||
symbol TEXT NOT NULL DEFAULT 'BTC',
|
||||
start_time TIMESTAMPTZ NOT NULL,
|
||||
end_time TIMESTAMPTZ NOT NULL,
|
||||
intervals TEXT[] NOT NULL,
|
||||
config JSONB,
|
||||
results JSONB,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- 20. Compression for decisions
|
||||
ALTER TABLE decisions SET (
|
||||
timescaledb.compress,
|
||||
timescaledb.compress_segmentby = 'symbol,interval,strategy'
|
||||
);
|
||||
|
||||
SELECT add_compression_policy('decisions', INTERVAL '7 days', if_not_exists => TRUE);
|
||||
|
||||
-- Success message
|
||||
SELECT 'Database schema initialized successfully' as status;
|
||||
43
docker/init-scripts/02-optimization.sql
Normal file
43
docker/init-scripts/02-optimization.sql
Normal file
@ -0,0 +1,43 @@
|
||||
-- Create a read-only user for API access (optional security)
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'btc_api') THEN
|
||||
CREATE USER btc_api WITH PASSWORD 'api_password_change_me';
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
|
||||
-- Grant read-only permissions
|
||||
GRANT CONNECT ON DATABASE btc_data TO btc_api;
|
||||
GRANT USAGE ON SCHEMA public TO btc_api;
|
||||
GRANT SELECT ON ALL TABLES IN SCHEMA public TO btc_api;
|
||||
|
||||
-- Grant sequence access for ID columns
|
||||
GRANT USAGE ON ALL SEQUENCES IN SCHEMA public TO btc_api;
|
||||
|
||||
-- Apply to future tables
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO btc_api;
|
||||
|
||||
-- Create continuous aggregate for hourly stats (optional optimization)
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS hourly_stats
|
||||
WITH (timescaledb.continuous) AS
|
||||
SELECT
|
||||
time_bucket('1 hour', time) as bucket,
|
||||
symbol,
|
||||
interval,
|
||||
FIRST(open, time) as first_open,
|
||||
MAX(high) as max_high,
|
||||
MIN(low) as min_low,
|
||||
LAST(close, time) as last_close,
|
||||
SUM(volume) as total_volume,
|
||||
COUNT(*) as candle_count
|
||||
FROM candles
|
||||
GROUP BY bucket, symbol, interval;
|
||||
|
||||
-- Add refresh policy for continuous aggregate
|
||||
SELECT add_continuous_aggregate_policy('hourly_stats',
|
||||
start_offset => INTERVAL '1 month',
|
||||
end_offset => INTERVAL '1 hour',
|
||||
schedule_interval => INTERVAL '1 hour',
|
||||
if_not_exists => TRUE
|
||||
);
|
||||
Reference in New Issue
Block a user