199 lines
6.0 KiB
PL/PgSQL
199 lines
6.0 KiB
PL/PgSQL
-- 1. Enable TimescaleDB extension
|
|
CREATE EXTENSION IF NOT EXISTS timescaledb;
|
|
|
|
-- 2. Create candles table (main data storage)
|
|
CREATE TABLE IF NOT EXISTS candles (
|
|
time TIMESTAMPTZ NOT NULL,
|
|
symbol TEXT NOT NULL,
|
|
interval TEXT NOT NULL,
|
|
open DECIMAL(18,8) NOT NULL,
|
|
high DECIMAL(18,8) NOT NULL,
|
|
low DECIMAL(18,8) NOT NULL,
|
|
close DECIMAL(18,8) NOT NULL,
|
|
volume DECIMAL(18,8) NOT NULL,
|
|
validated BOOLEAN DEFAULT FALSE,
|
|
source TEXT DEFAULT 'hyperliquid',
|
|
created_at TIMESTAMPTZ DEFAULT NOW()
|
|
);
|
|
|
|
-- 3. Convert to hypertable (partitioned by time)
|
|
SELECT create_hypertable('candles', 'time',
|
|
chunk_time_interval => INTERVAL '7 days',
|
|
if_not_exists => TRUE
|
|
);
|
|
|
|
-- 4. Create unique constraint for upserts (required by ON CONFLICT)
|
|
ALTER TABLE candles
|
|
ADD CONSTRAINT candles_unique_candle
|
|
UNIQUE (time, symbol, interval);
|
|
|
|
-- 5. Create indexes for efficient queries
|
|
CREATE INDEX IF NOT EXISTS idx_candles_symbol_time
|
|
ON candles (symbol, interval, time DESC);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_candles_validated
|
|
ON candles (validated) WHERE validated = FALSE;
|
|
|
|
-- 5. Create indicators table (computed values)
|
|
CREATE TABLE IF NOT EXISTS indicators (
|
|
time TIMESTAMPTZ NOT NULL,
|
|
symbol TEXT NOT NULL,
|
|
interval TEXT NOT NULL,
|
|
indicator_name TEXT NOT NULL,
|
|
value DECIMAL(18,8) NOT NULL,
|
|
parameters JSONB,
|
|
computed_at TIMESTAMPTZ DEFAULT NOW()
|
|
);
|
|
|
|
-- 6. Convert indicators to hypertable
|
|
SELECT create_hypertable('indicators', 'time',
|
|
chunk_time_interval => INTERVAL '7 days',
|
|
if_not_exists => TRUE
|
|
);
|
|
|
|
-- 7. Create unique constraint + index for indicators (required for upserts)
|
|
ALTER TABLE indicators
|
|
ADD CONSTRAINT indicators_unique
|
|
UNIQUE (time, symbol, interval, indicator_name);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_indicators_lookup
|
|
ON indicators (symbol, interval, indicator_name, time DESC);
|
|
|
|
-- 8. Create data quality log table
|
|
CREATE TABLE IF NOT EXISTS data_quality (
|
|
time TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
check_type TEXT NOT NULL,
|
|
severity TEXT NOT NULL,
|
|
symbol TEXT,
|
|
details JSONB,
|
|
resolved BOOLEAN DEFAULT FALSE
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_quality_unresolved
|
|
ON data_quality (resolved) WHERE resolved = FALSE;
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_quality_time
|
|
ON data_quality (time DESC);
|
|
|
|
-- 9. Create collector state tracking table
|
|
CREATE TABLE IF NOT EXISTS collector_state (
|
|
id SERIAL PRIMARY KEY,
|
|
symbol TEXT NOT NULL UNIQUE,
|
|
last_candle_time TIMESTAMPTZ,
|
|
last_validation_time TIMESTAMPTZ,
|
|
total_candles BIGINT DEFAULT 0,
|
|
updated_at TIMESTAMPTZ DEFAULT NOW()
|
|
);
|
|
|
|
-- 10. Insert initial state for cbBTC
|
|
INSERT INTO collector_state (symbol, last_candle_time)
|
|
VALUES ('cbBTC', NULL)
|
|
ON CONFLICT (symbol) DO NOTHING;
|
|
|
|
-- 11. Enable compression for old data (after 7 days)
|
|
ALTER TABLE candles SET (
|
|
timescaledb.compress,
|
|
timescaledb.compress_segmentby = 'symbol,interval'
|
|
);
|
|
|
|
ALTER TABLE indicators SET (
|
|
timescaledb.compress,
|
|
timescaledb.compress_segmentby = 'symbol,interval,indicator_name'
|
|
);
|
|
|
|
-- 12. Add compression policies
|
|
SELECT add_compression_policy('candles', INTERVAL '7 days', if_not_exists => TRUE);
|
|
SELECT add_compression_policy('indicators', INTERVAL '7 days', if_not_exists => TRUE);
|
|
|
|
-- 13. Create function to update collector state
|
|
CREATE OR REPLACE FUNCTION update_collector_state()
|
|
RETURNS TRIGGER AS $$
|
|
BEGIN
|
|
INSERT INTO collector_state (symbol, last_candle_time, total_candles)
|
|
VALUES (NEW.symbol, NEW.time, 1)
|
|
ON CONFLICT (symbol)
|
|
DO UPDATE SET
|
|
last_candle_time = NEW.time,
|
|
total_candles = collector_state.total_candles + 1,
|
|
updated_at = NOW();
|
|
RETURN NEW;
|
|
END;
|
|
$$ LANGUAGE plpgsql;
|
|
|
|
-- 14. Create trigger to auto-update state
|
|
DROP TRIGGER IF EXISTS trigger_update_state ON candles;
|
|
CREATE TRIGGER trigger_update_state
|
|
AFTER INSERT ON candles
|
|
FOR EACH ROW
|
|
EXECUTE FUNCTION update_collector_state();
|
|
|
|
-- 15. Create view for data health check
|
|
CREATE OR REPLACE VIEW data_health AS
|
|
SELECT
|
|
symbol,
|
|
COUNT(*) as total_candles,
|
|
COUNT(*) FILTER (WHERE validated) as validated_candles,
|
|
MAX(time) as latest_candle,
|
|
MIN(time) as earliest_candle,
|
|
NOW() - MAX(time) as time_since_last
|
|
FROM candles
|
|
GROUP BY symbol;
|
|
|
|
-- 16. Create decisions table (brain outputs - buy/sell/hold with full context)
|
|
CREATE TABLE IF NOT EXISTS decisions (
|
|
time TIMESTAMPTZ NOT NULL,
|
|
symbol TEXT NOT NULL,
|
|
interval TEXT NOT NULL,
|
|
decision_type TEXT NOT NULL,
|
|
strategy TEXT NOT NULL,
|
|
confidence DECIMAL(5,4),
|
|
price_at_decision DECIMAL(18,8),
|
|
indicator_snapshot JSONB NOT NULL,
|
|
candle_snapshot JSONB NOT NULL,
|
|
reasoning TEXT,
|
|
backtest_id TEXT,
|
|
executed BOOLEAN DEFAULT FALSE,
|
|
execution_price DECIMAL(18,8),
|
|
execution_time TIMESTAMPTZ,
|
|
created_at TIMESTAMPTZ DEFAULT NOW()
|
|
);
|
|
|
|
-- 17. Convert decisions to hypertable
|
|
SELECT create_hypertable('decisions', 'time',
|
|
chunk_time_interval => INTERVAL '7 days',
|
|
if_not_exists => TRUE
|
|
);
|
|
|
|
-- 18. Indexes for decisions - separate live from backtest queries
|
|
CREATE INDEX IF NOT EXISTS idx_decisions_live
|
|
ON decisions (symbol, interval, time DESC) WHERE backtest_id IS NULL;
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_decisions_backtest
|
|
ON decisions (backtest_id, symbol, time DESC) WHERE backtest_id IS NOT NULL;
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_decisions_type
|
|
ON decisions (symbol, decision_type, time DESC);
|
|
|
|
-- 19. Create backtest_runs metadata table
|
|
CREATE TABLE IF NOT EXISTS backtest_runs (
|
|
id TEXT PRIMARY KEY,
|
|
strategy TEXT NOT NULL,
|
|
symbol TEXT NOT NULL DEFAULT 'BTC',
|
|
start_time TIMESTAMPTZ NOT NULL,
|
|
end_time TIMESTAMPTZ NOT NULL,
|
|
intervals TEXT[] NOT NULL,
|
|
config JSONB,
|
|
results JSONB,
|
|
created_at TIMESTAMPTZ DEFAULT NOW()
|
|
);
|
|
|
|
-- 20. Compression for decisions
|
|
ALTER TABLE decisions SET (
|
|
timescaledb.compress,
|
|
timescaledb.compress_segmentby = 'symbol,interval,strategy'
|
|
);
|
|
|
|
SELECT add_compression_policy('decisions', INTERVAL '7 days', if_not_exists => TRUE);
|
|
|
|
-- Success message
|
|
SELECT 'Database schema initialized successfully' as status; |