diff --git a/BACKTESTING_GUIDE.md b/BACKTESTING_GUIDE.md new file mode 100644 index 0000000..bbbd011 --- /dev/null +++ b/BACKTESTING_GUIDE.md @@ -0,0 +1,56 @@ +# Ping-Pong Bot: Backtesting & Optimization Guide + +This guide explains how to use the local backtesting environment to test and optimize your BTC trading strategy using historical data from your PostgreSQL database. + +## 1. Prerequisites +The backtesting engine requires `pandas`, `numpy`, and `asyncpg`. These are already installed in your `btc_ping_pong_bot` Docker container. + +To run the backtester, use the following command: +```bash +docker exec -it btc_ping_pong_bot python src/strategies/backtest_engine.py +``` + +## 2. Backtest Engine (`backtest_engine.py`) +The backtest engine reuses the core logic from `ping_pong_bot.py` via the `PingPongStrategy` class. + +### Key Features: +* **Virtual Exchange:** Simulates a $1,000 account with customizable leverage and fees. +* **Fee Simulation:** Applies a 0.05% taker fee (configurable) to every entry and exit. +* **Mark-to-Market:** Calculates real-time equity based on current price and position size. +* **Data Sourcing:** Automatically pulls the last 10,000 candles from your `btc_data` database. + +### How to Run: +```bash +# From the project root +docker exec -it btc_ping_pong_bot python src/strategies/backtest_engine.py +``` + +## 3. Strategy Optimization (Optional) +To find the absolute best parameters for RSI and Hurst, you can use **Optuna**. + +### Installation: +Inside the Docker container: +```bash +docker exec -it btc_ping_pong_bot pip install optuna +``` + +### Planned Optimizer (`optimize_strategy.py`): +Once installed, we can implement an optimization script that searches for: +* `rsi_period`: 7 to 21 +* `hurst_multiplier`: 1.2 to 2.5 +* `partial_exit_pct`: 0.05 to 0.30 + +## 4. Local DB Data +The engine connects to your local PostgreSQL DB using the credentials in your `.env` file. It specifically queries the `candles` table for the symbol and interval defined in `config/ping_pong_config.yaml`. + +## 5. Interpreting Results +* **Final Equity:** Your simulated account balance after all trades. +* **ROI:** Return on Investment (Percentage). +* **Total Fees:** Total cost paid to the "Virtual Exchange". High fees indicate over-trading. +* **Trade Count:** Total number of Enter/Exit signals triggered. + +## 6. Next Steps +1. Run the backtester to see baseline performance. +2. Adjust parameters in `config/ping_pong_config.yaml`. +3. Rerun the backtest to see the impact of your changes. +4. (Optional) Ask me to implement the `optimize_strategy.py` script once you have Optuna installed. diff --git a/src/strategies/backtest_engine.py b/src/strategies/backtest_engine.py new file mode 100644 index 0000000..84dae04 --- /dev/null +++ b/src/strategies/backtest_engine.py @@ -0,0 +1,142 @@ +import pandas as pd +import numpy as np +import yaml +import os +import asyncio +import asyncpg +from datetime import datetime +from ping_pong_bot import PingPongStrategy + +class BacktestEngine: + def __init__(self, config_path="config/ping_pong_config.yaml"): + with open(config_path, 'r') as f: + self.config = yaml.safe_load(f) + + self.strategy = PingPongStrategy(self.config) + self.direction = self.config.get('direction', 'long') + self.strategy.direction = self.direction + + # Virtual Exchange State + self.balance = 1000.0 # Starting USD + self.equity = 1000.0 + self.position_size = 0.0 # BTC + self.position_value = 0.0 # USD + self.entry_price = 0.0 + + # Settings + self.fee_rate = 0.0005 # 0.05% Taker + self.leverage = float(self.config.get('leverage_long' if self.direction == 'long' else 'leverage_short', 5.0)) + self.pos_size_margin = float(self.config.get('pos_size_margin', 20.0)) + self.partial_exit_pct = float(self.config.get('partial_exit_pct', 0.15)) + + self.trades = [] + self.equity_curve = [] + + async def load_data(self, symbol, interval, limit=5000): + conn = await asyncpg.connect( + host=os.getenv('DB_HOST', '20.20.20.20'), + port=int(os.getenv('DB_PORT', 5433)), + user=os.getenv('DB_USER', 'btc_bot'), + password=os.getenv('DB_PASSWORD', ''), + database=os.getenv('DB_NAME', 'btc_data') + ) + rows = await conn.fetch(''' + SELECT time, open, high, low, close, volume + FROM candles + WHERE symbol = $1 AND interval = $2 + ORDER BY time ASC LIMIT $3 + ''', symbol, interval, limit) + await conn.close() + + df = pd.DataFrame([dict(r) for r in rows]) + df[['open', 'high', 'low', 'close', 'volume']] = df[['open', 'high', 'low', 'close', 'volume']].astype(float) + return df + + def run(self, df): + print(f"Starting backtest on {len(df)} candles...") + df = self.strategy.calculate_indicators(df) + + # Start after enough candles for indicators + start_idx = max(self.config['rsi']['period'], self.config['hurst']['period']) + 5 + + for i in range(start_idx, len(df)): + current_df = df.iloc[:i+1] + price = df.iloc[i]['close'] + time = df.iloc[i]['time'] + + signal = self.strategy.check_signals(current_df) + + if signal == "open": + # Entry Logic + qty_usd = self.pos_size_margin * self.leverage + qty_btc = qty_usd / price + fee = qty_usd * self.fee_rate + + self.balance -= fee + + if self.direction == "long": + self.position_size += qty_btc + else: # Short + self.position_size -= qty_btc + + self.entry_price = price # Simplified avg entry + self.trades.append({"time": time, "type": "Enter", "price": price, "fee": fee}) + + elif signal == "close" and abs(self.position_size) > 0: + # Exit Logic + qty_btc_exit = abs(self.position_size) * self.partial_exit_pct + qty_usd_exit = qty_btc_exit * price + fee = qty_usd_exit * self.fee_rate + + # Realized PnL + if self.direction == "long": + pnl = qty_btc_exit * (price - self.entry_price) + self.position_size -= qty_btc_exit + else: # Short + pnl = qty_btc_exit * (self.entry_price - price) + self.position_size += qty_btc_exit + + self.balance += (pnl - fee) + self.trades.append({"time": time, "type": "Exit", "price": price, "pnl": pnl, "fee": fee}) + + # Mark to Market Equity + unrealized = 0 + if self.direction == "long": + unrealized = self.position_size * (price - self.entry_price) if self.position_size > 0 else 0 + else: + unrealized = abs(self.position_size) * (self.entry_price - price) if self.position_size < 0 else 0 + + self.equity = self.balance + unrealized + self.equity_curve.append({"time": time, "equity": self.equity}) + + self.print_results() + + def print_results(self): + total_pnl = self.equity - 1000.0 + roi = (total_pnl / 1000.0) * 100 + fees = sum(t['fee'] for t in self.trades) + + print("\n" + "="*30) + print(" BACKTEST RESULTS ") + print("="*30) + print(f"Total Trades: {len(self.trades)}") + print(f"Final Equity: ${self.equity:.2f}") + print(f"Total PnL: ${total_pnl:.2f}") + print(f"ROI: {roi:.2f}%") + print(f"Total Fees: ${fees:.2f}") + print("="*30) + +async def main(): + engine = BacktestEngine() + # Assume BTC/1m for now + symbol = engine.config['symbol'].replace("USDT", "").replace("USD", "") + interval = engine.config['interval'] + "m" + + df = await engine.load_data(symbol, interval, limit=10000) + if not df.empty: + engine.run(df) + else: + print("No data found in DB.") + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/strategies/ping_pong_bot.py b/src/strategies/ping_pong_bot.py index f17b697..fd04369 100644 --- a/src/strategies/ping_pong_bot.py +++ b/src/strategies/ping_pong_bot.py @@ -84,12 +84,93 @@ class DatabaseManager: logger.error(f"DB Query Error for {symbol} {interval}: {e}") return [] +class PingPongStrategy: + """Core Strategy Logic for Ping-Pong Scalping""" + def __init__(self, config): + self.config = config + self.direction = config.get('direction', 'long') + + def rma(self, series, length): + alpha = 1 / length + return series.ewm(alpha=alpha, adjust=False).mean() + + def calculate_indicators(self, df): + # RSI + rsi_cfg = self.config['rsi'] + delta = df['close'].diff() + gain = delta.where(delta > 0, 0) + loss = -delta.where(delta < 0, 0) + df['rsi'] = 100 - (100 / (1 + (self.rma(gain, rsi_cfg['period']) / self.rma(loss, rsi_cfg['period'])))) + + # Hurst + hurst_cfg = self.config['hurst'] + mcl = hurst_cfg['period'] / 2 + mcl_2 = int(round(mcl / 2)) + df['tr'] = np.maximum(df['high'] - df['low'], np.maximum(abs(df['high'] - df['close'].shift(1)), abs(df['low'] - df['close'].shift(1)))) + df['ma_mcl'] = self.rma(df['close'], mcl) + df['atr_mcl'] = self.rma(df['tr'], mcl) + df['center'] = df['ma_mcl'].shift(mcl_2).fillna(df['ma_mcl']) + mcm_off = hurst_cfg['multiplier'] * df['atr_mcl'] + df['hurst_upper'] = df['center'] + mcm_off + df['hurst_lower'] = df['center'] - mcm_off + return df + + def check_signals(self, df): + if len(df) < 3: return None + # finished = candle that just closed (e.g. 10:30) + # prev = candle before that (e.g. 10:29) + finished = df.iloc[-2] + prev = df.iloc[-3] + + rsi_cfg, hurst_cfg = self.config['rsi'] or {}, self.config['hurst'] or {} + + def is_crossing_up(p_val, p_band, c_open, c_close, c_band): + # 1. Crossed up BETWEEN candles + between = p_val < p_band and c_close >= c_band + # 2. Crossed up WITHIN this candle + within = c_open is not None and c_open < c_band and c_close >= c_band + return between or within + + def is_crossing_down(p_val, p_band, c_open, c_close, c_band): + # 1. Crossed down BETWEEN candles + between = p_val > p_band and c_close <= c_band + # 2. Crossed down WITHIN this candle + within = c_open is not None and c_open > c_band and c_close <= c_band + return between or within + + # Hurst Signals + h_upper_cross_down = is_crossing_down(prev['close'], prev['hurst_upper'], finished['open'], finished['close'], finished['hurst_upper']) + h_lower_cross_down = is_crossing_down(prev['close'], prev['hurst_lower'], finished['open'], finished['close'], finished['hurst_lower']) + + # RSI Signals + rsi_cross_up = is_crossing_up(prev['rsi'], rsi_cfg.get('oversold', 30), None, finished['rsi'], rsi_cfg.get('oversold', 30)) + rsi_cross_down = is_crossing_down(prev['rsi'], rsi_cfg.get('overbought', 70), None, finished['rsi'], rsi_cfg.get('overbought', 70)) + + l_open = (rsi_cfg.get('enabled_for_open') and rsi_cross_up) or \ + (hurst_cfg.get('enabled_for_open') and h_lower_cross_down) + + l_close = (rsi_cfg.get('enabled_for_close') and rsi_cross_down) or \ + (hurst_cfg.get('enabled_for_close') and h_upper_cross_down) + + s_open = (rsi_cfg.get('enabled_for_open') and rsi_cross_down) or \ + (hurst_cfg.get('enabled_for_open') and h_upper_cross_down) + + s_close = (rsi_cfg.get('enabled_for_close') and rsi_cross_up) or \ + (hurst_cfg.get('enabled_for_close') and h_lower_cross_down) + + if self.direction == 'long': + return "open" if l_open else ("close" if l_close else None) + else: + return "open" if s_open else ("close" if s_close else None) + class PingPongBot: def __init__(self, config_path="config/ping_pong_config.yaml"): - self.version = "1.7.3" + self.version = "1.7.5" with open(config_path, 'r') as f: self.config = yaml.safe_load(f) + self.strategy = PingPongStrategy(self.config) + # Explicitly load from ENV to ensure they are available self.api_key = os.getenv("BYBIT_API_KEY") or os.getenv("API_KEY") self.api_secret = os.getenv("BYBIT_API_SECRET") or os.getenv("API_SECRET") @@ -201,30 +282,8 @@ class PingPongBot: except Exception as e: logger.error(f"Failed to write to CSV log: {e}") - def rma(self, series, length): - alpha = 1 / length - return series.ewm(alpha=alpha, adjust=False).mean() - def calculate_indicators(self, df): - # RSI - rsi_cfg = self.config['rsi'] - delta = df['close'].diff() - gain = delta.where(delta > 0, 0) - loss = -delta.where(delta < 0, 0) - df['rsi'] = 100 - (100 / (1 + (self.rma(gain, rsi_cfg['period']) / self.rma(loss, rsi_cfg['period'])))) - - # Hurst - hurst_cfg = self.config['hurst'] - mcl = hurst_cfg['period'] / 2 - mcl_2 = int(round(mcl / 2)) - df['tr'] = np.maximum(df['high'] - df['low'], np.maximum(abs(df['high'] - df['close'].shift(1)), abs(df['low'] - df['close'].shift(1)))) - df['ma_mcl'] = self.rma(df['close'], mcl) - df['atr_mcl'] = self.rma(df['tr'], mcl) - df['center'] = df['ma_mcl'].shift(mcl_2).fillna(df['ma_mcl']) - mcm_off = hurst_cfg['multiplier'] * df['atr_mcl'] - df['hurst_upper'] = df['center'] + mcm_off - df['hurst_lower'] = df['center'] - mcm_off - + df = self.strategy.calculate_indicators(df) last_row = df.iloc[-1] now_str = datetime.now().strftime("%H:%M:%S") self.current_indicators["rsi"] = {"value": float(last_row['rsi']), "timestamp": now_str} @@ -262,6 +321,7 @@ class PingPongBot: await self.close_all_positions() self.direction = new_direction + self.strategy.direction = new_direction if self.direction == "long": self.category = "inverse" self.symbol = f"{self.base_coin}USD" @@ -396,52 +456,7 @@ class PingPongBot: logger.error(f"Exchange Sync Error: {e}") def check_signals(self, df): - if len(df) < 3: return None - # finished = candle that just closed (e.g. 10:30) - # prev = candle before that (e.g. 10:29) - finished = df.iloc[-2] - prev = df.iloc[-3] - - rsi_cfg, hurst_cfg = self.config['rsi'] or {}, self.config['hurst'] or {} - - def is_crossing_up(p_val, p_band, c_open, c_close, c_band): - # 1. Crossed up BETWEEN candles - between = p_val < p_band and c_close >= c_band - # 2. Crossed up WITHIN this candle - within = c_open is not None and c_open < c_band and c_close >= c_band - return between or within - - def is_crossing_down(p_val, p_band, c_open, c_close, c_band): - # 1. Crossed down BETWEEN candles - between = p_val > p_band and c_close <= c_band - # 2. Crossed down WITHIN this candle - within = c_open is not None and c_open > c_band and c_close <= c_band - return between or within - - # Hurst Signals - Only using 'is_crossing_down' as requested - h_upper_cross_down = is_crossing_down(prev['close'], prev['hurst_upper'], finished['open'], finished['close'], finished['hurst_upper']) - h_lower_cross_down = is_crossing_down(prev['close'], prev['hurst_lower'], finished['open'], finished['close'], finished['hurst_lower']) - - # RSI Signals - rsi_cross_up = is_crossing_up(prev['rsi'], rsi_cfg.get('oversold', 30), None, finished['rsi'], rsi_cfg.get('oversold', 30)) - rsi_cross_down = is_crossing_down(prev['rsi'], rsi_cfg.get('overbought', 70), None, finished['rsi'], rsi_cfg.get('overbought', 70)) - - l_open = (rsi_cfg.get('enabled_for_open') and rsi_cross_up) or \ - (hurst_cfg.get('enabled_for_open') and h_lower_cross_down) - - l_close = (rsi_cfg.get('enabled_for_close') and rsi_cross_down) or \ - (hurst_cfg.get('enabled_for_close') and h_upper_cross_down) - - s_open = (rsi_cfg.get('enabled_for_open') and rsi_cross_down) or \ - (hurst_cfg.get('enabled_for_open') and h_upper_cross_down) - - s_close = (rsi_cfg.get('enabled_for_close') and rsi_cross_up) or \ - (hurst_cfg.get('enabled_for_close') and h_lower_cross_down) - - if self.direction == 'long': - return "open" if l_open else ("close" if l_close else None) - else: - return "open" if s_open else ("close" if s_close else None) + return self.strategy.check_signals(df) async def execute_trade(self, signal): if not signal or not self.market_price: return @@ -544,11 +559,14 @@ class PingPongBot: pnl_color = "green" if self.session_pnl >= 0 else "red" pnl_btc_color = "green" if self.session_pnl_btc >= 0 else "red" + net_realized_pnl = self.total_realized_pnl - self.total_fees + cfg_table.add_row("Running Time", runtime_str) cfg_table.add_row("Session PnL (USD)", f"[bold {pnl_color}]{'$' if self.session_pnl >= 0 else '-$'}{abs(self.session_pnl):.2f}[/]") cfg_table.add_row("Session PnL (BTC)", f"[bold {pnl_btc_color}]{'{:+.6f}'.format(self.session_pnl_btc)} BTC[/]") cfg_table.add_row("Total Fees", f"[bold red]-${self.total_fees:.2f}[/]") - cfg_table.add_row("Realized PnL", f"[bold {'green' if self.total_realized_pnl >= 0 else 'red'}]${self.total_realized_pnl:.2f}[/]") + cfg_table.add_row("Gross Realized PnL", f"[bold {'green' if self.total_realized_pnl >= 0 else 'red'}]${self.total_realized_pnl:.2f}[/]") + cfg_table.add_row("Net Realized PnL", f"[bold {'green' if net_realized_pnl >= 0 else 'red'}]${net_realized_pnl:.2f}[/]") ind_table = Table(title="INDICATORS", box=box.ROUNDED, expand=True) ind_table.add_column("Indicator"); ind_table.add_column("Value"); ind_table.add_column("Updated")