From 2a8ee9c8c5ac26eba255e9db6ec6cdf9c744c1c6 Mon Sep 17 00:00:00 2001 From: DiTus Date: Tue, 28 Jul 2026 09:43:39 +0200 Subject: [PATCH] Remove ASTER, PUMP, ZEC from dashboard and stop tracking their history - Remove ASTER, PUMP, ZEC from WATCHED_COINS in main_app.py - Remove ASTER, PUMP, ZEC from coin_id_map.json (stops market cap collection) - Remove ASTER, PUMP, ZEC from market_cap_data.json summary - Remove ASTER, PUMP, ZEC manual overrides from coin_id_map.py - Remove ASTER, PUMP, ZEC from resampling_status.json (gitignored) - Add WIKI/symbol_management.md documentation for adding/removing symbols Existing data in market_data.db is preserved; only new data collection stops. --- WIKI/symbol_management.md | 221 +++++++++++++++++++++++++++++++++++++ _data/coin_id_map.json | 3 - _data/market_cap_data.json | 15 --- coin_id_map.py | 3 - main_app.py | 2 +- 5 files changed, 222 insertions(+), 22 deletions(-) create mode 100644 WIKI/symbol_management.md diff --git a/WIKI/symbol_management.md b/WIKI/symbol_management.md new file mode 100644 index 0000000..7960cae --- /dev/null +++ b/WIKI/symbol_management.md @@ -0,0 +1,221 @@ +# Symbol Management Guide + +This guide explains how to add or remove Hyperliquid trading symbols (coins) from the trading bot's dashboard, data pipeline, and market cap tracking. + +## Overview + +The system tracks coins through multiple interconnected components. Each component reads its coin list from a specific source: + +| Component | Source | Purpose | +|-----------|--------|---------| +| Dashboard display | `WATCHED_COINS` in `main_app.py` | Shows live prices in terminal | +| Live candle fetcher | `--coins` CLI arg (from `WATCHED_COINS`) | Collects 1-minute candle data | +| Resampler | `--coins` CLI arg (from `WATCHED_COINS`) | Resamples 1m data to 15+ timeframes | +| Live price feed | `coins_to_watch` arg (from `WATCHED_COINS`) | WebSocket BBO/trade subscriptions | +| Market cap fetcher | `coin_id_map.json` | CoinGecko market cap data | +| Resampling status | `resampling_status.json` | Tracks progress per coin/timeframe | +| Market cap summary | `market_cap_data.json` | Aggregated market cap snapshots | + +## Data Pipeline + +``` +Hyperliquid WebSocket + | + +---> Live Candle Fetcher (1m candles) --> SQLite: {coin}_1m + | | + | +---> Resampler --> SQLite: {coin}_{3m,5m,15m,...,1M} + | + +---> Live Price Feed (BBO/trades) --> shared_prices dict --> Dashboard + +CoinGecko API + | + +---> Market Cap Fetcher --> SQLite: {coin}_market_cap + --> market_cap_data.json (summary) +``` + +All historical data is stored in `_data/market_data.db` (SQLite). Existing data is **preserved** when removing coins; only new data collection stops. + +--- + +## Adding a Symbol + +### Step 1: Add to the Watched Coins List + +Edit `main_app.py` (line 23): + +```python +WATCHED_COINS = ["BTC", "ETH", "SOL", "BNB", "HYPE", "NEW_COIN", "xyz:BRENTOIL", "xyz:CL"] +``` + +### Step 2: Add Display Name (Optional) + +If the symbol contains special characters or you want a custom display name, add it to `COIN_DISPLAY_NAMES` in `main_app.py` (lines 25-28): + +```python +COIN_DISPLAY_NAMES = { + "xyz:BRENTOIL": "BRENT", + "xyz:CL": "WTI", + "NEW_COIN": "NewCoin" +} +``` + +### Step 3: Add to Coin ID Map (for Market Cap) + +Edit `_data/coin_id_map.json` and add an entry mapping the Hyperliquid symbol to the CoinGecko ID: + +```json +"NEW_COIN": "new-coin-id-on-coingecko" +``` + +If the coin is already in the map (e.g., it was previously fetched), skip this step. + +### Step 4: Add to Manual Overrides (Optional) + +If the CoinGecko ID is ambiguous, add it to the `manual_overrides` dictionary in `coin_id_map.py` (lines 49-61): + +```python +manual_overrides = { + "BTC": "bitcoin", + "ETH": "ethereum", + "NEW_COIN": "new-coin-id-on-coingecko", + ... +} +``` + +### Step 5: Restart the Application + +Stop all running processes, then start `main_app.py`: + +```bash +python main_app.py +``` + +The system will automatically: +- Create new candle tables in `market_data.db` +- Begin collecting 1-minute candle data +- Begin resampling to all timeframes +- Begin collecting market cap data +- Display the coin on the dashboard + +--- + +## Removing a Symbol + +### Step 1: Stop All Running Processes + +Before making changes, stop all Python processes related to the project: + +```powershell +# Find running processes +Get-WmiObject Win32_Process | Where-Object { $_.ExecutablePath -like "*python*" -and $_.CommandLine -like "*hyper*" } + +# Stop them (replace PIDs with actual values) +Stop-Process -Id , , ... -Force +``` + +### Step 2: Remove from Watched Coins List + +Edit `main_app.py` (line 23) and remove the coin from `WATCHED_COINS`: + +```python +WATCHED_COINS = ["BTC", "ETH", "SOL", "BNB", "HYPE", "SUI", "xyz:BRENTOIL", "xyz:CL"] +``` + +### Step 3: Remove from Resampling Status + +Edit `_data/resampling_status.json` and delete the entire block for the coin, e.g.: + +```json +"REMOVED_COIN": { + "12h": { ... }, + "148m": { ... }, + ... +} +``` + +### Step 4: Remove from Coin ID Map + +Edit `_data/coin_id_map.json` and delete the entry: + +```json +"REMOVED_COIN": "coingecko-id" +``` + +### Step 5: Remove from Market Cap Summary + +Edit `_data/market_cap_data.json` and delete the entry: + +```json +"REMOVED_COIN_market_cap": { ... } +``` + +### Step 6: Remove from Manual Overrides (if present) + +Edit `coin_id_map.py` and remove the entry from `manual_overrides`: + +```python +manual_overrides = { + "BTC": "bitcoin", + "ETH": "ethereum", + ... +} +``` + +### Step 7: Restart the Application + +```bash +python main_app.py +``` + +**Note:** Existing data in `_data/market_data.db` (candle tables, market cap tables) is **not deleted**. The coin's data remains available for historical analysis; only new data collection stops. + +--- + +## File Reference + +### Core Configuration + +| File | Line | Description | +|------|------|-------------| +| `main_app.py` | 23 | `WATCHED_COINS` list - master coin list for dashboard, candle fetcher, resampler, and live feed | +| `main_app.py` | 25-28 | `COIN_DISPLAY_NAMES` dict - maps internal symbols to display names | +| `main_app.py` | 591-594 | `required_timeframes` list - timeframes for resampling | + +### Data Files + +| File | Description | +|------|-------------| +| `_data/market_data.db` | SQLite database with all candle and market cap data. Tables: `{coin}_1m`, `{coin}_{timeframe}`, `{coin}_market_cap` | +| `_data/resampling_status.json` | Tracks `last_candle_utc` and `total_candles` per coin/timeframe | +| `_data/coin_id_map.json` | Maps Hyperliquid symbols to CoinGecko IDs for market cap fetching | +| `_data/market_cap_data.json` | Summary of latest market cap data per coin | +| `_data/coin_precision.json` | All Hyperliquid coins with trade precision (reference only) | +| `_data/strategies.json` | Trading strategy configurations (separate from watched coins) | + +### Scripts + +| File | Description | +|------|-------------| +| `main_app.py` | Main orchestrator - starts all processes, renders dashboard | +| `live_candle_fetcher.py` | Collects 1-minute candles via WebSocket + historical catch-up | +| `resampler.py` | Resamples 1m candles to multiple timeframes using pandas | +| `live_market_utils.py` | WebSocket feed for live BBO (best bid/offer) and trade data | +| `market_cap_fetcher.py` | Fetches daily market cap data from CoinGecko API | +| `coin_id_map.py` | Generates `coin_id_map.json` from Hyperliquid + CoinGecko APIs | +| `dashboard_data_fetcher.py` | Fetches account balances and positions for dashboard | + +--- + +## Important Notes + +1. **Always stop processes before editing config files.** Running processes will overwrite changes to `resampling_status.json` and `market_data.db`. + +2. **Existing data is preserved.** Removing a coin from the lists stops new data collection but does not delete existing data from the SQLite database. + +3. **The `coin_id_map.json` is auto-generated.** Running `python coin_id_map.py` regenerates it from the Hyperliquid API. Manual overrides in `coin_id_map.py` ensure correct CoinGecko mappings. + +4. **Market cap fetcher is not auto-started.** The market cap fetcher process is currently disabled in `main_app.py` (line 614). It can be run manually: `python market_cap_fetcher.py`. + +5. **Strategy coins are separate.** Trading strategies in `_data/strategies.json` define their own coins independently of `WATCHED_COINS`. A coin can be traded by a strategy even if it's not in the watched list. + +6. **Special symbols.** Coins with the `xyz:` prefix (e.g., `xyz:BRENTOIL`, `xyz:CL`) are synthetic/derivative symbols on Hyperliquid. They follow the same management process as regular coins. diff --git a/_data/coin_id_map.json b/_data/coin_id_map.json index f6c9d78..aca6828 100644 --- a/_data/coin_id_map.json +++ b/_data/coin_id_map.json @@ -16,7 +16,6 @@ "AR": "arweave", "ARB": "osmosis-allarb", "ARK": "ark-3", - "ASTER": "astar", "ATOM": "lost-bitcoin-layer", "AVAX": "binance-peg-avalanche", "AVNT": "avantis", @@ -139,7 +138,6 @@ "POPCAT": "popcat", "PROMPT": "wayfinder", "PROVE": "succinct", - "PUMP": "pump-fun", "PURR": "purr-2", "PYTH": "pyth-network", "RDNT": "radiant-capital", @@ -198,7 +196,6 @@ "XRP": "ripple", "YGG": "yield-guild-games", "YZY": "yzy", - "ZEC": "zcash", "ZEN": "zenith-3", "ZEREBRO": "zerebro", "ZETA": "zeta", diff --git a/_data/market_cap_data.json b/_data/market_cap_data.json index 45b1096..4123a37 100644 --- a/_data/market_cap_data.json +++ b/_data/market_cap_data.json @@ -84,11 +84,6 @@ "timestamp_ms": 1762214400000, "market_cap": 411547691.74511635 }, - "ASTER_market_cap": { - "datetime_utc": "2025-11-04 00:00:00", - "timestamp_ms": 1762214400000, - "market_cap": 122331099.54500043 - }, "ATOM_market_cap": { "datetime_utc": "2025-11-04 00:00:00", "timestamp_ms": 1762214400000, @@ -699,11 +694,6 @@ "timestamp_ms": 1762214400000, "market_cap": 116187315.47981949 }, - "PUMP_market_cap": { - "datetime_utc": "2025-11-04 00:00:00", - "timestamp_ms": 1762214400000, - "market_cap": 1369591728.1563232 - }, "PURR_market_cap": { "datetime_utc": "2025-11-04 00:00:00", "timestamp_ms": 1762214400000, @@ -994,11 +984,6 @@ "timestamp_ms": 1762214400000, "market_cap": 49793986.29032182 }, - "ZEC_market_cap": { - "datetime_utc": "2025-11-04 00:00:00", - "timestamp_ms": 1762214400000, - "market_cap": 6917445577.244665 - }, "ZEN_market_cap": { "datetime_utc": "2025-11-04 00:00:00", "timestamp_ms": 1762214400000, diff --git a/coin_id_map.py b/coin_id_map.py index 91183bc..6951655 100644 --- a/coin_id_map.py +++ b/coin_id_map.py @@ -52,9 +52,6 @@ def update_coin_mapping(): "SOL": "solana", "BNB": "binancecoin", "HYPE": "hyperliquid", - "PUMP": "pump-fun", - "ASTER": "astar", - "ZEC": "zcash", "SUI": "sui", "ACE": "endurance", # Add other important ones you watch here diff --git a/main_app.py b/main_app.py index 75ed480..7535a1c 100644 --- a/main_app.py +++ b/main_app.py @@ -20,7 +20,7 @@ from live_market_utils import start_live_feed from strategies.base_strategy import BaseStrategy # --- Configuration --- -WATCHED_COINS = ["BTC", "ETH", "SOL", "BNB", "HYPE", "ASTER", "ZEC", "PUMP", "SUI", "xyz:BRENTOIL", "xyz:CL"] +WATCHED_COINS = ["BTC", "ETH", "SOL", "BNB", "HYPE", "SUI", "xyz:BRENTOIL", "xyz:CL"] # Display name mapping for dashboard (internal symbol -> display name) COIN_DISPLAY_NAMES = { "xyz:BRENTOIL": "BRENT",