save market cap of all coins

market_cap_fetcher.py

@@ -8,47 +8,52 @@ import requests
 import time
 from datetime import datetime, timezone, timedelta
 import json
+from dotenv import load_dotenv
+
+load_dotenv()
 
-# Assuming logging_utils.py is in the same directory
 from logging_utils import setup_logging
 
 class MarketCapFetcher:
     """
     Fetches historical daily market cap data from the CoinGecko API and
-    intelligently updates the SQLite database. It processes individual coins,
-    aggregates stablecoins, and captures total market cap metrics.
+    intelligently updates the SQLite database for all coins found in the coin map.
     """
 
-    COIN_ID_MAP = {
-        "BTC": "bitcoin",
-        "ETH": "ethereum",
-        "SOL": "solana",
-        "BNB": "binancecoin",
-        "HYPE": "hyperliquid",
-        "ASTER": "astar",
-        "ZEC": "zcash",
-        "PUMP": "pump-fun", # Correct ID is 'pump-fun'
-        "SUI": "sui"
-    }
-    
-    STABLECOIN_ID_MAP = {
-        "USDT": "tether",
-        "USDC": "usd-coin",
-        "USDE": "ethena-usde",
-        "DAI": "dai",
-        "PYUSD": "paypal-usd"
-    }
-
-    def __init__(self, log_level: str, coins: list):
+    def __init__(self, log_level: str):
         setup_logging(log_level, 'MarketCapFetcher')
-        self.coins_to_fetch = coins
         self.db_path = os.path.join("_data", "market_data.db")
         self.api_base_url = "https://api.coingecko.com/api/v3"
         self.api_key = os.environ.get("COINGECKO_API_KEY")
-        
         if not self.api_key:
             logging.error("CoinGecko API key not found. Please set the COINGECKO_API_KEY environment variable.")
             sys.exit(1)
+        
+        self.COIN_ID_MAP = self._load_coin_id_map()
+        if not self.COIN_ID_MAP:
+            logging.error("Coin ID map is empty. Run 'update_coin_map.py' to generate it.")
+            sys.exit(1)
+        
+        # --- FIX: The list of coins to fetch is now all coins from the map ---
+        self.coins_to_fetch = list(self.COIN_ID_MAP.keys())
+            
+        self.STABLECOIN_ID_MAP = {
+            "USDT": "tether",
+            "USDC": "usd-coin",
+            "USDE": "ethena-usde",
+            "DAI": "dai",
+            "PYUSD": "paypal-usd"
+        }
+
+    def _load_coin_id_map(self) -> dict:
+        """Loads the dynamically generated coin-to-id mapping."""
+        map_file_path = os.path.join("_data", "coin_id_map.json")
+        try:
+            with open(map_file_path, 'r') as f:
+                return json.load(f)
+        except (FileNotFoundError, json.JSONDecodeError) as e:
+            logging.error(f"Could not load '{map_file_path}'. Please run 'update_coin_map.py' first. Error: {e}")
+            return {}
 
     def run(self):
         """
@@ -58,7 +63,7 @@ class MarketCapFetcher:
         with sqlite3.connect(self.db_path) as conn:
             conn.execute("PRAGMA journal_mode=WAL;")
 
-            # 1. Process individual coins
+            # 1. Process individual coins from the map
             for coin_symbol in self.coins_to_fetch:
                 coin_id = self.COIN_ID_MAP.get(coin_symbol.upper())
                 if not coin_id:
@@ -123,7 +128,6 @@ class MarketCapFetcher:
         table_name = "TOTAL_market_cap_daily"
         
         try:
-            # --- FIX: Use the current date instead of yesterday's ---
             today_date = datetime.now(timezone.utc).date()
             
             cursor = conn.cursor()
@@ -131,7 +135,6 @@ class MarketCapFetcher:
             table_exists = cursor.fetchone()
 
             if table_exists:
-                # Check if we already have a record for today
                 cursor.execute(f"SELECT 1 FROM \"{table_name}\" WHERE date(datetime_utc) = ? LIMIT 1", (today_date.isoformat(),))
                 if cursor.fetchone():
                     logging.info(f"Total market cap for {today_date} already exists. Skipping.")
@@ -245,7 +248,7 @@ class MarketCapFetcher:
             
         try:
             logging.debug(f"Fetching last {days} days for {coin_id}...")
-            response = requests.get(url, headers=headers)
+            response = requests.get(url, headers=headers, params=params)
             response.raise_for_status()
             data = response.json()
 
@@ -264,12 +267,7 @@ class MarketCapFetcher:
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Fetch historical market cap data from CoinGecko.")
-    parser.add_argument(
-        "--coins",
-        nargs='+',
-        default=["BTC", "ETH", "SOL", "BNB", "HYPE", "ASTER", "ZEC", "PUMP", "SUI"],
-        help="List of coin symbols to fetch (e.g., BTC ETH)."
-    )
+    # --- FIX: The --coins argument is no longer needed as the script is now fully automated ---
     parser.add_argument(
         "--log-level",
         default="normal",
@@ -278,6 +276,7 @@ if __name__ == "__main__":
     )
     args = parser.parse_args()
     
-    fetcher = MarketCapFetcher(log_level=args.log_level, coins=args.coins)
+    # The 'coins' argument is no longer passed to the constructor
+    fetcher = MarketCapFetcher(log_level=args.log_level)
     fetcher.run()