Implement custom timeframes (37m, 148m) with real-time refresh for all intervals

scripts/generate_custom_timeframes.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""
+Generate custom timeframes (37m, 148m) from historical 1m data
+Run once to backfill all historical data
+"""
+
+import asyncio
+import argparse
+import logging
+import sys
+from pathlib import Path
+
+# Add parent to path
+sys.path.insert(0, str(Path(__file__).parent.parent / 'src'))
+
+from data_collector.database import DatabaseManager
+from data_collector.custom_timeframe_generator import CustomTimeframeGenerator
+
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+
+async def main():
+    parser = argparse.ArgumentParser(description='Generate custom timeframe candles')
+    parser.add_argument('--interval', 
+                       default='all',
+                       help='Which interval to generate (default: all, choices: 3m, 5m, 1h, 37m, etc.)')
+    parser.add_argument('--batch-size', type=int, default=5000,
+                       help='Number of source candles per batch')
+    parser.add_argument('--verify', action='store_true',
+                       help='Verify integrity after generation')
+    
+    args = parser.parse_args()
+    
+    # Initialize database
+    db = DatabaseManager()
+    await db.connect()
+    
+    try:
+        generator = CustomTimeframeGenerator(db)
+        await generator.initialize()
+        
+        if not generator.first_1m_time:
+            logger.error("No 1m data found in database. Cannot generate custom timeframes.")
+            return 1
+            
+        if args.interval == 'all':
+            intervals = list(generator.STANDARD_INTERVALS.keys()) + list(generator.CUSTOM_INTERVALS.keys())
+        else:
+            intervals = [args.interval]
+        
+        for interval in intervals:
+            logger.info(f"=" * 60)
+            logger.info(f"Generating {interval} candles")
+            logger.info(f"=" * 60)
+            
+            # Generate historical data
+            count = await generator.generate_historical(
+                interval=interval,
+                batch_size=args.batch_size
+            )
+            
+            logger.info(f"Generated {count} {interval} candles")
+            
+            # Verify if requested
+            if args.verify:
+                logger.info(f"Verifying {interval} integrity...")
+                stats = await generator.verify_integrity(interval)
+                logger.info(f"Stats: {stats}")
+                
+    except Exception as e:
+        logger.error(f"Error: {e}", exc_info=True)
+        return 1
+    finally:
+        await db.disconnect()
+        
+    logger.info("Custom timeframe generation complete!")
+    return 0
+
+
+if __name__ == '__main__':
+    exit_code = asyncio.run(main())
+    sys.exit(exit_code)