Clean up unused files, organize structure, update docs
- Delete obsolete files: data_fetcher_old.py, market_old.py, base_strategy.py (root), strategy_sma_cross.py, and old architecture remnants (address_monitor.py, position_monitor.py, trade_log.py, wallet_data.py, whale_tracker.py) - Delete zero-byte Docker artifacts and runtime files (clp_hedger.log, clp_hedger/hedge_status.json) - Move one-off utility scripts to scripts/ directory - Move example/template files to .temp/ directory - Update .gitignore: add entries for clp_hedger.log, clp_hedger/hedge_status.json, Docker layer hash files, Using, Running, and backups/ - Update .dockerignore: add clp_hedger.log, clp_hedger/hedge_status.json, backups/ - Create example config files: _data/strategies.json.example, _data/backtesting_conf.json.example, _data/coin_precision.json.example - Update GEMINI.md: remove outdated session summaries and duplicate review section - Update review.md: add cleanup status section, update remaining recommendations - Update MIGRATION_PLAN.md: mark completed phases, update file references - Update DOCKER_MIGRATION_GUIDE.md: update import_csv.py path reference
This commit is contained in:
15
resampler.py
15
resampler.py
@ -2,17 +2,20 @@ import argparse
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import warnings
|
||||
import db
|
||||
import pandas as pd
|
||||
import json
|
||||
from datetime import datetime, timezone, timedelta
|
||||
|
||||
warnings.filterwarnings("ignore", message="pandas only supports SQLAlchemy")
|
||||
|
||||
# Assuming logging_utils.py is in the same directory
|
||||
from logging_utils import setup_logging
|
||||
|
||||
class Resampler:
|
||||
"""
|
||||
Reads new 1-minute candle data from the SQLite database, resamples it to
|
||||
Reads new 1-minute candle data from the PostgreSQL database, resamples it to
|
||||
various timeframes, and upserts the new candles to the corresponding tables,
|
||||
preventing data duplication.
|
||||
"""
|
||||
@ -79,13 +82,13 @@ class Resampler:
|
||||
logging.debug(f"Processing {len(self.coins_to_process)} coins...")
|
||||
|
||||
for coin in self.coins_to_process:
|
||||
logging.debug(f"--- Processing {coin} ---")
|
||||
logging.info(f"--- Processing {coin} ---")
|
||||
|
||||
try:
|
||||
for tf_name, tf_code in self.timeframes.items():
|
||||
target_table_name = db.sanitize_table_name(coin, tf_name)
|
||||
source_table_name = db.sanitize_table_name(coin, "1m")
|
||||
logging.debug(f" Updating {tf_name} table...")
|
||||
logging.info(f" Resampling {coin} -> {tf_name}")
|
||||
|
||||
last_timestamp_ms = self._get_last_timestamp(conn, target_table_name)
|
||||
|
||||
@ -119,8 +122,8 @@ class Resampler:
|
||||
records_to_upsert.append((
|
||||
index.strftime('%Y-%m-%d %H:%M:%S'),
|
||||
int(index.timestamp() * 1000), # Generate timestamp_ms
|
||||
row['open'], row['high'], row['low'], row['close'],
|
||||
row['volume'], row['number_of_trades']
|
||||
float(row['open']), float(row['high']), float(row['low']), float(row['close']),
|
||||
float(row['volume']), int(row['number_of_trades'])
|
||||
))
|
||||
|
||||
db.upsert_candles(conn, target_table_name, records_to_upsert)
|
||||
@ -225,7 +228,7 @@ def parse_timeframes(tf_strings: list) -> dict:
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Resample 1-minute candle data from SQLite to other timeframes.")
|
||||
parser = argparse.ArgumentParser(description="Resample 1-minute candle data from PostgreSQL to other timeframes.")
|
||||
parser.add_argument("--coins", nargs='+', required=True, help="List of coins to process.")
|
||||
parser.add_argument("--timeframes", nargs='+', required=True, help="List of timeframes to generate.")
|
||||
parser.add_argument("--log-level", default="normal", choices=['off', 'normal', 'debug'])
|
||||
|
||||
Reference in New Issue
Block a user