p1otek/hyper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Clean up unused files, organize structure, update docs

Browse Source 
- Delete obsolete files: data_fetcher_old.py, market_old.py, base_strategy.py (root),
  strategy_sma_cross.py, and old architecture remnants (address_monitor.py,
  position_monitor.py, trade_log.py, wallet_data.py, whale_tracker.py)
- Delete zero-byte Docker artifacts and runtime files (clp_hedger.log,
  clp_hedger/hedge_status.json)
- Move one-off utility scripts to scripts/ directory
- Move example/template files to .temp/ directory
- Update .gitignore: add entries for clp_hedger.log, clp_hedger/hedge_status.json,
  Docker layer hash files, Using, Running, and backups/
- Update .dockerignore: add clp_hedger.log, clp_hedger/hedge_status.json, backups/
- Create example config files: _data/strategies.json.example,
  _data/backtesting_conf.json.example, _data/coin_precision.json.example
- Update GEMINI.md: remove outdated session summaries and duplicate review section
- Update review.md: add cleanup status section, update remaining recommendations
- Update MIGRATION_PLAN.md: mark completed phases, update file references
- Update DOCKER_MIGRATION_GUIDE.md: update import_csv.py path reference
This commit is contained in:
DiTus
2026-08-05 09:50:36 +02:00
parent 967c86e8e9
commit 1a95fe1caa
39 changed files with 599 additions and 3116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
scripts/check_wtioil.py Normal file
View File

@ -0,0 +1,81 @@
"""
Script to check if WTIOIL/CLUSD is available on Hyperliquid and add it to monitoring.
"""
import json
import logging
import requests
from hyperliquid.info import Info
from hyperliquid.utils import constants
from logging_utils import setup_logging
def check_and_add_wtioil():
"""Check if WTIOIL is available on Hyperliquid and add it to the precision file."""
setup_logging('normal', 'WTIOILChecker')
coin_name = "xyz:CLUSD" # Full HIP-3 format
alternative_names = ["WTIOIL", "CLUSD", "WTI"]
logging.info(f"Checking if {coin_name} is available on Hyperliquid...")
# Try direct HTTP API call for all mids
try:
url = 'https://api.hyperliquid.xyz/info'
payload = {"type": "allMids"}
response = requests.post(url, json=payload, timeout=10)
if response.status_code == 200:
result = response.json()
all_mids = result.get('mids', {})
print(f"\nTotal coins available: {len(all_mids)}")
# Look for oil-related coins
found = False
for name in all_mids.keys():
if 'oil' in name.lower() or 'wti' in name.lower() or 'cl' in name.lower() or 'xyz' in name.lower():
print(f"Found: {name} - Price: {all_mids[name]}")
found = True
if not found:
print("No oil-related coins found in all_mids.")
print("\nTrying alternative coin names...")
for alt_name in alternative_names:
try:
l2_payload = [{"type": "l2Book", "coin": alt_name}]
l2_response = requests.post(url, json=l2_payload, timeout=10)
if l2_response.status_code == 200:
l2_data = l2_response.json()
print(f"[OK] {alt_name} is available on Hyperliquid!")
print(f" L2 data: {l2_data}")
else:
print(f"[FAIL] {alt_name} not available (HTTP {l2_response.status_code})")
except Exception as e:
print(f"[ERROR] {alt_name}: {e}")
else:
print(f"Failed to get allMids: HTTP {response.status_code}")
print(f"Response: {response.text[:200]}")
except Exception as e:
logging.error(f"Error checking availability: {e}")
return
# Try to add to coin_precision.json
precision_file = "_data/coin_precision.json"
try:
with open(precision_file, 'r') as f:
precision_data = json.load(f)
# Add WTIOIL if not present
if coin_name not in precision_data:
precision_data[coin_name] = 2 # Default precision for commodities
with open(precision_file, 'w') as f:
json.dump(precision_data, f, indent=4, sort_keys=True)
logging.info(f"Added {coin_name} to {precision_file} with precision 2")
else:
logging.info(f"{coin_name} already exists in {precision_file}")
except Exception as e:
logging.error(f"Error updating precision file: {e}")
if __name__ == "__main__":
check_and_add_wtioil()

70
scripts/create_agent.py Normal file
View File

@ -0,0 +1,70 @@
import os
from eth_account import Account
from hyperliquid.exchange import Exchange
from hyperliquid.utils import constants
from dotenv import load_dotenv
from datetime import datetime, timedelta
import json
# Load environment variables from a .env file if it exists
load_dotenv()
def create_and_authorize_agent():
"""
Creates and authorizes a new agent key pair using your main wallet,
following the correct SDK pattern.
"""
# --- STEP 1: Load your main wallet ---
# This is the wallet that holds the funds and has been activated on Hyperliquid.
main_wallet_private_key = os.environ.get("MAIN_WALLET_PRIVATE_KEY")
if not main_wallet_private_key:
main_wallet_private_key = input("Please enter the private key of your MAIN trading wallet: ")
try:
main_account = Account.from_key(main_wallet_private_key)
print(f"\n✅ Loaded main wallet: {main_account.address}")
except Exception as e:
print(f"❌ Error: Invalid main wallet private key provided. Details: {e}")
return
# --- STEP 2: Initialize the Exchange with your MAIN account ---
# This object is used to send the authorization transaction.
exchange = Exchange(main_account, constants.MAINNET_API_URL, account_address=main_account.address)
# --- STEP 3: Create and approve the agent with a specific name ---
# agent name must be between 1 and 16 characters long
agent_name = "executor_SCALPER"
print(f"\n🔗 Authorizing a new agent named '{agent_name}'...")
try:
# --- FIX: Pass only the agent name string to the function ---
approve_result, agent_private_key = exchange.approve_agent(agent_name)
if approve_result.get("status") == "ok":
# Derive the agent's public address from the key we received
agent_account = Account.from_key(agent_private_key)
print("\n🎉 SUCCESS! Agent has been authorized on-chain.")
print("="*50)
print("SAVE THESE SECURELY. This is what your bot will use.")
print(f" Name: {agent_name}")
print(f" (Agent has a default long-term validity)")
print(f"🔑 Agent Private Key: {agent_private_key}")
print(f"🏠 Agent Address: {agent_account.address}")
print("="*50)
print("\nYou can now set this private key as the AGENT_PRIVATE_KEY environment variable.")
else:
print("\n❌ ERROR: Agent authorization failed.")
print(" Response:", approve_result)
if "Vault may not perform this action" in str(approve_result):
print("\n ACTION REQUIRED: This error means your main wallet (vault) has not been activated. "
"Please go to the Hyperliquid website, connect this wallet, and make a deposit to activate it.")
except Exception as e:
print(f"\nAn unexpected error occurred during authorization: {e}")
if __name__ == "__main__":
create_and_authorize_agent()

56
scripts/del_market_cap_tables.py Normal file
View File

@ -0,0 +1,56 @@
import sqlite3
import logging
import os
from logging_utils import setup_logging
def cleanup_market_cap_tables():
"""
Scans the database and drops all tables related to market cap data
to allow for a clean refresh.
"""
setup_logging('normal', 'DBCleanup')
db_path = os.path.join("_data", "market_data.db")
if not os.path.exists(db_path):
logging.error(f"Database file not found at '{db_path}'. Nothing to clean.")
return
logging.info(f"Connecting to database at '{db_path}'...")
try:
with sqlite3.connect(db_path) as conn:
cursor = conn.cursor()
# Find all tables that were created by the market cap fetcher
cursor.execute("""
SELECT name FROM sqlite_master
WHERE type='table'
AND (name LIKE '%_market_cap' OR name LIKE 'TOTAL_%')
""")
tables_to_drop = cursor.fetchall()
if not tables_to_drop:
logging.info("No market cap tables found to clean up. Database is already clean.")
return
logging.warning(f"Found {len(tables_to_drop)} market cap tables to remove...")
for table in tables_to_drop:
table_name = table[0]
try:
logging.info(f"Dropping table: {table_name}...")
conn.execute(f'DROP TABLE IF EXISTS "{table_name}"')
except Exception as e:
logging.error(f"Failed to drop table {table_name}: {e}")
conn.commit()
logging.info("--- Database cleanup complete ---")
except sqlite3.Error as e:
logging.error(f"A database error occurred: {e}")
except Exception as e:
logging.error(f"An unexpected error occurred: {e}")
if __name__ == "__main__":
cleanup_market_cap_tables()

118
scripts/fix_timestamps.py Normal file
View File

@ -0,0 +1,118 @@
import argparse
import logging
import os
import sys
import sqlite3
import pandas as pd
# script to fix missing millisecond timestamps in the database after import from CSVs (this is already fixed in import_csv.py)
# Assuming logging_utils.py is in the same directory
from logging_utils import setup_logging
class DatabaseFixer:
"""
Scans the SQLite database for rows with missing millisecond timestamps
and updates them based on the datetime_utc column.
"""
def __init__(self, log_level: str, coin: str):
setup_logging(log_level, 'TimestampFixer')
self.coin = coin
self.table_name = f"{self.coin}_1m"
self.db_path = os.path.join("_data", "market_data.db")
def run(self):
"""Orchestrates the entire database update and verification process."""
logging.info(f"Starting timestamp fix process for table '{self.table_name}'...")
if not os.path.exists(self.db_path):
logging.error(f"Database file not found at '{self.db_path}'. Exiting.")
sys.exit(1)
try:
with sqlite3.connect(self.db_path) as conn:
conn.execute("PRAGMA journal_mode=WAL;")
# 1. Check how many rows need fixing
rows_to_fix_count = self._count_rows_to_fix(conn)
if rows_to_fix_count == 0:
logging.info(f"No rows with missing timestamps found in '{self.table_name}'. No action needed.")
return
logging.info(f"Found {rows_to_fix_count:,} rows with missing timestamps to update.")
# 2. Process the table in chunks to conserve memory
updated_count = self._process_in_chunks(conn)
# 3. Provide a final summary
self._summarize_update(rows_to_fix_count, updated_count)
except Exception as e:
logging.error(f"A critical error occurred: {e}")
def _count_rows_to_fix(self, conn) -> int:
"""Counts the number of rows where timestamp_ms is NULL."""
try:
return pd.read_sql(f'SELECT COUNT(*) FROM "{self.table_name}" WHERE timestamp_ms IS NULL', conn).iloc[0, 0]
except pd.io.sql.DatabaseError:
logging.error(f"Table '{self.table_name}' not found in the database. Cannot fix timestamps.")
sys.exit(1)
def _process_in_chunks(self, conn) -> int:
"""Reads, calculates, and updates timestamps in manageable chunks."""
total_updated = 0
chunk_size = 50000 # Process 50,000 rows at a time
# We select the special 'rowid' column to uniquely identify each row for updating
query = f'SELECT rowid, datetime_utc FROM "{self.table_name}" WHERE timestamp_ms IS NULL'
for chunk_df in pd.read_sql_query(query, conn, chunksize=chunk_size):
if chunk_df.empty:
break
logging.info(f"Processing a chunk of {len(chunk_df)} rows...")
# Calculate the missing timestamps
chunk_df['datetime_utc'] = pd.to_datetime(chunk_df['datetime_utc'])
chunk_df['timestamp_ms'] = (chunk_df['datetime_utc'].astype('int64') // 10**6)
# Prepare data for the update command: a list of (timestamp, rowid) tuples
update_data = list(zip(chunk_df['timestamp_ms'], chunk_df['rowid']))
# Use executemany for a fast bulk update
cursor = conn.cursor()
cursor.executemany(f'UPDATE "{self.table_name}" SET timestamp_ms = ? WHERE rowid = ?', update_data)
conn.commit()
total_updated += len(chunk_df)
logging.info(f"Updated {total_updated} rows so far...")
return total_updated
def _summarize_update(self, expected_count: int, actual_count: int):
"""Prints a final summary of the update process."""
logging.info("--- Timestamp Fix Summary ---")
print(f"\n{'Status':<25}: COMPLETE")
print("-" * 40)
print(f"{'Table Processed':<25}: {self.table_name}")
print(f"{'Rows Needing Update':<25}: {expected_count:,}")
print(f"{'Rows Successfully Updated':<25}: {actual_count:,}")
if expected_count == actual_count:
logging.info("Verification successful: All necessary rows have been updated.")
else:
logging.warning("Verification warning: The number of updated rows does not match the expected count.")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Fix missing millisecond timestamps in the SQLite database.")
parser.add_argument("--coin", default="BTC", help="The coin symbol for the table to fix (e.g., BTC).")
parser.add_argument(
"--log-level",
default="normal",
choices=['off', 'normal', 'debug'],
help="Set the logging level for the script."
)
args = parser.parse_args()
fixer = DatabaseFixer(log_level=args.log_level, coin=args.coin)
fixer.run()

156
scripts/import_csv.py Normal file
View File

@ -0,0 +1,156 @@
import argparse
import logging
import os
import sys
import db
import pandas as pd
from datetime import datetime
# Assuming logging_utils.py is in the same directory
from logging_utils import setup_logging
class CsvImporter:
"""
Imports historical candle data from a large CSV file into the PostgreSQL database,
intelligently adding only the missing data.
"""
def __init__(self, log_level: str, csv_path: str, coin: str):
setup_logging(log_level, 'CsvImporter')
if not os.path.exists(csv_path):
logging.error(f"CSV file not found at '{csv_path}'. Please check the path.")
sys.exit(1)
self.csv_path = csv_path
self.coin = coin
# --- FIX: Corrected the f-string syntax for the table name ---
self.table_name = db.sanitize_table_name(self.coin, "1m")
self.db_path = os.environ.get("PG_CONN_STR", "postgresql://hyper:hyper@localhost:5432/hyper")
self.column_mapping = {
'Open time': 'datetime_utc',
'Open': 'open',
'High': 'high',
'Low': 'low',
'Close': 'close',
'Volume': 'volume',
'Number of trades': 'number_of_trades'
}
def run(self):
"""Orchestrates the entire import and verification process."""
logging.info(f"Starting import process for '{self.coin}' from '{self.csv_path}'...")
conn = db.get_connection()
try:
# 1. Get the current state of the database
db_oldest, db_newest, initial_row_count = self._get_db_state(conn)
# 2. Read, clean, and filter the CSV data
new_data_df = self._process_and_filter_csv(db_oldest, db_newest)
if new_data_df.empty:
logging.info("No new data to import. Database is already up-to-date with the CSV file.")
return
# 3. Append the new data to the database
self._append_to_db(new_data_df, conn)
# 4. Summarize and verify the import
self._summarize_import(initial_row_count, len(new_data_df), conn)
finally:
conn.close()
def _get_db_state(self, conn) -> (datetime, datetime, int):
"""Gets the oldest and newest timestamps and total row count from the DB table."""
try:
oldest = pd.read_sql(f'SELECT MIN(datetime_utc) FROM "{self.table_name}"', conn).iloc[0, 0]
newest = pd.read_sql(f'SELECT MAX(datetime_utc) FROM "{self.table_name}"', conn).iloc[0, 0]
count = pd.read_sql(f'SELECT COUNT(*) FROM "{self.table_name}"', conn).iloc[0, 0]
oldest_dt = pd.to_datetime(oldest) if oldest else None
newest_dt = pd.to_datetime(newest) if newest else None
if oldest_dt:
logging.info(f"Database contains data from {oldest_dt} to {newest_dt}.")
else:
logging.info("Database table is empty. A full import will be performed.")
return oldest_dt, newest_dt, count
except pd.io.sql.DatabaseError:
logging.info(f"Table '{self.table_name}' not found. It will be created.")
return None, None, 0
def _process_and_filter_csv(self, db_oldest: datetime, db_newest: datetime) -> pd.DataFrame:
"""Reads the CSV and returns a DataFrame of only the missing data."""
logging.info("Reading and processing CSV file. This may take a moment for large files...")
df = pd.read_csv(self.csv_path, usecols=self.column_mapping.keys())
# Clean and format the data
df.rename(columns=self.column_mapping, inplace=True)
df['datetime_utc'] = pd.to_datetime(df['datetime_utc'])
# --- FIX: Calculate the millisecond timestamp from the datetime column ---
# This converts the datetime to nanoseconds and then to milliseconds.
df['timestamp_ms'] = (df['datetime_utc'].astype('int64') // 10**6)
# Filter the data to find only rows that are outside the range of what's already in the DB
if db_oldest and db_newest:
# Get data from before the oldest record and after the newest record
df_filtered = df[(df['datetime_utc'] < db_oldest) | (df['datetime_utc'] > db_newest)]
else:
# If the DB is empty, all data is new
df_filtered = df
logging.info(f"Found {len(df_filtered):,} new rows to import.")
return df_filtered
def _append_to_db(self, df: pd.DataFrame, conn):
"""Appends the DataFrame to the database."""
logging.info(f"Appending {len(df):,} new rows to the database...")
records = list(df.itertuples(index=False, name=None))
db.upsert_candles(conn, self.table_name, records)
logging.info("Append operation complete.")
def _summarize_import(self, initial_count: int, added_count: int, conn):
"""Prints a final summary and verification of the import."""
logging.info("--- Import Summary & Verification ---")
try:
final_count = pd.read_sql(f'SELECT COUNT(*) FROM "{self.table_name}"', conn).iloc[0, 0]
new_oldest = pd.read_sql(f'SELECT MIN(datetime_utc) FROM "{self.table_name}"', conn).iloc[0, 0]
new_newest = pd.read_sql(f'SELECT MAX(datetime_utc) FROM "{self.table_name}"', conn).iloc[0, 0]
print(f"\n{'Status':<20}: SUCCESS")
print("-" * 40)
print(f"{'Initial Row Count':<20}: {initial_count:,}")
print(f"{'Rows Added':<20}: {added_count:,}")
print(f"{'Final Row Count':<20}: {final_count:,}")
print("-" * 40)
print(f"{'New Oldest Record':<20}: {new_oldest}")
print(f"{'New Newest Record':<20}: {new_newest}")
# Verification check
if final_count == initial_count + added_count:
logging.info("Verification successful: Final row count matches expected count.")
else:
logging.warning("Verification warning: Final row count does not match expected count.")
except Exception as e:
logging.error(f"Could not generate summary. Error: {e}")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Import historical CSV data into the PostgreSQL database.")
parser.add_argument("--file", required=True, help="Path to the large CSV file to import.")
parser.add_argument("--coin", default="BTC", help="The coin symbol for this data (e.g., BTC).")
parser.add_argument(
"--log-level",
default="normal",
choices=['off', 'normal', 'debug'],
help="Set the logging level for the script."
)
args = parser.parse_args()
importer = CsvImporter(log_level=args.log_level, csv_path=args.file, coin=args.coin)
importer.run()

61
scripts/list_coins.py Normal file
View File

@ -0,0 +1,61 @@
import json
import logging
from hyperliquid.info import Info
from hyperliquid.utils import constants
# Import the setup function from our new logging module
from logging_utils import setup_logging
def save_coin_precision_data():
"""
Connects to the Hyperliquid API, gets a list of all listed coins
and their trade size precision, and saves it to a JSON file.
"""
logging.info("Fetching asset information from Hyperliquid...")
try:
info = Info(constants.MAINNET_API_URL, skip_ws=True)
meta_data = info.meta_and_asset_ctxs()[0]
all_assets = meta_data.get("universe", [])
if not all_assets:
logging.error("Could not retrieve asset information from the meta object.")
return
# Create a dictionary mapping the coin name to its precision
coin_precision_map = {}
for asset in all_assets:
name = asset.get("name")
precision = asset.get("szDecimals")
if name is not None and precision is not None:
coin_precision_map[name] = precision
# Save the dictionary to a JSON file
file_name = "_data/coin_precision.json"
with open(file_name, 'w', encoding='utf-8') as f:
# indent=4 makes the file readable; sort_keys keeps it organized
json.dump(coin_precision_map, f, indent=4, sort_keys=True)
logging.info(f"Successfully saved coin precision data to '{file_name}'")
# Provide an example of how to use the generated file
# print("\n--- Example Usage in another script ---")
# print("import json")
# print("\n# Load the data from the file")
# print("with open('coin_precision.json', 'r') as f:")
# print(" precision_data = json.load(f)")
# print("\n# Access the precision for a specific coin")
# print("eth_precision = precision_data.get('ETH')")
# print("print(f'The size precision for ETH is: {eth_precision}')")
except Exception as e:
logging.error(f"An error occurred: {e}")
if __name__ == "__main__":
# Setup logging with a specified level and process name
setup_logging('off', 'CoinLister')
save_coin_precision_data()

92
scripts/migrate_to_sqlite.py Normal file
View File

@ -0,0 +1,92 @@
import argparse
import logging
import os
import sys
import sqlite3
import pandas as pd
# Assuming logging_utils.py is in the same directory
from logging_utils import setup_logging
class Migrator:
"""
Reads 1-minute candle data from CSV files and migrates it into an
SQLite database for improved performance and easier access.
"""
def __init__(self, log_level: str):
setup_logging(log_level, 'Migrator')
self.source_folder = os.path.join("_data", "candles")
self.db_path = os.path.join("_data", "market_data.db")
def run(self):
"""
Main execution function to find all CSV files and migrate them to the database.
"""
if not os.path.exists(self.source_folder):
logging.error(f"Source data folder '{self.source_folder}' not found. "
"Please ensure data has been fetched first.")
sys.exit(1)
csv_files = [f for f in os.listdir(self.source_folder) if f.endswith('_1m.csv')]
if not csv_files:
logging.warning("No 1-minute CSV files found in the source folder to migrate.")
return
logging.info(f"Found {len(csv_files)} source CSV files to migrate to SQLite.")
# Connect to the SQLite database (it will be created if it doesn't exist)
with sqlite3.connect(self.db_path) as conn:
for file_name in csv_files:
coin = file_name.split('_')[0]
table_name = f"{coin}_1m"
file_path = os.path.join(self.source_folder, file_name)
logging.info(f"Migrating '{file_name}' to table '{table_name}'...")
try:
# 1. Load the entire CSV file into a pandas DataFrame.
df = pd.read_csv(file_path)
if df.empty:
logging.warning(f"CSV file '{file_name}' is empty. Skipping.")
continue
# 2. Convert the timestamp column to a proper datetime object.
df['datetime_utc'] = pd.to_datetime(df['datetime_utc'])
# 3. Write the DataFrame to the SQLite database.
# 'replace' will drop the table first if it exists and create a new one.
# This is ideal for a migration script to ensure a clean import.
df.to_sql(
table_name,
conn,
if_exists='replace',
index=False # Do not write the pandas DataFrame index as a column
)
# 4. (Optional but Recommended) Create an index on the timestamp for fast queries.
logging.debug(f"Creating index on 'datetime_utc' for table '{table_name}'...")
conn.execute(f"CREATE INDEX IF NOT EXISTS idx_{table_name}_time ON {table_name}(datetime_utc);")
logging.info(f"Successfully migrated {len(df)} rows to '{table_name}'.")
except Exception as e:
logging.error(f"Failed to process and migrate file '{file_name}': {e}")
logging.info("--- Database migration complete ---")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Migrate 1-minute candle data from CSV files to an SQLite database.")
parser.add_argument(
"--log-level",
default="normal",
choices=['off', 'normal', 'debug'],
help="Set the logging level for the script."
)
args = parser.parse_args()
migrator = Migrator(log_level=args.log_level)
migrator.run()