102 lines
3.3 KiB
Python
102 lines
3.3 KiB
Python
import requests
|
|
import json
|
|
import sqlite3
|
|
import time
|
|
from datetime import datetime, timezone
|
|
|
|
DB_PATH = "_data/market_data.db"
|
|
URL = "https://api.hyperliquid.xyz/info"
|
|
|
|
def fetch_historical_candles(coin, start_ms, end_ms, interval="1m"):
|
|
"""Fetch historical candles using the raw HTTP API."""
|
|
candles = []
|
|
current_start = start_ms
|
|
while current_start < end_ms:
|
|
payload = {
|
|
"type": "candleSnapshot",
|
|
"req": {
|
|
"coin": coin,
|
|
"interval": interval,
|
|
"startTime": current_start,
|
|
"endTime": end_ms
|
|
}
|
|
}
|
|
resp = requests.post(URL, json=payload)
|
|
batch = resp.json()
|
|
if not batch:
|
|
break
|
|
for candle in batch:
|
|
candle['coin'] = coin
|
|
candles.append(candle)
|
|
last_ts = batch[-1]['t']
|
|
if last_ts < current_start:
|
|
break
|
|
current_start = last_ts + 1
|
|
time.sleep(0.5)
|
|
return candles
|
|
|
|
def write_candles_to_db(coin, candles, interval="1m"):
|
|
"""Write candles to the database."""
|
|
table_name = coin + "_" + interval
|
|
conn = sqlite3.connect(DB_PATH)
|
|
cursor = conn.cursor()
|
|
# Ensure table exists
|
|
cursor.execute(f'''
|
|
CREATE TABLE IF NOT EXISTS "{table_name}" (
|
|
datetime_utc TEXT,
|
|
timestamp_ms INTEGER PRIMARY KEY,
|
|
open REAL,
|
|
high REAL,
|
|
low REAL,
|
|
close REAL,
|
|
volume REAL,
|
|
number_of_trades INTEGER
|
|
)
|
|
''')
|
|
for candle in candles:
|
|
record = (
|
|
datetime.fromtimestamp(candle['t'] / 1000, tz=timezone.utc).strftime('%Y-%m-%d %H:%M:%S'),
|
|
candle['t'],
|
|
candle.get('o'), candle.get('h'), candle.get('l'), candle.get('c'),
|
|
candle.get('v'), candle.get('n')
|
|
)
|
|
cursor.execute(f'''
|
|
INSERT OR REPLACE INTO "{table_name}" (datetime_utc, timestamp_ms, open, high, low, close, volume, number_of_trades)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
''', record)
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
def get_last_timestamp(coin):
|
|
"""Get the most recent timestamp from the database."""
|
|
table_name = coin + "_1m"
|
|
conn = sqlite3.connect(DB_PATH)
|
|
cursor = conn.cursor()
|
|
try:
|
|
cursor.execute(f'SELECT MAX(timestamp_ms) FROM "{table_name}"')
|
|
result = cursor.fetchone()
|
|
return int(result[0]) if result and result[0] is not None else None
|
|
except:
|
|
return None
|
|
finally:
|
|
conn.close()
|
|
|
|
coins = ["mkts:USTECH", "xyz:XYZ100"]
|
|
now_ms = int(time.time() * 1000)
|
|
seven_days_ms = 7 * 24 * 60 * 60 * 1000
|
|
|
|
for coin in coins:
|
|
for tf in ["1m", "1d"]:
|
|
start_ts = now_ms - seven_days_ms
|
|
if start_ts >= now_ms:
|
|
print(f"{coin} ({tf}): Already up to date")
|
|
continue
|
|
|
|
print(f"{coin} ({tf}): Fetching historical candles from {datetime.fromtimestamp(start_ts/1000, tz=timezone.utc)} to {datetime.fromtimestamp(now_ms/1000, tz=timezone.utc)}...")
|
|
candles = fetch_historical_candles(coin, start_ts, now_ms, interval=tf)
|
|
print(f"{coin} ({tf}): Fetched {len(candles)} candles")
|
|
write_candles_to_db(coin, candles, interval=tf)
|
|
print(f"{coin} ({tf}): Written to database")
|
|
|
|
print("Done!")
|