first stategies, import script for BTC histry
This commit is contained in:
217
main_app.py
217
main_app.py
@ -17,121 +17,141 @@ WATCHED_COINS = ["BTC", "ETH", "SOL", "BNB", "HYPE", "ASTER", "ZEC", "PUMP", "SU
|
||||
COIN_LISTER_SCRIPT = "list_coins.py"
|
||||
MARKET_FEEDER_SCRIPT = "market.py"
|
||||
DATA_FETCHER_SCRIPT = "data_fetcher.py"
|
||||
RESAMPLER_SCRIPT = "resampler.py" # Restored resampler script
|
||||
RESAMPLER_SCRIPT = "resampler.py"
|
||||
STRATEGY_CONFIG_FILE = os.path.join("_data", "strategies.json")
|
||||
PRICE_DATA_FILE = os.path.join("_data", "current_prices.json")
|
||||
DB_PATH = os.path.join("_data", "market_data.db")
|
||||
STATUS_FILE = os.path.join("_data", "fetcher_status.json")
|
||||
LOGS_DIR = "_logs" # Directory to store logs from background processes
|
||||
|
||||
|
||||
def run_market_feeder():
|
||||
"""Target function to run the market.py script in a separate process."""
|
||||
setup_logging('off', 'MarketFeedProcess')
|
||||
logging.info("Market feeder process started.")
|
||||
try:
|
||||
# Pass the log level to the script
|
||||
subprocess.run([sys.executable, MARKET_FEEDER_SCRIPT, "--log-level", "off"], check=True)
|
||||
except subprocess.CalledProcessError as e:
|
||||
logging.error(f"Market feeder script failed with error: {e}")
|
||||
except KeyboardInterrupt:
|
||||
logging.info("Market feeder process stopping.")
|
||||
"""Target function to run market.py and redirect its output to a log file."""
|
||||
log_file = os.path.join(LOGS_DIR, "market_feeder.log")
|
||||
while True:
|
||||
try:
|
||||
with open(log_file, 'a') as f:
|
||||
subprocess.run(
|
||||
[sys.executable, MARKET_FEEDER_SCRIPT, "--log-level", "normal"],
|
||||
check=True, stdout=f, stderr=subprocess.STDOUT
|
||||
)
|
||||
except (subprocess.CalledProcessError, Exception) as e:
|
||||
logging.error(f"Market feeder script failed: {e}. Restarting...")
|
||||
time.sleep(5)
|
||||
|
||||
|
||||
def run_data_fetcher_job():
|
||||
"""Defines the job to be run by the scheduler for the data fetcher."""
|
||||
logging.info(f"Scheduler starting data_fetcher.py task for {', '.join(WATCHED_COINS)}...")
|
||||
"""Defines the job for the data fetcher, redirecting output to a log file."""
|
||||
log_file = os.path.join(LOGS_DIR, "data_fetcher.log")
|
||||
logging.info(f"Scheduler starting data_fetcher.py task...")
|
||||
try:
|
||||
command = [sys.executable, DATA_FETCHER_SCRIPT, "--coins"] + WATCHED_COINS + ["--days", "7", "--log-level", "off"]
|
||||
subprocess.run(command, check=True)
|
||||
logging.info("data_fetcher.py task finished successfully.")
|
||||
command = [sys.executable, DATA_FETCHER_SCRIPT, "--coins"] + WATCHED_COINS + ["--days", "7", "--log-level", "normal"]
|
||||
with open(log_file, 'a') as f:
|
||||
subprocess.run(command, check=True, stdout=f, stderr=subprocess.STDOUT)
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to run data_fetcher.py job: {e}")
|
||||
|
||||
|
||||
def data_fetcher_scheduler():
|
||||
"""Schedules and runs the data_fetcher.py script periodically."""
|
||||
"""Schedules the data_fetcher.py script."""
|
||||
setup_logging('off', 'DataFetcherScheduler')
|
||||
run_data_fetcher_job()
|
||||
schedule.every(1).minutes.do(run_data_fetcher_job)
|
||||
logging.info("Data fetcher scheduled to run every 1 minute.")
|
||||
while True:
|
||||
schedule.run_pending()
|
||||
time.sleep(1)
|
||||
|
||||
# --- Restored Resampler Functions ---
|
||||
|
||||
def run_resampler_job():
|
||||
"""Defines the job to be run by the scheduler for the resampler."""
|
||||
logging.info(f"Scheduler starting resampler.py task for {', '.join(WATCHED_COINS)}...")
|
||||
"""Defines the job for the resampler, redirecting output to a log file."""
|
||||
log_file = os.path.join(LOGS_DIR, "resampler.log")
|
||||
logging.info(f"Scheduler starting resampler.py task...")
|
||||
try:
|
||||
# Uses default timeframes configured within resampler.py
|
||||
command = [sys.executable, RESAMPLER_SCRIPT, "--coins"] + WATCHED_COINS + ["--log-level", "off"]
|
||||
subprocess.run(command, check=True)
|
||||
logging.info("resampler.py task finished successfully.")
|
||||
command = [sys.executable, RESAMPLER_SCRIPT, "--coins"] + WATCHED_COINS + ["--log-level", "normal"]
|
||||
with open(log_file, 'a') as f:
|
||||
subprocess.run(command, check=True, stdout=f, stderr=subprocess.STDOUT)
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to run resampler.py job: {e}")
|
||||
|
||||
|
||||
def resampler_scheduler():
|
||||
"""Schedules and runs the resampler.py script periodically."""
|
||||
"""Schedules the resampler.py script."""
|
||||
setup_logging('off', 'ResamplerScheduler')
|
||||
run_resampler_job()
|
||||
schedule.every(4).minutes.do(run_resampler_job)
|
||||
logging.info("Resampler scheduled to run every 4 minutes.")
|
||||
while True:
|
||||
schedule.run_pending()
|
||||
time.sleep(1)
|
||||
# --- End of Restored Functions ---
|
||||
|
||||
|
||||
def run_strategy(strategy_name: str, config: dict):
|
||||
"""Target function to run a strategy, redirecting its output to a log file."""
|
||||
log_file = os.path.join(LOGS_DIR, f"strategy_{strategy_name}.log")
|
||||
script_name = config['script']
|
||||
params_str = json.dumps(config['parameters'])
|
||||
command = [sys.executable, script_name, "--name", strategy_name, "--params", params_str, "--log-level", "normal"]
|
||||
while True:
|
||||
try:
|
||||
with open(log_file, 'a') as f:
|
||||
f.write(f"\n--- Starting strategy '{strategy_name}' at {datetime.now()} ---\n")
|
||||
subprocess.run(command, check=True, stdout=f, stderr=subprocess.STDOUT)
|
||||
except (subprocess.CalledProcessError, Exception) as e:
|
||||
logging.error(f"Strategy '{strategy_name}' failed: {e}. Restarting...")
|
||||
time.sleep(10)
|
||||
|
||||
|
||||
class MainApp:
|
||||
def __init__(self, coins_to_watch: list):
|
||||
def __init__(self, coins_to_watch: list, processes: dict):
|
||||
self.watched_coins = coins_to_watch
|
||||
self.prices = {}
|
||||
self.last_db_update_info = "Initializing..."
|
||||
self._lines_printed = 0 # To track how many lines we printed last time
|
||||
self._lines_printed = 0
|
||||
self.background_processes = processes
|
||||
self.process_status = {}
|
||||
|
||||
def read_prices(self):
|
||||
"""Reads the latest prices from the JSON file."""
|
||||
if not os.path.exists(PRICE_DATA_FILE):
|
||||
return
|
||||
try:
|
||||
with open(PRICE_DATA_FILE, 'r', encoding='utf-8') as f:
|
||||
self.prices = json.load(f)
|
||||
except (json.JSONDecodeError, IOError):
|
||||
logging.debug("Could not read price file (might be locked).")
|
||||
if os.path.exists(PRICE_DATA_FILE):
|
||||
try:
|
||||
with open(PRICE_DATA_FILE, 'r', encoding='utf-8') as f:
|
||||
self.prices = json.load(f)
|
||||
except (json.JSONDecodeError, IOError):
|
||||
logging.debug("Could not read price file.")
|
||||
|
||||
def get_overall_db_status(self):
|
||||
"""Reads the fetcher status from the status file."""
|
||||
if not os.path.exists(STATUS_FILE):
|
||||
self.last_db_update_info = "Status file not found."
|
||||
return
|
||||
try:
|
||||
with open(STATUS_FILE, 'r', encoding='utf-8') as f:
|
||||
status = json.load(f)
|
||||
coin = status.get("last_updated_coin")
|
||||
timestamp_utc_str = status.get("last_run_timestamp_utc")
|
||||
num_candles = status.get("num_updated_candles", 0)
|
||||
if os.path.exists(STATUS_FILE):
|
||||
try:
|
||||
with open(STATUS_FILE, 'r', encoding='utf-8') as f:
|
||||
status = json.load(f)
|
||||
coin = status.get("last_updated_coin")
|
||||
timestamp_utc_str = status.get("last_run_timestamp_utc")
|
||||
num_candles = status.get("num_updated_candles", 0)
|
||||
if timestamp_utc_str:
|
||||
dt_utc = datetime.fromisoformat(timestamp_utc_str.replace('Z', '+00:00')).replace(tzinfo=timezone.utc)
|
||||
dt_local = dt_utc.astimezone(None)
|
||||
|
||||
# --- FIX: Manually format the UTC offset ---
|
||||
offset = dt_local.utcoffset()
|
||||
offset_hours = int(offset.total_seconds() / 3600)
|
||||
sign = '+' if offset_hours >= 0 else ''
|
||||
offset_str = f"(UTC{sign}{offset_hours})"
|
||||
timestamp_display = f"{dt_local.strftime('%Y-%m-%d %H:%M:%S')} {offset_str}"
|
||||
else:
|
||||
timestamp_display = "N/A"
|
||||
self.last_db_update_info = f"{coin} at {timestamp_display} ({num_candles} candles)"
|
||||
except (IOError, json.JSONDecodeError):
|
||||
self.last_db_update_info = "Error reading status file."
|
||||
|
||||
if timestamp_utc_str:
|
||||
dt_naive = datetime.strptime(timestamp_utc_str, '%Y-%m-%d %H:%M:%S')
|
||||
dt_utc = dt_naive.replace(tzinfo=timezone.utc)
|
||||
dt_local = dt_utc.astimezone(None)
|
||||
timestamp_display = dt_local.strftime('%Y-%m-%d %H:%M:%S %Z')
|
||||
else:
|
||||
timestamp_display = "N/A"
|
||||
|
||||
self.last_db_update_info = f"{coin} at {timestamp_display} ({num_candles} candles)"
|
||||
except (IOError, json.JSONDecodeError) as e:
|
||||
self.last_db_update_info = "Error reading status file."
|
||||
logging.error(f"Could not read status file: {e}")
|
||||
def check_process_status(self):
|
||||
"""Checks if the background processes are still running."""
|
||||
for name, process in self.background_processes.items():
|
||||
self.process_status[name] = "Running" if process.is_alive() else "STOPPED"
|
||||
|
||||
def display_dashboard(self):
|
||||
"""Displays a formatted table for prices and DB status without blinking."""
|
||||
# Move the cursor up to overwrite the previous output
|
||||
if self._lines_printed > 0:
|
||||
print(f"\x1b[{self._lines_printed}A", end="")
|
||||
|
||||
# Build the output as a single string
|
||||
output_lines = []
|
||||
output_lines.append("--- Market Dashboard ---")
|
||||
"""Displays a formatted table without blinking."""
|
||||
if self._lines_printed > 0: print(f"\x1b[{self._lines_printed}A", end="")
|
||||
output_lines = ["--- Market Dashboard ---"]
|
||||
table_width = 26
|
||||
output_lines.append("-" * table_width)
|
||||
output_lines.append(f"{'#':<2} | {'Coin':<6} | {'Live Price':>10} |")
|
||||
@ -140,23 +160,25 @@ class MainApp:
|
||||
price = self.prices.get(coin, "Loading...")
|
||||
output_lines.append(f"{i:<2} | {coin:<6} | {price:>10} |")
|
||||
output_lines.append("-" * table_width)
|
||||
output_lines.append(f"DB Status: Last coin updated -> {self.last_db_update_info}")
|
||||
|
||||
# Join lines and add a code to clear from cursor to end of screen
|
||||
# This prevents artifacts if the new output is shorter than the old one.
|
||||
output_lines.append("DB Status:")
|
||||
output_lines.append(f" Last update -> {self.last_db_update_info}")
|
||||
|
||||
output_lines.append("--- Background Processes ---")
|
||||
for name, status in self.process_status.items():
|
||||
output_lines.append(f"{name:<25}: {status}")
|
||||
|
||||
final_output = "\n".join(output_lines) + "\n\x1b[J"
|
||||
print(final_output, end="")
|
||||
|
||||
# Store the number of lines printed for the next iteration
|
||||
self._lines_printed = len(output_lines)
|
||||
|
||||
sys.stdout.flush()
|
||||
|
||||
def run(self):
|
||||
"""Main loop to read and display data."""
|
||||
"""Main loop to read data, display dashboard, and check processes."""
|
||||
while True:
|
||||
self.read_prices()
|
||||
self.get_overall_db_status()
|
||||
self.check_process_status()
|
||||
self.display_dashboard()
|
||||
time.sleep(2)
|
||||
|
||||
@ -164,6 +186,10 @@ class MainApp:
|
||||
if __name__ == "__main__":
|
||||
setup_logging('normal', 'MainApp')
|
||||
|
||||
# Create logs directory if it doesn't exist
|
||||
if not os.path.exists(LOGS_DIR):
|
||||
os.makedirs(LOGS_DIR)
|
||||
|
||||
logging.info(f"Running coin lister: '{COIN_LISTER_SCRIPT}'...")
|
||||
try:
|
||||
subprocess.run([sys.executable, COIN_LISTER_SCRIPT], check=True, capture_output=True, text=True)
|
||||
@ -171,35 +197,40 @@ if __name__ == "__main__":
|
||||
logging.error(f"Failed to run '{COIN_LISTER_SCRIPT}'. Error: {e.stderr}")
|
||||
sys.exit(1)
|
||||
|
||||
logging.info(f"Starting market feeder ('{MARKET_FEEDER_SCRIPT}')...")
|
||||
market_process = multiprocessing.Process(target=run_market_feeder, daemon=True)
|
||||
market_process.start()
|
||||
processes = {}
|
||||
|
||||
logging.info(f"Starting historical data fetcher ('{DATA_FETCHER_SCRIPT}')...")
|
||||
fetcher_process = multiprocessing.Process(target=data_fetcher_scheduler, daemon=True)
|
||||
fetcher_process.start()
|
||||
# Start Data Pipeline Processes
|
||||
processes["Market Feeder"] = multiprocessing.Process(target=run_market_feeder, daemon=True)
|
||||
processes["Data Fetcher"] = multiprocessing.Process(target=data_fetcher_scheduler, daemon=True)
|
||||
processes["Resampler"] = multiprocessing.Process(target=resampler_scheduler, daemon=True)
|
||||
|
||||
# --- Restored Resampler Process Start ---
|
||||
logging.info(f"Starting resampler ('{RESAMPLER_SCRIPT}')...")
|
||||
resampler_process = multiprocessing.Process(target=resampler_scheduler, daemon=True)
|
||||
resampler_process.start()
|
||||
# --- End Resampler Process Start ---
|
||||
# Start Strategy Processes based on config
|
||||
try:
|
||||
with open(STRATEGY_CONFIG_FILE, 'r') as f:
|
||||
strategy_configs = json.load(f)
|
||||
for name, config in strategy_configs.items():
|
||||
if config.get("enabled", False):
|
||||
proc = multiprocessing.Process(target=run_strategy, args=(name, config), daemon=True)
|
||||
processes[f"Strategy: {name}"] = proc
|
||||
except (FileNotFoundError, json.JSONDecodeError) as e:
|
||||
logging.error(f"Could not load strategies from '{STRATEGY_CONFIG_FILE}': {e}")
|
||||
|
||||
# Launch all processes
|
||||
for name, proc in processes.items():
|
||||
logging.info(f"Starting process '{name}'...")
|
||||
proc.start()
|
||||
|
||||
time.sleep(3)
|
||||
|
||||
app = MainApp(coins_to_watch=WATCHED_COINS)
|
||||
app = MainApp(coins_to_watch=WATCHED_COINS, processes=processes)
|
||||
try:
|
||||
app.run()
|
||||
except KeyboardInterrupt:
|
||||
logging.info("Shutting down...")
|
||||
market_process.terminate()
|
||||
fetcher_process.terminate()
|
||||
# --- Restored Resampler Shutdown ---
|
||||
resampler_process.terminate()
|
||||
market_process.join()
|
||||
fetcher_process.join()
|
||||
resampler_process.join()
|
||||
# --- End Resampler Shutdown ---
|
||||
for proc in processes.values():
|
||||
if proc.is_alive(): proc.terminate()
|
||||
for proc in processes.values():
|
||||
if proc.is_alive(): proc.join()
|
||||
logging.info("Shutdown complete.")
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user