# source: https://raw.githubusercontent.com/ikonclast/bob_der_botmeister/da05d31149b2f7577580f600e6bb591310274fe0/scripts/attempt2_pair_expansion.py #!/usr/bin/env python3 """ 2024 Attempt #2 - Pair Expansion SINGLE CHANGE: 5 -> 15 pairs (80% top volume, 20% diversity) Ziel: TEACHER_ALIVE_2024 (trades>=200, PF>=0.85, DD<=12%, profit>=-6%) Safety: STOP if TPPPD > 0.5 or trades > 900 """ import json import subprocess import zipfile import time import random from pathlib import Path from datetime import datetime OUTPUT_DIR = Path("/home/node/.openclaw/workspace/bob_quant/anchor_2024_attempt2") RUNS_DIR = OUTPUT_DIR / "runs" OUTPUT_DIR.mkdir(parents=True, exist_ok=True) RUNS_DIR.mkdir(parents=True, exist_ok=True) FREQTRADE_BASE = Path("/opt/docker/freqtrade/shared_data/user_data") STRATEGIES_DIR = FREQTRADE_BASE / "strategies" CONFIGS_DIR = FREQTRADE_BASE / "configs" BACKTEST_BASE = FREQTRADE_BASE / "backtest_results" DOCKER_ENV = {"DOCKER_HOST": "tcp://socket-proxy:2375"} TIMERANGE_2024 = "20240101-20241231" # 60-Asset Pool (Top Volume + Diversity) TOP_VOLUME_PAIRS = [ "BTC/USDT", "ETH/USDT", "SOL/USDT", "XRP/USDT", "BNB/USDT", "DOGE/USDT", "ADA/USDT", "TRX/USDT", "AVAX/USDT", "LINK/USDT", "SUI/USDT", "TON/USDT", "XLM/USDT", "LTC/USDT", "BCH/USDT", "DOT/USDT", "UNI/USDT", "DAI/USDT", "MATIC/USDT", "ICP/USDT", "ATOM/USDT", "ETC/USDT", "HYPE/USDT", "APT/USDT", "ARB/USDT", "NEAR/USDT", "VET/USDT", "FIL/USDT", "ALGO/USDT", "HBAR/USDT" ] DIVERSITY_POOL = [ "CRV/USDT", "GRT/USDT", "AXS/USDT", "IMX/USDT", "OP/USDT", "INJ/USDT", "GALA/USDT", "FET/USDT", "SEI/USDT", "TIA/USDT", "AR/USDT", "APE/USDT", "STRK/USDT", "BEAM/USDT", "BERA/USDT", "LDO/USDT", "SAND/USDT", "RENDER/USDT", "FLOKI/USDT", "BONK/USDT", "ENS/USDT", "PYTH/USDT", "JASMY/USDT", "GMT/USDT", "WLD/USDT", "PENDLE/USDT", "TAO/USDT", "RUNE/USDT", "CELO/USDT", "MINA/USDT", "SNX/USDT", "KAVA/USDT", "KSM/USDT", "FLOW/USDT", "CHZ/USDT", "DYDX/USDT", "BOME/USDT", "W/USDT", "TIA/USDT", "RNDR/USDT", "STORJ/USDT", "IOTX/USDT", "ONT/USDT", "ZIL/USDT", "NKN/USDT", "CELR/USDT", "SKL/USDT", "CTSI/USDT", "LRC/USDT", "SUSHI/USDT", "COMP/USDT", "AAVE/USDT", "MKR/USDT", "YFI/USDT", "BAL/USDT", "DASH/USDT", "ZEC/USDT", "XMR/USDT", "XTZ/USDT", "EOS/USDT" ] def select_pairs(seed: int) -> list: """Select 15 pairs: 12 top volume + 3 random diversity""" random.seed(seed) # 80% top volume (12 pairs) top_selected = random.sample(TOP_VOLUME_PAIRS[:20], 12) # 20% diversity (3 pairs) diversity_selected = random.sample(DIVERSITY_POOL, 3) pairs = top_selected + diversity_selected random.shuffle(pairs) # Shuffle to avoid bias return pairs def generate_strategy(run_id: str, pairs: list) -> str: """Generate breakout strategy with given pairs (NO logic changes from Attempt #1!)n """ # Deterministic parameters - reuse best from Attempt #1 base_seed = hash(run_id) % 100000 random.seed(base_seed) params = { "stoploss": round(-0.045, 3), "adx_min": 30, # NO CHANGE from Attempt #1! "breakout_mult": round(0.999, 4), # NO CHANGE! "volume_mult": round(1.0, 2), # NO CHANGE! "trailing": round(0.016, 3), "trailing_offset": round(0.025, 3), "time_stop_hours": 6, "time_stop_profit": 0.004 } # Mutation around best seeds from Attempt #1 if "BO24_03" in run_id or run_id in ["P2_020", "P2_025"]: params = { "stoploss": round(-0.045, 3), "adx_min": 30, "breakout_mult": round(0.999, 4), "volume_mult": round(1.0, 2), "trailing": round(0.016, 3), "trailing_offset": round(0.025, 3), "time_stop_hours": 6, "time_stop_profit": 0.004 } elif "BO24_04" in run_id or "P2_027" in run_id: params = { "stoploss": round(-0.050, 3), "adx_min": 25, "breakout_mult": round(0.997, 4), "volume_mult": round(0.9, 2), "trailing": round(0.016, 3), "trailing_offset": round(0.025, 3), "time_stop_hours": 6, "time_stop_profit": 0.004 } elif "BO24_06" in run_id or "BO24_08" in run_id: params = { "stoploss": round(-0.050, 3), "adx_min": 30, "breakout_mult": round(0.995, 4), "volume_mult": round(0.9, 2), "trailing": round(0.012, 3), "trailing_offset": round(0.030, 3), "time_stop_hours": 6, "time_stop_profit": 0.004 } # Generate strategy (NO logic changes!) strat_code = f"""from freqtrade.strategy import IStrategy from pandas import DataFrame import talib.abstract as ta class Github_ikonclast_bob_der_botmeister__attempt2_pair_expansion__20260716_095920(IStrategy): '''Breakout Uptrend Attempt2 More Pairs''' timeframe = '4h' stoploss = {params['stoploss']} minimal_roi = {{"0": 0.03, "180": 0.015, "360": 0.0}} trailing_stop = True trailing_stop_positive = {params['trailing']} trailing_stop_positive_offset = {params['trailing_offset']} trailing_only_offset_is_reached = True max_trade_duration = 1440 max_open_trades = 5 # Increased for more pairs # Safety limits pair_count = {len(pairs)} def populate_indicators(self, dataframe: DataFrame, metadata: dict): try: bb = ta.BBANDS(dataframe, timeperiod=20, nbdevup=2.0, nbdevdn=2.0) dataframe['bb_upper'] = bb['upperband'] dataframe['bb_middle'] = bb['middleband'] dataframe['bb_lower'] = bb['lowerband'] except: dataframe['bb_middle'] = dataframe['close'].rolling(20).mean() std = dataframe['close'].rolling(20).std() dataframe['bb_upper'] = dataframe['bb_middle'] + std * 2 dataframe['bb_lower'] = dataframe['bb_middle'] - std * 2 dataframe['ema_50'] = ta.EMA(dataframe, timeperiod=50) dataframe['adx'] = ta.ADX(dataframe, timeperiod=14) dataframe['volume_ma'] = ta.SMA(dataframe['volume'], timeperiod=20) dataframe['volume_spike'] = dataframe['volume'] / dataframe['volume_ma'].replace(0, 1) return dataframe def custom_exit(self, pair: str, trade, current_time, current_rate, current_profit, **kwargs): trade_duration = (current_time - trade.open_date_utc).total_seconds() / 3600 if trade_duration >= {params['time_stop_hours']} and current_profit < {params['time_stop_profit']}: return "time_stop_no_progress" if trade_duration >= 24: return "time_stop_hard" return None def populate_buy_trend(self, dataframe: DataFrame, metadata: dict): dataframe.loc[:, 'buy'] = 0 # Uptrend filter (NO CHANGE!) uptrend = dataframe['adx'] >= {params['adx_min']} # Breakout condition (NO CHANGE!) breakout = dataframe['close'] > dataframe['bb_upper'] * {params['breakout_mult']} # Volume confirmation (NO CHANGE!) volume_ok = dataframe['volume_spike'] > {params['volume_mult']} # Price in uptrend price_up = dataframe['close'] > dataframe['ema_50'] * 0.98 buy = uptrend & breakout & volume_ok & price_up dataframe.loc[buy, 'buy'] = 1 return dataframe def populate_sell_trend(self, dataframe: DataFrame, metadata: dict): dataframe.loc[:, 'sell'] = 0 to_middle = dataframe['close'] < dataframe['bb_middle'] * 1.005 dataframe.loc[to_middle, 'sell'] = 1 return dataframe """ return strat_code, params def run_single(run_id: str, seed_ref: str, run_num: int) -> dict: """Execute single backtest with 15 pairs""" run_dir = RUNS_DIR / run_id run_dir.mkdir(exist_ok=True) docker_backtest_dir = f"/freqtrade/user_data/backtest_results/a2_Github_ikonclast_bob_der_botmeister__attempt2_pair_expansion__20260716_095920" # Select pairs seed_val = hash(f"Github_ikonclast_bob_der_botmeister__attempt2_pair_expansion__20260716_095920_{run_num}") % 100000 pairs = select_pairs(seed_val) # Generate strategy strat_code, params = generate_strategy(run_id, pairs) strat_file = STRATEGIES_DIR / f"Github_ikonclast_bob_der_botmeister__attempt2_pair_expansion__20260716_095920.py" strat_file.write_text(strat_code) # Save config config = { "run_id": run_id, "run_num": run_num, "seed_ref": seed_ref, "pair_count": len(pairs), "pairs": pairs, "params": params, "timestamp": datetime.now().isoformat() } with open(run_dir / "config.json", 'w') as f: json.dump(config, f, indent=2) # Freqtrade config freq_config = { "max_open_trades": 5, "stake_currency": "USDT", "stake_amount": 100, "dry_run": True, "dry_run_wallet": 1000, "timeframe": "4h", "fee": 0.0015, "trading_mode": "spot", "exchange": {"name": "binance", "pair_whitelist": pairs}, "pairlists": [{"method": "StaticPairList"}], "entry_pricing": {"price_side": "other", "use_order_book": False}, "exit_pricing": {"price_side": "other", "use_order_book": False}, "telegram": {"enabled": False, "token": "x", "chat_id": "1"}, "api_server": {"enabled": False, "listen_ip_address": "127.0.0.1", "listen_port": 8080, "username": "a", "password": "a"} } cfg_file = CONFIGS_DIR / f"cfgGithub_ikonclast_bob_der_botmeister__attempt2_pair_expansion__20260716_095920.json" cfg_file.write_text(json.dumps(freq_config, indent=2)) # Run command cmd = [ "docker", "run", "--rm", "--network", "host", "-v", f"{FREQTRADE_BASE}:/freqtrade/user_data:rw", "freqtradeorg/freqtrade:stable", "backtesting", "--strategy", run_id, "--config", f"/freqtrade/user_data/configs/cfgGithub_ikonclast_bob_der_botmeister__attempt2_pair_expansion__20260716_095920.json", "--timeframe", "4h", "--timerange", TIMERANGE_2024, "--fee", "0.0015", "--export", "trades", "--backtest-directory", docker_backtest_dir, "--notes", f"Github_ikonclast_bob_der_botmeister__attempt2_pair_expansion__20260716_095920_2024_a2", "--cache", "none", "--no-color" ] result = { "run_id": run_id, "run_num": run_num, "seed_ref": seed_ref, "pair_count": len(pairs), "pairs": pairs, "timerange": TIMERANGE_2024, "params": params, "status": "PENDING" } try: proc = subprocess.run(cmd, capture_output=True, text=True, timeout=240, env=DOCKER_ENV) time.sleep(0.5) # Save logs with open(run_dir / "stdout.txt", 'w') as f: f.write(proc.stdout) with open(run_dir / "stderr.txt", 'w') as f: f.write(proc.stderr) # Find ZIP zip_files = list(BACKTEST_BASE.glob(f"a2_Github_ikonclast_bob_der_botmeister__attempt2_pair_expansion__20260716_095920-*.zip")) if not zip_files: result.update({"status": "FAILED", "error": "No ZIP file found"}) return result zip_path = max(zip_files, key=lambda p: p.stat().st_mtime) # Parse JSON with zipfile.ZipFile(zip_path, 'r') as zf: json_files = [f for f in zf.namelist() if f.endswith('.json') and '_config' not in f] if not json_files: result.update({"status": "FAILED", "error": "No JSON in ZIP"}) return result with zf.open(json_files[0]) as f: data = json.load(f) strat_name = list(data["strategy"].keys())[0] s = data["strategy"][strat_name] trades = s.get("total_trades", 0) profit = (s.get("profit_total") or 0) * 100 pf = s.get("profit_factor", 0) dd = (s.get("max_drawdown_account") or 0) * 100 tpppd = (trades / 15) / 366 if trades > 0 else 0 # 15 pairs now! # Exit reasons exit_reasons = s.get("sell_reason", {}) # Win/loss wins = s.get("wins", 0) losses = s.get("losses", 0) avg_win = (s.get("avg_win", 0) * 100) if wins > 0 else 0 avg_loss = (s.get("avg_loss", 0) * 100) if losses > 0 else 0 # Trades per pair analysis (from trades data if available) trades_per_pair = {} try: # Try to parse trades data for per-pair stats for trade_file in zf.namelist(): if "trades" in trade_file.lower() and trade_file.endswith('.csv'): with zf.open(trade_file) as tf: # This is simplified - actual parsing would need pandas pass except: pass # Label determination label = "NEGATIVE" if trades >= 200 and pf >= 1.10 and profit >= 0: label = "POSITIVE" elif trades >= 200 and pf >= 0.95 and dd <= 15 and profit >= -1: label = "POSITIVE_WEAK" elif trades >= 200 and pf >= 0.85 and dd <= 12 and profit >= -6: label = "TEACHER_ALIVE_2024" elif trades < 200: label = "LOW_SAMPLE" result.update({ "status": "SUCCESS", "trades": trades, "profit_pct": round(profit, 2), "pf": round(pf, 2), "max_dd": round(dd, 2), "tpppd": round(tpppd, 3), "high_churn": tpppd > 0.5 or trades > 900, "exit_reasons": exit_reasons, "avg_win": round(avg_win, 2), "avg_loss": round(avg_loss, 2), "label": label }) # HARD SAFETY: STOP if churn detected if result["high_churn"]: result["warning"] = "CHURN_DETECTED - Consider reducing pairs or tightening entry" except Exception as e: result.update({"status": "FAILED", "error": str(e)}) # Save metrics with open(run_dir / "metrics.json", 'w') as f: json.dump(result, f, indent=2, default=str) return result def main(): print("=" * 70) print("🎯 2024 ATTEMPT #2 - Pair Expansion (5→15)") print("=" * 70) print(f"Timerange: {TIMERANGE_2024}") print(f"Pairs: 15 (12 top-volume + 3 diversity)") print(f"Safety: STOP if TPPPD > 0.5 or trades > 900") print() print("Ziel: TEACHER_ALIVE_2024") print(" trades>=200, PF>=0.85, DD<=12%, profit>=-6%") print("=" * 70) results = [] # Seeds from best Attempt #1 runs seeds = [ ("BO24_03_BEST", "BO24_03", 5), # ADX=30, breakout=0.999 ("P2_025_BEST", "P2_025", 3), # PF=0.94, profit=-0.86% ("P2_027_BEST", "P2_027", 3), # PF=1.04, profit=+0.48% ("BO24_04", "BO24_04", 3), # Classic seed ("BO24_06", "BO24_06", 3), # Conservative ("BO24_08", "BO24_08", 3), # Aggressive ] run_counter = 0 for seed_name, seed_ref, count in seeds: print(f"\n--- Seed: {seed_name} ({count} runs) ---") for i in range(count): run_counter += 1 run_id = f"A2_{seed_name}_R{i+1:02d}" print(f"[{run_counter}/20] Github_ikonclast_bob_der_botmeister__attempt2_pair_expansion__20260716_095920...", end=" ", flush=True) result = run_single(run_id, seed_ref, run_counter) results.append(result) label = result.get('label', 'FAILED') marker = "" if label == "POSITIVE_WEAK": marker = " 🎯" elif label == "TEACHER_ALIVE_2024": marker = " ✅ TEACHER!" elif result.get('high_churn'): marker = " ⚠️ CHURN!" print(f"{label}: {result.get('trades')}T, {result.get('profit_pct')}%, PF={result.get('pf')}, TPPPD={result.get('tpppd')}{marker}") # Check for early safety stop if result.get('high_churn'): print("\n" + "⚠️" * 35) print("SAFETY STOP: High churn detected!") print("⚠️" * 35) break # Summary successful = [r for r in results if r['status'] == 'SUCCESS'] low_sample = len([r for r in successful if r['label'] == 'LOW_SAMPLE']) teacher_alive = [r for r in successful if r['label'] == 'TEACHER_ALIVE_2024'] pos_weak = [r for r in successful if r['label'] == 'POSITIVE_WEAK'] print("\n" + "=" * 70) print("📊 ATTEMPT #2 RESULTS") print("=" * 70) print(f"Total runs: {len(results)}") print(f"Successful: {len(successful)}") print(f"LOW_SAMPLE (<200 trades): {low_sample}/{len(successful)} ({low_sample/len(successful)*100:.0f}%)") print(f"TEACHER_ALIVE_2024: {len(teacher_alive)}") print(f"POSITIVE_WEAK: {len(pos_weak)}") if low_sample / len(successful) > 0.5: print("\n⚠️ LOW_SAMPLE > 50% - Considering entry relaxation for Attempt #3") if teacher_alive: print("\n✅ TEACHER_ALIVE_2024 achieved! Ready for curriculum.") # Save manifest manifest = { "attempt": "2024_attempt2", "date": datetime.now().isoformat(), "timerange": TIMERANGE_2024, "pair_count": 15, "selection": "80% top-volume + 20% diversity", "total_runs": len(results), "successful": len(successful), "teacher_alive_2024": len(teacher_alive), "positive_weak": len(pos_weak), "low_sample_pct": round(low_sample / len(successful) * 100, 1) if successful else 0, "results": results } with open(OUTPUT_DIR / "manifest.json", 'w') as f: json.dump(manifest, f, indent=2, default=str) # Generate report with open(OUTPUT_DIR / "attempt2_report.md", 'w') as f: f.write(f"# 2024 Attempt #2 - Pair Expansion Report\n\n") f.write(f"**Date:** {datetime.now().isoformat()}\n") f.write(f"**Pairs:** 15 (12 top-volume + 3 diversity)\n") f.write(f"**Timerange:** {TIMERANGE_2024}\n\n") f.write("## Summary\n\n") f.write(f"- Total runs: {len(results)}\n") f.write(f"- TEACHER_ALIVE_2024: {len(teacher_alive)}\n") f.write(f"- POSITIVE_WEAK: {len(pos_weak)}\n") f.write(f"- LOW_SAMPLE: {low_sample} ({low_sample/len(successful)*100:.1f}%)\n\n") f.write("| Run | Seed | Pairs | Trades | Profit | PF | MaxDD | TPPPD | Label |\n") f.write("|-----|------|-------|--------|--------|----|-------|-------|-------|\n") for r in results: seed = r.get('seed_ref', '-') pairs = r.get('pair_count', 0) trades = r.get('trades', '-') profit = r.get('profit_pct', '-') pf = r.get('pf', '-') dd = r.get('max_dd', '-') tp = r.get('tpppd', '-') label = r.get('label', 'FAILED') f.write(f"| {r['run_id']} | {seed} | {pairs} | {trades} | {profit}% | {pf} | {dd}% | {tp} | {label} |\n") print("=" * 70) if __name__ == "__main__": main()