From 98daf71a525a0ca3c0c0df9dbcac80773cadabde Mon Sep 17 00:00:00 2001 From: harwinder Date: Thu, 14 May 2026 23:15:37 -0400 Subject: [PATCH] Add web dashboard (Redis + FastAPI + WebSocket), scope XAU penalties Signal daemon (signal_web.py): candle-boundary scanner mirroring telegram_bot.py, publishes serialized SignalReport JSON to Redis. Web API (web/app.py): thin FastAPI reader behind Apache, serves /api/signals REST + /ws WebSocket from Redis pub/sub. Frontend (web/static/index.html): 3-pair grid, color-coded signals, indicator breakdown per timeframe, trade setup display. Also: XAU-specific penalty scoping in signal_aggregator (softer confidence penalties for gold, original strict values for forex), pair-aware session filter routing XAU to its own blocked hours. Co-Authored-By: Claude Opus 4.6 --- CLAUDE.md | 110 ++++++++ backtest/evaluate.py | 618 ------------------------------------------ config.py | 48 +++- mt5.md | 185 ------------- requirements.txt | 3 + signal_aggregator.py | 405 ++++++++++++++++++++++----- signal_web.py | 303 +++++++++++++++++++++ test_all.py | 9 +- web/app.py | 121 +++++++++ web/static/index.html | 482 ++++++++++++++++++++++++++++++++ 10 files changed, 1401 insertions(+), 883 deletions(-) create mode 100644 CLAUDE.md delete mode 100644 backtest/evaluate.py delete mode 100644 mt5.md create mode 100644 signal_web.py create mode 100644 web/app.py create mode 100644 web/static/index.html diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..33c563c --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,110 @@ +# ForexBot2 + +Automated forex signal intelligence platform. Analyzes 4 currency pairs (EUR/USD, GBP/USD, USD/JPY, XAU/USD) across multiple timeframes using technical indicators, news sentiment, and economic calendar data. Delivers signals via Telegram with full trade setups (entry, SL, TP). + +## Architecture + +``` +Twelve Data REST API + WebSocket + | + data_provider.py (fetch, cache, validate OHLCV; rate-limited 55 calls/min) + | + engine.py (6 indicators per timeframe → composite score) + | + signal_aggregator.py (MTF agreement + news + final signal + trade setup) + | + +-----+--------+ + | | +telegram_bot.py main.py +(Telegram alerts, (CLI single-pair + auto-scanner, analysis) + WS monitor, + live dashboard) +``` + +## Core Files + +| File | Purpose | +|------|---------| +| `config.py` | All configuration dataclasses, pair definitions, strategy configs | +| `data_provider.py` | Twelve Data SDK wrapper with in-memory cache and rate limiting | +| `engine.py` | `TechnicalEngine` — computes EMA, RSI, BB, ADX, ATR, MACD, divergences, candlestick patterns, swing S/R | +| `signal_aggregator.py` | `SignalAggregator` — MTF analysis, news integration, final signal scoring, trade setup generation | +| `news_engine.py` | Economic calendar (Forex Factory JSON) + RSS sentiment (keyword-based) | +| `telegram_bot.py` | Async Telegram bot with auto-scanner (M5/M15/H1/H4), WebSocket price monitor, live dashboard | +| `main.py` | CLI entry point for single USD/JPY analysis | +| `test_all.py` | Comprehensive QA suite (~90 tests, no live API needed) | + +## Key Concepts + +**Signal Pipeline:** OHLCV → indicators → regime classification → regime-weighted composite score → MTF agreement check → news/event adjustment → confidence penalties → final signal (STRONG_BUY/BUY/NEUTRAL/SELL/STRONG_SELL) → quality grade (A+/A/B/C/D/F) → trade setup if grade C+. + +**Regime-Adaptive Weights:** Market classified as `trending_up`, `trending_down`, or `ranging` via ADX + EMA. Trending boosts EMA weight (2.0x), ranging boosts RSI/BB (2.0x). This is the core edge. + +**Pair-Specific Strategies:** XAU/USD (oversold-bounce: RSI<38 + triple-TF EMA200 confirmation), EUR/USD and GBP/USD (pullback-buy: RSI 30-55 + BB<0.75 + dual-TF EMA200). Each has hour-blocked windows from backtest optimization. + +**SELL Suppression:** Forex SELL signals are suppressed for all pairs except XAU/USD. Backtesting showed 0% win rate for forex shorts. + +**Trade Setup:** SL = nearest swing low/high ± 1×ATR buffer (min 1.5×ATR). TP = nearest swing level if RR≥1.2, else calculated from ATR. Minimum RR 1.2:1. + +## How to Run + +```bash +# CLI (single USD/JPY analysis) +python main.py # H1 default +python main.py --timeframe M15 --detail --json + +# Telegram bot (auto-scanner) +TELEGRAM_TOKEN= TELEGRAM_CHAT_ID= python telegram_bot.py + +# Tests +python test_all.py +``` + +## Environment Variables + +| Variable | Purpose | Default | +|----------|---------|---------| +| `TWELVE_DATA_API_KEY` | Market data API key | Hardcoded fallback in config | +| `TELEGRAM_TOKEN` | Telegram bot token | Hardcoded fallback | +| `TELEGRAM_CHAT_ID` | Auto-scan chat ID (enables boot auto-scan) | Hardcoded fallback | +| `FB2_EMA_LO` / `FB2_EMA_HI` | EMA200 distance thresholds (%) | 0.5 / 1.5 | + +## Dependencies + +`numpy`, `pandas`, `requests`, `twelvedata`, `python-telegram-bot`, `websockets` — see `requirements.txt`. + +## Data Provider + +- **Source:** Twelve Data REST API via `twelvedata` SDK +- **Rate limit:** 55 credits/min (Grow plan), enforced by sliding-window throttle +- **Cache:** In-memory dict, TTL = candle duration of the timeframe +- **Validation:** Drops NaN rows, forward-fills gaps (limit 3), fixes OHLC inconsistencies, removes outliers (>10σ), strips future timestamps, deduplicates +- **WebSocket:** EUR/USD and XAU/USD only (Grow plan WS limit); GBP/USD and USD/JPY via REST-only scans + +## Telegram Bot + +- **Auto-scan:** M5 (data-only, no alerts), M15, H1, H4 at candle boundaries + 10s buffer +- **WebSocket:** Streams live ticks for EUR/USD + XAU/USD; triggers immediate M15 scan on key level breach (swing S/R, BB, EMA200) with 5-min cooldown +- **Live dashboard:** Edit-in-place message, 30s refresh, shows all pairs/timeframes/prices/setups +- **Commands:** `/start`, `/autoscan`, `/status`, `/cache`, `/clearcache`, `/help` +- **Correlation warning:** Flags same-direction signals on EUR/USD + GBP/USD (USD-weakness group) + +## Backtesting + +The `backtest/` directory (gitignored) contains pair-specific backtesting scripts, GPU-accelerated optimizers, and historical parquet data. Strategies were optimized on 1-year M15 data. Results drove the current strategy configs in `config.py`. + +## Planned: MT5 Auto-Execution + +Documented in `mt5.md`. REST bridge architecture: Linux bot → HTTP → Windows FastAPI server wrapping MetaTrader5 COM. Not yet implemented. Key: `MT5Config` dataclass, idempotent orders, position-size from bridge-side tick values. + +## Code Conventions + +- Python 3.12, type hints on public methods +- Dataclasses for all config and output structures +- `SignalStrength` enum (-2 to +2) is the universal signal currency +- Indicators stored as `Dict[str, IndicatorSignal]` in `TechnicalAnalysis` +- Key levels stored as `Dict[str, float]` — prefixed `swing_high_`, `swing_low_`, `ema_`, `bb_`, `atr` +- Private indicator data attached as `result._rsi_series`, `result._macd_histogram` (underscore prefix, skipped by composite scorer) +- Blocking signal computation runs in `ThreadPoolExecutor(max_workers=4)` to avoid blocking the async Telegram event loop +- Shared `ForexDataProvider` instance across all scans for cache reuse diff --git a/backtest/evaluate.py b/backtest/evaluate.py deleted file mode 100644 index f04c2dc..0000000 --- a/backtest/evaluate.py +++ /dev/null @@ -1,618 +0,0 @@ -#!/usr/bin/env python3 -""" -ForexBot2 — Historical Signal Evaluation -========================================= -Walks the engine over the last 7 days of market data for all scan pairs, -records every signal and trade setup, then checks outcomes on M5 data. - -GPU-accelerated outcome evaluation via CuPy (RTX 4070 SUPER). -Falls back to NumPy if CuPy/CUDA is unavailable. - -Usage: - python backtest/evaluate.py # full 7-day eval, M15+H1 - python backtest/evaluate.py --tf H1 # single timeframe - python backtest/evaluate.py --days 3 # shorter window - python backtest/evaluate.py --no-cache # re-download all data -""" - -import argparse -import csv -import os -import sys -import time -from concurrent.futures import ProcessPoolExecutor, as_completed -from dataclasses import dataclass, field -from datetime import datetime, timezone, timedelta -from pathlib import Path -from typing import Dict, List, Optional, Tuple - -import numpy as np -import pandas as pd - -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) - -from config import ( - PlatformConfig, Timeframe, SignalStrength, - EURUSD, USDJPY, GBPUSD, XAUUSD, PAIR_INFO, -) -from data_provider import ForexDataProvider -from signal_aggregator import SignalAggregator - -# ── GPU setup ──────────────────────────────────────────────────────── -try: - if "CUDA_PATH" not in os.environ: - _nvidia_dir = os.path.join( - sys.prefix, "lib", "python3.12", "site-packages", "nvidia" - ) - if os.path.isdir(_nvidia_dir): - os.environ["CUDA_PATH"] = _nvidia_dir - import cupy as cp - _test = cp.array([1.0, 2.0]) - _ = cp.where(_test > 0.5) - GPU = True - GPU_NAME = cp.cuda.runtime.getDeviceProperties(0)["name"].decode() - print(f"GPU: {GPU_NAME} (CuPy {cp.__version__})") - del _test -except Exception: - import numpy as cp # type: ignore[no-redef] - GPU = False - print("GPU: not available — falling back to NumPy") - -# ── Constants ──────────────────────────────────────────────────────── -BACKTEST_DIR = Path(__file__).resolve().parent -DATA_DIR = BACKTEST_DIR / "data" -RESULTS_DIR = BACKTEST_DIR / "results" - -SCAN_PAIRS = [EURUSD, GBPUSD, XAUUSD] - -EVAL_TIMEFRAMES = [Timeframe.M15, Timeframe.H1] - -TF_CONFIRMATIONS = { - Timeframe.M5: [Timeframe.M15, Timeframe.H1], - Timeframe.M15: [Timeframe.M5, Timeframe.H1], - Timeframe.H1: [Timeframe.M15, Timeframe.H4], - Timeframe.H4: [Timeframe.H1, Timeframe.D1], -} - -ALL_TFS_NEEDED = { - Timeframe.M5, Timeframe.M15, Timeframe.H1, Timeframe.H4, Timeframe.D1, -} - -OUTCOME_TF = Timeframe.M5 - -MIN_LOOKBACK_BARS = 250 - - -@dataclass -class Trade: - pair: str - display: str - timeframe: str - entry_time: pd.Timestamp - direction: str - entry_price: float - stop_loss: float - take_profit_1: float - take_profit_2: Optional[float] - risk_reward: float - risk_pips: float - reward_pips: float - signal_quality: str - confidence: float - score: float - regime: str - outcome: str = "OPEN" - exit_price: float = 0.0 - exit_time: Optional[pd.Timestamp] = None - pnl_pips: float = 0.0 - bars_held: int = 0 - - -# ══════════════════════════════════════════════════════════════════════ -# DATA FETCHING & CACHING -# ══════════════════════════════════════════════════════════════════════ - -def _cache_path(symbol: str, tf: Timeframe) -> Path: - safe = symbol.replace("/", "_").replace("=", "") - return DATA_DIR / f"{safe}_{tf.value}.parquet" - - -def fetch_all_data( - pairs: List[str], - timeframes, - lookback_days: int = 67, - use_cache: bool = True, -) -> Dict[str, Dict[Timeframe, pd.DataFrame]]: - """Fetch OHLCV for all pair×TF combos, cache to parquet.""" - config = PlatformConfig() - config.data_lookback_days = lookback_days - dp = ForexDataProvider(config) - data: Dict[str, Dict[Timeframe, pd.DataFrame]] = {} - - total = len(pairs) * len(timeframes) - done = 0 - - for symbol in pairs: - data[symbol] = {} - display = PAIR_INFO.get(symbol, {}).get("display", symbol) - for tf in timeframes: - done += 1 - cache = _cache_path(symbol, tf) - if use_cache and cache.exists(): - mtime = datetime.fromtimestamp(cache.stat().st_mtime, tz=timezone.utc) - age_hours = (datetime.now(timezone.utc) - mtime).total_seconds() / 3600 - if age_hours < 4: - df = pd.read_parquet(cache) - data[symbol][tf] = df - print(f" [{done}/{total}] {display} {tf.value} — cached ({len(df)} bars)") - continue - - print(f" [{done}/{total}] {display} {tf.value} — fetching...", end="", flush=True) - try: - df = dp.fetch_ohlcv(symbol, tf, lookback_days=lookback_days) - data[symbol][tf] = df - try: - df.to_parquet(cache) - except Exception: - pass - print(f" {len(df)} bars") - except Exception as e: - print(f" FAILED: {e}") - - return data - - -# ══════════════════════════════════════════════════════════════════════ -# WALK-FORWARD EVALUATION -# ══════════════════════════════════════════════════════════════════════ - -def walk_forward( - pair: str, - eval_tf: Timeframe, - all_data: Dict[Timeframe, pd.DataFrame], - eval_days: int, -) -> List[Trade]: - """Walk through eval_days of candles, run the engine at each bar.""" - primary_df = all_data.get(eval_tf) - if primary_df is None or primary_df.empty: - return [] - - conf_tfs = TF_CONFIRMATIONS.get(eval_tf, [Timeframe.M15, Timeframe.H4]) - display = PAIR_INFO.get(pair, {}).get("display", pair) - - cutoff = primary_df.index[-1] - pd.Timedelta(days=eval_days) - eval_indices = primary_df.index[primary_df.index >= cutoff] - - if len(eval_indices) == 0: - return [] - - first_eval_pos = primary_df.index.get_loc(eval_indices[0]) - if first_eval_pos < MIN_LOOKBACK_BARS: - first_eval_pos = MIN_LOOKBACK_BARS - eval_indices = primary_df.index[first_eval_pos:] - - config = PlatformConfig() - config.primary_timeframe = eval_tf - config.confirmation_timeframes = conf_tfs - aggregator = SignalAggregator(config) - - cooldown_bars = { - Timeframe.M5: 24, Timeframe.M15: 10, - Timeframe.H1: 5, Timeframe.H4: 3, - }.get(eval_tf, 6) - - trades: List[Trade] = [] - total = len(eval_indices) - bars_until_next = 0 - - for i, eval_time in enumerate(eval_indices): - if (i + 1) % max(1, total // 10) == 0: - print(f" {display} {eval_tf.value}: {i+1}/{total} candles...", flush=True) - - if bars_until_next > 0: - bars_until_next -= 1 - continue - - dfs: Dict[Timeframe, pd.DataFrame] = {} - dfs[eval_tf] = primary_df.loc[:eval_time] - - for ctf in conf_tfs: - ctf_df = all_data.get(ctf) - if ctf_df is not None and not ctf_df.empty: - dfs[ctf] = ctf_df.loc[:eval_time] - - try: - report = aggregator.analyze_from_dataframes( - symbol=pair, dfs=dfs, use_news=False, - ) - except Exception: - continue - - if report.trade_setup and report.signal_quality in ("A+", "A", "B", "C"): - ts = report.trade_setup - trades.append(Trade( - pair=pair, - display=display, - timeframe=eval_tf.value, - entry_time=eval_time, - direction=ts.direction, - entry_price=ts.entry_price, - stop_loss=ts.stop_loss, - take_profit_1=ts.take_profit_1, - take_profit_2=ts.take_profit_2, - risk_reward=ts.risk_reward, - risk_pips=ts.risk_pips, - reward_pips=ts.reward_pips, - signal_quality=report.signal_quality, - confidence=report.final_confidence, - score=report.final_score, - regime=report.market_regime, - )) - bars_until_next = cooldown_bars - - return trades - - -def _walk_forward_worker(args): - """Top-level worker for multiprocessing (must be picklable).""" - pair, tf, pair_data, eval_days = args - display = PAIR_INFO.get(pair, {}).get("display", pair) - trades = walk_forward(pair, tf, pair_data, eval_days) - return pair, tf, display, trades - - -# ══════════════════════════════════════════════════════════════════════ -# GPU-ACCELERATED OUTCOME EVALUATION -# ══════════════════════════════════════════════════════════════════════ - -def evaluate_outcomes( - trades: List[Trade], - m5_data: Dict[str, pd.DataFrame], - max_bars: int = 500, - max_hours: int = 0, -) -> List[Trade]: - """ - For each trade, scan forward in M5 data to check if TP1 or SL hit first. - Uses GPU (CuPy) for vectorized parallel evaluation across all trades per pair. - """ - if not trades: - return trades - - pairs_in_trades = set(t.pair for t in trades) - label = "GPU" if GPU else "CPU" - print(f"\nEvaluating {len(trades)} trade outcomes on {label}...") - - for pair in pairs_in_trades: - df = m5_data.get(pair) - if df is None or df.empty: - continue - - pair_trades = [t for t in trades if t.pair == pair] - if not pair_trades: - continue - - highs_np = df["High"].values.astype(np.float64) - lows_np = df["Low"].values.astype(np.float64) - closes_np = df["Close"].values.astype(np.float64) - timestamps = df.index - - highs_gpu = cp.asarray(highs_np) - lows_gpu = cp.asarray(lows_np) - - for trade in pair_trades: - start_mask = timestamps > trade.entry_time - start_indices = np.where(start_mask)[0] - if len(start_indices) == 0: - continue - - start = int(start_indices[0]) - time_limit = max_hours * 12 if max_hours > 0 else max_bars - effective_max = min(time_limit, max_bars) - end = min(start + effective_max, len(highs_gpu)) - - if start >= end: - continue - - h_slice = highs_gpu[start:end] - l_slice = lows_gpu[start:end] - - if trade.direction == "BUY": - tp_mask = h_slice >= trade.take_profit_1 - sl_mask = l_slice <= trade.stop_loss - else: - tp_mask = l_slice <= trade.take_profit_1 - sl_mask = h_slice >= trade.stop_loss - - tp_hits = cp.where(tp_mask)[0] - sl_hits = cp.where(sl_mask)[0] - - tp_idx = int(tp_hits[0]) if len(tp_hits) > 0 else max_bars + 1 - sl_idx = int(sl_hits[0]) if len(sl_hits) > 0 else max_bars + 1 - - pip_val = PAIR_INFO.get(pair, {}).get("pip_value", 0.0001) - - if tp_idx <= sl_idx and tp_idx < max_bars + 1: - trade.outcome = "WIN" - trade.bars_held = tp_idx + 1 - real_idx = start + tp_idx - trade.exit_price = trade.take_profit_1 - trade.exit_time = timestamps[real_idx] - trade.pnl_pips = trade.reward_pips - elif sl_idx < tp_idx and sl_idx < max_bars + 1: - trade.outcome = "LOSS" - trade.bars_held = sl_idx + 1 - real_idx = start + sl_idx - trade.exit_price = trade.stop_loss - trade.exit_time = timestamps[real_idx] - trade.pnl_pips = -trade.risk_pips - else: - trade.outcome = "OPEN" - last_idx = end - 1 - trade.exit_time = timestamps[last_idx] - exit_p = float(closes_np[last_idx]) - trade.exit_price = exit_p - if trade.direction == "BUY": - trade.pnl_pips = (exit_p - trade.entry_price) / pip_val - else: - trade.pnl_pips = (trade.entry_price - exit_p) / pip_val - trade.bars_held = end - start - - return trades - - -# ══════════════════════════════════════════════════════════════════════ -# REPORTING -# ══════════════════════════════════════════════════════════════════════ - -def print_report(trades: List[Trade], eval_days: int): - print("\n") - print("=" * 90) - print(f" FOREXBOT2 — ENGINE EVALUATION REPORT ({eval_days}-DAY)") - print(f" {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}") - print("=" * 90) - - if not trades: - print("\n No trades generated.\n") - return - - closed = [t for t in trades if t.outcome in ("WIN", "LOSS")] - wins = [t for t in closed if t.outcome == "WIN"] - losses = [t for t in closed if t.outcome == "LOSS"] - still_open = [t for t in trades if t.outcome == "OPEN"] - - # ── Overall ────────────────────────────────────────────────── - print(f"\n OVERALL") - print(f" {'─' * 50}") - print(f" Total signals with trade setup: {len(trades)}") - print(f" Closed: {len(closed)} | Still open: {len(still_open)}") - - if closed: - win_rate = len(wins) / len(closed) * 100 - total_pnl = sum(t.pnl_pips for t in closed) - avg_win = np.mean([t.pnl_pips for t in wins]) if wins else 0 - avg_loss = np.mean([abs(t.pnl_pips) for t in losses]) if losses else 0 - gross_profit = sum(t.pnl_pips for t in wins) if wins else 0 - gross_loss = sum(abs(t.pnl_pips) for t in losses) if losses else 0 - profit_factor = gross_profit / gross_loss if gross_loss > 0 else float("inf") - avg_rr = np.mean([t.risk_reward for t in closed]) - avg_bars = np.mean([t.bars_held for t in closed]) - - print(f" Win rate: {win_rate:.1f}% ({len(wins)}W / {len(losses)}L)") - print(f" Profit factor: {profit_factor:.2f}") - print(f" Total P&L: {total_pnl:+.1f} pips") - print(f" Avg win: +{avg_win:.1f} pips | Avg loss: -{avg_loss:.1f} pips") - print(f" Avg R:R setup: 1:{avg_rr:.1f}") - print(f" Avg hold: {avg_bars:.0f} M5 bars ({avg_bars * 5:.0f} min)") - - # Consecutive streaks - max_wins = max_losses = cur_wins = cur_losses = 0 - for t in closed: - if t.outcome == "WIN": - cur_wins += 1 - cur_losses = 0 - max_wins = max(max_wins, cur_wins) - else: - cur_losses += 1 - cur_wins = 0 - max_losses = max(max_losses, cur_losses) - print(f" Max streak: {max_wins}W / {max_losses}L") - - # ── By pair ────────────────────────────────────────────────── - print(f"\n BY PAIR") - print(f" {'─' * 50}") - print(f" {'Pair':<12} {'Trades':>7} {'Closed':>7} {'Win%':>6} {'P&L':>10} {'PF':>7}") - - for pair in SCAN_PAIRS: - display = PAIR_INFO.get(pair, {}).get("display", pair) - pt = [t for t in trades if t.pair == pair] - pc = [t for t in pt if t.outcome in ("WIN", "LOSS")] - pw = [t for t in pc if t.outcome == "WIN"] - if not pt: - continue - wr = (len(pw) / len(pc) * 100) if pc else 0 - pnl = sum(t.pnl_pips for t in pc) - gp = sum(t.pnl_pips for t in pw) if pw else 0 - gl = sum(abs(t.pnl_pips) for t in pc if t.outcome == "LOSS") or 1 - pf = gp / gl if gl > 0 else 0 - print(f" {display:<12} {len(pt):>7} {len(pc):>7} {wr:>5.1f}% {pnl:>+9.1f}p {pf:>6.2f}") - - # ── By timeframe ───────────────────────────────────────────── - print(f"\n BY TIMEFRAME") - print(f" {'─' * 50}") - print(f" {'TF':<8} {'Trades':>7} {'Closed':>7} {'Win%':>6} {'P&L':>10} {'PF':>7}") - - tfs = sorted(set(t.timeframe for t in trades)) - for tf in tfs: - tt = [t for t in trades if t.timeframe == tf] - tc = [t for t in tt if t.outcome in ("WIN", "LOSS")] - tw = [t for t in tc if t.outcome == "WIN"] - if not tt: - continue - wr = (len(tw) / len(tc) * 100) if tc else 0 - pnl = sum(t.pnl_pips for t in tc) - gp = sum(t.pnl_pips for t in tw) if tw else 0 - gl = sum(abs(t.pnl_pips) for t in tc if t.outcome == "LOSS") or 1 - pf = gp / gl if gl > 0 else 0 - print(f" {tf:<8} {len(tt):>7} {len(tc):>7} {wr:>5.1f}% {pnl:>+9.1f}p {pf:>6.2f}") - - # ── By quality grade ───────────────────────────────────────── - print(f"\n BY QUALITY GRADE") - print(f" {'─' * 50}") - print(f" {'Grade':<8} {'Trades':>7} {'Closed':>7} {'Win%':>6} {'P&L':>10} {'Avg RR':>7}") - - for grade in ("A+", "A", "B", "C"): - gt = [t for t in trades if t.signal_quality == grade] - gc = [t for t in gt if t.outcome in ("WIN", "LOSS")] - gw = [t for t in gc if t.outcome == "WIN"] - if not gt: - continue - wr = (len(gw) / len(gc) * 100) if gc else 0 - pnl = sum(t.pnl_pips for t in gc) - arr = np.mean([t.risk_reward for t in gt]) - print(f" {grade:<8} {len(gt):>7} {len(gc):>7} {wr:>5.1f}% {pnl:>+9.1f}p {arr:>6.1f}") - - # ── By regime ──────────────────────────────────────────────── - print(f"\n BY MARKET REGIME") - print(f" {'─' * 50}") - print(f" {'Regime':<16} {'Trades':>7} {'Win%':>6} {'P&L':>10}") - - for regime in sorted(set(t.regime for t in trades)): - rt = [t for t in trades if t.regime == regime] - rc = [t for t in rt if t.outcome in ("WIN", "LOSS")] - rw = [t for t in rc if t.outcome == "WIN"] - wr = (len(rw) / len(rc) * 100) if rc else 0 - pnl = sum(t.pnl_pips for t in rc) - label = regime.replace("_", " ").title() - print(f" {label:<16} {len(rt):>7} {wr:>5.1f}% {pnl:>+9.1f}p") - - # ── Direction ──────────────────────────────────────────────── - print(f"\n BY DIRECTION") - print(f" {'─' * 50}") - for d in ("BUY", "SELL"): - dt = [t for t in trades if t.direction == d] - dc = [t for t in dt if t.outcome in ("WIN", "LOSS")] - dw = [t for t in dc if t.outcome == "WIN"] - wr = (len(dw) / len(dc) * 100) if dc else 0 - pnl = sum(t.pnl_pips for t in dc) - print(f" {d:<8} {len(dt):>7} trades {wr:>5.1f}% win {pnl:>+9.1f}p") - - print(f"\n{'=' * 90}\n") - - -def save_trades_csv(trades: List[Trade], filename: str): - path = RESULTS_DIR / filename - with open(path, "w", newline="") as f: - writer = csv.writer(f) - writer.writerow([ - "pair", "display", "timeframe", "entry_time", "direction", - "entry_price", "stop_loss", "take_profit_1", "risk_reward", - "risk_pips", "reward_pips", "signal_quality", "confidence", - "score", "regime", "outcome", "exit_price", "exit_time", - "pnl_pips", "bars_held", - ]) - for t in trades: - writer.writerow([ - t.pair, t.display, t.timeframe, t.entry_time, t.direction, - f"{t.entry_price:.5f}", f"{t.stop_loss:.5f}", f"{t.take_profit_1:.5f}", - f"{t.risk_reward:.2f}", f"{t.risk_pips:.1f}", f"{t.reward_pips:.1f}", - t.signal_quality, f"{t.confidence:.1f}", f"{t.score:.1f}", - t.regime, t.outcome, f"{t.exit_price:.5f}", - t.exit_time or "", f"{t.pnl_pips:.1f}", t.bars_held, - ]) - print(f"Trades saved to {path}") - - -# ══════════════════════════════════════════════════════════════════════ -# MAIN -# ══════════════════════════════════════════════════════════════════════ - -def main(): - parser = argparse.ArgumentParser(description="ForexBot2 signal engine evaluator") - parser.add_argument("--days", type=int, default=7, help="Evaluation window in days (default: 7)") - parser.add_argument("--tf", default=None, help="Single timeframe to evaluate (M15, H1)") - parser.add_argument("--no-cache", action="store_true", help="Re-download all data") - parser.add_argument("--max-outcome-bars", type=int, default=1000, - help="Max M5 bars to scan for SL/TP hit (default: 1000 = ~83h)") - parser.add_argument("--pair", default=None, - help="Single pair to evaluate (EURUSD, USDJPY, GBPUSD, XAUUSD)") - parser.add_argument("--max-hours", type=int, default=0, - help="Max hours to hold a trade before time-exit (0 = disabled)") - args = parser.parse_args() - - eval_tfs = EVAL_TIMEFRAMES - if args.tf: - tf_map = {"M5": Timeframe.M5, "M15": Timeframe.M15, "H1": Timeframe.H1, "H4": Timeframe.H4} - if args.tf.upper() not in tf_map: - sys.exit(f"Invalid timeframe: {args.tf}. Choose from: {list(tf_map.keys())}") - eval_tfs = [tf_map[args.tf.upper()]] - - pair_map = {"EURUSD": EURUSD, "USDJPY": USDJPY, "GBPUSD": GBPUSD, "XAUUSD": XAUUSD} - scan_pairs = SCAN_PAIRS - if args.pair: - key = args.pair.upper().replace("/", "") - if key not in pair_map: - sys.exit(f"Invalid pair: {args.pair}. Choose from: {list(pair_map.keys())}") - scan_pairs = [pair_map[key]] - - DATA_DIR.mkdir(parents=True, exist_ok=True) - RESULTS_DIR.mkdir(parents=True, exist_ok=True) - - # ── 1. Fetch data ──────────────────────────────────────────── - lookback = args.days + 60 - print(f"\n{'=' * 60}") - print(f" Fetching {lookback} days of data for {len(scan_pairs)} pair(s)...") - print(f"{'=' * 60}") - t0 = time.time() - all_data = fetch_all_data( - scan_pairs, ALL_TFS_NEEDED, - lookback_days=lookback, - use_cache=not args.no_cache, - ) - print(f" Data fetched in {time.time() - t0:.1f}s") - - # ── 2. Walk-forward evaluation (multi-core) ────────────────── - print(f"\n{'=' * 60}") - n_workers = min(os.cpu_count() or 4, 16) - print(f" Walk-forward evaluation: {args.days} days, TFs: {[tf.value for tf in eval_tfs]}") - print(f" Using {n_workers} CPU cores") - print(f"{'=' * 60}") - t1 = time.time() - - jobs = [] - for pair in scan_pairs: - pair_data = all_data.get(pair, {}) - if not pair_data: - display = PAIR_INFO.get(pair, {}).get("display", pair) - print(f" {display}: no data, skipping") - continue - for tf in eval_tfs: - jobs.append((pair, tf, pair_data, args.days)) - - all_trades: List[Trade] = [] - with ProcessPoolExecutor(max_workers=n_workers) as pool: - futures = {pool.submit(_walk_forward_worker, job): job for job in jobs} - for future in as_completed(futures): - pair, tf, display, trades = future.result() - print(f" {display} {tf.value}: {len(trades)} trade setups") - all_trades.extend(trades) - - print(f"\n Walk-forward complete in {time.time() - t1:.1f}s") - print(f" Total trade setups: {len(all_trades)}") - - # ── 3. Outcome evaluation (GPU) ────────────────────────────── - m5_data = {pair: all_data.get(pair, {}).get(Timeframe.M5) for pair in scan_pairs} - m5_data = {k: v for k, v in m5_data.items() if v is not None} - - t2 = time.time() - all_trades = evaluate_outcomes(all_trades, m5_data, max_bars=args.max_outcome_bars, max_hours=args.max_hours) - print(f" Outcomes evaluated in {time.time() - t2:.3f}s") - - # ── 4. Report ──────────────────────────────────────────────── - print_report(all_trades, args.days) - - ts = datetime.now().strftime("%Y%m%d_%H%M%S") - save_trades_csv(all_trades, f"eval_{args.days}d_{ts}.csv") - - -if __name__ == "__main__": - main() diff --git a/config.py b/config.py index 14ae41c..4dcd647 100755 --- a/config.py +++ b/config.py @@ -111,7 +111,7 @@ class SignalConfig: default_risk_pct: float = 1.0 default_rr_ratio: float = 2.0 atr_sl_multiplier: float = 1.5 - min_rr_ratio: float = 1.2 + min_rr_ratio: float = 0.5 @dataclass @@ -130,16 +130,31 @@ class NewsConfig: @dataclass class XauStrategyConfig: - """XAU/USD oversold-bounce strategy (BUY only). - 1-year optimized: 89% WR, PF 4.58, +5041p (118 trades). - Entry: M15 RSI<38 + price above EMA200 on M15, H1, H4. - Exit: TP=0.9×ATR, SL=2.0×ATR. + """XAU/USD v3 weighted confluence strategy. + 1-year grid-optimized (768 combos): 52.4% WR, PF 2.34, +40.3R (63 trades). + Entry: M15 score>=80 + H4 RSI confirms + H1 MACD confirms + RSI 40-65 zone. + Exit: Partial — 1R→BE, 2R→close 50%, 3R→close rest. SL=2.5×ATR. + Blocked: Fri/Sat/Sun, hours 1/3/9/10/19/22/23. """ enabled: bool = True - rsi_entry_threshold: float = 38.0 # Wilder RSI scale; equivalent to standard RSI ~30 + rsi_entry_threshold: float = 45.0 tp_atr_multiplier: float = 0.9 sl_atr_multiplier: float = 2.0 - blocked_hours_utc: List[int] = field(default_factory=lambda: [8, 9, 11, 14, 15, 20]) + min_ema200_tfs: int = 2 + allow_generic_trades: bool = True + blocked_hours_utc: List[int] = field(default_factory=lambda: [0, 1, 2, 5, 6, 21, 22, 23]) + # v2 confluence fixes + weekend_filter: bool = True + min_atr_threshold: float = 1.0 + v2_sl_atr_multiplier: float = 2.5 + buy_threshold_boost: int = 0 + m5_confirmation_bars: int = 0 + # v3 gates (grid-optimized on 1Y Mon-Thu data) + h4_rsi_gate: bool = True # require H4 RSI >55 for BUY, <45 for SELL + h1_macd_gate: bool = True # require H1 MACD histogram >0 for BUY, <0 for SELL + rsi_zone: tuple = (40.0, 65.0) # only enter when M15 RSI in this zone + allowed_weekdays: List[int] = field(default_factory=lambda: [0, 1, 2, 3, 4]) # Mon-Fri + sell_min_quality: List[str] = field(default_factory=lambda: ["A+", "A", "B"]) # SELL needs B+ @dataclass @@ -158,6 +173,16 @@ class EurStrategyConfig: tp_atr_multiplier: float = 0.7 sl_atr_multiplier: float = 1.0 blocked_hours_utc: List[int] = field(default_factory=lambda: [0, 1, 2, 3, 4, 5, 6, 16, 17, 18, 19, 21, 22, 23]) + # If False, suppress generic trade setups when the EUR pullback gate + # didn't fire. Backtest (90-day M15): True → +4.70R, False → +9.50R. + # Generic momentum buys at trending_up extremes are pure drag. + allow_generic_trades: bool = False + # Minimum number of M15 scored indicators (ema_trend, rsi, bollinger, + # adx, candlestick) that must independently signal BUY for the + # pullback gate to fire. The strategy hardcodes Quality=A and + # Score=+40 — without this, the "A" grade fires on weak technical + # confluence. 0 = disabled (legacy behavior). + min_indicators_agree: int = 2 @dataclass @@ -177,6 +202,10 @@ class GbpStrategyConfig: tp_atr_multiplier: float = 0.7 sl_atr_multiplier: float = 1.0 blocked_hours_utc: List[int] = field(default_factory=lambda: [0, 1, 2, 3, 4, 5, 6, 14, 15, 16, 17, 19, 21, 22, 23]) + # See EurStrategyConfig.allow_generic_trades — same rationale. + allow_generic_trades: bool = False + # See EurStrategyConfig.min_indicators_agree — same rationale. + min_indicators_agree: int = 2 @dataclass @@ -191,6 +220,11 @@ class FilterConfig: session_end_hour: int = 16 blocked_hours_utc: List[int] = field(default_factory=lambda: [0, 1, 2, 5, 6, 8, 12, 13, 14, 15, 16, 17, 21, 22, 23]) + # Universal weekday filter (0=Mon ... 6=Sun). Default Mon-Thu — Fridays + # have wide spreads from US session ending and weekend gap risk; XAU/USD + # has its own stricter allowed_weekdays in xau_strategy that overrides. + allowed_weekdays: List[int] = field(default_factory=lambda: [0, 1, 2, 3]) + sell_filter: bool = True sell_allowed_pairs: List[str] = field(default_factory=lambda: ["XAU/USD"]) diff --git a/mt5.md b/mt5.md deleted file mode 100644 index c5e2c84..0000000 --- a/mt5.md +++ /dev/null @@ -1,185 +0,0 @@ -# MT5 Auto-Execution Integration Plan - -## Context -ForexBot2 generates trade signals with full setups (entry, SL, TP) but currently only displays them in Telegram. The user wants fully automatic order execution via MetaTrader 5. - -**Constraint:** MT5 runs on a separate Windows machine. The Python `MetaTrader5` package requires Windows (COM interface). The bot runs on Linux. A network bridge is needed. - -**Broker:** TIO Markets (demo first, live later via config change). -**Execution mode:** Fully automatic for actionable signals (A+/A/B grades). - -## Architecture: REST API Bridge - -``` -Linux (telegram_bot.py) Windows (mt5_bridge.py) -======================== ======================== -Signal fires with TradeSetup FastAPI server running - -> mt5_client.execute_trade() -> Wraps MetaTrader5 package - POST /api/order ─────── HTTP ───────> -> mt5.order_send() - <- {ticket: 12345} <──── HTTP ────── <- Returns ticket or error - -> Telegram: "ORDER PLACED" -> Idempotency via order_id -``` - -**Why REST over WebSocket/file-sync:** Stateless, debuggable, retry-safe. For a bot placing 1-5 trades/day, HTTP latency (5-50ms LAN) is negligible. - -## Files to Create/Modify - -### New Files - -**1. `mt5_bridge/mt5_bridge.py`** — Windows-side FastAPI server (~200 lines) -- Wraps `MetaTrader5` Python package -- Endpoints: `/api/health`, `/api/account`, `/api/positions`, `/api/order` (POST), `/api/order/close` (POST) -- Lot size calculation on bridge side (has access to `mt5.symbol_info()` for tick value, contract size, volume step) -- Idempotency: stores `{order_id: ticket}` to prevent duplicate orders on retry -- API key auth via `X-API-Key` header -- MT5 credentials from env vars: `MT5_LOGIN`, `MT5_PASSWORD`, `MT5_SERVER`, `MT5_PATH` - -**2. `mt5_bridge/requirements.txt`** — Windows dependencies -- `fastapi`, `uvicorn`, `MetaTrader5`, `pydantic` - -**3. `mt5_client.py`** — Linux-side async HTTP client (~150 lines) -- `MT5ExecutionClient` class with `aiohttp` -- `execute_trade(setup: TradeSetup, quality: str)` — sends order to bridge -- Symbol mapping: `EURUSD=X` -> `EURUSD` (TIO Markets format, configurable) -- Duplicate prevention: tracks open positions per symbol -- Retry: 3 attempts, exponential backoff, same `order_id` for idempotency -- Position sync: periodically calls `/api/positions` to stay in sync with MT5 -- Graceful degradation: if bridge offline, logs warning, bot continues signal-only - -### Modified Files - -**4. `config.py`** — Add `MT5Config` dataclass -```python -@dataclass -class MT5Config: - enabled: bool = False - bridge_url: str = "http://192.168.1.100:8080" - bridge_api_key: str = "" - risk_pct: float = 1.0 # % of balance risked per trade - max_open_positions: int = 3 - max_daily_trades: int = 10 # Circuit breaker - magic_number: int = 234000 # Identifies ForexBot2 orders - slippage_points: int = 20 # 2 pips for 5-digit brokers - auto_execute_grades: List[str] # ["A+", "A", "B"] - symbol_map: Dict[str, str] # Internal -> broker symbol -``` -All overridable via env vars (`MT5_BRIDGE_URL`, `MT5_RISK_PCT`, etc.) - -**5. `telegram_bot.py`** — Wire execution into signal flow -- In `_scan_timeframe()`: after sending Telegram messages, call `mt5_client.execute_trade()` for actionable signals -- In `_handle_breach()`: same pattern after breach scan produces actionable signal -- Add background task `_mt5_position_sync_loop()` for periodic position sync -- Add Telegram commands: `/mt5status` (bridge health + account info), `/positions` (open trades) -- Register new commands in `set_my_commands()` -- Execution result messages sent to Telegram (ticket # on success, error on failure) - -**6. `requirements.txt`** — Add `aiohttp>=3.9` (async HTTP for Linux-side client) - -## Safety Layers - -1. **Demo account first** — `MT5_SERVER=TIOMarkets-Demo` -2. **`enabled: False` default** — must explicitly opt-in via `MT5_ENABLED=true` -3. **Duplicate prevention** — 3 layers: client-side tracking, bridge-side position check, idempotent `order_id` -4. **Circuit breaker** — `max_daily_trades` stops execution if too many orders fire -5. **Position limit** — `max_open_positions` caps concurrent exposure -6. **Risk-based sizing** — lot size from `risk_pct` of account balance, calculated on bridge with live tick values -7. **Graceful degradation** — bridge offline = signal-only mode, no crash - -## Position Sizing (calculated on Windows bridge) - -``` -risk_amount = balance * (risk_pct / 100) -tick_value = mt5.symbol_info(symbol).trade_tick_value -tick_size = mt5.symbol_info(symbol).trade_tick_size -sl_ticks = abs(entry - stop_loss) / tick_size -lot_size = risk_amount / (sl_ticks * tick_value) -lot_size = round_down(lot_size, volume_step) -lot_size = clamp(lot_size, volume_min, volume_max) -``` - -Bridge side is best because it has real-time `tick_value` (varies with exchange rates for cross pairs). - -## Symbol Mapping (TIO Markets) - -| Internal | MT5 Symbol | -|----------|-----------| -| `EURUSD=X` | `EURUSD` | -| `USDJPY=X` | `USDJPY` | -| `GBPUSD=X` | `GBPUSD` | -| `XAU/USD` | `XAUUSD` | - -Configurable via `MT5_SYMBOL_MAP` env var (JSON) for broker variations. - -## Integration Points - -### In `_scan_timeframe()` (after line ~406): -```python -mt5 = _get_mt5_client() -if mt5: - for report in actionable: - if report.trade_setup and report.signal_quality in mt5.config.auto_execute_grades: - result = await mt5.execute_trade(report.trade_setup, report.signal_quality) - # Send execution confirmation/error to Telegram -``` - -### In `_handle_breach()` (after line ~477): -```python -if report.trade_setup and report.signal_quality in mt5.config.auto_execute_grades: - mt5 = _get_mt5_client() - if mt5: - result = await mt5.execute_trade(report.trade_setup, report.signal_quality) - # Send execution confirmation/error to Telegram -``` - -## New Telegram Commands - -| Command | Description | -|---------|-------------| -| `/mt5status` | Bridge health, account balance/equity, open positions count | -| `/positions` | List all open MT5 positions with P/L | - -## Implementation Order - -1. `config.py` — Add `MT5Config` -2. `mt5_bridge/mt5_bridge.py` + `requirements.txt` — Windows bridge server -3. `mt5_client.py` — Linux HTTP client -4. `telegram_bot.py` — Wire execution + new commands -5. `requirements.txt` — Add `aiohttp` - -## Verification - -1. Start bridge on Windows: `python mt5_bridge.py` -> verify `curl /api/health` returns MT5 connected -2. Test order via curl: `POST /api/order` with demo credentials -> verify order appears in MT5 -3. Start bot on Linux with `MT5_ENABLED=true` -> verify `/mt5status` command works -4. Wait for actionable signal (or manually trigger scan) -> verify order placed + Telegram confirmation -5. Verify duplicate prevention: same signal shouldn't place second order -6. Kill bridge -> verify bot continues in signal-only mode without crashing - -## Windows Bridge Setup (one-time) - -```bash -# On Windows machine: -pip install fastapi uvicorn MetaTrader5 pydantic - -# Set environment variables: -set MT5_LOGIN=12345678 -set MT5_PASSWORD=your-password -set MT5_SERVER=TIOMarkets-Demo -set MT5_PATH=C:\Program Files\MetaTrader 5\terminal64.exe -set BRIDGE_API_KEY=your-random-32-char-key -set BRIDGE_PORT=8080 - -# Run: -python mt5_bridge.py -``` - -## Linux Bot Config (env vars) - -```bash -MT5_ENABLED=true -MT5_BRIDGE_URL=http://:8080 -MT5_BRIDGE_API_KEY=your-random-32-char-key -MT5_RISK_PCT=1.0 -MT5_MAX_OPEN_POSITIONS=3 -MT5_AUTO_EXECUTE_GRADES=A+,A,B -``` diff --git a/requirements.txt b/requirements.txt index 19ed6a7..305806c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,6 @@ requests>=2.28 twelvedata>=1.2.0 python-telegram-bot>=20.0 websockets>=12.0 +fastapi>=0.100 +uvicorn>=0.20 +redis>=5.0 diff --git a/signal_aggregator.py b/signal_aggregator.py index fcdd0e0..7e37ad2 100755 --- a/signal_aggregator.py +++ b/signal_aggregator.py @@ -101,6 +101,42 @@ class SignalAggregator: if symbol is None: symbol = self.config.symbol + pair_info = self.data_provider.get_pair_info(symbol) + now_utc = datetime.now(timezone.utc) + + if pair_info["display"] == "XAU/USD" and self.config.xau_strategy.enabled: + xau_cfg = self.config.xau_strategy + + if xau_cfg.weekend_filter and now_utc.weekday() >= 5: + report = SignalReport( + symbol=symbol, + display_name=pair_info["display"], + timestamp=str(now_utc), + ) + report.final_signal = SignalStrength.NEUTRAL + report.signal_quality = "F" + report.trade_setup = None + report.warnings.append("WEEKEND FILTER: XAU/USD market closed — no liquidity") + report.action_recommendation = "NO TRADE — Weekend (market closed)" + return report + + if xau_cfg.allowed_weekdays and now_utc.weekday() not in xau_cfg.allowed_weekdays: + day_names = {0: "Mon", 1: "Tue", 2: "Wed", 3: "Thu", 4: "Fri"} + allowed = "/".join(day_names.get(d, str(d)) for d in xau_cfg.allowed_weekdays) + report = SignalReport( + symbol=symbol, + display_name=pair_info["display"], + timestamp=str(now_utc), + ) + report.final_signal = SignalStrength.NEUTRAL + report.signal_quality = "F" + report.trade_setup = None + report.warnings.append( + f"DAY FILTER: {day_names.get(now_utc.weekday(), '?')} not in allowed days ({allowed})" + ) + report.action_recommendation = f"NO TRADE — Only trading {allowed}" + return report + timeframes = [self.config.primary_timeframe] + list( self.config.confirmation_timeframes ) @@ -339,30 +375,45 @@ class SignalAggregator: # ── 6b. Session filter (works for both live and backtest) ─ if self.config.backtest.session_filter: try: - bar_hour = pd.Timestamp(report.timestamp).hour - if bar_hour in self.config.backtest.blocked_hours_utc: + bar_ts = pd.Timestamp(report.timestamp) + bar_hour = bar_ts.hour + bar_wd = bar_ts.weekday() + _is_xau_pair = report.display_name == "XAU/USD" + _blocked_hours = (self.config.xau_strategy.blocked_hours_utc + if _is_xau_pair else self.config.backtest.blocked_hours_utc) + if bar_hour in _blocked_hours: report.final_signal = SignalStrength.NEUTRAL report.signal_quality = "F" report.warnings.append(f"SESSION FILTER: {bar_hour:02}:00 UTC blocked") + _allowed_wd = (self.config.xau_strategy.allowed_weekdays + if _is_xau_pair else self.config.backtest.allowed_weekdays) + if _allowed_wd and bar_wd not in _allowed_wd: + day_names = {0: "Mon", 1: "Tue", 2: "Wed", 3: "Thu", + 4: "Fri", 5: "Sat", 6: "Sun"} + report.final_signal = SignalStrength.NEUTRAL + report.signal_quality = "F" + report.warnings.append( + f"DAY FILTER: {day_names.get(bar_wd, '?')} not in allowed days" + ) except Exception: pass - # ── 6c. XAU/USD oversold bounce strategy ───────────────── - if report.display_name == "XAU/USD" and self.config.xau_strategy.enabled: - if self._check_xau_oversold_bounce(report, dfs): - report.final_signal = SignalStrength.STRONG_BUY - report.final_score = 50.0 - report.final_confidence = 85.0 - report.signal_quality = "A+" - report.action_recommendation = ( - "STRONG BUY — XAU oversold bounce " - "(RSI<30 + M15/H1/H4 all above EMA200)" - ) - report.bullish_reasons.insert( - 0, - "XAU STRATEGY: M15 RSI oversold + triple-TF bullish trend confirmation", - ) - report.trade_setup = self._generate_xau_trade_setup(report, dfs) + # ── 6c. XAU/USD oversold bounce strategy (DISABLED — replaced by v2 confluence) ── + # if report.display_name == "XAU/USD" and self.config.xau_strategy.enabled: + # if self._check_xau_oversold_bounce(report, dfs): + # report.final_signal = SignalStrength.STRONG_BUY + # report.final_score = 50.0 + # report.final_confidence = 85.0 + # report.signal_quality = "A+" + # report.action_recommendation = ( + # "STRONG BUY — XAU oversold bounce " + # "(RSI<30 + M15/H1/H4 all above EMA200)" + # ) + # report.bullish_reasons.insert( + # 0, + # "XAU STRATEGY: M15 RSI oversold + triple-TF bullish trend confirmation", + # ) + # report.trade_setup = self._generate_xau_trade_setup(report, dfs) # ── 6d. EUR/USD pullback-buy strategy ───────────────────── if report.display_name == "EUR/USD" and self.config.eur_strategy.enabled: @@ -398,8 +449,118 @@ class SignalAggregator: ) report.trade_setup = self._generate_gbp_trade_setup(report, dfs) + # ── 6d. XAU/USD BUY threshold boost (Fix 5) ────────────── + if (report.display_name == "XAU/USD" + and self.config.xau_strategy.buy_threshold_boost > 0 + and report.final_signal in (SignalStrength.BUY, SignalStrength.STRONG_BUY) + and report.signal_quality in ("B", "C")): + report.warnings.append( + f"XAU BUY FILTER: BUY requires higher confidence (quality {report.signal_quality} insufficient)" + ) + report.signal_quality = "D" + report.trade_setup = None + + # ── 6f. XAU/USD H4 RSI + H1 MACD directional gates ────── + if (report.display_name == "XAU/USD" + and self.config.xau_strategy.enabled + and report.signal_quality in ("A+", "A", "B", "C") + and report.final_signal != SignalStrength.NEUTRAL): + xau_cfg = self.config.xau_strategy + is_buy = report.final_signal in (SignalStrength.BUY, SignalStrength.STRONG_BUY) + + if xau_cfg.h4_rsi_gate and Timeframe.H4 in dfs: + h4_close = dfs[Timeframe.H4]["Close"] + delta = h4_close.diff() + gain = delta.clip(lower=0).ewm(alpha=1/14, min_periods=14, adjust=False).mean() + loss = (-delta.clip(upper=0)).ewm(alpha=1/14, min_periods=14, adjust=False).mean() + h4_rsi = 100 - (100 / (1 + gain / loss)) + if len(h4_rsi) > 0: + h4_rsi_val = float(h4_rsi.iloc[-1]) + if is_buy and h4_rsi_val <= 55: + report.warnings.append( + f"H4 RSI GATE: RSI {h4_rsi_val:.0f} ≤55 — no BUY confirmation" + ) + report.signal_quality = "D" + report.trade_setup = None + elif not is_buy and h4_rsi_val >= 45: + report.warnings.append( + f"H4 RSI GATE: RSI {h4_rsi_val:.0f} ≥45 — no SELL confirmation" + ) + report.signal_quality = "D" + report.trade_setup = None + + if (xau_cfg.h1_macd_gate + and Timeframe.H1 in dfs + and report.signal_quality in ("A+", "A", "B", "C")): + h1_df = dfs[Timeframe.H1] + h1_close = h1_df["Close"] + h1_ema_fast = h1_close.ewm(span=12, adjust=False).mean() + h1_ema_slow = h1_close.ewm(span=26, adjust=False).mean() + h1_macd_line = h1_ema_fast - h1_ema_slow + h1_signal_line = h1_macd_line.ewm(span=9, adjust=False).mean() + h1_macd_hist = h1_macd_line - h1_signal_line + if len(h1_macd_hist) > 0: + h1_hist_val = float(h1_macd_hist.iloc[-1]) + if is_buy and h1_hist_val <= 0: + report.warnings.append( + f"H1 MACD GATE: histogram {h1_hist_val:.2f} ≤0 — no BUY confirmation" + ) + report.signal_quality = "D" + report.trade_setup = None + elif not is_buy and h1_hist_val >= 0: + report.warnings.append( + f"H1 MACD GATE: histogram {h1_hist_val:.2f} ≥0 — no SELL confirmation" + ) + report.signal_quality = "D" + report.trade_setup = None + + # ── RSI zone gate ────────────────────────────────────── + if (xau_cfg.rsi_zone + and report.signal_quality in ("A+", "A", "B", "C")): + m15_analysis = report.primary_analysis + if m15_analysis and hasattr(m15_analysis, '_rsi_series'): + rsi_val = float(m15_analysis._rsi_series.iloc[-1]) + lo, hi = xau_cfg.rsi_zone + if not (lo <= rsi_val <= hi): + report.warnings.append( + f"RSI ZONE GATE: RSI {rsi_val:.0f} outside [{lo:.0f}-{hi:.0f}]" + ) + report.signal_quality = "D" + report.trade_setup = None + + # ── 6g. XAU/USD SELL quality gate ───────────────────────── + xau_is_sell_dir = ( + report.final_signal in (SignalStrength.SELL, SignalStrength.STRONG_SELL) + or (report.final_signal == SignalStrength.NEUTRAL and report.final_score < 0) + ) + if (report.display_name == "XAU/USD" + and self.config.xau_strategy.enabled + and xau_is_sell_dir + and report.signal_quality not in self.config.xau_strategy.sell_min_quality): + report.warnings.append( + f"XAU SELL FILTER: quality {report.signal_quality} below minimum for SELL" + ) + report.signal_quality = "D" + report.trade_setup = None + # ── 7. Trade setup for actionable signals ────────────────── - if report.trade_setup is None and report.signal_quality in ("A+", "A", "B", "C"): + # Pair-specific generic-trade suppression: when a pair has a dedicated + # strategy and that strategy didn't fire, falling through to a generic + # swing-based setup tends to chase momentum at unfavorable spots. + # 90-day EUR M15 backtest: allow=True → +4.7R, allow=False → +9.5R. + suppress_generic = ( + (report.display_name == "XAU/USD" + and not self.config.xau_strategy.allow_generic_trades) + or (report.display_name == "EUR/USD" + and self.config.eur_strategy.enabled + and not self.config.eur_strategy.allow_generic_trades) + or (report.display_name == "GBP/USD" + and self.config.gbp_strategy.enabled + and not self.config.gbp_strategy.allow_generic_trades) + ) + if report.trade_setup is None and suppress_generic: + pass # generic trade setup suppressed for this pair + elif report.trade_setup is None and report.signal_quality in ("A+", "A", "B", "C"): is_sell_direction = ( report.final_signal in (SignalStrength.SELL, SignalStrength.STRONG_SELL) or (report.final_signal == SignalStrength.NEUTRAL and report.final_score < 0) @@ -431,12 +592,10 @@ class SignalAggregator: ) -> bool: """Return True when XAU/USD oversold-bounce conditions are met. - Conditions (all must be true): - 1. M15 RSI(14) < 30 - 2. M15 price > M15 EMA(200) - 3. H1 price > H1 EMA(200) - 4. H4 price > H4 EMA(200) - 5. Current bar hour not in blocked list + Conditions: + 1. M15 RSI(14) < rsi_entry_threshold + 2. Price > EMA(200) on at least min_ema200_tfs of {M15, H1, H4} + 3. Current bar hour not in blocked list """ cfg = self.config.xau_strategy @@ -457,31 +616,35 @@ class SignalAggregator: h1_analysis = report.primary_analysis - if not m15_analysis or not h1_analysis or not h4_analysis: + if not m15_analysis or not h1_analysis: return False m15_rsi = m15_analysis.indicators.get("rsi") if not m15_rsi or m15_rsi.value >= cfg.rsi_entry_threshold: return False + ema200_count = 0 + m15_ema200 = m15_analysis.key_levels.get("ema_trend") m15_df = dfs.get(Timeframe.M15) - if m15_df is None or m15_ema200 is None: - return False - m15_price = float(m15_df["Close"].iloc[-1]) - if m15_price <= m15_ema200: - return False + if m15_df is not None and m15_ema200 is not None: + m15_price = float(m15_df["Close"].iloc[-1]) + if m15_price > m15_ema200: + ema200_count += 1 h1_ema200 = h1_analysis.key_levels.get("ema_trend") - if h1_ema200 is None or report.current_price <= h1_ema200: - return False + if h1_ema200 is not None and report.current_price > h1_ema200: + ema200_count += 1 - h4_ema200 = h4_analysis.key_levels.get("ema_trend") - h4_df = dfs.get(Timeframe.H4) - if h4_df is None or h4_ema200 is None: - return False - h4_price = float(h4_df["Close"].iloc[-1]) - if h4_price <= h4_ema200: + if h4_analysis: + h4_ema200 = h4_analysis.key_levels.get("ema_trend") + h4_df = dfs.get(Timeframe.H4) + if h4_df is not None and h4_ema200 is not None: + h4_price = float(h4_df["Close"].iloc[-1]) + if h4_price > h4_ema200: + ema200_count += 1 + + if ema200_count < cfg.min_ema200_tfs: return False return True @@ -530,12 +693,57 @@ class SignalAggregator: pip_value=pip_value, ) + def _generate_xau_atr_trade_setup( + self, report: SignalReport, atr: float + ) -> Optional[TradeSetup]: + """XAU-aware exits for generic signals: wide SL, tight TP via ATR.""" + cfg = self.config.xau_strategy + price = report.current_price + pip_value = 0.10 + + if report.final_signal in (SignalStrength.BUY, SignalStrength.STRONG_BUY): + is_buy = True + elif report.final_signal in (SignalStrength.SELL, SignalStrength.STRONG_SELL): + is_buy = False + else: + is_buy = report.final_score >= 0 + + if is_buy: + sl = price - cfg.sl_atr_multiplier * atr + tp1 = price + cfg.tp_atr_multiplier * atr + tp2 = price + cfg.tp_atr_multiplier * 2.0 * atr + direction = "BUY" + else: + sl = price + cfg.sl_atr_multiplier * atr + tp1 = price - cfg.tp_atr_multiplier * atr + tp2 = price - cfg.tp_atr_multiplier * 2.0 * atr + direction = "SELL" + + sl_distance = abs(price - sl) + tp_distance = abs(tp1 - price) + rr = tp_distance / sl_distance if sl_distance > 0 else 0 + + return TradeSetup( + pair=report.symbol, + display_name=report.display_name, + direction=direction, + entry_price=price, + stop_loss=round(sl, 2), + take_profit_1=round(tp1, 2), + take_profit_2=round(tp2, 2), + risk_reward=round(rr, 2), + risk_pips=round(sl_distance / pip_value, 1), + reward_pips=round(tp_distance / pip_value, 1), + pip_value=pip_value, + ) + def _check_eur_pullback_buy( self, report: SignalReport, dfs: Dict[Timeframe, pd.DataFrame] ) -> bool: """Return True when EUR/USD pullback-buy conditions are met. Conditions (all must be true): + 0. Bar's weekday is in the universal allowed_weekdays list 1. Current bar hour not in blocked list 2. M15 RSI(14) in 30–55 (pullback, not overbought) 3. M15 BB position < 0.75 (not at resistance) @@ -546,9 +754,16 @@ class SignalAggregator: cfg = self.config.eur_strategy try: - bar_hour = pd.Timestamp(report.timestamp).hour + bar_ts = pd.Timestamp(report.timestamp) + bar_hour = bar_ts.hour + bar_wd = bar_ts.weekday() if bar_hour in cfg.blocked_hours_utc: return False + # Universal day filter — must be respected here because the + # strategy hook overwrites the section-6b filter's quality reset. + allowed_wd = self.config.backtest.allowed_weekdays + if allowed_wd and bar_wd not in allowed_wd: + return False except Exception: return False @@ -602,6 +817,20 @@ class SignalAggregator: if dist > cfg.chase_max_atr_from_ema21: return False + # Minimum indicator confluence — rejects gate-passes with weak + # technical agreement. The pullback gate alone fires Quality=A + # even when M15 indicators are mostly neutral; this enforces + # actual confluence. + if cfg.min_indicators_agree > 0: + agreeing = sum( + 1 + for name in ("ema_trend", "rsi", "bollinger", "adx", "candlestick") + if (ind := m15_analysis.indicators.get(name)) is not None + and ind.signal in (SignalStrength.BUY, SignalStrength.STRONG_BUY) + ) + if agreeing < cfg.min_indicators_agree: + return False + return True def _generate_eur_trade_setup( @@ -658,9 +887,14 @@ class SignalAggregator: cfg = self.config.gbp_strategy try: - bar_hour = pd.Timestamp(report.timestamp).hour + bar_ts = pd.Timestamp(report.timestamp) + bar_hour = bar_ts.hour + bar_wd = bar_ts.weekday() if bar_hour in cfg.blocked_hours_utc: return False + allowed_wd = self.config.backtest.allowed_weekdays + if allowed_wd and bar_wd not in allowed_wd: + return False except Exception: return False @@ -714,6 +948,17 @@ class SignalAggregator: if dist > cfg.chase_max_atr_from_ema21: return False + # See EUR equivalent — enforce minimum indicator confluence. + if cfg.min_indicators_agree > 0: + agreeing = sum( + 1 + for name in ("ema_trend", "rsi", "bollinger", "adx", "candlestick") + if (ind := m15_analysis.indicators.get(name)) is not None + and ind.signal in (SignalStrength.BUY, SignalStrength.STRONG_BUY) + ) + if agreeing < cfg.min_indicators_agree: + return False + return True def _generate_gbp_trade_setup( @@ -824,6 +1069,8 @@ class SignalAggregator: elif report.volatility_regime == "low": report.warnings.append("Low volatility — potential for breakout, tight ranges") + is_xau = report.display_name == "XAU/USD" + primary = report.primary_analysis if primary is not None: ema200 = primary.key_levels.get("ema_trend") @@ -835,12 +1082,12 @@ class SignalAggregator: report.warnings.append( f"NEAR EMA200 ({ema_dist_pct:.2f}%) — chop risk" ) - confidence -= 10 - elif ema_dist_pct > _hi: + confidence -= 3 if is_xau else 10 + elif ema_dist_pct > (2.5 if is_xau else _hi): report.warnings.append( f"FAR FROM EMA200 ({ema_dist_pct:.2f}%) — over-extended" ) - confidence -= 10 + confidence -= 5 if is_xau else 10 atr = primary.key_levels.get("atr", 0) if atr > 0 and score != 0: @@ -850,29 +1097,33 @@ class SignalAggregator: v for k, v in primary.key_levels.items() if k.startswith("swing_low_") ) - if swing_lows and report.current_price - swing_lows[0] > 2.5 * atr: - confidence -= 15 - report.warnings.append("TREND EXHAUSTION — price extended >2.5 ATR past swing low") + _exh_mult = 3.5 if is_xau else 2.5 + if swing_lows and report.current_price - swing_lows[0] > _exh_mult * atr: + confidence -= 5 if is_xau else 15 + report.warnings.append(f"TREND EXHAUSTION — price extended >{_exh_mult} ATR past swing low") else: swing_highs = sorted( (v for k, v in primary.key_levels.items() if k.startswith("swing_high_")), reverse=True, ) - if swing_highs and swing_highs[0] - report.current_price > 2.5 * atr: - confidence -= 15 - report.warnings.append("TREND EXHAUSTION — price extended >2.5 ATR past swing high") + _exh_mult_s = 3.5 if is_xau else 2.5 + if swing_highs and swing_highs[0] - report.current_price > _exh_mult_s * atr: + confidence -= 5 if is_xau else 15 + report.warnings.append(f"TREND EXHAUSTION — price extended >{_exh_mult_s} ATR past swing high") ema_medium = primary.key_levels.get("ema_medium") if atr > 0 and ema_medium and report.market_regime.startswith("trending"): pullback_dist = abs(report.current_price - ema_medium) / atr - if pullback_dist > 2.0: - confidence -= 20 + _pb_severe = 2.5 if is_xau else 2.0 + _pb_mild = 2.0 if is_xau else 1.5 + if pullback_dist > _pb_severe: + confidence -= 8 if is_xau else 20 report.warnings.append( f"NO PULLBACK — price {pullback_dist:.1f} ATR from EMA21, chasing risk" ) - elif pullback_dist > 1.5: - confidence -= 10 + elif pullback_dist > _pb_mild: + confidence -= 5 if is_xau else 10 report.warnings.append( f"Weak pullback — price {pullback_dist:.1f} ATR from EMA21" ) @@ -883,9 +1134,10 @@ class SignalAggregator: if rsi_ind: rsi_val = rsi_ind.value + _rsi_pen = 5 if is_xau else 15 if report.market_regime in ("trending_up", "trending") and score > 0: if rsi_val > 65: - confidence -= 15 + confidence -= _rsi_pen report.warnings.append( f"Extended entry — RSI {rsi_val:.0f} in uptrend (prefer pullback to 40-55)" ) @@ -893,7 +1145,7 @@ class SignalAggregator: confidence += 5 elif report.market_regime == "trending_down" and score < 0: if rsi_val < 35: - confidence -= 15 + confidence -= _rsi_pen report.warnings.append( f"Extended entry — RSI {rsi_val:.0f} in downtrend (prefer bounce to 45-62)" ) @@ -916,16 +1168,16 @@ class SignalAggregator: confidence = max(0.0, min(95.0, confidence)) # Final signal - if score > 38: - confidence -= 20 - report.warnings.append("EXHAUSTION: score > 38 — trend overextended") - if score >= 40 and confidence >= 60: + if not is_xau and score > 42: + confidence -= 10 + report.warnings.append("EXHAUSTION: score > 42 — trend overextended") + if score >= 35 and confidence >= 55: report.final_signal = SignalStrength.STRONG_BUY - elif score >= 30 and confidence >= 55: + elif score >= 25 and confidence >= 50: report.final_signal = SignalStrength.BUY - elif score <= -40 and confidence >= 60: + elif score <= -35 and confidence >= 55: report.final_signal = SignalStrength.STRONG_SELL - elif score <= -30 and confidence >= 55: + elif score <= -25 and confidence >= 50: report.final_signal = SignalStrength.SELL else: report.final_signal = SignalStrength.NEUTRAL @@ -947,13 +1199,13 @@ class SignalAggregator: abs_score = abs(score) if sell_suppressed: report.signal_quality = "F" - elif abs_score >= 45 and confidence >= 70 and report.mtf_agreement: + elif abs_score >= 40 and confidence >= 65 and report.mtf_agreement: report.signal_quality = "A+" - elif abs_score >= 38 and confidence >= 60 and report.mtf_agreement: + elif abs_score >= 33 and confidence >= 55 and report.mtf_agreement: report.signal_quality = "A" - elif abs_score >= 33 and confidence >= 55: + elif abs_score >= 28 and confidence >= 50: report.signal_quality = "B" - elif abs_score >= 30 and confidence >= 50: + elif abs_score >= 25 and confidence >= 45: report.signal_quality = "C" elif abs_score >= 15: report.signal_quality = "D" @@ -980,6 +1232,13 @@ class SignalAggregator: if atr == 0: return None + # Fix 2: Minimum ATR threshold for XAU/USD + if report.display_name == "XAU/USD" and atr < self.config.xau_strategy.min_atr_threshold: + report.warnings.append( + f"LOW ATR FILTER: ATR ${atr:.2f} < ${self.config.xau_strategy.min_atr_threshold:.2f} minimum" + ) + return None + min_sl_pips = 8.0 if pip_value < 0.01 else 80.0 min_sl_price = min_sl_pips * pip_value @@ -993,8 +1252,12 @@ class SignalAggregator: min_rr = self.config.signals.min_rr_ratio mature_move = abs(report.final_score) > 30 - tp_mult = 1.5 if mature_move else 2.0 - tp_swing_cap = 2.5 if mature_move else 3.0 + tp_mult = 0.9 if mature_move else 1.2 + tp_swing_cap = 1.5 if mature_move else 2.0 + + # Fix 3: Wider SL multiplier for XAU/USD (1.8x vs default 1.5x) + sl_atr_floor = (self.config.xau_strategy.v2_sl_atr_multiplier + if report.display_name == "XAU/USD" else 1.5) if is_buy: swing_lows = sorted( @@ -1005,7 +1268,7 @@ class SignalAggregator: sl_price = swing_lows[0] - atr * 1.0 else: sl_price = price - atr * 2.0 - min_sl = max(atr * 1.5, min_sl_price) + min_sl = max(atr * sl_atr_floor, min_sl_price) if price - sl_price < min_sl: sl_price = price - min_sl @@ -1035,7 +1298,7 @@ class SignalAggregator: sl_price = swing_highs[0] + atr * 1.0 else: sl_price = price + atr * 2.0 - min_sl = max(atr * 1.5, min_sl_price) + min_sl = max(atr * sl_atr_floor, min_sl_price) if sl_price - price < min_sl: sl_price = price + min_sl diff --git a/signal_web.py b/signal_web.py new file mode 100644 index 0000000..3bf5428 --- /dev/null +++ b/signal_web.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +""" +ForexBot2 — Signal Daemon +========================== +Candle-boundary auto-scanner that publishes signal data to Redis. +Mirrors telegram_bot.py scan loops but outputs to Redis instead of Telegram. + +Scan loop: M15/H1/H4 at candle boundaries + 10s buffer. +Data flow: SignalAggregator → serialize → Redis key + pub/sub channel. + +Usage: + python signal_web.py +""" + +import asyncio +import json +import logging +import os +import signal as unix_signal +import sys +import time +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from typing import Dict, List, Optional + +import redis + +from config import ( + PlatformConfig, SignalStrength, Timeframe, + EURUSD, GBPUSD, XAUUSD, PAIR_INFO, +) +from data_provider import ForexDataProvider +from signal_aggregator import SignalAggregator, SignalReport + +if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8") + +logging.basicConfig( + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", + level=logging.INFO, + datefmt="%H:%M:%S", +) +logging.getLogger("httpx").setLevel(logging.WARNING) +logging.getLogger("httpcore").setLevel(logging.WARNING) +logger = logging.getLogger("signal_web") + +# ── Config ──────────────────────────────────────────────────────────── + +SCAN_PAIRS = [EURUSD, GBPUSD, XAUUSD] + +AUTOSCAN_CONFIRMATIONS = { + Timeframe.M15: [Timeframe.M5, Timeframe.H1], + Timeframe.H1: [Timeframe.M15, Timeframe.H4], + Timeframe.H4: [Timeframe.H1, Timeframe.D1], +} + +AUTOSCAN_LOOPS = [ + (Timeframe.M15, 15), + (Timeframe.H1, 60), + (Timeframe.H4, 240), +] + +REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379/0") +REDIS_KEY = "forexbot2:signals" +REDIS_CHANNEL = "forexbot2:updates" + +_executor = ThreadPoolExecutor(max_workers=4) + +# ── Shared data provider ────────────────────────────────────────────── + +_shared_dp: Optional[ForexDataProvider] = None + + +def _get_shared_dp() -> ForexDataProvider: + global _shared_dp + if _shared_dp is None: + _shared_dp = ForexDataProvider(PlatformConfig()) + return _shared_dp + + +# ── Redis client ────────────────────────────────────────────────────── + +_redis: Optional[redis.Redis] = None + + +def _init_redis() -> None: + global _redis + _redis = redis.from_url(REDIS_URL, decode_responses=True) + _redis.ping() + logger.info(f"Redis connected: {REDIS_URL}") + + +# ── Signal runner (mirrors telegram_bot.py) ─────────────────────────── + +TF_CONFIRMATIONS = { + Timeframe.M15: [Timeframe.M5, Timeframe.H1], + Timeframe.H1: [Timeframe.M15, Timeframe.H4], + Timeframe.H4: [Timeframe.H1, Timeframe.D1], +} + + +def _run_signal_sync( + symbol: str, + timeframe: Timeframe, + confirmation_tfs: Optional[List[Timeframe]] = None, +) -> SignalReport: + if confirmation_tfs is None: + confirmation_tfs = TF_CONFIRMATIONS.get( + timeframe, [Timeframe.M15, Timeframe.H4] + ) + config = PlatformConfig() + config.primary_timeframe = timeframe + config.confirmation_timeframes = confirmation_tfs + aggregator = SignalAggregator(config) + aggregator.data_provider = _get_shared_dp() + return aggregator.analyze_pair(symbol=symbol) + + +async def _run_signal( + symbol: str, + timeframe: Timeframe, + confirmation_tfs: Optional[List[Timeframe]] = None, +) -> SignalReport: + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + _executor, _run_signal_sync, symbol, timeframe, confirmation_tfs + ) + + +# ── Serialization ───────────────────────────────────────────────────── + +def _serialize_report(report: SignalReport) -> dict: + indicators = {} + if report.primary_analysis: + for name, ind in report.primary_analysis.indicators.items(): + if name.startswith("_") or ind.weight == 0: + continue + indicators[name] = { + "signal": ind.signal.name, + "value": round(ind.value, 4), + "confidence": round(ind.confidence, 1), + "description": ind.description, + } + + confirmation_data = [] + for ca in report.confirmation_analyses: + cf_indicators = {} + for name, ind in ca.indicators.items(): + if name.startswith("_") or ind.weight == 0: + continue + cf_indicators[name] = { + "signal": ind.signal.name, + "value": round(ind.value, 4), + "confidence": round(ind.confidence, 1), + "description": ind.description, + } + confirmation_data.append({ + "timeframe": ca.timeframe, + "signal": ca.composite_signal.name, + "score": round(ca.composite_score, 1), + "confidence": round(ca.confidence, 1), + "regime": ca.market_regime, + "indicators": cf_indicators, + }) + + trade = None + if report.trade_setup and report.final_signal not in (SignalStrength.NEUTRAL,): + ts = report.trade_setup + trade = { + "direction": ts.direction, + "entry": round(ts.entry_price, 5), + "sl": round(ts.stop_loss, 5), + "tp1": round(ts.take_profit_1, 5), + "tp2": round(ts.take_profit_2, 5) if ts.take_profit_2 else None, + "rr": round(ts.risk_reward, 2), + "risk_pips": round(ts.risk_pips, 1), + "reward_pips": round(ts.reward_pips, 1), + } + + return { + "pair": report.display_name, + "symbol": report.symbol, + "timestamp": report.timestamp, + "signal": report.final_signal.name, + "score": round(report.final_score, 1), + "confidence": round(report.final_confidence, 1), + "quality": report.signal_quality, + "regime": report.market_regime, + "volatility": report.volatility_regime, + "price": report.current_price, + "mtf_agreement": report.mtf_agreement, + "mtf_signals": {k: v.name for k, v in report.mtf_signals.items()}, + "mtf_scores": {k: round(v, 1) for k, v in report.mtf_scores.items()}, + "indicators": indicators, + "confirmations": confirmation_data, + "trade_setup": trade, + "bullish_reasons": report.bullish_reasons, + "bearish_reasons": report.bearish_reasons, + "warnings": report.warnings, + "action": report.action_recommendation, + } + + +# ── Redis publish ───────────────────────────────────────────────────── + +def _publish_to_redis(snapshot: dict) -> None: + payload = json.dumps(snapshot) + _redis.set(REDIS_KEY, payload) + _redis.publish(REDIS_CHANNEL, payload) + + +# ── Scan logic ──────────────────────────────────────────────────────── + +_signal_store: Dict[str, Dict[str, dict]] = {} + + +async def _scan_timeframe(tf: Timeframe, confirmation_tfs: List[Timeframe]) -> None: + tasks = [_run_signal(symbol, tf, confirmation_tfs) for symbol in SCAN_PAIRS] + results = await asyncio.gather(*tasks, return_exceptions=True) + + for i, result in enumerate(results): + if isinstance(result, Exception): + logger.exception(f"Scan failed for {SCAN_PAIRS[i]} {tf.value}: {result}") + continue + + report: SignalReport = result + display = report.display_name + if display not in _signal_store: + _signal_store[display] = {} + _signal_store[display][tf.value] = _serialize_report(report) + + snapshot = { + "type": "update", + "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC"), + "data": _signal_store, + } + _publish_to_redis(snapshot) + ok = len([r for r in results if not isinstance(r, Exception)]) + logger.info(f"{tf.value} scan complete — {ok} pairs updated") + + +# ── Candle-boundary scheduler ──────────────────────────────────────── + +def _seconds_until_boundary(interval_minutes: int, buffer: int = 10) -> float: + now = datetime.now(timezone.utc) + total_minutes = now.hour * 60 + now.minute + elapsed_minutes = total_minutes % interval_minutes + elapsed_seconds = elapsed_minutes * 60 + now.second + now.microsecond / 1e6 + interval_seconds = interval_minutes * 60 + remaining = interval_seconds - elapsed_seconds + buffer + if remaining <= buffer: + remaining += interval_seconds + return remaining + + +async def _candle_loop(tf: Timeframe, interval_minutes: int) -> None: + label = tf.value + try: + while True: + wait = _seconds_until_boundary(interval_minutes, buffer=10) + logger.info(f"{label} scan: next in {wait:.0f}s") + await asyncio.sleep(wait) + logger.info(f"{label} scan: firing") + await _scan_timeframe(tf, AUTOSCAN_CONFIRMATIONS[tf]) + except asyncio.CancelledError: + pass + finally: + logger.info(f"{label} scan loop stopped") + + +# ── Main ────────────────────────────────────────────────────────────── + +async def main() -> None: + _init_redis() + + logger.info("Running initial M15 scan…") + await _scan_timeframe(Timeframe.M15, AUTOSCAN_CONFIRMATIONS[Timeframe.M15]) + logger.info("Initial scan complete") + + tasks = [] + for tf, interval in AUTOSCAN_LOOPS: + task = asyncio.create_task(_candle_loop(tf, interval)) + tasks.append(task) + logger.info(f"Started {tf.value} scan loop (every {interval}min)") + + stop = asyncio.Event() + loop = asyncio.get_event_loop() + for sig in (unix_signal.SIGINT, unix_signal.SIGTERM): + loop.add_signal_handler(sig, stop.set) + + logger.info("Signal daemon running — Ctrl+C to stop") + await stop.wait() + + logger.info("Shutting down…") + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + _executor.shutdown(wait=False) + logger.info("Signal daemon stopped") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/test_all.py b/test_all.py index 4badacb..500527f 100644 --- a/test_all.py +++ b/test_all.py @@ -172,7 +172,7 @@ def _(): def _(): sc = SignalConfig() assert sc.min_confidence == 55.0 - assert sc.min_rr_ratio == 1.2 + assert sc.min_rr_ratio == 0.5 assert sc.atr_sl_multiplier == 1.5 assert sc.default_rr_ratio == 2.0 assert sc.cooldown_bars == 5 @@ -198,7 +198,9 @@ def _(): def _(): xc = XauStrategyConfig() assert xc.enabled is True - assert xc.rsi_entry_threshold == 38.0 + assert xc.rsi_entry_threshold == 45.0 + assert xc.min_ema200_tfs == 2 + assert xc.allow_generic_trades is True assert xc.tp_atr_multiplier == 0.9 assert xc.sl_atr_multiplier == 2.0 assert isinstance(xc.blocked_hours_utc, list) @@ -1123,6 +1125,9 @@ def _(): config = PlatformConfig() config.backtest.session_filter = True config.backtest.blocked_hours_utc = list(range(24)) # block all hours + config.eur_strategy.enabled = False + config.gbp_strategy.enabled = False + config.xau_strategy.enabled = False agg = SignalAggregator(config) dfs = { Timeframe.H1: make_ohlcv(300, seed=42), diff --git a/web/app.py b/web/app.py new file mode 100644 index 0000000..b0dcb4a --- /dev/null +++ b/web/app.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +""" +ForexBot2 — Web API +==================== +Thin FastAPI process that reads signal data from Redis and serves it +via REST + WebSocket. Apache reverse-proxies to this. + +Endpoints: + GET /api/signals — current snapshot from Redis + WS /ws — real-time updates via Redis pub/sub + +No signal computation here — signal_web.py daemon handles that. + +Usage: + uvicorn web.app:app --host 127.0.0.1 --port 8000 +""" + +import asyncio +import json +import logging +import os +from contextlib import asynccontextmanager +from typing import Optional, Set + +import redis.asyncio as aioredis +import uvicorn +from fastapi import FastAPI, WebSocket, WebSocketDisconnect +from fastapi.middleware.cors import CORSMiddleware + +logging.basicConfig( + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", + level=logging.INFO, + datefmt="%H:%M:%S", +) +logger = logging.getLogger("web_api") + +REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379/0") +REDIS_KEY = "forexbot2:signals" +REDIS_CHANNEL = "forexbot2:updates" + +_redis: Optional[aioredis.Redis] = None +_ws_clients: Set[WebSocket] = set() +_subscriber_task: Optional[asyncio.Task] = None + + +async def _redis_subscriber() -> None: + pubsub = _redis.pubsub() + await pubsub.subscribe(REDIS_CHANNEL) + try: + async for msg in pubsub.listen(): + if msg["type"] != "message": + continue + payload = msg["data"] + dead = [] + for ws in _ws_clients: + try: + await ws.send_text(payload) + except Exception: + dead.append(ws) + for ws in dead: + _ws_clients.discard(ws) + except asyncio.CancelledError: + pass + finally: + await pubsub.unsubscribe(REDIS_CHANNEL) + await pubsub.close() + + +@asynccontextmanager +async def lifespan(application: FastAPI): + global _redis, _subscriber_task + _redis = aioredis.from_url(REDIS_URL, decode_responses=True) + await _redis.ping() + logger.info(f"Redis connected: {REDIS_URL}") + _subscriber_task = asyncio.create_task(_redis_subscriber()) + logger.info("Redis subscriber started") + yield + _subscriber_task.cancel() + await asyncio.gather(_subscriber_task, return_exceptions=True) + await _redis.close() + logger.info("Web API stopped") + + +app = FastAPI(title="ForexBot2 API", lifespan=lifespan) + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["GET"], + allow_headers=["*"], +) + + +@app.get("/api/signals") +async def api_signals(): + raw = await _redis.get(REDIS_KEY) + if raw: + return json.loads(raw) + return {"type": "update", "timestamp": "", "data": {}} + + +@app.websocket("/ws") +async def ws_endpoint(ws: WebSocket): + await ws.accept() + _ws_clients.add(ws) + try: + raw = await _redis.get(REDIS_KEY) + if raw: + await ws.send_text(raw) + while True: + await ws.receive_text() + except WebSocketDisconnect: + pass + finally: + _ws_clients.discard(ws) + + +if __name__ == "__main__": + port = int(os.environ.get("WEB_PORT", "8000")) + logger.info(f"Starting ForexBot2 API on http://127.0.0.1:{port}") + uvicorn.run(app, host="127.0.0.1", port=port, log_level="info") diff --git a/web/static/index.html b/web/static/index.html new file mode 100644 index 0000000..d4075b0 --- /dev/null +++ b/web/static/index.html @@ -0,0 +1,482 @@ + + + + + +ForexBot2 — Live Dashboard + + + + +
+

ForexBot2 Live Dashboard

+
+ Connecting… + +
+
+ +
+
+ + Running initial scan — this takes 30–60 seconds… +
+
+ + + +