backtest cleanup: drop four research/one-time scripts; books and stale cache moved out (owner decisions 2026-09-15)
Discussion-first cleanup of backtest/ and scripts/. Nothing in the signal
core or the backtester changed.
Removed (last commit containing them: 931b265):
- backtest/diagnose_filters.py + its two Module 18 tests
- scripts/shadow_scan.py (Phase 4 shadow scan, never run)
- scripts/backfill_news_archive.py, scripts/migrate_signal_log_news.py
(one-time scripts; the interning test keeps its live-path half, the
prune test goes)
Kept: the replay-recording stack (replay_record.py, replay_scorecard.py,
ff_calendar_scraper.py, cache/calendar.parquet, Module 33).
Local gitignored data was moved to ~/Development/ForexBot2_trash_2026-09-15
(346 MB) for the owner to delete: all 35 pinned backtest/*.csv books, the
three research output folders, the BTC/ETH cache snapshots, the Tiingo
parquets and 13 uncited Oanda windows. The three research-folder ignore
rules are dropped with the folders.
Verification: all five gate books regenerate offline from the pruned cache
with the recorded MD5s (M5 W2 29120a1e, M15 W2 d185f099, M15 OOS 6f8f24b4,
H1 W3 3b45d12b, H4 W3 e75adae1; NumPy CPU, 0 cache fallbacks). Suite
506 passed, 0 failed, 33 modules.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Harwinder Sekhon
@@ -1,230 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Seed logs/newsdata.db from the headline blobs still on disk.
|
||||
==============================================================
|
||||
|
||||
The durable headline corpus (``news_archive.py``) only started collecting
|
||||
when its hook shipped. Everything before that survives only inside signal
|
||||
logs, in two different shapes:
|
||||
|
||||
* ``logs/signals.db`` — modern schema, headlines interned into
|
||||
``news_snapshots`` (kind ``headlines``) and referenced by hash.
|
||||
* an older orphan DB (default ``~/signals.db``) — pre-interning, blobs
|
||||
inline in ``signals.headlines_json``, short-form item keys with no
|
||||
``description``, ``link``, ``publisher`` or ``score_by_currency``.
|
||||
|
||||
Both are opened READ-ONLY. The script is idempotent: re-running it adds
|
||||
nothing new and never downgrades a score.
|
||||
|
||||
Provenance is RECOMPUTED, not trusted
|
||||
-------------------------------------
|
||||
Neither source records which scorer produced its ``score`` — that flag
|
||||
only exists from this change onwards. So each headline is re-scored with
|
||||
the live keyword scorer and compared:
|
||||
|
||||
* stored score differs from the keyword score -> ``llm`` (DeepSeek wrote it)
|
||||
* exact match on a non-zero score -> ``keyword``
|
||||
* both zero -> ``unknown`` (indistinguishable)
|
||||
|
||||
Rows are stamped ``origin='backfill'`` because they come from the
|
||||
post-filter pool: ``get_fundamental_context`` drops headlines inside
|
||||
±0.1 before they ever reach a signal row, so genuinely neutral headlines
|
||||
are under-represented in this history. Rows collected live by the hook
|
||||
in ``news_engine`` do not have that bias and are stamped ``'live'``.
|
||||
|
||||
Usage
|
||||
-----
|
||||
venv/bin/python scripts/backfill_news_archive.py --dry-run
|
||||
venv/bin/python scripts/backfill_news_archive.py
|
||||
venv/bin/python scripts/backfill_news_archive.py --legacy-db /path/to/old.db
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
from collections import Counter
|
||||
from typing import Any, Dict, Iterator, Tuple
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import app_config # noqa: E402
|
||||
import news_archive # noqa: E402
|
||||
from config import NewsConfig # noqa: E402
|
||||
from news_engine import NewsEngine # noqa: E402
|
||||
|
||||
|
||||
def _open_ro(path: str) -> sqlite3.Connection:
|
||||
"""Read-only, but NOT immutable=1 — that flag is blind to WAL rows."""
|
||||
return sqlite3.connect(f"file:{os.path.abspath(path)}?mode=ro", uri=True)
|
||||
|
||||
|
||||
def _has_table(con: sqlite3.Connection, name: str) -> bool:
|
||||
row = con.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?",
|
||||
(name,)).fetchone()
|
||||
return row is not None
|
||||
|
||||
|
||||
def iter_blobs(path: str) -> Iterator[Tuple[str, str]]:
|
||||
"""Yield (headlines_json, seen_at) from either DB shape."""
|
||||
con = _open_ro(path)
|
||||
try:
|
||||
if _has_table(con, "news_snapshots"):
|
||||
yield from con.execute(
|
||||
"SELECT body, first_seen FROM news_snapshots"
|
||||
" WHERE kind='headlines' ORDER BY first_seen")
|
||||
cols = {r[1] for r in con.execute("PRAGMA table_info(signals)")}
|
||||
if "headlines_json" in cols:
|
||||
order = "logged_at" if "logged_at" in cols else "id"
|
||||
yield from con.execute(
|
||||
"SELECT headlines_json, "
|
||||
+ ("logged_at" if "logged_at" in cols else "''")
|
||||
+ " FROM signals WHERE headlines_json IS NOT NULL"
|
||||
f" ORDER BY {order}")
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
class _Row:
|
||||
"""Duck-typed stand-in for NewsSentiment, so news_archive.build_row
|
||||
flattens a stored dict exactly the way it flattens a live headline."""
|
||||
|
||||
__slots__ = ("headline", "source", "timestamp", "currencies_mentioned",
|
||||
"sentiment_score", "keywords", "sentiment_by_currency",
|
||||
"link", "description", "publisher", "score_source")
|
||||
|
||||
def __init__(self, item: Dict[str, Any], score_source: str):
|
||||
self.headline = item.get("headline")
|
||||
self.source = item.get("source")
|
||||
self.timestamp = item.get("timestamp")
|
||||
self.currencies_mentioned = item.get("currencies") or []
|
||||
self.sentiment_score = item.get("score")
|
||||
self.keywords = item.get("keywords") or []
|
||||
self.sentiment_by_currency = item.get("score_by_currency") or {}
|
||||
self.link = item.get("link")
|
||||
self.description = item.get("description")
|
||||
self.publisher = item.get("publisher")
|
||||
self.score_source = score_source
|
||||
|
||||
|
||||
def classify(engine: NewsEngine, item: Dict[str, Any]) -> str:
|
||||
"""Recover which scorer produced this stored score."""
|
||||
text = item.get("headline")
|
||||
if not isinstance(text, str) or not text.strip():
|
||||
return "unknown"
|
||||
try:
|
||||
stored = float(item.get("score") or 0.0)
|
||||
except (TypeError, ValueError):
|
||||
return "unknown"
|
||||
keyword = engine._analyze_headline_sentiment(text)
|
||||
if abs(stored - keyword) > 1e-9:
|
||||
return "llm"
|
||||
if abs(stored) < 1e-9:
|
||||
return "unknown" # both zero — the two scorers are indistinguishable
|
||||
return "keyword"
|
||||
|
||||
|
||||
def collect(paths, engine: NewsEngine):
|
||||
"""Newest wins per headline: later blobs carry upgraded LLM scores."""
|
||||
best: Dict[str, Tuple[str, Dict[str, Any], str]] = {}
|
||||
stats = Counter()
|
||||
for path in paths:
|
||||
if not os.path.exists(path):
|
||||
print(f" skip (not found): {path}")
|
||||
continue
|
||||
blobs = 0
|
||||
for body, seen_at in iter_blobs(path):
|
||||
try:
|
||||
items = json.loads(body)
|
||||
except Exception:
|
||||
stats["bad_blob"] += 1
|
||||
continue
|
||||
blobs += 1
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
text = item.get("headline")
|
||||
if not isinstance(text, str) or not text.strip():
|
||||
continue
|
||||
source = classify(engine, item)
|
||||
prior = best.get(text)
|
||||
if prior is None or (
|
||||
news_archive._SOURCE_RANK.get(source, 0)
|
||||
>= news_archive._SOURCE_RANK.get(prior[0], 0)):
|
||||
best[text] = (source, item, seen_at or "")
|
||||
stats["items"] += 1
|
||||
print(f" {path}: {blobs} blobs")
|
||||
for source, _, _ in best.values():
|
||||
stats[source] += 1
|
||||
return best, stats
|
||||
|
||||
|
||||
def main() -> int:
|
||||
default_legacy = os.path.join(os.path.expanduser("~"), "signals.db")
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n")[1])
|
||||
ap.add_argument("--db", default=app_config.signal_log_path(),
|
||||
help="modern signal-log DB (default: config path)")
|
||||
ap.add_argument("--legacy-db", default=default_legacy,
|
||||
help=f"older inline-blob DB (default: {default_legacy})")
|
||||
ap.add_argument("--out", default=app_config.news_archive_path(),
|
||||
help="archive DB to write (default: config path)")
|
||||
ap.add_argument("--dry-run", action="store_true",
|
||||
help="count only, write nothing")
|
||||
args = ap.parse_args()
|
||||
|
||||
engine = NewsEngine(NewsConfig())
|
||||
print("Reading headline blobs (read-only):")
|
||||
best, stats = collect([args.db, args.legacy_db], engine)
|
||||
|
||||
print(f"\n headline items scanned : {stats['items']}")
|
||||
print(f" distinct headlines : {len(best)}")
|
||||
print(f" llm (DeepSeek) : {stats['llm']}")
|
||||
print(f" keyword : {stats['keyword']}")
|
||||
print(f" unknown (both zero) : {stats['unknown']}")
|
||||
if stats["bad_blob"]:
|
||||
print(f" unparseable blobs : {stats['bad_blob']}")
|
||||
|
||||
if args.dry_run:
|
||||
print("\n--dry-run: nothing written.")
|
||||
return 0
|
||||
if not best:
|
||||
print("\nNothing to write.")
|
||||
return 0
|
||||
|
||||
rows = []
|
||||
for text, (source, item, seen_at) in best.items():
|
||||
row = news_archive.build_row(
|
||||
_Row(item, source),
|
||||
teacher_model=None, # not recorded by either source
|
||||
teacher_prompt_sha1=None,
|
||||
origin="backfill",
|
||||
first_seen=seen_at or None,
|
||||
last_seen=seen_at or None)
|
||||
if row is not None:
|
||||
rows.append(row)
|
||||
|
||||
archive = news_archive.NewsArchive(args.out)
|
||||
try:
|
||||
written = archive.store(rows)
|
||||
finally:
|
||||
archive.close()
|
||||
print(f"\nWrote {written} rows to {args.out}")
|
||||
|
||||
con = _open_ro(args.out)
|
||||
try:
|
||||
print("\nArchive now holds:")
|
||||
for source, count in con.execute(
|
||||
"SELECT score_source, COUNT(*) FROM headlines"
|
||||
" GROUP BY score_source ORDER BY COUNT(*) DESC"):
|
||||
print(f" {source:8s} {count}")
|
||||
finally:
|
||||
con.close()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,177 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
migrate_signal_log_news.py — move inline news blobs into news_snapshots.
|
||||
|
||||
Legacy rows store the full RSS headline list (~33 KB) and calendar events
|
||||
(~1.2 KB) inline on EVERY scan row; the same blob repeats ~5× because the
|
||||
news caches outlive a candle. This script interns each distinct blob once
|
||||
(content hash), stamps `headlines_hash` / `events_hash` on the row and
|
||||
NULLs the inline columns. Readers use the `signals_with_news` view or
|
||||
`signal_log.read_news`.
|
||||
|
||||
Idempotent and resumable (batched transactions). Run --vacuum with BOTH
|
||||
live writers (telegram bot, dashboard publisher) stopped — VACUUM needs
|
||||
exclusive access and ~1× the final size of free disk.
|
||||
|
||||
python scripts/migrate_signal_log_news.py --dry-run
|
||||
python scripts/migrate_signal_log_news.py
|
||||
python scripts/migrate_signal_log_news.py --vacuum # bots stopped
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import app_config # noqa: E402
|
||||
from signal_log import SignalLogger # noqa: E402
|
||||
|
||||
|
||||
def _hash(text: str) -> str:
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def migrate(con: sqlite3.Connection, batch: int = 500) -> dict:
|
||||
"""Intern legacy inline blobs; returns counters. Caller owns the
|
||||
connection (schema must already exist — open a SignalLogger once)."""
|
||||
stats = {"rows": 0, "headlines_interned": 0, "events_interned": 0}
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
while True:
|
||||
rows = con.execute(
|
||||
"SELECT id, headlines_json, events_json FROM signals"
|
||||
" WHERE (headlines_json IS NOT NULL AND headlines_hash IS NULL)"
|
||||
" OR (events_json IS NOT NULL AND events_hash IS NULL)"
|
||||
" ORDER BY id LIMIT ?", (batch,)).fetchall()
|
||||
if not rows:
|
||||
break
|
||||
con.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
for rowid, headlines, events in rows:
|
||||
sets, params = [], []
|
||||
if headlines is not None:
|
||||
h = _hash(headlines)
|
||||
cur = con.execute(
|
||||
"INSERT OR IGNORE INTO news_snapshots"
|
||||
" (hash, kind, body, first_seen, last_seen)"
|
||||
" VALUES (?, 'headlines', ?, ?, ?)",
|
||||
(h, headlines, now, now))
|
||||
stats["headlines_interned"] += cur.rowcount
|
||||
sets += ["headlines_hash=?", "headlines_json=NULL"]
|
||||
params.append(h)
|
||||
if events is not None:
|
||||
h = _hash(events)
|
||||
cur = con.execute(
|
||||
"INSERT OR IGNORE INTO news_snapshots"
|
||||
" (hash, kind, body, first_seen, last_seen)"
|
||||
" VALUES (?, 'events', ?, ?, ?)",
|
||||
(h, events, now, now))
|
||||
stats["events_interned"] += cur.rowcount
|
||||
sets += ["events_hash=?", "events_json=NULL"]
|
||||
params.append(h)
|
||||
params.append(rowid)
|
||||
con.execute(
|
||||
f"UPDATE signals SET {', '.join(sets)} WHERE id=?",
|
||||
params)
|
||||
stats["rows"] += 1
|
||||
con.commit()
|
||||
except Exception:
|
||||
con.rollback()
|
||||
raise
|
||||
return stats
|
||||
|
||||
|
||||
def prune_orphans(con: sqlite3.Connection) -> int:
|
||||
cur = con.execute(
|
||||
"DELETE FROM news_snapshots WHERE hash NOT IN"
|
||||
" (SELECT headlines_hash FROM signals WHERE headlines_hash IS NOT NULL"
|
||||
" UNION SELECT events_hash FROM signals WHERE events_hash IS NOT NULL)")
|
||||
con.commit()
|
||||
return cur.rowcount
|
||||
|
||||
|
||||
def prune_news_older_than(con: sqlite3.Connection, days: int) -> int:
|
||||
"""Drop the news reference (not the row) from rows older than N days,
|
||||
then delete snapshots nothing references any more."""
|
||||
# logged_at is ISO8601 with a 'T' and a +00:00 offset; SQLite's
|
||||
# datetime() returns 'YYYY-MM-DD HH:MM:SS'. Comparing them as raw
|
||||
# strings put every row from the cutoff CALENDAR DAY on the wrong
|
||||
# side ('T' > ' '), so a full day never pruned. Normalise to the
|
||||
# same shape first — all logged_at values are UTC (see _iso()).
|
||||
cur = con.execute(
|
||||
"UPDATE signals SET headlines_hash=NULL, events_hash=NULL"
|
||||
" WHERE substr(replace(logged_at, 'T', ' '), 1, 19)"
|
||||
" < datetime('now', ?)", (f"-{int(days)} days",))
|
||||
con.commit()
|
||||
prune_orphans(con)
|
||||
return cur.rowcount
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[1])
|
||||
ap.add_argument("--db", default=None)
|
||||
ap.add_argument("--dry-run", action="store_true")
|
||||
ap.add_argument("--batch", type=int, default=500)
|
||||
ap.add_argument("--prune-news-older-than", type=int, default=None,
|
||||
metavar="DAYS")
|
||||
ap.add_argument("--prune-orphans", action="store_true")
|
||||
ap.add_argument("--vacuum", action="store_true",
|
||||
help="checkpoint + VACUUM (all writers must be stopped)")
|
||||
ap.add_argument("--no-backup", action="store_true")
|
||||
args = ap.parse_args(argv)
|
||||
path = args.db or app_config.signal_log_path()
|
||||
if not os.path.exists(path):
|
||||
print(f"no database at {path}")
|
||||
return 1
|
||||
size0 = os.path.getsize(path)
|
||||
if args.dry_run:
|
||||
con = sqlite3.connect(f"file:{os.path.abspath(path)}?mode=ro", uri=True)
|
||||
n = con.execute(
|
||||
"SELECT COUNT(*) FROM signals WHERE headlines_json IS NOT NULL"
|
||||
" OR events_json IS NOT NULL").fetchone()[0]
|
||||
d = con.execute(
|
||||
"SELECT COUNT(DISTINCT headlines_json) FROM signals"
|
||||
" WHERE headlines_json IS NOT NULL").fetchone()[0]
|
||||
print(f"{n} rows carry inline news blobs ({d} distinct headline "
|
||||
f"blobs); file size {size0 / 1e6:.1f} MB")
|
||||
con.close()
|
||||
return 0
|
||||
if not args.no_backup:
|
||||
backup = f"{path}.pre-news-{datetime.now(timezone.utc):%Y%m%d}"
|
||||
if not os.path.exists(backup):
|
||||
src = sqlite3.connect(path)
|
||||
dst = sqlite3.connect(backup)
|
||||
src.backup(dst)
|
||||
dst.close()
|
||||
src.close()
|
||||
print(f"backup written: {backup}")
|
||||
lg = SignalLogger(path) # applies the additive schema
|
||||
lg.close()
|
||||
con = sqlite3.connect(path)
|
||||
con.execute("PRAGMA busy_timeout=10000")
|
||||
try:
|
||||
print("migrate:", migrate(con, batch=args.batch))
|
||||
if args.prune_news_older_than is not None:
|
||||
print("rows de-referenced:",
|
||||
prune_news_older_than(con, args.prune_news_older_than))
|
||||
if args.prune_orphans:
|
||||
print("orphan snapshots removed:", prune_orphans(con))
|
||||
if args.vacuum:
|
||||
try:
|
||||
con.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
||||
con.execute("VACUUM")
|
||||
except sqlite3.OperationalError as exc:
|
||||
print(f"VACUUM failed ({exc}) — stop the bot and dashboard "
|
||||
"publisher first, then re-run with --vacuum")
|
||||
return 2
|
||||
finally:
|
||||
con.close()
|
||||
print(f"size {size0 / 1e6:.1f} MB -> {os.path.getsize(path) / 1e6:.1f} MB"
|
||||
+ ("" if args.vacuum else " (run --vacuum to reclaim space)"))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,192 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Phase 4 — live shadow scan (data source change test)
|
||||
=====================================================
|
||||
Runs TWO data providers side-by-side at every candle close (M5/M15/H1)
|
||||
and logs how their signals diverge, so the feed switch can be validated
|
||||
under real fetch depths / caching / boundary timing for ≥2 weeks.
|
||||
|
||||
Unlike the history studies, this uses the LIVE fetch path (fetch_ohlcv,
|
||||
forming-candle drop, freshness checks) for both providers, but calls
|
||||
`analyze_from_dataframes(use_news=False)` directly so the (provider-
|
||||
independent) news/DeepSeek budget is not burned on shadow scans. The
|
||||
weekend filter is applied here (skip the whole cycle), and the session/
|
||||
day/hour gates run inside analyze_from_dataframes as in the backtest.
|
||||
|
||||
No orders, no Telegram, no signal-log writes — output is a side-by-side
|
||||
CSV plus a rolling agreement tally.
|
||||
|
||||
Usage:
|
||||
venv/bin/python scripts/shadow_scan.py --a oanda --b tiingo \
|
||||
--tfs M5,M15,H1 --out logs/shadow_oanda_tiingo.csv --once
|
||||
# production (2+ weeks):
|
||||
nohup venv/bin/python scripts/shadow_scan.py --a oanda --b tiingo \
|
||||
--out logs/shadow_oanda_tiingo.csv >> logs/shadow_scan.log 2>&1 &
|
||||
|
||||
CSV columns (one row per TF per boundary):
|
||||
ts, tf, a_<field>, b_<field> for direction/grade/setup/tag/entry/sl/tp1,
|
||||
a_price, b_price, a_spread, b_spread, a_warnings, b_warnings,
|
||||
dir_agree, grade_agree, setup_agree
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from config import PlatformConfig, Timeframe, XAUUSD
|
||||
from pair_signals.xauusd import make_config
|
||||
from providers import make_data_provider
|
||||
from providers.base import _is_forex_weekend
|
||||
from signal_aggregator import SignalAggregator, SignalReport
|
||||
|
||||
_TF_LOOKUP = {"M5": Timeframe.M5, "M15": Timeframe.M15, "H1": Timeframe.H1,
|
||||
"H4": Timeframe.H4}
|
||||
_TF_MINUTES = {Timeframe.M5: 5, Timeframe.M15: 15, Timeframe.H1: 60,
|
||||
Timeframe.H4: 240}
|
||||
_CONFIRMS = {Timeframe.M5: [Timeframe.M15, Timeframe.H1],
|
||||
Timeframe.M15: [Timeframe.M5, Timeframe.H1],
|
||||
Timeframe.H1: [Timeframe.M15, Timeframe.H4],
|
||||
Timeframe.H4: [Timeframe.H1, Timeframe.D1]}
|
||||
|
||||
FIELDS = ["direction", "grade", "setup", "tag", "entry", "sl", "tp1"]
|
||||
|
||||
|
||||
def seconds_until_boundary(interval_minutes: int, buffer: int = 10) -> float:
|
||||
now = datetime.now(timezone.utc)
|
||||
elapsed = (now.minute % interval_minutes) * 60 + now.second \
|
||||
+ now.microsecond / 1e6
|
||||
return interval_minutes * 60 - elapsed + buffer
|
||||
|
||||
|
||||
def due_timeframes(wake: datetime, tfs) -> list:
|
||||
minutes = wake.hour * 60 + wake.minute
|
||||
floored = minutes - (minutes % 5)
|
||||
return [tf for tf in tfs if floored % _TF_MINUTES[tf] == 0]
|
||||
|
||||
|
||||
def tag_of(report) -> str:
|
||||
fb = report.bullish_reasons[0] if report.bullish_reasons else ""
|
||||
fbe = report.bearish_reasons[0] if report.bearish_reasons else ""
|
||||
if "XAU PULLBACK" in fb or "XAU PULLBACK" in fbe:
|
||||
return "XAU_PB"
|
||||
if "XAU CHART PATTERN" in fb or "XAU CHART PATTERN" in fbe:
|
||||
return "XAU_CP"
|
||||
if "XAU STRATEGY" in fb:
|
||||
return "XAU"
|
||||
return "GENERIC"
|
||||
|
||||
|
||||
def scan(provider, symbol, tf) -> SignalReport:
|
||||
confirms = _CONFIRMS[tf]
|
||||
cfg = make_config(tf, confirms)
|
||||
agg = SignalAggregator(cfg)
|
||||
agg.data_provider = provider
|
||||
dfs = {}
|
||||
for t in [tf] + confirms:
|
||||
dfs[t] = provider.fetch_ohlcv(symbol, t)
|
||||
return agg.analyze_from_dataframes(symbol, dfs, use_news=False)
|
||||
|
||||
|
||||
def extract(provider, report) -> dict:
|
||||
s = report.trade_setup
|
||||
return {
|
||||
"direction": report.final_signal.value,
|
||||
"grade": report.signal_quality,
|
||||
"setup": 1 if s is not None else 0,
|
||||
"tag": tag_of(report) if s else "",
|
||||
"entry": round(s.entry_price, 3) if s else "",
|
||||
"sl": round(s.stop_loss, 3) if s else "",
|
||||
"tp1": round(s.take_profit_1, 3) if s else "",
|
||||
"price": round(report.current_price, 3),
|
||||
"spread": round(provider.get_spread(XAUUSD), 3),
|
||||
"warnings": " | ".join(report.warnings)[:200],
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--a", default="oanda")
|
||||
ap.add_argument("--b", default="tiingo")
|
||||
ap.add_argument("--tfs", default="M5,M15,H1")
|
||||
ap.add_argument("--out", default="logs/shadow_scan.csv")
|
||||
ap.add_argument("--once", action="store_true",
|
||||
help="one immediate scan cycle, then exit")
|
||||
args = ap.parse_args()
|
||||
|
||||
tfs = [_TF_LOOKUP[s.strip()] for s in args.tfs.split(",")]
|
||||
pa = make_data_provider(PlatformConfig(), args.a)
|
||||
pb = make_data_provider(PlatformConfig(), args.b)
|
||||
|
||||
out_path = Path(args.out)
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
new_file = not out_path.exists()
|
||||
tally = {"cycles": 0, "dir": 0, "grade": 0, "setup": 0}
|
||||
|
||||
with open(out_path, "a", newline="") as fh:
|
||||
w = csv.writer(fh)
|
||||
if new_file:
|
||||
w.writerow(["ts", "tf"] + [f"a_{x}" for x in FIELDS + ["price", "spread", "warnings"]]
|
||||
+ [f"b_{x}" for x in FIELDS + ["price", "spread", "warnings"]]
|
||||
+ ["dir_agree", "grade_agree", "setup_agree"])
|
||||
|
||||
while True:
|
||||
if args.once:
|
||||
wake = datetime.now(timezone.utc)
|
||||
else:
|
||||
wait = seconds_until_boundary(5, buffer=10)
|
||||
time.sleep(wait)
|
||||
wake = datetime.now(timezone.utc)
|
||||
|
||||
if _is_forex_weekend():
|
||||
if args.once:
|
||||
print("weekend — nothing to scan")
|
||||
return
|
||||
continue
|
||||
|
||||
for tf in due_timeframes(wake, tfs):
|
||||
try:
|
||||
ra = scan(pa, XAUUSD, tf)
|
||||
except Exception as e:
|
||||
print(f"[{wake:%H:%M:%S}] {args.a} {tf.value} FAILED: {e}")
|
||||
continue
|
||||
try:
|
||||
rb = scan(pb, XAUUSD, tf)
|
||||
except Exception as e:
|
||||
print(f"[{wake:%H:%M:%S}] {args.b} {tf.value} FAILED: {e}")
|
||||
continue
|
||||
|
||||
ea, eb = extract(pa, ra), extract(pb, rb)
|
||||
da = ea["direction"] == eb["direction"]
|
||||
ga = ea["grade"] == eb["grade"]
|
||||
sa = ea["setup"] == eb["setup"]
|
||||
tally["cycles"] += 1
|
||||
tally["dir"] += da; tally["grade"] += ga; tally["setup"] += sa
|
||||
|
||||
row = [wake.isoformat(), tf.value]
|
||||
row += [ea[x] for x in FIELDS + ["price", "spread", "warnings"]]
|
||||
row += [eb[x] for x in FIELDS + ["price", "spread", "warnings"]]
|
||||
row += [int(da), int(ga), int(sa)]
|
||||
w.writerow(row)
|
||||
fh.flush()
|
||||
|
||||
print(f"[{wake:%Y-%m-%d %H:%M:%S}] {tf.value:<5} "
|
||||
f"{args.a}={ea['direction']:+d}/{ea['grade'] or '-':>2} "
|
||||
f"{args.b}={eb['direction']:+d}/{eb['grade'] or '-':>2} "
|
||||
f"dir={'OK' if da else 'FLIP'} "
|
||||
f"| Δprice {abs(ea['price']-eb['price']):.2f} "
|
||||
f"| agree dir {tally['dir']}/{tally['cycles']} "
|
||||
f"grade {tally['grade']}/{tally['cycles']} "
|
||||
f"setup {tally['setup']}/{tally['cycles']}")
|
||||
|
||||
if args.once:
|
||||
print("shadow scan complete (--once)")
|
||||
return
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user