harry/ForexBot2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

remove the AI market-outlook feature (delivery-side only; books byte-identical)

Browse Source 
Owner decision 2026-09-15. Gone: market_outlook.py, the post-H1-scan Telegram
outlook DM, main.py --outlook, the outlooks table DDL/migration/writer in
signal_log.py, the market_outlook config block + MARKET_OUTLOOK_ENABLED, and
test Module 26. Last commit containing the feature: 2836024.

- app_config: "market_outlook" stays in _KNOWN_TOP as a TOLERATED, ignored
  legacy key (oanda.account_id precedent) with one warning, so a live
  config.json that still carries the block keeps booting.
- signal_log: existing DBs keep their orphan outlooks table and rows (no
  DROP TABLE); fresh DBs never create it.
- tests: Module 26's 4-thread concurrent legacy-DB migration test (the
  suite's only one) moved to Module 31; new test for the tolerated block.
  519 -> 509 tests, 34 -> 33 modules, 0 failed.
- docs: CLAUDE.md usage/config/file-map lines and a dated state note.

Verification: W2 books (2024-09-05 -> 2026-09-05, $0.65, NumPy CPU, 0 cache
fallbacks) byte-identical before/after — M5 29120a1ee32979431d062f210d41a9d1
(781 / +146.07R / PF 1.52), M15 d185f099482a07f29502cb5a371a9bbf
(931 / +146.14R / PF 1.43).
This commit is contained in:
Harwinder Sekhon
2026-09-15 19:58:47 -04:00
parent 7936c2fe6e
commit c916ecf213
10 changed files with 103 additions and 1047 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test_all.py
+84 -405
View File
@@ -3323,6 +3323,36 @@ def _():
f"example config.json must be byte-identical to defaults on {tf}"
@test("app_config: a legacy market_outlook block is tolerated and ignored")
def _():
# The market-outlook feature was removed 2026-09-15; config.json files
# on the live box still carry its block and must keep loading (the
# oanda.account_id precedent), with one warning so the owner deletes it.
import logging
records = []
class _Grab(logging.Handler):
def emit(self, record):
records.append(record)
log = logging.getLogger("app_config")
handler = _Grab()
log.addHandler(handler)
path = _write_json({
"market_outlook": {"enabled": True, "telegram": True,
"max_per_day": 12, "candles_bars": 20},
})
try:
app = load_app_config(path)
finally:
log.removeHandler(handler)
os.remove(path)
assert not hasattr(app, "market_outlook_settings")
assert any(r.levelno == logging.WARNING
and "market_outlook" in r.getMessage() for r in records), \
"loading a legacy market_outlook block must log one warning"
@test("providers: api key resolves env → config.json (no code default)")
def _():
from providers import TwelveDataProvider
@@ -8528,409 +8558,6 @@ def _():
os.remove(path + suffix)
# ══════════════════════════════════════════════════════════════════════
# MODULE 26: market_outlook.py (advisory hourly market brief)
# ══════════════════════════════════════════════════════════════════════
print("\n═══ Module 26: AI market outlook ═══")
import market_outlook as mo
class _OutlookStub:
available = True
def __init__(self, reply=None):
self.reply = reply or {
"bias": "bullish", "setup_forming": "buy", "confidence": 68,
"watch": "Look for the price to hold above its recent low.",
"note": "The short-term chart is improving while the bigger trend remains supportive.",
}
self.calls = []
def chat_json(self, system_prompt, user_payload):
self.calls.append((system_prompt, user_payload))
if isinstance(self.reply, Exception):
raise self.reply
return self.reply
class _OutlookNews:
def __init__(self, config):
pass
def get_fundamental_context(self):
now = datetime.now(timezone.utc)
return FundamentalContext(
news_headlines=[NewsSentiment(
headline="Gold firms as the dollar weakens", source="test",
timestamp=now, currencies_mentioned=["XAU", "USD"],
sentiment_score=0.3, keywords=[])],
events_today=[EconomicEvent(
time=now - timedelta(minutes=5), currency="USD", impact="high",
event="Inflation", actual="0.2%", forecast="0.3%")],
)
def get_pair_sentiment(self, symbol, context):
return 0.25, "slightly bullish"
def get_week_events(self):
return []
_OUTLOOK_ON = {
"enabled": True,
"telegram": True,
"max_per_day": 12,
"candles_bars": 20,
}
@test("app_config: market_outlook defaults, env override, and validation")
def _():
defaults = load_app_config(_write_json({})).market_outlook_settings
assert defaults == {
"enabled": False, "telegram": False,
"max_per_day": 12, "candles_bars": 20,
}
app = load_app_config(_write_json({"market_outlook": _OUTLOOK_ON}))
assert app.market_outlook_settings == _OUTLOOK_ON
old = os.environ.get("MARKET_OUTLOOK_ENABLED")
try:
os.environ["MARKET_OUTLOOK_ENABLED"] = "false"
assert app_config.market_outlook_enabled(app) is False
os.environ["MARKET_OUTLOOK_ENABLED"] = "true"
assert app_config.market_outlook_enabled(app) is True
finally:
if old is None:
os.environ.pop("MARKET_OUTLOOK_ENABLED", None)
else:
os.environ["MARKET_OUTLOOK_ENABLED"] = old
for bad in (
{"market_outlook": {"enabld": True}},
{"market_outlook": {"enabled": "yes"}},
{"market_outlook": {"telegram": 1}},
{"market_outlook": {"max_per_day": 0}},
{"market_outlook": {"max_per_day": True}},
{"market_outlook": {"candles_bars": 2.5}},
):
try:
load_app_config(_write_json(bad))
assert False, f"should have rejected {bad}"
except AppConfigError:
pass
@test("signal_log: market outlook table roundtrip and old-db migration")
def _():
import sqlite3
import signal_log as slog
fd, path = tempfile.mkstemp(suffix=".db")
os.close(fd)
try:
con = sqlite3.connect(path)
# Legacy pre-panel schema: it has a signals table but no outlooks
# table. Keep the index columns from the original schema present.
con.execute(
"CREATE TABLE signals (id INTEGER PRIMARY KEY, logged_at TEXT,"
" symbol TEXT, timeframe TEXT)")
con.execute(
"CREATE TABLE outlooks (id INTEGER PRIMARY KEY, logged_at TEXT,"
" surface TEXT, bias TEXT, setup_forming TEXT, confidence REAL,"
" watch TEXT, note TEXT)")
con.commit()
con.close()
barrier = threading.Barrier(4)
loggers = []
errors = []
def open_logger():
try:
barrier.wait()
loggers.append(slog.SignalLogger(path))
except Exception as exc:
errors.append(exc)
threads = [threading.Thread(target=open_logger) for _ in range(4)]
for thread in threads:
thread.start()
for thread in threads:
thread.join(timeout=5)
assert not errors, f"concurrent additive migration failed: {errors}"
assert len(loggers) == 4
logger = loggers[0]
columns = {
row[1] for row in logger._conn.execute("PRAGMA table_info(signals)")
}
assert {
"indicators_json", "candles_json", "panel_created_at",
"panel_cache_status", "panel_payload_hash",
"advisory_outcome_status", "advisory_r_multiple",
"advisory_outcome_json",
} <= columns
outlook_columns = {
row[1] for row in logger._conn.execute("PRAGMA table_info(outlooks)")
}
assert {"news_json", "indicators_json", "candles_json"} <= \
outlook_columns
result = mo.OutlookResult(
bias="bearish", setup_forming="sell", confidence=71.5,
watch="Wait for price to fail at resistance.",
note="Momentum is fading after a sharp rise.", created_ts=time.time())
payload = {
"news": {"headlines": ["Gold firms"]},
"indicators": {"1h": {"rsi": {"value": 55.0}}},
"market_regime": {"1h": "trending_up"},
"volatility_regime": {"1h": "normal"},
"key_levels": {"1h": {"support": 4400.0}},
"candles": {"1h": {"bars": [[-0.2, 0.1, -0.3, 0.0]]}},
}
rowid = logger.log_outlook(result, surface="test", payload=payload)
for open_instance in loggers:
open_instance.close()
con = sqlite3.connect(path)
row = con.execute(
"SELECT surface, bias, setup_forming, confidence, watch, note, "
"news_json, indicators_json, candles_json "
"FROM outlooks WHERE id=?", (rowid,)).fetchone()
con.close()
assert row[:6] == ("test", "bearish", "sell", 71.5,
"Wait for price to fail at resistance.",
"Momentum is fading after a sharp rise.")
assert json.loads(row[6])["headlines"] == ["Gold firms"]
assert json.loads(row[7])["market_regime"]["1h"] == "trending_up"
assert len(json.loads(row[8])["1h"]["bars"]) == 1
finally:
slog.reset_loggers_for_tests()
os.remove(path)
@test("market_outlook: payload is finite, normalized, and news-safe")
def _():
try:
mo._dp = _FakeDP()
with patch("news_engine.NewsEngine", _OutlookNews):
payload = mo._collect_payload(_OUTLOOK_ON)
assert mo._normalize_candles is ap._normalize_candles
assert set(payload["candles"]) == {"15min", "1h", "4h"}
assert all(len(block["bars"]) == 20
for block in payload["candles"].values())
assert payload["indicators"]["1h"]
assert payload["news"]["headlines"] == ["Gold firms as the dollar weakens"]
json.dumps(payload, sort_keys=True, allow_nan=False)
with patch("news_engine.NewsEngine", side_effect=RuntimeError("down")):
no_news = mo._collect_payload(_OUTLOOK_ON)["news"]
assert no_news == mo._empty_news_payload()
finally:
mo._dp = None
@test("market_outlook: reply validation clamps and rejects malformed JSON")
def _():
validate = mo._validate_outlook
assert validate(None) is None
assert validate({"bias": "up", "setup_forming": "buy", "confidence": 50}) is None
assert validate({"bias": "bullish", "setup_forming": "wait", "confidence": 50}) is None
assert validate({"bias": "bullish", "setup_forming": "buy"}) is None
assert validate({"bias": "bullish", "setup_forming": "buy", "confidence": "hot"}) is None
result = validate({
"bias": "neutral", "setup_forming": "none", "confidence": 150,
"watch": "word " * 30, "note": "note " * 30,
})
assert result.confidence == 100.0
assert len(result.watch.split()) == 25 and len(result.note.split()) == 25
@test("market_outlook: disabled default spends no LLM request")
def _():
original_app = app_config.get_app_config()
old = os.environ.pop("MARKET_OUTLOOK_ENABLED", None)
try:
app_config.set_app_config(AppConfig())
stub = _OutlookStub()
lc._client = stub
mo.reset_outlook_state()
assert mo.maybe_run_outlook("telegram") is None
assert stub.calls == []
finally:
if old is not None:
os.environ["MARKET_OUTLOOK_ENABLED"] = old
lc._client = None
mo.reset_outlook_state()
app_config.set_app_config(original_app)
@test("market_outlook: hourly dedup prevents a second request")
def _():
original_app = app_config.get_app_config()
old = os.environ.pop("MARKET_OUTLOOK_ENABLED", None)
try:
app_config.set_app_config(AppConfig(market_outlook_settings=dict(_OUTLOOK_ON)))
stub = _OutlookStub()
lc._client = stub
mo.reset_outlook_state()
with patch("providers.base._is_forex_weekend", return_value=False), \
patch.object(mo, "_collect_payload", return_value={}), \
patch("signal_log.log_outlook"):
assert mo.maybe_run_outlook("telegram") is not None
assert mo.maybe_run_outlook("telegram") is None
assert len(stub.calls) == 1
finally:
if old is not None:
os.environ["MARKET_OUTLOOK_ENABLED"] = old
lc._client = None
mo.reset_outlook_state()
app_config.set_app_config(original_app)
@test("market_outlook: daily budget blocks force calls after the cap")
def _():
original_app = app_config.get_app_config()
try:
app_config.set_app_config(AppConfig(market_outlook_settings={
**_OUTLOOK_ON, "max_per_day": 1}))
stub = _OutlookStub()
lc._client = stub
mo.reset_outlook_state()
with patch("providers.base._is_forex_weekend", return_value=False), \
patch.object(mo, "_collect_payload", return_value={}), \
patch("signal_log.log_outlook"):
assert mo.maybe_run_outlook("cli", force=True) is not None
assert mo.maybe_run_outlook("cli", force=True) is None
assert len(stub.calls) == 1
finally:
lc._client = None
mo.reset_outlook_state()
app_config.set_app_config(original_app)
@test("market_outlook: LLM and persistence failures remain fail-safe")
def _():
original_app = app_config.get_app_config()
try:
app_config.set_app_config(AppConfig(market_outlook_settings=dict(_OUTLOOK_ON)))
lc._client = _OutlookStub(RuntimeError("LLM down"))
mo.reset_outlook_state()
with patch("providers.base._is_forex_weekend", return_value=False), \
patch.object(mo, "_collect_payload", return_value={}):
assert mo.maybe_run_outlook("telegram") is None
lc._client = _OutlookStub()
mo.reset_outlook_state()
with patch("providers.base._is_forex_weekend", return_value=False), \
patch.object(mo, "_collect_payload", return_value={}), \
patch("signal_log.log_outlook", side_effect=RuntimeError("db locked")):
assert mo.maybe_run_outlook("telegram") is not None
finally:
lc._client = None
mo.reset_outlook_state()
app_config.set_app_config(original_app)
@test("market_outlook: weekend gate spends nothing")
def _():
original_app = app_config.get_app_config()
try:
app_config.set_app_config(AppConfig(market_outlook_settings=dict(_OUTLOOK_ON)))
stub = _OutlookStub()
lc._client = stub
mo.reset_outlook_state()
with patch("providers.base._is_forex_weekend", return_value=True):
assert mo.maybe_run_outlook("telegram") is None
assert stub.calls == []
finally:
lc._client = None
mo.reset_outlook_state()
app_config.set_app_config(original_app)
@test("market_outlook: hour-bucket dedup runs consecutive hours 3,590 s apart")
def _():
original_app = app_config.get_app_config()
old = os.environ.pop("MARKET_OUTLOOK_ENABLED", None)
try:
app_config.set_app_config(AppConfig(market_outlook_settings=dict(_OUTLOOK_ON)))
stub = _OutlookStub()
lc._client = stub
mo.reset_outlook_state()
base = 1_800_000_000 - (1_800_000_000 % 3600) # an exact hour
clock = {"t": base + 25.0} # 10:00:25
with patch("providers.base._is_forex_weekend", return_value=False), \
patch.object(mo, "_collect_payload", return_value={}), \
patch("signal_log.log_outlook"), \
patch.object(mo.time, "time", lambda: clock["t"]):
assert mo.maybe_run_outlook("telegram") is not None
clock["t"] = base + 1800.0 # 10:30 same bucket
assert mo.maybe_run_outlook("telegram") is None
clock["t"] = base + 3600.0 + 15.0 # 11:00:15 — 3,590 s later
assert mo.maybe_run_outlook("telegram") is not None, \
"the old `now - last < 3600` rule dropped every other hour"
assert len(stub.calls) == 2
finally:
if old is not None:
os.environ["MARKET_OUTLOOK_ENABLED"] = old
lc._client = None
mo.reset_outlook_state()
app_config.set_app_config(original_app)
@test("market_outlook: every silent early return leaves a heartbeat log line")
def _():
original_app = app_config.get_app_config()
old = os.environ.pop("MARKET_OUTLOOK_ENABLED", None)
records = []
class _Grab(logging.Handler):
def emit(self, record):
records.append(record.getMessage())
handler = _Grab()
mo.logger.addHandler(handler)
prev_level = mo.logger.level
mo.logger.setLevel(logging.INFO)
try:
app_config.set_app_config(AppConfig(market_outlook_settings={
**_OUTLOOK_ON, "enabled": False}))
mo.reset_outlook_state()
assert mo.maybe_run_outlook("telegram") is None
assert any("skipped (disabled)" in m for m in records), records
records.clear()
app_config.set_app_config(AppConfig(market_outlook_settings=dict(_OUTLOOK_ON)))
with patch("providers.base._is_forex_weekend", return_value=True):
assert mo.maybe_run_outlook("telegram") is None
assert any("skipped (weekend)" in m for m in records), records
records.clear()
# LLM returns nothing → warning, nothing persisted
stub = _OutlookStub(reply=None)
stub.reply = None
lc._client = stub
persisted = []
with patch("providers.base._is_forex_weekend", return_value=False), \
patch.object(mo, "_collect_payload", return_value={}), \
patch("signal_log.log_outlook", persisted.append):
assert mo.maybe_run_outlook("telegram") is None
assert any("no outlook produced" in m for m in records), records
assert persisted == []
finally:
mo.logger.removeHandler(handler)
mo.logger.setLevel(prev_level)
if old is not None:
os.environ["MARKET_OUTLOOK_ENABLED"] = old
lc._client = None
mo.reset_outlook_state()
app_config.set_app_config(original_app)
@test("market_outlook isolation: signal path never imports the advisory module")
def _():
root = os.path.dirname(__file__)
for name in ("engine.py", "signal_aggregator.py"):
with open(os.path.join(root, name), encoding="utf-8") as fh:
assert "market_outlook" not in fh.read(), name
# ══════════════════════════════════════════════════════════════════════
# MODULE 27: signal-log observability (2026-09 audit)
# ══════════════════════════════════════════════════════════════════════
@@ -10536,6 +10163,59 @@ def _():
assert weighted_agg.gate_config_fingerprint() == m5.gate_config_fingerprint()
@test("signal_log: concurrent legacy-DB migration adds the panel/advisory columns")
def _():
# Four loggers open the same pre-panel database at once: the additive
# ALTER TABLE migration must survive the race (_with_lock_retry / WAL).
# Moved from the removed market-outlook module (2026-09-15); it was the
# suite's only concurrent-migration regression.
import sqlite3
import signal_log as slog
fd, path = tempfile.mkstemp(suffix=".db")
os.close(fd)
try:
con = sqlite3.connect(path)
# Legacy pre-panel schema. Keep the index columns from the original
# schema present.
con.execute(
"CREATE TABLE signals (id INTEGER PRIMARY KEY, logged_at TEXT,"
" symbol TEXT, timeframe TEXT)")
con.commit()
con.close()
barrier = threading.Barrier(4)
loggers = []
errors = []
def open_logger():
try:
barrier.wait()
loggers.append(slog.SignalLogger(path))
except Exception as exc:
errors.append(exc)
threads = [threading.Thread(target=open_logger) for _ in range(4)]
for thread in threads:
thread.start()
for thread in threads:
thread.join(timeout=5)
assert not errors, f"concurrent additive migration failed: {errors}"
assert len(loggers) == 4
columns = {
row[1] for row in loggers[0]._conn.execute("PRAGMA table_info(signals)")
}
assert {
"indicators_json", "candles_json", "panel_created_at",
"panel_cache_status", "panel_payload_hash",
"advisory_outcome_status", "advisory_r_multiple",
"advisory_outcome_json",
} <= columns
for open_instance in loggers:
open_instance.close()
finally:
slog.reset_loggers_for_tests()
os.remove(path)
@test("signal_log: migration adds the research columns, scan_outcomes and indexes")
def _():
import sqlite3
@@ -11906,8 +11586,7 @@ def _():
# the backtester.
banned = {"signal_log", "news_engine", "providers", "telegram_bot",
"news_archive", "agent_panel", "violent_watch",
"market_outlook", "gpu_backtest", "sklearn", "scipy",
"lightgbm", "torch"}
"gpu_backtest", "sklearn", "scipy", "lightgbm", "torch"}
path = Path("backtest/replay_record.py")
tree = ast.parse(path.read_text(encoding="utf-8"), str(path))
seen, offenders = set(), []