news: beginner-language summary, 4 h headline window, drop neutral headlines; --news log report
Verdict from the 3-week signal log (backtest/research/news_sentiment_value_2026-09-04.md): news_sentiment is a trailing-24h echo of price (r +0.67 with the past day, +0.03 with the next; most bullish 2 h after the 08-25 top, bearish 44 h later; most bearish at the 09-02 low, bullish 16 h later). Per-headline LLM labels are correct; the 12 h pool and the averaging are what destroy the information. Both gating flags stay False. - news_engine: _LLM_SUMMARY_SYSTEM rewritten for someone who has never traded (no acronyms, two worked examples); summary payload newest-first with the platform's own gold direction; reply cap 300 -> 400 chars. - config: headline_max_age_seconds 43200 -> 14400; RSS items without a parsable pubDate are dropped (they used to be stamped "now" and never aged out). - news_engine.get_fundamental_context drops headlines whose every relevant score is inside +-0.1 (_is_neutral_headline) before the per-currency mean; news_sentiment from this date is a mean over directional <=4 h headlines (log comparisons split here). - static: dashboard copy for the 4 h pool; news_summary.php prompt gets the same style rule. - analyze_signal_log: --news (under --stored-candles) prints sentiment lead/lag with split halves, sentiment at the window high/low with flip lags, and outcome buckets by direction x news x macro stance; LogRow gains news_sentiment/event_risk; _row_index shared. - tests: 406 -> 414 (age window, undated drop, neutral filter, summary prompt/payload/cap, --news helpers, load_rows round-trip); RSS fixtures now carry a pubDate. - docs: user/beginner guides, CLAUDE.md, deploy README. Claude-Session: https://claude.ai/code/session_0169BgFtWjp1SEAa5392Q9jH
This commit is contained in:
Harwinder Sekhon
+264
-5
@@ -4465,6 +4465,120 @@ class _StubLLM:
|
||||
return self.reply
|
||||
|
||||
|
||||
def _m19_pubdate(hours_ago=0.5):
|
||||
"""RFC-2822 pubDate bytes `hours_ago` in the past (RSS fixtures must be
|
||||
dated since 2026-09-04: undated items are dropped)."""
|
||||
from email.utils import format_datetime
|
||||
when = (_dt_m19.datetime.now(_dt_m19.timezone.utc)
|
||||
- _dt_m19.timedelta(hours=hours_ago))
|
||||
return format_datetime(when).encode()
|
||||
|
||||
|
||||
@test("NewsEngine: RSS age window is 4 h; older and undated items are dropped")
|
||||
def _():
|
||||
assert NewsConfig().headline_max_age_seconds == 14400, "4 h (2026-09-04)"
|
||||
cfg = NewsConfig()
|
||||
cfg.rss_feeds = ["https://feed.example/rss"]
|
||||
ne = NewsEngine(cfg)
|
||||
saved = NewsEngine._source_health
|
||||
NewsEngine._source_health = {}
|
||||
rss = (b"<rss><channel>"
|
||||
b"<item><title>Gold surges as Fed signals rate cuts</title>"
|
||||
b"<pubDate>" + _m19_pubdate(3) + b"</pubDate></item>"
|
||||
b"<item><title>Gold slips on strong dollar data</title>"
|
||||
b"<pubDate>" + _m19_pubdate(5) + b"</pubDate></item>"
|
||||
b"<item><title>Dollar steady before Fed minutes</title></item>"
|
||||
b"</channel></rss>")
|
||||
|
||||
class _Sess:
|
||||
def get(self, url, timeout=None):
|
||||
return type("R", (), {"status_code": 200, "content": rss})()
|
||||
|
||||
ne._session = _Sess()
|
||||
try:
|
||||
heads = ne._fetch_news_sentiment_raw()
|
||||
assert [h.headline for h in heads] == \
|
||||
["Gold surges as Fed signals rate cuts"], heads
|
||||
finally:
|
||||
NewsEngine._source_health = saved
|
||||
|
||||
|
||||
@test("NewsEngine: neutral headlines are dropped before the per-currency mean")
|
||||
def _():
|
||||
from news_engine import _is_neutral_headline, NEUTRAL_HEADLINE_THRESHOLD
|
||||
now = _dt_m19.datetime.now(_dt_m19.timezone.utc)
|
||||
|
||||
def head(overall, by, cur):
|
||||
return NewsSentiment(headline="h", source="s", timestamp=now,
|
||||
currencies_mentioned=cur, sentiment_score=overall,
|
||||
keywords=[], sentiment_by_currency=by)
|
||||
|
||||
assert NEUTRAL_HEADLINE_THRESHOLD == 0.1
|
||||
assert _is_neutral_headline(head(0.0, {"XAU": 0.0}, ["XAU"]))
|
||||
assert not _is_neutral_headline(head(0.0, {"XAU": -0.7}, ["XAU"])), \
|
||||
"overall 0.0 but gold -0.7 leans one way"
|
||||
assert _is_neutral_headline(head(float("nan"), {"XAU": float("nan")}, ["XAU"]))
|
||||
assert _is_neutral_headline(head(0.5, {}, [])), "no currencies → neutral"
|
||||
assert _is_neutral_headline(head(0.05, {"USD": 0.05, "XAU": 0.09},
|
||||
["USD", "XAU"]))
|
||||
|
||||
ne = NewsEngine(NewsConfig())
|
||||
ne._fetch_economic_calendar = lambda currencies: []
|
||||
ne._fetch_news_sentiment = lambda: [
|
||||
head(0.0, {"XAU": 0.0}, ["XAU"]), # neutral → dropped
|
||||
head(0.0, {"XAU": -0.7}, ["XAU"]), # gold-bearish → kept
|
||||
head(0.3, {"USD": 0.3}, ["USD"]), # dollar-bullish → kept
|
||||
head(float("nan"), {"USD": float("nan")}, ["USD"]), # unscored → dropped
|
||||
]
|
||||
NewsEngine._llm_summary_cache = (0.0, "")
|
||||
try:
|
||||
lc._client = _StubLLM(None)
|
||||
ctx = ne.get_fundamental_context()
|
||||
finally:
|
||||
lc._client = None
|
||||
NewsEngine._llm_summary_cache = (0.0, "")
|
||||
assert len(ctx.news_headlines) == 2, [h.sentiment_by_currency for h in ctx.news_headlines]
|
||||
assert abs(ctx.news_sentiment["XAU"] - (-0.7)) < 1e-9, \
|
||||
"mean over survivors only (the 0.0 must not dilute it)"
|
||||
assert abs(ctx.news_sentiment["USD"] - 0.3) < 1e-9
|
||||
|
||||
|
||||
@test("NewsEngine: summary prompt is beginner-language; payload newest-first with gold direction; 400-char cap")
|
||||
def _():
|
||||
from news_engine import _LLM_SUMMARY_SYSTEM, _gold_direction_label
|
||||
assert "never traded" in _LLM_SUMMARY_SYSTEM
|
||||
assert "no acronyms" in _LLM_SUMMARY_SYSTEM
|
||||
assert "max 55 words" in _LLM_SUMMARY_SYSTEM
|
||||
assert "the US central bank" in _LLM_SUMMARY_SYSTEM # worked example carries the style
|
||||
now = _dt_m19.datetime.now(_dt_m19.timezone.utc)
|
||||
|
||||
def head(title, ts, by, cur, overall=0.0):
|
||||
return NewsSentiment(headline=title, source="s", timestamp=ts,
|
||||
currencies_mentioned=cur, sentiment_score=overall,
|
||||
keywords=[], sentiment_by_currency=by)
|
||||
|
||||
older = head("old gold story", now - _dt_m19.timedelta(hours=3),
|
||||
{"XAU": 0.6}, ["XAU"])
|
||||
newer = head("dollar jumps", now, {"USD": 0.5}, ["USD"], overall=0.5)
|
||||
assert _gold_direction_label(older) == "up"
|
||||
assert _gold_direction_label(newer) == "down", "dollar-bullish → gold down"
|
||||
ctx = FundamentalContext(news_headlines=[older, newer])
|
||||
NewsEngine._llm_summary_cache = (0.0, "")
|
||||
try:
|
||||
stub = _StubLLM({"summary": "x" * 450})
|
||||
lc._client = stub
|
||||
out = NewsEngine(NewsConfig())._llm_market_summary(ctx)
|
||||
assert len(out) == 400, len(out)
|
||||
sent = stub.last_payload["headlines"]
|
||||
assert [h["headline"] for h in sent] == ["dollar jumps", "old gold story"], \
|
||||
"newest first"
|
||||
assert [h["gold"] for h in sent] == ["down", "up"]
|
||||
assert set(sent[0]) == {"headline", "gold"}
|
||||
finally:
|
||||
NewsEngine._llm_summary_cache = (0.0, "")
|
||||
lc._client = None
|
||||
|
||||
|
||||
@test("NewsEngine: _llm_headline_scores validates, clamps, rejects mismatch")
|
||||
def _():
|
||||
ne = NewsEngine(NewsConfig())
|
||||
@@ -4737,6 +4851,7 @@ def _():
|
||||
NewsEngine._source_health = {}
|
||||
rss = (b"<rss><channel><item>"
|
||||
b"<title>Fed signals rate cuts as dollar slides</title>"
|
||||
b"<pubDate>" + _m19_pubdate() + b"</pubDate>"
|
||||
b"</item></channel></rss>")
|
||||
|
||||
class _Sess:
|
||||
@@ -4771,6 +4886,7 @@ def _():
|
||||
# <source> carrying the real publisher.
|
||||
rss = (b"<rss><channel><item>"
|
||||
b"<title>Gold surges as Fed signals rate cuts - Reuters</title>"
|
||||
b"<pubDate>" + _m19_pubdate() + b"</pubDate>"
|
||||
b"<link>https://news.google.com/rss/articles/CBMiAbC?oc=5</link>"
|
||||
b"<description><a href=\"https://x\">Gold &amp; dollar "
|
||||
b"diverge as the Fed turns dovish.</a></description>"
|
||||
@@ -4802,15 +4918,16 @@ def _():
|
||||
saved = NewsEngine._source_health
|
||||
NewsEngine._source_health = {}
|
||||
long_link = b"https://a.example/" + b"x" * 2100
|
||||
stamp = b"<pubDate>" + _m19_pubdate() + b"</pubDate>"
|
||||
rss = (b"<rss><channel>"
|
||||
# bare item: no link/description/source at all
|
||||
b"<item><title>Dollar steady before Fed minutes</title></item>"
|
||||
b"<item><title>Dollar steady before Fed minutes</title>" + stamp + b"</item>"
|
||||
# javascript: link + description that merely echoes the title
|
||||
b"<item><title>Gold slips on strong dollar data</title>"
|
||||
b"<item><title>Gold slips on strong dollar data</title>" + stamp +
|
||||
b"<link>javascript:alert(1)</link>"
|
||||
b"<description>Gold slips on strong dollar data</description></item>"
|
||||
# over-long link
|
||||
b"<item><title>Fed officials split on rate cuts</title>"
|
||||
b"<item><title>Fed officials split on rate cuts</title>" + stamp +
|
||||
b"<link>" + long_link + b"</link></item>"
|
||||
b"</channel></rss>")
|
||||
|
||||
@@ -7738,7 +7855,7 @@ import analyze_signal_log as asl
|
||||
def _asl_row(id_, logged_at, tf="15min", source="telegram", signal="BUY",
|
||||
quality="B", entry=3400.0, sl=3392.0, strategy="GENERIC",
|
||||
bar_ts=None, action="BUY — Decent setup", fp="cfg1",
|
||||
setup=True, price=None):
|
||||
setup=True, price=None, news_sentiment=None):
|
||||
setup_d = None
|
||||
if setup:
|
||||
setup_d = {"direction": "BUY" if "BUY" in signal else "SELL",
|
||||
@@ -7753,7 +7870,7 @@ def _asl_row(id_, logged_at, tf="15min", source="telegram", signal="BUY",
|
||||
price=price if price is not None else entry,
|
||||
market_regime="trending_up", setup=setup_d, panel_verdict="",
|
||||
panel_confidence=None, action=action, bar_ts=bar_ts,
|
||||
config_fingerprint=fp)
|
||||
config_fingerprint=fp, news_sentiment=news_sentiment)
|
||||
|
||||
|
||||
@test("analyze_signal_log: entry_bar_open floors the log time to the TF boundary")
|
||||
@@ -7930,6 +8047,148 @@ def _():
|
||||
assert by_sig["BUY"] > 0 and by_sig["SELL"] < 0
|
||||
|
||||
|
||||
@test("analyze_signal_log: news_lead_lag separates trailing sentiment from a lead")
|
||||
def _():
|
||||
rng = np.random.default_rng(20260904)
|
||||
n = 200
|
||||
start = pd.Timestamp("2026-08-19 00:00", tz="UTC") # Wednesday
|
||||
opens = 3400.0 + np.cumsum(rng.normal(0, 2.0, n + 1))
|
||||
bars, rows = [], []
|
||||
for i in range(n):
|
||||
o, c = float(opens[i]), float(opens[i + 1])
|
||||
ts = start + pd.Timedelta(hours=i)
|
||||
bars.append({"timestamp": ts.isoformat(), "open": o,
|
||||
"high": max(o, c) + 1.0, "low": min(o, c) - 1.0,
|
||||
"close": c, "volume": 100})
|
||||
# The logged score IS a trailing 24h return, by construction.
|
||||
sent = 0.0 if i < 24 else float(np.sign(opens[i] - opens[i - 24]))
|
||||
rows.append(_asl_row(i + 1, ts.to_pydatetime() + timedelta(seconds=10),
|
||||
tf="1h", setup=False, price=o,
|
||||
news_sentiment=sent))
|
||||
ll = asl.news_lead_lag(rows, {"1h": bars})
|
||||
assert ll["n_hours"] == n
|
||||
assert set(ll["corr_past"]) == set(asl.NEWS_LAGS_H)
|
||||
assert set(ll["corr_fwd"]) == set(asl.NEWS_LAGS_H)
|
||||
assert set(ll["corr_fwd_split"]) == set(asl.NEWS_LAGS_H)
|
||||
assert ll["corr_past"][24] > 0.5, ll["corr_past"]
|
||||
assert abs(ll["corr_fwd"][24]) < 0.3, ll["corr_fwd"]
|
||||
assert set(ll["daily"]) == {"same", "prev", "next"}
|
||||
|
||||
|
||||
@test("analyze_signal_log: news_lead_lag needs 8 joined hours")
|
||||
def _():
|
||||
start = pd.Timestamp("2026-08-19 00:00", tz="UTC")
|
||||
bars = [{"timestamp": (start + pd.Timedelta(hours=i)).isoformat(),
|
||||
"open": 3400.0, "high": 3401.0, "low": 3399.0,
|
||||
"close": 3400.0, "volume": 1} for i in range(4)]
|
||||
rows = [_asl_row(i + 1, (start + pd.Timedelta(hours=i)).to_pydatetime(),
|
||||
tf="1h", setup=False, price=3400.0, news_sentiment=0.2)
|
||||
for i in range(4)]
|
||||
assert asl.news_lead_lag(rows, {"1h": bars}) == {"n_hours": 4}
|
||||
assert asl.news_lead_lag(rows, {}) == {"n_hours": 0}
|
||||
|
||||
|
||||
@test("analyze_signal_log: news_extremes times the first turn call after the high/low")
|
||||
def _():
|
||||
n = 100
|
||||
start = pd.Timestamp("2026-08-19 00:00", tz="UTC")
|
||||
bars, rows = [], []
|
||||
for i in range(n):
|
||||
ts = start + pd.Timedelta(hours=i)
|
||||
bars.append({"timestamp": ts.isoformat(), "open": 3400.0,
|
||||
"high": 3450.0 if i == 30 else 3405.0,
|
||||
"low": 3350.0 if i == 80 else 3395.0,
|
||||
"close": 3400.0, "volume": 100})
|
||||
sent = 0.3 if (i < 36 or i >= 86) else -0.3
|
||||
rows.append(_asl_row(i + 1, ts.to_pydatetime() + timedelta(seconds=10),
|
||||
tf="1h", setup=False, price=3400.0,
|
||||
news_sentiment=sent))
|
||||
ex = asl.news_extremes(rows, {"1h": bars})
|
||||
assert ex["high"]["price"] == 3450.0 and ex["low"]["price"] == 3350.0
|
||||
assert ex["high"]["ts"].startswith("2026-08-20T06:00") # hour 30
|
||||
assert ex["low"]["ts"].startswith("2026-08-22T08:00") # hour 80
|
||||
assert abs(ex["high"]["sentiment"] - 0.3) < 1e-9
|
||||
assert abs(ex["low"]["sentiment"] + 0.3) < 1e-9
|
||||
assert ex["bear_after_high"]["lag_h"] == 6.0
|
||||
assert ex["bull_after_low"]["lag_h"] == 6.0
|
||||
assert ex["bear_after_high"]["price_move"] == -50.0
|
||||
assert ex["bull_after_low"]["price_move"] == 50.0
|
||||
assert asl.news_extremes(rows, {}) == {}
|
||||
|
||||
|
||||
@test("analyze_signal_log: news_outcome_buckets splits R by news bucket x macro stance")
|
||||
def _():
|
||||
import sqlite3
|
||||
t0 = datetime(2026, 8, 19, 12, 0, 10, tzinfo=timezone.utc)
|
||||
rows = [
|
||||
_asl_row(1, t0, signal="BUY", entry=3400.0, sl=3392.0,
|
||||
news_sentiment=0.3),
|
||||
_asl_row(2, t0 + timedelta(hours=3), signal="BUY", entry=3410.0,
|
||||
sl=3402.0, news_sentiment=-0.3),
|
||||
_asl_row(3, t0 + timedelta(hours=6), signal="SELL", entry=3420.0,
|
||||
sl=3428.0, news_sentiment=0.0),
|
||||
]
|
||||
events = asl.cluster_setup_rows(rows)
|
||||
assert len(events) == 3
|
||||
for ev, r in zip(events, (2.0, -1.0, 0.5)):
|
||||
ev.r_headline = r
|
||||
fd, path = tempfile.mkstemp(suffix=".db")
|
||||
os.close(fd)
|
||||
try:
|
||||
con = sqlite3.connect(path)
|
||||
con.execute("CREATE TABLE signals(id INTEGER PRIMARY KEY,"
|
||||
" panel_takes_json TEXT)")
|
||||
con.execute("INSERT INTO signals VALUES (?, ?)",
|
||||
(1, json.dumps([{"persona": "technical",
|
||||
"stance": "oppose"},
|
||||
{"persona": "macro",
|
||||
"stance": "support"}])))
|
||||
con.commit()
|
||||
con.close()
|
||||
buckets = asl.news_outcome_buckets(rows, events, path)
|
||||
finally:
|
||||
os.remove(path)
|
||||
assert sum(b["n"] for b in buckets) == 3
|
||||
by_key = {(b["direction"], b["bucket"], b["macro"]): b for b in buckets}
|
||||
assert set(by_key) == {("BUY", "bull", "support"),
|
||||
("BUY", "bear", "none"),
|
||||
("SELL", "neutral", "none")}
|
||||
assert by_key[("BUY", "bull", "support")]["sum_r"] == 2.0
|
||||
assert by_key[("BUY", "bull", "support")]["win_rate"] == 1.0
|
||||
assert by_key[("BUY", "bear", "none")]["sum_r"] == -1.0
|
||||
assert by_key[("SELL", "neutral", "none")]["sum_r"] == 0.5
|
||||
# print_news must survive every shape it can be handed
|
||||
import contextlib
|
||||
import io as _io
|
||||
buf = _io.StringIO()
|
||||
with contextlib.redirect_stdout(buf):
|
||||
asl.print_news(asl.news_lead_lag(rows, {}), {}, buckets)
|
||||
asl.print_news({"n_hours": 0}, {}, [])
|
||||
out = buf.getvalue()
|
||||
assert "need 8+" in out and "BUY bull support" in out
|
||||
assert "no resolved events to bucket yet" in out
|
||||
|
||||
|
||||
@test("analyze_signal_log: load_rows round-trips news_sentiment and event_risk")
|
||||
def _():
|
||||
import signal_log as slog
|
||||
fd, path = tempfile.mkstemp(suffix=".db")
|
||||
os.close(fd)
|
||||
try:
|
||||
lg = slog.SignalLogger(path)
|
||||
lg.log(_make_log_report(), source="telegram", timeframe="5min")
|
||||
lg.close()
|
||||
rows = asl.load_rows(path)
|
||||
assert len(rows) == 1
|
||||
assert abs(rows[0].news_sentiment - (-0.4)) < 1e-9, rows[0].news_sentiment
|
||||
assert rows[0].event_risk == "caution", rows[0].event_risk
|
||||
finally:
|
||||
slog.reset_loggers_for_tests()
|
||||
for suffix in ("", "-wal", "-shm"):
|
||||
if os.path.exists(path + suffix):
|
||||
os.remove(path + suffix)
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# MODULE 26: market_outlook.py (advisory hourly market brief)
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
Reference in New Issue
Block a user