Cleanup: remove dead code, stale docs, and all non-main branches
- Delete back-compat shims data_provider.py / oanda_provider.py; test_all.py now imports providers.* directly (ForexDataProvider name retired; the two shim-only tests removed, 268 -> 266 tests) - Delete backtest/research/calendar_store.py (no importer, no entry point); config.py audit comment reworded - Delete executed plan doc (2026-07-21 centralized-config) and the two unbuildable guide PDFs; HTML guides remain the source of truth - OANDA_MIGRATION_ROADMAP.md status header corrected: phases 1-2 shipped 2026-07-28, feed is live on Oanda; phases 3-6 still unbuilt - Branches deleted locally + on both remotes: oanda_demo (merged), web-dashboard, eurusd-modular, reversal-exit-alert (owner-approved permanent deletion); July stash dropped Claude-Session: https://claude.ai/code/session_019zR58p8ACW1oJ5QXVUZ5es
This commit is contained in:
Harwinder Sekhon
+70
-101
@@ -214,29 +214,29 @@ def _():
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# MODULE 2: data_provider.py
|
||||
# MODULE 2: providers/twelve_data.py
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
print("\n═══ Module 2: data_provider.py ═══")
|
||||
print("\n═══ Module 2: providers/twelve_data.py ═══")
|
||||
|
||||
from data_provider import ForexDataProvider, DataIntegrityError
|
||||
from providers import TwelveDataProvider, DataIntegrityError
|
||||
|
||||
|
||||
@test("ForexDataProvider._to_td_symbol conversion")
|
||||
@test("TwelveDataProvider._to_td_symbol conversion")
|
||||
def _():
|
||||
assert ForexDataProvider._to_td_symbol("XAU/USD") == "XAU/USD"
|
||||
assert TwelveDataProvider._to_td_symbol("XAU/USD") == "XAU/USD"
|
||||
|
||||
|
||||
@test("ForexDataProvider TD_INTERVALS complete mapping")
|
||||
@test("TwelveDataProvider TD_INTERVALS complete mapping")
|
||||
def _():
|
||||
dp = ForexDataProvider(PlatformConfig())
|
||||
dp = TwelveDataProvider(PlatformConfig())
|
||||
for tf in Timeframe:
|
||||
assert tf in dp.TD_INTERVALS, f"Missing TD_INTERVALS mapping for {tf}"
|
||||
|
||||
|
||||
@test("ForexDataProvider BARS_PER_DAY mapping")
|
||||
@test("TwelveDataProvider BARS_PER_DAY mapping")
|
||||
def _():
|
||||
dp = ForexDataProvider(PlatformConfig())
|
||||
dp = TwelveDataProvider(PlatformConfig())
|
||||
for tf in Timeframe:
|
||||
assert tf in dp.BARS_PER_DAY, f"Missing BARS_PER_DAY for {tf}"
|
||||
assert dp.BARS_PER_DAY[Timeframe.H1] == 24
|
||||
@@ -244,9 +244,9 @@ def _():
|
||||
assert dp.BARS_PER_DAY[Timeframe.D1] == 1
|
||||
|
||||
|
||||
@test("ForexDataProvider CANDLE_SECONDS mapping")
|
||||
@test("TwelveDataProvider CANDLE_SECONDS mapping")
|
||||
def _():
|
||||
dp = ForexDataProvider(PlatformConfig())
|
||||
dp = TwelveDataProvider(PlatformConfig())
|
||||
for tf in Timeframe:
|
||||
assert tf in dp.CANDLE_SECONDS, f"Missing CANDLE_SECONDS for {tf}"
|
||||
assert dp.CANDLE_SECONDS[Timeframe.M1] == 60
|
||||
@@ -254,9 +254,9 @@ def _():
|
||||
assert dp.CANDLE_SECONDS[Timeframe.D1] == 86400
|
||||
|
||||
|
||||
@test("ForexDataProvider._validate_and_clean basic valid data")
|
||||
@test("TwelveDataProvider._validate_and_clean basic valid data")
|
||||
def _():
|
||||
dp = ForexDataProvider(PlatformConfig())
|
||||
dp = TwelveDataProvider(PlatformConfig())
|
||||
df = make_ohlcv(300)
|
||||
result = dp._validate_and_clean(df, "XAU/USD", Timeframe.H1)
|
||||
assert len(result) > 0
|
||||
@@ -264,9 +264,9 @@ def _():
|
||||
assert not result.index.has_duplicates
|
||||
|
||||
|
||||
@test("ForexDataProvider._validate_and_clean empty df raises error")
|
||||
@test("TwelveDataProvider._validate_and_clean empty df raises error")
|
||||
def _():
|
||||
dp = ForexDataProvider(PlatformConfig())
|
||||
dp = TwelveDataProvider(PlatformConfig())
|
||||
try:
|
||||
dp._validate_and_clean(pd.DataFrame(), "XAU/USD", Timeframe.H1)
|
||||
assert False, "Should have raised DataIntegrityError"
|
||||
@@ -274,9 +274,9 @@ def _():
|
||||
pass
|
||||
|
||||
|
||||
@test("ForexDataProvider._validate_and_clean None df raises error")
|
||||
@test("TwelveDataProvider._validate_and_clean None df raises error")
|
||||
def _():
|
||||
dp = ForexDataProvider(PlatformConfig())
|
||||
dp = TwelveDataProvider(PlatformConfig())
|
||||
try:
|
||||
dp._validate_and_clean(None, "XAU/USD", Timeframe.H1)
|
||||
assert False, "Should have raised DataIntegrityError"
|
||||
@@ -284,18 +284,18 @@ def _():
|
||||
pass
|
||||
|
||||
|
||||
@test("ForexDataProvider._validate_and_clean fixes OHLC inconsistencies")
|
||||
@test("TwelveDataProvider._validate_and_clean fixes OHLC inconsistencies")
|
||||
def _():
|
||||
dp = ForexDataProvider(PlatformConfig())
|
||||
dp = TwelveDataProvider(PlatformConfig())
|
||||
df = make_ohlcv(300)
|
||||
df.iloc[10, df.columns.get_loc("High")] = df.iloc[10]["Low"] - 1
|
||||
result = dp._validate_and_clean(df, "XAU/USD", Timeframe.H1)
|
||||
assert (result["High"] >= result["Low"]).all()
|
||||
|
||||
|
||||
@test("ForexDataProvider._validate_and_clean drops future timestamps")
|
||||
@test("TwelveDataProvider._validate_and_clean drops future timestamps")
|
||||
def _():
|
||||
dp = ForexDataProvider(PlatformConfig())
|
||||
dp = TwelveDataProvider(PlatformConfig())
|
||||
df = make_ohlcv(300)
|
||||
future = pd.Timestamp.now(tz="UTC") + pd.Timedelta(hours=10)
|
||||
df.loc[future] = df.iloc[-1]
|
||||
@@ -303,9 +303,9 @@ def _():
|
||||
assert (result.index <= pd.Timestamp.utcnow() + pd.Timedelta(minutes=5)).all()
|
||||
|
||||
|
||||
@test("ForexDataProvider._validate_and_clean removes duplicates")
|
||||
@test("TwelveDataProvider._validate_and_clean removes duplicates")
|
||||
def _():
|
||||
dp = ForexDataProvider(PlatformConfig())
|
||||
dp = TwelveDataProvider(PlatformConfig())
|
||||
df = make_ohlcv(300)
|
||||
dup = pd.concat([df, df.iloc[-5:]])
|
||||
assert dup.index.has_duplicates, "Setup: should have duplicates"
|
||||
@@ -313,9 +313,9 @@ def _():
|
||||
assert not result.index.has_duplicates, "Should remove duplicate indices"
|
||||
|
||||
|
||||
@test("ForexDataProvider._validate_and_clean forward fills NaN (limit 3)")
|
||||
@test("TwelveDataProvider._validate_and_clean forward fills NaN (limit 3)")
|
||||
def _():
|
||||
dp = ForexDataProvider(PlatformConfig())
|
||||
dp = TwelveDataProvider(PlatformConfig())
|
||||
df = make_ohlcv(300)
|
||||
df.iloc[100, df.columns.get_loc("Close")] = np.nan
|
||||
df.iloc[101, df.columns.get_loc("Close")] = np.nan
|
||||
@@ -323,9 +323,9 @@ def _():
|
||||
assert not result["Close"].isna().any()
|
||||
|
||||
|
||||
@test("ForexDataProvider._is_data_fresh weekend override")
|
||||
@test("TwelveDataProvider._is_data_fresh weekend override")
|
||||
def _():
|
||||
dp = ForexDataProvider(PlatformConfig())
|
||||
dp = TwelveDataProvider(PlatformConfig())
|
||||
# Create data that ends on a Friday, call _is_data_fresh on a Saturday
|
||||
friday = pd.Timestamp("2025-05-09 16:00:00", tz="UTC")
|
||||
old_dates = pd.date_range(end=friday, periods=300, freq="h")
|
||||
@@ -339,16 +339,16 @@ def _():
|
||||
assert result is True, "Weekend data should always be considered fresh"
|
||||
|
||||
|
||||
@test("ForexDataProvider._is_data_fresh returns False for empty df")
|
||||
@test("TwelveDataProvider._is_data_fresh returns False for empty df")
|
||||
def _():
|
||||
dp = ForexDataProvider(PlatformConfig())
|
||||
dp = TwelveDataProvider(PlatformConfig())
|
||||
assert dp._is_data_fresh(pd.DataFrame(), "XAU/USD", Timeframe.H1) is False
|
||||
assert dp._is_data_fresh(None, "XAU/USD", Timeframe.H1) is False
|
||||
|
||||
|
||||
@test("ForexDataProvider.clear_cache clears all entries")
|
||||
@test("TwelveDataProvider.clear_cache clears all entries")
|
||||
def _():
|
||||
dp = ForexDataProvider(PlatformConfig())
|
||||
dp = TwelveDataProvider(PlatformConfig())
|
||||
dp._cache["test_key"] = make_ohlcv(10)
|
||||
dp._cache_timestamps["test_key"] = time.time()
|
||||
count = dp.clear_cache()
|
||||
@@ -357,9 +357,9 @@ def _():
|
||||
assert len(dp._cache_timestamps) == 0
|
||||
|
||||
|
||||
@test("ForexDataProvider.get_pair_info returns correct info")
|
||||
@test("TwelveDataProvider.get_pair_info returns correct info")
|
||||
def _():
|
||||
dp = ForexDataProvider(PlatformConfig())
|
||||
dp = TwelveDataProvider(PlatformConfig())
|
||||
info = dp.get_pair_info(XAUUSD)
|
||||
assert info["symbol"] == XAUUSD
|
||||
assert info["display"] == "XAU/USD"
|
||||
@@ -369,18 +369,18 @@ def _():
|
||||
assert info_unknown["display"] == "XAU/USD"
|
||||
|
||||
|
||||
@test("ForexDataProvider._throttle rate limiting logic")
|
||||
@test("TwelveDataProvider._throttle rate limiting logic")
|
||||
def _():
|
||||
dp = ForexDataProvider(PlatformConfig())
|
||||
dp = TwelveDataProvider(PlatformConfig())
|
||||
dp._request_times.clear()
|
||||
for _ in range(10):
|
||||
dp._throttle(credits=1)
|
||||
assert len(dp._request_times) == 10
|
||||
|
||||
|
||||
@test("ForexDataProvider caching - in-memory cache returns same data")
|
||||
@test("TwelveDataProvider caching - in-memory cache returns same data")
|
||||
def _():
|
||||
dp = ForexDataProvider(PlatformConfig())
|
||||
dp = TwelveDataProvider(PlatformConfig())
|
||||
df = make_ohlcv(300)
|
||||
dp._cache["XAU/USD_1h"] = df
|
||||
dp._cache_timestamps["XAU/USD_1h"] = time.time()
|
||||
@@ -2267,7 +2267,7 @@ def _():
|
||||
|
||||
@test("Outlier filter drops the spike bar but keeps the snap-back bar")
|
||||
def _():
|
||||
provider = ForexDataProvider(PlatformConfig())
|
||||
provider = TwelveDataProvider(PlatformConfig())
|
||||
df = make_ohlcv(n=300, seed=3)
|
||||
k = 150
|
||||
spike_ts, after_ts = df.index[k], df.index[k + 1]
|
||||
@@ -2288,7 +2288,7 @@ def _():
|
||||
"Close": v, "Volume": 1000.0}, index=idx)
|
||||
|
||||
for _attempt in range(2):
|
||||
provider = ForexDataProvider(PlatformConfig())
|
||||
provider = TwelveDataProvider(PlatformConfig())
|
||||
started = pd.Timestamp.now(tz="UTC").floor("15min")
|
||||
last_closed = started - pd.Timedelta(minutes=15)
|
||||
stale = bars_ending(last_closed - pd.Timedelta(minutes=15))
|
||||
@@ -2323,13 +2323,13 @@ def _():
|
||||
"Close": v, "Volume": 1000.0}, index=idx)
|
||||
|
||||
for _attempt in range(2):
|
||||
provider = ForexDataProvider(PlatformConfig())
|
||||
provider = TwelveDataProvider(PlatformConfig())
|
||||
started = pd.Timestamp.now(tz="UTC").floor("15min")
|
||||
last_closed = started - pd.Timedelta(minutes=15)
|
||||
lagging = bars_ending(last_closed - pd.Timedelta(minutes=15))
|
||||
complete = bars_ending(last_closed)
|
||||
with patch("providers.base._is_forex_weekend", return_value=False), \
|
||||
patch.object(ForexDataProvider, "FRESH_FETCH_RETRY_SECONDS", 0.0):
|
||||
patch.object(TwelveDataProvider, "FRESH_FETCH_RETRY_SECONDS", 0.0):
|
||||
# (a) lagging first response → one retry returns the closed bar.
|
||||
with patch.object(provider, "_fetch_twelve_data",
|
||||
side_effect=[lagging.copy(), complete.copy()]) as mf:
|
||||
@@ -2966,30 +2966,30 @@ def _():
|
||||
f"example config.json must be byte-identical to defaults on {tf}"
|
||||
|
||||
|
||||
@test("data_provider: api key resolves env → config.json (no code default)")
|
||||
@test("providers: api key resolves env → config.json (no code default)")
|
||||
def _():
|
||||
from data_provider import ForexDataProvider
|
||||
from providers import TwelveDataProvider
|
||||
app = AppConfig(provider_settings={"twelve_data": {"api_key": "json-key"}})
|
||||
set_app_config(app)
|
||||
try:
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
dp = ForexDataProvider(PlatformConfig())
|
||||
dp = TwelveDataProvider(PlatformConfig())
|
||||
assert dp._api_key == "json-key"
|
||||
with patch.dict(os.environ, {"TWELVE_DATA_API_KEY": "env-key"}):
|
||||
dp = ForexDataProvider(PlatformConfig())
|
||||
dp = TwelveDataProvider(PlatformConfig())
|
||||
assert dp._api_key == "env-key"
|
||||
set_app_config(AppConfig()) # no key anywhere
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
dp = ForexDataProvider(PlatformConfig())
|
||||
dp = TwelveDataProvider(PlatformConfig())
|
||||
assert dp._api_key is None and dp._td is None
|
||||
finally:
|
||||
set_app_config(_TEST_APP)
|
||||
|
||||
|
||||
@test("data_provider: make_data_provider dispatches on active provider")
|
||||
@test("providers: make_data_provider dispatches on active provider")
|
||||
def _():
|
||||
from data_provider import ForexDataProvider, make_data_provider
|
||||
assert isinstance(make_data_provider(PlatformConfig()), ForexDataProvider)
|
||||
from providers import TwelveDataProvider, make_data_provider
|
||||
assert isinstance(make_data_provider(PlatformConfig()), TwelveDataProvider)
|
||||
set_app_config(AppConfig(active_provider="polygon",
|
||||
provider_settings={"polygon": {"api_key": "k"}}))
|
||||
try:
|
||||
@@ -3132,30 +3132,12 @@ def _():
|
||||
|
||||
print("\n═══ Module 15: Oanda provider / executor / daemon ═══")
|
||||
|
||||
import oanda_provider
|
||||
import providers.oanda
|
||||
import oanda_executor
|
||||
import oanda_daemon
|
||||
|
||||
|
||||
@test("oanda_provider: _is_forex_weekend matches data_provider's")
|
||||
def _():
|
||||
from data_provider import _is_forex_weekend as dp_weekend
|
||||
from oanda_provider import _is_forex_weekend as oa_weekend
|
||||
import pandas as pd
|
||||
for ts_str in [
|
||||
"2026-07-17 12:00", # Friday noon
|
||||
"2026-07-17 22:30", # Friday after close
|
||||
"2026-07-18 15:00", # Saturday
|
||||
"2026-07-19 12:00", # Sunday before open
|
||||
"2026-07-19 22:30", # Sunday after open
|
||||
"2026-07-20 08:00", # Monday
|
||||
]:
|
||||
ts = pd.Timestamp(ts_str, tz="UTC")
|
||||
assert dp_weekend(ts) == oa_weekend(ts), \
|
||||
f"Weekend check mismatch at {ts_str}"
|
||||
|
||||
|
||||
@test("oanda_provider: OandaDataProvider parses Oanda candle JSON into correct DataFrame")
|
||||
@test("providers.oanda: OandaDataProvider parses Oanda candle JSON into correct DataFrame")
|
||||
def _():
|
||||
sample_candles = [
|
||||
{
|
||||
@@ -3190,8 +3172,8 @@ def _():
|
||||
"raise_for_status": lambda self: None,
|
||||
})()
|
||||
|
||||
provider2 = oanda_provider.OandaDataProvider.__new__(
|
||||
oanda_provider.OandaDataProvider)
|
||||
provider2 = providers.oanda.OandaDataProvider.__new__(
|
||||
providers.oanda.OandaDataProvider)
|
||||
provider2.config = PlatformConfig()
|
||||
provider2._base_url = "https://test"
|
||||
provider2._cache = {}
|
||||
@@ -3219,25 +3201,25 @@ def _():
|
||||
assert abs(spread - 0.30) < 0.01, f"Expected spread ~0.30, got {spread}"
|
||||
|
||||
|
||||
@test("oanda_provider: instrument name mapping")
|
||||
@test("providers.oanda: instrument name mapping")
|
||||
def _():
|
||||
provider = oanda_provider.OandaDataProvider.__new__(
|
||||
oanda_provider.OandaDataProvider)
|
||||
provider = providers.oanda.OandaDataProvider.__new__(
|
||||
providers.oanda.OandaDataProvider)
|
||||
assert provider._to_oanda_instrument("XAU/USD") == "XAU_USD"
|
||||
assert provider._to_oanda_instrument("EUR/USD") == "EUR_USD"
|
||||
|
||||
|
||||
@test("oanda_provider: granularity mapping covers all scan timeframes")
|
||||
@test("providers.oanda: granularity mapping covers all scan timeframes")
|
||||
def _():
|
||||
for tf in [Timeframe.M5, Timeframe.M15, Timeframe.H1, Timeframe.H4]:
|
||||
assert tf in oanda_provider.OandaDataProvider.OANDA_GRANULARITY, \
|
||||
assert tf in providers.oanda.OandaDataProvider.OANDA_GRANULARITY, \
|
||||
f"{tf} missing from OANDA_GRANULARITY"
|
||||
|
||||
|
||||
@test("oanda_provider: get_spread returns default when no cache")
|
||||
@test("providers.oanda: get_spread returns default when no cache")
|
||||
def _():
|
||||
provider = oanda_provider.OandaDataProvider.__new__(
|
||||
oanda_provider.OandaDataProvider)
|
||||
provider = providers.oanda.OandaDataProvider.__new__(
|
||||
providers.oanda.OandaDataProvider)
|
||||
provider._spread_cache = {}
|
||||
spread = provider.get_spread("XAU/USD")
|
||||
assert spread == 0.30, f"Expected default spread 0.30, got {spread}"
|
||||
@@ -3820,15 +3802,15 @@ def _():
|
||||
assert "oanda.com" in url, f"Expected Oanda URL, got {url}"
|
||||
|
||||
|
||||
@test("oanda_provider: HTTP error surfaces status code and body")
|
||||
@test("providers.oanda: HTTP error surfaces status code and body")
|
||||
def _():
|
||||
mock_resp = type("MockResp", (), {
|
||||
"status_code": 401,
|
||||
"text": '{"errorMessage":"Insufficient authorization to perform request"}',
|
||||
})()
|
||||
|
||||
provider = oanda_provider.OandaDataProvider.__new__(
|
||||
oanda_provider.OandaDataProvider)
|
||||
provider = providers.oanda.OandaDataProvider.__new__(
|
||||
providers.oanda.OandaDataProvider)
|
||||
provider._base_url = "https://test"
|
||||
provider._spread_cache = {}
|
||||
provider._session = type("S", (), {
|
||||
@@ -3839,17 +3821,17 @@ def _():
|
||||
try:
|
||||
provider._fetch_oanda_candles("XAU/USD", Timeframe.H1, 300)
|
||||
assert False, "Should have raised OandaDataError"
|
||||
except oanda_provider.OandaDataError as e:
|
||||
except providers.oanda.OandaDataError as e:
|
||||
msg = str(e)
|
||||
assert "401" in msg, f"Status code missing from error: {msg}"
|
||||
assert "Insufficient authorization" in msg, \
|
||||
f"Body detail missing from error: {msg}"
|
||||
|
||||
|
||||
@test("oanda_provider: boundary-retry failure keeps original data")
|
||||
@test("providers.oanda: boundary-retry failure keeps original data")
|
||||
def _():
|
||||
provider = oanda_provider.OandaDataProvider.__new__(
|
||||
oanda_provider.OandaDataProvider)
|
||||
provider = providers.oanda.OandaDataProvider.__new__(
|
||||
providers.oanda.OandaDataProvider)
|
||||
provider.config = PlatformConfig()
|
||||
provider._cache = {}
|
||||
provider._cache_timestamps = {}
|
||||
@@ -3868,7 +3850,7 @@ def _():
|
||||
call_count["n"] += 1
|
||||
if call_count["n"] == 1:
|
||||
return original_df
|
||||
raise oanda_provider.OandaDataError("boom")
|
||||
raise providers.oanda.OandaDataError("boom")
|
||||
|
||||
def _validate_and_clean(df, symbol, timeframe):
|
||||
return df
|
||||
@@ -4029,19 +4011,6 @@ def _():
|
||||
assert TiingoDataProvider.CANDLE_SECONDS is BaseDataProvider.CANDLE_SECONDS
|
||||
|
||||
|
||||
@test("providers: back-compat shims re-export the real classes")
|
||||
def _():
|
||||
import data_provider as dp_shim
|
||||
import oanda_provider as oa_shim
|
||||
assert dp_shim.ForexDataProvider is TwelveDataProvider
|
||||
assert dp_shim.make_data_provider is make_data_provider
|
||||
assert oa_shim.OandaDataProvider is _POanda
|
||||
from providers.base import DataIntegrityError as base_die
|
||||
assert dp_shim.DataIntegrityError is base_die
|
||||
# Provider-specific errors are catchable as the shared type.
|
||||
assert issubclass(oa_shim.OandaDataError, base_die)
|
||||
|
||||
|
||||
@test("providers: fetch_range paginates, dedupes, and trims to the window")
|
||||
def _():
|
||||
class FakeProvider(BaseDataProvider):
|
||||
@@ -4288,7 +4257,7 @@ def _():
|
||||
# must never change the session clock.
|
||||
import signal_aggregator as sa
|
||||
assert sa.BaseDataProvider.CANDLE_SECONDS[Timeframe.M15] == 900
|
||||
assert not hasattr(sa, "ForexDataProvider"), \
|
||||
assert not hasattr(sa, "TwelveDataProvider"), \
|
||||
"aggregator must not depend on a concrete provider class"
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user