Align with Twelve Data API docs and add Telegram command menu

data_provider:
- Switch to Authorization header (recommended over apikey query param)
- Enforce 800 credits/day daily limit with warning at 80%
- Fall back to synthetic data when daily limit is reached
- Fix null OHLC field handling (API can return null for any field)

telegram_bot:
- Register commands with set_my_commands so Telegram shows / menu
- Add /credits command to show daily API credit usage and limits

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

data_provider.py

@@ -60,9 +60,10 @@ class ForexDataProvider:
         Timeframe.W1:  0.143,
     }
 
-    # Free plan: 8 API credits per minute (each symbol = 1 credit)
+    # Free plan: 8 credits/minute, 800 credits/day
     RATE_LIMIT = 8
     RATE_WINDOW = 60.0
+    DAILY_LIMIT = 800
 
     def __init__(self, config: PlatformConfig):
         self.config = config
@@ -72,12 +73,17 @@ class ForexDataProvider:
         )
         self._cache: Dict[str, pd.DataFrame] = {}
         self._cache_timestamps: Dict[str, float] = {}
-        self._request_times: deque = deque()  # timestamps of recent API credits used
+        self._request_times: deque = deque()  # timestamps of recent per-minute credits
+        self._daily_credits: int = 0           # credits used today
+        self._daily_reset: float = time.time() + 86400  # next midnight reset
         self._no_api_data: set = set()  # td_symbols with no values from API (use synthetic)
         self._session = requests.Session()
+        # Docs recommend Authorization header over apikey query param
         self._session.headers.update({
-            "User-Agent": "Mozilla/5.0 (compatible; ForexIntel/1.0)"
+            "User-Agent": "Mozilla/5.0 (compatible; ForexIntel/1.0)",
         })
+        if self._api_key:
+            self._session.headers["Authorization"] = f"apikey {self._api_key}"
 
     @staticmethod
     def _to_td_symbol(symbol: str) -> str:
@@ -199,27 +205,48 @@ class ForexDataProvider:
 
     # ── Private Methods ────────────────────────────────────────────
 
-    def _throttle(self, credits: int = 1) -> None:
+    def _throttle(self, credits: int = 1) -> bool:
         """
-        Block until `credits` API credits are available.
+        Block until `credits` per-minute API credits are available.
+        Returns False if the daily limit is reached (caller should use synthetic data).
 
-        Twelve Data free tier: 8 credits/minute. Each symbol in any request
+        Twelve Data free tier: 8 credits/minute, 800 credits/day.
-        (single or batch) consumes 1 credit. This method enforces the limit
+        Each symbol in any request (single or batch) consumes 1 credit.
-        regardless of whether requests are batched or individual.
         """
+        # Reset daily counter after 24 hours
+        now = time.time()
+        if now >= self._daily_reset:
+            self._daily_credits = 0
+            self._daily_reset = now + 86400
+
+        # Daily limit check
+        if self._daily_credits + credits > self.DAILY_LIMIT:
+            logger.warning(
+                f"Daily API limit reached ({self._daily_credits}/{self.DAILY_LIMIT} credits) "
+                "— using cached/synthetic data for remaining requests today."
+            )
+            return False
+
+        # Warn when approaching daily limit
+        if self._daily_credits + credits > self.DAILY_LIMIT * 0.8:
+            logger.warning(
+                f"Approaching daily API limit: {self._daily_credits}/{self.DAILY_LIMIT} credits used."
+            )
+
+        # Per-minute rate limit
         while True:
             now = time.time()
-            # Evict credits older than the rate window
             while self._request_times and now - self._request_times[0] >= self.RATE_WINDOW:
                 self._request_times.popleft()
 
             available = self.RATE_LIMIT - len(self._request_times)
             if available >= credits:
+                ts = time.time()
                 for _ in range(credits):
-                    self._request_times.append(time.time())
+                    self._request_times.append(ts)
-                return
+                self._daily_credits += credits
+                return True
 
-            # Wait until the oldest credit expires
             wait = (self._request_times[0] + self.RATE_WINDOW) - now + 0.2
             logger.info(
                 f"Rate limit: {len(self._request_times)}/{self.RATE_LIMIT} credits used "
@@ -246,7 +273,8 @@ class ForexDataProvider:
         outputsize = max(outputsize, 300)
 
         # Each symbol in a batch request costs 1 credit
-        self._throttle(credits=len(symbols))
+        if not self._throttle(credits=len(symbols)):
+            return  # daily limit reached — callers will use synthetic data
 
         resp = self._session.get(
             f"{self.TD_BASE_URL}/time_series",
@@ -254,7 +282,6 @@ class ForexDataProvider:
                 "symbol": ",".join(td_symbols),
                 "interval": interval,
                 "outputsize": outputsize,
-                "apikey": self._api_key,
                 "timezone": "UTC",
                 "format": "json",
             },
@@ -283,7 +310,7 @@ class ForexDataProvider:
                 values = sym_data.get("values")
                 if not values:
                     if td_sym not in self._no_api_data:
-                        logger.warning(
+                        logger.info(
                             f"No API data for {td_sym} — using synthetic data"
                         )
                         self._no_api_data.add(td_sym)
@@ -301,13 +328,14 @@ class ForexDataProvider:
                 records = [
                     {
                         "Datetime": pd.Timestamp(v["datetime"], tz="UTC"),
-                        "Open":   float(v["open"]),
+                        "Open":   float(v["open"])   if v.get("open")   is not None else None,
-                        "High":   float(v["high"]),
+                        "High":   float(v["high"])   if v.get("high")   is not None else None,
-                        "Low":    float(v["low"]),
+                        "Low":    float(v["low"])    if v.get("low")    is not None else None,
-                        "Close":  float(v["close"]),
+                        "Close":  float(v["close"])  if v.get("close")  is not None else None,
-                        "Volume": float(v.get("volume") or 0),
+                        "Volume": float(v["volume"]) if v.get("volume") is not None else 0.0,
                     }
                     for v in reversed(values)
+                    if v.get("datetime")
                 ]
                 df = pd.DataFrame(records).set_index("Datetime")
                 df.index.name = "Datetime"
@@ -341,7 +369,8 @@ class ForexDataProvider:
         )
         outputsize = max(outputsize, 300)
 
-        self._throttle(credits=1)
+        if not self._throttle(credits=1):
+            return self._generate_synthetic_data(symbol, timeframe, lookback_days)
 
         try:
             resp = self._session.get(
@@ -350,7 +379,6 @@ class ForexDataProvider:
                     "symbol": td_symbol,
                     "interval": interval,
                     "outputsize": outputsize,
-                    "apikey": self._api_key,
                     "timezone": "UTC",
                     "format": "json",
                 },
@@ -369,16 +397,18 @@ class ForexDataProvider:
                 raise DataIntegrityError(f"Empty response from Twelve Data for {td_symbol}")
 
             # Values arrive newest-first; reverse to chronological order
+            # API may return null for any OHLC field — map to None for pandas NaN handling
             records = [
                 {
                     "Datetime": pd.Timestamp(v["datetime"], tz="UTC"),
-                    "Open":   float(v["open"]),
+                    "Open":   float(v["open"])   if v.get("open")   is not None else None,
-                    "High":   float(v["high"]),
+                    "High":   float(v["high"])   if v.get("high")   is not None else None,
-                    "Low":    float(v["low"]),
+                    "Low":    float(v["low"])    if v.get("low")    is not None else None,
-                    "Close":  float(v["close"]),
+                    "Close":  float(v["close"])  if v.get("close")  is not None else None,
-                    "Volume": float(v.get("volume") or 0),
+                    "Volume": float(v["volume"]) if v.get("volume") is not None else 0.0,
                 }
                 for v in reversed(values)
+                if v.get("datetime")
             ]
 
             df = pd.DataFrame(records).set_index("Datetime")

telegram_bot.py

@@ -27,7 +27,7 @@ from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime, timezone
 from typing import Dict, Optional
 
-from telegram import Update
+from telegram import BotCommand, Update
 from telegram.constants import ParseMode
 from telegram.ext import Application, CallbackContext, CommandHandler, ContextTypes
 
@@ -514,6 +514,29 @@ class ForexTelegramBot:
             job.schedule_removal()
         await update.message.reply_text("❌ Watch alerts disabled.")
 
+    async def cmd_credits(self, update: Update, context: ContextTypes.DEFAULT_TYPE):
+        """Show API credit usage against the free-tier daily limit."""
+        from data_provider import ForexDataProvider
+        config = self._build_config(self._state(update.effective_chat.id))
+        provider = ForexDataProvider(config)
+        used = provider._daily_credits
+        limit = provider.DAILY_LIMIT
+        remaining = limit - used
+        pct = (used / limit) * 100 if limit else 0
+        bar_filled = int(pct / 5)
+        bar = "█" * bar_filled + "░" * (20 - bar_filled)
+        status = "🟢" if pct < 60 else ("🟡" if pct < 80 else "🔴")
+        text = (
+            "<b>📊 API Credit Usage</b>\n\n"
+            f"<code>{bar}</code> {status}\n"
+            f"<b>Used:</b>      {used} / {limit} credits\n"
+            f"<b>Remaining:</b> {remaining} credits\n\n"
+            "<i>Free tier: 8 credits/min, 800 credits/day\n"
+            "Each full scan uses ~21 credits (7 pairs × 3 timeframes)\n"
+            "≈ 38 scans available per day</i>"
+        )
+        await update.message.reply_text(text, parse_mode=ParseMode.HTML)
+
     # ── Watch job ──────────────────────────────────────────────────
 
     async def _watch_job(self, context: CallbackContext):
@@ -548,9 +571,25 @@ class ForexTelegramBot:
                 "  export TELEGRAM_BOT_TOKEN='your_token_here'"
             )
 
+        async def _register_commands(app: Application) -> None:
+            """Register commands with Telegram so they appear in the / menu."""
+            await app.bot.set_my_commands([
+                BotCommand("scan",    "Scan all major pairs"),
+                BotCommand("signals", "Actionable signals only (grade B+)"),
+                BotCommand("pair",    "Detailed pair analysis — /pair EURUSD"),
+                BotCommand("all",     "Scan major + cross pairs"),
+                BotCommand("tf",      "Set timeframe — /tf H4"),
+                BotCommand("watch",   "Enable periodic alerts"),
+                BotCommand("unwatch", "Disable alerts"),
+                BotCommand("credits", "View API credit usage"),
+                BotCommand("status",  "View current settings"),
+                BotCommand("help",    "Full command reference"),
+            ])
+
         app = (
             Application.builder()
             .token(token)
+            .post_init(_register_commands)
             .build()
         )
 
@@ -564,6 +603,7 @@ class ForexTelegramBot:
         app.add_handler(CommandHandler("all",     self.cmd_all))
         app.add_handler(CommandHandler("watch",   self.cmd_watch))
         app.add_handler(CommandHandler("unwatch", self.cmd_unwatch))
+        app.add_handler(CommandHandler("credits", self.cmd_credits))
 
         return app