Fix API base URL, rate-limit paging, background refresh; verify live against official API

- Base URL corrected to https://www.fuel-finder.service.gov.uk (the
  api.fuelfinder.service.gov.uk host in the portal's examples is stale —
  it does not resolve; www.fuel-finder... resolves to CloudFront and is
  what the live spec uses). Verified: /health now reports source=api,
  8,010 stations, 0 failures.
- Fuel Finder is strictly sequential (30 rpm per client, 429 on overlap):
  added FUEL_API_BATCH_SLEEP (default 4.0s) between pages ≈ 15 rpm.
- Stale-cache refresh moved off the request path into a background
  thread — a multi-minute rate-limited sync no longer blows the app's
  5s timeout; stale data is served while it catches up.
- Multiple diesel variants (B7_STANDARD, B7_PREMIUM, B10) now map to
  DIESEL as the MINIMUM price, so a premium price can't inflate the
  cheapest-diesel reference (CSV relay took first-wins; min is safer).
- Added B7_PREMIUM (underscore) to the fuel map — official key.
- scripts/run.sh now prefers the project venv (bare python3 missed
  uvicorn when run outside an activated env).
- Tests updated for min-diesel + B7_PREMIUM; 12/12 pass.

.env.example

@@ -1,9 +1,11 @@
-# Copy to .env and fill in — NEVER commit .env
+# Copy of .env.example with REAL credentials — NEVER commit this file.
+# .gitignore line 1 excludes it; git check-ignore -v .env confirms.
 FUEL_API_CLIENT_ID=
 FUEL_API_CLIENT_SECRET=
-FUEL_API_TOKEN_URL=https://api.fuelfinder.service.gov.uk/api/v1/oauth/generate_access_token
+FUEL_API_BASE=https://www.fuel-finder.service.gov.uk
-FUEL_API_PFS_URL=https://api.fuelfinder.service.gov.uk/api/v1/pfs
+FUEL_API_TOKEN_URL=https://www.fuel-finder.service.gov.uk/api/v1/oauth/generate_access_token
-FUEL_API_PRICES_URL=https://api.fuelfinder.service.gov.uk/api/v1/pfs/fuel-prices
+FUEL_API_PFS_URL=https://www.fuel-finder.service.gov.uk/api/v1/pfs
+FUEL_API_PRICES_URL=https://www.fuel-finder.service.gov.uk/api/v1/pfs/fuel-prices
 FUEL_CACHE_TTL=300
+FUEL_API_BATCH_SLEEP=4.0
 FUEL_RELAY_PORT=8789
-# FUEL_SOURCE=auto   # auto (API if creds, CSV fallback) | csv | api

README.md

@@ -26,16 +26,21 @@ FuelBoard app / widget
 - Token refreshed automatically before expiry (60s buffer); reused across the
   two batched loops (docs: reuse tokens until near expiry).
 - Both endpoints paginated 500 records/page via `batch-number` until a short
-  page is returned.
+  page is returned; pages are spaced `FUEL_API_BATCH_SLEEP` (default 4s) apart
+  to stay inside the documented 30 rpm sequential-only rate limit.
 - Prices arrive as **decimal strings in pence** (`"0120.0000"` = 120.0p); some
   stations report pounds (values < 2.0 are multiplied by 100). Out-of-band
   values are dropped (50–500p sane band) so a pounds-denominated column can't
   poison the nationwide cheapest reference.
-- Fuel types mapped: `E5`/`E10` → same, `B7`/`B7_STANDARD`/`B7S`/`B7P`/`B10` →
+- Fuel types mapped: `E5`/`E10` → same, `B7`/`B7_STANDARD`/`B7S`/`B7P`/
-  `DIESEL`. Others (`HVO`, `SDV`, …) are ignored — the app tracks exactly
+  `B7_PREMIUM`/`B10` → `DIESEL`. Others (`HVO`, `SDV`, …) are ignored — the
-  E5/E10/DIESEL.
+  app tracks exactly E5/E10/DIESEL.
-- Station snapshot cached `FUEL_CACHE_TTL` (default 300s); stale cache is
+- Station snapshot cached `FUEL_CACHE_TTL` (default 300s); a stale cache is
-  served on upstream failure so the app never sees a hard outage.
+  refreshed in the background on the next request and stale data is served in
+  the meantime, so a multi-minute sync never blocks the app's 5s timeout.
+- Base URL is `https://www.fuel-finder.service.gov.uk` (NOT
+  `api.fuelfinder...` — that host doesn't resolve). Overridable per-endpoint
+  via env.
 - No credentials are exposed to clients — the app calls the relay keyless.
 
 ## Setup

app/main.py

@@ -47,19 +47,25 @@ from fastapi import FastAPI, HTTPException, Query
 
 CLIENT_ID = os.environ.get("FUEL_API_CLIENT_ID", "")
 CLIENT_SECRET = os.environ.get("FUEL_API_CLIENT_SECRET", "")
-TOKEN_URL = os.environ.get(
+# Base host for the official Fuel Finder API (note: NOT api.fuelfinder... —
-    "FUEL_API_TOKEN_URL",
+# the real host is www.fuel-finder.service.gov.uk).
-    "https://api.fuelfinder.service.gov.uk/api/v1/oauth/generate_access_token",
+API_BASE = os.environ.get(
+    "FUEL_API_BASE",
+    "https://www.fuel-finder.service.gov.uk",
 )
+TOKEN_URL = os.environ.get("FUEL_API_TOKEN_URL", f"{API_BASE}/api/v1/oauth/generate_access_token")
 # Official Fuel Finder endpoints (batched, 500 records/page).
-PFS_URL = os.environ.get("FUEL_API_PFS_URL", "https://api.fuelfinder.service.gov.uk/api/v1/pfs")
+PFS_URL = os.environ.get("FUEL_API_PFS_URL", f"{API_BASE}/api/v1/pfs")
 PRICES_URL = os.environ.get(
     "FUEL_API_PRICES_URL",
-    "https://api.fuelfinder.service.gov.uk/api/v1/pfs/fuel-prices",
+    f"{API_BASE}/api/v1/pfs/fuel-prices",
 )
 PAGE_SIZE = int(os.environ.get("FUEL_API_PAGE_SIZE", "500"))
 CACHE_TTL = int(os.environ.get("FUEL_CACHE_TTL", "300"))       # seconds (5 min)
 TOKEN_EXPIRY_BUFFER = int(os.environ.get("FUEL_TOKEN_BUFFER", "60"))  # seconds
+# Fuel Finder rate limit: 30 rpm per client, sequential only (429 on overlap).
+# uff.py's default of 4.0s between batch pages ≈ 15 rpm — stay inside.
+BATCH_SLEEP = float(os.environ.get("FUEL_API_BATCH_SLEEP", "4.0"))
 
 # Public full-UK mirror of the Fuel Finder dataset (GitHub Actions, hourly),
 # used as the fallback source when OAuth credentials are not configured so the
@@ -105,6 +111,7 @@ FUEL_MAP = {
     "B7_STANDARD": "DIESEL",
     "B7S": "DIESEL",
     "B7P": "DIESEL",
+    "B7_PREMIUM": "DIESEL",
     "B10": "DIESEL",
 }
 
@@ -197,6 +204,12 @@ def _normalise_api_payload(stations_info: list, fuel_prices: list) -> list:
             price = _parse_price(entry.get("price"))
             if price is None:
                 continue
+            # Multiple diesel variants (B7_STANDARD, B7_PREMIUM, B10) map to
+            # one DIESEL grade — take the MINIMUM so the cheapest-diesel
+            # reference is never inflated by a premium price.
+            if grade == "DIESEL":
+                prices[grade] = min(prices.get(grade, float("inf")), price)
+            else:
                 prices[grade] = price
             updated = entry.get("price_last_updated")
             if updated and (latest_updated is None or updated > latest_updated):
@@ -326,7 +339,12 @@ def _get_token(client: httpx.Client) -> str:
 
 
 def _fetch_all_pages(client: httpx.Client, url: str) -> list:
-    """Fetch every page of a batched Fuel Finder endpoint (500/page)."""
+    """Fetch every page of a batched Fuel Finder endpoint (500/page).
+
+    Fuel Finder is strictly sequential (30 rpm per client): a request sent
+    before the previous one completes gets a 429. We therefore sleep
+    BATCH_SLEEP between pages and never issue a second in-flight request.
+    """
     results = []
     batch = 1
     while True:
@@ -345,6 +363,7 @@ def _fetch_all_pages(client: httpx.Client, url: str) -> list:
         batch += 1
         if batch > 200:  # safety cap (~100k records)
             break
+        time.sleep(BATCH_SLEEP)
     return results
 
 
@@ -371,8 +390,24 @@ def _fetch_api() -> list:
 # Cache refresh
 # ---------------------------------------------------------------------------
 
+import threading
+
+_refresh_lock = threading.Lock()
+
+
 def _refresh_cache() -> None:
-    """Refresh the station cache if stale (or on first run)."""
+    """Refresh the station cache (called synchronously or from a thread)."""
+    with _refresh_lock:
+        _refresh_cache_locked()
+
+
+def _refresh_cache_locked() -> None:
+    """Refresh the station cache if stale (or on first run).
+
+    Callers hold _refresh_lock. Full API sync takes ~1-2 min (17+ pages × 2
+    endpoints × BATCH_SLEEP), so this must NEVER run inline in a request path
+    — see _ensure_fresh_cache().
+    """
     _state["sync_attempts"] += 1
     try:
         if SOURCE == "csv":
@@ -395,6 +430,22 @@ def _refresh_cache() -> None:
         _state["last_error"] = f"{type(exc).__name__}: {exc}"
 
 
+def _ensure_fresh_cache() -> None:
+    """Non-blocking staleness check: serve stale data, refresh in background.
+
+    A full API sync takes minutes (rate-limited batched paging); blocking a
+    request on it would blow past the app/widget's 5-second timeout. We return
+    whatever we have immediately and let the sync catch up on a daemon thread.
+    """
+    if _state["stations"] is None:
+        # First run: block (usually the lifespan warm-up, but guards a race
+        # where a request beats it). CSV fallback is fast; API takes a while.
+        _refresh_cache()
+        return
+    if time.time() - _state["stations_fetched_at"] > CACHE_TTL:
+        threading.Thread(target=_refresh_cache, daemon=True).start()
+
+
 @asynccontextmanager
 async def lifespan(_: FastAPI):
     # Always try to warm the cache. In auto mode with no credentials this
@@ -471,9 +522,9 @@ def stations(
     if _state["stations"] is None:
         raise HTTPException(status_code=502, detail="No data cached yet and upstream fetch failed")
 
-    # Trigger a refresh if stale; keep serving stale cache if it fails.
+    # Refresh in the background if stale; never block a request on a
-    if time.time() - _state["stations_fetched_at"] > CACHE_TTL:
+    # multi-minute rate-limited API sync.
-        _refresh_cache()
+    _ensure_fresh_cache()
 
     rows = []
     for station in _stations_list():

scripts/run.sh

@@ -12,4 +12,9 @@ set -a
 set +a
 
 PORT="${FUEL_RELAY_PORT:-8789}"
+
+# Prefer the project venv so the bare `python3` fallback can't miss uvicorn.
+if [ -x ".venv/bin/python" ]; then
+  exec .venv/bin/python -m uvicorn app.main:app --host 0.0.0.0 --port "$PORT"
+fi
 exec python3 -m uvicorn app.main:app --host 0.0.0.0 --port "$PORT"

tests/test_api_adapter.py

@@ -70,6 +70,7 @@ class TestNormaliseApiPayload:
                 {"price": "137.9000", "fuel_type": "E10", "price_last_updated": "2026-08-12T08:15:23"},
                 {"price": "144.9000", "fuel_type": "E5", "price_last_updated": "2026-08-12T08:15:24"},
                 {"price": "149.9000", "fuel_type": "B7_STANDARD", "price_last_updated": "2026-08-12T08:15:25"},
+                {"price": "151.9000", "fuel_type": "B7_PREMIUM", "price_last_updated": "2026-08-12T08:16:00"},
             ],
         }]
 
@@ -82,7 +83,7 @@ class TestNormaliseApiPayload:
         assert s["lat"] == 53.7265
         assert s["lng"] == -1.8580
         assert s["prices"] == {"E10": 137.9, "E5": 144.9, "DIESEL": 149.9}
-        assert s["price_updated"] == "2026-08-12T08:15:25"
+        assert s["price_updated"] == "2026-08-12T08:16:00"
 
     def test_unpriced_fuel_dropped(self):
         stations_info = [{