From bd3d3862ec7f85540f6f4175946073262a6355b7 Mon Sep 17 00:00:00 2001 From: FuelBoard Contributor Date: Wed, 12 Aug 2026 15:50:08 +0100 Subject: [PATCH] Fix API base URL, rate-limit paging, background refresh; verify live against official API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Base URL corrected to https://www.fuel-finder.service.gov.uk (the api.fuelfinder.service.gov.uk host in the portal's examples is stale — it does not resolve; www.fuel-finder... resolves to CloudFront and is what the live spec uses). Verified: /health now reports source=api, 8,010 stations, 0 failures. - Fuel Finder is strictly sequential (30 rpm per client, 429 on overlap): added FUEL_API_BATCH_SLEEP (default 4.0s) between pages ≈ 15 rpm. - Stale-cache refresh moved off the request path into a background thread — a multi-minute rate-limited sync no longer blows the app's 5s timeout; stale data is served while it catches up. - Multiple diesel variants (B7_STANDARD, B7_PREMIUM, B10) now map to DIESEL as the MINIMUM price, so a premium price can't inflate the cheapest-diesel reference (CSV relay took first-wins; min is safer). - Added B7_PREMIUM (underscore) to the fuel map — official key. - scripts/run.sh now prefers the project venv (bare python3 missed uvicorn when run outside an activated env). - Tests updated for min-diesel + B7_PREMIUM; 12/12 pass. --- .env.example | 12 ++++--- README.md | 17 +++++---- app/main.py | 73 +++++++++++++++++++++++++++++++++------ scripts/run.sh | 5 +++ tests/test_api_adapter.py | 3 +- 5 files changed, 87 insertions(+), 23 deletions(-) diff --git a/.env.example b/.env.example index 1a41b63..874fae1 100644 --- a/.env.example +++ b/.env.example @@ -1,9 +1,11 @@ -# Copy to .env and fill in — NEVER commit .env +# Copy of .env.example with REAL credentials — NEVER commit this file. +# .gitignore line 1 excludes it; git check-ignore -v .env confirms. FUEL_API_CLIENT_ID= FUEL_API_CLIENT_SECRET= -FUEL_API_TOKEN_URL=https://api.fuelfinder.service.gov.uk/api/v1/oauth/generate_access_token -FUEL_API_PFS_URL=https://api.fuelfinder.service.gov.uk/api/v1/pfs -FUEL_API_PRICES_URL=https://api.fuelfinder.service.gov.uk/api/v1/pfs/fuel-prices +FUEL_API_BASE=https://www.fuel-finder.service.gov.uk +FUEL_API_TOKEN_URL=https://www.fuel-finder.service.gov.uk/api/v1/oauth/generate_access_token +FUEL_API_PFS_URL=https://www.fuel-finder.service.gov.uk/api/v1/pfs +FUEL_API_PRICES_URL=https://www.fuel-finder.service.gov.uk/api/v1/pfs/fuel-prices FUEL_CACHE_TTL=300 +FUEL_API_BATCH_SLEEP=4.0 FUEL_RELAY_PORT=8789 -# FUEL_SOURCE=auto # auto (API if creds, CSV fallback) | csv | api diff --git a/README.md b/README.md index 9956652..90c2b8e 100644 --- a/README.md +++ b/README.md @@ -26,16 +26,21 @@ FuelBoard app / widget - Token refreshed automatically before expiry (60s buffer); reused across the two batched loops (docs: reuse tokens until near expiry). - Both endpoints paginated 500 records/page via `batch-number` until a short - page is returned. + page is returned; pages are spaced `FUEL_API_BATCH_SLEEP` (default 4s) apart + to stay inside the documented 30 rpm sequential-only rate limit. - Prices arrive as **decimal strings in pence** (`"0120.0000"` = 120.0p); some stations report pounds (values < 2.0 are multiplied by 100). Out-of-band values are dropped (50–500p sane band) so a pounds-denominated column can't poison the nationwide cheapest reference. -- Fuel types mapped: `E5`/`E10` → same, `B7`/`B7_STANDARD`/`B7S`/`B7P`/`B10` → - `DIESEL`. Others (`HVO`, `SDV`, …) are ignored — the app tracks exactly - E5/E10/DIESEL. -- Station snapshot cached `FUEL_CACHE_TTL` (default 300s); stale cache is - served on upstream failure so the app never sees a hard outage. +- Fuel types mapped: `E5`/`E10` → same, `B7`/`B7_STANDARD`/`B7S`/`B7P`/ + `B7_PREMIUM`/`B10` → `DIESEL`. Others (`HVO`, `SDV`, …) are ignored — the + app tracks exactly E5/E10/DIESEL. +- Station snapshot cached `FUEL_CACHE_TTL` (default 300s); a stale cache is + refreshed in the background on the next request and stale data is served in + the meantime, so a multi-minute sync never blocks the app's 5s timeout. +- Base URL is `https://www.fuel-finder.service.gov.uk` (NOT + `api.fuelfinder...` — that host doesn't resolve). Overridable per-endpoint + via env. - No credentials are exposed to clients — the app calls the relay keyless. ## Setup diff --git a/app/main.py b/app/main.py index f053ccc..f817186 100644 --- a/app/main.py +++ b/app/main.py @@ -47,19 +47,25 @@ from fastapi import FastAPI, HTTPException, Query CLIENT_ID = os.environ.get("FUEL_API_CLIENT_ID", "") CLIENT_SECRET = os.environ.get("FUEL_API_CLIENT_SECRET", "") -TOKEN_URL = os.environ.get( - "FUEL_API_TOKEN_URL", - "https://api.fuelfinder.service.gov.uk/api/v1/oauth/generate_access_token", +# Base host for the official Fuel Finder API (note: NOT api.fuelfinder... — +# the real host is www.fuel-finder.service.gov.uk). +API_BASE = os.environ.get( + "FUEL_API_BASE", + "https://www.fuel-finder.service.gov.uk", ) +TOKEN_URL = os.environ.get("FUEL_API_TOKEN_URL", f"{API_BASE}/api/v1/oauth/generate_access_token") # Official Fuel Finder endpoints (batched, 500 records/page). -PFS_URL = os.environ.get("FUEL_API_PFS_URL", "https://api.fuelfinder.service.gov.uk/api/v1/pfs") +PFS_URL = os.environ.get("FUEL_API_PFS_URL", f"{API_BASE}/api/v1/pfs") PRICES_URL = os.environ.get( "FUEL_API_PRICES_URL", - "https://api.fuelfinder.service.gov.uk/api/v1/pfs/fuel-prices", + f"{API_BASE}/api/v1/pfs/fuel-prices", ) PAGE_SIZE = int(os.environ.get("FUEL_API_PAGE_SIZE", "500")) CACHE_TTL = int(os.environ.get("FUEL_CACHE_TTL", "300")) # seconds (5 min) TOKEN_EXPIRY_BUFFER = int(os.environ.get("FUEL_TOKEN_BUFFER", "60")) # seconds +# Fuel Finder rate limit: 30 rpm per client, sequential only (429 on overlap). +# uff.py's default of 4.0s between batch pages ≈ 15 rpm — stay inside. +BATCH_SLEEP = float(os.environ.get("FUEL_API_BATCH_SLEEP", "4.0")) # Public full-UK mirror of the Fuel Finder dataset (GitHub Actions, hourly), # used as the fallback source when OAuth credentials are not configured so the @@ -105,6 +111,7 @@ FUEL_MAP = { "B7_STANDARD": "DIESEL", "B7S": "DIESEL", "B7P": "DIESEL", + "B7_PREMIUM": "DIESEL", "B10": "DIESEL", } @@ -197,7 +204,13 @@ def _normalise_api_payload(stations_info: list, fuel_prices: list) -> list: price = _parse_price(entry.get("price")) if price is None: continue - prices[grade] = price + # Multiple diesel variants (B7_STANDARD, B7_PREMIUM, B10) map to + # one DIESEL grade — take the MINIMUM so the cheapest-diesel + # reference is never inflated by a premium price. + if grade == "DIESEL": + prices[grade] = min(prices.get(grade, float("inf")), price) + else: + prices[grade] = price updated = entry.get("price_last_updated") if updated and (latest_updated is None or updated > latest_updated): latest_updated = updated @@ -326,7 +339,12 @@ def _get_token(client: httpx.Client) -> str: def _fetch_all_pages(client: httpx.Client, url: str) -> list: - """Fetch every page of a batched Fuel Finder endpoint (500/page).""" + """Fetch every page of a batched Fuel Finder endpoint (500/page). + + Fuel Finder is strictly sequential (30 rpm per client): a request sent + before the previous one completes gets a 429. We therefore sleep + BATCH_SLEEP between pages and never issue a second in-flight request. + """ results = [] batch = 1 while True: @@ -345,6 +363,7 @@ def _fetch_all_pages(client: httpx.Client, url: str) -> list: batch += 1 if batch > 200: # safety cap (~100k records) break + time.sleep(BATCH_SLEEP) return results @@ -371,8 +390,24 @@ def _fetch_api() -> list: # Cache refresh # --------------------------------------------------------------------------- +import threading + +_refresh_lock = threading.Lock() + + def _refresh_cache() -> None: - """Refresh the station cache if stale (or on first run).""" + """Refresh the station cache (called synchronously or from a thread).""" + with _refresh_lock: + _refresh_cache_locked() + + +def _refresh_cache_locked() -> None: + """Refresh the station cache if stale (or on first run). + + Callers hold _refresh_lock. Full API sync takes ~1-2 min (17+ pages × 2 + endpoints × BATCH_SLEEP), so this must NEVER run inline in a request path + — see _ensure_fresh_cache(). + """ _state["sync_attempts"] += 1 try: if SOURCE == "csv": @@ -395,6 +430,22 @@ def _refresh_cache() -> None: _state["last_error"] = f"{type(exc).__name__}: {exc}" +def _ensure_fresh_cache() -> None: + """Non-blocking staleness check: serve stale data, refresh in background. + + A full API sync takes minutes (rate-limited batched paging); blocking a + request on it would blow past the app/widget's 5-second timeout. We return + whatever we have immediately and let the sync catch up on a daemon thread. + """ + if _state["stations"] is None: + # First run: block (usually the lifespan warm-up, but guards a race + # where a request beats it). CSV fallback is fast; API takes a while. + _refresh_cache() + return + if time.time() - _state["stations_fetched_at"] > CACHE_TTL: + threading.Thread(target=_refresh_cache, daemon=True).start() + + @asynccontextmanager async def lifespan(_: FastAPI): # Always try to warm the cache. In auto mode with no credentials this @@ -471,9 +522,9 @@ def stations( if _state["stations"] is None: raise HTTPException(status_code=502, detail="No data cached yet and upstream fetch failed") - # Trigger a refresh if stale; keep serving stale cache if it fails. - if time.time() - _state["stations_fetched_at"] > CACHE_TTL: - _refresh_cache() + # Refresh in the background if stale; never block a request on a + # multi-minute rate-limited API sync. + _ensure_fresh_cache() rows = [] for station in _stations_list(): diff --git a/scripts/run.sh b/scripts/run.sh index 2af30ca..3802435 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -12,4 +12,9 @@ set -a set +a PORT="${FUEL_RELAY_PORT:-8789}" + +# Prefer the project venv so the bare `python3` fallback can't miss uvicorn. +if [ -x ".venv/bin/python" ]; then + exec .venv/bin/python -m uvicorn app.main:app --host 0.0.0.0 --port "$PORT" +fi exec python3 -m uvicorn app.main:app --host 0.0.0.0 --port "$PORT" diff --git a/tests/test_api_adapter.py b/tests/test_api_adapter.py index 5a24bad..8a86637 100644 --- a/tests/test_api_adapter.py +++ b/tests/test_api_adapter.py @@ -70,6 +70,7 @@ class TestNormaliseApiPayload: {"price": "137.9000", "fuel_type": "E10", "price_last_updated": "2026-08-12T08:15:23"}, {"price": "144.9000", "fuel_type": "E5", "price_last_updated": "2026-08-12T08:15:24"}, {"price": "149.9000", "fuel_type": "B7_STANDARD", "price_last_updated": "2026-08-12T08:15:25"}, + {"price": "151.9000", "fuel_type": "B7_PREMIUM", "price_last_updated": "2026-08-12T08:16:00"}, ], }] @@ -82,7 +83,7 @@ class TestNormaliseApiPayload: assert s["lat"] == 53.7265 assert s["lng"] == -1.8580 assert s["prices"] == {"E10": 137.9, "E5": 144.9, "DIESEL": 149.9} - assert s["price_updated"] == "2026-08-12T08:15:25" + assert s["price_updated"] == "2026-08-12T08:16:00" def test_unpriced_fuel_dropped(self): stations_info = [{