Fix API base URL, rate-limit paging, background refresh; verify live against official API
- Base URL corrected to https://www.fuel-finder.service.gov.uk (the api.fuelfinder.service.gov.uk host in the portal's examples is stale — it does not resolve; www.fuel-finder... resolves to CloudFront and is what the live spec uses). Verified: /health now reports source=api, 8,010 stations, 0 failures. - Fuel Finder is strictly sequential (30 rpm per client, 429 on overlap): added FUEL_API_BATCH_SLEEP (default 4.0s) between pages ≈ 15 rpm. - Stale-cache refresh moved off the request path into a background thread — a multi-minute rate-limited sync no longer blows the app's 5s timeout; stale data is served while it catches up. - Multiple diesel variants (B7_STANDARD, B7_PREMIUM, B10) now map to DIESEL as the MINIMUM price, so a premium price can't inflate the cheapest-diesel reference (CSV relay took first-wins; min is safer). - Added B7_PREMIUM (underscore) to the fuel map — official key. - scripts/run.sh now prefers the project venv (bare python3 missed uvicorn when run outside an activated env). - Tests updated for min-diesel + B7_PREMIUM; 12/12 pass.
This commit is contained in:
FuelBoard Contributor
+62
-11
@@ -47,19 +47,25 @@ from fastapi import FastAPI, HTTPException, Query
|
||||
|
||||
CLIENT_ID = os.environ.get("FUEL_API_CLIENT_ID", "")
|
||||
CLIENT_SECRET = os.environ.get("FUEL_API_CLIENT_SECRET", "")
|
||||
TOKEN_URL = os.environ.get(
|
||||
"FUEL_API_TOKEN_URL",
|
||||
"https://api.fuelfinder.service.gov.uk/api/v1/oauth/generate_access_token",
|
||||
# Base host for the official Fuel Finder API (note: NOT api.fuelfinder... —
|
||||
# the real host is www.fuel-finder.service.gov.uk).
|
||||
API_BASE = os.environ.get(
|
||||
"FUEL_API_BASE",
|
||||
"https://www.fuel-finder.service.gov.uk",
|
||||
)
|
||||
TOKEN_URL = os.environ.get("FUEL_API_TOKEN_URL", f"{API_BASE}/api/v1/oauth/generate_access_token")
|
||||
# Official Fuel Finder endpoints (batched, 500 records/page).
|
||||
PFS_URL = os.environ.get("FUEL_API_PFS_URL", "https://api.fuelfinder.service.gov.uk/api/v1/pfs")
|
||||
PFS_URL = os.environ.get("FUEL_API_PFS_URL", f"{API_BASE}/api/v1/pfs")
|
||||
PRICES_URL = os.environ.get(
|
||||
"FUEL_API_PRICES_URL",
|
||||
"https://api.fuelfinder.service.gov.uk/api/v1/pfs/fuel-prices",
|
||||
f"{API_BASE}/api/v1/pfs/fuel-prices",
|
||||
)
|
||||
PAGE_SIZE = int(os.environ.get("FUEL_API_PAGE_SIZE", "500"))
|
||||
CACHE_TTL = int(os.environ.get("FUEL_CACHE_TTL", "300")) # seconds (5 min)
|
||||
TOKEN_EXPIRY_BUFFER = int(os.environ.get("FUEL_TOKEN_BUFFER", "60")) # seconds
|
||||
# Fuel Finder rate limit: 30 rpm per client, sequential only (429 on overlap).
|
||||
# uff.py's default of 4.0s between batch pages ≈ 15 rpm — stay inside.
|
||||
BATCH_SLEEP = float(os.environ.get("FUEL_API_BATCH_SLEEP", "4.0"))
|
||||
|
||||
# Public full-UK mirror of the Fuel Finder dataset (GitHub Actions, hourly),
|
||||
# used as the fallback source when OAuth credentials are not configured so the
|
||||
@@ -105,6 +111,7 @@ FUEL_MAP = {
|
||||
"B7_STANDARD": "DIESEL",
|
||||
"B7S": "DIESEL",
|
||||
"B7P": "DIESEL",
|
||||
"B7_PREMIUM": "DIESEL",
|
||||
"B10": "DIESEL",
|
||||
}
|
||||
|
||||
@@ -197,7 +204,13 @@ def _normalise_api_payload(stations_info: list, fuel_prices: list) -> list:
|
||||
price = _parse_price(entry.get("price"))
|
||||
if price is None:
|
||||
continue
|
||||
prices[grade] = price
|
||||
# Multiple diesel variants (B7_STANDARD, B7_PREMIUM, B10) map to
|
||||
# one DIESEL grade — take the MINIMUM so the cheapest-diesel
|
||||
# reference is never inflated by a premium price.
|
||||
if grade == "DIESEL":
|
||||
prices[grade] = min(prices.get(grade, float("inf")), price)
|
||||
else:
|
||||
prices[grade] = price
|
||||
updated = entry.get("price_last_updated")
|
||||
if updated and (latest_updated is None or updated > latest_updated):
|
||||
latest_updated = updated
|
||||
@@ -326,7 +339,12 @@ def _get_token(client: httpx.Client) -> str:
|
||||
|
||||
|
||||
def _fetch_all_pages(client: httpx.Client, url: str) -> list:
|
||||
"""Fetch every page of a batched Fuel Finder endpoint (500/page)."""
|
||||
"""Fetch every page of a batched Fuel Finder endpoint (500/page).
|
||||
|
||||
Fuel Finder is strictly sequential (30 rpm per client): a request sent
|
||||
before the previous one completes gets a 429. We therefore sleep
|
||||
BATCH_SLEEP between pages and never issue a second in-flight request.
|
||||
"""
|
||||
results = []
|
||||
batch = 1
|
||||
while True:
|
||||
@@ -345,6 +363,7 @@ def _fetch_all_pages(client: httpx.Client, url: str) -> list:
|
||||
batch += 1
|
||||
if batch > 200: # safety cap (~100k records)
|
||||
break
|
||||
time.sleep(BATCH_SLEEP)
|
||||
return results
|
||||
|
||||
|
||||
@@ -371,8 +390,24 @@ def _fetch_api() -> list:
|
||||
# Cache refresh
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
import threading
|
||||
|
||||
_refresh_lock = threading.Lock()
|
||||
|
||||
|
||||
def _refresh_cache() -> None:
|
||||
"""Refresh the station cache if stale (or on first run)."""
|
||||
"""Refresh the station cache (called synchronously or from a thread)."""
|
||||
with _refresh_lock:
|
||||
_refresh_cache_locked()
|
||||
|
||||
|
||||
def _refresh_cache_locked() -> None:
|
||||
"""Refresh the station cache if stale (or on first run).
|
||||
|
||||
Callers hold _refresh_lock. Full API sync takes ~1-2 min (17+ pages × 2
|
||||
endpoints × BATCH_SLEEP), so this must NEVER run inline in a request path
|
||||
— see _ensure_fresh_cache().
|
||||
"""
|
||||
_state["sync_attempts"] += 1
|
||||
try:
|
||||
if SOURCE == "csv":
|
||||
@@ -395,6 +430,22 @@ def _refresh_cache() -> None:
|
||||
_state["last_error"] = f"{type(exc).__name__}: {exc}"
|
||||
|
||||
|
||||
def _ensure_fresh_cache() -> None:
|
||||
"""Non-blocking staleness check: serve stale data, refresh in background.
|
||||
|
||||
A full API sync takes minutes (rate-limited batched paging); blocking a
|
||||
request on it would blow past the app/widget's 5-second timeout. We return
|
||||
whatever we have immediately and let the sync catch up on a daemon thread.
|
||||
"""
|
||||
if _state["stations"] is None:
|
||||
# First run: block (usually the lifespan warm-up, but guards a race
|
||||
# where a request beats it). CSV fallback is fast; API takes a while.
|
||||
_refresh_cache()
|
||||
return
|
||||
if time.time() - _state["stations_fetched_at"] > CACHE_TTL:
|
||||
threading.Thread(target=_refresh_cache, daemon=True).start()
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_: FastAPI):
|
||||
# Always try to warm the cache. In auto mode with no credentials this
|
||||
@@ -471,9 +522,9 @@ def stations(
|
||||
if _state["stations"] is None:
|
||||
raise HTTPException(status_code=502, detail="No data cached yet and upstream fetch failed")
|
||||
|
||||
# Trigger a refresh if stale; keep serving stale cache if it fails.
|
||||
if time.time() - _state["stations_fetched_at"] > CACHE_TTL:
|
||||
_refresh_cache()
|
||||
# Refresh in the background if stale; never block a request on a
|
||||
# multi-minute rate-limited API sync.
|
||||
_ensure_fresh_cache()
|
||||
|
||||
rows = []
|
||||
for station in _stations_list():
|
||||
|
||||
Reference in New Issue
Block a user