"""SearXNG search provider (self-hosted meta-search, no API key).
SearXNG aggregates 70+ upstream engines behind one JSON endpoint
(``GET /search?q=...&format=json``). It is the fallback when the
rsearch backend fails: a local SearXNG instance keeps evidence
gathering alive. Results are normalised to the standard shape
(title/url/date/summary) plus the ``fallback_provider`` marker; cost
is recorded as reported (self-hosted search costs nothing per call).
Default base is a local instance (``http://localhost:8888``, the
stock Docker mapping); override with ``SEARXNG_BASE_URL``. Public
instances work too but may rate-limit or block bot traffic —
self-hosting is the reliable path.
"""
from __future__ import annotations
# retoor <retoor@molodetz.nl>
import json
import logging
import os
import urllib.parse
from typing import Any
import aiohttp
from blackcat.client import DeepSeekError
from blackcat.costs import get_tracker
log = logging.getLogger(__name__)
DEFAULT_BASE = "http://localhost:8888"
class SearXNGClient:
def __init__(self, base_url: str = "", timeout: float = 30.0):
self.base_url = (base_url or os.environ.get(
"SEARXNG_BASE_URL") or DEFAULT_BASE).rstrip("/")
self.timeout = timeout
self._http: aiohttp.ClientSession | None = None
async def _session(self) -> aiohttp.ClientSession:
"""One shared session, created lazily in the running loop."""
if self._http is None or self._http.closed:
from blackcat.http import open_session
self._http = open_session(self.timeout)
return self._http
async def aclose(self) -> None:
"""Close the shared session; safe to call twice."""
session, self._http = self._http, None
if session is not None and not session.closed:
await session.close()
async def search(self, query: str, count: int = 10) -> dict[str, Any]:
query = query.strip()
if not query:
raise DeepSeekError("searxng query must be non-empty")
count = max(1, min(50, count))
url = self.base_url + "/search?" + urllib.parse.urlencode(
{"q": query, "format": "json", "categories": "general"})
log.info("searxng search: %r (count=%d)", query, count)
try:
session = await self._session()
async with session.get(
url, headers={"Accept": "application/json"}) as resp:
raw = await resp.read()
if resp.status >= 400:
raise DeepSeekError(
f"searxng error {resp.status}", resp.status)
payload = json.loads(raw.decode("utf-8"))
except DeepSeekError:
raise
except (aiohttp.ClientError, TimeoutError, OSError,
ValueError) as exc:
raise DeepSeekError(f"searxng unreachable: {exc}") from exc
if not isinstance(payload, dict):
raise DeepSeekError("searxng failed: bad response")
results = []
for item in (payload.get("results") or [])[:count]:
if not isinstance(item, dict):
continue
results.append({
"title": item.get("title") or "",
"url": item.get("url") or "",
"summary": item.get("content") or "",
"date": item.get("publishedDate"),
"source_provider": item.get("engine") or ""})
get_tracker().record_call("search", "searxng", cost_usd=0.0)
log.info("searxng done: %r -> %d result(s) (self-hosted, $0)",
query, len(results))
return {"results": results, "fallback_provider": "searxng"}