Skip to content
Repos

blackcat

Python 3.6 KB92 lines Opens in site

Directory Raw Download

"""SearXNG search provider (self-hosted meta-search, no API key).

SearXNG aggregates 70+ upstream engines behind one JSON endpoint
(``GET /search?q=...&format=json``). It is the fallback when the
rsearch backend fails: a local SearXNG instance keeps evidence
gathering alive. Results are normalised to the standard shape
(title/url/date/summary) plus the ``fallback_provider`` marker; cost
is recorded as reported (self-hosted search costs nothing per call).

Default base is a local instance (``http://localhost:8888``, the
stock Docker mapping); override with ``SEARXNG_BASE_URL``. Public
instances work too but may rate-limit or block bot traffic —
self-hosting is the reliable path.
"""

from __future__ import annotations

# retoor <retoor@molodetz.nl>
import json
import logging
import os
import urllib.parse
from typing import Any

import aiohttp

from blackcat.client import DeepSeekError
from blackcat.costs import get_tracker

log = logging.getLogger(__name__)

DEFAULT_BASE = "http://localhost:8888"


class SearXNGClient:
    def __init__(self, base_url: str = "", timeout: float = 30.0):
        self.base_url = (base_url or os.environ.get(
            "SEARXNG_BASE_URL") or DEFAULT_BASE).rstrip("/")
        self.timeout = timeout
        self._http: aiohttp.ClientSession | None = None

    async def _session(self) -> aiohttp.ClientSession:
        """One shared session, created lazily in the running loop."""
        if self._http is None or self._http.closed:
            from blackcat.http import open_session
            self._http = open_session(self.timeout)
        return self._http

    async def aclose(self) -> None:
        """Close the shared session; safe to call twice."""
        session, self._http = self._http, None
        if session is not None and not session.closed:
            await session.close()

    async def search(self, query: str, count: int = 10) -> dict[str, Any]:
        query = query.strip()
        if not query:
            raise DeepSeekError("searxng query must be non-empty")
        count = max(1, min(50, count))
        url = self.base_url + "/search?" + urllib.parse.urlencode(
            {"q": query, "format": "json", "categories": "general"})
        log.info("searxng search: %r (count=%d)", query, count)
        try:
            session = await self._session()
            async with session.get(
                    url, headers={"Accept": "application/json"}) as resp:
                raw = await resp.read()
                if resp.status >= 400:
                    raise DeepSeekError(
                        f"searxng error {resp.status}", resp.status)
                payload = json.loads(raw.decode("utf-8"))
        except DeepSeekError:
            raise
        except (aiohttp.ClientError, TimeoutError, OSError,
                ValueError) as exc:
            raise DeepSeekError(f"searxng unreachable: {exc}") from exc
        if not isinstance(payload, dict):
            raise DeepSeekError("searxng failed: bad response")
        results = []
        for item in (payload.get("results") or [])[:count]:
            if not isinstance(item, dict):
                continue
            results.append({
                "title": item.get("title") or "",
                "url": item.get("url") or "",
                "summary": item.get("content") or "",
                "date": item.get("publishedDate"),
                "source_provider": item.get("engine") or ""})
        get_tracker().record_call("search", "searxng", cost_usd=0.0)
        log.info("searxng done: %r -> %d result(s) (self-hosted, $0)",
                 query, len(results))
        return {"results": results, "fallback_provider": "searxng"}
74.8ms