Skip to content
Repos

blackcat

Python 22.4 KB588 lines Opens in site

Directory Raw Download

"""Committed tests for the search family (stdlib unittest, no network).

rsearch plus the fallback chain (SearXNG, Jina Reader), the stealthii
first fetch step, DeepSeek Fetch, the scholarly vertical (arXiv +
Semantic Scholar), and the persistent FTS5 repo index - all asserted
offline via fake sessions and tmp directories.
"""
# retoor <retoor@molodetz.nl>

import io
import json
import os
import tempfile
import unittest
from contextlib import redirect_stdout
from unittest import mock

from blackcat.agent import BlackCatBot
from blackcat.cli import build_parser, main
from blackcat.client import DeepSeekError
from blackcat.costs import get_tracker
from blackcat.fstate import FileState, FileToolError
from blackcat.http import StealthFetcher
from blackcat.reader import JinaReaderClient
from blackcat.repo_index import RepoIndex
from blackcat.scholar import ScholarClient
from blackcat.searxng import SearXNGClient
from blackcat.tools import FileTools
from tests.test_blackcat import (
    _FakeResp,
    _FakeSession,
    _json_handler,
)

ARXIV_FEED = """<feed xmlns="http://www.w3.org/2005/Atom">
<entry><title>  Deep
Learning </title><id>https://arxiv.org/abs/1234.5678v1</id>
<summary>An abstract here.</summary><published>2024-01-02T00:00:00Z</published>
<author><name>Jane Doe</name></author></entry>
<entry><title>Second Paper</title><id>https://arxiv.org/abs/9999.1</id>
<summary>More.</summary><published>2023-05-01T00:00:00Z</published>
</entry></feed>"""

S2_PAYLOAD = {"data": [
    {"paperId": "abc", "title": "Second Paper",
     "abstract": "S2 abstract.", "year": 2023, "url": None,
     "openAccessPdf": {"url": "https://arxiv.org/abs/9999.1"},
     "authors": [{"name": "A U"}], "citationCount": 5},
    {"paperId": "def", "title": "Third Paper", "abstract": "",
     "year": None, "url": None, "openAccessPdf": None,
     "authors": [], "citationCount": 0}]}


def _session_for(handler):
    return _FakeSession(handler)


def _payload_handler(payload, status=200):
    return _json_handler(payload, status=status)


class CostCase(unittest.IsolatedAsyncioTestCase):
    def setUp(self):
        get_tracker().reset()
        self.addCleanup(get_tracker().reset)


class TestSearXNG(CostCase):
    async def test_normalises_and_trims(self):
        payload = {"results": [
            {"title": "T%d" % i, "url": "https://a.test/%d" % i,
             "content": "s%d" % i, "publishedDate": "2024-01-0%d" % i,
             "engine": "brave"}
            for i in range(1, 6)]}
        client = SearXNGClient(base_url="https://sx.test")
        client._http = _session_for(_payload_handler(payload))
        out = await client.search("q", count=3)
        self.assertEqual(out["fallback_provider"], "searxng")
        self.assertEqual(len(out["results"]), 3)
        first = out["results"][0]
        self.assertEqual(
            (first["title"], first["url"], first["summary"],
             first["date"], first["source_provider"]),
            ("T1", "https://a.test/1", "s1", "2024-01-01", "brave"))
        snap = get_tracker().snapshot()
        self.assertEqual(snap["by_provider"]["searxng"]["calls"], 1)
        self.assertEqual(snap["by_provider"]["searxng"]["cost_usd"], 0.0)

    def test_base_url_env_override(self):
        with mock.patch.dict(os.environ,
                             {"SEARXNG_BASE_URL": "https://env.test/"}):
            self.assertEqual(SearXNGClient().base_url,
                             "https://env.test")

    async def test_failures_raise(self):
        client = SearXNGClient(base_url="https://example.invalid")
        with self.assertRaises(DeepSeekError):
            await client.search("  ")
        bad = SearXNGClient(base_url="https://sx.test")
        bad._http = _session_for(_payload_handler([1, 2]))
        with self.assertRaises(DeepSeekError):
            await bad.search("q")

        def fail_429(_method, _url, _kwargs):
            return _FakeResp(status=429,
                             payload={"error": "slow down"})

        limited = SearXNGClient(base_url="https://sx.test")
        limited._http = _session_for(fail_429)
        with self.assertRaises(DeepSeekError) as ctx:
            await limited.search("q")
        self.assertEqual(ctx.exception.status, 429)

        def fail_down(_method, _url, _kwargs):
            raise ConnectionError("down")

        down = SearXNGClient(base_url="https://sx.test")
        down._http = _session_for(fail_down)
        with self.assertRaises(DeepSeekError):
            await down.search("q")


class TestScholar(CostCase):
    def handler_for(self, s2=True):
        def handle(_method, url, _kwargs):
            if "arxiv" in url:
                return _FakeResp(raw=ARXIV_FEED.encode("utf-8"))
            if not s2:
                raise ConnectionError("down")
            return _FakeResp(payload=S2_PAYLOAD)
        return handle

    async def test_merges_both_upstreams(self):
        client = ScholarClient()
        client._http = _session_for(self.handler_for())
        out = await client.search("deep learning", count=10)
        self.assertEqual(out["provider"], "scholar")
        self.assertNotIn("partial", out)
        self.assertEqual(out["sources"],
                         {"arxiv": 2, "semantic_scholar": 2})
        titles = [r["title"] for r in out["results"]]
        self.assertEqual(titles, ["Deep Learning", "Second Paper",
                                  "Third Paper"])
        first = out["results"][0]
        self.assertEqual(first["url"], "https://arxiv.org/abs/1234.5678v1")
        self.assertEqual(first["summary"], "An abstract here.")
        self.assertEqual(first["date"], "2024-01-02T00:00:00Z")
        self.assertEqual(first["authors"], ["Jane Doe"])
        self.assertEqual(first["source_provider"], "arxiv")
        second = out["results"][1]
        self.assertEqual(second["source_provider"], "semantic_scholar")
        self.assertEqual(second["url"], "https://arxiv.org/abs/9999.1")
        self.assertEqual(second["citations"], 5)
        third = out["results"][2]
        self.assertTrue(third["url"].startswith(
            "https://www.semanticscholar.org/paper/def"))
        snap = get_tracker().snapshot()
        self.assertEqual(snap["by_provider"]["arxiv"]["calls"], 1)
        self.assertEqual(
            snap["by_provider"]["semantic_scholar"]["calls"], 1)

    async def test_trims_to_count(self):
        client = ScholarClient()
        client._http = _session_for(self.handler_for())
        out = await client.search("q", count=2)
        self.assertEqual(len(out["results"]), 2)

    async def test_partial_when_one_upstream_fails(self):
        client = ScholarClient()
        client._http = _session_for(self.handler_for(s2=False))
        out = await client.search("q")
        self.assertTrue(out["partial"])
        self.assertIn("semantic_scholar", out["errors"])
        self.assertTrue(all(r["source_provider"] == "arxiv"
                            for r in out["results"]))

    async def test_both_failing_raises(self):
        def fail(_method, _url, _kwargs):
            raise ConnectionError("down")

        client = ScholarClient()
        client._http = _session_for(fail)
        with self.assertRaises(DeepSeekError):
            await client.search("q")

    async def test_bad_payloads_raise(self):
        def fake(_method, url, _kwargs):
            if "arxiv" in url:
                return _FakeResp(raw=b"<feed><entry><unclosed>")
            return _FakeResp(payload=[1, 2])

        client = ScholarClient()
        client._http = _session_for(fake)
        with self.assertRaises(DeepSeekError):
            await client.search("q")
        with self.assertRaises(DeepSeekError):
            await client.search("  ")


class TestJina(CostCase):
    async def test_markdown_passthrough(self):
        client = JinaReaderClient()
        client._http = _session_for(
            lambda _m, _u, _k: _FakeResp(raw=b"# Hi\n\nbody"))
        out = await client.fetch("https://a.test/p")
        self.assertEqual(out["url"], "https://a.test/p")
        self.assertEqual(out["content"], "# Hi\n\nbody")
        self.assertEqual(out["fallback_provider"], "jina")
        snap = get_tracker().snapshot()
        self.assertEqual(snap["by_provider"]["jina"]["calls"], 1)
        self.assertEqual(
            snap["by_category"]["fetch"]["cost_usd"], 0.0)

    async def test_auth_header_and_target(self):
        seen = {}

        def fake(_method, url, kwargs):
            seen["url"] = url
            seen["headers"] = kwargs.get("headers")
            return _FakeResp(raw=b"x")

        with mock.patch.dict(os.environ, {"JINA_API_KEY": "k"}):
            client = JinaReaderClient()
            client._http = _session_for(fake)
            await client.fetch("https://a.test/p")
        self.assertEqual(seen["url"], "https://r.jina.ai/https://a.test/p")
        self.assertEqual(seen["headers"]["Authorization"], "Bearer k")

    async def test_failures_raise(self):
        client = JinaReaderClient()
        with self.assertRaises(DeepSeekError):
            await client.fetch("ftp://a.test/x")

        def fail_402(_method, _url, _kwargs):
            return _FakeResp(status=402, payload={"error": "paid"})

        paid = JinaReaderClient()
        paid._http = _session_for(fail_402)
        with self.assertRaises(DeepSeekError) as ctx:
            await paid.fetch("https://a.test/x")
        self.assertEqual(ctx.exception.status, 402)

        def fail_down(_method, _url, _kwargs):
            raise ConnectionError("down")

        down = JinaReaderClient()
        down._http = _session_for(fail_down)
        with self.assertRaises(DeepSeekError):
            await down.fetch("https://a.test/x")


class _FakeStealthResp:
    def __init__(self, status=200, text="",
                 headers=None):
        self.status = status
        self._text = text
        self.headers = headers or {}

    async def text(self):
        return self._text


class _FakeStealth:
    def __init__(self, resp=None, error=None):
        self.resp = resp
        self.error = error
        self.urls = []
        self.closed = False

    async def get(self, url, headers=None, timeout=None):
        self.urls.append(url)
        if self.error is not None:
            raise self.error
        return self.resp

    async def shutdown(self):
        self.closed = True


class _FailStealth:
    async def get(self, url, headers=None, timeout=None):
        from blackcat.client import DeepSeekError
        raise DeepSeekError("stealthii fetch unreachable: down")

    async def shutdown(self):
        pass


class FailingClient:
    model = "m"

    def __init__(self, status=503):
        self.status = status

    async def fetch(self, url):
        raise DeepSeekError("down", self.status)

    async def aclose(self):
        pass


def _stub_search(result=None, error=None):
    class Stub:
        async def search(self, query, count=10):
            if error is not None:
                raise error
            return result
    return Stub()


def _stub_fetch(result=None, error=None):
    class Stub:
        async def fetch(self, url):
            if error is not None:
                raise error
            return result
    return Stub()


class ChainCase(unittest.IsolatedAsyncioTestCase):
    def bot(self, **kwargs):
        kwargs.setdefault("system", "t")
        bot = BlackCatBot(provider="deepseek", api_key="k", **kwargs)
        bot._client = FailingClient()
        bot._stealth = StealthFetcher(stealth=_FailStealth())
        self.addAsyncCleanup(bot.aclose)
        return bot


class TestSearchChain(ChainCase):
    async def test_rsearch_primary(self):
        bot = self.bot()
        bot._rsearch = _stub_search({"results": [], "fallback_provider":
                                     "rsearch"})
        bot._searxng = _stub_search({"results": [],
                                     "fallback_provider": "searxng"})
        out = await bot.search("q")
        self.assertEqual(out["fallback_provider"], "rsearch")
        frames = [f for f in bot.transcript
                  if f["kind"] == "search_fallback"]
        self.assertEqual(frames, [])
        kinds = [f["kind"] for f in bot.transcript]
        self.assertIn("search", kinds)

    async def test_searxng_second_when_rsearch_fails(self):
        bot = self.bot()
        bot._rsearch = _stub_search(
            error=DeepSeekError("rsearch down", 503))
        bot._searxng = _stub_search({"results": [],
                                     "fallback_provider": "searxng"})
        out = await bot.search("q")
        self.assertEqual(out["fallback_provider"], "searxng")
        frames = [f for f in bot.transcript
                  if f["kind"] == "search_fallback"]
        self.assertEqual(frames[0]["provider"], "searxng")
        self.assertIn("rsearch down", frames[0]["reason"])

    async def test_all_failing_raises_last_error(self):
        bot = self.bot()
        bot._rsearch = _stub_search(
            error=DeepSeekError("rsearch down", 503))
        bot._searxng = _stub_search(
            error=DeepSeekError("searxng down"))
        with self.assertRaises(DeepSeekError) as ctx:
            await bot.search("q")
        self.assertIn("searxng down", str(ctx.exception))

    async def test_chain_disabled_raises_original(self):
        bot = self.bot(search_fallback=False)
        bot._rsearch = _stub_search(
            error=DeepSeekError("rsearch down", 503))
        with self.assertRaises(DeepSeekError) as ctx:
            await bot.search("q")
        self.assertEqual(ctx.exception.status, 503)


class TestFetchFallback(ChainCase):
    async def test_stealthii_first_wins(self):
        bot = self.bot()
        bot._stealth = StealthFetcher(stealth=_FakeStealth(
            resp=_FakeStealthResp(
                text="<html><body>" + "w" * 500 + "</body></html>",
                headers={"content-type": "text/html"})))
        got = await bot._fetch_chain("https://a.test", n=1)
        self.assertTrue(got["ok"])
        self.assertEqual(got["method"], "stealthii")
        self.assertEqual(
            [a["method"] for a in got["attempts"]], ["stealthii"])

    async def test_jina_when_stealthii_and_provider_fail(self):
        bot = self.bot()
        bot._jina = _stub_fetch({"url": "https://a.test",
                                 "content": "w" * 500,
                                 "fallback_provider": "jina"})
        got = await bot._fetch_chain("https://a.test", n=2)
        self.assertTrue(got["ok"])
        self.assertEqual(got["method"], "jina")
        self.assertEqual(
            [a["method"] for a in got["attempts"]],
            ["stealthii", "jina"])

    async def test_all_failing_reports(self):
        bot = self.bot()
        bot._jina = _stub_fetch(error=DeepSeekError("jina down"))
        got = await bot._fetch_chain("https://a.test", n=3)
        self.assertFalse(got["ok"])
        self.assertEqual(got["method"], "none")
        self.assertEqual(
            [a["method"] for a in got["attempts"]],
            ["stealthii", "provider", "jina", "direct"])

    async def test_openrouter_search_uses_rsearch(self):
        bot = BlackCatBot(provider="openrouter", api_key="k", system="t")
        self.addAsyncCleanup(bot.aclose)
        bot._rsearch = _stub_search({"results": [],
                                     "fallback_provider": "rsearch"})
        out = await bot.search("q")
        self.assertEqual(out["fallback_provider"], "rsearch")


class TestStealthFetcher(unittest.IsolatedAsyncioTestCase):
    async def test_html_becomes_text(self):
        fetcher = StealthFetcher(stealth=_FakeStealth(
            resp=_FakeStealthResp(
                text="<html><body><p>Hello world</p></body></html>",
                headers={"content-type": "text/html"})))
        out = await fetcher.fetch("https://a.test/p")
        self.assertEqual(out["fallback_provider"], "stealthii")
        self.assertIn("Hello world", out["content"])
        self.assertNotIn("<p>", out["content"])

    async def test_non_html_passthrough(self):
        fetcher = StealthFetcher(stealth=_FakeStealth(
            resp=_FakeStealthResp(text="# title\n\nbody",
                                  headers={"content-type":
                                           "text/markdown"})))
        out = await fetcher.fetch("https://a.test/p")
        self.assertEqual(out["content"], "# title\n\nbody")

    async def test_status_error(self):
        fetcher = StealthFetcher(stealth=_FakeStealth(
            resp=_FakeStealthResp(status=403, text="denied")))
        with self.assertRaises(DeepSeekError) as ctx:
            await fetcher.fetch("https://a.test/p")
        self.assertEqual(ctx.exception.status, 403)

    async def test_unreachable_becomes_deepseek_error(self):
        fetcher = StealthFetcher(
            stealth=_FakeStealth(error=ConnectionError("down")))
        with self.assertRaises(DeepSeekError):
            await fetcher.fetch("https://a.test/p")

    async def test_rejects_non_http(self):
        fetcher = StealthFetcher(stealth=_FakeStealth())
        with self.assertRaises(DeepSeekError):
            await fetcher.fetch("ftp://a.test/x")

    async def test_aclose_idempotent(self):
        fake = _FakeStealth()
        fetcher = StealthFetcher(stealth=fake)
        await fetcher.aclose()
        await fetcher.aclose()
        self.assertTrue(fake.closed)


class TestScholarBot(ChainCase):
    async def test_live_and_echo(self):
        bot = self.bot()
        bot._scholar = _stub_search({"results": [{"title": "T"}],
                                     "provider": "scholar"})
        out = await bot.scholar("q")
        self.assertEqual(out["results"], [{"title": "T"}])
        kinds = [f["kind"] for f in bot.transcript]
        self.assertIn("scholar", kinds)
        async with BlackCatBot(provider="echo", system="t") as echo:
            self.assertEqual((await echo.scholar("q"))["results"], [])


class TestRepoIndex(unittest.IsolatedAsyncioTestCase):
    def setUp(self):
        self.tmp = tempfile.TemporaryDirectory()
        self.addCleanup(self.tmp.cleanup)
        self.root = self.tmp.name
        self.write("a.py", "def hello():\n    return 'world'\n")
        self.write("b.md", "# Title\n\nhello world again\n")
        self.write("sub/c.txt", "nothing relevant here\n")

    def write(self, rel, content):
        path = os.path.join(self.root, rel)
        os.makedirs(os.path.dirname(path) or self.root, exist_ok=True)
        mode = "wb" if isinstance(content, bytes) else "w"
        with open(path, mode) as fh:
            fh.write(content)
        return path

    def index(self):
        return RepoIndex(self.root)

    async def test_ranked_hits_with_lines(self):
        out = await self.index().search("hello world")
        self.assertEqual(out["indexed_files"], 3)
        paths = [r["path"] for r in out["results"]]
        self.assertEqual(sorted(paths), ["a.py", "b.md"])
        by_path = {r["path"]: r for r in out["results"]}
        self.assertEqual(by_path["a.py"]["line"], 1)
        self.assertEqual(by_path["b.md"]["line"], 3)
        for hit in out["results"]:
            self.assertGreaterEqual(hit["score"], 0.0)
            self.assertLessEqual(hit["score"], 1.0)
            self.assertIn("[", hit["snippet"])
        scores = [r["score"] for r in out["results"]]
        self.assertEqual(scores, sorted(scores, reverse=True))

    async def test_or_fallback_when_and_misses(self):
        out = await self.index().search("hello zebras")
        self.assertTrue(out["results"])
        self.assertFalse((await self.index().search("zebras"))["results"])
        self.assertFalse((await self.index().search("  "))["results"])

    async def test_freshness_add_modify_delete(self):
        index = self.index()
        self.assertEqual((await index.search("freshfruit"))["results"],
                         [])
        self.write("d.txt", "freshfruit grows here\n")
        hits = (await index.search("freshfruit"))["results"]
        self.assertEqual([h["path"] for h in hits], ["d.txt"])
        self.write("d.txt", "now it mentions allotment instead\n")
        self.assertEqual((await index.search("freshfruit"))["results"],
                         [])
        self.assertEqual((await index.search("allotment"))["results"]
                         [0]["path"], "d.txt")
        os.remove(os.path.join(self.root, "d.txt"))
        self.assertEqual((await index.search("allotment"))["results"],
                         [])

    async def test_skips_binary_hidden_heavy_and_itself(self):
        self.write("bin.dat", b"\x00\x01binary\x00blob")
        self.write(".hidden/x.txt", "hidden treasure here\n")
        self.write("node_modules/dep/index.js", "hidden treasure here\n")
        out = await self.index().search("treasure")
        self.assertEqual(out["results"], [])
        self.assertEqual((await self.index().search("treasure"))
                         ["indexed_files"], 3)
        self.assertTrue(os.path.exists(
            os.path.join(self.root, ".blackcat-index.sqlite3")))

    async def test_bot_repo_search_and_errors(self):
        bot = BlackCatBot(provider="echo", system="t",
                          tools=FileTools(FileState(self.root)))
        self.addAsyncCleanup(bot.aclose)
        out = await bot.repo_search("hello")
        self.assertGreaterEqual(len(out["results"]), 1)
        kinds = [f["kind"] for f in bot.transcript]
        self.assertIn("repo_search", kinds)
        with self.assertRaises(FileToolError):
            RepoIndex(os.path.join(self.root, "no-such-dir"))
        with self.assertRaises(FileToolError):
            await self.index().search("hello", count="many")


class TestSearchCLI(unittest.TestCase):
    def test_scholar_echo(self):
        out = io.StringIO()
        with redirect_stdout(out):
            code = main(["--provider", "echo", "scholar", "q"])
        self.assertEqual(code, 0)
        self.assertEqual(json.loads(out.getvalue())["results"], [])

    def test_repo_search_command(self):
        with tempfile.TemporaryDirectory() as tmp:
            with open(os.path.join(tmp, "f.txt"), "w") as fh:
                fh.write("pineapples are tasty\n")
            out = io.StringIO()
            with redirect_stdout(out):
                code = main(["--provider", "echo", "--tools-root", tmp,
                             "repo-search", "pineapples"])
        self.assertEqual(code, 0)
        payload = json.loads(out.getvalue())
        self.assertEqual(payload["results"][0]["path"], "f.txt")

    def test_flags(self):
        args = build_parser().parse_args(
            ["--no-fetch-fallback", "fetch", "https://a.test"])
        self.assertTrue(args.no_fetch_fallback)


if __name__ == "__main__":
    unittest.main()
26.1ms