"""Committed tests for the search family (stdlib unittest, no network).
rsearch plus the fallback chain (SearXNG, Jina Reader), the stealthii
first fetch step, DeepSeek Fetch, the scholarly vertical (arXiv +
Semantic Scholar), and the persistent FTS5 repo index - all asserted
offline via fake sessions and tmp directories.
"""
# retoor <retoor@molodetz.nl>
import io
import json
import os
import tempfile
import unittest
from contextlib import redirect_stdout
from unittest import mock
from blackcat.agent import BlackCatBot
from blackcat.cli import build_parser, main
from blackcat.client import DeepSeekError
from blackcat.costs import get_tracker
from blackcat.fstate import FileState, FileToolError
from blackcat.http import StealthFetcher
from blackcat.reader import JinaReaderClient
from blackcat.repo_index import RepoIndex
from blackcat.scholar import ScholarClient
from blackcat.searxng import SearXNGClient
from blackcat.tools import FileTools
from tests.test_blackcat import (
_FakeResp,
_FakeSession,
_json_handler,
)
ARXIV_FEED = """<feed xmlns="http://www.w3.org/2005/Atom">
<entry><title> Deep
Learning </title><id>https://arxiv.org/abs/1234.5678v1</id>
<summary>An abstract here.</summary><published>2024-01-02T00:00:00Z</published>
<author><name>Jane Doe</name></author></entry>
<entry><title>Second Paper</title><id>https://arxiv.org/abs/9999.1</id>
<summary>More.</summary><published>2023-05-01T00:00:00Z</published>
</entry></feed>"""
S2_PAYLOAD = {"data": [
{"paperId": "abc", "title": "Second Paper",
"abstract": "S2 abstract.", "year": 2023, "url": None,
"openAccessPdf": {"url": "https://arxiv.org/abs/9999.1"},
"authors": [{"name": "A U"}], "citationCount": 5},
{"paperId": "def", "title": "Third Paper", "abstract": "",
"year": None, "url": None, "openAccessPdf": None,
"authors": [], "citationCount": 0}]}
def _session_for(handler):
return _FakeSession(handler)
def _payload_handler(payload, status=200):
return _json_handler(payload, status=status)
class CostCase(unittest.IsolatedAsyncioTestCase):
def setUp(self):
get_tracker().reset()
self.addCleanup(get_tracker().reset)
class TestSearXNG(CostCase):
async def test_normalises_and_trims(self):
payload = {"results": [
{"title": "T%d" % i, "url": "https://a.test/%d" % i,
"content": "s%d" % i, "publishedDate": "2024-01-0%d" % i,
"engine": "brave"}
for i in range(1, 6)]}
client = SearXNGClient(base_url="https://sx.test")
client._http = _session_for(_payload_handler(payload))
out = await client.search("q", count=3)
self.assertEqual(out["fallback_provider"], "searxng")
self.assertEqual(len(out["results"]), 3)
first = out["results"][0]
self.assertEqual(
(first["title"], first["url"], first["summary"],
first["date"], first["source_provider"]),
("T1", "https://a.test/1", "s1", "2024-01-01", "brave"))
snap = get_tracker().snapshot()
self.assertEqual(snap["by_provider"]["searxng"]["calls"], 1)
self.assertEqual(snap["by_provider"]["searxng"]["cost_usd"], 0.0)
def test_base_url_env_override(self):
with mock.patch.dict(os.environ,
{"SEARXNG_BASE_URL": "https://env.test/"}):
self.assertEqual(SearXNGClient().base_url,
"https://env.test")
async def test_failures_raise(self):
client = SearXNGClient(base_url="https://example.invalid")
with self.assertRaises(DeepSeekError):
await client.search(" ")
bad = SearXNGClient(base_url="https://sx.test")
bad._http = _session_for(_payload_handler([1, 2]))
with self.assertRaises(DeepSeekError):
await bad.search("q")
def fail_429(_method, _url, _kwargs):
return _FakeResp(status=429,
payload={"error": "slow down"})
limited = SearXNGClient(base_url="https://sx.test")
limited._http = _session_for(fail_429)
with self.assertRaises(DeepSeekError) as ctx:
await limited.search("q")
self.assertEqual(ctx.exception.status, 429)
def fail_down(_method, _url, _kwargs):
raise ConnectionError("down")
down = SearXNGClient(base_url="https://sx.test")
down._http = _session_for(fail_down)
with self.assertRaises(DeepSeekError):
await down.search("q")
class TestScholar(CostCase):
def handler_for(self, s2=True):
def handle(_method, url, _kwargs):
if "arxiv" in url:
return _FakeResp(raw=ARXIV_FEED.encode("utf-8"))
if not s2:
raise ConnectionError("down")
return _FakeResp(payload=S2_PAYLOAD)
return handle
async def test_merges_both_upstreams(self):
client = ScholarClient()
client._http = _session_for(self.handler_for())
out = await client.search("deep learning", count=10)
self.assertEqual(out["provider"], "scholar")
self.assertNotIn("partial", out)
self.assertEqual(out["sources"],
{"arxiv": 2, "semantic_scholar": 2})
titles = [r["title"] for r in out["results"]]
self.assertEqual(titles, ["Deep Learning", "Second Paper",
"Third Paper"])
first = out["results"][0]
self.assertEqual(first["url"], "https://arxiv.org/abs/1234.5678v1")
self.assertEqual(first["summary"], "An abstract here.")
self.assertEqual(first["date"], "2024-01-02T00:00:00Z")
self.assertEqual(first["authors"], ["Jane Doe"])
self.assertEqual(first["source_provider"], "arxiv")
second = out["results"][1]
self.assertEqual(second["source_provider"], "semantic_scholar")
self.assertEqual(second["url"], "https://arxiv.org/abs/9999.1")
self.assertEqual(second["citations"], 5)
third = out["results"][2]
self.assertTrue(third["url"].startswith(
"https://www.semanticscholar.org/paper/def"))
snap = get_tracker().snapshot()
self.assertEqual(snap["by_provider"]["arxiv"]["calls"], 1)
self.assertEqual(
snap["by_provider"]["semantic_scholar"]["calls"], 1)
async def test_trims_to_count(self):
client = ScholarClient()
client._http = _session_for(self.handler_for())
out = await client.search("q", count=2)
self.assertEqual(len(out["results"]), 2)
async def test_partial_when_one_upstream_fails(self):
client = ScholarClient()
client._http = _session_for(self.handler_for(s2=False))
out = await client.search("q")
self.assertTrue(out["partial"])
self.assertIn("semantic_scholar", out["errors"])
self.assertTrue(all(r["source_provider"] == "arxiv"
for r in out["results"]))
async def test_both_failing_raises(self):
def fail(_method, _url, _kwargs):
raise ConnectionError("down")
client = ScholarClient()
client._http = _session_for(fail)
with self.assertRaises(DeepSeekError):
await client.search("q")
async def test_bad_payloads_raise(self):
def fake(_method, url, _kwargs):
if "arxiv" in url:
return _FakeResp(raw=b"<feed><entry><unclosed>")
return _FakeResp(payload=[1, 2])
client = ScholarClient()
client._http = _session_for(fake)
with self.assertRaises(DeepSeekError):
await client.search("q")
with self.assertRaises(DeepSeekError):
await client.search(" ")
class TestJina(CostCase):
async def test_markdown_passthrough(self):
client = JinaReaderClient()
client._http = _session_for(
lambda _m, _u, _k: _FakeResp(raw=b"# Hi\n\nbody"))
out = await client.fetch("https://a.test/p")
self.assertEqual(out["url"], "https://a.test/p")
self.assertEqual(out["content"], "# Hi\n\nbody")
self.assertEqual(out["fallback_provider"], "jina")
snap = get_tracker().snapshot()
self.assertEqual(snap["by_provider"]["jina"]["calls"], 1)
self.assertEqual(
snap["by_category"]["fetch"]["cost_usd"], 0.0)
async def test_auth_header_and_target(self):
seen = {}
def fake(_method, url, kwargs):
seen["url"] = url
seen["headers"] = kwargs.get("headers")
return _FakeResp(raw=b"x")
with mock.patch.dict(os.environ, {"JINA_API_KEY": "k"}):
client = JinaReaderClient()
client._http = _session_for(fake)
await client.fetch("https://a.test/p")
self.assertEqual(seen["url"], "https://r.jina.ai/https://a.test/p")
self.assertEqual(seen["headers"]["Authorization"], "Bearer k")
async def test_failures_raise(self):
client = JinaReaderClient()
with self.assertRaises(DeepSeekError):
await client.fetch("ftp://a.test/x")
def fail_402(_method, _url, _kwargs):
return _FakeResp(status=402, payload={"error": "paid"})
paid = JinaReaderClient()
paid._http = _session_for(fail_402)
with self.assertRaises(DeepSeekError) as ctx:
await paid.fetch("https://a.test/x")
self.assertEqual(ctx.exception.status, 402)
def fail_down(_method, _url, _kwargs):
raise ConnectionError("down")
down = JinaReaderClient()
down._http = _session_for(fail_down)
with self.assertRaises(DeepSeekError):
await down.fetch("https://a.test/x")
class _FakeStealthResp:
def __init__(self, status=200, text="",
headers=None):
self.status = status
self._text = text
self.headers = headers or {}
async def text(self):
return self._text
class _FakeStealth:
def __init__(self, resp=None, error=None):
self.resp = resp
self.error = error
self.urls = []
self.closed = False
async def get(self, url, headers=None, timeout=None):
self.urls.append(url)
if self.error is not None:
raise self.error
return self.resp
async def shutdown(self):
self.closed = True
class _FailStealth:
async def get(self, url, headers=None, timeout=None):
from blackcat.client import DeepSeekError
raise DeepSeekError("stealthii fetch unreachable: down")
async def shutdown(self):
pass
class FailingClient:
model = "m"
def __init__(self, status=503):
self.status = status
async def fetch(self, url):
raise DeepSeekError("down", self.status)
async def aclose(self):
pass
def _stub_search(result=None, error=None):
class Stub:
async def search(self, query, count=10):
if error is not None:
raise error
return result
return Stub()
def _stub_fetch(result=None, error=None):
class Stub:
async def fetch(self, url):
if error is not None:
raise error
return result
return Stub()
class ChainCase(unittest.IsolatedAsyncioTestCase):
def bot(self, **kwargs):
kwargs.setdefault("system", "t")
bot = BlackCatBot(provider="deepseek", api_key="k", **kwargs)
bot._client = FailingClient()
bot._stealth = StealthFetcher(stealth=_FailStealth())
self.addAsyncCleanup(bot.aclose)
return bot
class TestSearchChain(ChainCase):
async def test_rsearch_primary(self):
bot = self.bot()
bot._rsearch = _stub_search({"results": [], "fallback_provider":
"rsearch"})
bot._searxng = _stub_search({"results": [],
"fallback_provider": "searxng"})
out = await bot.search("q")
self.assertEqual(out["fallback_provider"], "rsearch")
frames = [f for f in bot.transcript
if f["kind"] == "search_fallback"]
self.assertEqual(frames, [])
kinds = [f["kind"] for f in bot.transcript]
self.assertIn("search", kinds)
async def test_searxng_second_when_rsearch_fails(self):
bot = self.bot()
bot._rsearch = _stub_search(
error=DeepSeekError("rsearch down", 503))
bot._searxng = _stub_search({"results": [],
"fallback_provider": "searxng"})
out = await bot.search("q")
self.assertEqual(out["fallback_provider"], "searxng")
frames = [f for f in bot.transcript
if f["kind"] == "search_fallback"]
self.assertEqual(frames[0]["provider"], "searxng")
self.assertIn("rsearch down", frames[0]["reason"])
async def test_all_failing_raises_last_error(self):
bot = self.bot()
bot._rsearch = _stub_search(
error=DeepSeekError("rsearch down", 503))
bot._searxng = _stub_search(
error=DeepSeekError("searxng down"))
with self.assertRaises(DeepSeekError) as ctx:
await bot.search("q")
self.assertIn("searxng down", str(ctx.exception))
async def test_chain_disabled_raises_original(self):
bot = self.bot(search_fallback=False)
bot._rsearch = _stub_search(
error=DeepSeekError("rsearch down", 503))
with self.assertRaises(DeepSeekError) as ctx:
await bot.search("q")
self.assertEqual(ctx.exception.status, 503)
class TestFetchFallback(ChainCase):
async def test_stealthii_first_wins(self):
bot = self.bot()
bot._stealth = StealthFetcher(stealth=_FakeStealth(
resp=_FakeStealthResp(
text="<html><body>" + "w" * 500 + "</body></html>",
headers={"content-type": "text/html"})))
got = await bot._fetch_chain("https://a.test", n=1)
self.assertTrue(got["ok"])
self.assertEqual(got["method"], "stealthii")
self.assertEqual(
[a["method"] for a in got["attempts"]], ["stealthii"])
async def test_jina_when_stealthii_and_provider_fail(self):
bot = self.bot()
bot._jina = _stub_fetch({"url": "https://a.test",
"content": "w" * 500,
"fallback_provider": "jina"})
got = await bot._fetch_chain("https://a.test", n=2)
self.assertTrue(got["ok"])
self.assertEqual(got["method"], "jina")
self.assertEqual(
[a["method"] for a in got["attempts"]],
["stealthii", "jina"])
async def test_all_failing_reports(self):
bot = self.bot()
bot._jina = _stub_fetch(error=DeepSeekError("jina down"))
got = await bot._fetch_chain("https://a.test", n=3)
self.assertFalse(got["ok"])
self.assertEqual(got["method"], "none")
self.assertEqual(
[a["method"] for a in got["attempts"]],
["stealthii", "provider", "jina", "direct"])
async def test_openrouter_search_uses_rsearch(self):
bot = BlackCatBot(provider="openrouter", api_key="k", system="t")
self.addAsyncCleanup(bot.aclose)
bot._rsearch = _stub_search({"results": [],
"fallback_provider": "rsearch"})
out = await bot.search("q")
self.assertEqual(out["fallback_provider"], "rsearch")
class TestStealthFetcher(unittest.IsolatedAsyncioTestCase):
async def test_html_becomes_text(self):
fetcher = StealthFetcher(stealth=_FakeStealth(
resp=_FakeStealthResp(
text="<html><body><p>Hello world</p></body></html>",
headers={"content-type": "text/html"})))
out = await fetcher.fetch("https://a.test/p")
self.assertEqual(out["fallback_provider"], "stealthii")
self.assertIn("Hello world", out["content"])
self.assertNotIn("<p>", out["content"])
async def test_non_html_passthrough(self):
fetcher = StealthFetcher(stealth=_FakeStealth(
resp=_FakeStealthResp(text="# title\n\nbody",
headers={"content-type":
"text/markdown"})))
out = await fetcher.fetch("https://a.test/p")
self.assertEqual(out["content"], "# title\n\nbody")
async def test_status_error(self):
fetcher = StealthFetcher(stealth=_FakeStealth(
resp=_FakeStealthResp(status=403, text="denied")))
with self.assertRaises(DeepSeekError) as ctx:
await fetcher.fetch("https://a.test/p")
self.assertEqual(ctx.exception.status, 403)
async def test_unreachable_becomes_deepseek_error(self):
fetcher = StealthFetcher(
stealth=_FakeStealth(error=ConnectionError("down")))
with self.assertRaises(DeepSeekError):
await fetcher.fetch("https://a.test/p")
async def test_rejects_non_http(self):
fetcher = StealthFetcher(stealth=_FakeStealth())
with self.assertRaises(DeepSeekError):
await fetcher.fetch("ftp://a.test/x")
async def test_aclose_idempotent(self):
fake = _FakeStealth()
fetcher = StealthFetcher(stealth=fake)
await fetcher.aclose()
await fetcher.aclose()
self.assertTrue(fake.closed)
class TestScholarBot(ChainCase):
async def test_live_and_echo(self):
bot = self.bot()
bot._scholar = _stub_search({"results": [{"title": "T"}],
"provider": "scholar"})
out = await bot.scholar("q")
self.assertEqual(out["results"], [{"title": "T"}])
kinds = [f["kind"] for f in bot.transcript]
self.assertIn("scholar", kinds)
async with BlackCatBot(provider="echo", system="t") as echo:
self.assertEqual((await echo.scholar("q"))["results"], [])
class TestRepoIndex(unittest.IsolatedAsyncioTestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.root = self.tmp.name
self.write("a.py", "def hello():\n return 'world'\n")
self.write("b.md", "# Title\n\nhello world again\n")
self.write("sub/c.txt", "nothing relevant here\n")
def write(self, rel, content):
path = os.path.join(self.root, rel)
os.makedirs(os.path.dirname(path) or self.root, exist_ok=True)
mode = "wb" if isinstance(content, bytes) else "w"
with open(path, mode) as fh:
fh.write(content)
return path
def index(self):
return RepoIndex(self.root)
async def test_ranked_hits_with_lines(self):
out = await self.index().search("hello world")
self.assertEqual(out["indexed_files"], 3)
paths = [r["path"] for r in out["results"]]
self.assertEqual(sorted(paths), ["a.py", "b.md"])
by_path = {r["path"]: r for r in out["results"]}
self.assertEqual(by_path["a.py"]["line"], 1)
self.assertEqual(by_path["b.md"]["line"], 3)
for hit in out["results"]:
self.assertGreaterEqual(hit["score"], 0.0)
self.assertLessEqual(hit["score"], 1.0)
self.assertIn("[", hit["snippet"])
scores = [r["score"] for r in out["results"]]
self.assertEqual(scores, sorted(scores, reverse=True))
async def test_or_fallback_when_and_misses(self):
out = await self.index().search("hello zebras")
self.assertTrue(out["results"])
self.assertFalse((await self.index().search("zebras"))["results"])
self.assertFalse((await self.index().search(" "))["results"])
async def test_freshness_add_modify_delete(self):
index = self.index()
self.assertEqual((await index.search("freshfruit"))["results"],
[])
self.write("d.txt", "freshfruit grows here\n")
hits = (await index.search("freshfruit"))["results"]
self.assertEqual([h["path"] for h in hits], ["d.txt"])
self.write("d.txt", "now it mentions allotment instead\n")
self.assertEqual((await index.search("freshfruit"))["results"],
[])
self.assertEqual((await index.search("allotment"))["results"]
[0]["path"], "d.txt")
os.remove(os.path.join(self.root, "d.txt"))
self.assertEqual((await index.search("allotment"))["results"],
[])
async def test_skips_binary_hidden_heavy_and_itself(self):
self.write("bin.dat", b"\x00\x01binary\x00blob")
self.write(".hidden/x.txt", "hidden treasure here\n")
self.write("node_modules/dep/index.js", "hidden treasure here\n")
out = await self.index().search("treasure")
self.assertEqual(out["results"], [])
self.assertEqual((await self.index().search("treasure"))
["indexed_files"], 3)
self.assertTrue(os.path.exists(
os.path.join(self.root, ".blackcat-index.sqlite3")))
async def test_bot_repo_search_and_errors(self):
bot = BlackCatBot(provider="echo", system="t",
tools=FileTools(FileState(self.root)))
self.addAsyncCleanup(bot.aclose)
out = await bot.repo_search("hello")
self.assertGreaterEqual(len(out["results"]), 1)
kinds = [f["kind"] for f in bot.transcript]
self.assertIn("repo_search", kinds)
with self.assertRaises(FileToolError):
RepoIndex(os.path.join(self.root, "no-such-dir"))
with self.assertRaises(FileToolError):
await self.index().search("hello", count="many")
class TestSearchCLI(unittest.TestCase):
def test_scholar_echo(self):
out = io.StringIO()
with redirect_stdout(out):
code = main(["--provider", "echo", "scholar", "q"])
self.assertEqual(code, 0)
self.assertEqual(json.loads(out.getvalue())["results"], [])
def test_repo_search_command(self):
with tempfile.TemporaryDirectory() as tmp:
with open(os.path.join(tmp, "f.txt"), "w") as fh:
fh.write("pineapples are tasty\n")
out = io.StringIO()
with redirect_stdout(out):
code = main(["--provider", "echo", "--tools-root", tmp,
"repo-search", "pineapples"])
self.assertEqual(code, 0)
payload = json.loads(out.getvalue())
self.assertEqual(payload["results"][0]["path"], "f.txt")
def test_flags(self):
args = build_parser().parse_args(
["--no-fetch-fallback", "fetch", "https://a.test"])
self.assertTrue(args.no_fetch_fallback)
if __name__ == "__main__":
unittest.main()