Skip to content
Repos

blackcat

Python 9.3 KB227 lines Opens in site

Directory Raw Download

"""Tests for the deep_research tool (stdlib unittest, no network).

Search, fetch, Jina, and direct HTTP are all replaced with fakes on
the bot instance, so the whole fallback chain runs offline.
"""

# retoor <retoor@molodetz.nl>
import unittest
from unittest import mock

from blackcat import research as R
from blackcat.agent import BlackCatBot
from blackcat.client import DeepSeekError

PAGE_A = (
    "WidgetOS handbook chapter on performance tuning and scheduling. "
    "WidgetOS 3.2 ships a new scheduler that cuts latency by 30 "
    "percent on servers under load. The scheduler uses deadline "
    "queues and per-core load balancing for predictable latency. "
    "Independent benchmarks confirm the latency improvement across "
    "database and web workloads with default settings enabled. "
    "Operators can tune the scheduler through the performance "
    "profile that ships with the standard server installation. "
    "The release also improves latency for desktop clients during "
    "background synchronization and large file transfers overall.")
PAGE_B = (
    "Release coverage from the community news desk this week. "
    "WidgetOS 3.2 ships a new scheduler that cuts latency by 30 "
    "percent on servers under load. Reporters measured smoother "
    "dashboards after the upgrade on two production clusters. "
    "The project says the scheduler work took three release cycles "
    "and involved the core platform team plus outside reviewers. "
    "Desktop builds receive the same kernel with a different "
    "default profile tuned for battery life and responsiveness.")


def _results(*rows):
    return {"results": [{"title": t, "url": u, "summary": s}
                        for t, u, s in rows]}


class TestHelpers(unittest.TestCase):
    def test_plan_queries_angles_and_cap(self):
        queries = R.plan_queries("quantum chips", max_queries=3)
        self.assertEqual(len(queries), 3)
        self.assertEqual(queries[0], "quantum chips")
        self.assertIn("official documentation", queries[1])
        self.assertEqual(len(R.plan_queries("x", max_queries=99)), 5)

    def test_canonical_url_strips_tracking(self):
        url = R.canonical_url(
            "HTTPS://Example.COM/a/b/?utm_source=x&q=1#frag")
        self.assertEqual(url, "https://example.com/a/b?q=1")

    def test_dedupe_by_canonical_url(self):
        rows = [{"url": "https://a.com/x?utm_campaign=y"},
                {"url": "https://a.com/x"},
                {"url": "https://b.com/y"}, {"url": ""}, "junk"]
        out = R.dedupe_results(rows)
        self.assertEqual(len(out), 2)

    def test_domain_tiers(self):
        self.assertEqual(R.domain_tier("docs.python.org"), 2)
        self.assertEqual(R.domain_tier("agency.gov"), 3)
        self.assertEqual(R.domain_tier("blog.example.com"), 1)

    def test_rank_sources_domain_cap(self):
        rows = [{"title": "latency widget", "url": "https://a.com/1",
                 "summary": "latency"},
                {"title": "latency widget", "url": "https://a.com/2",
                 "summary": "latency"},
                {"title": "latency widget", "url": "https://a.com/3",
                 "summary": "latency"},
                {"title": "latency widget", "url": "https://b.org/1",
                 "summary": "latency"}]
        picked = R.rank_sources("latency widget", rows, max_sources=6,
                                per_domain=2)
        self.assertEqual(len(picked), 3)
        self.assertEqual([p["n"] for p in picked], [1, 2, 3])

    def test_extract_and_merge_claims(self):
        terms = R.topic_terms("WidgetOS latency improvements")
        per_source = [
            (1, "a.com", R.extract_claims(PAGE_A, terms)),
            (2, "b.org", R.extract_claims(PAGE_B, terms))]
        self.assertTrue(per_source[0][2])
        merged = R.merge_claims(per_source)
        top = merged[0]
        self.assertEqual(top["confidence"], "high")
        self.assertEqual(len(top["domains"]), 2)
        self.assertIn("cuts latency by 30", top["text"])


class _FakeJina:
    def __init__(self, content="", error=None):
        self.content = content
        self.error = error

    async def fetch(self, url):
        if self.error:
            raise self.error
        return {"url": url, "content": self.content,
                "fallback_provider": "jina"}


class _FakeStealth:
    async def fetch(self, url):
        raise DeepSeekError("stealthii down")


class ToolCase(unittest.IsolatedAsyncioTestCase):
    def bot(self):
        bot = BlackCatBot(provider="echo", system="t")
        self.addAsyncCleanup(bot.aclose)

        async def _search(query, count=10):
            return _results(
            ("WidgetOS latency guide", "https://docs.example.org/w",
             "WidgetOS 3.2 ships a new scheduler that cuts latency "
             "by 30 percent on servers."),
            ("WidgetOS latency news", "https://news.example.com/s",
             "WidgetOS 3.2 ships a new scheduler that cuts latency "
             "by 30 percent on servers."))
        bot.search = _search
        bot._jina = _FakeJina(error=DeepSeekError("jina down"))
        bot._stealth = _FakeStealth()
        return bot


class TestTool(ToolCase):
    async def test_provider_fetch_happy_path(self):
        bot = self.bot()
        pages = {"https://docs.example.org/w": PAGE_A,
                 "https://news.example.com/s": PAGE_B}
        async def _fetch(url):
            return {"url": url, "content": pages[url]}
        bot.fetch = _fetch
        events = []
        bot._research_event = lambda kind, **kw: events.append(kind)
        out = await bot.run_tool("deep_research",
                           topic="WidgetOS latency scheduler",
                           max_queries=2, max_sources=4)
        self.assertTrue(out["ok"])
        self.assertEqual(out["stats"]["fetched"], 2)
        self.assertEqual({s["method"] for s in out["sources"]},
                         {"provider"})
        self.assertTrue(out["claims"])
        self.assertEqual(out["claims"][0]["confidence"], "high")
        self.assertIn("[1", out["report"])
        core = [e for e in events if e != "fetch_attempt"]
        self.assertEqual(core, ["plan", "query", "query", "ranked",
                                "fetch", "fetch", "synth"])
        self.assertIn("fetch_attempt", events)
        self.assertLess(events.index("fetch_attempt"),
                        events.index("fetch"))

    async def test_jina_fallback_when_provider_short(self):
        bot = self.bot()
        async def _fetch(url):
            return {"url": url, "content": "tiny"}
        bot.fetch = _fetch
        bot._jina = _FakeJina(content=PAGE_A)
        out = await bot.run_tool("deep_research",
                           topic="WidgetOS latency scheduler",
                           max_queries=1, max_sources=2)
        self.assertTrue(out["ok"])
        first = out["sources"][0]
        self.assertEqual(first["method"], "jina")
        self.assertEqual([a["method"] for a in first["attempts"]],
                         ["stealthii", "provider", "jina"])
        self.assertGreaterEqual(first["chars"], 400)

    async def test_direct_fallback_when_provider_and_jina_fail(self):
        bot = self.bot()

        async def boom(url):
            raise DeepSeekError("provider down")

        bot.fetch = boom
        with mock.patch("blackcat.reader.direct_fetch",
                        new=mock.AsyncMock(
                            return_value={"url": "x", "content": PAGE_B,
                                          "fallback_provider": "direct"})):
            out = await bot.run_tool("deep_research",
                               topic="WidgetOS latency scheduler",
                               max_queries=1, max_sources=2)
        self.assertTrue(out["ok"])
        self.assertEqual(out["sources"][0]["method"], "direct")
        methods = [a["method"] for a in out["sources"][0]["attempts"]]
        self.assertEqual(
            methods, ["stealthii", "provider", "jina", "direct"])

    async def test_summary_only_marks_unknowns_and_low_confidence(self):
        bot = self.bot()
        async def _fetch(url):
            return {"url": url, "content": ""}
        bot.fetch = _fetch
        with mock.patch("blackcat.reader.direct_fetch",
                        new=mock.AsyncMock(
                            side_effect=DeepSeekError("no network"))):
            out = await bot.run_tool("deep_research",
                               topic="WidgetOS latency scheduler",
                               max_queries=1, max_sources=2)
        self.assertTrue(out["ok"])
        self.assertEqual(out["stats"]["fetched"], 0)
        self.assertTrue(out["unknowns"])
        self.assertTrue(all(c["confidence"] == "low"
                            for c in out["claims"]))

    async def test_no_sources_is_clean_error(self):
        bot = self.bot()
        async def _search(query, count=10):
            return {"results": []}
        bot.search = _search
        out = await bot.run_tool("deep_research", topic="nothing anywhere")
        self.assertFalse(out["ok"])
        self.assertEqual(out["code"], "no_evidence")

    async def test_bad_topic_rejected(self):
        bot = self.bot()
        out = await bot.run_tool("deep_research", topic="  ")
        self.assertFalse(out["ok"])
        self.assertEqual(out["code"], "bad_args")


if __name__ == "__main__":
    unittest.main()
26.9ms