Skip to content
Repos

blackcat

Python 7.7 KB180 lines Opens in site

Directory Raw Download

"""Tests for cost awareness, the rsearch backend, and review fixes."""
# retoor <retoor@molodetz.nl>

import io
import unittest
from contextlib import redirect_stdout
from unittest import mock

from blackcat.agent import BlackCatBot
from blackcat.cli import run_chat
from blackcat.client import DeepSeekClient, DeepSeekError
from blackcat.converge import (
    Claim,
    EvidenceState,
    iterative_evidence_convergence,
)
from blackcat.costs import chat_cost_usd, format_cost_report, get_tracker
from tests.test_blackcat import _FakeSession, _json_handler
from tests.test_converge import TOPIC, extract, make_search


class TestCosts(unittest.IsolatedAsyncioTestCase):
    def setUp(self):
        get_tracker().reset()
        self.addCleanup(get_tracker().reset)

    def test_chat_pricing(self):
        cost = chat_cost_usd("deepseek-flash", {
            "prompt_tokens": 1_000_000, "completion_tokens": 1_000_000})
        self.assertAlmostEqual(cost, 0.30 + 1.20, places=4)
        cached = chat_cost_usd("deepseek-flash", {
            "prompt_tokens": 1_000_000, "completion_tokens": 0,
            "prompt_tokens_details": {"cached_tokens": 1_000_000}})
        self.assertAlmostEqual(cached, 0.006, places=4)

    async def test_tracker_categories_and_delta(self):
        client = DeepSeekClient(api_key="k")
        client._http = _FakeSession(_json_handler(
            {"choices": [{"message": {"content": "hi"}}],
             "usage": {"prompt_tokens": 100, "completion_tokens": 10}}))
        self.addAsyncCleanup(client.aclose)
        await client.chat([{"role": "user", "content": "hi"}])
        from blackcat.rsearch import RSearchClient
        rsearch = RSearchClient(base_url="https://example.invalid")
        rsearch._http = _FakeSession(_json_handler(
            {"success": True, "results": []}))
        self.addAsyncCleanup(rsearch.aclose)
        await rsearch.search("q")
        snap = get_tracker().snapshot()
        self.assertEqual(snap["by_category"]["chat"]["calls"], 1)
        self.assertEqual(snap["by_category"]["search"]["calls"], 1)
        self.assertEqual(snap["by_provider"]["rsearch"]["calls"], 1)
        self.assertAlmostEqual(
            snap["by_category"]["search"]["cost_usd"], 0.0)
        self.assertGreater(snap["total_usd"], 0)
        report = format_cost_report(snap)
        self.assertIn("chat", report)
        self.assertIn("TOTAL", report)

    async def test_openrouter_chat_priced_at_default_rate(self):
        from blackcat.client import OpenRouterClient
        client = OpenRouterClient(api_key="k")
        client._http = _FakeSession(_json_handler(
            {"choices": [{"message": {"content": "hi"}}],
             "usage": {"prompt_tokens": 1_000_000,
                       "completion_tokens": 1_000_000}}))
        self.addAsyncCleanup(client.aclose)
        await client.chat([{"role": "user", "content": "hi"}])
        snap = get_tracker().snapshot()
        self.assertEqual(snap["by_provider"]["openrouter"]["calls"], 1)
        self.assertAlmostEqual(snap["total_usd"], 0.30 + 1.20, places=4)

    async def test_costs_command_in_chat(self):
        get_tracker().record_call("search", "deepseek")
        async with BlackCatBot(provider="echo", system="t") as bot:
            out = io.StringIO()
            with mock.patch("builtins.input",
                            side_effect=["/costs", "/quit"]), \
                    redirect_stdout(out):
                await run_chat(bot)
        self.assertIn("Costs this session", out.getvalue())
        self.assertIn("TOTAL", out.getvalue())


class TestFallback(unittest.IsolatedAsyncioTestCase):
    def setUp(self):
        get_tracker().reset()
        self.addCleanup(get_tracker().reset)

    async def test_rsearch_primary(self):
        bot = BlackCatBot(provider="deepseek", api_key="k", system="t")
        self.addAsyncCleanup(bot.aclose)

        class FakeRSearch:
            async def search(self, q, count=10):
                return {"results": [{"title": "T", "url": "https://a.org",
                                     "summary": "s"}],
                        "fallback_provider": "rsearch"}
        bot._rsearch = FakeRSearch()
        out = await bot.search("anything")
        self.assertEqual(out["fallback_provider"], "rsearch")
        kinds = [f["kind"] for f in bot.transcript]
        self.assertIn("search", kinds)
        self.assertNotIn("search_fallback", kinds)

    async def test_fallback_disabled_raises(self):
        bot = BlackCatBot(provider="deepseek", api_key="k", system="t",
                      search_fallback=False)
        self.addAsyncCleanup(bot.aclose)

        class Failing:
            async def search(self, q, count=10):
                raise DeepSeekError("down", 503)
        bot._rsearch = Failing()
        with self.assertRaises(DeepSeekError):
            await bot.search("anything")

    async def test_rsearch_client_normalises(self):
        from blackcat.rsearch import RSearchClient
        client = RSearchClient(base_url="https://example.invalid")
        payload = {"success": True, "source": "brave", "results": [
            {"title": "T", "url": "https://a.org", "description": "d",
             "source": "brave"}]}
        client._http = _FakeSession(_json_handler(payload))
        self.addAsyncCleanup(client.aclose)
        out = await client.search("q", count=3)
        self.assertEqual(out["results"][0]["summary"], "d")
        self.assertEqual(out["fallback_provider"], "rsearch")
        snap = get_tracker().snapshot()
        self.assertEqual(snap["by_provider"]["rsearch"]["calls"], 1)


class TestReviewFixes(unittest.IsolatedAsyncioTestCase):
    async def test_reconciled_accounting(self):
        out = await iterative_evidence_convergence(
            TOPIC, min_iterations=1, max_iterations=2, early_stop=False,
            search_fn=make_search({}), extract_fn=extract)
        acc = out["evidence_accounting"]
        linked = acc["total_items"] - acc["unlinked"]
        self.assertGreaterEqual(
            acc["unique_supporting"] + acc["unique_opposing"]
            + acc["unique_neutral"], linked)
        self.assertEqual(acc["stance_mentions"],
                         acc["unique_supporting"] + acc["unique_opposing"]
                         + acc["unique_neutral"])
        self.assertIn("sources", out)
        self.assertIn("candidate_weighting", out)
        self.assertIn("headline heuristic confidence",
                      out["conclusion"])

    def test_candidate_kinds_grouped_and_underexposed(self):
        from blackcat.converge import (
            synthesize_conclusion,
            update_confidence_and_graph,
        )
        state = EvidenceState(topic="who started the valley conflict")
        state.candidates = [
            {"name": "1880 first clash", "kind": "date",
             "description": "a date"},
            {"name": "Group A thesis", "kind": "cause",
             "description": "a cause"}]
        claim = Claim(id="C1", text="1880 first clash started the "
                                    "valley conflict")
        state.claims.append(claim)
        from blackcat.converge import EvidenceItem
        item = EvidenceItem("E1", "https://a.org", claim.text, 0.8, 1.0,
                            "2026-01-01", 0)
        state.evidence_items.append(item)
        state.claim_evidence_graph = {"C1": ["E1"]}
        state.evidence_stances = {"E1@C1": "supporting"}
        update_confidence_and_graph(state)
        out = synthesize_conclusion(state)
        self.assertEqual(set(out["candidates_by_kind"]),
                         {"date", "cause"})
        statuses = {c["name"]: c["status"] for c in out["candidates"]}
        self.assertEqual(statuses["Group A thesis"], "underexposed")


if __name__ == "__main__":
    unittest.main()
28.1ms