Skip to content
Repos

blackcat

Python 21.1 KB510 lines Opens in site

Directory Raw Download

"""Committed tests for the DeepSeek tool-calling agent loop (no network).

A scripted stub client stands in for DeepSeekClient: it returns canned
chat responses (tool_calls, then final text) and records every wire it
received, so loop shape, error feedback, and safeguards are asserted
offline against a tmp-root FileTools.
"""
# retoor <retoor@molodetz.nl>

import inspect
import json
import os
import tempfile
import unittest
from unittest import mock

from blackcat.agent import (
    MUTATING_TOOLS,
    TOOL_NAMES,
    TOOL_SCHEMAS,
    BlackCatBot,
)
from blackcat.cli import build_parser, main
from blackcat.client import DeepSeekClient, DeepSeekError
from blackcat.fstate import FileState
from blackcat.tools import FileTools


def _call(cid, name, arguments):
    return {"id": cid, "type": "function",
            "function": {"name": name, "arguments": arguments}}


def _tool_resp(*calls):
    return {"choices": [{"message": {"role": "assistant", "content": None,
                                     "tool_calls": list(calls)},
                         "finish_reason": "tool_calls"}],
            "usage": {"prompt_tokens": 10}}


def _text_resp(text):
    return {"choices": [{"message": {"role": "assistant", "content": text},
                         "finish_reason": "stop"}],
            "usage": {"prompt_tokens": 10}}


class StubClient:
    model = "stub-model"

    def __init__(self, script):
        self.script = list(script)
        self.calls = []

    async def chat(self, messages, **kwargs):
        self.calls.append((messages, kwargs))
        item = self.script.pop(0)
        if isinstance(item, Exception):
            raise item
        return item


class LoopCase(unittest.IsolatedAsyncioTestCase):
    def setUp(self):
        self.tmp = tempfile.TemporaryDirectory()
        self.addCleanup(self.tmp.cleanup)

    def bot(self, script, **kwargs):
        kwargs.setdefault("system", "t")
        bot = BlackCatBot(provider="deepseek", api_key="k",
                      tools=FileTools(FileState(self.tmp.name)), **kwargs)
        bot._client = StubClient(script)
        self.addAsyncCleanup(bot.aclose)
        return bot

    def path(self, name):
        return os.path.join(self.tmp.name, name)


class TestSchemas(unittest.IsolatedAsyncioTestCase):
    def test_names_are_canonical_tools(self):
        self.assertEqual(TOOL_NAMES, ("read", "grep", "outline", "edit",
                                      "multi_edit", "write_file",
                                      "apply_patch", "bash", "browse",
                                      "download",
                                      "search", "fetch",
                                      "iterative_evidence_convergence",
                                      "deep_research", "todo",
                                      "delegate", "task_output",
                                      "read_spill"))
        self.assertNotIn("write", TOOL_NAMES)

    def test_schemas_match_signatures(self):
        for schema in TOOL_SCHEMAS:
            fn = schema["function"]
            self.assertEqual(schema["type"], "function")
            if fn["name"] == "iterative_evidence_convergence":
                method = BlackCatBot.iterative_evidence_convergence
            elif fn["name"] == "bash":
                method = BlackCatBot.bash
            elif fn["name"] == "browse":
                method = BlackCatBot.browse
            elif fn["name"] == "download":
                method = BlackCatBot.download
            elif fn["name"] == "fetch":
                method = BlackCatBot.web_fetch
            elif fn["name"] == "search":
                method = BlackCatBot.web_search
            elif fn["name"] == "todo":
                method = BlackCatBot.todo
            elif fn["name"] == "delegate":
                method = BlackCatBot.delegate
            elif fn["name"] == "task_output":
                method = BlackCatBot.task_output
            elif fn["name"] == "read_spill":
                method = BlackCatBot.read_spill
            elif fn["name"] == "deep_research":
                method = BlackCatBot.deep_research
            else:
                method = getattr(FileTools, fn["name"])
            params = inspect.signature(method).parameters
            required = [n for n, p in params.items()
                        if n != "self" and p.default is inspect._empty]
            self.assertEqual(fn["parameters"]["required"], required,
                             fn["name"])
            self.assertFalse(fn["parameters"]["additionalProperties"])
            for name in fn["parameters"]["properties"]:
                self.assertIn(name, params, fn["name"])


class TestLoop(LoopCase):
    async def test_write_file_then_answer(self):
        bot = self.bot([
            _tool_resp(_call("c1", "write_file", json.dumps(
                {"path": "aa.md", "content": "xx\n"}))),
            _text_resp("done")])
        self.assertEqual(await bot.prompt("Write xx to aa.md"), "done")
        with open(self.path("aa.md"), encoding="utf-8") as fh:
            self.assertEqual(fh.read(), "xx\n")
        wire, kwargs = bot._client.calls[0]
        self.assertEqual(kwargs["tools"], TOOL_SCHEMAS)
        self.assertEqual(kwargs["tool_choice"], "auto")
        roles = [m["role"] for m in bot.messages]
        self.assertEqual(roles, ["user", "assistant", "tool", "assistant"])
        tool_wire = bot._wire_messages()[2]
        self.assertEqual(set(tool_wire),
                         {"role", "tool_call_id", "content"})
        self.assertEqual(tool_wire["tool_call_id"], "c1")
        self.assertIsNone(BlackCatBot.validate_wire(bot._wire_messages()))
        self.assertTrue(json.loads(tool_wire["content"])["ok"])

    async def test_read_then_answer(self):
        with open(self.path("f.txt"), "w", encoding="utf-8") as fh:
            fh.write("hello\n")
        bot = self.bot([
            _tool_resp(_call("c1", "read", '{"path": "f.txt"}')),
            _text_resp("it says hello")])
        self.assertEqual(await bot.prompt("read f"), "it says hello")
        self.assertIn("hello", bot.messages[2]["content"])

    async def test_malformed_arguments_fed_back(self):
        bot = self.bot([
            _tool_resp(_call("c1", "write_file", "{not json")),
            _text_resp("recovered")])
        self.assertEqual(await bot.prompt("write"), "recovered")
        self.assertFalse(os.path.exists(self.path("aa.md")))
        result = json.loads(bot.messages[2]["content"])
        self.assertEqual(result["code"], "bad_arguments")

    async def test_unknown_tool_and_bad_args_fed_back(self):
        bot = self.bot([
            _tool_resp(_call("c1", "nope", "{}"),
                       _call("c2", "read", '{"file_path": "x"}')),
            _text_resp("ok")])
        self.assertEqual(await bot.prompt("go"), "ok")
        codes = [json.loads(m["content"])["code"]
                 for m in bot.messages if m.get("role") == "tool"]
        self.assertEqual(codes, ["unknown_tool", "bad_args"])

    async def test_tool_exception_fed_back_pair_closed(self):
        with open(self.path("f.txt"), "w", encoding="utf-8") as fh:
            fh.write("a\n")
        bot = self.bot([
            _tool_resp(_call("c1", "read", '{"path": "f.txt"}')),
            _tool_resp(_call("c2", "edit", json.dumps(
                {"path": "f.txt", "old_string": 123, "new_string": "b"}))),
            _text_resp("survived")])
        self.assertEqual(await bot.prompt("edit"), "survived")
        result = json.loads(bot.messages[4]["content"])
        self.assertEqual(result["code"], "tool_exception")
        self.assertIsNone(BlackCatBot.validate_wire(bot._wire_messages()))

    async def test_read_only_tools_refuse_writes(self):
        bot = BlackCatBot(provider="deepseek", api_key="k", system="t",
                      tools=FileTools(FileState(self.tmp.name,
                                                write=False)))
        self.addAsyncCleanup(bot.aclose)
        bot._client = StubClient([
            _tool_resp(_call("c1", "write_file", json.dumps(
                {"path": "aa.md", "content": "xx"}))),
            _text_resp("cannot")])
        self.assertEqual(await bot.prompt("write"), "cannot")
        self.assertFalse(os.path.exists(self.path("aa.md")))
        result = json.loads(bot.messages[2]["content"])
        self.assertEqual(result["code"], "security")


class TestLoopSafeguards(LoopCase):
    async def test_max_iterations_forces_final_without_tools(self):
        bot = self.bot([
            _tool_resp(_call("c1", "read", '{"path": "missing"}')),
            _tool_resp(_call("c2", "read", '{"path": "missing"}')),
            _text_resp("final")], max_tool_iterations=2)
        self.assertEqual(await bot.prompt("loop"), "final")
        self.assertEqual(len(bot._client.calls), 3)
        self.assertNotIn("tools", bot._client.calls[-1][1])
        events = [f.get("event") for f in bot.transcript
                  if f["kind"] == "tool_loop"]
        self.assertIn("tool_loop_limit", events)

    async def test_identical_failure_breaker(self):
        script = [_tool_resp(_call("c%d" % i, "read",
                                  '{"path": "missing"}'))
                  for i in range(3)] + [_text_resp("gave up")]
        bot = self.bot(script, max_tool_iterations=16)
        self.assertEqual(await bot.prompt("loop"), "gave up")
        self.assertEqual(len(bot._client.calls), 4)
        events = [f.get("event") for f in bot.transcript
                  if f["kind"] == "tool_loop"]
        self.assertIn("loop_breaker", events)

    async def test_missing_tool_call_id_raises_history_valid(self):
        bot = self.bot([_tool_resp({"type": "function", "function": {
            "name": "read", "arguments": "{}"}})])
        with self.assertRaises(DeepSeekError):
            await bot.prompt("go")
        self.assertIsNone(BlackCatBot.validate_wire(bot._wire_messages()))
        self.assertEqual([m["role"] for m in bot.messages], ["user"])

    async def test_429_retried_then_success(self):
        bot = self.bot([DeepSeekError("rate", 429), _text_resp("ok")])
        with mock.patch("blackcat.agent.asyncio.sleep", new=mock.AsyncMock()):
            self.assertEqual(await bot.prompt("hi"), "ok")
        self.assertEqual(len(bot._client.calls), 2)

    async def test_401_raised_without_retry(self):
        bot = self.bot([DeepSeekError("auth", 401)])
        with self.assertRaises(DeepSeekError):
            await bot.prompt("hi")
        self.assertEqual(len(bot._client.calls), 1)

    async def test_no_tools_is_plain_chat(self):
        bot = self.bot([_text_resp("plain")], enable_tools=False)
        self.assertEqual(await bot.prompt("hi"), "plain")
        self.assertNotIn("tools", bot._client.calls[0][1])

    async def test_echo_never_loops(self):
        async with BlackCatBot(provider="echo", system="t",
                     tools=FileTools(FileState(self.tmp.name))) as bot:
            self.assertEqual(await bot.prompt("Write xx to aa.md"),
                             "echo: Write xx to aa.md")
        self.assertFalse(os.path.exists(self.path("aa.md")))


class TestFetchTool(LoopCase):
    async def test_fetch_then_answer(self):
        page = {"url": "https://a.test/p", "content": "live page words"}
        bot = self.bot([
            _tool_resp(_call("c1", "fetch", json.dumps(
                {"url": "https://a.test/p"}))),
            _text_resp("got it")])
        with mock.patch.object(BlackCatBot, "fetch", new=mock.AsyncMock(return_value=page)):
            self.assertEqual(await bot.prompt("is https://a.test/p secure?"),
                             "got it")
        result = json.loads(bot.messages[2]["content"])
        self.assertTrue(result["ok"])
        self.assertIn("live page words", bot.messages[2]["content"])
        self.assertIsNone(BlackCatBot.validate_wire(bot._wire_messages()))
        system = bot._client.calls[0][0][0]["content"]
        self.assertIn("Use fetch for live web pages", system)

    async def test_fetch_failure_fed_back(self):
        bot = self.bot([])
        with mock.patch.object(BlackCatBot, "fetch",
                               new=mock.AsyncMock(side_effect=DeepSeekError("down", 503))):
            result = await bot.run_tool("fetch", url="https://a.test/p")
        self.assertEqual(result["code"], "fetch_failed")
        self.assertIn("down", result["error"])


class TestSearchTool(LoopCase):
    hits = {"query": "deepseek v4", "results": [
        {"title": "changelog", "url": "https://a.test/c",
         "date": "2026-11-01", "summary": "flash shipped"}]}

    async def test_search_then_answer(self):
        bot = self.bot([
            _tool_resp(_call("c1", "search", json.dumps(
                {"query": "deepseek v4", "count": 3}))),
            _text_resp("found it")])
        with mock.patch.object(BlackCatBot, "search", new=mock.AsyncMock(return_value=self.hits)) as m:
            self.assertEqual(await bot.prompt("search deepseek v4"), "found it")
        m.assert_called_once_with("deepseek v4", 3)
        result = json.loads(bot.messages[2]["content"])
        self.assertTrue(result["ok"])
        self.assertIn("flash shipped", bot.messages[2]["content"])
        self.assertIsNone(BlackCatBot.validate_wire(bot._wire_messages()))

    async def test_search_failure_fed_back(self):
        bot = self.bot([])
        with mock.patch.object(BlackCatBot, "search",
                               new=mock.AsyncMock(side_effect=DeepSeekError("down", 503))):
            result = await bot.run_tool("search", query="x")
        self.assertEqual(result["code"], "search_failed")
        self.assertIn("down", result["error"])

    async def test_search_is_not_mutating(self):
        bot = self.bot([])
        self.assertNotIn("search", MUTATING_TOOLS)
        with mock.patch.object(BlackCatBot, "search", new=mock.AsyncMock(return_value=self.hits)):
            self.assertTrue((await bot.run_tool("search", query="q"))["ok"])


class TestBashTool(LoopCase):
    async def test_bash_is_mutating(self):
        self.assertIn("bash", MUTATING_TOOLS)

    async def test_bash_runs_and_reports(self):
        bot = self.bot([])
        result = await bot.run_tool("bash", command="echo hi")
        self.assertTrue(result["ok"])
        self.assertEqual(result["exit_code"], 0)
        self.assertEqual(result["stdout"], "hi\n")
        self.assertFalse(result["timed_out"])
        self.assertIsNone(BlackCatBot.validate_wire(bot._wire_messages()))

    async def test_bash_failure_fed_back(self):
        bot = self.bot([])
        result = await bot.run_tool("bash", command="exit 2")
        self.assertFalse(result["ok"])
        self.assertEqual(result["exit_code"], 2)
        self.assertEqual(result["code"], "nonzero_exit")

    async def test_confirm_deny_blocks_bash(self):
        bot = self.bot([], tool_confirm=lambda name, args: False)
        result = await bot.run_tool("bash", command="echo hi")
        self.assertFalse(result["ok"])
        self.assertEqual(result["code"], "rejected")

    async def test_yolo_bypasses_confirm_for_bash(self):
        bot = self.bot([], tool_confirm=lambda name, args: False,
                       yolo=True)
        result = await bot.run_tool("bash", command="echo hi")
        self.assertTrue(result["ok"])

    async def test_yolo_bash_leaves_root(self):
        bot = self.bot([], yolo=True)
        result = await bot.run_tool("bash", command="pwd", cwd="/tmp")
        self.assertTrue(result["ok"])
        self.assertEqual(result["cwd"], "/tmp")

    async def test_sandboxed_bash_cannot_leave_root(self):
        bot = self.bot([])
        result = await bot.run_tool("bash", command="pwd", cwd="/tmp")
        self.assertFalse(result["ok"])
        self.assertEqual(result["code"], "security")

    async def test_read_only_bash_refuses(self):
        bot = BlackCatBot(provider="deepseek", api_key="k", system="t",
                          tools=FileTools(FileState(self.tmp.name,
                                                    write=False)))
        self.addAsyncCleanup(bot.aclose)
        result = await bot.run_tool("bash", command="echo hi")
        self.assertFalse(result["ok"])
        self.assertEqual(result["code"], "security")

    async def test_env_values_never_reach_transcript(self):
        bot = self.bot([])
        secret = "transcript-secret-abc"
        result = await bot.run_tool(
            "bash", command="echo $BLACKCAT_T",
            env={"BLACKCAT_T": secret})
        self.assertTrue(result["ok"])
        blob = json.dumps(bot.transcript)
        self.assertNotIn(secret, blob)
        self.assertIn("BLACKCAT_T", blob)


class TestCompactionAtomic(unittest.IsolatedAsyncioTestCase):
    async def test_clearing_keeps_pairs_valid(self):
        bot = BlackCatBot(provider="echo", system="t", keep_recent_tools=1)
        self.addAsyncCleanup(bot.aclose)
        for i in range(3):
            bot.messages.append({
                "role": "assistant", "content": None,
                "tool_calls": [_call("c%d" % i, "read", "{}")]})
            bot.messages.append({"role": "tool", "tool_call_id": "c%d" % i,
                                 "content": "result %d" % i})
        bot._clear_old_tool_results()
        self.assertIsNone(BlackCatBot.validate_wire(bot._wire_messages()))
        self.assertIn("cleared", bot.messages[1]["content"])
        self.assertEqual(bot.messages[-1]["content"], "result 2")


class TestClientHardening(unittest.IsolatedAsyncioTestCase):
    async def test_message_of_keeps_tool_calls(self):
        resp = _tool_resp(_call("c1", "read", "{}"))
        msg = DeepSeekClient.message_of(resp)
        self.assertEqual(msg["tool_calls"][0]["id"], "c1")
        with self.assertRaises(DeepSeekError):
            DeepSeekClient.message_of({"choices": []})

    async def test_malformed_json_body_is_deepseek_error(self):
        client = DeepSeekClient(api_key="k")

        class _Raw:
            status = 200
            headers = {}

            async def __aenter__(self):
                return self

            async def __aexit__(self, *exc):
                return False

            async def read(self):
                return b"not json"

        class _Session:
            closed = False

            def post(self, _url, **_kwargs):
                return _Raw()

            async def close(self):
                pass

        client._http = _Session()
        with self.assertRaises(DeepSeekError):
            await client.chat([{"role": "user", "content": "hi"}])


class TestYolo(unittest.IsolatedAsyncioTestCase):
    def setUp(self):
        self.tmp = tempfile.TemporaryDirectory()
        self.addCleanup(self.tmp.cleanup)
        self.root = os.path.join(self.tmp.name, "ws")
        os.mkdir(self.root)

    def bot(self, **kwargs):
        kwargs.setdefault("system", "t")
        bot = BlackCatBot(provider="echo",
                          tools=FileTools(FileState(self.root)), **kwargs)
        self.addAsyncCleanup(bot.aclose)
        return bot

    async def test_confirm_deny_blocks_without_yolo(self):
        bot = self.bot(tool_confirm=lambda name, args: False)
        self.assertFalse(bot.yolo)
        result = await bot.run_tool("write_file", path="a.txt", content="x")
        self.assertFalse(result["ok"])
        self.assertEqual(result["code"], "rejected")
        self.assertFalse(os.path.exists(os.path.join(self.root, "a.txt")))

    async def test_yolo_bypasses_confirm(self):
        bot = self.bot(tool_confirm=lambda name, args: False, yolo=True)
        self.assertTrue(bot.yolo)
        result = await bot.run_tool("write_file", path="a.txt", content="x")
        self.assertTrue(result["ok"])
        self.assertTrue(os.path.exists(os.path.join(self.root, "a.txt")))

    async def test_yolo_unsandboxes_and_enables_writes(self):
        tools = FileTools(FileState(self.root, write=False))
        bot = BlackCatBot(provider="echo", system="t", tools=tools,
                          yolo=True)
        self.addAsyncCleanup(bot.aclose)
        self.assertFalse(tools.state.sandboxed)
        self.assertTrue(tools.state.write_enabled)
        outside = os.path.join(self.tmp.name, "outside.txt")
        result = await bot.run_tool("write_file", path=outside, content="free")
        self.assertTrue(result["ok"])
        self.assertTrue(os.path.exists(outside))

    async def test_yolo_lazy_tools_unsandboxed(self):
        bot = BlackCatBot(provider="echo", system="t", yolo=True)
        self.addAsyncCleanup(bot.aclose)
        self.assertFalse(bot._file_tools().state.sandboxed)


class TestCLIFlags(unittest.IsolatedAsyncioTestCase):
    def test_parser_flags(self):
        args = build_parser().parse_args(
            ["--no-tools", "--max-tool-steps", "3", "--tools-root", "/tmp",
             "--read-only-tools", "prompt", "hi"])
        self.assertTrue(args.no_tools)
        self.assertEqual(args.max_tool_steps, 3)
        self.assertTrue(args.read_only_tools)

    def test_yolo_flag(self):
        args = build_parser().parse_args(["--yolo", "prompt", "hi"])
        self.assertTrue(args.yolo)
        args = build_parser().parse_args(["prompt", "hi"])
        self.assertFalse(args.yolo)

    def test_bad_max_steps_exit_2(self):
        self.assertEqual(main(["--provider", "echo", "--max-tool-steps",
                               "0", "prompt", "hi"]), 2)


if __name__ == "__main__":
    unittest.main()
30.7ms