"""Committed tests for the DeepSeek tool-calling agent loop (no network). A scripted stub client stands in for DeepSeekClient: it returns canned chat responses (tool_calls, then final text) and records every wire it received, so loop shape, error feedback, and safeguards are asserted offline against a tmp-root FileTools. """ # retoor import inspect import json import os import tempfile import unittest from unittest import mock from blackcat.agent import ( MUTATING_TOOLS, TOOL_NAMES, TOOL_SCHEMAS, BlackCatBot, ) from blackcat.cli import build_parser, main from blackcat.client import DeepSeekClient, DeepSeekError from blackcat.fstate import FileState from blackcat.tools import FileTools def _call(cid, name, arguments): return {"id": cid, "type": "function", "function": {"name": name, "arguments": arguments}} def _tool_resp(*calls): return {"choices": [{"message": {"role": "assistant", "content": None, "tool_calls": list(calls)}, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 10}} def _text_resp(text): return {"choices": [{"message": {"role": "assistant", "content": text}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 10}} class StubClient: model = "stub-model" def __init__(self, script): self.script = list(script) self.calls = [] async def chat(self, messages, **kwargs): self.calls.append((messages, kwargs)) item = self.script.pop(0) if isinstance(item, Exception): raise item return item class LoopCase(unittest.IsolatedAsyncioTestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) def bot(self, script, **kwargs): kwargs.setdefault("system", "t") bot = BlackCatBot(provider="deepseek", api_key="k", tools=FileTools(FileState(self.tmp.name)), **kwargs) bot._client = StubClient(script) self.addAsyncCleanup(bot.aclose) return bot def path(self, name): return os.path.join(self.tmp.name, name) class TestSchemas(unittest.IsolatedAsyncioTestCase): def test_names_are_canonical_tools(self): self.assertEqual(TOOL_NAMES, ("read", "grep", "outline", "edit", "multi_edit", "write_file", "apply_patch", "bash", "browse", "download", "search", "fetch", "iterative_evidence_convergence", "deep_research", "todo", "delegate", "task_output", "read_spill")) self.assertNotIn("write", TOOL_NAMES) def test_schemas_match_signatures(self): for schema in TOOL_SCHEMAS: fn = schema["function"] self.assertEqual(schema["type"], "function") if fn["name"] == "iterative_evidence_convergence": method = BlackCatBot.iterative_evidence_convergence elif fn["name"] == "bash": method = BlackCatBot.bash elif fn["name"] == "browse": method = BlackCatBot.browse elif fn["name"] == "download": method = BlackCatBot.download elif fn["name"] == "fetch": method = BlackCatBot.web_fetch elif fn["name"] == "search": method = BlackCatBot.web_search elif fn["name"] == "todo": method = BlackCatBot.todo elif fn["name"] == "delegate": method = BlackCatBot.delegate elif fn["name"] == "task_output": method = BlackCatBot.task_output elif fn["name"] == "read_spill": method = BlackCatBot.read_spill elif fn["name"] == "deep_research": method = BlackCatBot.deep_research else: method = getattr(FileTools, fn["name"]) params = inspect.signature(method).parameters required = [n for n, p in params.items() if n != "self" and p.default is inspect._empty] self.assertEqual(fn["parameters"]["required"], required, fn["name"]) self.assertFalse(fn["parameters"]["additionalProperties"]) for name in fn["parameters"]["properties"]: self.assertIn(name, params, fn["name"]) class TestLoop(LoopCase): async def test_write_file_then_answer(self): bot = self.bot([ _tool_resp(_call("c1", "write_file", json.dumps( {"path": "aa.md", "content": "xx\n"}))), _text_resp("done")]) self.assertEqual(await bot.prompt("Write xx to aa.md"), "done") with open(self.path("aa.md"), encoding="utf-8") as fh: self.assertEqual(fh.read(), "xx\n") wire, kwargs = bot._client.calls[0] self.assertEqual(kwargs["tools"], TOOL_SCHEMAS) self.assertEqual(kwargs["tool_choice"], "auto") roles = [m["role"] for m in bot.messages] self.assertEqual(roles, ["user", "assistant", "tool", "assistant"]) tool_wire = bot._wire_messages()[2] self.assertEqual(set(tool_wire), {"role", "tool_call_id", "content"}) self.assertEqual(tool_wire["tool_call_id"], "c1") self.assertIsNone(BlackCatBot.validate_wire(bot._wire_messages())) self.assertTrue(json.loads(tool_wire["content"])["ok"]) async def test_read_then_answer(self): with open(self.path("f.txt"), "w", encoding="utf-8") as fh: fh.write("hello\n") bot = self.bot([ _tool_resp(_call("c1", "read", '{"path": "f.txt"}')), _text_resp("it says hello")]) self.assertEqual(await bot.prompt("read f"), "it says hello") self.assertIn("hello", bot.messages[2]["content"]) async def test_malformed_arguments_fed_back(self): bot = self.bot([ _tool_resp(_call("c1", "write_file", "{not json")), _text_resp("recovered")]) self.assertEqual(await bot.prompt("write"), "recovered") self.assertFalse(os.path.exists(self.path("aa.md"))) result = json.loads(bot.messages[2]["content"]) self.assertEqual(result["code"], "bad_arguments") async def test_unknown_tool_and_bad_args_fed_back(self): bot = self.bot([ _tool_resp(_call("c1", "nope", "{}"), _call("c2", "read", '{"file_path": "x"}')), _text_resp("ok")]) self.assertEqual(await bot.prompt("go"), "ok") codes = [json.loads(m["content"])["code"] for m in bot.messages if m.get("role") == "tool"] self.assertEqual(codes, ["unknown_tool", "bad_args"]) async def test_tool_exception_fed_back_pair_closed(self): with open(self.path("f.txt"), "w", encoding="utf-8") as fh: fh.write("a\n") bot = self.bot([ _tool_resp(_call("c1", "read", '{"path": "f.txt"}')), _tool_resp(_call("c2", "edit", json.dumps( {"path": "f.txt", "old_string": 123, "new_string": "b"}))), _text_resp("survived")]) self.assertEqual(await bot.prompt("edit"), "survived") result = json.loads(bot.messages[4]["content"]) self.assertEqual(result["code"], "tool_exception") self.assertIsNone(BlackCatBot.validate_wire(bot._wire_messages())) async def test_read_only_tools_refuse_writes(self): bot = BlackCatBot(provider="deepseek", api_key="k", system="t", tools=FileTools(FileState(self.tmp.name, write=False))) self.addAsyncCleanup(bot.aclose) bot._client = StubClient([ _tool_resp(_call("c1", "write_file", json.dumps( {"path": "aa.md", "content": "xx"}))), _text_resp("cannot")]) self.assertEqual(await bot.prompt("write"), "cannot") self.assertFalse(os.path.exists(self.path("aa.md"))) result = json.loads(bot.messages[2]["content"]) self.assertEqual(result["code"], "security") class TestLoopSafeguards(LoopCase): async def test_max_iterations_forces_final_without_tools(self): bot = self.bot([ _tool_resp(_call("c1", "read", '{"path": "missing"}')), _tool_resp(_call("c2", "read", '{"path": "missing"}')), _text_resp("final")], max_tool_iterations=2) self.assertEqual(await bot.prompt("loop"), "final") self.assertEqual(len(bot._client.calls), 3) self.assertNotIn("tools", bot._client.calls[-1][1]) events = [f.get("event") for f in bot.transcript if f["kind"] == "tool_loop"] self.assertIn("tool_loop_limit", events) async def test_identical_failure_breaker(self): script = [_tool_resp(_call("c%d" % i, "read", '{"path": "missing"}')) for i in range(3)] + [_text_resp("gave up")] bot = self.bot(script, max_tool_iterations=16) self.assertEqual(await bot.prompt("loop"), "gave up") self.assertEqual(len(bot._client.calls), 4) events = [f.get("event") for f in bot.transcript if f["kind"] == "tool_loop"] self.assertIn("loop_breaker", events) async def test_missing_tool_call_id_raises_history_valid(self): bot = self.bot([_tool_resp({"type": "function", "function": { "name": "read", "arguments": "{}"}})]) with self.assertRaises(DeepSeekError): await bot.prompt("go") self.assertIsNone(BlackCatBot.validate_wire(bot._wire_messages())) self.assertEqual([m["role"] for m in bot.messages], ["user"]) async def test_429_retried_then_success(self): bot = self.bot([DeepSeekError("rate", 429), _text_resp("ok")]) with mock.patch("blackcat.agent.asyncio.sleep", new=mock.AsyncMock()): self.assertEqual(await bot.prompt("hi"), "ok") self.assertEqual(len(bot._client.calls), 2) async def test_401_raised_without_retry(self): bot = self.bot([DeepSeekError("auth", 401)]) with self.assertRaises(DeepSeekError): await bot.prompt("hi") self.assertEqual(len(bot._client.calls), 1) async def test_no_tools_is_plain_chat(self): bot = self.bot([_text_resp("plain")], enable_tools=False) self.assertEqual(await bot.prompt("hi"), "plain") self.assertNotIn("tools", bot._client.calls[0][1]) async def test_echo_never_loops(self): async with BlackCatBot(provider="echo", system="t", tools=FileTools(FileState(self.tmp.name))) as bot: self.assertEqual(await bot.prompt("Write xx to aa.md"), "echo: Write xx to aa.md") self.assertFalse(os.path.exists(self.path("aa.md"))) class TestFetchTool(LoopCase): async def test_fetch_then_answer(self): page = {"url": "https://a.test/p", "content": "live page words"} bot = self.bot([ _tool_resp(_call("c1", "fetch", json.dumps( {"url": "https://a.test/p"}))), _text_resp("got it")]) with mock.patch.object(BlackCatBot, "fetch", new=mock.AsyncMock(return_value=page)): self.assertEqual(await bot.prompt("is https://a.test/p secure?"), "got it") result = json.loads(bot.messages[2]["content"]) self.assertTrue(result["ok"]) self.assertIn("live page words", bot.messages[2]["content"]) self.assertIsNone(BlackCatBot.validate_wire(bot._wire_messages())) system = bot._client.calls[0][0][0]["content"] self.assertIn("Use fetch for live web pages", system) async def test_fetch_failure_fed_back(self): bot = self.bot([]) with mock.patch.object(BlackCatBot, "fetch", new=mock.AsyncMock(side_effect=DeepSeekError("down", 503))): result = await bot.run_tool("fetch", url="https://a.test/p") self.assertEqual(result["code"], "fetch_failed") self.assertIn("down", result["error"]) class TestSearchTool(LoopCase): hits = {"query": "deepseek v4", "results": [ {"title": "changelog", "url": "https://a.test/c", "date": "2026-11-01", "summary": "flash shipped"}]} async def test_search_then_answer(self): bot = self.bot([ _tool_resp(_call("c1", "search", json.dumps( {"query": "deepseek v4", "count": 3}))), _text_resp("found it")]) with mock.patch.object(BlackCatBot, "search", new=mock.AsyncMock(return_value=self.hits)) as m: self.assertEqual(await bot.prompt("search deepseek v4"), "found it") m.assert_called_once_with("deepseek v4", 3) result = json.loads(bot.messages[2]["content"]) self.assertTrue(result["ok"]) self.assertIn("flash shipped", bot.messages[2]["content"]) self.assertIsNone(BlackCatBot.validate_wire(bot._wire_messages())) async def test_search_failure_fed_back(self): bot = self.bot([]) with mock.patch.object(BlackCatBot, "search", new=mock.AsyncMock(side_effect=DeepSeekError("down", 503))): result = await bot.run_tool("search", query="x") self.assertEqual(result["code"], "search_failed") self.assertIn("down", result["error"]) async def test_search_is_not_mutating(self): bot = self.bot([]) self.assertNotIn("search", MUTATING_TOOLS) with mock.patch.object(BlackCatBot, "search", new=mock.AsyncMock(return_value=self.hits)): self.assertTrue((await bot.run_tool("search", query="q"))["ok"]) class TestBashTool(LoopCase): async def test_bash_is_mutating(self): self.assertIn("bash", MUTATING_TOOLS) async def test_bash_runs_and_reports(self): bot = self.bot([]) result = await bot.run_tool("bash", command="echo hi") self.assertTrue(result["ok"]) self.assertEqual(result["exit_code"], 0) self.assertEqual(result["stdout"], "hi\n") self.assertFalse(result["timed_out"]) self.assertIsNone(BlackCatBot.validate_wire(bot._wire_messages())) async def test_bash_failure_fed_back(self): bot = self.bot([]) result = await bot.run_tool("bash", command="exit 2") self.assertFalse(result["ok"]) self.assertEqual(result["exit_code"], 2) self.assertEqual(result["code"], "nonzero_exit") async def test_confirm_deny_blocks_bash(self): bot = self.bot([], tool_confirm=lambda name, args: False) result = await bot.run_tool("bash", command="echo hi") self.assertFalse(result["ok"]) self.assertEqual(result["code"], "rejected") async def test_yolo_bypasses_confirm_for_bash(self): bot = self.bot([], tool_confirm=lambda name, args: False, yolo=True) result = await bot.run_tool("bash", command="echo hi") self.assertTrue(result["ok"]) async def test_yolo_bash_leaves_root(self): bot = self.bot([], yolo=True) result = await bot.run_tool("bash", command="pwd", cwd="/tmp") self.assertTrue(result["ok"]) self.assertEqual(result["cwd"], "/tmp") async def test_sandboxed_bash_cannot_leave_root(self): bot = self.bot([]) result = await bot.run_tool("bash", command="pwd", cwd="/tmp") self.assertFalse(result["ok"]) self.assertEqual(result["code"], "security") async def test_read_only_bash_refuses(self): bot = BlackCatBot(provider="deepseek", api_key="k", system="t", tools=FileTools(FileState(self.tmp.name, write=False))) self.addAsyncCleanup(bot.aclose) result = await bot.run_tool("bash", command="echo hi") self.assertFalse(result["ok"]) self.assertEqual(result["code"], "security") async def test_env_values_never_reach_transcript(self): bot = self.bot([]) secret = "transcript-secret-abc" result = await bot.run_tool( "bash", command="echo $BLACKCAT_T", env={"BLACKCAT_T": secret}) self.assertTrue(result["ok"]) blob = json.dumps(bot.transcript) self.assertNotIn(secret, blob) self.assertIn("BLACKCAT_T", blob) class TestCompactionAtomic(unittest.IsolatedAsyncioTestCase): async def test_clearing_keeps_pairs_valid(self): bot = BlackCatBot(provider="echo", system="t", keep_recent_tools=1) self.addAsyncCleanup(bot.aclose) for i in range(3): bot.messages.append({ "role": "assistant", "content": None, "tool_calls": [_call("c%d" % i, "read", "{}")]}) bot.messages.append({"role": "tool", "tool_call_id": "c%d" % i, "content": "result %d" % i}) bot._clear_old_tool_results() self.assertIsNone(BlackCatBot.validate_wire(bot._wire_messages())) self.assertIn("cleared", bot.messages[1]["content"]) self.assertEqual(bot.messages[-1]["content"], "result 2") class TestClientHardening(unittest.IsolatedAsyncioTestCase): async def test_message_of_keeps_tool_calls(self): resp = _tool_resp(_call("c1", "read", "{}")) msg = DeepSeekClient.message_of(resp) self.assertEqual(msg["tool_calls"][0]["id"], "c1") with self.assertRaises(DeepSeekError): DeepSeekClient.message_of({"choices": []}) async def test_malformed_json_body_is_deepseek_error(self): client = DeepSeekClient(api_key="k") class _Raw: status = 200 headers = {} async def __aenter__(self): return self async def __aexit__(self, *exc): return False async def read(self): return b"not json" class _Session: closed = False def post(self, _url, **_kwargs): return _Raw() async def close(self): pass client._http = _Session() with self.assertRaises(DeepSeekError): await client.chat([{"role": "user", "content": "hi"}]) class TestYolo(unittest.IsolatedAsyncioTestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) self.root = os.path.join(self.tmp.name, "ws") os.mkdir(self.root) def bot(self, **kwargs): kwargs.setdefault("system", "t") bot = BlackCatBot(provider="echo", tools=FileTools(FileState(self.root)), **kwargs) self.addAsyncCleanup(bot.aclose) return bot async def test_confirm_deny_blocks_without_yolo(self): bot = self.bot(tool_confirm=lambda name, args: False) self.assertFalse(bot.yolo) result = await bot.run_tool("write_file", path="a.txt", content="x") self.assertFalse(result["ok"]) self.assertEqual(result["code"], "rejected") self.assertFalse(os.path.exists(os.path.join(self.root, "a.txt"))) async def test_yolo_bypasses_confirm(self): bot = self.bot(tool_confirm=lambda name, args: False, yolo=True) self.assertTrue(bot.yolo) result = await bot.run_tool("write_file", path="a.txt", content="x") self.assertTrue(result["ok"]) self.assertTrue(os.path.exists(os.path.join(self.root, "a.txt"))) async def test_yolo_unsandboxes_and_enables_writes(self): tools = FileTools(FileState(self.root, write=False)) bot = BlackCatBot(provider="echo", system="t", tools=tools, yolo=True) self.addAsyncCleanup(bot.aclose) self.assertFalse(tools.state.sandboxed) self.assertTrue(tools.state.write_enabled) outside = os.path.join(self.tmp.name, "outside.txt") result = await bot.run_tool("write_file", path=outside, content="free") self.assertTrue(result["ok"]) self.assertTrue(os.path.exists(outside)) async def test_yolo_lazy_tools_unsandboxed(self): bot = BlackCatBot(provider="echo", system="t", yolo=True) self.addAsyncCleanup(bot.aclose) self.assertFalse(bot._file_tools().state.sandboxed) class TestCLIFlags(unittest.IsolatedAsyncioTestCase): def test_parser_flags(self): args = build_parser().parse_args( ["--no-tools", "--max-tool-steps", "3", "--tools-root", "/tmp", "--read-only-tools", "prompt", "hi"]) self.assertTrue(args.no_tools) self.assertEqual(args.max_tool_steps, 3) self.assertTrue(args.read_only_tools) def test_yolo_flag(self): args = build_parser().parse_args(["--yolo", "prompt", "hi"]) self.assertTrue(args.yolo) args = build_parser().parse_args(["prompt", "hi"]) self.assertFalse(args.yolo) def test_bad_max_steps_exit_2(self): self.assertEqual(main(["--provider", "echo", "--max-tool-steps", "0", "prompt", "hi"]), 2) if __name__ == "__main__": unittest.main()