"""Committed tests for the DeepSeek tool-calling agent loop (no network).
A scripted stub client stands in for DeepSeekClient: it returns canned
chat responses (tool_calls, then final text) and records every wire it
received, so loop shape, error feedback, and safeguards are asserted
offline against a tmp-root FileTools.
"""
# retoor <retoor@molodetz.nl>
import inspect
import json
import os
import tempfile
import unittest
from unittest import mock
from blackcat.agent import (
MUTATING_TOOLS,
TOOL_NAMES,
TOOL_SCHEMAS,
BlackCatBot,
)
from blackcat.cli import build_parser, main
from blackcat.client import DeepSeekClient, DeepSeekError
from blackcat.fstate import FileState
from blackcat.tools import FileTools
def _call(cid, name, arguments):
return {"id": cid, "type": "function",
"function": {"name": name, "arguments": arguments}}
def _tool_resp(*calls):
return {"choices": [{"message": {"role": "assistant", "content": None,
"tool_calls": list(calls)},
"finish_reason": "tool_calls"}],
"usage": {"prompt_tokens": 10}}
def _text_resp(text):
return {"choices": [{"message": {"role": "assistant", "content": text},
"finish_reason": "stop"}],
"usage": {"prompt_tokens": 10}}
class StubClient:
model = "stub-model"
def __init__(self, script):
self.script = list(script)
self.calls = []
async def chat(self, messages, **kwargs):
self.calls.append((messages, kwargs))
item = self.script.pop(0)
if isinstance(item, Exception):
raise item
return item
class LoopCase(unittest.IsolatedAsyncioTestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
def bot(self, script, **kwargs):
kwargs.setdefault("system", "t")
bot = BlackCatBot(provider="deepseek", api_key="k",
tools=FileTools(FileState(self.tmp.name)), **kwargs)
bot._client = StubClient(script)
self.addAsyncCleanup(bot.aclose)
return bot
def path(self, name):
return os.path.join(self.tmp.name, name)
class TestSchemas(unittest.IsolatedAsyncioTestCase):
def test_names_are_canonical_tools(self):
self.assertEqual(TOOL_NAMES, ("read", "grep", "outline", "edit",
"multi_edit", "write_file",
"apply_patch", "bash", "browse",
"download",
"search", "fetch",
"iterative_evidence_convergence",
"deep_research", "todo",
"delegate", "task_output",
"read_spill"))
self.assertNotIn("write", TOOL_NAMES)
def test_schemas_match_signatures(self):
for schema in TOOL_SCHEMAS:
fn = schema["function"]
self.assertEqual(schema["type"], "function")
if fn["name"] == "iterative_evidence_convergence":
method = BlackCatBot.iterative_evidence_convergence
elif fn["name"] == "bash":
method = BlackCatBot.bash
elif fn["name"] == "browse":
method = BlackCatBot.browse
elif fn["name"] == "download":
method = BlackCatBot.download
elif fn["name"] == "fetch":
method = BlackCatBot.web_fetch
elif fn["name"] == "search":
method = BlackCatBot.web_search
elif fn["name"] == "todo":
method = BlackCatBot.todo
elif fn["name"] == "delegate":
method = BlackCatBot.delegate
elif fn["name"] == "task_output":
method = BlackCatBot.task_output
elif fn["name"] == "read_spill":
method = BlackCatBot.read_spill
elif fn["name"] == "deep_research":
method = BlackCatBot.deep_research
else:
method = getattr(FileTools, fn["name"])
params = inspect.signature(method).parameters
required = [n for n, p in params.items()
if n != "self" and p.default is inspect._empty]
self.assertEqual(fn["parameters"]["required"], required,
fn["name"])
self.assertFalse(fn["parameters"]["additionalProperties"])
for name in fn["parameters"]["properties"]:
self.assertIn(name, params, fn["name"])
class TestLoop(LoopCase):
async def test_write_file_then_answer(self):
bot = self.bot([
_tool_resp(_call("c1", "write_file", json.dumps(
{"path": "aa.md", "content": "xx\n"}))),
_text_resp("done")])
self.assertEqual(await bot.prompt("Write xx to aa.md"), "done")
with open(self.path("aa.md"), encoding="utf-8") as fh:
self.assertEqual(fh.read(), "xx\n")
wire, kwargs = bot._client.calls[0]
self.assertEqual(kwargs["tools"], TOOL_SCHEMAS)
self.assertEqual(kwargs["tool_choice"], "auto")
roles = [m["role"] for m in bot.messages]
self.assertEqual(roles, ["user", "assistant", "tool", "assistant"])
tool_wire = bot._wire_messages()[2]
self.assertEqual(set(tool_wire),
{"role", "tool_call_id", "content"})
self.assertEqual(tool_wire["tool_call_id"], "c1")
self.assertIsNone(BlackCatBot.validate_wire(bot._wire_messages()))
self.assertTrue(json.loads(tool_wire["content"])["ok"])
async def test_read_then_answer(self):
with open(self.path("f.txt"), "w", encoding="utf-8") as fh:
fh.write("hello\n")
bot = self.bot([
_tool_resp(_call("c1", "read", '{"path": "f.txt"}')),
_text_resp("it says hello")])
self.assertEqual(await bot.prompt("read f"), "it says hello")
self.assertIn("hello", bot.messages[2]["content"])
async def test_malformed_arguments_fed_back(self):
bot = self.bot([
_tool_resp(_call("c1", "write_file", "{not json")),
_text_resp("recovered")])
self.assertEqual(await bot.prompt("write"), "recovered")
self.assertFalse(os.path.exists(self.path("aa.md")))
result = json.loads(bot.messages[2]["content"])
self.assertEqual(result["code"], "bad_arguments")
async def test_unknown_tool_and_bad_args_fed_back(self):
bot = self.bot([
_tool_resp(_call("c1", "nope", "{}"),
_call("c2", "read", '{"file_path": "x"}')),
_text_resp("ok")])
self.assertEqual(await bot.prompt("go"), "ok")
codes = [json.loads(m["content"])["code"]
for m in bot.messages if m.get("role") == "tool"]
self.assertEqual(codes, ["unknown_tool", "bad_args"])
async def test_tool_exception_fed_back_pair_closed(self):
with open(self.path("f.txt"), "w", encoding="utf-8") as fh:
fh.write("a\n")
bot = self.bot([
_tool_resp(_call("c1", "read", '{"path": "f.txt"}')),
_tool_resp(_call("c2", "edit", json.dumps(
{"path": "f.txt", "old_string": 123, "new_string": "b"}))),
_text_resp("survived")])
self.assertEqual(await bot.prompt("edit"), "survived")
result = json.loads(bot.messages[4]["content"])
self.assertEqual(result["code"], "tool_exception")
self.assertIsNone(BlackCatBot.validate_wire(bot._wire_messages()))
async def test_read_only_tools_refuse_writes(self):
bot = BlackCatBot(provider="deepseek", api_key="k", system="t",
tools=FileTools(FileState(self.tmp.name,
write=False)))
self.addAsyncCleanup(bot.aclose)
bot._client = StubClient([
_tool_resp(_call("c1", "write_file", json.dumps(
{"path": "aa.md", "content": "xx"}))),
_text_resp("cannot")])
self.assertEqual(await bot.prompt("write"), "cannot")
self.assertFalse(os.path.exists(self.path("aa.md")))
result = json.loads(bot.messages[2]["content"])
self.assertEqual(result["code"], "security")
class TestLoopSafeguards(LoopCase):
async def test_max_iterations_forces_final_without_tools(self):
bot = self.bot([
_tool_resp(_call("c1", "read", '{"path": "missing"}')),
_tool_resp(_call("c2", "read", '{"path": "missing"}')),
_text_resp("final")], max_tool_iterations=2)
self.assertEqual(await bot.prompt("loop"), "final")
self.assertEqual(len(bot._client.calls), 3)
self.assertNotIn("tools", bot._client.calls[-1][1])
events = [f.get("event") for f in bot.transcript
if f["kind"] == "tool_loop"]
self.assertIn("tool_loop_limit", events)
async def test_identical_failure_breaker(self):
script = [_tool_resp(_call("c%d" % i, "read",
'{"path": "missing"}'))
for i in range(3)] + [_text_resp("gave up")]
bot = self.bot(script, max_tool_iterations=16)
self.assertEqual(await bot.prompt("loop"), "gave up")
self.assertEqual(len(bot._client.calls), 4)
events = [f.get("event") for f in bot.transcript
if f["kind"] == "tool_loop"]
self.assertIn("loop_breaker", events)
async def test_missing_tool_call_id_raises_history_valid(self):
bot = self.bot([_tool_resp({"type": "function", "function": {
"name": "read", "arguments": "{}"}})])
with self.assertRaises(DeepSeekError):
await bot.prompt("go")
self.assertIsNone(BlackCatBot.validate_wire(bot._wire_messages()))
self.assertEqual([m["role"] for m in bot.messages], ["user"])
async def test_429_retried_then_success(self):
bot = self.bot([DeepSeekError("rate", 429), _text_resp("ok")])
with mock.patch("blackcat.agent.asyncio.sleep", new=mock.AsyncMock()):
self.assertEqual(await bot.prompt("hi"), "ok")
self.assertEqual(len(bot._client.calls), 2)
async def test_401_raised_without_retry(self):
bot = self.bot([DeepSeekError("auth", 401)])
with self.assertRaises(DeepSeekError):
await bot.prompt("hi")
self.assertEqual(len(bot._client.calls), 1)
async def test_no_tools_is_plain_chat(self):
bot = self.bot([_text_resp("plain")], enable_tools=False)
self.assertEqual(await bot.prompt("hi"), "plain")
self.assertNotIn("tools", bot._client.calls[0][1])
async def test_echo_never_loops(self):
async with BlackCatBot(provider="echo", system="t",
tools=FileTools(FileState(self.tmp.name))) as bot:
self.assertEqual(await bot.prompt("Write xx to aa.md"),
"echo: Write xx to aa.md")
self.assertFalse(os.path.exists(self.path("aa.md")))
class TestFetchTool(LoopCase):
async def test_fetch_then_answer(self):
page = {"url": "https://a.test/p", "content": "live page words"}
bot = self.bot([
_tool_resp(_call("c1", "fetch", json.dumps(
{"url": "https://a.test/p"}))),
_text_resp("got it")])
with mock.patch.object(BlackCatBot, "fetch", new=mock.AsyncMock(return_value=page)):
self.assertEqual(await bot.prompt("is https://a.test/p secure?"),
"got it")
result = json.loads(bot.messages[2]["content"])
self.assertTrue(result["ok"])
self.assertIn("live page words", bot.messages[2]["content"])
self.assertIsNone(BlackCatBot.validate_wire(bot._wire_messages()))
system = bot._client.calls[0][0][0]["content"]
self.assertIn("Use fetch for live web pages", system)
async def test_fetch_failure_fed_back(self):
bot = self.bot([])
with mock.patch.object(BlackCatBot, "fetch",
new=mock.AsyncMock(side_effect=DeepSeekError("down", 503))):
result = await bot.run_tool("fetch", url="https://a.test/p")
self.assertEqual(result["code"], "fetch_failed")
self.assertIn("down", result["error"])
class TestSearchTool(LoopCase):
hits = {"query": "deepseek v4", "results": [
{"title": "changelog", "url": "https://a.test/c",
"date": "2026-11-01", "summary": "flash shipped"}]}
async def test_search_then_answer(self):
bot = self.bot([
_tool_resp(_call("c1", "search", json.dumps(
{"query": "deepseek v4", "count": 3}))),
_text_resp("found it")])
with mock.patch.object(BlackCatBot, "search", new=mock.AsyncMock(return_value=self.hits)) as m:
self.assertEqual(await bot.prompt("search deepseek v4"), "found it")
m.assert_called_once_with("deepseek v4", 3)
result = json.loads(bot.messages[2]["content"])
self.assertTrue(result["ok"])
self.assertIn("flash shipped", bot.messages[2]["content"])
self.assertIsNone(BlackCatBot.validate_wire(bot._wire_messages()))
async def test_search_failure_fed_back(self):
bot = self.bot([])
with mock.patch.object(BlackCatBot, "search",
new=mock.AsyncMock(side_effect=DeepSeekError("down", 503))):
result = await bot.run_tool("search", query="x")
self.assertEqual(result["code"], "search_failed")
self.assertIn("down", result["error"])
async def test_search_is_not_mutating(self):
bot = self.bot([])
self.assertNotIn("search", MUTATING_TOOLS)
with mock.patch.object(BlackCatBot, "search", new=mock.AsyncMock(return_value=self.hits)):
self.assertTrue((await bot.run_tool("search", query="q"))["ok"])
class TestBashTool(LoopCase):
async def test_bash_is_mutating(self):
self.assertIn("bash", MUTATING_TOOLS)
async def test_bash_runs_and_reports(self):
bot = self.bot([])
result = await bot.run_tool("bash", command="echo hi")
self.assertTrue(result["ok"])
self.assertEqual(result["exit_code"], 0)
self.assertEqual(result["stdout"], "hi\n")
self.assertFalse(result["timed_out"])
self.assertIsNone(BlackCatBot.validate_wire(bot._wire_messages()))
async def test_bash_failure_fed_back(self):
bot = self.bot([])
result = await bot.run_tool("bash", command="exit 2")
self.assertFalse(result["ok"])
self.assertEqual(result["exit_code"], 2)
self.assertEqual(result["code"], "nonzero_exit")
async def test_confirm_deny_blocks_bash(self):
bot = self.bot([], tool_confirm=lambda name, args: False)
result = await bot.run_tool("bash", command="echo hi")
self.assertFalse(result["ok"])
self.assertEqual(result["code"], "rejected")
async def test_yolo_bypasses_confirm_for_bash(self):
bot = self.bot([], tool_confirm=lambda name, args: False,
yolo=True)
result = await bot.run_tool("bash", command="echo hi")
self.assertTrue(result["ok"])
async def test_yolo_bash_leaves_root(self):
bot = self.bot([], yolo=True)
result = await bot.run_tool("bash", command="pwd", cwd="/tmp")
self.assertTrue(result["ok"])
self.assertEqual(result["cwd"], "/tmp")
async def test_sandboxed_bash_cannot_leave_root(self):
bot = self.bot([])
result = await bot.run_tool("bash", command="pwd", cwd="/tmp")
self.assertFalse(result["ok"])
self.assertEqual(result["code"], "security")
async def test_read_only_bash_refuses(self):
bot = BlackCatBot(provider="deepseek", api_key="k", system="t",
tools=FileTools(FileState(self.tmp.name,
write=False)))
self.addAsyncCleanup(bot.aclose)
result = await bot.run_tool("bash", command="echo hi")
self.assertFalse(result["ok"])
self.assertEqual(result["code"], "security")
async def test_env_values_never_reach_transcript(self):
bot = self.bot([])
secret = "transcript-secret-abc"
result = await bot.run_tool(
"bash", command="echo $BLACKCAT_T",
env={"BLACKCAT_T": secret})
self.assertTrue(result["ok"])
blob = json.dumps(bot.transcript)
self.assertNotIn(secret, blob)
self.assertIn("BLACKCAT_T", blob)
class TestCompactionAtomic(unittest.IsolatedAsyncioTestCase):
async def test_clearing_keeps_pairs_valid(self):
bot = BlackCatBot(provider="echo", system="t", keep_recent_tools=1)
self.addAsyncCleanup(bot.aclose)
for i in range(3):
bot.messages.append({
"role": "assistant", "content": None,
"tool_calls": [_call("c%d" % i, "read", "{}")]})
bot.messages.append({"role": "tool", "tool_call_id": "c%d" % i,
"content": "result %d" % i})
bot._clear_old_tool_results()
self.assertIsNone(BlackCatBot.validate_wire(bot._wire_messages()))
self.assertIn("cleared", bot.messages[1]["content"])
self.assertEqual(bot.messages[-1]["content"], "result 2")
class TestClientHardening(unittest.IsolatedAsyncioTestCase):
async def test_message_of_keeps_tool_calls(self):
resp = _tool_resp(_call("c1", "read", "{}"))
msg = DeepSeekClient.message_of(resp)
self.assertEqual(msg["tool_calls"][0]["id"], "c1")
with self.assertRaises(DeepSeekError):
DeepSeekClient.message_of({"choices": []})
async def test_malformed_json_body_is_deepseek_error(self):
client = DeepSeekClient(api_key="k")
class _Raw:
status = 200
headers = {}
async def __aenter__(self):
return self
async def __aexit__(self, *exc):
return False
async def read(self):
return b"not json"
class _Session:
closed = False
def post(self, _url, **_kwargs):
return _Raw()
async def close(self):
pass
client._http = _Session()
with self.assertRaises(DeepSeekError):
await client.chat([{"role": "user", "content": "hi"}])
class TestYolo(unittest.IsolatedAsyncioTestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.root = os.path.join(self.tmp.name, "ws")
os.mkdir(self.root)
def bot(self, **kwargs):
kwargs.setdefault("system", "t")
bot = BlackCatBot(provider="echo",
tools=FileTools(FileState(self.root)), **kwargs)
self.addAsyncCleanup(bot.aclose)
return bot
async def test_confirm_deny_blocks_without_yolo(self):
bot = self.bot(tool_confirm=lambda name, args: False)
self.assertFalse(bot.yolo)
result = await bot.run_tool("write_file", path="a.txt", content="x")
self.assertFalse(result["ok"])
self.assertEqual(result["code"], "rejected")
self.assertFalse(os.path.exists(os.path.join(self.root, "a.txt")))
async def test_yolo_bypasses_confirm(self):
bot = self.bot(tool_confirm=lambda name, args: False, yolo=True)
self.assertTrue(bot.yolo)
result = await bot.run_tool("write_file", path="a.txt", content="x")
self.assertTrue(result["ok"])
self.assertTrue(os.path.exists(os.path.join(self.root, "a.txt")))
async def test_yolo_unsandboxes_and_enables_writes(self):
tools = FileTools(FileState(self.root, write=False))
bot = BlackCatBot(provider="echo", system="t", tools=tools,
yolo=True)
self.addAsyncCleanup(bot.aclose)
self.assertFalse(tools.state.sandboxed)
self.assertTrue(tools.state.write_enabled)
outside = os.path.join(self.tmp.name, "outside.txt")
result = await bot.run_tool("write_file", path=outside, content="free")
self.assertTrue(result["ok"])
self.assertTrue(os.path.exists(outside))
async def test_yolo_lazy_tools_unsandboxed(self):
bot = BlackCatBot(provider="echo", system="t", yolo=True)
self.addAsyncCleanup(bot.aclose)
self.assertFalse(bot._file_tools().state.sandboxed)
class TestCLIFlags(unittest.IsolatedAsyncioTestCase):
def test_parser_flags(self):
args = build_parser().parse_args(
["--no-tools", "--max-tool-steps", "3", "--tools-root", "/tmp",
"--read-only-tools", "prompt", "hi"])
self.assertTrue(args.no_tools)
self.assertEqual(args.max_tool_steps, 3)
self.assertTrue(args.read_only_tools)
def test_yolo_flag(self):
args = build_parser().parse_args(["--yolo", "prompt", "hi"])
self.assertTrue(args.yolo)
args = build_parser().parse_args(["prompt", "hi"])
self.assertFalse(args.yolo)
def test_bad_max_steps_exit_2(self):
self.assertEqual(main(["--provider", "echo", "--max-tool-steps",
"0", "prompt", "hi"]), 2)
if __name__ == "__main__":
unittest.main()