"""Committed tests for DeepSeek DSML leak recovery (no network).
DeepSeek V4/V4.1 natively emits tool calls as DSML markup; when the
provider-side parser fails, raw tags leak into assistant ``content``
instead of arriving as ``tool_calls``. ``blackcat.dsml`` recovers those
calls client-side and strips the markup so it never poisons history.
Live DSML tokens are never pasted into this file: the ``M()`` helper
builds every tag programmatically (``\\uFF5C`` bars), so the source
itself cannot become a phantom tool call.
"""
# retoor <retoor@molodetz.nl>
import json
import os
import tempfile
import unittest
from blackcat.agent import TOOL_SCHEMAS, BlackCatBot
from blackcat.dsml import looks_like_dsml, recover_dsml
from blackcat.fstate import FileState
from blackcat.tools import FileTools
FW = "\uFF5C"
def M(tag, attrs="", closer=False, doubled=False, ascii=False, space=False,
marker=True):
"""Build one DSML (or bare) tag without pasting a live token."""
if not marker:
if closer:
return "</%s>" % tag
return "<%s%s>" % (tag, attrs)
bar = "|" if ascii else FW
bars = bar * (2 if doubled else 1)
if closer:
if not tag:
return "</%sDSML%s>" % (bars, bars)
return "</%sDSML%s%s%s>" % (bars, bars, " " if space else "",
tag)
return "<%sDSML%s%s%s%s>" % (bars, bars, " " if space else "", tag,
attrs)
def EOS(kind="end"):
"""A DeepSeek begin/end-of-sentence residue token."""
return "<%s%s\u2581of\u2581sentence%s>" % (FW, kind, FW)
def _text_resp(text):
return {"choices": [{"message": {"role": "assistant",
"content": text},
"finish_reason": "stop"}],
"usage": {"prompt_tokens": 10}}
def _tool_resp(content, *calls):
return {"choices": [{"message": {"role": "assistant",
"content": content,
"tool_calls": list(calls)},
"finish_reason": "tool_calls"}],
"usage": {"prompt_tokens": 10}}
def _call(cid, name, arguments):
return {"id": cid, "type": "function",
"function": {"name": name, "arguments": arguments}}
class StubClient:
model = "stub-model"
def __init__(self, script):
self.script = list(script)
self.calls = []
async def chat(self, messages, **kwargs):
self.calls.append((messages, kwargs))
item = self.script.pop(0)
if isinstance(item, Exception):
raise item
return item
def read_block(path="f.txt", **kw):
"""A canonical read call inside a tool_calls block."""
return (M("tool_calls", **kw)
+ M("invoke", ' name="read"', **kw)
+ M("parameter", ' name="path"', **kw) + path
+ M("parameter", closer=True, **kw)
+ M("invoke", closer=True, **kw)
+ M("tool_calls", closer=True, **kw))
class TestFastPath(unittest.TestCase):
def test_plain_prose_is_passthrough(self):
for text in ("Hello world", "", "a < b > c", "DSMLess prose"):
rec = recover_dsml(text, TOOL_SCHEMAS)
self.assertEqual(rec["calls"], [])
self.assertEqual(rec["removed"], 0)
self.assertEqual(rec["text"], text)
self.assertEqual(rec["noted_text"], text)
def test_looks_like_dsml(self):
self.assertFalse(looks_like_dsml("plain prose"))
self.assertFalse(looks_like_dsml(""))
self.assertTrue(looks_like_dsml(M("invoke", closer=True)))
self.assertTrue(looks_like_dsml("x " + EOS()))
self.assertTrue(looks_like_dsml("x </parameter> y"))
def test_hyphen_block_tag_is_left_alone(self):
text = ("<%sDSML%stool-calls\nrecord_item\n</plan>"
% (FW, FW))
rec = recover_dsml(text, TOOL_SCHEMAS)
self.assertEqual(rec["calls"], [])
self.assertEqual(rec["removed"], 0)
self.assertEqual(rec["text"], text)
class TestCanonicalShapes(unittest.TestCase):
def test_canonical_single_bar(self):
rec = recover_dsml(read_block(), TOOL_SCHEMAS)
self.assertEqual(rec["calls"],
[{"name": "read", "arguments": {"path": "f.txt"}}])
self.assertEqual(rec["text"].strip(), "")
self.assertGreaterEqual(rec["removed"], 1)
self.assertEqual(rec["names"], ["read"])
def test_doubled_bars_with_spaced_closers(self):
text = (M("tool_calls", doubled=True)
+ M("invoke", ' name="write_file"', doubled=True)
+ M("parameter", ' name="content" string="true"',
doubled=True)
+ '{"failed": 0, "written": 3}'
+ M("parameter", closer=True, doubled=True, space=True)
+ M("invoke", closer=True, doubled=True, space=True)
+ M("calls", closer=True, doubled=True, space=True))
rec = recover_dsml(text, TOOL_SCHEMAS)
self.assertEqual(rec["calls"], [{
"name": "write_file",
"arguments": {"content": '{"failed": 0, "written": 3}'}}])
self.assertEqual(rec["text"].strip(), "")
def test_ascii_pipes(self):
rec = recover_dsml(read_block(ascii=True), TOOL_SCHEMAS)
self.assertEqual(rec["calls"],
[{"name": "read", "arguments": {"path": "f.txt"}}])
def test_block_tag_variants(self):
for block in ("tool_calls", "tool_call", "function_calls",
"calls"):
text = (M(block) + M("invoke", ' name="read"')
+ M("parameter", ' name="path"') + "f.txt"
+ M("parameter", closer=True)
+ M("invoke", closer=True) + M(block, closer=True))
rec = recover_dsml(text, TOOL_SCHEMAS)
self.assertEqual(rec["calls"], [
{"name": "read", "arguments": {"path": "f.txt"}}], block)
def test_orphan_invoke_without_block(self):
text = (M("invoke", ' name="read"')
+ M("parameter", ' name="path"') + "f.txt"
+ M("parameter", closer=True)
+ M("invoke", closer=True))
rec = recover_dsml(text, TOOL_SCHEMAS)
self.assertEqual(rec["calls"],
[{"name": "read", "arguments": {"path": "f.txt"}}])
def test_name_spacing_and_quote_variants(self):
for attrs in (' name = "read"', " name='read'", " name=read"):
text = (M("invoke", attrs)
+ M("parameter", ' name = "path" ') + "f.txt"
+ M("parameter", closer=True)
+ M("invoke", closer=True))
rec = recover_dsml(text, TOOL_SCHEMAS)
self.assertEqual(rec["calls"], [
{"name": "read", "arguments": {"path": "f.txt"}}], attrs)
def test_surrounding_prose_preserved(self):
text = "Let me read it.\n" + read_block() + "\nDone."
rec = recover_dsml(text, TOOL_SCHEMAS)
self.assertEqual(len(rec["calls"]), 1)
self.assertEqual(rec["text"], "Let me read it.\n\nDone.")
def test_multiple_invokes_in_one_block(self):
text = (M("tool_calls")
+ M("invoke", ' name="read"')
+ M("parameter", ' name="path"') + "a.txt"
+ M("parameter", closer=True)
+ M("invoke", closer=True)
+ M("invoke", ' name="read"')
+ M("parameter", ' name="path"') + "b.txt"
+ M("parameter", closer=True)
+ M("invoke", closer=True)
+ M("tool_calls", closer=True))
rec = recover_dsml(text, TOOL_SCHEMAS)
self.assertEqual([c["arguments"]["path"] for c in rec["calls"]],
["a.txt", "b.txt"])
def test_idempotent(self):
text = "Prose\n" + read_block() + "\nTail " + EOS()
once = recover_dsml(text, TOOL_SCHEMAS)
twice = recover_dsml(once["text"], TOOL_SCHEMAS)
self.assertEqual(twice["removed"], 0)
self.assertEqual(twice["text"], once["text"])
class TestValues(unittest.TestCase):
def test_multiline_value_keeps_indentation(self):
text = (M("invoke", ' name="write_file"')
+ M("parameter", ' name="content"')
+ "\nline1\n line2\n"
+ M("parameter", closer=True)
+ M("invoke", closer=True))
rec = recover_dsml(text, TOOL_SCHEMAS)
self.assertEqual(rec["calls"][0]["arguments"]["content"],
"line1\n line2")
def test_entities_decoded(self):
text = (M("invoke", ' name="read"')
+ M("parameter", ' name="path"')
+ "a "q" & b"
+ M("parameter", closer=True)
+ M("invoke", closer=True))
rec = recover_dsml(text, TOOL_SCHEMAS)
self.assertEqual(rec["calls"][0]["arguments"]["path"],
'a "q" & b')
def test_json_param_parsed_and_fallback(self):
text = (M("invoke", ' name="multi_edit"')
+ M("parameter", ' name="edits" string="false"')
+ '[{"a": 1}]' + M("parameter", closer=True)
+ M("invoke", closer=True))
rec = recover_dsml(text, TOOL_SCHEMAS)
self.assertEqual(rec["calls"][0]["arguments"]["edits"],
[{"a": 1}])
bad = (M("invoke", ' name="read"')
+ M("parameter", ' name="path" string="false"')
+ "not json" + M("parameter", closer=True)
+ M("invoke", closer=True))
rec = recover_dsml(bad, TOOL_SCHEMAS)
self.assertEqual(rec["calls"][0]["arguments"]["path"],
"not json")
def test_raw_json_body(self):
text = (M("invoke", ' name="read"') + '{"path": "f.txt"}'
+ M("invoke", closer=True))
rec = recover_dsml(text, TOOL_SCHEMAS)
self.assertEqual(rec["calls"],
[{"name": "read", "arguments": {"path": "f.txt"}}])
bad = (M("invoke", ' name="read"') + "{nope"
+ M("invoke", closer=True))
rec = recover_dsml(bad, TOOL_SCHEMAS)
self.assertEqual(rec["calls"], [])
self.assertGreater(rec["removed"], 0)
def test_eos_tokens_stripped(self):
rec = recover_dsml("answer " + EOS(), TOOL_SCHEMAS)
self.assertEqual(rec["calls"], [])
self.assertEqual(rec["removed"], 1)
self.assertNotIn("sentence", rec["text"])
class TestDegradedShapes(unittest.TestCase):
def test_misspelled_closer_drops_alpha_keeps_beta(self):
text = (M("invoke", ' name="record_item"')
+ M("parameter", ' name="alpha" string="true"') + "first"
+ M("", closer=True)
+ M("parameter", ' name="beta" string="true"') + "second"
+ M("parameter", closer=True)
+ M("invoke", closer=True))
rec = recover_dsml(text, TOOL_SCHEMAS)
self.assertEqual(rec["calls"], [
{"name": "record_item", "arguments": {"beta": "second"}}])
def test_runaway_invoke_name_rejected_and_stripped(self):
text = (M("tool_calls")
+ M("invoke", ' name="record_item {"')
+ "category: Dexes"
+ M("invoke", closer=True) + M("tool_calls", closer=True))
rec = recover_dsml(text, TOOL_SCHEMAS)
self.assertEqual(rec["calls"], [])
self.assertGreater(rec["removed"], 0)
self.assertNotIn("DSML", rec["text"])
def test_unterminated_opener_line_stripped(self):
text = ("Hello\n<%sDSML%sinvoke name=\"read\" {\n"
"category: Dexes" % (FW, FW))
rec = recover_dsml(text, TOOL_SCHEMAS)
self.assertEqual(rec["calls"], [])
self.assertIn("Hello", rec["text"])
self.assertIn("category: Dexes", rec["text"])
self.assertNotIn("DSML", rec["text"])
def test_undeclared_tool_kept_for_runtime(self):
text = (M("invoke", ' name="nope"')
+ M("parameter", ' name="x"') + "1"
+ M("parameter", closer=True)
+ M("invoke", closer=True))
rec = recover_dsml(text, TOOL_SCHEMAS)
self.assertEqual(rec["calls"],
[{"name": "nope", "arguments": {"x": "1"}}])
def test_empty_invoke_name_rejected(self):
text = (M("invoke", ' name=""')
+ M("parameter", ' name="path"') + "f.txt"
+ M("parameter", closer=True)
+ M("invoke", closer=True))
rec = recover_dsml(text, TOOL_SCHEMAS)
self.assertEqual(rec["calls"], [])
self.assertGreater(rec["removed"], 0)
def test_empty_args_rejected(self):
text = M("invoke", ' name="read"') + M("invoke", closer=True)
rec = recover_dsml(text, TOOL_SCHEMAS)
self.assertEqual(rec["calls"], [])
self.assertGreater(rec["removed"], 0)
self.assertIn("read", rec["names"])
def test_unclosed_invoke_dropped_complete_kept(self):
text = (read_block()
+ M("invoke", ' name="read"')
+ M("parameter", ' name="path"') + "half.txt")
rec = recover_dsml(text, TOOL_SCHEMAS)
self.assertEqual(len(rec["calls"]), 1)
self.assertEqual(rec["calls"][0]["arguments"]["path"], "f.txt")
self.assertNotIn("DSML", rec["text"])
def test_stray_closers_stripped_nothing_recovered(self):
text = ('{"failed": 1, "written": 2} '
+ M("parameter", closer=True, doubled=True, space=True)
+ " " + M("invoke", closer=True, doubled=True, space=True)
+ " " + M("calls", closer=True, doubled=True, space=True))
rec = recover_dsml(text, TOOL_SCHEMAS)
self.assertEqual(rec["calls"], [])
self.assertEqual(rec["removed"], 1)
self.assertIn('{"failed": 1, "written": 2}', rec["text"])
self.assertNotIn("DSML", rec["text"])
self.assertIn("[removed malformed tool markup]",
rec["noted_text"])
def test_code_fence_ignored(self):
text = ("Here:\n```\n" + read_block() + "\n```\nDone")
rec = recover_dsml(text, TOOL_SCHEMAS)
self.assertEqual(rec["calls"], [])
self.assertEqual(rec["removed"], 0)
self.assertEqual(rec["text"], text)
def test_unclosed_fence_ignored(self):
rec = recover_dsml("```\n" + read_block(), TOOL_SCHEMAS)
self.assertEqual(rec["calls"], [])
self.assertEqual(rec["removed"], 0)
class TestOrphanParameters(unittest.TestCase):
def test_wrapped_tool_bare_outer(self):
text = ('<parameter name="read">'
+ M("parameter", ' name="path"') + "f.txt"
+ M("parameter", closer=True) + "</parameter>")
rec = recover_dsml(text, TOOL_SCHEMAS)
self.assertEqual(rec["calls"],
[{"name": "read", "arguments": {"path": "f.txt"}}])
def test_wrapped_tool_marked_outer(self):
text = (M("parameter", ' name="read"')
+ M("parameter", ' name="path"') + "f.txt"
+ M("parameter", closer=True)
+ M("parameter", closer=True))
rec = recover_dsml(text, TOOL_SCHEMAS)
self.assertEqual(rec["calls"],
[{"name": "read", "arguments": {"path": "f.txt"}}])
def test_bare_value_with_derivable_key(self):
text = (M("parameter", ' name="read"') + "f.txt"
+ M("parameter", closer=True)
+ M("invoke", closer=True))
rec = recover_dsml(text, TOOL_SCHEMAS)
self.assertEqual(rec["calls"],
[{"name": "read", "arguments": {"path": "f.txt"}}])
def test_bare_value_without_key_redacted(self):
text = (M("parameter", ' name="write_file"') + "blob"
+ M("parameter", closer=True)
+ M("invoke", closer=True))
rec = recover_dsml(text, TOOL_SCHEMAS)
self.assertEqual(rec["calls"], [])
self.assertGreater(rec["removed"], 0)
self.assertIn("[removed malformed write_file call]",
rec["noted_text"])
def test_bare_value_without_schema_redacted(self):
text = (M("parameter", ' name="read"') + "f.txt"
+ M("parameter", closer=True)
+ M("invoke", closer=True))
rec = recover_dsml(text, None)
self.assertEqual(rec["calls"], [])
self.assertGreater(rec["removed"], 0)
def test_bare_value_without_closer_context_left_careful(self):
text = (M("parameter", ' name="read"') + "f.txt"
+ M("parameter", closer=True))
rec = recover_dsml(text, TOOL_SCHEMAS)
self.assertEqual(rec["calls"], [])
self.assertNotIn("DSML", rec["text"])
class LoopCase(unittest.IsolatedAsyncioTestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
def bot(self, script, **kwargs):
kwargs.setdefault("system", "t")
bot = BlackCatBot(provider="deepseek", api_key="k",
tools=FileTools(FileState(self.tmp.name)), **kwargs)
bot._client = StubClient(script)
self.addAsyncCleanup(bot.aclose)
return bot
def path(self, name):
return os.path.join(self.tmp.name, name)
class TestAgentRecovery(LoopCase):
async def test_leaked_call_executes_and_history_stays_clean(self):
with open(self.path("f.txt"), "w", encoding="utf-8") as fh:
fh.write("hello\n")
leak = "I will read it.\n" + read_block()
bot = self.bot([_text_resp(leak), _text_resp("it says hello")])
self.assertEqual(await bot.prompt("read f"), "it says hello")
roles = [m["role"] for m in bot.messages]
self.assertEqual(roles, ["user", "assistant", "tool", "assistant"])
tool_call = bot.messages[1]["tool_calls"][0]
self.assertEqual(tool_call["function"]["name"], "read")
self.assertEqual(json.loads(tool_call["function"]["arguments"]),
{"path": "f.txt"})
self.assertTrue(tool_call["id"].startswith("dsml-"))
self.assertIn("hello", bot.messages[2]["content"])
self.assertNotIn("DSML", bot.messages[1]["content"] or "")
self.assertIsNone(BlackCatBot.validate_wire(bot._wire_messages()))
for msg in bot._wire_messages():
self.assertNotIn("DSML", msg.get("content") or "")
events = [f for f in bot.transcript if f["kind"] == "dsml"]
self.assertEqual(events[0]["event"], "recovered")
self.assertEqual(events[0]["tools"], ["read"])
async def test_native_calls_win_markup_only_stripped(self):
with open(self.path("f.txt"), "w", encoding="utf-8") as fh:
fh.write("hello\n")
noise = "working " + M("invoke", closer=True, doubled=True,
space=True)
bot = self.bot([
_tool_resp(noise, _call("c1", "read", '{"path": "f.txt"}')),
_text_resp("done")])
self.assertEqual(await bot.prompt("read f"), "done")
tools = [m for m in bot.messages if m.get("role") == "tool"]
self.assertEqual(len(tools), 1)
self.assertEqual(tools[0]["tool_call_id"], "c1")
self.assertNotIn("DSML", bot.messages[1]["content"] or "")
events = [f for f in bot.transcript if f["kind"] == "dsml"]
self.assertEqual(events[0]["event"], "stripped")
async def test_unrecoverable_sanitized_then_nudged_once(self):
tail = ('{"failed": 1} '
+ M("parameter", closer=True, doubled=True, space=True)
+ " " + M("invoke", closer=True, doubled=True, space=True))
bot = self.bot([_text_resp(tail), _text_resp("fine")])
self.assertEqual(await bot.prompt("go"), "fine")
self.assertEqual(len(bot._client.calls), 2)
roles = [m["role"] for m in bot.messages]
self.assertEqual(roles, ["user", "assistant", "user", "assistant"])
self.assertIn("[removed malformed tool markup]",
bot.messages[1]["content"])
self.assertIn("broken tool-call markup", bot.messages[2]["content"])
self.assertNotIn("DSML", bot.messages[2]["content"])
for msg in bot._wire_messages():
self.assertNotIn("DSML", msg.get("content") or "")
async def test_nudges_are_bounded(self):
tail = '{"x": 1} ' + M("invoke", closer=True)
bot = self.bot([_text_resp(tail), _text_resp(tail),
_text_resp(tail)])
reply = await bot.prompt("go")
self.assertEqual(len(bot._client.calls), 3)
self.assertIn("[removed malformed tool markup]", reply)
self.assertNotIn("DSML", reply)
async def test_recovery_opt_out_keeps_verbatim(self):
bot = self.bot([_text_resp(read_block())], dsml_recovery=False)
reply = await bot.prompt("go")
self.assertIn("DSML", reply)
self.assertEqual([f["kind"] for f in bot.transcript
if f["kind"] == "dsml"], [])
async def test_plain_chat_answer_sanitized(self):
bot = self.bot([_text_resp("text " + M("invoke", closer=True))],
enable_tools=False)
reply = await bot.prompt("hi")
self.assertNotIn("DSML", reply)
self.assertIn("[removed malformed tool markup]", reply)
if __name__ == "__main__":
unittest.main()