"""Committed tests for DeepSeek DSML leak recovery (no network). DeepSeek V4/V4.1 natively emits tool calls as DSML markup; when the provider-side parser fails, raw tags leak into assistant ``content`` instead of arriving as ``tool_calls``. ``blackcat.dsml`` recovers those calls client-side and strips the markup so it never poisons history. Live DSML tokens are never pasted into this file: the ``M()`` helper builds every tag programmatically (``\\uFF5C`` bars), so the source itself cannot become a phantom tool call. """ # retoor import json import os import tempfile import unittest from blackcat.agent import TOOL_SCHEMAS, BlackCatBot from blackcat.dsml import looks_like_dsml, recover_dsml from blackcat.fstate import FileState from blackcat.tools import FileTools FW = "\uFF5C" def M(tag, attrs="", closer=False, doubled=False, ascii=False, space=False, marker=True): """Build one DSML (or bare) tag without pasting a live token.""" if not marker: if closer: return "" % tag return "<%s%s>" % (tag, attrs) bar = "|" if ascii else FW bars = bar * (2 if doubled else 1) if closer: if not tag: return "" % (bars, bars) return "" % (bars, bars, " " if space else "", tag) return "<%sDSML%s%s%s%s>" % (bars, bars, " " if space else "", tag, attrs) def EOS(kind="end"): """A DeepSeek begin/end-of-sentence residue token.""" return "<%s%s\u2581of\u2581sentence%s>" % (FW, kind, FW) def _text_resp(text): return {"choices": [{"message": {"role": "assistant", "content": text}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 10}} def _tool_resp(content, *calls): return {"choices": [{"message": {"role": "assistant", "content": content, "tool_calls": list(calls)}, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 10}} def _call(cid, name, arguments): return {"id": cid, "type": "function", "function": {"name": name, "arguments": arguments}} class StubClient: model = "stub-model" def __init__(self, script): self.script = list(script) self.calls = [] async def chat(self, messages, **kwargs): self.calls.append((messages, kwargs)) item = self.script.pop(0) if isinstance(item, Exception): raise item return item def read_block(path="f.txt", **kw): """A canonical read call inside a tool_calls block.""" return (M("tool_calls", **kw) + M("invoke", ' name="read"', **kw) + M("parameter", ' name="path"', **kw) + path + M("parameter", closer=True, **kw) + M("invoke", closer=True, **kw) + M("tool_calls", closer=True, **kw)) class TestFastPath(unittest.TestCase): def test_plain_prose_is_passthrough(self): for text in ("Hello world", "", "a < b > c", "DSMLess prose"): rec = recover_dsml(text, TOOL_SCHEMAS) self.assertEqual(rec["calls"], []) self.assertEqual(rec["removed"], 0) self.assertEqual(rec["text"], text) self.assertEqual(rec["noted_text"], text) def test_looks_like_dsml(self): self.assertFalse(looks_like_dsml("plain prose")) self.assertFalse(looks_like_dsml("")) self.assertTrue(looks_like_dsml(M("invoke", closer=True))) self.assertTrue(looks_like_dsml("x " + EOS())) self.assertTrue(looks_like_dsml("x y")) def test_hyphen_block_tag_is_left_alone(self): text = ("<%sDSML%stool-calls\nrecord_item\n" % (FW, FW)) rec = recover_dsml(text, TOOL_SCHEMAS) self.assertEqual(rec["calls"], []) self.assertEqual(rec["removed"], 0) self.assertEqual(rec["text"], text) class TestCanonicalShapes(unittest.TestCase): def test_canonical_single_bar(self): rec = recover_dsml(read_block(), TOOL_SCHEMAS) self.assertEqual(rec["calls"], [{"name": "read", "arguments": {"path": "f.txt"}}]) self.assertEqual(rec["text"].strip(), "") self.assertGreaterEqual(rec["removed"], 1) self.assertEqual(rec["names"], ["read"]) def test_doubled_bars_with_spaced_closers(self): text = (M("tool_calls", doubled=True) + M("invoke", ' name="write_file"', doubled=True) + M("parameter", ' name="content" string="true"', doubled=True) + '{"failed": 0, "written": 3}' + M("parameter", closer=True, doubled=True, space=True) + M("invoke", closer=True, doubled=True, space=True) + M("calls", closer=True, doubled=True, space=True)) rec = recover_dsml(text, TOOL_SCHEMAS) self.assertEqual(rec["calls"], [{ "name": "write_file", "arguments": {"content": '{"failed": 0, "written": 3}'}}]) self.assertEqual(rec["text"].strip(), "") def test_ascii_pipes(self): rec = recover_dsml(read_block(ascii=True), TOOL_SCHEMAS) self.assertEqual(rec["calls"], [{"name": "read", "arguments": {"path": "f.txt"}}]) def test_block_tag_variants(self): for block in ("tool_calls", "tool_call", "function_calls", "calls"): text = (M(block) + M("invoke", ' name="read"') + M("parameter", ' name="path"') + "f.txt" + M("parameter", closer=True) + M("invoke", closer=True) + M(block, closer=True)) rec = recover_dsml(text, TOOL_SCHEMAS) self.assertEqual(rec["calls"], [ {"name": "read", "arguments": {"path": "f.txt"}}], block) def test_orphan_invoke_without_block(self): text = (M("invoke", ' name="read"') + M("parameter", ' name="path"') + "f.txt" + M("parameter", closer=True) + M("invoke", closer=True)) rec = recover_dsml(text, TOOL_SCHEMAS) self.assertEqual(rec["calls"], [{"name": "read", "arguments": {"path": "f.txt"}}]) def test_name_spacing_and_quote_variants(self): for attrs in (' name = "read"', " name='read'", " name=read"): text = (M("invoke", attrs) + M("parameter", ' name = "path" ') + "f.txt" + M("parameter", closer=True) + M("invoke", closer=True)) rec = recover_dsml(text, TOOL_SCHEMAS) self.assertEqual(rec["calls"], [ {"name": "read", "arguments": {"path": "f.txt"}}], attrs) def test_surrounding_prose_preserved(self): text = "Let me read it.\n" + read_block() + "\nDone." rec = recover_dsml(text, TOOL_SCHEMAS) self.assertEqual(len(rec["calls"]), 1) self.assertEqual(rec["text"], "Let me read it.\n\nDone.") def test_multiple_invokes_in_one_block(self): text = (M("tool_calls") + M("invoke", ' name="read"') + M("parameter", ' name="path"') + "a.txt" + M("parameter", closer=True) + M("invoke", closer=True) + M("invoke", ' name="read"') + M("parameter", ' name="path"') + "b.txt" + M("parameter", closer=True) + M("invoke", closer=True) + M("tool_calls", closer=True)) rec = recover_dsml(text, TOOL_SCHEMAS) self.assertEqual([c["arguments"]["path"] for c in rec["calls"]], ["a.txt", "b.txt"]) def test_idempotent(self): text = "Prose\n" + read_block() + "\nTail " + EOS() once = recover_dsml(text, TOOL_SCHEMAS) twice = recover_dsml(once["text"], TOOL_SCHEMAS) self.assertEqual(twice["removed"], 0) self.assertEqual(twice["text"], once["text"]) class TestValues(unittest.TestCase): def test_multiline_value_keeps_indentation(self): text = (M("invoke", ' name="write_file"') + M("parameter", ' name="content"') + "\nline1\n line2\n" + M("parameter", closer=True) + M("invoke", closer=True)) rec = recover_dsml(text, TOOL_SCHEMAS) self.assertEqual(rec["calls"][0]["arguments"]["content"], "line1\n line2") def test_entities_decoded(self): text = (M("invoke", ' name="read"') + M("parameter", ' name="path"') + "a "q" & b" + M("parameter", closer=True) + M("invoke", closer=True)) rec = recover_dsml(text, TOOL_SCHEMAS) self.assertEqual(rec["calls"][0]["arguments"]["path"], 'a "q" & b') def test_json_param_parsed_and_fallback(self): text = (M("invoke", ' name="multi_edit"') + M("parameter", ' name="edits" string="false"') + '[{"a": 1}]' + M("parameter", closer=True) + M("invoke", closer=True)) rec = recover_dsml(text, TOOL_SCHEMAS) self.assertEqual(rec["calls"][0]["arguments"]["edits"], [{"a": 1}]) bad = (M("invoke", ' name="read"') + M("parameter", ' name="path" string="false"') + "not json" + M("parameter", closer=True) + M("invoke", closer=True)) rec = recover_dsml(bad, TOOL_SCHEMAS) self.assertEqual(rec["calls"][0]["arguments"]["path"], "not json") def test_raw_json_body(self): text = (M("invoke", ' name="read"') + '{"path": "f.txt"}' + M("invoke", closer=True)) rec = recover_dsml(text, TOOL_SCHEMAS) self.assertEqual(rec["calls"], [{"name": "read", "arguments": {"path": "f.txt"}}]) bad = (M("invoke", ' name="read"') + "{nope" + M("invoke", closer=True)) rec = recover_dsml(bad, TOOL_SCHEMAS) self.assertEqual(rec["calls"], []) self.assertGreater(rec["removed"], 0) def test_eos_tokens_stripped(self): rec = recover_dsml("answer " + EOS(), TOOL_SCHEMAS) self.assertEqual(rec["calls"], []) self.assertEqual(rec["removed"], 1) self.assertNotIn("sentence", rec["text"]) class TestDegradedShapes(unittest.TestCase): def test_misspelled_closer_drops_alpha_keeps_beta(self): text = (M("invoke", ' name="record_item"') + M("parameter", ' name="alpha" string="true"') + "first" + M("", closer=True) + M("parameter", ' name="beta" string="true"') + "second" + M("parameter", closer=True) + M("invoke", closer=True)) rec = recover_dsml(text, TOOL_SCHEMAS) self.assertEqual(rec["calls"], [ {"name": "record_item", "arguments": {"beta": "second"}}]) def test_runaway_invoke_name_rejected_and_stripped(self): text = (M("tool_calls") + M("invoke", ' name="record_item {"') + "category: Dexes" + M("invoke", closer=True) + M("tool_calls", closer=True)) rec = recover_dsml(text, TOOL_SCHEMAS) self.assertEqual(rec["calls"], []) self.assertGreater(rec["removed"], 0) self.assertNotIn("DSML", rec["text"]) def test_unterminated_opener_line_stripped(self): text = ("Hello\n<%sDSML%sinvoke name=\"read\" {\n" "category: Dexes" % (FW, FW)) rec = recover_dsml(text, TOOL_SCHEMAS) self.assertEqual(rec["calls"], []) self.assertIn("Hello", rec["text"]) self.assertIn("category: Dexes", rec["text"]) self.assertNotIn("DSML", rec["text"]) def test_undeclared_tool_kept_for_runtime(self): text = (M("invoke", ' name="nope"') + M("parameter", ' name="x"') + "1" + M("parameter", closer=True) + M("invoke", closer=True)) rec = recover_dsml(text, TOOL_SCHEMAS) self.assertEqual(rec["calls"], [{"name": "nope", "arguments": {"x": "1"}}]) def test_empty_invoke_name_rejected(self): text = (M("invoke", ' name=""') + M("parameter", ' name="path"') + "f.txt" + M("parameter", closer=True) + M("invoke", closer=True)) rec = recover_dsml(text, TOOL_SCHEMAS) self.assertEqual(rec["calls"], []) self.assertGreater(rec["removed"], 0) def test_empty_args_rejected(self): text = M("invoke", ' name="read"') + M("invoke", closer=True) rec = recover_dsml(text, TOOL_SCHEMAS) self.assertEqual(rec["calls"], []) self.assertGreater(rec["removed"], 0) self.assertIn("read", rec["names"]) def test_unclosed_invoke_dropped_complete_kept(self): text = (read_block() + M("invoke", ' name="read"') + M("parameter", ' name="path"') + "half.txt") rec = recover_dsml(text, TOOL_SCHEMAS) self.assertEqual(len(rec["calls"]), 1) self.assertEqual(rec["calls"][0]["arguments"]["path"], "f.txt") self.assertNotIn("DSML", rec["text"]) def test_stray_closers_stripped_nothing_recovered(self): text = ('{"failed": 1, "written": 2} ' + M("parameter", closer=True, doubled=True, space=True) + " " + M("invoke", closer=True, doubled=True, space=True) + " " + M("calls", closer=True, doubled=True, space=True)) rec = recover_dsml(text, TOOL_SCHEMAS) self.assertEqual(rec["calls"], []) self.assertEqual(rec["removed"], 1) self.assertIn('{"failed": 1, "written": 2}', rec["text"]) self.assertNotIn("DSML", rec["text"]) self.assertIn("[removed malformed tool markup]", rec["noted_text"]) def test_code_fence_ignored(self): text = ("Here:\n```\n" + read_block() + "\n```\nDone") rec = recover_dsml(text, TOOL_SCHEMAS) self.assertEqual(rec["calls"], []) self.assertEqual(rec["removed"], 0) self.assertEqual(rec["text"], text) def test_unclosed_fence_ignored(self): rec = recover_dsml("```\n" + read_block(), TOOL_SCHEMAS) self.assertEqual(rec["calls"], []) self.assertEqual(rec["removed"], 0) class TestOrphanParameters(unittest.TestCase): def test_wrapped_tool_bare_outer(self): text = ('' + M("parameter", ' name="path"') + "f.txt" + M("parameter", closer=True) + "") rec = recover_dsml(text, TOOL_SCHEMAS) self.assertEqual(rec["calls"], [{"name": "read", "arguments": {"path": "f.txt"}}]) def test_wrapped_tool_marked_outer(self): text = (M("parameter", ' name="read"') + M("parameter", ' name="path"') + "f.txt" + M("parameter", closer=True) + M("parameter", closer=True)) rec = recover_dsml(text, TOOL_SCHEMAS) self.assertEqual(rec["calls"], [{"name": "read", "arguments": {"path": "f.txt"}}]) def test_bare_value_with_derivable_key(self): text = (M("parameter", ' name="read"') + "f.txt" + M("parameter", closer=True) + M("invoke", closer=True)) rec = recover_dsml(text, TOOL_SCHEMAS) self.assertEqual(rec["calls"], [{"name": "read", "arguments": {"path": "f.txt"}}]) def test_bare_value_without_key_redacted(self): text = (M("parameter", ' name="write_file"') + "blob" + M("parameter", closer=True) + M("invoke", closer=True)) rec = recover_dsml(text, TOOL_SCHEMAS) self.assertEqual(rec["calls"], []) self.assertGreater(rec["removed"], 0) self.assertIn("[removed malformed write_file call]", rec["noted_text"]) def test_bare_value_without_schema_redacted(self): text = (M("parameter", ' name="read"') + "f.txt" + M("parameter", closer=True) + M("invoke", closer=True)) rec = recover_dsml(text, None) self.assertEqual(rec["calls"], []) self.assertGreater(rec["removed"], 0) def test_bare_value_without_closer_context_left_careful(self): text = (M("parameter", ' name="read"') + "f.txt" + M("parameter", closer=True)) rec = recover_dsml(text, TOOL_SCHEMAS) self.assertEqual(rec["calls"], []) self.assertNotIn("DSML", rec["text"]) class LoopCase(unittest.IsolatedAsyncioTestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) def bot(self, script, **kwargs): kwargs.setdefault("system", "t") bot = BlackCatBot(provider="deepseek", api_key="k", tools=FileTools(FileState(self.tmp.name)), **kwargs) bot._client = StubClient(script) self.addAsyncCleanup(bot.aclose) return bot def path(self, name): return os.path.join(self.tmp.name, name) class TestAgentRecovery(LoopCase): async def test_leaked_call_executes_and_history_stays_clean(self): with open(self.path("f.txt"), "w", encoding="utf-8") as fh: fh.write("hello\n") leak = "I will read it.\n" + read_block() bot = self.bot([_text_resp(leak), _text_resp("it says hello")]) self.assertEqual(await bot.prompt("read f"), "it says hello") roles = [m["role"] for m in bot.messages] self.assertEqual(roles, ["user", "assistant", "tool", "assistant"]) tool_call = bot.messages[1]["tool_calls"][0] self.assertEqual(tool_call["function"]["name"], "read") self.assertEqual(json.loads(tool_call["function"]["arguments"]), {"path": "f.txt"}) self.assertTrue(tool_call["id"].startswith("dsml-")) self.assertIn("hello", bot.messages[2]["content"]) self.assertNotIn("DSML", bot.messages[1]["content"] or "") self.assertIsNone(BlackCatBot.validate_wire(bot._wire_messages())) for msg in bot._wire_messages(): self.assertNotIn("DSML", msg.get("content") or "") events = [f for f in bot.transcript if f["kind"] == "dsml"] self.assertEqual(events[0]["event"], "recovered") self.assertEqual(events[0]["tools"], ["read"]) async def test_native_calls_win_markup_only_stripped(self): with open(self.path("f.txt"), "w", encoding="utf-8") as fh: fh.write("hello\n") noise = "working " + M("invoke", closer=True, doubled=True, space=True) bot = self.bot([ _tool_resp(noise, _call("c1", "read", '{"path": "f.txt"}')), _text_resp("done")]) self.assertEqual(await bot.prompt("read f"), "done") tools = [m for m in bot.messages if m.get("role") == "tool"] self.assertEqual(len(tools), 1) self.assertEqual(tools[0]["tool_call_id"], "c1") self.assertNotIn("DSML", bot.messages[1]["content"] or "") events = [f for f in bot.transcript if f["kind"] == "dsml"] self.assertEqual(events[0]["event"], "stripped") async def test_unrecoverable_sanitized_then_nudged_once(self): tail = ('{"failed": 1} ' + M("parameter", closer=True, doubled=True, space=True) + " " + M("invoke", closer=True, doubled=True, space=True)) bot = self.bot([_text_resp(tail), _text_resp("fine")]) self.assertEqual(await bot.prompt("go"), "fine") self.assertEqual(len(bot._client.calls), 2) roles = [m["role"] for m in bot.messages] self.assertEqual(roles, ["user", "assistant", "user", "assistant"]) self.assertIn("[removed malformed tool markup]", bot.messages[1]["content"]) self.assertIn("broken tool-call markup", bot.messages[2]["content"]) self.assertNotIn("DSML", bot.messages[2]["content"]) for msg in bot._wire_messages(): self.assertNotIn("DSML", msg.get("content") or "") async def test_nudges_are_bounded(self): tail = '{"x": 1} ' + M("invoke", closer=True) bot = self.bot([_text_resp(tail), _text_resp(tail), _text_resp(tail)]) reply = await bot.prompt("go") self.assertEqual(len(bot._client.calls), 3) self.assertIn("[removed malformed tool markup]", reply) self.assertNotIn("DSML", reply) async def test_recovery_opt_out_keeps_verbatim(self): bot = self.bot([_text_resp(read_block())], dsml_recovery=False) reply = await bot.prompt("go") self.assertIn("DSML", reply) self.assertEqual([f["kind"] for f in bot.transcript if f["kind"] == "dsml"], []) async def test_plain_chat_answer_sanitized(self): bot = self.bot([_text_resp("text " + M("invoke", closer=True))], enable_tools=False) reply = await bot.prompt("hi") self.assertNotIn("DSML", reply) self.assertIn("[removed malformed tool markup]", reply) if __name__ == "__main__": unittest.main()