Skip to content
Repos

blackcat

Python 21.1 KB504 lines Opens in site

Directory Raw Download

"""Committed tests for DeepSeek DSML leak recovery (no network).

DeepSeek V4/V4.1 natively emits tool calls as DSML markup; when the
provider-side parser fails, raw tags leak into assistant ``content``
instead of arriving as ``tool_calls``. ``blackcat.dsml`` recovers those
calls client-side and strips the markup so it never poisons history.

Live DSML tokens are never pasted into this file: the ``M()`` helper
builds every tag programmatically (``\\uFF5C`` bars), so the source
itself cannot become a phantom tool call.
"""
# retoor <retoor@molodetz.nl>

import json
import os
import tempfile
import unittest

from blackcat.agent import TOOL_SCHEMAS, BlackCatBot
from blackcat.dsml import looks_like_dsml, recover_dsml
from blackcat.fstate import FileState
from blackcat.tools import FileTools

FW = "\uFF5C"


def M(tag, attrs="", closer=False, doubled=False, ascii=False, space=False,
      marker=True):
    """Build one DSML (or bare) tag without pasting a live token."""
    if not marker:
        if closer:
            return "</%s>" % tag
        return "<%s%s>" % (tag, attrs)
    bar = "|" if ascii else FW
    bars = bar * (2 if doubled else 1)
    if closer:
        if not tag:
            return "</%sDSML%s>" % (bars, bars)
        return "</%sDSML%s%s%s>" % (bars, bars, " " if space else "",
                                   tag)
    return "<%sDSML%s%s%s%s>" % (bars, bars, " " if space else "", tag,
                                attrs)


def EOS(kind="end"):
    """A DeepSeek begin/end-of-sentence residue token."""
    return "<%s%s\u2581of\u2581sentence%s>" % (FW, kind, FW)


def _text_resp(text):
    return {"choices": [{"message": {"role": "assistant",
                                     "content": text},
                         "finish_reason": "stop"}],
            "usage": {"prompt_tokens": 10}}


def _tool_resp(content, *calls):
    return {"choices": [{"message": {"role": "assistant",
                                     "content": content,
                                     "tool_calls": list(calls)},
                         "finish_reason": "tool_calls"}],
            "usage": {"prompt_tokens": 10}}


def _call(cid, name, arguments):
    return {"id": cid, "type": "function",
            "function": {"name": name, "arguments": arguments}}


class StubClient:
    model = "stub-model"

    def __init__(self, script):
        self.script = list(script)
        self.calls = []

    async def chat(self, messages, **kwargs):
        self.calls.append((messages, kwargs))
        item = self.script.pop(0)
        if isinstance(item, Exception):
            raise item
        return item


def read_block(path="f.txt", **kw):
    """A canonical read call inside a tool_calls block."""
    return (M("tool_calls", **kw)
            + M("invoke", ' name="read"', **kw)
            + M("parameter", ' name="path"', **kw) + path
            + M("parameter", closer=True, **kw)
            + M("invoke", closer=True, **kw)
            + M("tool_calls", closer=True, **kw))


class TestFastPath(unittest.TestCase):
    def test_plain_prose_is_passthrough(self):
        for text in ("Hello world", "", "a < b > c", "DSMLess prose"):
            rec = recover_dsml(text, TOOL_SCHEMAS)
            self.assertEqual(rec["calls"], [])
            self.assertEqual(rec["removed"], 0)
            self.assertEqual(rec["text"], text)
            self.assertEqual(rec["noted_text"], text)

    def test_looks_like_dsml(self):
        self.assertFalse(looks_like_dsml("plain prose"))
        self.assertFalse(looks_like_dsml(""))
        self.assertTrue(looks_like_dsml(M("invoke", closer=True)))
        self.assertTrue(looks_like_dsml("x " + EOS()))
        self.assertTrue(looks_like_dsml("x </parameter> y"))

    def test_hyphen_block_tag_is_left_alone(self):
        text = ("<%sDSML%stool-calls\nrecord_item\n</plan>"
                % (FW, FW))
        rec = recover_dsml(text, TOOL_SCHEMAS)
        self.assertEqual(rec["calls"], [])
        self.assertEqual(rec["removed"], 0)
        self.assertEqual(rec["text"], text)


class TestCanonicalShapes(unittest.TestCase):
    def test_canonical_single_bar(self):
        rec = recover_dsml(read_block(), TOOL_SCHEMAS)
        self.assertEqual(rec["calls"],
                         [{"name": "read", "arguments": {"path": "f.txt"}}])
        self.assertEqual(rec["text"].strip(), "")
        self.assertGreaterEqual(rec["removed"], 1)
        self.assertEqual(rec["names"], ["read"])

    def test_doubled_bars_with_spaced_closers(self):
        text = (M("tool_calls", doubled=True)
                + M("invoke", ' name="write_file"', doubled=True)
                + M("parameter", ' name="content" string="true"',
                    doubled=True)
                + '{"failed": 0, "written": 3}'
                + M("parameter", closer=True, doubled=True, space=True)
                + M("invoke", closer=True, doubled=True, space=True)
                + M("calls", closer=True, doubled=True, space=True))
        rec = recover_dsml(text, TOOL_SCHEMAS)
        self.assertEqual(rec["calls"], [{
            "name": "write_file",
            "arguments": {"content": '{"failed": 0, "written": 3}'}}])
        self.assertEqual(rec["text"].strip(), "")

    def test_ascii_pipes(self):
        rec = recover_dsml(read_block(ascii=True), TOOL_SCHEMAS)
        self.assertEqual(rec["calls"],
                         [{"name": "read", "arguments": {"path": "f.txt"}}])

    def test_block_tag_variants(self):
        for block in ("tool_calls", "tool_call", "function_calls",
                      "calls"):
            text = (M(block) + M("invoke", ' name="read"')
                    + M("parameter", ' name="path"') + "f.txt"
                    + M("parameter", closer=True)
                    + M("invoke", closer=True) + M(block, closer=True))
            rec = recover_dsml(text, TOOL_SCHEMAS)
            self.assertEqual(rec["calls"], [
                {"name": "read", "arguments": {"path": "f.txt"}}], block)

    def test_orphan_invoke_without_block(self):
        text = (M("invoke", ' name="read"')
                + M("parameter", ' name="path"') + "f.txt"
                + M("parameter", closer=True)
                + M("invoke", closer=True))
        rec = recover_dsml(text, TOOL_SCHEMAS)
        self.assertEqual(rec["calls"],
                         [{"name": "read", "arguments": {"path": "f.txt"}}])

    def test_name_spacing_and_quote_variants(self):
        for attrs in (' name = "read"', " name='read'", " name=read"):
            text = (M("invoke", attrs)
                    + M("parameter", ' name = "path" ') + "f.txt"
                    + M("parameter", closer=True)
                    + M("invoke", closer=True))
            rec = recover_dsml(text, TOOL_SCHEMAS)
            self.assertEqual(rec["calls"], [
                {"name": "read", "arguments": {"path": "f.txt"}}], attrs)

    def test_surrounding_prose_preserved(self):
        text = "Let me read it.\n" + read_block() + "\nDone."
        rec = recover_dsml(text, TOOL_SCHEMAS)
        self.assertEqual(len(rec["calls"]), 1)
        self.assertEqual(rec["text"], "Let me read it.\n\nDone.")

    def test_multiple_invokes_in_one_block(self):
        text = (M("tool_calls")
                + M("invoke", ' name="read"')
                + M("parameter", ' name="path"') + "a.txt"
                + M("parameter", closer=True)
                + M("invoke", closer=True)
                + M("invoke", ' name="read"')
                + M("parameter", ' name="path"') + "b.txt"
                + M("parameter", closer=True)
                + M("invoke", closer=True)
                + M("tool_calls", closer=True))
        rec = recover_dsml(text, TOOL_SCHEMAS)
        self.assertEqual([c["arguments"]["path"] for c in rec["calls"]],
                         ["a.txt", "b.txt"])

    def test_idempotent(self):
        text = "Prose\n" + read_block() + "\nTail " + EOS()
        once = recover_dsml(text, TOOL_SCHEMAS)
        twice = recover_dsml(once["text"], TOOL_SCHEMAS)
        self.assertEqual(twice["removed"], 0)
        self.assertEqual(twice["text"], once["text"])


class TestValues(unittest.TestCase):
    def test_multiline_value_keeps_indentation(self):
        text = (M("invoke", ' name="write_file"')
                + M("parameter", ' name="content"')
                + "\nline1\n  line2\n"
                + M("parameter", closer=True)
                + M("invoke", closer=True))
        rec = recover_dsml(text, TOOL_SCHEMAS)
        self.assertEqual(rec["calls"][0]["arguments"]["content"],
                         "line1\n  line2")

    def test_entities_decoded(self):
        text = (M("invoke", ' name="read"')
                + M("parameter", ' name="path"')
                + "a &quot;q&quot; &amp; b"
                + M("parameter", closer=True)
                + M("invoke", closer=True))
        rec = recover_dsml(text, TOOL_SCHEMAS)
        self.assertEqual(rec["calls"][0]["arguments"]["path"],
                         'a "q" & b')

    def test_json_param_parsed_and_fallback(self):
        text = (M("invoke", ' name="multi_edit"')
                + M("parameter", ' name="edits" string="false"')
                + '[{"a": 1}]' + M("parameter", closer=True)
                + M("invoke", closer=True))
        rec = recover_dsml(text, TOOL_SCHEMAS)
        self.assertEqual(rec["calls"][0]["arguments"]["edits"],
                         [{"a": 1}])
        bad = (M("invoke", ' name="read"')
               + M("parameter", ' name="path" string="false"')
               + "not json" + M("parameter", closer=True)
               + M("invoke", closer=True))
        rec = recover_dsml(bad, TOOL_SCHEMAS)
        self.assertEqual(rec["calls"][0]["arguments"]["path"],
                         "not json")

    def test_raw_json_body(self):
        text = (M("invoke", ' name="read"') + '{"path": "f.txt"}'
                + M("invoke", closer=True))
        rec = recover_dsml(text, TOOL_SCHEMAS)
        self.assertEqual(rec["calls"],
                         [{"name": "read", "arguments": {"path": "f.txt"}}])
        bad = (M("invoke", ' name="read"') + "{nope"
               + M("invoke", closer=True))
        rec = recover_dsml(bad, TOOL_SCHEMAS)
        self.assertEqual(rec["calls"], [])
        self.assertGreater(rec["removed"], 0)

    def test_eos_tokens_stripped(self):
        rec = recover_dsml("answer " + EOS(), TOOL_SCHEMAS)
        self.assertEqual(rec["calls"], [])
        self.assertEqual(rec["removed"], 1)
        self.assertNotIn("sentence", rec["text"])


class TestDegradedShapes(unittest.TestCase):
    def test_misspelled_closer_drops_alpha_keeps_beta(self):
        text = (M("invoke", ' name="record_item"')
                + M("parameter", ' name="alpha" string="true"') + "first"
                + M("", closer=True)
                + M("parameter", ' name="beta" string="true"') + "second"
                + M("parameter", closer=True)
                + M("invoke", closer=True))
        rec = recover_dsml(text, TOOL_SCHEMAS)
        self.assertEqual(rec["calls"], [
            {"name": "record_item", "arguments": {"beta": "second"}}])

    def test_runaway_invoke_name_rejected_and_stripped(self):
        text = (M("tool_calls")
                + M("invoke", ' name="record_item {"')
                + "category: Dexes"
                + M("invoke", closer=True) + M("tool_calls", closer=True))
        rec = recover_dsml(text, TOOL_SCHEMAS)
        self.assertEqual(rec["calls"], [])
        self.assertGreater(rec["removed"], 0)
        self.assertNotIn("DSML", rec["text"])

    def test_unterminated_opener_line_stripped(self):
        text = ("Hello\n<%sDSML%sinvoke name=\"read\" {\n"
                "category: Dexes" % (FW, FW))
        rec = recover_dsml(text, TOOL_SCHEMAS)
        self.assertEqual(rec["calls"], [])
        self.assertIn("Hello", rec["text"])
        self.assertIn("category: Dexes", rec["text"])
        self.assertNotIn("DSML", rec["text"])

    def test_undeclared_tool_kept_for_runtime(self):
        text = (M("invoke", ' name="nope"')
                + M("parameter", ' name="x"') + "1"
                + M("parameter", closer=True)
                + M("invoke", closer=True))
        rec = recover_dsml(text, TOOL_SCHEMAS)
        self.assertEqual(rec["calls"],
                         [{"name": "nope", "arguments": {"x": "1"}}])

    def test_empty_invoke_name_rejected(self):
        text = (M("invoke", ' name=""')
                + M("parameter", ' name="path"') + "f.txt"
                + M("parameter", closer=True)
                + M("invoke", closer=True))
        rec = recover_dsml(text, TOOL_SCHEMAS)
        self.assertEqual(rec["calls"], [])
        self.assertGreater(rec["removed"], 0)

    def test_empty_args_rejected(self):
        text = M("invoke", ' name="read"') + M("invoke", closer=True)
        rec = recover_dsml(text, TOOL_SCHEMAS)
        self.assertEqual(rec["calls"], [])
        self.assertGreater(rec["removed"], 0)
        self.assertIn("read", rec["names"])

    def test_unclosed_invoke_dropped_complete_kept(self):
        text = (read_block()
                + M("invoke", ' name="read"')
                + M("parameter", ' name="path"') + "half.txt")
        rec = recover_dsml(text, TOOL_SCHEMAS)
        self.assertEqual(len(rec["calls"]), 1)
        self.assertEqual(rec["calls"][0]["arguments"]["path"], "f.txt")
        self.assertNotIn("DSML", rec["text"])

    def test_stray_closers_stripped_nothing_recovered(self):
        text = ('{"failed": 1, "written": 2} '
                + M("parameter", closer=True, doubled=True, space=True)
                + " " + M("invoke", closer=True, doubled=True, space=True)
                + " " + M("calls", closer=True, doubled=True, space=True))
        rec = recover_dsml(text, TOOL_SCHEMAS)
        self.assertEqual(rec["calls"], [])
        self.assertEqual(rec["removed"], 1)
        self.assertIn('{"failed": 1, "written": 2}', rec["text"])
        self.assertNotIn("DSML", rec["text"])
        self.assertIn("[removed malformed tool markup]",
                      rec["noted_text"])

    def test_code_fence_ignored(self):
        text = ("Here:\n```\n" + read_block() + "\n```\nDone")
        rec = recover_dsml(text, TOOL_SCHEMAS)
        self.assertEqual(rec["calls"], [])
        self.assertEqual(rec["removed"], 0)
        self.assertEqual(rec["text"], text)

    def test_unclosed_fence_ignored(self):
        rec = recover_dsml("```\n" + read_block(), TOOL_SCHEMAS)
        self.assertEqual(rec["calls"], [])
        self.assertEqual(rec["removed"], 0)


class TestOrphanParameters(unittest.TestCase):
    def test_wrapped_tool_bare_outer(self):
        text = ('<parameter name="read">'
                + M("parameter", ' name="path"') + "f.txt"
                + M("parameter", closer=True) + "</parameter>")
        rec = recover_dsml(text, TOOL_SCHEMAS)
        self.assertEqual(rec["calls"],
                         [{"name": "read", "arguments": {"path": "f.txt"}}])

    def test_wrapped_tool_marked_outer(self):
        text = (M("parameter", ' name="read"')
                + M("parameter", ' name="path"') + "f.txt"
                + M("parameter", closer=True)
                + M("parameter", closer=True))
        rec = recover_dsml(text, TOOL_SCHEMAS)
        self.assertEqual(rec["calls"],
                         [{"name": "read", "arguments": {"path": "f.txt"}}])

    def test_bare_value_with_derivable_key(self):
        text = (M("parameter", ' name="read"') + "f.txt"
                + M("parameter", closer=True)
                + M("invoke", closer=True))
        rec = recover_dsml(text, TOOL_SCHEMAS)
        self.assertEqual(rec["calls"],
                         [{"name": "read", "arguments": {"path": "f.txt"}}])

    def test_bare_value_without_key_redacted(self):
        text = (M("parameter", ' name="write_file"') + "blob"
                + M("parameter", closer=True)
                + M("invoke", closer=True))
        rec = recover_dsml(text, TOOL_SCHEMAS)
        self.assertEqual(rec["calls"], [])
        self.assertGreater(rec["removed"], 0)
        self.assertIn("[removed malformed write_file call]",
                      rec["noted_text"])

    def test_bare_value_without_schema_redacted(self):
        text = (M("parameter", ' name="read"') + "f.txt"
                + M("parameter", closer=True)
                + M("invoke", closer=True))
        rec = recover_dsml(text, None)
        self.assertEqual(rec["calls"], [])
        self.assertGreater(rec["removed"], 0)

    def test_bare_value_without_closer_context_left_careful(self):
        text = (M("parameter", ' name="read"') + "f.txt"
                + M("parameter", closer=True))
        rec = recover_dsml(text, TOOL_SCHEMAS)
        self.assertEqual(rec["calls"], [])
        self.assertNotIn("DSML", rec["text"])


class LoopCase(unittest.IsolatedAsyncioTestCase):
    def setUp(self):
        self.tmp = tempfile.TemporaryDirectory()
        self.addCleanup(self.tmp.cleanup)

    def bot(self, script, **kwargs):
        kwargs.setdefault("system", "t")
        bot = BlackCatBot(provider="deepseek", api_key="k",
                      tools=FileTools(FileState(self.tmp.name)), **kwargs)
        bot._client = StubClient(script)
        self.addAsyncCleanup(bot.aclose)
        return bot

    def path(self, name):
        return os.path.join(self.tmp.name, name)


class TestAgentRecovery(LoopCase):
    async def test_leaked_call_executes_and_history_stays_clean(self):
        with open(self.path("f.txt"), "w", encoding="utf-8") as fh:
            fh.write("hello\n")
        leak = "I will read it.\n" + read_block()
        bot = self.bot([_text_resp(leak), _text_resp("it says hello")])
        self.assertEqual(await bot.prompt("read f"), "it says hello")
        roles = [m["role"] for m in bot.messages]
        self.assertEqual(roles, ["user", "assistant", "tool", "assistant"])
        tool_call = bot.messages[1]["tool_calls"][0]
        self.assertEqual(tool_call["function"]["name"], "read")
        self.assertEqual(json.loads(tool_call["function"]["arguments"]),
                         {"path": "f.txt"})
        self.assertTrue(tool_call["id"].startswith("dsml-"))
        self.assertIn("hello", bot.messages[2]["content"])
        self.assertNotIn("DSML", bot.messages[1]["content"] or "")
        self.assertIsNone(BlackCatBot.validate_wire(bot._wire_messages()))
        for msg in bot._wire_messages():
            self.assertNotIn("DSML", msg.get("content") or "")
        events = [f for f in bot.transcript if f["kind"] == "dsml"]
        self.assertEqual(events[0]["event"], "recovered")
        self.assertEqual(events[0]["tools"], ["read"])

    async def test_native_calls_win_markup_only_stripped(self):
        with open(self.path("f.txt"), "w", encoding="utf-8") as fh:
            fh.write("hello\n")
        noise = "working " + M("invoke", closer=True, doubled=True,
                               space=True)
        bot = self.bot([
            _tool_resp(noise, _call("c1", "read", '{"path": "f.txt"}')),
            _text_resp("done")])
        self.assertEqual(await bot.prompt("read f"), "done")
        tools = [m for m in bot.messages if m.get("role") == "tool"]
        self.assertEqual(len(tools), 1)
        self.assertEqual(tools[0]["tool_call_id"], "c1")
        self.assertNotIn("DSML", bot.messages[1]["content"] or "")
        events = [f for f in bot.transcript if f["kind"] == "dsml"]
        self.assertEqual(events[0]["event"], "stripped")

    async def test_unrecoverable_sanitized_then_nudged_once(self):
        tail = ('{"failed": 1} '
                + M("parameter", closer=True, doubled=True, space=True)
                + " " + M("invoke", closer=True, doubled=True, space=True))
        bot = self.bot([_text_resp(tail), _text_resp("fine")])
        self.assertEqual(await bot.prompt("go"), "fine")
        self.assertEqual(len(bot._client.calls), 2)
        roles = [m["role"] for m in bot.messages]
        self.assertEqual(roles, ["user", "assistant", "user", "assistant"])
        self.assertIn("[removed malformed tool markup]",
                      bot.messages[1]["content"])
        self.assertIn("broken tool-call markup", bot.messages[2]["content"])
        self.assertNotIn("DSML", bot.messages[2]["content"])
        for msg in bot._wire_messages():
            self.assertNotIn("DSML", msg.get("content") or "")

    async def test_nudges_are_bounded(self):
        tail = '{"x": 1} ' + M("invoke", closer=True)
        bot = self.bot([_text_resp(tail), _text_resp(tail),
                        _text_resp(tail)])
        reply = await bot.prompt("go")
        self.assertEqual(len(bot._client.calls), 3)
        self.assertIn("[removed malformed tool markup]", reply)
        self.assertNotIn("DSML", reply)

    async def test_recovery_opt_out_keeps_verbatim(self):
        bot = self.bot([_text_resp(read_block())], dsml_recovery=False)
        reply = await bot.prompt("go")
        self.assertIn("DSML", reply)
        self.assertEqual([f["kind"] for f in bot.transcript
                          if f["kind"] == "dsml"], [])

    async def test_plain_chat_answer_sanitized(self):
        bot = self.bot([_text_resp("text " + M("invoke", closer=True))],
                       enable_tools=False)
        reply = await bot.prompt("hi")
        self.assertNotIn("DSML", reply)
        self.assertIn("[removed malformed tool markup]", reply)


if __name__ == "__main__":
    unittest.main()
29.2ms