"""Tests for concurrent multi-file downloads (stdlib, offline). The engine runs against fake stealth/direct byte sources; the bot method runs against an echo bot with an injected fake fetcher. No Chromium, no network. """ # retoor import os import tempfile import unittest from blackcat.agent import BlackCatBot from blackcat.client import DeepSeekError from blackcat.fstate import FileState from blackcat.http import StealthFetcher from blackcat.tools import DownloadTools, FileTools class _FakeResp: def __init__(self, status=200, data=b"", headers=None): self.status = status self._data = data self.headers = headers or {} async def read(self): return self._data class _FakeStealth: def __init__(self, handler): self.handler = handler self.calls = [] async def get(self, url, timeout=None): self.calls.append(url) return self.handler(url) async def shutdown(self): pass def _resp(data=b"bytes", headers=None, status=200): def handle(url): return _FakeResp(status=status, data=data, headers=headers) return handle def _fail(error): def handle(url): raise error return handle def _engine(tmp, stealth=None, direct=None, sandbox=True): state = FileState(tmp, sandbox=sandbox) return DownloadTools(state, stealth=stealth, direct=direct) class TestFilenames(unittest.TestCase): def test_disposition_wins(self): name = DownloadTools.filename_for( "https://a.org/get?id=1", {"Content-Disposition": 'attachment; filename="r.pdf"'}, 0) self.assertEqual(name, "r.pdf") def test_disposition_case_insensitive(self): name = DownloadTools.filename_for( "https://a.org/x", {"content-disposition": "a; filename=b"}, 0) self.assertEqual(name, "b") def test_utf8_prefix_stripped(self): name = DownloadTools.filename_for( "https://a.org/x", {"content-disposition": "attachment; filename*=utf-8''r%C3%A9.pdf"}, 0) self.assertEqual(name, "r%C3%A9.pdf") def test_url_basename(self): name = DownloadTools.filename_for( "https://a.org/dir/a.bin?v=2", None, 0) self.assertEqual(name, "a.bin") def test_traversal_flattened(self): name = DownloadTools.filename_for("https://a.org/x", None, 0) self.assertNotIn("/", name) evil = DownloadTools.filename_for( "https://a.org/x", {"content-disposition": 'attachment; filename="../../etc/passwd"'}, 0) self.assertEqual(evil, "passwd") def test_fallback_and_cap(self): self.assertEqual( DownloadTools.filename_for("https://a.org/", None, 3), "file-3") long_name = "n" * 300 + ".bin" name = DownloadTools.filename_for( "https://a.org/" + long_name, None, 0) self.assertLessEqual(len(name), 128) class TestEngine(unittest.IsolatedAsyncioTestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) async def test_stealthii_first(self): stealth = _FakeStealth(_resp(b"one")) engine = _engine(self.tmp.name, stealth=StealthFetcher(stealth=stealth)) out = await engine.run(["https://a.org/one.bin"]) self.assertTrue(out["ok"]) self.assertEqual(out["code"], "ok") self.assertEqual(out["saved"], 1) self.assertEqual(out["bytes"], 3) row = out["files"][0] self.assertEqual(row["method"], "stealthii") self.assertEqual(row["path"], "one.bin") with open(os.path.join(self.tmp.name, "one.bin"), "rb") as fh: self.assertEqual(fh.read(), b"one") async def test_direct_fallback_when_stealthii_down(self): stealth = _FakeStealth(_fail(DeepSeekError("down"))) async def direct(url, timeout): return {"content": b"two", "headers": {}} engine = _engine(self.tmp.name, stealth=StealthFetcher(stealth=stealth), direct=direct) out = await engine.run(["https://a.org/2.bin"]) self.assertTrue(out["ok"]) self.assertEqual(out["files"][0]["method"], "direct") async def test_partial_and_failed_codes(self): stealth = _FakeStealth(_fail(DeepSeekError("down"))) async def direct(url, timeout): if url.endswith("good.bin"): return {"content": b"g", "headers": {}} raise DeepSeekError("unreachable") engine = _engine(self.tmp.name, stealth=StealthFetcher(stealth=stealth), direct=direct) out = await engine.run(["https://a.org/good.bin", "https://a.org/bad.bin"]) self.assertFalse(out["ok"]) self.assertEqual(out["code"], "partial") self.assertEqual((out["saved"], out["failed"]), (1, 1)) out = await engine.run(["https://a.org/bad.bin"]) self.assertFalse(out["ok"]) self.assertEqual(out["code"], "failed") async def test_bad_url_rows(self): engine = _engine(self.tmp.name, stealth=StealthFetcher( stealth=_FakeStealth(_resp()))) out = await engine.run(["ftp://a.org/x"]) self.assertFalse(out["ok"]) self.assertEqual(out["files"][0]["code"], "bad_args") async def test_empty_and_oversize_rejected(self): engine = _engine(self.tmp.name) self.assertEqual((await engine.run([]))["code"], "bad_args") self.assertEqual( (await engine.run("https://a.org/x"))["code"], "bad_args") many = ["https://a.org/%d" % i for i in range(21)] self.assertEqual( (await engine.run(many))["code"], "bad_args") self.assertEqual( (await engine.run(["https://a.org/x"], concurrency="lots"))["code"], "bad_args") async def test_file_too_large(self): engine = _engine(self.tmp.name, stealth=StealthFetcher( stealth=_FakeStealth(_resp(b"12345")))) out = await engine.run(["https://a.org/big.bin"], max_bytes=3) self.assertFalse(out["ok"]) self.assertEqual(out["files"][0]["code"], "file_too_large") self.assertFalse(os.path.exists( os.path.join(self.tmp.name, "big.bin"))) async def test_skip_then_overwrite(self): engine = _engine(self.tmp.name, stealth=StealthFetcher( stealth=_FakeStealth(_resp(b"v1")))) first = await engine.run(["https://a.org/a.bin"]) self.assertEqual(first["files"][0]["code"], "ok") second = await engine.run(["https://a.org/a.bin"]) self.assertTrue(second["ok"]) self.assertEqual(second["skipped"], 1) self.assertEqual(second["files"][0]["code"], "skipped") engine2 = _engine(self.tmp.name, stealth=StealthFetcher( stealth=_FakeStealth(_resp(b"v2")))) third = await engine2.run(["https://a.org/a.bin"], overwrite=True) self.assertEqual(third["saved"], 1) with open(os.path.join(self.tmp.name, "a.bin"), "rb") as fh: self.assertEqual(fh.read(), b"v2") async def test_dest_subdir_created(self): engine = _engine(self.tmp.name, stealth=StealthFetcher( stealth=_FakeStealth(_resp(b"d")))) out = await engine.run(["https://a.org/f.bin"], dest="sub/d") self.assertTrue(out["ok"]) self.assertEqual(out["dest"], os.path.join("sub", "d")) self.assertTrue(os.path.exists( os.path.join(self.tmp.name, "sub", "d", "f.bin"))) async def test_escape_rejected(self): engine = _engine(self.tmp.name, stealth=StealthFetcher( stealth=_FakeStealth(_resp()))) out = await engine.run(["https://a.org/f.bin"], dest="/etc") self.assertEqual(out["code"], "security") async def test_read_only_refused(self): state = FileState(self.tmp.name, write=False) engine = DownloadTools(state, stealth=StealthFetcher( stealth=_FakeStealth(_resp()))) out = await engine.run(["https://a.org/f.bin"]) self.assertEqual(out["code"], "security") async def test_concurrent_fetches(self): import asyncio in_flight = 0 peak = 0 async def direct(url, timeout): nonlocal in_flight, peak in_flight += 1 peak = max(peak, in_flight) await asyncio.sleep(0.02) in_flight -= 1 return {"content": b"x", "headers": {}} engine = _engine( self.tmp.name, stealth=StealthFetcher( stealth=_FakeStealth(_fail(DeepSeekError("down")))), direct=direct) urls = ["https://a.org/%d.bin" % i for i in range(6)] out = await engine.run(urls, concurrency=3) self.assertTrue(out["ok"]) self.assertEqual(peak, 3) async def test_on_file_called_per_row(self): seen = [] stealth = _FakeStealth(_resp(b"z")) engine = _engine(self.tmp.name, stealth=StealthFetcher(stealth=stealth)) out = await engine.run( ["https://a.org/1.bin", "https://a.org/2.bin"], on_file=lambda url, row: seen.append((url, row["ok"]))) self.assertTrue(out["ok"]) self.assertEqual(len(seen), 2) self.assertTrue(all(ok for _, ok in seen)) class TestBotDownload(unittest.IsolatedAsyncioTestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) def bot(self): bot = BlackCatBot(provider="echo", system="t", tools=FileTools(FileState(self.tmp.name))) bot._stealth = StealthFetcher( stealth=_FakeStealth(_resp(b"data"))) self.addAsyncCleanup(bot.aclose) return bot async def test_download_saves(self): bot = self.bot() out = await bot.download(["https://a.org/f.bin"], dest="dl") self.assertTrue(out["ok"]) self.assertEqual(out["saved"], 1) kinds = [f["kind"] for f in bot.transcript] self.assertIn("download", kinds) self.assertTrue(os.path.exists( os.path.join(self.tmp.name, "dl", "f.bin"))) async def test_run_tool_dispatch(self): bot = self.bot() out = await bot.run_tool( "download", urls=["https://a.org/f.bin"]) self.assertTrue(out["ok"]) async def test_empty_urls_rejected(self): bot = self.bot() out = await bot.download([]) self.assertEqual(out["code"], "bad_args") async def test_events_fire(self): bot = self.bot() kinds = [] orig = bot._download_event async def spy(kind, **payload): kinds.append(kind) await orig(kind, **payload) bot._download_event = spy await bot.download(["https://a.org/f.bin"]) self.assertEqual(kinds, ["start", "file", "done"]) if __name__ == "__main__": unittest.main()