"""Tests for concurrent multi-file downloads (stdlib, offline).
The engine runs against fake stealth/direct byte sources; the bot
method runs against an echo bot with an injected fake fetcher. No
Chromium, no network.
"""
# retoor <retoor@molodetz.nl>
import os
import tempfile
import unittest
from blackcat.agent import BlackCatBot
from blackcat.client import DeepSeekError
from blackcat.fstate import FileState
from blackcat.http import StealthFetcher
from blackcat.tools import DownloadTools, FileTools
class _FakeResp:
def __init__(self, status=200, data=b"", headers=None):
self.status = status
self._data = data
self.headers = headers or {}
async def read(self):
return self._data
class _FakeStealth:
def __init__(self, handler):
self.handler = handler
self.calls = []
async def get(self, url, timeout=None):
self.calls.append(url)
return self.handler(url)
async def shutdown(self):
pass
def _resp(data=b"bytes", headers=None, status=200):
def handle(url):
return _FakeResp(status=status, data=data, headers=headers)
return handle
def _fail(error):
def handle(url):
raise error
return handle
def _engine(tmp, stealth=None, direct=None, sandbox=True):
state = FileState(tmp, sandbox=sandbox)
return DownloadTools(state, stealth=stealth, direct=direct)
class TestFilenames(unittest.TestCase):
def test_disposition_wins(self):
name = DownloadTools.filename_for(
"https://a.org/get?id=1",
{"Content-Disposition": 'attachment; filename="r.pdf"'},
0)
self.assertEqual(name, "r.pdf")
def test_disposition_case_insensitive(self):
name = DownloadTools.filename_for(
"https://a.org/x", {"content-disposition": "a; filename=b"},
0)
self.assertEqual(name, "b")
def test_utf8_prefix_stripped(self):
name = DownloadTools.filename_for(
"https://a.org/x",
{"content-disposition":
"attachment; filename*=utf-8''r%C3%A9.pdf"}, 0)
self.assertEqual(name, "r%C3%A9.pdf")
def test_url_basename(self):
name = DownloadTools.filename_for(
"https://a.org/dir/a.bin?v=2", None, 0)
self.assertEqual(name, "a.bin")
def test_traversal_flattened(self):
name = DownloadTools.filename_for("https://a.org/x", None, 0)
self.assertNotIn("/", name)
evil = DownloadTools.filename_for(
"https://a.org/x",
{"content-disposition":
'attachment; filename="../../etc/passwd"'}, 0)
self.assertEqual(evil, "passwd")
def test_fallback_and_cap(self):
self.assertEqual(
DownloadTools.filename_for("https://a.org/", None, 3),
"file-3")
long_name = "n" * 300 + ".bin"
name = DownloadTools.filename_for(
"https://a.org/" + long_name, None, 0)
self.assertLessEqual(len(name), 128)
class TestEngine(unittest.IsolatedAsyncioTestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
async def test_stealthii_first(self):
stealth = _FakeStealth(_resp(b"one"))
engine = _engine(self.tmp.name,
stealth=StealthFetcher(stealth=stealth))
out = await engine.run(["https://a.org/one.bin"])
self.assertTrue(out["ok"])
self.assertEqual(out["code"], "ok")
self.assertEqual(out["saved"], 1)
self.assertEqual(out["bytes"], 3)
row = out["files"][0]
self.assertEqual(row["method"], "stealthii")
self.assertEqual(row["path"], "one.bin")
with open(os.path.join(self.tmp.name, "one.bin"), "rb") as fh:
self.assertEqual(fh.read(), b"one")
async def test_direct_fallback_when_stealthii_down(self):
stealth = _FakeStealth(_fail(DeepSeekError("down")))
async def direct(url, timeout):
return {"content": b"two", "headers": {}}
engine = _engine(self.tmp.name,
stealth=StealthFetcher(stealth=stealth),
direct=direct)
out = await engine.run(["https://a.org/2.bin"])
self.assertTrue(out["ok"])
self.assertEqual(out["files"][0]["method"], "direct")
async def test_partial_and_failed_codes(self):
stealth = _FakeStealth(_fail(DeepSeekError("down")))
async def direct(url, timeout):
if url.endswith("good.bin"):
return {"content": b"g", "headers": {}}
raise DeepSeekError("unreachable")
engine = _engine(self.tmp.name,
stealth=StealthFetcher(stealth=stealth),
direct=direct)
out = await engine.run(["https://a.org/good.bin",
"https://a.org/bad.bin"])
self.assertFalse(out["ok"])
self.assertEqual(out["code"], "partial")
self.assertEqual((out["saved"], out["failed"]), (1, 1))
out = await engine.run(["https://a.org/bad.bin"])
self.assertFalse(out["ok"])
self.assertEqual(out["code"], "failed")
async def test_bad_url_rows(self):
engine = _engine(self.tmp.name,
stealth=StealthFetcher(
stealth=_FakeStealth(_resp())))
out = await engine.run(["ftp://a.org/x"])
self.assertFalse(out["ok"])
self.assertEqual(out["files"][0]["code"], "bad_args")
async def test_empty_and_oversize_rejected(self):
engine = _engine(self.tmp.name)
self.assertEqual((await engine.run([]))["code"], "bad_args")
self.assertEqual(
(await engine.run("https://a.org/x"))["code"],
"bad_args")
many = ["https://a.org/%d" % i for i in range(21)]
self.assertEqual(
(await engine.run(many))["code"], "bad_args")
self.assertEqual(
(await engine.run(["https://a.org/x"],
concurrency="lots"))["code"],
"bad_args")
async def test_file_too_large(self):
engine = _engine(self.tmp.name,
stealth=StealthFetcher(
stealth=_FakeStealth(_resp(b"12345"))))
out = await engine.run(["https://a.org/big.bin"],
max_bytes=3)
self.assertFalse(out["ok"])
self.assertEqual(out["files"][0]["code"], "file_too_large")
self.assertFalse(os.path.exists(
os.path.join(self.tmp.name, "big.bin")))
async def test_skip_then_overwrite(self):
engine = _engine(self.tmp.name,
stealth=StealthFetcher(
stealth=_FakeStealth(_resp(b"v1"))))
first = await engine.run(["https://a.org/a.bin"])
self.assertEqual(first["files"][0]["code"], "ok")
second = await engine.run(["https://a.org/a.bin"])
self.assertTrue(second["ok"])
self.assertEqual(second["skipped"], 1)
self.assertEqual(second["files"][0]["code"], "skipped")
engine2 = _engine(self.tmp.name,
stealth=StealthFetcher(
stealth=_FakeStealth(_resp(b"v2"))))
third = await engine2.run(["https://a.org/a.bin"],
overwrite=True)
self.assertEqual(third["saved"], 1)
with open(os.path.join(self.tmp.name, "a.bin"), "rb") as fh:
self.assertEqual(fh.read(), b"v2")
async def test_dest_subdir_created(self):
engine = _engine(self.tmp.name,
stealth=StealthFetcher(
stealth=_FakeStealth(_resp(b"d"))))
out = await engine.run(["https://a.org/f.bin"], dest="sub/d")
self.assertTrue(out["ok"])
self.assertEqual(out["dest"], os.path.join("sub", "d"))
self.assertTrue(os.path.exists(
os.path.join(self.tmp.name, "sub", "d", "f.bin")))
async def test_escape_rejected(self):
engine = _engine(self.tmp.name,
stealth=StealthFetcher(
stealth=_FakeStealth(_resp())))
out = await engine.run(["https://a.org/f.bin"], dest="/etc")
self.assertEqual(out["code"], "security")
async def test_read_only_refused(self):
state = FileState(self.tmp.name, write=False)
engine = DownloadTools(state, stealth=StealthFetcher(
stealth=_FakeStealth(_resp())))
out = await engine.run(["https://a.org/f.bin"])
self.assertEqual(out["code"], "security")
async def test_concurrent_fetches(self):
import asyncio
in_flight = 0
peak = 0
async def direct(url, timeout):
nonlocal in_flight, peak
in_flight += 1
peak = max(peak, in_flight)
await asyncio.sleep(0.02)
in_flight -= 1
return {"content": b"x", "headers": {}}
engine = _engine(
self.tmp.name,
stealth=StealthFetcher(
stealth=_FakeStealth(_fail(DeepSeekError("down")))),
direct=direct)
urls = ["https://a.org/%d.bin" % i for i in range(6)]
out = await engine.run(urls, concurrency=3)
self.assertTrue(out["ok"])
self.assertEqual(peak, 3)
async def test_on_file_called_per_row(self):
seen = []
stealth = _FakeStealth(_resp(b"z"))
engine = _engine(self.tmp.name,
stealth=StealthFetcher(stealth=stealth))
out = await engine.run(
["https://a.org/1.bin", "https://a.org/2.bin"],
on_file=lambda url, row: seen.append((url, row["ok"])))
self.assertTrue(out["ok"])
self.assertEqual(len(seen), 2)
self.assertTrue(all(ok for _, ok in seen))
class TestBotDownload(unittest.IsolatedAsyncioTestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
def bot(self):
bot = BlackCatBot(provider="echo", system="t",
tools=FileTools(FileState(self.tmp.name)))
bot._stealth = StealthFetcher(
stealth=_FakeStealth(_resp(b"data")))
self.addAsyncCleanup(bot.aclose)
return bot
async def test_download_saves(self):
bot = self.bot()
out = await bot.download(["https://a.org/f.bin"],
dest="dl")
self.assertTrue(out["ok"])
self.assertEqual(out["saved"], 1)
kinds = [f["kind"] for f in bot.transcript]
self.assertIn("download", kinds)
self.assertTrue(os.path.exists(
os.path.join(self.tmp.name, "dl", "f.bin")))
async def test_run_tool_dispatch(self):
bot = self.bot()
out = await bot.run_tool(
"download", urls=["https://a.org/f.bin"])
self.assertTrue(out["ok"])
async def test_empty_urls_rejected(self):
bot = self.bot()
out = await bot.download([])
self.assertEqual(out["code"], "bad_args")
async def test_events_fire(self):
bot = self.bot()
kinds = []
orig = bot._download_event
async def spy(kind, **payload):
kinds.append(kind)
await orig(kind, **payload)
bot._download_event = spy
await bot.download(["https://a.org/f.bin"])
self.assertEqual(kinds, ["start", "file", "done"])
if __name__ == "__main__":
unittest.main()