Skip to content
Repos

blackcat

Python 11.3 KB319 lines Opens in site

Directory Raw Download

"""Tests for concurrent multi-file downloads (stdlib, offline).

The engine runs against fake stealth/direct byte sources; the bot
method runs against an echo bot with an injected fake fetcher. No
Chromium, no network.
"""
# retoor <retoor@molodetz.nl>
import os
import tempfile
import unittest

from blackcat.agent import BlackCatBot
from blackcat.client import DeepSeekError
from blackcat.fstate import FileState
from blackcat.http import StealthFetcher
from blackcat.tools import DownloadTools, FileTools


class _FakeResp:
    def __init__(self, status=200, data=b"", headers=None):
        self.status = status
        self._data = data
        self.headers = headers or {}

    async def read(self):
        return self._data


class _FakeStealth:
    def __init__(self, handler):
        self.handler = handler
        self.calls = []

    async def get(self, url, timeout=None):
        self.calls.append(url)
        return self.handler(url)

    async def shutdown(self):
        pass


def _resp(data=b"bytes", headers=None, status=200):
    def handle(url):
        return _FakeResp(status=status, data=data, headers=headers)
    return handle


def _fail(error):
    def handle(url):
        raise error
    return handle


def _engine(tmp, stealth=None, direct=None, sandbox=True):
    state = FileState(tmp, sandbox=sandbox)
    return DownloadTools(state, stealth=stealth, direct=direct)


class TestFilenames(unittest.TestCase):
    def test_disposition_wins(self):
        name = DownloadTools.filename_for(
            "https://a.org/get?id=1",
            {"Content-Disposition": 'attachment; filename="r.pdf"'},
            0)
        self.assertEqual(name, "r.pdf")

    def test_disposition_case_insensitive(self):
        name = DownloadTools.filename_for(
            "https://a.org/x", {"content-disposition": "a; filename=b"},
            0)
        self.assertEqual(name, "b")

    def test_utf8_prefix_stripped(self):
        name = DownloadTools.filename_for(
            "https://a.org/x",
            {"content-disposition":
             "attachment; filename*=utf-8''r%C3%A9.pdf"}, 0)
        self.assertEqual(name, "r%C3%A9.pdf")

    def test_url_basename(self):
        name = DownloadTools.filename_for(
            "https://a.org/dir/a.bin?v=2", None, 0)
        self.assertEqual(name, "a.bin")

    def test_traversal_flattened(self):
        name = DownloadTools.filename_for("https://a.org/x", None, 0)
        self.assertNotIn("/", name)
        evil = DownloadTools.filename_for(
            "https://a.org/x",
            {"content-disposition":
             'attachment; filename="../../etc/passwd"'}, 0)
        self.assertEqual(evil, "passwd")

    def test_fallback_and_cap(self):
        self.assertEqual(
            DownloadTools.filename_for("https://a.org/", None, 3),
            "file-3")
        long_name = "n" * 300 + ".bin"
        name = DownloadTools.filename_for(
            "https://a.org/" + long_name, None, 0)
        self.assertLessEqual(len(name), 128)


class TestEngine(unittest.IsolatedAsyncioTestCase):
    def setUp(self):
        self.tmp = tempfile.TemporaryDirectory()
        self.addCleanup(self.tmp.cleanup)

    async def test_stealthii_first(self):
        stealth = _FakeStealth(_resp(b"one"))
        engine = _engine(self.tmp.name,
                         stealth=StealthFetcher(stealth=stealth))
        out = await engine.run(["https://a.org/one.bin"])
        self.assertTrue(out["ok"])
        self.assertEqual(out["code"], "ok")
        self.assertEqual(out["saved"], 1)
        self.assertEqual(out["bytes"], 3)
        row = out["files"][0]
        self.assertEqual(row["method"], "stealthii")
        self.assertEqual(row["path"], "one.bin")
        with open(os.path.join(self.tmp.name, "one.bin"), "rb") as fh:
            self.assertEqual(fh.read(), b"one")

    async def test_direct_fallback_when_stealthii_down(self):
        stealth = _FakeStealth(_fail(DeepSeekError("down")))

        async def direct(url, timeout):
            return {"content": b"two", "headers": {}}

        engine = _engine(self.tmp.name,
                         stealth=StealthFetcher(stealth=stealth),
                         direct=direct)
        out = await engine.run(["https://a.org/2.bin"])
        self.assertTrue(out["ok"])
        self.assertEqual(out["files"][0]["method"], "direct")

    async def test_partial_and_failed_codes(self):
        stealth = _FakeStealth(_fail(DeepSeekError("down")))

        async def direct(url, timeout):
            if url.endswith("good.bin"):
                return {"content": b"g", "headers": {}}
            raise DeepSeekError("unreachable")

        engine = _engine(self.tmp.name,
                         stealth=StealthFetcher(stealth=stealth),
                         direct=direct)
        out = await engine.run(["https://a.org/good.bin",
                                "https://a.org/bad.bin"])
        self.assertFalse(out["ok"])
        self.assertEqual(out["code"], "partial")
        self.assertEqual((out["saved"], out["failed"]), (1, 1))
        out = await engine.run(["https://a.org/bad.bin"])
        self.assertFalse(out["ok"])
        self.assertEqual(out["code"], "failed")

    async def test_bad_url_rows(self):
        engine = _engine(self.tmp.name,
                         stealth=StealthFetcher(
                             stealth=_FakeStealth(_resp())))
        out = await engine.run(["ftp://a.org/x"])
        self.assertFalse(out["ok"])
        self.assertEqual(out["files"][0]["code"], "bad_args")

    async def test_empty_and_oversize_rejected(self):
        engine = _engine(self.tmp.name)
        self.assertEqual((await engine.run([]))["code"], "bad_args")
        self.assertEqual(
            (await engine.run("https://a.org/x"))["code"],
            "bad_args")
        many = ["https://a.org/%d" % i for i in range(21)]
        self.assertEqual(
            (await engine.run(many))["code"], "bad_args")
        self.assertEqual(
            (await engine.run(["https://a.org/x"],
                              concurrency="lots"))["code"],
            "bad_args")

    async def test_file_too_large(self):
        engine = _engine(self.tmp.name,
                         stealth=StealthFetcher(
                             stealth=_FakeStealth(_resp(b"12345"))))
        out = await engine.run(["https://a.org/big.bin"],
                               max_bytes=3)
        self.assertFalse(out["ok"])
        self.assertEqual(out["files"][0]["code"], "file_too_large")
        self.assertFalse(os.path.exists(
            os.path.join(self.tmp.name, "big.bin")))

    async def test_skip_then_overwrite(self):
        engine = _engine(self.tmp.name,
                         stealth=StealthFetcher(
                             stealth=_FakeStealth(_resp(b"v1"))))
        first = await engine.run(["https://a.org/a.bin"])
        self.assertEqual(first["files"][0]["code"], "ok")
        second = await engine.run(["https://a.org/a.bin"])
        self.assertTrue(second["ok"])
        self.assertEqual(second["skipped"], 1)
        self.assertEqual(second["files"][0]["code"], "skipped")
        engine2 = _engine(self.tmp.name,
                          stealth=StealthFetcher(
                              stealth=_FakeStealth(_resp(b"v2"))))
        third = await engine2.run(["https://a.org/a.bin"],
                                  overwrite=True)
        self.assertEqual(third["saved"], 1)
        with open(os.path.join(self.tmp.name, "a.bin"), "rb") as fh:
            self.assertEqual(fh.read(), b"v2")

    async def test_dest_subdir_created(self):
        engine = _engine(self.tmp.name,
                         stealth=StealthFetcher(
                             stealth=_FakeStealth(_resp(b"d"))))
        out = await engine.run(["https://a.org/f.bin"], dest="sub/d")
        self.assertTrue(out["ok"])
        self.assertEqual(out["dest"], os.path.join("sub", "d"))
        self.assertTrue(os.path.exists(
            os.path.join(self.tmp.name, "sub", "d", "f.bin")))

    async def test_escape_rejected(self):
        engine = _engine(self.tmp.name,
                         stealth=StealthFetcher(
                             stealth=_FakeStealth(_resp())))
        out = await engine.run(["https://a.org/f.bin"], dest="/etc")
        self.assertEqual(out["code"], "security")

    async def test_read_only_refused(self):
        state = FileState(self.tmp.name, write=False)
        engine = DownloadTools(state, stealth=StealthFetcher(
            stealth=_FakeStealth(_resp())))
        out = await engine.run(["https://a.org/f.bin"])
        self.assertEqual(out["code"], "security")

    async def test_concurrent_fetches(self):
        import asyncio
        in_flight = 0
        peak = 0

        async def direct(url, timeout):
            nonlocal in_flight, peak
            in_flight += 1
            peak = max(peak, in_flight)
            await asyncio.sleep(0.02)
            in_flight -= 1
            return {"content": b"x", "headers": {}}

        engine = _engine(
            self.tmp.name,
            stealth=StealthFetcher(
                stealth=_FakeStealth(_fail(DeepSeekError("down")))),
            direct=direct)
        urls = ["https://a.org/%d.bin" % i for i in range(6)]
        out = await engine.run(urls, concurrency=3)
        self.assertTrue(out["ok"])
        self.assertEqual(peak, 3)

    async def test_on_file_called_per_row(self):
        seen = []
        stealth = _FakeStealth(_resp(b"z"))
        engine = _engine(self.tmp.name,
                         stealth=StealthFetcher(stealth=stealth))
        out = await engine.run(
            ["https://a.org/1.bin", "https://a.org/2.bin"],
            on_file=lambda url, row: seen.append((url, row["ok"])))
        self.assertTrue(out["ok"])
        self.assertEqual(len(seen), 2)
        self.assertTrue(all(ok for _, ok in seen))


class TestBotDownload(unittest.IsolatedAsyncioTestCase):
    def setUp(self):
        self.tmp = tempfile.TemporaryDirectory()
        self.addCleanup(self.tmp.cleanup)

    def bot(self):
        bot = BlackCatBot(provider="echo", system="t",
                          tools=FileTools(FileState(self.tmp.name)))
        bot._stealth = StealthFetcher(
            stealth=_FakeStealth(_resp(b"data")))
        self.addAsyncCleanup(bot.aclose)
        return bot

    async def test_download_saves(self):
        bot = self.bot()
        out = await bot.download(["https://a.org/f.bin"],
                                 dest="dl")
        self.assertTrue(out["ok"])
        self.assertEqual(out["saved"], 1)
        kinds = [f["kind"] for f in bot.transcript]
        self.assertIn("download", kinds)
        self.assertTrue(os.path.exists(
            os.path.join(self.tmp.name, "dl", "f.bin")))

    async def test_run_tool_dispatch(self):
        bot = self.bot()
        out = await bot.run_tool(
            "download", urls=["https://a.org/f.bin"])
        self.assertTrue(out["ok"])

    async def test_empty_urls_rejected(self):
        bot = self.bot()
        out = await bot.download([])
        self.assertEqual(out["code"], "bad_args")

    async def test_events_fire(self):
        bot = self.bot()
        kinds = []
        orig = bot._download_event

        async def spy(kind, **payload):
            kinds.append(kind)
            await orig(kind, **payload)

        bot._download_event = spy
        await bot.download(["https://a.org/f.bin"])
        self.assertEqual(kinds, ["start", "file", "done"])


if __name__ == "__main__":
    unittest.main()
68.8ms