# retoor <retoor@molodetz.nl>
import argparse
import json
from html.parser import HTMLParser
from urllib.parse import urljoin

MAX_IMAGES = 10
MIN_TEXT_WORDS = 50
SEMANTIC_SELECTORS = ("article", "main", "[role=main]", ".post-body", ".article-body")


class _ImageScan(HTMLParser):
    def __init__(self):
        super().__init__()
        self.found = []

    def handle_starttag(self, tag, attrs):
        if tag != "img":
            return
        values = dict(attrs)
        src = (values.get("src") or "").strip()
        if not src or src.startswith("data:"):
            return
        try:
            tiny = int(values.get("width", "0") or 0) <= 1 and int(values.get("height", "0") or 0) <= 1
        except ValueError:
            tiny = False
        if tiny and (values.get("width") or values.get("height")):
            return
        self.found.append(src)


def page_images(html, base_url="", cap=MAX_IMAGES):
    scan = _ImageScan()
    scan.feed(html)
    seen, out = set(), []
    for src in scan.found:
        absolute = urljoin(base_url, src)
        if absolute in seen:
            continue
        seen.add(absolute)
        out.append(absolute)
        if len(out) >= cap:
            break
    return out


def _trafilatura_text(html, url):
    import trafilatura

    text = trafilatura.extract(html, include_comments=False, url=url)
    if text and len(text.split()) >= MIN_TEXT_WORDS:
        return text.strip()
    return ""


def _semantic_text(html):
    from bs4 import BeautifulSoup

    soup = BeautifulSoup(html, "lxml")
    for selector in SEMANTIC_SELECTORS:
        node = soup.select_one(selector)
        if node is None:
            continue
        text = node.get_text(" ", strip=True)
        if len(text.split()) >= MIN_TEXT_WORDS:
            return text
    paragraphs = [part.get_text(" ", strip=True) for part in soup.find_all("p")]
    text = " ".join(part for part in paragraphs if part)
    if len(text.split()) >= MIN_TEXT_WORDS:
        return text
    return ""


def _metadata_text(html):
    from bs4 import BeautifulSoup

    soup = BeautifulSoup(html, "lxml")
    parts = []
    if soup.title and soup.title.string:
        parts.append(soup.title.string.strip())
    for node in soup.find_all("meta"):
        name = (node.get("name") or node.get("property") or "").lower()
        if name in ("description", "og:description", "twitter:description") and node.get("content"):
            parts.append(node["content"].strip())
    return " ".join(dict.fromkeys(part for part in parts if part))


def extract(html, url=None, backend="auto"):
    images = page_images(html, base_url=url or "")
    if backend in ("auto", "trafilatura"):
        try:
            text = _trafilatura_text(html, url)
        except ImportError:
            text = ""
        if text:
            return {"text": text, "backend": "trafilatura", "images": images}
        if backend == "trafilatura":
            return {"text": "", "backend": "trafilatura", "images": images}
    if backend in ("auto", "bs4"):
        try:
            text = _semantic_text(html)
        except ImportError:
            text = ""
        if text:
            return {"text": text, "backend": "bs4", "images": images}
        if backend == "bs4":
            return {"text": "", "backend": "bs4", "images": images}
    try:
        text = _metadata_text(html)
    except ImportError:
        text = ""
    return {"text": text, "backend": "metadata", "images": images}


def batch(directory, manifest_path=""):
    from pathlib import Path

    urls = {}
    if manifest_path:
        with open(manifest_path, encoding="utf-8") as handle:
            payload = json.load(handle)
        rows = payload.get("manifest", payload) if isinstance(payload, dict) else payload
        for row in rows:
            if row.get("ok") and row.get("path"):
                urls[Path(row["path"]).name] = row.get("url", "")
    out = []
    for path in sorted(Path(directory).glob("*.html")):
        found = extract(path.read_text(encoding="utf-8"), url=urls.get(path.name) or None, backend="auto")
        found["file"] = path.name
        found["url"] = urls.get(path.name, "")
        out.append(found)
    return out


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Extract text and images from saved page HTML.")
    parser.add_argument("page", nargs="?", default="", help="HTML file to extract")
    parser.add_argument("--url", default="", help="Page URL for resolving relative image URLs")
    parser.add_argument("--backend", choices=("auto", "trafilatura", "bs4", "metadata"), default="auto")
    parser.add_argument("--batch", default="", help="Directory of HTML files to extract in one call")
    parser.add_argument("--manifest", default="", help="fetch.py manifest JSON joining files back to URLs")
    args = parser.parse_args()
    if args.batch:
        print(json.dumps(batch(args.batch, args.manifest)))
    else:
        with open(args.page, encoding="utf-8") as handle:
            print(json.dumps(extract(handle.read(), url=args.url or None, backend=args.backend)))