#!/usr/bin/env python3
"""
rip.py — deterministic Phase-1 pre-pass for the HYBRID clone pipeline.

Does the mechanical, error-prone work that a local LLM does unreliably:
  wget mirror → index.html → organize into assets/ → rewrite EVERY reference →
  download external CDN assets locally → strip KNOWN tracking scripts (safe regex).

After this runs, OpenHands/Qwen only does judgment work (unusual trackers, brand/
CTA/text/media changes, QA) on an already-correct, fully-localized page.

This is the battle-tested logic from the original bot.py Phase-1, lifted verbatim
into a reusable function. Raises RuntimeError on fatal failures (wget / no HTML).
"""

import re
import shutil
import subprocess
import urllib.request
from pathlib import Path
from urllib.parse import urlparse, unquote

try:
    from playwright.sync_api import sync_playwright
except ImportError:
    sync_playwright = None

# ── Known tracking (deterministic, never breaks JS) ───────────────────────────

_TRACKING_DOMAINS = [
    r'googletagmanager\.com', r'google-analytics\.com', r'googletagservices\.com',
    r'connect\.facebook\.net', r'sc-static\.net', r'static\.hotjar\.com',
    r'cdn\.segment\.com', r'clarity\.ms', r'cdn\.amplitude\.com',
    r'cdn\.heapanalytics\.com', r'fullstory\.com', r'a\.klaviyo\.com',
    r'static\.klaviyo\.com', r'config-security\.com', r'cdn-cgi/scripts',
    r'snap\.licdn\.com', r'snapchat\.com', r'archive-digger\.com',
    r'ds0wlyksfn0sb\.cloudfront\.net', r'elevateab\.app',
]

_TRACKING_INLINE_PATTERNS = [
    r'function\(w,\s*d,\s*s,\s*l,\s*i\)\s*\{',       # GTM snippet
    r'window\.dataLayer\s*=\s*window\.dataLayer',     # dataLayer init
    r'function\s+gtag\s*\(\)',                         # gtag function
    r'gtag\s*\(\s*[\'"]config[\'"]',                   # gtag config call
    r'gtag\s*\(\s*[\'"]js[\'"]',                       # gtag js call
    r'fbq\s*\(',                                       # Meta Pixel
    r'snaptr\s*\(',                                    # Snap Pixel
    r'function\(h,\s*o,\s*t,\s*j,\s*a,\s*r\)',        # Hotjar
    r'function\(c,\s*l,\s*a,\s*r,\s*i,\s*t,\s*y\)',   # Clarity
    r'sentSmartNewEvent', r'sendVoluumEvent', r'taboolaId',
]


def _strip_tracking(html_path: Path) -> int:
    """Remove all tracking scripts/pixels from an HTML file. Returns count of removals."""
    content = html_path.read_text(encoding="utf-8", errors="ignore")
    removals = 0

    def _remove_script_tag(match):
        nonlocal removals
        tag = match.group(0)
        for domain in _TRACKING_DOMAINS:
            if re.search(domain, tag, re.IGNORECASE):
                removals += 1
                return ""
        return tag

    content = re.sub(
        r'<script\b[^>]*\bsrc\s*=\s*["\'][^"\']*["\'][^>]*>.*?</script>',
        _remove_script_tag, content, flags=re.DOTALL | re.IGNORECASE,
    )

    def _remove_inline_script(match):
        nonlocal removals
        block = match.group(0)
        if re.search(r'\bsrc\s*=', match.group(1) or ""):
            return block
        for pattern in _TRACKING_INLINE_PATTERNS:
            if re.search(pattern, block, re.IGNORECASE):
                removals += 1
                return ""
        for domain in _TRACKING_DOMAINS:
            if re.search(domain, block, re.IGNORECASE):
                removals += 1
                return ""
        return block

    content = re.sub(
        r'<script(\b[^>]*)>(.*?)</script>',
        _remove_inline_script, content, flags=re.DOTALL | re.IGNORECASE,
    )

    def _remove_noscript(match):
        nonlocal removals
        block = match.group(0)
        for domain in _TRACKING_DOMAINS:
            if re.search(domain, block, re.IGNORECASE):
                removals += 1
                return ""
        if re.search(r'facebook\.com/tr', block, re.IGNORECASE):
            removals += 1
            return ""
        return block

    content = re.sub(
        r'<noscript>.*?</noscript>',
        _remove_noscript, content, flags=re.DOTALL | re.IGNORECASE,
    )

    def _remove_tracking_img(match):
        nonlocal removals
        tag = match.group(0)
        for domain in (r'facebook\.com', r'doubleclick\.net', r'google-analytics\.com',
                       r'googletagmanager\.com', r'google\.com/pagead'):
            if re.search(domain, tag, re.IGNORECASE):
                removals += 1
                return ""
        if re.search(r'width=["\']1["\']', tag) and re.search(r'height=["\']1["\']', tag):
            removals += 1
            return ""
        return tag

    content = re.sub(r'<img\b[^>]*>', _remove_tracking_img, content, flags=re.IGNORECASE)

    content = re.sub(
        r'<!--\s*Start of Shoplift scripts\s*-->.*?<!--\s*End of Shoplift scripts\s*-->',
        '', content, flags=re.DOTALL | re.IGNORECASE,
    )
    content = re.sub(
        r'<!--\s*(?:End )?Google Tag Manager(?:\s*\(noscript\))?\s*-->',
        '', content, flags=re.IGNORECASE,
    )
    content = re.sub(r'\n{3,}', '\n\n', content)

    html_path.write_text(content, encoding="utf-8")
    return removals


_EXT_MAP = {
    "css":    [".css"],
    "js":     [".js"],
    "fonts":  [".woff", ".woff2", ".ttf", ".eot", ".otf"],
    "images": [".jpg", ".jpeg", ".png", ".gif", ".svg", ".webp", ".ico", ".avif"],
    "media":  [".mp4", ".webm", ".ogg", ".mp3"],
}


def _find_external_urls(text):
    """Extract all external asset URLs from HTML or CSS content."""
    urls = set()
    patterns = [
        r'(?:src|href|poster|content|data-src|data-lazy|data-bg)\s*=\s*["\']?(https?://[^"\'>\s]+)',
        r'srcset\s*=\s*["\']([^"\']+)',
        r'url\(\s*["\']?(https?://[^"\')\s]+)',
        r'["\']+(https?://[^"\'>\s]+\.(?:jpg|jpeg|png|gif|svg|webp|ico|avif|mp4|webm|woff2?|ttf|eot|otf|css|js))["\']',
    ]
    for pat in patterns:
        for match in re.finditer(pat, text, re.IGNORECASE):
            val = match.group(1)
            if 'srcset' in pat:
                for part in val.split(','):
                    part = part.strip().split()[0] if part.strip() else ""
                    if part.startswith('http'):
                        urls.add(part)
            else:
                urls.add(val)
    return urls


def _fetch(url, dest):
    """Download url→dest robustly. Try urllib, then fall back to wget (a different TLS stack that
    often succeeds where urllib transiently fails). Returns True only if a non-empty file landed —
    so callers can REPORT misses instead of silently leaving an external CDN reference behind."""
    try:
        req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
        with urllib.request.urlopen(req, timeout=20) as resp:
            data = resp.read()
        if data:
            dest.write_bytes(data)
            return True
    except Exception:
        pass
    try:
        subprocess.run(["wget", "-q", "-O", str(dest), url], timeout=40, check=True)
        if dest.exists() and dest.stat().st_size > 0:
            return True
    except Exception:
        pass
    if dest.exists() and dest.stat().st_size == 0:
        dest.unlink()
    return False


def _rewrite_by_basename(text, basename_map, for_css=False):
    """Rewrite asset references (src/href/poster/data-*/srcset/url()) to their new assets/ path,
    matched by FILENAME. Handles any relative style: bare 'foo.webp', './foo', '../foo', 'dir/foo'."""
    def newpath(p):
        base = p.split('?')[0].split('#')[0].rsplit('/', 1)[-1]
        nr = basename_map.get(base)
        if not nr:
            return None
        return nr.replace('assets/', '../') if for_css else nr

    def attr_repl(m):
        np = newpath(m.group(2))
        return (m.group(1) + np + m.group(3)) if np else m.group(0)

    text = re.sub(r'(\b(?:src|href|poster|data-src|data-lazy|data-bg)\s*=\s*["\'])([^"\']+)(["\'])',
                  attr_repl, text, flags=re.IGNORECASE)
    text = re.sub(r'(url\(\s*["\']?)([^"\')]+)(["\']?\s*\))', attr_repl, text, flags=re.IGNORECASE)

    def srcset_repl(m):
        out = []
        for item in m.group(2).split(','):
            seg = item.strip().split()
            if seg:
                np = newpath(seg[0])
                if np:
                    seg[0] = np
                out.append(' '.join(seg))
        return m.group(1) + ', '.join(out) + m.group(3)

    text = re.sub(r'(\bsrcset\s*=\s*["\'])([^"\']+)(["\'])', srcset_repl, text, flags=re.IGNORECASE)
    return text


def _render_with_playwright(url: str, timeout_ms: int = 45000) -> str | None:
    """Load url in headless Chromium and return the post-JS DOM. Returns None on any failure
    (missing playwright install, navigation timeout, cloaking/bot-block, etc.) so callers can
    fall back to the wget-captured HTML rather than fail the whole rip."""
    if sync_playwright is None:
        return None
    try:
        with sync_playwright() as p:
            browser = p.chromium.launch(headless=True)
            try:
                context = browser.new_context(
                    user_agent=("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
                                "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"),
                    viewport={"width": 1920, "height": 1080},
                )
                page = context.new_page()
                page.goto(url, wait_until="networkidle", timeout=timeout_ms)
                page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
                page.wait_for_timeout(2000)  # let lazy-loaded/scroll-triggered content settle
                return page.content()
            finally:
                browser.close()
    except Exception:
        return None


def rip(job_dir: Path, url: str, post=None, wget_timeout: int = 300) -> dict:
    """
    Deterministic rip + organize + rewrite + CDN-download + known-tracker strip.
    Posts progress via post(msg) if given. Returns a summary dict.
    Raises RuntimeError if wget fails or no HTML is found.
    """
    job_dir = Path(job_dir)

    def _say(msg):
        if post:
            try:
                post(msg)
            except Exception:
                pass

    # Step 1 — wget mirror
    try:
        r = subprocess.run(
            ["wget", "--mirror", "--convert-links", "--adjust-extension",
             "--page-requisites", "--no-parent", "--reject", "*.map",
             "-P", str(job_dir), url],
            capture_output=True, text=True, timeout=wget_timeout,
        )
        _say(f"📥 Downloaded page (wget rc={r.returncode})")
    except subprocess.TimeoutExpired:
        raise RuntimeError(f"wget timed out after {wget_timeout}s")
    except Exception as e:
        raise RuntimeError(f"wget failed: {e}")

    # Step 2 — find downloaded HTML → index.html
    parsed   = urlparse(url)
    hostname = parsed.netloc
    url_path = parsed.path.strip("/")
    candidates = [
        job_dir / hostname / f"{url_path}.html",
        job_dir / hostname / url_path / "index.html",
        job_dir / hostname / f"{url_path}",
    ]
    html_src = next((c for c in candidates if c.exists() and c.is_file()), None)
    if not html_src and (job_dir / hostname).exists():
        htmls = list((job_dir / hostname).rglob("*.html"))
        html_src = htmls[0] if htmls else None
    if not html_src:
        raise RuntimeError("Could not find downloaded HTML file")

    shutil.copy2(str(html_src), str(job_dir / "index.html"))
    _say(f"📄 Created index.html from `{html_src.relative_to(job_dir)}`")

    # Step 2b — re-render with headless Chromium and swap in the post-JS DOM. wget only sees the
    # pre-JS shell, which leaves cloaked/dynamic landers (Taboola redirects, client-side templating)
    # with missing text/CTA/assets downstream. Non-fatal: keep the wget copy if this fails.
    rendered = _render_with_playwright(url)
    if rendered and len(rendered) > 500:
        (job_dir / "index.html").write_text(rendered, encoding="utf-8")
        _say("🖥️ Re-rendered with headless Chromium (captured post-JS content)")
    else:
        _say("⚠️ Headless render unavailable/failed — using wget's static capture")

    # Step 3 — asset dirs
    for sub in _EXT_MAP:
        (job_dir / "assets" / sub).mkdir(parents=True, exist_ok=True)

    # Step 4 — move files by extension into assets/
    moved_files = {}  # old rel path (from job_dir) → new assets/ path
    wget_dir = job_dir / hostname
    if wget_dir.exists():
        for f in wget_dir.rglob("*"):
            if not f.is_file():
                continue
            suffix = f.suffix.lower()
            for folder, exts in _EXT_MAP.items():
                if suffix in exts:
                    dest = job_dir / "assets" / folder / f.name
                    if dest.exists():
                        dest = job_dir / "assets" / folder / f"{f.stem}_{hash(str(f)) % 10000}{f.suffix}"
                    old_rel = str(f.relative_to(job_dir)).replace("\\", "/")
                    new_rel = f"assets/{folder}/{dest.name}"
                    shutil.move(str(f), str(dest))
                    moved_files[old_rel] = new_rel
                    break

    asset_count = sum(1 for _ in (job_dir / "assets").rglob("*") if _.is_file())
    _say(f"📁 Organized {asset_count} assets into `assets/`")

    # Step 5 — rewrite asset paths in index.html + CSS
    idx = job_dir / "index.html"
    content = idx.read_text(encoding="utf-8", errors="ignore")
    for old_rel, new_rel in moved_files.items():
        content = content.replace(old_rel, new_rel)
        content = content.replace(f"/{old_rel}", new_rel)
        tail = old_rel.split("/", 1)[-1] if "/" in old_rel else old_rel
        content = re.sub(r'(?:\.\./)+' + re.escape(tail), new_rel, content)
    content = content.replace(f"{hostname}/", "")
    idx.write_text(content, encoding="utf-8")

    for css_file in (job_dir / "assets" / "css").rglob("*.css"):
        css_text = css_file.read_text(encoding="utf-8", errors="ignore")
        css_changed = False
        for old_rel, new_rel in moved_files.items():
            css_rel = f"../{'/'.join(new_rel.split('/')[1:])}"
            old_dir_parts = old_rel.split("/")
            for part_idx in range(len(old_dir_parts)):
                old_css_ref = "../" + "/".join(old_dir_parts[part_idx:])
                if old_css_ref in css_text:
                    css_text = css_text.replace(old_css_ref, css_rel)
                    css_changed = True
        if "../font/" in css_text and not (job_dir / "assets" / "font").exists():
            css_text = css_text.replace("../font/", "../fonts/")
            css_changed = True
        if css_changed:
            css_file.write_text(css_text, encoding="utf-8")

    # Robust catch-all: rewrite ANY remaining asset reference by filename (covers templates that
    # reference assets by bare name, e.g. <img src="epicooler_product.webp">).
    basename_map = {old.rsplit('/', 1)[-1]: new for old, new in moved_files.items()}
    idx.write_text(_rewrite_by_basename(idx.read_text(encoding="utf-8", errors="ignore"), basename_map),
                   encoding="utf-8")
    for css_file in (job_dir / "assets" / "css").rglob("*.css"):
        css_file.write_text(
            _rewrite_by_basename(css_file.read_text(encoding="utf-8", errors="ignore"),
                                 basename_map, for_css=True),
            encoding="utf-8")

    _say("🔗 Rewrote asset paths in index.html and CSS")

    # Download external CDN assets and rewrite references
    content = idx.read_text(encoding="utf-8", errors="ignore")
    all_urls = _find_external_urls(content)
    for css_file in (job_dir / "assets" / "css").rglob("*.css"):
        all_urls |= _find_external_urls(css_file.read_text(encoding="utf-8", errors="ignore"))

    external_urls = set()
    for u in all_urls:
        clean = u.split('?')[0].split('#')[0]
        if hostname in u:
            continue
        if any(clean.lower().endswith(ext) for exts in _EXT_MAP.values() for ext in exts):
            external_urls.add(u)

    downloaded_cdn = 0
    failed_cdn = []
    for ext_url in external_urls:
        clean_url = ext_url.split('?')[0].split('#')[0]
        cdn_filename = unquote(Path(clean_url).name)
        if not cdn_filename or len(cdn_filename) > 200:
            continue
        cdn_ext = Path(cdn_filename).suffix.lower()
        target_folder = next((folder for folder, exts in _EXT_MAP.items() if cdn_ext in exts), None)
        if not target_folder:
            continue
        dest = job_dir / "assets" / target_folder / cdn_filename
        if dest.exists():
            content = content.replace(ext_url, f"assets/{target_folder}/{cdn_filename}")
            continue
        if _fetch(ext_url, dest):
            content = content.replace(ext_url, f"assets/{target_folder}/{cdn_filename}")
            downloaded_cdn += 1
        else:
            failed_cdn.append(ext_url)  # don't swallow — report so it's never silently external

    for css_file in (job_dir / "assets" / "css").rglob("*.css"):
        css_content = css_file.read_text(encoding="utf-8", errors="ignore")
        changed = False
        for ext_url in external_urls:
            clean_url = ext_url.split('?')[0].split('#')[0]
            cdn_filename = Path(clean_url).name
            if not cdn_filename:
                continue
            cdn_ext = Path(cdn_filename).suffix.lower()
            for folder, exts in _EXT_MAP.items():
                if cdn_ext in exts:
                    local_path = f"../{folder}/{cdn_filename}"
                    if ext_url in css_content:
                        css_content = css_content.replace(ext_url, local_path)
                        changed = True
                    break
        if changed:
            css_file.write_text(css_content, encoding="utf-8")

    idx.write_text(content, encoding="utf-8")
    if downloaded_cdn:
        _say(f"🌐 Downloaded {downloaded_cdn} external CDN assets")
    if failed_cdn:
        _say("⚠️ Could not localize (still external): " + ", ".join(Path(u).name for u in failed_cdn))

    # Step 6 — strip known trackers (safe regex pass)
    tracking_removed = _strip_tracking(idx)
    _say(f"🧹 Removed {tracking_removed} known tracking scripts/pixels")

    return {"asset_count": asset_count, "downloaded_cdn": downloaded_cdn,
            "tracking_removed": tracking_removed}


if __name__ == "__main__":
    # CLI so the agent can invoke it as its first tool call: python3 rip.py "<url>" [job_dir]
    import sys
    argv = sys.argv[1:]
    if not argv:
        print("usage: rip.py <url> [job_dir]")
        sys.exit(2)
    target_url = argv[0]
    target_dir = Path(argv[1]) if len(argv) > 1 else Path.cwd()
    result = rip(target_dir, target_url, post=print)
    print(f"[rip] complete: {result}")