#!/usr/bin/env python3
import csv
import hashlib
import json
import os
import re
import shutil
import sys
import time
import xml.etree.ElementTree as ET
from collections import deque
from html import unescape
from urllib.parse import quote, unquote, urljoin, urlparse, urlsplit, urlunsplit
from urllib.request import Request, urlopen

BASE = "/root/MARKETING_DAM"
USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36"
TIMEOUT = 30
PAGE_LIMIT = 400
ASSET_EXTS = {
    ".png", ".jpg", ".jpeg", ".webp", ".gif", ".svg", ".ico", ".bmp", ".avif",
    ".css", ".js", ".mjs", ".map", ".json", ".xml", ".txt", ".pdf",
    ".mp4", ".webm", ".mov", ".m4v", ".mp3", ".wav", ".ogg",
    ".woff", ".woff2", ".ttf", ".otf", ".eot"
}
FONT_EXTS = {".woff", ".woff2", ".ttf", ".otf", ".eot"}
TEXT_LIKE_EXTS = {".html", ".htm", ".css", ".js", ".mjs", ".json", ".xml", ".txt", ".map", ".svg"}
BLOCKED_HOST_PARTS = (
    "googletagmanager.com",
    "google-analytics.com",
    "hotjar.com",
    "facebook.com",
    "instagram.com",
    "tiktok.com",
    "twitter.com",
    "youtube.com",
    "schema.org",
    "w3.org",
    "whatsapp.com",
    "notifications-icommkt.com",
)
SITE_CONFIGS = [
    {
        "name": "EL_CORRAL",
        "root": "https://www.elcorral.com/",
        "sitemap": "https://www.elcorral.com/sitemap.xml",
        "site_hosts": {"www.elcorral.com", "elcorral.com"},
        "extra_asset_hosts": {"d7364jmfys2bj.cloudfront.net", "gruponutresa.com", "unpkg.com"},
    },
    {
        "name": "EL_CORRAL_GOURMET",
        "root": "https://www.elcorralgourmet.com/",
        "sitemap": "https://www.elcorralgourmet.com/sitemap.xml",
        "site_hosts": {"www.elcorralgourmet.com", "elcorralgourmet.com"},
        "extra_asset_hosts": {"cdn.jsdelivr.net", "drive.google.com"},
    },
]


def normalize_url(raw_url, base_url=None):
    if not raw_url:
        return None
    raw_url = unescape(raw_url).strip().strip('"\'<>')
    raw_url = raw_url.rstrip('\\')
    if not raw_url or raw_url.startswith(("data:", "mailto:", "tel:", "javascript:")):
        return None
    if base_url:
        raw_url = urljoin(base_url, raw_url)
    parsed = urlsplit(raw_url)
    if parsed.scheme not in {"http", "https"} or not parsed.netloc:
        return None
    path = quote(unquote(parsed.path), safe="/%:@-._~!$&'()*+,;=")
    query = quote(unquote(parsed.query), safe="=&?/:;+,%@-._~")
    return urlunsplit((parsed.scheme, parsed.netloc, path, query, ""))


def get_extension(url):
    return os.path.splitext(urlparse(url).path)[1].lower()


def content_type_to_ext(content_type):
    mapping = {
        "text/html": ".html",
        "text/css": ".css",
        "application/javascript": ".js",
        "text/javascript": ".js",
        "application/json": ".json",
        "image/svg+xml": ".svg",
        "font/otf": ".otf",
        "font/ttf": ".ttf",
        "font/woff": ".woff",
        "font/woff2": ".woff2",
        "application/font-woff": ".woff",
        "application/pdf": ".pdf",
        "image/webp": ".webp",
        "image/png": ".png",
        "image/jpeg": ".jpg",
        "image/x-icon": ".ico",
    }
    return mapping.get((content_type or "").split(";")[0].strip().lower(), "")


def is_blocked_host(host):
    host = host.lower()
    return any(part in host for part in BLOCKED_HOST_PARTS)


def is_same_site_page(url, cfg):
    parsed = urlparse(url)
    if parsed.netloc.lower() not in cfg["site_hosts"]:
        return False
    ext = get_extension(url)
    return ext in {"", ".html", ".htm", ".php"}


def should_fetch_asset(url, cfg):
    parsed = urlparse(url)
    host = parsed.netloc.lower()
    if is_blocked_host(host):
        return False
    ext = get_extension(url)
    if host in cfg["site_hosts"]:
        if ext in ASSET_EXTS:
            return True
        return parsed.path.startswith("/_next/") or parsed.path.startswith("/static/") or parsed.path.startswith("/assets/")
    if host in cfg["extra_asset_hosts"]:
        return ext in ASSET_EXTS or any(token in parsed.path.lower() for token in ["/static/", "/uploads/", "/wp-content/", "/dist/"])
    return False


def ensure_dir(path):
    os.makedirs(path, exist_ok=True)


def safe_segment(segment):
    segment = segment.strip().replace(" ", "_")
    segment = re.sub(r"[^A-Za-z0-9._-]", "_", segment)
    return segment or "_"


def local_path_for(base_dir, bucket, url, content_type):
    parsed = urlparse(url)
    host = safe_segment(parsed.netloc)
    raw_path = unquote(parsed.path or "/")
    ext = get_extension(url) or content_type_to_ext(content_type)
    if bucket == "pages":
        if raw_path.endswith("/"):
            raw_path = raw_path + "index.html"
        elif raw_path in {"", "/"}:
            raw_path = "/index.html"
        elif not os.path.splitext(raw_path)[1]:
            raw_path = raw_path + "/index.html"
    else:
        if raw_path.endswith("/"):
            raw_path = raw_path + "index"
        elif raw_path in {"", "/"}:
            raw_path = "/index"
        if not os.path.splitext(raw_path)[1]:
            raw_path = raw_path + (ext or ".bin")
    parts = [safe_segment(part) for part in raw_path.split("/") if part]
    if not parts:
        parts = ["index.html" if bucket == "pages" else f"index{ext or '.bin'}"]
    filename = parts[-1]
    stem, old_ext = os.path.splitext(filename)
    if parsed.query:
        digest = hashlib.sha1(parsed.query.encode("utf-8", "ignore")).hexdigest()[:10]
        filename = f"{stem}__q_{digest}{old_ext}"
        parts[-1] = filename
    target = os.path.join(base_dir, bucket, host, *parts)
    ensure_dir(os.path.dirname(target))
    return target


def fetch(url):
    req = Request(url, headers={"User-Agent": USER_AGENT})
    with urlopen(req, timeout=TIMEOUT) as resp:
        data = resp.read()
        content_type = resp.headers.get_content_type()
        final_url = normalize_url(resp.geturl()) or url
        return final_url, content_type, data


def decode_text(data):
    return data.decode("utf-8", "ignore")


def extract_candidates(text, base_url):
    found = set()
    for match in re.findall(r'https?://[^\s"\'<>\\)]+', text):
        found.add(match)
    attr_pattern = re.compile(r'(?:href|src|poster|content|data-src|data-href|data-background|data-bg)=["\']([^"\']+)["\']', re.I)
    for match in attr_pattern.findall(text):
        found.add(match)
    srcset_pattern = re.compile(r'(?:srcset|data-srcset)=["\']([^"\']+)["\']', re.I)
    for value in srcset_pattern.findall(text):
        for chunk in value.split(','):
            url_part = chunk.strip().split()[0]
            if url_part:
                found.add(url_part)
    for match in re.findall(r'url\(([^)]+)\)', text, re.I):
        found.add(match.strip())
    path_pattern = re.compile(r'["\']((?:/|\.\.?/)[^"\']+\.(?:png|jpg|jpeg|webp|gif|svg|ico|css|js|mjs|woff2?|ttf|otf|eot|pdf|mp4|webm|mp3|wav|json))(?:\?[^"\']*)?["\']', re.I)
    for match in path_pattern.findall(text):
        found.add(match)
    normalized = set()
    for raw in found:
        norm = normalize_url(raw, base_url=base_url)
        if norm:
            normalized.add(norm)
    return normalized


def should_parse_text(url, content_type, local_path):
    ext = get_extension(url) or os.path.splitext(local_path)[1].lower() or content_type_to_ext(content_type)
    if ext in TEXT_LIKE_EXTS:
        return True
    return (content_type or "").startswith("text/") or content_type in {"application/json", "image/svg+xml"}


def copy_unique(src_path, dest_dir):
    ensure_dir(dest_dir)
    base = os.path.basename(src_path)
    target = os.path.join(dest_dir, base)
    if not os.path.exists(target):
        shutil.copy2(src_path, target)
        return target
    if os.path.getsize(src_path) == os.path.getsize(target):
        return target
    stem, ext = os.path.splitext(base)
    digest = hashlib.sha1(src_path.encode("utf-8", "ignore")).hexdigest()[:8]
    target = os.path.join(dest_dir, f"{stem}_{digest}{ext}")
    shutil.copy2(src_path, target)
    return target


def write_csv(path, rows, header):
    ensure_dir(os.path.dirname(path))
    with open(path, "w", newline="", encoding="utf-8") as fh:
        writer = csv.DictWriter(fh, fieldnames=header)
        writer.writeheader()
        writer.writerows(rows)


def fetch_sitemap_urls(sitemap_url):
    final_url, content_type, data = fetch(sitemap_url)
    root = ET.fromstring(data)
    ns = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"}
    urls = []
    for node in root.findall("sm:url", ns):
        loc = node.find("sm:loc", ns)
        if loc is not None and loc.text:
            normalized = normalize_url(loc.text)
            if normalized:
                urls.append(normalized)
    return list(dict.fromkeys(urls))


def crawl_site(cfg):
    site_dir = os.path.join(BASE, "INBOX", "WEBSITE_IMPORTS", cfg["name"])
    pages_dir = os.path.join(site_dir, "pages")
    assets_dir = os.path.join(site_dir, "assets")
    manifests_dir = os.path.join(site_dir, "manifests")
    fonts_dir = os.path.join(BASE, "FONTS", "BRANDS", cfg["name"], "WEB_IMPORTED")
    logos_dir = os.path.join(BASE, "LOGOS", "BRANDS", cfg["name"], "WEB_IMPORTED")
    for path in [pages_dir, assets_dir, manifests_dir, fonts_dir, logos_dir]:
        ensure_dir(path)

    seeds = fetch_sitemap_urls(cfg["sitemap"])
    root_url = normalize_url(cfg["root"])
    if root_url and root_url not in seeds:
        seeds.insert(0, root_url)

    queue = deque(("page", url, "") for url in seeds)
    seen = set()
    manifest_rows = []
    font_rows = []
    logo_rows = []
    errors = []
    pages_downloaded = 0
    assets_downloaded = 0

    print(f"[{cfg['name']}] seeds: {len(seeds)}")
    while queue and pages_downloaded <= PAGE_LIMIT:
        kind, url, referer = queue.popleft()
        norm = normalize_url(url)
        if not norm or norm in seen:
            continue
        if kind == "page" and not is_same_site_page(norm, cfg):
            continue
        if kind == "asset" and not should_fetch_asset(norm, cfg):
            continue
        seen.add(norm)
        try:
            final_url, content_type, data = fetch(norm)
            bucket = "pages" if kind == "page" or (content_type or "").startswith("text/html") else "assets"
            local_path = local_path_for(site_dir, bucket, final_url, content_type)
            with open(local_path, "wb") as fh:
                fh.write(data)
            if bucket == "pages":
                pages_downloaded += 1
            else:
                assets_downloaded += 1
            manifest_rows.append({
                "source_site": cfg["name"],
                "kind": bucket[:-1],
                "url": final_url,
                "referer": referer,
                "content_type": content_type,
                "bytes": len(data),
                "local_path": local_path,
            })
            ext = get_extension(final_url) or os.path.splitext(local_path)[1].lower() or content_type_to_ext(content_type)
            if ext in FONT_EXTS or (content_type or "").startswith("font/"):
                copied = copy_unique(local_path, fonts_dir)
                font_rows.append({"source_site": cfg["name"], "url": final_url, "local_path": copied})
            lower_url = final_url.lower()
            lower_name = os.path.basename(urlparse(final_url).path).lower()
            if "logo" in lower_url or lower_name in {"favicon.ico", "apple-touch-icon.png", "apple-touch-icon-precomposed.png"}:
                copied = copy_unique(local_path, logos_dir)
                logo_rows.append({"source_site": cfg["name"], "url": final_url, "local_path": copied})
            if should_parse_text(final_url, content_type, local_path):
                text = decode_text(data)
                for discovered in extract_candidates(text, final_url):
                    if is_same_site_page(discovered, cfg):
                        queue.append(("page", discovered, final_url))
                    elif should_fetch_asset(discovered, cfg):
                        queue.append(("asset", discovered, final_url))
        except Exception as exc:
            errors.append({"source_site": cfg["name"], "kind": kind, "url": norm, "referer": referer, "error": str(exc)})

    write_csv(
        os.path.join(manifests_dir, "download_manifest.csv"),
        manifest_rows,
        ["source_site", "kind", "url", "referer", "content_type", "bytes", "local_path"],
    )
    write_csv(
        os.path.join(manifests_dir, "font_manifest.csv"),
        font_rows,
        ["source_site", "url", "local_path"],
    )
    write_csv(
        os.path.join(manifests_dir, "logo_manifest.csv"),
        logo_rows,
        ["source_site", "url", "local_path"],
    )
    write_csv(
        os.path.join(manifests_dir, "errors.csv"),
        errors,
        ["source_site", "kind", "url", "referer", "error"],
    )
    summary = {
        "source_site": cfg["name"],
        "seed_pages": len(seeds),
        "downloaded_pages": pages_downloaded,
        "downloaded_assets": assets_downloaded,
        "detected_fonts": len(font_rows),
        "detected_logos": len(logo_rows),
        "errors": len(errors),
        "generated_at_unix": int(time.time()),
        "site_dir": site_dir,
    }
    with open(os.path.join(manifests_dir, "summary.json"), "w", encoding="utf-8") as fh:
        json.dump(summary, fh, indent=2)
    print(json.dumps(summary, indent=2))


def main():
    ensure_dir(os.path.join(BASE, "INBOX", "WEBSITE_IMPORTS"))
    for cfg in SITE_CONFIGS:
        crawl_site(cfg)


if __name__ == "__main__":
    sys.exit(main())