#!/usr/bin/env python3
"""
Live eBay sold-listings scraper (Playwright + BeautifulSoup).

WHY SCRAPING: eBay's official APIs do NOT expose sold/completed history
(Finding API retired, Browse API = active listings only, Terapeak discontinued).
Sold listings ARE public on eBay's own search pages:
    https://www.ebay.com/sch/i.html?_nkw=<query>&LH_Sold=1&LH_Complete=1
so this module drives a real browser over those pages — exactly the view in
the user's screenshots — and caches the parsed results.

CAVEATS (be a good citizen):
  - eBay serves 403/captcha to datacenter IPs and plain HTTP clients.
    Run this on a residential connection. Optional: route via a residential
    proxy by setting EBAY_PROXY=http://user:pass@host:port
  - eBay's ToS discourages scraping; keep request rates low (we do 2-6s/page).
  - Alternatively, replace this source with a licensed data provider
    (e.g. Worthpoint-style sold-price DB); the DataSource interface in
    server.py is identical.

Setup (on YOUR machine):
    pip install playwright beautifulsoup4
    playwright install chromium
Run:
    DATA_SOURCE=live python server.py
"""
import hashlib
import json
import os
import re
import time
import random
from datetime import datetime, date
from pathlib import Path
from urllib.parse import quote_plus

from bs4 import BeautifulSoup

BASE = Path(__file__).parent
CACHE_DIR = BASE / "data" / "live"
CACHE_DIR.mkdir(parents=True, exist_ok=True)

UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
      "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")

# eBay category IDs (_sacat) come straight from data/categories.json, so the
# sidebar tree and the scraper always use the same real eBay category numbers.
def _load_cat_maps():
    tree = json.loads((BASE / "data" / "categories.json").read_text())
    sacat, names = {}, {}
    def walk(nodes):
        for n in nodes:
            sacat[str(n["id"])] = int(n["id"])
            names[str(n["id"])] = n["name"]
            if "children" in n:
                walk(n["children"])
    walk(tree)
    return sacat, names
SACAT, CAT_NAME = _load_cat_maps()

MPN_RE = re.compile(r"\b[A-Z0-9]{6,}(?:-[A-Z0-9]+)+\b|\b[A-Z0-9]{10,}\b")
PRICE_RE = re.compile(r"([\d,]+(?:\.\d{1,2})?)")

# spec/format tokens that must never be mistaken for an MPN
SPEC_RE = re.compile(r"^(DDR\d|PC\d|LGA\d+|AM\d+|SP\d+|TR\d+|GDDR\d|LPDDR\d|NVMe|SATA|USB[\d.]*|"
                      r"HDMI\d*|DP\d*|\d+(GB|MB|TB)|[\d.]+(GHZ|MHZ)|SODIMM|RDIMM|UDIMM|ECC|CL\d+|CAS|"
                      r"PCIE|M2|SSD|HDD|RAM|CPU|GPU|OEM|NEW|USED)$", re.I)


def extract_mpn(title, text=""):
    """Best-effort MPN/model token: hyphenated codes first, then any token that
    mixes letters+digits (ZX10, Z80, 9700K...) while skipping spec words."""
    blob = f"{title} {text}".upper()
    for m in re.finditer(r"[A-Z0-9]+(?:-[A-Z0-9]+)*", blob):
        t = m.group(0)
        if "-" in t or (len(t) >= 3 and any(c.isdigit() for c in t) and any(c.isalpha() for c in t)):
            if not SPEC_RE.match(t) and not re.match(
                    r"^(DDR\d|PC\d|LGA\d|AM\d|SP\d|TR\d|GDDR\d|LPDDR\d|NVME|SATA|USB|HDMI|DP\d|"
                    r"\d+RX\d|\d+(GB|MB|TB)|[\d.]+(GHZ|MHZ)|SODIMM|RDIMM|UDIMM|ECC|CL\d|PCIE|M2|SSD|HDD|RAM|CPU|GPU|OEM|NEW|USED)",
                    t, re.I):
                return t
    return ""

BRAND_ALIASES = {"XEON": "Intel", "EPYC": "AMD", "RYZEN": "AMD", "ATHLON": "AMD", "OPTERON": "AMD",
                 "PENTIUM": "Intel", "CELERON": "Intel", "GEFORCE": "NVIDIA", "RADEON": "AMD",
                 "CORE": "Intel", "CELERON": "Intel"}

KNOWN_BRANDS = ["SK hynix", "G.Skill", "Western Digital", "Intel", "AMD", "Hynix", "Samsung",
                "Kingston", "Corsair", "Crucial", "Micron", "EVGA", "Gigabyte", "ASUS", "MSI",
                "ASRock", "Seagate", "Toshiba", "SanDisk", "PNY", "Patriot", "Teamgroup", "Adata",
                "Dell", "Lenovo", "HP", "Acer", "Razer", "Logitech", "Broadcom", "Marvell", "NVIDIA"]
NON_BRAND_WORDS = {"used", "new", "pre-owned", "open", "box", "lot", "refurbished", "sale",
                   "bundle", "kit", "set", "only", "hot", "great", "combo", "combination", "i7", "i9"}


def normalize_brand(title):
    """Detect brand: known brands, then product-line aliases (Xeon->Intel),
    then first non-filler word. Never blindly 'first word of title'."""
    t = title or ""
    for b in KNOWN_BRANDS:
        if re.search(rf"\b{re.escape(b)}\b", t, re.I):
            return b
    for alias, brand in BRAND_ALIASES.items():
        if re.search(rf"\b{alias}\b", t, re.I):
            return brand
    for w in t.split()[:3]:
        w2 = w.strip(",.:;()")
        if w2.lower() not in NON_BRAND_WORDS and len(w2) > 1:
            return w2
    return t.split()[0].strip(",.:;()") if t.split() else ""


def build_url(q, category_id, page=1):
    params = {"_ipg": "100", "LH_Sold": "1", "LH_Complete": "1", "_sop": "10"}
    if q:
        params["_nkw"] = q
    sacat = SACAT.get(category_id)
    if sacat:
        params["_sacat"] = str(sacat)
    if page > 1:
        params["_pgn"] = str(page)
    qs = "&".join(f"{k}={quote_plus(str(v))}" for k, v in params.items())
    return f"https://www.ebay.com/sch/i.html?{qs}"


def parse_sold_date(text, now=None):
    """'Sold Oct 30, 2026' / 'Sold 10/30/26' -> '2026-10-30' (best effort)."""
    now = now or date.today()
    m = re.search(r"([A-Z][a-z]{2})\s+(\d{1,2}),?\s+(\d{4})", text)
    if m:
        try:
            return datetime.strptime(f"{m.group(1)} {m.group(2)} {m.group(3)}", "%b %d %Y").date().isoformat()
        except ValueError:
            pass
    m = re.search(r"(\d{1,2})/(\d{1,2})/(\d{2,4})", text)
    if m:
        y = int(m.group(3)); y = 2000 + y if y < 100 else y
        try:
            return date(y, int(m.group(1)), int(m.group(2))).isoformat()
        except ValueError:
            pass
    return now.isoformat()


ITEM_SEL = "li.s-item, div.s-card, [data-testid='s-card'], .su-card"
TITLE_SEL = ".s-item__title, .s-card__title, [data-testid='s-card-title'], h3.s-card__title, h3"
PRICE_SEL = ".s-item__price, .s-card__price, .s-price, [class*='price']"
LINK_SEL = "a.s-item__link, .s-card__link, a[href*='/itm/']"
DATE_SEL = ".s-item__endedDate, .s-card__endedDate, [class*='endedDate'], [class*='soldDate']"
SELLER_SEL = ".s-item__sellerInfo a, .s-card__sellerInfo, .su-card__seller a, [class*='seller'] a"
IMG_SEL = ".s-item__image-img, .s-card__image img, img[src*='ebayimg']"


def _pick(soup_or_el, selector_list):
    for sel in selector_list.split(", "):
        el = soup_or_el.select_one(sel)
        if el:
            return el
    return None


def parse_items(html, category_id, now=None):
    """Parse one eBay sold page (/sch/ search or /b/ category browse) into dicts."""
    now = now or date.today()
    soup = BeautifulSoup(html, "html.parser")
    items = []
    for li in soup.select(ITEM_SEL):
        try:
            title_el = _pick(li, TITLE_SEL)
            if not title_el or "Shop on eBay" in title_el.get_text():
                continue
            title = title_el.get_text(strip=True)
            price_el = _pick(li, PRICE_SEL)
            price = 0.0
            if price_el:
                pm = PRICE_RE.search(price_el.get_text())
                if pm:
                    price = float(pm.group(1).replace(",", ""))
            link_el = _pick(li, LINK_SEL)
            url = link_el["href"] if link_el and link_el.has_attr("href") else ""
            # sold date: several layouts over the years
            sold_txt = ""
            date_el = _pick(li, DATE_SEL)
            if date_el and "sold" in date_el.get_text().lower():
                sold_txt = date_el.get_text(" ", strip=True)
            if not sold_txt:
                m = re.search(r"Sold[^<]{0,30}", li.get_text(" ", strip=True))
                sold_txt = m.group(0) if m else ""
            sold_date = parse_sold_date(sold_txt, now)
            # seller
            seller = ""
            sel_el = _pick(li, SELLER_SEL)
            if sel_el:
                seller = sel_el.get_text(strip=True)
            # condition
            cond = ""
            sub = li.select_one(".s-item__subtitle, .s-item__details, .s-item__trendingContents")
            blob = (sub.get_text(" ", strip=True) if sub else li.get_text(" ", strip=True))
            for c in ("New", "Open Box", "Pre-Owned", "Used", "Refurbished"):
                if re.search(rf"\b{c}\b", blob, re.I):
                    cond = c if c != "Refurbished" else "Used"; break
            mpn = extract_mpn(title, li.get_text(" ", strip=True))
            img_el = _pick(li, IMG_SEL)
            img = img_el.get("src", "") if img_el else ""
            items.append({
                "title": title, "mpn": mpn,
                "brand": normalize_brand(title),
                "model": mpn, "category_id": category_id,
                "category": CAT_NAME.get(category_id, "eBay Sold"),
                "price": round(price, 2), "currency": "USD",
                "sold_date": sold_date, "condition": cond or "Pre-Owned",
                "format": "Sold", "seller": seller,
                "seller_rating": None, "seller_feedback": None,
                "location": "", "shipping": None, "url": url, "image": img,
            })
        except Exception:
            continue
    if not items:
        items = _parse_link_centric(soup, category_id, now)
    return items


def _parse_link_centric(soup, category_id, now):
    """Fallback for layouts without known card classes: anchor every item to its
    /itm/ link (universal across eBay layouts) and read the enclosing card."""
    items, seen = [], set()
    for a in soup.select('a[href*="/itm/"]'):
        try:
            href = a.get("href", "")
            if not href or href in seen:
                continue
            seen.add(href)
            card = a.find_parent("li") or a
            for _ in range(6):
                if card is None:
                    break
                if len(card.get_text(strip=True)) > 150 or card.find("img"):
                    break
                card = card.parent
            if card is None or card.name in ("body", "html", "[document]"):
                continue
            text = re.sub(r"\s+", " ", card.get_text(" ", strip=True))
            h3 = card.find("h3")
            title = (a.get_text(strip=True) or (h3.get_text(strip=True) if h3 else "")).strip()[:300]
            if not title or "shop on ebay" in title.lower():
                continue
            pm = re.search(r"(?:US\s*)?\$([\d,]+\.\d{2})", text)
            price = float(pm.group(1).replace(",", "")) if pm else 0.0
            dm = re.search(r"Sold[^.]{0,30}", text, re.I)
            sold_date = parse_sold_date(dm.group(0) if dm else "", now)
            sm = re.search(r"([A-Za-z0-9_.\-]{2,30})\s*\(?[\d.]+%\s*positive", text, re.I)
            seller = sm.group(1) if sm else ""
            cm = None
            for c in ("New", "Open Box", "Pre-Owned", "Refurbished", "Used"):
                if re.search(rf"\b{c}\b", text):
                    cm = "Used" if c == "Refurbished" else c
                    break
            img = card.find("img")
            img_src = img.get("src", "") if img else ""
            mpn_m = MPN_RE.search(title)
            items.append({
                "title": title, "mpn": mpn_m.group(0) if mpn_m else "",
                "brand": normalize_brand(title),
                "model": mpn_m.group(0) if mpn_m else "", "category_id": category_id,
                "category": CAT_NAME.get(category_id, "eBay Sold"),
                "price": round(price, 2), "currency": "USD", "sold_date": sold_date,
                "condition": cm or "Pre-Owned", "format": "Sold", "seller": seller,
                "seller_rating": None, "seller_feedback": None,
                "location": "", "shipping": None, "url": href, "image": img_src,
            })
        except Exception:
            continue
    return items


def cache_key(q, category_id):
    return hashlib.md5(f"{q}|{category_id}".encode()).hexdigest()[:16]


def cache_path(q, category_id):
    return CACHE_DIR / f"{cache_key(q, category_id)}.json"


def read_cache(q, category_id):
    p = cache_path(q, category_id)
    if p.is_file():
        return json.loads(p.read_text())
    return None


def _check_proxy(proxy):
    """Fail fast with a clear message if EBAY_PROXY is a placeholder/unreachable."""
    import socket
    from urllib.parse import urlparse
    u = urlparse(proxy)
    host, port = u.hostname, u.port or (443 if u.scheme == "https" else 80)
    if not host or host in ("host", "proxy.example.com"):
        raise RuntimeError("EBAY_PROXY looks like a placeholder. Unset it (PowerShell: Remove-Item Env:EBAY_PROXY) "
                           "to scrape directly, or set a real proxy like http://user:pass@1.2.3.4:8080")
    try:
        s = socket.create_connection((host, port), timeout=6)
        s.close()
    except Exception as e:
        raise RuntimeError(f"EBAY_PROXY {host}:{port} is unreachable ({e}). Unset it (PowerShell: "
                           f"Remove-Item Env:EBAY_PROXY) to scrape directly, or fix the proxy address.")


def _resolve_proxy(proxy):
    """Return a usable proxy, or None with a warning (auto-fallback to direct)."""
    if not proxy:
        return None
    try:
        _check_proxy(proxy)
        return proxy
    except RuntimeError as e:
        print(f"[scraper] WARNING: {e}  -> falling back to DIRECT connection (no proxy).")
        return None


def scrape_sold(q="", category_id="", max_pages=3, headless=None):
    """Drive a real browser over eBay sold-search pages. Returns item list."""
    try:
        from playwright.sync_api import sync_playwright
    except ImportError:
        raise RuntimeError("Playwright not installed. Run: pip install playwright && playwright install chromium")
    if headless is None:
        headless = os.environ.get("HEADLESS", "1") != "0"  # set HEADLESS=0 to watch the browser
    proxy = _resolve_proxy(os.environ.get("EBAY_PROXY"))
    all_items, seen = [], set()
    profile = BASE / "data" / "browser-profile"  # persistent profile keeps eBay cookies between runs
    with sync_playwright() as pw:
        kwargs = {"headless": headless, "user_agent": UA, "locale": "en-US",
                  "viewport": {"width": 1366, "height": 900},
                  "args": ["--disable-blink-features=AutomationControlled"]}
        if proxy:
            kwargs["proxy"] = {"server": proxy}
        browser = pw.chromium.launch_persistent_context(str(profile), **kwargs)
        page = browser.new_page()
        for pg in range(1, max_pages + 1):
            url = build_url(q, category_id, pg)
            try:
                page.goto(url, wait_until="domcontentloaded", timeout=60000)
            except Exception as e:
                msg = str(e)
                if "PROXY" in msg:
                    raise RuntimeError("Browser could not reach EBAY_PROXY — it is unset/invalid. "
                                       "PowerShell: Remove-Item Env:EBAY_PROXY  (then restart server), or set a real proxy.")
                if "ABORTED" in msg:
                    raise RuntimeError("Navigation aborted (often caused by an invalid EBAY_PROXY or eBay bot-wall). "
                                       "Try: Remove-Item Env:EBAY_PROXY, restart server, retry.")
                raise
            # cookie/GDPR consent wall
            for sel in ('#gdpr-banner-accept', 'button:has-text("Accept all")', 'button:has-text("Accept All")'):
                try:
                    page.click(sel, timeout=2500); break
                except Exception:
                    pass
            try:
                page.wait_for_selector("li.s-item", timeout=25000)
            except Exception:
                # likely bot-wall (403/captcha/sign-in) — stop and report
                body = page.inner_text("body")[:200].replace("\n", " ")
                raise RuntimeError(f"eBay returned a bot/sign-in wall on page {pg}: {body!r}. "
                                   "Run from a residential IP, accept cookies in a real browser first, "
                                   "or set EBAY_PROXY to a residential proxy.")
            items = parse_items(page.content(), category_id)
            for it in items:
                k = it["url"] or it["title"]
                if k not in seen:
                    seen.add(k); all_items.append(it)
            # human-like pacing: move mouse, scroll, then wait
            try:
                page.mouse.move(random.randint(100, 1200), random.randint(100, 800))
                page.mouse.wheel(0, random.randint(300, 900))
            except Exception:
                pass
            time.sleep(random.uniform(3.0, 8.0))  # be polite & human-like
        browser.close()
    return all_items


def refresh_cache(q="", category_id="", max_pages=None):
    """Scrape and persist; returns (items, status_dict)."""
    if max_pages is None:
        max_pages = int(os.environ.get("MAX_PAGES", "2"))
    p = cache_path(q, category_id)
    try:
        items = scrape_sold(q, category_id, max_pages)
        payload = {"q": q, "category_id": category_id, "scraped_at": datetime.now().isoformat(),
                   "count": len(items), "items": items}
        p.write_text(json.dumps(payload))
        return items, {"ok": True, "count": len(items), "scraped_at": payload["scraped_at"]}
    except Exception as e:
        cached = read_cache(q, category_id)
        return (cached["items"] if cached else []), {"ok": False, "error": str(e),
                                                     "using_cache": bool(cached)}


class LiveScraperSource:
    """DataSource backed by scraped+ cached eBay sold pages."""
    def items(self):
        out = []
        for f in sorted(CACHE_DIR.glob("*.json")):
            try:
                out.extend(json.loads(f.read_text()).get("items", []))
            except Exception:
                pass
        for i, it in enumerate(out, 1):
            it.setdefault("id", i)
            it["brand"] = normalize_brand(it.get("title", ""))
        return out

    def categories(self):
        return json.loads((BASE / "data" / "categories.json").read_text())