#!/usr/bin/env python3
"""
eBay Sold-Items Research — backend API (zero-dependency, stdlib only).

DATA-SOURCE ABSTRACTION:
  eBay's official APIs no longer expose sold/completed listing history
  (Finding API retired; Browse API has no sold data; Terapeak discontinued).
  Therefore `DataSource` is an interface: swap `JsonFileSource` for a
  scraper pipeline, a licensed data provider, or CSV import — the API
  contract and UI stay identical.

Endpoints:
  GET /api/categories                 category tree with sold counts (90d)
  GET /api/sold                       filtered sold items (paginated, sortable)
  GET /api/analytics                  KPIs + trend + top products/sellers/brands
  GET /api/export.csv                 same filters, CSV download
  GET /api/health
  GET /                              dashboard UI (static/index.html)
"""
import csv
import io
import json
import os
import re
import time
from datetime import date, datetime, timedelta
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from urllib.parse import urlparse, parse_qs

BASE = Path(__file__).parent
DATA = BASE / "data"
STATIC = BASE / "static"
TODAY = date(2026, 11, 5)
WINDOW_DAYS = 90
CACHE_TTL = int(os.environ.get("CACHE_TTL_SECONDS", "3600"))  # re-scrape at most hourly
MODE = "live"  # live eBay sold-listing scraping is the ONLY mode; no demo data


class DataSource:
    """Interface for sold-items data. Replace with scraper/provider/CSV."""
    def items(self):
        raise NotImplementedError
    def categories(self):
        raise NotImplementedError


from scraper_source import LiveScraperSource, read_cache, refresh_cache, cache_path, normalize_brand, extract_mpn
SRC = LiveScraperSource()  # real eBay sold pages only — no demo data anywhere
FAIL_TTL = 120  # after a failed scrape, don't re-scrape the same query for 2 min
REPORTS = DATA / "reports"
REPORTS.mkdir(exist_ok=True)


def handle_ingest(payload):
    """Normalize + cache sold items sent by the browser bookmarklet. Returns (obj, http_code)."""
    from scraper_source import parse_sold_date, CAT_NAME
    q = (payload.get("q") or "").strip()
    cat = str(payload.get("category_id") or "")
    raw = payload.get("items") or []
    norm = []
    for it in raw:
        title = (it.get("title") or "").strip()
        if not title:
            continue
        cond = it.get("condition") or ""
        if not cond:
            blob = (it.get("text") or title)
            for c in ("New", "Open Box", "Pre-Owned", "Refurbished", "Used"):
                if re.search(rf"\b{c}\b", blob, re.I):
                    cond = "Used" if c == "Refurbished" else c
                    break
        norm.append({
            "title": title, "mpn": it.get("mpn") or extract_mpn(title, it.get("text", "")),
            "brand": normalize_brand(title), "model": it.get("mpn") or extract_mpn(title, it.get("text", "")),
            "category_id": cat, "category": CAT_NAME.get(cat, "eBay Sold"),
            "price": round(float(it.get("price") or 0), 2), "currency": "USD",
            "sold_date": parse_sold_date(it.get("sold_date") or ""),
            "condition": cond or "Pre-Owned", "format": "Sold",
            "seller": it.get("seller") or "", "seller_rating": None, "seller_feedback": None,
            "location": "", "shipping": None, "url": it.get("url") or "", "image": it.get("image") or "",
        })
    cache_path(q, cat).write_text(json.dumps({
        "q": q, "category_id": cat, "scraped_at": datetime.now().isoformat(),
        "count": len(norm), "total_results": payload.get("total_results"),
        "items": norm, "source": "browser-bookmarklet"}))
    return {"status": "ok", "ingested": len(norm), "q": q, "category_id": cat}, 200


def save_report(payload):
    name = (payload.get("name") or "report").strip()[:80]
    slug = re.sub(r"[^\w-]+", "-", name.lower()).strip("-") or "report"
    rid = datetime.now().strftime("%Y%m%d%H%M%S") + "_" + slug
    doc = {"id": rid, "name": name, "created_at": datetime.now().isoformat(),
           "params": {k: payload.get(k, "") for k in ("q", "category_id", "days", "condition", "format",
                                                      "min_price", "max_price", "min_rating", "sort", "order")},
           "analytics": payload.get("analytics", {})}
    (REPORTS / f"{rid}.json").write_text(json.dumps(doc, indent=1))
    return {"status": "ok", "id": rid}, 200


def list_reports():
    out = []
    for f in sorted(REPORTS.glob("*.json"), reverse=True):
        try:
            d = json.loads(f.read_text())
            k = d.get("analytics", {}).get("kpis", {})
            top = (d.get("analytics", {}).get("top_products") or [{}])[0]
            out.append({"id": d["id"], "name": d["name"], "created_at": d["created_at"],
                        "params": d.get("params", {}), "kpis": k, "top_product": top})
        except Exception:
            pass
    return out


def get_report(rid):
    if not re.fullmatch(r"[\w-]+", rid or ""):
        return None
    f = REPORTS / f"{rid}.json"
    return json.loads(f.read_text()) if f.is_file() else None


def maybe_refresh_live(qs):
    """Re-scrape the current query from eBay if its cache is stale/missing."""
    q = (qs.get("q", [""])[0] or "").strip()
    cat = (qs.get("category_id", [""])[0] or "")
    if not q and not cat:
        return {"refreshed": False, "reason": "no query — pick a category or search an MPN/brand first"}
    force = qs.get("force", ["0"])[0] == "1"
    fail_marker = cache_path(q, cat).with_suffix(".fail")
    if not force and fail_marker.is_file():
        age = time.time() - fail_marker.stat().st_mtime
        if age < FAIL_TTL:
            return {"refreshed": False, "reason": "recent scrape failed; cooling down (use force=1 to retry now)",
                    "age_seconds": int(age)}
    cached = read_cache(q, cat)
    if not force and cached:
        try:
            age = time.time() - datetime.fromisoformat(cached["scraped_at"]).timestamp()
            if age < CACHE_TTL:
                return {"refreshed": False, "reason": "cache fresh", "age_seconds": int(age)}
        except Exception:
            pass
    _, status = refresh_cache(q, cat)
    if not status.get("ok"):
        fail_marker.write_text(json.dumps(status))
    elif fail_marker.exists():
        fail_marker.unlink()
    return {"refreshed": True, **status}


def parse_filters(qs):
    def one(k, default=None):
        v = qs.get(k, [default])
        return v[0] if v else default
    f = {
        "q": (one("q") or "").strip().lower(),
        "category_id": one("category_id") or "",
        "days": int(one("days") or WINDOW_DAYS),
        "condition": one("condition") or "",
        "format": one("format") or "",
        "min_price": float(one("min_price") or 0),
        "max_price": float(one("max_price") or 1e12),
        "min_rating": float(one("min_rating") or 0),
        "sort": one("sort") or "sold_date",
        "order": one("order") or "desc",
        "page": int(one("page") or 1),
        "per_page": min(int(one("per_page") or 25), 200),
    }
    return f


SORT_KEYS = {
    "sold_date": lambda x: x["sold_date"],
    "price": lambda x: x["price"],
    "title": lambda x: x["title"].lower(),
    "brand": lambda x: x["brand"].lower(),
    "seller": lambda x: x["seller"].lower(),
    "shipping": lambda x: x["shipping"],
}


def filter_items(f):
    cutoff = (TODAY - timedelta(days=f["days"])).isoformat()
    q = f["q"]
    out = []
    for it in SRC.items():
        if it["sold_date"] < cutoff:
            continue
        if f["category_id"] and it["category_id"] != f["category_id"]:
            continue
        if q and not any(q in str(it[k]).lower() for k in ("mpn", "title", "brand", "model")):
            continue
        if f["condition"] and it["condition"] != f["condition"]:
            continue
        if f["format"] and it["format"] != f["format"]:
            continue
        if not (f["min_price"] <= it["price"] <= f["max_price"]):
            continue
        if (it.get("seller_rating") or 0) < f["min_rating"]:
            continue
        out.append(it)
    key = SORT_KEYS.get(f["sort"], SORT_KEYS["sold_date"])
    out.sort(key=key, reverse=(f["order"] == "desc"))
    return out


def analytics(rows):
    if not rows:
        return {"kpis": {}, "trend": [], "top_products": [], "top_sellers": [],
                "brands": [], "conditions": [], "formats": []}
    prices = sorted(r["price"] for r in rows)
    n = len(prices)
    kpis = {
        "total_sold": len(rows),
        "revenue": round(sum(prices), 2),
        "avg_price": round(sum(prices) / n, 2),
        "median_price": round(prices[n // 2] if n % 2 else (prices[n // 2 - 1] + prices[n // 2]) / 2, 2),
        "min_price": prices[0],
        "max_price": prices[-1],
        "unique_products": len({r["mpn"] for r in rows}),
        "unique_sellers": len({r["seller"] for r in rows}),
        "avg_shipping": round(sum((r.get("shipping") or 0) for r in rows) / n, 2),
    }

    def group(keyfn):
        g = {}
        for r in rows:
            g.setdefault(keyfn(r), []).append(r)
        return g

    # daily avg price trend
    by_day = group(lambda r: r["sold_date"])
    trend = [{"date": d, "avg": round(sum(x["price"] for x in v) / len(v), 2), "count": len(v)}
             for d, v in sorted(by_day.items())]

    by_prod = group(lambda r: (r["mpn"] or r["title"].lower(), r["title"]))
    top_products = sorted(
        [{"mpn": k[0], "title": k[1], "units": len(v),
          "revenue": round(sum(x["price"] for x in v), 2),
          "avg_price": round(sum(x["price"] for x in v) / len(v), 2)} for k, v in by_prod.items()],
        key=lambda x: x["units"], reverse=True)[:10]

    by_seller = group(lambda r: r["seller"])
    top_sellers = sorted(
        [{"seller": k, "units": len(v), "revenue": round(sum(x["price"] for x in v), 2)} for k, v in by_seller.items()],
        key=lambda x: x["units"], reverse=True)[:8]

    def share(keyfn):
        g = group(keyfn)
        total = len(rows)
        return [{"name": k, "units": len(v), "share": round(100 * len(v) / total, 1)} for k, v in
                sorted(g.items(), key=lambda kv: len(kv[1]), reverse=True)]

    return {"kpis": kpis, "trend": trend, "top_products": top_products,
            "top_sellers": top_sellers, "brands": share(lambda r: r["brand"])[:8],
            "conditions": share(lambda r: r["condition"]), "formats": share(lambda r: r["format"])}


def category_tree_with_counts():
    counts = {}
    cutoff = (TODAY - timedelta(days=WINDOW_DAYS)).isoformat()
    for it in SRC.items():
        if it["sold_date"] >= cutoff:
            counts[it["category_id"]] = counts.get(it["category_id"], 0) + 1

    def walk(nodes):
        out = []
        for n in nodes:
            node = dict(n)
            node["sold_90d"] = counts.get(n["id"], 0)
            if "children" in n:
                node["children"] = walk(n["children"])
                node["sold_90d"] += sum(c.get("sold_90d", 0) for c in node["children"])
            out.append(node)
        return out
    return walk(SRC.categories())


class Handler(BaseHTTPRequestHandler):
    def log_message(self, *a):  # quiet
        pass

    def _safe_write(self, body):
        try:
            self.wfile.write(body)
        except (ConnectionError, BrokenPipeError):
            pass  # browser refreshed/navigated away mid-response — harmless

    def _json(self, obj, status=200):
        body = json.dumps(obj).encode()
        self.send_response(status)
        self.send_header("Content-Type", "application/json")
        self.send_header("Access-Control-Allow-Origin", "*")  # allow the eBay-page bookmarklet to POST
        self.send_header("Content-Length", str(len(body)))
        self.end_headers()
        self._safe_write(body)

    def _csv(self, rows):
        buf = io.StringIO()
        w = csv.writer(buf)
        w.writerow(["id", "sold_date", "title", "mpn", "brand", "model", "category",
                    "price_usd", "shipping_usd", "condition", "format", "seller",
                    "seller_rating", "location"])
        for r in rows:
            w.writerow([r["id"], r["sold_date"], r["title"], r["mpn"], r["brand"], r["model"],
                        r["category"], r["price"], r["shipping"], r["condition"], r["format"],
                        r["seller"], r["seller_rating"], r["location"]])
        body = buf.getvalue().encode()
        self.send_response(200)
        self.send_header("Content-Type", "text/csv")
        self.send_header("Content-Disposition", 'attachment; filename="sold_items.csv"')
        self.send_header("Content-Length", str(len(body)))
        self.end_headers()
        self._safe_write(body)

    def _static(self, path):
        if path in ("/", ""):
            path = "/index.html"
        f = STATIC / path.lstrip("/")
        if not f.is_file() or ".." in path:
            self.send_response(404); self.end_headers(); return
        ctype = {".html": "text/html", ".js": "text/javascript", ".css": "text/css",
                 ".svg": "image/svg+xml", ".png": "image/png"}.get(f.suffix, "application/octet-stream")
        body = f.read_bytes()
        self.send_response(200)
        self.send_header("Content-Type", ctype)
        self.send_header("Content-Length", str(len(body)))
        self.end_headers()
        self._safe_write(body)

    def do_DELETE(self):
        u = urlparse(self.path)
        m = re.fullmatch(r"/api/reports/([\w-]+)", u.path)
        if m and get_report(m.group(1)):
            (REPORTS / f"{m.group(1)}.json").unlink()
            return self._json({"status": "deleted", "id": m.group(1)})
        self.send_response(404)
        self.send_header("Access-Control-Allow-Origin", "*")
        self.end_headers()

    def do_OPTIONS(self):
        self.send_response(204)
        self.send_header("Access-Control-Allow-Origin", "*")
        self.send_header("Access-Control-Allow-Methods", "POST, GET, OPTIONS")
        self.send_header("Access-Control-Allow-Headers", "Content-Type")
        self.end_headers()

    def do_POST(self):
        """Ingest sold items extracted by the bookmarklet running on eBay's own page."""
        u = urlparse(self.path)
        if u.path == "/api/ingest":
            length = int(self.headers.get("Content-Length", 0) or 0)
            try:
                payload = json.loads(self.rfile.read(length) or b"{}")
            except Exception:
                return self._json({"status": "error", "detail": "invalid JSON"}, 400)
            obj, code = handle_ingest(payload)
            return self._json(obj, code)
        if u.path == "/api/reports/save":
            length = int(self.headers.get("Content-Length", 0) or 0)
            try:
                payload = json.loads(self.rfile.read(length) or b"{}")
            except Exception:
                return self._json({"status": "error", "detail": "invalid JSON"}, 400)
            obj, code = save_report(payload)
            return self._json(obj, code)
        self.send_response(404)
        self.send_header("Access-Control-Allow-Origin", "*")
        self.end_headers()

    def do_GET(self):
        u = urlparse(self.path)
        qs = parse_qs(u.query)
        if u.path == "/api/health":
            return self._json({"status": "ok", "mode": MODE,
                               "source": "LiveScraperSource (eBay public sold-search pages, anonymous, no login)",
                               "items_cached": len(SRC.items()), "as_of": TODAY.isoformat(),
                               "note": "Only real eBay sold data is served. First search/category click scrapes eBay live (~10-40s)."})
        if u.path == "/api/refresh":
            info = maybe_refresh_live(qs)
            return self._json({"mode": MODE, "refresh": info})
        if u.path == "/api/categories":
            return self._json({"tree": category_tree_with_counts(), "window_days": WINDOW_DAYS})
        if u.path == "/api/sold":
            live_info = maybe_refresh_live(qs)
            f = parse_filters(qs)
            rows = filter_items(f)
            total = len(rows)
            page, per = f["page"], f["per_page"]
            q_raw = (qs.get("q", [""])[0] or "").strip()
            cached = read_cache(q_raw, f["category_id"])
            ebay_total = cached.get("total_results") if cached else None
            return self._json({"total": total, "page": page, "per_page": per,
                               "items": rows[(page - 1) * per: page * per],
                               "ebay_total": ebay_total, "live": live_info})
        if u.path == "/api/reports":
            return self._json({"reports": list_reports()})
        m = re.fullmatch(r"/api/reports/([\w-]+)", u.path)
        if m:
            rep = get_report(m.group(1))
            return self._json(rep) if rep else self._json({"error": "not found"}, 404)
        if u.path == "/api/analytics":
            maybe_refresh_live(qs)
            return self._json(analytics(filter_items(parse_filters(qs))))
        if u.path == "/api/export.csv":
            maybe_refresh_live(qs)
            return self._csv(filter_items(parse_filters(qs)))
        if u.path.startswith("/api/"):
            self.send_response(404); self.end_headers(); return
        return self._static(u.path)


if __name__ == "__main__":
    port = 8000
    srv = ThreadingHTTPServer(("0.0.0.0", port), Handler)
    print(f"eBay Sold-Items Research running at http://0.0.0.0:{port}")
    srv.serve_forever()