docparser.py

"""
docparser.py
------------
Terminal-menu SDK documentation extractor.

Pipeline:  fetch → extract → emit

Receipts are stored in receipts.json (next to this script by default) and are
never hard-coded here.  Edit that file — or use the interactive commands — to
add, update, or remove receipt profiles without touching this source.

Dependencies:
    Managed with uv (pyproject.toml / uv.lock):
      uv sync                              # create .venv, install dependencies
      uv run playwright install chromium   # only needed for js_render receipts

Usage:
    uv run docparser.py [--receipts PATH] [--output-dir PATH] [--no-color]

Flags:
    --receipts PATH     Path to receipts JSON file (default: ./receipts.json)
    --output-dir PATH   Directory for emitted .md files  (default: ./doc_output)
    --no-color          Disable ANSI colour output

Receipt schema (receipts.json → receipts.<key>):
    name                Human-readable label
    language            Language tag injected into output header and code blocks
    urls                Ordered list of URLs to fetch and concatenate
    selectors           CSS selectors tried per page — first match wins
    strip_tags          CSS selectors for elements to remove before conversion
    section             Restrict to content after this H2 text (null = whole page)
    js_render           bool — use Playwright (headless Chromium) instead of requests
    markdown_passthrough  bool — treat extracted <pre> text as clean markdown directly
    notes               Free-text notes about the site / selector quirks
    last_fetched        ISO date of last successful parse (managed automatically)
    last_output         Filename of last emitted .md file  (managed automatically)
"""

from __future__ import annotations

import argparse
import json
import os
import re
import sys
from datetime import date, datetime
from functools import partial
from pathlib import Path
from typing import Callable

import requests
from bs4 import BeautifulSoup, Tag
import markdownify


# ---------------------------------------------------------------------------
# ANSI colour helpers
# ---------------------------------------------------------------------------

_USE_COLOR = sys.stdout.isatty()


def _c(code: str, text: str) -> str:
    """Wrap *text* in an ANSI escape if colour is enabled."""
    if not _USE_COLOR:
        return text
    return f"\033[{code}m{text}\033[0m"


def info(msg: str)    -> None: print(_c("36", f"  {msg}"))        # cyan
def ok(msg: str)      -> None: print(_c("32", f"  ✓ {msg}"))      # green
def warn(msg: str)    -> None: print(_c("33", f"  ⚠ {msg}"))      # yellow
def err(msg: str)     -> None: print(_c("31", f"  ✗ {msg}"))      # red
def header(msg: str)  -> None: print(_c("1;34", f"\n{msg}"))      # bold blue
def dim(msg: str)     -> None: print(_c("2", f"    {msg}"))       # dim


# ---------------------------------------------------------------------------
# Receipt registry  (load / save / validate)
# ---------------------------------------------------------------------------

#: Required keys every receipt must have.
_REQUIRED_KEYS: set[str] = {"name", "language", "urls", "selectors"}

#: Keys that are managed automatically and should always be present after normalisation.
_AUTO_KEYS: dict[str, object] = {
    "strip_tags":           [],
    "section":              None,
    "js_render":            False,
    "markdown_passthrough": False,
    "notes":                "",
    "last_fetched":         None,
    "last_output":          None,
}


def _normalise_receipt(raw: dict) -> dict:
    """Fill in optional keys with their defaults so callers can always rely on them."""
    receipt = dict(raw)
    for key, default in _AUTO_KEYS.items():
        receipt.setdefault(key, default)
    return receipt


def _validate_receipt(key: str, receipt: dict) -> list[str]:
    """Return a list of validation error strings (empty = valid)."""
    errors: list[str] = []
    for k in _REQUIRED_KEYS:
        if k not in receipt:
            errors.append(f"missing required field '{k}'")
    if "urls" in receipt and not isinstance(receipt["urls"], list):
        errors.append("'urls' must be a list")
    if "selectors" in receipt and not isinstance(receipt["selectors"], list):
        errors.append("'selectors' must be a list")
    return errors


class ReceiptRegistry:
    """
    Thin wrapper around receipts.json.

    Keeps an in-memory dict that is read from / written to disk on demand.
    All mutations go through this class so the file stays in sync.
    """

    def __init__(self, path: Path) -> None:
        self.path = path
        self._data: dict[str, dict] = {}
        self._load()

    # ------------------------------------------------------------------
    # I/O
    # ------------------------------------------------------------------

    def _load(self) -> None:
        """Read receipts.json from disk, creating an empty registry if absent."""
        if not self.path.exists():
            warn(f"Receipts file not found at {self.path} — starting with empty registry.")
            self._data = {}
            return
        try:
            raw = json.loads(self.path.read_text(encoding="utf-8"))
            receipts_block = raw.get("receipts", raw)   # tolerate bare dict
            self._data = {k: _normalise_receipt(v) for k, v in receipts_block.items()}
            ok(f"Loaded {len(self._data)} receipt(s) from {self.path}")
        except json.JSONDecodeError as exc:
            err(f"Could not parse {self.path}: {exc}")
            self._data = {}

    def save(self) -> None:
        """Persist in-memory registry to disk, preserving schema_version and comment."""
        self.path.parent.mkdir(parents=True, exist_ok=True)
        payload = {
            "_comment":        "docparser receipt registry — edit this file or use the CLI",
            "_schema_version": "2",
            "receipts":        self._data,
        }
        self.path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8")
        ok(f"Saved {len(self._data)} receipt(s) to {self.path}")

    def reload(self) -> None:
        """Re-read from disk (useful after an external editor session)."""
        self._load()

    # ------------------------------------------------------------------
    # Queries
    # ------------------------------------------------------------------

    def keys(self) -> list[str]:
        return sorted(self._data.keys())

    def get(self, key: str) -> dict | None:
        return self._data.get(key)

    def all(self) -> dict[str, dict]:
        return dict(self._data)

    def __contains__(self, key: str) -> bool:
        return key in self._data

    # ------------------------------------------------------------------
    # Mutations
    # ------------------------------------------------------------------

    def upsert(self, key: str, receipt: dict, *, save: bool = True) -> list[str]:
        """
        Insert or replace a receipt.  Validates before writing.
        Returns a list of validation errors (empty = success).
        """
        errors = _validate_receipt(key, receipt)
        if errors:
            return errors
        self._data[key] = _normalise_receipt(receipt)
        if save:
            self.save()
        return []

    def delete(self, key: str, *, save: bool = True) -> bool:
        """Remove a receipt by key.  Returns True if it existed."""
        existed = key in self._data
        if existed:
            del self._data[key]
            if save:
                self.save()
        return existed

    def update_fields(self, key: str, fields: dict, *, save: bool = True) -> bool:
        """Patch specific fields of an existing receipt.  Returns False if key missing."""
        if key not in self._data:
            return False
        self._data[key].update(fields)
        if save:
            self.save()
        return True

    def mark_fetched(self, key: str, output_filename: str) -> None:
        """Record a successful parse — updates last_fetched and last_output."""
        self.update_fields(key, {
            "last_fetched": date.today().isoformat(),
            "last_output":  output_filename,
        })


# ---------------------------------------------------------------------------
# Fetch layer
# ---------------------------------------------------------------------------

_REQUEST_HEADERS = {
    "User-Agent": (
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
        "AppleWebKit/537.36 (KHTML, like Gecko) "
        "Chrome/120.0.0.0 Safari/537.36"
    )
}


def fetch_static(url: str) -> BeautifulSoup | None:
    """Fetch a page with requests and return a BeautifulSoup DOM, or None on error."""
    try:
        response = requests.get(url, headers=_REQUEST_HEADERS, timeout=15)
        response.raise_for_status()
        return BeautifulSoup(response.text, "html.parser")
    except requests.RequestException as exc:
        err(f"Fetch error: {exc}")
        return None


def fetch_js(url: str) -> BeautifulSoup | None:
    """
    Fetch a JS-rendered page using headless Chromium via Playwright.

    Strategy (in order):
      1. Click the "Copy as Markdown" button if present — cleanest output.
      2. Fall back to reading .markdown-body inner text from the DOM.

    Returns a minimal BeautifulSoup wrapping the content in a <pre> tag so
    the downstream extract_content() can recover it via markdown_passthrough.
    """
    try:
        from playwright.sync_api import sync_playwright, TimeoutError as PWTimeout
    except ImportError:
        err("playwright not installed.  Run: uv sync && uv run playwright install chromium")
        return None

    try:
        with sync_playwright() as pw:
            browser = pw.chromium.launch(headless=True)
            context = browser.new_context(permissions=["clipboard-read", "clipboard-write"])
            page    = context.new_page()

            info("playwright: navigating …")
            try:
                page.goto(url, wait_until="networkidle", timeout=45_000)
                info("playwright: networkidle reached")
            except PWTimeout:
                warn("playwright: networkidle timed out — continuing with current DOM")

            # ---- Strategy 1: Copy-as-Markdown button ----
            try:
                page.wait_for_selector("button:has-text('Markdown')", timeout=12_000)
                info("playwright: clicking 'Copy as Markdown' button …")
                page.click("button:has-text('Markdown')")
                page.wait_for_timeout(800)
                content = page.evaluate("navigator.clipboard.readText()")
                info(f"playwright: clipboard yielded {len(content):,} chars")
                context.close()
                browser.close()
                return BeautifulSoup(f"<div><pre>{content}</pre></div>", "html.parser")
            except PWTimeout:
                warn("playwright: no 'Copy as Markdown' button found — falling back to DOM")

            # ---- Strategy 2: .markdown-body innerText ----
            content = page.evaluate("document.querySelector('.markdown-body')?.innerText ?? ''")
            info(f"playwright: DOM fallback yielded {len(content):,} chars")
            context.close()
            browser.close()
            return BeautifulSoup(f"<div><pre>{content}</pre></div>", "html.parser")

    except Exception as exc:  # noqa: BLE001
        err(f"playwright error: {exc}")
        return None


def fetch(url: str, js_render: bool = False) -> BeautifulSoup | None:
    """Dispatch to the correct fetcher based on the js_render flag."""
    if js_render:
        info(f"js-render  → {url}")
        return fetch_js(url)
    info(f"fetching   → {url}")
    return fetch_static(url)


# ---------------------------------------------------------------------------
# Extract layer
# ---------------------------------------------------------------------------

def extract_content(soup: BeautifulSoup, receipt: dict) -> str:
    """
    Locate the best content block, strip noise, and return clean markdown.

    Steps:
      1. If markdown_passthrough is set, treat the first <pre> as raw markdown.
      2. Walk the selector list — first CSS selector that matches wins.
      3. Remove strip_tags elements in place.
      4. Optionally restrict to a named H2 section.
      5. Convert HTML → markdown via markdownify.
      6. Collapse excessive blank lines.
    """
    # ---- Passthrough: content is already markdown (e.g. from Playwright clipboard) ----
    if receipt.get("markdown_passthrough"):
        pre = soup.find("pre")
        return pre.get_text() if pre else soup.get_text()

    # ---- Selector walk ----
    content_block: Tag | None = None
    for selector in receipt["selectors"]:
        found = soup.select_one(selector)
        if found:
            content_block = found
            break

    if content_block is None:
        return "_No content block matched any selector for this page._\n"

    # ---- Strip noise ----
    for selector in receipt.get("strip_tags", []):
        for element in content_block.select(selector):
            element.decompose()

    # ---- Optional H2 section filter ----
    section_filter: str | None = receipt.get("section")
    if section_filter:
        capturing = False
        kept: list[str] = []
        for tag in content_block.find_all(True, recursive=False):
            if tag.name == "h2" and section_filter.lower() in tag.get_text().lower():
                capturing = True
                continue
            if tag.name == "h2" and capturing:
                break
            if capturing:
                kept.append(str(tag))
        html_fragment = "\n".join(kept) if kept else str(content_block)
    else:
        html_fragment = str(content_block)

    # ---- HTML → Markdown ----
    raw_md = markdownify.markdownify(
        html_fragment,
        heading_style="ATX",
        bullets="-",
        code_language=receipt["language"],
    )
    return re.sub(r"\n{3,}", "\n\n", raw_md).strip()


# ---------------------------------------------------------------------------
# Emit layer
# ---------------------------------------------------------------------------

def emit_markdown(
    receipt: dict,
    sections: list[tuple[str, str]],
    output_dir: Path,
) -> Path:
    """
    Write a timestamped .md file to output_dir.

    Each (url, content) pair in sections becomes a <!-- SOURCE: url --> block.
    Returns the path of the written file.
    """
    output_dir.mkdir(parents=True, exist_ok=True)

    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
    safe_name = re.sub(r"[^a-z0-9]+", "_", receipt["name"].lower()).strip("_")
    filepath  = output_dir / f"{safe_name}_{timestamp}.md"

    lines: list[str] = [
        f"# {receipt['name']}",
        "",
        f"- **Language:** `{receipt['language']}`",
        f"- **Fetched:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
        f"- **Sources ({len(sections)}):**",
    ]
    for url, _ in sections:
        lines.append(f"  - {url}")

    notes = receipt.get("notes", "").strip()
    if notes:
        lines += ["", f"> {notes}"]

    lines += ["", "---", ""]

    for url, content in sections:
        lines.append(f"<!-- SOURCE: {url} -->")
        lines.append("")
        lines.append(content)
        lines.append("")
        lines.append("---")
        lines.append("")

    filepath.write_text("\n".join(lines), encoding="utf-8")
    return filepath


# ---------------------------------------------------------------------------
# Core parse action
# ---------------------------------------------------------------------------

def parse_receipt(
    key: str,
    registry: ReceiptRegistry,
    output_dir: Path,
    *,
    dry_run: bool = False,
) -> None:
    """
    Fetch all URLs in a receipt, extract content, emit a .md file,
    and update the registry with last_fetched / last_output.

    dry_run=True prints what would be done without fetching anything.
    """
    receipt = registry.get(key)
    if receipt is None:
        err(f"Receipt '{key}' not found.  Run -list to see available receipts.")
        return

    urls      = receipt.get("urls", [])
    js_render = receipt.get("js_render", False)

    if not urls:
        err(f"Receipt '{key}' has no URLs defined.")
        return

    header(f"Parsing: {receipt['name']}")
    info(f"URLs: {len(urls)}  |  JS-render: {js_render}  |  Passthrough: {receipt.get('markdown_passthrough', False)}")

    if dry_run:
        warn("Dry-run mode — no network requests made.")
        for url in urls:
            dim(url)
        return

    sections: list[tuple[str, str]] = []
    for url in urls:
        soup = fetch(url, js_render=js_render)
        if soup is None:
            warn(f"Skipping {url} — fetch failed")
            continue
        content = extract_content(soup, receipt)
        sections.append((url, content))
        info(f"extracted {len(content):,} chars")

    if not sections:
        err("No content extracted — check receipt URLs and selectors.")
        return

    filepath = emit_markdown(receipt, sections, output_dir)
    registry.mark_fetched(key, filepath.name)
    ok(f"Output: {filepath}")


# ---------------------------------------------------------------------------
# Probe — selector discovery for an unknown page
# ---------------------------------------------------------------------------

_PROBE_SELECTORS = [
    "#content", ".prose", "article", "main", "[role='main']",
    ".content", ".docs-content", ".markdown-body",
    ".md-content", ".page-content", "[class*='mdx']",
    "[class*='article']", "[class*='doc-content']",
]


def _analyse_probe_soup(soup: BeautifulSoup, *, no_match_hint: str) -> dict:
    """
    Shared probe analysis: walk _PROBE_SELECTORS against a parsed DOM and
    report hits, H2 structure, and sample links.  Used by -probe and -probe-js.
    """
    hits: list[str]   = []
    misses: list[str] = []
    first_match: Tag | None = None

    for sel in _PROBE_SELECTORS:
        el = soup.select_one(sel)
        if el:
            classes = " ".join(el.get("class", []))[:60]
            print(f"    {_c('32','HIT')}  {sel:30s} → <{el.name}> {classes}")
            hits.append(sel)
            if first_match is None:
                first_match = el
        else:
            print(f"    {_c('2','miss')} {sel}")

    h2s: list[str] = []
    links: list[str] = []
    if first_match is not None:
        h2s = [h.get_text().strip()[:70] for h in first_match.find_all("h2")][:15]
        if h2s:
            info(f"H2 sections ({len(h2s)}):")
            for h in h2s:
                dim(h)
        links = [a["href"] for a in first_match.find_all("a", href=True)
                 if a["href"].startswith("http")][:10]
        if links:
            info("Internal links (sample):")
            for lnk in links:
                dim(lnk)
    else:
        warn(f"No selector matched — {no_match_hint}")

    return {
        "hits":          hits,
        "misses":        misses,
        "h2s":           h2s,
        "links":         links,
        "best_selector": hits[0] if hits else None,
    }


def probe_url(url: str) -> dict | None:
    """
    Fetch a page and report which CSS selectors match and what H2 structure exists.
    Returns a dict of probe findings, or None if the page could not be fetched.
    Findings can be used directly to draft a new receipt.
    """
    header(f"Probing: {url}")
    soup = fetch_static(url)
    if soup is None:
        return None

    findings = _analyse_probe_soup(
        soup, no_match_hint="page may be JS-rendered; try -probe-js."
    )
    findings["url"]         = url
    findings["js"]          = False
    findings["copy_button"] = None
    return findings


def probe_url_js(url: str) -> dict | None:
    """
    Probe a JS-rendered page with headless Chromium.

    Loads the URL in Playwright, waits for the network to settle, then runs
    the same selector report as probe_url() over the *rendered* DOM.  Also
    detects a 'Copy as Markdown' button so a saved receipt can pre-fill
    markdown_passthrough automatically.
    """
    try:
        from playwright.sync_api import sync_playwright, TimeoutError as PWTimeout
    except ImportError:
        err("playwright not installed.  Run: uv sync && uv run playwright install chromium")
        return None

    header(f"Probing (JS): {url}")

    copy_button = False
    try:
        with sync_playwright() as pw:
            browser = pw.chromium.launch(headless=True)
            context = browser.new_context()
            page    = context.new_page()

            info("playwright: navigating …")
            try:
                page.goto(url, wait_until="networkidle", timeout=45_000)
                info("playwright: networkidle reached")
            except PWTimeout:
                warn("playwright: networkidle timed out — continuing with current DOM")

            page.wait_for_timeout(1_000)   # let late hydration settle

            copy_button = page.locator("button:has-text('Markdown')").count() > 0
            if copy_button:
                ok("'Copy as Markdown' button detected — saved receipt will use markdown_passthrough.")

            html = page.content()
            info(f"playwright: captured {len(html):,} chars of rendered HTML")

            context.close()
            browser.close()

    except Exception as exc:  # noqa: BLE001
        err(f"playwright error: {exc}")
        return None

    soup     = BeautifulSoup(html, "html.parser")
    findings = _analyse_probe_soup(
        soup, no_match_hint="even after JS render — inspect the page manually."
    )
    findings["url"]         = url
    findings["js"]          = True
    findings["copy_button"] = copy_button
    return findings


# ---------------------------------------------------------------------------
# Interactive command handlers
# ---------------------------------------------------------------------------

def cmd_list(registry: ReceiptRegistry, **_) -> None:
    """List all registered receipts."""
    data = registry.all()
    if not data:
        warn("No receipts registered yet.  Use -add to create one.")
        return
    header(f"Receipts ({len(data)}):")
    for key, rec in sorted(data.items()):
        urls        = rec.get("urls", [])
        js_flag     = _c("33", " [js]") if rec.get("js_render") else ""
        last        = rec.get("last_fetched") or "never"
        url_display = urls[0] if urls else "(no URLs)"
        extra       = f" +{len(urls)-1} more" if len(urls) > 1 else ""
        print(
            f"  {_c('1', key):30s}  {rec['language']:12s}{js_flag}\n"
            f"    {_c('2', url_display + extra)}\n"
            f"    last fetched: {last}"
        )


def cmd_parse(registry: ReceiptRegistry, output_dir: Path, **_) -> None:
    """Interactively select and parse a receipt."""
    cmd_list(registry)
    key = input("\n  Receipt key to parse: >>> ").strip()
    if not key:
        warn("Aborted.")
        return
    dry = input("  Dry run? [y/N]: >>> ").strip().lower() == "y"
    parse_receipt(key, registry, output_dir, dry_run=dry)


def cmd_parse_key(key: str, registry: ReceiptRegistry, output_dir: Path, **_) -> None:
    """Parse a specific receipt directly (used by shortcut commands)."""
    parse_receipt(key, registry, output_dir)


def cmd_url(registry: ReceiptRegistry, output_dir: Path, **_) -> None:
    """Parse an arbitrary URL using an existing receipt as a selector/language template."""
    cmd_list(registry)
    key = input("\n  Receipt key to use as template: >>> ").strip()
    if key not in registry:
        err(f"Unknown key: '{key}'")
        return
    url = input("  URL to fetch: >>> ").strip()
    if not url.startswith("http"):
        err("URL must start with http:// or https://")
        return
    # Build a throwaway receipt — do not persist it
    receipt = dict(registry.get(key))
    receipt["name"] = f"Custom — {url[:60]}"
    receipt["urls"] = [url]
    soup = fetch(url, js_render=False)
    if soup is None:
        return
    content  = extract_content(soup, receipt)
    filepath = emit_markdown(receipt, [(url, content)], output_dir)
    ok(f"Output: {filepath}")


def _save_probe_receipt(registry: ReceiptRegistry, url: str, findings: dict) -> None:
    """Interactively turn probe findings into a saved receipt."""
    save = input("\n  Save as new receipt? [y/N]: >>> ").strip().lower()
    if save != "y":
        return

    js          = findings.get("js", False)
    passthrough = bool(findings.get("copy_button"))

    # Pre-fill from probe findings, let user refine
    suggested_key = re.sub(r"[^a-z0-9]+", "-", url.split("//")[-1].split("/")[0]).strip("-")
    key      = input(f"  Receipt key [{suggested_key}]: >>> ").strip() or suggested_key
    name     = input(f"  Display name: >>> ").strip()
    language = input("  Language (python/typescript/etc): >>> ").strip() or "text"
    best     = findings["best_selector"]
    info(f"  Using best selector: {best}")
    use_best = input("  Accept this selector? [Y/n]: >>> ").strip().lower()
    selectors = [best] if use_best != "n" else []
    if not selectors:
        raw = input("  Selector(s) comma-separated: >>> ").strip()
        selectors = [s.strip() for s in raw.split(",") if s.strip()]

    extra_urls: list[str] = []
    info("Add more URLs for this receipt (blank line to finish):")
    extra_urls.append(url)
    while True:
        u = input("    url: >>> ").strip()
        if not u:
            break
        extra_urls.append(u)

    receipt = {
        "name":                name,
        "language":            language,
        "urls":                extra_urls,
        "selectors":           selectors,
        "strip_tags":          ["nav", "footer", "header", "script", "style"],
        "section":             None,
        "js_render":           js,
        "markdown_passthrough": passthrough,
        "notes":               f"Created via {'-probe-js' if js else '-probe'} from {url}",
        "last_fetched":        None,
        "last_output":         None,
    }
    errors = registry.upsert(key, receipt)
    if errors:
        err(f"Validation failed: {'; '.join(errors)}")
    else:
        ok(f"Receipt '{key}' saved.")


def cmd_probe(registry: ReceiptRegistry, **_) -> None:
    """Probe a URL to discover its selector structure, then optionally save as a receipt."""
    url = input("\n  URL to probe: >>> ").strip()
    if not url.startswith("http"):
        err("URL must start with http:// or https://")
        return
    findings = probe_url(url)
    if findings is None or not findings["best_selector"]:
        warn("Probe inconclusive.  The page may need js_render — try -probe-js.")
        return
    _save_probe_receipt(registry, url, findings)


def cmd_probe_js(registry: ReceiptRegistry, **_) -> None:
    """Probe a JS-rendered URL via headless Chromium, then optionally save as a receipt."""
    url = input("\n  URL to probe (JS render): >>> ").strip()
    if not url.startswith("http"):
        err("URL must start with http:// or https://")
        return
    findings = probe_url_js(url)
    if findings is None or not findings["best_selector"]:
        warn("JS probe inconclusive — inspect the page manually.")
        return
    _save_probe_receipt(registry, url, findings)


def cmd_add(registry: ReceiptRegistry, **_) -> None:
    """Interactively build and register a new receipt from scratch."""
    header("Add receipt")
    key = input("  Key (no spaces, e.g. mylib-python): >>> ").strip()
    if not key:
        warn("Aborted.")
        return
    if key in registry:
        err(f"Key '{key}' already exists.  Use -edit to modify it.")
        return

    name     = input("  Display name: >>> ").strip()
    language = input("  Language tag (python/typescript/etc): >>> ").strip() or "text"

    info("URLs (one per line, blank to finish):")
    urls: list[str] = []
    while True:
        u = input("    url: >>> ").strip()
        if not u:
            break
        urls.append(u)
    if not urls:
        warn("No URLs entered — aborted.")
        return

    info("CSS selectors (one per line, blank to finish — leave empty for defaults):")
    selectors: list[str] = []
    while True:
        s = input("    selector: >>> ").strip()
        if not s:
            break
        selectors.append(s)
    if not selectors:
        selectors = [".prose", "article", "main", "body"]
        info(f"Using default selectors: {selectors}")

    js_render = input("  JS-rendered? Playwright required [y/N]: >>> ").strip().lower() == "y"
    passthrough = False
    if js_render:
        passthrough = input("  Markdown passthrough (clipboard copy)? [y/N]: >>> ").strip().lower() == "y"

    notes = input("  Notes (optional): >>> ").strip()

    receipt = {
        "name":                name,
        "language":            language,
        "urls":                urls,
        "selectors":           selectors,
        "strip_tags":          ["nav", "footer", "header", "script", "style"],
        "section":             None,
        "js_render":           js_render,
        "markdown_passthrough": passthrough,
        "notes":               notes,
        "last_fetched":        None,
        "last_output":         None,
    }
    errors = registry.upsert(key, receipt)
    if errors:
        err(f"Validation failed: {'; '.join(errors)}")
    else:
        ok(f"Receipt '{key}' added and saved to {registry.path}")


def cmd_edit(registry: ReceiptRegistry, **_) -> None:
    """Edit individual fields of an existing receipt."""
    cmd_list(registry)
    key = input("\n  Receipt key to edit: >>> ").strip()
    if key not in registry:
        err(f"Key '{key}' not found.")
        return

    receipt = dict(registry.get(key))
    EDITABLE = ["name", "language", "urls", "selectors", "strip_tags",
                "section", "js_render", "markdown_passthrough", "notes"]

    info("Editable fields:")
    for i, field in enumerate(EDITABLE, 1):
        print(f"    {i:2}. {field:25s} = {json.dumps(receipt.get(field))}")

    choice = input("\n  Field number to edit (or blank to cancel): >>> ").strip()
    if not choice.isdigit() or not (1 <= int(choice) <= len(EDITABLE)):
        warn("Aborted.")
        return

    field  = EDITABLE[int(choice) - 1]
    current = receipt.get(field)
    info(f"Current value: {json.dumps(current)}")
    raw = input("  New value (JSON — use [] for list, true/false for bool): >>> ").strip()

    try:
        new_value = json.loads(raw)
    except json.JSONDecodeError:
        # Treat as plain string if JSON parse fails
        new_value = raw

    registry.update_fields(key, {field: new_value})
    ok(f"Updated '{field}' on receipt '{key}'.")


def cmd_delete(registry: ReceiptRegistry, **_) -> None:
    """Delete a receipt after confirmation."""
    cmd_list(registry)
    key = input("\n  Receipt key to delete: >>> ").strip()
    if key not in registry:
        err(f"Key '{key}' not found.")
        return
    confirm = input(f"  Really delete '{key}'? [y/N]: >>> ").strip().lower()
    if confirm == "y":
        registry.delete(key)
        ok(f"Receipt '{key}' deleted.")
    else:
        warn("Aborted.")


def cmd_show(registry: ReceiptRegistry, **_) -> None:
    """Pretty-print a single receipt's full JSON."""
    cmd_list(registry)
    key = input("\n  Receipt key to show: >>> ").strip()
    receipt = registry.get(key)
    if receipt is None:
        err(f"Key '{key}' not found.")
        return
    print(json.dumps({key: receipt}, indent=2, ensure_ascii=False))


def cmd_export(registry: ReceiptRegistry, **_) -> None:
    """Export receipts to an arbitrary file path (for sharing)."""
    dest = input("\n  Export path (e.g. ~/my-receipts.json): >>> ").strip()
    dest = os.path.expanduser(dest)
    try:
        payload = {
            "_comment":        "docparser receipt export",
            "_schema_version": "2",
            "receipts":        registry.all(),
        }
        Path(dest).write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8")
        ok(f"Exported {len(registry.keys())} receipts to {dest}")
    except OSError as exc:
        err(f"Export failed: {exc}")


def cmd_import(registry: ReceiptRegistry, **_) -> None:
    """Import receipts from an external file, merging into the current registry."""
    src = input("\n  Import from path: >>> ").strip()
    src = os.path.expanduser(src)
    try:
        raw    = json.loads(Path(src).read_text(encoding="utf-8"))
        items  = raw.get("receipts", raw)
        added  = 0
        skipped = 0
        for key, receipt in items.items():
            if key.startswith("_"):
                continue
            if key in registry:
                overwrite = input(f"  '{key}' already exists — overwrite? [y/N]: >>> ").strip().lower()
                if overwrite != "y":
                    skipped += 1
                    continue
            errors = registry.upsert(key, receipt, save=False)
            if errors:
                warn(f"Skipping '{key}': {'; '.join(errors)}")
                skipped += 1
            else:
                added += 1
        registry.save()
        ok(f"Import complete — {added} added, {skipped} skipped.")
    except (OSError, json.JSONDecodeError) as exc:
        err(f"Import failed: {exc}")


def cmd_reload(registry: ReceiptRegistry, **_) -> None:
    """Reload receipts.json from disk (pick up external edits)."""
    registry.reload()


def cmd_out(output_dir: Path, **_) -> None:
    """Show the output directory and list .md files already emitted."""
    header(f"Output directory: {output_dir}")
    if output_dir.exists():
        files = sorted(output_dir.glob("*.md"))
        if files:
            info(f"{len(files)} .md file(s):")
            for f in files:
                size = f.stat().st_size
                print(f"    {f.name}  {_c('2', f'{size:,} bytes')}")
        else:
            warn("No .md files yet.")
    else:
        warn("Directory does not exist yet — created on first parse.")


def cmd_help(**_) -> None:
    """Print command reference."""
    print("""
  ─── Receipt management ───────────────────────────────────────────
  -list           List all registered receipts (key, language, last fetched)
  -show           Pretty-print a single receipt's full JSON
  -add            Add a new receipt interactively
  -edit           Edit a single field of an existing receipt
  -delete         Delete a receipt after confirmation
  -reload         Re-read receipts.json from disk (pick up external edits)
  -export         Export all receipts to a file for sharing
  -import         Merge receipts from an external file

  ─── Parsing ──────────────────────────────────────────────────────
  -parse          Select a receipt and parse it (prompts for key)
  -parse:<key>    Parse receipt <key> directly, e.g. -parse:memvid-python
  -url            Fetch a custom URL using an existing receipt as a template

  ─── Discovery ────────────────────────────────────────────────────
  -probe          Probe a URL: find matching selectors and H2 structure
                  Optionally saves findings as a new receipt
  -probe-js       Probe a JS-rendered URL via headless Chromium
                  Same report as -probe, plus Copy-as-Markdown detection
                  Optionally saves findings as a new receipt (js_render prefilled)

  ─── Utility ──────────────────────────────────────────────────────
  -out            Show output directory and list emitted .md files
  -help           Show this help
  -exit           Quit

  ─── Tips ─────────────────────────────────────────────────────────
  Workflow for a new site:
    1. -probe   → discover selector and page structure
       (prompts to save as receipt when done)
    2. -edit    → refine URLs, strip_tags, section filter, etc.
    3. -parse   → generate the .md output

  Receipts are stored in receipts.json.  You can edit that file
  directly in any text editor, then -reload to pick up the changes.
""")


# ---------------------------------------------------------------------------
# Command registry builder
# ---------------------------------------------------------------------------

def build_commands(
    registry: ReceiptRegistry,
    output_dir: Path,
) -> dict[str, Callable]:
    """Build the full command dispatch table, including per-receipt -parse:<key> shortcuts."""
    ctx = dict(registry=registry, output_dir=output_dir)

    base: dict[str, Callable] = {
        "-list":   lambda: cmd_list(**ctx),
        "-show":   lambda: cmd_show(**ctx),
        "-add":    lambda: cmd_add(**ctx),
        "-edit":   lambda: cmd_edit(**ctx),
        "-delete": lambda: cmd_delete(**ctx),
        "-reload": lambda: cmd_reload(**ctx),
        "-export": lambda: cmd_export(**ctx),
        "-import": lambda: cmd_import(**ctx),
        "-parse":  lambda: cmd_parse(**ctx),
        "-url":    lambda: cmd_url(**ctx),
        "-probe":  lambda: cmd_probe(**ctx),
        "-probe-js": lambda: cmd_probe_js(**ctx),
        "-out":    lambda: cmd_out(output_dir=output_dir),
        "-help":   cmd_help,
    }

    # Dynamic per-receipt parse shortcuts
    for key in registry.keys():
        base[f"-parse:{key}"] = partial(parse_receipt, key, registry, output_dir)

    return base


# ---------------------------------------------------------------------------
# CLI argument parsing
# ---------------------------------------------------------------------------

def parse_args() -> argparse.Namespace:
    p = argparse.ArgumentParser(
        prog="docparser",
        description="SDK documentation extractor — fetch → extract → emit",
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    p.add_argument(
        "--receipts",
        metavar="PATH",
        default=str(Path(__file__).parent / "receipts.json"),
        help="Path to receipts JSON file (default: ./receipts.json)",
    )
    p.add_argument(
        "--output-dir",
        metavar="PATH",
        default=str(Path(__file__).parent / "doc_output"),
        help="Directory for emitted .md files (default: ./doc_output)",
    )
    p.add_argument(
        "--no-color",
        action="store_true",
        help="Disable ANSI colour output",
    )
    return p.parse_args()


# ---------------------------------------------------------------------------
# Main REPL loop
# ---------------------------------------------------------------------------

def loop(registry: ReceiptRegistry, output_dir: Path) -> None:
    print(_c("1;34", "=" * 62))
    print(_c("1;34", "  docparser — SDK documentation extractor"))
    print(_c("2",    "  receipts:   " + str(registry.path)))
    print(_c("2",    "  output dir: " + str(output_dir)))
    print(_c("1;34", "  type -help for commands · -list for receipts · -exit to quit"))
    print(_c("1;34", "=" * 62))

    while True:
        commands = build_commands(registry, output_dir)
        try:
            raw = input(_c("36", "\n$ >>> ")).strip().lower()
        except (EOFError, KeyboardInterrupt):
            print()
            ok("bye.")
            break

        if raw in ("-exit", "exit", "quit", "q"):
            ok("bye.")
            break
        elif raw in commands:
            commands[raw]()
        elif raw == "":
            pass
        else:
            err(f"Unknown command: '{raw}' — try -help")


# ---------------------------------------------------------------------------
# Entry point
# ---------------------------------------------------------------------------

def main() -> None:
    args = parse_args()

    global _USE_COLOR
    if args.no_color:
        _USE_COLOR = False

    receipts_path = Path(args.receipts).expanduser().resolve()
    output_dir    = Path(args.output_dir).expanduser().resolve()

    registry = ReceiptRegistry(receipts_path)
    loop(registry, output_dir)


if __name__ == "__main__":
    main()