test(docs): automated documentation quality checks (ENG-23)

.github/workflows/test-snippets.yml

@@ -6,7 +6,10 @@ on:
     paths:
       - 'v3/**/*.mdx'
       - '*.mdx'
+      - 'docs.json'
       - 'tests/**'
+  schedule:
+    - cron: '0 6 * * 1'
 
 # These tests create shared, named resources on one Eden AI account (custom
 # tokens, uploaded files), so two runs of the same ref must not overlap.
@@ -24,23 +27,31 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
-  execute:
-    name: Execution Tests
+  python-tests:
+    name: Python Tests
     runs-on: ubuntu-latest
+    env:
+      EDEN_AI_BASE_URL: ${{ vars.EDEN_AI_BASE_URL || 'https://staging-api.edenai.run' }}
+      EDEN_AI_SANDBOX_API_TOKEN: ${{ secrets.EDEN_AI_SANDBOX_TOKEN }}
+      EDEN_AI_PRODUCTION_API_TOKEN: ${{ secrets.EDEN_AI_PRODUCTION_TOKEN }}
     steps:
       - uses: actions/checkout@v4
       - uses: astral-sh/setup-uv@v5
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
+          python-version: '3.12'
       - name: Install dependencies
         run: uv pip install -r tests/requirements-lock.txt --system
-      - name: Run tests
+      - name: Run snippet execution tests
         run: pytest tests/test_snippets_execute.py
-        env:
-          EDEN_AI_BASE_URL: ${{ vars.EDEN_AI_BASE_URL || 'https://staging-api.edenai.run' }}
-          EDEN_AI_SANDBOX_API_TOKEN: ${{ secrets.EDEN_AI_SANDBOX_TOKEN }}
-          EDEN_AI_PRODUCTION_API_TOKEN: ${{ secrets.EDEN_AI_PRODUCTION_TOKEN }}
+      - name: Run validators
+        if: always()
+        run: |
+          pytest \
+            tests/config_validator.py \
+            tests/api_reference_validator.py \
+            tests/model_provider_validator.py \
+            tests/link_checker.py
       - name: Coverage summary
         if: always()
         run: |
@@ -50,3 +61,26 @@ jobs:
           coverage report -m >> "$GITHUB_STEP_SUMMARY"
           echo '```' >> "$GITHUB_STEP_SUMMARY"
 
+  ts-tests:
+    name: TypeScript Tests
+    runs-on: ubuntu-latest
+    env:
+      EDEN_AI_BASE_URL: ${{ vars.EDEN_AI_BASE_URL || 'https://staging-api.edenai.run' }}
+      EDEN_AI_SANDBOX_API_TOKEN: ${{ secrets.EDEN_AI_SANDBOX_TOKEN }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - uses: astral-sh/setup-uv@v5
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: latest
+      - name: Install Python deps (needed for the snippet extractor)
+        run: uv pip install -r tests/requirements-lock.txt --system
+      - name: Install TS deps
+        working-directory: tests/ts
+        run: bun install --frozen-lockfile
+      - name: Run TS snippet tests
+        working-directory: tests/ts
+        run: bun test

.gitignore

@@ -5,6 +5,9 @@ __pycache__/
 
 # Generated test modules
 tests/generated/
+tests/generated_ts/
+tests/ts/node_modules/
+
 
 # Coverage reports
 htmlcov/

index.mdx

@@ -43,4 +43,4 @@ import { SiteSchema } from "/snippets/SiteSchema.mdx";
 
 <br />
 <br /><br /><br />
-<Note>If you were a user before 2026/01/05, you still have access to the previous version: [https://old-app.edenai.run/](https://old-app.edenai.run/). We'll continue supporting the old version until the end of 2026. If you're looking for the documentation, you can find it [here](https://old-docs.edenai.co)</Note>
+<Note>If you were a user before 2026/01/05, you still have access to the previous version: [https://old-app.edenai.run/](https://old-app.edenai.run/). We'll continue supporting the old version until the end of 2026. If you're looking for the documentation, you can find it [here](https://www.edenai.co/docs/v2)</Note>

tests/README.md

@@ -112,15 +112,54 @@ This also works with `<CodeGroup>` blocks — place the comment before the `<Cod
 The comment is invisible in rendered docs. The extractor checks the 3 lines preceding each ` ```python ` fence for the marker. Skipped blocks still appear in test output (as `SKIPPED`) rather than being silently excluded, so you can track how many snippets are skipped.
 
 
+## TypeScript snippets
+
+Guides that ship TypeScript examples (`v3/integrations/openai-sdk-typescript.mdx`, `langchain.mdx`, `pi.mdx`) go through a parallel bun-native runner in `tests/ts/`.
+
+```bash
+# One-time install
+bun install --cwd tests/ts
+
+# Run all TS snippets (bunfig.toml preload script auto-invokes the Python extractor first)
+cd tests/ts && bun test
+```
+
+Same skip/fixtures mechanics as Python: `{/* skip-test */}` in the `.mdx` produces a `.skip.<ext>` filename that `bun:test` routes to `test.skip`. `tests/generated_ts/fixtures/` is populated via the shared `populate_fixtures_dir()` helper so `image.jpg`, `document.pdf`, etc. are present.
+
+## Validators
+
+Beyond snippet execution, four validators enforce doc/API consistency. All run under the same `pytest -n auto`.
+
+| File | Checks |
+|------|--------|
+| `tests/config_validator.py` | Parses fenced JSON/YAML/TOML config blocks in tool-integration guides; validates Eden AI URL hosts + endpoint prefixes; cross-checks every `` `provider/model` `` string against the live inventory |
+| `tests/api_reference_validator.py` | Fetches the 3 remote OpenAPI specs referenced in `docs.json`, asserts they're reachable and valid, cross-checks every `https://api.edenai.run/v[23]/…` URL in prose against the specs |
+| `tests/model_provider_validator.py` | Scans every `.mdx` for backticked `provider/model` references; cross-checks against `/v3/models` + `/v3/info` + probed embeddings inventory |
+| `tests/link_checker.py` | Extracts markdown links, JSX `href="…"`, `<TechArticleSchema path="…">`, and bare URLs; verifies internal targets exist and external URLs return 2xx-3xx (or a non-404/410 4xx). Also checks every `docs.json` nav path resolves to an `.mdx` file |
+
+Model/provider lookup is powered by `tests/helpers/edenai_inventory.py`, a session-cached inventory of LLM models (`/v3/models`), expert models (`/v3/info`), and verified embeddings.
+
+Unknown `provider/model` references fail the test unless they match `DOCUMENTATION_PLACEHOLDERS` (e.g. `provider/model` used as a format placeholder), match an `UNAMBIGUOUS_MIME_PREFIXES` prefix (e.g. `application/json`), or resolve after `strip_tool_alias` removes a leading `edenai/` tool alias. There is no per-file allowlist.
+
 ## CI (GitHub Actions)
 
-The workflow at `.github/workflows/test-snippets.yml` runs on PRs that touch `v3/**/*.mdx` or `tests/**`:
+The workflow at `.github/workflows/test-snippets.yml` runs on:
+- PRs that touch `v3/**/*.mdx`, root `*.mdx`, `docs.json`, or `tests/**`
+- Weekly cron (`0 6 * * 1`)
+- Manual dispatch
+
+Two jobs, both with `cancel-in-progress: false` (session cleanup runs at pytest_sessionfinish; cancelling a started run orphans account resources):
+
+1. **Python Tests**: snippet execution + all four validators.
+2. **TypeScript Tests**: installs bun + runs `bun test` in `tests/ts/`.
+
+Both consume `EDEN_AI_SANDBOX_TOKEN` and (Python job only) `EDEN_AI_PRODUCTION_TOKEN` from repository secrets. `EDEN_AI_BASE_URL` is set from the `EDEN_AI_BASE_URL` repository variable if defined (defaults to staging).
 
-1. **Execution job**: runs execution tests with `EDEN_AI_SANDBOX_TOKEN` and `EDEN_AI_PRODUCTION_TOKEN` secrets
+Python deps install from `requirements-lock.txt` for reproducibility. TS deps install from `tests/ts/bun.lock`.
 
-Installs from `requirements-lock.txt` for reproducible builds.
+## Disabling the doc-tests workflow
 
-To set up: add `EDEN_AI_SANDBOX_TOKEN` and `EDEN_AI_PRODUCTION_TOKEN` as repository secrets in GitHub.
+If the docs need to ship despite a failing test run (broken external link, upstream API drift, etc.), disable via GitHub UI: **Actions → Test Documentation Snippets → ⋯ → Disable workflow**. Mintlify's own build pipeline is independent, so the site continues to publish.
 
 ## Common Failure Patterns
 

tests/api_reference_validator.py

@@ -0,0 +1,100 @@
+import re
+from pathlib import Path
+from urllib.parse import urlparse
+
+import pytest
+import requests
+
+DOCS_ROOT = Path(__file__).resolve().parent.parent
+
+OPENAPI_SPEC_URLS = [
+    "https://api.edenai.run/v3/docs/openapi.json",
+    "https://api.edenai.run/v2/info/splitted-schema/cost_management/openapi.json",
+    "https://api.edenai.run/v2/info/splitted-schema/user/openapi.json",
+]
+
+ENDPOINT_URL_RE = re.compile(
+    r"https://api\.edenai\.run(?P<path>/v[23][^\s,'\"`)>]*)"
+)
+
+_NON_ENDPOINT_SUFFIXES = (".json", ".yaml", ".yml", ".txt", ".md")
+
+
+@pytest.fixture(scope="session")
+def openapi_specs() -> dict[str, dict]:
+    specs: dict[str, dict] = {}
+    for url in OPENAPI_SPEC_URLS:
+        resp = requests.get(url, timeout=30)
+        resp.raise_for_status()
+        specs[url] = resp.json()
+    return specs
+
+
+@pytest.fixture(scope="session")
+def openapi_paths(openapi_specs: dict[str, dict]) -> set[str]:
+    all_paths: set[str] = set()
+    for spec in openapi_specs.values():
+        servers = spec.get("servers") or [{"url": ""}]
+        server_prefix = urlparse(servers[0].get("url", "")).path.rstrip("/")
+        for spec_path in spec.get("paths", {}).keys():
+            all_paths.add((server_prefix + spec_path).rstrip("/"))
+    return all_paths
+
+
+@pytest.mark.parametrize("spec_url", OPENAPI_SPEC_URLS)
+def test_openapi_spec_reachable_and_valid(spec_url: str) -> None:
+    resp = requests.get(spec_url, timeout=30)
+    assert resp.status_code == 200, f"{spec_url} returned {resp.status_code}"
+    spec = resp.json()
+    assert isinstance(spec, dict), f"{spec_url} did not return a JSON object"
+    assert "openapi" in spec or "swagger" in spec, (
+        f"{spec_url} is not an OpenAPI/Swagger document"
+    )
+    paths = spec.get("paths")
+    assert isinstance(paths, dict) and paths, f"{spec_url} has no paths"
+
+
+def _prose_pages() -> list[str]:
+    return sorted(
+        str(p.relative_to(DOCS_ROOT))
+        for p in list(DOCS_ROOT.glob("v3/**/*.mdx")) + list(DOCS_ROOT.glob("*.mdx"))
+        if not str(p.relative_to(DOCS_ROOT)).startswith("api-reference/")
+    )
+
+
+def _is_endpoint_path(path: str) -> bool:
+    normalized = path.rstrip("/")
+    if normalized in ("/v2", "/v3"):
+        return False
+    if "..." in normalized:
+        return False
+    return not normalized.endswith(_NON_ENDPOINT_SUFFIXES)
+
+
+def _matches_openapi_path(path: str, openapi_paths: set[str]) -> bool:
+    normalized = path.split("?")[0].split("#")[0].rstrip("/")
+    if normalized in openapi_paths:
+        return True
+    for spec_path in openapi_paths:
+        if "{" not in spec_path:
+            continue
+        pattern = re.sub(r"\{[^/]+\}", r"[^/]+", spec_path.rstrip("/"))
+        if re.fullmatch(pattern, normalized):
+            return True
+    return False
+
+
+@pytest.mark.parametrize("page", _prose_pages())
+def test_prose_endpoint_urls_match_openapi(page: str, openapi_paths: set[str]) -> None:
+    content = (DOCS_ROOT / page).read_text(encoding="utf-8")
+    unknown: list[str] = []
+    for match in ENDPOINT_URL_RE.finditer(content):
+        raw_path = match.group("path").split("?")[0].split("#")[0]
+        if not _is_endpoint_path(raw_path):
+            continue
+        if _matches_openapi_path(raw_path, openapi_paths):
+            continue
+        line = content[: match.start()].count("\n") + 1
+        unknown.append(f"line {line}: {match.group(0)}")
+    if unknown:
+        pytest.fail(f"{page}:\n  " + "\n  ".join(unknown))

tests/config_validator.py

@@ -0,0 +1,143 @@
+import json
+import os
+import re
+import tomllib
+from pathlib import Path
+
+import pytest
+import yaml
+
+from tests.helpers.edenai_inventory import get_model_inventory
+from tests.helpers.model_names import (
+    BACKTICKED_MODEL_RE,
+    DOCUMENTATION_PLACEHOLDERS,
+    strip_tool_alias,
+)
+from tests.snippet_extractor import SKIP_COMMENT_RE
+
+CONFIG_GUIDES = [
+    "v3/integrations/bifrost.mdx",
+    "v3/integrations/claude-code.mdx",
+    "v3/integrations/cline.mdx",
+    "v3/integrations/codex-cli.mdx",
+    "v3/integrations/continue-dev.mdx",
+    "v3/integrations/hermes.mdx",
+    "v3/integrations/librechat.mdx",
+    "v3/integrations/n8n.mdx",
+    "v3/integrations/open-code-review.mdx",
+    "v3/integrations/open-webui.mdx",
+    "v3/integrations/openclaw.mdx",
+    "v3/integrations/opencode.mdx",
+    "v3/integrations/pi.mdx",
+]
+
+DOCS_ROOT = Path(__file__).resolve().parent.parent
+
+ALLOWED_EDEN_HOSTS = {
+    "api.edenai.run",
+    "app.edenai.run",
+    "docs.edenai.co",
+    "app-edenai.instatus.com",
+}
+
+KNOWN_API_ENDPOINT_PREFIXES = ("/v2", "/v3")
+
+FENCED_BLOCK_RE = re.compile(
+    r"^```(?P<lang>[a-zA-Z]+)(?:[ \t]+[^\n]*)?[ \t]*\n(?P<body>.*?)^\s*```",
+    re.MULTILINE | re.DOTALL,
+)
+
+EDEN_URL_RE = re.compile(
+    r"https://(?P<host>[a-z0-9][a-z0-9\-]*(?:\.[a-z0-9\-]+)+)(?P<path>/[^\s'\"`)>]*)?"
+)
+
+CONFIG_PARSERS = {
+    "json": json.loads,
+    "yaml": yaml.safe_load,
+    "yml": yaml.safe_load,
+    "toml": tomllib.loads,
+}
+
+
+def parse_fenced_config_blocks(content: str) -> list[dict]:
+    blocks = []
+    for m in FENCED_BLOCK_RE.finditer(content):
+        lang = m.group("lang").lower()
+        if lang not in CONFIG_PARSERS:
+            continue
+        preceding = content[: m.start()]
+        recent_lines = preceding.rsplit("\n", 3)[-3:]
+        if any(SKIP_COMMENT_RE.search(line) for line in recent_lines):
+            continue
+        blocks.append(
+            {
+                "lang": lang,
+                "body": m.group("body"),
+                "line": preceding.count("\n") + 1,
+            }
+        )
+    return blocks
+
+
+def find_eden_urls(content: str) -> list[dict]:
+    urls = []
+    for m in EDEN_URL_RE.finditer(content):
+        urls.append(
+            {
+                "url": m.group(0),
+                "host": m.group("host"),
+                "path": (m.group("path") or "").split("?")[0].split("#")[0].rstrip("/"),
+                "line": content[: m.start()].count("\n") + 1,
+            }
+        )
+    return urls
+
+
+def find_provider_model_strings(content: str) -> list[str]:
+    return [
+        f"{m.group('provider')}/{m.group('rest')}"
+        for m in BACKTICKED_MODEL_RE.finditer(content)
+    ]
+
+
+@pytest.mark.parametrize("guide", CONFIG_GUIDES, ids=lambda p: Path(p).stem)
+def test_config_guide(guide: str) -> None:
+    path = DOCS_ROOT / guide
+    assert path.exists(), f"Guide not found: {path}"
+    content = path.read_text(encoding="utf-8")
+    errors: list[str] = []
+
+    for block in parse_fenced_config_blocks(content):
+        parser = CONFIG_PARSERS[block["lang"]]
+        try:
+            parser(block["body"])
+        except Exception as exc:
+            errors.append(
+                f"line {block['line']}: {block['lang']} config block failed to parse: {exc}"
+            )
+
+    for u in find_eden_urls(content):
+        if "edenai" not in u["host"] and "instatus" not in u["host"]:
+            continue
+        if u["host"] not in ALLOWED_EDEN_HOSTS:
+            errors.append(
+                f"line {u['line']}: unexpected Eden AI host `{u['host']}` in {u['url']}"
+            )
+            continue
+        if u["host"] == "api.edenai.run" and u["path"]:
+            if not any(u["path"].startswith(prefix) for prefix in KNOWN_API_ENDPOINT_PREFIXES):
+                errors.append(
+                    f"line {u['line']}: unknown API endpoint path `{u['path']}` in {u['url']}"
+                )
+
+    if os.environ.get("EDEN_AI_SANDBOX_API_TOKEN"):
+        inventory = get_model_inventory()
+        for pm in find_provider_model_strings(content):
+            if pm in DOCUMENTATION_PLACEHOLDERS:
+                continue
+            if pm in inventory or strip_tool_alias(pm) in inventory:
+                continue
+            errors.append(f"unknown model `{pm}` (not in live inventory)")
+
+    if errors:
+        pytest.fail(f"{guide}:\n  " + "\n  ".join(errors))