Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/_publish-code.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,15 @@ jobs:
env:
PYTHON_VERSION: "3.12"
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v7

- name: Set up Python ${{ env.PYTHON_VERSION }}
uses: actions/setup-python@v6
with:
python-version: ${{ env.PYTHON_VERSION }}

- name: Cache dependencies
uses: actions/cache@v5
uses: actions/cache@v6
with:
path: ~/.cache/pip
key: ${{ runner.os }}-build-${{ hashFiles('pyproject.toml') }}
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/_publish-docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,15 @@ jobs:
matrix:
python-version: ["3.12"]
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v7

- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}

- name: Cache dependencies
uses: actions/cache@v5
uses: actions/cache@v6
with:
path: ~/.cache/pip
key: ${{ runner.os }}-docs-${{ hashFiles('pyproject.toml') }}
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/_smoke-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ jobs:
- "3.14"
runs-on: "ubuntu-22.04"
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v7
with:
submodules: recursive

Expand All @@ -35,7 +35,7 @@ jobs:
python-version: ${{ matrix.python-version }}

- name: Cache dependencies
uses: actions/cache@v5
uses: actions/cache@v6
with:
path: ~/.cache/pip
key: ${{ runner.os }}-samples-${{ hashFiles('**/pyproject.toml') }}
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/_static-analysis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,15 @@ jobs:
matrix:
python-version: ["3.12"]
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v7

- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}

- name: Cache dependencies
uses: actions/cache@v5
uses: actions/cache@v6
with:
path: ~/.cache/pip
key: ${{ runner.os }}-lint-${{ hashFiles('pyproject.toml') }}
Expand All @@ -44,7 +44,7 @@ jobs:
pip install -e '.[lint]'

- name: Cache pre-commit
uses: actions/cache@v5
uses: actions/cache@v6
with:
path: ~/.cache/pre-commit
key: ${{ runner.os }}-prec-${{ hashFiles('.pre-commit-config.yaml') }}
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/_test-cli.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ jobs:
- "3.14"
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v7
with:
submodules: recursive

Expand All @@ -41,7 +41,7 @@ jobs:
python-version: ${{ matrix.python-version }}

- name: Cache dependencies
uses: actions/cache@v5
uses: actions/cache@v6
with:
path: ~/.cache/pip
key: ${{ runner.os }}-cli-${{ hashFiles('**/pyproject.toml') }}
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/_test-integrations.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ jobs:
- "3.14"
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v7
with:
submodules: recursive

Expand All @@ -34,7 +34,7 @@ jobs:
python-version: ${{ matrix.python-version }}

- name: Cache dependencies
uses: actions/cache@v5
uses: actions/cache@v6
with:
path: ~/.cache/pip
key: ${{ runner.os }}-test-${{ hashFiles('pyproject.toml') }}
Expand Down Expand Up @@ -80,7 +80,7 @@ jobs:
- "3.10"
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v7
with:
submodules: recursive

Expand All @@ -90,7 +90,7 @@ jobs:
python-version: ${{ matrix.python-version }}

- name: Cache dependencies
uses: actions/cache@v5
uses: actions/cache@v6
with:
path: ~/.cache/pip
key: ${{ runner.os }}-test-${{ hashFiles('pyproject.toml') }}
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/_test-regressions.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ jobs:
- "3.14"
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v7
with:
submodules: recursive

Expand All @@ -31,7 +31,7 @@ jobs:
python-version: ${{ matrix.python-version }}

- name: Cache dependencies
uses: actions/cache@v5
uses: actions/cache@v6
with:
path: ~/.cache/pip
key: ${{ runner.os }}-test-${{ hashFiles('pyproject.toml') }}
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/_test-units.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ jobs:
- "3.14"
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v7
with:
submodules: recursive

Expand All @@ -33,7 +33,7 @@ jobs:
python-version: ${{ matrix.python-version }}

- name: Cache dependencies
uses: actions/cache@v5
uses: actions/cache@v6
with:
path: ~/.cache/pip
key: ${{ runner.os }}-test-${{ hashFiles('pyproject.toml') }}
Expand Down Expand Up @@ -65,7 +65,7 @@ jobs:
- "3.14"
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v7
with:
submodules: recursive

Expand All @@ -75,7 +75,7 @@ jobs:
python-version: ${{ matrix.python-version }}

- name: Cache dependencies
uses: actions/cache@v5
uses: actions/cache@v6
with:
path: ~/.cache/pip
key: ${{ runner.os }}-test-${{ hashFiles('pyproject.toml') }}
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/_workflow_lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ jobs:
actionlint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v7
- name: Download actionlint
id: get_actionlint
run: bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash)
Expand Down
6 changes: 3 additions & 3 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
repos:
- repo: https://github.com/charliermarsh/ruff-pre-commit
rev: v0.15.18
rev: v0.15.21
hooks:
- id: ruff-check
args: [ --fix, --exit-non-zero-on-fix]
- id: ruff-format

- repo: https://github.com/betterleaks/betterleaks
rev: v1.5.0
rev: v1.6.1
hooks:
- id: betterleaks

Expand All @@ -31,7 +31,7 @@ repos:
stages: [pre-push]

- repo: https://github.com/pre-commit/mirrors-mypy
rev: v2.1.0
rev: v2.2.0
hooks:
- id: mypy
args: []
Expand Down
4 changes: 2 additions & 2 deletions mindee/input/url_input_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from mindee.error.mindee_error import MindeeSourceError
from mindee.input.bytes_input import BytesInput
from mindee.logger import logger
from mindee.mindee_http.response_validation import validate_url_for_source
from mindee.parsing.common.string_dict import StringDict


Expand All @@ -25,8 +26,7 @@ def __init__(self, url: str) -> None:

:param url: URL to send, must be HTTPS.
"""
if not url.lower().startswith("https"):
raise MindeeSourceError("URL must be HTTPS")
validate_url_for_source(url)

logger.debug("URL input: %s", url)

Expand Down
15 changes: 15 additions & 0 deletions mindee/mindee_http/cancellation_token.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
class CancellationToken:
"""Custom cancellation token that can be used to cancel a polling request."""

is_canceled: bool
"""A cancellation token that can be used to cancel a request."""

def __init__(
self,
is_canceled: bool = False,
):
self.is_canceled = is_canceled

def cancel(self):
"""Cancel the request."""
self.is_canceled = True
Comment thread
sebastianMindee marked this conversation as resolved.
70 changes: 70 additions & 0 deletions mindee/mindee_http/response_validation.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,79 @@
import ipaddress
import json
from urllib.parse import urlparse

import httpx

from mindee.error.mindee_error import MindeeSourceError
from mindee.parsing.common.string_dict import StringDict

_CGNAT_BLOCK = ipaddress.IPv4Network("100.64.0.0/10")
_IPV6_UNIQUE_LOCAL = ipaddress.IPv6Network("fc00::/7")


def validate_url_for_source(url: str) -> None:
"""
Validates that a URL is safe to send to the Mindee server.

Rejects any URL that could be used for Server-Side Request Forgery (SSRF):

- non-HTTPS schemes,
- embedded userinfo (e.g. ``https://user:pass@host``),
- loopback hostnames (``localhost``, ``*.localhost``),
- literal IP addresses that are loopback, link-local, private (RFC 1918),
any-local (``0.0.0.0``), multicast, IPv6 unique-local (``fc00::/7``),
or carrier-grade NAT (``100.64.0.0/10``).
Comment thread
sebastianMindee marked this conversation as resolved.

Note: DNS resolution is not performed. A hostname that resolves to a
private IP will not be caught here.

:param url: The URL string to validate.
:raises MindeeSourceError: If the URL fails any security check.
"""
try:
parsed = urlparse(url)
except Exception as exc:
raise MindeeSourceError("Invalid URL") from exc

if parsed.scheme.lower() != "https":
raise MindeeSourceError("URL must be HTTPS")

if parsed.username or parsed.password:
raise MindeeSourceError("Source URLs must not embed user credentials")

host = parsed.hostname
if not host:
raise MindeeSourceError("Source URL is missing a host")

lower_host = host.lower()
if (
lower_host == "localhost"
or lower_host.endswith(".localhost")
or lower_host == "ip6-localhost"
or lower_host == "ip6-loopback"
):
raise MindeeSourceError(f"Loopback hostnames are not allowed: {host}")

try:
addr = ipaddress.ip_address(lower_host)
except ValueError:
return

if (
addr.is_loopback
or addr.is_link_local
or addr.is_private
or addr.is_unspecified
or addr.is_multicast
):
raise MindeeSourceError(f"URL host resolves to a disallowed address: {addr}")

if isinstance(addr, ipaddress.IPv4Address) and addr in _CGNAT_BLOCK:
raise MindeeSourceError(f"URL host resolves to a disallowed address: {addr}")

if isinstance(addr, ipaddress.IPv6Address) and addr in _IPV6_UNIQUE_LOCAL:
raise MindeeSourceError(f"URL host resolves to a disallowed address: {addr}")


def is_valid_sync_response(response: httpx.Response) -> bool:
"""
Expand Down
8 changes: 8 additions & 0 deletions mindee/v2/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from mindee.input import URLInputSource
from mindee.input.local_input_source import LocalInputSource
from mindee.logger import logger
from mindee.mindee_http.cancellation_token import CancellationToken
from mindee.parsing.common.common_response import CommonStatus
from mindee.v2.client_options.base_parameters import BaseParameters
from mindee.v2.mindee_http.mindee_api_v2 import MindeeAPIV2
Expand Down Expand Up @@ -104,13 +105,16 @@ def enqueue_and_get_result(
response_type: type[TypeBaseResponse],
input_source: LocalInputSource | URLInputSource,
params: BaseParameters,
cancellation_token: CancellationToken | None = None,
) -> TypeBaseResponse:
"""
Enqueues to an asynchronous endpoint and automatically polls for a response.

:param input_source: The document/source file to use. Can be local or remote.
:param params: Parameters to set when sending a file.
:param response_type: The product class to use for the response object.
:param cancellation_token: A cancellation token that can be used to cancel the
request.
Comment thread
sebastianMindee marked this conversation as resolved.

:return: A valid inference response.
"""
Expand All @@ -125,9 +129,13 @@ def enqueue_and_get_result(
logger.debug(
"Successfully enqueued document with job ID: %s", enqueue_response.job.id
)
if cancellation_token and cancellation_token.is_canceled:
raise MindeeError("Request canceled through cancellation token.")
sleep(params.polling_options.initial_delay_sec)
try_counter = 0
while try_counter < params.polling_options.max_retries:
if cancellation_token and cancellation_token.is_canceled:
raise MindeeError("Request canceled through cancellation token.")
job_response = self.get_job(enqueue_response.job.id)
assert isinstance(job_response, JobResponse)
if job_response.job.status == CommonStatus.FAILED.value:
Expand Down
3 changes: 2 additions & 1 deletion mindee/v2/mindee_http/mindee_api_v2.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import json
import os
from collections.abc import Callable
from typing import TypeVar
Expand Down Expand Up @@ -297,7 +298,7 @@ def get_models(self, name: str | None, model_type: str | None):
def _response_json(response: httpx.Response) -> StringDict:
try:
return response.json()
except httpx.DecodingError as e:
except (httpx.DecodingError, json.JSONDecodeError) as e:
raise MindeeHTTPUnknownErrorV2(
f"HTTP {response.status_code} response is not valid JSON: "
f"{response.text}"
Expand Down
Loading