From 94bae488f06be1025c657891ac38e23af8a0e9d1 Mon Sep 17 00:00:00 2001 From: EssKayz Date: Thu, 25 Jun 2026 16:28:38 +0300 Subject: [PATCH 1/9] [SIS-Import] add params and fix overloads --- .../sis_integration/async_imports.py | 52 +++++++++++-------- funidata_utils/sis_integration/exports.py | 49 +++++++++++------ 2 files changed, 63 insertions(+), 38 deletions(-) diff --git a/funidata_utils/sis_integration/async_imports.py b/funidata_utils/sis_integration/async_imports.py index ed3e924..045ac99 100644 --- a/funidata_utils/sis_integration/async_imports.py +++ b/funidata_utils/sis_integration/async_imports.py @@ -22,11 +22,12 @@ async def import_to_sisu( resource: SisImportable, use_legacy_import: Literal[False], fp: IO, - batch_size: int | None, - binary_search_max_depth: int | None, - group_by_key: str | None, - binary_err_search_sublists: bool, - max_parallel_requests: int, + batch_size: int | None = UNSET_BATCH_SIZE, + binary_search_max_depth: int | None = 0, + group_by_key: str | None = None, + binary_err_search_sublists: bool = False, + max_parallel_requests: int = 1, + params: dict | None = None, ) -> list[httpx.Response]: ... @@ -37,11 +38,12 @@ async def import_to_sisu( resource: SisLegacyImportable, use_legacy_import: Literal[True], fp: IO, - batch_size: int | None, - binary_search_max_depth: int | None, - group_by_key: str | None, - binary_err_search_sublists: bool, - max_parallel_requests: int, + batch_size: int | None = UNSET_BATCH_SIZE, + binary_search_max_depth: int | None = 0, + group_by_key: str | None = None, + binary_err_search_sublists: bool = False, + max_parallel_requests: int = 1, + params: dict | None = None, ) -> list[httpx.Response]: ... @@ -52,11 +54,12 @@ async def import_to_sisu( resource: SisImportable, use_legacy_import: Literal[False], data: list[dict], - batch_size: int | None, - binary_search_max_depth: int | None, - group_by_key: str | None, - binary_err_search_sublists: bool, - max_parallel_requests: int, + batch_size: int | None = UNSET_BATCH_SIZE, + binary_search_max_depth: int | None = 0, + group_by_key: str | None = None, + binary_err_search_sublists: bool = False, + max_parallel_requests: int = 1, + params: dict | None = None, ) -> list[httpx.Response]: ... @@ -67,11 +70,12 @@ async def import_to_sisu( resource: SisLegacyImportable, use_legacy_import: Literal[True], data: list[dict], - batch_size: int | None, - binary_search_max_depth: int | None, - group_by_key: str | None, - binary_err_search_sublists: bool, - max_parallel_requests: int, + batch_size: int | None = UNSET_BATCH_SIZE, + binary_search_max_depth: int | None = 0, + group_by_key: str | None = None, + binary_err_search_sublists: bool = False, + max_parallel_requests: int = 1, + params: dict | None = None, ) -> list[httpx.Response]: ... @@ -86,7 +90,8 @@ async def import_to_sisu( binary_search_max_depth: int | None = 0, group_by_key: str | None = None, binary_err_search_sublists: bool = False, - max_parallel_requests: int = 1 + max_parallel_requests: int = 1, + params: dict | None = None, ) -> list[httpx.Response]: if fp: raise NotImplementedError("Not yet implemented") @@ -112,6 +117,7 @@ async def import_to_sisu( group_by_key=group_by_key, method='POST', max_parallel_requests=max_parallel_requests, + params=params, ) return responses @@ -127,7 +133,8 @@ async def patch_to_sisu( binary_search_max_depth: int | None = 0, group_by_key: str | None = None, binary_err_search_sublists: bool = False, - max_parallel_requests: int = 1 + max_parallel_requests: int = 1, + params: dict | None = None, ) -> list[httpx.Response]: if fp: raise NotImplementedError("Not yet implemented") @@ -153,6 +160,7 @@ async def patch_to_sisu( group_by_key=group_by_key, method='PATCH', max_parallel_requests=max_parallel_requests, + params=params, ) return responses diff --git a/funidata_utils/sis_integration/exports.py b/funidata_utils/sis_integration/exports.py index 8857554..df50af3 100644 --- a/funidata_utils/sis_integration/exports.py +++ b/funidata_utils/sis_integration/exports.py @@ -15,7 +15,8 @@ def _export_from_endpoint( fp: None, since_ordinal: int = 0, export_limit: int = 1000, - since: str = 'since' + since: str = 'since', + params: dict | None = None, ) -> list[dict]: ... @@ -27,7 +28,8 @@ def _export_from_endpoint( fp: IO, since_ordinal: int = 0, export_limit: int = 1000, - since: str = 'since' + since: str = 'since', + params: dict | None = None, ) -> TextIO: ... @@ -38,8 +40,12 @@ def _export_from_endpoint( fp: IO | None, since_ordinal: int = 0, export_limit: int = 1000, - since: str = 'since' + since: str = 'since', + params: dict | None = None, ) -> IO | list[dict]: + if not params: + params = {} + exported_entities = [] for entities in export_from_endpoint_generator( sis_settings=sis_settings, @@ -47,6 +53,7 @@ def _export_from_endpoint( since_ordinal=since_ordinal, export_limit=export_limit, since=since, + params=params, ): if fp is None: exported_entities += entities @@ -69,16 +76,19 @@ def export_from_endpoint_generator( endpoint: str, since_ordinal: int = 0, export_limit: int = 1000, - since: str = 'since' + since: str = 'since', + params: dict | None = None, ) -> Generator[list[dict], None, None]: greatest_ordinal = since_ordinal export_limit = export_limit + if not params: + params = {} while True: sis_response = send_get_httpx( path=f"{sis_settings.host}{endpoint}", auth=sis_settings.get_export_auth(), - params={since: greatest_ordinal, 'limit': export_limit}, + params={since: greatest_ordinal, 'limit': export_limit} | params, proxies=sis_settings.proxies, ) if sis_response.status_code == 200: @@ -99,10 +109,11 @@ def export_from_endpoint_generator( def export_from_sisu( sisu_config: SupportsExportAuthentication, resource: SisExportable, - fp: None, since_ordinal: int, - as_generator: Literal[False] + as_generator: Literal[False], + params: dict | None = None, ) -> list[dict]: + # Regular call, no generator or FP reference ... @@ -110,10 +121,11 @@ def export_from_sisu( def export_from_sisu( sisu_config: SupportsExportAuthentication, resource: SisExportable, - fp: IO, since_ordinal: int, - as_generator: Literal[False] -) -> IO: + as_generator: Literal[True], + params: dict | None = None, +) -> Generator[list[dict], None, None]: + # Call with as_generator does not allow FP reference ... @@ -121,10 +133,11 @@ def export_from_sisu( def export_from_sisu( sisu_config: SupportsExportAuthentication, resource: SisExportable, - fp: None, + fp: IO, since_ordinal: int, - as_generator: Literal[True] -) -> Generator[list[dict], None, None]: + params: dict | None = None, +) -> IO: + # Call with FP reference does not allow as_generator ... @@ -133,7 +146,8 @@ def export_from_sisu( resource: SisExportable, fp: IO | None = None, since_ordinal: int = 0, - as_generator: bool = False + as_generator: bool = False, + params: dict | None = None, ) -> list[dict] | IO | Generator[list[dict], None, None]: if as_generator: return export_from_endpoint_generator( @@ -142,6 +156,7 @@ def export_from_sisu( sis_settings=sisu_config, since_ordinal=since_ordinal, since=resource.exports.since, + params=params, ) if fp: @@ -151,7 +166,8 @@ def export_from_sisu( sis_settings=sisu_config, since_ordinal=since_ordinal, since=resource.exports.since, - fp=fp + fp=fp, + params=params, ) return _export_from_endpoint( @@ -160,5 +176,6 @@ def export_from_sisu( sis_settings=sisu_config, since_ordinal=since_ordinal, since=resource.exports.since, - fp=None + fp=None, + params=params ) From d6314ee37569d8c2940cb5ca4c5e0a912e6df1f5 Mon Sep 17 00:00:00 2001 From: EssKayz Date: Thu, 25 Jun 2026 20:48:03 +0300 Subject: [PATCH 2/9] [sis-integration] Tests for integration --- requirements-tests.txt | 3 +- tests/test_import_batching.py | 185 ++++++++++++++++++++++++++++++++++ 2 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 tests/test_import_batching.py diff --git a/requirements-tests.txt b/requirements-tests.txt index dd54f63..8de32f3 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -1,3 +1,4 @@ -e .[all] pytest >=8.4.1,<9.0.0 -coverage[toml] >= 7.9.2,< 8.0 \ No newline at end of file +coverage[toml] >= 7.9.2,< 8.0 +pytest-asyncio==1.4.0 \ No newline at end of file diff --git a/tests/test_import_batching.py b/tests/test_import_batching.py new file mode 100644 index 0000000..a8cc899 --- /dev/null +++ b/tests/test_import_batching.py @@ -0,0 +1,185 @@ +import json +from collections import defaultdict + +import pytest + +from funidata_utils.request_utils.async_httpx_requests import _binary_search_enabled_post_httpx + +import httpx + +from funidata_utils.utils import group_by + + +def invalid_handler(request: httpx.Request): + _content = json.loads(request.content) + _failing_ids = [_x['id'] for _x in _content if _x.get('invalid')] + if _failing_ids: + return httpx.Response( + status_code=422, json={"failingIds": _failing_ids} + ) + return httpx.Response(200, json={"diu": "OK"}) + + +def get_entity_counts_by_status_code(responses: list[httpx.Response]): + counts_by_status_code = defaultdict(int) + for response in responses: + counts_by_status_code[response.status_code] += len(json.loads(response.request.content)) + + return counts_by_status_code + + +@pytest.fixture +def mock_client(): + return httpx.AsyncClient( + transport=httpx.MockTransport(invalid_handler) + ) + + +@pytest.mark.asyncio +async def test_recursive_import_batching_with_sublists_off(mock_client): + test_data = [ + [ + { + "id": 2, + "person": 1, + "invalid": True + }, + { + "id": 3, + "person": 1 + } + ], + [ + { + "id": 3, + "person": 2 + }, + { + "id": 4, + "person": 2 + } + ], + [ + { + "id": 4, + "person": 3 + }, + { + "id": 5, + "person": 3 + } + ] + ] + + results = await _binary_search_enabled_post_httpx( + path="http://localhost", + payload=test_data, + auth=None, + client=mock_client, + binary_search_depth=0, + binary_err_search_sublists=False, + binary_search_max_depth=None, + ) + assert get_entity_counts_by_status_code(results)[200] == 4 + assert get_entity_counts_by_status_code(results)[422] == 2 + + +@pytest.mark.asyncio +async def test_recursive_import_batching_with_sublists_off_multiple_fails(mock_client): + test_data = [ + [ + { + "id": 2, + "person": 1, + "invalid": True + }, + { + "id": 3, + "person": 1, + "invalid": True, + } + ], + [ + { + "id": 3, + "person": 2 + }, + { + "id": 4, + "person": 2 + } + ], + [ + { + "id": 4, + "person": 3 + }, + { + "id": 5, + "person": 3, + "invalid": True, + } + ] + ] + + results = await _binary_search_enabled_post_httpx( + path="http://localhost", + payload=test_data, + auth=None, + client=mock_client, + binary_search_depth=0, + binary_err_search_sublists=False, + binary_search_max_depth=None, + ) + + assert get_entity_counts_by_status_code(results)[200] == 4 + assert get_entity_counts_by_status_code(results)[422] == 2 + + +@pytest.mark.asyncio +async def test_recursive_import_batching_with_sublists_on(mock_client): + test_data = [ + [ + { + "id": 2, + "person": 1, + "invalid": True + }, + { + "id": 3, + "person": 1 + } + ], + [ + { + "id": 3, + "person": 2 + }, + { + "id": 4, + "person": 2 + } + ], + [ + { + "id": 4, + "person": 3 + }, + { + "id": 5, + "person": 3 + } + ] + ] + + results = await _binary_search_enabled_post_httpx( + path="http://localhost", + payload=test_data, + auth=None, + client=mock_client, + binary_err_search_sublists=True, + binary_search_depth=0, + binary_search_max_depth=None + ) + assert get_entity_counts_by_status_code(results)[200] == 5 + assert get_entity_counts_by_status_code(results)[422] == 1 From 2a48b520f3eac2f0efee519c624efc0b723a45bb Mon Sep 17 00:00:00 2001 From: EssKayz Date: Thu, 25 Jun 2026 16:58:03 +0300 Subject: [PATCH 3/9] [SIS-integration] add overload for calling without as_generator --- funidata_utils/sis_integration/exports.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/funidata_utils/sis_integration/exports.py b/funidata_utils/sis_integration/exports.py index df50af3..03adea0 100644 --- a/funidata_utils/sis_integration/exports.py +++ b/funidata_utils/sis_integration/exports.py @@ -110,13 +110,24 @@ def export_from_sisu( sisu_config: SupportsExportAuthentication, resource: SisExportable, since_ordinal: int, - as_generator: Literal[False], params: dict | None = None, ) -> list[dict]: # Regular call, no generator or FP reference ... +@overload +def export_from_sisu( + sisu_config: SupportsExportAuthentication, + resource: SisExportable, + since_ordinal: int, + as_generator: Literal[False], + params: dict | None = None, +) -> list[dict]: + # Regular call, generator explicit false, no FP reference + ... + + @overload def export_from_sisu( sisu_config: SupportsExportAuthentication, From 6f409d8b6b1c503338a90b843ef122985ba93d8c Mon Sep 17 00:00:00 2001 From: EssKayz Date: Thu, 25 Jun 2026 20:54:28 +0300 Subject: [PATCH 4/9] [sis-integration] Refactor tests --- ....py => test_import_batching_no_sublist.py} | 105 +++++---- tests/test_import_batching_with_sublist.py | 208 ++++++++++++++++++ tests/test_utils.py | 30 +++ 3 files changed, 302 insertions(+), 41 deletions(-) rename tests/{test_import_batching.py => test_import_batching_no_sublist.py} (62%) create mode 100644 tests/test_import_batching_with_sublist.py create mode 100644 tests/test_utils.py diff --git a/tests/test_import_batching.py b/tests/test_import_batching_no_sublist.py similarity index 62% rename from tests/test_import_batching.py rename to tests/test_import_batching_no_sublist.py index a8cc899..26f4141 100644 --- a/tests/test_import_batching.py +++ b/tests/test_import_batching_no_sublist.py @@ -1,42 +1,59 @@ -import json -from collections import defaultdict - import pytest from funidata_utils.request_utils.async_httpx_requests import _binary_search_enabled_post_httpx - -import httpx - -from funidata_utils.utils import group_by - - -def invalid_handler(request: httpx.Request): - _content = json.loads(request.content) - _failing_ids = [_x['id'] for _x in _content if _x.get('invalid')] - if _failing_ids: - return httpx.Response( - status_code=422, json={"failingIds": _failing_ids} - ) - return httpx.Response(200, json={"diu": "OK"}) +from tests.test_utils import mock_client, get_entity_counts_by_status_code -def get_entity_counts_by_status_code(responses: list[httpx.Response]): - counts_by_status_code = defaultdict(int) - for response in responses: - counts_by_status_code[response.status_code] += len(json.loads(response.request.content)) - - return counts_by_status_code - +@pytest.mark.asyncio +async def test_recursive_import_batching_with_sublists_off_no_fails(mock_client): + test_data = [ + [ + { + "id": 2, + "person": 1, + }, + { + "id": 3, + "person": 1 + } + ], + [ + { + "id": 3, + "person": 2 + }, + { + "id": 4, + "person": 2 + } + ], + [ + { + "id": 4, + "person": 3 + }, + { + "id": 5, + "person": 3 + } + ] + ] -@pytest.fixture -def mock_client(): - return httpx.AsyncClient( - transport=httpx.MockTransport(invalid_handler) + results = await _binary_search_enabled_post_httpx( + path="http://localhost", + payload=test_data, + auth=None, + client=mock_client, + binary_search_depth=0, + binary_err_search_sublists=False, + binary_search_max_depth=None, ) + assert get_entity_counts_by_status_code(results)[200] == 6 + assert get_entity_counts_by_status_code(results).get(422) is None @pytest.mark.asyncio -async def test_recursive_import_batching_with_sublists_off(mock_client): +async def test_recursive_import_batching_with_sublists_off_one_fail(mock_client): test_data = [ [ { @@ -132,12 +149,12 @@ async def test_recursive_import_batching_with_sublists_off_multiple_fails(mock_c binary_search_max_depth=None, ) - assert get_entity_counts_by_status_code(results)[200] == 4 - assert get_entity_counts_by_status_code(results)[422] == 2 + assert get_entity_counts_by_status_code(results)[200] == 2 + assert get_entity_counts_by_status_code(results)[422] == 4 @pytest.mark.asyncio -async def test_recursive_import_batching_with_sublists_on(mock_client): +async def test_recursive_import_batching_with_sublists_off_all_fails(mock_client): test_data = [ [ { @@ -147,27 +164,32 @@ async def test_recursive_import_batching_with_sublists_on(mock_client): }, { "id": 3, - "person": 1 + "person": 1, + "invalid": True, } ], [ { "id": 3, - "person": 2 + "person": 2, + "invalid": True, }, { "id": 4, - "person": 2 + "person": 2, + "invalid": True, } ], [ { "id": 4, - "person": 3 + "person": 3, + "invalid": True, }, { "id": 5, - "person": 3 + "person": 3, + "invalid": True, } ] ] @@ -177,9 +199,10 @@ async def test_recursive_import_batching_with_sublists_on(mock_client): payload=test_data, auth=None, client=mock_client, - binary_err_search_sublists=True, binary_search_depth=0, - binary_search_max_depth=None + binary_err_search_sublists=False, + binary_search_max_depth=None, ) - assert get_entity_counts_by_status_code(results)[200] == 5 - assert get_entity_counts_by_status_code(results)[422] == 1 + + assert get_entity_counts_by_status_code(results).get(200) is None + assert get_entity_counts_by_status_code(results)[422] == 6 diff --git a/tests/test_import_batching_with_sublist.py b/tests/test_import_batching_with_sublist.py new file mode 100644 index 0000000..cf64d1f --- /dev/null +++ b/tests/test_import_batching_with_sublist.py @@ -0,0 +1,208 @@ +import pytest + +from funidata_utils.request_utils.async_httpx_requests import _binary_search_enabled_post_httpx +from tests.test_utils import mock_client, get_entity_counts_by_status_code + + +@pytest.mark.asyncio +async def test_recursive_import_batching_with_sublists_on_no_fails(mock_client): + test_data = [ + [ + { + "id": 2, + "person": 1, + }, + { + "id": 3, + "person": 1 + } + ], + [ + { + "id": 3, + "person": 2 + }, + { + "id": 4, + "person": 2 + } + ], + [ + { + "id": 4, + "person": 3 + }, + { + "id": 5, + "person": 3 + } + ] + ] + + results = await _binary_search_enabled_post_httpx( + path="http://localhost", + payload=test_data, + auth=None, + client=mock_client, + binary_search_depth=0, + binary_err_search_sublists=False, + binary_search_max_depth=None, + ) + assert get_entity_counts_by_status_code(results)[200] == 6 + assert get_entity_counts_by_status_code(results).get(422) is None + + +@pytest.mark.asyncio +async def test_recursive_import_batching_with_sublists_on_one_fail(mock_client): + test_data = [ + [ + { + "id": 2, + "person": 1, + "invalid": True + }, + { + "id": 3, + "person": 1 + } + ], + [ + { + "id": 3, + "person": 2 + }, + { + "id": 4, + "person": 2 + } + ], + [ + { + "id": 4, + "person": 3 + }, + { + "id": 5, + "person": 3 + } + ] + ] + + results = await _binary_search_enabled_post_httpx( + path="http://localhost", + payload=test_data, + auth=None, + client=mock_client, + binary_search_depth=0, + binary_err_search_sublists=True, + binary_search_max_depth=None, + ) + assert get_entity_counts_by_status_code(results)[200] == 5 + assert get_entity_counts_by_status_code(results)[422] == 1 + + +@pytest.mark.asyncio +async def test_recursive_import_batching_with_sublists_on_multiple_fails(mock_client): + test_data = [ + [ + { + "id": 2, + "person": 1, + "invalid": True + }, + { + "id": 3, + "person": 1, + "invalid": True, + } + ], + [ + { + "id": 3, + "person": 2 + }, + { + "id": 4, + "person": 2 + } + ], + [ + { + "id": 4, + "person": 3 + }, + { + "id": 5, + "person": 3, + "invalid": True, + } + ] + ] + + results = await _binary_search_enabled_post_httpx( + path="http://localhost", + payload=test_data, + auth=None, + client=mock_client, + binary_search_depth=0, + binary_err_search_sublists=True, + binary_search_max_depth=None, + ) + + assert get_entity_counts_by_status_code(results)[200] == 3 + assert get_entity_counts_by_status_code(results)[422] == 3 + + +@pytest.mark.asyncio +async def test_recursive_import_batching_with_sublists_on_all_fails(mock_client): + test_data = [ + [ + { + "id": 2, + "person": 1, + "invalid": True + }, + { + "id": 3, + "person": 1, + "invalid": True, + } + ], + [ + { + "id": 3, + "person": 2, + "invalid": True, + }, + { + "id": 4, + "person": 2, + "invalid": True, + } + ], + [ + { + "id": 4, + "person": 3, + "invalid": True, + }, + { + "id": 5, + "person": 3, + "invalid": True, + } + ] + ] + + results = await _binary_search_enabled_post_httpx( + path="http://localhost", + payload=test_data, + auth=None, + client=mock_client, + binary_search_depth=0, + binary_err_search_sublists=True, + binary_search_max_depth=None, + ) + + assert get_entity_counts_by_status_code(results).get(200) is None + assert get_entity_counts_by_status_code(results)[422] == 6 diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..e60b464 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,30 @@ +import json +from collections import defaultdict + +import httpx +import pytest + + +@pytest.fixture +def mock_client(): + return httpx.AsyncClient( + transport=httpx.MockTransport(invalid_handler) + ) + + +def invalid_handler(request: httpx.Request): + _content = json.loads(request.content) + _failing_ids = [_x['id'] for _x in _content if _x.get('invalid')] + if _failing_ids: + return httpx.Response( + status_code=422, json={"failingIds": _failing_ids} + ) + return httpx.Response(200, json={"diu": "OK"}) + + +def get_entity_counts_by_status_code(responses: list[httpx.Response]): + counts_by_status_code = defaultdict(int) + for response in responses: + counts_by_status_code[response.status_code] += len(json.loads(response.request.content)) + + return counts_by_status_code From e9ac72dd25d941f1a89b8ec4490558c264c73c98 Mon Sep 17 00:00:00 2001 From: EssKayz Date: Thu, 25 Jun 2026 21:10:39 +0300 Subject: [PATCH 5/9] [workflow] Add gh workflow for tests --- .github/workflows/pytest-verify.yml | 31 +++++++++++++++++++++++++++++ pyproject.toml | 6 ++---- 2 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/pytest-verify.yml diff --git a/.github/workflows/pytest-verify.yml b/.github/workflows/pytest-verify.yml new file mode 100644 index 0000000..4db607c --- /dev/null +++ b/.github/workflows/pytest-verify.yml @@ -0,0 +1,31 @@ +name: Run pytest on supported python versions + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.12", "3.13"] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install flake8 pytest + if [ -f requirements-tests.txt ]; then pip install -r requirements-tests.txt; fi + - name: Test with pytest + run: | + pytest \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index ba08b76..4dc298c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ classifiers = [ "Development Status :: 3 - Alpha", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python", "Framework :: Pydantic", "Framework :: Pydantic :: 2", @@ -50,10 +51,7 @@ version = { source = "scm", fallback_version = "0.0.0" } [tool.pdm.build] -source-includes = [ - "tests/", - "requirements*.txt", -] +excludes = ["tests/"] [tool.pytest.ini_options] addopts = [ From 2740b4153cdb7ba5410d7adf026727e6081d87e5 Mon Sep 17 00:00:00 2001 From: EssKayz Date: Thu, 25 Jun 2026 21:47:26 +0300 Subject: [PATCH 6/9] Remove redundant cross-referencing from requirements files --- .github/workflows/pytest-verify.yml | 3 +-- README.md | 6 ++++++ requirements-tests.txt | 4 ---- requirements.txt | 4 +++- 4 files changed, 10 insertions(+), 7 deletions(-) delete mode 100644 requirements-tests.txt diff --git a/.github/workflows/pytest-verify.yml b/.github/workflows/pytest-verify.yml index 4db607c..9b61aae 100644 --- a/.github/workflows/pytest-verify.yml +++ b/.github/workflows/pytest-verify.yml @@ -24,8 +24,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install flake8 pytest - if [ -f requirements-tests.txt ]; then pip install -r requirements-tests.txt; fi + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Test with pytest run: | pytest \ No newline at end of file diff --git a/README.md b/README.md index 9085429..5ce216b 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,12 @@ A Collection of python utility functions Primarily focused on integration with Funidata APIs +Install dependencies in local virtual environment using the pyproject.toml : +`pip install .` + +Include dev dependencies for testing etc: +`pip install -r requirements.txt` + ### TODO Finish readme documentation \ No newline at end of file diff --git a/requirements-tests.txt b/requirements-tests.txt deleted file mode 100644 index 8de32f3..0000000 --- a/requirements-tests.txt +++ /dev/null @@ -1,4 +0,0 @@ --e .[all] -pytest >=8.4.1,<9.0.0 -coverage[toml] >= 7.9.2,< 8.0 -pytest-asyncio==1.4.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 37158c8..163b13c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ -e .[all] --r requirements-tests.txt \ No newline at end of file +pytest>=9.1.1, <10.0.0 +coverage[toml] >= 7.9.2,< 8.0 +pytest-asyncio==1.4.0 \ No newline at end of file From 635ebb0d382b3c05bfdff5e6b8bba34a90d5cff7 Mon Sep 17 00:00:00 2001 From: EssKayz Date: Thu, 25 Jun 2026 22:44:54 +0300 Subject: [PATCH 7/9] [feat] python-3.11 support --- .github/workflows/pytest-verify.yml | 2 +- funidata_utils/compat/utils_312.py | 10 ++++++++++ funidata_utils/compat/utils_legacy.py | 13 +++++++++++++ funidata_utils/schemas/common_serializers.py | 15 +++++---------- .../schemas/compat/common_serializers_312.py | 12 ++++++++++++ .../schemas/compat/common_serializers_legacy.py | 15 +++++++++++++++ funidata_utils/utils.py | 15 +++++++-------- pyproject.toml | 3 ++- tests/{test_utils.py => helpers.py} | 0 tests/test_import_batching_no_sublist.py | 2 +- tests/test_import_batching_with_sublist.py | 2 +- tests/test_util.py | 10 ++++++++++ 12 files changed, 77 insertions(+), 22 deletions(-) create mode 100644 funidata_utils/compat/utils_312.py create mode 100644 funidata_utils/compat/utils_legacy.py create mode 100644 funidata_utils/schemas/compat/common_serializers_312.py create mode 100644 funidata_utils/schemas/compat/common_serializers_legacy.py rename tests/{test_utils.py => helpers.py} (100%) create mode 100644 tests/test_util.py diff --git a/.github/workflows/pytest-verify.yml b/.github/workflows/pytest-verify.yml index 9b61aae..01d6aa3 100644 --- a/.github/workflows/pytest-verify.yml +++ b/.github/workflows/pytest-verify.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.12", "3.13"] + python-version: ["3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 diff --git a/funidata_utils/compat/utils_312.py b/funidata_utils/compat/utils_312.py new file mode 100644 index 0000000..7ce0371 --- /dev/null +++ b/funidata_utils/compat/utils_312.py @@ -0,0 +1,10 @@ +from collections import defaultdict +from functools import reduce +from typing import Callable, Any + + +def group_by[T]( + seq: list[T], + key: Callable +) -> dict[Any, list[T]]: + return reduce(lambda grp, val: grp[key(val)].append(val) or grp, seq, defaultdict(list)) diff --git a/funidata_utils/compat/utils_legacy.py b/funidata_utils/compat/utils_legacy.py new file mode 100644 index 0000000..843eaf0 --- /dev/null +++ b/funidata_utils/compat/utils_legacy.py @@ -0,0 +1,13 @@ +from collections import defaultdict +from functools import reduce +from typing import Callable, Any, TypeVar + + +T = TypeVar('T') + + +def group_by( + seq: list[T], + key: Callable +) -> dict[Any, list[T]]: + return reduce(lambda grp, val: grp[key(val)].append(val) or grp, seq, defaultdict(list)) diff --git a/funidata_utils/schemas/common_serializers.py b/funidata_utils/schemas/common_serializers.py index c55f679..1af1546 100644 --- a/funidata_utils/schemas/common_serializers.py +++ b/funidata_utils/schemas/common_serializers.py @@ -1,12 +1,7 @@ -from typing import Iterable +import sys -def serialize_as_list[typevar](v: set[typevar] | list[typevar] | None) -> list[typevar] | None: - if v is None: - return None - - try: - return list(set(v)) - except Exception as e: - # If it can't be hashed to set, just return as list - return list(v) +if sys.version_info >= (3, 12): + from .compat.common_serializers_312 import serialize_as_list # noqa: F401 ("Unused import") +else: + from .compat.common_serializers_legacy import serialize_as_list # noqa: F401 ("Unused import") diff --git a/funidata_utils/schemas/compat/common_serializers_312.py b/funidata_utils/schemas/compat/common_serializers_312.py new file mode 100644 index 0000000..e148608 --- /dev/null +++ b/funidata_utils/schemas/compat/common_serializers_312.py @@ -0,0 +1,12 @@ +# Python > 3.12 implementations + + +def serialize_as_list[typevar](v: set[typevar] | list[typevar] | None) -> list[typevar] | None: + if v is None: + return None + + try: + return list(set(v)) + except Exception as e: + # If it can't be hashed to set, just return as list + return list(v) diff --git a/funidata_utils/schemas/compat/common_serializers_legacy.py b/funidata_utils/schemas/compat/common_serializers_legacy.py new file mode 100644 index 0000000..e80cbc4 --- /dev/null +++ b/funidata_utils/schemas/compat/common_serializers_legacy.py @@ -0,0 +1,15 @@ +from typing import TypeVar + + +T = TypeVar('T') + + +def serialize_as_list(v: set[T] | list[T] | None) -> list[T] | None: + if v is None: + return None + + try: + return list(set(v)) + except Exception as e: + # If it can't be hashed to set, just return as list + return list(v) diff --git a/funidata_utils/utils.py b/funidata_utils/utils.py index a9dcd16..0a234ce 100644 --- a/funidata_utils/utils.py +++ b/funidata_utils/utils.py @@ -1,7 +1,7 @@ # Copyright (c) 2025 Funidata Oy. # All rights reserved. # ------------------------------------------------------------------------------ - +import sys from collections import defaultdict from functools import reduce from statistics import mean, stdev @@ -10,6 +10,12 @@ import httpx +if sys.version_info >= (3, 12): + from .compat.utils_312 import group_by # noqa: F401 ("Unused import") +else: + from .compat.utils_legacy import group_by # noqa: F401 ("Unused import") + + def _recursive_flatten( lst: list ) -> Generator: @@ -27,13 +33,6 @@ def flatten( return list(_recursive_flatten(lst)) -def group_by[T]( - seq: list[T], - key: Callable -) -> dict[Any, list[T]]: - return reduce(lambda grp, val: grp[key(val)].append(val) or grp, seq, defaultdict(list)) - - def group_indexes_by( seq: list, key: Callable diff --git a/pyproject.toml b/pyproject.toml index 4dc298c..1463d9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,11 +6,12 @@ build-backend = "pdm.backend" name = "funidata-utils" dynamic = ["version"] description = "A collection of utility packages for interacting with Funidata products" -requires-python = ">=3.12" +requires-python = ">=3.11" readme = "README.md" classifiers = [ "Development Status :: 3 - Alpha", "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python", diff --git a/tests/test_utils.py b/tests/helpers.py similarity index 100% rename from tests/test_utils.py rename to tests/helpers.py diff --git a/tests/test_import_batching_no_sublist.py b/tests/test_import_batching_no_sublist.py index 26f4141..a3550cb 100644 --- a/tests/test_import_batching_no_sublist.py +++ b/tests/test_import_batching_no_sublist.py @@ -1,7 +1,7 @@ import pytest from funidata_utils.request_utils.async_httpx_requests import _binary_search_enabled_post_httpx -from tests.test_utils import mock_client, get_entity_counts_by_status_code +from tests.helpers import mock_client, get_entity_counts_by_status_code @pytest.mark.asyncio diff --git a/tests/test_import_batching_with_sublist.py b/tests/test_import_batching_with_sublist.py index cf64d1f..0c2e420 100644 --- a/tests/test_import_batching_with_sublist.py +++ b/tests/test_import_batching_with_sublist.py @@ -1,7 +1,7 @@ import pytest from funidata_utils.request_utils.async_httpx_requests import _binary_search_enabled_post_httpx -from tests.test_utils import mock_client, get_entity_counts_by_status_code +from tests.helpers import mock_client, get_entity_counts_by_status_code @pytest.mark.asyncio diff --git a/tests/test_util.py b/tests/test_util.py new file mode 100644 index 0000000..3441656 --- /dev/null +++ b/tests/test_util.py @@ -0,0 +1,10 @@ +import pytest +from funidata_utils.utils import group_by + + +def test_group_by(): + data = [dict(id=1, type=1), dict(id=2, type=1), dict(id=3, type=2)] + + grouping = group_by(data, lambda x: x['type'] == 1) + + assert grouping[1] == [dict(id=1, type=1), dict(id=2, type=1)] From da4038b6b739616b9d7bcd6914649182d3d645a6 Mon Sep 17 00:00:00 2001 From: EssKayz Date: Thu, 25 Jun 2026 22:53:12 +0300 Subject: [PATCH 8/9] [tests] dockerized tests for python 3.11 ... 3.13 --- Dockerfile | 18 ++++++++++++++++++ docker-compose.yaml | 27 +++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 Dockerfile create mode 100644 docker-compose.yaml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..abc7ed2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +ARG PYTHON_VERSION=3.11 + + +FROM python:$PYTHON_VERSION + +WORKDIR /app + +COPY pyproject.toml . +COPY README.md . +COPY requirements* . + +RUN pip install -r requirements.txt +RUN pip install . + + +ENTRYPOINT ["python", "-m", "pytest"] + + diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..8a055ea --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,27 @@ +x-tests-common: &tests-common + volumes: + - ./tests:/app/tests + - ./funidata_utils:/app/funidata_utils + + +services: + pytest_311: + build: + context: . + args: + - PYTHON_VERSION=3.11 + <<: *tests-common + + pytest_312: + build: + context: . + args: + - PYTHON_VERSION=3.12 + <<: *tests-common + + pytest_313: + build: + context: . + args: + - PYTHON_VERSION=3.13 + <<: *tests-common From 3805e353af2820b827c3d26b38e52dfdf68e71f1 Mon Sep 17 00:00:00 2001 From: EssKayz Date: Fri, 26 Jun 2026 14:58:47 +0300 Subject: [PATCH 9/9] [sis-integration] flip param merge to not allow since/limit overriding --- funidata_utils/sis_integration/exports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/funidata_utils/sis_integration/exports.py b/funidata_utils/sis_integration/exports.py index 03adea0..d7b56b5 100644 --- a/funidata_utils/sis_integration/exports.py +++ b/funidata_utils/sis_integration/exports.py @@ -88,7 +88,7 @@ def export_from_endpoint_generator( sis_response = send_get_httpx( path=f"{sis_settings.host}{endpoint}", auth=sis_settings.get_export_auth(), - params={since: greatest_ordinal, 'limit': export_limit} | params, + params=params | {since: greatest_ordinal, 'limit': export_limit}, proxies=sis_settings.proxies, ) if sis_response.status_code == 200: