From ed66f61eaf8b8dc1e6ffe6a7593caf925c19dbed Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 19:08:31 +0000 Subject: [PATCH 1/2] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/pre-commit-hooks: v5.0.0 → v6.0.0](https://github.com/pre-commit/pre-commit-hooks/compare/v5.0.0...v6.0.0) - [github.com/astral-sh/ruff-pre-commit: v0.3.4 → v0.16.8](https://github.com/astral-sh/ruff-pre-commit/compare/v0.3.4...v0.16.8) - [github.com/asottile/pyupgrade: v3.17.0 → v3.21.2](https://github.com/asottile/pyupgrade/compare/v3.17.0...v3.21.2) - [github.com/igorshubovych/markdownlint-cli: v0.42.0 → v0.49.1](https://github.com/igorshubovych/markdownlint-cli/compare/v0.42.0...v0.49.1) - [github.com/gitleaks/gitleaks: v8.18.2 → v8.30.0](https://github.com/gitleaks/gitleaks/compare/v8.18.2...v8.30.0) - [github.com/hadolint/hadolint: v2.12.1-beta → v2.15.1](https://github.com/hadolint/hadolint/compare/v2.12.1-beta...v2.15.1) --- .pre-commit-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c709dded37..3860fe421d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ ci: - hadolint-docker # Fails in pre-commit CI repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: trailing-whitespace exclude_types: @@ -38,7 +38,7 @@ repos: - id: no-commit-to-branch - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.4 + rev: v0.16.8 hooks: - id: ruff args: [--fix] @@ -57,7 +57,7 @@ repos: # language: python - repo: https://github.com/asottile/pyupgrade - rev: v3.17.0 + rev: v3.21.2 hooks: - id: pyupgrade entry: pyupgrade --py39-plus --keep-runtime-typing @@ -111,7 +111,7 @@ repos: stages: [pre-commit] - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.42.0 + rev: v0.49.1 hooks: - id: markdownlint args: [--disable, MD013] @@ -119,7 +119,7 @@ repos: args: [--disable, MD013] - repo: https://github.com/gitleaks/gitleaks - rev: v8.18.2 + rev: v8.30.0 hooks: - id: gitleaks @@ -129,7 +129,7 @@ repos: - id: htmlhint - repo: https://github.com/hadolint/hadolint - rev: v2.12.1-beta + rev: v2.15.1 hooks: - id: hadolint-docker args: From 741d4c8a175a3693190321dd0c5003e3c254c91c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 19:09:10 +0000 Subject: [PATCH 2/2] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .claude/skills/adapter-ops/SKILL.md | 9 +++- .../references/adapter_patterns.md | 25 ++++++----- .../references/connector_patterns.md | 45 +++++++++++-------- .../connector-ops/references/test_patterns.md | 3 ++ backend/account_v2/custom_auth_middleware.py | 4 +- backend/account_v2/models.py | 3 +- .../adapter_processor_v2/adapter_processor.py | 14 +++--- backend/adapter_processor_v2/exceptions.py | 2 +- .../commands/manage_deprecated_adapters.py | 4 +- backend/adapter_processor_v2/models.py | 7 ++- backend/adapter_processor_v2/serializers.py | 8 ++-- .../commands/delete_org_rate_limit.py | 2 +- .../commands/generate_docstudio_spec.py | 2 +- .../management/commands/get_org_rate_limit.py | 4 +- .../commands/list_org_rate_limits.py | 2 +- .../management/commands/set_org_rate_limit.py | 2 +- backend/api_v2/serializers.py | 2 +- backend/backend/exceptions.py | 1 - backend/configuration/config_registry.py | 6 +-- backend/configuration/enums.py | 6 +-- backend/connector_auth_v2/pipeline/google.py | 1 - .../connector_processor.py | 15 +++---- backend/connector_processor/exceptions.py | 3 +- backend/connector_processor/serializers.py | 2 +- .../0003_migrate_to_centralized_connectors.py | 2 +- backend/connector_v2/models.py | 3 +- backend/connector_v2/serializers.py | 4 +- backend/connector_v2/views.py | 6 +-- backend/dashboard_metrics/README.md | 29 ++++++------ backend/dashboard_metrics/services.py | 2 +- backend/file_management/exceptions.py | 3 +- .../file_management/file_management_helper.py | 4 +- backend/file_management/views.py | 2 +- .../global_api_deployment_key/serializers.py | 2 +- backend/middleware/exception.py | 1 - .../v2/management/commands/migrate_to_v2.py | 2 +- backend/notification_v2/clubbed_renderer.py | 3 +- backend/notification_v2/helper.py | 2 +- backend/notification_v2/internal_api_views.py | 2 +- backend/notification_v2/internal_views.py | 2 +- backend/notification_v2/serializers.py | 3 +- backend/notification_v2/tasks.py | 1 + backend/pg_queue/executor_rpc.py | 6 +-- backend/pg_queue/models.py | 1 - backend/pg_queue/producer.py | 4 +- backend/pipeline_v2/exceptions.py | 2 +- backend/pipeline_v2/manager.py | 2 +- backend/pipeline_v2/models.py | 3 +- backend/pipeline_v2/serializers/crud.py | 5 +-- backend/platform_api/serializers.py | 2 +- backend/platform_settings_v2/serializers.py | 3 +- .../prompt_profile_manager_v2/serializers.py | 2 +- .../prompt_studio_core_v2/internal_views.py | 16 +++---- .../prompt_studio_core_v2/migration_utils.py | 3 +- .../prompt_studio_core_v2/models.py | 4 +- .../prompt_ide_base_tool.py | 6 +-- .../prompt_studio_helper.py | 21 +++++---- .../prompt_studio_core_v2/serializers.py | 4 +- .../prompt_studio_core_v2/views.py | 4 +- .../prompt_studio_index_manager_v2/models.py | 4 +- .../serializers.py | 2 +- .../prompt_studio_registry_helper.py | 2 +- .../prompt_studio_registry_v2/serializers.py | 3 +- .../prompt_studio_v2/serializers.py | 3 +- backend/scheduler/serializer.py | 2 +- backend/tenant_account_v2/models.py | 3 +- backend/tenant_account_v2/serializer.py | 2 +- backend/tool_instance_v2/serializers.py | 10 ++--- .../tool_instance_v2/tool_instance_helper.py | 10 ++--- backend/tool_instance_v2/tool_processor.py | 4 +- backend/tool_instance_v2/views.py | 5 +-- backend/usage_v2/internal_views.py | 3 +- backend/utils/FileValidator.py | 4 +- backend/utils/cache_service.py | 1 - backend/utils/dto.py | 1 - .../helpers/prompt_studio_file_helper.py | 2 +- backend/utils/log_events.py | 2 +- .../endpoint_v2/base_connector.py | 5 +-- .../endpoint_v2/database_utils.py | 9 ++-- .../endpoint_v2/destination.py | 10 ++--- .../endpoint_v2/queue_utils.py | 5 +-- .../workflow_manager/endpoint_v2/source.py | 6 +-- .../workflow_manager/internal_api_views.py | 2 +- .../workflow_manager/internal_serializers.py | 1 + backend/workflow_manager/internal_views.py | 10 +++-- .../workflow_manager/utils/workflow_log.py | 3 +- .../workflow_manager/workflow_v2/execution.py | 8 ++-- .../workflow_v2/filters/execution_log.py | 2 +- .../workflow_v2/serializers.py | 4 +- .../workflow_v2/undispatched_sweep.py | 1 - backend/workflow_manager/workflow_v2/views.py | 4 +- .../workflow_v2/workflow_helper.py | 12 ++--- .../release-notes/print_release_notes.py | 3 +- docs/API_DEPLOYMENT_RATE_LIMITING.md | 8 +++- .../src/unstract/platform_service/config.py | 1 - .../platform_service/controller/__init__.py | 3 +- .../platform_service/controller/platform.py | 1 - .../unstract/platform_service/extensions.py | 1 - runner/src/unstract/runner/__init__.py | 1 - .../unstract/runner/clients/docker_client.py | 3 +- .../unstract/runner/clients/test_docker.py | 1 - runner/src/unstract/runner/controller/run.py | 1 - runner/src/unstract/runner/runner.py | 1 - runner/src/unstract/runner/utils.py | 1 - tools/classifier/src/main.py | 1 - .../filesystems/google_drive/google_drive.py | 3 +- .../connectors/filesystems/http/http.py | 3 +- .../local_storage/local_storage.py | 1 - .../connectors/filesystems/sftp/sftp.py | 3 +- .../core/src/unstract/core/plugins/README.md | 1 - .../core/src/unstract/core/worker_models.py | 3 +- .../filesystem/file_storage_config.py | 3 +- .../src/unstract/flags/flipt_grpc/README.md | 41 +++++++---------- .../src/unstract/sdk1/adapters/adapterkit.py | 1 + .../sdk1/src/unstract/sdk1/adapters/base1.py | 1 + .../src/google_document_ai.py | 1 + .../sdk1/adapters/vectordb/exceptions.py | 1 + .../unstract/sdk1/adapters/vectordb/helper.py | 1 + .../adapters/vectordb/milvus/src/milvus.py | 1 + .../vectordb/no_op/src/no_op_vectordb.py | 1 + .../vectordb/pinecone/src/pinecone.py | 1 + .../vectordb/postgres/src/postgres.py | 1 + .../adapters/vectordb/qdrant/src/qdrant.py | 1 + .../vectordb/supabase/src/supabase.py | 1 + .../adapters/vectordb/vectordb_adapter.py | 1 + .../vectordb/weaviate/src/weaviate.py | 5 ++- .../unstract/sdk1/adapters/x2text/helper.py | 1 + .../x2text/llama_parse/src/llama_parse.py | 1 + .../x2text/llm_whisperer/src/llm_whisperer.py | 1 + .../x2text/llm_whisperer_v2/src/helper.py | 1 + unstract/sdk1/src/unstract/sdk1/audit.py | 1 + unstract/sdk1/src/unstract/sdk1/cache.py | 1 + unstract/sdk1/src/unstract/sdk1/embedding.py | 3 +- .../src/unstract/sdk1/file_storage/helper.py | 1 + .../src/unstract/sdk1/file_storage/impl.py | 1 + .../unstract/sdk1/file_storage/interface.py | 1 + .../unstract/sdk1/file_storage/permanent.py | 1 + unstract/sdk1/src/unstract/sdk1/index.py | 1 + unstract/sdk1/src/unstract/sdk1/llm.py | 1 + unstract/sdk1/src/unstract/sdk1/platform.py | 1 + unstract/sdk1/src/unstract/sdk1/tool/cache.py | 1 + .../sdk1/src/unstract/sdk1/tool/parser.py | 1 + .../sdk1/src/unstract/sdk1/tool/stream.py | 1 + .../sdk1/src/unstract/sdk1/tool/validator.py | 1 + .../sdk1/src/unstract/sdk1/usage_handler.py | 1 + .../unstract/sdk1/utils/callback_manager.py | 1 + unstract/sdk1/src/unstract/sdk1/vector_db.py | 1 + unstract/sdk1/src/unstract/sdk1/x2txt.py | 1 + .../sdk1/tests/file_storage/test_impl_rm.py | 5 ++- .../patches/test_litellm_cohere_timeout.py | 13 +++--- .../tests/test_anthropic_stream_complete.py | 1 + unstract/sdk1/tests/test_bedrock_adapter.py | 1 + .../tests/test_branded_openai_adapters.py | 1 + unstract/sdk1/tests/test_execution.py | 1 + unstract/sdk1/tests/test_gemini_adapter.py | 1 + unstract/sdk1/tests/test_gemini_embedding.py | 1 + unstract/sdk1/tests/test_llm_compat.py | 3 +- .../tests/test_llm_whisperer_v2_params.py | 1 + unstract/sdk1/tests/test_mistral_embedding.py | 1 + .../tests/test_openai_compatible_adapter.py | 13 +++--- unstract/sdk1/tests/test_platform.py | 1 + unstract/sdk1/tests/test_prompt_caching.py | 1 + unstract/sdk1/tests/test_sampling_strip.py | 7 +-- .../tests/utils/test_collect_with_retry.py | 1 + unstract/sdk1/tests/utils/test_retry_utils.py | 1 + workers/api-deployment/tasks.py | 3 +- workers/callback/tasks.py | 3 +- workers/executor/executors/file_utils.py | 1 - workers/executor/executors/index.py | 1 - workers/executor/executors/legacy_executor.py | 4 -- .../executor/executors/lookup_enrichment.py | 1 - workers/executor/executors/postprocessor.py | 1 - .../executors/retrievers/retriever_llm.py | 1 - workers/executor/tasks.py | 1 - .../file_processing/structure_tool_task.py | 1 - workers/file_processing/tasks.py | 1 - workers/general/tasks.py | 8 ++-- workers/log_consumer/process_log_history.py | 1 - workers/log_consumer/redis_stream_consumer.py | 1 - workers/log_consumer/tasks.py | 1 - workers/notification/providers/api_webhook.py | 1 - workers/notification/providers/registry.py | 1 - .../notification/providers/slack_webhook.py | 1 - .../providers/webhook_provider.py | 1 - workers/notification/tasks.py | 1 - workers/queue_backend/pg_barrier.py | 1 - workers/queue_backend/pg_queue/consumer.py | 1 - workers/queue_backend/pg_queue/metrics.py | 3 +- .../queue_backend/pg_queue/pg_scheduler.py | 1 - .../queue_backend/pg_queue/result_backend.py | 1 - workers/scheduler/tasks.py | 1 - workers/shared/api/internal_client.py | 1 - workers/shared/clients/usage_client.py | 1 - workers/shared/clients/workflow_client.py | 1 - .../infrastructure/caching/cache_utils.py | 1 - .../infrastructure/config/worker_config.py | 32 +++++++------ .../shared/infrastructure/logging/logger.py | 4 +- workers/shared/models/file_processing.py | 1 - workers/shared/models/result_models.py | 2 +- workers/shared/patterns/worker_patterns.py | 3 +- workers/shared/processing/files/processor.py | 1 - workers/shared/processing/files/utils.py | 3 +- workers/shared/utils/api_result_cache.py | 1 - workers/shared/utils/manual_review_factory.py | 1 - .../shared/workflow/destination_connector.py | 4 +- .../workflow/execution/active_file_manager.py | 2 +- .../workflow/execution/orchestration_utils.py | 1 - workers/shared/workflow/execution/service.py | 1 + .../workflow/execution/tool_validation.py | 1 - workers/shared/workflow/source_connector.py | 1 - 210 files changed, 374 insertions(+), 386 deletions(-) diff --git a/.claude/skills/adapter-ops/SKILL.md b/.claude/skills/adapter-ops/SKILL.md index 1a75912c5d..112e4ccb54 100644 --- a/.claude/skills/adapter-ops/SKILL.md +++ b/.claude/skills/adapter-ops/SKILL.md @@ -90,7 +90,9 @@ LiteLLM requires provider prefixes on model names: @staticmethod def validate(adapter_metadata: dict[str, "Any"]) -> dict[str, "Any"]: - adapter_metadata["model"] = NewProviderLLMParameters.validate_model(adapter_metadata) + adapter_metadata["model"] = NewProviderLLMParameters.validate_model( + adapter_metadata + ) return NewProviderLLMParameters(**adapter_metadata).model_dump() @staticmethod @@ -114,6 +116,7 @@ LiteLLM requires provider prefixes on model names: 5. **Test the adapter**: ```python from unstract.sdk1.adapters.adapterkit import Adapterkit + kit = Adapterkit() adapters = kit.get_adapters_list() # Verify new adapter appears @@ -142,7 +145,9 @@ LiteLLM requires provider prefixes on model names: @staticmethod def validate(adapter_metadata: dict[str, "Any"]) -> dict[str, "Any"]: - adapter_metadata["model"] = NewProviderEmbeddingParameters.validate_model(adapter_metadata) + adapter_metadata["model"] = NewProviderEmbeddingParameters.validate_model( + adapter_metadata + ) return NewProviderEmbeddingParameters(**adapter_metadata).model_dump() @staticmethod diff --git a/.claude/skills/adapter-ops/references/adapter_patterns.md b/.claude/skills/adapter-ops/references/adapter_patterns.md index c8e7e9a790..cdb432f3d8 100644 --- a/.claude/skills/adapter-ops/references/adapter_patterns.md +++ b/.claude/skills/adapter-ops/references/adapter_patterns.md @@ -253,7 +253,8 @@ def validate(adapter_metadata: dict[str, "Any"]) -> dict[str, "Any"]: # Exclude control fields before validation validation_metadata = { - k: v for k, v in adapter_metadata.items() + k: v + for k, v in adapter_metadata.items() if k not in ("enable_reasoning", "reasoning_effort") } @@ -327,8 +328,7 @@ def validate(adapter_metadata: dict[str, "Any"]) -> dict[str, "Any"]: # Check if thinking was previously enabled has_thinking_config = ( - "thinking" in adapter_metadata - and adapter_metadata.get("thinking") is not None + "thinking" in adapter_metadata and adapter_metadata.get("thinking") is not None ) if not enable_thinking and has_thinking_config: enable_thinking = True @@ -348,7 +348,8 @@ def validate(adapter_metadata: dict[str, "Any"]) -> dict[str, "Any"]: # Exclude control fields from validation validation_metadata = { - k: v for k, v in result_metadata.items() + k: v + for k, v in result_metadata.items() if k not in ("enable_thinking", "budget_tokens", "thinking") } @@ -570,10 +571,12 @@ adapter_class = kit.get_adapter_class_by_adapter_id( ) # Validate metadata -validated = adapter_class.validate({ - "model": "my-model", - "api_key": "sk-xxx", -}) +validated = adapter_class.validate( + { + "model": "my-model", + "api_key": "sk-xxx", + } +) print(validated) # Get JSON schema @@ -645,9 +648,11 @@ unconditionally prepends `custom_openai/`, and **no** LiteLLM cost-map key uses class MiniMaxLLMParameters(OpenAICompatibleLLMParameters): api_base: str = "https://api.minimax.io/v1" + # CORRECT - emits "minimax/MiniMax-M3", priced at $0.30 / $1.20 per 1M tokens -class MiniMaxLLMParameters(BaseChatCompletionParameters): - ... # follow OpenRouterLLMParameters +class MiniMaxLLMParameters( + BaseChatCompletionParameters +): ... # follow OpenRouterLLMParameters ``` Note that `get_provider()` returns `"minimax"` in *both* cases and matches `litellm_provider` diff --git a/.claude/skills/connector-ops/references/connector_patterns.md b/.claude/skills/connector-ops/references/connector_patterns.md index 2d8fa4f0ba..a2ab3b2856 100644 --- a/.claude/skills/connector-ops/references/connector_patterns.md +++ b/.claude/skills/connector-ops/references/connector_patterns.md @@ -28,6 +28,7 @@ class GoogleDriveFS(UnstractFileSystem): if self._client is None: # Double-check # Import heavy libraries here, not at module level from google.oauth2.credentials import Credentials + self._client = self._create_client() return self._client ``` @@ -77,6 +78,7 @@ def __init__(self, settings: dict[str, Any]): self.ssl_key = settings.get("sslKey", "") self.ssl_ca = settings.get("sslCA", "") + def get_engine(self): conn_params = {...} @@ -128,14 +130,10 @@ def test_credentials(self) -> bool: conn.close() return True except AuthenticationError as e: - raise ConnectorError( - f"Authentication failed: {e}", - treat_as_user_message=True - ) + raise ConnectorError(f"Authentication failed: {e}", treat_as_user_message=True) except ConnectionRefusedError as e: raise ConnectorError( - f"Connection refused - check host and port: {e}", - treat_as_user_message=True + f"Connection refused - check host and port: {e}", treat_as_user_message=True ) except Exception as e: raise ConnectorError(f"Connection error: {e}") @@ -186,9 +184,11 @@ def sql_to_db_mapping(self, value: Any, column_name: str | None = None) -> str: ```python from fsspec import AbstractFileSystem + class MyStorageFS(UnstractFileSystem): def get_fsspec_fs(self) -> AbstractFileSystem: from myfs import MyFileSystem + return MyFileSystem( key=self.access_key, secret=self.secret_key, @@ -202,7 +202,10 @@ class MyStorageFS(UnstractFileSystem): def is_dir_by_metadata(self, metadata: dict[str, Any]) -> bool: """Check if path is directory from metadata.""" - return metadata.get("type") == "directory" or metadata.get("StorageClass") == "DIRECTORY" + return ( + metadata.get("type") == "directory" + or metadata.get("StorageClass") == "DIRECTORY" + ) ``` --- @@ -220,8 +223,7 @@ SharePoint returned plain lists instead of dicts, breaking file processing silen **Bad** — reimplementing methods the base class already provides: ```python class MyFileSystem(AbstractFileSystem): - def ls(self, path, detail=True, **kwargs): - ... # Core method - MUST implement + def ls(self, path, detail=True, **kwargs): ... # Core method - MUST implement # BAD: These are all redundant reimplementations def listdir(self, path, detail=True, **kwargs): @@ -240,8 +242,9 @@ class MyFileSystem(AbstractFileSystem): def isdir(self, path): return self.info(path)["type"] == "directory" # Base already does this - def walk(self, path, maxdepth=None, **kwargs): - ... # Base already delegates to ls() with full detail/on_error support + def walk( + self, path, maxdepth=None, **kwargs + ): ... # Base already delegates to ls() with full detail/on_error support def delete(self, path, **kwargs): self.rm(path, **kwargs) # Base already does this @@ -249,8 +252,9 @@ class MyFileSystem(AbstractFileSystem): def read_bytes(self, path): return self.cat_file(path) # Base already does this - def write_bytes(self, path, data, **kwargs): - ... # Base already delegates to pipe_file() + def write_bytes( + self, path, data, **kwargs + ): ... # Base already delegates to pipe_file() ``` **Good** — implement only the core methods, let fsspec handle the rest: @@ -258,7 +262,7 @@ class MyFileSystem(AbstractFileSystem): class MyFileSystem(AbstractFileSystem): # REQUIRED: Core methods that talk to the service API def ls(self, path, detail=True, **kwargs): ... - def info(self, path, **kwargs): ... # Optional optimization + def info(self, path, **kwargs): ... # Optional optimization def _open(self, path, mode="rb", **kwargs): ... def cat_file(self, path, **kwargs): ... def pipe_file(self, path, value, **kwargs): ... # NOT write_bytes @@ -291,19 +295,21 @@ lists when False) and the `on_error` callback. from google.cloud import bigquery from google.oauth2 import service_account -class BigQueryDB(UnstractDB): - ... + +class BigQueryDB(UnstractDB): ... ``` **Good**: ```python # No heavy imports at module level + class BigQueryDB(UnstractDB): def get_engine(self): # Import when needed from google.cloud import bigquery from google.oauth2 import service_account + ... ``` @@ -341,6 +347,7 @@ def __init__(self, settings: dict[str, Any]): def get_id() -> str: return "postgres|abc123" + # Version 2.0 - DON'T DO THIS @staticmethod def get_id() -> str: @@ -362,9 +369,9 @@ def get_engine(self): conn_params = { # TCP keepalive settings "keepalives": 1, - "keepalives_idle": 30, # Seconds before sending keepalive + "keepalives_idle": 30, # Seconds before sending keepalive "keepalives_interval": 10, # Seconds between keepalives - "keepalives_count": 5, # Failed keepalives before disconnect + "keepalives_count": 5, # Failed keepalives before disconnect "connect_timeout": 30, # ... } @@ -441,7 +448,7 @@ def execute_batch(self, query: str, data: list[tuple], batch_size: int = 1000) - try: with engine.cursor() as cursor: for i in range(0, len(data), batch_size): - batch = data[i:i + batch_size] + batch = data[i : i + batch_size] cursor.executemany(query, batch) total_rows += len(batch) engine.commit() diff --git a/.claude/skills/connector-ops/references/test_patterns.md b/.claude/skills/connector-ops/references/test_patterns.md index 8943dc84fe..1162e3c52f 100644 --- a/.claude/skills/connector-ops/references/test_patterns.md +++ b/.claude/skills/connector-ops/references/test_patterns.md @@ -501,16 +501,19 @@ self.assertRegex(connector.get_id(), r"^[a-z_]+\|[a-f0-9-]+$") # JSON schema validity import json + schema = json.loads(connector.get_json_schema()) self.assertIn("title", schema) self.assertIn("type", schema) # Connection returns correct type from psycopg2.extensions import connection + self.assertIsInstance(connector.get_engine(), connection) # Exception handling from unstract.connectors.exceptions import ConnectorError + with self.assertRaises(ConnectorError) as ctx: connector.test_credentials() self.assertIn("expected message", str(ctx.exception)) diff --git a/backend/account_v2/custom_auth_middleware.py b/backend/account_v2/custom_auth_middleware.py index b69174fddb..1d4c18ceae 100644 --- a/backend/account_v2/custom_auth_middleware.py +++ b/backend/account_v2/custom_auth_middleware.py @@ -2,6 +2,8 @@ import logging import uuid +from backend.constants import RequestHeader +from backend.internal_api_constants import INTERNAL_API_PREFIX from django.conf import settings from django.http import HttpRequest, HttpResponse, JsonResponse from utils.constants import Account @@ -11,8 +13,6 @@ from account_v2.authentication_plugin_registry import AuthenticationPluginRegistry from account_v2.authentication_service import AuthenticationService from account_v2.constants import Common -from backend.constants import RequestHeader -from backend.internal_api_constants import INTERNAL_API_PREFIX logger = logging.getLogger(__name__) diff --git a/backend/account_v2/models.py b/backend/account_v2/models.py index e532a9a456..416b088847 100644 --- a/backend/account_v2/models.py +++ b/backend/account_v2/models.py @@ -1,10 +1,9 @@ import uuid +from backend.constants import FieldLengthConstants as FieldLength from django.contrib.auth.models import AbstractUser, Group, Permission from django.db import models -from backend.constants import FieldLengthConstants as FieldLength - NAME_SIZE = 64 KEY_SIZE = 64 diff --git a/backend/adapter_processor_v2/adapter_processor.py b/backend/adapter_processor_v2/adapter_processor.py index a9c7f40e32..61acbe5c7c 100644 --- a/backend/adapter_processor_v2/adapter_processor.py +++ b/backend/adapter_processor_v2/adapter_processor.py @@ -8,6 +8,13 @@ from django.core.exceptions import ObjectDoesNotExist from platform_settings_v2.platform_auth_service import PlatformAuthenticationService from tenant_account_v2.organization_member_service import OrganizationMemberService +from unstract.sdk1.adapters.adapterkit import Adapterkit +from unstract.sdk1.adapters.base import Adapter +from unstract.sdk1.adapters.x2text.constants import X2TextConstants +from unstract.sdk1.constants import AdapterTypes +from unstract.sdk1.embedding import EmbeddingCompat +from unstract.sdk1.exceptions import SdkError +from unstract.sdk1.llm import LLM from adapter_processor_v2.constants import AdapterKeys, AllowedDomains from adapter_processor_v2.exceptions import ( @@ -16,13 +23,6 @@ InValidAdapterId, TestAdapterError, ) -from unstract.sdk1.adapters.adapterkit import Adapterkit -from unstract.sdk1.adapters.base import Adapter -from unstract.sdk1.adapters.x2text.constants import X2TextConstants -from unstract.sdk1.constants import AdapterTypes -from unstract.sdk1.embedding import EmbeddingCompat -from unstract.sdk1.exceptions import SdkError -from unstract.sdk1.llm import LLM from .models import AdapterInstance, UserDefaultAdapter diff --git a/backend/adapter_processor_v2/exceptions.py b/backend/adapter_processor_v2/exceptions.py index 7aa2e586f4..da6901e244 100644 --- a/backend/adapter_processor_v2/exceptions.py +++ b/backend/adapter_processor_v2/exceptions.py @@ -1,9 +1,9 @@ import logging from rest_framework.exceptions import APIException +from unstract.sdk1.exceptions import SdkError from adapter_processor_v2.constants import AdapterKeys -from unstract.sdk1.exceptions import SdkError logger = logging.getLogger(__name__) diff --git a/backend/adapter_processor_v2/management/commands/manage_deprecated_adapters.py b/backend/adapter_processor_v2/management/commands/manage_deprecated_adapters.py index 609d16a990..9077157a5f 100644 --- a/backend/adapter_processor_v2/management/commands/manage_deprecated_adapters.py +++ b/backend/adapter_processor_v2/management/commands/manage_deprecated_adapters.py @@ -88,7 +88,7 @@ def _list_deprecated(self) -> None: ) for adapter in deprecated_adapters: - self.stdout.write(f"\n{'='*80}") + self.stdout.write(f"\n{'=' * 80}") self.stdout.write(f"UUID: {adapter.id}") self.stdout.write(f"Name: {adapter.adapter_name}") self.stdout.write(f"Type: {adapter.adapter_type}") @@ -261,7 +261,7 @@ def _generate_report(self) -> None: self.stdout.write(f"Total adapters: {total_adapters}") self.stdout.write(f"Deprecated adapters: {deprecated_count}") self.stdout.write( - f"Deprecation rate: {(deprecated_count/total_adapters*100):.1f}%\n" + f"Deprecation rate: {(deprecated_count / total_adapters * 100):.1f}%\n" if total_adapters > 0 else "N/A\n" ) diff --git a/backend/adapter_processor_v2/models.py b/backend/adapter_processor_v2/models.py index a2287aa4a3..744e79ca76 100644 --- a/backend/adapter_processor_v2/models.py +++ b/backend/adapter_processor_v2/models.py @@ -16,6 +16,9 @@ resources_visible_via_groups, resources_visible_via_memberships, ) +from unstract.sdk1.constants import AdapterTypes +from unstract.sdk1.exceptions import SdkError +from unstract.sdk1.llm import LLM from utils.exceptions import InvalidEncryptionKey from utils.models.base_model import BaseModel, BaseModelManager from utils.models.organization_mixin import ( @@ -23,10 +26,6 @@ DefaultOrganizationMixin, ) -from unstract.sdk1.constants import AdapterTypes -from unstract.sdk1.exceptions import SdkError -from unstract.sdk1.llm import LLM - logger = logging.getLogger(__name__) ADAPTER_NAME_SIZE = 128 diff --git a/backend/adapter_processor_v2/serializers.py b/backend/adapter_processor_v2/serializers.py index 3fee1f0cec..f9f6432deb 100644 --- a/backend/adapter_processor_v2/serializers.py +++ b/backend/adapter_processor_v2/serializers.py @@ -2,6 +2,8 @@ from typing import Any from account_v2.serializer import UserSerializer +from backend.constants import FieldLengthConstants as FLC +from backend.serializers import AuditSerializer from cryptography.fernet import Fernet from django.conf import settings from rest_framework import serializers @@ -10,14 +12,12 @@ serialize_group_refs, serialize_owner_refs, ) +from unstract.sdk1.constants import AdapterTypes +from unstract.sdk1.constants import Common as common from utils.input_sanitizer import validate_name_field, validate_no_html_tags from adapter_processor_v2.adapter_processor import AdapterProcessor from adapter_processor_v2.constants import AdapterKeys -from backend.constants import FieldLengthConstants as FLC -from backend.serializers import AuditSerializer -from unstract.sdk1.constants import AdapterTypes -from unstract.sdk1.constants import Common as common from .models import AdapterInstance, UserDefaultAdapter diff --git a/backend/api_v2/management/commands/delete_org_rate_limit.py b/backend/api_v2/management/commands/delete_org_rate_limit.py index dbd9de859b..bdec54804c 100644 --- a/backend/api_v2/management/commands/delete_org_rate_limit.py +++ b/backend/api_v2/management/commands/delete_org_rate_limit.py @@ -86,7 +86,7 @@ def handle(self, *args, **options): try: usage = APIDeploymentRateLimiter.get_current_usage(organization) self.stdout.write( - f'\nCurrent usage: {usage["org_count"]}/{default_limit} concurrent requests' + f"\nCurrent usage: {usage['org_count']}/{default_limit} concurrent requests" ) if usage["org_count"] >= default_limit: diff --git a/backend/api_v2/management/commands/generate_docstudio_spec.py b/backend/api_v2/management/commands/generate_docstudio_spec.py index be2278e806..4cb6a76f8e 100644 --- a/backend/api_v2/management/commands/generate_docstudio_spec.py +++ b/backend/api_v2/management/commands/generate_docstudio_spec.py @@ -85,7 +85,7 @@ def _restore_organisation_segment(schema: dict[str, Any]) -> None: for method, operation in item.items(): if method in HTTP_METHODS: operation.setdefault("parameters", []).append(dict(ORG_SEGMENT_PARAMETER)) - schema["paths"][f"{TENANT_MOUNT}{ORG_SEGMENT}/{url[len(TENANT_MOUNT):]}"] = item + schema["paths"][f"{TENANT_MOUNT}{ORG_SEGMENT}/{url[len(TENANT_MOUNT) :]}"] = item class SpecGenerationFailed(CommandError): diff --git a/backend/api_v2/management/commands/get_org_rate_limit.py b/backend/api_v2/management/commands/get_org_rate_limit.py index ab93073a30..183d8f8d98 100644 --- a/backend/api_v2/management/commands/get_org_rate_limit.py +++ b/backend/api_v2/management/commands/get_org_rate_limit.py @@ -65,10 +65,10 @@ def handle(self, *args, **options): self.stdout.write("\n--- Current Usage ---") self.stdout.write( - f'Organization: {usage["org_count"]}/{usage["org_limit"]} concurrent requests' + f"Organization: {usage['org_count']}/{usage['org_limit']} concurrent requests" ) self.stdout.write( - f'Global System: {usage["global_count"]}/{usage["global_limit"]} concurrent requests' + f"Global System: {usage['global_count']}/{usage['global_limit']} concurrent requests" ) # Usage percentage diff --git a/backend/api_v2/management/commands/list_org_rate_limits.py b/backend/api_v2/management/commands/list_org_rate_limits.py index 7c2d9284b6..3a9b543e8b 100644 --- a/backend/api_v2/management/commands/list_org_rate_limits.py +++ b/backend/api_v2/management/commands/list_org_rate_limits.py @@ -37,7 +37,7 @@ def handle(self, *args, **options): usage = APIDeploymentRateLimiter.get_current_usage(org) pct = (usage["org_count"] / limit * 100) if limit > 0 else 0 self.stdout.write( - f' Usage: {usage["org_count"]}/{limit} ({pct:.1f}%)' + f" Usage: {usage['org_count']}/{limit} ({pct:.1f}%)" ) except Exception as e: self.stdout.write(f" Usage: Error - {e}") diff --git a/backend/api_v2/management/commands/set_org_rate_limit.py b/backend/api_v2/management/commands/set_org_rate_limit.py index 6652ea775b..59aeeecc3b 100644 --- a/backend/api_v2/management/commands/set_org_rate_limit.py +++ b/backend/api_v2/management/commands/set_org_rate_limit.py @@ -55,7 +55,7 @@ def handle(self, *args, **options): usage = APIDeploymentRateLimiter.get_current_usage(organization) self.stdout.write( self.style.WARNING( - f'Current usage: {usage["org_count"]}/{limit} concurrent requests' + f"Current usage: {usage['org_count']}/{limit} concurrent requests" ) ) diff --git a/backend/api_v2/serializers.py b/backend/api_v2/serializers.py index 9e7bdf9369..92a3aef41a 100644 --- a/backend/api_v2/serializers.py +++ b/backend/api_v2/serializers.py @@ -4,6 +4,7 @@ from typing import Any from urllib.parse import urlparse +from backend.serializers import AuditSerializer from django.apps import apps from django.core.validators import RegexValidator from drf_spectacular.types import OpenApiTypes @@ -39,7 +40,6 @@ from api_v2.constants import ApiExecution from api_v2.models import APIDeployment, APIKey -from backend.serializers import AuditSerializer class APIDeploymentSerializer(IntegrityErrorMixin, AuditSerializer): diff --git a/backend/backend/exceptions.py b/backend/backend/exceptions.py index caf4df575b..c7ca940752 100644 --- a/backend/backend/exceptions.py +++ b/backend/backend/exceptions.py @@ -3,7 +3,6 @@ from rest_framework.exceptions import APIException from rest_framework.response import Response from rest_framework.views import exception_handler - from unstract.connectors.exceptions import ConnectorBaseException, ConnectorError diff --git a/backend/configuration/config_registry.py b/backend/configuration/config_registry.py index 2d9eef31da..022a00daef 100644 --- a/backend/configuration/config_registry.py +++ b/backend/configuration/config_registry.py @@ -165,9 +165,9 @@ def cast_value(cls, key_name: str, raw_value: Any) -> Any: converters = { ConfigType.INT: int, - ConfigType.BOOL: lambda v: v.lower() in ("true", "1") - if isinstance(v, str) - else bool(v), + ConfigType.BOOL: lambda v: ( + v.lower() in ("true", "1") if isinstance(v, str) else bool(v) + ), ConfigType.JSON: lambda v: __import__("json").loads(v), ConfigType.STRING: str, } diff --git a/backend/configuration/enums.py b/backend/configuration/enums.py index 78bbeb7dfd..c13172a8b4 100644 --- a/backend/configuration/enums.py +++ b/backend/configuration/enums.py @@ -72,9 +72,9 @@ class ConfigKey(Enum): def cast_value(self, raw_value: Any): converters = { ConfigType.INT: int, - ConfigType.BOOL: lambda v: v.lower() in ("true", "1") - if isinstance(v, str) - else bool(v), + ConfigType.BOOL: lambda v: ( + v.lower() in ("true", "1") if isinstance(v, str) else bool(v) + ), ConfigType.JSON: json.loads, ConfigType.STRING: str, } diff --git a/backend/connector_auth_v2/pipeline/google.py b/backend/connector_auth_v2/pipeline/google.py index d585bd0857..9b6bec3ec3 100644 --- a/backend/connector_auth_v2/pipeline/google.py +++ b/backend/connector_auth_v2/pipeline/google.py @@ -3,7 +3,6 @@ from connector_auth_v2.constants import SocialAuthConstants as AuthConstants from connector_auth_v2.exceptions import EnrichConnectorMetadataException from connector_processor.constants import ConnectorKeys - from unstract.connectors.filesystems.google_drive.constants import GDriveConstants diff --git a/backend/connector_processor/connector_processor.py b/backend/connector_processor/connector_processor.py index 332aaaeb48..9f0ecaa247 100644 --- a/backend/connector_processor/connector_processor.py +++ b/backend/connector_processor/connector_processor.py @@ -3,11 +3,16 @@ import logging from typing import Any +from backend.exceptions import UnstractFSException from connector_auth_v2.constants import ConnectorAuthKey from connector_auth_v2.pipeline.common import ConnectorAuthHelper from connector_v2.constants import ConnectorInstanceKey as CIKey +from unstract.connectors.base import UnstractConnector +from unstract.connectors.connectorkit import Connectorkit +from unstract.connectors.enums import ConnectorMode +from unstract.connectors.exceptions import ConnectorError, FSAccessDeniedError +from unstract.connectors.filesystems.ucs import UnstractCloudStorage -from backend.exceptions import UnstractFSException from connector_processor.constants import ConnectorKeys from connector_processor.exceptions import ( InvalidConnectorID, @@ -15,11 +20,6 @@ OAuthTimeOut, TestConnectorInputError, ) -from unstract.connectors.base import UnstractConnector -from unstract.connectors.connectorkit import Connectorkit -from unstract.connectors.enums import ConnectorMode -from unstract.connectors.exceptions import ConnectorError, FSAccessDeniedError -from unstract.connectors.filesystems.ucs import UnstractCloudStorage logger = logging.getLogger(__name__) @@ -145,8 +145,7 @@ def test_connectors(connector_id: str, credentials: dict[str, Any]) -> bool: ) except Exception as exc: logger.error( - "Error while testing file based OAuth supported " - "connectors: %s", + "Error while testing file based OAuth supported connectors: %s", exc, ) raise OAuthTimeOut() diff --git a/backend/connector_processor/exceptions.py b/backend/connector_processor/exceptions.py index 2deb364699..2b9cbc0b00 100644 --- a/backend/connector_processor/exceptions.py +++ b/backend/connector_processor/exceptions.py @@ -1,6 +1,5 @@ -from rest_framework.exceptions import APIException - from backend.exceptions import UnstractBaseException +from rest_framework.exceptions import APIException from unstract.connectors.exceptions import ConnectorError diff --git a/backend/connector_processor/serializers.py b/backend/connector_processor/serializers.py index bd0bcf0b8f..a2ca7d0075 100644 --- a/backend/connector_processor/serializers.py +++ b/backend/connector_processor/serializers.py @@ -1,6 +1,6 @@ +from backend.constants import FieldLengthConstants as FLC from rest_framework import serializers -from backend.constants import FieldLengthConstants as FLC from connector_processor.constants import ConnectorKeys diff --git a/backend/connector_v2/migrations/0003_migrate_to_centralized_connectors.py b/backend/connector_v2/migrations/0003_migrate_to_centralized_connectors.py index 73dd12f04b..9269767e47 100644 --- a/backend/connector_v2/migrations/0003_migrate_to_centralized_connectors.py +++ b/backend/connector_v2/migrations/0003_migrate_to_centralized_connectors.py @@ -221,7 +221,7 @@ def _fix_remaining_duplicate_names(connector_instance_model: Any) -> int: for i in range(0, len(duplicates), batch_size): batch = duplicates[i : i + batch_size] logger.info( - f"Processing batch {i//batch_size + 1}/{(len(duplicates)-1)//batch_size + 1}" + f"Processing batch {i // batch_size + 1}/{(len(duplicates) - 1) // batch_size + 1}" ) for dup_info in batch: diff --git a/backend/connector_v2/models.py b/backend/connector_v2/models.py index e0790c052c..855e27359c 100644 --- a/backend/connector_v2/models.py +++ b/backend/connector_v2/models.py @@ -3,6 +3,7 @@ from typing import Any from account_v2.models import User +from backend.constants import FieldLengthConstants as FLC from connector_auth_v2.models import ConnectorAuth from connector_processor.connector_processor import ConnectorProcessor from connector_processor.constants import ConnectorKeys @@ -21,8 +22,6 @@ DefaultOrganizationMixin, ) -from backend.constants import FieldLengthConstants as FLC - CONNECTOR_NAME_SIZE = 128 VERSION_NAME_SIZE = 64 diff --git a/backend/connector_v2/serializers.py b/backend/connector_v2/serializers.py index 8e5583e889..1fe475e259 100644 --- a/backend/connector_v2/serializers.py +++ b/backend/connector_v2/serializers.py @@ -2,6 +2,7 @@ from collections import OrderedDict from typing import Any +from backend.serializers import AuditSerializer from connector_auth_v2.constants import OAUTH_TOKEN_KEYS from connector_auth_v2.models import ConnectorAuth from connector_auth_v2.pipeline.common import ConnectorAuthHelper @@ -19,12 +20,11 @@ serialize_group_refs, serialize_owner_refs, ) +from unstract.connectors.filesystems.ucs import UnstractCloudStorage from utils.fields import EncryptedBinaryFieldSerializer from utils.input_sanitizer import validate_name_field -from backend.serializers import AuditSerializer from connector_v2.constants import ConnectorInstanceKey as CIKey -from unstract.connectors.filesystems.ucs import UnstractCloudStorage from .models import ConnectorInstance diff --git a/backend/connector_v2/views.py b/backend/connector_v2/views.py index 28f8038213..9c6d785a21 100644 --- a/backend/connector_v2/views.py +++ b/backend/connector_v2/views.py @@ -2,6 +2,7 @@ from typing import Any from account_v2.custom_exceptions import DuplicateData +from backend.constants import RequestKey from connector_auth_v2.constants import ConnectorAuthKey from connector_auth_v2.exceptions import CacheMissException, MissingParamException from connector_auth_v2.pipeline.common import ConnectorAuthHelper @@ -19,14 +20,13 @@ from rest_framework.response import Response from rest_framework.versioning import URLPathVersioning from tenant_account_v2.organization_member_service import OrganizationMemberService +from unstract.connectors.connectorkit import Connectorkit +from unstract.connectors.enums import ConnectorMode from utils.filtering import FilterHelper from utils.pagination import OptionalPagination from utils.user_context import UserContext -from backend.constants import RequestKey from connector_v2.constants import ConnectorInstanceKey as CIKey -from unstract.connectors.connectorkit import Connectorkit -from unstract.connectors.enums import ConnectorMode from .exceptions import DeleteConnectorInUseError from .models import ConnectorInstance diff --git a/backend/dashboard_metrics/README.md b/backend/dashboard_metrics/README.md index 61a61c40dc..b39e4387d3 100644 --- a/backend/dashboard_metrics/README.md +++ b/backend/dashboard_metrics/README.md @@ -322,11 +322,11 @@ All three tables share a similar schema: class EventMetricsHourly(Model): id = UUIDField(primary_key=True) organization = ForeignKey(Organization) - timestamp = DateTimeField() # Hour bucket (truncated) + timestamp = DateTimeField() # Hour bucket (truncated) metric_name = CharField(max_length=64) # e.g., "documents_processed" metric_type = CharField(choices=["counter", "histogram"]) - metric_value = FloatField() # Aggregated value - metric_count = IntegerField() # Number of events aggregated + metric_value = FloatField() # Aggregated value + metric_count = IntegerField() # Number of events aggregated project = CharField(default="default") tag = CharField(blank=True) created_at = DateTimeField(auto_now_add=True) @@ -399,9 +399,9 @@ The `aggregate_metrics_from_sources` task: ```python # Query windows -hourly_start = end_date - timedelta(hours=24) # Last 24 hours +hourly_start = end_date - timedelta(hours=24) # Last 24 hours daily_start = truncate_to_day(end_date - source_window_days) # 2 days, 7 on reconcile -monthly_start = first_of_previous_month # summed from daily +monthly_start = first_of_previous_month # summed from daily ``` The monthly tier has no source queries of its own. `backfill_metrics` still computes @@ -695,8 +695,7 @@ Used for `/overview/`, `/live-summary/`, `/live-series/` ```python @cache_metrics_response(endpoint="overview") -def overview(self, request): - ... +def overview(self, request): ... ``` TTLs: @@ -731,11 +730,11 @@ metrics:bucket:hourly::[:metric_name] In `settings.py`: ```python -DASHBOARD_CACHE_TTL_CURRENT_HOUR = 30 # 30 seconds -DASHBOARD_CACHE_TTL_HISTORICAL = 28800 # 8 hours -DASHBOARD_CACHE_TTL_OVERVIEW = 300 # 5 minutes -DASHBOARD_CACHE_TTL_SUMMARY = 900 # 15 minutes -DASHBOARD_CACHE_TTL_SERIES = 1800 # 30 minutes +DASHBOARD_CACHE_TTL_CURRENT_HOUR = 30 # 30 seconds +DASHBOARD_CACHE_TTL_HISTORICAL = 28800 # 8 hours +DASHBOARD_CACHE_TTL_OVERVIEW = 300 # 5 minutes +DASHBOARD_CACHE_TTL_SUMMARY = 900 # 15 minutes +DASHBOARD_CACHE_TTL_SERIES = 1800 # 30 minutes ``` --- @@ -1068,15 +1067,13 @@ When adding new metrics: 1. **Add query method** in `services.py`: ```python @staticmethod - def get_new_metric(org_id, start, end, granularity): - ... + def get_new_metric(org_id, start, end, granularity): ... ``` 2. **Add to metric configs** in `tasks.py`: ```python metric_configs = [ - ... - ("new_metric", MetricsQueryService.get_new_metric, False), + ...("new_metric", MetricsQueryService.get_new_metric, False), ] ``` diff --git a/backend/dashboard_metrics/services.py b/backend/dashboard_metrics/services.py index f8591c4f07..4e0185811f 100644 --- a/backend/dashboard_metrics/services.py +++ b/backend/dashboard_metrics/services.py @@ -17,6 +17,7 @@ from django.db.models import CharField, Count, OuterRef, Q, Subquery, Sum from django.db.models.functions import Cast, Coalesce, TruncDay, TruncHour, TruncWeek from pipeline_v2.models import Pipeline +from unstract.core.data_models import ExecutionStatus from usage_v2.models import Usage from workflow_manager.execution.enum import ExecutionEntity from workflow_manager.file_execution.models import WorkflowFileExecution @@ -24,7 +25,6 @@ from workflow_manager.workflow_v2.models.workflow import Workflow from dashboard_metrics.models import Granularity -from unstract.core.data_models import ExecutionStatus def _get_hitl_queue_model(): diff --git a/backend/file_management/exceptions.py b/backend/file_management/exceptions.py index 75f45ed30a..b7254be9f0 100644 --- a/backend/file_management/exceptions.py +++ b/backend/file_management/exceptions.py @@ -1,6 +1,5 @@ -from rest_framework.exceptions import APIException - from backend.exceptions import UnstractBaseException +from rest_framework.exceptions import APIException class MissingConnectorParams(APIException): diff --git a/backend/file_management/file_management_helper.py b/backend/file_management/file_management_helper.py index 51cc141809..871c11dc3c 100644 --- a/backend/file_management/file_management_helper.py +++ b/backend/file_management/file_management_helper.py @@ -12,6 +12,8 @@ from django.http import StreamingHttpResponse from fsspec import AbstractFileSystem from pydrive2.files import ApiRequestError +from unstract.connectors.filesystems import connectors as fs_connectors +from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem from file_management.exceptions import ( ConnectorApiRequestError, @@ -25,8 +27,6 @@ TenantDirCreationError, ) from file_management.file_management_dto import FileInformation -from unstract.connectors.filesystems import connectors as fs_connectors -from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem logger = logging.getLogger(__name__) diff --git a/backend/file_management/views.py b/backend/file_management/views.py index ea39cb1d28..f09acf61a2 100644 --- a/backend/file_management/views.py +++ b/backend/file_management/views.py @@ -8,6 +8,7 @@ from rest_framework.decorators import action from rest_framework.response import Response from rest_framework.versioning import URLPathVersioning +from unstract.connectors.exceptions import ConnectorError from file_management.exceptions import ( ConnectorInstanceNotFound, @@ -20,7 +21,6 @@ FileListRequestSerializer, FileUploadSerializer, ) -from unstract.connectors.exceptions import ConnectorError logger = logging.getLogger(__name__) diff --git a/backend/global_api_deployment_key/serializers.py b/backend/global_api_deployment_key/serializers.py index 0f22f83f73..f9dcfaac60 100644 --- a/backend/global_api_deployment_key/serializers.py +++ b/backend/global_api_deployment_key/serializers.py @@ -1,9 +1,9 @@ from api_v2.models import APIDeployment +from backend.serializers import AuditSerializer from rest_framework import serializers from utils.input_sanitizer import validate_safe_text from utils.user_context import UserContext -from backend.serializers import AuditSerializer from global_api_deployment_key.models import GlobalApiDeploymentKey ALLOW_ALL_WITH_LIST_ERROR = { diff --git a/backend/middleware/exception.py b/backend/middleware/exception.py index ce6b1abadf..aeeb815acf 100644 --- a/backend/middleware/exception.py +++ b/backend/middleware/exception.py @@ -8,7 +8,6 @@ from drf_standardized_errors.handler import exception_handler from rest_framework.request import Request from rest_framework.response import Response - from unstract.sdk1.exceptions import SdkError logger = logging.getLogger(__name__) diff --git a/backend/migrating/v2/management/commands/migrate_to_v2.py b/backend/migrating/v2/management/commands/migrate_to_v2.py index 1b8d5a8bc7..513923cade 100644 --- a/backend/migrating/v2/management/commands/migrate_to_v2.py +++ b/backend/migrating/v2/management/commands/migrate_to_v2.py @@ -367,7 +367,7 @@ def _bump_auto_increment_id( if max_id is None: logger.info( - f"Table '{dest_table}' is empty. No need to adjust " "auto-increment." + f"Table '{dest_table}' is empty. No need to adjust auto-increment." ) return diff --git a/backend/notification_v2/clubbed_renderer.py b/backend/notification_v2/clubbed_renderer.py index b94667c190..c132ed89d9 100644 --- a/backend/notification_v2/clubbed_renderer.py +++ b/backend/notification_v2/clubbed_renderer.py @@ -13,13 +13,14 @@ import logging from typing import Any -from notification_v2.enums import PlatformType from unstract.core.notification_clubbed_renderer import ( MAX_BATCH_SIZE, build_envelope, render_slack_text, ) +from notification_v2.enums import PlatformType + logger = logging.getLogger(__name__) # build_envelope is imported for internal use below; it is intentionally not diff --git a/backend/notification_v2/helper.py b/backend/notification_v2/helper.py index c50159def0..346a837fb0 100644 --- a/backend/notification_v2/helper.py +++ b/backend/notification_v2/helper.py @@ -7,6 +7,7 @@ from account_v2.models import Organization from django.utils import timezone +from unstract.core.data_models import is_failure_run as _core_is_failure_run from notification_v2.enums import ( AuthorizationType, @@ -14,7 +15,6 @@ PlatformType, ) from notification_v2.models import Notification, NotificationBuffer -from unstract.core.data_models import is_failure_run as _core_is_failure_run logger = logging.getLogger(__name__) diff --git a/backend/notification_v2/internal_api_views.py b/backend/notification_v2/internal_api_views.py index 4b24cf98df..938b663e33 100644 --- a/backend/notification_v2/internal_api_views.py +++ b/backend/notification_v2/internal_api_views.py @@ -62,7 +62,7 @@ def _load_execution(execution_id: str | None) -> WorkflowExecution | None: if not execution_id: return None try: - return cast(WorkflowExecution, WorkflowExecution.objects.get(id=execution_id)) + return cast("WorkflowExecution", WorkflowExecution.objects.get(id=execution_id)) except WorkflowExecution.DoesNotExist: # Catch ONLY DoesNotExist: a missing row is the fail-closed case # (_apply_failure_filter then drops notify_on_failures rows). A malformed diff --git a/backend/notification_v2/internal_views.py b/backend/notification_v2/internal_views.py index 507fb17c6d..1eb602a77f 100644 --- a/backend/notification_v2/internal_views.py +++ b/backend/notification_v2/internal_views.py @@ -11,6 +11,7 @@ from rest_framework.decorators import action from rest_framework.response import Response from rest_framework.views import APIView +from unstract.core.network.ssrf import is_safe_webhook_url from utils.organization_utils import organization_from_request from notification_v2.enums import AuthorizationType, NotificationType, PlatformType @@ -23,7 +24,6 @@ WebhookTestSerializer, ) from notification_v2.models import Notification -from unstract.core.network.ssrf import is_safe_webhook_url logger = logging.getLogger(__name__) diff --git a/backend/notification_v2/serializers.py b/backend/notification_v2/serializers.py index efa2a6abcb..35d9bc5c78 100644 --- a/backend/notification_v2/serializers.py +++ b/backend/notification_v2/serializers.py @@ -1,9 +1,8 @@ from rest_framework import serializers +from unstract.core.network.ssrf import is_safe_webhook_url from utils.input_sanitizer import validate_name_field from utils.serializer import ModelSerializer -from unstract.core.network.ssrf import is_safe_webhook_url - from .enums import AuthorizationType, NotificationType, PlatformType from .models import Notification diff --git a/backend/notification_v2/tasks.py b/backend/notification_v2/tasks.py index fc98cbb0da..734edd825e 100644 --- a/backend/notification_v2/tasks.py +++ b/backend/notification_v2/tasks.py @@ -22,6 +22,7 @@ from typing import Any from backend.celery_service import app as celery_app + from notification_v2.enums import BufferStatus from notification_v2.models import NotificationBuffer diff --git a/backend/pg_queue/executor_rpc.py b/backend/pg_queue/executor_rpc.py index b76b057871..57e7acbe00 100644 --- a/backend/pg_queue/executor_rpc.py +++ b/backend/pg_queue/executor_rpc.py @@ -17,9 +17,6 @@ from typing import TYPE_CHECKING from django.db import close_old_connections - -from pg_queue.models import PgTaskResult -from pg_queue.producer import enqueue_task from unstract.core.polling import poll_for_row from unstract.workflow_execution.executor_rpc import ( EXECUTE_TASK, @@ -28,6 +25,9 @@ QueueTransport, ) +from pg_queue.models import PgTaskResult +from pg_queue.producer import enqueue_task + if TYPE_CHECKING: from unstract.core.data_models import ContinuationSpec from unstract.sdk1.execution.context import ExecutionContext diff --git a/backend/pg_queue/models.py b/backend/pg_queue/models.py index 5720cc2b6c..4110bdb4b2 100644 --- a/backend/pg_queue/models.py +++ b/backend/pg_queue/models.py @@ -1,7 +1,6 @@ from django.db import models from django.db.models import F from django.utils import timezone - from unstract.core.data_models import QueueMessageState # Local aliases so the model reads cleanly; the enum is the single source of truth diff --git a/backend/pg_queue/producer.py b/backend/pg_queue/producer.py index 58cbac5a7e..6ac8debe82 100644 --- a/backend/pg_queue/producer.py +++ b/backend/pg_queue/producer.py @@ -22,8 +22,6 @@ from typing import Any from django.utils import timezone - -from pg_queue.models import PgQueueMessage from unstract.core.data_models import ( FAIRNESS_DEFAULT_PRIORITY, FAIRNESS_MAX_PRIORITY, @@ -35,6 +33,8 @@ ) from unstract.core.jsonb import dumps_for_jsonb +from pg_queue.models import PgQueueMessage + _READY = QueueMessageState.READY.value _SCHEDULED = QueueMessageState.SCHEDULED.value diff --git a/backend/pipeline_v2/exceptions.py b/backend/pipeline_v2/exceptions.py index 8b87f661a5..011bf74db5 100644 --- a/backend/pipeline_v2/exceptions.py +++ b/backend/pipeline_v2/exceptions.py @@ -28,7 +28,7 @@ def __init__( ): if pipeline_name: self.default_detail = ( - f"Pipeline '{pipeline_name}' is inactive, " "please activate the pipeline" + f"Pipeline '{pipeline_name}' is inactive, please activate the pipeline" ) super().__init__(detail, code) diff --git a/backend/pipeline_v2/manager.py b/backend/pipeline_v2/manager.py index 9be9d80317..e979b74a26 100644 --- a/backend/pipeline_v2/manager.py +++ b/backend/pipeline_v2/manager.py @@ -1,6 +1,7 @@ import logging from typing import Any +from backend.constants import RequestHeader from django.conf import settings from django.urls import reverse from rest_framework.request import Request @@ -9,7 +10,6 @@ from workflow_manager.workflow_v2.constants import WorkflowExecutionKey, WorkflowKey from workflow_manager.workflow_v2.views import WorkflowViewSet -from backend.constants import RequestHeader from pipeline_v2.constants import PipelineKey, PipelineURL from pipeline_v2.models import Pipeline from pipeline_v2.pipeline_processor import PipelineProcessor diff --git a/backend/pipeline_v2/models.py b/backend/pipeline_v2/models.py index a92f030272..eceb348b25 100644 --- a/backend/pipeline_v2/models.py +++ b/backend/pipeline_v2/models.py @@ -1,6 +1,7 @@ import uuid from account_v2.models import User +from backend.constants import FieldLengthConstants as FieldLength from django.conf import settings from django.contrib.contenttypes.fields import GenericRelation from django.db import models @@ -19,8 +20,6 @@ from utils.user_context import UserContext from workflow_manager.workflow_v2.models.workflow import Workflow -from backend.constants import FieldLengthConstants as FieldLength - APP_ID_LENGTH = 32 PIPELINE_NAME_LENGTH = 32 diff --git a/backend/pipeline_v2/serializers/crud.py b/backend/pipeline_v2/serializers/crud.py index 1d93fdd123..3592c1c491 100644 --- a/backend/pipeline_v2/serializers/crud.py +++ b/backend/pipeline_v2/serializers/crud.py @@ -3,6 +3,7 @@ from datetime import datetime from typing import Any +from backend.serializers import AuditSerializer from croniter import croniter from django.conf import settings from django.utils import timezone @@ -14,15 +15,13 @@ from rest_framework import serializers from rest_framework.serializers import SerializerMethodField, ValidationError from scheduler.helper import SchedulerHelper +from unstract.connectors.connectorkit import Connectorkit from utils.serializer.integrity_error_mixin import IntegrityErrorMixin from utils.serializer_utils import SerializerUtils from workflow_manager.endpoint_v2.models import WorkflowEndpoint from workflow_manager.workflow_v2.models.execution import WorkflowExecution from workflow_manager.workflow_v2.models.workflow import Workflow -from backend.serializers import AuditSerializer -from unstract.connectors.connectorkit import Connectorkit - logger = logging.getLogger(__name__) DEPLOYMENT_ENDPOINT = settings.API_DEPLOYMENT_PATH_PREFIX + "/pipeline" diff --git a/backend/platform_api/serializers.py b/backend/platform_api/serializers.py index e3edadabd5..a9f750f6bc 100644 --- a/backend/platform_api/serializers.py +++ b/backend/platform_api/serializers.py @@ -1,8 +1,8 @@ +from backend.serializers import AuditSerializer from rest_framework import serializers from utils.input_sanitizer import validate_safe_text from utils.user_context import UserContext -from backend.serializers import AuditSerializer from platform_api.models import PlatformApiKey diff --git a/backend/platform_settings_v2/serializers.py b/backend/platform_settings_v2/serializers.py index 24bd93ec5c..3595020213 100644 --- a/backend/platform_settings_v2/serializers.py +++ b/backend/platform_settings_v2/serializers.py @@ -1,7 +1,6 @@ from account_v2.models import PlatformKey -from rest_framework import serializers - from backend.serializers import AuditSerializer +from rest_framework import serializers class PlatformKeySerializer(AuditSerializer): diff --git a/backend/prompt_studio/prompt_profile_manager_v2/serializers.py b/backend/prompt_studio/prompt_profile_manager_v2/serializers.py index e26b20fd1f..2a7e9edf45 100644 --- a/backend/prompt_studio/prompt_profile_manager_v2/serializers.py +++ b/backend/prompt_studio/prompt_profile_manager_v2/serializers.py @@ -3,9 +3,9 @@ from adapter_processor_v2.adapter_processor import AdapterProcessor from adapter_processor_v2.models import AdapterInstance +from backend.serializers import AuditSerializer from rest_framework.serializers import ValidationError -from backend.serializers import AuditSerializer from prompt_studio.prompt_profile_manager_v2.constants import ProfileManagerKeys from .models import ProfileManager diff --git a/backend/prompt_studio/prompt_studio_core_v2/internal_views.py b/backend/prompt_studio/prompt_studio_core_v2/internal_views.py index 7d93e298aa..c49eb97ea2 100644 --- a/backend/prompt_studio/prompt_studio_core_v2/internal_views.py +++ b/backend/prompt_studio/prompt_studio_core_v2/internal_views.py @@ -46,7 +46,7 @@ def _resolve_profile(profile_manager_id): return ProfileManager.objects.get(pk=profile_manager_id), None except (ProfileManager.DoesNotExist, ValidationError): logger.error( - "Profile manager %s not found or not visible in the current " "organization.", + "Profile manager %s not found or not visible in the current organization.", profile_manager_id, ) return None, JsonResponse( @@ -89,8 +89,7 @@ def _validated_uuids(raw_ids, field_name): { "success": False, "error": ( - f"'{field_name}' must be a JSON array, got " - f"{type(raw_ids).__name__}." + f"'{field_name}' must be a JSON array, got {type(raw_ids).__name__}." ), }, status=status.HTTP_400_BAD_REQUEST, @@ -337,8 +336,7 @@ def extraction_status(request): { "success": False, "error": ( - "document_id, profile_manager_id, and x2text_config_hash " - "are required" + "document_id, profile_manager_id, and x2text_config_hash are required" ), }, status=status.HTTP_400_BAD_REQUEST, @@ -590,15 +588,15 @@ def summary_index_key(request): ) try: + from unstract.sdk1.constants import LogLevel + from unstract.sdk1.file_storage.constants import StorageType + from unstract.sdk1.file_storage.env_helper import EnvHelper + from unstract.sdk1.utils.indexing import IndexingUtils from utils.file_storage.constants import FileStorageKeys from prompt_studio.prompt_studio_core_v2.prompt_ide_base_tool import ( PromptIdeBaseTool, ) - from unstract.sdk1.constants import LogLevel - from unstract.sdk1.file_storage.constants import StorageType - from unstract.sdk1.file_storage.env_helper import EnvHelper - from unstract.sdk1.utils.indexing import IndexingUtils profile, err = _resolve_profile(summary_profile_id) if err: diff --git a/backend/prompt_studio/prompt_studio_core_v2/migration_utils.py b/backend/prompt_studio/prompt_studio_core_v2/migration_utils.py index 500f4ff4f5..a03558bb59 100644 --- a/backend/prompt_studio/prompt_studio_core_v2/migration_utils.py +++ b/backend/prompt_studio/prompt_studio_core_v2/migration_utils.py @@ -76,8 +76,7 @@ def migrate_tool_to_adapter_based( ) else: logger.info( - "No summarize profile found for tool %s, skipping " - "migration", + "No summarize profile found for tool %s, skipping migration", tool_instance.tool_id, ) return False diff --git a/backend/prompt_studio/prompt_studio_core_v2/models.py b/backend/prompt_studio/prompt_studio_core_v2/models.py index 4403de75db..5646fbc994 100644 --- a/backend/prompt_studio/prompt_studio_core_v2/models.py +++ b/backend/prompt_studio/prompt_studio_core_v2/models.py @@ -13,6 +13,8 @@ resources_visible_via_groups, resources_visible_via_memberships, ) +from unstract.sdk1.file_storage.constants import StorageType +from unstract.sdk1.file_storage.env_helper import EnvHelper from utils.file_storage.constants import FileStorageKeys from utils.file_storage.helpers.prompt_studio_file_helper import PromptStudioFileHelper from utils.models.base_model import BaseModel, BaseModelManager @@ -22,8 +24,6 @@ ) from prompt_studio.prompt_studio_core_v2.constants import DefaultPrompts -from unstract.sdk1.file_storage.constants import StorageType -from unstract.sdk1.file_storage.env_helper import EnvHelper logger = logging.getLogger(__name__) diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_ide_base_tool.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_ide_base_tool.py index 536bb16eb1..c1e0c46aab 100644 --- a/backend/prompt_studio/prompt_studio_core_v2/prompt_ide_base_tool.py +++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_ide_base_tool.py @@ -1,13 +1,13 @@ import os from platform_settings_v2.platform_auth_service import PlatformAuthenticationService -from utils.file_storage.constants import FileStorageKeys - -from prompt_studio.prompt_studio_core_v2.constants import ToolStudioKeys from unstract.sdk1.constants import LogLevel from unstract.sdk1.file_storage.constants import StorageType from unstract.sdk1.file_storage.env_helper import EnvHelper from unstract.sdk1.tool.stream import StreamMixin +from utils.file_storage.constants import FileStorageKeys + +from prompt_studio.prompt_studio_core_v2.constants import ToolStudioKeys class PromptIdeBaseTool(StreamMixin): diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py index a1d1cfb543..77b8fa0a47 100644 --- a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py +++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py @@ -21,6 +21,14 @@ from rest_framework.exceptions import APIException from rest_framework.request import Request from tenant_account_v2.organization_member_service import OrganizationMemberService +from unstract.core.pubsub_helper import LogPublisher +from unstract.sdk1.constants import LogLevel +from unstract.sdk1.exceptions import IndexingError, SdkError +from unstract.sdk1.execution.context import ExecutionContext +from unstract.sdk1.file_storage.constants import StorageType +from unstract.sdk1.file_storage.env_helper import EnvHelper +from unstract.sdk1.utils.indexing import IndexingUtils +from unstract.sdk1.utils.tool import ToolUtils from utils.file_storage.constants import FileStorageKeys from utils.file_storage.helpers.prompt_studio_file_helper import PromptStudioFileHelper from utils.local_context import StateStore @@ -75,14 +83,6 @@ OutputManagerHelper, ) from prompt_studio.prompt_studio_v2.models import ToolStudioPrompt -from unstract.core.pubsub_helper import LogPublisher -from unstract.sdk1.constants import LogLevel -from unstract.sdk1.exceptions import IndexingError, SdkError -from unstract.sdk1.execution.context import ExecutionContext -from unstract.sdk1.file_storage.constants import StorageType -from unstract.sdk1.file_storage.env_helper import EnvHelper -from unstract.sdk1.utils.indexing import IndexingUtils -from unstract.sdk1.utils.tool import ToolUtils logger = logging.getLogger(__name__) @@ -272,7 +272,7 @@ def validate_profile_manager_owner_access( # The requester IS the creator — "created by another user" # would be false, and they can be addressed directly. adapter_ref = ( - f"the adapter '{denied_names}', which you no longer have" f" access to" + f"the adapter '{denied_names}', which you no longer have access to" if len(denied) == 1 else f"adapters you no longer have access to: {denied_names}" ) @@ -3107,8 +3107,7 @@ def import_prompts(prompts_data: list, new_tool: CustomTool, user) -> None: ) except Exception as e: logger.warning( - f"Failed to import settings for prompt " - f"{created.prompt_id}: {e}" + f"Failed to import settings for prompt {created.prompt_id}: {e}" ) @staticmethod diff --git a/backend/prompt_studio/prompt_studio_core_v2/serializers.py b/backend/prompt_studio/prompt_studio_core_v2/serializers.py index 3ee9291065..4051e7840e 100644 --- a/backend/prompt_studio/prompt_studio_core_v2/serializers.py +++ b/backend/prompt_studio/prompt_studio_core_v2/serializers.py @@ -3,6 +3,7 @@ from account_v2.serializer import UserSerializer from adapter_processor_v2.models import AdapterInstance +from backend.serializers import AuditSerializer from django.core.exceptions import ObjectDoesNotExist from rest_framework import serializers from rest_framework.exceptions import PermissionDenied, ValidationError @@ -11,11 +12,11 @@ serialize_group_refs, serialize_owner_refs, ) +from unstract.sdk1.adapters.enums import AdapterTypes from utils.FileValidator import FileValidator from utils.input_sanitizer import validate_name_field from utils.serializer.integrity_error_mixin import IntegrityErrorMixin -from backend.serializers import AuditSerializer from prompt_studio.prompt_profile_manager_v2.models import ProfileManager from prompt_studio.prompt_studio_core_v2.constants import ToolStudioKeys as TSKeys from prompt_studio.prompt_studio_core_v2.exceptions import DefaultProfileError @@ -24,7 +25,6 @@ ) from prompt_studio.prompt_studio_v2.models import ToolStudioPrompt from prompt_studio.prompt_studio_v2.serializers import ToolStudioPromptSerializer -from unstract.sdk1.adapters.enums import AdapterTypes from .models import CustomTool diff --git a/backend/prompt_studio/prompt_studio_core_v2/views.py b/backend/prompt_studio/prompt_studio_core_v2/views.py index 2179253ec8..421c647286 100644 --- a/backend/prompt_studio/prompt_studio_core_v2/views.py +++ b/backend/prompt_studio/prompt_studio_core_v2/views.py @@ -26,6 +26,8 @@ from rest_framework.request import Request from rest_framework.response import Response from rest_framework.versioning import URLPathVersioning +from unstract.core.data_models import PgTaskStatus +from unstract.sdk1.utils.common import Utils as CommonUtils from utils.file_storage.helpers.prompt_studio_file_helper import PromptStudioFileHelper from utils.hubspot_notify import notify_hubspot_event from utils.pagination import OptionalPagination @@ -84,8 +86,6 @@ deployment_types_for, join_deployment_types, ) -from unstract.core.data_models import PgTaskStatus -from unstract.sdk1.utils.common import Utils as CommonUtils from .models import CustomTool from .serializers import ( diff --git a/backend/prompt_studio/prompt_studio_index_manager_v2/models.py b/backend/prompt_studio/prompt_studio_index_manager_v2/models.py index 6d84ed832b..456703bafb 100644 --- a/backend/prompt_studio/prompt_studio_index_manager_v2/models.py +++ b/backend/prompt_studio/prompt_studio_index_manager_v2/models.py @@ -6,6 +6,8 @@ from django.db import models from django.db.models.signals import pre_delete from django.dispatch import receiver +from unstract.sdk1.constants import LogLevel +from unstract.sdk1.vector_db import VectorDB from utils.models.base_model import BaseModel from utils.models.org_aware_manager import OrgAwareManager from utils.user_context import UserContext @@ -13,8 +15,6 @@ from prompt_studio.prompt_profile_manager_v2.models import ProfileManager from prompt_studio.prompt_studio_core_v2.prompt_ide_base_tool import PromptIdeBaseTool from prompt_studio.prompt_studio_document_manager_v2.models import DocumentManager -from unstract.sdk1.constants import LogLevel -from unstract.sdk1.vector_db import VectorDB logger = logging.getLogger(__name__) diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py b/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py index d484391a6d..f2efb9eaf3 100644 --- a/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py +++ b/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py @@ -1,9 +1,9 @@ import json import logging +from backend.serializers import AuditSerializer from usage_v2.helper import UsageHelper -from backend.serializers import AuditSerializer from prompt_studio.lookup_utils import enrich_prompt_output from .models import PromptStudioOutputManager diff --git a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py index f514eda95c..3831ffc794 100644 --- a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py +++ b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py @@ -7,6 +7,7 @@ from django.conf import settings from django.db import IntegrityError from plugins import get_plugin +from unstract.tool_registry.dto import Properties, Spec, Tool from prompt_studio.lookup_utils import validate_lookups_for_export from prompt_studio.prompt_profile_manager_v2.models import ProfileManager @@ -16,7 +17,6 @@ PromptStudioOutputManager, ) from prompt_studio.prompt_studio_v2.models import ToolStudioPrompt -from unstract.tool_registry.dto import Properties, Spec, Tool from .constants import JsonSchemaKey, PromptStudioRegistryKeys from .exceptions import ( diff --git a/backend/prompt_studio/prompt_studio_registry_v2/serializers.py b/backend/prompt_studio/prompt_studio_registry_v2/serializers.py index f1be6fcfed..57f95d444e 100644 --- a/backend/prompt_studio/prompt_studio_registry_v2/serializers.py +++ b/backend/prompt_studio/prompt_studio_registry_v2/serializers.py @@ -1,9 +1,8 @@ from typing import Any from account_v2.serializer import UserSerializer -from rest_framework import serializers - from backend.serializers import AuditSerializer +from rest_framework import serializers from .models import PromptStudioRegistry diff --git a/backend/prompt_studio/prompt_studio_v2/serializers.py b/backend/prompt_studio/prompt_studio_v2/serializers.py index 82d11dac85..b3a4ca2f8f 100644 --- a/backend/prompt_studio/prompt_studio_v2/serializers.py +++ b/backend/prompt_studio/prompt_studio_v2/serializers.py @@ -1,8 +1,7 @@ +from backend.serializers import AuditSerializer from rest_framework import serializers from rest_framework.serializers import ValidationError -from backend.serializers import AuditSerializer - from .models import ToolStudioPrompt diff --git a/backend/scheduler/serializer.py b/backend/scheduler/serializer.py index 2cb0d61502..d2b7abab22 100644 --- a/backend/scheduler/serializer.py +++ b/backend/scheduler/serializer.py @@ -1,10 +1,10 @@ import logging from typing import Any +from backend.constants import FieldLengthConstants as FieldLength from pipeline_v2.manager import PipelineManager from rest_framework import serializers -from backend.constants import FieldLengthConstants as FieldLength from scheduler.constants import SchedulerConstants as SC logger = logging.getLogger(__name__) diff --git a/backend/tenant_account_v2/models.py b/backend/tenant_account_v2/models.py index f569f50f6e..e70f30f660 100644 --- a/backend/tenant_account_v2/models.py +++ b/backend/tenant_account_v2/models.py @@ -33,8 +33,7 @@ class OrganizationMember(DefaultOrganizationMixin): def __str__(self): # type: ignore return ( - f"OrganizationMember(" - f"{self.member_id}, role: {self.role}, user: {self.user})" + f"OrganizationMember({self.member_id}, role: {self.role}, user: {self.user})" ) objects = OrganizationMemberModelManager() diff --git a/backend/tenant_account_v2/serializer.py b/backend/tenant_account_v2/serializer.py index 6851d88d03..0cd55c6296 100644 --- a/backend/tenant_account_v2/serializer.py +++ b/backend/tenant_account_v2/serializer.py @@ -106,7 +106,7 @@ class RemoveUserFromOrganizationSerializer(serializers.Serializer): ) def get_user_emails(self, validated_data: dict[str, list[str] | None]) -> list[str]: - return cast(list[str], validated_data.get(Common.USER_EMAILS, [])) + return cast("list[str]", validated_data.get(Common.USER_EMAILS, [])) class ChangeUserRoleRequestSerializer(serializers.Serializer): diff --git a/backend/tool_instance_v2/serializers.py b/backend/tool_instance_v2/serializers.py index 3edffea8b9..141fd2a8b8 100644 --- a/backend/tool_instance_v2/serializers.py +++ b/backend/tool_instance_v2/serializers.py @@ -5,6 +5,8 @@ from account_v2.models import User from adapter_processor_v2.adapter_processor import AdapterProcessor from adapter_processor_v2.models import AdapterInstance +from backend.constants import RequestKey +from backend.serializers import AuditSerializer from prompt_studio.prompt_studio_registry_v2.constants import ( JsonSchemaKey, PromptStudioRegistryKeys, @@ -13,20 +15,18 @@ PromptStudioRegistryHelper, ) from rest_framework.serializers import ListField, Serializer, UUIDField, ValidationError +from unstract.sdk1.adapters.enums import AdapterTypes +from unstract.tool_registry.dto import Tool +from unstract.tool_registry.tool_utils import ToolUtils from workflow_manager.workflow_v2.constants import WorkflowKey from workflow_manager.workflow_v2.models.workflow import Workflow -from backend.constants import RequestKey -from backend.serializers import AuditSerializer from tool_instance_v2.constants import ToolInstanceKey as TIKey from tool_instance_v2.constants import ToolKey from tool_instance_v2.exceptions import ToolDoesNotExist from tool_instance_v2.models import ToolInstance from tool_instance_v2.tool_instance_helper import ToolInstanceHelper from tool_instance_v2.tool_processor import ToolProcessor -from unstract.sdk1.adapters.enums import AdapterTypes -from unstract.tool_registry.dto import Tool -from unstract.tool_registry.tool_utils import ToolUtils logger = logging.getLogger(__name__) diff --git a/backend/tool_instance_v2/tool_instance_helper.py b/backend/tool_instance_v2/tool_instance_helper.py index 0669048a3b..b329100b1b 100644 --- a/backend/tool_instance_v2/tool_instance_helper.py +++ b/backend/tool_instance_v2/tool_instance_helper.py @@ -16,17 +16,17 @@ ) from prompt_studio.prompt_studio_registry_v2.models import PromptStudioRegistry from tenant_account_v2.organization_member_service import OrganizationMemberService +from unstract.sdk1.constants import AdapterTypes +from unstract.sdk1.tool.validator import DefaultsGeneratingValidator +from unstract.tool_registry.constants import AdapterPropertyKey +from unstract.tool_registry.dto import Spec, Tool +from unstract.tool_registry.tool_utils import ToolUtils from workflow_manager.workflow_v2.constants import WorkflowKey from tool_instance_v2.constants import JsonSchemaKey from tool_instance_v2.exceptions import ToolSettingValidationError from tool_instance_v2.models import ToolInstance from tool_instance_v2.tool_processor import ToolProcessor -from unstract.sdk1.constants import AdapterTypes -from unstract.sdk1.tool.validator import DefaultsGeneratingValidator -from unstract.tool_registry.constants import AdapterPropertyKey -from unstract.tool_registry.dto import Spec, Tool -from unstract.tool_registry.tool_utils import ToolUtils # Import agentic registry if available (cloud-only feature) try: diff --git a/backend/tool_instance_v2/tool_processor.py b/backend/tool_instance_v2/tool_processor.py index 5e33cceec4..3c25643835 100644 --- a/backend/tool_instance_v2/tool_processor.py +++ b/backend/tool_instance_v2/tool_processor.py @@ -8,13 +8,13 @@ from prompt_studio.prompt_studio_registry_v2.prompt_studio_registry_helper import ( PromptStudioRegistryHelper, ) - -from tool_instance_v2.exceptions import ToolDoesNotExist from unstract.sdk1.constants import AdapterTypes from unstract.tool_registry.dto import Spec, Tool from unstract.tool_registry.tool_registry import ToolRegistry from unstract.tool_registry.tool_utils import ToolUtils +from tool_instance_v2.exceptions import ToolDoesNotExist + # Import agentic registry if available (cloud-only feature) try: from pluggable_apps.agentic_studio_registry.registry_helper import ( diff --git a/backend/tool_instance_v2/views.py b/backend/tool_instance_v2/views.py index 0488354a6f..d5f95b7d0f 100644 --- a/backend/tool_instance_v2/views.py +++ b/backend/tool_instance_v2/views.py @@ -3,6 +3,7 @@ from typing import Any from account_v2.custom_exceptions import DuplicateData +from backend.constants import RequestKey from django.db import IntegrityError from django.db.models import Q from django.db.models.query import QuerySet @@ -19,7 +20,6 @@ from workflow_manager.workflow_v2.constants import WorkflowKey from workflow_manager.workflow_v2.models.workflow import Workflow -from backend.constants import RequestKey from tool_instance_v2.constants import ToolInstanceErrors, ToolKey from tool_instance_v2.constants import ToolInstanceKey as TIKey from tool_instance_v2.exceptions import FetchToolListFailed, ToolFunctionIsMandatory @@ -143,8 +143,7 @@ def create(self, request: Any) -> Response: ) if not is_workflow_mutator(request, workflow): raise PermissionDenied( - "Only the workflow owner or an organization admin can " - "add a tool to it." + "Only the workflow owner or an organization admin can add a tool to it." ) try: diff --git a/backend/usage_v2/internal_views.py b/backend/usage_v2/internal_views.py index 85c20c16e4..270a334ee0 100644 --- a/backend/usage_v2/internal_views.py +++ b/backend/usage_v2/internal_views.py @@ -12,9 +12,8 @@ from rest_framework.request import Request from rest_framework.response import Response from rest_framework.views import APIView -from utils.user_context import UserContext - from unstract.core.data_models import UsageResponseData +from utils.user_context import UserContext from .helper import UsageHelper from .hooks import run_post_write_hooks diff --git a/backend/utils/FileValidator.py b/backend/utils/FileValidator.py index c4c66b473d..a6242bbace 100644 --- a/backend/utils/FileValidator.py +++ b/backend/utils/FileValidator.py @@ -7,7 +7,6 @@ from django.core.files.uploadedfile import InMemoryUploadedFile from django.template.defaultfilters import filesizeformat from django.utils.translation import gettext_lazy as _ - from unstract.sdk1.file_storage.constants import FileOperationParams @@ -37,8 +36,7 @@ class FileValidator: "Allowed extensions are: '%(allowed_extensions)s.'" ) mime_message = _( - "MIME type '%(mimetype)s' is not valid. " - "Allowed types are: %(allowed_mimetypes)s." + "MIME type '%(mimetype)s' is not valid. Allowed types are: %(allowed_mimetypes)s." ) min_size_message = _( "The current file %(size)s, which is too small. " diff --git a/backend/utils/cache_service.py b/backend/utils/cache_service.py index ea6e4278bf..f8110076c3 100644 --- a/backend/utils/cache_service.py +++ b/backend/utils/cache_service.py @@ -6,7 +6,6 @@ from django.conf import settings from django.core.cache import cache from django_redis import get_redis_connection - from unstract.core.cache.redis_client import create_redis_client redis_cache = get_redis_connection("default") diff --git a/backend/utils/dto.py b/backend/utils/dto.py index af10f475fe..7c6217ee5f 100644 --- a/backend/utils/dto.py +++ b/backend/utils/dto.py @@ -6,7 +6,6 @@ from typing import Any from django.utils import timezone as dj_timezone - from unstract.core.constants import LogFieldName from unstract.workflow_execution.enums import LogType diff --git a/backend/utils/file_storage/helpers/prompt_studio_file_helper.py b/backend/utils/file_storage/helpers/prompt_studio_file_helper.py index c5e9cfebd7..74332273af 100644 --- a/backend/utils/file_storage/helpers/prompt_studio_file_helper.py +++ b/backend/utils/file_storage/helpers/prompt_studio_file_helper.py @@ -6,11 +6,11 @@ from file_management.exceptions import InvalidFileType from file_management.file_management_helper import FileManagerHelper - from unstract.core.utilities import UnstractUtils from unstract.sdk1.file_storage import FileStorage from unstract.sdk1.file_storage.constants import StorageType from unstract.sdk1.file_storage.env_helper import EnvHelper + from utils.file_storage.constants import FileStorageConstants, FileStorageKeys from utils.file_storage.helpers.streaming_writer import write_streaming diff --git a/backend/utils/log_events.py b/backend/utils/log_events.py index dcb66ed81c..c5a4533d1c 100644 --- a/backend/utils/log_events.py +++ b/backend/utils/log_events.py @@ -6,10 +6,10 @@ import socketio from django.conf import settings from django.core.wsgi import WSGIHandler - from unstract.core.cache.redis_client import create_redis_client from unstract.core.data_models import LogDataDTO from unstract.core.log_utils import get_validated_log_data, store_execution_log + from utils.constants import ExecutionLogConstants from utils.cors_origin import RegexOrigin diff --git a/backend/workflow_manager/endpoint_v2/base_connector.py b/backend/workflow_manager/endpoint_v2/base_connector.py index 66fec358f7..92bf693d7f 100644 --- a/backend/workflow_manager/endpoint_v2/base_connector.py +++ b/backend/workflow_manager/endpoint_v2/base_connector.py @@ -2,12 +2,11 @@ from typing import Any from fsspec import AbstractFileSystem -from utils.constants import Common -from utils.user_context import UserContext - from unstract.connectors.filesystems import connectors from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem from unstract.workflow_execution.execution_file_handler import ExecutionFileHandler +from utils.constants import Common +from utils.user_context import UserContext class BaseConnector(ExecutionFileHandler): diff --git a/backend/workflow_manager/endpoint_v2/database_utils.py b/backend/workflow_manager/endpoint_v2/database_utils.py index a89594e186..2330b42fd8 100644 --- a/backend/workflow_manager/endpoint_v2/database_utils.py +++ b/backend/workflow_manager/endpoint_v2/database_utils.py @@ -3,17 +3,16 @@ import logging from typing import Any +from unstract.connectors.databases import connectors as db_connectors +from unstract.connectors.databases.exceptions import UnstractDBConnectorException +from unstract.connectors.databases.unstract_db import UnstractDB +from unstract.connectors.exceptions import ConnectorError from utils.constants import Common from workflow_manager.endpoint_v2.constants import TableColumns from workflow_manager.endpoint_v2.enums import FileProcessingStatus from workflow_manager.endpoint_v2.exceptions import UnstractDBException from workflow_manager.workflow_v2.enums import AgentName, ColumnModes -from unstract.connectors.databases import connectors as db_connectors -from unstract.connectors.databases.exceptions import UnstractDBConnectorException -from unstract.connectors.databases.unstract_db import UnstractDB -from unstract.connectors.exceptions import ConnectorError - logger = logging.getLogger(__name__) diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.py index fbfb17c518..f18cc294d6 100644 --- a/backend/workflow_manager/endpoint_v2/destination.py +++ b/backend/workflow_manager/endpoint_v2/destination.py @@ -5,9 +5,14 @@ import os from typing import Any +from backend.exceptions import UnstractFSException from connector_v2.models import ConnectorInstance from plugins.workflow_manager.workflow_v2.utils import WorkflowUtil from rest_framework.exceptions import APIException +from unstract.connectors.exceptions import ConnectorError +from unstract.sdk1.constants import ToolExecKey +from unstract.sdk1.tool.mime_types import EXT_MIME_MAP +from unstract.workflow_execution.constants import ToolOutputType from usage_v2.helper import UsageHelper from utils.user_context import UserContext from workflow_manager.endpoint_v2.base_connector import BaseConnector @@ -34,12 +39,7 @@ from workflow_manager.workflow_v2.models.file_history import FileHistory from workflow_manager.workflow_v2.models.workflow import Workflow -from backend.exceptions import UnstractFSException -from unstract.connectors.exceptions import ConnectorError from unstract.filesystem import FileStorageType, FileSystem -from unstract.sdk1.constants import ToolExecKey -from unstract.sdk1.tool.mime_types import EXT_MIME_MAP -from unstract.workflow_execution.constants import ToolOutputType logger = logging.getLogger(__name__) diff --git a/backend/workflow_manager/endpoint_v2/queue_utils.py b/backend/workflow_manager/endpoint_v2/queue_utils.py index 5e44998be5..9bbe529480 100644 --- a/backend/workflow_manager/endpoint_v2/queue_utils.py +++ b/backend/workflow_manager/endpoint_v2/queue_utils.py @@ -5,11 +5,10 @@ from typing import Any from django.conf import settings -from utils.constants import Common -from workflow_manager.endpoint_v2.exceptions import UnstractQueueException - from unstract.connectors.queues import connectors as queue_connectors from unstract.connectors.queues.unstract_queue import UnstractQueue +from utils.constants import Common +from workflow_manager.endpoint_v2.exceptions import UnstractQueueException logger = logging.getLogger(__name__) diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.py index 23b0d04f2c..7a515d8ba8 100644 --- a/backend/workflow_manager/endpoint_v2/source.py +++ b/backend/workflow_manager/endpoint_v2/source.py @@ -15,6 +15,9 @@ from connector_v2.models import ConnectorInstance from django.core.files.uploadedfile import UploadedFile from django.db.models import Q +from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem +from unstract.sdk1.file_storage import FileStorage +from unstract.workflow_execution.enums import LogLevel, LogStage, LogState from utils.user_context import UserContext from workflow_manager.endpoint_v2.base_connector import BaseConnector from workflow_manager.endpoint_v2.constants import ( @@ -45,10 +48,7 @@ from workflow_manager.workflow_v2.models.file_history import FileHistory from workflow_manager.workflow_v2.models.workflow import Workflow -from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem from unstract.filesystem import FileStorageType, FileSystem -from unstract.sdk1.file_storage import FileStorage -from unstract.workflow_execution.enums import LogLevel, LogStage, LogState logger = logging.getLogger(__name__) diff --git a/backend/workflow_manager/internal_api_views.py b/backend/workflow_manager/internal_api_views.py index 15559a71c4..f970f5f94e 100644 --- a/backend/workflow_manager/internal_api_views.py +++ b/backend/workflow_manager/internal_api_views.py @@ -17,8 +17,8 @@ from rest_framework.decorators import api_view from rest_framework.response import Response from tool_instance_v2.models import ToolInstance - from unstract.core.data_models import LEGACY_TRANSPORT_KEY, LEGACY_TRANSPORT_VALUE + from workflow_manager.workflow_v2.enums import ExecutionStatus from workflow_manager.workflow_v2.models import Workflow, WorkflowExecution diff --git a/backend/workflow_manager/internal_serializers.py b/backend/workflow_manager/internal_serializers.py index 9127763873..6835a88a41 100644 --- a/backend/workflow_manager/internal_serializers.py +++ b/backend/workflow_manager/internal_serializers.py @@ -12,6 +12,7 @@ FileExecutionStatusUpdateRequest, WorkflowFileExecutionData, ) + from workflow_manager.file_execution.models import WorkflowFileExecution from workflow_manager.workflow_v2.enums import ExecutionStatus from workflow_manager.workflow_v2.models.execution import WorkflowExecution diff --git a/backend/workflow_manager/internal_views.py b/backend/workflow_manager/internal_views.py index 66b22d3216..6c1b06ee73 100644 --- a/backend/workflow_manager/internal_views.py +++ b/backend/workflow_manager/internal_views.py @@ -13,9 +13,6 @@ from rest_framework.response import Response from rest_framework.views import APIView from tool_instance_v2.models import ToolInstance -from utils.constants import Account -from utils.local_context import StateStore -from utils.organization_utils import filter_queryset_by_organization # Import new dataclasses for WorkflowDefinitionAPIView from unstract.core.data_models import ( @@ -25,6 +22,10 @@ WorkflowEndpointConfigData, WorkflowEndpointConfigResponseData, ) +from utils.constants import Account +from utils.local_context import StateStore +from utils.organization_utils import filter_queryset_by_organization + from workflow_manager.endpoint_v2.endpoint_utils import WorkflowEndpointUtils from workflow_manager.endpoint_v2.models import WorkflowEndpoint from workflow_manager.file_execution.models import WorkflowFileExecution @@ -1714,9 +1715,9 @@ class WorkflowSourceFilesAPIView(APIView): def post(self, request, workflow_id): """Get source files for a workflow execution.""" try: + from unstract.workflow_execution.enums import LogStage from utils.user_context import UserContext - from unstract.workflow_execution.enums import LogStage from workflow_manager.endpoint_v2.source import SourceConnector from workflow_manager.utils.workflow_log import WorkflowLog @@ -2841,6 +2842,7 @@ def post(self, request): # Create file history entry using the FileHistoryHelper from unstract.core.data_models import FileHashData + from workflow_manager.workflow_v2.enums import ExecutionStatus from workflow_manager.workflow_v2.file_history_helper import FileHistoryHelper diff --git a/backend/workflow_manager/utils/workflow_log.py b/backend/workflow_manager/utils/workflow_log.py index cbd351c91a..1f7d698468 100644 --- a/backend/workflow_manager/utils/workflow_log.py +++ b/backend/workflow_manager/utils/workflow_log.py @@ -1,10 +1,9 @@ import logging from account_v2.constants import Common -from utils.local_context import StateStore - from unstract.core.pubsub_helper import LogPublisher from unstract.workflow_execution.enums import LogComponent, LogLevel, LogStage, LogState +from utils.local_context import StateStore class WorkflowLog: diff --git a/backend/workflow_manager/workflow_v2/execution.py b/backend/workflow_manager/workflow_v2/execution.py index 213c81302b..186df50bff 100644 --- a/backend/workflow_manager/workflow_v2/execution.py +++ b/backend/workflow_manager/workflow_v2/execution.py @@ -7,10 +7,6 @@ from tags.models import Tag from tool_instance_v2.models import ToolInstance from tool_instance_v2.tool_processor import ToolProcessor -from usage_v2.helper import UsageHelper -from utils.local_context import StateStore -from utils.user_context import UserContext - from unstract.tool_registry.dto import Tool from unstract.workflow_execution import WorkflowExecutionService from unstract.workflow_execution.dto import ToolInstance as ToolInstanceDataClass @@ -22,6 +18,10 @@ LogState, ) from unstract.workflow_execution.exceptions import StopExecution +from usage_v2.helper import UsageHelper +from utils.local_context import StateStore +from utils.user_context import UserContext + from workflow_manager.file_execution.models import WorkflowFileExecution from workflow_manager.workflow_v2.constants import WorkflowKey from workflow_manager.workflow_v2.enums import ExecutionStatus diff --git a/backend/workflow_manager/workflow_v2/filters/execution_log.py b/backend/workflow_manager/workflow_v2/filters/execution_log.py index 96a95aaf52..7418807b11 100644 --- a/backend/workflow_manager/workflow_v2/filters/execution_log.py +++ b/backend/workflow_manager/workflow_v2/filters/execution_log.py @@ -2,8 +2,8 @@ from django.db.models.query import QuerySet from django_filters import CharFilter, FilterSet - from unstract.sdk1.constants import LogLevel + from workflow_manager.workflow_v2.models.execution_log import ExecutionLog diff --git a/backend/workflow_manager/workflow_v2/serializers.py b/backend/workflow_manager/workflow_v2/serializers.py index 7ee69b1375..6c0544d507 100644 --- a/backend/workflow_manager/workflow_v2/serializers.py +++ b/backend/workflow_manager/workflow_v2/serializers.py @@ -1,6 +1,8 @@ import logging from typing import Any +from backend.constants import RequestKey +from backend.serializers import AuditSerializer from django.conf import settings from rest_framework import serializers from rest_framework.serializers import ( @@ -22,8 +24,6 @@ from utils.input_sanitizer import validate_name_field from utils.serializer.integrity_error_mixin import IntegrityErrorMixin -from backend.constants import RequestKey -from backend.serializers import AuditSerializer from workflow_manager.workflow_v2.constants import WorkflowExecutionKey, WorkflowKey from workflow_manager.workflow_v2.models.execution import WorkflowExecution from workflow_manager.workflow_v2.models.execution_log import ExecutionLog diff --git a/backend/workflow_manager/workflow_v2/undispatched_sweep.py b/backend/workflow_manager/workflow_v2/undispatched_sweep.py index 290f635226..89921dfd01 100644 --- a/backend/workflow_manager/workflow_v2/undispatched_sweep.py +++ b/backend/workflow_manager/workflow_v2/undispatched_sweep.py @@ -44,7 +44,6 @@ import os from django.utils import timezone - from unstract.core.data_models import ExecutionStatus logger = logging.getLogger(__name__) diff --git a/backend/workflow_manager/workflow_v2/views.py b/backend/workflow_manager/workflow_v2/views.py index 845b254177..1df924540e 100644 --- a/backend/workflow_manager/workflow_v2/views.py +++ b/backend/workflow_manager/workflow_v2/views.py @@ -2,6 +2,7 @@ import uuid from typing import Any +from backend.constants import RequestKey from django.conf import settings from django.db import transaction from django.db.models.query import QuerySet @@ -19,12 +20,11 @@ from rest_framework.response import Response from rest_framework.versioning import URLPathVersioning from rest_framework.views import APIView +from unstract.core.data_models import FileHistoryCreateRequest from utils.filtering import FilterHelper from utils.organization_utils import filter_queryset_by_organization, resolve_organization from utils.pagination import OptionalPagination -from backend.constants import RequestKey -from unstract.core.data_models import FileHistoryCreateRequest from workflow_manager.endpoint_v2.destination import DestinationConnector from workflow_manager.endpoint_v2.dto import FileHash from workflow_manager.endpoint_v2.endpoint_utils import WorkflowEndpointUtils diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.py index 2c926d0dcd..467ec078e2 100644 --- a/backend/workflow_manager/workflow_v2/workflow_helper.py +++ b/backend/workflow_manager/workflow_v2/workflow_helper.py @@ -7,6 +7,7 @@ from account_v2.constants import Common from api_v2.models import APIDeployment +from backend.celery_service import app as celery_app from celery import chord, current_task from celery.result import AsyncResult from configuration.enums import ConfigKey @@ -21,18 +22,17 @@ from tool_instance_v2.constants import ToolInstanceKey from tool_instance_v2.models import ToolInstance from tool_instance_v2.tool_instance_helper import ToolInstanceHelper -from utils.cache_service import CacheService -from utils.constants import Account, CeleryQueue, FileProcessingQueue -from utils.local_context import StateStore -from utils.user_context import UserContext - -from backend.celery_service import app as celery_app from unstract.core.data_models import ( LEGACY_TRANSPORT_KEY, LEGACY_TRANSPORT_VALUE, WorkloadType, ) from unstract.workflow_execution.enums import LogStage +from utils.cache_service import CacheService +from utils.constants import Account, CeleryQueue, FileProcessingQueue +from utils.local_context import StateStore +from utils.user_context import UserContext + from workflow_manager.endpoint_v2.destination import DestinationConnector from workflow_manager.endpoint_v2.dto import FileHash from workflow_manager.endpoint_v2.models import WorkflowEndpoint diff --git a/docker/scripts/release-notes/print_release_notes.py b/docker/scripts/release-notes/print_release_notes.py index 6c278d88ab..b4c076e2bb 100644 --- a/docker/scripts/release-notes/print_release_notes.py +++ b/docker/scripts/release-notes/print_release_notes.py @@ -62,8 +62,7 @@ def print_release_notes(current_version, target_version, release_notes): f"{colour('WARNING', Colour.YELLOW)}: " if release_note["warning"] else "" ) message += ( - f"[from {colour(note_version, Colour.BLUE)}]\n" - f"{release_note['message']}" + f"[from {colour(note_version, Colour.BLUE)}]\n{release_note['message']}" ) messages_to_print.append(message) diff --git a/docs/API_DEPLOYMENT_RATE_LIMITING.md b/docs/API_DEPLOYMENT_RATE_LIMITING.md index ee5ec8c48a..c469b7b88f 100644 --- a/docs/API_DEPLOYMENT_RATE_LIMITING.md +++ b/docs/API_DEPLOYMENT_RATE_LIMITING.md @@ -182,8 +182,12 @@ API_DEPLOYMENT_RATE_LIMIT_LOCK_BLOCKING_TIMEOUT=5 These environment variables are loaded in `backend/settings/base.py`: ```python -API_DEPLOYMENT_DEFAULT_RATE_LIMIT = int(os.environ.get("API_DEPLOYMENT_DEFAULT_RATE_LIMIT", 20)) -API_DEPLOYMENT_GLOBAL_RATE_LIMIT = int(os.environ.get("API_DEPLOYMENT_GLOBAL_RATE_LIMIT", 100)) +API_DEPLOYMENT_DEFAULT_RATE_LIMIT = int( + os.environ.get("API_DEPLOYMENT_DEFAULT_RATE_LIMIT", 20) +) +API_DEPLOYMENT_GLOBAL_RATE_LIMIT = int( + os.environ.get("API_DEPLOYMENT_GLOBAL_RATE_LIMIT", 100) +) # ... etc ``` diff --git a/platform-service/src/unstract/platform_service/config.py b/platform-service/src/unstract/platform_service/config.py index 08fff41307..043d317c83 100644 --- a/platform-service/src/unstract/platform_service/config.py +++ b/platform-service/src/unstract/platform_service/config.py @@ -4,7 +4,6 @@ from dotenv import load_dotenv from flask import Flask - from unstract.core.flask import ( PluginManager, register_error_handlers, diff --git a/platform-service/src/unstract/platform_service/controller/__init__.py b/platform-service/src/unstract/platform_service/controller/__init__.py index 88d501cf2b..7c287fa3aa 100644 --- a/platform-service/src/unstract/platform_service/controller/__init__.py +++ b/platform-service/src/unstract/platform_service/controller/__init__.py @@ -2,11 +2,10 @@ import traceback from flask import Blueprint, Response, jsonify, request -from werkzeug.exceptions import HTTPException - from unstract.core.flask.exceptions import APIError, ErrorResponse from unstract.platform_service.controller.health import health_bp from unstract.platform_service.controller.platform import platform_bp +from werkzeug.exceptions import HTTPException api = Blueprint("api", __name__) api.register_blueprint(platform_bp) diff --git a/platform-service/src/unstract/platform_service/controller/platform.py b/platform-service/src/unstract/platform_service/controller/platform.py index fbc8cdd8bb..937190af57 100644 --- a/platform-service/src/unstract/platform_service/controller/platform.py +++ b/platform-service/src/unstract/platform_service/controller/platform.py @@ -7,7 +7,6 @@ from cryptography.fernet import Fernet, InvalidToken from flask import Blueprint, Request, jsonify, make_response, request from flask import current_app as app - from unstract.core.flask import PluginManager from unstract.core.flask.exceptions import APIError from unstract.platform_service.constants import DBTable diff --git a/platform-service/src/unstract/platform_service/extensions.py b/platform-service/src/unstract/platform_service/extensions.py index 75a073a5fe..8613d5c211 100644 --- a/platform-service/src/unstract/platform_service/extensions.py +++ b/platform-service/src/unstract/platform_service/extensions.py @@ -4,7 +4,6 @@ import redis from peewee import PostgresqlDatabase - from unstract.core.cache.redis_client import create_redis_client db = PostgresqlDatabase(None) diff --git a/runner/src/unstract/runner/__init__.py b/runner/src/unstract/runner/__init__.py index 54041b09fc..b54d23871e 100644 --- a/runner/src/unstract/runner/__init__.py +++ b/runner/src/unstract/runner/__init__.py @@ -2,7 +2,6 @@ import os from flask import Flask - from unstract.core.flask import register_error_handlers, register_request_id_middleware from unstract.core.flask.logging import setup_logging from unstract.runner.constants import LogLevel diff --git a/runner/src/unstract/runner/clients/docker_client.py b/runner/src/unstract/runner/clients/docker_client.py index 735f0f3415..69c635eada 100644 --- a/runner/src/unstract/runner/clients/docker_client.py +++ b/runner/src/unstract/runner/clients/docker_client.py @@ -4,7 +4,6 @@ from collections.abc import Iterator from typing import Any -from docker import DockerClient from docker.errors import APIError, ImageNotFound, NotFound from docker.models.containers import Container from unstract.core.runner.enum import ContainerStatus @@ -17,6 +16,8 @@ from unstract.runner.exception import ToolImageNotFoundError from unstract.runner.utils import Utils +from docker import DockerClient + class DockerContainer(ContainerInterface): def __init__(self, container: Container, logger: logging.Logger) -> None: diff --git a/runner/src/unstract/runner/clients/test_docker.py b/runner/src/unstract/runner/clients/test_docker.py index 6841c4e2e8..643d09ff75 100644 --- a/runner/src/unstract/runner/clients/test_docker.py +++ b/runner/src/unstract/runner/clients/test_docker.py @@ -3,7 +3,6 @@ from unittest.mock import MagicMock import pytest - from docker.errors import ImageNotFound from unstract.runner.constants import Env diff --git a/runner/src/unstract/runner/controller/run.py b/runner/src/unstract/runner/controller/run.py index dc681630c1..692e26aba8 100644 --- a/runner/src/unstract/runner/controller/run.py +++ b/runner/src/unstract/runner/controller/run.py @@ -2,7 +2,6 @@ from flask import Blueprint, abort, request from flask import current_app as app - from unstract.runner.runner import UnstractRunner # Define a Blueprint with a root URL path diff --git a/runner/src/unstract/runner/runner.py b/runner/src/unstract/runner/runner.py index 2414ac7548..4468df80d3 100644 --- a/runner/src/unstract/runner/runner.py +++ b/runner/src/unstract/runner/runner.py @@ -7,7 +7,6 @@ from dotenv import load_dotenv from flask import Flask - from unstract.core.constants import LogFieldName from unstract.core.file_execution_tracker import ( FileExecutionStage, diff --git a/runner/src/unstract/runner/utils.py b/runner/src/unstract/runner/utils.py index 4bec83a2cb..d021feebd8 100644 --- a/runner/src/unstract/runner/utils.py +++ b/runner/src/unstract/runner/utils.py @@ -2,7 +2,6 @@ import os from dotenv import load_dotenv - from unstract.runner.constants import Env from unstract.runner.enum import LogLevel diff --git a/tools/classifier/src/main.py b/tools/classifier/src/main.py index 4903d727b2..77ff9356a1 100644 --- a/tools/classifier/src/main.py +++ b/tools/classifier/src/main.py @@ -5,7 +5,6 @@ ClassifierHelper, # type: ignore ReservedBins, ) - from unstract.sdk1.constants import ( LogLevel, LogState, diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py index f73e3b731c..e99937de39 100644 --- a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py +++ b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py @@ -272,8 +272,7 @@ def test_credentials(self) -> bool: ) from e if not is_dir: raise ConnectorError( - "Unable to connect to Google Drive, " - "please check the connection settings." + "Unable to connect to Google Drive, please check the connection settings." ) return True diff --git a/unstract/connectors/src/unstract/connectors/filesystems/http/http.py b/unstract/connectors/src/unstract/connectors/filesystems/http/http.py index 87e786a7a3..a3350b9c94 100644 --- a/unstract/connectors/src/unstract/connectors/filesystems/http/http.py +++ b/unstract/connectors/src/unstract/connectors/filesystems/http/http.py @@ -119,7 +119,6 @@ def test_credentials(self) -> bool: ) from e if not is_dir: raise ConnectorError( - "Unable to connect to HTTP server, " - "please check the connection settings." + "Unable to connect to HTTP server, please check the connection settings." ) return True diff --git a/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py index 34cfa79659..4fb1943746 100644 --- a/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py +++ b/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py @@ -4,7 +4,6 @@ from typing import Any from fsspec.implementations.local import LocalFileSystem - from unstract.connectors.exceptions import ConnectorError from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem diff --git a/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py b/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py index 3e3915debc..e0f3989027 100644 --- a/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py +++ b/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py @@ -140,8 +140,7 @@ def test_credentials(self) -> bool: ) from e if not is_dir: raise ConnectorError( - "Unable to connect to SFTP server, " - "please check the connection settings." + "Unable to connect to SFTP server, please check the connection settings." ) return True diff --git a/unstract/core/src/unstract/core/plugins/README.md b/unstract/core/src/unstract/core/plugins/README.md index aaa2113df7..55b63b587e 100644 --- a/unstract/core/src/unstract/core/plugins/README.md +++ b/unstract/core/src/unstract/core/plugins/README.md @@ -40,7 +40,6 @@ metadata = { # Required "name": "my_plugin", "version": "1.0.0", - # Optional "description": "Description of what this plugin does", "is_active": True, diff --git a/unstract/core/src/unstract/core/worker_models.py b/unstract/core/src/unstract/core/worker_models.py index 4f3e48d99b..8771b073d7 100644 --- a/unstract/core/src/unstract/core/worker_models.py +++ b/unstract/core/src/unstract/core/worker_models.py @@ -347,8 +347,7 @@ def _parse_skipped( return SkipReason(raw) except ValueError: logger.warning( - "Unknown SkipReason on wire: %r (file_execution_id=%s); " - "treating as None", + "Unknown SkipReason on wire: %r (file_execution_id=%s); treating as None", raw, file_execution_id, ) diff --git a/unstract/filesystem/src/unstract/filesystem/file_storage_config.py b/unstract/filesystem/src/unstract/filesystem/file_storage_config.py index 870ed30a02..303138b206 100644 --- a/unstract/filesystem/src/unstract/filesystem/file_storage_config.py +++ b/unstract/filesystem/src/unstract/filesystem/file_storage_config.py @@ -18,8 +18,7 @@ def get_provider(var_name: str, default: str = "minio") -> FileStorageProvider: except KeyError: allowed_providers = ", ".join([provider.name for provider in FileStorageProvider]) logger.error( - f"Invalid provider '{provider_name}'. Allowed providers: " - f"{allowed_providers}" + f"Invalid provider '{provider_name}'. Allowed providers: {allowed_providers}" ) raise ProviderNotFound(f"Provider '{provider_name}' not found") diff --git a/unstract/flags/src/unstract/flags/flipt_grpc/README.md b/unstract/flags/src/unstract/flags/flipt_grpc/README.md index 0df7721f08..7d065f0b17 100644 --- a/unstract/flags/src/unstract/flags/flipt_grpc/README.md +++ b/unstract/flags/src/unstract/flags/flipt_grpc/README.md @@ -30,17 +30,13 @@ from flipt_grpc.client import FliptGrpcClient, GrpcClientOptions # Create a client client = FliptGrpcClient( opts=GrpcClientOptions( - address="localhost:9000", - namespace_key="default", - environment_key="production" + address="localhost:9000", namespace_key="default", environment_key="production" ) ) # Evaluate a boolean flag result = client.evaluate_boolean( - flag_key="my-feature", - entity_id="user-123", - context={"region": "us-west"} + flag_key="my-feature", entity_id="user-123", context={"region": "us-west"} ) print(f"Feature enabled: {result.enabled}") @@ -72,7 +68,7 @@ client = FliptGrpcClient(opts=GrpcClientOptions(address="localhost:9000")) result = client.evaluate_boolean( flag_key="enable-feature-x", entity_id="user-456", - context={"plan": "premium", "region": "us-east"} + context={"plan": "premium", "region": "us-east"}, ) print(f"Enabled: {result.enabled}") @@ -86,9 +82,7 @@ client.close() ```python result = client.evaluate_variant( - flag_key="color-theme", - entity_id="user-789", - context={"device": "mobile"} + flag_key="color-theme", entity_id="user-789", context={"device": "mobile"} ) print(f"Variant: {result.variant_key}") @@ -112,25 +106,21 @@ Evaluate multiple flags in a single request: ```python batch_requests = [ - { - "flag_key": "feature-a", - "entity_id": "user-123", - "context": {"role": "admin"} - }, - { - "flag_key": "feature-b", - "entity_id": "user-123", - "context": {"role": "admin"} - } + {"flag_key": "feature-a", "entity_id": "user-123", "context": {"role": "admin"}}, + {"flag_key": "feature-b", "entity_id": "user-123", "context": {"role": "admin"}}, ] result = client.evaluate_batch(requests=batch_requests) for response in result.responses: if response.HasField("boolean_response"): - print(f"Flag: {response.boolean_response.flag_key}, Enabled: {response.boolean_response.enabled}") + print( + f"Flag: {response.boolean_response.flag_key}, Enabled: {response.boolean_response.enabled}" + ) elif response.HasField("variant_response"): - print(f"Flag: {response.variant_response.flag_key}, Variant: {response.variant_response.variant_key}") + print( + f"Flag: {response.variant_response.flag_key}, Variant: {response.variant_response.variant_key}" + ) ``` ### Context Manager @@ -152,7 +142,7 @@ client = FliptGrpcClient( address="flipt.example.com:443", namespace_key="production", secure=True, - client_token="your-client-token" + client_token="your-client-token", ) ) @@ -165,9 +155,7 @@ client.close() ```python client = FliptGrpcClient( opts=GrpcClientOptions( - address="flipt.internal:9000", - secure=True, - ssl_cert_path="/path/to/ca-cert.pem" + address="flipt.internal:9000", secure=True, ssl_cert_path="/path/to/ca-cert.pem" ) ) ``` @@ -245,6 +233,7 @@ Then fix the imports in the generated `*_grpc.py` files to use relative imports: ```python # Change this: from flipt import flipt_simple_pb2 + # To this: from . import flipt_simple_pb2 ``` diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/adapterkit.py b/unstract/sdk1/src/unstract/sdk1/adapters/adapterkit.py index 330ad2db92..201c2c3182 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/adapterkit.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/adapterkit.py @@ -5,6 +5,7 @@ from typing import Any from singleton_decorator import singleton + from unstract.sdk1.adapters import AdapterDict from unstract.sdk1.adapters.base import Adapter from unstract.sdk1.adapters.constants import Common diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/base1.py b/unstract/sdk1/src/unstract/sdk1/adapters/base1.py index 716587a23e..1d7a603d3d 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/base1.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/base1.py @@ -12,6 +12,7 @@ from typing import Any from pydantic import BaseModel, Field, model_validator + from unstract.sdk1.adapters.constants import AdapterDocs, Common from unstract.sdk1.adapters.enums import AdapterTypes diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/ocr/google_document_ai/src/google_document_ai.py b/unstract/sdk1/src/unstract/sdk1/adapters/ocr/google_document_ai/src/google_document_ai.py index a36d3ae357..327db3b89b 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/ocr/google_document_ai/src/google_document_ai.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/ocr/google_document_ai/src/google_document_ai.py @@ -8,6 +8,7 @@ from filetype import filetype from google.auth.transport import requests as google_requests from google.oauth2.service_account import Credentials + from unstract.sdk1.adapters.exceptions import AdapterError from unstract.sdk1.adapters.ocr.constants import FileType from unstract.sdk1.adapters.ocr.ocr_adapter import OCRAdapter diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/exceptions.py b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/exceptions.py index e44784671e..82aee35d18 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/exceptions.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/exceptions.py @@ -23,6 +23,7 @@ def parse_vector_db_err(e: Exception, vector_db: VectorDBAdapter) -> VectorDBErr # protobuf runtime version (KeyError: '_POINTID'). try: from qdrant_client.http.exceptions import ApiException as QdrantAPIException + from unstract.sdk1.adapters.vectordb.qdrant.src import Qdrant if isinstance(e, QdrantAPIException): diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/helper.py b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/helper.py index ed2b4921e9..5e77d5da9d 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/helper.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/helper.py @@ -9,6 +9,7 @@ ) from llama_index.core.llms import MockLLM from llama_index.core.vector_stores.types import BasePydanticVectorStore + from unstract.sdk1.adapters.vectordb.constants import VectorDbConstants from unstract.sdk1.exceptions import VectorDBError diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/milvus/src/milvus.py b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/milvus/src/milvus.py index 85d0a2ec6a..cff646801b 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/milvus/src/milvus.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/milvus/src/milvus.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING from llama_index.vector_stores.milvus import MilvusVectorStore + from unstract.sdk1.adapters.exceptions import AdapterError from unstract.sdk1.adapters.vectordb.constants import VectorDbConstants from unstract.sdk1.adapters.vectordb.helper import VectorDBHelper diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/no_op/src/no_op_vectordb.py b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/no_op/src/no_op_vectordb.py index 0f396a79e5..edbbc76716 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/no_op/src/no_op_vectordb.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/no_op/src/no_op_vectordb.py @@ -3,6 +3,7 @@ from llama_index.core.schema import BaseNode from llama_index.core.vector_stores.types import VectorStore + from unstract.sdk1.adapters.exceptions import AdapterError from unstract.sdk1.adapters.vectordb.constants import VectorDbConstants from unstract.sdk1.adapters.vectordb.helper import VectorDBHelper diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/pinecone/src/pinecone.py b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/pinecone/src/pinecone.py index f8e6aa990e..6b727a62b4 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/pinecone/src/pinecone.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/pinecone/src/pinecone.py @@ -6,6 +6,7 @@ from llama_index.vector_stores.pinecone import PineconeVectorStore from pinecone import NotFoundException, PodSpec, ServerlessSpec from pinecone import Pinecone as LLamaIndexPinecone + from unstract.sdk1.adapters.exceptions import AdapterError from unstract.sdk1.adapters.vectordb.constants import VectorDbConstants from unstract.sdk1.adapters.vectordb.helper import VectorDBHelper diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/postgres/src/postgres.py b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/postgres/src/postgres.py index 7182f4d42b..232584e04d 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/postgres/src/postgres.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/postgres/src/postgres.py @@ -6,6 +6,7 @@ import psycopg2 from llama_index.vector_stores.postgres import PGVectorStore + from unstract.sdk1.adapters.exceptions import AdapterError from unstract.sdk1.adapters.vectordb.constants import VectorDbConstants from unstract.sdk1.adapters.vectordb.helper import VectorDBHelper diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/qdrant/src/qdrant.py b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/qdrant/src/qdrant.py index ed435dacaf..55d30fece1 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/qdrant/src/qdrant.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/qdrant/src/qdrant.py @@ -5,6 +5,7 @@ from llama_index.vector_stores.qdrant import QdrantVectorStore from qdrant_client import QdrantClient from qdrant_client.http.exceptions import UnexpectedResponse + from unstract.sdk1.adapters.vectordb.constants import VectorDbConstants from unstract.sdk1.adapters.vectordb.helper import VectorDBHelper from unstract.sdk1.adapters.vectordb.vectordb_adapter import VectorDBAdapter diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/supabase/src/supabase.py b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/supabase/src/supabase.py index 6508eaad93..347fd6c729 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/supabase/src/supabase.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/supabase/src/supabase.py @@ -5,6 +5,7 @@ from urllib.parse import quote_plus from llama_index.vector_stores.supabase import SupabaseVectorStore + from unstract.sdk1.adapters.exceptions import AdapterError from unstract.sdk1.adapters.vectordb.constants import VectorDbConstants from unstract.sdk1.adapters.vectordb.helper import VectorDBHelper diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/vectordb_adapter.py b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/vectordb_adapter.py index 786ab11457..9ccee85ba3 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/vectordb_adapter.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/vectordb_adapter.py @@ -4,6 +4,7 @@ from llama_index.core.schema import BaseNode from llama_index.core.vector_stores import SimpleVectorStore from llama_index.core.vector_stores.types import BasePydanticVectorStore, VectorStore + from unstract.sdk1.adapters.base import Adapter from unstract.sdk1.adapters.enums import AdapterTypes from unstract.sdk1.exceptions import VectorDBError diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/weaviate/src/weaviate.py b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/weaviate/src/weaviate.py index 1c08892f6b..a14a660e54 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/weaviate/src/weaviate.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/weaviate/src/weaviate.py @@ -4,12 +4,13 @@ import weaviate from llama_index.core.vector_stores.types import BasePydanticVectorStore from llama_index.vector_stores.weaviate import WeaviateVectorStore +from weaviate.classes.init import Auth +from weaviate.exceptions import UnexpectedStatusCodeException + from unstract.sdk1.adapters.exceptions import AdapterError from unstract.sdk1.adapters.vectordb.constants import VectorDbConstants from unstract.sdk1.adapters.vectordb.helper import VectorDBHelper from unstract.sdk1.adapters.vectordb.vectordb_adapter import VectorDBAdapter -from weaviate.classes.init import Auth -from weaviate.exceptions import UnexpectedStatusCodeException logger = logging.getLogger(__name__) diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/helper.py b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/helper.py index 0b9376e95d..a7bdb2ecf5 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/helper.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/helper.py @@ -6,6 +6,7 @@ import requests from requests import Response from requests.exceptions import ConnectionError, HTTPError, Timeout + from unstract.sdk1.adapters.exceptions import AdapterError from unstract.sdk1.adapters.utils import AdapterUtils from unstract.sdk1.adapters.x2text.constants import X2TextConstants diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llama_parse/src/llama_parse.py b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llama_parse/src/llama_parse.py index e41a584ca9..59c348d1a3 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llama_parse/src/llama_parse.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llama_parse/src/llama_parse.py @@ -5,6 +5,7 @@ from httpx import ConnectError from llama_parse import LlamaParse + from unstract.sdk1.adapters.exceptions import AdapterError from unstract.sdk1.adapters.x2text.dto import TextExtractionResult from unstract.sdk1.adapters.x2text.llama_parse.src.constants import LlamaParseConfig diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer/src/llm_whisperer.py b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer/src/llm_whisperer.py index 1d8f1ee388..abecb4ce64 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer/src/llm_whisperer.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer/src/llm_whisperer.py @@ -8,6 +8,7 @@ import requests from requests import Response from requests.exceptions import ConnectionError, HTTPError, Timeout + from unstract.sdk1.adapters.exceptions import ExtractorError from unstract.sdk1.adapters.utils import AdapterUtils from unstract.sdk1.adapters.x2text.constants import X2TextConstants diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/helper.py b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/helper.py index 019f4a0c34..6a0a3e1dae 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/helper.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/helper.py @@ -11,6 +11,7 @@ LLMWhispererClientException, LLMWhispererClientV2, ) + from unstract.sdk1.adapters.exceptions import ExtractorError from unstract.sdk1.adapters.utils import AdapterUtils from unstract.sdk1.adapters.x2text.constants import X2TextConstants diff --git a/unstract/sdk1/src/unstract/sdk1/audit.py b/unstract/sdk1/src/unstract/sdk1/audit.py index abd764763d..d6bf31643b 100644 --- a/unstract/sdk1/src/unstract/sdk1/audit.py +++ b/unstract/sdk1/src/unstract/sdk1/audit.py @@ -4,6 +4,7 @@ import requests from litellm import cost_per_token from llama_index.core.callbacks import CBEventType, TokenCountingHandler + from unstract.sdk1.constants import LogLevel, ToolEnv from unstract.sdk1.platform import PlatformHelper from unstract.sdk1.tool.stream import StreamMixin diff --git a/unstract/sdk1/src/unstract/sdk1/cache.py b/unstract/sdk1/src/unstract/sdk1/cache.py index 9a85659209..ab22a9e54d 100644 --- a/unstract/sdk1/src/unstract/sdk1/cache.py +++ b/unstract/sdk1/src/unstract/sdk1/cache.py @@ -1,4 +1,5 @@ import requests + from unstract.sdk1.constants import LogLevel from unstract.sdk1.platform import PlatformHelper from unstract.sdk1.tool.base import BaseTool diff --git a/unstract/sdk1/src/unstract/sdk1/embedding.py b/unstract/sdk1/src/unstract/sdk1/embedding.py index 1e02944f83..982bb5063a 100644 --- a/unstract/sdk1/src/unstract/sdk1/embedding.py +++ b/unstract/sdk1/src/unstract/sdk1/embedding.py @@ -5,9 +5,10 @@ from typing import TYPE_CHECKING import litellm -import unstract.sdk1.patches.litellm_cohere_timeout # noqa: F401 from llama_index.core.embeddings import BaseEmbedding from pydantic import ValidationError + +import unstract.sdk1.patches.litellm_cohere_timeout # noqa: F401 from unstract.sdk1.adapters.constants import Common from unstract.sdk1.adapters.embedding1 import adapters from unstract.sdk1.constants import Common as SdkCommon diff --git a/unstract/sdk1/src/unstract/sdk1/file_storage/helper.py b/unstract/sdk1/src/unstract/sdk1/file_storage/helper.py index 8fdf7e1069..4a343da7d9 100644 --- a/unstract/sdk1/src/unstract/sdk1/file_storage/helper.py +++ b/unstract/sdk1/src/unstract/sdk1/file_storage/helper.py @@ -2,6 +2,7 @@ import fsspec from fsspec import AbstractFileSystem + from unstract.sdk1.exceptions import FileOperationError, FileStorageError from unstract.sdk1.file_storage.provider import FileStorageProvider diff --git a/unstract/sdk1/src/unstract/sdk1/file_storage/impl.py b/unstract/sdk1/src/unstract/sdk1/file_storage/impl.py index 198e838f8c..ee7c98fd66 100644 --- a/unstract/sdk1/src/unstract/sdk1/file_storage/impl.py +++ b/unstract/sdk1/src/unstract/sdk1/file_storage/impl.py @@ -7,6 +7,7 @@ import fsspec import magic import yaml + from unstract.sdk1.exceptions import FileOperationError from unstract.sdk1.file_storage.constants import FileOperationParams, FileSeekPosition from unstract.sdk1.file_storage.helper import FileStorageHelper, skip_local_cache diff --git a/unstract/sdk1/src/unstract/sdk1/file_storage/interface.py b/unstract/sdk1/src/unstract/sdk1/file_storage/interface.py index d053f149f5..c3a171cf7f 100644 --- a/unstract/sdk1/src/unstract/sdk1/file_storage/interface.py +++ b/unstract/sdk1/src/unstract/sdk1/file_storage/interface.py @@ -3,6 +3,7 @@ from typing import Any from fsspec import AbstractFileSystem + from unstract.sdk1.file_storage.constants import FileOperationParams, FileSeekPosition diff --git a/unstract/sdk1/src/unstract/sdk1/file_storage/permanent.py b/unstract/sdk1/src/unstract/sdk1/file_storage/permanent.py index a21fe54315..0d5ce03eca 100644 --- a/unstract/sdk1/src/unstract/sdk1/file_storage/permanent.py +++ b/unstract/sdk1/src/unstract/sdk1/file_storage/permanent.py @@ -3,6 +3,7 @@ import filetype import magic + from unstract.sdk1.exceptions import FileOperationError, FileStorageError from unstract.sdk1.file_storage.constants import FileOperationParams from unstract.sdk1.file_storage.impl import FileStorage diff --git a/unstract/sdk1/src/unstract/sdk1/index.py b/unstract/sdk1/src/unstract/sdk1/index.py index 92220f0a87..01e27c5f3f 100644 --- a/unstract/sdk1/src/unstract/sdk1/index.py +++ b/unstract/sdk1/src/unstract/sdk1/index.py @@ -13,6 +13,7 @@ VectorStoreQuery, VectorStoreQueryResult, ) + from unstract.sdk1.adapters.exceptions import AdapterError from unstract.sdk1.adapters.vectordb.no_op.src.no_op_custom_vectordb import ( NoOpCustomVectorDB, diff --git a/unstract/sdk1/src/unstract/sdk1/llm.py b/unstract/sdk1/src/unstract/sdk1/llm.py index 2082c24d9a..e34000aeb0 100644 --- a/unstract/sdk1/src/unstract/sdk1/llm.py +++ b/unstract/sdk1/src/unstract/sdk1/llm.py @@ -11,6 +11,7 @@ # from litellm import get_supported_openai_params from litellm import get_max_tokens + from unstract.sdk1.adapters.constants import Common from unstract.sdk1.adapters.llm1 import adapters from unstract.sdk1.constants import Common as SdkCommon diff --git a/unstract/sdk1/src/unstract/sdk1/platform.py b/unstract/sdk1/src/unstract/sdk1/platform.py index 2d3d95616a..c673d5a281 100644 --- a/unstract/sdk1/src/unstract/sdk1/platform.py +++ b/unstract/sdk1/src/unstract/sdk1/platform.py @@ -5,6 +5,7 @@ import requests from requests import RequestException, Response from requests.exceptions import ConnectionError, HTTPError + from unstract.sdk1.constants import ( AdapterKeys, Common, diff --git a/unstract/sdk1/src/unstract/sdk1/tool/cache.py b/unstract/sdk1/src/unstract/sdk1/tool/cache.py index 159f905c50..737787960e 100644 --- a/unstract/sdk1/src/unstract/sdk1/tool/cache.py +++ b/unstract/sdk1/src/unstract/sdk1/tool/cache.py @@ -1,4 +1,5 @@ import requests + from unstract.sdk1.constants import LogLevel from unstract.sdk1.platform import PlatformHelper from unstract.sdk1.tool.base import BaseTool diff --git a/unstract/sdk1/src/unstract/sdk1/tool/parser.py b/unstract/sdk1/src/unstract/sdk1/tool/parser.py index ed439c2615..787c49ebf0 100644 --- a/unstract/sdk1/src/unstract/sdk1/tool/parser.py +++ b/unstract/sdk1/src/unstract/sdk1/tool/parser.py @@ -1,6 +1,7 @@ import argparse from dotenv import find_dotenv, load_dotenv + from unstract.sdk1.constants import LogLevel diff --git a/unstract/sdk1/src/unstract/sdk1/tool/stream.py b/unstract/sdk1/src/unstract/sdk1/tool/stream.py index 92e3b67356..6cc387d331 100644 --- a/unstract/sdk1/src/unstract/sdk1/tool/stream.py +++ b/unstract/sdk1/src/unstract/sdk1/tool/stream.py @@ -5,6 +5,7 @@ from typing import Any from deprecated import deprecated + from unstract.sdk1.constants import Command, LogLevel, LogStage, ToolEnv from unstract.sdk1.exceptions import SdkError from unstract.sdk1.utils.common import Utils diff --git a/unstract/sdk1/src/unstract/sdk1/tool/validator.py b/unstract/sdk1/src/unstract/sdk1/tool/validator.py index 30d6ae475e..b55bf9a555 100644 --- a/unstract/sdk1/src/unstract/sdk1/tool/validator.py +++ b/unstract/sdk1/src/unstract/sdk1/tool/validator.py @@ -3,6 +3,7 @@ from pathlib import Path from jsonschema import Draft202012Validator, ValidationError, validators + from unstract.sdk1.constants import MetadataKey, PropKey from unstract.sdk1.tool.base import BaseTool from unstract.sdk1.tool.mime_types import EXT_MIME_MAP diff --git a/unstract/sdk1/src/unstract/sdk1/usage_handler.py b/unstract/sdk1/src/unstract/sdk1/usage_handler.py index 0d1abf8aeb..c8d1b949bb 100644 --- a/unstract/sdk1/src/unstract/sdk1/usage_handler.py +++ b/unstract/sdk1/src/unstract/sdk1/usage_handler.py @@ -5,6 +5,7 @@ from llama_index.core.callbacks import CBEventType, TokenCountingHandler from llama_index.core.callbacks.base_handler import BaseCallbackHandler from llama_index.core.embeddings import BaseEmbedding + from unstract.sdk1.constants import LogLevel from unstract.sdk1.tool.stream import StreamMixin diff --git a/unstract/sdk1/src/unstract/sdk1/utils/callback_manager.py b/unstract/sdk1/src/unstract/sdk1/utils/callback_manager.py index b5a1c5578f..2bcdad78cb 100644 --- a/unstract/sdk1/src/unstract/sdk1/utils/callback_manager.py +++ b/unstract/sdk1/src/unstract/sdk1/utils/callback_manager.py @@ -7,6 +7,7 @@ from llama_index.core.callbacks import TokenCountingHandler from llama_index.core.embeddings import BaseEmbedding from llama_index.core.llms import LLM + from unstract.sdk1.usage_handler import UsageHandler logger = logging.getLogger(__name__) diff --git a/unstract/sdk1/src/unstract/sdk1/vector_db.py b/unstract/sdk1/src/unstract/sdk1/vector_db.py index ac991e0dec..ab4d1c039d 100644 --- a/unstract/sdk1/src/unstract/sdk1/vector_db.py +++ b/unstract/sdk1/src/unstract/sdk1/vector_db.py @@ -11,6 +11,7 @@ VectorStore, VectorStoreQueryResult, ) + from unstract.sdk1.adapters.vectordb import adapters from unstract.sdk1.adapters.vectordb.constants import VectorDbConstants from unstract.sdk1.adapters.vectordb.exceptions import parse_vector_db_err diff --git a/unstract/sdk1/src/unstract/sdk1/x2txt.py b/unstract/sdk1/src/unstract/sdk1/x2txt.py index 2024f8cdbc..8d4127344e 100644 --- a/unstract/sdk1/src/unstract/sdk1/x2txt.py +++ b/unstract/sdk1/src/unstract/sdk1/x2txt.py @@ -2,6 +2,7 @@ from typing import Any import pdfplumber + from unstract.sdk1.adapters.constants import Common from unstract.sdk1.adapters.x2text import adapters from unstract.sdk1.adapters.x2text.constants import X2TextConstants diff --git a/unstract/sdk1/tests/file_storage/test_impl_rm.py b/unstract/sdk1/tests/file_storage/test_impl_rm.py index 7d4610720f..bb295df3f7 100644 --- a/unstract/sdk1/tests/file_storage/test_impl_rm.py +++ b/unstract/sdk1/tests/file_storage/test_impl_rm.py @@ -11,6 +11,7 @@ from unittest.mock import MagicMock, patch import pytest + from unstract.sdk1.exceptions import FileOperationError from unstract.sdk1.file_storage.impl import FileStorage from unstract.sdk1.file_storage.provider import FileStorageProvider @@ -149,8 +150,8 @@ def test_only_singular_delete_called(self, s3_file_storage: FileStorage) -> None boto3_client.delete_object.return_value = {"ResponseMetadata": {}} # Top-level fs.rm routes to bulk DeleteObjects (broken on MinIO). - s3_file_storage.fs.rm.side_effect = ( - lambda path, recursive: boto3_client.delete_objects( # noqa: ARG005 + s3_file_storage.fs.rm.side_effect = lambda path, recursive: ( + boto3_client.delete_objects( # noqa: ARG005 Bucket="bucket", Delete={"Objects": []} ) ) diff --git a/unstract/sdk1/tests/patches/test_litellm_cohere_timeout.py b/unstract/sdk1/tests/patches/test_litellm_cohere_timeout.py index dc9f7c7517..70b84b4349 100644 --- a/unstract/sdk1/tests/patches/test_litellm_cohere_timeout.py +++ b/unstract/sdk1/tests/patches/test_litellm_cohere_timeout.py @@ -4,6 +4,7 @@ import httpx import pytest + from unstract.sdk1.patches.litellm_cohere_timeout import ( _patched_async_embedding, _patched_embedding, @@ -105,12 +106,12 @@ def test_none_timeout_passed_to_client_post( ) call_kwargs = mock_http_handler.post.call_args - assert ( - "timeout" in call_kwargs.kwargs - ), "timeout kwarg must always be passed to client.post()" - assert ( - call_kwargs.kwargs["timeout"] is None - ), f"Expected timeout=None, got timeout={call_kwargs.kwargs['timeout']}" + assert "timeout" in call_kwargs.kwargs, ( + "timeout kwarg must always be passed to client.post()" + ) + assert call_kwargs.kwargs["timeout"] is None, ( + f"Expected timeout=None, got timeout={call_kwargs.kwargs['timeout']}" + ) def test_httpx_timeout_object_forwarded( self, diff --git a/unstract/sdk1/tests/test_anthropic_stream_complete.py b/unstract/sdk1/tests/test_anthropic_stream_complete.py index 6fa2234a1c..cb9df529b6 100644 --- a/unstract/sdk1/tests/test_anthropic_stream_complete.py +++ b/unstract/sdk1/tests/test_anthropic_stream_complete.py @@ -25,6 +25,7 @@ import litellm import pytest + from unstract.sdk1.utils import retry_utils if TYPE_CHECKING: diff --git a/unstract/sdk1/tests/test_bedrock_adapter.py b/unstract/sdk1/tests/test_bedrock_adapter.py index 6e462084bf..89c3b0b6a3 100644 --- a/unstract/sdk1/tests/test_bedrock_adapter.py +++ b/unstract/sdk1/tests/test_bedrock_adapter.py @@ -13,6 +13,7 @@ import logging import pytest + from unstract.sdk1.adapters.base1 import ( AWSBedrockEmbeddingParameters, AWSBedrockLLMParameters, diff --git a/unstract/sdk1/tests/test_branded_openai_adapters.py b/unstract/sdk1/tests/test_branded_openai_adapters.py index c1556a3dd7..f93ee51539 100644 --- a/unstract/sdk1/tests/test_branded_openai_adapters.py +++ b/unstract/sdk1/tests/test_branded_openai_adapters.py @@ -1,6 +1,7 @@ import json import pytest + from unstract.sdk1.adapters.base1 import ( MiniMaxLLMParameters, NvidiaBuildEmbeddingParameters, diff --git a/unstract/sdk1/tests/test_execution.py b/unstract/sdk1/tests/test_execution.py index 1dcb8b18a9..9238a286bd 100644 --- a/unstract/sdk1/tests/test_execution.py +++ b/unstract/sdk1/tests/test_execution.py @@ -7,6 +7,7 @@ from typing import Any, Self import pytest + from unstract.sdk1.constants import LogLevel, ToolEnv from unstract.sdk1.exceptions import SdkError from unstract.sdk1.execution.context import ( diff --git a/unstract/sdk1/tests/test_gemini_adapter.py b/unstract/sdk1/tests/test_gemini_adapter.py index fb59bcf611..daed5917e5 100644 --- a/unstract/sdk1/tests/test_gemini_adapter.py +++ b/unstract/sdk1/tests/test_gemini_adapter.py @@ -4,6 +4,7 @@ from pathlib import Path import pytest + from unstract.sdk1.adapters.base1 import GeminiLLMParameters from unstract.sdk1.adapters.llm1.gemini import GeminiLLMAdapter diff --git a/unstract/sdk1/tests/test_gemini_embedding.py b/unstract/sdk1/tests/test_gemini_embedding.py index 3bdc0f6d70..4f331838ad 100644 --- a/unstract/sdk1/tests/test_gemini_embedding.py +++ b/unstract/sdk1/tests/test_gemini_embedding.py @@ -1,6 +1,7 @@ import json import pytest + from unstract.sdk1.adapters.embedding1.gemini import GeminiEmbeddingAdapter from unstract.sdk1.adapters.enums import AdapterTypes diff --git a/unstract/sdk1/tests/test_llm_compat.py b/unstract/sdk1/tests/test_llm_compat.py index c9eac4095d..d7d3a6b17e 100644 --- a/unstract/sdk1/tests/test_llm_compat.py +++ b/unstract/sdk1/tests/test_llm_compat.py @@ -4,6 +4,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest + from unstract.sdk1.llm import ( LLM, ChatMessage, @@ -358,7 +359,7 @@ def test_multi_turn_conversation(self: Self) -> None: ] result = LLMCompat._messages_to_prompt(messages) assert result == ( - "system: Be concise\n" "user: First\n" "assistant: Answer\n" "user: Follow-up" + "system: Be concise\nuser: First\nassistant: Answer\nuser: Follow-up" ) def test_empty_messages_returns_empty_string(self: Self) -> None: diff --git a/unstract/sdk1/tests/test_llm_whisperer_v2_params.py b/unstract/sdk1/tests/test_llm_whisperer_v2_params.py index d8ca030e31..352571b148 100644 --- a/unstract/sdk1/tests/test_llm_whisperer_v2_params.py +++ b/unstract/sdk1/tests/test_llm_whisperer_v2_params.py @@ -1,6 +1,7 @@ """Tests for the query params the LLMWhisperer V2 adapter sends.""" import pytest + from unstract.sdk1.adapters.x2text.llm_whisperer_v2.src.dto import ( WhispererRequestParams, ) diff --git a/unstract/sdk1/tests/test_mistral_embedding.py b/unstract/sdk1/tests/test_mistral_embedding.py index 6569d0ec78..16eef99284 100644 --- a/unstract/sdk1/tests/test_mistral_embedding.py +++ b/unstract/sdk1/tests/test_mistral_embedding.py @@ -1,6 +1,7 @@ import json import pytest + from unstract.sdk1.adapters.embedding1.mistral import MistralEmbeddingAdapter from unstract.sdk1.adapters.enums import AdapterTypes diff --git a/unstract/sdk1/tests/test_openai_compatible_adapter.py b/unstract/sdk1/tests/test_openai_compatible_adapter.py index d1dd631d2b..8bdcc2da35 100644 --- a/unstract/sdk1/tests/test_openai_compatible_adapter.py +++ b/unstract/sdk1/tests/test_openai_compatible_adapter.py @@ -4,6 +4,7 @@ from unittest.mock import MagicMock, patch import pytest + from unstract.sdk1.adapters.base1 import OpenAICompatibleLLMParameters from unstract.sdk1.adapters.constants import Common from unstract.sdk1.adapters.llm1 import adapters @@ -113,9 +114,9 @@ def test_openai_compatible_schema_exposes_reasoning_toggle() -> None: # vacuously when the property is omitted from the submitted instance # (JSON Schema treats a `properties`-only `if` as valid in that case). for branch in schema["allOf"]: - assert branch["if"].get("required") == [ - "enable_reasoning" - ], "if/then branches must anchor on the property being present" + assert branch["if"].get("required") == ["enable_reasoning"], ( + "if/then branches must anchor on the property being present" + ) def test_openai_compatible_validate_auto_detects_reasoning_for_known_families() -> None: @@ -165,9 +166,9 @@ def test_openai_compatible_validate_preserves_non_reasoning_models() -> None: } ) - assert ( - validated["temperature"] == _DEFAULT_TEMPERATURE - ), f"{model} should keep temperature" + assert validated["temperature"] == _DEFAULT_TEMPERATURE, ( + f"{model} should keep temperature" + ) assert validated["max_tokens"] == 1024, f"{model} should keep max_tokens" assert "extra_body" not in validated, f"{model} should not set extra_body" diff --git a/unstract/sdk1/tests/test_platform.py b/unstract/sdk1/tests/test_platform.py index f318175063..b390f94562 100644 --- a/unstract/sdk1/tests/test_platform.py +++ b/unstract/sdk1/tests/test_platform.py @@ -5,6 +5,7 @@ import pytest from _pytest.monkeypatch import MonkeyPatch from requests.exceptions import ConnectionError, HTTPError + from unstract.sdk1.exceptions import SdkError from unstract.sdk1.platform import PlatformHelper diff --git a/unstract/sdk1/tests/test_prompt_caching.py b/unstract/sdk1/tests/test_prompt_caching.py index 22912525ef..774f74282b 100644 --- a/unstract/sdk1/tests/test_prompt_caching.py +++ b/unstract/sdk1/tests/test_prompt_caching.py @@ -13,6 +13,7 @@ from typing import Any import pytest + from unstract.sdk1.adapters.base1 import ( AnthropicLLMParameters, AWSBedrockLLMParameters, diff --git a/unstract/sdk1/tests/test_sampling_strip.py b/unstract/sdk1/tests/test_sampling_strip.py index d6d792a3d7..432fab4fe8 100644 --- a/unstract/sdk1/tests/test_sampling_strip.py +++ b/unstract/sdk1/tests/test_sampling_strip.py @@ -18,6 +18,7 @@ from typing import Any import pytest + from unstract.sdk1.adapters.base1 import ( _DEPRECATED_SAMPLING_PARAMS, AnthropicLLMParameters, @@ -219,9 +220,9 @@ def test_strip_skipped_when_both_fields_opaque_and_logs_debug( out = _strip_deprecated_sampling_params(inp) # Documented limitation: not stripped when no field carries the model id. assert out["temperature"] == pytest.approx(0.5) - assert any( - "Sampling-param strip skipped" in rec.message for rec in caplog.records - ), "expected debug breadcrumb when strip is a no-op" + assert any("Sampling-param strip skipped" in rec.message for rec in caplog.records), ( + "expected debug breadcrumb when strip is a no-op" + ) def test_strip_does_not_log_when_no_sampling_params_present( diff --git a/unstract/sdk1/tests/utils/test_collect_with_retry.py b/unstract/sdk1/tests/utils/test_collect_with_retry.py index 974a6e6e00..64ae11cb49 100644 --- a/unstract/sdk1/tests/utils/test_collect_with_retry.py +++ b/unstract/sdk1/tests/utils/test_collect_with_retry.py @@ -13,6 +13,7 @@ from unittest.mock import patch import pytest + from unstract.sdk1.utils import retry_utils from unstract.sdk1.utils.retry_utils import collect_with_retry diff --git a/unstract/sdk1/tests/utils/test_retry_utils.py b/unstract/sdk1/tests/utils/test_retry_utils.py index 5d001d85cc..e289a9181f 100644 --- a/unstract/sdk1/tests/utils/test_retry_utils.py +++ b/unstract/sdk1/tests/utils/test_retry_utils.py @@ -8,6 +8,7 @@ import pytest from _pytest.monkeypatch import MonkeyPatch from requests.exceptions import ConnectionError, HTTPError, Timeout + from unstract.sdk1.utils.retry_utils import ( calculate_delay, create_retry_decorator, diff --git a/workers/api-deployment/tasks.py b/workers/api-deployment/tasks.py index 4d4b56b3b5..cca0a32118 100644 --- a/workers/api-deployment/tasks.py +++ b/workers/api-deployment/tasks.py @@ -24,14 +24,13 @@ from shared.processing.files import FileProcessingUtils from shared.workflow.execution import WorkerExecutionContext, WorkflowOrchestrationUtils from shared.workflow.execution.tool_validation import validate_workflow_tool_instances -from worker import app - from unstract.core.data_models import ( ExecutionStatus, FileHashData, WorkerFileData, ) from unstract.core.worker_models import ApiDeploymentResultStatus +from worker import app logger = WorkerLogger.get_logger(__name__) diff --git a/workers/callback/tasks.py b/workers/callback/tasks.py index 8259fff591..ebee461bd9 100644 --- a/workers/callback/tasks.py +++ b/workers/callback/tasks.py @@ -1050,9 +1050,10 @@ def _get_execution_directories(context: CallbackContext) -> list[tuple[str, any, Raises: ValueError: If execution type cannot be determined """ - from unstract.filesystem import FileStorageType from unstract.workflow_execution.execution_file_handler import ExecutionFileHandler + from unstract.filesystem import FileStorageType + # Determine if this is an API or workflow execution is_api_execution = context.pipeline_data and context.pipeline_data.get( "is_api", False diff --git a/workers/executor/executors/file_utils.py b/workers/executor/executors/file_utils.py index 92f80d6d76..3741aa26d4 100644 --- a/workers/executor/executors/file_utils.py +++ b/workers/executor/executors/file_utils.py @@ -5,7 +5,6 @@ """ from executor.executors.constants import ExecutionSource, FileStorageKeys - from unstract.sdk1.file_storage import FileStorage from unstract.sdk1.file_storage.constants import StorageType from unstract.sdk1.file_storage.env_helper import EnvHelper diff --git a/workers/executor/executors/index.py b/workers/executor/executors/index.py index 4bbbaf8839..6ae395718c 100644 --- a/workers/executor/executors/index.py +++ b/workers/executor/executors/index.py @@ -20,7 +20,6 @@ InstanceIdentifiers, ProcessingOptions, ) - from unstract.sdk1.constants import LogLevel from unstract.sdk1.exceptions import SdkError, parse_litellm_err from unstract.sdk1.file_storage.impl import FileStorage diff --git a/workers/executor/executors/legacy_executor.py b/workers/executor/executors/legacy_executor.py index 7daaeda6ed..8230d10d3d 100644 --- a/workers/executor/executors/legacy_executor.py +++ b/workers/executor/executors/legacy_executor.py @@ -26,7 +26,6 @@ run_lookup_enrichment, run_webhook_postprocessing, ) - from unstract.sdk1.adapters.exceptions import AdapterError from unstract.sdk1.adapters.x2text.constants import X2TextConstants from unstract.sdk1.adapters.x2text.llm_whisperer.src import LLMWhisperer @@ -352,7 +351,6 @@ def _get_indexing_deps(): Wrapped in a method so tests can mock it cleanly. """ from executor.executors.index import Index - from unstract.sdk1.embedding import EmbeddingCompat from unstract.sdk1.vector_db import VectorDB @@ -1301,7 +1299,6 @@ def _get_prompt_deps(): from executor.executors.variable_replacement import ( VariableReplacementService, ) - from unstract.sdk1.embedding import EmbeddingCompat from unstract.sdk1.llm import LLM from unstract.sdk1.vector_db import VectorDB @@ -1664,7 +1661,6 @@ def _execute_single_prompt( """Run one prompt end-to-end; return its usage rows.""" from executor.executors.constants import PromptServiceConstants as PSKeys from executor.executors.constants import RetrievalStrategy - from unstract.sdk1.utils.indexing import IndexingUtils ( diff --git a/workers/executor/executors/lookup_enrichment.py b/workers/executor/executors/lookup_enrichment.py index 3cfbe6b767..990d2c011e 100644 --- a/workers/executor/executors/lookup_enrichment.py +++ b/workers/executor/executors/lookup_enrichment.py @@ -11,7 +11,6 @@ from executor.executors.constants import PromptServiceConstants as PSKeys from executor.executors.plugins import ExecutorPluginLoader - from unstract.sdk1.constants import LogLevel logger = logging.getLogger(__name__) diff --git a/workers/executor/executors/postprocessor.py b/workers/executor/executors/postprocessor.py index d179ca9895..1f98996097 100644 --- a/workers/executor/executors/postprocessor.py +++ b/workers/executor/executors/postprocessor.py @@ -8,7 +8,6 @@ from typing import Any import requests - from unstract.core.network.ssrf import is_safe_webhook_url logger = logging.getLogger(__name__) diff --git a/workers/executor/executors/retrievers/retriever_llm.py b/workers/executor/executors/retrievers/retriever_llm.py index c2038f9181..36285271ee 100644 --- a/workers/executor/executors/retrievers/retriever_llm.py +++ b/workers/executor/executors/retrievers/retriever_llm.py @@ -14,7 +14,6 @@ ) from llama_index.core.llms.llm import LLM as LlamaIndexBaseLLM # noqa: N811 from pydantic import PrivateAttr - from unstract.sdk1.llm import LLM, LLMCompat diff --git a/workers/executor/tasks.py b/workers/executor/tasks.py index 1244e60b9c..bd40d672e1 100644 --- a/workers/executor/tasks.py +++ b/workers/executor/tasks.py @@ -20,7 +20,6 @@ from shared.enums.task_enums import TaskName from shared.infrastructure.config import WorkerConfig from shared.infrastructure.logging import WorkerLogger - from unstract.sdk1.execution.context import ExecutionContext from unstract.sdk1.execution.orchestrator import ExecutionOrchestrator from unstract.sdk1.execution.result import ExecutionResult diff --git a/workers/file_processing/structure_tool_task.py b/workers/file_processing/structure_tool_task.py index 95d7452116..cc7bcfb389 100644 --- a/workers/file_processing/structure_tool_task.py +++ b/workers/file_processing/structure_tool_task.py @@ -28,7 +28,6 @@ ) from shared.enums.task_enums import TaskName from shared.infrastructure.context import StateStore - from unstract.sdk1.constants import ToolEnv, UsageKwargs from unstract.sdk1.execution.context import ExecutionContext from unstract.sdk1.execution.result import ExecutionResult diff --git a/workers/file_processing/tasks.py b/workers/file_processing/tasks.py index 764953834b..6edb56952c 100644 --- a/workers/file_processing/tasks.py +++ b/workers/file_processing/tasks.py @@ -46,7 +46,6 @@ create_organization_context, ) from shared.processing.files.processor import FileProcessor - from unstract.core.data_models import ( ExecutionStatus, FileBatchData, diff --git a/workers/general/tasks.py b/workers/general/tasks.py index 861ddc7ddc..1967dc6f4c 100644 --- a/workers/general/tasks.py +++ b/workers/general/tasks.py @@ -50,10 +50,6 @@ ) from shared.workflow.execution.tool_validation import validate_workflow_tool_instances -# File management handled by StreamingFileDiscovery -# Import from local worker module (avoid circular import) -from worker import app, config - # Import shared data models for type safety from unstract.core.data_models import ( ExecutionStatus, @@ -65,6 +61,10 @@ # Import common workflow utilities from unstract.core.workflow_utils import WorkflowTypeDetector +# File management handled by StreamingFileDiscovery +# Import from local worker module (avoid circular import) +from worker import app, config + logger = WorkerLogger.get_logger(__name__) diff --git a/workers/log_consumer/process_log_history.py b/workers/log_consumer/process_log_history.py index a941bb0609..8a534633c5 100755 --- a/workers/log_consumer/process_log_history.py +++ b/workers/log_consumer/process_log_history.py @@ -17,7 +17,6 @@ import sys import httpx - from unstract.core.cache.redis_queue_client import RedisQueueClient logger = logging.getLogger(__name__) diff --git a/workers/log_consumer/redis_stream_consumer.py b/workers/log_consumer/redis_stream_consumer.py index a2242f11f2..349119f650 100644 --- a/workers/log_consumer/redis_stream_consumer.py +++ b/workers/log_consumer/redis_stream_consumer.py @@ -37,7 +37,6 @@ from shared.enums.worker_enums import WorkerType from shared.infrastructure.config.builder import WorkerBuilder from shared.infrastructure.logging import WorkerLogger - from unstract.core.cache.redis_client import create_redis_client from unstract.core.constants import LogProcessingTask diff --git a/workers/log_consumer/tasks.py b/workers/log_consumer/tasks.py index 809b63c081..bfa3a803a1 100644 --- a/workers/log_consumer/tasks.py +++ b/workers/log_consumer/tasks.py @@ -13,7 +13,6 @@ from shared.infrastructure.config import WorkerConfig from shared.infrastructure.logging import WorkerLogger from shared.utils.api_client_singleton import get_singleton_api_client - from unstract.core.cache.redis_queue_client import RedisQueueClient from unstract.core.constants import LogEventArgument, LogProcessingTask from unstract.core.log_utils import store_execution_log diff --git a/workers/notification/providers/api_webhook.py b/workers/notification/providers/api_webhook.py index 3acee31fa4..9f33aa36bb 100644 --- a/workers/notification/providers/api_webhook.py +++ b/workers/notification/providers/api_webhook.py @@ -11,7 +11,6 @@ from notification.providers.webhook_provider import WebhookProvider from shared.infrastructure.logging import WorkerLogger - from unstract.core.notification_clubbed_renderer import build_envelope logger = WorkerLogger.get_logger(__name__) diff --git a/workers/notification/providers/registry.py b/workers/notification/providers/registry.py index b6f4c58ac9..43b9a82da3 100644 --- a/workers/notification/providers/registry.py +++ b/workers/notification/providers/registry.py @@ -9,7 +9,6 @@ from notification.providers.base_provider import BaseNotificationProvider from notification.providers.slack_webhook import SlackWebhook from shared.infrastructure.logging import WorkerLogger - from unstract.core.notification_enums import NotificationType logger = WorkerLogger.get_logger(__name__) diff --git a/workers/notification/providers/slack_webhook.py b/workers/notification/providers/slack_webhook.py index 6d88564124..e63447f2ed 100644 --- a/workers/notification/providers/slack_webhook.py +++ b/workers/notification/providers/slack_webhook.py @@ -11,7 +11,6 @@ from notification.providers.webhook_provider import WebhookProvider from shared.infrastructure.logging import WorkerLogger - from unstract.core.notification_clubbed_renderer import ( build_envelope, render_slack_text, diff --git a/workers/notification/providers/webhook_provider.py b/workers/notification/providers/webhook_provider.py index 57a6382d55..8675151768 100644 --- a/workers/notification/providers/webhook_provider.py +++ b/workers/notification/providers/webhook_provider.py @@ -13,7 +13,6 @@ ValidationError, ) from shared.infrastructure.logging import WorkerLogger - from unstract.core.notification_utils import ( build_webhook_headers, send_webhook_request, diff --git a/workers/notification/tasks.py b/workers/notification/tasks.py index 10c908b119..07cfa66d15 100644 --- a/workers/notification/tasks.py +++ b/workers/notification/tasks.py @@ -25,7 +25,6 @@ from queue_backend import worker_task from shared.infrastructure.config import WorkerConfig from shared.infrastructure.logging import WorkerLogger - from unstract.core.network.ssrf import safe_host from unstract.core.notification_enums import NotificationType diff --git a/workers/queue_backend/pg_barrier.py b/workers/queue_backend/pg_barrier.py index 08e0a1d464..54c430e231 100644 --- a/workers/queue_backend/pg_barrier.py +++ b/workers/queue_backend/pg_barrier.py @@ -72,7 +72,6 @@ import psycopg2 import psycopg2.errors import psycopg2.extensions - from unstract.core.data_models import LEGACY_TRANSPORT_KEY, LEGACY_TRANSPORT_VALUE from unstract.core.jsonb import dumps_for_jsonb diff --git a/workers/queue_backend/pg_queue/consumer.py b/workers/queue_backend/pg_queue/consumer.py index 9f0e27fb9e..10971b7d93 100644 --- a/workers/queue_backend/pg_queue/consumer.py +++ b/workers/queue_backend/pg_queue/consumer.py @@ -35,7 +35,6 @@ from typing import TYPE_CHECKING, Any, TypeVar from celery import current_app - from unstract.core.data_models import ContinuationSpec, TaskPayload from unstract.core.jsonb import dumps_for_jsonb diff --git a/workers/queue_backend/pg_queue/metrics.py b/workers/queue_backend/pg_queue/metrics.py index d34c7d885d..af35fd3ccc 100644 --- a/workers/queue_backend/pg_queue/metrics.py +++ b/workers/queue_backend/pg_queue/metrics.py @@ -268,8 +268,7 @@ def __init__( ) self.claim_recovered = Counter( "pg_reaper_claim_recovered_total", - "Orphan orchestration claims recovered (crash-window execution " - "marked ERROR)", + "Orphan orchestration claims recovered (crash-window execution marked ERROR)", registry=self.registry, ) self.claim_gc = Counter( diff --git a/workers/queue_backend/pg_queue/pg_scheduler.py b/workers/queue_backend/pg_queue/pg_scheduler.py index 13ddd0d679..12535fb9ae 100644 --- a/workers/queue_backend/pg_queue/pg_scheduler.py +++ b/workers/queue_backend/pg_queue/pg_scheduler.py @@ -36,7 +36,6 @@ from typing import TYPE_CHECKING, NamedTuple from croniter import croniter - from unstract.core.data_models import TaskPayload from unstract.core.jsonb import dumps_for_jsonb diff --git a/workers/queue_backend/pg_queue/result_backend.py b/workers/queue_backend/pg_queue/result_backend.py index 16f8eefd8c..ed9af4b289 100644 --- a/workers/queue_backend/pg_queue/result_backend.py +++ b/workers/queue_backend/pg_queue/result_backend.py @@ -73,7 +73,6 @@ from typing import TYPE_CHECKING, Any, Final, Self import redis - from unstract.core.cache.redis_client import create_redis_client from unstract.core.data_models import PgTaskStatus from unstract.core.jsonb import dumps_for_jsonb, sanitize_for_jsonb diff --git a/workers/scheduler/tasks.py b/workers/scheduler/tasks.py index d79f7bf758..03d5a20074 100644 --- a/workers/scheduler/tasks.py +++ b/workers/scheduler/tasks.py @@ -37,7 +37,6 @@ ) from shared.patterns.notification.helper import trigger_notification from shared.utils.api_client_singleton import get_singleton_api_client - from unstract.core.data_models import ( LEGACY_TRANSPORT_KEY, LEGACY_TRANSPORT_VALUE, diff --git a/workers/shared/api/internal_client.py b/workers/shared/api/internal_client.py index ee9fbca7ac..5655c8298d 100644 --- a/workers/shared/api/internal_client.py +++ b/workers/shared/api/internal_client.py @@ -58,7 +58,6 @@ WebhookNotificationRequest, ) from shared.models.scheduler_models import SchedulerExecutionResult - from unstract.core.data_models import ( ExecutionStatus, FileExecutionCreateRequest, diff --git a/workers/shared/clients/usage_client.py b/workers/shared/clients/usage_client.py index 0bbaba9f67..635796738a 100644 --- a/workers/shared/clients/usage_client.py +++ b/workers/shared/clients/usage_client.py @@ -16,7 +16,6 @@ from uuid import UUID from shared.data.response_models import APIResponse, ResponseStatus - from unstract.core.data_models import UsageResponseData from .base_client import BaseAPIClient diff --git a/workers/shared/clients/workflow_client.py b/workers/shared/clients/workflow_client.py index 42c1ab549f..3faabac809 100644 --- a/workers/shared/clients/workflow_client.py +++ b/workers/shared/clients/workflow_client.py @@ -19,7 +19,6 @@ from uuid import UUID from shared.data.response_models import APIResponse, ResponseStatus - from unstract.core.data_models import ( WorkflowDefinitionResponseData, WorkflowEndpointConfigResponseData, diff --git a/workers/shared/infrastructure/caching/cache_utils.py b/workers/shared/infrastructure/caching/cache_utils.py index 28441a6fc9..05fdfd0175 100644 --- a/workers/shared/infrastructure/caching/cache_utils.py +++ b/workers/shared/infrastructure/caching/cache_utils.py @@ -9,7 +9,6 @@ import time from shared.infrastructure.config import WorkerConfig - from unstract.core.cache.redis_client import create_redis_client logger = logging.getLogger(__name__) diff --git a/workers/shared/infrastructure/config/worker_config.py b/workers/shared/infrastructure/config/worker_config.py index e1e11997c2..b3d29afa5f 100644 --- a/workers/shared/infrastructure/config/worker_config.py +++ b/workers/shared/infrastructure/config/worker_config.py @@ -247,14 +247,14 @@ class WorkerConfig: # by directly assigning shared_session to client.session, which breaks fork detection. # Only enable if you understand the fork-safety implications and are not using prefork pool. enable_api_client_singleton: bool = field( - default_factory=lambda: os.getenv("ENABLE_API_CLIENT_SINGLETON", "false").lower() - == "true" + default_factory=lambda: ( + os.getenv("ENABLE_API_CLIENT_SINGLETON", "false").lower() == "true" + ) ) enable_organization_context_cache: bool = field( - default_factory=lambda: os.getenv( - "ENABLE_ORGANIZATION_CONTEXT_CACHE", "true" - ).lower() - == "true" + default_factory=lambda: ( + os.getenv("ENABLE_ORGANIZATION_CONTEXT_CACHE", "true").lower() == "true" + ) ) api_client_pool_size: int = field( default_factory=lambda: int(os.getenv("API_CLIENT_POOL_SIZE", "10")) @@ -277,16 +277,19 @@ class WorkerConfig: # Debug Logging Control (Performance Optimization) enable_debug_logging: bool = field( - default_factory=lambda: os.getenv("ENABLE_DEBUG_LOGGING", "false").lower() - == "true" + default_factory=lambda: ( + os.getenv("ENABLE_DEBUG_LOGGING", "false").lower() == "true" + ) ) debug_api_client_init: bool = field( - default_factory=lambda: os.getenv("DEBUG_API_CLIENT_INIT", "false").lower() - == "true" + default_factory=lambda: ( + os.getenv("DEBUG_API_CLIENT_INIT", "false").lower() == "true" + ) ) debug_organization_context: bool = field( - default_factory=lambda: os.getenv("DEBUG_ORGANIZATION_CONTEXT", "false").lower() - == "true" + default_factory=lambda: ( + os.getenv("DEBUG_ORGANIZATION_CONTEXT", "false").lower() == "true" + ) ) # Task Timeout Settings (in seconds) @@ -301,8 +304,9 @@ class WorkerConfig: default_factory=lambda: os.getenv("ENABLE_METRICS", "true").lower() == "true" ) enable_health_server: bool = field( - default_factory=lambda: os.getenv("ENABLE_HEALTH_SERVER", "true").lower() - == "true" + default_factory=lambda: ( + os.getenv("ENABLE_HEALTH_SERVER", "true").lower() == "true" + ) ) metrics_port: int = field( default_factory=lambda: int(os.getenv("METRICS_PORT", "8080")) diff --git a/workers/shared/infrastructure/logging/logger.py b/workers/shared/infrastructure/logging/logger.py index c82619acf9..9b695d39e7 100644 --- a/workers/shared/infrastructure/logging/logger.py +++ b/workers/shared/infrastructure/logging/logger.py @@ -798,7 +798,9 @@ def ensure_dict_access(obj: Any, keys: list, default: Any = None) -> dict[str, A Dictionary with extracted values Example: - >>> ensure_dict_access(file_data, ["execution_id", "workflow_id", "organization_id"]) + >>> ensure_dict_access( + ... file_data, ["execution_id", "workflow_id", "organization_id"] + ... ) {'execution_id': '123', 'workflow_id': '456', 'organization_id': 'org_789'} """ return {key: safe_get_attr(obj, key, default) for key in keys} diff --git a/workers/shared/models/file_processing.py b/workers/shared/models/file_processing.py index af40624176..613139646c 100644 --- a/workers/shared/models/file_processing.py +++ b/workers/shared/models/file_processing.py @@ -12,7 +12,6 @@ # from unstract.api.internal_client import InternalAPIClient # from unstract.infrastructure.logging.workflow_logger import WorkerWorkflowLogger from shared.infrastructure.logging import WorkerLogger - from unstract.core.data_models import FileHashData, WorkerFileData logger = WorkerLogger.get_logger(__name__) diff --git a/workers/shared/models/result_models.py b/workers/shared/models/result_models.py index 29a75cabf0..2fa0cd6c8a 100644 --- a/workers/shared/models/result_models.py +++ b/workers/shared/models/result_models.py @@ -13,9 +13,9 @@ # Import shared domain models from core sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../../unstract/core/src")) from shared.enums import QueueResultStatus +from unstract.core.worker_models import FileExecutionResult from unstract.core import ExecutionStatus, serialize_dataclass_to_dict -from unstract.core.worker_models import FileExecutionResult # Import worker enums from ..enums import WebhookStatus diff --git a/workers/shared/patterns/worker_patterns.py b/workers/shared/patterns/worker_patterns.py index fbd915efe9..208a09c688 100644 --- a/workers/shared/patterns/worker_patterns.py +++ b/workers/shared/patterns/worker_patterns.py @@ -17,9 +17,10 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../unstract/core/src")) -from unstract.core import ExecutionStatus, serialize_dataclass_to_dict from unstract.core.worker_models import FileExecutionResult +from unstract.core import ExecutionStatus, serialize_dataclass_to_dict + logger = logging.getLogger(__name__) diff --git a/workers/shared/processing/files/processor.py b/workers/shared/processing/files/processor.py index 90009d3a55..72dca7496f 100644 --- a/workers/shared/processing/files/processor.py +++ b/workers/shared/processing/files/processor.py @@ -24,7 +24,6 @@ from typing import Any from shared.models.file_processing import FileProcessingContext - from unstract.core.data_models import ExecutionStatus, FileHashData, WorkerFileData from unstract.core.worker_models import FileProcessingResult, WorkflowExecutionResult diff --git a/workers/shared/processing/files/utils.py b/workers/shared/processing/files/utils.py index a423b7e274..0cecd09081 100644 --- a/workers/shared/processing/files/utils.py +++ b/workers/shared/processing/files/utils.py @@ -409,8 +409,7 @@ def handle_file_format_variations( else: # Single item - wrap in dict logger.warning( - f"Unexpected file data type: {type(files_data)}. " - "Wrapping as single file." + f"Unexpected file data type: {type(files_data)}. Wrapping as single file." ) return {"single_file": {"file_data": files_data}} diff --git a/workers/shared/utils/api_result_cache.py b/workers/shared/utils/api_result_cache.py index 8fcec05beb..61deefdaf9 100644 --- a/workers/shared/utils/api_result_cache.py +++ b/workers/shared/utils/api_result_cache.py @@ -9,7 +9,6 @@ from typing import Any from shared.utils.api_hub_factory import APIHubUsageUtil - from unstract.core.data_models import FileHashData from unstract.core.worker_models import ( ApiDeploymentResultStatus, diff --git a/workers/shared/utils/manual_review_factory.py b/workers/shared/utils/manual_review_factory.py index 69c1011553..7b2ff13c21 100644 --- a/workers/shared/utils/manual_review_factory.py +++ b/workers/shared/utils/manual_review_factory.py @@ -8,7 +8,6 @@ from typing import Any, Protocol from client_plugin_registry import get_client_plugin, has_client_plugin - from unstract.core.data_models import WorkerFileData from ..infrastructure.logging import WorkerLogger diff --git a/workers/shared/workflow/destination_connector.py b/workers/shared/workflow/destination_connector.py index 7d0d62ebeb..51722f3996 100644 --- a/workers/shared/workflow/destination_connector.py +++ b/workers/shared/workflow/destination_connector.py @@ -34,7 +34,6 @@ ) from shared.workflow.connectors.service import WorkerConnectorService from shared.workflow.logger_helper import WorkflowLoggerHelper - from unstract.connectors.connectorkit import Connectorkit from unstract.connectors.exceptions import ConnectorError from unstract.core.data_models import ConnectionType as CoreConnectionType @@ -46,7 +45,6 @@ FileExecutionStageStatus, FileExecutionStatusTracker, ) -from unstract.filesystem import FileStorageType, FileSystem from unstract.sdk1.constants import ToolExecKey from unstract.sdk1.tool.mime_types import EXT_MIME_MAP from unstract.workflow_execution.constants import ( @@ -58,6 +56,8 @@ ExecutionFileHandler, ) +from unstract.filesystem import FileStorageType, FileSystem + if TYPE_CHECKING: from ..api_client import InternalAPIClient diff --git a/workers/shared/workflow/execution/active_file_manager.py b/workers/shared/workflow/execution/active_file_manager.py index be056b40aa..75c4dc4179 100644 --- a/workers/shared/workflow/execution/active_file_manager.py +++ b/workers/shared/workflow/execution/active_file_manager.py @@ -653,7 +653,7 @@ def cleanup_cache_entries( deleted_count = cache.redis_client.delete(*batch_keys) cleaned_count += deleted_count logger_instance.debug( - f"Cleaned up {deleted_count} cache entries for {provider_uuid} (batch {i//batch_size + 1})" + f"Cleaned up {deleted_count} cache entries for {provider_uuid} (batch {i // batch_size + 1})" ) except Exception as batch_error: logger_instance.warning( diff --git a/workers/shared/workflow/execution/orchestration_utils.py b/workers/shared/workflow/execution/orchestration_utils.py index d40ac36636..0520bedac4 100644 --- a/workers/shared/workflow/execution/orchestration_utils.py +++ b/workers/shared/workflow/execution/orchestration_utils.py @@ -11,7 +11,6 @@ from queue_backend import BarrierHandle, FairnessKey from queue_backend.pg_barrier import PgBarrier - from unstract.core.data_models import ExecutionStatus from ...enums import FileDestinationType, PipelineType diff --git a/workers/shared/workflow/execution/service.py b/workers/shared/workflow/execution/service.py index a62253d51c..c4576b15e5 100644 --- a/workers/shared/workflow/execution/service.py +++ b/workers/shared/workflow/execution/service.py @@ -1171,6 +1171,7 @@ def _copy_filesystem_file( from unstract.connectors.constants import Common from unstract.connectors.filesystems import connectors + from unstract.filesystem import FileStorageType, FileSystem # Get workflow file storage diff --git a/workers/shared/workflow/execution/tool_validation.py b/workers/shared/workflow/execution/tool_validation.py index ed40b6554c..623782fd63 100644 --- a/workers/shared/workflow/execution/tool_validation.py +++ b/workers/shared/workflow/execution/tool_validation.py @@ -9,7 +9,6 @@ from shared.api.internal_client import InternalAPIClient from shared.infrastructure.logging.workflow_logger import WorkerWorkflowLogger - from unstract.core.data_models import ExecutionStatus logger = logging.getLogger(__name__) diff --git a/workers/shared/workflow/source_connector.py b/workers/shared/workflow/source_connector.py index 950b54e462..683c7aa4bc 100644 --- a/workers/shared/workflow/source_connector.py +++ b/workers/shared/workflow/source_connector.py @@ -15,7 +15,6 @@ from shared.infrastructure.logging.logger import WorkerLogger from shared.workflow.logger_helper import WorkflowLoggerHelper - from unstract.core.data_models import ConnectionType as CoreConnectionType logger = WorkerLogger.get_logger(__name__)