diff --git a/docs/02_concepts/12_storage_clients.mdx b/docs/02_concepts/12_storage_clients.mdx index 7b953fd93..5e33bed1c 100644 --- a/docs/02_concepts/12_storage_clients.mdx +++ b/docs/02_concepts/12_storage_clients.mdx @@ -17,7 +17,7 @@ Storage clients are the components that read and write your [storages](./storage By default, the Actor uses a `SmartApifyStorageClient`, a hybrid client that delegates to one of two underlying clients depending on the environment: - When running on the Apify platform (detected automatically), or when you pass `force_cloud=True`, it uses the cloud client, `ApifyStorageClient`, which persists data through the Apify API. -- When running locally, it uses the local client, `FileSystemStorageClient`, which emulates platform storages on your filesystem under the `storage` folder. +- When running locally, it uses the local client, `ApifyFileSystemStorageClient`, which emulates platform storages on your filesystem under the `storage` folder. As a result, the same Actor code can run unchanged both locally and on the platform. @@ -27,7 +27,7 @@ The `apify.storage_clients` module provides the following clients: - `SmartApifyStorageClient` - the default hybrid client. It wraps a `cloud_storage_client` and a `local_storage_client` and routes each call to the right one. - `ApifyStorageClient` - talks to the Apify API. Used as the cloud client. -- `FileSystemStorageClient` - persists data to the local filesystem. Used as the default local client. +- `ApifyFileSystemStorageClient` - persists data to the local filesystem. Used as the default local client. - `MemoryStorageClient` - keeps everything in memory and persists nothing. Useful for tests and short-lived runs. All of these clients implement Crawlee's `StorageClient` interface, so any of them can be used as a sub-client of `SmartApifyStorageClient`. For details, see [customizing the storage client](#customizing-the-storage-client). diff --git a/docs/04_upgrading/upgrading_to_v4.md b/docs/04_upgrading/upgrading_to_v4.md index edb105a7e..663d3b112 100644 --- a/docs/04_upgrading/upgrading_to_v4.md +++ b/docs/04_upgrading/upgrading_to_v4.md @@ -85,6 +85,24 @@ run = await Actor.start('my-actor-id', wait_for_finish=60) run = await Actor.call('my-actor-id', wait=timedelta(seconds=60)) ``` +## Storage client exports + +In v3, `apify.storage_clients.FileSystemStorageClient` was an alias for the Apify-specific file-system client, which shadowed Crawlee's distinct class of the same name. In v4 the name refers to Crawlee's `FileSystemStorageClient` (the base client), and the Apify variant is exported under its own name, `ApifyFileSystemStorageClient`. + +`ApifyFileSystemStorageClient` stays the default local client, so most Actors need no change. Update your imports only if you construct the local client explicitly and rely on its Apify behavior, such as preserving the Actor `INPUT` record on purge or local `PAY_PER_EVENT` charging. + +```python +# Before (v3) +from apify.storage_clients import FileSystemStorageClient + +storage_client = FileSystemStorageClient() + +# After (v4) +from apify.storage_clients import ApifyFileSystemStorageClient + +storage_client = ApifyFileSystemStorageClient() +``` + ## Built on apify-client v3 The SDK is now built on [`apify-client`](https://docs.apify.com/api/client/python) v3 and no longer depends on `apify-shared`. The sections below cover the user-visible consequences; see the client's [Upgrading to v3](https://docs.apify.com/api/client/python/docs/upgrading/upgrading-to-v3) guide for the full list of changes in the client itself. diff --git a/src/apify/_configuration.py b/src/apify/_configuration.py index 9731696b0..f76595931 100644 --- a/src/apify/_configuration.py +++ b/src/apify/_configuration.py @@ -109,7 +109,7 @@ class Configuration(CrawleeConfiguration): """A class for specifying the configuration of an Actor. Can be used either globally via `Configuration.get_global_configuration()`, - or it can be specific to each `Actor` instance on the `actor.config` property. + or it can be specific to each `Actor` instance on the `Actor.configuration` property. """ # Fields are validated from environment variables via their `validation_alias`, but serialized under a diff --git a/src/apify/storage_clients/__init__.py b/src/apify/storage_clients/__init__.py index fef5dde83..a2f649d59 100644 --- a/src/apify/storage_clients/__init__.py +++ b/src/apify/storage_clients/__init__.py @@ -1,10 +1,14 @@ -from crawlee.storage_clients import MemoryStorageClient, StorageClient +# `FileSystemStorageClient`, `MemoryStorageClient`, and `StorageClient` are re-exported from Crawlee. Only the +# file-system client has an Apify variant: `ApifyFileSystemStorageClient`, the default local client, which adds +# `INPUT` preservation on purge and local `PAY_PER_EVENT` charging. +from crawlee.storage_clients import FileSystemStorageClient, MemoryStorageClient, StorageClient from ._apify import ApifyStorageClient -from ._file_system import ApifyFileSystemStorageClient as FileSystemStorageClient +from ._file_system import ApifyFileSystemStorageClient from ._smart_apify import SmartApifyStorageClient __all__ = [ + 'ApifyFileSystemStorageClient', 'ApifyStorageClient', 'FileSystemStorageClient', 'MemoryStorageClient', diff --git a/src/apify/storage_clients/_file_system/__init__.py b/src/apify/storage_clients/_file_system/__init__.py index b18af53b8..354906c86 100644 --- a/src/apify/storage_clients/_file_system/__init__.py +++ b/src/apify/storage_clients/_file_system/__init__.py @@ -1,2 +1,3 @@ +from ._dataset_client import ApifyFileSystemDatasetClient from ._key_value_store_client import ApifyFileSystemKeyValueStoreClient from ._storage_client import ApifyFileSystemStorageClient diff --git a/src/apify/storage_clients/_file_system/_storage_client.py b/src/apify/storage_clients/_file_system/_storage_client.py index be31dc298..2bf458ac6 100644 --- a/src/apify/storage_clients/_file_system/_storage_client.py +++ b/src/apify/storage_clients/_file_system/_storage_client.py @@ -19,11 +19,16 @@ @docs_group('Storage clients') class ApifyFileSystemStorageClient(FileSystemStorageClient): - """Apify-specific implementation of the file system storage client. + """Apify SDK variant of Crawlee's `FileSystemStorageClient`, used as the default local storage client. - The only difference is that it uses `ApifyFileSystemKeyValueStoreClient` for key-value stores, - which overrides the `purge` method to delete all files in the key-value store directory - except for the metadata file and the `INPUT.json` file. + It extends the Crawlee file-system client with Apify-specific behavior that keeps local runs consistent + with the Apify platform: + + - Key-value stores use `ApifyFileSystemKeyValueStoreClient`, which preserves the Actor input file (e.g. + `INPUT.json`) and the metadata file when purging, and maps the logical `INPUT` key to the input file on + disk. + - Datasets use `ApifyFileSystemDatasetClient`, which charges for the `PAY_PER_EVENT` pricing model so it + can be exercised locally. """ @override diff --git a/src/apify/storage_clients/_smart_apify/_storage_client.py b/src/apify/storage_clients/_smart_apify/_storage_client.py index e2c1df4ad..00267e9e0 100644 --- a/src/apify/storage_clients/_smart_apify/_storage_client.py +++ b/src/apify/storage_clients/_smart_apify/_storage_client.py @@ -8,7 +8,7 @@ from apify._configuration import Configuration as ApifyConfiguration from apify._utils import docs_group -from apify.storage_clients import ApifyStorageClient, FileSystemStorageClient +from apify.storage_clients import ApifyFileSystemStorageClient, ApifyStorageClient if TYPE_CHECKING: from collections.abc import Hashable @@ -44,10 +44,10 @@ def __init__( cloud_storage_client: Storage client used when an Actor is running on the Apify platform, or when explicitly enabled via the `force_cloud` argument. Defaults to `ApifyStorageClient`. local_storage_client: Storage client used when an Actor is not running on the Apify platform and when - `force_cloud` flag is not set. Defaults to `FileSystemStorageClient`. + `force_cloud` flag is not set. Defaults to `ApifyFileSystemStorageClient`. """ self._cloud_storage_client = cloud_storage_client or ApifyStorageClient() - self._local_storage_client = local_storage_client or FileSystemStorageClient() + self._local_storage_client = local_storage_client or ApifyFileSystemStorageClient() def __str__(self) -> str: return ( diff --git a/tests/unit/storage_clients/test_smart_apify_storage_client.py b/tests/unit/storage_clients/test_smart_apify_storage_client.py index 874022bc4..6e779be28 100644 --- a/tests/unit/storage_clients/test_smart_apify_storage_client.py +++ b/tests/unit/storage_clients/test_smart_apify_storage_client.py @@ -5,7 +5,7 @@ import pytest from apify._configuration import Configuration -from apify.storage_clients import ApifyStorageClient, FileSystemStorageClient +from apify.storage_clients import ApifyFileSystemStorageClient, ApifyStorageClient from apify.storage_clients._smart_apify._storage_client import SmartApifyStorageClient @@ -44,7 +44,7 @@ def test_local_returns_local_client(monkeypatch: pytest.MonkeyPatch) -> None: """Test that local environment returns local client.""" monkeypatch.delenv('APIFY_IS_AT_HOME', raising=False) - local_client = MagicMock(spec=FileSystemStorageClient) + local_client = MagicMock(spec=ApifyFileSystemStorageClient) client = SmartApifyStorageClient(local_storage_client=local_client) result = client.get_suitable_storage_client() assert result is local_client @@ -54,7 +54,7 @@ def test_default_clients_initialized() -> None: """Test that default cloud and local clients are created when not provided.""" client = SmartApifyStorageClient() assert isinstance(client._cloud_storage_client, ApifyStorageClient) - assert isinstance(client._local_storage_client, FileSystemStorageClient) + assert isinstance(client._local_storage_client, ApifyFileSystemStorageClient) def test_str_representation() -> None: @@ -63,7 +63,7 @@ def test_str_representation() -> None: result = str(client) assert 'SmartApifyStorageClient' in result assert 'ApifyStorageClient' in result - assert 'FileSystemStorageClient' in result + assert 'ApifyFileSystemStorageClient' in result def test_cache_key_at_home(monkeypatch: pytest.MonkeyPatch) -> None: @@ -85,7 +85,7 @@ def test_cache_key_local(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv('APIFY_IS_AT_HOME', raising=False) config = Configuration() - local_client = MagicMock(spec=FileSystemStorageClient) + local_client = MagicMock(spec=ApifyFileSystemStorageClient) local_client.get_storage_client_cache_key.return_value = 'local-key' client = SmartApifyStorageClient(local_storage_client=local_client) key = client.get_storage_client_cache_key(config)