Add pagination support for generic table API

CHANGELOG.md

@@ -86,7 +86,6 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti
 - Semantic models now support dedicated privileges for listing, creating, reading, updating,
   and dropping. Privileges can be granted to catalog roles on individual models or at namespace
   or catalog scope, with separate controls for managing model grants.
-
 - Python CLI: `catalogs update` now supports `--no-sts` and `--no-kms` to toggle STS/KMS availability on an existing S3 catalog. Previously these were only settable at `catalogs create` time.
 - Python CLI: added `gcp` as an external catalog authentication type for Iceberg REST federation, enabling CLI creation of GCP-authenticated catalogs such as BigLake without passing Google credential secrets through command-line flags.
 - Python CLI: added a global `--page-size` option to paginate list calls internally on Iceberg endpoints. Requires the server-side `LIST_PAGINATION_ENABLED` feature flag.
@@ -222,6 +221,7 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti
   backend, since `S3FileIO`, `GCSFileIO`, `ADLSFileIO` and `HadoopFileIO` all implement
   `DelegateFileIO`.
 - Async task retries no longer fail with a `NullPointerException` when the task entity has already been dropped by a previous attempt. Such a retry is now recognized as an already-completed task and exits cleanly, instead of exhausting all retry attempts and logging a `NullPointerException` on each one.
+- Honored pagination for generic table API.
 - JDBC optimized location-overlap queries no longer include the lone `/` prefix term produced by
   scheme stripping (e.g. `s3://bucket/path` → `//bucket/path`). `//` and `///` are retained so
   scheme-root ancestors remain visible to the overlap check.

client/python/apache_polaris/cli/command/__init__.py

@@ -263,6 +263,7 @@ def options_get(key: str, f: Callable[[Any], Any] = lambda x: x) -> Any:
                     Arguments.NAMESPACE, lambda x: x.split(".") if x else None
                 ),
                 generic_table_name=options_get(Arguments.GENERIC_TABLE),
+                page_size=options_get(Arguments.PAGE_SIZE),
             )
         elif options.command == Commands.FIND:
             from apache_polaris.cli.command.find import FindCommand

client/python/apache_polaris/cli/command/generic_tables.py

@@ -20,7 +20,7 @@
 from typing import List, Optional, cast
 
 from apache_polaris.cli.command import Command
-from apache_polaris.cli.command.utils import get_catalog_api_client
+from apache_polaris.cli.command.utils import get_catalog_api_client, paginate
 from apache_polaris.cli.exceptions import CliError
 from apache_polaris.cli.constants import Subcommands, Arguments, UNIT_SEPARATOR
 from apache_polaris.cli.options.option_tree import Argument
@@ -43,6 +43,7 @@ class GenericTableCommand(Command):
     catalog_name: Optional[str] = None
     namespace: Optional[List[str]] = field(default_factory=list)
     generic_table_name: Optional[str] = None
+    page_size: Optional[int] = None
 
     def validate(self) -> None:
         if not self.catalog_name:
@@ -68,11 +69,14 @@ def execute(self, api: PolarisDefaultApi) -> None:
         ns_str = UNIT_SEPARATOR.join(namespace_list)
 
         if self.generic_tables_subcommand == Subcommands.LIST:
-            result = generic_api.list_generic_tables(
-                prefix=catalog_name, namespace=ns_str
-            )
-            for table_identifier in result.identifiers:
-                print(table_identifier.to_json())
+            for resp in paginate(
+                generic_api.list_generic_tables,
+                page_size=self.page_size,
+                prefix=catalog_name,
+                namespace=ns_str,
+            ):
+                for table_identifier in resp.identifiers or []:
+                    print(table_identifier.to_json())
         elif self.generic_tables_subcommand == Subcommands.GET:
             print(
                 generic_api.load_generic_table(

client/python/tests/test_generic_tables_command.py

@@ -70,6 +70,37 @@ def test_generic_table_list(self, mock_generic_api_class: MagicMock) -> None:
             prefix="my-catalog", namespace=UNIT_SEPARATOR.join(["ns1", "ns2"])
         )
 
+    @patch("apache_polaris.cli.command.generic_tables.GenericTableAPI")
+    def test_generic_table_list_with_paginate(
+        self, mock_generic_api_class: MagicMock
+    ) -> None:
+        mock_client = self.build_mock_client()
+        mock_generic_api = mock_generic_api_class.return_value
+        ids = [MagicMock(to_json=MagicMock(return_value="{}")) for _ in range(3)]
+        page1 = MagicMock(identifiers=ids[:2], next_page_token="token")
+        page2 = MagicMock(identifiers=[ids[2]], next_page_token=None)
+        mock_generic_api.list_generic_tables.side_effect = [page1, page2]
+        self.mock_execute(
+            mock_client,
+            [
+                "generic-tables",
+                "list",
+                "--catalog",
+                "my-catalog",
+                "--namespace",
+                "ns1",
+                "--page-size",
+                "2",
+            ],
+        )
+        self.assertEqual(mock_generic_api.list_generic_tables.call_count, 2)
+        mock_generic_api.list_generic_tables.assert_any_call(
+            prefix="my-catalog", namespace="ns1", page_size=2, page_token=""
+        )
+        mock_generic_api.list_generic_tables.assert_any_call(
+            prefix="my-catalog", namespace="ns1", page_size=2, page_token="token"
+        )
+
     @patch("apache_polaris.cli.command.generic_tables.GenericTableAPI")
     def test_generic_table_get(self, mock_generic_api_class: MagicMock) -> None:
         mock_client = self.build_mock_client()

integration-tests/src/main/java/org/apache/polaris/service/it/env/GenericTableApi.java

@@ -25,6 +25,8 @@
 import jakarta.ws.rs.client.Entity;
 import jakarta.ws.rs.core.Response;
 import java.net.URI;
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import org.apache.iceberg.catalog.Namespace;
@@ -51,11 +53,40 @@ public void purge(String catalog, Namespace ns) {
 
   public List<TableIdentifier> listGenericTables(String catalog, Namespace namespace) {
     String ns = NamespaceUtils.joinNamespace(namespace, NamespaceUtils.DEFAULT_NAMESPACE_SEPARATOR);
+    Map<String, String> templateValues = Map.of("cat", catalog, "ns", ns);
+    List<TableIdentifier> identifiers = new ArrayList<>();
+    Map<String, String> queryParams = new HashMap<>();
+    String nextPageToken = null;
+    do {
+      if (nextPageToken != null) {
+        queryParams.put("pageToken", nextPageToken);
+      }
+      try (Response res =
+          request("polaris/v1/{cat}/namespaces/{ns}/generic-tables", templateValues, queryParams)
+              .get()) {
+        assertThat(res.getStatus()).isEqualTo(Response.Status.OK.getStatusCode());
+        ListGenericTablesResponse response = res.readEntity(ListGenericTablesResponse.class);
+        identifiers.addAll(response.getIdentifiers());
+        nextPageToken = response.getNextPageToken();
+      }
+    } while (nextPageToken != null);
+    return identifiers;
+  }
+
+  public ListGenericTablesResponse listGenericTables(
+      String catalog, Namespace namespace, String pageToken, String pageSize) {
+    String ns = NamespaceUtils.joinNamespace(namespace, NamespaceUtils.DEFAULT_NAMESPACE_SEPARATOR);
+    Map<String, String> queryParams = new HashMap<>();
+    queryParams.put("pageToken", pageToken);
+    queryParams.put("pageSize", pageSize);
     try (Response res =
-        request("polaris/v1/{cat}/namespaces/{ns}/generic-tables", Map.of("cat", catalog, "ns", ns))
+        request(
+                "polaris/v1/{cat}/namespaces/{ns}/generic-tables",
+                Map.of("cat", catalog, "ns", ns),
+                queryParams)
             .get()) {
       assertThat(res.getStatus()).isEqualTo(Response.Status.OK.getStatusCode());
-      return res.readEntity(ListGenericTablesResponse.class).getIdentifiers().stream().toList();
+      return res.readEntity(ListGenericTablesResponse.class);
     }
   }
 

integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationBase.java

@@ -116,6 +116,7 @@
 import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension;
 import org.apache.polaris.service.types.CreateGenericTableRequest;
 import org.apache.polaris.service.types.GenericTable;
+import org.apache.polaris.service.types.ListGenericTablesResponse;
 import org.assertj.core.api.Assertions;
 import org.assertj.core.api.Assumptions;
 import org.assertj.core.api.InstanceOfAssertFactories;
@@ -2961,6 +2962,36 @@ public void testNonPaginatedListTablesViewNamespaces() {
     assertThat(nsResponse.nextPageToken()).isNull();
   }
 
+  @Test
+  public void testPaginatedListGenericTables() {
+    String prefix = "testPaginatedListGenericTables";
+    Namespace namespace = Namespace.of(prefix);
+    restCatalog.createNamespace(namespace);
+    for (int i = 0; i < 30; i++) {
+      genericTableApi.createGenericTable(
+          currentCatalogName, TableIdentifier.of(namespace, prefix + i), "format", Map.of());
+    }
+
+    try {
+      assertThat(genericTableApi.listGenericTables(currentCatalogName, namespace)).hasSize(30);
+      for (var pageSize : List.of(1, 2, 3, 9, 10, 11, 19, 20, 21, 25, 2000)) {
+        int total = 0;
+        String pageToken = null;
+        do {
+          ListGenericTablesResponse response =
+              genericTableApi.listGenericTables(
+                  currentCatalogName, namespace, pageToken, String.valueOf(pageSize));
+          assertThat(response.getIdentifiers().size()).isLessThanOrEqualTo(pageSize);
+          total += response.getIdentifiers().size();
+          pageToken = response.getNextPageToken();
+        } while (pageToken != null);
+        assertThat(total).as("Total paginated results for pageSize = " + pageSize).isEqualTo(30);
+      }
+    } finally {
+      genericTableApi.purge(currentCatalogName, namespace);
+    }
+  }
+
   @ParameterizedTest
   @MethodSource("invalidEntityNames")
   public void testCreateNamespaceRejectsInvalidName(String badName) {

polaris-core/src/main/java/org/apache/polaris/core/catalog/GenericTableCatalog.java

@@ -18,11 +18,12 @@
  */
 package org.apache.polaris.core.catalog;
 
-import java.util.List;
 import java.util.Map;
 import org.apache.iceberg.catalog.Namespace;
 import org.apache.iceberg.catalog.TableIdentifier;
 import org.apache.polaris.core.entity.table.GenericTableEntity;
+import org.apache.polaris.core.persistence.pagination.Page;
+import org.apache.polaris.core.persistence.pagination.PageToken;
 
 /** A catalog for managing `GenericTableEntity` instances */
 public interface GenericTableCatalog {
@@ -44,6 +45,6 @@ GenericTableEntity createGenericTable(
   /** Drop a generic table entity with a given identifier */
   boolean dropGenericTable(TableIdentifier tableIdentifier);
 
-  /** List all generic tables under a specific namespace */
-  List<TableIdentifier> listGenericTables(Namespace namespace);
+  /** List generic tables under a specific namespace, paginated according to {@code pageToken} */
+  Page<TableIdentifier> listGenericTables(Namespace namespace, PageToken pageToken);
 }

runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogAdapter.java

@@ -118,7 +118,9 @@ public Response listGenericTables(
     GenericTableCatalogHandler handler = newHandler(securityContext, prefix);
     ListGenericTablesResponse response =
         handler.listGenericTables(
-            NamespaceUtils.splitNamespace(namespace, NamespaceUtils.DEFAULT_NAMESPACE_SEPARATOR));
+            NamespaceUtils.splitNamespace(namespace, NamespaceUtils.DEFAULT_NAMESPACE_SEPARATOR),
+            pageToken,
+            pageSize);
     return Response.ok(response).build();
   }
 

runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogHandler.java

@@ -35,6 +35,8 @@
 import org.apache.polaris.core.entity.CatalogEntity;
 import org.apache.polaris.core.entity.PolarisEntitySubType;
 import org.apache.polaris.core.entity.table.GenericTableEntity;
+import org.apache.polaris.core.persistence.pagination.Page;
+import org.apache.polaris.core.persistence.pagination.PageToken;
 import org.apache.polaris.immutables.PolarisImmutable;
 import org.apache.polaris.service.catalog.common.CatalogHandler;
 import org.apache.polaris.service.types.GenericTable;
@@ -97,12 +99,17 @@ protected void initializeCatalog() {
     }
   }
 
-  public ListGenericTablesResponse listGenericTables(Namespace parent) {
+  public ListGenericTablesResponse listGenericTables(
+      Namespace parent, String pageToken, Integer pageSize) {
     PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LIST_TABLES;
     authorizeBasicNamespaceOperationOrThrow(op, parent);
 
+    PageToken pageRequest =
+        PageToken.build(pageToken, pageSize, maxPageSize(), this::shouldDecodeToken);
+    Page<TableIdentifier> page = genericTableCatalog.listGenericTables(parent, pageRequest);
     return ListGenericTablesResponse.builder()
-        .setIdentifiers(new LinkedHashSet<>(genericTableCatalog.listGenericTables(parent)))
+        .setIdentifiers(new LinkedHashSet<>(page.items()))
+        .setNextPageToken(page.encodedResponseToken())
         .build();
   }
 
@@ -155,4 +162,19 @@ public LoadGenericTableResponse loadGenericTable(TableIdentifier identifier) {
 
     return LoadGenericTableResponse.builder().setTable(loadedTable).build();
   }
+
+  private boolean shouldDecodeToken() {
+    CatalogEntity catalogEntity = resolutionManifest.getResolvedCatalogEntity();
+    return catalogEntity == null
+        ? realmConfig().getConfig(FeatureConfiguration.LIST_PAGINATION_ENABLED)
+        : realmConfig().getConfig(FeatureConfiguration.LIST_PAGINATION_ENABLED, catalogEntity);
+  }
+
+  private int maxPageSize() {
+    CatalogEntity catalogEntity = resolutionManifest.getResolvedCatalogEntity();
+    return catalogEntity == null
+        ? realmConfig().getConfig(FeatureConfiguration.LIST_PAGINATION_MAX_PAGE_SIZE)
+        : realmConfig()
+            .getConfig(FeatureConfiguration.LIST_PAGINATION_MAX_PAGE_SIZE, catalogEntity);
+  }
 }

runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalog.java

@@ -41,6 +41,7 @@
 import org.apache.polaris.core.persistence.dao.entity.BaseResult;
 import org.apache.polaris.core.persistence.dao.entity.DropEntityResult;
 import org.apache.polaris.core.persistence.dao.entity.EntityResult;
+import org.apache.polaris.core.persistence.pagination.Page;
 import org.apache.polaris.core.persistence.pagination.PageToken;
 import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView;
 import org.apache.polaris.core.persistence.resolver.ResolvedPathKey;
@@ -185,24 +186,23 @@ public boolean dropGenericTable(TableIdentifier tableIdentifier) {
   }
 
   @Override
-  public List<TableIdentifier> listGenericTables(Namespace namespace) {
+  public Page<TableIdentifier> listGenericTables(Namespace namespace, PageToken pageToken) {
     PolarisResolvedPathWrapper resolvedEntities =
         resolvedEntityView.getResolvedPath(ResolvedPathKey.ofNamespace(namespace));
     if (resolvedEntities == null) {
       throw noSuchNamespaceException(namespace);
     }
 
     List<PolarisEntity> catalogPath = resolvedEntities.getRawFullPath();
-    List<PolarisEntity.NameAndId> entities =
-        PolarisEntity.toNameAndIdList(
-            this.metaStoreManager
-                .listEntities(
-                    this.callContext.getPolarisCallContext(),
-                    PolarisEntity.toCoreList(catalogPath),
-                    PolarisEntityType.TABLE_LIKE,
-                    PolarisEntitySubType.GENERIC_TABLE,
-                    PageToken.readEverything())
-                .getEntities());
-    return PolarisCatalogHelpers.nameAndIdToTableIdentifiers(catalogPath, entities);
+    Namespace parentNamespace = PolarisCatalogHelpers.parentNamespace(catalogPath);
+    return this.metaStoreManager
+        .listEntities(
+            this.callContext.getPolarisCallContext(),
+            PolarisEntity.toCoreList(catalogPath),
+            PolarisEntityType.TABLE_LIKE,
+            PolarisEntitySubType.GENERIC_TABLE,
+            pageToken)
+        .getPage()
+        .map(record -> TableIdentifier.of(parentNamespace, record.getName()));
   }
 }