From 939e392a7a8b64a3f8632253ebf2bcc71a3b5157 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Tue, 11 Aug 2026 17:45:57 +0800 Subject: [PATCH 1/4] Add DuckDB platform support --- .../duckdb-setup/.gitignore | 16 + platforms-setup-guides/duckdb-setup/README.md | 204 +++++ platforms-setup-guides/duckdb-setup/demo.sh | 116 +++ .../duckdb-setup/docker-compose.yml | 29 + platforms-setup-guides/duckdb-setup/pom.xml | 81 ++ .../profiling/ga-relaxed.properties | 28 + .../duckdb-setup/scripts/check.sql | 24 + .../duckdb-setup/scripts/init.sql | 46 ++ .../duckdb-setup/scripts/run-duckdb-ga.ps1 | 121 +++ .../wayang/duckdb/DuckDBIntegrationTest.java | 244 ++++++ wayang-platforms/pom.xml | 1 + wayang-platforms/wayang-duckdb/README.md | 199 +++++ wayang-platforms/wayang-duckdb/pom.xml | 91 ++ .../java/org/apache/wayang/duckdb/DuckDB.java | 62 ++ .../org/apache/wayang/duckdb/DuckDBDemo.java | 160 ++++ .../duckdb/channels/ChannelConversions.java | 55 ++ .../wayang/duckdb/mapping/FilterMapping.java | 63 ++ .../duckdb/mapping/GlobalReduceMapping.java | 63 ++ .../wayang/duckdb/mapping/JoinMapping.java | 75 ++ .../wayang/duckdb/mapping/Mappings.java | 42 + .../duckdb/mapping/ParquetSourceMapping.java | 59 ++ .../duckdb/mapping/ProjectionMapping.java | 67 ++ .../duckdb/mapping/ReduceByMapping.java | 66 ++ .../wayang/duckdb/mapping/SortMapping.java | 65 ++ .../duckdb/mapping/TableSinkMapping.java | 60 ++ .../operators/DuckDBExecutionOperator.java | 34 + .../operators/DuckDBFilterOperator.java | 49 ++ .../operators/DuckDBGlobalReduceOperator.java | 50 ++ .../duckdb/operators/DuckDBJoinOperator.java | 49 ++ .../duckdb/operators/DuckDBParquetSource.java | 36 + .../operators/DuckDBProjectionOperator.java | 49 ++ .../operators/DuckDBReduceByOperator.java | 52 ++ .../duckdb/operators/DuckDBSortOperator.java | 49 ++ .../operators/DuckDBTableSinkOperator.java | 48 ++ .../duckdb/operators/DuckDBTableSource.java | 55 ++ .../duckdb/platform/DuckDBPlatform.java | 54 ++ .../plugin/DuckDBConversionsPlugin.java | 59 ++ .../wayang/duckdb/plugin/DuckDBPlugin.java | 58 ++ .../wayang-duckdb-defaults.properties | 228 +++++ .../wayang/duckdb/DuckDBCostPilotIT.java | 777 ++++++++++++++++++ .../wayang/duckdb/DuckDBOperatorsIT.java | 636 ++++++++++++++ .../wayang/duckdb/DuckDBParquetSourceIT.java | 197 +++++ .../test/resources/duckdb-ga-smoke.properties | 28 + .../wayang/jdbc/execution/JdbcExecutor.java | 216 +++-- .../jdbc/operators/JdbcExecutionOperator.java | 8 + .../jdbc/operators/JdbcJoinOperator.java | 10 +- .../jdbc/operators/JdbcParquetSource.java | 410 +++++++++ .../jdbc/operators/JdbcSourceOperator.java | 66 ++ .../jdbc/operators/JdbcTableSource.java | 6 +- .../jdbc/execution/JdbcExecutorTest.java | 8 +- .../execution/JdbcTableSinkExecutorTest.java | 41 +- 51 files changed, 5213 insertions(+), 97 deletions(-) create mode 100644 platforms-setup-guides/duckdb-setup/.gitignore create mode 100644 platforms-setup-guides/duckdb-setup/README.md create mode 100644 platforms-setup-guides/duckdb-setup/demo.sh create mode 100644 platforms-setup-guides/duckdb-setup/docker-compose.yml create mode 100644 platforms-setup-guides/duckdb-setup/pom.xml create mode 100644 platforms-setup-guides/duckdb-setup/profiling/ga-relaxed.properties create mode 100644 platforms-setup-guides/duckdb-setup/scripts/check.sql create mode 100644 platforms-setup-guides/duckdb-setup/scripts/init.sql create mode 100644 platforms-setup-guides/duckdb-setup/scripts/run-duckdb-ga.ps1 create mode 100644 platforms-setup-guides/duckdb-setup/src/test/java/org/apache/wayang/duckdb/DuckDBIntegrationTest.java create mode 100644 wayang-platforms/wayang-duckdb/README.md create mode 100644 wayang-platforms/wayang-duckdb/pom.xml create mode 100644 wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/DuckDB.java create mode 100644 wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/DuckDBDemo.java create mode 100644 wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/channels/ChannelConversions.java create mode 100644 wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/FilterMapping.java create mode 100644 wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/GlobalReduceMapping.java create mode 100644 wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/JoinMapping.java create mode 100644 wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/Mappings.java create mode 100644 wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/ParquetSourceMapping.java create mode 100644 wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/ProjectionMapping.java create mode 100644 wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/ReduceByMapping.java create mode 100644 wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/SortMapping.java create mode 100644 wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/TableSinkMapping.java create mode 100644 wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBExecutionOperator.java create mode 100644 wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBFilterOperator.java create mode 100644 wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBGlobalReduceOperator.java create mode 100644 wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBJoinOperator.java create mode 100644 wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBParquetSource.java create mode 100644 wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBProjectionOperator.java create mode 100644 wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBReduceByOperator.java create mode 100644 wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBSortOperator.java create mode 100644 wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBTableSinkOperator.java create mode 100644 wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBTableSource.java create mode 100644 wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/platform/DuckDBPlatform.java create mode 100644 wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/plugin/DuckDBConversionsPlugin.java create mode 100644 wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/plugin/DuckDBPlugin.java create mode 100644 wayang-platforms/wayang-duckdb/src/main/resources/wayang-duckdb-defaults.properties create mode 100644 wayang-platforms/wayang-duckdb/src/test/java/org/apache/wayang/duckdb/DuckDBCostPilotIT.java create mode 100644 wayang-platforms/wayang-duckdb/src/test/java/org/apache/wayang/duckdb/DuckDBOperatorsIT.java create mode 100644 wayang-platforms/wayang-duckdb/src/test/java/org/apache/wayang/duckdb/DuckDBParquetSourceIT.java create mode 100644 wayang-platforms/wayang-duckdb/src/test/resources/duckdb-ga-smoke.properties create mode 100644 wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcParquetSource.java create mode 100644 wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcSourceOperator.java diff --git a/platforms-setup-guides/duckdb-setup/.gitignore b/platforms-setup-guides/duckdb-setup/.gitignore new file mode 100644 index 000000000..b4185cd23 --- /dev/null +++ b/platforms-setup-guides/duckdb-setup/.gitignore @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +data/ diff --git a/platforms-setup-guides/duckdb-setup/README.md b/platforms-setup-guides/duckdb-setup/README.md new file mode 100644 index 000000000..ef1819112 --- /dev/null +++ b/platforms-setup-guides/duckdb-setup/README.md @@ -0,0 +1,204 @@ +# DuckDB Local Setup + +Local DuckDB setup for the Wayang DuckDB platform. + +DuckDB is embedded: there is no coordinator or long-running database service to +start. The Wayang platform connects directly to a DuckDB database file through +the DuckDB JDBC driver. For a Trino-like reproducible local workflow, this setup +uses the official DuckDB CLI Docker image to create and inspect a local +`data/wayang.duckdb` file, then runs the Wayang operator tests against that +same file. + +Run the commands below from the repository root. Java 17 and Docker with Docker +Compose are required; Maven is provided by the repository wrapper. + +## Stack + +| Component | Image | Role | +|-----------|-------|------| +| DuckDB CLI | `duckdb/duckdb:1.5.5` | Creates and inspects the local database file | +| DuckDB JDBC | `org.duckdb:duckdb_jdbc:1.5.5.1` | Runs Wayang plans against that file | + +The Docker service is a one-shot CLI container. It exits after running the SQL +command; that is expected. + +## 1. Create The Local DuckDB File + +```bash +mkdir -p platforms-setup-guides/duckdb-setup/data +docker compose -f platforms-setup-guides/duckdb-setup/docker-compose.yml run --rm duckdb +``` + +On PowerShell: + +```powershell +New-Item -ItemType Directory -Force platforms-setup-guides/duckdb-setup/data +docker compose -f platforms-setup-guides/duckdb-setup/docker-compose.yml run --rm duckdb +``` + +Expected output includes grouped totals for `APAC`, `AMER`, and `EMEA`. + +## 2. Inspect The File Directly + +```bash +docker run --rm -i \ + -v "$PWD/platforms-setup-guides/duckdb-setup:/workspace" \ + duckdb/duckdb:1.5.5 \ + duckdb /workspace/data/wayang.duckdb < platforms-setup-guides/duckdb-setup/scripts/check.sql +``` + +On PowerShell: + +```powershell +Get-Content -Raw platforms-setup-guides/duckdb-setup/scripts/check.sql | + docker run --rm -i -v "${PWD}/platforms-setup-guides/duckdb-setup:/workspace" duckdb/duckdb:1.5.5 duckdb /workspace/data/wayang.duckdb +``` + +## 3. Run Wayang Tests Against The Docker-Created File + +```bash +./mvnw -Pskip-prerequisite-check -pl wayang-platforms/wayang-duckdb -am \ + -Dtest=DuckDBOperatorsIT \ + -Dduckdb.url=jdbc:duckdb:platforms-setup-guides/duckdb-setup/data/wayang.duckdb \ + -Dsurefire.failIfNoSpecifiedTests=false -DfailIfNoTests=false \ + -Drat.skip=true -Dlicense.skip=true test +``` + +On PowerShell: + +```powershell +.\mvnw.cmd --% -Pskip-prerequisite-check -pl wayang-platforms/wayang-duckdb -am -Dtest=DuckDBOperatorsIT -Dduckdb.url=jdbc:duckdb:platforms-setup-guides/duckdb-setup/data/wayang.duckdb -Dsurefire.failIfNoSpecifiedTests=false -DfailIfNoTests=false -Drat.skip=true -Dlicense.skip=true test +``` + +Expected result: + +```text +Tests run: 15, Failures: 0, Errors: 0, Skipped: 0 +BUILD SUCCESS +``` + +`DuckDBOperatorsIT` recreates the `wayang_it` fixtures before it runs, so the +test is deterministic even if the local database file already exists. + +## 4. Run Parquet And GCS Tests + +`DuckDBParquetSourceIT` creates a local Parquet file, reads it through DuckDB +auto-created `read_parquet(...)` views, checks URI-to-relation mappings, and +tries a public GCS Parquet smoke through DuckDB `httpfs`. + +```bash +./mvnw -Pskip-prerequisite-check -pl wayang-platforms/wayang-duckdb -am \ + -Dtest=DuckDBParquetSourceIT \ + -Dsurefire.failIfNoSpecifiedTests=false -DfailIfNoTests=false \ + -Drat.skip=true -Dlicense.skip=true test +``` + +On PowerShell: + +```powershell +.\mvnw.cmd --% -Pskip-prerequisite-check -pl wayang-platforms/wayang-duckdb -am -Dtest=DuckDBParquetSourceIT -Dsurefire.failIfNoSpecifiedTests=false -DfailIfNoTests=false -Drat.skip=true -Dlicense.skip=true test +``` + +Override the GCS object with `-Dduckdb.gcs.parquet.uri=gs://bucket/path/file.parquet`. +If DuckDB cannot install/load `httpfs` or reach the object, the GCS smoke is +skipped; the local Parquet tests still run. + +## 5. Run Cost Profiling Smoke + +For a fast local check, run two profiling plans over two small cardinalities: + +```bash +./mvnw -Pskip-prerequisite-check -pl wayang-platforms/wayang-duckdb -am \ + -Dtest=DuckDBCostPilotIT#runPilot \ + -Dduckdb.profile.rowCounts=100,1000 \ + -Dduckdb.profile.plans=S01,S02 \ + -Dduckdb.profile.repetitions=2 \ + -Drat.skip=true -Dlicense.skip=true test +``` + +The Trino Week8-style reference pilot is S01-S13 over four cardinalities and +six repetitions, producing 312 Wayang executions: + +```bash +./mvnw -Pskip-prerequisite-check -pl wayang-platforms/wayang-duckdb -am \ + -Dtest=DuckDBCostPilotIT#runPilot \ + -Dduckdb.profile.rowCounts=10000,50000,100000,250000 \ + -Dduckdb.profile.plans=S01,S02,S03,S04,S05,S06,S07,S08,S09,S10,S11,S12,S13 \ + -Dduckdb.profile.repetitions=6 \ + -Drat.skip=true -Dlicense.skip=true test +``` + +Outputs are written under `wayang-platforms/wayang-duckdb/target/cost-profiling/`. +Run the GA optimizer outside Surefire, matching the Trino profiling workflow: + +```powershell +.\platforms-setup-guides\duckdb-setup\scripts\run-duckdb-ga.ps1 +``` + +The script writes +`wayang-platforms/wayang-duckdb/target/cost-profiling/duckdb/learned-duckdb-relaxed.properties`. +S14-S16 are implemented for optional expanded runs, but the checked-in reference +parameters are learned from S01-S13. The GA optimizer is stochastic, so repeated +runs over the same execution log can produce slightly different coefficients. + +## 6. Run The Standalone Setup Integration Tests + +The setup directory includes a small Maven project that validates the local +DuckDB database independently of Wayang. Tests are skipped by default; enable +them with `-Pintegration`. + +```bash +./mvnw -f platforms-setup-guides/duckdb-setup/pom.xml \ + -Pintegration -Dtest=DuckDBIntegrationTest test +``` + +On PowerShell: + +```powershell +.\mvnw.cmd --% -f platforms-setup-guides/duckdb-setup/pom.xml -Pintegration -Dtest=DuckDBIntegrationTest test +``` + +Expected result: + +```text +Tests run: 10, Failures: 0, Errors: 0, Skipped: 0 +BUILD SUCCESS +``` + +Override the database file: + +```bash +DUCKDB_JDBC_URL=jdbc:duckdb:/tmp/wayang.duckdb ./mvnw -f platforms-setup-guides/duckdb-setup/pom.xml -Pintegration -Dtest=DuckDBIntegrationTest test +``` + +On PowerShell: + +```powershell +$env:DUCKDB_JDBC_URL="jdbc:duckdb:C:/tmp/wayang.duckdb" +.\mvnw.cmd --% -f platforms-setup-guides/duckdb-setup/pom.xml -Pintegration -Dtest=DuckDBIntegrationTest test +Remove-Item Env:DUCKDB_JDBC_URL +``` + +## 7. Run The Walkthrough Demo + +The optional demo script creates the local DuckDB file, runs the Wayang DuckDB +operator tests against it, runs the standalone JDBC integration tests, and +executes `org.apache.wayang.duckdb.DuckDBDemo`. + +```bash +bash platforms-setup-guides/duckdb-setup/demo.sh +``` + +Set `WAYANG_DEMO_AUTO=true` to skip the interactive pauses. + +## 8. Clean Up + +```bash +rm -f platforms-setup-guides/duckdb-setup/data/wayang.duckdb* +``` + +On PowerShell: + +```powershell +Remove-Item platforms-setup-guides/duckdb-setup/data/wayang.duckdb* -Force +``` diff --git a/platforms-setup-guides/duckdb-setup/demo.sh b/platforms-setup-guides/duckdb-setup/demo.sh new file mode 100644 index 000000000..2228f5cc0 --- /dev/null +++ b/platforms-setup-guides/duckdb-setup/demo.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WAYANG_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +DB_FILE="$SCRIPT_DIR/data/wayang.duckdb" +MAVEN_FLAGS="-Pskip-prerequisite-check -Drat.skip=true -Dlicense.skip=true" + +banner() { + echo + echo "============================================================" + printf " %s\n" "$*" + echo "============================================================" + echo +} + +step() { + echo + echo "-- $*" + echo +} + +pause() { + if [[ "${WAYANG_DEMO_AUTO:-false}" != "true" ]]; then + echo + read -rp "Press ENTER to continue..." _ || true + echo + fi +} + +banner "ACT 1: Create a local DuckDB file with Docker" + +step "1a. Running the DuckDB CLI container" +mkdir -p "$SCRIPT_DIR/data" +docker compose -f "$SCRIPT_DIR/docker-compose.yml" run --rm duckdb + +step "1b. Inspecting the local database file" +docker run --rm -i \ + -v "$SCRIPT_DIR:/workspace" \ + duckdb/duckdb:1.5.5 \ + duckdb /workspace/data/wayang.duckdb < "$SCRIPT_DIR/scripts/check.sql" + +pause + +banner "ACT 2: Run Wayang DuckDB tests against that file" + +cd "$WAYANG_ROOT" +./mvnw ${MAVEN_FLAGS} -pl wayang-platforms/wayang-duckdb -am \ + -Dtest=DuckDBOperatorsIT \ + -Dduckdb.url="jdbc:duckdb:$DB_FILE" \ + -Dsurefire.failIfNoSpecifiedTests=false \ + -DfailIfNoTests=false \ + test + +pause + +banner "ACT 3: Run DuckDB Parquet and GCS tests" + +./mvnw ${MAVEN_FLAGS} -pl wayang-platforms/wayang-duckdb -am \ + -Dtest=DuckDBParquetSourceIT \ + -Dsurefire.failIfNoSpecifiedTests=false \ + -DfailIfNoTests=false \ + test + +pause + +banner "ACT 4: Run a DuckDB cost-profiling smoke" + +./mvnw ${MAVEN_FLAGS} -pl wayang-platforms/wayang-duckdb -am \ + -Dtest=DuckDBCostPilotIT#runPilot \ + -Dduckdb.profile.rowCounts=100,1000 \ + -Dduckdb.profile.plans=S01,S02 \ + -Dduckdb.profile.repetitions=2 \ + -Dsurefire.failIfNoSpecifiedTests=false \ + -DfailIfNoTests=false \ + test + +pause + +banner "ACT 5: Run the standalone DuckDB setup integration tests" + +./mvnw -f platforms-setup-guides/duckdb-setup/pom.xml \ + -Pintegration \ + -Dtest=DuckDBIntegrationTest \ + -Dduckdb.url="jdbc:duckdb:$DB_FILE" \ + test + +pause + +banner "ACT 6: Run the Wayang DuckDB demo" + +./mvnw ${MAVEN_FLAGS} -pl wayang-platforms/wayang-duckdb \ + -DskipTests \ + exec:java \ + -Dexec.mainClass=org.apache.wayang.duckdb.DuckDBDemo \ + -Dduckdb.url="jdbc:duckdb:$DB_FILE" + +banner "Demo complete" +echo "DuckDB file: $DB_FILE" diff --git a/platforms-setup-guides/duckdb-setup/docker-compose.yml b/platforms-setup-guides/duckdb-setup/docker-compose.yml new file mode 100644 index 000000000..9b169fdd3 --- /dev/null +++ b/platforms-setup-guides/duckdb-setup/docker-compose.yml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +services: + duckdb: + image: duckdb/duckdb:1.5.5 + working_dir: /workspace + volumes: + - ./data:/workspace/data + - ./scripts:/workspace/scripts:ro + command: + - duckdb + - /workspace/data/wayang.duckdb + - -init + - /workspace/scripts/init.sql + - -c + - SELECT region, SUM(amount) AS total_amount FROM wayang_it.orders GROUP BY region ORDER BY region; diff --git a/platforms-setup-guides/duckdb-setup/pom.xml b/platforms-setup-guides/duckdb-setup/pom.xml new file mode 100644 index 000000000..08ef0eb85 --- /dev/null +++ b/platforms-setup-guides/duckdb-setup/pom.xml @@ -0,0 +1,81 @@ + + + + 4.0.0 + + org.apache.wayang + duckdb-setup + 1.0-SNAPSHOT + jar + + DuckDB Local Setup - Integration Tests + + Standalone integration tests for a local DuckDB database file. + Independent of the Wayang codebase. + + + + 17 + 17 + UTF-8 + 1.5.5.1 + 5.10.2 + true + + + + + org.duckdb + duckdb_jdbc + ${duckdb.version} + test + + + org.junit.jupiter + junit-jupiter + ${junit.version} + test + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.2.5 + + ${skipIntegrationTests} + + + + + + + + integration + + false + + + + diff --git a/platforms-setup-guides/duckdb-setup/profiling/ga-relaxed.properties b/platforms-setup-guides/duckdb-setup/profiling/ga-relaxed.properties new file mode 100644 index 000000000..16001ffb4 --- /dev/null +++ b/platforms-setup-guides/duckdb-setup/profiling/ga-relaxed.properties @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +wayang.profiler.ga.timelimit.ms = 120000 +wayang.profiler.ga.maxgenerations = 800 +wayang.profiler.ga.maxstablegenerations = 150 +wayang.profiler.ga.superoptimizations = 1 +wayang.profiler.ga.intermediateupdate = 200 +wayang.profiler.ga.min-exec-time = 1 +wayang.profiler.ga.max-cardinality-spread = 100 +wayang.profiler.ga.min-cardinality-confidence = 0 +wayang.profiler.ga.binning = 1.0 +wayang.profiler.ga.output-file = wayang-platforms/wayang-duckdb/target/cost-profiling/duckdb/learned-duckdb-relaxed.properties +wayang.profiler.ga.noise-filter.max = 0 diff --git a/platforms-setup-guides/duckdb-setup/scripts/check.sql b/platforms-setup-guides/duckdb-setup/scripts/check.sql new file mode 100644 index 000000000..f225f44ec --- /dev/null +++ b/platforms-setup-guides/duckdb-setup/scripts/check.sql @@ -0,0 +1,24 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +SELECT count(*) AS order_count FROM wayang_it.orders; +SELECT region, SUM(amount) AS total_amount +FROM wayang_it.orders +GROUP BY region +ORDER BY region; +SELECT count(*) AS joined_rows +FROM wayang_it.orders +JOIN wayang_it.customers + ON customers.cust_id = orders.customer_id; diff --git a/platforms-setup-guides/duckdb-setup/scripts/init.sql b/platforms-setup-guides/duckdb-setup/scripts/init.sql new file mode 100644 index 000000000..8a71d376f --- /dev/null +++ b/platforms-setup-guides/duckdb-setup/scripts/init.sql @@ -0,0 +1,46 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +CREATE SCHEMA IF NOT EXISTS wayang_it; + +DROP TABLE IF EXISTS wayang_it.operator_result; +DROP TABLE IF EXISTS wayang_it.orders; +DROP TABLE IF EXISTS wayang_it.customers; + +CREATE TABLE wayang_it.orders ( + order_id BIGINT, + customer_id BIGINT, + region VARCHAR, + amount DOUBLE +); + +INSERT INTO wayang_it.orders VALUES + (1, 100, 'AMER', 2200.0), + (2, 101, 'EMEA', 800.5), + (3, 100, 'AMER', 680.5), + (4, 102, 'APAC', 1500.0), + (5, 101, 'EMEA', 1100.0), + (6, 100, 'AMER', 950.25); + +CREATE TABLE wayang_it.customers ( + cust_id BIGINT, + name VARCHAR, + tier VARCHAR +); + +INSERT INTO wayang_it.customers VALUES + (100, 'Acme', 'GOLD'), + (101, 'Globex', 'SILVER'), + (102, 'Initech','BRONZE'); diff --git a/platforms-setup-guides/duckdb-setup/scripts/run-duckdb-ga.ps1 b/platforms-setup-guides/duckdb-setup/scripts/run-duckdb-ga.ps1 new file mode 100644 index 000000000..df81fb4ea --- /dev/null +++ b/platforms-setup-guides/duckdb-setup/scripts/run-duckdb-ga.ps1 @@ -0,0 +1,121 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +param( + [string]$Config = "platforms-setup-guides/duckdb-setup/profiling/ga-relaxed.properties", + [string]$Executions = "wayang-platforms/wayang-duckdb/target/cost-profiling/duckdb/executions.json", + [string]$Log = "wayang-platforms/wayang-duckdb/target/cost-profiling/duckdb/ga-relaxed-run.log" +) + +$ErrorActionPreference = "Stop" + +$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$root = (Resolve-Path (Join-Path $scriptDir "../../..")).Path +$mvnw = Join-Path $root "mvnw.cmd" + +function Resolve-WayangFileUrl([string]$Path) { + $resolved = (Resolve-Path (Join-Path $root $Path)).Path + return ([System.Uri]$resolved).AbsoluteUri +} + +function Resolve-RepoPath([string]$Path) { + return (Resolve-Path (Join-Path $root $Path)).Path +} + +function Add-IfExists([System.Collections.Generic.List[string]]$Items, [string]$Path) { + if (Test-Path $Path) { + $Items.Add((Resolve-Path $Path).Path) + } +} + +Push-Location $root +try { + $compileArgs = @( + "-Pskip-prerequisite-check", + "-pl", "wayang-profiler,wayang-platforms/wayang-duckdb", + "-am", + "-DskipTests", + "-Drat.skip=true", + "-Dlicense.skip=true", + "compile" + ) + & $mvnw @compileArgs + + if ($LASTEXITCODE -ne 0) { + throw "Maven compile failed with exit code $LASTEXITCODE." + } + + $classpathArgs = @( + "-Pskip-prerequisite-check", + "-pl", "wayang-profiler", + "-DincludeScope=runtime", + "-Dmdep.outputFile=target/duckdb-ga-profiler-classpath.txt", + "-Drat.skip=true", + "-Dlicense.skip=true", + "dependency:build-classpath" + ) + & $mvnw @classpathArgs + + if ($LASTEXITCODE -ne 0) { + throw "Maven classpath generation failed with exit code $LASTEXITCODE." + } + + $dependencyClasspath = Get-Content "wayang-profiler/target/duckdb-ga-profiler-classpath.txt" + $classpathItems = [System.Collections.Generic.List[string]]::new() + + Add-IfExists $classpathItems "$env:USERPROFILE/.m2/repository/org/antlr/antlr4-runtime/4.13.1/antlr4-runtime-4.13.1.jar" + Add-IfExists $classpathItems "$env:USERPROFILE/.m2/repository/com/fasterxml/jackson/core/jackson-core/2.18.6/jackson-core-2.18.6.jar" + + foreach ($classesDir in @( + "wayang-profiler/target/classes", + "wayang-platforms/wayang-duckdb/target/classes", + "wayang-platforms/wayang-jdbc-template/target/classes", + "wayang-platforms/wayang-java/target/classes", + "wayang-platforms/wayang-spark/target/classes", + "wayang-platforms/wayang-postgres/target/classes", + "wayang-platforms/wayang-sqlite3/target/classes", + "wayang-commons/wayang-core/target/classes", + "wayang-commons/wayang-basic/target/classes", + "wayang-commons/wayang-utils-profile-db/target/classes" + )) { + Add-IfExists $classpathItems (Join-Path $root $classesDir) + } + + $classpathItems.Add($dependencyClasspath) + $classpath = [string]::Join([System.IO.Path]::PathSeparator, $classpathItems) + + $argsFile = Join-Path (Split-Path -Parent (Resolve-RepoPath $Executions)) "duckdb-ga.args" + @( + "-cp", + $classpath, + "org.apache.wayang.profiler.log.GeneticOptimizerApp", + (Resolve-WayangFileUrl $Config), + (Resolve-RepoPath $Executions) + ) | Set-Content -Encoding ASCII $argsFile + + & java "@$argsFile" *> (Join-Path $root $Log) + if ($LASTEXITCODE -ne 0) { + Get-Content (Join-Path $root $Log) -Tail 80 + throw "DuckDB GA profiler failed with exit code $LASTEXITCODE." + } + + Write-Host "DuckDB GA profiler completed." + Write-Host "Log: $Log" +} +finally { + Pop-Location +} diff --git a/platforms-setup-guides/duckdb-setup/src/test/java/org/apache/wayang/duckdb/DuckDBIntegrationTest.java b/platforms-setup-guides/duckdb-setup/src/test/java/org/apache/wayang/duckdb/DuckDBIntegrationTest.java new file mode 100644 index 000000000..c1ad0aaa2 --- /dev/null +++ b/platforms-setup-guides/duckdb-setup/src/test/java/org/apache/wayang/duckdb/DuckDBIntegrationTest.java @@ -0,0 +1,244 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Standalone JDBC integration tests for the local DuckDB setup. + * + *

Run from the repository root with: + * 

+ *   ./mvnw -f platforms-setup-guides/duckdb-setup/pom.xml \
+ *     -Pintegration -Dtest=DuckDBIntegrationTest test
+ *
+ */ +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +class DuckDBIntegrationTest { + + private static final String JDBC_URL = System.getenv().getOrDefault( + "DUCKDB_JDBC_URL", + System.getProperty("duckdb.url", "jdbc:duckdb:data/wayang.duckdb")); + + private static Connection connection; + + @BeforeAll + static void openConnectionAndLoadFixture() throws Exception { + connection = DriverManager.getConnection(JDBC_URL); + executeSqlScript(Path.of("scripts", "init.sql")); + } + + @AfterAll + static void closeConnection() throws Exception { + if (connection != null && !connection.isClosed()) { + connection.close(); + } + } + + @Test + @Order(1) + @DisplayName("DuckDB responds to SELECT 1") + void connectivity() throws SQLException { + List> rows = query("SELECT 1"); + assertEquals(1, rows.size()); + assertEquals(1, ((Number) rows.get(0).get(0)).intValue()); + } + + @Test + @Order(2) + @DisplayName("Fixture tables are visible") + void fixtureTablesVisible() throws SQLException { + List> rows = query(""" + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'wayang_it' + ORDER BY table_name + """); + assertEquals(2, rows.size()); + assertEquals("customers", rows.get(0).get(0)); + assertEquals("orders", rows.get(1).get(0)); + } + + @Test + @Order(3) + @DisplayName("Orders table full scan") + void ordersFullScan() throws SQLException { + assertEquals(6L, scalarLong("SELECT COUNT(*) FROM wayang_it.orders")); + } + + @Test + @Order(4) + @DisplayName("Filter by region") + void filterByRegion() throws SQLException { + List> rows = query(""" + SELECT order_id, region + FROM wayang_it.orders + WHERE region = 'AMER' + ORDER BY order_id + """); + assertEquals(3, rows.size()); + rows.forEach(row -> assertEquals("AMER", row.get(1))); + } + + @Test + @Order(5) + @DisplayName("Project subset of columns") + void projection() throws SQLException { + List> rows = query(""" + SELECT region, amount + FROM wayang_it.orders + ORDER BY order_id + LIMIT 3 + """); + assertEquals(3, rows.size()); + assertEquals(2, rows.get(0).size()); + } + + @Test + @Order(6) + @DisplayName("Join orders and customers") + void join() throws SQLException { + assertEquals(6L, scalarLong(""" + SELECT COUNT(*) + FROM wayang_it.orders o + JOIN wayang_it.customers c ON o.customer_id = c.cust_id + """)); + } + + @Test + @Order(7) + @DisplayName("Aggregate total amount by region") + void aggregateByRegion() throws SQLException { + List> rows = query(""" + SELECT region, SUM(amount) AS total_amount + FROM wayang_it.orders + GROUP BY region + ORDER BY region + """); + assertEquals(3, rows.size()); + assertEquals("AMER", rows.get(0).get(0)); + assertEquals(3830.75, ((Number) rows.get(0).get(1)).doubleValue(), 0.01); + } + + @Test + @Order(8) + @DisplayName("Filter by amount threshold") + void filterByAmount() throws SQLException { + List> rows = query(""" + SELECT amount + FROM wayang_it.orders + WHERE amount > 1000.0 + """); + assertFalse(rows.isEmpty()); + rows.forEach(row -> assertTrue(((Number) row.get(0)).doubleValue() > 1000.0)); + } + + @Test + @Order(9) + @DisplayName("Sort by amount") + void sortByAmount() throws SQLException { + List> rows = query(""" + SELECT order_id, amount + FROM wayang_it.orders + ORDER BY amount DESC + LIMIT 1 + """); + assertEquals(1, rows.size()); + assertEquals(1L, ((Number) rows.get(0).get(0)).longValue()); + } + + @Test + @Order(10) + @DisplayName("Create table as select") + void createTableAsSelect() throws SQLException { + try (Statement statement = connection.createStatement()) { + statement.execute("DROP TABLE IF EXISTS wayang_it.operator_result"); + statement.execute(""" + CREATE TABLE wayang_it.operator_result AS + SELECT * FROM wayang_it.orders WHERE region = 'AMER' + """); + } + assertEquals(3L, scalarLong("SELECT COUNT(*) FROM wayang_it.operator_result")); + } + + private static void executeSqlScript(Path script) throws Exception { + String sql = Files.readString(script); + StringBuilder statement = new StringBuilder(); + try (Statement jdbcStatement = connection.createStatement()) { + for (String line : sql.split("\\R")) { + String trimmed = line.trim(); + if (trimmed.startsWith("--") || trimmed.isEmpty()) { + continue; + } + statement.append(line).append('\n'); + if (trimmed.endsWith(";")) { + jdbcStatement.execute(statement.toString()); + statement.setLength(0); + } + } + if (statement.length() > 0) { + jdbcStatement.execute(statement.toString()); + } + } + } + + private static long scalarLong(String sql) throws SQLException { + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) { + resultSet.next(); + return resultSet.getLong(1); + } + } + + private static List> query(String sql) throws SQLException { + List> rows = new ArrayList<>(); + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) { + int columns = resultSet.getMetaData().getColumnCount(); + while (resultSet.next()) { + List row = new ArrayList<>(); + for (int i = 1; i <= columns; i++) { + row.add(resultSet.getObject(i)); + } + rows.add(row); + } + } + return rows; + } +} diff --git a/wayang-platforms/pom.xml b/wayang-platforms/pom.xml index 06b34ee77..b5cdd37b4 100644 --- a/wayang-platforms/pom.xml +++ b/wayang-platforms/pom.xml @@ -45,6 +45,7 @@ wayang-generic-jdbc wayang-presto wayang-trino + wayang-duckdb wayang-tensorflow diff --git a/wayang-platforms/wayang-duckdb/README.md b/wayang-platforms/wayang-duckdb/README.md new file mode 100644 index 000000000..17b267173 --- /dev/null +++ b/wayang-platforms/wayang-duckdb/README.md @@ -0,0 +1,199 @@ +# Wayang Platform DuckDB + +Wayang platform adapter for [DuckDB](https://duckdb.org/) through JDBC. + +DuckDB is embedded, so the platform does not require a coordinator, worker, or +long-running database service. Use `jdbc:duckdb:` for an in-memory database, or +`jdbc:duckdb:/path/to/database.duckdb` for a persistent database file. + +## Usage + +Register the DuckDB plugin in a Wayang context: + +```java +Configuration config = new Configuration(); +config.setProperty("wayang.duckdb.jdbc.url", "jdbc:duckdb:/tmp/wayang.duckdb"); + +WayangContext wayang = new WayangContext(config) + .withPlugin(DuckDB.plugin()); +``` + +The default configuration lives in +`src/main/resources/wayang-duckdb-defaults.properties`. Important properties: + +```properties +wayang.duckdb.jdbc.url = jdbc:duckdb: +wayang.duckdb.jdbc.user = +wayang.duckdb.jdbc.password = +``` + +## Supported Operators + +The DuckDB platform follows the same JDBC pushdown model as Trino and Presto: + +| Operator | SQL shape | +|----------|-----------| +| `TableSource` | `SELECT * FROM table` | +| `Filter` | `WHERE ...` | +| `Projection` | `SELECT col1, col2, ...` | +| `Join` | `JOIN ... ON ...` | +| `GlobalReduce` | aggregate projection such as `SUM(amount)` | +| `ReduceBy` | aggregate projection plus `GROUP BY` | +| `Sort` | `ORDER BY ...` | +| `TableSink` | `CREATE TABLE ... AS SELECT ...` or `INSERT INTO ... SELECT ...` | +| `ParquetSource` | DuckDB relation, configured mapping, or auto-created `read_parquet(...)` view | + +## Parquet And GCS + +DuckDB can read local Parquet files directly through `read_parquet(...)`. The +Wayang adapter supports two Trino/Presto-style modes: + +```properties +# Keep the logical Parquet URI and map it to an existing DuckDB relation. +wayang.duckdb.parquetsource.mappings = file:///data/orders.parquet=wayang_parquet.orders + +# Or let DuckDB create a view over the Parquet location before execution. +wayang.duckdb.parquetsource.auto-create = true +wayang.duckdb.parquetsource.auto-create.template = CREATE OR REPLACE VIEW ${relation} AS SELECT * FROM read_parquet('${uri}') +``` + +For GCS Parquet files, load DuckDB's `httpfs` extension before the view is +created: + +```properties +wayang.duckdb.parquetsource.prepare-sql = INSTALL httpfs; LOAD httpfs +``` + +Private GCS buckets can use the same hook to create a DuckDB secret with HMAC +credentials, as documented in `wayang-duckdb-defaults.properties`. + +## Tests + +The embedded operator suite mirrors `TrinoOperatorsIT` / `PrestoOperatorsIT`, +but runs against a temporary DuckDB database file. The Parquet and cost pilots +mirror the separate Trino/Presto feature branches for those pieces. + +Run the DuckDB operator suite while compiling the required reactor modules: + +```bash +./mvnw -Pskip-prerequisite-check -pl wayang-platforms/wayang-duckdb -am \ + -Dtest=DuckDBOperatorsIT \ + -Dsurefire.failIfNoSpecifiedTests=false -DfailIfNoTests=false \ + -Drat.skip=true -Dlicense.skip=true test +``` + +On PowerShell: + +```powershell +.\mvnw.cmd --% -Pskip-prerequisite-check -pl wayang-platforms/wayang-duckdb -am -Dtest=DuckDBOperatorsIT -Dsurefire.failIfNoSpecifiedTests=false -DfailIfNoTests=false -Drat.skip=true -Dlicense.skip=true test +``` + +Expected result: + +```text +Tests run: 15, Failures: 0, Errors: 0, Skipped: 0 +``` + +Run the Parquet suite, including a public GCS smoke when DuckDB `httpfs` can +reach the configured object: + +```bash +./mvnw -Pskip-prerequisite-check -pl wayang-platforms/wayang-duckdb -am \ + -Dtest=DuckDBParquetSourceIT \ + -Dsurefire.failIfNoSpecifiedTests=false -DfailIfNoTests=false \ + -Drat.skip=true -Dlicense.skip=true test +``` + +Override the public GCS file with: + +```bash +-Dduckdb.gcs.parquet.uri=gs://bucket/path/file.parquet +``` + +## Test Coverage + +| Test | What it checks | +|------|----------------| +| `loadsDuckDbDriverAndRunsQuery` | DuckDB JDBC driver sanity check | +| `tableSource` | Full table scan through `DuckDBTableSource` | +| `filter` | Wayang `FilterOperator` and SQL `WHERE` pushdown | +| `projection` | Column projection through SQL `SELECT` | +| `join` | DuckDB join plus a test-only flatten projection | +| `globalReduce` | Global aggregation such as `SUM` | +| `reduceBy` | Grouped aggregation and SQL `GROUP BY` | +| `sort` | Wayang sort and SQL `ORDER BY` | +| `tableSink` | Filtered result written with `CREATE TABLE AS` | +| `javaPlanBuilderReadTableFilterProjection` | `readTable -> filter -> projection -> writeTable` | +| `javaPlanBuilderReadTableFilterGlobalReduce` | `readTable -> filter -> globalReduce -> writeTable` | +| `javaPlanBuilderReadTableReduceBySort` | `readTable -> reduceByKey -> sort -> writeTable` | +| `javaPlanBuilderReadTableFilterProjectionTableSink` | `readTable -> filter -> projection -> writeTable` | +| `javaPlanBuilderReadTableJoin` | `readTable + readTable -> join -> writeTable` | +| `generatedSqlContainsPushdownShapes` | Captured SQL contains `WHERE`, `JOIN`, `GROUP BY`, and `ORDER BY` | + +## Cost Profiling + +`DuckDBCostPilotIT` follows the Trino Week8 cost-pilot shape and writes Wayang +execution/cardinality logs plus a manifest under `target/cost-profiling/duckdb`. +The default/reference workload is S01-S13 over 10k, 50k, 100k, and 250k rows +with six repetitions, producing 312 Wayang executions: + +```bash +./mvnw -Pskip-prerequisite-check -pl wayang-platforms/wayang-duckdb -am \ + -Dtest=DuckDBCostPilotIT#runPilot \ + -Dduckdb.profile.rowCounts=10000,50000,100000,250000 \ + -Dduckdb.profile.plans=S01,S02,S03,S04,S05,S06,S07,S08,S09,S10,S11,S12,S13 \ + -Dduckdb.profile.repetitions=6 \ + -Drat.skip=true -Dlicense.skip=true test +``` + +S14-S16 are implemented as optional expanded/join-heavy plans and can be passed +explicitly, but the checked-in reference parameters come from the Week8 S01-S13 +run. + +For a quick local smoke: + +```bash +./mvnw -Pskip-prerequisite-check -pl wayang-platforms/wayang-duckdb -am \ + -Dtest=DuckDBCostPilotIT#runPilot \ + -Dduckdb.profile.rowCounts=100,1000 \ + -Dduckdb.profile.plans=S01,S02 \ + -Dduckdb.profile.repetitions=2 \ + -Drat.skip=true -Dlicense.skip=true test +``` + +Then run the GA profiler outside Surefire, as in the Trino profiling branch. +This avoids Maven/Surefire dependency-ordering conflicts around Jackson and +ANTLR. On PowerShell: + +```powershell +.\platforms-setup-guides\duckdb-setup\scripts\run-duckdb-ga.ps1 +``` + +The GA settings live in +`platforms-setup-guides/duckdb-setup/profiling/ga-relaxed.properties`. The +learned output is written to +`wayang-platforms/wayang-duckdb/target/cost-profiling/duckdb/learned-duckdb-relaxed.properties`. +The GA optimizer is stochastic; repeated runs over the same execution log can +produce slightly different coefficients. + +## Demo + +`DuckDBDemo` creates a small local fixture and runs two Wayang plans that end in +DuckDB table sinks: + +| Segment | Pushdown shape | +|---------|----------------| +| Filter | `SELECT * FROM wayang_demo.orders WHERE region = 'AMER'` | +| Projection + filter | `SELECT region, amount FROM wayang_demo.orders WHERE region = 'AMER'` | + +Run from the repository root: + +```bash +./mvnw -Pskip-prerequisite-check -pl wayang-platforms/wayang-duckdb -am \ + -DskipTests -Drat.skip=true -Dlicense.skip=true compile + +./mvnw -Pskip-prerequisite-check -pl wayang-platforms/wayang-duckdb \ + -DskipTests -Drat.skip=true -Dlicense.skip=true exec:java \ + -Dexec.mainClass=org.apache.wayang.duckdb.DuckDBDemo \ + -Dduckdb.url=jdbc:duckdb:target/duckdb-demo.duckdb +``` diff --git a/wayang-platforms/wayang-duckdb/pom.xml b/wayang-platforms/wayang-duckdb/pom.xml new file mode 100644 index 000000000..b86ac3d76 --- /dev/null +++ b/wayang-platforms/wayang-duckdb/pom.xml @@ -0,0 +1,91 @@ + + + + 4.0.0 + + + wayang-platforms + org.apache.wayang + 1.1.2-SNAPSHOT + + + wayang-duckdb + + Wayang Platform DuckDB + + Wayang implementation of the operators to be working with the platform "DuckDB" + + + + org.apache.wayang.platform.duckdb + 1.5.5.1 + + + + + org.duckdb + duckdb_jdbc + ${duckdb.version} + + + org.antlr + antlr4-runtime + 4.13.1 + + + org.apache.wayang + wayang-basic + 1.1.2-SNAPSHOT + + + org.apache.wayang + wayang-jdbc-template + 1.1.2-SNAPSHOT + + + org.apache.wayang + wayang-spark + 1.1.2-SNAPSHOT + + + org.apache.wayang + wayang-api-scala-java + 1.1.2-SNAPSHOT + test + + + org.junit.jupiter + junit-jupiter + 5.10.2 + test + + + + + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + + + + + diff --git a/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/DuckDB.java b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/DuckDB.java new file mode 100644 index 000000000..5fcd8796c --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/DuckDB.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb; + +import org.apache.wayang.duckdb.platform.DuckDBPlatform; +import org.apache.wayang.duckdb.plugin.DuckDBConversionsPlugin; +import org.apache.wayang.duckdb.plugin.DuckDBPlugin; + +/** + * Entry point that exposes the relevant components of the DuckDB platform. + * + *

Typical usage: + * 

{@code
+ *   new WayangContext(config)
+ *       .withPlugin(Java.basicPlugin())
+ *       .withPlugin(DuckDB.plugin());
+ * }
+ */ +public class DuckDB { + + private static final DuckDBPlugin PLUGIN = new DuckDBPlugin(); + + private static final DuckDBConversionsPlugin CONVERSIONS_PLUGIN = new DuckDBConversionsPlugin(); + + /** + * @return the {@link DuckDBPlugin} (operator mappings + channel conversions) + */ + public static DuckDBPlugin plugin() { + return PLUGIN; + } + + /** + * @return the {@link DuckDBConversionsPlugin} (channel conversions only) + */ + public static DuckDBConversionsPlugin conversionPlugin() { + return CONVERSIONS_PLUGIN; + } + + /** + * @return the {@link DuckDBPlatform} + */ + public static DuckDBPlatform platform() { + return DuckDBPlatform.getInstance(); + } + +} diff --git a/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/DuckDBDemo.java b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/DuckDBDemo.java new file mode 100644 index 000000000..fee667513 --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/DuckDBDemo.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.function.ProjectionDescriptor; +import org.apache.wayang.basic.operators.FilterOperator; +import org.apache.wayang.basic.operators.MapOperator; +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.basic.types.RecordType; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.WayangContext; +import org.apache.wayang.core.function.PredicateDescriptor; +import org.apache.wayang.core.plan.wayangplan.WayangPlan; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.duckdb.operators.DuckDBTableSource; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Properties; + +/** + * Standalone demo for the Wayang DuckDB platform. + * + *

Run from the repository root with: + * 

+ *   ./mvnw -Pskip-prerequisite-check -pl wayang-platforms/wayang-duckdb \
+ *     -DskipTests -Drat.skip=true -Dlicense.skip=true exec:java \
+ *     -Dexec.mainClass=org.apache.wayang.duckdb.DuckDBDemo
+ *
+ */ +public class DuckDBDemo { + + private static final String JDBC_URL = System.getProperty("duckdb.url", "jdbc:duckdb:target/duckdb-demo.duckdb"); + private static final String SCHEMA = "wayang_demo"; + private static final String ORDERS = SCHEMA + ".orders"; + private static final String FILTER_RESULT = SCHEMA + ".filter_result"; + private static final String PROJECTION_RESULT = SCHEMA + ".projection_result"; + + public static void main(String[] args) throws Exception { + createFixture(); + runFilterPushdown(); + runProjectionPushdown(); + } + + private static void runFilterPushdown() throws Exception { + System.out.println(); + System.out.println("DuckDB demo: Filter pushdown"); + System.out.println("SQL shape: SELECT * FROM " + ORDERS + " WHERE region = 'AMER'"); + + DuckDBTableSource source = new DuckDBTableSource( + ORDERS, "order_id", "customer_id", "region", "amount"); + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>( + (Record record) -> "AMER".equals(record.getField(2)), Record.class) + .withSqlImplementation("region = 'AMER'")); + TableSink sink = new TableSink<>( + new Properties(), "overwrite", FILTER_RESULT, + "order_id", "customer_id", "region", "amount"); + + source.connectTo(0, filter, 0); + filter.connectTo(0, sink, 0); + wayangContext().execute("DuckDB filter demo", new WayangPlan(sink)); + + printQuery("SELECT order_id, region, amount FROM " + FILTER_RESULT + " ORDER BY order_id"); + } + + private static void runProjectionPushdown() throws Exception { + System.out.println(); + System.out.println("DuckDB demo: Projection + filter pushdown"); + System.out.println("SQL shape: SELECT region, amount FROM " + ORDERS + " WHERE region = 'AMER'"); + + DuckDBTableSource source = new DuckDBTableSource( + ORDERS, "order_id", "customer_id", "region", "amount"); + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>( + (Record record) -> "AMER".equals(record.getField(2)), Record.class) + .withSqlImplementation("region = 'AMER'")); + MapOperator projection = new MapOperator<>( + ProjectionDescriptor.createForRecords( + new RecordType("order_id", "customer_id", "region", "amount"), + "region", "amount"), + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class)); + TableSink sink = new TableSink<>( + new Properties(), "overwrite", PROJECTION_RESULT, + "region", "amount"); + + source.connectTo(0, filter, 0); + filter.connectTo(0, projection, 0); + projection.connectTo(0, sink, 0); + wayangContext().execute("DuckDB projection demo", new WayangPlan(sink)); + + printQuery("SELECT region, amount FROM " + PROJECTION_RESULT + " ORDER BY amount DESC"); + } + + private static WayangContext wayangContext() { + Configuration configuration = new Configuration(); + configuration.setProperty("wayang.duckdb.jdbc.url", JDBC_URL); + configuration.setProperty("wayang.duckdb.jdbc.user", ""); + configuration.setProperty("wayang.duckdb.jdbc.password", ""); + return new WayangContext(configuration) + .withPlugin(DuckDB.plugin()); + } + + private static void createFixture() throws Exception { + try (Connection connection = DriverManager.getConnection(JDBC_URL); + Statement statement = connection.createStatement()) { + statement.execute("CREATE SCHEMA IF NOT EXISTS " + SCHEMA); + statement.execute("DROP TABLE IF EXISTS " + FILTER_RESULT); + statement.execute("DROP TABLE IF EXISTS " + PROJECTION_RESULT); + statement.execute("DROP TABLE IF EXISTS " + ORDERS); + statement.execute("CREATE TABLE " + ORDERS + " (" + + "order_id BIGINT, customer_id BIGINT, region VARCHAR, amount DOUBLE)"); + statement.execute("INSERT INTO " + ORDERS + " VALUES " + + "(1, 100, 'AMER', 2200.0)," + + "(2, 101, 'EMEA', 800.5)," + + "(3, 100, 'AMER', 680.5)," + + "(4, 102, 'APAC', 1500.0)," + + "(5, 101, 'EMEA', 1100.0)," + + "(6, 100, 'AMER', 950.25)"); + } + } + + private static void printQuery(String sql) throws Exception { + try (Connection connection = DriverManager.getConnection(JDBC_URL); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) { + int columns = resultSet.getMetaData().getColumnCount(); + while (resultSet.next()) { + StringBuilder row = new StringBuilder(" "); + for (int i = 1; i <= columns; i++) { + if (i > 1) { + row.append(" | "); + } + row.append(resultSet.getObject(i)); + } + System.out.println(row); + } + } + } +} diff --git a/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/channels/ChannelConversions.java b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/channels/ChannelConversions.java new file mode 100644 index 000000000..9fe60ffea --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/channels/ChannelConversions.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb.channels; + +import org.apache.wayang.core.optimizer.channels.ChannelConversion; +import org.apache.wayang.core.optimizer.channels.DefaultChannelConversion; +import org.apache.wayang.java.channels.StreamChannel; +import org.apache.wayang.jdbc.operators.SqlToRddOperator; +import org.apache.wayang.jdbc.operators.SqlToStreamOperator; +import org.apache.wayang.duckdb.platform.DuckDBPlatform; +import org.apache.wayang.spark.channels.RddChannel; + +import java.util.Arrays; +import java.util.Collection; + +/** + * {@link ChannelConversion}s that materialise a DuckDB SQL query result into a + * Java {@link StreamChannel} or a Spark {@link RddChannel}. + */ +public class ChannelConversions { + + public static final ChannelConversion SQL_TO_STREAM_CONVERSION = new DefaultChannelConversion( + DuckDBPlatform.getInstance().getSqlQueryChannelDescriptor(), + StreamChannel.DESCRIPTOR, + () -> new SqlToStreamOperator(DuckDBPlatform.getInstance()) + ); + + public static final ChannelConversion SQL_TO_UNCACHED_RDD_CONVERSION = new DefaultChannelConversion( + DuckDBPlatform.getInstance().getSqlQueryChannelDescriptor(), + RddChannel.UNCACHED_DESCRIPTOR, + () -> new SqlToRddOperator(DuckDBPlatform.getInstance()) + ); + + public static final Collection ALL = Arrays.asList( + SQL_TO_STREAM_CONVERSION, + SQL_TO_UNCACHED_RDD_CONVERSION + ); + +} diff --git a/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/FilterMapping.java b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/FilterMapping.java new file mode 100644 index 000000000..0addc6dbf --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/FilterMapping.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.FilterOperator; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.duckdb.operators.DuckDBFilterOperator; +import org.apache.wayang.duckdb.platform.DuckDBPlatform; + +import java.util.Collection; +import java.util.Collections; + +/** + * Maps a {@link FilterOperator} (with a SQL-implementable predicate) to a + * {@link DuckDBFilterOperator}. + */ +@SuppressWarnings("unchecked") +public class FilterMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + DuckDBPlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern> operatorPattern = new OperatorPattern<>( + "filter", new FilterOperator<>(null, DataSetType.createDefault(Record.class)), false + ).withAdditionalTest(op -> op.getPredicateDescriptor().getSqlImplementation() != null); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators( + (matchedOperator, epoch) -> new DuckDBFilterOperator(matchedOperator).at(epoch) + ); + } +} diff --git a/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/GlobalReduceMapping.java b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/GlobalReduceMapping.java new file mode 100644 index 000000000..ccea6326d --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/GlobalReduceMapping.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.GlobalReduceOperator; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.duckdb.operators.DuckDBGlobalReduceOperator; +import org.apache.wayang.duckdb.platform.DuckDBPlatform; + +import java.util.Collection; +import java.util.Collections; + +/** + * Maps a {@link GlobalReduceOperator} (with a SQL-implementable reduction) to a + * {@link DuckDBGlobalReduceOperator}. + */ +@SuppressWarnings("unchecked") +public class GlobalReduceMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + DuckDBPlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern> operatorPattern = new OperatorPattern<>( + "reduce", new GlobalReduceOperator(null, DataSetType.createDefault(Record.class)), false) + .withAdditionalTest(op -> op.getReduceDescriptor().getSqlImplementation() != null); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators>( + (matchedOperator, epoch) -> new DuckDBGlobalReduceOperator(matchedOperator).at(epoch) + ); + } +} diff --git a/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/JoinMapping.java b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/JoinMapping.java new file mode 100644 index 000000000..f5b4863e1 --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/JoinMapping.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.JoinOperator; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.duckdb.operators.DuckDBJoinOperator; +import org.apache.wayang.duckdb.platform.DuckDBPlatform; + +import java.util.Collection; +import java.util.Collections; + +/** + * Maps a {@link JoinOperator} whose key descriptors are SQL-implementable to a + * {@link DuckDBJoinOperator}. + */ +@SuppressWarnings("unchecked") +public class JoinMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + DuckDBPlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + OperatorPattern> operatorPattern = new OperatorPattern<>( + "join", + new JoinOperator( + null, + null, + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class) + ), + false + ) + .withAdditionalTest(op -> op.getKeyDescriptor0() instanceof TransformationDescriptor) + .withAdditionalTest(op -> op.getKeyDescriptor1() instanceof TransformationDescriptor) + .withAdditionalTest(op -> op.getKeyDescriptor0().getSqlImplementation() != null) + .withAdditionalTest(op -> op.getKeyDescriptor1().getSqlImplementation() != null); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators>( + (matchedOperator, epoch) -> new DuckDBJoinOperator(matchedOperator).at(epoch) + ); + } +} diff --git a/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/Mappings.java b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/Mappings.java new file mode 100644 index 000000000..459e84b9e --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/Mappings.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb.mapping; + +import org.apache.wayang.core.mapping.Mapping; + +import java.util.Arrays; +import java.util.Collection; + +/** + * Register of the {@link Mapping}s supported on the DuckDB platform. + */ +public class Mappings { + + public static final Collection ALL = Arrays.asList( + new FilterMapping(), + new GlobalReduceMapping(), + new JoinMapping(), + new ParquetSourceMapping(), + new ProjectionMapping(), + new ReduceByMapping(), + new SortMapping(), + new TableSinkMapping() + ); + +} diff --git a/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/ParquetSourceMapping.java b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/ParquetSourceMapping.java new file mode 100644 index 000000000..5b2842e1c --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/ParquetSourceMapping.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb.mapping; + +import org.apache.wayang.basic.operators.ParquetSource; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.duckdb.operators.DuckDBParquetSource; +import org.apache.wayang.duckdb.platform.DuckDBPlatform; + +import java.util.Collection; +import java.util.Collections; + +/** + * Mapping from {@link ParquetSource} to {@link DuckDBParquetSource}. + */ +public class ParquetSourceMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + DuckDBPlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern operatorPattern = new OperatorPattern( + "source", new ParquetSource((String) null, (String[]) null), false + ); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators( + (matchedOperator, epoch) -> new DuckDBParquetSource(matchedOperator).at(epoch) + ); + } +} diff --git a/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/ProjectionMapping.java b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/ProjectionMapping.java new file mode 100644 index 000000000..4946cfd5c --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/ProjectionMapping.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.function.ProjectionDescriptor; +import org.apache.wayang.basic.operators.MapOperator; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.duckdb.operators.DuckDBProjectionOperator; +import org.apache.wayang.duckdb.platform.DuckDBPlatform; + +import java.util.Collection; +import java.util.Collections; + +/** + * Maps a {@link MapOperator} that carries a {@link ProjectionDescriptor} to a + * {@link DuckDBProjectionOperator}. + */ +public class ProjectionMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + DuckDBPlatform.getInstance())); + } + + private SubplanPattern createSubplanPattern() { + OperatorPattern> operatorPattern = new OperatorPattern<>( + "projection", + new MapOperator<>( + null, + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class)), + false) + .withAdditionalTest(op -> op.getFunctionDescriptor() instanceof ProjectionDescriptor) + .withAdditionalTest(op -> op.getNumInputs() == 1); // No broadcasts. + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators>( + (matchedOperator, epoch) -> new DuckDBProjectionOperator(matchedOperator).at(epoch)); + } +} diff --git a/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/ReduceByMapping.java b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/ReduceByMapping.java new file mode 100644 index 000000000..0be3437cc --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/ReduceByMapping.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.ReduceByOperator; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.duckdb.operators.DuckDBReduceByOperator; +import org.apache.wayang.duckdb.platform.DuckDBPlatform; + +import java.util.Collection; +import java.util.Collections; + +/** + * Maps a {@link ReduceByOperator} (with SQL-implementable key and reduction) to + * a {@link DuckDBReduceByOperator}. + */ +@SuppressWarnings("unchecked") +public class ReduceByMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + DuckDBPlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern> operatorPattern = new OperatorPattern<>( + "reduceBy", + new ReduceByOperator(null, null, DataSetType.createDefault(Record.class)), + false) + .withAdditionalTest(op -> op.getKeyDescriptor().getSqlImplementation() != null + && op.getReduceDescriptor().getSqlImplementation() != null); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators>( + (matchedOperator, epoch) -> new DuckDBReduceByOperator(matchedOperator).at(epoch) + ); + } +} diff --git a/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/SortMapping.java b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/SortMapping.java new file mode 100644 index 000000000..8c316472c --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/SortMapping.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb.mapping; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.SortOperator; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.duckdb.operators.DuckDBSortOperator; +import org.apache.wayang.duckdb.platform.DuckDBPlatform; + +import java.util.Collection; +import java.util.Collections; + +/** + * Maps a {@link SortOperator} (with a SQL-implementable sort key) to a + * {@link DuckDBSortOperator}. + */ +@SuppressWarnings("unchecked") +public class SortMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + DuckDBPlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern> operatorPattern = new OperatorPattern<>( + "sort", + new SortOperator(null, DataSetType.createDefault(Record.class)), + false) + .withAdditionalTest(op -> op.getKeyDescriptor().getSqlImplementation() != null); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators>( + (matchedOperator, epoch) -> new DuckDBSortOperator(matchedOperator).at(epoch) + ); + } +} diff --git a/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/TableSinkMapping.java b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/TableSinkMapping.java new file mode 100644 index 000000000..ff65c8e0d --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/mapping/TableSinkMapping.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb.mapping; + +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.duckdb.operators.DuckDBTableSinkOperator; +import org.apache.wayang.duckdb.platform.DuckDBPlatform; + +import java.util.Collection; +import java.util.Collections; + +/** + * Maps a {@link TableSink} to a {@link DuckDBTableSinkOperator}. + */ +@SuppressWarnings("unchecked") +public class TableSinkMapping implements Mapping { + + @Override + public Collection getTransformations() { + return Collections.singleton(new PlanTransformation( + this.createSubplanPattern(), + this.createReplacementSubplanFactory(), + DuckDBPlatform.getInstance() + )); + } + + private SubplanPattern createSubplanPattern() { + final OperatorPattern operatorPattern = new OperatorPattern<>( + "sink", new TableSink<>(null, null, null), false + ); + return SubplanPattern.createSingleton(operatorPattern); + } + + private ReplacementSubplanFactory createReplacementSubplanFactory() { + return new ReplacementSubplanFactory.OfSingleOperators( + (matchedOperator, epoch) -> new DuckDBTableSinkOperator(matchedOperator).at(epoch) + ); + } +} diff --git a/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBExecutionOperator.java b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBExecutionOperator.java new file mode 100644 index 000000000..bc0c272f8 --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBExecutionOperator.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb.operators; + +import org.apache.wayang.jdbc.operators.JdbcExecutionOperator; +import org.apache.wayang.duckdb.platform.DuckDBPlatform; + +/** + * Marker for {@link JdbcExecutionOperator}s that run on the {@link DuckDBPlatform}. + */ +public interface DuckDBExecutionOperator extends JdbcExecutionOperator { + + @Override + default DuckDBPlatform getPlatform() { + return DuckDBPlatform.getInstance(); + } + +} diff --git a/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBFilterOperator.java b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBFilterOperator.java new file mode 100644 index 000000000..bbdf2b40a --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBFilterOperator.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.FilterOperator; +import org.apache.wayang.core.function.PredicateDescriptor; +import org.apache.wayang.jdbc.operators.JdbcFilterOperator; + +/** + * DuckDB implementation of the {@link FilterOperator}. The predicate is pushed + * down as a SQL {@code WHERE} clause via its {@code sqlImplementation}. + */ +public class DuckDBFilterOperator extends JdbcFilterOperator implements DuckDBExecutionOperator { + + public DuckDBFilterOperator(PredicateDescriptor predicateDescriptor) { + super(predicateDescriptor); + } + + /** + * Copies an instance (exclusive of broadcasts). + * + * @param that that should be copied + */ + public DuckDBFilterOperator(FilterOperator that) { + super(that); + } + + @Override + protected DuckDBFilterOperator createCopy() { + return new DuckDBFilterOperator(this); + } +} diff --git a/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBGlobalReduceOperator.java b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBGlobalReduceOperator.java new file mode 100644 index 000000000..33f72851f --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBGlobalReduceOperator.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.GlobalReduceOperator; +import org.apache.wayang.core.function.ReduceDescriptor; +import org.apache.wayang.jdbc.operators.JdbcGlobalReduceOperator; + +/** + * DuckDB implementation of the {@link GlobalReduceOperator}. The reduction is + * pushed down as a SQL aggregate (e.g. {@code SUM(amount)}) via its + * {@code sqlImplementation}. + */ +public class DuckDBGlobalReduceOperator extends JdbcGlobalReduceOperator implements DuckDBExecutionOperator { + + public DuckDBGlobalReduceOperator(ReduceDescriptor reduceDescriptor) { + super(reduceDescriptor); + } + + /** + * Copies an instance (exclusive of broadcasts). + * + * @param that that should be copied + */ + public DuckDBGlobalReduceOperator(GlobalReduceOperator that) { + super(that); + } + + @Override + protected DuckDBGlobalReduceOperator createCopy() { + return new DuckDBGlobalReduceOperator(this); + } +} diff --git a/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBJoinOperator.java b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBJoinOperator.java new file mode 100644 index 000000000..8a9c80286 --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBJoinOperator.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.JoinOperator; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.jdbc.operators.JdbcJoinOperator; + +/** + * DuckDB implementation of the {@link JoinOperator}. The two key descriptors + * carry the {@code (table, keyColumns)} SQL implementation that the base class + * renders into a {@code JOIN ... ON ...} clause. + * + * @param type of the join key + */ +public class DuckDBJoinOperator extends JdbcJoinOperator implements DuckDBExecutionOperator { + + public DuckDBJoinOperator( + TransformationDescriptor keyDescriptor0, + TransformationDescriptor keyDescriptor1) { + super(keyDescriptor0, keyDescriptor1); + } + + public DuckDBJoinOperator(JoinOperator that) { + super(that); + } + + @Override + protected DuckDBJoinOperator createCopy() { + return new DuckDBJoinOperator(this); + } +} diff --git a/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBParquetSource.java b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBParquetSource.java new file mode 100644 index 000000000..5af9bbdd0 --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBParquetSource.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb.operators; + +import org.apache.wayang.basic.operators.ParquetSource; +import org.apache.wayang.jdbc.operators.JdbcParquetSource; + +/** + * DuckDB implementation for Parquet-backed SQL relations. + */ +public class DuckDBParquetSource extends JdbcParquetSource implements DuckDBExecutionOperator { + + public DuckDBParquetSource(String sourceName, String[] projection, String... columnNames) { + super(sourceName, projection, columnNames); + } + + public DuckDBParquetSource(ParquetSource that) { + super(that); + } +} diff --git a/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBProjectionOperator.java b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBProjectionOperator.java new file mode 100644 index 000000000..511c56771 --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBProjectionOperator.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.function.ProjectionDescriptor; +import org.apache.wayang.basic.operators.MapOperator; +import org.apache.wayang.jdbc.operators.JdbcProjectionOperator; + +/** + * DuckDB implementation of a column projection. The selected fields are pushed + * down as the SQL {@code SELECT} list. + */ +public class DuckDBProjectionOperator extends JdbcProjectionOperator implements DuckDBExecutionOperator { + + public DuckDBProjectionOperator(String... fieldNames) { + super(fieldNames); + } + + public DuckDBProjectionOperator(ProjectionDescriptor functionDescriptor) { + super(functionDescriptor); + } + + public DuckDBProjectionOperator(MapOperator that) { + super(that); + } + + @Override + protected DuckDBProjectionOperator createCopy() { + return new DuckDBProjectionOperator(this); + } + +} diff --git a/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBReduceByOperator.java b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBReduceByOperator.java new file mode 100644 index 000000000..1e7664703 --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBReduceByOperator.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.ReduceByOperator; +import org.apache.wayang.core.function.ReduceDescriptor; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.jdbc.operators.JdbcReduceByOperator; + +/** + * DuckDB implementation of the {@link ReduceByOperator}. The grouping key and + * the reduction are pushed down as a SQL {@code GROUP BY} plus aggregate (e.g. + * {@code SELECT region, SUM(amount) ... GROUP BY region}). + */ +public class DuckDBReduceByOperator extends JdbcReduceByOperator implements DuckDBExecutionOperator { + + public DuckDBReduceByOperator(TransformationDescriptor keyDescriptor, + ReduceDescriptor reduceDescriptor) { + super(keyDescriptor, reduceDescriptor); + } + + /** + * Copies an instance (exclusive of broadcasts). + * + * @param that that should be copied + */ + public DuckDBReduceByOperator(ReduceByOperator that) { + super(that); + } + + @Override + protected DuckDBReduceByOperator createCopy() { + return new DuckDBReduceByOperator(this); + } +} diff --git a/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBSortOperator.java b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBSortOperator.java new file mode 100644 index 000000000..697a2605c --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBSortOperator.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.SortOperator; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.jdbc.operators.JdbcSortOperator; + +/** + * DuckDB implementation of the {@link SortOperator}. The sort key and direction + * are pushed down as a SQL {@code ORDER BY} clause via its {@code sqlImplementation}. + */ +public class DuckDBSortOperator extends JdbcSortOperator implements DuckDBExecutionOperator { + + public DuckDBSortOperator(TransformationDescriptor keyDescriptor) { + super(keyDescriptor); + } + + /** + * Copies an instance (exclusive of broadcasts). + * + * @param that that should be copied + */ + public DuckDBSortOperator(SortOperator that) { + super(that); + } + + @Override + protected DuckDBSortOperator createCopy() { + return new DuckDBSortOperator(this); + } +} diff --git a/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBTableSinkOperator.java b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBTableSinkOperator.java new file mode 100644 index 000000000..35977c920 --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBTableSinkOperator.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb.operators; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.jdbc.operators.JdbcTableSinkOperator; + +/** + * DuckDB implementation of the {@link JdbcTableSinkOperator}. The sink stays + * entirely within DuckDB: the composed query is wrapped in a + * {@code CREATE TABLE ... AS} (mode {@code overwrite}) or {@code INSERT INTO ...} + * statement. + * + *

Table names can be unqualified or schema-qualified, e.g. + * {@code orders} or {@code main.orders}. + */ +public class DuckDBTableSinkOperator extends JdbcTableSinkOperator implements DuckDBExecutionOperator { + + public DuckDBTableSinkOperator(String tableName, String[] columnNames) { + super(tableName, columnNames); + } + + /** + * Copies an instance (exclusive of broadcasts). + * + * @param that that should be copied + */ + public DuckDBTableSinkOperator(TableSink that) { + super(that); + } +} diff --git a/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBTableSource.java b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBTableSource.java new file mode 100644 index 000000000..47eab390d --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/operators/DuckDBTableSource.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb.operators; + +import org.apache.wayang.basic.operators.TableSource; +import org.apache.wayang.core.platform.ChannelDescriptor; +import org.apache.wayang.jdbc.operators.JdbcTableSource; + +import java.util.List; + +/** + * DuckDB implementation of the {@link TableSource}. + * + *

Table names can be unqualified or schema-qualified, e.g. + * {@code orders} or {@code main.orders}. + */ +public class DuckDBTableSource extends JdbcTableSource implements DuckDBExecutionOperator { + + /** + * @see TableSource#TableSource(String, String...) + */ + public DuckDBTableSource(String tableName, String... columnNames) { + super(tableName, columnNames); + } + + /** + * Copies an instance (exclusive of broadcasts). + * + * @param that that should be copied + */ + public DuckDBTableSource(JdbcTableSource that) { + super(that); + } + + @Override + public List getSupportedInputChannels(int index) { + throw new UnsupportedOperationException("This operator has no input channels."); + } +} diff --git a/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/platform/DuckDBPlatform.java b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/platform/DuckDBPlatform.java new file mode 100644 index 000000000..8ad24135d --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/platform/DuckDBPlatform.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb.platform; + +import org.apache.wayang.core.platform.Platform; +import org.apache.wayang.jdbc.platform.JdbcPlatformTemplate; + +/** + * {@link Platform} implementation for DuckDB. + * + *

The {@code configName} {@code "duckdb"} makes Wayang resolve every property + * with the {@code wayang.duckdb.*} prefix. + */ +public class DuckDBPlatform extends JdbcPlatformTemplate { + + private static final String PLATFORM_NAME = "DuckDB"; + + private static final String CONFIG_NAME = "duckdb"; + + private static DuckDBPlatform instance = null; + + public static DuckDBPlatform getInstance() { + if (instance == null) { + instance = new DuckDBPlatform(); + } + return instance; + } + + protected DuckDBPlatform() { + super(PLATFORM_NAME, CONFIG_NAME); + } + + @Override + public String getJdbcDriverClassName() { + return "org.duckdb.DuckDBDriver"; + } + +} diff --git a/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/plugin/DuckDBConversionsPlugin.java b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/plugin/DuckDBConversionsPlugin.java new file mode 100644 index 000000000..aa797fd71 --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/plugin/DuckDBConversionsPlugin.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb.plugin; + +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.optimizer.channels.ChannelConversion; +import org.apache.wayang.core.platform.Platform; +import org.apache.wayang.core.plugin.Plugin; +import org.apache.wayang.java.platform.JavaPlatform; +import org.apache.wayang.duckdb.channels.ChannelConversions; +import org.apache.wayang.duckdb.platform.DuckDBPlatform; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; + +/** + * Provides only the {@link ChannelConversion}s for the {@link DuckDBPlatform} + * (no operator {@link Mapping}s), used to make DuckDB results consumable by + * other platforms without enabling operator pushdown. + */ +public class DuckDBConversionsPlugin implements Plugin { + + @Override + public Collection getRequiredPlatforms() { + return Arrays.asList(DuckDBPlatform.getInstance(), JavaPlatform.getInstance()); + } + + @Override + public Collection getMappings() { + return Collections.emptyList(); + } + + @Override + public Collection getChannelConversions() { + return ChannelConversions.ALL; + } + + @Override + public void setProperties(Configuration configuration) { + } +} diff --git a/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/plugin/DuckDBPlugin.java b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/plugin/DuckDBPlugin.java new file mode 100644 index 000000000..aaa6a0fa4 --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/plugin/DuckDBPlugin.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb.plugin; + +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.optimizer.channels.ChannelConversion; +import org.apache.wayang.core.plan.wayangplan.Operator; +import org.apache.wayang.core.platform.Platform; +import org.apache.wayang.core.plugin.Plugin; +import org.apache.wayang.java.platform.JavaPlatform; +import org.apache.wayang.duckdb.channels.ChannelConversions; +import org.apache.wayang.duckdb.mapping.Mappings; +import org.apache.wayang.duckdb.platform.DuckDBPlatform; + +import java.util.Arrays; +import java.util.Collection; + +/** + * Enables Wayang {@link Operator}s to be pushed down onto the {@link DuckDBPlatform}. + */ +public class DuckDBPlugin implements Plugin { + + @Override + public Collection getRequiredPlatforms() { + return Arrays.asList(DuckDBPlatform.getInstance(), JavaPlatform.getInstance()); + } + + @Override + public Collection getMappings() { + return Mappings.ALL; + } + + @Override + public Collection getChannelConversions() { + return ChannelConversions.ALL; + } + + @Override + public void setProperties(Configuration configuration) { + } +} diff --git a/wayang-platforms/wayang-duckdb/src/main/resources/wayang-duckdb-defaults.properties b/wayang-platforms/wayang-duckdb/src/main/resources/wayang-duckdb-defaults.properties new file mode 100644 index 000000000..5c8843f8d --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/main/resources/wayang-duckdb-defaults.properties @@ -0,0 +1,228 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Connection (override per deployment in wayang.properties). +# Use jdbc:duckdb: for a persistent database file. +wayang.duckdb.jdbc.url = jdbc:duckdb: +wayang.duckdb.jdbc.user = +wayang.duckdb.jdbc.password = +wayang.duckdb.jdbc.driverName = org.duckdb.DuckDBDriver + +# ParquetSource support. +# Use mappings to keep ParquetSource inputUrl as a canonical Parquet URI while +# resolving it to a DuckDB-visible SQL relation during execution. +# Format: parquet-uri=duckdb-relation;parquet-uri-2=duckdb-relation-2 +# wayang.duckdb.parquetsource.mappings = file:///data/orders.parquet=wayang_parquet.orders +# +# Optional auto-create support. When enabled, DuckDB creates a view over +# read_parquet('${uri}') before composing the SQL stage. +wayang.duckdb.parquetsource.auto-create = false +wayang.duckdb.parquetsource.auto-create.relation-prefix = wayang_parquet_ +wayang.duckdb.parquetsource.auto-create.template = CREATE OR REPLACE VIEW ${relation} AS SELECT * FROM read_parquet('${uri}') +# +# Optional prepare SQL executed before DuckDB creates the Parquet view. Keep +# statements idempotent: they can run during optimization cardinality estimation +# and during execution. This is the hook for cloud storage extensions/secrets. +# Environment and JVM system properties can be injected via ${env:NAME} and +# ${sys:name}. +# +# Example for private GCS Parquet files: +# wayang.duckdb.parquetsource.prepare-sql = INSTALL httpfs; LOAD httpfs; CREATE OR REPLACE SECRET wayang_gcs (TYPE gcs, KEY_ID '${env:GCS_HMAC_KEY_ID}', SECRET '${env:GCS_HMAC_SECRET}') + +# Hardware profile used by LoadProfileToTimeConverter. +wayang.duckdb.cpu.mhz = 2700 +wayang.duckdb.cores = 4 +wayang.duckdb.costs.fix = 0.0 +wayang.duckdb.costs.per-ms = 1.0 + +# Reference cost model learned from the Trino Week8-style DuckDB pilot +# (S01-S13, 10k/50k/100k/250k rows, 6 repetitions). +# Re-profile for deployment-specific hardware, data shape, and PRAGMA threads. + +wayang.duckdb.tablesource.load.template = {\ + "type":"mathex", "in":0, "out":1,\ + "cpu":"?*out0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.duckdb.tablesource.load = {\ + "type":"mathex",\ + "in":0,\ + "out":1,\ + "cpu":"((10.472865061115924)*(out0))+(14042.766808200337)",\ + "ram":"0",\ + "disk":"0",\ + "net":"0",\ + "p":0.9\ +} + +wayang.duckdb.parquetsource.load.template = {\ + "type":"mathex", "in":0, "out":1,\ + "cpu":"?*out0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.duckdb.parquetsource.load = {\ + "type":"mathex",\ + "in":0,\ + "out":1,\ + "cpu":"((10.472865061115924)*(out0))+(14042.766808200337)",\ + "ram":"0",\ + "disk":"0",\ + "net":"0",\ + "p":0.9\ +} + +wayang.duckdb.filter.load.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.duckdb.filter.load = {\ + "type":"mathex",\ + "in":1,\ + "out":1,\ + "cpu":"((55.53629275089796)*(in0))+(9844596.873788856)",\ + "ram":"0",\ + "disk":"0",\ + "net":"0",\ + "p":0.9\ +} + +wayang.duckdb.projection.load.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.duckdb.projection.load = {\ + "type":"mathex",\ + "in":1,\ + "out":1,\ + "cpu":"((24.619202590290122)*(in0))+(2184.9837993317874)",\ + "ram":"0",\ + "disk":"0",\ + "net":"0",\ + "p":0.9\ +} + +wayang.duckdb.join.load.template = {\ + "type":"mathex", "in":2, "out":1,\ + "cpu":"?*in0 + ?*in1 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.duckdb.join.load = {\ + "type":"mathex",\ + "in":2,\ + "out":1,\ + "cpu":"(((101.30552469545577)*(in0))+((30.896968724733654)*(in1)))+(53.481690928306364)",\ + "ram":"0",\ + "disk":"0",\ + "net":"0",\ + "p":0.9\ +} + +wayang.duckdb.globalreduce.load.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.duckdb.globalreduce.load = {\ + "type":"mathex",\ + "in":1,\ + "out":1,\ + "cpu":"((70.40718624423555)*(in0))+(8101.054898948163)",\ + "ram":"0",\ + "disk":"0",\ + "net":"0",\ + "p":0.9\ +} + +wayang.duckdb.reduceby.load.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.duckdb.reduceby.load = {\ + "type":"mathex",\ + "in":1,\ + "out":1,\ + "cpu":"((1.836076012140386)*(in0))+(20.74663077429906)",\ + "ram":"0",\ + "disk":"0",\ + "net":"0",\ + "p":0.9\ +} + +wayang.duckdb.sort.load.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.duckdb.sort.load = {\ + "type":"mathex",\ + "in":1,\ + "out":1,\ + "cpu":"((2622.8815961367363)*(in0))+(2537.744683180438)",\ + "ram":"0",\ + "disk":"0",\ + "net":"0",\ + "p":0.9\ +} + +wayang.duckdb.tablesink.load.template = {\ + "type":"mathex", "in":1, "out":0,\ + "cpu":"?*in0 + ?",\ + "ram":"0",\ + "p":0.9\ +} +wayang.duckdb.tablesink.load = {\ + "type":"mathex",\ + "in":1,\ + "out":0,\ + "cpu":"((2240.3138798129726)*(in0))+(0.7832474099914776)",\ + "ram":"0",\ + "disk":"0",\ + "net":"0",\ + "p":0.9\ +} + +wayang.duckdb.sqltostream.load.query.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*out0 + ?"\ +} +wayang.duckdb.sqltostream.load.query = {\ + "in":1, "out":1,\ + "cpu":"${20*out0 + 200000}",\ + "ram":"0",\ + "p":0.9\ +} +wayang.duckdb.sqltostream.load.output.template = {\ + "type":"mathex", "in":1, "out":1,\ + "cpu":"?*out0"\ +} +wayang.duckdb.sqltostream.load.output = {\ + "in":1, "out":1,\ + "cpu":"${20*out0}",\ + "ram":"0",\ + "p":0.9\ +} diff --git a/wayang-platforms/wayang-duckdb/src/test/java/org/apache/wayang/duckdb/DuckDBCostPilotIT.java b/wayang-platforms/wayang-duckdb/src/test/java/org/apache/wayang/duckdb/DuckDBCostPilotIT.java new file mode 100644 index 000000000..e9f7f8868 --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/test/java/org/apache/wayang/duckdb/DuckDBCostPilotIT.java @@ -0,0 +1,777 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + */ + +package org.apache.wayang.duckdb; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.data.Tuple2; +import org.apache.wayang.basic.function.ProjectionDescriptor; +import org.apache.wayang.basic.operators.FilterOperator; +import org.apache.wayang.basic.operators.GlobalReduceOperator; +import org.apache.wayang.basic.operators.JoinOperator; +import org.apache.wayang.basic.operators.MapOperator; +import org.apache.wayang.basic.operators.ReduceByOperator; +import org.apache.wayang.basic.operators.SortOperator; +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.basic.types.RecordType; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.WayangContext; +import org.apache.wayang.core.function.FunctionDescriptor; +import org.apache.wayang.core.function.PredicateDescriptor; +import org.apache.wayang.core.function.ReduceDescriptor; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.plan.wayangplan.WayangPlan; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.core.types.DataUnitType; +import org.apache.wayang.duckdb.operators.DuckDBProjectionOperator; +import org.apache.wayang.duckdb.operators.DuckDBTableSource; +import org.apache.wayang.duckdb.platform.DuckDBPlatform; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.io.BufferedWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Small DuckDB cost-profiling pilot. + */ +class DuckDBCostPilotIT { + + private static final String SCHEMA = "wayang_profile"; + private static final String CUSTOMERS_1K = SCHEMA + ".customers_1k"; + private static final int[] ROW_COUNTS = parseIntList(System.getProperty( + "duckdb.profile.rowCounts", + "10000,50000,100000,250000" + )); + private static final String[] COLUMNS = {"order_id", "customer_id", "region", "amount", "bucket"}; + private static final String[] JOIN_COLUMNS = { + "order_id", "customer_id", "region", "amount", "bucket", "cust_id", "tier" + }; + private static final String[] JOIN_ORDER_TIER_AMOUNT_COLUMNS = {"order_id", "tier", "amount"}; + private static final String[] JOIN_TIER_AMOUNT_COLUMNS = {"tier", "amount"}; + private static final String JOIN_FLATTEN_NAME = "DuckDB profile join flatten"; + private static final String JOIN_ORDER_TIER_AMOUNT_FLATTEN_NAME = "DuckDB profile join order tier amount flatten"; + private static final String JOIN_TIER_AMOUNT_FLATTEN_NAME = "DuckDB profile join tier amount flatten"; + private static final Path OUTPUT_DIR = Paths.get(System.getProperty( + "duckdb.profile.outputDir", + "target/cost-profiling/duckdb" + )); + private static final Path EXECUTIONS_PATH = OUTPUT_DIR.resolve("executions.json"); + private static final Path CARDINALITIES_PATH = OUTPUT_DIR.resolve("cardinalities.json"); + private static final Path MANIFEST_PATH = OUTPUT_DIR.resolve("manifest.csv"); + private static final List PLAN_IDS = Arrays.asList( + System.getProperty( + "duckdb.profile.plans", + "S01,S02,S03,S04,S05,S06,S07,S08,S09,S10,S11,S12,S13" + ).split(",") + ); + private static final int REPETITIONS = Integer.parseInt( + System.getProperty("duckdb.profile.repetitions", "6") + ); + private static final boolean RESET_OUTPUT = Boolean.parseBoolean( + System.getProperty("duckdb.profile.reset", "true") + ); + + private static Path databaseFile; + private static String jdbcUrl; + + @AfterEach + void cleanUpDatabaseFile() throws Exception { + if (databaseFile != null) { + Files.deleteIfExists(databaseFile); + Files.deleteIfExists(databaseFile.resolveSibling(databaseFile.getFileName() + ".wal")); + databaseFile = null; + } + } + + @Test + void runPilot() throws Exception { + openFreshDatabase(); + Files.createDirectories(OUTPUT_DIR); + initializeOutputFiles(); + + prepareTables(); + + for (int rowCount : ROW_COUNTS) { + for (String planId : PLAN_IDS) { + String normalizedPlanId = planId.trim(); + runPlan( + normalizedPlanId, + getOperatorChain(normalizedPlanId), + rowCount, + getExpectedRows(normalizedPlanId, rowCount) + ); + } + } + + assertTrue(Files.exists(EXECUTIONS_PATH), "execution log should be written"); + if (Files.exists(CARDINALITIES_PATH)) { + assertFalse(Files.readAllLines(CARDINALITIES_PATH).isEmpty(), "cardinality log should contain rows"); + } + assertTrue(Files.exists(MANIFEST_PATH), "manifest should be written"); + assertFalse(Files.readAllLines(MANIFEST_PATH).isEmpty(), "manifest should contain rows"); + } + + private void runPlan(String planId, String operatorChain, int rowCount, long expectedRows) throws Exception { + for (int repetition = 0; repetition < REPETITIONS; repetition++) { + boolean isWarmup = repetition == 0; + String runId = String.format("%s_%s_r%02d", planId, formatRows(rowCount), repetition); + String sourceTable = SCHEMA + ".orders_" + formatRows(rowCount); + String sinkTable = SCHEMA + ".sink_" + runId.toLowerCase(); + + dropTable(sinkTable); + WayangPlan plan = createPlan(planId, sourceTable, sinkTable); + wayangContext().execute(runId, plan); + + long actualRows = queryLong("SELECT count(*) FROM " + sinkTable); + assertEquals(expectedRows, actualRows, runId + " row count"); + appendManifest(runId, planId, operatorChain, rowCount, expectedRows, repetition, isWarmup, sinkTable); + dropTable(sinkTable); + } + } + + private WayangPlan createPlan(String planId, String sourceTable, String sinkTable) { + DuckDBTableSource source = new DuckDBTableSource(sourceTable, COLUMNS); + + if ("S01".equals(planId)) { + TableSink sink = new TableSink<>(new Properties(), "overwrite", sinkTable, COLUMNS); + source.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S02".equals(planId)) { + TableSink sink = new TableSink<>(new Properties(), "overwrite", sinkTable, COLUMNS); + FilterOperator filter = createAmerFilter(); + source.connectTo(0, filter, 0); + filter.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S03".equals(planId)) { + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, "order_id", "amount"); + MapOperator projection = createOrderAmountProjection(); + source.connectTo(0, projection, 0); + projection.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S04".equals(planId)) { + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, "order_id", "amount"); + FilterOperator filter = createAmerFilter(); + MapOperator projection = createOrderAmountProjection(); + source.connectTo(0, filter, 0); + filter.connectTo(0, projection, 0); + projection.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S05".equals(planId)) { + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, "total_amount"); + GlobalReduceOperator reduce = createGlobalAmountReduceOperator(); + source.connectTo(0, reduce, 0); + reduce.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S06".equals(planId)) { + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, "bucket", "total_amount"); + ReduceByOperator reduceBy = createBucketReduceByOperator(); + source.connectTo(0, reduceBy, 0); + reduceBy.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S07".equals(planId)) { + TableSink sink = new TableSink<>(new Properties(), "overwrite", sinkTable, COLUMNS); + SortOperator sort = createAmountSortOperator(3); + source.connectTo(0, sort, 0); + sort.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S08".equals(planId)) { + DuckDBTableSource customers = new DuckDBTableSource(CUSTOMERS_1K, "cust_id", "tier"); + JoinOperator join = createCustomerJoinOperator(sourceTable); + MapOperator, Record> flatten = createJoinFlattenOperator(); + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, JOIN_COLUMNS); + source.connectTo(0, join, 0); + customers.connectTo(0, join, 1); + join.connectTo(0, flatten, 0); + flatten.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S09".equals(planId)) { + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, "total_amount"); + FilterOperator filter = createAmerFilter(); + GlobalReduceOperator reduce = createGlobalAmountReduceOperator(); + source.connectTo(0, filter, 0); + filter.connectTo(0, reduce, 0); + reduce.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S10".equals(planId)) { + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, "bucket", "total_amount"); + FilterOperator filter = createAmerFilter(); + ReduceByOperator reduceBy = createBucketReduceByOperator(); + source.connectTo(0, filter, 0); + filter.connectTo(0, reduceBy, 0); + reduceBy.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S11".equals(planId)) { + TableSink sink = new TableSink<>(new Properties(), "overwrite", sinkTable, COLUMNS); + FilterOperator filter = createAmerFilter(); + SortOperator sort = createAmountSortOperator(3); + source.connectTo(0, filter, 0); + filter.connectTo(0, sort, 0); + sort.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S12".equals(planId)) { + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, "order_id", "amount"); + MapOperator projection = createOrderAmountProjection(); + SortOperator sort = createAmountSortOperator(1); + source.connectTo(0, projection, 0); + projection.connectTo(0, sort, 0); + sort.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S13".equals(planId)) { + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, "order_id", "amount"); + FilterOperator filter = createAmerFilter(); + MapOperator projection = createOrderAmountProjection(); + SortOperator sort = createAmountSortOperator(1); + source.connectTo(0, filter, 0); + filter.connectTo(0, projection, 0); + projection.connectTo(0, sort, 0); + sort.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S14".equals(planId)) { + DuckDBTableSource customers = new DuckDBTableSource(CUSTOMERS_1K, "cust_id", "tier"); + FilterOperator filter = createAmerFilter(); + JoinOperator join = createCustomerJoinOperator(sourceTable); + MapOperator, Record> flatten = createJoinFlattenOperator(); + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, JOIN_COLUMNS); + source.connectTo(0, filter, 0); + filter.connectTo(0, join, 0); + customers.connectTo(0, join, 1); + join.connectTo(0, flatten, 0); + flatten.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S15".equals(planId)) { + DuckDBTableSource customers = new DuckDBTableSource(CUSTOMERS_1K, "cust_id", "tier"); + JoinOperator join = createCustomerJoinOperator(sourceTable); + MapOperator, Record> flatten = createJoinOrderTierAmountFlattenOperator(); + SortOperator sort = createAmountSortOperator(2); + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, JOIN_ORDER_TIER_AMOUNT_COLUMNS); + source.connectTo(0, join, 0); + customers.connectTo(0, join, 1); + join.connectTo(0, flatten, 0); + flatten.connectTo(0, sort, 0); + sort.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + if ("S16".equals(planId)) { + DuckDBTableSource customers = new DuckDBTableSource(CUSTOMERS_1K, "cust_id", "tier"); + JoinOperator join = createCustomerJoinOperator(sourceTable); + MapOperator, Record> flatten = createJoinTierAmountFlattenOperator(); + ReduceByOperator reduceBy = createTierReduceByOperator(); + TableSink sink = new TableSink<>( + new Properties(), "overwrite", sinkTable, "tier", "total_amount"); + source.connectTo(0, join, 0); + customers.connectTo(0, join, 1); + join.connectTo(0, flatten, 0); + flatten.connectTo(0, reduceBy, 0); + reduceBy.connectTo(0, sink, 0); + return new WayangPlan(sink); + } + + throw new IllegalArgumentException("Unsupported pilot plan: " + planId); + } + + private static GlobalReduceOperator createGlobalAmountReduceOperator() { + return new GlobalReduceOperator<>( + new ReduceDescriptor<>((left, right) -> left, Record.class) + .withSqlImplementation("SUM(amount) AS total_amount"), + DataSetType.createDefault(Record.class)); + } + + private static ReduceByOperator createBucketReduceByOperator() { + return new ReduceByOperator<>( + new TransformationDescriptor<>( + record -> new Record(record.getField(4)), + Record.class, + Record.class + ).withSqlImplementation("bucket", "bucket"), + new ReduceDescriptor<>((left, right) -> left, Record.class) + .withSqlImplementation("SUM(amount) AS total_amount"), + DataSetType.createDefault(Record.class)); + } + + private static ReduceByOperator createTierReduceByOperator() { + return new ReduceByOperator<>( + new TransformationDescriptor<>( + record -> new Record(record.getField(0)), + Record.class, + Record.class + ).withSqlImplementation("tier", "tier"), + new ReduceDescriptor<>((left, right) -> left, Record.class) + .withSqlImplementation("SUM(amount) AS total_amount"), + DataSetType.createDefault(Record.class)); + } + + private static SortOperator createAmountSortOperator(int amountFieldIndex) { + return new SortOperator<>( + new TransformationDescriptor<>( + record -> new Record(record.getField(amountFieldIndex)), + Record.class, + Record.class + ).withSqlImplementation("amount", "ASC"), + DataSetType.createDefault(Record.class)); + } + + private static JoinOperator createCustomerJoinOperator(String sourceTable) { + return new JoinOperator<>( + new TransformationDescriptor<>( + record -> new Record(record.getField(1)), + Record.class, + Record.class + ).withSqlImplementation(sourceTable, "customer_id"), + new TransformationDescriptor<>( + record -> new Record(record.getField(0)), + Record.class, + Record.class + ).withSqlImplementation(CUSTOMERS_1K, "cust_id")); + } + + private static FilterOperator createAmerFilter() { + return new FilterOperator<>( + new PredicateDescriptor<>( + (Record record) -> "AMER".equals(record.getField(2)), + Record.class + ).withSqlImplementation("region = 'AMER'") + ); + } + + private static MapOperator createOrderAmountProjection() { + return new MapOperator<>( + ProjectionDescriptor.createForRecords( + new RecordType(COLUMNS), + "order_id", "amount"), + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class)); + } + + private static MapOperator, Record> createJoinFlattenOperator() { + return createJoinFlattenOperator(new JoinFlattenFunction(), JOIN_FLATTEN_NAME); + } + + private static MapOperator, Record> createJoinOrderTierAmountFlattenOperator() { + return createJoinFlattenOperator(new JoinOrderTierAmountFlattenFunction(), JOIN_ORDER_TIER_AMOUNT_FLATTEN_NAME); + } + + private static MapOperator, Record> createJoinTierAmountFlattenOperator() { + return createJoinFlattenOperator(new JoinTierAmountFlattenFunction(), JOIN_TIER_AMOUNT_FLATTEN_NAME); + } + + private static MapOperator, Record> createJoinFlattenOperator( + FunctionDescriptor.SerializableFunction, Record> function, + String name) { + MapOperator, Record> operator = new MapOperator<>( + new TransformationDescriptor<>( + function, + DataUnitType.createBasicUnchecked(Tuple2.class), + DataUnitType.createBasic(Record.class)), + DataSetType.createDefaultUnchecked(Tuple2.class), + DataSetType.createDefault(Record.class)); + operator.setName(name); + return operator; + } + + private WayangContext wayangContext() { + Configuration configuration = new Configuration(); + configuration.setProperty("wayang.duckdb.jdbc.url", jdbcUrl); + configuration.setProperty("wayang.duckdb.jdbc.user", ""); + configuration.setProperty("wayang.duckdb.jdbc.password", ""); + configuration.setProperty("wayang.core.log.enabled", "true"); + configuration.setProperty("wayang.core.explain.enabled", "false"); + configuration.setProperty("wayang.core.log.executions", EXECUTIONS_PATH.toString().replace('\\', '/')); + configuration.setProperty("wayang.core.log.cardinalities", CARDINALITIES_PATH.toString().replace('\\', '/')); + configuration.getMappingProvider().addAllToWhitelist( + Collections.singleton(new JoinFlattenMapping())); + return new WayangContext(configuration).withPlugin(DuckDB.plugin()); + } + + private static void prepareTables() throws Exception { + try (Connection connection = jdbc(); Statement statement = connection.createStatement()) { + statement.execute("CREATE SCHEMA IF NOT EXISTS " + SCHEMA); + for (int rowCount : ROW_COUNTS) { + String table = SCHEMA + ".orders_" + formatRows(rowCount); + statement.execute("DROP TABLE IF EXISTS " + table); + statement.execute("CREATE TABLE " + table + " AS " + + "SELECT " + + "CAST(n AS BIGINT) AS order_id, " + + "CAST(n % 1000 AS BIGINT) AS customer_id, " + + "CASE WHEN n % 2 = 0 THEN 'AMER' ELSE 'EMEA' END AS region, " + + "CAST(n % 10000 AS DOUBLE) AS amount, " + + "CAST(n % 100 AS BIGINT) AS bucket " + + "FROM range(1, " + (rowCount + 1) + ") AS t(n)"); + assertEquals(rowCount, queryLong("SELECT count(*) FROM " + table), table + " row count"); + assertEquals(rowCount / 2, queryLong("SELECT count(*) FROM " + table + " WHERE region = 'AMER'"), + table + " AMER row count"); + } + statement.execute("DROP TABLE IF EXISTS " + CUSTOMERS_1K); + statement.execute("CREATE TABLE " + CUSTOMERS_1K + " AS " + + "SELECT " + + "CAST(n - 1 AS BIGINT) AS cust_id, " + + "CASE WHEN n % 2 = 0 THEN 'GOLD' ELSE 'SILVER' END AS tier " + + "FROM range(1, 1001) AS t(n)"); + assertEquals(1000, queryLong("SELECT count(*) FROM " + CUSTOMERS_1K), CUSTOMERS_1K + " row count"); + } + } + + private static String formatRows(int rowCount) { + if (rowCount % 1000 == 0) { + return (rowCount / 1000) + "k"; + } + return String.valueOf(rowCount); + } + + private static void initializeOutputFiles() throws Exception { + if (RESET_OUTPUT) { + Files.deleteIfExists(EXECUTIONS_PATH); + Files.deleteIfExists(CARDINALITIES_PATH); + writeManifestHeader(); + } else if (!Files.exists(MANIFEST_PATH)) { + writeManifestHeader(); + } + } + + private static void writeManifestHeader() throws Exception { + try (BufferedWriter writer = Files.newBufferedWriter(MANIFEST_PATH, StandardCharsets.UTF_8)) { + writer.write("run_id,plan_id,operator_chain,input_rows_left,input_rows_right,expected_output_rows," + + "selectivity,repetition,is_warmup,sink_table,status,notes"); + writer.newLine(); + } + } + + private static void appendManifest( + String runId, + String planId, + String operatorChain, + int inputRows, + long expectedOutputRows, + int repetition, + boolean isWarmup, + String sinkTable) throws Exception { + try (BufferedWriter writer = Files.newBufferedWriter( + MANIFEST_PATH, + StandardCharsets.UTF_8, + java.nio.file.StandardOpenOption.APPEND)) { + writer.write(String.join(",", + runId, + planId, + operatorChain, + String.valueOf(inputRows), + hasJoin(planId) ? "1000" : "", + String.valueOf(expectedOutputRows), + hasFilter(planId) ? "0.5" : "1.0", + String.valueOf(repetition), + String.valueOf(isWarmup), + sinkTable, + "ok", + "")); + writer.newLine(); + } + } + + private static String getOperatorChain(String planId) { + switch (planId) { + case "S01": + return "TableSource->TableSink"; + case "S02": + return "TableSource->Filter(50%)->TableSink"; + case "S03": + return "TableSource->Projection->TableSink"; + case "S04": + return "TableSource->Filter(50%)->Projection->TableSink"; + case "S05": + return "TableSource->GlobalReduce->TableSink"; + case "S06": + return "TableSource->ReduceBy(bucket)->TableSink"; + case "S07": + return "TableSource->Sort(amount)->TableSink"; + case "S08": + return "Orders->Join(Customers 1k)->Projection->TableSink"; + case "S09": + return "TableSource->Filter(50%)->GlobalReduce->TableSink"; + case "S10": + return "TableSource->Filter(50%)->ReduceBy(bucket)->TableSink"; + case "S11": + return "TableSource->Filter(50%)->Sort(amount)->TableSink"; + case "S12": + return "TableSource->Projection(order_id,amount)->Sort(amount)->TableSink"; + case "S13": + return "TableSource->Filter(50%)->Projection(order_id,amount)->Sort(amount)->TableSink"; + case "S14": + return "Orders->Filter(50%)->Join(Customers 1k)->Projection->TableSink"; + case "S15": + return "Orders->Join(Customers 1k)->Projection(order_id,tier,amount)->Sort(amount)->TableSink"; + case "S16": + return "Orders->Join(Customers 1k)->Projection(tier,amount)->ReduceBy(tier)->TableSink"; + default: + throw new IllegalArgumentException("Unsupported pilot plan: " + planId); + } + } + + private static long getExpectedRows(String planId, int rowCount) { + if ("S05".equals(planId) || "S09".equals(planId)) { + return 1; + } + if ("S06".equals(planId)) { + return 100; + } + if ("S10".equals(planId)) { + return 50; + } + if ("S16".equals(planId)) { + return 2; + } + return hasFilter(planId) ? rowCount / 2 : rowCount; + } + + private static boolean hasFilter(String planId) { + return "S02".equals(planId) + || "S04".equals(planId) + || "S09".equals(planId) + || "S10".equals(planId) + || "S11".equals(planId) + || "S13".equals(planId) + || "S14".equals(planId); + } + + private static boolean hasJoin(String planId) { + return "S08".equals(planId) + || "S14".equals(planId) + || "S15".equals(planId) + || "S16".equals(planId); + } + + private static int[] parseIntList(String value) { + return Arrays.stream(value.split(",")) + .map(String::trim) + .filter(token -> !token.isEmpty()) + .mapToInt(Integer::parseInt) + .toArray(); + } + + private static long queryLong(String sql) throws Exception { + try (Connection connection = jdbc(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) { + resultSet.next(); + return resultSet.getLong(1); + } + } + + private static void dropTable(String table) throws Exception { + try (Connection connection = jdbc(); Statement statement = connection.createStatement()) { + statement.execute("DROP TABLE IF EXISTS " + table); + } + } + + private static Connection jdbc() throws Exception { + return DriverManager.getConnection(jdbcUrl); + } + + private static void openFreshDatabase() throws Exception { + databaseFile = Files.createTempFile("wayang-duckdb-cost-", ".duckdb"); + Files.deleteIfExists(databaseFile); + jdbcUrl = "jdbc:duckdb:" + databaseFile.toAbsolutePath(); + } + + private static Record flattenJoinResult(Object joinResult) { + if (joinResult instanceof Record) { + return (Record) joinResult; + } + Tuple2 pair = (Tuple2) joinResult; + Record left = (Record) pair.field0; + Record right = (Record) pair.field1; + return new Record( + left.getField(0), + left.getField(1), + left.getField(2), + left.getField(3), + left.getField(4), + right.getField(0), + right.getField(1)); + } + + private static Record flattenJoinOrderTierAmountResult(Object joinResult) { + if (joinResult instanceof Record) { + Record record = (Record) joinResult; + return new Record(record.getField(0), record.getField(6), record.getField(3)); + } + Tuple2 pair = (Tuple2) joinResult; + Record left = (Record) pair.field0; + Record right = (Record) pair.field1; + return new Record(left.getField(0), right.getField(1), left.getField(3)); + } + + private static Record flattenJoinTierAmountResult(Object joinResult) { + if (joinResult instanceof Record) { + Record record = (Record) joinResult; + return new Record(record.getField(6), record.getField(3)); + } + Tuple2 pair = (Tuple2) joinResult; + Record left = (Record) pair.field0; + Record right = (Record) pair.field1; + return new Record(right.getField(1), left.getField(3)); + } + + private static final class JoinFlattenFunction implements + FunctionDescriptor.SerializableFunction, Record> { + + @Override + public Record apply(Tuple2 tuple) { + return flattenJoinResult(tuple); + } + } + + private static final class JoinOrderTierAmountFlattenFunction implements + FunctionDescriptor.SerializableFunction, Record> { + + @Override + public Record apply(Tuple2 tuple) { + return flattenJoinOrderTierAmountResult(tuple); + } + } + + private static final class JoinTierAmountFlattenFunction implements + FunctionDescriptor.SerializableFunction, Record> { + + @Override + public Record apply(Tuple2 tuple) { + return flattenJoinTierAmountResult(tuple); + } + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static final class JoinFlattenMapping implements Mapping { + + @Override + public java.util.Collection getTransformations() { + OperatorPattern pattern = new OperatorPattern( + "joinFlatten", + new MapOperator(null, DataSetType.none(), DataSetType.createDefault(Record.class)), + false) + .withAdditionalTest(operator -> isJoinFlattenName(((MapOperator) operator).getName())); + + ReplacementSubplanFactory factory = new ReplacementSubplanFactory.OfSingleOperators( + (matchedOperator, epoch) -> createDuckDBProjection(matchedOperator.getName()).at(epoch)); + + return Collections.singleton(new PlanTransformation( + SubplanPattern.createSingleton(pattern), + factory, + DuckDBPlatform.getInstance())); + } + + private static DuckDBProjectionOperator createDuckDBProjection(String operatorName) { + ProjectionDescriptor, Record> descriptor = new ProjectionDescriptor<>( + getJoinFlattenFunction(operatorName), + Arrays.asList(getJoinFlattenColumns(operatorName)), + DataUnitType.createBasicUnchecked(Tuple2.class), + DataUnitType.createBasic(Record.class)); + MapOperator, Record> projection = new MapOperator<>( + descriptor, + DataSetType.createDefaultUnchecked(Tuple2.class), + DataSetType.createDefault(Record.class)); + projection.setName(operatorName); + return new DuckDBProjectionOperator((MapOperator) (MapOperator) projection); + } + + private static boolean isJoinFlattenName(String operatorName) { + return JOIN_FLATTEN_NAME.equals(operatorName) + || JOIN_ORDER_TIER_AMOUNT_FLATTEN_NAME.equals(operatorName) + || JOIN_TIER_AMOUNT_FLATTEN_NAME.equals(operatorName); + } + + private static String[] getJoinFlattenColumns(String operatorName) { + if (JOIN_ORDER_TIER_AMOUNT_FLATTEN_NAME.equals(operatorName)) { + return JOIN_ORDER_TIER_AMOUNT_COLUMNS; + } + if (JOIN_TIER_AMOUNT_FLATTEN_NAME.equals(operatorName)) { + return JOIN_TIER_AMOUNT_COLUMNS; + } + return JOIN_COLUMNS; + } + + private static FunctionDescriptor.SerializableFunction, Record> getJoinFlattenFunction( + String operatorName) { + if (JOIN_ORDER_TIER_AMOUNT_FLATTEN_NAME.equals(operatorName)) { + return new JoinOrderTierAmountFlattenFunction(); + } + if (JOIN_TIER_AMOUNT_FLATTEN_NAME.equals(operatorName)) { + return new JoinTierAmountFlattenFunction(); + } + return new JoinFlattenFunction(); + } + } +} diff --git a/wayang-platforms/wayang-duckdb/src/test/java/org/apache/wayang/duckdb/DuckDBOperatorsIT.java b/wayang-platforms/wayang-duckdb/src/test/java/org/apache/wayang/duckdb/DuckDBOperatorsIT.java new file mode 100644 index 000000000..9483fe0b7 --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/test/java/org/apache/wayang/duckdb/DuckDBOperatorsIT.java @@ -0,0 +1,636 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb; + +import org.apache.wayang.api.DataQuantaBuilder; +import org.apache.wayang.api.JavaPlanBuilder; +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.data.Tuple2; +import org.apache.wayang.basic.function.ProjectionDescriptor; +import org.apache.wayang.basic.operators.FilterOperator; +import org.apache.wayang.basic.operators.GlobalReduceOperator; +import org.apache.wayang.basic.operators.JoinOperator; +import org.apache.wayang.basic.operators.MapOperator; +import org.apache.wayang.basic.operators.ReduceByOperator; +import org.apache.wayang.basic.operators.SortOperator; +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.basic.types.RecordType; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.WayangContext; +import org.apache.wayang.core.function.FunctionDescriptor; +import org.apache.wayang.core.function.PredicateDescriptor; +import org.apache.wayang.core.function.ReduceDescriptor; +import org.apache.wayang.core.function.TransformationDescriptor; +import org.apache.wayang.core.mapping.Mapping; +import org.apache.wayang.core.mapping.OperatorPattern; +import org.apache.wayang.core.mapping.PlanTransformation; +import org.apache.wayang.core.mapping.ReplacementSubplanFactory; +import org.apache.wayang.core.mapping.SubplanPattern; +import org.apache.wayang.core.plan.wayangplan.WayangPlan; +import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.core.types.DataUnitType; +import org.apache.wayang.duckdb.operators.DuckDBProjectionOperator; +import org.apache.wayang.duckdb.operators.DuckDBTableSource; +import org.apache.wayang.duckdb.platform.DuckDBPlatform; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Embedded end-to-end tests for every operator the DuckDB platform implements. + * + *

Coverage mirrors {@code TrinoOperatorsIT}: {@code TableSource}, + * {@code Filter}, {@code Projection}, {@code Join}, {@code GlobalReduce}, + * {@code ReduceBy}, {@code Sort}, and {@code TableSink}, plus five + * JavaPlanBuilder combinations. Each Wayang plan registers only + * {@link DuckDB#plugin()} and ends in a DuckDB table sink, so no Java-side + * operator implementation is needed to compute the result. + */ +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +class DuckDBOperatorsIT { + + private static final String SCHEMA = "wayang_it"; + private static final String ORDERS = SCHEMA + ".orders"; + private static final String CUSTOMERS = SCHEMA + ".customers"; + private static final String SINK_TABLE_NAME = "operator_result"; + private static final String SINK_TABLE = SCHEMA + "." + SINK_TABLE_NAME; + private static final String[] JOIN_COLUMNS = { + "order_id", "customer_id", "region", "amount", "cust_id", "name", "tier" + }; + private static final String JOIN_FLATTEN_NAME = "DuckDB test-only join flatten"; + + private static Path databaseFile; + private static String jdbcUrl; + private static boolean deleteDatabaseFile; + + @BeforeAll + static void setUp() throws Exception { + jdbcUrl = System.getProperty( + "duckdb.url", + System.getenv().getOrDefault("DUCKDB_JDBC_URL", "")); + if (jdbcUrl.isEmpty()) { + databaseFile = Files.createTempFile("wayang-duckdb-operators-", ".duckdb"); + Files.deleteIfExists(databaseFile); + jdbcUrl = "jdbc:duckdb:" + databaseFile.toAbsolutePath(); + deleteDatabaseFile = true; + } else { + jdbcUrl = normalizeDuckDbUrl(jdbcUrl); + } + + try (Connection connection = jdbc(); Statement statement = connection.createStatement()) { + statement.execute("CREATE SCHEMA IF NOT EXISTS " + SCHEMA); + statement.execute("DROP TABLE IF EXISTS " + SINK_TABLE); + statement.execute("DROP TABLE IF EXISTS " + ORDERS); + statement.execute("DROP TABLE IF EXISTS " + CUSTOMERS); + statement.execute("CREATE TABLE " + ORDERS + " (" + + "order_id BIGINT, customer_id BIGINT, region VARCHAR, amount DOUBLE)"); + statement.execute("INSERT INTO " + ORDERS + " VALUES " + + "(1, 100, 'AMER', 2200.0)," + + "(2, 101, 'EMEA', 800.5)," + + "(3, 100, 'AMER', 680.5)," + + "(4, 102, 'APAC', 1500.0)," + + "(5, 101, 'EMEA', 1100.0)," + + "(6, 100, 'AMER', 950.25)"); + + statement.execute("CREATE TABLE " + CUSTOMERS + " (" + + "cust_id BIGINT, name VARCHAR, tier VARCHAR)"); + statement.execute("INSERT INTO " + CUSTOMERS + " VALUES " + + "(100, 'Acme', 'GOLD')," + + "(101, 'Globex', 'SILVER')," + + "(102, 'Initech','BRONZE')"); + } + } + + @AfterAll + static void tearDown() throws Exception { + if (deleteDatabaseFile && databaseFile != null) { + Files.deleteIfExists(databaseFile); + Files.deleteIfExists(databaseFile.resolveSibling(databaseFile.getFileName() + ".wal")); + } + } + + @Test + @Order(1) + void loadsDuckDbDriverAndRunsQuery() throws Exception { + Class.forName(DuckDBPlatform.getInstance().getJdbcDriverClassName()); + + try (Connection connection = DriverManager.getConnection("jdbc:duckdb:"); + ResultSet resultSet = connection.createStatement().executeQuery("SELECT 1")) { + resultSet.next(); + assertEquals(1, resultSet.getInt(1)); + } + } + + @Test + @Order(2) + void tableSource() { + DuckDBTableSource src = new DuckDBTableSource(ORDERS, "order_id", "customer_id", "region", "amount"); + TableSink sink = tableSink("order_id", "customer_id", "region", "amount"); + src.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + + assertEquals(6, queryLong("SELECT count(*) FROM " + SINK_TABLE)); + } + + @Test + @Order(3) + void filter() { + DuckDBTableSource src = new DuckDBTableSource(ORDERS, "order_id", "customer_id", "region", "amount"); + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>( + (Record record) -> "AMER".equals(record.getField(2)), Record.class) + .withSqlImplementation("region = 'AMER'")); + TableSink sink = tableSink("order_id", "customer_id", "region", "amount"); + src.connectTo(0, filter, 0); + filter.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + + assertEquals(3, queryLong("SELECT count(*) FROM " + SINK_TABLE)); + assertEquals(0, queryLong("SELECT SUM(CASE WHEN region <> 'AMER' THEN 1 ELSE 0 END) FROM " + SINK_TABLE)); + } + + @Test + @Order(4) + void projection() { + DuckDBTableSource src = new DuckDBTableSource(ORDERS, "order_id", "customer_id", "region", "amount"); + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>( + (Record record) -> "AMER".equals(record.getField(2)), Record.class) + .withSqlImplementation("region = 'AMER'")); + MapOperator projection = new MapOperator<>( + ProjectionDescriptor.createForRecords( + new RecordType("order_id", "customer_id", "region", "amount"), + "region", "amount"), + DataSetType.createDefault(Record.class), + DataSetType.createDefault(Record.class)); + TableSink sink = tableSink("region", "amount"); + src.connectTo(0, filter, 0); + filter.connectTo(0, projection, 0); + projection.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + + assertEquals(3, queryLong("SELECT count(*) FROM " + SINK_TABLE)); + assertEquals(2, queryLong( + "SELECT count(*) FROM information_schema.columns " + + "WHERE table_schema = '" + SCHEMA + "' AND table_name = '" + SINK_TABLE_NAME + "'")); + } + + @Test + @Order(5) + void join() { + DuckDBTableSource orders = new DuckDBTableSource( + ORDERS, "order_id", "customer_id", "region", "amount"); + DuckDBTableSource customers = new DuckDBTableSource( + CUSTOMERS, "cust_id", "name", "tier"); + JoinOperator join = new JoinOperator<>( + new TransformationDescriptor<>( + (Record record) -> new Record(record.getField(1)), Record.class, Record.class) + .withSqlImplementation(ORDERS, "customer_id"), + new TransformationDescriptor<>( + (Record record) -> new Record(record.getField(0)), Record.class, Record.class) + .withSqlImplementation(CUSTOMERS, "cust_id")); + MapOperator, Record> flatten = joinFlattenOperator(); + TableSink sink = tableSink(JOIN_COLUMNS); + orders.connectTo(0, join, 0); + customers.connectTo(0, join, 1); + join.connectTo(0, flatten, 0); + flatten.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + + assertEquals(6, queryLong("SELECT count(*) FROM " + SINK_TABLE)); + assertEquals(0, queryLong("SELECT SUM(CASE WHEN customer_id <> cust_id THEN 1 ELSE 0 END) FROM " + + SINK_TABLE)); + } + + @Test + @Order(6) + void globalReduce() { + DuckDBTableSource src = new DuckDBTableSource(ORDERS, "order_id", "customer_id", "region", "amount"); + GlobalReduceOperator reduce = new GlobalReduceOperator<>( + new ReduceDescriptor<>((left, right) -> left, Record.class) + .withSqlImplementation("SUM(amount) AS total_amount"), + DataSetType.createDefault(Record.class)); + TableSink sink = tableSink("total_amount"); + src.connectTo(0, reduce, 0); + reduce.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + + assertSingleDoubleResult(7231.25); + } + + @Test + @Order(7) + void reduceBy() { + DuckDBTableSource src = new DuckDBTableSource(ORDERS, "order_id", "customer_id", "region", "amount"); + ReduceByOperator reduceBy = new ReduceByOperator<>( + new TransformationDescriptor<>( + (Record record) -> new Record(record.getField(2)), Record.class, Record.class) + .withSqlImplementation("region", "region"), + new ReduceDescriptor<>((left, right) -> left, Record.class) + .withSqlImplementation("SUM(amount) AS total_amount"), + DataSetType.createDefault(Record.class)); + TableSink sink = tableSink("region", "total_amount"); + src.connectTo(0, reduceBy, 0); + reduceBy.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + + Map sums = readRegionSums(); + assertEquals(3, sums.size()); + assertEquals(3830.75, sums.get("AMER"), 0.01); + assertEquals(1900.5, sums.get("EMEA"), 0.01); + assertEquals(1500.0, sums.get("APAC"), 0.01); + } + + @Test + @Order(8) + void sort() { + DuckDBTableSource src = new DuckDBTableSource(ORDERS, "order_id", "customer_id", "region", "amount"); + SortOperator sort = new SortOperator<>( + new TransformationDescriptor<>( + (Record record) -> new Record(record.getField(3)), Record.class, Record.class) + .withSqlImplementation("amount", "ASC"), + DataSetType.createDefault(Record.class)); + TableSink sink = tableSink("order_id", "customer_id", "region", "amount"); + src.connectTo(0, sort, 0); + sort.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + + assertEquals(6, queryLong("SELECT count(*) FROM " + SINK_TABLE)); + assertEquals(680.5, queryDouble("SELECT min(amount) FROM " + SINK_TABLE), 0.001); + assertEquals(2200.0, queryDouble("SELECT max(amount) FROM " + SINK_TABLE), 0.001); + } + + @Test + @Order(9) + void tableSink() { + DuckDBTableSource src = new DuckDBTableSource(ORDERS, "order_id", "customer_id", "region", "amount"); + FilterOperator filter = new FilterOperator<>( + new PredicateDescriptor<>( + (Record record) -> "AMER".equals(record.getField(2)), Record.class) + .withSqlImplementation("region = 'AMER'")); + TableSink sink = new TableSink<>( + new Properties(), "overwrite", SINK_TABLE, + "order_id", "customer_id", "region", "amount"); + src.connectTo(0, filter, 0); + filter.connectTo(0, sink, 0); + + wayangContext().execute(new WayangPlan(sink)); + + assertEquals(3, queryLong("SELECT count(*) FROM " + SINK_TABLE)); + assertEquals(0, queryLong("SELECT SUM(CASE WHEN region <> 'AMER' THEN 1 ELSE 0 END) FROM " + SINK_TABLE)); + } + + @Test + @Order(10) + void javaPlanBuilderReadTableFilterProjection() { + new JavaPlanBuilder( + wayangContext(), "DuckDB JavaPlanBuilder readTable integration test") + .readTable(new DuckDBTableSource( + ORDERS, "order_id", "customer_id", "region", "amount")) + .filter(record -> "AMER".equals(record.getField(2))) + .withSqlUdf("region = 'AMER'") + .asRecords() + .projectRecords(new String[]{"order_id", "amount"}) + .writeTable(SINK_TABLE, "overwrite", new String[]{"order_id", "amount"}, new Properties()); + + assertEquals(3, queryLong("SELECT count(*) FROM " + SINK_TABLE)); + assertEquals(0, queryLong( + "SELECT SUM(CASE WHEN amount NOT IN (2200.0, 680.5, 950.25) THEN 1 ELSE 0 END) FROM " + + SINK_TABLE)); + } + + @Test + @Order(11) + void javaPlanBuilderReadTableFilterGlobalReduce() { + new JavaPlanBuilder( + wayangContext(), "DuckDB JavaPlanBuilder global reduce integration test") + .readTable(new DuckDBTableSource( + ORDERS, "order_id", "customer_id", "region", "amount")) + .filter(record -> "AMER".equals(record.getField(2))) + .withSqlUdf("region = 'AMER'") + .reduce((left, right) -> left) + .withSqlUdf("SUM(amount) AS total_amount") + .writeTable(SINK_TABLE, "overwrite", new String[]{"total_amount"}, new Properties()); + + assertSingleDoubleResult(3830.75); + } + + @Test + @Order(12) + void javaPlanBuilderReadTableReduceBySort() { + new JavaPlanBuilder( + wayangContext(), "DuckDB JavaPlanBuilder reduce-by and sort integration test") + .readTable(new DuckDBTableSource( + ORDERS, "order_id", "customer_id", "region", "amount")) + .reduceByKey( + record -> new Record(record.getField(2)), + (left, right) -> left) + .withSqlUdfs("region", "SUM(amount) AS total_amount") + .sort(record -> new Record(record.getField(0))) + .withSqlUdf("region", "ASC") + .writeTable(SINK_TABLE, "overwrite", new String[]{"region", "total_amount"}, new Properties()); + + assertEquals("AMER,APAC,EMEA", queryString( + "SELECT string_agg(region, ',' ORDER BY region) FROM " + SINK_TABLE)); + } + + @Test + @Order(13) + void javaPlanBuilderReadTableFilterProjectionTableSink() { + new JavaPlanBuilder(wayangContext(), "DuckDB JavaPlanBuilder table sink integration test") + .readTable(new DuckDBTableSource( + ORDERS, "order_id", "customer_id", "region", "amount")) + .filter(record -> "AMER".equals(record.getField(2))) + .withSqlUdf("region = 'AMER'") + .asRecords() + .projectRecords(new String[]{"order_id", "amount"}) + .writeTable( + SINK_TABLE, + "overwrite", + new String[]{"order_id", "amount"}, + new Properties()); + + assertEquals(3, queryLong("SELECT count(*) FROM " + SINK_TABLE)); + assertEquals(2, queryLong( + "SELECT count(*) FROM information_schema.columns " + + "WHERE table_schema = '" + SCHEMA + "' AND table_name = '" + SINK_TABLE_NAME + "'")); + } + + @Test + @Order(14) + void javaPlanBuilderReadTableJoin() { + JavaPlanBuilder plan = new JavaPlanBuilder( + wayangContext(), "DuckDB JavaPlanBuilder join integration test"); + DataQuantaBuilder orders = plan.readTable(new DuckDBTableSource( + ORDERS, "order_id", "customer_id", "region", "amount")); + DataQuantaBuilder customers = plan.readTable(new DuckDBTableSource( + CUSTOMERS, "cust_id", "name", "tier")); + + orders + .join( + record -> new Record(record.getField(1)), + customers, + record -> new Record(record.getField(0))) + .withSqlUdfs(ORDERS, "customer_id", CUSTOMERS, "cust_id") + .map(new JoinFlattenFunction()) + .withName(JOIN_FLATTEN_NAME) + .writeTable(SINK_TABLE, "overwrite", JOIN_COLUMNS, new Properties()); + + assertEquals(6, queryLong("SELECT count(*) FROM " + SINK_TABLE)); + assertEquals(0, queryLong("SELECT SUM(CASE WHEN customer_id <> cust_id THEN 1 ELSE 0 END) FROM " + + SINK_TABLE)); + } + + @Test + @Order(15) + void generatedSqlContainsPushdownShapes() { + String filterSql = captureStdout(this::filter); + assertTrue(filterSql.contains("WHERE region = 'AMER'")); + + String projectionSql = captureStdout(this::projection); + assertTrue(projectionSql.contains("SELECT region, amount FROM " + ORDERS)); + assertTrue(projectionSql.contains("WHERE region = 'AMER'")); + + String joinSql = captureStdout(this::join); + assertTrue(joinSql.contains("JOIN " + CUSTOMERS)); + assertTrue(joinSql.contains(CUSTOMERS + ".cust_id=" + ORDERS + ".customer_id")); + + String reduceBySql = captureStdout(this::reduceBy); + assertTrue(reduceBySql.contains("GROUP BY region")); + + String sortSql = captureStdout(this::sort); + assertTrue(sortSql.contains("ORDER BY amount ASC")); + } + + private WayangContext wayangContext() { + Configuration config = new Configuration(); + config.setProperty("wayang.duckdb.jdbc.url", jdbcUrl); + config.setProperty("wayang.duckdb.jdbc.user", ""); + config.setProperty("wayang.duckdb.jdbc.password", ""); + config.getMappingProvider().addAllToWhitelist( + Collections.singleton(new JoinFlattenMapping())); + return new WayangContext(config) + .withPlugin(DuckDB.plugin()); + } + + private TableSink tableSink(String... columnNames) { + return new TableSink<>(new Properties(), "overwrite", SINK_TABLE, columnNames); + } + + private static MapOperator, Record> joinFlattenOperator() { + MapOperator, Record> operator = new MapOperator<>( + new TransformationDescriptor<>( + new JoinFlattenFunction(), + DataUnitType.createBasicUnchecked(Tuple2.class), + DataUnitType.createBasic(Record.class)), + DataSetType.createDefaultUnchecked(Tuple2.class), + DataSetType.createDefault(Record.class)); + operator.setName(JOIN_FLATTEN_NAME); + return operator; + } + + private static Record flattenJoinResult(Object joinResult) { + if (joinResult instanceof Record) { + return (Record) joinResult; + } + Tuple2 pair = (Tuple2) joinResult; + Record left = (Record) pair.field0; + Record right = (Record) pair.field1; + return new Record( + left.getField(0), + left.getField(1), + left.getField(2), + left.getField(3), + right.getField(0), + right.getField(1), + right.getField(2)); + } + + private long queryLong(String sql) { + try (Connection connection = jdbc(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) { + resultSet.next(); + return resultSet.getLong(1); + } catch (Exception e) { + throw new RuntimeException("query failed: " + sql, e); + } + } + + private double queryDouble(String sql) { + try (Connection connection = jdbc(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) { + resultSet.next(); + return resultSet.getDouble(1); + } catch (Exception e) { + throw new RuntimeException("query failed: " + sql, e); + } + } + + private String queryString(String sql) { + try (Connection connection = jdbc(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) { + resultSet.next(); + return resultSet.getString(1); + } catch (Exception e) { + throw new RuntimeException("query failed: " + sql, e); + } + } + + private static String captureStdout(Runnable runnable) { + PrintStream originalOut = System.out; + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + try (PrintStream capture = new PrintStream(buffer, true, StandardCharsets.UTF_8)) { + System.setOut(capture); + runnable.run(); + } finally { + System.setOut(originalOut); + } + return buffer.toString(StandardCharsets.UTF_8); + } + + private void assertSingleDoubleResult(double expected) { + try (Connection connection = jdbc(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("SELECT * FROM " + SINK_TABLE)) { + assertTrue(resultSet.next()); + assertEquals(expected, resultSet.getDouble(1), 0.01); + assertFalse(resultSet.next()); + } catch (Exception e) { + throw new RuntimeException("query failed: SELECT * FROM " + SINK_TABLE, e); + } + } + + private Map readRegionSums() { + Map sums = new HashMap<>(); + try (Connection connection = jdbc(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("SELECT * FROM " + SINK_TABLE)) { + while (resultSet.next()) { + sums.put(resultSet.getString(1), resultSet.getDouble(2)); + } + return sums; + } catch (Exception e) { + throw new RuntimeException("query failed: SELECT * FROM " + SINK_TABLE, e); + } + } + + private static final class JoinFlattenFunction implements + FunctionDescriptor.SerializableFunction, Record> { + + @Override + public Record apply(Tuple2 tuple) { + return flattenJoinResult(tuple); + } + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static final class JoinFlattenMapping implements Mapping { + + @Override + public java.util.Collection getTransformations() { + OperatorPattern pattern = new OperatorPattern( + "joinFlatten", + new MapOperator(null, DataSetType.none(), DataSetType.createDefault(Record.class)), + false) + .withAdditionalTest(operator -> JOIN_FLATTEN_NAME.equals(((MapOperator) operator).getName())); + + ReplacementSubplanFactory factory = new ReplacementSubplanFactory.OfSingleOperators( + (matchedOperator, epoch) -> createDuckDBProjection().at(epoch)); + + return Collections.singleton(new PlanTransformation( + SubplanPattern.createSingleton(pattern), + factory, + DuckDBPlatform.getInstance())); + } + + private static DuckDBProjectionOperator createDuckDBProjection() { + ProjectionDescriptor, Record> descriptor = new ProjectionDescriptor<>( + new JoinFlattenFunction(), + Arrays.asList(JOIN_COLUMNS), + DataUnitType.createBasicUnchecked(Tuple2.class), + DataUnitType.createBasic(Record.class)); + MapOperator, Record> projection = new MapOperator<>( + descriptor, + DataSetType.createDefaultUnchecked(Tuple2.class), + DataSetType.createDefault(Record.class)); + projection.setName(JOIN_FLATTEN_NAME); + return new DuckDBProjectionOperator((MapOperator) (MapOperator) projection); + } + } + + private static Connection jdbc() throws Exception { + return DriverManager.getConnection(jdbcUrl); + } + + private static String normalizeDuckDbUrl(String url) { + String prefix = "jdbc:duckdb:"; + if (!url.startsWith(prefix) || url.length() == prefix.length()) { + return url; + } + + String databasePath = url.substring(prefix.length()); + Path path = Path.of(databasePath); + if (path.isAbsolute()) { + return url; + } + + Path current = Path.of("").toAbsolutePath(); + while (current != null) { + Path candidate = current.resolve(path).normalize(); + if (Files.exists(candidate)) { + return prefix + candidate; + } + current = current.getParent(); + } + + return prefix + Path.of("").toAbsolutePath().resolve(path).normalize(); + } +} diff --git a/wayang-platforms/wayang-duckdb/src/test/java/org/apache/wayang/duckdb/DuckDBParquetSourceIT.java b/wayang-platforms/wayang-duckdb/src/test/java/org/apache/wayang/duckdb/DuckDBParquetSourceIT.java new file mode 100644 index 000000000..0ff93d36a --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/test/java/org/apache/wayang/duckdb/DuckDBParquetSourceIT.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.duckdb; + +import org.apache.wayang.basic.data.Record; +import org.apache.wayang.basic.operators.TableSink; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.WayangContext; +import org.apache.wayang.core.plan.wayangplan.WayangPlan; +import org.apache.wayang.duckdb.operators.DuckDBParquetSource; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Integration tests for {@link DuckDBParquetSource}. + */ +class DuckDBParquetSourceIT { + + private static final String SCHEMA = "wayang_parquet_it"; + private static final String SINK_TABLE = SCHEMA + ".orders_parquet_copy"; + private static final String MAPPED_SINK_TABLE = SCHEMA + ".orders_mapped_copy"; + private static final String GCS_SINK_TABLE = SCHEMA + ".orders_gcs_copy"; + private static final String SOURCE_VIEW = SCHEMA + ".orders_parquet"; + private static final String[] COLUMNS = {"order_id", "region", "amount"}; + private static final String DEFAULT_GCS_URI = + "gs://anaconda-public-data/nyc-taxi/nyc.parquet/part.0.parquet"; + + private static Path databaseFile; + private static Path parquetFile; + private static String jdbcUrl; + + @BeforeAll + static void setUp() throws Exception { + databaseFile = Files.createTempFile("wayang-duckdb-parquet-", ".duckdb"); + parquetFile = Files.createTempFile("wayang-duckdb-orders-", ".parquet"); + Files.deleteIfExists(databaseFile); + Files.deleteIfExists(parquetFile); + jdbcUrl = "jdbc:duckdb:" + databaseFile.toAbsolutePath(); + + try (Connection connection = jdbc(); Statement statement = connection.createStatement()) { + statement.execute("CREATE SCHEMA IF NOT EXISTS " + SCHEMA); + statement.execute("DROP TABLE IF EXISTS " + SINK_TABLE); + statement.execute("DROP TABLE IF EXISTS " + MAPPED_SINK_TABLE); + statement.execute("DROP TABLE IF EXISTS " + GCS_SINK_TABLE); + statement.execute("DROP VIEW IF EXISTS " + SOURCE_VIEW); + statement.execute("COPY (" + + "SELECT * FROM (VALUES " + + "(CAST(1 AS BIGINT), 'AMER', CAST(10.0 AS DOUBLE)), " + + "(CAST(2 AS BIGINT), 'EMEA', CAST(20.0 AS DOUBLE)), " + + "(CAST(3 AS BIGINT), 'APAC', CAST(30.0 AS DOUBLE))" + + ") AS t(order_id, region, amount)" + + ") TO '" + parquetUri() + "' (FORMAT PARQUET)"); + statement.execute("CREATE VIEW " + SOURCE_VIEW + " AS SELECT * FROM read_parquet('" + + parquetUri() + "')"); + } + } + + @AfterAll + static void tearDown() throws Exception { + if (databaseFile != null) { + Files.deleteIfExists(databaseFile); + Files.deleteIfExists(databaseFile.resolveSibling(databaseFile.getFileName() + ".wal")); + } + if (parquetFile != null) { + Files.deleteIfExists(parquetFile); + } + } + + @Test + void readsLocalParquetFileViaAutoCreatedDuckDbView() throws Exception { + DuckDBParquetSource source = new DuckDBParquetSource(parquetUri(), null, COLUMNS); + TableSink sink = new TableSink<>(new Properties(), "overwrite", SINK_TABLE, COLUMNS); + source.connectTo(0, sink, 0); + + wayangContext(true, "").execute(new WayangPlan(sink)); + + assertEquals(3, queryLong("SELECT count(*) FROM " + SINK_TABLE)); + assertEquals(60.0, queryDouble("SELECT sum(amount) FROM " + SINK_TABLE), 0.01); + } + + @Test + void mapsParquetUriToExistingDuckDbRelation() throws Exception { + DuckDBParquetSource source = new DuckDBParquetSource(parquetUri(), null, COLUMNS); + TableSink sink = new TableSink<>(new Properties(), "overwrite", MAPPED_SINK_TABLE, COLUMNS); + source.connectTo(0, sink, 0); + + Configuration configuration = baseConfiguration(); + configuration.setProperty("wayang.duckdb.parquetsource.mappings", parquetUri() + "=" + SOURCE_VIEW); + new WayangContext(configuration).withPlugin(DuckDB.plugin()).execute(new WayangPlan(sink)); + + assertEquals(3, queryLong("SELECT count(*) FROM " + MAPPED_SINK_TABLE)); + assertEquals(60.0, queryDouble("SELECT sum(amount) FROM " + MAPPED_SINK_TABLE), 0.01); + } + + @Test + void readsPublicGcsParquetFileViaDuckDbHttpfs() throws Exception { + String gcsUri = System.getProperty("duckdb.gcs.parquet.uri", + System.getenv().getOrDefault("DUCKDB_GCS_PARQUET_URI", DEFAULT_GCS_URI)); + Assumptions.assumeTrue(isGcsParquetReachable(gcsUri), "DuckDB httpfs cannot reach " + gcsUri); + + DuckDBParquetSource source = new DuckDBParquetSource(gcsUri, null); + TableSink sink = new TableSink<>(new Properties(), "overwrite", GCS_SINK_TABLE); + source.connectTo(0, sink, 0); + + wayangContext(true, "INSTALL httpfs; LOAD httpfs").execute(new WayangPlan(sink)); + + assertEquals( + queryLong("SELECT count(*) FROM read_parquet('" + gcsUri + "')"), + queryLong("SELECT count(*) FROM " + GCS_SINK_TABLE)); + } + + private WayangContext wayangContext(boolean autoCreate, String prepareSql) { + Configuration configuration = baseConfiguration(); + configuration.setProperty("wayang.duckdb.parquetsource.auto-create", Boolean.toString(autoCreate)); + if (!prepareSql.isEmpty()) { + configuration.setProperty("wayang.duckdb.parquetsource.prepare-sql", prepareSql); + } + return new WayangContext(configuration).withPlugin(DuckDB.plugin()); + } + + private static Configuration baseConfiguration() { + Configuration configuration = new Configuration(); + configuration.setProperty("wayang.duckdb.jdbc.url", jdbcUrl); + configuration.setProperty("wayang.duckdb.jdbc.user", ""); + configuration.setProperty("wayang.duckdb.jdbc.password", ""); + return configuration; + } + + private static boolean isGcsParquetReachable(String gcsUri) { + try (Connection connection = jdbc(); Statement statement = connection.createStatement()) { + statement.execute("INSTALL httpfs"); + statement.execute("LOAD httpfs"); + return queryLong(statement, "SELECT count(*) FROM read_parquet('" + gcsUri + "')") > 0; + } catch (Exception e) { + System.err.println("[DuckDBParquetSourceIT] GCS Parquet unavailable: " + e.getMessage()); + return false; + } + } + + private static long queryLong(String sql) throws Exception { + try (Connection connection = jdbc(); Statement statement = connection.createStatement()) { + return queryLong(statement, sql); + } + } + + private static long queryLong(Statement statement, String sql) throws Exception { + try (ResultSet resultSet = statement.executeQuery(sql)) { + resultSet.next(); + return resultSet.getLong(1); + } + } + + private static double queryDouble(String sql) throws Exception { + try (Connection connection = jdbc(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) { + resultSet.next(); + return resultSet.getDouble(1); + } + } + + private static Connection jdbc() throws Exception { + return DriverManager.getConnection(jdbcUrl); + } + + private static String parquetUri() { + return parquetFile.toAbsolutePath().toString().replace('\\', '/'); + } +} diff --git a/wayang-platforms/wayang-duckdb/src/test/resources/duckdb-ga-smoke.properties b/wayang-platforms/wayang-duckdb/src/test/resources/duckdb-ga-smoke.properties new file mode 100644 index 000000000..a7a4c1e57 --- /dev/null +++ b/wayang-platforms/wayang-duckdb/src/test/resources/duckdb-ga-smoke.properties @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +wayang.profiler.ga.timelimit.ms = 1000 +wayang.profiler.ga.maxgenerations = 1 +wayang.profiler.ga.maxstablegenerations = 1 +wayang.profiler.ga.superoptimizations = 1 +wayang.profiler.ga.intermediateupdate = 1 +wayang.profiler.ga.min-exec-time = 1 +wayang.profiler.ga.max-cardinality-spread = 100 +wayang.profiler.ga.min-cardinality-confidence = 0 +wayang.profiler.ga.binning = 1.0 +wayang.profiler.ga.output-file = wayang-platforms/wayang-duckdb/target/cost-profiling/duckdb-ga-smoke/learned.properties +wayang.profiler.ga.noise-filter.max = 0 diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java index 4b816b2eb..e59545eb4 100644 --- a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java +++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/execution/JdbcExecutor.java @@ -1,4 +1,3 @@ -package org.apache.wayang.jdbc.execution; /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -17,48 +16,7 @@ * limitations under the License. */ -import org.apache.wayang.basic.channels.FileChannel; -import org.apache.wayang.basic.data.Tuple2; -import org.apache.wayang.basic.operators.SpatialFilterOperator; -import org.apache.wayang.basic.operators.SpatialJoinOperator; -import org.apache.wayang.basic.operators.FilterOperator; -import org.apache.wayang.basic.operators.JoinOperator; -import org.apache.wayang.basic.operators.TableSource; -import org.apache.wayang.core.api.Job; -import org.apache.wayang.core.api.exception.WayangException; -import org.apache.wayang.core.optimizer.OptimizationContext; -import org.apache.wayang.core.plan.executionplan.Channel; -import org.apache.wayang.core.plan.executionplan.ExecutionStage; -import org.apache.wayang.core.plan.executionplan.ExecutionTask; -import org.apache.wayang.core.platform.ExecutionState; -import org.apache.wayang.core.platform.ExecutorTemplate; -import org.apache.wayang.core.platform.Platform; -import org.apache.wayang.core.util.fs.FileSystem; -import org.apache.wayang.core.util.fs.FileSystems; -import org.apache.wayang.jdbc.channels.SqlQueryChannel; -import org.apache.wayang.jdbc.compiler.FunctionCompiler; - -import org.apache.wayang.jdbc.operators.JdbcExecutionOperator; -import org.apache.wayang.jdbc.operators.JdbcFilterOperator; -import org.apache.wayang.jdbc.operators.JdbcJoinOperator; -import org.apache.wayang.jdbc.operators.JdbcProjectionOperator; -import org.apache.wayang.jdbc.operators.JdbcTableSinkOperator; -import org.apache.wayang.jdbc.operators.JdbcTableSource; - -import org.apache.wayang.jdbc.platform.JdbcPlatformTemplate; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.UncheckedIOException; -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Set; -import java.util.stream.Collectors; +package org.apache.wayang.jdbc.execution; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -67,7 +25,7 @@ import org.apache.wayang.basic.operators.JoinOperator; import org.apache.wayang.basic.operators.SpatialFilterOperator; import org.apache.wayang.basic.operators.SpatialJoinOperator; -import org.apache.wayang.basic.operators.TableSource; +import org.apache.wayang.core.api.Configuration; import org.apache.wayang.core.api.Job; import org.apache.wayang.core.api.exception.WayangException; import org.apache.wayang.core.optimizer.OptimizationContext; @@ -78,8 +36,9 @@ import org.apache.wayang.core.platform.ExecutionState; import org.apache.wayang.core.platform.Executor; import org.apache.wayang.core.platform.ExecutorTemplate; +import org.apache.wayang.core.platform.PartialExecution; import org.apache.wayang.core.platform.Platform; -import org.apache.wayang.core.util.WayangCollections; +import org.apache.wayang.core.platform.lineage.ExecutionLineageNode; import org.apache.wayang.jdbc.channels.SqlQueryChannel; import org.apache.wayang.jdbc.compiler.FunctionCompiler; import org.apache.wayang.jdbc.operators.JdbcExecutionOperator; @@ -89,48 +48,70 @@ import org.apache.wayang.jdbc.operators.JdbcProjectionOperator; import org.apache.wayang.jdbc.operators.JdbcReduceByOperator; import org.apache.wayang.jdbc.operators.JdbcSortOperator; +import org.apache.wayang.jdbc.operators.JdbcSourceOperator; import org.apache.wayang.jdbc.operators.JdbcTableSinkOperator; -import org.apache.wayang.jdbc.operators.JdbcTableSource; import org.apache.wayang.jdbc.platform.JdbcPlatformTemplate; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Set; +import java.util.stream.Collectors; + /** * {@link Executor} implementation for the {@link JdbcPlatformTemplate}. */ public class JdbcExecutor extends ExecutorTemplate { - public static StringBuilder createSqlString(final JdbcExecutor jdbcExecutor, final JdbcTableSource tableOp, + public static StringBuilder createSqlString(final JdbcExecutor jdbcExecutor, final JdbcSourceOperator sourceOp, final Collection filterTasks, final JdbcProjectionOperator projectionTask, final JdbcGlobalReduceOperator globalReduceTask, final JdbcReduceByOperator reduceByTask, final JdbcSortOperator sortTask, final Collection joinTasks) { - final String tableName = tableOp.createSqlClause(jdbcExecutor.connection, jdbcExecutor.functionCompiler); + return createSqlString(jdbcExecutor, sourceOp, filterTasks, projectionTask, globalReduceTask, reduceByTask, + sortTask, joinTasks, null); + } + + public static StringBuilder createSqlString(final JdbcExecutor jdbcExecutor, final JdbcSourceOperator sourceOp, + final Collection filterTasks, final JdbcProjectionOperator projectionTask, final JdbcGlobalReduceOperator globalReduceTask, final JdbcReduceByOperator reduceByTask, final JdbcSortOperator sortTask, + final Collection joinTasks, final Configuration configuration) { + final String sourceName = sourceOp.createSqlClause( + jdbcExecutor.connection, + jdbcExecutor.functionCompiler, + configuration + ); final Collection conditions = filterTasks.stream() - .map(op -> op.createSqlClause(jdbcExecutor.connection, jdbcExecutor.functionCompiler)) + .map(op -> op.createSqlClause(jdbcExecutor.connection, jdbcExecutor.functionCompiler, configuration)) .collect(Collectors.toList()); final Collection joins = joinTasks.stream() - .map(op -> op.createSqlClause(jdbcExecutor.connection, jdbcExecutor.functionCompiler)) + .map(op -> op.createSqlClause(jdbcExecutor.connection, jdbcExecutor.functionCompiler, configuration)) .collect(Collectors.toList()); final String selectClause; if (globalReduceTask != null) { selectClause = globalReduceTask.createSqlClause( jdbcExecutor.connection, - jdbcExecutor.functionCompiler + jdbcExecutor.functionCompiler, + configuration ); } else if (reduceByTask != null) { selectClause = reduceByTask.createSqlClause( jdbcExecutor.connection, - jdbcExecutor.functionCompiler + jdbcExecutor.functionCompiler, + configuration ); } else if (projectionTask != null) { selectClause = projectionTask.createSqlClause( jdbcExecutor.connection, - jdbcExecutor.functionCompiler + jdbcExecutor.functionCompiler, + configuration ); } else { selectClause = "*"; } final StringBuilder sb = new StringBuilder(1000); - sb.append("SELECT ").append(selectClause).append(" FROM ").append(tableName); + sb.append("SELECT ").append(selectClause).append(" FROM ").append(sourceName); if (!joins.isEmpty()) { final String separator = " "; for (final String join : joins) { @@ -147,24 +128,16 @@ public static StringBuilder createSqlString(final JdbcExecutor jdbcExecutor, fin if (sortTask != null) { sb.append(sortTask.createSqlClause( jdbcExecutor.connection, - jdbcExecutor.functionCompiler + jdbcExecutor.functionCompiler, + configuration )); } - appendStatementTerminator(sb); + // A trailing semicolon is unnecessary for single-statement JDBC calls and + // strict SQL parsers such as Trino and BigQuery reject it. return sb; } - private static void appendStatementTerminator(final StringBuilder query) { - int i = query.length() - 1; - while (i >= 0 && Character.isWhitespace(query.charAt(i))) { - i--; - } - if (i < 0 || query.charAt(i) != ';') { - query.append(';'); - } - } - /** * Creates a query channel and the sql statement * @@ -177,12 +150,13 @@ protected static Tuple2 createSqlQuery(final E final Collection startTasks = stage.getStartTasks(); // Verify that we can handle this instance. - final ExecutionTask startTask = JdbcExecutor.selectStartTask(startTasks, stage); - assert startTask.getOperator() instanceof TableSource - : "Invalid JDBC stage: Start task has to be a TableSource"; + JdbcExecutor.prepareSourceTasks(startTasks, jdbcExecutor, context.getConfiguration()); + final ExecutionTask startTask = JdbcExecutor.selectStartTask(startTasks, stage, context.getConfiguration()); + assert startTask.getOperator() instanceof JdbcSourceOperator + : "Invalid JDBC stage: Start task has to be a JDBC source"; // Extract the different types of ExecutionOperators from the stage. - final JdbcTableSource tableOp = (JdbcTableSource) startTask.getOperator(); + final JdbcSourceOperator sourceOp = (JdbcSourceOperator) startTask.getOperator(); SqlQueryChannel.Instance tipChannelInstance = JdbcExecutor.instantiateOutboundChannel(startTask, context, jdbcExecutor); final Collection filterTasks = new ArrayList<>(4); @@ -229,7 +203,8 @@ protected static Tuple2 createSqlQuery(final E } // Create the SQL query. - final StringBuilder query = createSqlString(jdbcExecutor, tableOp, filterTasks, projectionTask, globalReduceTask, reduceByTask, sortTask, joinTasks); + final StringBuilder query = createSqlString(jdbcExecutor, sourceOp, filterTasks, projectionTask, + globalReduceTask, reduceByTask, sortTask, joinTasks, context.getConfiguration()); return new Tuple2<>(query.toString(), tipChannelInstance); } @@ -240,6 +215,12 @@ protected static Tuple2 createSqlQuery(final E * assumes its first key descriptor's table is used in the {@code FROM} clause. */ private static ExecutionTask selectStartTask(final Collection startTasks, final ExecutionStage stage) { + return selectStartTask(startTasks, stage, null); + } + + private static ExecutionTask selectStartTask(final Collection startTasks, + final ExecutionStage stage, + final Configuration configuration) { if (startTasks.size() == 1) { return (ExecutionTask) startTasks.iterator().next(); } @@ -250,9 +231,12 @@ private static ExecutionTask selectStartTask(final Collection startTasks, fin final String leftTableName = joinOperator.getKeyDescriptor0().getSqlImplementation().field0; for (Object startTaskObject : startTasks) { final ExecutionTask startTask = (ExecutionTask) startTaskObject; - if (startTask.getOperator() instanceof JdbcTableSource - && ((JdbcTableSource) startTask.getOperator()).getTableName().equals(leftTableName)) { - return startTask; + if (startTask.getOperator() instanceof JdbcSourceOperator) { + final JdbcSourceOperator sourceOperator = (JdbcSourceOperator) startTask.getOperator(); + if (sourceOperator.getSourceName().equals(leftTableName) + || sourceOperator.getSourceName(configuration).equals(leftTableName)) { + return startTask; + } } } } @@ -261,6 +245,21 @@ private static ExecutionTask selectStartTask(final Collection startTasks, fin throw new WayangException("Could not determine the left table source for JDBC stage."); } + private static void prepareSourceTasks(final Collection startTasks, + final JdbcExecutor jdbcExecutor, + final Configuration configuration) { + for (Object startTaskObject : startTasks) { + final ExecutionTask startTask = (ExecutionTask) startTaskObject; + if (startTask.getOperator() instanceof JdbcSourceOperator) { + ((JdbcSourceOperator) startTask.getOperator()).prepareSource( + jdbcExecutor.connection, + jdbcExecutor.functionCompiler, + configuration + ); + } + } + } + /** * Handles execution stages that end with a {@link JdbcTableSinkOperator}. * Composes a SQL query from the stage's operators and executes it directly on @@ -270,21 +269,26 @@ private static ExecutionTask selectStartTask(final Collection startTasks, fin * @param optimizationContext provides optimization information * @param jdbcExecutor the executor with the database connection */ - private static void executeSinkStage(final ExecutionStage stage, final OptimizationContext optimizationContext, + private static long executeSinkStage(final ExecutionStage stage, final OptimizationContext optimizationContext, final JdbcExecutor jdbcExecutor) { final Collection startTasks = stage.getStartTasks(); final Collection termTasks = stage.getTerminalTasks(); + JdbcExecutor.prepareSourceTasks(startTasks, jdbcExecutor, optimizationContext.getConfiguration()); - final ExecutionTask startTask = JdbcExecutor.selectStartTask(startTasks, stage); + final ExecutionTask startTask = JdbcExecutor.selectStartTask( + startTasks, + stage, + optimizationContext.getConfiguration() + ); assert termTasks.size() == 1 : "Invalid JDBC stage: multiple terminal tasks are not currently supported."; final ExecutionTask termTask = (ExecutionTask) termTasks.toArray()[0]; - assert startTask.getOperator() instanceof TableSource - : "Invalid JDBC stage: Start task has to be a TableSource"; + assert startTask.getOperator() instanceof JdbcSourceOperator + : "Invalid JDBC stage: Start task has to be a JDBC source"; assert termTask.getOperator() instanceof JdbcTableSinkOperator : "Invalid JDBC stage: Terminal task has to be a JdbcTableSinkOperator"; // Extract operators from the stage - final JdbcTableSource tableOp = (JdbcTableSource) startTask.getOperator(); + final JdbcSourceOperator sourceOp = (JdbcSourceOperator) startTask.getOperator(); final JdbcTableSinkOperator sinkOp = (JdbcTableSinkOperator) termTask.getOperator(); final Collection filterTasks = new ArrayList<>(4); JdbcProjectionOperator projectionTask = null; @@ -325,14 +329,8 @@ private static void executeSinkStage(final ExecutionStage stage, final Optimizat } // Compose the SELECT query - final StringBuilder selectQuery = createSqlString(jdbcExecutor, tableOp, filterTasks, projectionTask, - globalReduceTask, reduceByTask, sortTask, joinTasks); - - // Remove trailing semicolon from SELECT - String selectSql = selectQuery.toString(); - if (selectSql.endsWith(";")) { - selectSql = selectSql.substring(0, selectSql.length() - 1); - } + final String selectSql = createSqlString(jdbcExecutor, sourceOp, filterTasks, projectionTask, + globalReduceTask, reduceByTask, sortTask, joinTasks, optimizationContext.getConfiguration()).toString(); // Get the sink's SQL clause final String sinkClause = sinkOp.createSqlClause(jdbcExecutor.connection, jdbcExecutor.functionCompiler); @@ -346,15 +344,44 @@ private static void executeSinkStage(final ExecutionStage stage, final Optimizat // Execute the composed query: CREATE TABLE x AS SELECT ... or INSERT INTO x // SELECT ... final String fullSql = sinkClause + " " + selectSql + sinkOp.createSqlSuffix(); + final long startTime = System.currentTimeMillis(); stmt.execute(fullSql); + final long executionDuration = System.currentTimeMillis() - startTime; jdbcExecutor.logger.info("Executed SQL sink: {}", fullSql); System.out.println("Executed sql sink: " + fullSql); + return executionDuration; } catch (final SQLException e) { throw new WayangException("Failed to execute SQL sink on table: " + sinkOp.getTableName(), e); } } + /** + * Creates lineage nodes for the JDBC operators that were executed as one SQL + * statement. Operators without load estimators are skipped so JDBC platforms + * without cost specifications can still execute normally. + */ + private Collection createExecutionLineageNodes( + final ExecutionStage stage, + final OptimizationContext optimizationContext) { + final Collection executionLineageNodes = new ArrayList<>(); + for (ExecutionTask task : stage.getAllTasks()) { + final OptimizationContext.OperatorContext operatorContext = + optimizationContext.getOperatorContext(task.getOperator()); + if (operatorContext == null) { + this.logger.warn("Cannot profile {} because its optimization context is missing.", task); + continue; + } + if (operatorContext.getLoadProfileEstimator() == null) { + this.logger.warn("Cannot profile {} because its load profile estimator is missing.", task); + continue; + } + executionLineageNodes.add( + new ExecutionLineageNode(operatorContext).addAtomicExecutionFromOperatorContext()); + } + return executionLineageNodes; + } + /** * Retrieves the follow-up {@link ExecutionTask} of the given {@code task} * unless it is not comprising a {@link JdbcExecutionOperator} and/or not in the @@ -428,7 +455,16 @@ public void execute(final ExecutionStage stage, final OptimizationContext optimi final ExecutionTask termTask = (ExecutionTask) termTasks.toArray()[0]; if (termTask.getOperator() instanceof JdbcTableSinkOperator) { - JdbcExecutor.executeSinkStage(stage, optimizationContext, this); + final long executionDuration = JdbcExecutor.executeSinkStage(stage, optimizationContext, this); + if (this.isProfilingEnabled()) { + final PartialExecution partialExecution = this.createPartialExecution( + this.createExecutionLineageNodes(stage, optimizationContext), + executionDuration + ); + if (partialExecution != null) { + executionState.add(partialExecution); + } + } } else { // If it is normal stage: compose SQL and store in channel for downstream // consumption @@ -441,6 +477,10 @@ public void execute(final ExecutionStage stage, final OptimizationContext optimi } } + private boolean isProfilingEnabled() { + return this.getConfiguration().getBooleanProperty("wayang.core.log.enabled", false); + } + @Override public void dispose() { try { diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcExecutionOperator.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcExecutionOperator.java index 570897aed..e8457f508 100644 --- a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcExecutionOperator.java +++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcExecutionOperator.java @@ -19,6 +19,7 @@ package org.apache.wayang.jdbc.operators; import org.apache.wayang.basic.operators.TableSource; +import org.apache.wayang.core.api.Configuration; import org.apache.wayang.core.plan.wayangplan.ExecutionOperator; import org.apache.wayang.core.platform.ChannelDescriptor; import org.apache.wayang.jdbc.compiler.FunctionCompiler; @@ -42,6 +43,13 @@ public interface JdbcExecutionOperator extends ExecutionOperator { */ String createSqlClause(Connection connection, FunctionCompiler compiler); + /** + * Creates a SQL clause under the given configuration. + */ + default String createSqlClause(Connection connection, FunctionCompiler compiler, Configuration configuration) { + return this.createSqlClause(connection, compiler); + } + @Override JdbcPlatformTemplate getPlatform(); diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcJoinOperator.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcJoinOperator.java index 6ef378422..1368dde66 100644 --- a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcJoinOperator.java +++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcJoinOperator.java @@ -66,11 +66,17 @@ public JdbcJoinOperator(JoinOperator that) { @Override public String createSqlClause(Connection connection, FunctionCompiler compiler) { + return this.createSqlClause(connection, compiler, null); + } + + @Override + public String createSqlClause(Connection connection, FunctionCompiler compiler, Configuration configuration) { final Tuple left = this.keyDescriptor0.getSqlImplementation(); final Tuple right = this.keyDescriptor1.getSqlImplementation(); - final String leftTableName = left.field0; + final String platformId = this.getPlatform().getPlatformId(); + final String leftTableName = JdbcParquetSource.resolveSourceName(configuration, platformId, left.field0); final String leftKeys = left.field1; - final String rightTableName = right.field0; + final String rightTableName = JdbcParquetSource.resolveSourceName(configuration, platformId, right.field0); final String rightKeys = right.field1; if (leftKeys.contains(",") && rightKeys.contains(",")) { diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcParquetSource.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcParquetSource.java new file mode 100644 index 000000000..4bea068df --- /dev/null +++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcParquetSource.java @@ -0,0 +1,410 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.jdbc.operators; + +import org.apache.logging.log4j.LogManager; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.Type; +import org.apache.wayang.basic.operators.ParquetSource; +import org.apache.wayang.commons.util.profiledb.model.measurement.TimeMeasurement; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.exception.WayangException; +import org.apache.wayang.core.optimizer.OptimizationContext; +import org.apache.wayang.core.optimizer.cardinality.CardinalityEstimate; +import org.apache.wayang.jdbc.compiler.FunctionCompiler; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * JDBC implementation for {@link ParquetSource}s exposed by an engine as a SQL + * relation, such as a Hive/Iceberg table, BigQuery external table, or DuckDB + * view over {@code read_parquet(...)}. + */ +public abstract class JdbcParquetSource extends ParquetSource implements JdbcSourceOperator { + + private static final Pattern PROPERTY_PLACEHOLDER = Pattern.compile("\\$\\{(env|sys):([^}]+)}"); + + public JdbcParquetSource(String sourceName, String[] projection, String... columnNames) { + super(sourceName, projection, columnNames); + } + + public JdbcParquetSource(ParquetSource that) { + super(that); + } + + @Override + public String getSourceName() { + return this.getInputUrl(); + } + + @Override + public String getSourceName(Configuration configuration) { + return this.resolveSourceName(configuration); + } + + @Override + public String createSqlClause(Connection connection, FunctionCompiler compiler) { + return this.getSourceName(); + } + + @Override + public String createSqlClause(Connection connection, FunctionCompiler compiler, Configuration configuration) { + return this.resolveSourceName(connection, configuration); + } + + @Override + public void prepareSource(Connection connection, FunctionCompiler compiler, Configuration configuration) { + this.resolveSourceName(connection, configuration); + } + + @Override + public String getLoadProfileEstimatorConfigurationKey() { + return String.format("wayang.%s.parquetsource.load", this.getPlatform().getPlatformId()); + } + + @Override + public org.apache.wayang.core.optimizer.cardinality.CardinalityEstimator getCardinalityEstimator(int outputIndex) { + assert outputIndex == 0; + return new org.apache.wayang.core.optimizer.cardinality.CardinalityEstimator() { + @Override + public CardinalityEstimate estimate(OptimizationContext optimizationContext, + CardinalityEstimate... inputEstimates) { + final TimeMeasurement timeMeasurement = optimizationContext.getJob().getStopWatch().start( + "Optimization", "Cardinality&Load Estimation", "Push Estimation", "Estimate source cardinalities" + ); + + try (Connection connection = JdbcParquetSource.this.getPlatform() + .createDatabaseDescriptor(optimizationContext.getConfiguration()) + .createJdbcConnection()) { + final String sql = String.format("SELECT count(*) FROM %s", + JdbcParquetSource.this.resolveSourceName( + connection, + optimizationContext.getConfiguration() + )); + final ResultSet resultSet = connection.createStatement().executeQuery(sql); + if (!resultSet.next()) { + throw new SQLException("No query result for \"" + sql + "\"."); + } + long cardinality = resultSet.getLong(1); + return new CardinalityEstimate(cardinality, cardinality, 1d); + } catch (Exception e) { + LogManager.getLogger(this.getClass()).error( + "Could not estimate cardinality for {}.", JdbcParquetSource.this, e + ); + return new CardinalityEstimate(10, 10000000, 0.9); + } finally { + timeMeasurement.stop(); + } + } + }; + } + + @Override + public Optional createCardinalityEstimator( + int outputIndex, + Configuration configuration) { + return Optional.of(this.getCardinalityEstimator(outputIndex)); + } + + private String resolveSourceName(Configuration configuration) { + if (configuration == null) { + return this.getSourceName(); + } + + final String platformId = this.getPlatform().getPlatformId(); + return resolveSourceName(configuration, platformId, this.getInputUrl()); + } + + private String resolveSourceName(Connection connection, Configuration configuration) { + final String sourceName = this.resolveSourceName(configuration); + if (configuration == null) { + return sourceName; + } + + final String platformId = this.getPlatform().getPlatformId(); + this.executePrepareSql(connection, configuration, platformId); + if (isAutoCreateEnabled(configuration, platformId)) { + this.createExternalRelation(connection, configuration, platformId, sourceName); + } + + return sourceName; + } + + public static String resolveSourceName(Configuration configuration, String platformId, String inputUrl) { + if (configuration == null) { + return inputUrl; + } + + return findMappedRelation(configuration, platformId, inputUrl) + .orElseGet(() -> isAutoCreateEnabled(configuration, platformId) && isParquetLocation(inputUrl) + ? createGeneratedRelationName(configuration, platformId, inputUrl) + : inputUrl); + } + + private static boolean isParquetLocation(String inputUrl) { + return inputUrl.contains("://") + || inputUrl.startsWith("/") + || inputUrl.startsWith("\\") + || inputUrl.matches("^[A-Za-z]:[\\\\/].*") + || inputUrl.contains(".parquet"); + } + + private static Optional findMappedRelation(Configuration configuration, String platformId, String inputUrl) { + final String mappingKey = String.format("wayang.%s.parquetsource.mappings", platformId); + final Optional mapping = configuration.getOptionalStringProperty(mappingKey); + if (mapping.isEmpty()) { + return Optional.empty(); + } + + for (String entry : mapping.get().split(";")) { + final String trimmedEntry = entry.trim(); + if (trimmedEntry.isEmpty()) { + continue; + } + + final int separator = trimmedEntry.indexOf('='); + if (separator < 0) { + LogManager.getLogger(JdbcParquetSource.class).warn( + "Ignoring invalid Parquet source mapping entry '{}' for {}.", trimmedEntry, mappingKey + ); + continue; + } + + final String sourceUri = trimmedEntry.substring(0, separator).trim(); + final String relationName = trimmedEntry.substring(separator + 1).trim(); + if (sourceUri.equals(inputUrl) && !relationName.isEmpty()) { + return Optional.of(relationName); + } + } + + return Optional.empty(); + } + + private void executePrepareSql(Connection connection, Configuration configuration, String platformId) { + final String prepareSqlKey = String.format("wayang.%s.parquetsource.prepare-sql", platformId); + final Optional optionalPrepareSql = configuration.getOptionalStringProperty(prepareSqlKey); + if (optionalPrepareSql.isEmpty() || optionalPrepareSql.get().trim().isEmpty()) { + return; + } + + try (Statement statement = connection.createStatement()) { + for (String sql : splitSqlStatements(resolvePlaceholders(optionalPrepareSql.get()))) { + statement.execute(sql); + } + } catch (SQLException e) { + throw new WayangException(String.format( + "Could not execute Parquet source prepare SQL configured by '%s'.", + prepareSqlKey + ), e); + } + } + + private static List splitSqlStatements(String sql) { + final List statements = new ArrayList<>(); + final StringBuilder current = new StringBuilder(sql.length()); + boolean insideSingleQuote = false; + for (int i = 0; i < sql.length(); i++) { + char currentChar = sql.charAt(i); + if (currentChar == '\'') { + current.append(currentChar); + if (insideSingleQuote && i + 1 < sql.length() && sql.charAt(i + 1) == '\'') { + current.append(sql.charAt(++i)); + } else { + insideSingleQuote = !insideSingleQuote; + } + } else if (currentChar == ';' && !insideSingleQuote) { + addStatement(statements, current); + } else { + current.append(currentChar); + } + } + addStatement(statements, current); + return statements; + } + + private static void addStatement(List statements, StringBuilder statement) { + final String sql = statement.toString().trim(); + if (!sql.isEmpty()) { + statements.add(sql); + } + statement.setLength(0); + } + + private static String resolvePlaceholders(String sql) { + Matcher matcher = PROPERTY_PLACEHOLDER.matcher(sql); + StringBuffer resolved = new StringBuffer(); + while (matcher.find()) { + final String type = matcher.group(1); + final String name = matcher.group(2); + final String value = "env".equals(type) ? System.getenv(name) : System.getProperty(name); + if (value == null) { + throw new WayangException(String.format( + "Could not resolve Parquet source prepare SQL placeholder '${%s:%s}'.", + type, + name + )); + } + matcher.appendReplacement(resolved, Matcher.quoteReplacement(value)); + } + matcher.appendTail(resolved); + return resolved.toString(); + } + + private static boolean isAutoCreateEnabled(Configuration configuration, String platformId) { + return configuration.getBooleanProperty( + String.format("wayang.%s.parquetsource.auto-create", platformId), + false + ); + } + + private static String createGeneratedRelationName(Configuration configuration, String platformId, String inputUrl) { + final String prefix = configuration.getStringProperty( + String.format("wayang.%s.parquetsource.auto-create.relation-prefix", platformId), + "wayang_parquet_" + ); + return prefix + shortHash(inputUrl); + } + + private static String shortHash(String value) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] hash = digest.digest(value.getBytes(StandardCharsets.UTF_8)); + StringBuilder sb = new StringBuilder(16); + for (int i = 0; i < 8; i++) { + sb.append(String.format("%02x", hash[i])); + } + return sb.toString(); + } catch (NoSuchAlgorithmException e) { + throw new WayangException("Could not create stable Parquet relation name.", e); + } + } + + private void createExternalRelation(Connection connection, + Configuration configuration, + String platformId, + String relationName) { + final String templateKey = String.format("wayang.%s.parquetsource.auto-create.template", platformId); + final Optional optionalTemplate = configuration.getOptionalStringProperty(templateKey); + if (optionalTemplate.isEmpty()) { + throw new WayangException(String.format( + "Parquet auto-create is enabled for platform '%s', but '%s' is not configured.", + platformId, + templateKey + )); + } + + final String template = optionalTemplate.get(); + final String ddl = template + .replace("${relation}", relationName) + .replace("${uri}", this.escapeSqlString(this.getInputUrl())) + .replace("${columns}", this.createColumnDefinitions(templateKey, template)); + + try (Statement statement = connection.createStatement()) { + statement.execute(ddl); + } catch (SQLException e) { + throw new WayangException(String.format( + "Could not create Parquet SQL relation '%s' for '%s'.", + relationName, + this.getInputUrl() + ), e); + } + } + + private String escapeSqlString(String value) { + return value.replace("'", "''"); + } + + private String createColumnDefinitions(String templateKey, String template) { + if (!template.contains("${columns}")) { + return ""; + } + + if (this.getSchema() == null || this.getSchema().getFields().isEmpty()) { + throw new WayangException(String.format( + "Parquet source auto-create template '%s' uses ${columns}, but no Parquet schema is available. " + + "Create the source with ParquetSource.create(...) or configure a template that does not " + + "need explicit columns.", + templateKey + )); + } + + return this.getSchema().getFields().stream() + .map(field -> field.getName() + " " + this.toSqlType(field)) + .reduce((left, right) -> left + ", " + right) + .orElse(""); + } + + private String toSqlType(Type field) { + if (!field.isPrimitive()) { + return "VARCHAR"; + } + + final PrimitiveType primitiveType = field.asPrimitiveType(); + final LogicalTypeAnnotation logicalType = primitiveType.getLogicalTypeAnnotation(); + if (logicalType instanceof LogicalTypeAnnotation.StringLogicalTypeAnnotation + || logicalType instanceof LogicalTypeAnnotation.EnumLogicalTypeAnnotation + || logicalType instanceof LogicalTypeAnnotation.UUIDLogicalTypeAnnotation) { + return "VARCHAR"; + } + if (logicalType instanceof LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) { + LogicalTypeAnnotation.DecimalLogicalTypeAnnotation decimal = + (LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) logicalType; + return String.format("DECIMAL(%d,%d)", decimal.getPrecision(), decimal.getScale()); + } + if (logicalType instanceof LogicalTypeAnnotation.DateLogicalTypeAnnotation) { + return "DATE"; + } + if (logicalType instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) { + return "TIMESTAMP"; + } + + switch (primitiveType.getPrimitiveTypeName()) { + case BOOLEAN: + return "BOOLEAN"; + case INT32: + return "INTEGER"; + case INT64: + return "BIGINT"; + case FLOAT: + return "REAL"; + case DOUBLE: + return "DOUBLE"; + case BINARY: + return "VARCHAR"; + case FIXED_LEN_BYTE_ARRAY: + return "VARBINARY"; + case INT96: + return "TIMESTAMP"; + default: + return "VARCHAR"; + } + } +} diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcSourceOperator.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcSourceOperator.java new file mode 100644 index 000000000..b7934ac89 --- /dev/null +++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcSourceOperator.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.jdbc.operators; + +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.platform.ChannelDescriptor; +import org.apache.wayang.jdbc.compiler.FunctionCompiler; + +import java.sql.Connection; +import java.util.List; + +/** + * Marks JDBC operators that can start a SQL stage and provide a relation for a + * {@code FROM} clause. + */ +public interface JdbcSourceOperator extends JdbcExecutionOperator { + + /** + * Name or expression used to identify this source in SQL metadata, e.g., in + * join descriptors. + */ + String getSourceName(); + + /** + * Name or expression used for this source under the given configuration. + */ + default String getSourceName(Configuration configuration) { + return this.getSourceName(); + } + + /** + * Creates a SQL clause for this source under the given configuration. + */ + default String createSqlClause(Connection connection, FunctionCompiler compiler, Configuration configuration) { + return this.createSqlClause(connection, compiler); + } + + /** + * Prepares this source for SQL generation, e.g., by registering a temporary + * relation. Implementations can keep this as a no-op when no preparation is + * required. + */ + default void prepareSource(Connection connection, FunctionCompiler compiler, Configuration configuration) { + } + + @Override + default List getSupportedInputChannels(int index) { + throw new UnsupportedOperationException("JDBC source operators have no input channels."); + } +} diff --git a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcTableSource.java b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcTableSource.java index 4d7096649..ac414b564 100644 --- a/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcTableSource.java +++ b/wayang-platforms/wayang-jdbc-template/src/main/java/org/apache/wayang/jdbc/operators/JdbcTableSource.java @@ -33,7 +33,7 @@ /** * PostgreSQL implementation for the {@link TableSource}. */ -public abstract class JdbcTableSource extends TableSource implements JdbcExecutionOperator { +public abstract class JdbcTableSource extends TableSource implements JdbcSourceOperator { /** * Creates a new instance. @@ -58,6 +58,10 @@ public String createSqlClause(Connection connection, FunctionCompiler compiler) return this.getTableName(); } + @Override + public String getSourceName() { + return this.getTableName(); + } @Override public String getLoadProfileEstimatorConfigurationKey() { diff --git a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcExecutorTest.java b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcExecutorTest.java index 0dfd8b698..8f7b3d8a2 100644 --- a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcExecutorTest.java +++ b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcExecutorTest.java @@ -81,7 +81,7 @@ void testExecuteWithPlainTableSource() throws SQLException { SqlQueryChannel.Instance sqlQueryChannelInstance = (SqlQueryChannel.Instance) job.getCrossPlatformExecutor().getChannelInstance(sqlToStreamTask.getInputChannel(0)); assertEquals( - "SELECT * FROM customer;", + "SELECT * FROM customer", sqlQueryChannelInstance.getSqlQuery() ); } @@ -130,7 +130,7 @@ void testExecuteWithFilter() throws SQLException { SqlQueryChannel.Instance sqlQueryChannelInstance = (SqlQueryChannel.Instance) job.getCrossPlatformExecutor().getChannelInstance(sqlToStreamTask.getInputChannel(0)); assertEquals( - "SELECT * FROM customer WHERE age >= 18;", + "SELECT * FROM customer WHERE age >= 18", sqlQueryChannelInstance.getSqlQuery() ); } @@ -172,7 +172,7 @@ void testExecuteWithProjection() throws SQLException { SqlQueryChannel.Instance sqlQueryChannelInstance = (SqlQueryChannel.Instance) job.getCrossPlatformExecutor().getChannelInstance(sqlToStreamTask.getInputChannel(0)); assertEquals( - "SELECT name, age FROM customer;", + "SELECT name, age FROM customer", sqlQueryChannelInstance.getSqlQuery() ); } @@ -240,7 +240,7 @@ void testExecuteWithProjectionAndFilters() throws SQLException { SqlQueryChannel.Instance sqlQueryChannelInstance = (SqlQueryChannel.Instance) job.getCrossPlatformExecutor().getChannelInstance(sqlToStreamTask.getInputChannel(0)); assertEquals( - "SELECT name, age FROM customer WHERE age >= 18 AND name IS NOT NULL;", + "SELECT name, age FROM customer WHERE age >= 18 AND name IS NOT NULL", sqlQueryChannelInstance.getSqlQuery() ); } diff --git a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcTableSinkExecutorTest.java b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcTableSinkExecutorTest.java index 263730629..46262743d 100644 --- a/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcTableSinkExecutorTest.java +++ b/wayang-platforms/wayang-jdbc-template/src/test/java/org/apache/wayang/jdbc/execution/JdbcTableSinkExecutorTest.java @@ -23,7 +23,9 @@ import org.apache.wayang.core.optimizer.DefaultOptimizationContext; import org.apache.wayang.core.plan.executionplan.ExecutionStage; import org.apache.wayang.core.plan.executionplan.ExecutionTask; +import org.apache.wayang.core.platform.AtomicExecution; import org.apache.wayang.core.platform.CrossPlatformExecutor; +import org.apache.wayang.core.platform.PartialExecution; import org.apache.wayang.core.profiling.NoInstrumentationStrategy; import org.apache.wayang.jdbc.channels.SqlQueryChannel; import org.apache.wayang.jdbc.operators.JdbcTableSinkOperator; @@ -37,7 +39,11 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; @@ -51,6 +57,17 @@ class JdbcTableSinkExecutorTest { @Test void testOverwriteModeCreatesNewTable() throws SQLException { Configuration configuration = new Configuration(); + configuration.setProperty("wayang.core.log.enabled", "true"); + configuration.setProperty("wayang.hsqldb.cpu.mhz", "2700"); + configuration.setProperty("wayang.hsqldb.cores", "1"); + configuration.setProperty( + "wayang.hsqldb.tablesource.load", + "{\"in\":0,\"out\":1,\"cpu\":\"${1}\",\"ram\":\"0\",\"p\":1.0}" + ); + configuration.setProperty( + "wayang.hsqldb.tablesink.load", + "{\"in\":1,\"out\":0,\"cpu\":\"${1}\",\"ram\":\"0\",\"p\":1.0}" + ); HsqldbPlatform hsqldbPlatform = new HsqldbPlatform(); // Create source table with data @@ -89,10 +106,30 @@ void testOverwriteModeCreatesNewTable() throws SQLException { when(sqlStage.getStartTasks()).thenReturn(Collections.singleton(tableSourceTask)); when(sqlStage.getTerminalTasks()).thenReturn(Collections.singleton(sinkTask)); + when(sqlStage.getAllTasks()).thenReturn(new HashSet<>(Arrays.asList(tableSourceTask, sinkTask))); // Execute JdbcExecutor executor = new JdbcExecutor(HsqldbPlatform.getInstance(), job); - executor.execute(sqlStage, new DefaultOptimizationContext(job), job.getCrossPlatformExecutor()); + DefaultOptimizationContext optimizationContext = new DefaultOptimizationContext(job); + optimizationContext.addOneTimeOperator(tableSource); + optimizationContext.addOneTimeOperator(sinkOp); + executor.execute(sqlStage, optimizationContext, job.getCrossPlatformExecutor()); + + assertEquals(1, job.getCrossPlatformExecutor().getPartialExecutions().size()); + PartialExecution partialExecution = + job.getCrossPlatformExecutor().getPartialExecutions().iterator().next(); + Set estimatorKeys = partialExecution.getAtomicExecutionGroups().stream() + .flatMap(group -> group.getAtomicExecutions().stream()) + .map(AtomicExecution::getLoadProfileEstimator) + .map(estimator -> estimator.getConfigurationKey()) + .collect(Collectors.toSet()); + assertEquals( + new HashSet<>(Arrays.asList( + "wayang.hsqldb.tablesource.load", + "wayang.hsqldb.tablesink.load" + )), + estimatorKeys + ); // Verify table was created and contains all 3 rows try (Connection conn = hsqldbPlatform.createDatabaseDescriptor(configuration).createJdbcConnection()) { @@ -252,4 +289,4 @@ void testAppendClauseGeneration() { sinkOp.setMode("append"); assertEquals("INSERT INTO my_table", sinkOp.createSqlClause(null, null)); } -} \ No newline at end of file +} From 5ae23749ff606cc29862a268cdbc366b4e876f47 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Tue, 11 Aug 2026 20:18:49 +0800 Subject: [PATCH 2/4] Tidy DuckDB README test wording --- wayang-platforms/wayang-duckdb/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wayang-platforms/wayang-duckdb/README.md b/wayang-platforms/wayang-duckdb/README.md index 17b267173..7f93c02fe 100644 --- a/wayang-platforms/wayang-duckdb/README.md +++ b/wayang-platforms/wayang-duckdb/README.md @@ -70,8 +70,8 @@ credentials, as documented in `wayang-duckdb-defaults.properties`. ## Tests The embedded operator suite mirrors `TrinoOperatorsIT` / `PrestoOperatorsIT`, -but runs against a temporary DuckDB database file. The Parquet and cost pilots -mirror the separate Trino/Presto feature branches for those pieces. +but runs against a temporary DuckDB database file. Separate Parquet and cost +profiling suites cover DuckDB-specific file access and calibration workflows. Run the DuckDB operator suite while compiling the required reactor modules: From ba1290bef6917e95d26d8bf0f2674a54e82c5507 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Wed, 19 Aug 2026 00:40:10 +0800 Subject: [PATCH 3/4] Align DuckDB docs with Parquet platform scope --- platforms-setup-guides/duckdb-setup/README.md | 2 ++ wayang-platforms/wayang-duckdb/README.md | 5 ++--- .../src/main/resources/wayang-duckdb-defaults.properties | 7 +------ 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/platforms-setup-guides/duckdb-setup/README.md b/platforms-setup-guides/duckdb-setup/README.md index ef1819112..8c923fa5e 100644 --- a/platforms-setup-guides/duckdb-setup/README.md +++ b/platforms-setup-guides/duckdb-setup/README.md @@ -113,6 +113,7 @@ For a fast local check, run two profiling plans over two small cardinalities: -Dduckdb.profile.rowCounts=100,1000 \ -Dduckdb.profile.plans=S01,S02 \ -Dduckdb.profile.repetitions=2 \ + -Dsurefire.failIfNoSpecifiedTests=false -DfailIfNoTests=false \ -Drat.skip=true -Dlicense.skip=true test ``` @@ -125,6 +126,7 @@ six repetitions, producing 312 Wayang executions: -Dduckdb.profile.rowCounts=10000,50000,100000,250000 \ -Dduckdb.profile.plans=S01,S02,S03,S04,S05,S06,S07,S08,S09,S10,S11,S12,S13 \ -Dduckdb.profile.repetitions=6 \ + -Dsurefire.failIfNoSpecifiedTests=false -DfailIfNoTests=false \ -Drat.skip=true -Dlicense.skip=true test ``` diff --git a/wayang-platforms/wayang-duckdb/README.md b/wayang-platforms/wayang-duckdb/README.md index 7f93c02fe..ac439d7ee 100644 --- a/wayang-platforms/wayang-duckdb/README.md +++ b/wayang-platforms/wayang-duckdb/README.md @@ -64,9 +64,6 @@ created: wayang.duckdb.parquetsource.prepare-sql = INSTALL httpfs; LOAD httpfs ``` -Private GCS buckets can use the same hook to create a DuckDB secret with HMAC -credentials, as documented in `wayang-duckdb-defaults.properties`. - ## Tests The embedded operator suite mirrors `TrinoOperatorsIT` / `PrestoOperatorsIT`, @@ -143,6 +140,7 @@ with six repetitions, producing 312 Wayang executions: -Dduckdb.profile.rowCounts=10000,50000,100000,250000 \ -Dduckdb.profile.plans=S01,S02,S03,S04,S05,S06,S07,S08,S09,S10,S11,S12,S13 \ -Dduckdb.profile.repetitions=6 \ + -Dsurefire.failIfNoSpecifiedTests=false -DfailIfNoTests=false \ -Drat.skip=true -Dlicense.skip=true test ``` @@ -158,6 +156,7 @@ For a quick local smoke: -Dduckdb.profile.rowCounts=100,1000 \ -Dduckdb.profile.plans=S01,S02 \ -Dduckdb.profile.repetitions=2 \ + -Dsurefire.failIfNoSpecifiedTests=false -DfailIfNoTests=false \ -Drat.skip=true -Dlicense.skip=true test ``` diff --git a/wayang-platforms/wayang-duckdb/src/main/resources/wayang-duckdb-defaults.properties b/wayang-platforms/wayang-duckdb/src/main/resources/wayang-duckdb-defaults.properties index 5c8843f8d..8aec8f440 100644 --- a/wayang-platforms/wayang-duckdb/src/main/resources/wayang-duckdb-defaults.properties +++ b/wayang-platforms/wayang-duckdb/src/main/resources/wayang-duckdb-defaults.properties @@ -36,12 +36,7 @@ wayang.duckdb.parquetsource.auto-create.template = CREATE OR REPLACE VIEW ${rela # # Optional prepare SQL executed before DuckDB creates the Parquet view. Keep # statements idempotent: they can run during optimization cardinality estimation -# and during execution. This is the hook for cloud storage extensions/secrets. -# Environment and JVM system properties can be injected via ${env:NAME} and -# ${sys:name}. -# -# Example for private GCS Parquet files: -# wayang.duckdb.parquetsource.prepare-sql = INSTALL httpfs; LOAD httpfs; CREATE OR REPLACE SECRET wayang_gcs (TYPE gcs, KEY_ID '${env:GCS_HMAC_KEY_ID}', SECRET '${env:GCS_HMAC_SECRET}') +# and during execution. This can load DuckDB extensions such as httpfs. # Hardware profile used by LoadProfileToTimeConverter. wayang.duckdb.cpu.mhz = 2700 From 49bf1ae3a54b1b9c7c562f7434d4de03abdeb44e Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Fri, 4 Sep 2026 22:38:23 -0400 Subject: [PATCH 4/4] Align DuckDB example with application conventions --- .../duckdb-setup/.gitignore | 16 -- platforms-setup-guides/duckdb-setup/README.md | 206 --------------- platforms-setup-guides/duckdb-setup/demo.sh | 116 --------- .../duckdb-setup/docker-compose.yml | 29 --- platforms-setup-guides/duckdb-setup/pom.xml | 81 ------ .../duckdb-setup/scripts/check.sql | 24 -- .../duckdb-setup/scripts/init.sql | 46 ---- .../duckdb-setup/scripts/run-duckdb-ga.ps1 | 121 --------- .../wayang/duckdb/DuckDBIntegrationTest.java | 244 ------------------ wayang-applications/README.md | 3 + wayang-applications/duckdb.md | 70 +++++ wayang-applications/pom.xml | 10 + .../wayang/applications}/DuckDBDemo.java | 97 ++++--- .../wayang/applications/DuckDBDemoTest.java | 70 +++++ wayang-platforms/wayang-duckdb/README.md | 41 +-- wayang-profiler/duckdb.md | 46 ++++ wayang-profiler/pom.xml | 27 ++ .../src/main/resources/duckdb-ga.properties | 0 18 files changed, 288 insertions(+), 959 deletions(-) delete mode 100644 platforms-setup-guides/duckdb-setup/.gitignore delete mode 100644 platforms-setup-guides/duckdb-setup/README.md delete mode 100644 platforms-setup-guides/duckdb-setup/demo.sh delete mode 100644 platforms-setup-guides/duckdb-setup/docker-compose.yml delete mode 100644 platforms-setup-guides/duckdb-setup/pom.xml delete mode 100644 platforms-setup-guides/duckdb-setup/scripts/check.sql delete mode 100644 platforms-setup-guides/duckdb-setup/scripts/init.sql delete mode 100644 platforms-setup-guides/duckdb-setup/scripts/run-duckdb-ga.ps1 delete mode 100644 platforms-setup-guides/duckdb-setup/src/test/java/org/apache/wayang/duckdb/DuckDBIntegrationTest.java create mode 100644 wayang-applications/duckdb.md rename {wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb => wayang-applications/src/main/java/org/apache/wayang/applications}/DuckDBDemo.java (62%) create mode 100644 wayang-applications/src/test/java/org/apache/wayang/applications/DuckDBDemoTest.java create mode 100644 wayang-profiler/duckdb.md rename platforms-setup-guides/duckdb-setup/profiling/ga-relaxed.properties => wayang-profiler/src/main/resources/duckdb-ga.properties (100%) diff --git a/platforms-setup-guides/duckdb-setup/.gitignore b/platforms-setup-guides/duckdb-setup/.gitignore deleted file mode 100644 index b4185cd23..000000000 --- a/platforms-setup-guides/duckdb-setup/.gitignore +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -data/ diff --git a/platforms-setup-guides/duckdb-setup/README.md b/platforms-setup-guides/duckdb-setup/README.md deleted file mode 100644 index 8c923fa5e..000000000 --- a/platforms-setup-guides/duckdb-setup/README.md +++ /dev/null @@ -1,206 +0,0 @@ -# DuckDB Local Setup - -Local DuckDB setup for the Wayang DuckDB platform. - -DuckDB is embedded: there is no coordinator or long-running database service to -start. The Wayang platform connects directly to a DuckDB database file through -the DuckDB JDBC driver. For a Trino-like reproducible local workflow, this setup -uses the official DuckDB CLI Docker image to create and inspect a local -`data/wayang.duckdb` file, then runs the Wayang operator tests against that -same file. - -Run the commands below from the repository root. Java 17 and Docker with Docker -Compose are required; Maven is provided by the repository wrapper. - -## Stack - -| Component | Image | Role | -|-----------|-------|------| -| DuckDB CLI | `duckdb/duckdb:1.5.5` | Creates and inspects the local database file | -| DuckDB JDBC | `org.duckdb:duckdb_jdbc:1.5.5.1` | Runs Wayang plans against that file | - -The Docker service is a one-shot CLI container. It exits after running the SQL -command; that is expected. - -## 1. Create The Local DuckDB File - -```bash -mkdir -p platforms-setup-guides/duckdb-setup/data -docker compose -f platforms-setup-guides/duckdb-setup/docker-compose.yml run --rm duckdb -``` - -On PowerShell: - -```powershell -New-Item -ItemType Directory -Force platforms-setup-guides/duckdb-setup/data -docker compose -f platforms-setup-guides/duckdb-setup/docker-compose.yml run --rm duckdb -``` - -Expected output includes grouped totals for `APAC`, `AMER`, and `EMEA`. - -## 2. Inspect The File Directly - -```bash -docker run --rm -i \ - -v "$PWD/platforms-setup-guides/duckdb-setup:/workspace" \ - duckdb/duckdb:1.5.5 \ - duckdb /workspace/data/wayang.duckdb < platforms-setup-guides/duckdb-setup/scripts/check.sql -``` - -On PowerShell: - -```powershell -Get-Content -Raw platforms-setup-guides/duckdb-setup/scripts/check.sql | - docker run --rm -i -v "${PWD}/platforms-setup-guides/duckdb-setup:/workspace" duckdb/duckdb:1.5.5 duckdb /workspace/data/wayang.duckdb -``` - -## 3. Run Wayang Tests Against The Docker-Created File - -```bash -./mvnw -Pskip-prerequisite-check -pl wayang-platforms/wayang-duckdb -am \ - -Dtest=DuckDBOperatorsIT \ - -Dduckdb.url=jdbc:duckdb:platforms-setup-guides/duckdb-setup/data/wayang.duckdb \ - -Dsurefire.failIfNoSpecifiedTests=false -DfailIfNoTests=false \ - -Drat.skip=true -Dlicense.skip=true test -``` - -On PowerShell: - -```powershell -.\mvnw.cmd --% -Pskip-prerequisite-check -pl wayang-platforms/wayang-duckdb -am -Dtest=DuckDBOperatorsIT -Dduckdb.url=jdbc:duckdb:platforms-setup-guides/duckdb-setup/data/wayang.duckdb -Dsurefire.failIfNoSpecifiedTests=false -DfailIfNoTests=false -Drat.skip=true -Dlicense.skip=true test -``` - -Expected result: - -```text -Tests run: 15, Failures: 0, Errors: 0, Skipped: 0 -BUILD SUCCESS -``` - -`DuckDBOperatorsIT` recreates the `wayang_it` fixtures before it runs, so the -test is deterministic even if the local database file already exists. - -## 4. Run Parquet And GCS Tests - -`DuckDBParquetSourceIT` creates a local Parquet file, reads it through DuckDB -auto-created `read_parquet(...)` views, checks URI-to-relation mappings, and -tries a public GCS Parquet smoke through DuckDB `httpfs`. - -```bash -./mvnw -Pskip-prerequisite-check -pl wayang-platforms/wayang-duckdb -am \ - -Dtest=DuckDBParquetSourceIT \ - -Dsurefire.failIfNoSpecifiedTests=false -DfailIfNoTests=false \ - -Drat.skip=true -Dlicense.skip=true test -``` - -On PowerShell: - -```powershell -.\mvnw.cmd --% -Pskip-prerequisite-check -pl wayang-platforms/wayang-duckdb -am -Dtest=DuckDBParquetSourceIT -Dsurefire.failIfNoSpecifiedTests=false -DfailIfNoTests=false -Drat.skip=true -Dlicense.skip=true test -``` - -Override the GCS object with `-Dduckdb.gcs.parquet.uri=gs://bucket/path/file.parquet`. -If DuckDB cannot install/load `httpfs` or reach the object, the GCS smoke is -skipped; the local Parquet tests still run. - -## 5. Run Cost Profiling Smoke - -For a fast local check, run two profiling plans over two small cardinalities: - -```bash -./mvnw -Pskip-prerequisite-check -pl wayang-platforms/wayang-duckdb -am \ - -Dtest=DuckDBCostPilotIT#runPilot \ - -Dduckdb.profile.rowCounts=100,1000 \ - -Dduckdb.profile.plans=S01,S02 \ - -Dduckdb.profile.repetitions=2 \ - -Dsurefire.failIfNoSpecifiedTests=false -DfailIfNoTests=false \ - -Drat.skip=true -Dlicense.skip=true test -``` - -The Trino Week8-style reference pilot is S01-S13 over four cardinalities and -six repetitions, producing 312 Wayang executions: - -```bash -./mvnw -Pskip-prerequisite-check -pl wayang-platforms/wayang-duckdb -am \ - -Dtest=DuckDBCostPilotIT#runPilot \ - -Dduckdb.profile.rowCounts=10000,50000,100000,250000 \ - -Dduckdb.profile.plans=S01,S02,S03,S04,S05,S06,S07,S08,S09,S10,S11,S12,S13 \ - -Dduckdb.profile.repetitions=6 \ - -Dsurefire.failIfNoSpecifiedTests=false -DfailIfNoTests=false \ - -Drat.skip=true -Dlicense.skip=true test -``` - -Outputs are written under `wayang-platforms/wayang-duckdb/target/cost-profiling/`. -Run the GA optimizer outside Surefire, matching the Trino profiling workflow: - -```powershell -.\platforms-setup-guides\duckdb-setup\scripts\run-duckdb-ga.ps1 -``` - -The script writes -`wayang-platforms/wayang-duckdb/target/cost-profiling/duckdb/learned-duckdb-relaxed.properties`. -S14-S16 are implemented for optional expanded runs, but the checked-in reference -parameters are learned from S01-S13. The GA optimizer is stochastic, so repeated -runs over the same execution log can produce slightly different coefficients. - -## 6. Run The Standalone Setup Integration Tests - -The setup directory includes a small Maven project that validates the local -DuckDB database independently of Wayang. Tests are skipped by default; enable -them with `-Pintegration`. - -```bash -./mvnw -f platforms-setup-guides/duckdb-setup/pom.xml \ - -Pintegration -Dtest=DuckDBIntegrationTest test -``` - -On PowerShell: - -```powershell -.\mvnw.cmd --% -f platforms-setup-guides/duckdb-setup/pom.xml -Pintegration -Dtest=DuckDBIntegrationTest test -``` - -Expected result: - -```text -Tests run: 10, Failures: 0, Errors: 0, Skipped: 0 -BUILD SUCCESS -``` - -Override the database file: - -```bash -DUCKDB_JDBC_URL=jdbc:duckdb:/tmp/wayang.duckdb ./mvnw -f platforms-setup-guides/duckdb-setup/pom.xml -Pintegration -Dtest=DuckDBIntegrationTest test -``` - -On PowerShell: - -```powershell -$env:DUCKDB_JDBC_URL="jdbc:duckdb:C:/tmp/wayang.duckdb" -.\mvnw.cmd --% -f platforms-setup-guides/duckdb-setup/pom.xml -Pintegration -Dtest=DuckDBIntegrationTest test -Remove-Item Env:DUCKDB_JDBC_URL -``` - -## 7. Run The Walkthrough Demo - -The optional demo script creates the local DuckDB file, runs the Wayang DuckDB -operator tests against it, runs the standalone JDBC integration tests, and -executes `org.apache.wayang.duckdb.DuckDBDemo`. - -```bash -bash platforms-setup-guides/duckdb-setup/demo.sh -``` - -Set `WAYANG_DEMO_AUTO=true` to skip the interactive pauses. - -## 8. Clean Up - -```bash -rm -f platforms-setup-guides/duckdb-setup/data/wayang.duckdb* -``` - -On PowerShell: - -```powershell -Remove-Item platforms-setup-guides/duckdb-setup/data/wayang.duckdb* -Force -``` diff --git a/platforms-setup-guides/duckdb-setup/demo.sh b/platforms-setup-guides/duckdb-setup/demo.sh deleted file mode 100644 index 2228f5cc0..000000000 --- a/platforms-setup-guides/duckdb-setup/demo.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -WAYANG_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" -DB_FILE="$SCRIPT_DIR/data/wayang.duckdb" -MAVEN_FLAGS="-Pskip-prerequisite-check -Drat.skip=true -Dlicense.skip=true" - -banner() { - echo - echo "============================================================" - printf " %s\n" "$*" - echo "============================================================" - echo -} - -step() { - echo - echo "-- $*" - echo -} - -pause() { - if [[ "${WAYANG_DEMO_AUTO:-false}" != "true" ]]; then - echo - read -rp "Press ENTER to continue..." _ || true - echo - fi -} - -banner "ACT 1: Create a local DuckDB file with Docker" - -step "1a. Running the DuckDB CLI container" -mkdir -p "$SCRIPT_DIR/data" -docker compose -f "$SCRIPT_DIR/docker-compose.yml" run --rm duckdb - -step "1b. Inspecting the local database file" -docker run --rm -i \ - -v "$SCRIPT_DIR:/workspace" \ - duckdb/duckdb:1.5.5 \ - duckdb /workspace/data/wayang.duckdb < "$SCRIPT_DIR/scripts/check.sql" - -pause - -banner "ACT 2: Run Wayang DuckDB tests against that file" - -cd "$WAYANG_ROOT" -./mvnw ${MAVEN_FLAGS} -pl wayang-platforms/wayang-duckdb -am \ - -Dtest=DuckDBOperatorsIT \ - -Dduckdb.url="jdbc:duckdb:$DB_FILE" \ - -Dsurefire.failIfNoSpecifiedTests=false \ - -DfailIfNoTests=false \ - test - -pause - -banner "ACT 3: Run DuckDB Parquet and GCS tests" - -./mvnw ${MAVEN_FLAGS} -pl wayang-platforms/wayang-duckdb -am \ - -Dtest=DuckDBParquetSourceIT \ - -Dsurefire.failIfNoSpecifiedTests=false \ - -DfailIfNoTests=false \ - test - -pause - -banner "ACT 4: Run a DuckDB cost-profiling smoke" - -./mvnw ${MAVEN_FLAGS} -pl wayang-platforms/wayang-duckdb -am \ - -Dtest=DuckDBCostPilotIT#runPilot \ - -Dduckdb.profile.rowCounts=100,1000 \ - -Dduckdb.profile.plans=S01,S02 \ - -Dduckdb.profile.repetitions=2 \ - -Dsurefire.failIfNoSpecifiedTests=false \ - -DfailIfNoTests=false \ - test - -pause - -banner "ACT 5: Run the standalone DuckDB setup integration tests" - -./mvnw -f platforms-setup-guides/duckdb-setup/pom.xml \ - -Pintegration \ - -Dtest=DuckDBIntegrationTest \ - -Dduckdb.url="jdbc:duckdb:$DB_FILE" \ - test - -pause - -banner "ACT 6: Run the Wayang DuckDB demo" - -./mvnw ${MAVEN_FLAGS} -pl wayang-platforms/wayang-duckdb \ - -DskipTests \ - exec:java \ - -Dexec.mainClass=org.apache.wayang.duckdb.DuckDBDemo \ - -Dduckdb.url="jdbc:duckdb:$DB_FILE" - -banner "Demo complete" -echo "DuckDB file: $DB_FILE" diff --git a/platforms-setup-guides/duckdb-setup/docker-compose.yml b/platforms-setup-guides/duckdb-setup/docker-compose.yml deleted file mode 100644 index 9b169fdd3..000000000 --- a/platforms-setup-guides/duckdb-setup/docker-compose.yml +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -services: - duckdb: - image: duckdb/duckdb:1.5.5 - working_dir: /workspace - volumes: - - ./data:/workspace/data - - ./scripts:/workspace/scripts:ro - command: - - duckdb - - /workspace/data/wayang.duckdb - - -init - - /workspace/scripts/init.sql - - -c - - SELECT region, SUM(amount) AS total_amount FROM wayang_it.orders GROUP BY region ORDER BY region; diff --git a/platforms-setup-guides/duckdb-setup/pom.xml b/platforms-setup-guides/duckdb-setup/pom.xml deleted file mode 100644 index 08ef0eb85..000000000 --- a/platforms-setup-guides/duckdb-setup/pom.xml +++ /dev/null @@ -1,81 +0,0 @@ - - - - 4.0.0 - - org.apache.wayang - duckdb-setup - 1.0-SNAPSHOT - jar - - DuckDB Local Setup - Integration Tests - - Standalone integration tests for a local DuckDB database file. - Independent of the Wayang codebase. - - - - 17 - 17 - UTF-8 - 1.5.5.1 - 5.10.2 - true - - - - - org.duckdb - duckdb_jdbc - ${duckdb.version} - test - - - org.junit.jupiter - junit-jupiter - ${junit.version} - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.2.5 - - ${skipIntegrationTests} - - - - - - - - integration - - false - - - - diff --git a/platforms-setup-guides/duckdb-setup/scripts/check.sql b/platforms-setup-guides/duckdb-setup/scripts/check.sql deleted file mode 100644 index f225f44ec..000000000 --- a/platforms-setup-guides/duckdb-setup/scripts/check.sql +++ /dev/null @@ -1,24 +0,0 @@ --- Licensed to the Apache Software Foundation (ASF) under one or more --- contributor license agreements. See the NOTICE file distributed with --- this work for additional information regarding copyright ownership. --- The ASF licenses this file to You under the Apache License, Version 2.0 --- (the "License"); you may not use this file except in compliance with --- the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -SELECT count(*) AS order_count FROM wayang_it.orders; -SELECT region, SUM(amount) AS total_amount -FROM wayang_it.orders -GROUP BY region -ORDER BY region; -SELECT count(*) AS joined_rows -FROM wayang_it.orders -JOIN wayang_it.customers - ON customers.cust_id = orders.customer_id; diff --git a/platforms-setup-guides/duckdb-setup/scripts/init.sql b/platforms-setup-guides/duckdb-setup/scripts/init.sql deleted file mode 100644 index 8a71d376f..000000000 --- a/platforms-setup-guides/duckdb-setup/scripts/init.sql +++ /dev/null @@ -1,46 +0,0 @@ --- Licensed to the Apache Software Foundation (ASF) under one or more --- contributor license agreements. See the NOTICE file distributed with --- this work for additional information regarding copyright ownership. --- The ASF licenses this file to You under the Apache License, Version 2.0 --- (the "License"); you may not use this file except in compliance with --- the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -CREATE SCHEMA IF NOT EXISTS wayang_it; - -DROP TABLE IF EXISTS wayang_it.operator_result; -DROP TABLE IF EXISTS wayang_it.orders; -DROP TABLE IF EXISTS wayang_it.customers; - -CREATE TABLE wayang_it.orders ( - order_id BIGINT, - customer_id BIGINT, - region VARCHAR, - amount DOUBLE -); - -INSERT INTO wayang_it.orders VALUES - (1, 100, 'AMER', 2200.0), - (2, 101, 'EMEA', 800.5), - (3, 100, 'AMER', 680.5), - (4, 102, 'APAC', 1500.0), - (5, 101, 'EMEA', 1100.0), - (6, 100, 'AMER', 950.25); - -CREATE TABLE wayang_it.customers ( - cust_id BIGINT, - name VARCHAR, - tier VARCHAR -); - -INSERT INTO wayang_it.customers VALUES - (100, 'Acme', 'GOLD'), - (101, 'Globex', 'SILVER'), - (102, 'Initech','BRONZE'); diff --git a/platforms-setup-guides/duckdb-setup/scripts/run-duckdb-ga.ps1 b/platforms-setup-guides/duckdb-setup/scripts/run-duckdb-ga.ps1 deleted file mode 100644 index df81fb4ea..000000000 --- a/platforms-setup-guides/duckdb-setup/scripts/run-duckdb-ga.ps1 +++ /dev/null @@ -1,121 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -param( - [string]$Config = "platforms-setup-guides/duckdb-setup/profiling/ga-relaxed.properties", - [string]$Executions = "wayang-platforms/wayang-duckdb/target/cost-profiling/duckdb/executions.json", - [string]$Log = "wayang-platforms/wayang-duckdb/target/cost-profiling/duckdb/ga-relaxed-run.log" -) - -$ErrorActionPreference = "Stop" - -$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path -$root = (Resolve-Path (Join-Path $scriptDir "../../..")).Path -$mvnw = Join-Path $root "mvnw.cmd" - -function Resolve-WayangFileUrl([string]$Path) { - $resolved = (Resolve-Path (Join-Path $root $Path)).Path - return ([System.Uri]$resolved).AbsoluteUri -} - -function Resolve-RepoPath([string]$Path) { - return (Resolve-Path (Join-Path $root $Path)).Path -} - -function Add-IfExists([System.Collections.Generic.List[string]]$Items, [string]$Path) { - if (Test-Path $Path) { - $Items.Add((Resolve-Path $Path).Path) - } -} - -Push-Location $root -try { - $compileArgs = @( - "-Pskip-prerequisite-check", - "-pl", "wayang-profiler,wayang-platforms/wayang-duckdb", - "-am", - "-DskipTests", - "-Drat.skip=true", - "-Dlicense.skip=true", - "compile" - ) - & $mvnw @compileArgs - - if ($LASTEXITCODE -ne 0) { - throw "Maven compile failed with exit code $LASTEXITCODE." - } - - $classpathArgs = @( - "-Pskip-prerequisite-check", - "-pl", "wayang-profiler", - "-DincludeScope=runtime", - "-Dmdep.outputFile=target/duckdb-ga-profiler-classpath.txt", - "-Drat.skip=true", - "-Dlicense.skip=true", - "dependency:build-classpath" - ) - & $mvnw @classpathArgs - - if ($LASTEXITCODE -ne 0) { - throw "Maven classpath generation failed with exit code $LASTEXITCODE." - } - - $dependencyClasspath = Get-Content "wayang-profiler/target/duckdb-ga-profiler-classpath.txt" - $classpathItems = [System.Collections.Generic.List[string]]::new() - - Add-IfExists $classpathItems "$env:USERPROFILE/.m2/repository/org/antlr/antlr4-runtime/4.13.1/antlr4-runtime-4.13.1.jar" - Add-IfExists $classpathItems "$env:USERPROFILE/.m2/repository/com/fasterxml/jackson/core/jackson-core/2.18.6/jackson-core-2.18.6.jar" - - foreach ($classesDir in @( - "wayang-profiler/target/classes", - "wayang-platforms/wayang-duckdb/target/classes", - "wayang-platforms/wayang-jdbc-template/target/classes", - "wayang-platforms/wayang-java/target/classes", - "wayang-platforms/wayang-spark/target/classes", - "wayang-platforms/wayang-postgres/target/classes", - "wayang-platforms/wayang-sqlite3/target/classes", - "wayang-commons/wayang-core/target/classes", - "wayang-commons/wayang-basic/target/classes", - "wayang-commons/wayang-utils-profile-db/target/classes" - )) { - Add-IfExists $classpathItems (Join-Path $root $classesDir) - } - - $classpathItems.Add($dependencyClasspath) - $classpath = [string]::Join([System.IO.Path]::PathSeparator, $classpathItems) - - $argsFile = Join-Path (Split-Path -Parent (Resolve-RepoPath $Executions)) "duckdb-ga.args" - @( - "-cp", - $classpath, - "org.apache.wayang.profiler.log.GeneticOptimizerApp", - (Resolve-WayangFileUrl $Config), - (Resolve-RepoPath $Executions) - ) | Set-Content -Encoding ASCII $argsFile - - & java "@$argsFile" *> (Join-Path $root $Log) - if ($LASTEXITCODE -ne 0) { - Get-Content (Join-Path $root $Log) -Tail 80 - throw "DuckDB GA profiler failed with exit code $LASTEXITCODE." - } - - Write-Host "DuckDB GA profiler completed." - Write-Host "Log: $Log" -} -finally { - Pop-Location -} diff --git a/platforms-setup-guides/duckdb-setup/src/test/java/org/apache/wayang/duckdb/DuckDBIntegrationTest.java b/platforms-setup-guides/duckdb-setup/src/test/java/org/apache/wayang/duckdb/DuckDBIntegrationTest.java deleted file mode 100644 index c1ad0aaa2..000000000 --- a/platforms-setup-guides/duckdb-setup/src/test/java/org/apache/wayang/duckdb/DuckDBIntegrationTest.java +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.wayang.duckdb; - -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.MethodOrderer; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestMethodOrder; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * Standalone JDBC integration tests for the local DuckDB setup. - * - *

Run from the repository root with: - * 

- *   ./mvnw -f platforms-setup-guides/duckdb-setup/pom.xml \
- *     -Pintegration -Dtest=DuckDBIntegrationTest test
- *
- */ -@TestMethodOrder(MethodOrderer.OrderAnnotation.class) -class DuckDBIntegrationTest { - - private static final String JDBC_URL = System.getenv().getOrDefault( - "DUCKDB_JDBC_URL", - System.getProperty("duckdb.url", "jdbc:duckdb:data/wayang.duckdb")); - - private static Connection connection; - - @BeforeAll - static void openConnectionAndLoadFixture() throws Exception { - connection = DriverManager.getConnection(JDBC_URL); - executeSqlScript(Path.of("scripts", "init.sql")); - } - - @AfterAll - static void closeConnection() throws Exception { - if (connection != null && !connection.isClosed()) { - connection.close(); - } - } - - @Test - @Order(1) - @DisplayName("DuckDB responds to SELECT 1") - void connectivity() throws SQLException { - List> rows = query("SELECT 1"); - assertEquals(1, rows.size()); - assertEquals(1, ((Number) rows.get(0).get(0)).intValue()); - } - - @Test - @Order(2) - @DisplayName("Fixture tables are visible") - void fixtureTablesVisible() throws SQLException { - List> rows = query(""" - SELECT table_name - FROM information_schema.tables - WHERE table_schema = 'wayang_it' - ORDER BY table_name - """); - assertEquals(2, rows.size()); - assertEquals("customers", rows.get(0).get(0)); - assertEquals("orders", rows.get(1).get(0)); - } - - @Test - @Order(3) - @DisplayName("Orders table full scan") - void ordersFullScan() throws SQLException { - assertEquals(6L, scalarLong("SELECT COUNT(*) FROM wayang_it.orders")); - } - - @Test - @Order(4) - @DisplayName("Filter by region") - void filterByRegion() throws SQLException { - List> rows = query(""" - SELECT order_id, region - FROM wayang_it.orders - WHERE region = 'AMER' - ORDER BY order_id - """); - assertEquals(3, rows.size()); - rows.forEach(row -> assertEquals("AMER", row.get(1))); - } - - @Test - @Order(5) - @DisplayName("Project subset of columns") - void projection() throws SQLException { - List> rows = query(""" - SELECT region, amount - FROM wayang_it.orders - ORDER BY order_id - LIMIT 3 - """); - assertEquals(3, rows.size()); - assertEquals(2, rows.get(0).size()); - } - - @Test - @Order(6) - @DisplayName("Join orders and customers") - void join() throws SQLException { - assertEquals(6L, scalarLong(""" - SELECT COUNT(*) - FROM wayang_it.orders o - JOIN wayang_it.customers c ON o.customer_id = c.cust_id - """)); - } - - @Test - @Order(7) - @DisplayName("Aggregate total amount by region") - void aggregateByRegion() throws SQLException { - List> rows = query(""" - SELECT region, SUM(amount) AS total_amount - FROM wayang_it.orders - GROUP BY region - ORDER BY region - """); - assertEquals(3, rows.size()); - assertEquals("AMER", rows.get(0).get(0)); - assertEquals(3830.75, ((Number) rows.get(0).get(1)).doubleValue(), 0.01); - } - - @Test - @Order(8) - @DisplayName("Filter by amount threshold") - void filterByAmount() throws SQLException { - List> rows = query(""" - SELECT amount - FROM wayang_it.orders - WHERE amount > 1000.0 - """); - assertFalse(rows.isEmpty()); - rows.forEach(row -> assertTrue(((Number) row.get(0)).doubleValue() > 1000.0)); - } - - @Test - @Order(9) - @DisplayName("Sort by amount") - void sortByAmount() throws SQLException { - List> rows = query(""" - SELECT order_id, amount - FROM wayang_it.orders - ORDER BY amount DESC - LIMIT 1 - """); - assertEquals(1, rows.size()); - assertEquals(1L, ((Number) rows.get(0).get(0)).longValue()); - } - - @Test - @Order(10) - @DisplayName("Create table as select") - void createTableAsSelect() throws SQLException { - try (Statement statement = connection.createStatement()) { - statement.execute("DROP TABLE IF EXISTS wayang_it.operator_result"); - statement.execute(""" - CREATE TABLE wayang_it.operator_result AS - SELECT * FROM wayang_it.orders WHERE region = 'AMER' - """); - } - assertEquals(3L, scalarLong("SELECT COUNT(*) FROM wayang_it.operator_result")); - } - - private static void executeSqlScript(Path script) throws Exception { - String sql = Files.readString(script); - StringBuilder statement = new StringBuilder(); - try (Statement jdbcStatement = connection.createStatement()) { - for (String line : sql.split("\\R")) { - String trimmed = line.trim(); - if (trimmed.startsWith("--") || trimmed.isEmpty()) { - continue; - } - statement.append(line).append('\n'); - if (trimmed.endsWith(";")) { - jdbcStatement.execute(statement.toString()); - statement.setLength(0); - } - } - if (statement.length() > 0) { - jdbcStatement.execute(statement.toString()); - } - } - } - - private static long scalarLong(String sql) throws SQLException { - try (Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery(sql)) { - resultSet.next(); - return resultSet.getLong(1); - } - } - - private static List> query(String sql) throws SQLException { - List> rows = new ArrayList<>(); - try (Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery(sql)) { - int columns = resultSet.getMetaData().getColumnCount(); - while (resultSet.next()) { - List row = new ArrayList<>(); - for (int i = 1; i <= columns; i++) { - row.add(resultSet.getObject(i)); - } - rows.add(row); - } - } - return rows; - } -} diff --git a/wayang-applications/README.md b/wayang-applications/README.md index 3332b3caa..43c361a7c 100644 --- a/wayang-applications/README.md +++ b/wayang-applications/README.md @@ -76,4 +76,7 @@ The file _env.demo1.sh_ contains additional properties, need in a particlar demo In this file we will never see cluster or user specific details, only properties which are specific to the particular application are listed here. +## DuckDB Example +The [DuckDB example](duckdb.md) demonstrates filter and projection plans against +a configurable DuckDB database. diff --git a/wayang-applications/duckdb.md b/wayang-applications/duckdb.md new file mode 100644 index 000000000..54f0a720a --- /dev/null +++ b/wayang-applications/duckdb.md @@ -0,0 +1,70 @@ + + +# DuckDB Example + +`org.apache.wayang.applications.DuckDBDemo` runs filter and projection plans +through Wayang against an embedded DuckDB database. Java 17 is required. No +Docker installation or database server is needed. + +Create a properties file, for example `/tmp/duckdb-example.properties`: + +```properties +wayang.duckdb.jdbc.url = jdbc:duckdb:/tmp/wayang-example.duckdb +wayang.duckdb.demo.orders = wayang_demo.orders +wayang.duckdb.demo.filter-result = wayang_demo.filter_result +wayang.duckdb.demo.projection-result = wayang_demo.projection_result +``` + +Use an absolute database path with an existing parent directory. The example +uses multiple JDBC connections, so it requires a persistent file. On Windows, +use a path such as `C:/Temp/wayang-example.duckdb`. + +From the repository root, build and install the application and its dependencies: + +```bash +./mvnw -Pskip-prerequisite-check -pl wayang-applications -am \ + -DskipTests -Dpython.worker.tests.skip=true install +``` + +To create sample data and run the plans: + +```bash +./mvnw -Pskip-prerequisite-check -pl wayang-applications exec:java \ + -Dexec.mainClass=org.apache.wayang.applications.DuckDBDemo \ + "-Dexec.args=file:///tmp/duckdb-example.properties --init" +``` + +The `--init` option creates six sample orders. It fails if the input table +already exists, so it cannot replace existing input data. If a different input +schema is configured, create that schema before initialization. + +To use an existing table, omit `--init`: + +```bash +./mvnw -Pskip-prerequisite-check -pl wayang-applications exec:java \ + -Dexec.mainClass=org.apache.wayang.applications.DuckDBDemo \ + "-Dexec.args=file:///tmp/duckdb-example.properties" +``` + +The input table must contain `order_id BIGINT`, `customer_id BIGINT`, +`region VARCHAR`, and `amount DOUBLE`. Configure distinct input and output table +names, and create any custom output schemas before running the example. Each run +replaces the two configured output tables. + +On Windows, replace `./mvnw` with `.\mvnw.cmd` and use a configuration URL such +as `file:///C:/Temp/duckdb-example.properties`. diff --git a/wayang-applications/pom.xml b/wayang-applications/pom.xml index 28fba545d..d5ca494a1 100644 --- a/wayang-applications/pom.xml +++ b/wayang-applications/pom.xml @@ -56,6 +56,11 @@ + + org.apache.wayang + wayang-duckdb + ${project.version} + org.apache.wayang wayang-core @@ -104,6 +109,11 @@ 3.9.2 + + com.fasterxml.jackson.core + jackson-core + 2.18.8 + com.fasterxml.jackson.core jackson-databind diff --git a/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/DuckDBDemo.java b/wayang-applications/src/main/java/org/apache/wayang/applications/DuckDBDemo.java similarity index 62% rename from wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/DuckDBDemo.java rename to wayang-applications/src/main/java/org/apache/wayang/applications/DuckDBDemo.java index fee667513..44e9ac8ad 100644 --- a/wayang-platforms/wayang-duckdb/src/main/java/org/apache/wayang/duckdb/DuckDBDemo.java +++ b/wayang-applications/src/main/java/org/apache/wayang/applications/DuckDBDemo.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.wayang.duckdb; +package org.apache.wayang.applications; import org.apache.wayang.basic.data.Record; import org.apache.wayang.basic.function.ProjectionDescriptor; @@ -29,6 +29,7 @@ import org.apache.wayang.core.function.PredicateDescriptor; import org.apache.wayang.core.plan.wayangplan.WayangPlan; import org.apache.wayang.core.types.DataSetType; +import org.apache.wayang.duckdb.DuckDB; import org.apache.wayang.duckdb.operators.DuckDBTableSource; import java.sql.Connection; @@ -38,58 +39,77 @@ import java.util.Properties; /** - * Standalone demo for the Wayang DuckDB platform. - * - *

Run from the repository root with: - * 

- *   ./mvnw -Pskip-prerequisite-check -pl wayang-platforms/wayang-duckdb \
- *     -DskipTests -Drat.skip=true -Dlicense.skip=true exec:java \
- *     -Dexec.mainClass=org.apache.wayang.duckdb.DuckDBDemo
- *
+ * Configurable DuckDB filter and projection example. + * See {@code wayang-applications/duckdb.md} for usage and fixture initialization. */ public class DuckDBDemo { - private static final String JDBC_URL = System.getProperty("duckdb.url", "jdbc:duckdb:target/duckdb-demo.duckdb"); - private static final String SCHEMA = "wayang_demo"; - private static final String ORDERS = SCHEMA + ".orders"; - private static final String FILTER_RESULT = SCHEMA + ".filter_result"; - private static final String PROJECTION_RESULT = SCHEMA + ".projection_result"; + private final Configuration configuration; + private final String jdbcUrl; + private final String orders; + private final String filterResult; + private final String projectionResult; + + private DuckDBDemo(Configuration configuration) { + this.configuration = configuration; + this.jdbcUrl = configuration.getStringProperty("wayang.duckdb.jdbc.url"); + if ("jdbc:duckdb:".equals(jdbcUrl)) { + throw new IllegalArgumentException("This example requires a database file shared by its JDBC connections."); + } + this.orders = configuration.getStringProperty("wayang.duckdb.demo.orders", "wayang_demo.orders"); + this.filterResult = configuration.getStringProperty( + "wayang.duckdb.demo.filter-result", "wayang_demo.filter_result" + ); + this.projectionResult = configuration.getStringProperty( + "wayang.duckdb.demo.projection-result", "wayang_demo.projection_result" + ); + if (orders.equalsIgnoreCase(filterResult) || orders.equalsIgnoreCase(projectionResult) + || filterResult.equalsIgnoreCase(projectionResult)) { + throw new IllegalArgumentException("Input and output tables must have distinct names."); + } + } public static void main(String[] args) throws Exception { - createFixture(); - runFilterPushdown(); - runProjectionPushdown(); + if (args.length < 1 || args.length > 2 || (args.length == 2 && !"--init".equals(args[1]))) { + throw new IllegalArgumentException("Usage: DuckDBDemo [--init]"); + } + DuckDBDemo demo = new DuckDBDemo(new Configuration(args[0])); + if (args.length == 2) { + demo.createFixture(); + } + demo.runFilterPushdown(); + demo.runProjectionPushdown(); } - private static void runFilterPushdown() throws Exception { + private void runFilterPushdown() throws Exception { System.out.println(); System.out.println("DuckDB demo: Filter pushdown"); - System.out.println("SQL shape: SELECT * FROM " + ORDERS + " WHERE region = 'AMER'"); + System.out.println("SQL shape: SELECT * FROM " + orders + " WHERE region = 'AMER'"); DuckDBTableSource source = new DuckDBTableSource( - ORDERS, "order_id", "customer_id", "region", "amount"); + orders, "order_id", "customer_id", "region", "amount"); FilterOperator filter = new FilterOperator<>( new PredicateDescriptor<>( (Record record) -> "AMER".equals(record.getField(2)), Record.class) .withSqlImplementation("region = 'AMER'")); TableSink sink = new TableSink<>( - new Properties(), "overwrite", FILTER_RESULT, + new Properties(), "overwrite", filterResult, "order_id", "customer_id", "region", "amount"); source.connectTo(0, filter, 0); filter.connectTo(0, sink, 0); wayangContext().execute("DuckDB filter demo", new WayangPlan(sink)); - printQuery("SELECT order_id, region, amount FROM " + FILTER_RESULT + " ORDER BY order_id"); + printQuery("SELECT order_id, region, amount FROM " + filterResult + " ORDER BY order_id"); } - private static void runProjectionPushdown() throws Exception { + private void runProjectionPushdown() throws Exception { System.out.println(); System.out.println("DuckDB demo: Projection + filter pushdown"); - System.out.println("SQL shape: SELECT region, amount FROM " + ORDERS + " WHERE region = 'AMER'"); + System.out.println("SQL shape: SELECT region, amount FROM " + orders + " WHERE region = 'AMER'"); DuckDBTableSource source = new DuckDBTableSource( - ORDERS, "order_id", "customer_id", "region", "amount"); + orders, "order_id", "customer_id", "region", "amount"); FilterOperator filter = new FilterOperator<>( new PredicateDescriptor<>( (Record record) -> "AMER".equals(record.getField(2)), Record.class) @@ -101,7 +121,7 @@ private static void runProjectionPushdown() throws Exception { DataSetType.createDefault(Record.class), DataSetType.createDefault(Record.class)); TableSink sink = new TableSink<>( - new Properties(), "overwrite", PROJECTION_RESULT, + new Properties(), "overwrite", projectionResult, "region", "amount"); source.connectTo(0, filter, 0); @@ -109,28 +129,21 @@ private static void runProjectionPushdown() throws Exception { projection.connectTo(0, sink, 0); wayangContext().execute("DuckDB projection demo", new WayangPlan(sink)); - printQuery("SELECT region, amount FROM " + PROJECTION_RESULT + " ORDER BY amount DESC"); + printQuery("SELECT region, amount FROM " + projectionResult + " ORDER BY amount DESC"); } - private static WayangContext wayangContext() { - Configuration configuration = new Configuration(); - configuration.setProperty("wayang.duckdb.jdbc.url", JDBC_URL); - configuration.setProperty("wayang.duckdb.jdbc.user", ""); - configuration.setProperty("wayang.duckdb.jdbc.password", ""); + private WayangContext wayangContext() { return new WayangContext(configuration) .withPlugin(DuckDB.plugin()); } - private static void createFixture() throws Exception { - try (Connection connection = DriverManager.getConnection(JDBC_URL); + private void createFixture() throws Exception { + try (Connection connection = DriverManager.getConnection(jdbcUrl); Statement statement = connection.createStatement()) { - statement.execute("CREATE SCHEMA IF NOT EXISTS " + SCHEMA); - statement.execute("DROP TABLE IF EXISTS " + FILTER_RESULT); - statement.execute("DROP TABLE IF EXISTS " + PROJECTION_RESULT); - statement.execute("DROP TABLE IF EXISTS " + ORDERS); - statement.execute("CREATE TABLE " + ORDERS + " (" + statement.execute("CREATE SCHEMA IF NOT EXISTS wayang_demo"); + statement.execute("CREATE TABLE " + orders + " (" + "order_id BIGINT, customer_id BIGINT, region VARCHAR, amount DOUBLE)"); - statement.execute("INSERT INTO " + ORDERS + " VALUES " + statement.execute("INSERT INTO " + orders + " VALUES " + "(1, 100, 'AMER', 2200.0)," + "(2, 101, 'EMEA', 800.5)," + "(3, 100, 'AMER', 680.5)," @@ -140,8 +153,8 @@ private static void createFixture() throws Exception { } } - private static void printQuery(String sql) throws Exception { - try (Connection connection = DriverManager.getConnection(JDBC_URL); + private void printQuery(String sql) throws Exception { + try (Connection connection = DriverManager.getConnection(jdbcUrl); Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery(sql)) { int columns = resultSet.getMetaData().getColumnCount(); diff --git a/wayang-applications/src/test/java/org/apache/wayang/applications/DuckDBDemoTest.java b/wayang-applications/src/test/java/org/apache/wayang/applications/DuckDBDemoTest.java new file mode 100644 index 000000000..0112e1e39 --- /dev/null +++ b/wayang-applications/src/test/java/org/apache/wayang/applications/DuckDBDemoTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.applications; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.DriverManager; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class DuckDBDemoTest { + @TempDir + Path directory; + + @Test + void initializesOnlyOnRequestAndUsesExistingData() throws Exception { + String jdbcUrl = "jdbc:duckdb:" + directory.resolve("example.duckdb"); + Path config = directory.resolve("example.properties"); + Files.writeString(config, "wayang.duckdb.jdbc.url = " + jdbcUrl.replace("\\", "/") + "\n"); + String configUrl = config.toUri().toString(); + DuckDBDemo.main(new String[]{configUrl, "--init"}); + try (var connection = DriverManager.getConnection(jdbcUrl); + var statement = connection.createStatement()) { + statement.execute("INSERT INTO wayang_demo.orders VALUES (7, 103, 'AMER', 42.0)"); + } + DuckDBDemo.main(new String[]{configUrl}); + try (var connection = DriverManager.getConnection(jdbcUrl); + var statement = connection.createStatement()) { + try (var result = statement.executeQuery("SELECT count(*) FROM wayang_demo.orders")) { + assertTrue(result.next()); + assertEquals(7, result.getInt(1)); + } + try (var result = statement.executeQuery( + "SELECT count(*), sum(amount) FROM wayang_demo.projection_result" + )) { + assertTrue(result.next()); + assertEquals(4, result.getInt(1)); + assertEquals(3872.75, result.getDouble(2), 0.001); + } + } + assertThrows(Exception.class, () -> DuckDBDemo.main(new String[]{configUrl, "--init"})); + try (var connection = DriverManager.getConnection(jdbcUrl); + var statement = connection.createStatement(); + var result = statement.executeQuery("SELECT count(*) FROM wayang_demo.orders")) { + assertTrue(result.next()); + assertEquals(7, result.getInt(1)); + } + } +} diff --git a/wayang-platforms/wayang-duckdb/README.md b/wayang-platforms/wayang-duckdb/README.md index ac439d7ee..b01470911 100644 --- a/wayang-platforms/wayang-duckdb/README.md +++ b/wayang-platforms/wayang-duckdb/README.md @@ -160,39 +160,12 @@ For a quick local smoke: -Drat.skip=true -Dlicense.skip=true test ``` -Then run the GA profiler outside Surefire, as in the Trino profiling branch. -This avoids Maven/Surefire dependency-ordering conflicts around Jackson and -ANTLR. On PowerShell: +The GA configuration is maintained in +[`wayang-profiler`](../../wayang-profiler/duckdb.md), which documents how to run +calibration with the generated execution log on Linux, macOS, and Windows. -```powershell -.\platforms-setup-guides\duckdb-setup\scripts\run-duckdb-ga.ps1 -``` - -The GA settings live in -`platforms-setup-guides/duckdb-setup/profiling/ga-relaxed.properties`. The -learned output is written to -`wayang-platforms/wayang-duckdb/target/cost-profiling/duckdb/learned-duckdb-relaxed.properties`. -The GA optimizer is stochastic; repeated runs over the same execution log can -produce slightly different coefficients. - -## Demo - -`DuckDBDemo` creates a small local fixture and runs two Wayang plans that end in -DuckDB table sinks: - -| Segment | Pushdown shape | -|---------|----------------| -| Filter | `SELECT * FROM wayang_demo.orders WHERE region = 'AMER'` | -| Projection + filter | `SELECT region, amount FROM wayang_demo.orders WHERE region = 'AMER'` | +## Example -Run from the repository root: - -```bash -./mvnw -Pskip-prerequisite-check -pl wayang-platforms/wayang-duckdb -am \ - -DskipTests -Drat.skip=true -Dlicense.skip=true compile - -./mvnw -Pskip-prerequisite-check -pl wayang-platforms/wayang-duckdb \ - -DskipTests -Drat.skip=true -Dlicense.skip=true exec:java \ - -Dexec.mainClass=org.apache.wayang.duckdb.DuckDBDemo \ - -Dduckdb.url=jdbc:duckdb:target/duckdb-demo.duckdb -``` +The configurable filter and projection example lives in +[`wayang-applications`](../../wayang-applications/duckdb.md). +It connects directly through JDBC; no Docker setup is required. diff --git a/wayang-profiler/duckdb.md b/wayang-profiler/duckdb.md new file mode 100644 index 000000000..a0fb9ecbf --- /dev/null +++ b/wayang-profiler/duckdb.md @@ -0,0 +1,46 @@ + + +# DuckDB Cost Calibration + +Generate execution logs using `DuckDBCostPilotIT` as described in the +[platform README](../wayang-platforms/wayang-duckdb/README.md#cost-profiling). +The pilot remains a platform test; the genetic optimizer is provided by this +module. + +From the repository root, install the profiler with the optional DuckDB runtime +(the profile makes DuckDB operators available when reading execution logs): + +```sh +./mvnw -Pskip-prerequisite-check,duckdb -pl wayang-profiler -am -DskipTests -Dpython.worker.tests.skip=true install +``` + +Run the existing optimizer with the calibration configuration and execution log: + +```sh +./mvnw -Pskip-prerequisite-check,duckdb -pl wayang-profiler exec:java -Dexec.mainClass=org.apache.wayang.profiler.log.GeneticOptimizerApp "-Dexec.args=file:///absolute/path/to/wayang/wayang-profiler/src/main/resources/duckdb-ga.properties wayang-platforms/wayang-duckdb/target/cost-profiling/duckdb/executions.json" +``` + +On Windows, replace `./mvnw` with `.\mvnw.cmd`. Paths are relative to the +repository root. Replace the configuration URL with the absolute file URL for +your checkout (on Windows, for example, `file:///C:/src/wayang/wayang-profiler/src/main/resources/duckdb-ga.properties`). No platform-specific launch script or manually assembled Java +classpath is required. + +The settings in `src/main/resources/duckdb-ga.properties` control the run limits +and output location. By default, learned coefficients are written to +`wayang-platforms/wayang-duckdb/target/cost-profiling/duckdb/learned-duckdb-relaxed.properties`. +The optimizer is stochastic, so successive runs can produce different coefficients. diff --git a/wayang-profiler/pom.xml b/wayang-profiler/pom.xml index 77fa88ec5..024740e5a 100644 --- a/wayang-profiler/pom.xml +++ b/wayang-profiler/pom.xml @@ -100,4 +100,31 @@ + + + duckdb + + + org.antlr + antlr4-runtime + 4.13.1 + + + com.fasterxml.jackson.core + jackson-core + 2.18.8 + + + com.fasterxml.jackson.core + jackson-databind + 2.18.9 + + + org.apache.wayang + wayang-duckdb + ${project.version} + + + + diff --git a/platforms-setup-guides/duckdb-setup/profiling/ga-relaxed.properties b/wayang-profiler/src/main/resources/duckdb-ga.properties similarity index 100% rename from platforms-setup-guides/duckdb-setup/profiling/ga-relaxed.properties rename to wayang-profiler/src/main/resources/duckdb-ga.properties