From 52052df84f2a7dae6c3ce7cc8426644d18e3c4f1 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Wed, 2 Sep 2026 16:30:43 -0700 Subject: [PATCH] test(verify): give every operator the same table to read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One table, thirty-four columns and fifteen rows, holding a column of every type an operator can ask for and the awkward values each type admits: an empty cell, a zero, a negative, a string that looks like a number, a date at a boundary. An operator that needs something the table cannot hold — a pair of tables sharing a key, a numeric projection — names a projection of it rather than a fixture of its own, so what every operator reads stays one thing. Co-Authored-By: Claude Opus 5 (1M context) --- .../resources/verify/canonical_fixture.json | 542 ++++++++++++++++++ .../translator/verify/CanonicalFixture.scala | 249 ++++++++ .../verify/CanonicalFixtureSpec.scala | 302 ++++++++++ .../translator/verify/SharedFixture.scala | 178 ++++++ 4 files changed, 1271 insertions(+) create mode 100644 workflow-compiling-service/src/test/resources/verify/canonical_fixture.json create mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CanonicalFixture.scala create mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CanonicalFixtureSpec.scala create mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/SharedFixture.scala diff --git a/workflow-compiling-service/src/test/resources/verify/canonical_fixture.json b/workflow-compiling-service/src/test/resources/verify/canonical_fixture.json new file mode 100644 index 00000000000..59dc4823c14 --- /dev/null +++ b/workflow-compiling-service/src/test/resources/verify/canonical_fixture.json @@ -0,0 +1,542 @@ +[ + { + "id": 7, + "name": "eve", + "score": 0.9, + "open": 7.0, + "high": 8.5, + "low": 6.0, + "close": 7.5, + "iso_country": "IND", + "trade_date": "2024-01-07", + "pvalue": 0.2597402597402597, + "log2fc": 1.4, + "comp_a": 3.0, + "comp_b": 1.0, + "comp_c": 2.0, + "uvec": -3.0, + "edge_pair": "[0, 7]", + "node_src": "n3", + "node_dst": "n4", + "start_ts": "2024-01-07 00:00:00.0", + "finish_ts": "2024-01-07 08:00:00.0", + "uniq_name": "cat_7", + "simplex_a": 35.0, + "simplex_b": 30.0, + "simplex_c": 35.0, + "short_text": "The meeting is scheduled for three o'clock tomorrow.", + "long_text": "The city council voted on Tuesday to approve a new public transit plan. The proposal adds three bus routes and extends weekend service hours. Officials estimate the changes will serve ten thousand additional riders each month. The measure passed by a vote of seven to two after a long public comment period.", + "petal_length": 3.9000000000000004, + "petal_width": 1.3, + "species": 1, + "csv_list": "a7", + "mixed_case": "abacus", + "species_pred": 1, + "species_name": "versicolor", + "species_name_pred": "versicolor" + }, + { + "id": 3, + "name": "bob", + "score": 1.2, + "open": 3.0, + "high": 4.5, + "low": 2.0, + "close": 3.5, + "iso_country": "JPN", + "trade_date": "2024-01-03", + "pvalue": 0.11188811188811189, + "log2fc": -1.4, + "comp_a": 4.0, + "comp_b": 4.0, + "comp_c": 1.0, + "uvec": 0.0, + "edge_pair": "[0, 3]", + "node_src": "n3", + "node_dst": "n4", + "start_ts": "2024-01-03 00:00:00.0", + "finish_ts": "2024-01-03 04:00:00.0", + "uniq_name": "cat_3", + "simplex_a": 35.0, + "simplex_b": 25.0, + "simplex_c": 40.0, + "short_text": "The meeting is scheduled for three o'clock tomorrow.", + "long_text": "The city council voted on Tuesday to approve a new public transit plan. The proposal adds three bus routes and extends weekend service hours. Officials estimate the changes will serve ten thousand additional riders each month. The measure passed by a vote of seven to two after a long public comment period.", + "petal_length": 5.2, + "petal_width": 1.85, + "species": 1, + "csv_list": "a3,b3", + "mixed_case": "ABBEY", + "species_pred": 1, + "species_name": "versicolor", + "species_name_pred": "versicolor" + }, + { + "id": 11, + "name": "1", + "score": 1.7, + "open": 11.0, + "high": 12.5, + "low": 10.0, + "close": 11.5, + "iso_country": "USA", + "trade_date": "2024-01-11", + "pvalue": 0.4075924075924076, + "log2fc": -3.5, + "comp_a": 2.0, + "comp_b": 5.0, + "comp_c": 3.0, + "uvec": 1.0, + "edge_pair": "[0, 11]", + "node_src": "n3", + "node_dst": "n4", + "start_ts": "2024-01-11 00:00:00.0", + "finish_ts": "2024-01-11 04:00:00.0", + "uniq_name": "cat_11", + "simplex_a": 35.0, + "simplex_b": 35.0, + "simplex_c": 30.0, + "short_text": "URGENT: your account needs verification, click the link immediately.", + "long_text": "The city council voted on Tuesday to approve a new public transit plan. The proposal adds three bus routes and extends weekend service hours. Officials estimate the changes will serve ten thousand additional riders each month. The measure passed by a vote of seven to two after a long public comment period.", + "petal_length": 2.6, + "petal_width": 0.75, + "species": 0, + "csv_list": "a11,b11,c11", + "mixed_case": "101", + "species_pred": 0, + "species_name": "setosa", + "species_name_pred": "setosa" + }, + { + "id": 1, + "name": "1", + "score": 0.5, + "open": 1.0, + "high": 2.5, + "low": 0.0, + "close": 1.5, + "iso_country": "USA", + "trade_date": "2024-01-01", + "pvalue": 0.03796203796203796, + "log2fc": -2.8, + "comp_a": 2.0, + "comp_b": 2.0, + "comp_c": 2.0, + "uvec": -2.0, + "edge_pair": "[0, 1]", + "node_src": "n1", + "node_dst": "n2", + "start_ts": "2024-01-01 00:00:00.0", + "finish_ts": "2024-01-01 02:00:00.0", + "uniq_name": "cat_1", + "simplex_a": 25.0, + "simplex_b": 30.0, + "simplex_c": 45.0, + "short_text": "I'm really not sure how I feel about it.", + "long_text": "The city council voted on Tuesday to approve a new public transit plan. The proposal adds three bus routes and extends weekend service hours. Officials estimate the changes will serve ten thousand additional riders each month. The measure passed by a vote of seven to two after a long public comment period.", + "petal_length": 2.6, + "petal_width": 0.75, + "species": 0, + "csv_list": "a1,b1,c1,d1", + "mixed_case": "abdomen", + "species_pred": 1, + "species_name": "setosa", + "species_name_pred": "versicolor" + }, + { + "id": 14, + "name": "carol", + "score": 4.5, + "open": 14.0, + "high": 15.5, + "low": 13.0, + "close": 14.5, + "iso_country": "DEU", + "trade_date": "2024-01-14", + "pvalue": 0.5184815184815185, + "log2fc": -1.4, + "comp_a": 5.0, + "comp_b": 1.0, + "comp_c": 3.0, + "uvec": -3.0, + "edge_pair": "[0, 14]", + "node_src": "n2", + "node_dst": "n3", + "start_ts": "2024-01-14 00:00:00.0", + "finish_ts": "2024-01-14 07:00:00.0", + "uniq_name": "cat_14", + "simplex_a": 30.0, + "simplex_b": 35.0, + "simplex_c": 35.0, + "short_text": "I absolutely love this, it completely made my day!", + "long_text": "Researchers published a study this week describing a new way to recycle lithium batteries. The technique recovers most of the metal at a lower cost than existing methods. The team says a pilot plant could open within two years if funding is secured. Analysts called the results promising but cautioned that scaling up remains difficult.", + "petal_length": 6.5, + "petal_width": 2.4000000000000004, + "species": 1, + "csv_list": "a14", + "mixed_case": "ABILITY", + "species_pred": 1, + "species_name": "versicolor", + "species_name_pred": "versicolor" + }, + { + "id": 5, + "name": "dave", + "score": 2.0, + "open": 5.0, + "high": 6.5, + "low": 4.0, + "close": 5.5, + "iso_country": "GBR", + "trade_date": "2024-01-05", + "pvalue": 0.18581418581418582, + "log2fc": 0.0, + "comp_a": 1.0, + "comp_b": 6.0, + "comp_c": 3.0, + "uvec": 2.0, + "edge_pair": "[0, 5]", + "node_src": "n1", + "node_dst": "n2", + "start_ts": "2024-01-05 00:00:00.0", + "finish_ts": "2024-01-05 06:00:00.0", + "uniq_name": "cat_5", + "simplex_a": 25.0, + "simplex_b": 35.0, + "simplex_c": 40.0, + "short_text": "This is the worst experience I have ever had.", + "long_text": "The city council voted on Tuesday to approve a new public transit plan. The proposal adds three bus routes and extends weekend service hours. Officials estimate the changes will serve ten thousand additional riders each month. The measure passed by a vote of seven to two after a long public comment period.", + "petal_length": 1.3, + "petal_width": 0.2, + "species": 0, + "csv_list": "a5,b5", + "mixed_case": "202", + "species_pred": 0, + "species_name": "setosa", + "species_name_pred": "setosa" + }, + { + "id": 9, + "name": "grace", + "score": 2.4, + "open": 9.0, + "high": 10.5, + "low": 8.0, + "close": 9.5, + "iso_country": "CAN", + "trade_date": "2024-01-09", + "pvalue": 0.3336663336663337, + "log2fc": 2.8, + "comp_a": 5.0, + "comp_b": 3.0, + "comp_c": 1.0, + "uvec": -1.0, + "edge_pair": "[0, 9]", + "node_src": "n1", + "node_dst": "n2", + "start_ts": "2024-01-09 00:00:00.0", + "finish_ts": "2024-01-09 02:00:00.0", + "uniq_name": "cat_9", + "simplex_a": 25.0, + "simplex_b": 25.0, + "simplex_c": 50.0, + "short_text": "I absolutely love this, it completely made my day!", + "long_text": "The city council voted on Tuesday to approve a new public transit plan. The proposal adds three bus routes and extends weekend service hours. Officials estimate the changes will serve ten thousand additional riders each month. The measure passed by a vote of seven to two after a long public comment period.", + "petal_length": 6.5, + "petal_width": 2.4000000000000004, + "species": 1, + "csv_list": "a9,b9,c9", + "mixed_case": "abstract", + "species_pred": 0, + "species_name": "versicolor", + "species_name_pred": "setosa" + }, + { + "id": 2, + "name": "alice", + "score": 3.1, + "open": 2.0, + "high": 3.5, + "low": 1.0, + "close": 2.5, + "iso_country": "CHN", + "trade_date": "2024-01-02", + "pvalue": 0.07492507492507493, + "log2fc": -2.0999999999999996, + "comp_a": 3.0, + "comp_b": 3.0, + "comp_c": 3.0, + "uvec": -1.0, + "edge_pair": "[0, 2]", + "node_src": "n2", + "node_dst": "n3", + "start_ts": "2024-01-02 00:00:00.0", + "finish_ts": "2024-01-02 03:00:00.0", + "uniq_name": "cat_2", + "simplex_a": 30.0, + "simplex_b": 35.0, + "simplex_c": 35.0, + "short_text": "Thank you so much, everything was absolutely perfect.", + "long_text": "Researchers published a study this week describing a new way to recycle lithium batteries. The technique recovers most of the metal at a lower cost than existing methods. The team says a pilot plant could open within two years if funding is secured. Analysts called the results promising but cautioned that scaling up remains difficult.", + "petal_length": 3.9000000000000004, + "petal_width": 1.3, + "species": 1, + "csv_list": "a2,b2,c2,d2", + "mixed_case": "ABACUS", + "species_pred": 1, + "species_name": "versicolor", + "species_name_pred": "versicolor" + }, + { + "id": 13, + "name": "bob", + "score": 2.8, + "open": 13.0, + "high": 14.5, + "low": 12.0, + "close": 13.5, + "iso_country": "JPN", + "trade_date": "2024-01-13", + "pvalue": 0.48151848151848153, + "log2fc": -2.0999999999999996, + "comp_a": 4.0, + "comp_b": 7.0, + "comp_c": 2.0, + "uvec": 3.0, + "edge_pair": "[0, 13]", + "node_src": "n1", + "node_dst": "n2", + "start_ts": "2024-01-13 00:00:00.0", + "finish_ts": "2024-01-13 06:00:00.0", + "uniq_name": "cat_13", + "simplex_a": 25.0, + "simplex_b": 30.0, + "simplex_c": 45.0, + "short_text": "Thank you so much, everything was absolutely perfect.", + "long_text": "The city council voted on Tuesday to approve a new public transit plan. The proposal adds three bus routes and extends weekend service hours. Officials estimate the changes will serve ten thousand additional riders each month. The measure passed by a vote of seven to two after a long public comment period.", + "petal_length": 5.2, + "petal_width": 1.85, + "species": 1, + "csv_list": "a13", + "mixed_case": "303", + "species_pred": 1, + "species_name": "versicolor", + "species_name_pred": "versicolor" + }, + { + "id": 6, + "name": "1", + "score": 5.5, + "open": 6.0, + "high": 7.5, + "low": 5.0, + "close": 6.5, + "iso_country": "FRA", + "trade_date": "2024-01-06", + "pvalue": 0.22277722277722278, + "log2fc": 0.7, + "comp_a": 2.0, + "comp_b": 7.0, + "comp_c": 1.0, + "uvec": 3.0, + "edge_pair": "[0, 6]", + "node_src": "n2", + "node_dst": "n3", + "start_ts": "2024-01-06 00:00:00.0", + "finish_ts": "2024-01-06 07:00:00.0", + "uniq_name": "cat_6", + "simplex_a": 30.0, + "simplex_b": 25.0, + "simplex_c": 45.0, + "short_text": "I'm really not sure how I feel about it.", + "long_text": "Researchers published a study this week describing a new way to recycle lithium batteries. The technique recovers most of the metal at a lower cost than existing methods. The team says a pilot plant could open within two years if funding is secured. Analysts called the results promising but cautioned that scaling up remains difficult.", + "petal_length": 2.6, + "petal_width": 0.75, + "species": 0, + "csv_list": "a6,b6", + "mixed_case": "abbey", + "species_pred": 0, + "species_name": "setosa", + "species_name_pred": "setosa" + }, + { + "id": 10, + "name": "heidi", + "score": 4.2, + "open": 10.0, + "high": 11.5, + "low": 9.0, + "close": 10.5, + "iso_country": "AUS", + "trade_date": "2024-01-10", + "pvalue": 0.3706293706293706, + "log2fc": 3.5, + "comp_a": 1.0, + "comp_b": 4.0, + "comp_c": 2.0, + "uvec": 0.0, + "edge_pair": "[0, 10]", + "node_src": "n2", + "node_dst": "n3", + "start_ts": "2024-01-10 00:00:00.0", + "finish_ts": "2024-01-10 03:00:00.0", + "uniq_name": "cat_10", + "simplex_a": 30.0, + "simplex_b": 30.0, + "simplex_c": 40.0, + "short_text": "This is the worst experience I have ever had.", + "long_text": "Researchers published a study this week describing a new way to recycle lithium batteries. The technique recovers most of the metal at a lower cost than existing methods. The team says a pilot plant could open within two years if funding is secured. Analysts called the results promising but cautioned that scaling up remains difficult.", + "petal_length": 1.3, + "petal_width": 0.2, + "species": 0, + "csv_list": "a10,b10,c10", + "mixed_case": "ABDOMEN", + "species_pred": 1, + "species_name": "setosa", + "species_name_pred": "versicolor" + }, + { + "id": 4, + "name": "carol", + "score": 4.8, + "open": 4.0, + "high": 5.5, + "low": 3.0, + "close": 4.5, + "iso_country": "DEU", + "trade_date": "2024-01-04", + "pvalue": 0.14885114885114886, + "log2fc": -0.7, + "comp_a": 5.0, + "comp_b": 5.0, + "comp_c": 2.0, + "uvec": 1.0, + "edge_pair": "[0, 4]", + "node_src": "n0", + "node_dst": "n1", + "start_ts": "2024-01-04 00:00:00.0", + "finish_ts": "2024-01-04 05:00:00.0", + "uniq_name": "cat_4", + "simplex_a": 20.0, + "simplex_b": 30.0, + "simplex_c": 50.0, + "short_text": "Congratulations! You have won a free prize, reply now to claim it.", + "long_text": "Researchers published a study this week describing a new way to recycle lithium batteries. The technique recovers most of the metal at a lower cost than existing methods. The team says a pilot plant could open within two years if funding is secured. Analysts called the results promising but cautioned that scaling up remains difficult.", + "petal_length": 6.5, + "petal_width": 2.4000000000000004, + "species": 1, + "csv_list": "a4,b4,c4,d4", + "mixed_case": "404", + "species_pred": 1, + "species_name": "versicolor", + "species_name_pred": "versicolor" + }, + { + "id": 15, + "name": "dave", + "score": 3.3, + "open": 15.0, + "high": 16.5, + "low": 14.0, + "close": 15.5, + "iso_country": "GBR", + "trade_date": "2024-01-15", + "pvalue": 0.5554445554445554, + "log2fc": -0.7, + "comp_a": 1.0, + "comp_b": 2.0, + "comp_c": 1.0, + "uvec": -2.0, + "edge_pair": "[0, 15]", + "node_src": "n3", + "node_dst": "n4", + "start_ts": "2024-01-15 00:00:00.0", + "finish_ts": "2024-01-15 08:00:00.0", + "uniq_name": "cat_15", + "simplex_a": 35.0, + "simplex_b": 25.0, + "simplex_c": 40.0, + "short_text": "URGENT: your account needs verification, click the link immediately.", + "long_text": "The city council voted on Tuesday to approve a new public transit plan. The proposal adds three bus routes and extends weekend service hours. Officials estimate the changes will serve ten thousand additional riders each month. The measure passed by a vote of seven to two after a long public comment period.", + "petal_length": 1.3, + "petal_width": 0.2, + "species": 0, + "csv_list": "a15", + "mixed_case": "ability", + "species_pred": 0, + "species_name": "setosa", + "species_name_pred": "setosa" + }, + { + "id": 8, + "name": "frank", + "score": 3.7, + "open": 8.0, + "high": 9.5, + "low": 7.0, + "close": 8.5, + "iso_country": "BRA", + "trade_date": "2024-01-08", + "pvalue": 0.2967032967032967, + "log2fc": 2.0999999999999996, + "comp_a": 4.0, + "comp_b": 2.0, + "comp_c": 3.0, + "uvec": -2.0, + "edge_pair": "[0, 8]", + "node_src": "n0", + "node_dst": "n1", + "start_ts": "2024-01-08 00:00:00.0", + "finish_ts": "2024-01-08 01:00:00.0", + "uniq_name": "cat_8", + "simplex_a": 20.0, + "simplex_b": 35.0, + "simplex_c": 45.0, + "short_text": "See you at lunch, save me a seat by the window.", + "long_text": "Researchers published a study this week describing a new way to recycle lithium batteries. The technique recovers most of the metal at a lower cost than existing methods. The team says a pilot plant could open within two years if funding is secured. Analysts called the results promising but cautioned that scaling up remains difficult.", + "petal_length": 5.2, + "petal_width": 1.85, + "species": 1, + "csv_list": "a8,b8", + "mixed_case": "ABSTRACT", + "species_pred": 1, + "species_name": "versicolor", + "species_name_pred": "versicolor" + }, + { + "id": 12, + "name": "alice", + "score": 5.1, + "open": 12.0, + "high": 13.5, + "low": 11.0, + "close": 12.5, + "iso_country": "CHN", + "trade_date": "2024-01-12", + "pvalue": 0.44455544455544455, + "log2fc": -2.8, + "comp_a": 3.0, + "comp_b": 6.0, + "comp_c": 1.0, + "uvec": 2.0, + "edge_pair": "[0, 12]", + "node_src": "n0", + "node_dst": "n1", + "start_ts": "2024-01-12 00:00:00.0", + "finish_ts": "2024-01-12 05:00:00.0", + "uniq_name": "cat_12", + "simplex_a": 20.0, + "simplex_b": 25.0, + "simplex_c": 55.0, + "short_text": "Congratulations! You have won a free prize, reply now to claim it.", + "long_text": "Researchers published a study this week describing a new way to recycle lithium batteries. The technique recovers most of the metal at a lower cost than existing methods. The team says a pilot plant could open within two years if funding is secured. Analysts called the results promising but cautioned that scaling up remains difficult.", + "petal_length": 3.9000000000000004, + "petal_width": 1.3, + "species": 1, + "csv_list": "a12,b12,c12", + "mixed_case": "505", + "species_pred": 1, + "species_name": "versicolor", + "species_name_pred": "versicolor" + } +] diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CanonicalFixture.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CanonicalFixture.scala new file mode 100644 index 00000000000..a3e4f6a22bd --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CanonicalFixture.scala @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator.verify + +import com.fasterxml.jackson.databind.ObjectMapper +import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple} + +import java.sql.Timestamp +import scala.jdk.CollectionConverters._ + +/** + * The shared input dataset for auto-configured transform verification. + * Properties are deliberate (see CanonicalFixtureSpec): enough rows and + * partial port-0/port-1 overlap to defeat hash-coincidence false passes on + * set ops and joins, and the canonical value "1" present in some-but-not-all + * rows so ConfigGenerator-filled free-form predicates match a proper subset. + */ +object CanonicalFixture extends SharedFixture { + + // Columns are semantically named and type-correct so @SampleColumn-tagged or + // type-constrained fields can be filled with realistic input (a valid OHLC + // block, real ISO country codes, real dates) instead of a degenerate + // first-column pick. Ordering is deliberate: id/name/score lead so the + // first-column fallback AND the type-rule tier ("first column of a matching + // type") are unchanged for un-annotated fields — the domain-specific columns + // that follow are only reached via an explicit @SampleColumn. + val schema: Schema = new Schema( + new Attribute("id", AttributeType.INTEGER), + new Attribute("name", AttributeType.STRING), + new Attribute("score", AttributeType.DOUBLE), + new Attribute("open", AttributeType.DOUBLE), + new Attribute("high", AttributeType.DOUBLE), + new Attribute("low", AttributeType.DOUBLE), + new Attribute("close", AttributeType.DOUBLE), + new Attribute("iso_country", AttributeType.STRING), + new Attribute("trade_date", AttributeType.STRING), + // --- Domain-specific columns (reached only via @SampleColumn) --- + new Attribute("pvalue", AttributeType.DOUBLE), // strictly in (0,1): p-values + new Attribute("log2fc", AttributeType.DOUBLE), // signed, centered on 0: fold-change + new Attribute("comp_a", AttributeType.DOUBLE), // >0 ternary simplex component + new Attribute("comp_b", AttributeType.DOUBLE), // >0 ternary simplex component + new Attribute("comp_c", AttributeType.DOUBLE), // >0 ternary simplex component + new Attribute("uvec", AttributeType.DOUBLE), // any real: a 4th numeric (Quiver u/v) + new Attribute( + "edge_pair", + AttributeType.STRING + ), // "[parent, child]" literals, single-rooted tree + new Attribute("node_src", AttributeType.STRING), // edge source id (Sankey/Network) + new Attribute("node_dst", AttributeType.STRING), // edge target id, overlaps node_src (a DAG) + new Attribute( + "start_ts", + AttributeType.TIMESTAMP + ), // real timestamp; Gantt start / TimeSeries axis + new Attribute( + "finish_ts", + AttributeType.TIMESTAMP + ), // always > start_ts; Gantt finish (bar width) + new Attribute( + "uniq_name", + AttributeType.STRING + ), // distinct per row: Pie/name-keyed ops need no duplicates + new Attribute( + "simplex_a", + AttributeType.DOUBLE + ), // >0 and simplex_a+simplex_b+simplex_c == 100 (ternary-contour) + new Attribute("simplex_b", AttributeType.DOUBLE), // >0 simplex component summing to 100 + new Attribute("simplex_c", AttributeType.DOUBLE), // >0 simplex component summing to 100 + // ── text + iris-numeric columns for Hugging Face model operators ── + new Attribute( + "short_text", + AttributeType.STRING + ), // one sentence: sentiment / spam-detection input + new Attribute( + "long_text", + AttributeType.STRING + ), // a multi-sentence paragraph: summarization input + new Attribute("petal_length", AttributeType.DOUBLE), // iris petal length in cm (~1.3–6.5) + new Attribute("petal_width", AttributeType.DOUBLE), // iris petal width in cm (~0.2–2.4) + new Attribute( + "species", + AttributeType.INTEGER + ), // the 0/1 iris class, exactly `petal_length >= 3.9`: the label the sklearn + // families fit against, separable by the two petal columns above + new Attribute( + "csv_list", + AttributeType.STRING + ), // comma-delimited, 1–4 tokens per row: split/explode ops need real fan-out + new Attribute( + "mixed_case", + AttributeType.STRING + ), // a third lower-case, a third upper, a third letterless: a case flag has to + // change WHICH rows match, and on any other column it changes nothing + new Attribute( + "species_pred", + AttributeType.INTEGER + ), // a predictor's guess at `species`: the same 0/1 domain, wrong on a few rows. + // Scoring compares a PAIR of columns, and no single label column supplies one. + // Last so the first-unused fallback reaches it only after every other column + new Attribute("species_name", AttributeType.STRING), // `species` spelled out + new Attribute( + "species_name_pred", + AttributeType.STRING + ) // `species_pred` spelled out. A scorer takes a string label as readily as a + // numeric one and names the class after it rather than after its position, so + // the pair exists a second time in text + ) + + // ── Data source ── + // The rows are NOT generated at runtime — they live in a single, checked-in, + // human-readable JSON file that IS the source of truth: + // src/test/resources/verify/canonical_fixture.json (15 rows, ids 1..15) + // Open it to see the exact table; edit it to change the data. The + // CanonicalFixtureSpec invariants guard every semantic constraint (valid OHLC + // block, pvalue ∈ (0,1), ternary parts summing to 100, finish_ts > start_ts, + // etc.), so a hand-edit that breaks one fails the build. `schema` above stays + // authoritative for column types: JSON has no TIMESTAMP, so start_ts/finish_ts + // are stored as JDBC strings ("2024-01-01 00:00:00.0") and coerced back here. + private val fixtureResource = "/verify/canonical_fixture.json" + + override val allRows: Vector[Tuple] = { + val stream = Option(getClass.getResourceAsStream(fixtureResource)) + .getOrElse(sys.error(s"canonical fixture not found on classpath: $fixtureResource")) + val root = + try new ObjectMapper().readTree(stream) + finally stream.close() + root + .elements() + .asScala + .map { node => + val b = Tuple.builder(schema) + schema.getAttributes.foreach { attr => + val cell = node.get(attr.getName) + require(cell != null, s"fixture row missing column '${attr.getName}'") + val value: AnyRef = attr.getType match { + case AttributeType.INTEGER => Int.box(cell.asInt()) + case AttributeType.LONG => Long.box(cell.asLong()) + case AttributeType.DOUBLE => Double.box(cell.asDouble()) + case AttributeType.BOOLEAN => Boolean.box(cell.asBoolean()) + case AttributeType.TIMESTAMP => Timestamp.valueOf(cell.asText()) + case _ => cell.asText() // STRING + } + b.add(attr, value) + } + b.build() + } + .toVector + } + + // Each port takes two thirds of the table, from opposite ends, so the ports + // overlap by the remaining third and no row sits outside both. The overlap is + // what stops joins and set ops passing by hash coincidence: ports holding the + // same rows make intersect and union the same answer, and disjoint ports make + // both empty, which a broken operator produces too. Two thirds rather than a + // fixed count so a row added to the JSON widens the windows instead of falling + // off the end, and a port stays well under the whole table — these windows are + // what every per-test Python run is sized by. At 15 rows this is positions 0-9 + // and 5-14. Rows sit out of id order in the file, so the windows are + // positional — not id ranges. + private def windowSize: Int = allRows.size * 2 / 3 + def port0Rows: Seq[Tuple] = allRows.take(windowSize) + def port1Rows: Seq[Tuple] = allRows.takeRight(windowSize) + + override def rowsFor(port: Int): Seq[Tuple] = if (port == 0) port0Rows else port1Rows + + /** `id` keeps every value: it is what joins and set operations match on, and + * emptying it would change which rows pair up rather than what a null does. + */ + override val keepFilled: Set[String] = Set("id") + + /** This table as the sklearn families read it: the two petal columns and the + * `species` label, and nothing else, because `X = table.drop(target, axis=1)` + * hands `fit` every column that is not the target. The two features separate + * the classes exactly, so an estimator fits them without a tie to break. + */ + val sklearnNumeric: SharedFixture = ProjectedFixture( + this, + Seq("petal_length", "petal_width", "species"), + keepFilled = Set("species") + ) + + /** [[sklearnNumeric]] plus a column an estimator cannot fit. The families that + * narrow `X` to the fittable columns drop nothing on the numeric table, so the + * narrowing runs there with nothing to do. Here it has a column to drop, and + * the two paths narrow in different places: the operator once, ahead of the + * port branch; the standalone script once per port. Each has to drop it on its + * own. + * + * The text column carries no signal about the label, so the fit is the one the + * two petal columns give on their own. + */ + /** This table minus `score`. An operator whose output column is named `score` + * by default cannot run here otherwise: it would create a column the input + * already holds, and the schema refuses the duplicate before the operator + * runs. Dropping the one column puts the DEFAULT config under test, which is + * the config a user gets, rather than a hand-written name chosen to dodge the + * clash. + */ + val withoutScore: SharedFixture = ProjectedFixture( + this, + schema.getAttributeNames.filterNot(_ == "score"), + keepFilled = keepFilled + ) + + /** This table as a scorer reads it when the labels are text: the same pair as + * `species` / `species_pred`, spelled out, and nothing else. The scenario that + * takes it names the two columns itself, since the operator's `@SampleColumn`s + * name the numeric pair this projection does not carry. + */ + val scorerTextLabels: SharedFixture = ProjectedFixture( + this, + Seq("species_name", "species_name_pred"), + keepFilled = Set.empty + ) + + val sklearnNumericWithText: SharedFixture = ProjectedFixture( + this, + Seq("petal_length", "petal_width", "short_text", "species"), + keepFilled = Set("species") + ) + + /** This table as the `countVectorizer=true` path reads it: one text column and + * the same label. `short_text` leads because the model probe feeds a text + * pipeline the frame's first column as a Series. Every row carrying a given + * sentence carries the same `species` (an invariant of the table), so the + * vectorized classes separate exactly, as the numeric pair does. + */ + val sklearnText: SharedFixture = ProjectedFixture( + this, + Seq("short_text", "long_text", "species"), + keepFilled = Set("species") + ) +} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CanonicalFixtureSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CanonicalFixtureSpec.scala new file mode 100644 index 00000000000..aa471e30f1d --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CanonicalFixtureSpec.scala @@ -0,0 +1,302 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator.verify + +import org.apache.texera.amber.core.workflow.PortIdentity +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.nio.file.Files + +class CanonicalFixtureSpec extends AnyFlatSpec with Matchers { + + "CanonicalFixture" should "have at least 10 rows per port with partial id overlap" in { + CanonicalFixture.port0Rows.size should be >= 10 + CanonicalFixture.port1Rows.size should be >= 10 + val ids0 = CanonicalFixture.port0Rows.map(_.getField[Integer]("id")).toSet + val ids1 = CanonicalFixture.port1Rows.map(_.getField[Integer]("id")).toSet + (ids0 intersect ids1) should not be empty + (ids0 diff ids1) should not be empty + (ids1 diff ids0) should not be empty + } + + // The windows are a rule over the table's size rather than fixed indices, so + // that a row added to the JSON is read by a port instead of falling off the + // end. Fixed indices made that silent: the row was simply never fed to + // anything, and every assertion above still held. + it should "leave no row outside both ports" in { + val onAPort = (CanonicalFixture.port0Rows ++ CanonicalFixture.port1Rows) + .map(_.getField[Integer]("id")) + .toSet + onAPort shouldBe CanonicalFixture.allRows.map(_.getField[Integer]("id")).toSet + } + + it should "contain the canonical value \"1\" in some but not all name cells" in { + val names = CanonicalFixture.port0Rows.map(_.getField[String]("name")) + names.count(_ == "1") should be > 0 + names.count(_ == "1") should be < names.size + } + + it should "expose a valid OHLC block (high >= open/close >= low) for candlestick-style ops" in { + CanonicalFixture.port0Rows.foreach { t => + val o = t.getField[java.lang.Double]("open").doubleValue + val h = t.getField[java.lang.Double]("high").doubleValue + val l = t.getField[java.lang.Double]("low").doubleValue + val c = t.getField[java.lang.Double]("close").doubleValue + h should be >= math.max(o, c) + l should be <= math.min(o, c) + } + } + + it should "keep pvalue strictly inside (0, 1) for probability-domain fields" in { + CanonicalFixture.port0Rows.foreach { t => + val p = t.getField[java.lang.Double]("pvalue").doubleValue + p should (be > 0.0 and be < 1.0) + } + } + + it should "keep log2fc signed and centered (both a negative and a positive present)" in { + val vals = CanonicalFixture.port0Rows.map(_.getField[java.lang.Double]("log2fc").doubleValue) + vals.min should be < 0.0 + vals.max should be > 0.0 + } + + it should "keep ternary components strictly positive" in { + CanonicalFixture.port0Rows.foreach { t => + t.getField[java.lang.Double]("comp_a").doubleValue should be > 0.0 + t.getField[java.lang.Double]("comp_b").doubleValue should be > 0.0 + t.getField[java.lang.Double]("comp_c").doubleValue should be > 0.0 + } + } + + it should "expose uniq_name as globally distinct so name-keyed ops have no duplicates" in { + val names = CanonicalFixture.port0Rows.map(_.getField[String]("uniq_name")) + names.distinct.size shouldBe names.size + } + + it should "expose a valid ternary simplex (positive parts summing to 100)" in { + CanonicalFixture.port0Rows.foreach { t => + val a = t.getField[java.lang.Double]("simplex_a").doubleValue + val b = t.getField[java.lang.Double]("simplex_b").doubleValue + val c = t.getField[java.lang.Double]("simplex_c").doubleValue + a should be > 0.0 + b should be > 0.0 + c should be > 0.0 + (a + b + c) shouldBe 100.0 +- 1e-9 + } + } + + it should "expose trade_date as a real ISO-8601 date (parseable, not the old day-N)" in { + CanonicalFixture.port0Rows.foreach { t => + val d = t.getField[String]("trade_date") + noException should be thrownBy java.time.LocalDate.parse(d) + } + } + + it should "expose edge_pair as single-rooted 2-element list literals" in { + // Every cell is "[0, child]" → parses to a 2-list rooted at 0, so TreePlot + // builds one connected tree instead of an error page. + CanonicalFixture.port0Rows.foreach { t => + t.getField[String]("edge_pair") should fullyMatch regex """\[0, \d+\]""" + } + } + + it should "expose overlapping node_src/node_dst so graph ops have drawable edges" in { + val src = CanonicalFixture.port0Rows.map(_.getField[String]("node_src")).toSet + val dst = CanonicalFixture.port0Rows.map(_.getField[String]("node_dst")).toSet + (src intersect dst) should not be empty + } + + it should "expose finish_ts strictly after start_ts (non-degenerate Gantt bar)" in { + CanonicalFixture.port0Rows.foreach { t => + val s = t.getField[java.sql.Timestamp]("start_ts") + val f = t.getField[java.sql.Timestamp]("finish_ts") + f.after(s) shouldBe true + } + } + + it should "round-trip TIMESTAMP columns losslessly through TupleIO (write then read)" in { + val root = Files.createTempDirectory("canonical-fixture-ts-") + val path = CanonicalFixture.writeInputs(root, inputPortCount = 1)(PortIdentity(0)) + val schema = TupleIO.readSchemaSidecar(path) + val rows = TupleIO.readTuples(path, schema).toList + rows should not be empty + val read = rows.head + val orig = CanonicalFixture.port0Rows.head + // The JDBC-string codec is the exact inverse of Timestamp.toString, so the + // value read back equals the value written — no timezone drift. + read.getField[java.sql.Timestamp]("start_ts") shouldBe orig.getField[java.sql.Timestamp]( + "start_ts" + ) + read.getField[java.sql.Timestamp]("finish_ts") shouldBe orig.getField[java.sql.Timestamp]( + "finish_ts" + ) + } + + it should "expose non-empty short_text sentences for text-classification ops" in { + CanonicalFixture.port0Rows.foreach { t => + t.getField[String]("short_text").trim should not be empty + } + } + + it should "expose long_text with several sentences so summarization is non-trivial" in { + CanonicalFixture.port0Rows.foreach { t => + val txt = t.getField[String]("long_text") + // multiple sentence-terminating periods → real content to condense + txt.count(_ == '.') should be >= 2 + } + } + + // The sklearn families fit `species` against the petal columns, so the table + // has to hold up as training data. The estimators that cross-validate pass no + // fold count and so take sklearn's default of five, and a class of fewer than + // five rows leaves a fold holding none of it — no error, just a warning and a + // fold that asks nothing. A label the features cannot separate is the other + // half: it leaves the fit breaking ties, which is where two paths drift apart. + it should "expose species as a petal-separable label with enough members to fold on" in { + val rows = CanonicalFixture.allRows + val byClass = rows.groupBy(_.getField[java.lang.Integer]("species").intValue) + byClass.keySet shouldBe Set(0, 1) + byClass.values.foreach(_.size should be >= 5) + rows.foreach { t => + val large = t.getField[java.lang.Double]("petal_length").doubleValue >= 3.9 + t.getField[java.lang.Integer]("species").intValue shouldBe (if (large) 1 else 0) + } + } + + // `species_pred` exists so the scorer has a real pair to compare. All four + // cells of the confusion matrix have to be occupied: a perfect prediction + // scores every metric at 1.0, and one that never calls a class leaves that + // class's precision undefined — either way the metrics stop telling the two + // code paths apart. + it should "hold a species_pred that is right on most rows and wrong on some" in { + val cells = CanonicalFixture.allRows + .map(t => + ( + t.getField[java.lang.Integer]("species").intValue, + t.getField[java.lang.Integer]("species_pred").intValue + ) + ) + .distinct + cells should contain theSameElementsAs Seq((0, 0), (0, 1), (1, 0), (1, 1)) + } + + // The text pair exists to run the scorer's string-label path on the same + // arrangement the numeric pair gives it. Spelling out a different prediction + // would make the two paths score differently for a reason that has nothing to + // do with the label being text. + it should "spell out the species pair without changing what it says" in { + val name = Map(0 -> "setosa", 1 -> "versicolor") + CanonicalFixture.allRows.foreach { t => + t.getField[String]("species_name") shouldBe name( + t.getField[java.lang.Integer]("species").intValue + ) + t.getField[String]("species_name_pred") shouldBe name( + t.getField[java.lang.Integer]("species_pred").intValue + ) + } + } + + // The countVectorizer=true path fits the same label on short_text alone, and a + // sentence appearing under both labels makes that set unlearnable. + it should "keep every short_text sentence inside one species" in { + CanonicalFixture.allRows + .groupBy(_.getField[String]("short_text")) + .foreach { + case (sentence, rows) => + withClue(s"$sentence: ") { + rows.map(_.getField[java.lang.Integer]("species")).distinct.size shouldBe 1 + } + } + } + + it should "expose iris petal columns in a realistic centimetre range" in { + CanonicalFixture.port0Rows.foreach { t => + val len = t.getField[java.lang.Double]("petal_length").doubleValue + val wid = t.getField[java.lang.Double]("petal_width").doubleValue + len should (be > 0.0 and be < 8.0) + wid should (be > 0.0 and be < 3.0) + } + } + + // A single-token row would make split/explode a no-op, so both windows need + // rows that fan out AND a row that doesn't — the two branches of an unnest. + it should "expose csv_list as a clean delimited list with varying token counts" in { + Seq(CanonicalFixture.port0Rows, CanonicalFixture.port1Rows).foreach { rows => + val tokenCounts = rows.map { t => + val raw = t.getField[String]("csv_list") + raw should not startWith "," + raw should not endWith "," + val tokens = raw.split(",", -1) + tokens.foreach(_.trim should not be empty) + tokens.length + } + tokenCounts.min shouldBe 1 + tokenCounts.max should be > 1 + } + } + + // A case flag is only worth sweeping where flipping it changes WHICH rows match, + // which needs rows of all three kinds in EVERY window a test reads — hence the + // per-port check rather than one over the whole table. + it should "expose mixed_case with lower, upper and letterless rows on every port" in { + Seq(CanonicalFixture.port0Rows, CanonicalFixture.port1Rows).foreach { rows => + val values = rows.map(_.getField[String]("mixed_case")) + values.count(v => v.exists(_.isLower)) should be > 0 + values.count(v => v.exists(_.isUpper) && !v.exists(_.isLower)) should be > 0 + values.count(v => !v.exists(_.isLetter)) should be > 0 + } + } + + it should "write one JSONL fixture per requested input port" in { + val root = Files.createTempDirectory("canonical-fixture-") + val inputs = CanonicalFixture.writeInputs(root, inputPortCount = 2) + inputs.keySet shouldBe Set(PortIdentity(0), PortIdentity(1)) + inputs.values.foreach(p => Files.size(p) should be > 0L) + } + + it should "reject unsupported port counts" in { + val root = Files.createTempDirectory("canonical-fixture-") + an[IllegalArgumentException] should be thrownBy + CanonicalFixture.writeInputs(root, inputPortCount = 3) + } + + it should "empty every column but id exactly once in the gapped table" in { + val rows = CanonicalFixture.emptyOneCellPerColumn(CanonicalFixture.port0Rows) + rows.size shouldBe CanonicalFixture.port0Rows.size + + CanonicalFixture.schema.getAttributes.foreach { attr => + val empties = rows.count(_.getField[AnyRef](attr.getName) == null) + // id carries the joins, so it keeps every value; everything else gets one + // hole, which is what makes the case a null case rather than an empty table. + if (attr.getName == "id") empties shouldBe 0 + else empties shouldBe 1 + } + } + + it should "leave every row in the gapped table with something in it" in { + val names = CanonicalFixture.schema.getAttributes.map(_.getName) + CanonicalFixture.emptyOneCellPerColumn(CanonicalFixture.port0Rows).foreach { t => + // A wholly empty row would test the operator's handling of an empty table + // instead, and would say nothing about a null beside a filled neighbour. + names.count(n => t.getField[AnyRef](n) != null) should be > 0 + } + } +} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/SharedFixture.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/SharedFixture.scala new file mode 100644 index 00000000000..f42c1ca6897 --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/SharedFixture.scala @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator.verify + +import org.apache.texera.amber.core.tuple.{Schema, Tuple} +import org.apache.texera.amber.core.workflow.PortIdentity + +import java.nio.file.Path + +/** + * A checked-in table a whole FAMILY of operators runs on, as opposed to one + * written for a single operator. + * + * Which table an operator runs on is its own axis, separate from who writes its + * config. [[CanonicalFixture]] is the wide mixed-type table every operator takes; + * the sklearn families take [[ProjectedFixture]] views of it, since + * `X = table.drop(target)` feeds every remaining column to `fit`, which a string + * or timestamp column ends. + */ +trait SharedFixture { + + def schema: Schema + + /** The rows port `port` gets. Ports may take different windows of the table + * (canonical overlaps them partially, to defeat hash-coincidence passes on + * joins) or the same rows twice. + */ + def rowsFor(port: Int): Seq[Tuple] + + /** Every row of the table, ports aside — what a [[ProjectedFixture]] of it + * narrows. A table whose ports read windows says so by overriding; by default + * a port already sees the whole of it. + */ + def allRows: Seq[Tuple] = rowsFor(0) + + /** Columns [[write]] never empties, because their VALUE is what the table was + * built to arrange rather than data under test: canonical's `id` is what joins + * and set operations pair rows on, and a sklearn table's label is what its + * estimator fits against. Emptying one of those changes what the test asks + * instead of asking what an operator does with a null. + */ + def keepFilled: Set[String] + + /** Write one JSONL file per 0-based input port under `dir`. At most 2 ports. */ + final def write( + dir: Path, + inputPortCount: Int, + withGaps: Boolean + ): Map[PortIdentity, Path] = { + require( + inputPortCount >= 1 && inputPortCount <= 2, + s"unsupported input port count: $inputPortCount" + ) + (0 until inputPortCount).map { port => + val rows = rowsFor(port) + val path = dir.resolve(s"input_port_$port.jsonl") + TupleIO.writeTuples( + path, + (if (withGaps) emptyOneCellPerColumn(rows) else rows).iterator, + schema + ) + PortIdentity(port) -> path + }.toMap + } + + /** Schemas ConfigGenerator resolves @AutofillAttributeName fields against. + * Every port sees the same columns: a fixture's ports differ in which ROWS + * they get, not in shape. + */ + final def schemasByPort: Map[Int, Schema] = Map(0 -> schema, 1 -> schema) + + /** Write one JSONL fixture per 0-based input port, every cell filled. */ + final def writeInputs(dir: Path, inputPortCount: Int): Map[PortIdentity, Path] = + write(dir, inputPortCount, withGaps = false) + + /** How many rows port 0 gets — what a row-count-sensitive knob (`limit`, + * `offset`) is sized against so its value keeps some rows and drops some. + */ + final def port0RowCount: Int = rowsFor(0).size + + /** This table's rows with [[SharedFixture.emptyOneCellPerColumn]] applied. */ + private[verify] def emptyOneCellPerColumn(rows: Seq[Tuple]): Seq[Tuple] = + SharedFixture.emptyOneCellPerColumn(rows, schema, keepFilled) +} + +/** + * A column subset of another table: the same rows in the same order, keeping + * only the named columns, in the order named. + * + * The sklearn families need one. Their generated code is + * `X = table.drop(target, axis=1)`, so every column that is not the target + * reaches `fit`, and a string or a timestamp ends it. A projection hands them a + * table an estimator can fit without a second dataset to keep in step: the rows + * are still [[CanonicalFixture]]'s, only narrower. + */ +final case class ProjectedFixture( + source: SharedFixture, + columns: Seq[String], + keepFilled: Set[String] +) extends SharedFixture { + + val schema: Schema = new Schema(columns.map(c => source.schema.getAttribute(c)): _*) + + private val rows: Vector[Tuple] = source.allRows.map { t => + val b = Tuple.builder(schema) + schema.getAttributes.foreach(a => b.add(a, t.getField[AnyRef](a.getName))) + b.build() + }.toVector + + /** Every port gets the whole table. An estimator pair trains on port 0 and + * tests on port 1, and the point of the pair is the two ports rather than two + * datasets: what the comparison sees is the fitted model, which port 1 has no + * hand in, so giving the ports different rows buys nothing. + * + * The whole table rather than the source's ten-row window, because the + * estimators that cross-validate pass no fold count and so take sklearn's + * default of five: the window would leave the smaller class at four, and one + * fold holding none of a class is a fold that asks nothing (sklearn warns and + * splits anyway rather than refusing). + */ + override def rowsFor(port: Int): Seq[Tuple] = rows +} + +object SharedFixture { + + /** One empty cell per column, spread across rows so no row is wholly empty — an + * operator that reads two columns should still meet a row where one is filled + * and the other is not. Placement is by column position, so it is the same on + * every run. + * + * Free-standing rather than a member, because a curated handler's table has no + * [[SharedFixture]] behind it: the runner reads back the rows the handler wrote + * and punches the holes here. + */ + def emptyOneCellPerColumn( + rows: Seq[Tuple], + schema: Schema, + keepFilled: Set[String] + ): Seq[Tuple] = { + if (rows.isEmpty) return rows + val holes: Map[Int, Set[String]] = schema.getAttributes.zipWithIndex + .filterNot { case (attr, _) => keepFilled.contains(attr.getName) } + .map { case (attr, i) => (i % rows.size) -> attr.getName } + .groupBy(_._1) + .map { case (row, pairs) => row -> pairs.map(_._2).toSet } + rows.zipWithIndex.map { + case (t, rowIdx) => + val emptied = holes.getOrElse(rowIdx, Set.empty) + if (emptied.isEmpty) t + else { + val b = Tuple.builder(schema) + schema.getAttributes.foreach { attr => + val v: AnyRef = + if (emptied.contains(attr.getName)) null else t.getField[AnyRef](attr.getName) + b.add(attr, v) + } + b.build() + } + } + } +}