From 9cd782b9ca759cee54c9692ba51b4b291046636a Mon Sep 17 00:00:00 2001 From: kary zheng Date: Wed, 2 Sep 2026 16:30:45 -0700 Subject: [PATCH 01/10] test(verify): run every operator both ways and report what it did The runner discovers the operators that implement the trait, configures each from its schema, runs it through the engine and through its generated script, and compares. An operator that cannot be run is reported as a row with the reason rather than passed over, since an operator missing from the report is a fact the report has to carry. `OperatorBehaviorSpec` is what turns that into tests: one per operator per configuration its schema offers, named so a single operator can be run alone. Co-Authored-By: Claude Opus 5 (1M context) --- .../verify/OperatorBehaviorSpec.scala | 151 +++ .../verify/TransformVerificationRunner.scala | 962 ++++++++++++++++++ .../TransformVerificationRunnerSpec.scala | 76 ++ 3 files changed, 1189 insertions(+) create mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/OperatorBehaviorSpec.scala create mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala create mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunnerSpec.scala diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/OperatorBehaviorSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/OperatorBehaviorSpec.scala new file mode 100644 index 00000000000..f6d8b6759be --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/OperatorBehaviorSpec.scala @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator.verify + +import com.fasterxml.jackson.annotation.JsonSubTypes +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} +import org.apache.texera.amber.operator.source.SourceOperatorDescriptor +import org.apache.texera.amber.translator.verify.tags.IntegrationTest +import org.scalatest.ParallelTestExecution +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** + * Auto-discovered behavioral-parity tests: for every operator registered + * with [[LogicalOp]]'s `@JsonSubTypes` that implements + * [[StandaloneCodeGenerator]], emit a test that runs both Path A (Texera + * exec) and Path B (translator-generated Python via [[StandaloneRunner]]) + * and asserts their outputs are equivalent. + * + * Dispatch is auto-first: [[TransformVerificationRunner]] classifies each + * non-source transform as `Runnable("auto")` (auto-configured fixture), + * `Runnable("curated")` (hand-written fixture from [[CuratedHandlers]]), + * or `Flagged(reason)` (shown as ignored with the reason in the test name). + * Sources route to [[SourceCategoryRunner]] unchanged. + * + * No edits to this spec are needed when a new operator is added — reflection + * discovers it automatically via `@JsonSubTypes`. The tier label appears in + * the test name so the report shows which path exercised each operator. + * + * Requires Python 3 with pandas on the [[Comparator]] / [[StandaloneRunner]] + * resolution chain (`UDF_PYTHON_PATH` env var, then `python3.12`). + */ +// Tagged @IntegrationTest: this is the only verify spec that forks a real +// Python process end-to-end, so CI routes it to the Python-provisioned +// integration job (see workflow-compiling-service/build.sbt WCS_TEST_FILTER). +@IntegrationTest +class OperatorBehaviorSpec extends AnyFlatSpec with Matchers with ParallelTestExecution { + + // Build the test list at class construction. Each branch below registers + // one test (`in` for runnable, `ignore` for skipped) so the test report + // shows every translator-eligible operator and why it did or didn't run. + OperatorBehaviorSpec.discoverStandaloneOperators().foreach { opClass => + val name = opClass.getSimpleName + + if (!OperatorBehaviorSpec.isSelected(name)) { + // Narrowed out by VERIFY_ONLY / VERIFY_SKIP, which only a local run sets. + // Still registered, as an `ignore`, so the report lists every operator + // rather than reading as though the narrowed-out ones do not exist. + name should "NARROWED OUT — outside this run's VERIFY_ONLY / VERIFY_SKIP" ignore {} + } else if (classOf[SourceOperatorDescriptor].isAssignableFrom(opClass)) { + // Sources keep their handler-per-source design: each needs a real file + // in its specific format, which a generic fixture can't supply. + if (SourceCategoryRunner.canRun(opClass)) { + name should "produce equivalent output in Texera and standalone Python (source)" in { + SourceCategoryRunner.run(opClass) + } + } else { + name should s"FLAGGED — ${SourceCategoryRunner.flagReason(opClass)}" ignore {} + } + } else { + TransformVerificationRunner.disposition(opClass) match { + case TransformVerificationRunner.Runnable(tier) => + name should s"produce equivalent output in Texera and standalone Python ($tier)" in { + TransformVerificationRunner.run(opClass) + } + case TransformVerificationRunner.Flagged(reason) => + name should s"FLAGGED — $reason" ignore { + // Reason is in the test name so the report carries it; the + // coverage table in ConfigCoverageSpec aggregates these. + } + } + } + } + + // Not one test per operator like the rest of this spec: it is one assertion + // over all of them, and it deliberately ignores the selection knobs above so a + // VERIFY_ONLY run still cannot hide a broken splice site. + "Generated standalone code" should "stay parseable when the column names are hostile" in { + StandaloneEscapingCheck.run() shouldBe empty + } +} + +object OperatorBehaviorSpec { + + // Narrowing knobs for a local run, both unset by default, so the default run + // is every operator: VERIFY_ONLY names the only ones to run, VERIFY_SKIP the + // ones to leave out. Case-sensitive substrings against the operator's simple + // name, comma-separated. Neither is set in CI, which therefore runs the lot. + // + // There is deliberately no third list withholding operators by default. What + // stays withheld is narrower than an operator and lives where it can say why: + // a single variant in [[TransformVerificationRunner.variantsNotRun]], or an + // operator that cannot be run at all in its `knownIssues`, each against an + // issue or a reason. A name here would withdraw an operator's every variant + // and record nothing about what is wrong with it. + private def patterns(envVar: String): Seq[String] = + sys.env.getOrElse(envVar, "").split(",").iterator.map(_.trim).filter(_.nonEmpty).toSeq + + private lazy val onlyPatterns: Seq[String] = patterns("VERIFY_ONLY") + private lazy val skipPatterns: Seq[String] = patterns("VERIFY_SKIP") + + /** True if `name` should run: in VERIFY_ONLY when that is set, and not in + * VERIFY_SKIP. True for everything when neither is set. + */ + def isSelected(name: String): Boolean = { + val included = onlyPatterns.isEmpty || onlyPatterns.exists(name.contains) + val excluded = skipPatterns.exists(name.contains) + included && !excluded + } + + /** + * Enumerates every concrete subclass of [[LogicalOp]] declared in its + * `@JsonSubTypes` annotation, filters to those implementing + * [[StandaloneCodeGenerator]], and returns them sorted by simple name + * (stable test report order). + * + * Uses the same registry Jackson uses to deserialize operators — no + * separate discovery mechanism needed. Adding an operator to + * `LogicalOp.@JsonSubTypes` makes it visible here automatically. + */ + def discoverStandaloneOperators(): Seq[Class[_ <: LogicalOp]] = { + val annotation = classOf[LogicalOp].getAnnotation(classOf[JsonSubTypes]) + if (annotation == null) Seq.empty + else + annotation + .value() + .toSeq + .map(_.value()) + .filter(classOf[StandaloneCodeGenerator].isAssignableFrom) + .map(_.asInstanceOf[Class[_ <: LogicalOp]]) + .distinct + .sortBy(_.getSimpleName) + } +} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala new file mode 100644 index 00000000000..590e0c361ce --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala @@ -0,0 +1,962 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator.verify + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.{BooleanNode, IntNode, TextNode} +import org.apache.texera.amber.core.tuple.{AttributeType, Schema} +import org.apache.texera.amber.core.workflow.PortIdentity +import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.{ + LogicalOp, + PythonOperatorDescriptor, + StandaloneCodeGenerator +} +import org.apache.texera.amber.operator.aggregate.AggregateOpDesc +import org.apache.texera.amber.operator.dummy.DummyOpDesc +import org.apache.texera.amber.operator.filter.SpecializedFilterOpDesc +import org.apache.texera.amber.operator.sleep.SleepOpDesc +import org.apache.texera.amber.operator.split.SplitOpDesc +import org.apache.texera.amber.operator.sklearn.SklearnPredictionOpDesc +import org.apache.texera.amber.operator.sklearn.SklearnClassifierOpDesc +import org.apache.texera.amber.operator.sklearn.SklearnGaussianNaiveBayesOpDesc +import org.apache.texera.amber.operator.sklearn.SklearnLinearRegressionOpDesc +import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.base.SklearnMLOperatorDescriptor +import org.apache.texera.amber.operator.machineLearning.Scorer.MachineLearningScorerOpDesc +import org.apache.texera.amber.operator.huggingFace.HuggingFaceSpamSMSDetectionOpDesc +import org.apache.texera.amber.operator.sklearn.training.SklearnTrainingOpDesc +import org.apache.texera.amber.operator.sklearn.training.SklearnTrainingGaussianNaiveBayesOpDesc +import org.apache.texera.amber.operator.regex.RegexOpDesc +import org.apache.texera.amber.operator.sklearn.testing.SklearnTestingOpDesc +import org.apache.texera.amber.operator.typecasting.TypeCastingOpDesc +import org.apache.texera.amber.operator.visualization.wordCloud.WordCloudOpDesc +import org.apache.texera.amber.operator.visualization.DotPlot.DotPlotOpDesc +import org.apache.texera.amber.operator.visualization.barChart.BarChartOpDesc +import org.apache.texera.amber.operator.visualization.boxViolinPlot.BoxViolinPlotOpDesc +import org.apache.texera.amber.operator.visualization.ImageViz.ImageVisualizerOpDesc +import org.apache.texera.amber.operator.visualization.IcicleChart.IcicleChartOpDesc +import org.apache.texera.amber.operator.visualization.bubbleChart.BubbleChartOpDesc +import org.apache.texera.amber.operator.visualization.bulletChart.BulletChartOpDesc +import org.apache.texera.amber.operator.visualization.candlestickChart.CandlestickChartOpDesc +import org.apache.texera.amber.operator.visualization.carpetPlot.CarpetPlotOpDesc +import org.apache.texera.amber.operator.visualization.choroplethMap.ChoroplethMapOpDesc +import org.apache.texera.amber.operator.visualization.continuousErrorBands.ContinuousErrorBandsOpDesc +import org.apache.texera.amber.operator.visualization.contourPlot.ContourPlotOpDesc +import org.apache.texera.amber.operator.visualization.dendrogram.DendrogramOpDesc +import org.apache.texera.amber.operator.visualization.dumbbellPlot.DumbbellPlotOpDesc +import org.apache.texera.amber.operator.visualization.ecdfPlot.ECDFPlotOpDesc +import org.apache.texera.amber.operator.visualization.figureFactoryTable.FigureFactoryTableOpDesc +import org.apache.texera.amber.operator.visualization.filledAreaPlot.FilledAreaPlotOpDesc +import org.apache.texera.amber.operator.visualization.funnelPlot.FunnelPlotOpDesc +import org.apache.texera.amber.operator.visualization.ganttChart.GanttChartOpDesc +import org.apache.texera.amber.operator.visualization.gaugeChart.GaugeChartOpDesc +import org.apache.texera.amber.operator.visualization.ScatterMatrixChart.ScatterMatrixChartOpDesc + +import org.apache.texera.amber.operator.visualization.heatMap.HeatMapOpDesc +import org.apache.texera.amber.operator.visualization.hierarchychart.HierarchyChartOpDesc +import org.apache.texera.amber.operator.visualization.histogram2d.Histogram2DOpDesc +import org.apache.texera.amber.operator.visualization.histogram.HistogramChartOpDesc +import org.apache.texera.amber.operator.visualization.lineChart.LineChartOpDesc +import org.apache.texera.amber.operator.visualization.nestedTable.NestedTableOpDesc +import org.apache.texera.amber.operator.visualization.networkGraph.NetworkGraphOpDesc +import org.apache.texera.amber.operator.visualization.parallelCoordinatesPlot.ParallelCoordinatesPlotOpDesc +import org.apache.texera.amber.operator.visualization.pieChart.PieChartOpDesc +import org.apache.texera.amber.operator.visualization.polarChart.PolarChartOpDesc +import org.apache.texera.amber.operator.visualization.quiverPlot.QuiverPlotOpDesc +import org.apache.texera.amber.operator.visualization.radarChart.RadarChartOpDesc +import org.apache.texera.amber.operator.visualization.radarPlot.RadarPlotOpDesc +import org.apache.texera.amber.operator.visualization.rangeSlider.RangeSliderOpDesc +import org.apache.texera.amber.operator.visualization.sankeyDiagram.SankeyDiagramOpDesc +import org.apache.texera.amber.operator.visualization.scatter3DChart.Scatter3dChartOpDesc +import org.apache.texera.amber.operator.visualization.scatterplot.ScatterplotOpDesc +import org.apache.texera.amber.operator.visualization.stripChart.StripChartOpDesc +import org.apache.texera.amber.operator.visualization.tablesChart.TablesPlotOpDesc +import org.apache.texera.amber.operator.visualization.ternaryContour.TernaryContourOpDesc +import org.apache.texera.amber.operator.visualization.ternaryPlot.TernaryPlotOpDesc +import org.apache.texera.amber.operator.visualization.timeSeriesplot.TimeSeriesOpDesc +import org.apache.texera.amber.operator.visualization.treeplot.TreePlotOpDesc +import org.apache.texera.amber.operator.visualization.volcanoPlot.VolcanoPlotOpDesc +import org.apache.texera.amber.operator.visualization.waterfallChart.WaterfallChartOpDesc +import org.apache.texera.amber.operator.visualization.windRoseChart.WindRoseChartOpDesc +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path} +import scala.jdk.CollectionConverters._ +import scala.util.{Failure, Success, Try} + +/** + * Unified verification runner for non-source operators implementing + * [[StandaloneCodeGenerator]]. Resolves, per operator: + * - Path A engine: [[PyOpExecHarness]] for PythonOperatorDescriptor, + * [[OpExecHarness]] otherwise; Path B is always [[StandaloneRunner]]. + * - Config + fixture: curated handler ([[CuratedHandlers]]) if registered, + * else [[ConfigGenerator]] against the [[CanonicalFixture]] schemas. + * - Comparison: order-insensitive by default (parallel output order isn't a + * contract); strict positional only when the operator declares + * `orderSensitive = true` (the sort family). All output ports are compared. + * Operators that can't be run are Flagged with a reason — never silently + * skipped. + */ +object TransformVerificationRunner { + + /** + * Per-operator knob handling: a value this operator's variants must carry, + * and where it applies. Two needs, one table, because both answer the same + * question — what does the generator have to be told about this operator's + * knobs that its metadata does not say. + * + * `Pinned` holds a knob at one value and keeps it out of the sweep, for a + * knob whose other value selects non-determinism rather than a different + * behavior to check. Split's "Auto-Generate Seed" is the case: with it on the + * executor seeds from the clock, so that run agrees with nothing — its own + * previous run included — and there is no output for a script to reproduce. + * Everything else about the operator is deterministic, so pinning covers the + * partition rather than abandoning the operator over one switch. The value + * reaches the test name via [[pinnedTierNote]], so the run does not read as + * full coverage. + * + * `WithOptionals` sets a knob inside the `optionals` variant, for a branch + * that needs a switch AND the field it governs. Ternary Plot colours its + * points only when `colorEnabled` is on and `colorDataField` is set, and the + * two belong to different mechanisms: the sweep turns the switch on with the + * column empty, the optional fill supplies the column with the switch off, so + * neither variant generated the coloured branch. Naming the switch here puts + * it in the variant that fills the column. + * + * Named per operator rather than applied wholesale, because switches are not + * generally independent: turning every Boolean on in that variant paired + * Sklearn's `countVectorizer` with `tfidfTransformer` (mutually exclusive + * text pipelines), asked File Scan to extract an archive from a plain file, + * and re-enabled the very auto-seed switch the first scope holds off. + * + * Distinct from an `enumSweep` row in [[variantsNotRun]], which is about an + * operator's enums as a whole rather than one named knob. + */ + sealed trait KnobScope + object KnobScope { + case object Pinned extends KnobScope + case object WithOptionals extends KnobScope + } + + final case class Knob(field: String, value: JsonNode, scope: KnobScope) + + val knobOverrides: Map[Class[_], Seq[Knob]] = Map( + classOf[SplitOpDesc] -> Seq(Knob("random", BooleanNode.FALSE, KnobScope.Pinned)), + // `SleepOpExec` sleeps this many seconds per tuple, and the generator fills a + // required Int with half the row count, so the fixture would spend tens of + // seconds asleep for nothing: the delay never reaches the output being + // compared, and the standalone translation is a passthrough by design. + classOf[SleepOpDesc] -> Seq(Knob("sleepTime", IntNode.valueOf(0), KnobScope.Pinned)), + classOf[TernaryPlotOpDesc] -> Seq( + Knob("colorEnabled", BooleanNode.TRUE, KnobScope.WithOptionals) + ), + // Both text switches are pinned off on the numeric table: the pipeline they + // build reads a column that table does not have, and `tfidfTransformer` has + // no meaning at all outside a CountVectorizer pipeline (the schema hides it + // when the vectorizer is off). Their branches are generated against the text + // table by the [[AltScenario]]s instead. + classOf[SklearnClassifierOpDesc] -> Seq( + Knob("countVectorizer", BooleanNode.FALSE, KnobScope.Pinned), + Knob("tfidfTransformer", BooleanNode.FALSE, KnobScope.Pinned) + ), + classOf[SklearnTrainingOpDesc] -> Seq( + Knob("countVectorizer", BooleanNode.FALSE, KnobScope.Pinned), + Knob("tfidfTransformer", BooleanNode.FALSE, KnobScope.Pinned) + ) + ) + + /** This operator's overrides for one scope, as the generator takes them. + * Exact class first, then base class, so a family can be named once instead + * of per estimator. + */ + private def knobsFor(opClass: Class[_ <: LogicalOp], scope: KnobScope): Map[String, JsonNode] = + knobOverrides + .get(opClass) + .orElse(knobOverrides.collectFirst { + case (base, knobs) if base.isAssignableFrom(opClass) => knobs + }) + .getOrElse(Seq.empty) + .filter(_.scope == scope) + .map(k => k.field -> k.value) + .toMap + + /** A second generation pass for a branch the base config cannot reach. Usually + * that is a branch needing a DIFFERENT table: a swept variant cannot switch + * tables, since [[ConfigGenerator]] resolves every column picker against ONE + * schema, so the branch is generated separately against the table it needs + * with its switch pinned on. + * + * It also reaches a branch whose own knobs the base config leaves empty. The + * sweep offers the values a config already holds, so a list filled only on the + * far side of a switch has nothing to offer until the switch is pinned — and + * then the second pass may well take the same table as the first. + * + * The auto-tier twin of [[TransformHandler.extraScenarios]]: it names only the + * table and the pins, and the generator writes the config. + */ + final case class AltScenario( + label: String, + fixture: SharedFixture, + pinned: Map[String, JsonNode] + ) + + /** Every kind of run a [[variantsNotRun]] row can name: the derived variants, + * plus the [[AltScenario]] labels (an alt scenario IS one kind of run). Named + * so a row cannot misspell one and silently stop suppressing anything. + */ + object RunKind { + val Nulls = "nulls" + val EnumSweep = "enumSweep" + val HostileText = "hostileText" + val CountVectorizerText = "countVectorizer_text" + val TfidfText = "tfidf_text" + val NonFeatureColumn = "nonFeatureColumn" + val TextLabels = "textLabels" + val RegressionBranch = "regressionBranch" + + /** One swept hyperparameter of an advanced trainer, which the sweep labels by the + * pointer it flips. Built here rather than spelled out at each row, since a row + * that misspelled the pointer would suppress nothing and say nothing. + */ + def hyperParameter(name: String): String = s"paraList/0/parameter=$name" + } + + /** The [[RunKind]] a generated variant's label names. A `merged` variant labels + * itself `kind(fields…)`, and the fields it happened to move are not part of + * what is being withheld. + */ + private def kindOf(label: String): String = label.takeWhile(_ != '(') + + /** Keyed by BASE class, not by concrete operator: a newly registered sklearn + * estimator is covered with no entry of its own, matching how the families + * themselves are discovered. + */ + val altFixtureScenarios: Map[Class[_], Seq[AltScenario]] = { + // One scenario per text pipeline rather than a sweep inside one: which + // pipeline is built is the branch under test, and the two are alternatives, + // not a knob crossed with everything else. + val countVectorizerText = AltScenario( + label = RunKind.CountVectorizerText, + fixture = CanonicalFixture.sklearnText, + pinned = Map( + "countVectorizer" -> BooleanNode.TRUE, + "tfidfTransformer" -> BooleanNode.FALSE + ) + ) + val tfidfText = countVectorizerText.copy( + label = RunKind.TfidfText, + pinned = Map( + "countVectorizer" -> BooleanNode.TRUE, + "tfidfTransformer" -> BooleanNode.TRUE + ) + ) + // The vectorizer stays off: what this scenario covers is the branch that + // narrows `X` for an estimator, and Count Vectorizer replaces it rather than + // feeding it, naming the text columns the narrowing would otherwise drop. + val nonFeatureColumn = AltScenario( + label = RunKind.NonFeatureColumn, + fixture = CanonicalFixture.sklearnNumericWithText, + pinned = Map( + "countVectorizer" -> BooleanNode.FALSE, + "tfidfTransformer" -> BooleanNode.FALSE + ) + ) + // The advanced trainers are the ones left out: they name the feature columns + // themselves rather than taking every column but the target, so a column an + // estimator cannot fit is not reachable for them. + Map( + classOf[SklearnClassifierOpDesc] -> Seq(countVectorizerText, tfidfText, nonFeatureColumn), + classOf[SklearnTrainingOpDesc] -> Seq(countVectorizerText, tfidfText, nonFeatureColumn), + // No text scenarios, and nothing pinned: this operator declares neither + // switch, and a pin is set on the config whether or not the field exists, + // so pinning one here would hand it a property it cannot read back. + classOf[SklearnLinearRegressionOpDesc] -> Seq(nonFeatureColumn.copy(pinned = Map.empty)), + // A scorer reads a text label as readily as a numeric one, and names the + // class after the label rather than after its position. Regression is + // pinned off rather than swept: a regression metric puts both columns + // through `float()`, so on this table the sweep would generate the one + // configuration the operator is right to refuse. The two columns are + // pinned because the operator's `@SampleColumn`s name the numeric pair, + // and an annotation naming a column the table does not hold ends the run + // rather than falling back — which is what catches a misspelling. + classOf[MachineLearningScorerOpDesc] -> Seq( + AltScenario( + label = RunKind.TextLabels, + fixture = CanonicalFixture.scorerTextLabels, + pinned = Map( + "isRegression" -> BooleanNode.FALSE, + "actualValueColumn" -> TextNode.valueOf("species_name"), + "predictValueColumn" -> TextNode.valueOf("species_name_pred") + ) + ), + // The regression metrics are unreachable from the base config: the sweep + // reads the sites the config already holds, and on the classification + // branch the regression list is empty, so it offers none. Pinned on, the + // list is filled before the sweep looks, and the other three metrics + // become variants like any other enum. Same table as the default runs — + // this scenario is here for the branch, not for a different set of rows. + AltScenario( + label = RunKind.RegressionBranch, + fixture = CanonicalFixture, + pinned = Map("isRegression" -> BooleanNode.TRUE) + ) + ) + ) + } + + /** The alternate-table scenarios this operator takes, resolved by family and + * minus any [[variantsNotRun]] names. + */ + private def altScenariosFor(opClass: Class[_ <: LogicalOp]): Seq[AltScenario] = + altFixtureScenarios + .collectFirst { case (base, scenarios) if base.isAssignableFrom(opClass) => scenarios } + .getOrElse(Seq.empty) + .filterNot(alt => notRun(opClass, alt.label)) + + /** How a pinned operator's tier reads in the report, e.g. `auto, random=false`. */ + private def pinnedTierNote(opClass: Class[_ <: LogicalOp]): String = { + val pinned = knobsFor(opClass, KnobScope.Pinned) + if (pinned.isEmpty) "" + else pinned.map { case (field, value) => s"$field=${value.asText}" }.mkString(", ", ", ", "") + } + + /** Why one kind of run is left out for an operator. The distinction is whether + * anyone should be waiting for it: [[PendingFix]] is a debt someone closes, + * [[ByDesign]] is an answer that will not change. + */ + sealed trait NotRunReason + final case class PendingFix(issue: String) extends NotRunReason + final case class ByDesign(why: String) extends NotRunReason + + final case class NotRun(op: Class[_], kind: String, reason: NotRunReason) + + /** The runs an operator does not get, and why. + * + * One table rather than one per kind. Every row makes the same statement, so + * the coverage report can print them together, and the next exemption has an + * obvious home instead of arriving as another set somewhere else. + * + * `op` matches its subclasses, so one row covers a family. `kind` is a + * [[RunKind]]. + * + * A curated handler's own [[TransformHandler.unfillableVariants]] stays where + * it is: those describe the table that handler wrote rather than the operator, + * and change when the fixture is rewritten. + */ + val variantsNotRun: Seq[NotRun] = { + // The platform raises on an empty cell, so the two paths cannot be compared + // on one until it stops. + val emptyCellRaises: Seq[(Class[_], String)] = Seq( + // Regex alone: apache/texera#7566 answers the empty cell in Substring Search + // and Unnest String, and this one was not part of it. + classOf[RegexOpDesc] -> "apache/texera#7548" + ) + + // The operator refuses the text pipeline in `getOutputSchemas`, so there is no + // configuration to compare: neither path is generated. An invalid configuration + // rather than a translation gap. + // + // Not sklearn raising, which is what the estimator's own limitation would look + // like. This fixture's word counts repeat enough that `ColumnTransformer` stays + // above its 0.3 sparse threshold and hands over a dense array, which GaussianNB + // fits without complaint. Only a wider vocabulary would reach the limitation + // the operator is guarding against. + val dense = ByDesign("the operator refuses Count Vectorizer at compile time") + val denseOnly = for { + op <- Seq( + classOf[SklearnGaussianNaiveBayesOpDesc], + classOf[SklearnTrainingGaussianNaiveBayesOpDesc] + ) + label <- Seq(RunKind.CountVectorizerText, RunKind.TfidfText) + } yield NotRun(op, label, dense) + + emptyCellRaises.map { + case (op, issue) => NotRun(op, RunKind.Nulls, PendingFix(issue)) + } ++ Seq( + // An enum whose legal values depend on a sibling field: flipping it alone + // builds a config the curated fixture already covers properly. + NotRun( + classOf[TypeCastingOpDesc], + RunKind.EnumSweep, + ByDesign( + "resultType is legal only for certain source column types, and the native " + + "executor throws on an illegal cast; the fixture pairs each type with a " + + "compatible column already" + ) + ), + NotRun( + classOf[AggregateOpDesc], + RunKind.EnumSweep, + ByDesign( + "aggFunction is cross-constrained with its attribute's type and with " + + "COUNT(*)'s empty attribute; the fixture pairs each function with a " + + "compatible column already" + ) + ), + // Stated about the operator's own fixture rather than about the operator: a + // predicate over a string column takes the hostile value fine, so this row goes + // the day that fixture filters on one. + NotRun( + classOf[SpecializedFilterOpDesc], + RunKind.HostileText, + ByDesign( + "`id > 8` compares against an INTEGER column, and the platform parses the " + + "predicate value as that column's type, so the number parser refuses the " + + "hostile string before any escaping could matter" + ) + ), + NotRun( + classOf[SklearnMLOperatorDescriptor[_]], + RunKind.HostileText, + ByDesign( + "what a hyperparameter's value may hold is decided by the parameter beside " + + "it, and every one of those is a number or a word from a fixed set, so a " + + "spliced a\"b fails at the conversion rather than at any escaping" + ) + ) + ) ++ denseOnly + } + + /** Every kind of run withheld from this operator, with why. This is the whole of + * what the coverage report needs, so it never walks the table itself. One entry + * per kind: a family row and an operator row for the same kind are the same + * statement twice, and the first one wins. + */ + def withheldRunsFor(opClass: Class[_ <: LogicalOp]): Seq[(String, NotRunReason)] = + variantsNotRun + .collect { case NotRun(op, kind, reason) if op.isAssignableFrom(opClass) => kind -> reason } + .distinctBy(_._1) + + private def notRun(opClass: Class[_ <: LogicalOp], kind: String): Boolean = + withheldRunsFor(opClass).exists(_._1 == kind) + + /** Visualization operators with deterministic Plotly JSON validation. */ + val visualizationJsonOps: Set[Class[_]] = Set( + classOf[RangeSliderOpDesc], + classOf[HeatMapOpDesc], + classOf[HierarchyChartOpDesc], + classOf[HistogramChartOpDesc], + classOf[Histogram2DOpDesc], + classOf[LineChartOpDesc], + classOf[ParallelCoordinatesPlotOpDesc], + classOf[PieChartOpDesc], + classOf[PolarChartOpDesc], + classOf[QuiverPlotOpDesc], + classOf[RadarChartOpDesc], + classOf[RadarPlotOpDesc], + classOf[SankeyDiagramOpDesc], + classOf[Scatter3dChartOpDesc], + classOf[ScatterplotOpDesc], + classOf[StripChartOpDesc], + classOf[TablesPlotOpDesc], + classOf[TernaryContourOpDesc], + classOf[TernaryPlotOpDesc], + classOf[TimeSeriesOpDesc], + classOf[TreePlotOpDesc], + classOf[VolcanoPlotOpDesc], + classOf[WaterfallChartOpDesc], + classOf[WindRoseChartOpDesc], + classOf[BarChartOpDesc], + classOf[BulletChartOpDesc], + classOf[CandlestickChartOpDesc], + classOf[CarpetPlotOpDesc], + classOf[ChoroplethMapOpDesc], + classOf[ContinuousErrorBandsOpDesc], + classOf[ContourPlotOpDesc], + classOf[DendrogramOpDesc], + classOf[DumbbellPlotOpDesc], + classOf[ECDFPlotOpDesc], + classOf[FigureFactoryTableOpDesc], + classOf[FilledAreaPlotOpDesc], + classOf[FunnelPlotOpDesc], + classOf[GanttChartOpDesc], + classOf[GaugeChartOpDesc], + classOf[DotPlotOpDesc], + classOf[IcicleChartOpDesc], + classOf[BubbleChartOpDesc], + classOf[ScatterMatrixChartOpDesc], + classOf[BoxViolinPlotOpDesc] + ) + + /** Visualization operators with deterministic HTML validation. */ + val visualizationHtmlOps: Set[Class[_]] = Set( + classOf[ImageVisualizerOpDesc], + classOf[NestedTableOpDesc] + ) + + /** Triaged, explicitly-not-run operators: class → honest reason, shown in + * the test report and coverage table. + */ + val knownIssues: Map[Class[_], String] = Map( + classOf[DummyOpDesc] -> + ("harness gap: placeholder operator with no physical execution — " + + "LogicalOp.getPhysicalOp throws NotImplementedError"), + classOf[SklearnPredictionOpDesc] -> + ("trained-model input: the operator consumes a fitted sklearn model on " + + "its model port; a JSONL fixture written from the JVM cannot carry a " + + "live model object, so the operator cannot be run in isolation here"), + classOf[SklearnTestingOpDesc] -> + ("trained-model input: scores a fitted sklearn model read from its model " + + "port; a JVM-written JSONL fixture cannot carry a live model object, so " + + "the operator cannot be run in isolation here"), + classOf[WordCloudOpDesc] -> + ("non-deterministic image: emits a base64 PNG from the wordcloud library " + + "whose word placement is randomized (no seed), so the two paths' images " + + "never match byte-for-byte"), + classOf[NetworkGraphOpDesc] -> + ("non-deterministic layout: the native path calls nx.spring_layout with no " + + "seed, so node coordinates are random per run and differ from the seeded " + + "standalone path, and the two paths' Plotly figures never match numerically") + ) + + sealed trait Disposition + final case class Runnable(tier: String) extends Disposition // "auto" | "curated" + final case class Flagged(reason: String) extends Disposition + + /** When `VERIFY_FORCE_AUTO=1`, ignore CuratedHandlers so every operator is + * exercised through the shared-CSV auto path instead. Lets us measure how + * much of the hand-written curated set the auto tier can now replace: an op + * that stays RUNNABLE/passes under force-auto no longer needs its curated + * handler. + */ + private def forceAuto: Boolean = sys.env.get("VERIFY_FORCE_AUTO").contains("1") + + /** The shared table an operator runs on in the AUTO tier. Which table an + * operator takes is its own axis (see [[SharedFixture]]); the auto tier used + * to be pinned to the whole of [[CanonicalFixture]], which is why an operator + * needing a narrower table had to be curated just to name one. sklearn cannot + * fit canonical's string columns — `X = table.drop(target)` feeds every + * remaining column to `fit` — so its families take the petal-and-label view + * of that same table. + */ + private[verify] def fixtureFor(opClass: Class[_ <: LogicalOp]): SharedFixture = + if (CuratedHandlers.sklearnNumericClasses.contains(opClass)) CanonicalFixture.sklearnNumeric + else if (opClass == classOf[HuggingFaceSpamSMSDetectionOpDesc]) CanonicalFixture.withoutScore + else CanonicalFixture + + /** Static classification — cheap (reflection only, no subprocesses), called + * at spec construction time to decide test-vs-ignore. + */ + def disposition(opClass: Class[_ <: LogicalOp]): Disposition = + knownIssues.get(opClass) match { + case Some(reason) => Flagged(s"known issue: $reason") + case None => + Try(opClass.getDeclaredConstructor().newInstance()) match { + case Failure(e) => Flagged(s"cannot instantiate: ${e.getMessage}") + case Success(op: StandaloneCodeGenerator) => + if (!op.producesDataFrame()) + if (visualizationJsonOps.contains(opClass) || visualizationHtmlOps.contains(opClass)) + Runnable("visualization") + else Flagged("visualization: no DataFrame output to compare") + else if (!forceAuto && CuratedHandlers.byClass.contains(opClass)) + Runnable("curated") + else + ConfigGenerator.generate(opClass, fixtureFor(opClass).schemasByPort) match { + case Left(reason) => Flagged(s"cannot auto-configure: $reason") + case Right(configured) => + Try(configured.operatorInfo.inputPorts.size) match { + case Failure(e) => + Flagged(s"operatorInfo failed on generated config: ${e.getMessage}") + case Success(n) if n < 1 || n > 2 => + Flagged(s"unsupported input port count: $n") + case Success(_) + if outputHasBinaryColumn(configured, fixtureFor(opClass)) && + fixtureFor(opClass) == CanonicalFixture => + // A trained-model (BINARY) output cannot be fit on the + // canonical table, whose string columns reach `fit`. The + // model itself is not byte-comparable either, but that is + // handled for every tier alike (see modelColumns in run). + // An op that names a numeric fixture is fine here. + Flagged( + "model output: emits a BINARY (trained-model) column; " + + "requires a numeric fixture, not the canonical table" + ) + case Success(_) => Runnable(s"auto${pinnedTierNote(opClass)}") + } + } + case Success(_) => + Flagged("does not implement StandaloneCodeGenerator") + } + } + + /** True if the configured operator declares a BINARY output column (e.g. a + * serialized trained model). Best-effort: only Python descriptors expose + * getOutputSchemas, and a throw (schema needs real inputs) reads as "no + * detectable BINARY column" so the op falls through to its normal tier. + */ + private def outputHasBinaryColumn(configured: LogicalOp, fixture: SharedFixture): Boolean = + configured match { + case p: PythonOperatorDescriptor => + val inputSchemas = fixture.schemasByPort.map { + case (port, schema) => PortIdentity(port) -> schema + } + Try(p.getOutputSchemas(inputSchemas)).toOption + .exists(_.values.exists(_.getAttributes.exists(_.getType == AttributeType.BINARY))) + case _ => false + } + + /** Execute both paths and assert parity on every declared output port. + * Precondition: disposition(opClass) returned Runnable. + */ + def run(opClass: Class[_ <: LogicalOp]): Unit = { + val testRoot = Files.createTempDirectory(s"verify-${opClass.getSimpleName}-") + + // Resolve the run list: each entry is (label, configured op, its inputs). + // Both tiers yield the base config PLUS one variant per enum value, so each + // enum branch (e.g. a line chart's mode = line / dots / line+dots) is + // exercised, not just the default, PLUS the `optionals` and `hostileText` + // variants. Variants of one fixture share input files; a curated handler's + // extraScenarios carry their own (structurally different) inputs. + val runs: Seq[(String, LogicalOp, Map[PortIdentity, Path])] = + (if (forceAuto) None else CuratedHandlers.byClass.get(opClass)) match { + case Some(handler) => + val (op, in) = handler.fixture(testRoot) + // The variants are derived against the handler's OWN fixture, not the + // canonical one — a curated handler writes the table its operator needs, + // so that is what an optional column knob has to resolve against. + // + // An enum-sweep-exempt op still gets the fills: what is cross-constrained + // with a sibling field is its ENUM values, so a blind sweep produces invalid + // configs — filling an optional knob or splicing a quote does not. + // + // Fall back to the single curated config if it can't be varied at all. + val primary = + ConfigGenerator + .fullVariantsOf( + op, + schemasOf(in), + rowCountOf(in), + sweepEnums = !notRun(opClass, RunKind.EnumSweep) + ) + .fold(_ => Seq("default" -> op), identity) + // A variant this operator does not get, named in [[variantsNotRun]]. + .filterNot { case (label, _) => notRun(opClass, kindOf(label)) } + primary.map { case (label, o) => (label, o, in) } ++ + handler.extraScenarios(testRoot) ++ + handler.nullsKeepFilled.toSeq.flatMap(curatedNullsCase(opClass, op, in, testRoot, _)) + case None => + val fixture = fixtureFor(opClass) + val vs = ConfigGenerator + .generateVariants( + opClass, + fixture.schemasByPort, + fixture.port0RowCount, + knobsFor(opClass, KnobScope.Pinned), + knobsFor(opClass, KnobScope.WithOptionals) + ) + .fold( + reason => throw new IllegalStateException(s"cannot auto-configure: $reason"), + identity + ) + val inputPortCount = vs.head._2.operatorInfo.inputPorts.size + val in = fixture.writeInputs(testRoot, inputPortCount) + // A variant the operator itself cannot take, named in [[variantsNotRun]]: + // for a swept hyperparameter that is one row per parameter, so the sweep + // keeps covering the rest. + vs.filterNot { case (label, _) => notRun(opClass, kindOf(label)) } + .map { case (label, o) => (label, o, in) } ++ + nullsCase(opClass, vs.head._2, testRoot, fixture) ++ + altScenariosFor(opClass).flatMap { alt => + // Each scenario writes under its own directory: two tables in one + // testRoot would otherwise both claim input_port_0.jsonl. + val dir = testRoot.resolve(alt.label) + Files.createDirectories(dir) + ConfigGenerator + .generateVariants( + opClass, + alt.fixture.schemasByPort, + alt.fixture.port0RowCount, + pinned = alt.pinned, + switches = knobsFor(opClass, KnobScope.WithOptionals) + ) + .fold( + reason => + throw new IllegalStateException( + s"cannot auto-configure ${alt.label}: $reason" + ), + identity + ) + // The base variant carries the branch's own column knob: the pins + // are visible while the config is built, so the knob the schema + // requires under them is filled like any other required field. + .map { + case (label, o) => + (s"${alt.label}/$label", o, alt.fixture.writeInputs(dir, inputPortCount)) + } + } + } + + runs.foreach { + case (label, opDesc, inputs) => + val workDir = + if (runs.size == 1) testRoot + else testRoot.resolve(label.replaceAll("[^A-Za-z0-9]+", "_")) + Files.createDirectories(workDir) + try runVariant(opClass, opDesc, inputs, workDir) + catch { + case e: Throwable => + throw new AssertionError(s"[variant: $label] ${e.getMessage}", e) + } + } + } + + /** One extra run per operator, on `fixture` with one empty cell per column (see + * [[SharedFixture.emptyOneCellPerColumn]]). It takes the base config rather than + * crossing with the other variants: what an operator does with a null is a + * property of the operator, and multiplying it across every knob would buy more + * runtime than signal. + * + * The auto tier's form: the table is the shared one [[fixtureFor]] resolves, so + * the holes come from the fixture itself. See [[curatedNullsCase]] for the other + * tier, which has no fixture object to ask. + */ + private def nullsCase( + opClass: Class[_ <: LogicalOp], + base: LogicalOp, + testRoot: Path, + fixture: SharedFixture + ): Seq[(String, LogicalOp, Map[PortIdentity, Path])] = + if (notRun(opClass, RunKind.Nulls)) Seq.empty + else { + val dir = testRoot.resolve("nulls-input") + Files.createDirectories(dir) + val in = fixture.write(dir, base.operatorInfo.inputPorts.size, withGaps = true) + Seq(("nulls", base, in)) + } + + /** [[nullsCase]] for the curated tier, where there is no fixture object to write + * a second time: the handler's own files are read back, holed, and rewritten. + * So a handler opts in by naming its load-bearing columns and nothing else, and + * [[TransformHandler.fixture]] keeps returning paths. + */ + private def curatedNullsCase( + opClass: Class[_ <: LogicalOp], + base: LogicalOp, + inputs: Map[PortIdentity, Path], + testRoot: Path, + keepFilled: Set[String] + ): Seq[(String, LogicalOp, Map[PortIdentity, Path])] = + if (notRun(opClass, RunKind.Nulls)) Seq.empty + else { + val dir = testRoot.resolve("nulls-input") + Files.createDirectories(dir) + val holed = inputs.map { + case (portId, path) => + val schema = TupleIO.readSchemaSidecar(path) + val rows = TupleIO.readTuples(path, schema).toSeq + val out = dir.resolve(path.getFileName.toString) + TupleIO.writeTuples( + out, + SharedFixture.emptyOneCellPerColumn(rows, schema, keepFilled).iterator, + schema + ) + portId -> out + } + Seq(("nulls", base, holed)) + } + + /** The schema of each input file, keyed by port index — what a curated handler + * actually wrote, read back off the sidecar its writer drops. A file without one + * contributes no schema, so a column knob resolved against that port simply finds + * nothing to fill and the variant is skipped rather than built on a guess. + */ + private def schemasOf(inputs: Map[PortIdentity, Path]): Map[Int, Schema] = + inputs.flatMap { + case (portId, path) => Try(TupleIO.readSchemaSidecar(path)).toOption.map(portId.id -> _) + } + + /** How many rows port 0 holds — the hint a numeric knob's fill is scaled against + * (a `limit` worth running is one that keeps some rows and drops some). + */ + private def rowCountOf(inputs: Map[PortIdentity, Path]): Int = + inputs + .get(PortIdentity(0)) + .flatMap(path => Try(Files.readAllLines(path).asScala.count(_.trim.nonEmpty)).toOption) + .filter(_ > 0) + .getOrElse(ConfigGenerator.DefaultRowCount) + + /** Run one configured variant of `opDesc` through both paths against `inputs`, + * writing all intermediate/output files under `workDir`, and assert parity on + * every declared output port. + */ + private def runVariant( + opClass: Class[_ <: LogicalOp], + opDesc: LogicalOp, + inputs: Map[PortIdentity, Path], + workDir: Path + ): Unit = { + val outputPortCount = opDesc.operatorInfo.outputPorts.size + val actualDir = workDir.resolve("actual") + Files.createDirectories(actualDir) + + if (!opDesc.asInstanceOf[StandaloneCodeGenerator].producesDataFrame()) { + runVisualization(opClass, opDesc, inputs, outputPortCount, actualDir, workDir) + return + } + + // Path A's getPhysicalPlan/getPhysicalOp may mutate the OpDesc in place — + // AggregateOpDesc rewrites its `aggregations` to the final stage (COUNT→SUM) + // via getFinal. Run Path A on an isolated deep copy (same JSON round-trip the + // executor itself uses) so the shared instance stays pristine for Path B, + // whose generateStandaloneCode reads the original fields directly. + val opDescForPathA = + objectMapper + .readValue(objectMapper.writeValueAsString(opDesc), opClass) + .asInstanceOf[LogicalOp] + val (pathAOutputs, pathAOutputSchemas): (Map[PortIdentity, Path], Map[PortIdentity, Schema]) = + if (classOf[PythonOperatorDescriptor].isAssignableFrom(opClass)) { + val r = PyOpExecHarness.execute(opDescForPathA, inputs = inputs, outputDir = actualDir) + (r.outputs, r.outputSchemas) + } else { + val r = OpExecHarness.execute(opDescForPathA, inputs = inputs, outputDir = actualDir) + (r.outputs, r.outputSchemas) + } + + // StandaloneRunner keys inputs by 1-based port index (the inNdf convention). + val standaloneInputs: Map[Int, Path] = + inputs.toSeq + .sortBy(_._1.id) + .zipWithIndex + .map { + case ((_, path), idx) => (idx + 1) -> path + } + .toMap + + val pathB = StandaloneRunner.run( + opDesc = opDesc, + inputs = standaloneInputs, + outputPortCount = outputPortCount, + workDir = workDir + ) + + // The operator declares whether its output row order is meaningful via + // LogicalOp.orderSensitive (true only for the sort family); default unordered. + val orderSensitive = opDesc.orderSensitive + (0 until outputPortCount).foreach { port => + val actual = pathAOutputs.getOrElse( + PortIdentity(port), + throw new AssertionError(s"Texera path produced no output for port $port") + ) + val expected = pathB.outputs.getOrElse( + port + 1, + throw new AssertionError(s"standalone path produced no output for port $port") + ) + // A BINARY column holds a trained model: the two paths produce + // behaviorally-equivalent but not bit-identical models, so the comparator + // unpickles both and asserts their predictions on the training features + // (the probe) match — verifying behavior, not just completion. + val modelColumns: Seq[String] = pathAOutputSchemas + .get(PortIdentity(port)) + .map(_.getAttributes.filter(_.getType == AttributeType.BINARY).map(_.getName)) + .getOrElse(Seq.empty) + val probePath: Option[Path] = + if (modelColumns.nonEmpty) inputs.toSeq.sortBy(_._1.id).headOption.map(_._2) else None + Comparator.assertEqual( + actual, + expected, + orderSensitive = orderSensitive, + modelColumns = modelColumns, + probePath = probePath + ) + } + } + + private def runVisualization( + opClass: Class[_ <: LogicalOp], + opDesc: LogicalOp, + inputs: Map[PortIdentity, Path], + outputPortCount: Int, + actualDir: Path, + testRoot: Path + ): Unit = { + require( + visualizationJsonOps.contains(opClass) || visualizationHtmlOps.contains(opClass), + s"${opClass.getSimpleName} is not registered for visualization validation" + ) + require( + outputPortCount == 1, + "visualization JSON validation currently supports one output port" + ) + require( + classOf[PythonOperatorDescriptor].isAssignableFrom(opClass), + "visualization JSON validation currently supports Python visualization operators" + ) + + val actual = PyOpExecHarness + .execute(opDesc, inputs = inputs, outputDir = actualDir) + .outputs + .getOrElse( + PortIdentity(0), + throw new AssertionError("Texera path produced no visualization output for port 0") + ) + + val standaloneInputs: Map[Int, Path] = + inputs.toSeq + .sortBy(_._1.id) + .zipWithIndex + .map { + case ((_, path), idx) => (idx + 1) -> path + } + .toMap + + StandaloneRunner.run( + opDesc = opDesc, + inputs = standaloneInputs, + outputPortCount = outputPortCount, + workDir = testRoot + ) + + // A JSON-compared operator can still legitimately render its own error page + // instead of a figure (a non-numeric threshold, no non-null rows). There is + // then no Plotly payload to compare on either path, so compare what the user + // actually sees — the HTML. + if (visualizationJsonOps.contains(opClass) && hasPlotlyFigure(actual)) { + val expected = testRoot.resolve("output.json") + if (!Files.exists(expected)) { + throw new AssertionError(s"standalone visualization path did not produce $expected") + } + VisualizationJsonComparator.assertEqual(actual, expected) + } else { + val expected = testRoot.resolve("output.html") + if (!Files.exists(expected)) { + throw new AssertionError(s"standalone visualization path did not produce $expected") + } + VisualizationHtmlComparator.assertEqual(actual, expected) + } + } + + /** True if the runtime path's visualization output carries a Plotly figure — + * either a `json-content` payload or an `html-content` holding a + * `Plotly.newPlot(...)` call. False for an operator's own error page. + */ + private def hasPlotlyFigure(visualizationJsonl: Path): Boolean = { + val line = Files + .readAllLines(visualizationJsonl, StandardCharsets.UTF_8) + .asScala + .find(_.trim.nonEmpty) + .getOrElse(throw new AssertionError(s"$visualizationJsonl is empty")) + val node = objectMapper.readTree(line) + val json = node.get("json-content") + if (json != null && !json.isNull && json.asText().nonEmpty) true + else { + val html = node.get("html-content") + html != null && !html.isNull && html.asText().contains("Plotly.newPlot(") + } + } +} diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunnerSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunnerSpec.scala new file mode 100644 index 00000000000..38bb7a81962 --- /dev/null +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunnerSpec.scala @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.translator.verify + +// This spec pins the tier-routing logic (disposition). Per-operator end-to-end +// runs are NOT duplicated here: OperatorBehaviorSpec auto-discovers every +// registered operator and runs TransformVerificationRunner.run on each, and a +// single operator can be run in isolation with e.g. +// sbt "WorkflowCompilingService/testOnly *OperatorBehaviorSpec -- -z LimitOpDesc" +// (the auto-generated test name starts with the operator's simple name). What +// disposition asserts — which tier an operator routes to — is the one thing +// OperatorBehaviorSpec does not check, so it lives here. + +import org.apache.texera.amber.operator.dummy.DummyOpDesc +import org.apache.texera.amber.operator.limit.LimitOpDesc +import org.apache.texera.amber.operator.udf.python.PythonUDFOpDescV2 +import org.apache.texera.amber.operator.union.UnionOpDesc +import org.apache.texera.amber.operator.sklearn.SklearnPredictionOpDesc +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +class TransformVerificationRunnerSpec extends AnyFlatSpec with Matchers { + import TransformVerificationRunner._ + + "disposition" should "flag knownIssues operators with the triage reason" in { + // The prediction op consumes a trained model on its input port, which a + // JVM-written JSONL fixture can't carry; triaged as a known issue, not run. + disposition(classOf[SklearnPredictionOpDesc]) match { + case Flagged(reason) => reason should include("trained-model") + case other => fail(s"expected Flagged, got $other") + } + // The other kind of row: a placeholder with no physical execution, so the + // harness has nothing to run either path against. + disposition(classOf[DummyOpDesc]) match { + case Flagged(reason) => reason should include("known issue") + case other => fail(s"expected Flagged, got $other") + } + } + + it should "run the union now that its code names every upstream" in { + // It used to be flagged for naming exactly two, which was wrong in both + // directions: a third link was dropped and a lone link left the second + // frame unbound. The runner draws one link per port, so what runs here is + // the one-upstream case — the one the old code got wrong. + disposition(classOf[UnionOpDesc]) shouldBe Runnable("auto") + } + + it should "route auto-configurable operators to the auto tier" in { + disposition(classOf[LimitOpDesc]) shouldBe Runnable("auto") + } + + // A UDF's body is written by whoever drops the operator, so there is nothing + // for a generator to emit. It stands here for the shape of the report: an + // operator that cannot be exported is carried as a row, not passed over. + it should "flag an operator that has no standalone generator" in { + disposition(classOf[PythonUDFOpDescV2]) shouldBe + Flagged("does not implement StandaloneCodeGenerator") + } +} From e7d36c7e43a8735a5aadbafd155d0799a6130d8d Mon Sep 17 00:00:00 2001 From: kary zheng Date: Wed, 2 Sep 2026 16:51:31 -0700 Subject: [PATCH 02/10] docs: shorten the knob-scope note to the cases that earn their place Three examples where one carries the point, and a cross-reference the reader can follow without being told to. Co-Authored-By: Claude Opus 5 (1M context) --- .../verify/TransformVerificationRunner.scala | 31 ++++++------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala index 590e0c361ce..5e69b0c30e0 100644 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala @@ -122,31 +122,20 @@ object TransformVerificationRunner { * knobs that its metadata does not say. * * `Pinned` holds a knob at one value and keeps it out of the sweep, for a - * knob whose other value selects non-determinism rather than a different - * behavior to check. Split's "Auto-Generate Seed" is the case: with it on the - * executor seeds from the clock, so that run agrees with nothing — its own - * previous run included — and there is no output for a script to reproduce. - * Everything else about the operator is deterministic, so pinning covers the - * partition rather than abandoning the operator over one switch. The value - * reaches the test name via [[pinnedTierNote]], so the run does not read as - * full coverage. + * knob whose other value selects non-determinism rather than another + * behaviour to check. Split's "Auto-Generate Seed" is the case: with it on + * the executor seeds from the clock, so that run agrees with nothing, its own + * previous run included. The value reaches the test name via + * [[pinnedTierNote]], so the run does not read as full coverage. * * `WithOptionals` sets a knob inside the `optionals` variant, for a branch - * that needs a switch AND the field it governs. Ternary Plot colours its - * points only when `colorEnabled` is on and `colorDataField` is set, and the - * two belong to different mechanisms: the sweep turns the switch on with the - * column empty, the optional fill supplies the column with the switch off, so - * neither variant generated the coloured branch. Naming the switch here puts - * it in the variant that fills the column. + * that needs a switch and the field it governs together. Ternary Plot colours + * its points only when both are set, and the sweep and the optional fill each + * supply one, so neither variant reached the coloured branch. * * Named per operator rather than applied wholesale, because switches are not - * generally independent: turning every Boolean on in that variant paired - * Sklearn's `countVectorizer` with `tfidfTransformer` (mutually exclusive - * text pipelines), asked File Scan to extract an archive from a plain file, - * and re-enabled the very auto-seed switch the first scope holds off. - * - * Distinct from an `enumSweep` row in [[variantsNotRun]], which is about an - * operator's enums as a whole rather than one named knob. + * generally independent: turning every Boolean on at once paired Sklearn's + * two mutually exclusive text pipelines, among others. */ sealed trait KnobScope object KnobScope { From 40e57404b32a2e852334012d34d661a753c74e60 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Wed, 2 Sep 2026 16:55:18 -0700 Subject: [PATCH 03/10] fix(verify): say why Regex is withheld, rather than naming another operator's issue The row pointed at the issue for Substring Search and Unnest String, which this operator was never part of. That issue closes with the change that answers those two, and this one still raises. Co-Authored-By: Claude Opus 5 (1M context) --- .../translator/verify/TransformVerificationRunner.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala index 5e69b0c30e0..d778626dc97 100644 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala @@ -352,9 +352,9 @@ object TransformVerificationRunner { // The platform raises on an empty cell, so the two paths cannot be compared // on one until it stops. val emptyCellRaises: Seq[(Class[_], String)] = Seq( - // Regex alone: apache/texera#7566 answers the empty cell in Substring Search - // and Unnest String, and this one was not part of it. - classOf[RegexOpDesc] -> "apache/texera#7548" + // Regex alone. Substring Search and Unnest String answer an empty cell + // rather than raising on it; this one still raises. + classOf[RegexOpDesc] -> "Regex raises a NullPointerException on an empty cell" ) // The operator refuses the text pipeline in `getOutputSchemas`, so there is no From 673465058981e597e30fc1c60dcbf24149f16b9b Mon Sep 17 00:00:00 2001 From: kary zheng Date: Wed, 2 Sep 2026 17:01:22 -0700 Subject: [PATCH 04/10] test(verify): run Regex on the nulls variant now that it answers them The row said the platform raised on an empty cell. The operator answers one now, so the variant runs like every other. Co-Authored-By: Claude Opus 5 (1M context) --- .../verify/TransformVerificationRunner.scala | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala index d778626dc97..170a2678fb7 100644 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala @@ -43,7 +43,6 @@ import org.apache.texera.amber.operator.machineLearning.Scorer.MachineLearningSc import org.apache.texera.amber.operator.huggingFace.HuggingFaceSpamSMSDetectionOpDesc import org.apache.texera.amber.operator.sklearn.training.SklearnTrainingOpDesc import org.apache.texera.amber.operator.sklearn.training.SklearnTrainingGaussianNaiveBayesOpDesc -import org.apache.texera.amber.operator.regex.RegexOpDesc import org.apache.texera.amber.operator.sklearn.testing.SklearnTestingOpDesc import org.apache.texera.amber.operator.typecasting.TypeCastingOpDesc import org.apache.texera.amber.operator.visualization.wordCloud.WordCloudOpDesc @@ -349,14 +348,6 @@ object TransformVerificationRunner { * and change when the fixture is rewritten. */ val variantsNotRun: Seq[NotRun] = { - // The platform raises on an empty cell, so the two paths cannot be compared - // on one until it stops. - val emptyCellRaises: Seq[(Class[_], String)] = Seq( - // Regex alone. Substring Search and Unnest String answer an empty cell - // rather than raising on it; this one still raises. - classOf[RegexOpDesc] -> "Regex raises a NullPointerException on an empty cell" - ) - // The operator refuses the text pipeline in `getOutputSchemas`, so there is no // configuration to compare: neither path is generated. An invalid configuration // rather than a translation gap. @@ -375,9 +366,7 @@ object TransformVerificationRunner { label <- Seq(RunKind.CountVectorizerText, RunKind.TfidfText) } yield NotRun(op, label, dense) - emptyCellRaises.map { - case (op, issue) => NotRun(op, RunKind.Nulls, PendingFix(issue)) - } ++ Seq( + Seq( // An enum whose legal values depend on a sibling field: flipping it alone // builds a config the curated fixture already covers properly. NotRun( From 1ba3ca8e08661cd19714b5be92254d37c0ec4c5f Mon Sep 17 00:00:00 2001 From: kary zheng Date: Wed, 2 Sep 2026 17:17:36 -0700 Subject: [PATCH 05/10] test(verify): leave the per-operator spec to the change its callees are in Co-Authored-By: Claude Opus 5 (1M context) --- .../verify/OperatorBehaviorSpec.scala | 151 ------------------ 1 file changed, 151 deletions(-) delete mode 100644 workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/OperatorBehaviorSpec.scala diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/OperatorBehaviorSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/OperatorBehaviorSpec.scala deleted file mode 100644 index f6d8b6759be..00000000000 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/OperatorBehaviorSpec.scala +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.texera.amber.translator.verify - -import com.fasterxml.jackson.annotation.JsonSubTypes -import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} -import org.apache.texera.amber.operator.source.SourceOperatorDescriptor -import org.apache.texera.amber.translator.verify.tags.IntegrationTest -import org.scalatest.ParallelTestExecution -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -/** - * Auto-discovered behavioral-parity tests: for every operator registered - * with [[LogicalOp]]'s `@JsonSubTypes` that implements - * [[StandaloneCodeGenerator]], emit a test that runs both Path A (Texera - * exec) and Path B (translator-generated Python via [[StandaloneRunner]]) - * and asserts their outputs are equivalent. - * - * Dispatch is auto-first: [[TransformVerificationRunner]] classifies each - * non-source transform as `Runnable("auto")` (auto-configured fixture), - * `Runnable("curated")` (hand-written fixture from [[CuratedHandlers]]), - * or `Flagged(reason)` (shown as ignored with the reason in the test name). - * Sources route to [[SourceCategoryRunner]] unchanged. - * - * No edits to this spec are needed when a new operator is added — reflection - * discovers it automatically via `@JsonSubTypes`. The tier label appears in - * the test name so the report shows which path exercised each operator. - * - * Requires Python 3 with pandas on the [[Comparator]] / [[StandaloneRunner]] - * resolution chain (`UDF_PYTHON_PATH` env var, then `python3.12`). - */ -// Tagged @IntegrationTest: this is the only verify spec that forks a real -// Python process end-to-end, so CI routes it to the Python-provisioned -// integration job (see workflow-compiling-service/build.sbt WCS_TEST_FILTER). -@IntegrationTest -class OperatorBehaviorSpec extends AnyFlatSpec with Matchers with ParallelTestExecution { - - // Build the test list at class construction. Each branch below registers - // one test (`in` for runnable, `ignore` for skipped) so the test report - // shows every translator-eligible operator and why it did or didn't run. - OperatorBehaviorSpec.discoverStandaloneOperators().foreach { opClass => - val name = opClass.getSimpleName - - if (!OperatorBehaviorSpec.isSelected(name)) { - // Narrowed out by VERIFY_ONLY / VERIFY_SKIP, which only a local run sets. - // Still registered, as an `ignore`, so the report lists every operator - // rather than reading as though the narrowed-out ones do not exist. - name should "NARROWED OUT — outside this run's VERIFY_ONLY / VERIFY_SKIP" ignore {} - } else if (classOf[SourceOperatorDescriptor].isAssignableFrom(opClass)) { - // Sources keep their handler-per-source design: each needs a real file - // in its specific format, which a generic fixture can't supply. - if (SourceCategoryRunner.canRun(opClass)) { - name should "produce equivalent output in Texera and standalone Python (source)" in { - SourceCategoryRunner.run(opClass) - } - } else { - name should s"FLAGGED — ${SourceCategoryRunner.flagReason(opClass)}" ignore {} - } - } else { - TransformVerificationRunner.disposition(opClass) match { - case TransformVerificationRunner.Runnable(tier) => - name should s"produce equivalent output in Texera and standalone Python ($tier)" in { - TransformVerificationRunner.run(opClass) - } - case TransformVerificationRunner.Flagged(reason) => - name should s"FLAGGED — $reason" ignore { - // Reason is in the test name so the report carries it; the - // coverage table in ConfigCoverageSpec aggregates these. - } - } - } - } - - // Not one test per operator like the rest of this spec: it is one assertion - // over all of them, and it deliberately ignores the selection knobs above so a - // VERIFY_ONLY run still cannot hide a broken splice site. - "Generated standalone code" should "stay parseable when the column names are hostile" in { - StandaloneEscapingCheck.run() shouldBe empty - } -} - -object OperatorBehaviorSpec { - - // Narrowing knobs for a local run, both unset by default, so the default run - // is every operator: VERIFY_ONLY names the only ones to run, VERIFY_SKIP the - // ones to leave out. Case-sensitive substrings against the operator's simple - // name, comma-separated. Neither is set in CI, which therefore runs the lot. - // - // There is deliberately no third list withholding operators by default. What - // stays withheld is narrower than an operator and lives where it can say why: - // a single variant in [[TransformVerificationRunner.variantsNotRun]], or an - // operator that cannot be run at all in its `knownIssues`, each against an - // issue or a reason. A name here would withdraw an operator's every variant - // and record nothing about what is wrong with it. - private def patterns(envVar: String): Seq[String] = - sys.env.getOrElse(envVar, "").split(",").iterator.map(_.trim).filter(_.nonEmpty).toSeq - - private lazy val onlyPatterns: Seq[String] = patterns("VERIFY_ONLY") - private lazy val skipPatterns: Seq[String] = patterns("VERIFY_SKIP") - - /** True if `name` should run: in VERIFY_ONLY when that is set, and not in - * VERIFY_SKIP. True for everything when neither is set. - */ - def isSelected(name: String): Boolean = { - val included = onlyPatterns.isEmpty || onlyPatterns.exists(name.contains) - val excluded = skipPatterns.exists(name.contains) - included && !excluded - } - - /** - * Enumerates every concrete subclass of [[LogicalOp]] declared in its - * `@JsonSubTypes` annotation, filters to those implementing - * [[StandaloneCodeGenerator]], and returns them sorted by simple name - * (stable test report order). - * - * Uses the same registry Jackson uses to deserialize operators — no - * separate discovery mechanism needed. Adding an operator to - * `LogicalOp.@JsonSubTypes` makes it visible here automatically. - */ - def discoverStandaloneOperators(): Seq[Class[_ <: LogicalOp]] = { - val annotation = classOf[LogicalOp].getAnnotation(classOf[JsonSubTypes]) - if (annotation == null) Seq.empty - else - annotation - .value() - .toSeq - .map(_.value()) - .filter(classOf[StandaloneCodeGenerator].isAssignableFrom) - .map(_.asInstanceOf[Class[_ <: LogicalOp]]) - .distinct - .sortBy(_.getSimpleName) - } -} From a78e443624937425fe8c3a4a9165fb57cffe587e Mon Sep 17 00:00:00 2001 From: kary zheng Date: Wed, 2 Sep 2026 17:44:54 -0700 Subject: [PATCH 06/10] test(verify): register the two seeded visualizations here Both had been withheld for drawing a different picture on every run, which stopped being true once their placement was seeded. The registration is inert until the operator implements the trait, so it can sit here rather than making the change that gives them their generator depend on this one. Co-Authored-By: Claude Opus 5 (1M context) --- .../verify/TransformVerificationRunner.scala | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala index 170a2678fb7..a4f4b3287d9 100644 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala @@ -426,6 +426,9 @@ object TransformVerificationRunner { /** Visualization operators with deterministic Plotly JSON validation. */ val visualizationJsonOps: Set[Class[_]] = Set( + // Its layout is seeded, so both paths place the nodes identically and the two + // figures can be compared number by number. + classOf[NetworkGraphOpDesc], classOf[RangeSliderOpDesc], classOf[HeatMapOpDesc], classOf[HierarchyChartOpDesc], @@ -475,6 +478,10 @@ object TransformVerificationRunner { /** Visualization operators with deterministic HTML validation. */ val visualizationHtmlOps: Set[Class[_]] = Set( classOf[ImageVisualizerOpDesc], + // A word cloud is a picture, not a figure with values to read, so the two + // paths are compared as the HTML they emit. Its placement is seeded, which + // is what makes that comparison mean anything. + classOf[WordCloudOpDesc], classOf[NestedTableOpDesc] ) @@ -493,14 +500,6 @@ object TransformVerificationRunner { ("trained-model input: scores a fitted sklearn model read from its model " + "port; a JVM-written JSONL fixture cannot carry a live model object, so " + "the operator cannot be run in isolation here"), - classOf[WordCloudOpDesc] -> - ("non-deterministic image: emits a base64 PNG from the wordcloud library " + - "whose word placement is randomized (no seed), so the two paths' images " + - "never match byte-for-byte"), - classOf[NetworkGraphOpDesc] -> - ("non-deterministic layout: the native path calls nx.spring_layout with no " + - "seed, so node coordinates are random per run and differ from the seeded " + - "standalone path, and the two paths' Plotly figures never match numerically") ) sealed trait Disposition From 94c65c5980d9bb19a2df6395bd0c3d0b609701b3 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Wed, 2 Sep 2026 17:45:33 -0700 Subject: [PATCH 07/10] style: drop the trailing comma the removed rows left behind --- .../amber/translator/verify/TransformVerificationRunner.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala index a4f4b3287d9..5383fd777a8 100644 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala @@ -499,7 +499,7 @@ object TransformVerificationRunner { classOf[SklearnTestingOpDesc] -> ("trained-model input: scores a fitted sklearn model read from its model " + "port; a JVM-written JSONL fixture cannot carry a live model object, so " + - "the operator cannot be run in isolation here"), + "the operator cannot be run in isolation here") ) sealed trait Disposition From 9793f0a188f0a3d8729e3a6a7093ff395d50ae87 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Wed, 2 Sep 2026 17:50:32 -0700 Subject: [PATCH 08/10] test(verify): assert the tier every operator routes to Which tier an operator lands in is the one thing the per-operator runs do not check, so it is what this spec is for. Every visualization, the scorer, an estimator and an advanced trainer each state where they route and why that is not the tier they used to be in. All of them land before this change does, so the assertions hold when it merges rather than needing to be added back afterwards. Co-Authored-By: Claude Opus 5 (1M context) --- .../TransformVerificationRunnerSpec.scala | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunnerSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunnerSpec.scala index 38bb7a81962..4aedc11fa57 100644 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunnerSpec.scala +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunnerSpec.scala @@ -29,10 +29,37 @@ package org.apache.texera.amber.translator.verify // OperatorBehaviorSpec does not check, so it lives here. import org.apache.texera.amber.operator.dummy.DummyOpDesc +import org.apache.texera.amber.operator.hashJoin.HashJoinOpDesc import org.apache.texera.amber.operator.limit.LimitOpDesc import org.apache.texera.amber.operator.udf.python.PythonUDFOpDescV2 import org.apache.texera.amber.operator.union.UnionOpDesc +import org.apache.texera.amber.operator.machineLearning.Scorer.MachineLearningScorerOpDesc +import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.SVCTrainer.SklearnAdvancedSVCTrainerOpDesc import org.apache.texera.amber.operator.sklearn.SklearnPredictionOpDesc +import org.apache.texera.amber.operator.sklearn.training.SklearnTrainingLogisticRegressionOpDesc +import org.apache.texera.amber.operator.visualization.DotPlot.DotPlotOpDesc +import org.apache.texera.amber.operator.visualization.IcicleChart.IcicleChartOpDesc +import org.apache.texera.amber.operator.visualization.ImageViz.ImageVisualizerOpDesc +import org.apache.texera.amber.operator.visualization.ScatterMatrixChart.ScatterMatrixChartOpDesc +import org.apache.texera.amber.operator.visualization.barChart.BarChartOpDesc +import org.apache.texera.amber.operator.visualization.boxViolinPlot.BoxViolinPlotOpDesc +import org.apache.texera.amber.operator.visualization.bubbleChart.BubbleChartOpDesc +import org.apache.texera.amber.operator.visualization.bulletChart.BulletChartOpDesc +import org.apache.texera.amber.operator.visualization.candlestickChart.CandlestickChartOpDesc +import org.apache.texera.amber.operator.visualization.carpetPlot.CarpetPlotOpDesc +import org.apache.texera.amber.operator.visualization.choroplethMap.ChoroplethMapOpDesc +import org.apache.texera.amber.operator.visualization.continuousErrorBands.ContinuousErrorBandsOpDesc +import org.apache.texera.amber.operator.visualization.contourPlot.ContourPlotOpDesc +import org.apache.texera.amber.operator.visualization.dendrogram.DendrogramOpDesc +import org.apache.texera.amber.operator.visualization.dumbbellPlot.DumbbellPlotOpDesc +import org.apache.texera.amber.operator.visualization.ecdfPlot.ECDFPlotOpDesc +import org.apache.texera.amber.operator.visualization.figureFactoryTable.FigureFactoryTableOpDesc +import org.apache.texera.amber.operator.visualization.filledAreaPlot.FilledAreaPlotOpDesc +import org.apache.texera.amber.operator.visualization.funnelPlot.FunnelPlotOpDesc +import org.apache.texera.amber.operator.visualization.ganttChart.GanttChartOpDesc +import org.apache.texera.amber.operator.visualization.gaugeChart.GaugeChartOpDesc +import org.apache.texera.amber.operator.visualization.networkGraph.NetworkGraphOpDesc +import org.apache.texera.amber.operator.visualization.wordCloud.WordCloudOpDesc import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -66,6 +93,68 @@ class TransformVerificationRunnerSpec extends AnyFlatSpec with Matchers { disposition(classOf[LimitOpDesc]) shouldBe Runnable("auto") } + // Both were withheld for drawing a different picture on every run, which + // stopped being true once their placement was seeded. + it should "run the two seeded visualizations" in { + disposition(classOf[WordCloudOpDesc]) shouldBe Runnable("visualization") + disposition(classOf[NetworkGraphOpDesc]) shouldBe Runnable("visualization") + } + + it should "route the visualizations to the visualization tier" in { + val visualizations = Seq( + classOf[BarChartOpDesc], + classOf[BoxViolinPlotOpDesc], + classOf[BubbleChartOpDesc], + classOf[BulletChartOpDesc], + classOf[CandlestickChartOpDesc], + classOf[CarpetPlotOpDesc], + classOf[ChoroplethMapOpDesc], + classOf[ContinuousErrorBandsOpDesc], + classOf[ContourPlotOpDesc], + classOf[DendrogramOpDesc], + classOf[DotPlotOpDesc], + classOf[DumbbellPlotOpDesc], + classOf[ECDFPlotOpDesc], + classOf[FigureFactoryTableOpDesc], + classOf[FilledAreaPlotOpDesc], + classOf[FunnelPlotOpDesc], + classOf[GanttChartOpDesc], + classOf[GaugeChartOpDesc], + classOf[IcicleChartOpDesc], + classOf[ImageVisualizerOpDesc], + classOf[ScatterMatrixChartOpDesc] + ) + visualizations.foreach(op => + withClue(op.getSimpleName)(disposition(op) shouldBe Runnable("visualization")) + ) + } + + it should "route genuine one-off curated ops to the curated tier" in { + disposition(classOf[HashJoinOpDesc[_]]) shouldBe Runnable("curated") + } + + it should "route the scorer to the auto tier now the table holds a label pair" in { + // What kept it curated was the canonical table, not the operator: scoring reads + // one label through two columns, and until `species_pred` joined `species` there + // was no such pair for @SampleColumn to name. + disposition(classOf[MachineLearningScorerOpDesc]) shouldBe Runnable("auto") + } + + it should "route a sklearn estimator to the auto tier on the numeric projection" in { + disposition(classOf[SklearnTrainingLogisticRegressionOpDesc]) shouldBe + Runnable("auto, countVectorizer=false, tfidfTransformer=false") + fixtureFor(classOf[SklearnTrainingLogisticRegressionOpDesc]) shouldBe + CanonicalFixture.sklearnNumeric + } + + it should "route an advanced trainer to the auto tier on the same projection" in { + // Its `paraList` holds a row whose `parameter` is the operator's own enum, named + // only on the generic supertype. The generator resolves it, so the hand-written + // handler these four used to need is gone. + disposition(classOf[SklearnAdvancedSVCTrainerOpDesc]) shouldBe Runnable("auto") + fixtureFor(classOf[SklearnAdvancedSVCTrainerOpDesc]) shouldBe CanonicalFixture.sklearnNumeric + } + // A UDF's body is written by whoever drops the operator, so there is nothing // for a generator to emit. It stands here for the shape of the report: an // operator that cannot be exported is carried as a row, not passed over. From 97f71dd26bfbe7b32a6baa1cd8df8a98acebac7c Mon Sep 17 00:00:00 2001 From: kary zheng Date: Wed, 2 Sep 2026 17:56:37 -0700 Subject: [PATCH 09/10] style: assert the visualizations the way the rest of the file does One assertion per line, like every other test here. The loop it replaced was the only one of its kind in the file. Co-Authored-By: Claude Opus 5 (1M context) --- .../TransformVerificationRunnerSpec.scala | 47 +++++++++---------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunnerSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunnerSpec.scala index 4aedc11fa57..1dc73521e79 100644 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunnerSpec.scala +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunnerSpec.scala @@ -101,32 +101,27 @@ class TransformVerificationRunnerSpec extends AnyFlatSpec with Matchers { } it should "route the visualizations to the visualization tier" in { - val visualizations = Seq( - classOf[BarChartOpDesc], - classOf[BoxViolinPlotOpDesc], - classOf[BubbleChartOpDesc], - classOf[BulletChartOpDesc], - classOf[CandlestickChartOpDesc], - classOf[CarpetPlotOpDesc], - classOf[ChoroplethMapOpDesc], - classOf[ContinuousErrorBandsOpDesc], - classOf[ContourPlotOpDesc], - classOf[DendrogramOpDesc], - classOf[DotPlotOpDesc], - classOf[DumbbellPlotOpDesc], - classOf[ECDFPlotOpDesc], - classOf[FigureFactoryTableOpDesc], - classOf[FilledAreaPlotOpDesc], - classOf[FunnelPlotOpDesc], - classOf[GanttChartOpDesc], - classOf[GaugeChartOpDesc], - classOf[IcicleChartOpDesc], - classOf[ImageVisualizerOpDesc], - classOf[ScatterMatrixChartOpDesc] - ) - visualizations.foreach(op => - withClue(op.getSimpleName)(disposition(op) shouldBe Runnable("visualization")) - ) + disposition(classOf[BarChartOpDesc]) shouldBe Runnable("visualization") + disposition(classOf[BoxViolinPlotOpDesc]) shouldBe Runnable("visualization") + disposition(classOf[BubbleChartOpDesc]) shouldBe Runnable("visualization") + disposition(classOf[BulletChartOpDesc]) shouldBe Runnable("visualization") + disposition(classOf[CandlestickChartOpDesc]) shouldBe Runnable("visualization") + disposition(classOf[CarpetPlotOpDesc]) shouldBe Runnable("visualization") + disposition(classOf[ChoroplethMapOpDesc]) shouldBe Runnable("visualization") + disposition(classOf[ContinuousErrorBandsOpDesc]) shouldBe Runnable("visualization") + disposition(classOf[ContourPlotOpDesc]) shouldBe Runnable("visualization") + disposition(classOf[DendrogramOpDesc]) shouldBe Runnable("visualization") + disposition(classOf[DotPlotOpDesc]) shouldBe Runnable("visualization") + disposition(classOf[DumbbellPlotOpDesc]) shouldBe Runnable("visualization") + disposition(classOf[ECDFPlotOpDesc]) shouldBe Runnable("visualization") + disposition(classOf[FigureFactoryTableOpDesc]) shouldBe Runnable("visualization") + disposition(classOf[FilledAreaPlotOpDesc]) shouldBe Runnable("visualization") + disposition(classOf[FunnelPlotOpDesc]) shouldBe Runnable("visualization") + disposition(classOf[GanttChartOpDesc]) shouldBe Runnable("visualization") + disposition(classOf[GaugeChartOpDesc]) shouldBe Runnable("visualization") + disposition(classOf[IcicleChartOpDesc]) shouldBe Runnable("visualization") + disposition(classOf[ImageVisualizerOpDesc]) shouldBe Runnable("visualization") + disposition(classOf[ScatterMatrixChartOpDesc]) shouldBe Runnable("visualization") } it should "route genuine one-off curated ops to the curated tier" in { From 3549b51afefd794a289a6e3c5e755561e662e721 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Thu, 3 Sep 2026 15:28:15 -0700 Subject: [PATCH 10/10] test(verify): correct two comments that name things no longer there One said the coverage report prints the withheld runs together. It no longer prints them, so what is left is the reason that does not depend on a reader: one table means the next exemption has an obvious home. The other pointed at `TransformHandler.unfillableVariants`, which does not exist. What it meant is `nullsKeepFilled`, the one thing a handler still says about its own table rather than about its operator. `withheldRunsFor` was public for the report that read it. Its only caller now is beside it, so it says so. Co-Authored-By: Claude Opus 5 (1M context) --- .../verify/TransformVerificationRunner.scala | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala index 5383fd777a8..4c36d15ec8f 100644 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/TransformVerificationRunner.scala @@ -336,16 +336,16 @@ object TransformVerificationRunner { /** The runs an operator does not get, and why. * - * One table rather than one per kind. Every row makes the same statement, so - * the coverage report can print them together, and the next exemption has an - * obvious home instead of arriving as another set somewhere else. + * One table rather than one per kind: every row makes the same statement, so + * the next exemption has an obvious home instead of arriving as another set + * somewhere else. * * `op` matches its subclasses, so one row covers a family. `kind` is a * [[RunKind]]. * - * A curated handler's own [[TransformHandler.unfillableVariants]] stays where - * it is: those describe the table that handler wrote rather than the operator, - * and change when the fixture is rewritten. + * A curated handler's own [[TransformHandler.nullsKeepFilled]] stays where it + * is: it describes the table that handler wrote rather than the operator, and + * changes when the fixture is rewritten. */ val variantsNotRun: Seq[NotRun] = { // The operator refuses the text pipeline in `getOutputSchemas`, so there is no @@ -411,12 +411,11 @@ object TransformVerificationRunner { ) ++ denseOnly } - /** Every kind of run withheld from this operator, with why. This is the whole of - * what the coverage report needs, so it never walks the table itself. One entry - * per kind: a family row and an operator row for the same kind are the same - * statement twice, and the first one wins. + /** Every kind of run withheld from this operator, with why. One entry per kind: + * a family row and an operator row for the same kind are the same statement + * twice, and the first one wins. */ - def withheldRunsFor(opClass: Class[_ <: LogicalOp]): Seq[(String, NotRunReason)] = + private def withheldRunsFor(opClass: Class[_ <: LogicalOp]): Seq[(String, NotRunReason)] = variantsNotRun .collect { case NotRun(op, kind, reason) if op.isAssignableFrom(opClass) => kind -> reason } .distinctBy(_._1)