diff --git a/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java b/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java index 09ad6dba0..a995655c9 100644 --- a/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java +++ b/src/main/java/org/apache/datasketches/tdigest/TDigestDouble.java @@ -192,7 +192,8 @@ public double getRank(final double value) { if (value < firstMean) { if ((firstMean - minValue_) > 0) { if (value == minValue_) { return 0.5 / centroidsWeight_; } - return (1.0 + (((value - minValue_) / (firstMean - minValue_)) * ((centroidWeights_[0] / 2.0) - 1.0))); + return (1.0 + (((value - minValue_) / (firstMean - minValue_)) + * ((centroidWeights_[0] / 2.0) - 1.0))) / centroidsWeight_; } return 0; // should never happen } @@ -255,7 +256,7 @@ public double getQuantile(final double rank) { } final double lastWeight = centroidWeights_[numCentroids_ - 1]; if ((lastWeight > 1) && ((centroidsWeight_ - weight) <= (lastWeight / 2.0))) { - return maxValue_ + (((centroidsWeight_ - weight - 1.0) / ((lastWeight / 2.0) - 1.0)) + return maxValue_ - (((centroidsWeight_ - weight - 1.0) / ((lastWeight / 2.0) - 1.0)) * (maxValue_ - centroidMeans_[numCentroids_ - 1])); } @@ -277,7 +278,7 @@ public double getQuantile(final double rank) { } final double w1 = weight - weightSoFar - leftWeight; final double w2 = (weightSoFar + dw) - weight - rightWeight; - return weightedAverage(centroidMeans_[i], w1, centroidMeans_[i + 1], w2); + return weightedAverage(centroidMeans_[i], w2, centroidMeans_[i + 1], w1); } weightSoFar += dw; } diff --git a/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java b/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java index 18e4103cf..b97a4e918 100644 --- a/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java +++ b/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java @@ -236,6 +236,59 @@ public void deserializeNaNSingleValue() { assertThrows(SketchesArgumentException.class, () -> TDigestDouble.heapify(MemorySegment.ofArray(bytes))); } + @Test + public void rankBelowFirstCentroidMean() { + // the format allows a first centroid of weight greater than 1, so the left tail of + // getRank() must stay normalized just like the right tail + final byte[] bytes = serializeNonEmpty(); + MemorySegment.ofArray(bytes).set(ValueLayout.JAVA_DOUBLE_UNALIGNED, 16, -1); // min + MemorySegment.ofArray(bytes).set(ValueLayout.JAVA_LONG_UNALIGNED, 40, 100L); // first weight + final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes)); + final double totalWeight = td.getTotalWeight(); + assertEquals(td.getRank(-1), 0.5 / totalWeight); + assertEquals(td.getRank(-0.5), (1.0 + (((100 / 2.0) - 1.0) * 0.5)) / totalWeight); + double previous = 0; + for (int i = 0; i <= 100; i++) { + final double rank = td.getRank(-1 + (i / 100.0)); + assertTrue((rank >= 0) && (rank <= 1), "rank out of [0, 1]: " + rank); + assertTrue(rank >= previous, "rank not monotonic: " + rank + " after " + previous); + previous = rank; + } + } + + @Test + public void quantilesAreMonotonic() { + final TDigestDouble td = new TDigestDouble((short) 100); + for (int i = 0; i < 10000; i++) { + td.update(i); + } + double previous = td.getMinValue(); + for (int i = 0; i <= 1000; i++) { + final double quantile = td.getQuantile(i / 1000.0); + assertTrue(quantile >= previous, "quantile not monotonic: " + quantile + " after " + previous); + assertTrue((quantile >= td.getMinValue()) && (quantile <= td.getMaxValue()), + "quantile out of [min, max]: " + quantile); + previous = quantile; + } + } + + @Test + public void quantileAboveLastCentroidMean() { + final byte[] bytes = serializeNonEmpty(); + final MemorySegment seg = MemorySegment.ofArray(bytes); + final int numCentroids = seg.get(ValueLayout.JAVA_INT_UNALIGNED, 8); + final long lastWeightOffset = 40 + ((numCentroids - 1) * 16L); + seg.set(ValueLayout.JAVA_LONG_UNALIGNED, lastWeightOffset, 100L); + final TDigestDouble td = TDigestDouble.heapify(seg); + double previous = td.getMinValue(); + for (int i = 0; i <= 1000; i++) { + final double quantile = td.getQuantile(i / 1000.0); + assertTrue(quantile >= previous, "quantile not monotonic: " + quantile + " after " + previous); + assertTrue(quantile <= td.getMaxValue(), "quantile above max: " + quantile); + previous = quantile; + } + } + @Test public void deserializeFromReferenceImplementationDouble() { final byte[] bytes = TestUtil.getFileBytes(resPath, "tdigest_ref_k100_n10000_double.sk");