From 61fdf3ea773bef6785da0596a738223b3c920241 Mon Sep 17 00:00:00 2001 From: Johannes Schmidt <89488492+johannesparty@users.noreply.github.com> Date: Fri, 26 Jun 2026 11:10:00 -0700 Subject: [PATCH 1/2] fix: handle empty-feature depth slices in global rank MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rank_soils_global builds a per-depth-slice feature matrix and passes it to gower_distances. When a depth slice has zero usable feature columns — e.g. the user recorded a depth interval but left texture/rock-fragment/color blank, which the bedrock column-filter can reduce to nothing — gower_distances fed a shape=(n, 0) array into SimpleImputer, raising "Found array with 0 feature(s)… minimum of 1 is required" and failing the whole ranking. Guard the gower call: for a 0-feature slice emit an all-NaN (n, n) distance matrix instead. The existing masked-average and NaN-infill steps already treat NaN distances as "no information", so the slice is ignored and components are ranked on the depths that do have data. An all-NaN matrix (rather than skipping the slice) keeps dis_mat_list positionally aligned with soil_matrix rows. Also make dis_max a single NaN-aware reduction over the stack so an all-NaN slice can't turn the max NaN via max()'s ordering. Co-Authored-By: Claude Opus 4.8 --- soil_id/global_soil.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/soil_id/global_soil.py b/soil_id/global_soil.py index 3155ad5..eeb088c 100644 --- a/soil_id/global_soil.py +++ b/soil_id/global_soil.py @@ -829,7 +829,19 @@ def rank_soils_global( slice_mat = slice_df.drop("compname", axis=1) # Compute the Gower distance on the prepared slice matrix. - D = gower_distances(slice_mat) + # A slice can end up with zero feature columns when this depth has no + # usable measurements (e.g. the user recorded a depth interval but left + # texture/rock-fragment/color blank). gower_distances would crash its + # mean-imputer on a 0-feature array, so emit an all-NaN distance matrix + # instead. The masked average below and the NaN-infill loop already treat + # NaN distances as "no information", so this slice is simply ignored and + # components are ranked on the depths that do have data. An all-NaN matrix + # (rather than skipping) keeps dis_mat_list aligned with soil_matrix rows. + if slice_mat.shape[1] == 0: + n = slice_mat.shape[0] + D = np.full((n, n), np.nan) + else: + D = gower_distances(slice_mat) dis_mat_list.append(D) @@ -840,8 +852,10 @@ def rank_soils_global( "Not Ranked" if np.ma.is_masked(x) else "Ranked" for x in D_check[0][1:] ] - # Calculate max dissimilarity per depth slice - dis_max = max(map(np.nanmax, dis_mat_list)) + # Calculate max dissimilarity across all depth slices. Use a single + # NaN-aware reduction over the stack so an all-NaN slice (a depth with no + # usable data) can't make the result NaN via max()'s ordering. + dis_max = np.nanmax(dis_mat_list) # Apply depth weight depth_weight = np.concatenate([np.repeat(0.2, 20), np.repeat(1.0, 180)]) From 237c9d92343940fbf04252cccbb157de84f6ae5c Mon Sep 17 00:00:00 2001 From: Johannes Schmidt <89488492+johannesparty@users.noreply.github.com> Date: Fri, 26 Jun 2026 11:51:47 -0700 Subject: [PATCH 2/2] test: cover the empty-feature slice guard in global rank Extract the per-slice gower computation into _slice_gower_distance so the empty-feature guard is unit-testable without a database or network, then add fast regression tests: - a 0-feature slice returns an all-NaN (n, n) matrix instead of crashing - the raw gower path still raises on a 0-feature array (characterization, so the guard's rationale is revisited if that ever changes) - a non-empty slice passes through to gower_distances unchanged Behavior of rank_soils_global is unchanged; this only names the guard and gives it a seam to test against. Co-Authored-By: Claude Opus 4.8 --- soil_id/global_soil.py | 40 ++++++++---- soil_id/tests/test_global_rank_empty_slice.py | 64 +++++++++++++++++++ 2 files changed, 90 insertions(+), 14 deletions(-) create mode 100644 soil_id/tests/test_global_rank_empty_slice.py diff --git a/soil_id/global_soil.py b/soil_id/global_soil.py index eeb088c..c4e415c 100644 --- a/soil_id/global_soil.py +++ b/soil_id/global_soil.py @@ -555,6 +555,27 @@ def convert_to_serializable(obj): ) +def _slice_gower_distance(slice_mat): + """Gower distance matrix for a single depth slice. + + A slice can have zero feature columns when a depth has no usable measurements + — e.g. the user recorded a depth interval but left texture/rock-fragment/color + blank, or (after depth-aligning) the horizon data simply does not reach this + depth. ``gower_distances`` would feed a shape ``(n, 0)`` array into its mean + imputer and raise "Found array with 0 feature(s)…", failing the whole ranking. + + For such a slice, return an all-NaN ``(n, n)`` matrix instead. Callers treat NaN + distances as "no information" (the masked average and NaN-infill steps), so the + slice is ignored and components rank on the depths that do have data. Returning a + matrix (rather than skipping) keeps the per-slice list aligned with soil_matrix + rows. + """ + if slice_mat.shape[1] == 0: + n = slice_mat.shape[0] + return np.full((n, n), np.nan) + return gower_distances(slice_mat) + + ############################################################################################## # rankPredictionGlobal # ############################################################################################## @@ -828,20 +849,11 @@ def rank_soils_global( else: slice_mat = slice_df.drop("compname", axis=1) - # Compute the Gower distance on the prepared slice matrix. - # A slice can end up with zero feature columns when this depth has no - # usable measurements (e.g. the user recorded a depth interval but left - # texture/rock-fragment/color blank). gower_distances would crash its - # mean-imputer on a 0-feature array, so emit an all-NaN distance matrix - # instead. The masked average below and the NaN-infill loop already treat - # NaN distances as "no information", so this slice is simply ignored and - # components are ranked on the depths that do have data. An all-NaN matrix - # (rather than skipping) keeps dis_mat_list aligned with soil_matrix rows. - if slice_mat.shape[1] == 0: - n = slice_mat.shape[0] - D = np.full((n, n), np.nan) - else: - D = gower_distances(slice_mat) + # Compute the Gower distance on the prepared slice matrix. A slice with + # zero usable feature columns is handled inside the helper (see its + # docstring) so the per-depth mean imputer can't crash on a 0-feature + # array; it returns an all-NaN matrix that the steps below then ignore. + D = _slice_gower_distance(slice_mat) dis_mat_list.append(D) diff --git a/soil_id/tests/test_global_rank_empty_slice.py b/soil_id/tests/test_global_rank_empty_slice.py new file mode 100644 index 0000000..eeb654d --- /dev/null +++ b/soil_id/tests/test_global_rank_empty_slice.py @@ -0,0 +1,64 @@ +# Copyright © 2026 Technology Matters +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see https://www.gnu.org/licenses/. + +"""Regression tests for the empty-feature depth-slice guard in global rank. + +rank_soils_global builds a per-depth-slice feature matrix and feeds it to +gower_distances. When a depth slice has zero usable feature columns (e.g. a gap in +the user's horizons, or horizon data that doesn't reach the slice depth), the raw +gower path passes a shape ``(n, 0)`` array into SimpleImputer and raises +"Found array with 0 feature(s)…", failing the whole ranking. ``_slice_gower_distance`` +guards that case. These tests cover the guard directly so they need no database or +network — unlike the live ``test_global_integration`` path. +""" + +import numpy as np +import pandas as pd +import pytest + +from soil_id.global_soil import _slice_gower_distance +from soil_id.utils import gower_distances + + +def test_zero_feature_slice_returns_all_nan_matrix(): + # 11 components, but no feature columns — the exact shape from the Sentry crash. + slice_mat = pd.DataFrame(index=range(11)) + assert slice_mat.shape == (11, 0) + + result = _slice_gower_distance(slice_mat) + + # An (n, n) all-NaN matrix: same shape gower would have returned, and the + # downstream masked-average/NaN-infill steps treat it as "no information". + assert result.shape == (11, 11) + assert np.isnan(result).all() + + +def test_zero_feature_slice_would_otherwise_crash_gower(): + # Characterization: the raw gower path crashes on a 0-feature array. This is + # exactly what the guard above prevents; if gower ever stops raising here, the + # guard's rationale should be revisited. + slice_mat = pd.DataFrame(index=range(11)) + with pytest.raises(ValueError): + gower_distances(slice_mat) + + +def test_nonempty_slice_passes_through_to_gower(): + # With at least one feature column, the helper must be a transparent passthrough. + slice_mat = pd.DataFrame({"sand": [10.0, 20.0, 30.0], "clay": [5.0, 15.0, 25.0]}) + + result = _slice_gower_distance(slice_mat) + expected = gower_distances(slice_mat) + + assert np.array_equal(result, expected, equal_nan=True)