From d14f1c38659a978aa99c0e253aaf5e1d65aaf9bd Mon Sep 17 00:00:00 2001 From: Jared Lockhart <119884+jaredlockhart@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:39:27 -0400 Subject: [PATCH] fix(nimbus): distinguish errored metrics from not-yet-computed metrics on results Because * The results page branches on has_errors or not has_data, so a metric Jetstream failed on and a metric whose analysis window has not run yet render identically. * Every weekly and overall outcome metric on a recently launched experiment hits the second state, so the page offers a support-contact banner for an analysis that is working correctly. This commit * Splits the two states in the results template so errors keep the amber warning treatment and the support button. * Renders a blue informational state for metrics with no data yet, without a support button. * Moves the results availability copy into NimbusUIConstants. * Adds view tests covering both states. Fixes #17155 --- .../experimenter/nimbus_ui/constants.py | 9 ++ .../nimbus_experiments/results-inner.html | 30 ++++--- .../nimbus_ui/tests/test_views.py | 88 +++++++++++++++++++ 3 files changed, 117 insertions(+), 10 deletions(-) diff --git a/experimenter/experimenter/nimbus_ui/constants.py b/experimenter/experimenter/nimbus_ui/constants.py index 45f141f563..bc1235cb38 100644 --- a/experimenter/experimenter/nimbus_ui/constants.py +++ b/experimenter/experimenter/nimbus_ui/constants.py @@ -274,6 +274,15 @@ class NimbusUIConstants: "All statistically significant changes that have occurred in the experiment" ) NOTABLE_CHANGES_ABSENT_TEXT = "There are no notable changes in this experiment" + METRIC_ERRORS_TOOLTIP = "No data available" + METRIC_ERRORS_TITLE = "Metric unavailable" + METRIC_ERRORS_TEXT = "Other metrics may not be affected." + METRIC_NO_DATA_TOOLTIP = "Not available yet" + METRIC_NO_DATA_TITLE = "Not available yet" + METRIC_NO_DATA_TEXT = ( + "The analysis window for this metric has not been computed yet. " + "Results will appear once it has." + ) FEATURE_MONITORING_CARD_TITLE = "Feature Monitoring" FEATURE_MONITORING_OPEN_DASHBOARD_TEXT = "Open in Grafana" FEATURE_MONITORING_DASHBOARD_INFO = ( diff --git a/experimenter/experimenter/nimbus_ui/templates/nimbus_experiments/results-inner.html b/experimenter/experimenter/nimbus_ui/templates/nimbus_experiments/results-inner.html index a5bd292b32..7ed798e13b 100644 --- a/experimenter/experimenter/nimbus_ui/templates/nimbus_experiments/results-inner.html +++ b/experimenter/experimenter/nimbus_ui/templates/nimbus_experiments/results-inner.html @@ -272,9 +272,9 @@

{{ area }}

{{ metric.friendly_name }} - + title="{% if metric.has_errors %}{{ NimbusUIConstants.METRIC_ERRORS_TOOLTIP }}{% elif not metric.has_data %}{{ NimbusUIConstants.METRIC_NO_DATA_TOOLTIP }}{% elif metric.overall_change == 'positive' %}Positive changes{% elif metric.overall_change == 'negative' %}Negative changes{% elif metric.overall_change == 'mixed' %}Mixed changes{% elif metric.overall_change == 'neutral' %}No notable change{% endif %}"> {% if not forloop.last %}·{% endif %} @@ -389,14 +389,24 @@

{{ area }}

{% if metric.has_errors or not metric.has_data %} {% if forloop.parentloop.parentloop.first %}
-
- -

Metric unavailable

-

Other metrics may not be affected.

- Contact Experimenter Support -
+ {% if metric.has_errors %} +
+ +

{{ NimbusUIConstants.METRIC_ERRORS_TITLE }}

+

{{ NimbusUIConstants.METRIC_ERRORS_TEXT }}

+ Contact Experimenter Support +
+ {% else %} +
+ +

{{ NimbusUIConstants.METRIC_NO_DATA_TITLE }}

+

{{ NimbusUIConstants.METRIC_NO_DATA_TEXT }}

+
+ {% endif %}
{% else %} diff --git a/experimenter/experimenter/nimbus_ui/tests/test_views.py b/experimenter/experimenter/nimbus_ui/tests/test_views.py index 376dd21327..930fc35c83 100644 --- a/experimenter/experimenter/nimbus_ui/tests/test_views.py +++ b/experimenter/experimenter/nimbus_ui/tests/test_views.py @@ -3864,6 +3864,94 @@ def test_results_template_fragment_used_with_htmx(self): self.assertEqual(response.status_code, 200) self.assertTemplateUsed(response, "nimbus_experiments/results-fragment.html") + def create_experiment_with_kpi_results(self, errors): + experiment = NimbusExperimentFactory.create_with_lifecycle( + NimbusExperimentFactory.Lifecycles.ENDING_APPROVE_APPROVE, + application=NimbusExperiment.Application.DESKTOP, + primary_outcomes=[], + secondary_outcomes=[], + is_rollout=False, + ) + experiment.delete_branches() + experiment.reference_branch = NimbusBranchFactory.create( + experiment=experiment, name="Control", slug="control" + ) + NimbusBranchFactory.create( + experiment=experiment, name="Treatment A", slug="treatment-a" + ) + empty_metric = { + "absolute": {"all": [], "first": {}}, + "difference": {"control": {"all": [], "first": {}}}, + "relative_uplift": {"control": {"all": [], "first": {}}}, + } + branch_data = { + "branch_data": { + "other_metrics": {NimbusConstants.RETENTION: empty_metric}, + } + } + experiment.results_data = { + "v3": { + "metadata": { + "metrics": { + NimbusConstants.RETENTION: {"friendly_name": "Retention"}, + } + }, + "errors": errors, + "overall": { + "enrollments": { + "all": { + "control": branch_data, + "treatment-a": branch_data, + } + } + }, + } + } + experiment.save() + return experiment + + def test_results_view_renders_error_state_for_metric_with_errors(self): + experiment = self.create_experiment_with_kpi_results( + { + NimbusConstants.RETENTION: [ + {"analysis_basis": "enrollments", "segment": "all"} + ] + } + ) + + response = self.client.get( + reverse( + "nimbus-ui-results", + kwargs={"slug": experiment.slug}, + query={"reference_branch": "control"}, + ), + ) + + self.assertEqual(response.status_code, 200) + self.assertContains(response, "fa-triangle-exclamation text-warning") + self.assertContains(response, NimbusUIConstants.METRIC_ERRORS_TOOLTIP) + self.assertContains(response, NimbusUIConstants.METRIC_ERRORS_TITLE) + self.assertContains(response, escape(NimbusUIConstants.METRIC_ERRORS_TEXT)) + self.assertContains(response, "Contact Experimenter Support") + + def test_results_view_renders_no_data_state_for_metric_without_data(self): + experiment = self.create_experiment_with_kpi_results({}) + + response = self.client.get( + reverse( + "nimbus-ui-results", + kwargs={"slug": experiment.slug}, + query={"reference_branch": "control"}, + ), + ) + + self.assertEqual(response.status_code, 200) + self.assertContains(response, "fa-circle-info text-info") + self.assertContains(response, NimbusUIConstants.METRIC_NO_DATA_TOOLTIP) + self.assertContains(response, NimbusUIConstants.METRIC_NO_DATA_TEXT) + self.assertNotContains(response, NimbusUIConstants.METRIC_ERRORS_TITLE) + self.assertNotContains(response, "Contact Experimenter Support") + @parameterized.expand( [ (