From ca9586ca4b80c95068403ff9c2b2d14d64682d5b Mon Sep 17 00:00:00 2001
From: PritamSGB <29685062+PritamSGB@users.noreply.github.com>
Date: Thu, 10 Sep 2026 20:59:51 +0530
Subject: [PATCH 1/6] feat(eval): add combined binary metrics and validator
 config to PII eval output

Surfaces overall precision/recall/f1/accuracy (not just per-entity) and the
PIIRemover config (entity types, threshold, NLP engine, model, on_fail) used
for the run, so metrics.json is self-describing for external readers.
---
 backend/app/evaluation/pii/run.py | 27 +++++++++++++++++++++------
 1 file changed, 21 insertions(+), 6 deletions(-)

diff --git a/backend/app/evaluation/pii/run.py b/backend/app/evaluation/pii/run.py
index ccd669c1..9e5c84e9 100644
--- a/backend/app/evaluation/pii/run.py
+++ b/backend/app/evaluation/pii/run.py
@@ -6,6 +6,7 @@
 from app.evaluation.common.helper import (
     Profiler,
     build_evaluation_report,
+    compute_binary_metrics,
     write_csv,
     write_json,
 )
@@ -18,24 +19,36 @@
 
 validator = PIIRemover()
 
+config = {
+    "entity_types": validator.entity_types,
+    "threshold": validator.threshold,
+    "nlp_engine_type": validator.nlp_engine_type,
+    "model_name": validator.model_name,
+    "on_fail": validator.on_fail,
+    "language": "en",  # hardcoded in PIIRemover._validate; not a constructor param
+}
 
-def run_pii(text: str) -> str:
+
+def run_pii(text: str) -> tuple[str, int]:
     result = validator._validate(text)
     if isinstance(result, FailResult):
-        return result.fix_value
-    return text
+        return result.fix_value, 1
+    return text, 0
 
 
 with Profiler() as p:
-    df["anonymized"] = (
-        df["source_text"].astype(str).apply(lambda x: p.record(run_pii, x))
-    )
+    results = df["source_text"].astype(str).apply(lambda x: p.record(run_pii, x))
+    df["anonymized"] = results.apply(lambda r: r[0])
+    df["pii_detected"] = results.apply(lambda r: r[1])
 
 entity_report = compute_entity_metrics(
     df["target_text"],
     df["anonymized"],
 )
 
+y_true = (df["label"] == "pii").astype(int)
+binary_report = compute_binary_metrics(y_true, df["pii_detected"])
+
 # ---- Save outputs ----
 write_csv(df, OUT_DIR / "predictions.csv")
 
@@ -44,7 +57,9 @@ def run_pii(text: str) -> str:
         guardrail="pii_remover",
         num_samples=len(df),
         profiler=p,
+        config=config,
         entity_metrics=entity_report,
+        binary_metrics=binary_report,
     ),
     OUT_DIR / "metrics.json",
 )

From d2a1effbb7cdcafc5de4268e98016b4b5c6b48d3 Mon Sep 17 00:00:00 2001
From: PritamSGB <29685062+PritamSGB@users.noreply.github.com>
Date: Thu, 10 Sep 2026 21:05:25 +0530
Subject: [PATCH 2/6] refactor(eval): rename PII binary_metrics output key to
 combined_metrics

Clarifies that this is PII's overall detection-level score alongside
entity_metrics, distinct from the shared compute_binary_metrics() helper
used as the sole metric in other eval scripts.
---
 backend/app/evaluation/pii/run.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/backend/app/evaluation/pii/run.py b/backend/app/evaluation/pii/run.py
index 9e5c84e9..97678b76 100644
--- a/backend/app/evaluation/pii/run.py
+++ b/backend/app/evaluation/pii/run.py
@@ -47,7 +47,7 @@ def run_pii(text: str) -> tuple[str, int]:
 )
 
 y_true = (df["label"] == "pii").astype(int)
-binary_report = compute_binary_metrics(y_true, df["pii_detected"])
+combined_report = compute_binary_metrics(y_true, df["pii_detected"])
 
 # ---- Save outputs ----
 write_csv(df, OUT_DIR / "predictions.csv")
@@ -59,7 +59,7 @@ def run_pii(text: str) -> tuple[str, int]:
         profiler=p,
         config=config,
         entity_metrics=entity_report,
-        binary_metrics=binary_report,
+        combined_metrics=combined_report,
     ),
     OUT_DIR / "metrics.json",
 )

From c91514debfee233e94acae14b27c5c67e0084d1f Mon Sep 17 00:00:00 2001
From: PritamSGB <29685062+PritamSGB@users.noreply.github.com>
Date: Thu, 10 Sep 2026 21:23:20 +0530
Subject: [PATCH 3/6] feat(eval): include validator config in gender assumption
 bias metrics output

Mirrors the pii_remover eval script, which already records the config used
(entity types, threshold, etc.) alongside the metrics for reproducibility.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 backend/app/evaluation/gender_assumption_bias/run.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/backend/app/evaluation/gender_assumption_bias/run.py b/backend/app/evaluation/gender_assumption_bias/run.py
index 301f4503..679a1345 100644
--- a/backend/app/evaluation/gender_assumption_bias/run.py
+++ b/backend/app/evaluation/gender_assumption_bias/run.py
@@ -18,6 +18,12 @@
 
 validator = GenderAssumptionBias()
 
+config = {
+    "categories": [category.value for category in validator.categories],
+    "on_fail": validator.on_fail_descriptor,
+    "num_bias_words_loaded": len(validator.gender_bias_list),
+}
+
 with Profiler() as p:
     df["biased_result"] = (
         df["biased input"]
@@ -57,6 +63,7 @@
         guardrail="gender_assumption_bias",
         num_samples=len(df) * 2,
         profiler=p,
+        config=config,
         metrics=metrics,
     ),
     OUT_DIR / "metrics.json",

From 350b81e2b68b394e2c7918b50a9e1eade3abcc6b Mon Sep 17 00:00:00 2001
From: PritamSGB <29685062+PritamSGB@users.noreply.github.com>
Date: Thu, 10 Sep 2026 21:36:23 +0530
Subject: [PATCH 4/6] feat(eval): add shared validator config helper, apply
 config to all evaluations

Introduces build_validator_config() in common/helper.py, which reads on_fail
from validator.on_fail_descriptor (set by every guardrails Validator base
class) and normalizes any extra constructor params passed in (enums -> their
.value). PII and gender_assumption_bias now use it instead of hand-rolled
dicts, and ban_list, lexical_slur, topic_relevance, and toxicity gain a
config block in their metrics.json for the first time.
---
 backend/app/evaluation/ban_list/run.py        |  4 +-
 backend/app/evaluation/common/helper.py       | 28 +++++++++++++
 .../evaluation/gender_assumption_bias/run.py  | 11 ++---
 backend/app/evaluation/lexical_slur/run.py    |  9 ++++
 backend/app/evaluation/pii/run.py             | 17 ++++----
 backend/app/evaluation/topic_relevance/run.py | 16 +++----
 backend/app/evaluation/toxicity/run.py        | 42 +++++++++++++------
 7 files changed, 94 insertions(+), 33 deletions(-)

diff --git a/backend/app/evaluation/ban_list/run.py b/backend/app/evaluation/ban_list/run.py
index 2d630991..03620da8 100644
--- a/backend/app/evaluation/ban_list/run.py
+++ b/backend/app/evaluation/ban_list/run.py
@@ -6,6 +6,7 @@
 
 from app.evaluation.common.helper import (
     build_evaluation_report,
+    build_validator_config,
     Profiler,
     compute_binary_metrics,
     write_csv,
@@ -35,6 +36,7 @@ def run_evaluation(config: dict):
     dataset = pd.read_csv(DATASET_PATH)
 
     validator = BanList(banned_words=banned_words)
+    validator_config = build_validator_config(validator, banned_words=banned_words)
 
     def run_ban_list(text: str) -> tuple[str, int]:
         """Validate a single text and return the (possibly redacted) text and a binary prediction label."""
@@ -81,7 +83,7 @@ def run_ban_list(text: str) -> tuple[str, int]:
             guardrail="ban_list",
             num_samples=len(dataset),
             profiler=p,
-            banned_words=banned_words,
+            config=validator_config,
             dataset=str(DATASET_PATH.name),
             metrics=metrics,
         ),
diff --git a/backend/app/evaluation/common/helper.py b/backend/app/evaluation/common/helper.py
index a72c83c1..b24c6fa6 100644
--- a/backend/app/evaluation/common/helper.py
+++ b/backend/app/evaluation/common/helper.py
@@ -1,3 +1,4 @@
+from enum import Enum
 from pathlib import Path
 from typing import Any
 import json
@@ -52,6 +53,33 @@ def build_evaluation_report(
     }
 
 
+def _to_jsonable(value: Any) -> Any:
+    """Recursively normalize enums (and lists/dicts containing them) to JSON-safe values."""
+    if isinstance(value, Enum):
+        return value.value
+    if isinstance(value, (list, tuple)):
+        return [_to_jsonable(v) for v in value]
+    if isinstance(value, dict):
+        return {k: _to_jsonable(v) for k, v in value.items()}
+    return value
+
+
+def build_validator_config(validator: Any, **fields: Any) -> dict[str, Any]:
+    """
+    Build the `config` block for an evaluation's metrics.json.
+
+    `on_fail` is read from `validator.on_fail_descriptor`, which every
+    guardrails Validator subclass sets in its base __init__, so it's always
+    available without each evaluation script re-deriving it. Any
+    validator-specific constructor params (entity_types, threshold,
+    categories, ...) are passed in as keyword args and normalized the same
+    way (e.g. enums -> their .value).
+    """
+    config = {"on_fail": _to_jsonable(validator.on_fail_descriptor)}
+    config.update({key: _to_jsonable(value) for key, value in fields.items()})
+    return config
+
+
 def compute_binary_metrics(y_true, y_pred):
     tp = sum((yt == 1 and yp == 1) for yt, yp in zip(y_true, y_pred, strict=True))
     tn = sum((yt == 0 and yp == 0) for yt, yp in zip(y_true, y_pred, strict=True))
diff --git a/backend/app/evaluation/gender_assumption_bias/run.py b/backend/app/evaluation/gender_assumption_bias/run.py
index 679a1345..9d9ae11d 100644
--- a/backend/app/evaluation/gender_assumption_bias/run.py
+++ b/backend/app/evaluation/gender_assumption_bias/run.py
@@ -5,6 +5,7 @@
 from app.core.validators.gender_assumption_bias import GenderAssumptionBias
 from app.evaluation.common.helper import (
     build_evaluation_report,
+    build_validator_config,
     compute_binary_metrics,
     Profiler,
     write_csv,
@@ -18,11 +19,11 @@
 
 validator = GenderAssumptionBias()
 
-config = {
-    "categories": [category.value for category in validator.categories],
-    "on_fail": validator.on_fail_descriptor,
-    "num_bias_words_loaded": len(validator.gender_bias_list),
-}
+config = build_validator_config(
+    validator,
+    categories=validator.categories,
+    num_bias_words_loaded=len(validator.gender_bias_list),
+)
 
 with Profiler() as p:
     df["biased_result"] = (
diff --git a/backend/app/evaluation/lexical_slur/run.py b/backend/app/evaluation/lexical_slur/run.py
index 9f1808d5..7cd74a01 100644
--- a/backend/app/evaluation/lexical_slur/run.py
+++ b/backend/app/evaluation/lexical_slur/run.py
@@ -5,6 +5,7 @@
 from app.core.validators.lexical_slur import LexicalSlur
 from app.evaluation.common.helper import (
     build_evaluation_report,
+    build_validator_config,
     Profiler,
     compute_binary_metrics,
     write_csv,
@@ -18,6 +19,13 @@
 
 validator = LexicalSlur()
 
+config = build_validator_config(
+    validator,
+    severity=validator.severity,
+    languages=validator.languages,
+    num_slurs_loaded=len(validator.slur_list),
+)
+
 with Profiler() as p:
     df["result"] = (
         df["commentText"]
@@ -38,6 +46,7 @@
         guardrail="lexical_slur",
         num_samples=len(df),
         profiler=p,
+        config=config,
         metrics=metrics,
     ),
     OUT_DIR / "metrics.json",
diff --git a/backend/app/evaluation/pii/run.py b/backend/app/evaluation/pii/run.py
index 97678b76..fd989038 100644
--- a/backend/app/evaluation/pii/run.py
+++ b/backend/app/evaluation/pii/run.py
@@ -6,6 +6,7 @@
 from app.evaluation.common.helper import (
     Profiler,
     build_evaluation_report,
+    build_validator_config,
     compute_binary_metrics,
     write_csv,
     write_json,
@@ -19,14 +20,14 @@
 
 validator = PIIRemover()
 
-config = {
-    "entity_types": validator.entity_types,
-    "threshold": validator.threshold,
-    "nlp_engine_type": validator.nlp_engine_type,
-    "model_name": validator.model_name,
-    "on_fail": validator.on_fail,
-    "language": "en",  # hardcoded in PIIRemover._validate; not a constructor param
-}
+config = build_validator_config(
+    validator,
+    entity_types=validator.entity_types,
+    threshold=validator.threshold,
+    nlp_engine_type=validator.nlp_engine_type,
+    model_name=validator.model_name,
+    language="en",  # hardcoded in PIIRemover._validate; not a constructor param
+)
 
 
 def run_pii(text: str) -> tuple[str, int]:
diff --git a/backend/app/evaluation/topic_relevance/run.py b/backend/app/evaluation/topic_relevance/run.py
index d450e117..52fa1b5f 100644
--- a/backend/app/evaluation/topic_relevance/run.py
+++ b/backend/app/evaluation/topic_relevance/run.py
@@ -11,6 +11,7 @@
 from app.evaluation.common.helper import (
     Profiler,
     build_evaluation_report,
+    build_validator_config,
     compute_binary_metrics,
     write_csv,
     write_json,
@@ -42,9 +43,9 @@
             prompt_schema_version=1,
             llm_callable=settings.DEFAULT_LLM_CALLABLE,
         ),
-        "report_extra": {
-            "llm_callable": settings.DEFAULT_LLM_CALLABLE,
-            "prompt_schema_version": 1,
+        "config_fields": lambda v: {
+            "llm_callable": v.llm_callable,
+            "prompt_schema_version": v.prompt_schema_version,
         },
     },
     {
@@ -55,9 +56,9 @@
             llm_callable=settings.DEFAULT_LLM_CALLABLE,
             threshold=settings.TOPIC_RELEVANCE_LLM_THRESHOLD,
         ),
-        "report_extra": {
-            "llm_callable": settings.DEFAULT_LLM_CALLABLE,
-            "threshold": settings.TOPIC_RELEVANCE_LLM_THRESHOLD,
+        "config_fields": lambda v: {
+            "llm_callable": v.llm_callable,
+            "threshold": v.threshold,
         },
     },
 ]
@@ -73,6 +74,7 @@ def run_evaluation(dataset: dict, backend: dict) -> None:
 
     df = pd.read_csv(dataset_path)
     validator = backend["build"](topic_config)
+    config = build_validator_config(validator, **backend["config_fields"](validator))
 
     normalized_df = pd.DataFrame(
         {
@@ -115,7 +117,7 @@ def run_evaluation(dataset: dict, backend: dict) -> None:
             num_samples=len(normalized_df),
             profiler=p,
             dataset=str(dataset_path),
-            **backend["report_extra"],
+            config=config,
             metrics=metrics,
         ),
         out_dir / f"{domain}-metrics.json",
diff --git a/backend/app/evaluation/toxicity/run.py b/backend/app/evaluation/toxicity/run.py
index faf537cb..619c47a9 100644
--- a/backend/app/evaluation/toxicity/run.py
+++ b/backend/app/evaluation/toxicity/run.py
@@ -7,6 +7,7 @@
 
 from app.evaluation.common.helper import (
     build_evaluation_report,
+    build_validator_config,
     compute_binary_metrics,
     Profiler,
     write_csv,
@@ -32,16 +33,31 @@
 }
 
 VALIDATORS = {
-    "llamaguard_7b": lambda: LlamaGuard7B(on_fail="noop"),
-    "nsfw_text": lambda: NSFWText(
-        threshold=0.8,
-        validation_method="sentence",
-        device="cpu",
-        model_name="textdetox/xlmr-large-toxicity-classifier",
-        on_fail="noop",
-        use_local=True,
-    ),
-    "profanity_free": lambda: ProfanityFree(on_fail="noop"),
+    "llamaguard_7b": {
+        "build": lambda: LlamaGuard7B(on_fail="noop"),
+        "config_fields": {},
+    },
+    "nsfw_text": {
+        "build": lambda: NSFWText(
+            threshold=0.8,
+            validation_method="sentence",
+            device="cpu",
+            model_name="textdetox/xlmr-large-toxicity-classifier",
+            on_fail="noop",
+            use_local=True,
+        ),
+        "config_fields": {
+            "threshold": 0.8,
+            "validation_method": "sentence",
+            "device": "cpu",
+            "model_name": "textdetox/xlmr-large-toxicity-classifier",
+            "use_local": True,
+        },
+    },
+    "profanity_free": {
+        "build": lambda: ProfanityFree(on_fail="noop"),
+        "config_fields": {},
+    },
 }
 
 
@@ -67,9 +83,10 @@ def run_dataset(dataset_name: str, dataset_cfg: dict):
 
     all_metrics = {}
 
-    for validator_name, build_fn in VALIDATORS.items():
+    for validator_name, spec in VALIDATORS.items():
         print(f"  Running {validator_name} on {dataset_name}...")
-        validator = build_fn()
+        validator = spec["build"]()
+        config = build_validator_config(validator, **spec["config_fields"])
 
         with Profiler() as p:
             df[f"{validator_name}_result"] = df[text_col].apply(
@@ -89,6 +106,7 @@ def run_dataset(dataset_name: str, dataset_cfg: dict):
             dataset=dataset_name,
             num_samples=len(df),
             profiler=p,
+            config=config,
             metrics=metrics,
         )
 

From 87b81365687a75a51cc9f1913e44bbe54aca1139 Mon Sep 17 00:00:00 2001
From: PritamSGB <29685062+PritamSGB@users.noreply.github.com>
Date: Sun, 20 Sep 2026 22:14:28 +0530
Subject: [PATCH 5/6] feat(eval): add lexical_slur to toxicity eval,
 standardize dataset

Adds LexicalSlur alongside the existing toxicity validators with
--validators/--sources filtering and a combined (OR) metric per source
(computed from already-produced predictions, no re-running), now against
a single standardized dataset (text/label/language/dataset) instead of
three separate files. Also lazily imports nsfw_text/profanity_free from
their new guardrails_ai.* packages and llamaguard_7b from guardrails.hub,
so selecting a subset of validators doesn't require llamaguard_7b's
still-unmigrated hub path to be importable at all.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 backend/app/evaluation/README.md       |  64 +++++----
 backend/app/evaluation/toxicity/run.py | 172 ++++++++++++++++---------
 2 files changed, 153 insertions(+), 83 deletions(-)

diff --git a/backend/app/evaluation/README.md b/backend/app/evaluation/README.md
index 639bae5b..c34123ea 100644
--- a/backend/app/evaluation/README.md
+++ b/backend/app/evaluation/README.md
@@ -22,9 +22,8 @@ backend/app/evaluation/
 │   │   ├── education_topic_config.txt
 │   │   ├── healthcare-topic-relevance-dataset.csv
 │   │   └── healthcare_topic_config.txt
-│   └── toxicity/                          # Toxicity evaluation datasets
-│       ├── toxicity_test_hasoc.csv
-│       └── toxicity_test_sharechat.csv
+│   └── toxicity/                          # Toxicity evaluation dataset
+│       └── toxicity_test_combined.csv     # Standardized input consumed by run.py (see Toxicity section for its source mapping)
 ├── gender_assumption_bias/
 │   └── run.py                             # Gender assumption bias evaluation script
 ├── lexical_slur/
@@ -49,7 +48,7 @@ backend/app/evaluation/
 ├── topic_relevance/
 │   └── run.py                             # Topic relevance evaluation script
 └── toxicity/
-    └── run.py                             # Toxicity evaluation script (LlamaGuard7B, NSFWText, ProfanityFree)
+    └── run.py                             # Toxicity evaluation script (LlamaGuard7B, NSFWText, ProfanityFree, LexicalSlur)
 ```
 
 ## Prerequisites
@@ -263,38 +262,53 @@ python3 app/evaluation/topic_relevance/run.py
 
 ---
 
-### Toxicity (`llamaguard_7b`, `nsfw_text`, `profanity_free`)
+### Toxicity (`llamaguard_7b`, `nsfw_text`, `profanity_free`, `lexical_slur`)
 
 **Script:** `app/evaluation/toxicity/run.py`
 
-**Datasets:**
-- `datasets/toxicity/toxicity_test_hasoc.csv` (The HASOC (Hate Speech and Offensive Content) dataset is a multilingual benchmark (mainly Hindi, English, and code-mixed text) used to train and evaluate models for detecting hate speech, offensive language, and abusive content in social media.)
-- `datasets/toxicity/toxicity_test_sharechat.csv`
+**Dataset:** `datasets/toxicity/toxicity_test_combined.csv` — one standardized file combining three original sources (HASOC, ShareChat, and a lexical-slur set), tagged by origin.
 
-Expected columns — HASOC dataset:
+Expected columns:
 
-- `text` — tweet/comment text to validate
-- `task1` — ground truth label (`HOF` = hate/offensive/profanity → `1`, `NOT` → `0`)
-- `lang` — language code (informational)
+- `text` — text to validate
+- `label` — binary ground truth (`1` = toxic/abusive, `0` = not)
+- `language` — language label (informational)
+- `dataset` — which original source the row came from (`hasoc`, `sharechat`, or `lexical`) — used for the per-source metrics breakdown and the `--sources` filter, not for validation itself
 
-Expected columns — ShareChat dataset:
+This file was assembled from three original per-source CSVs, each normalized into the schema above via a one-off local data-prep step (not a script checked into the repo) using this column mapping:
 
-- `commentText` — comment text to validate
-- `label` — binary ground truth (`1` = toxic, `0` = not toxic)
-- `language` — language label (informational)
+- HASOC: `text` → `text`, `task1` (`HOF`/`NOT`) → `label` (`1`/`0`), `lang` → `language`
+- ShareChat: `commentText` → `text`, `label` (already `1`/`0`) → `label`, `language` → `language`
+- Lexical: `commentText` → `text`, `label` (already `1.0`/`0.0`) → `label`, `language` → `language`
+
+Each row is tagged with `dataset` = its source name. To add or refresh a source, re-apply this same mapping and re-concatenate into `toxicity_test_combined.csv`.
+
+> HASOC (Hate Speech and Offensive Content) is a multilingual benchmark (mainly Hindi, English, and code-mixed text) for detecting hate speech, offensive language, and abusive content in social media.
+
+**What it does:** Runs four validators — `LlamaGuard7B`, `NSFWText`, `ProfanityFree`, and `LexicalSlur` — against the combined dataset. For each validator, a binary prediction is recorded (`1` if `FailResult`, `0` otherwise) and compared against the ground truth label to compute classification metrics, further broken down per original `dataset` source (`source_metrics`). A `combined_pred` column (logical OR across whichever validators ran, computed from their already-produced `*_pred` columns without re-running anything) is also recorded, with its own `combined` entry (and `source_metrics` breakdown) in the metrics JSON — useful for seeing each validator's standalone effectiveness alongside what running them together would catch, both overall and per source.
+
+Use `--validators` to run only a subset of validators instead of all four:
+
+```bash
+python3 app/evaluation/toxicity/run.py --validators lexical_slur profanity_free
+```
 
-**What it does:** Runs three validators — `LlamaGuard7B`, `NSFWText`, and `ProfanityFree` — across both datasets independently. For each validator, a binary prediction is recorded (`1` if `FailResult`, `0` otherwise) and compared against the ground truth label to compute classification metrics.
+Use `--sources` to run only rows from a subset of original sources instead of all three:
 
-**Output per dataset:**
+```bash
+python3 app/evaluation/toxicity/run.py --sources hasoc lexical
+```
+
+The two flags can be combined. `--validators` is useful for skipping `llamaguard_7b` or `nsfw_text` when only exercising `lexical_slur`/`profanity_free` locally.
+
+**Output:**
 
 ```
-outputs/toxicity/predictions_hasoc.csv
-outputs/toxicity/metrics_hasoc.json
-outputs/toxicity/predictions_sharechat.csv
-outputs/toxicity/metrics_sharechat.json
+outputs/toxicity/predictions.csv
+outputs/toxicity/metrics.json
 ```
 
-Each predictions CSV contains the source text, ground truth (`y_true`), and one `*_pred` column per validator. Each metrics JSON contains accuracy, precision, recall, F1, and performance stats broken down per validator.
+`predictions.csv` contains the source text, its origin `dataset`, `language`, ground truth (`y_true`), one `*_pred` column per validator that ran, and a `combined_pred` column. `metrics.json` contains accuracy, precision, recall, F1, and performance stats per validator (each with a `source_metrics` breakdown by origin dataset), plus a `combined` entry for the OR-combined prediction (with its own `source_metrics` breakdown).
 
 **Run:**
 
@@ -302,7 +316,7 @@ Each predictions CSV contains the source text, ground truth (`y_true`), and one
 python3 app/evaluation/toxicity/run.py
 ```
 
-> **Note:** `LlamaGuard7B` uses remote inferencing — requires a valid `GUARDRAILS_HUB_API_KEY` and internet access. `NSFWText` downloads the `textdetox/xlmr-large-toxicity-classifier` model on first run.
+> **Note:** `llamaguard_7b` is not yet migrated off the sunset Guardrails Hub install path (see `backend/app/core/validators/README.md`) — running it will currently fail regardless of credentials. Use `--validators` to exclude it until that migration lands. `NSFWText` downloads the `textdetox/xlmr-large-toxicity-classifier` model on first run.
 
 ---
 
@@ -437,7 +451,7 @@ Each evaluation script expects a specific filename — files must be named exact
 | Ban List               | `ban_list_testing_dataset.csv`                                                                                      |
 | Multiple Validators    | `multi_validator_whatsapp_dataset.csv`                                                                              |
 | Topic Relevance        | `topic_relevance/education-topic-relevance-dataset.csv`, `topic_relevance/healthcare-topic-relevance-dataset.csv` |
-| Toxicity               | `toxicity/toxicity_test_hasoc.csv`, `toxicity/toxicity_test_sharechat.csv`                                        |
+| Toxicity               | `toxicity/toxicity_test_combined.csv` (see Toxicity section for the raw sources and column mapping it's derived from) |
 
 Topic relevance also requires plain-text topic config files alongside each dataset:
 
diff --git a/backend/app/evaluation/toxicity/run.py b/backend/app/evaluation/toxicity/run.py
index 619c47a9..2065a822 100644
--- a/backend/app/evaluation/toxicity/run.py
+++ b/backend/app/evaluation/toxicity/run.py
@@ -1,10 +1,9 @@
+import argparse
 from pathlib import Path
 import pandas as pd
-from guardrails.hub import LlamaGuard7B  # noqa: not yet migrated, see backend README
-from guardrails_ai.nsfw_text import NSFWText
-from guardrails_ai.profanity_free import ProfanityFree
 from guardrails.validators import FailResult
 
+from app.core.validators.lexical_slur import LexicalSlur
 from app.evaluation.common.helper import (
     build_evaluation_report,
     build_validator_config,
@@ -17,36 +16,52 @@
 BASE_DIR = Path(__file__).resolve().parent.parent
 OUT_DIR = BASE_DIR / "outputs" / "toxicity"
 
-DATASETS = {
-    "hasoc": {
-        "path": BASE_DIR / "datasets" / "toxicity" / "toxicity_test_hasoc.csv",
-        "text_col": "text",
-        "label_col": "task1",
-        "label_map": {"HOF": 1, "NOT": 0},
-    },
-    "sharechat": {
-        "path": BASE_DIR / "datasets" / "toxicity" / "toxicity_test_sharechat.csv",
-        "text_col": "commentText",
-        "label_col": "label",
-        "label_map": None,  # already binary int
-    },
-}
+DATASET_PATH = BASE_DIR / "datasets" / "toxicity" / "toxicity_test_combined.csv"
+# Standardized columns: text, label (0/1), language, dataset (source name).
+SOURCE_NAMES = ["hasoc", "sharechat", "lexical"]
+
+
+def _build_llamaguard_7b():
+    # Imported lazily: llamaguard_7b is not yet migrated off guardrails.hub (see
+    # backend/app/core/validators/README.md), and its local .guardrails/hub_registry.json
+    # entry still points at the now-defunct guardrails_grhub_llamaguard_7b package, so a
+    # top-level import would break every run of this script, not just ones selecting it.
+    from guardrails.hub import LlamaGuard7B  # noqa: not yet migrated, see backend README
+
+    return LlamaGuard7B(on_fail="noop")
+
+
+def _build_nsfw_text():
+    # Imported lazily so selecting other validators via --validators doesn't require
+    # the nsfw_text package (and its model download) to be usable.
+    from guardrails_ai.nsfw_text import NSFWText
+
+    return NSFWText(
+        threshold=0.8,
+        validation_method="sentence",
+        device="cpu",
+        model_name="textdetox/xlmr-large-toxicity-classifier",
+        on_fail="noop",
+        use_local=True,
+    )
+
+
+def _build_profanity_free():
+    # Imported lazily so selecting other validators via --validators doesn't require
+    # the profanity_free package to be usable.
+    from guardrails_ai.profanity_free import ProfanityFree
+
+    return ProfanityFree(on_fail="noop")
+
 
 VALIDATORS = {
     "llamaguard_7b": {
-        "build": lambda: LlamaGuard7B(on_fail="noop"),
-        "config_fields": {},
+        "build": _build_llamaguard_7b,
+        "config_fields": lambda v: {},
     },
     "nsfw_text": {
-        "build": lambda: NSFWText(
-            threshold=0.8,
-            validation_method="sentence",
-            device="cpu",
-            model_name="textdetox/xlmr-large-toxicity-classifier",
-            on_fail="noop",
-            use_local=True,
-        ),
-        "config_fields": {
+        "build": _build_nsfw_text,
+        "config_fields": lambda v: {
             "threshold": 0.8,
             "validation_method": "sentence",
             "device": "cpu",
@@ -55,41 +70,47 @@
         },
     },
     "profanity_free": {
-        "build": lambda: ProfanityFree(on_fail="noop"),
-        "config_fields": {},
+        "build": _build_profanity_free,
+        "config_fields": lambda v: {},
+    },
+    "lexical_slur": {
+        "build": lambda: LexicalSlur(on_fail="noop"),
+        "config_fields": lambda v: {
+            "severity": v.severity,
+            "languages": v.languages,
+            "num_slurs_loaded": len(v.slur_list),
+        },
     },
 }
 
 
-def run_dataset(dataset_name: str, dataset_cfg: dict):
-    df = pd.read_csv(dataset_cfg["path"])
-    text_col = dataset_cfg["text_col"]
-    label_col = dataset_cfg["label_col"]
-    label_map = dataset_cfg["label_map"]
+def _source_metrics(df: pd.DataFrame, pred_col: str) -> dict:
+    return {
+        str(source): compute_binary_metrics(group["y_true"], group[pred_col])
+        for source, group in df.groupby("dataset", dropna=False)
+    }
 
-    if label_map is not None:
-        df["y_true"] = df[label_col].map(label_map)
-        unmapped = df.loc[df["y_true"].isna(), label_col].unique().tolist()
-        if unmapped:
-            raise ValueError(
-                f"[{dataset_name}] label_col '{label_col}' contains values not in label_map: {unmapped}"
-            )
-    else:
-        df["y_true"] = df[label_col].astype(int)
 
-    missing_text = df[text_col].isna()
+def run_evaluation(validators: dict, sources: list[str] | None):
+    df = pd.read_csv(DATASET_PATH)
+    df["y_true"] = df["label"].astype(int)
+
+    if sources:
+        df = df[df["dataset"].isin(sources)].reset_index(drop=True)
+
+    missing_text = df["text"].isna()
     if missing_text.any():
         df = df[~missing_text].copy()
 
     all_metrics = {}
 
-    for validator_name, spec in VALIDATORS.items():
-        print(f"  Running {validator_name} on {dataset_name}...")
+    for validator_name, spec in validators.items():
+        print(f"Running {validator_name}...")
         validator = spec["build"]()
-        config = build_validator_config(validator, **spec["config_fields"])
+        config = build_validator_config(validator, **spec["config_fields"](validator))
 
         with Profiler() as p:
-            df[f"{validator_name}_result"] = df[text_col].apply(
+            df[f"{validator_name}_result"] = df["text"].apply(
                 lambda x: p.record(lambda t: validator.validate(t, metadata={}), x)
             )
 
@@ -101,9 +122,9 @@ def run_dataset(dataset_name: str, dataset_cfg: dict):
             df["llamaguard_7b_latency_ms"] = p.latencies
 
         metrics = compute_binary_metrics(df["y_true"], df[f"{validator_name}_pred"])
+        metrics["source_metrics"] = _source_metrics(df, f"{validator_name}_pred")
         all_metrics[validator_name] = build_evaluation_report(
             guardrail=validator_name,
-            dataset=dataset_name,
             num_samples=len(df),
             profiler=p,
             config=config,
@@ -112,19 +133,54 @@ def run_dataset(dataset_name: str, dataset_cfg: dict):
 
         df = df.drop(columns=[f"{validator_name}_result"])
 
-    pred_cols = ["y_true"] + [f"{v}_pred" for v in VALIDATORS]
+    validator_pred_cols = [f"{name}_pred" for name in validators]
+    df["combined_pred"] = df[validator_pred_cols].max(axis=1)
+    combined_metrics = compute_binary_metrics(df["y_true"], df["combined_pred"])
+    combined_metrics["source_metrics"] = _source_metrics(df, "combined_pred")
+    all_metrics["combined"] = {
+        "guardrail": "combined",
+        "validators": list(validators.keys()),
+        "num_samples": len(df),
+        "metrics": combined_metrics,
+    }
+
+    pred_cols = ["y_true"] + validator_pred_cols + ["combined_pred"]
     latency_cols = (
         ["llamaguard_7b_latency_ms"] if "llamaguard_7b_latency_ms" in df.columns else []
     )
     write_csv(
-        df[[text_col, *pred_cols, *latency_cols]],
-        OUT_DIR / f"predictions_{dataset_name}.csv",
+        df[["text", "dataset", "language", *pred_cols, *latency_cols]],
+        OUT_DIR / "predictions.csv",
     )
-    write_json(all_metrics, OUT_DIR / f"metrics_{dataset_name}.json")
+    write_json(all_metrics, OUT_DIR / "metrics.json")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--validators",
+        nargs="+",
+        choices=list(VALIDATORS.keys()),
+        help="Only run these validators (default: run all validators)",
+    )
+    parser.add_argument(
+        "--sources",
+        nargs="+",
+        choices=SOURCE_NAMES,
+        help="Only run rows from these original dataset sources (default: run all)",
+    )
+    args = parser.parse_args()
+
+    selected_validators = (
+        {name: VALIDATORS[name] for name in args.validators}
+        if args.validators
+        else VALIDATORS
+    )
+
+    run_evaluation(selected_validators, args.sources)
 
+    print("Done. Results saved to", OUT_DIR)
 
-for dataset_name, dataset_cfg in DATASETS.items():
-    print(f"Evaluating dataset: {dataset_name}")
-    run_dataset(dataset_name, dataset_cfg)
 
-print("Done. Results saved to", OUT_DIR)
+if __name__ == "__main__":
+    main()

From 93269487e3fdda72bf9a244b23472440905068d9 Mon Sep 17 00:00:00 2001
From: PritamSGB <29685062+PritamSGB@users.noreply.github.com>
Date: Wed, 23 Sep 2026 20:12:37 +0530
Subject: [PATCH 6/6] refactor(eval): drop standalone lexical slur eval, now
 covered by toxicity eval

The toxicity eval runs uli_slur_match over a superset of the rows the
standalone script used, producing identical metrics and config for that
subset (--validators lexical_slur --sources lexical reproduces it exactly),
so the separate script and its dataset are redundant.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 backend/app/evaluation/README.md           | 39 ++--------------
 backend/app/evaluation/lexical_slur/run.py | 53 ----------------------
 backend/scripts/run_all_evaluations.sh     |  1 -
 3 files changed, 4 insertions(+), 89 deletions(-)
 delete mode 100644 backend/app/evaluation/lexical_slur/run.py

diff --git a/backend/app/evaluation/README.md b/backend/app/evaluation/README.md
index c34123ea..4d7b79ea 100644
--- a/backend/app/evaluation/README.md
+++ b/backend/app/evaluation/README.md
@@ -13,7 +13,6 @@ backend/app/evaluation/
 ├── datasets/                              # Evaluation datasets (downloaded separately)
 │   ├── ban_list_testing_dataset.csv
 │   ├── gender_bias_assumption_dataset.csv
-│   ├── lexical_slur_testing_dataset.csv
 │   ├── multi_validator_whatsapp_dataset.csv
 │   ├── pii_detection_testing_dataset.csv
 │   ├── sharechat_toxic_dataset.csv
@@ -26,15 +25,12 @@ backend/app/evaluation/
 │       └── toxicity_test_combined.csv     # Standardized input consumed by run.py (see Toxicity section for its source mapping)
 ├── gender_assumption_bias/
 │   └── run.py                             # Gender assumption bias evaluation script
-├── lexical_slur/
-│   └── run.py                             # Lexical slur evaluation script
 ├── multiple_validators/
 │   ├── config.json                        # Multi-validator run configuration
 │   └── run.py                             # End-to-end multi-validator evaluation script
 ├── outputs/                               # Generated outputs (created at runtime)
 │   ├── ban_list/
 │   ├── gender_assumption_bias/
-│   ├── lexical_slur/
 │   ├── multi_validator_whatsapp/
 │   ├── multiple_validators/
 │   ├── pii_remover/
@@ -91,7 +87,7 @@ Validators that use LLM-as-judge approach will require credentials for LLM provi
 
 ## Running All Evaluations
 
-To run all individual validator evaluations in sequence (lexical slur, PII, gender assumption bias, ban list, topic relevance, toxicity):
+To run all individual validator evaluations in sequence (PII, gender assumption bias, ban list, topic relevance, toxicity):
 
 ```bash
 bash scripts/run_all_evaluations.sh
@@ -109,34 +105,6 @@ Run any individual evaluation from the `backend/` directory:
 python3 app/evaluation/<validator_folder>/run.py
 ```
 
-### Lexical Slur (`uli_slur_match`)
-
-**Script:** `app/evaluation/lexical_slur/run.py`
-
-**Dataset:** `datasets/lexical_slur_testing_dataset.csv`
-
-Expected columns in input csv:
-
-- `commentText` — text to validate
-- `label` — ground truth (`1` = abusive, `0` = not abusive)
-
-**What it does:** Runs each row through the `LexicalSlur` validator and records a binary prediction (`1` if `FailResult`, `0` otherwise). Computes binary classification metrics against the ground truth labels.
-
-**Output:**
-
-```
-outputs/lexical_slur/predictions.csv
-outputs/lexical_slur/metrics.json
-```
-
-**Run:**
-
-```bash
-python3 app/evaluation/lexical_slur/run.py
-```
-
----
-
 ### PII Remover (`pii_remover`)
 
 **Script:** `app/evaluation/pii/run.py`
@@ -287,6 +255,8 @@ Each row is tagged with `dataset` = its source name. To add or refresh a source,
 
 **What it does:** Runs four validators — `LlamaGuard7B`, `NSFWText`, `ProfanityFree`, and `LexicalSlur` — against the combined dataset. For each validator, a binary prediction is recorded (`1` if `FailResult`, `0` otherwise) and compared against the ground truth label to compute classification metrics, further broken down per original `dataset` source (`source_metrics`). A `combined_pred` column (logical OR across whichever validators ran, computed from their already-produced `*_pred` columns without re-running anything) is also recorded, with its own `combined` entry (and `source_metrics` breakdown) in the metrics JSON — useful for seeing each validator's standalone effectiveness alongside what running them together would catch, both overall and per source.
 
+`LexicalSlur` (`uli_slur_match`) is evaluated here rather than in its own script — this eval covers it over a superset of the rows its standalone eval used, so to reproduce that older run exactly, use `--validators lexical_slur --sources lexical`.
+
 Use `--validators` to run only a subset of validators instead of all four:
 
 ```bash
@@ -385,7 +355,7 @@ The output CSV contains `ID`, `text`, `validators_present`, and `response` (the
 
 ### Binary Classification Metrics (`metrics.json`)
 
-Used by lexical slur, gender assumption bias, ban list, and topic relevance evaluations.
+Used by the gender assumption bias, ban list, topic relevance, and toxicity evaluations.
 
 | Metric             | Description                                               |
 | ------------------ | --------------------------------------------------------- |
@@ -445,7 +415,6 @@ Each evaluation script expects a specific filename — files must be named exact
 
 | Validator              | Expected filename                                                                                                     |
 | ---------------------- | --------------------------------------------------------------------------------------------------------------------- |
-| Lexical Slur           | `lexical_slur_testing_dataset.csv`                                                                                  |
 | PII Remover            | `pii_detection_testing_dataset.csv`                                                                                 |
 | Gender Assumption Bias | `gender_bias_assumption_dataset.csv`                                                                                |
 | Ban List               | `ban_list_testing_dataset.csv`                                                                                      |
diff --git a/backend/app/evaluation/lexical_slur/run.py b/backend/app/evaluation/lexical_slur/run.py
deleted file mode 100644
index 7cd74a01..00000000
--- a/backend/app/evaluation/lexical_slur/run.py
+++ /dev/null
@@ -1,53 +0,0 @@
-from pathlib import Path
-import pandas as pd
-from guardrails.validators import FailResult
-
-from app.core.validators.lexical_slur import LexicalSlur
-from app.evaluation.common.helper import (
-    build_evaluation_report,
-    build_validator_config,
-    Profiler,
-    compute_binary_metrics,
-    write_csv,
-    write_json,
-)
-
-BASE_DIR = Path(__file__).resolve().parent.parent
-OUT_DIR = BASE_DIR / "outputs" / "lexical_slur"
-
-df = pd.read_csv(BASE_DIR / "datasets" / "lexical_slur_testing_dataset.csv")
-
-validator = LexicalSlur()
-
-config = build_validator_config(
-    validator,
-    severity=validator.severity,
-    languages=validator.languages,
-    num_slurs_loaded=len(validator.slur_list),
-)
-
-with Profiler() as p:
-    df["result"] = (
-        df["commentText"]
-        .astype(str)
-        .apply(lambda x: p.record(lambda t: validator.validate(t, metadata=None), x))
-    )
-
-df["y_pred"] = df["result"].apply(lambda r: int(isinstance(r, FailResult)))
-df["y_true"] = df["label"]
-
-metrics = compute_binary_metrics(df["y_true"], df["y_pred"])
-
-# ---- Save outputs ----
-write_csv(df.drop(columns=["result"]), OUT_DIR / "predictions.csv")
-
-write_json(
-    build_evaluation_report(
-        guardrail="lexical_slur",
-        num_samples=len(df),
-        profiler=p,
-        config=config,
-        metrics=metrics,
-    ),
-    OUT_DIR / "metrics.json",
-)
diff --git a/backend/scripts/run_all_evaluations.sh b/backend/scripts/run_all_evaluations.sh
index 0da24020..96a7c90c 100755
--- a/backend/scripts/run_all_evaluations.sh
+++ b/backend/scripts/run_all_evaluations.sh
@@ -6,7 +6,6 @@ BACKEND_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 EVAL_DIR="$BACKEND_DIR/app/evaluation"
 
 RUNNERS=(
-  "$EVAL_DIR/lexical_slur/run.py"
   "$EVAL_DIR/pii/run.py"
   "$EVAL_DIR/gender_assumption_bias/run.py"
   "$EVAL_DIR/ban_list/run.py"