From 5ac9123ecb4a004c022df81f35c33da9802c2cc5 Mon Sep 17 00:00:00 2001 From: Egemen Tuncarslan Date: Tue, 1 Sep 2026 15:39:05 +0300 Subject: [PATCH] kitti_eval: name class 3 the class clean_data actually filters for `clean_data` decides which annotations belong to a class index: CLASS_NAMES = ['car', 'pedestrian', 'cyclist', 'van', 'person_sitting', 'truck'] `get_official_eval_result` names the same indices for the results table, and inverts that map so a caller can pass a class name: 3: 'Sign', #'Van', Index by index, that is the only one of the six that disagrees: 0: filter=car label=Car same 1: filter=pedestrian label=Pedestrian same 2: filter=cyclist label=Cyclist same 3: filter=van label=Sign <-- differs 4: filter=person_sitting label=Person_sitting same 5: filter=truck label=Truck same Two consequences: * `name_to_class['Sign'] = 3`, so a caller asking for signs is given the van numbers, under the heading "Sign". * `name_to_class['Van']` is absent, so a caller using KITTI's own class name gets a KeyError. Three things in the repository say which one was meant. `get_coco_eval_result`, 120 lines further down the same file, has `3: 'Van'`. `clean_data` line 50 treats Van as a KITTI class in its own right (`current_cls_name == "car" and gt_name == "van"` -> neighbour, not a false positive). And min_overlaps gives index 3 a 0.7 IoU threshold -- the vehicle threshold, where pedestrian and cyclist get 0.5. `Sign` is a Waymo class, not a KITTI one, and this repository already says so: detection3d/waymo2kitti_async.py:87 is `'SIGN': 'Sign' # not in kitti`. The commented-out `#'Van'` on the same line is what the fix restores. Adds kitti_eval/tests/test_class_names_agree.py -- 10 tests. They read the source with `ast` rather than importing eval.py, which compiles its kernels with `@numba.jit` at import time; nothing here makes a claim about numba. Standard library only, no dataset, no GPU. Against this branch: 10 passed. Against the file as it stands on main: 4 failed, 6 passed -- FAILED test_every_index_names_the_class_clean_data_filters_for[get_official_eval_result] FAILED test_the_two_maps_agree_with_each_other FAILED test_a_caller_can_name_every_class_the_filter_accepts[get_official_eval_result] FAILED test_no_index_map_carries_a_class_kitti_does_not_have The same two parametrised tests pass for `get_coco_eval_result` either way, which is the internal contradiction showing up in the report rather than in prose. Co-Authored-By: Claude Opus 5 --- .../ngperception/detection/kitti_eval/eval.py | 2 +- .../tests/test_class_names_agree.py | 125 ++++++++++++++++++ 2 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 DeepDataMiningLearning/ngperception/detection/kitti_eval/tests/test_class_names_agree.py diff --git a/DeepDataMiningLearning/ngperception/detection/kitti_eval/eval.py b/DeepDataMiningLearning/ngperception/detection/kitti_eval/eval.py index 958fa525..45b54a40 100644 --- a/DeepDataMiningLearning/ngperception/detection/kitti_eval/eval.py +++ b/DeepDataMiningLearning/ngperception/detection/kitti_eval/eval.py @@ -649,7 +649,7 @@ def get_official_eval_result(gt_annos, dt_annos, current_classes, class_to_name= 0: 'Car', 1: 'Pedestrian', 2: 'Cyclist', - 3: 'Sign', #'Van', + 3: 'Van', 4: 'Person_sitting', 5: 'Truck' } diff --git a/DeepDataMiningLearning/ngperception/detection/kitti_eval/tests/test_class_names_agree.py b/DeepDataMiningLearning/ngperception/detection/kitti_eval/tests/test_class_names_agree.py new file mode 100644 index 00000000..6ac4c9d9 --- /dev/null +++ b/DeepDataMiningLearning/ngperception/detection/kitti_eval/tests/test_class_names_agree.py @@ -0,0 +1,125 @@ +"""The class index -> name maps in eval.py have to agree with the filter. + +`clean_data` decides which annotations belong to a class index, from + + CLASS_NAMES = ['car', 'pedestrian', 'cyclist', 'van', 'person_sitting', 'truck'] + +and two other maps in the same file turn indices into the names that appear in +the results and accept class names from callers: one in +`get_official_eval_result`, one in `get_coco_eval_result`. If those disagree +with `CLASS_NAMES`, a result is filed under a class it was not measured on, and +a caller naming a class gets a different one -- or a KeyError. + +Reads the source with `ast` rather than importing it: eval.py compiles its +kernels with `@numba.jit` at import, and this makes no claim about numba. +Standard library only, no dataset, no GPU. + + pytest DeepDataMiningLearning/ngperception/detection/kitti_eval/tests +""" +import ast +import io +from pathlib import Path + +import pytest + +EVAL_PY = Path(__file__).resolve().parents[1] / "eval.py" + + +def _tree(): + return ast.parse(io.open(EVAL_PY, encoding="utf-8", errors="replace").read()) + + +def _class_names(): + """CLASS_NAMES as assigned inside clean_data.""" + for node in ast.walk(_tree()): + if isinstance(node, ast.FunctionDef) and node.name == "clean_data": + for stmt in node.body: + if (isinstance(stmt, ast.Assign) + and getattr(stmt.targets[0], "id", None) == "CLASS_NAMES"): + return ast.literal_eval(stmt.value) + pytest.fail("clean_data no longer assigns CLASS_NAMES") + + +def _index_maps(): + """Every `class_to_name = {...}` literal, keyed by the function holding it.""" + maps = {} + for node in ast.walk(_tree()): + if not isinstance(node, ast.FunctionDef): + continue + for stmt in ast.walk(node): + if (isinstance(stmt, ast.Assign) + and getattr(stmt.targets[0], "id", None) == "class_to_name" + and isinstance(stmt.value, ast.Dict)): + try: + maps[node.name] = ast.literal_eval(stmt.value) + except ValueError: + pass + return maps + + +def test_the_file_is_where_it_is_expected(): + assert EVAL_PY.is_file() + + +def test_clean_data_still_defines_the_filter_list(): + names = _class_names() + assert names[:3] == ["car", "pedestrian", "cyclist"] + assert len(names) == 6 + + +def test_both_index_maps_were_found(): + maps = _index_maps() + assert "get_official_eval_result" in maps + assert "get_coco_eval_result" in maps + + +@pytest.mark.parametrize("func", ["get_official_eval_result", "get_coco_eval_result"]) +def test_every_index_names_the_class_clean_data_filters_for(func): + names = _class_names() + mapping = _index_maps()[func] + mismatches = [ + f"{i}: filter={names[i]!r} label={label!r}" + for i, label in sorted(mapping.items()) + if i < len(names) and label.lower() != names[i] + ] + assert mismatches == [] + + +def test_the_two_maps_agree_with_each_other(): + a = _index_maps()["get_official_eval_result"] + b = _index_maps()["get_coco_eval_result"] + shared = set(a) & set(b) + assert shared, "the two maps share no index" + assert [i for i in sorted(shared) if a[i] != b[i]] == [] + + +@pytest.mark.parametrize("func", ["get_official_eval_result", "get_coco_eval_result"]) +def test_a_caller_can_name_every_class_the_filter_accepts(func): + """`name_to_class` is built by inverting the map; a caller passes a name.""" + names = _class_names() + mapping = _index_maps()[func] + name_to_class = {v: k for k, v in mapping.items()} + for i in sorted(mapping): + if i >= len(names): + continue + assert any(k.lower() == names[i] for k in name_to_class), ( + f"no caller-facing name resolves to {names[i]!r} (index {i})" + ) + + +def test_no_index_map_carries_a_class_kitti_does_not_have(): + """`Sign` is a Waymo class. waymo2kitti_async.py:87 says so: 'not in kitti'.""" + names = {n.lower() for n in _class_names()} + strays = [] + for func, mapping in _index_maps().items(): + for i, label in sorted(mapping.items()): + if i < len(_class_names()) and label.lower() not in names: + strays.append(f"{func}[{i}] = {label!r}") + assert strays == [] + + +def test_the_inverted_map_has_no_duplicate_names(): + """Two indices sharing a name would make the caller's choice ambiguous.""" + for func, mapping in _index_maps().items(): + labels = [v for _, v in sorted(mapping.items())] + assert len(labels) == len(set(labels)), f"{func} repeats a name"