From 702af049968523c4e0b79841c5f3ab48a7edf0ae Mon Sep 17 00:00:00 2001 From: Egemen Tuncarslan Date: Tue, 1 Sep 2026 15:39:10 +0300 Subject: [PATCH] dataset_nuscenes: give each sample a COCO image id that survives a new process dataset_nuscenes.py:433 builds every target's COCO image id as 'image_id': torch.tensor([hash(sample_token) % 1000000]), and cocoevaluator.py keys both halves of the evaluation on it: the ground truth at convert_to_coco_api line 300, the predictions at `res[image_id] = out` on line 370. `hash()` on a str is salted per interpreter (PEP 456). One token, six fresh interpreters: PYTHONHASHSEED=0 -> 489194 PYTHONHASHSEED=1 -> 812316 PYTHONHASHSEED=2 -> 270356 PYTHONHASHSEED=random -> 473704 PYTHONHASHSEED=random -> 887793 PYTHONHASHSEED=random -> 561686 Six ids for one sample. So a prediction file written in one run cannot be scored against ground truth rebuilt in another, and the two processes of a distributed run disagree about which image is which. That is the part of this that does not get smaller with a bigger machine or a smaller dataset. The 1e6 range is the second, milder half. Over the 34,149 keyframes of v1.0-trainval, a 1e6 range loses about 555 ids to collisions -- "about", because the exact count depends on the salt: two runs here gave 564 and 594. The instability shows up even in the measurement of the collision rate. I would rather give the honest size of that half than the dramatic one. Run through this repository's own convert_to_coco_api and CocoEvaluator, a detector whose predictions ARE the ground truth scores one id per sample (control) mAP 1.0000 ids drawn from 1e6, 4,000 samples mAP 0.9901 (0.99% low) every id shared by two samples mAP 0.5050 (49.5% low) The last row is not the real rate -- it is an exaggeration that isolates the mechanism, since `res[image_id] = out` keeps only the last of a colliding pair while the ground truth keeps both. At realistic rates the collision costs about a point; the reproducibility is the reason to change the line. `stable_image_id` takes 13 hex digits of sha256: identical in every process, and 52 bits keeps every id an exact JavaScript integer for COCO JSON consumers while putting collisions out of reach -- 0 over 1,000,000 tokens, measured. Considered and rejected: crc32 (stable, but starts colliding above ~100k), and passing the sample index (unique by construction, but it changes get_target's signature and makes the id depend on --max-samples rather than on the sample). Adds detection/test_nuscenes_image_id.py -- 12 tests, no download, no GPU: 12 passed. One thing this suite cannot do, and I would rather say so than imply otherwise: it does not run red against main. `stable_image_id` is new, so against the current file the suite fails to import and pytest reports a collection error rather than a set of behavioural failures. What stands in for that is test_the_builtin_hash_really_is_unstable_here, which measures `hash()` across seeds directly and would fail if the premise of this change were wrong, and the mAP figures above, which come from the repository's own evaluator. `cv2` is stubbed in the test only when genuinely absent -- dataset_nuscenes imports it at module scope and no code under test touches it. A test reports whether the stub was used, so a stubbed run cannot be mistaken for a clean one. Co-Authored-By: Claude Opus 5 --- .../detection/dataset_nuscenes.py | 22 ++- .../detection/test_nuscenes_image_id.py | 155 ++++++++++++++++++ 2 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 DeepDataMiningLearning/detection/test_nuscenes_image_id.py diff --git a/DeepDataMiningLearning/detection/dataset_nuscenes.py b/DeepDataMiningLearning/detection/dataset_nuscenes.py index 6b1b9412..562e60d2 100644 --- a/DeepDataMiningLearning/detection/dataset_nuscenes.py +++ b/DeepDataMiningLearning/detection/dataset_nuscenes.py @@ -9,6 +9,7 @@ import os import json +import hashlib import numpy as np from typing import Dict, List, Tuple, Optional, Any from collections import defaultdict @@ -90,6 +91,25 @@ class BoxVisibility(IntEnum): DEFAULT_NUSCENES_ROOT = "/DATA10T/Datasets/nuScenes/v1.0-trainval" + +def stable_image_id(sample_token: str) -> int: + """A COCO image id for a sample token that is the same in every process. + + `hash()` on a str is salted per interpreter (PEP 456), so an id built from + it changes between runs: predictions written today cannot be scored against + a ground-truth file rebuilt tomorrow, and two processes of one distributed + run disagree about which image is which. + + Thirteen hex digits give 52 bits, which keeps every id an exact JavaScript + integer for COCO JSON consumers while putting collisions out of reach at + nuScenes scale: 0 collisions over 1,000,000 tokens, against ~555 ids lost to + a 1e6 range over the 34,149 keyframes of v1.0-trainval. The exact figure for + `hash()` moves between processes (564 and 594 in two runs here), which is + the first half of the problem showing up in the second. + """ + return int(hashlib.sha256(sample_token.encode("utf-8")).hexdigest()[:13], 16) + + class NuScenesDataset(Dataset): """ Simplified NuScenes Dataset for PyTorch object detection training. @@ -430,7 +450,7 @@ def get_target(self, sample_token: str, sample_data: Dict[str, Any]) -> Dict[str target = { 'boxes': boxes, 'labels': labels, - 'image_id': torch.tensor([hash(sample_token) % 1000000]), + 'image_id': torch.tensor([stable_image_id(sample_token)]), 'area': (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0]) if len(boxes) > 0 else torch.tensor([]), 'iscrowd': torch.zeros((len(boxes),), dtype=torch.int64) } diff --git a/DeepDataMiningLearning/detection/test_nuscenes_image_id.py b/DeepDataMiningLearning/detection/test_nuscenes_image_id.py new file mode 100644 index 00000000..fd1d568f --- /dev/null +++ b/DeepDataMiningLearning/detection/test_nuscenes_image_id.py @@ -0,0 +1,155 @@ +"""The COCO image id built for a nuScenes sample has to survive a new process. + +`get_target` puts an image id on every target, and `cocoevaluator.py` keys both +the ground truth (`convert_to_coco_api`, line 300) and the predictions +(`res[image_id] = out`, line 370) on it. Two properties matter and neither is +about the model: the id must not change between runs, and two samples must not +share one. + +No dataset download, no GPU. `cv2` is stubbed only if it is genuinely missing, +because dataset_nuscenes imports it at module scope and none of the code under +test touches it -- the stub is reported by test_cv2_stub_is_declared so a run +that used one cannot be mistaken for one that did not. + + pytest DeepDataMiningLearning/detection/test_nuscenes_image_id.py +""" +import os +import subprocess +import sys +import types + +import pytest + +# --------------------------------------------------------------- import setup +_CV2_STUBBED = False +try: # pragma: no cover - env probe + import cv2 # noqa: F401 +except ImportError: + sys.modules["cv2"] = types.ModuleType("cv2") + _CV2_STUBBED = True + +_REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if _REPO not in sys.path: + sys.path.insert(0, _REPO) + +from DeepDataMiningLearning.detection.dataset_nuscenes import ( # noqa: E402 + stable_image_id, +) + +# A placeholder in the shape of a nuScenes sample token: 32 lowercase hex. +# Deliberately low-entropy -- a realistic-looking random token reads as a +# credential to secret scanners, and nothing here depends on its entropy. +TOKEN = "0" * 31 + "1" +JS_SAFE_MAX = 2 ** 53 - 1 + + +def _tokens(n, seed=0): + import numpy as np + rng = np.random.default_rng(seed) + return [rng.bytes(16).hex() for _ in range(n)] + + +def test_cv2_stub_is_declared(): + """Fails nothing; records in the report whether cv2 was real.""" + assert _CV2_STUBBED in (True, False) + if _CV2_STUBBED: + print("\n[note] cv2 was absent and stubbed; it is not on any path under test") + + +# ------------------------------------------------------------- determinism +def test_the_same_token_gives_the_same_id_within_a_process(): + assert stable_image_id(TOKEN) == stable_image_id(TOKEN) + + +def test_the_same_token_gives_the_same_id_in_a_fresh_interpreter(): + """The property `hash()` does not have: PEP 456 salts str hashing.""" + code = ( + "import sys; sys.path.insert(0, %r)\n" + "import types; sys.modules.setdefault('cv2', types.ModuleType('cv2'))\n" + "from DeepDataMiningLearning.detection.dataset_nuscenes import stable_image_id\n" + # the module prints environment warnings on import, so tag the answer + "print('ID=%%d' %% stable_image_id(%r))" % (_REPO, TOKEN) + ) + seen = set() + for salt in ("0", "1", "2", "random", "random", "random"): + env = dict(os.environ, PYTHONHASHSEED=salt) + r = subprocess.run([sys.executable, "-c", code], capture_output=True, + text=True, env=env, encoding="utf-8") + assert r.returncode == 0, r.stderr + tagged = [l for l in r.stdout.splitlines() if l.startswith("ID=")] + assert len(tagged) == 1, f"expected one tagged line, got {r.stdout!r}" + seen.add(tagged[0][3:]) + assert len(seen) == 1, f"id moved between processes: {seen}" + assert int(seen.pop()) == stable_image_id(TOKEN) + + +def test_the_builtin_hash_really_is_unstable_here(): + """Guards the premise: if hash() were stable this change would be pointless.""" + seen = set() + for salt in ("0", "1", "2"): + env = dict(os.environ, PYTHONHASHSEED=salt) + r = subprocess.run([sys.executable, "-c", f"print(hash({TOKEN!r}) % 1000000)"], + capture_output=True, text=True, env=env, encoding="utf-8") + seen.add(r.stdout.strip()) + assert len(seen) > 1, "hash() was stable across seeds; the premise needs rechecking" + + +# -------------------------------------------------------------- collisions +@pytest.mark.parametrize("n", [1_000, 34_149]) +def test_no_collisions_at_nuscenes_scale(n): + """34,149 is the keyframe count of nuScenes v1.0-trainval.""" + ids = [stable_image_id(t) for t in _tokens(n, seed=n)] + assert len(set(ids)) == n + + +def test_the_previous_id_function_would_have_collided(): + """The comparison the change is justified by, measured rather than asserted.""" + import zlib + n = 34_149 + toks = _tokens(n, seed=n) + old = {zlib.crc32(t.encode()) % 1_000_000 for t in toks} # stands in for hash()%1e6 + new = {stable_image_id(t) for t in toks} + assert len(old) < n, "the 1e6 range collided in no sample; recheck the range" + assert len(new) == n + + +# ------------------------------------------------------------- id shape +def test_ids_are_exact_javascript_integers(): + """COCO JSON is routinely read by JS tooling, which loses precision past 2^53.""" + for t in _tokens(2_000, seed=1): + i = stable_image_id(t) + assert 0 <= i <= JS_SAFE_MAX + + +def test_ids_are_plain_python_ints(): + assert isinstance(stable_image_id(TOKEN), int) + assert not isinstance(stable_image_id(TOKEN), bool) + + +def test_non_hex_and_unicode_tokens_are_accepted(): + """The simplified dataset layout does not promise hex tokens.""" + for t in ("sample-0001", "", "ünïcode-token", "0" * 64): + assert isinstance(stable_image_id(t), int) + + +def test_different_tokens_give_different_ids(): + assert stable_image_id("a") != stable_image_id("b") + + +# --------------------------------------------------- the call site uses it +def test_get_target_builds_its_image_id_from_this_function(): + """A regression guard on the line this change exists for.""" + import ast + import inspect + from DeepDataMiningLearning.detection import dataset_nuscenes + + src = inspect.getsource(dataset_nuscenes) + tree = ast.parse(src) + calls = { + node.func.id + for node in ast.walk(tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) + } + assert "stable_image_id" in calls + # and nothing in the module builds an id out of the salted builtin any more + assert "hash(sample_token)" not in src