From df1a5b9c033e93537ef769ef62eb3deb3eb69431 Mon Sep 17 00:00:00 2001 From: Egemen Tuncarslan Date: Tue, 1 Sep 2026 15:39:09 +0300 Subject: [PATCH] mixed_dataset: let the shuffle reach training instead of being discarded build_mixed shuffles the train image list, with the reason stated in the comment above it: # Shuffle the train order so that taking the first --max-images during # training yields a balanced (cross-source) subset -- needed for clean # data-size studies. The order does not survive. img_id is assigned inside the per-source loop, so each source owns one contiguous block, and the consumer re-sorts: torchvision's CocoDetection does self.ids = list(sorted(self.coco.imgs.keys())) after which datasets.py takes the first --max-images in id order. Sorting therefore front-loads whichever source was written first. Measured by running build_mixed with three sources x 400 train images and reading the result back through CocoDetection: shuffle achieved, in the JSON list : waymo 148, kitti 124, nuimages 128 ids as assigned : kitti 0..399, waymo 550..949, nuimages 1100..1499 what --max-images 200 loads : kitti 200 what --max-images 400 loads : kitti 400 what --max-images 800 loads : kitti 400, waymo 400 train.py's default is max_images = 400, so `--dataset mixed` with nothing else set trains on one source. The held-out block has the same shape. It is written as val.json and labelled "held-out mixed", and run_eval defaults to --mixed-split val with --max-images 200, which read 150 KITTI + 50 Waymo and never reached nuImages. Both lists are now shuffled (with separate seeds) and the image ids are renumbered to follow that order, with annotations remapped. Train takes 0..T-1 and the held-out block the range above it, so the two stay disjoint by construction the way this function's docstring promises. After the change, same build: train --max-images 200 : waymo 81, kitti 57, nuimages 62 train --max-images 400 : waymo 148, kitti 124, nuimages 128 val --max-images 200 : waymo 72, kitti 65, nuimages 63 and the invariants hold: train/val id intersection 0, every annotation still resolves inside its own split, and test_.json is unchanged at 150 images of one source each -- those filter on "source", not on id. One thing I could not settle and am not asserting: TUTORIAL.md 18.3 runs --dataset mixed at the 400 default and reports a fine-tune result whose shape (kitti up, waymo and nuimages down) is what training on KITTI alone would look like, while 18.4 passes --max-images 99999 and recovers the other two. Whether that section is measuring catastrophic forgetting or this bug is a question for you, not a claim from me. Co-Authored-By: Claude Opus 5 --- DeepDataMiningLearning/ngdet/mixed_dataset.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/DeepDataMiningLearning/ngdet/mixed_dataset.py b/DeepDataMiningLearning/ngdet/mixed_dataset.py index 20712327..5db7b283 100644 --- a/DeepDataMiningLearning/ngdet/mixed_dataset.py +++ b/DeepDataMiningLearning/ngdet/mixed_dataset.py @@ -116,6 +116,29 @@ def _write(name, keep_imgs): # Shuffle the train order so that taking the first --max-images during training # yields a balanced (cross-source) subset -- needed for clean data-size studies. random.Random(seed + 1).shuffle(train_imgs) + + # The shuffle only reaches the trainer if the image *ids* follow it. + # torchvision's CocoDetection does `self.ids = list(sorted(self.coco.imgs.keys()))`, + # so it discards the JSON list order, and img_id above is assigned per source in + # one contiguous block. Sorting therefore front-loads whichever source came first: + # --max-images 400 over a 3x400 mixed base read 400 KITTI images and nothing else. + # The held-out block needs the same treatment: it is written as val.json and + # labelled "held-out mixed", and run_eval defaults to --mixed-split val with + # --max-images 200, which over a 3x150 block read 150 KITTI + 50 Waymo and never + # reached nuImages. Shuffled with its own seed so the two orders stay independent. + random.Random(seed + 2).shuffle(test_imgs) + + # Renumber so id order IS the shuffled order. Train takes 0..T-1 and the held-out + # block the range above it, which keeps the two disjoint by construction the way + # this function's docstring promises. The per-source test_.json files filter + # on "source", not on id, so they are unaffected. + _remap = {im["id"]: new_id for new_id, im in enumerate(train_imgs)} + _remap.update({im["id"]: len(train_imgs) + off + for off, im in enumerate(test_imgs)}) + for _im in images: + _im["id"] = _remap[_im["id"]] + for _a in annotations: + _a["image_id"] = _remap[_a["image_id"]] splits = {"train": _write("train", train_imgs), "val": _write("val", test_imgs)} # held-out mixed for src in sources: # held-out per source