diff --git a/scripts/build_yolo_dataset.py b/scripts/build_yolo_dataset.py index 55253d1..1af58d0 100644 --- a/scripts/build_yolo_dataset.py +++ b/scripts/build_yolo_dataset.py @@ -60,6 +60,12 @@ logger = None EMPTY = "(无框)" +#: 坐标的容忍量。Label Studio 按百分比存框,导出时 /100 会有浮点误差, +#: 实测出现过 height=1.0000000000000002。ultralytics 对 >1 的坐标直接报错, +#: 所以这点溢出必须钳掉;超出这个量级的才是真的坏标签。 +COORDINATE_TOLERANCE = 1e-6 + + class Sample(NamedTuple): """一张图、它的标签文本,以及分层划分用的类别。""" @@ -105,12 +111,18 @@ def read_export(path: Path) -> tuple[list[str], dict[str, str]]: return classes, labels -def parse_boxes(text: str, stem: str, num_classes: int) -> list[int]: - """校验一个标签文件,返回其中所有框的类别下标。 +def parse_boxes(text: str, stem: str, num_classes: int) -> tuple[list[int], str, int]: + """校验并规整一个标签文件。 - 坐标越界或类别越界都直接报错:这类问题在训练时不会崩,只会悄悄拉低指标。 + Returns: + (类别下标列表, 规整后的标签文本, 被钳位的坐标个数) + + 字段数、类别下标、明显越界的坐标都直接报错:这类问题在训练时不会崩, + 只会悄悄拉低指标。只有浮点误差量级的溢出才钳到 [0,1]。 """ class_ids = [] + lines = [] + clamped = 0 for number, line in enumerate(text.splitlines(), start=1): line = line.strip() if not line: @@ -125,18 +137,22 @@ def parse_boxes(text: str, stem: str, num_classes: int) -> list[int]: raise ExportError(f"{stem}.txt 第 {number} 行无法解析:{line!r}") from exc if not 0 <= class_id < num_classes: raise ExportError(f"{stem}.txt 第 {number} 行的类别下标 {class_id} 超出 0~{num_classes - 1}") - if not all(0.0 <= v <= 1.0 for v in coordinates): + if not all(-COORDINATE_TOLERANCE <= v <= 1 + COORDINATE_TOLERANCE for v in coordinates): raise ExportError(f"{stem}.txt 第 {number} 行的坐标不在 [0,1] 内:{coordinates}") + clamped += sum(not 0.0 <= v <= 1.0 for v in coordinates) + coordinates = [min(max(v, 0.0), 1.0) for v in coordinates] class_ids.append(class_id) - return class_ids + lines.append(" ".join([str(class_id), *(f"{v:.6f}" for v in coordinates)])) + return class_ids, "".join(f"{line}\n" for line in lines), clamped def collect( classes: list[str], labels: dict[str, str], images_dir: Path -) -> tuple[list[Sample], list[str]]: - """把标签与图片配对,返回 (样本列表, 多余的图片名)。""" +) -> tuple[list[Sample], list[str], int]: + """把标签与图片配对,返回 (样本列表, 多余的图片名, 钳位的坐标个数)。""" samples = [] missing = [] + clamped = 0 for stem, text in labels.items(): candidates = [ images_dir / f"{stem}{suffix}" for suffix in (".png", *IMAGE_SUFFIXES) @@ -145,10 +161,11 @@ def collect( if image is None: missing.append(stem) continue - class_ids = parse_boxes(text, stem, len(classes)) + class_ids, cleaned, clamped_here = parse_boxes(text, stem, len(classes)) + clamped += clamped_here # 分层用"框最多的那个类别":一张图可能有多个类别的框,人工复核后尤其常见 stratum = classes[Counter(class_ids).most_common(1)[0][0]] if class_ids else EMPTY - samples.append(Sample(image, text, stratum)) + samples.append(Sample(image, cleaned, stratum)) if missing: raise ExportError( @@ -160,7 +177,7 @@ def collect( p.stem for p in images_dir.iterdir() if p.suffix.lower() in IMAGE_SUFFIXES and p.stem not in known ) - return samples, extra + return samples, extra, clamped def split(samples: list[Sample], val_ratio: float, seed: int) -> dict[str, list[Sample]]: @@ -205,12 +222,17 @@ def main() -> int: try: classes, labels = read_export(Path(args.export)) - samples, extra = collect(classes, labels, Path(args.images)) + samples, extra, clamped = collect(classes, labels, Path(args.images)) except (ExportError, FileNotFoundError) as exc: logger.error("%s", exc) return 1 logger.info("导出: %s(%d 个标签,%d 类)", args.export, len(labels), len(classes)) + if clamped: + logger.info( + "%d 个坐标有浮点误差量级的溢出(Label Studio 按百分比存框,/100 会有误差),已钳到 [0,1];" + "不钳的话 ultralytics 读标签时会直接报错", clamped, + ) if extra: logger.warning( "%s 里有 %d 张图片没有对应的标签,不会进入数据集(例如 %s)", @@ -274,6 +296,7 @@ def main() -> int: "empty_labels": len(empty), "dropped_empty": args.drop_empty, "images_without_label": len(extra), + "clamped_coordinates": clamped, "val_ratio": args.val_ratio, "seed": args.seed, "split_sizes": {k: len(v) for k, v in splits.items()}, diff --git a/tests/test_build_yolo_dataset.py b/tests/test_build_yolo_dataset.py index f3a9eda..603c2e8 100644 --- a/tests/test_build_yolo_dataset.py +++ b/tests/test_build_yolo_dataset.py @@ -76,7 +76,10 @@ def test_builds_a_split_dataset(export, tmp_path): images, labels = outputs(out, "images"), outputs(out, "labels") assert set(images) == set(LABELS) == set(labels) for stem, path in labels.items(): - assert path.read_text(encoding="utf-8") == LABELS[stem] + # 内容等价,但坐标统一成 6 位小数 + written = [line.split() for line in path.read_text(encoding="utf-8").splitlines()] + expected = [line.split() for line in LABELS[stem].splitlines()] + assert [[v[0], *(f"{float(x):.6f}" for x in v[1:])] for v in expected] == written # 图与标签必须落在同一个 split,否则验证集会混进训练图 assert path.parent.name == images[stem].parent.name @@ -227,3 +230,41 @@ def test_refuses_to_clobber_and_dry_run_writes_nothing(export, tmp_path): assert run(export, out) == 0 assert run(export, out) != 0 # 已存在且非空 assert run(export, out, "--overwrite") == 0 + + +def _repack(archive, tmp_path, replacements): + """复制一份导出,替换其中几个标签文件。""" + broken = tmp_path / "repacked.zip" + with zipfile.ZipFile(archive) as source, zipfile.ZipFile(broken, "w") as target: + for item in source.namelist(): + data = replacements.get(item, source.read(item)) + target.writestr(item, data.encode("utf-8") if isinstance(data, str) else data) + return broken + + +def test_float_overshoot_is_clamped(export, tmp_path, capsys): + """Label Studio 按百分比存框,/100 后会出现 1.0000000000000002。 + + 回归:不钳位的话 ultralytics 读标签时会直接报"坐标未归一化"而拒绝训练。 + """ + archive, images = export + overshoot = "1 0.49999999999999994 0.5000000000000001 0.9999999999999999 1.0000000000000002\n" + repacked = _repack(archive, tmp_path, {"labels/grease_0001.txt": overshoot}) + + out = tmp_path / "out" + assert run((repacked, images), out) == 0 + assert "已钳到 [0,1]" in capsys.readouterr().out + + written = outputs(out, "labels")["grease_0001"].read_text(encoding="utf-8").split() + assert written == ["1", "0.500000", "0.500000", "1.000000", "1.000000"] + assert all(0.0 <= float(v) <= 1.0 for v in written[1:]) + # 四个值里只有 1.0000000000000002 真的越界,其余虽有浮点误差但仍在 [0,1] 内 + assert json.loads((out / "summary.json").read_text(encoding="utf-8"))["clamped_coordinates"] == 1 + + +def test_real_out_of_range_is_still_rejected(export, tmp_path, capsys): + """只钳浮点误差量级;真正越界的坐标仍要报错。""" + archive, images = export + repacked = _repack(archive, tmp_path, {"labels/grease_0001.txt": "1 0.5 1.02 0.1 0.1\n"}) + assert run((repacked, images), tmp_path / "out") != 0 + assert "不在 [0,1]" in capsys.readouterr().out