fix(detect): 钳掉 Label Studio 导出坐标的浮点溢出
真实导出里出现 height=1.0000000000000002：Label Studio 按百分比存框，导出时 /100 留下浮点误差。ultralytics 读标签时对 >1 的坐标直接报错，所以这点溢出必须钳掉， 否则装配出的数据集根本训不起来。 只钳 1e-6 量级的溢出，真正越界的坐标仍然报错。顺带把坐标统一写成 6 位小数。 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Shaun Zhou
co-authored by
Claude Opus 5
Claude Opus 5
parent
998e6601d4
commit
59d4b8a9c4
@@ -60,6 +60,12 @@ logger = None
|
||||
EMPTY = "(无框)"
|
||||
|
||||
|
||||
#: 坐标的容忍量。Label Studio 按百分比存框，导出时 /100 会有浮点误差，
|
||||
#: 实测出现过 height=1.0000000000000002。ultralytics 对 >1 的坐标直接报错，
|
||||
#: 所以这点溢出必须钳掉；超出这个量级的才是真的坏标签。
|
||||
COORDINATE_TOLERANCE = 1e-6
|
||||
|
||||
|
||||
class Sample(NamedTuple):
|
||||
"""一张图、它的标签文本，以及分层划分用的类别。"""
|
||||
|
||||
@@ -105,12 +111,18 @@ def read_export(path: Path) -> tuple[list[str], dict[str, str]]:
|
||||
return classes, labels
|
||||
|
||||
|
||||
def parse_boxes(text: str, stem: str, num_classes: int) -> list[int]:
|
||||
"""校验一个标签文件，返回其中所有框的类别下标。
|
||||
def parse_boxes(text: str, stem: str, num_classes: int) -> tuple[list[int], str, int]:
|
||||
"""校验并规整一个标签文件。
|
||||
|
||||
坐标越界或类别越界都直接报错：这类问题在训练时不会崩，只会悄悄拉低指标。
|
||||
Returns:
|
||||
(类别下标列表, 规整后的标签文本, 被钳位的坐标个数)
|
||||
|
||||
字段数、类别下标、明显越界的坐标都直接报错：这类问题在训练时不会崩，
|
||||
只会悄悄拉低指标。只有浮点误差量级的溢出才钳到 [0,1]。
|
||||
"""
|
||||
class_ids = []
|
||||
lines = []
|
||||
clamped = 0
|
||||
for number, line in enumerate(text.splitlines(), start=1):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
@@ -125,18 +137,22 @@ def parse_boxes(text: str, stem: str, num_classes: int) -> list[int]:
|
||||
raise ExportError(f"{stem}.txt 第 {number} 行无法解析：{line!r}") from exc
|
||||
if not 0 <= class_id < num_classes:
|
||||
raise ExportError(f"{stem}.txt 第 {number} 行的类别下标 {class_id} 超出 0~{num_classes - 1}")
|
||||
if not all(0.0 <= v <= 1.0 for v in coordinates):
|
||||
if not all(-COORDINATE_TOLERANCE <= v <= 1 + COORDINATE_TOLERANCE for v in coordinates):
|
||||
raise ExportError(f"{stem}.txt 第 {number} 行的坐标不在 [0,1] 内：{coordinates}")
|
||||
clamped += sum(not 0.0 <= v <= 1.0 for v in coordinates)
|
||||
coordinates = [min(max(v, 0.0), 1.0) for v in coordinates]
|
||||
class_ids.append(class_id)
|
||||
return class_ids
|
||||
lines.append(" ".join([str(class_id), *(f"{v:.6f}" for v in coordinates)]))
|
||||
return class_ids, "".join(f"{line}\n" for line in lines), clamped
|
||||
|
||||
|
||||
def collect(
|
||||
classes: list[str], labels: dict[str, str], images_dir: Path
|
||||
) -> tuple[list[Sample], list[str]]:
|
||||
"""把标签与图片配对，返回 (样本列表, 多余的图片名)。"""
|
||||
) -> tuple[list[Sample], list[str], int]:
|
||||
"""把标签与图片配对，返回 (样本列表, 多余的图片名, 钳位的坐标个数)。"""
|
||||
samples = []
|
||||
missing = []
|
||||
clamped = 0
|
||||
for stem, text in labels.items():
|
||||
candidates = [
|
||||
images_dir / f"{stem}{suffix}" for suffix in (".png", *IMAGE_SUFFIXES)
|
||||
@@ -145,10 +161,11 @@ def collect(
|
||||
if image is None:
|
||||
missing.append(stem)
|
||||
continue
|
||||
class_ids = parse_boxes(text, stem, len(classes))
|
||||
class_ids, cleaned, clamped_here = parse_boxes(text, stem, len(classes))
|
||||
clamped += clamped_here
|
||||
# 分层用"框最多的那个类别"：一张图可能有多个类别的框，人工复核后尤其常见
|
||||
stratum = classes[Counter(class_ids).most_common(1)[0][0]] if class_ids else EMPTY
|
||||
samples.append(Sample(image, text, stratum))
|
||||
samples.append(Sample(image, cleaned, stratum))
|
||||
|
||||
if missing:
|
||||
raise ExportError(
|
||||
@@ -160,7 +177,7 @@ def collect(
|
||||
p.stem for p in images_dir.iterdir()
|
||||
if p.suffix.lower() in IMAGE_SUFFIXES and p.stem not in known
|
||||
)
|
||||
return samples, extra
|
||||
return samples, extra, clamped
|
||||
|
||||
|
||||
def split(samples: list[Sample], val_ratio: float, seed: int) -> dict[str, list[Sample]]:
|
||||
@@ -205,12 +222,17 @@ def main() -> int:
|
||||
|
||||
try:
|
||||
classes, labels = read_export(Path(args.export))
|
||||
samples, extra = collect(classes, labels, Path(args.images))
|
||||
samples, extra, clamped = collect(classes, labels, Path(args.images))
|
||||
except (ExportError, FileNotFoundError) as exc:
|
||||
logger.error("%s", exc)
|
||||
return 1
|
||||
|
||||
logger.info("导出: %s（%d 个标签，%d 类）", args.export, len(labels), len(classes))
|
||||
if clamped:
|
||||
logger.info(
|
||||
"%d 个坐标有浮点误差量级的溢出（Label Studio 按百分比存框，/100 会有误差），已钳到 [0,1]；"
|
||||
"不钳的话 ultralytics 读标签时会直接报错", clamped,
|
||||
)
|
||||
if extra:
|
||||
logger.warning(
|
||||
"%s 里有 %d 张图片没有对应的标签，不会进入数据集（例如 %s）",
|
||||
@@ -274,6 +296,7 @@ def main() -> int:
|
||||
"empty_labels": len(empty),
|
||||
"dropped_empty": args.drop_empty,
|
||||
"images_without_label": len(extra),
|
||||
"clamped_coordinates": clamped,
|
||||
"val_ratio": args.val_ratio,
|
||||
"seed": args.seed,
|
||||
"split_sizes": {k: len(v) for k, v in splits.items()},
|
||||
|
||||
@@ -76,7 +76,10 @@ def test_builds_a_split_dataset(export, tmp_path):
|
||||
images, labels = outputs(out, "images"), outputs(out, "labels")
|
||||
assert set(images) == set(LABELS) == set(labels)
|
||||
for stem, path in labels.items():
|
||||
assert path.read_text(encoding="utf-8") == LABELS[stem]
|
||||
# 内容等价，但坐标统一成 6 位小数
|
||||
written = [line.split() for line in path.read_text(encoding="utf-8").splitlines()]
|
||||
expected = [line.split() for line in LABELS[stem].splitlines()]
|
||||
assert [[v[0], *(f"{float(x):.6f}" for x in v[1:])] for v in expected] == written
|
||||
# 图与标签必须落在同一个 split，否则验证集会混进训练图
|
||||
assert path.parent.name == images[stem].parent.name
|
||||
|
||||
@@ -227,3 +230,41 @@ def test_refuses_to_clobber_and_dry_run_writes_nothing(export, tmp_path):
|
||||
assert run(export, out) == 0
|
||||
assert run(export, out) != 0 # 已存在且非空
|
||||
assert run(export, out, "--overwrite") == 0
|
||||
|
||||
|
||||
def _repack(archive, tmp_path, replacements):
|
||||
"""复制一份导出，替换其中几个标签文件。"""
|
||||
broken = tmp_path / "repacked.zip"
|
||||
with zipfile.ZipFile(archive) as source, zipfile.ZipFile(broken, "w") as target:
|
||||
for item in source.namelist():
|
||||
data = replacements.get(item, source.read(item))
|
||||
target.writestr(item, data.encode("utf-8") if isinstance(data, str) else data)
|
||||
return broken
|
||||
|
||||
|
||||
def test_float_overshoot_is_clamped(export, tmp_path, capsys):
|
||||
"""Label Studio 按百分比存框，/100 后会出现 1.0000000000000002。
|
||||
|
||||
回归：不钳位的话 ultralytics 读标签时会直接报"坐标未归一化"而拒绝训练。
|
||||
"""
|
||||
archive, images = export
|
||||
overshoot = "1 0.49999999999999994 0.5000000000000001 0.9999999999999999 1.0000000000000002\n"
|
||||
repacked = _repack(archive, tmp_path, {"labels/grease_0001.txt": overshoot})
|
||||
|
||||
out = tmp_path / "out"
|
||||
assert run((repacked, images), out) == 0
|
||||
assert "已钳到 [0,1]" in capsys.readouterr().out
|
||||
|
||||
written = outputs(out, "labels")["grease_0001"].read_text(encoding="utf-8").split()
|
||||
assert written == ["1", "0.500000", "0.500000", "1.000000", "1.000000"]
|
||||
assert all(0.0 <= float(v) <= 1.0 for v in written[1:])
|
||||
# 四个值里只有 1.0000000000000002 真的越界，其余虽有浮点误差但仍在 [0,1] 内
|
||||
assert json.loads((out / "summary.json").read_text(encoding="utf-8"))["clamped_coordinates"] == 1
|
||||
|
||||
|
||||
def test_real_out_of_range_is_still_rejected(export, tmp_path, capsys):
|
||||
"""只钳浮点误差量级；真正越界的坐标仍要报错。"""
|
||||
archive, images = export
|
||||
repacked = _repack(archive, tmp_path, {"labels/grease_0001.txt": "1 0.5 1.02 0.1 0.1\n"})
|
||||
assert run((repacked, images), tmp_path / "out") != 0
|
||||
assert "不在 [0,1]" in capsys.readouterr().out
|
||||
|
||||
Reference in New Issue
Block a user