refactor: prepare_label_studio.py 的内容改动（补上 a3e08a9 漏掉的部分）
a3e08a9 只提交了 git mv 的改名，内容修改因 git add 的路径报错没有进入暂存区。
本提交补上：
- 缺省输出 runs/<source>_label_studio/，不再写到 data/；该目录同时是 Label Studio
的 DOCUMENT_ROOT，打印的导入步骤里提醒复核期间不要删除或 --overwrite。
- 总是生成 tasks.json 与 labeling config，去掉 --label-studio 开关。
- 不再写 data.yaml：复核材料不是训练集，没有 data.yaml 就不会被误拿去训练，
原先"train 与 val 相同"的告警也随之移除（detect.py 里的同类告警保留）。
- detect.py、configs/detect/film.yaml、README 里对旧脚本的引用改为新流程。
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>This commit is contained in:
Shaun Zhou
co-authored by
Claude Opus 5
Claude Opus 5
parent
a3e08a9bbc
commit
891f930de2
@@ -1,4 +1,4 @@
|
||||
"""把 generate_boxes.py 的产出整理成 YOLO 训练集。"""
|
||||
"""把 generate_boxes.py 的产出整理成 Label Studio 复核材料。"""
|
||||
|
||||
import csv
|
||||
import json
|
||||
@@ -14,7 +14,7 @@ import yaml # noqa: E402
|
||||
from PIL import Image # noqa: E402
|
||||
|
||||
sys.path.insert(0, "scripts")
|
||||
import prepare_yolo_dataset as pyd # noqa: E402
|
||||
import prepare_label_studio as pls # noqa: E402
|
||||
|
||||
CLASSES = ["grease", "normal", "whitedot"]
|
||||
|
||||
@@ -55,17 +55,17 @@ def output_of(out, source_stem):
|
||||
|
||||
|
||||
def run(source, out, *extra):
|
||||
argv = ["prepare_yolo_dataset.py", "--source", str(source), "--out", str(out), *extra]
|
||||
argv = ["prepare_label_studio.py", "--source", str(source), "--out", str(out), *extra]
|
||||
original, sys.argv = sys.argv, argv
|
||||
try:
|
||||
return pyd.main()
|
||||
return pls.main()
|
||||
finally:
|
||||
sys.argv = original
|
||||
|
||||
|
||||
def test_flat_layout_without_split(source, tmp_path):
|
||||
"""类别目录被拍平、不划分 train/val：images/ 与 labels/ 下直接是文件，一一对应。"""
|
||||
out = tmp_path / "data" / "boxes"
|
||||
def test_flat_layout(source, tmp_path):
|
||||
"""类别目录被拍平：images/ 与 labels/ 下直接是文件，一一对应（converter 不递归）。"""
|
||||
out = tmp_path / "runs" / "boxes_label_studio"
|
||||
assert run(source, out) == 0
|
||||
|
||||
for kind in ("images", "labels"):
|
||||
@@ -74,12 +74,11 @@ def test_flat_layout_without_split(source, tmp_path):
|
||||
labels = sorted(p.stem for p in (out / "labels").iterdir())
|
||||
assert images == labels
|
||||
assert len(images) == 13
|
||||
assert json.loads((out / "summary.json").read_text(encoding="utf-8"))["split"] is False
|
||||
|
||||
|
||||
def test_hard_links_share_inode(source, tmp_path):
|
||||
"""用硬链接而不是复制，不占额外磁盘。"""
|
||||
out = tmp_path / "data" / "boxes"
|
||||
out = tmp_path / "runs" / "boxes_label_studio"
|
||||
assert run(source, out) == 0
|
||||
original = source / "images" / "grease" / "grease_0.png"
|
||||
linked = output_of(out, "grease_0")
|
||||
@@ -87,21 +86,15 @@ def test_hard_links_share_inode(source, tmp_path):
|
||||
|
||||
|
||||
def test_per_class_counts_in_summary(source, tmp_path):
|
||||
out = tmp_path / "data" / "boxes"
|
||||
out = tmp_path / "runs" / "boxes_label_studio"
|
||||
assert run(source, out) == 0
|
||||
per_class = json.loads((out / "summary.json").read_text(encoding="utf-8"))["per_class"]
|
||||
assert per_class == {"grease": 6, "normal": 4, "(无框)": 1, "whitedot": 2}
|
||||
|
||||
|
||||
def test_warns_that_data_is_not_split(source, tmp_path, capsys):
|
||||
"""train 与 val 指向同一目录，直接训练验证指标会虚高，必须提醒。"""
|
||||
assert run(source, tmp_path / "data" / "boxes") == 0
|
||||
assert "未划分" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_empty_labels_kept_as_negatives_by_default(source, tmp_path):
|
||||
"""无框的图默认保留——YOLO 里它们是压低误检的负样本。"""
|
||||
out = tmp_path / "data" / "boxes"
|
||||
out = tmp_path / "runs" / "boxes_label_studio"
|
||||
assert run(source, out) == 0
|
||||
summary = json.loads((out / "summary.json").read_text(encoding="utf-8"))
|
||||
assert summary["empty_labels"] == {"normal": {"policy": "keep", "count": 1}}
|
||||
@@ -109,7 +102,7 @@ def test_empty_labels_kept_as_negatives_by_default(source, tmp_path):
|
||||
|
||||
|
||||
def test_empty_policy_drop_removes_the_image(source, tmp_path):
|
||||
out = tmp_path / "data" / "boxes"
|
||||
out = tmp_path / "runs" / "boxes_label_studio"
|
||||
assert run(source, out, "--empty-policy", "drop") == 0
|
||||
summary = json.loads((out / "summary.json").read_text(encoding="utf-8"))
|
||||
assert summary["num_images"] == 12
|
||||
@@ -119,7 +112,7 @@ def test_empty_policy_drop_removes_the_image(source, tmp_path):
|
||||
|
||||
def test_empty_policy_is_per_class(source, tmp_path):
|
||||
"""缺陷类的空标签丢掉，normal 的留作负样本——两种处置能共存。"""
|
||||
out = tmp_path / "data" / "boxes"
|
||||
out = tmp_path / "runs" / "boxes_label_studio"
|
||||
assert run(source, out, "--empty-policy", "drop", "normal=keep") == 0
|
||||
summary = json.loads((out / "summary.json").read_text(encoding="utf-8"))
|
||||
assert summary["empty_labels"]["normal"]["policy"] == "keep"
|
||||
@@ -128,7 +121,7 @@ def test_empty_policy_is_per_class(source, tmp_path):
|
||||
|
||||
def test_empty_policy_full_writes_a_whole_image_box(source, tmp_path):
|
||||
"""整图属性的类别退化成整图框：标签要重写，不能链接源文件（内容不同）。"""
|
||||
out = tmp_path / "data" / "boxes"
|
||||
out = tmp_path / "runs" / "boxes_label_studio"
|
||||
assert run(source, out, "--empty-policy", "normal=full") == 0
|
||||
|
||||
image = output_of(out, "normal_0")
|
||||
@@ -156,17 +149,25 @@ def test_empty_policy_rejects_typos(source, tmp_path, capsys, tokens, expected):
|
||||
assert expected in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_data_yaml_uses_absolute_path_and_source_names(source, tmp_path):
|
||||
"""data.yaml 要能被 ultralytics 从任意工作目录读到，类别顺序沿用来源。"""
|
||||
out = tmp_path / "data" / "boxes"
|
||||
def test_no_data_yaml_so_it_cannot_be_trained_on_by_mistake(source, tmp_path):
|
||||
"""复核材料不是训练集：不写 data.yaml，免得没修过的弱监督框被直接拿去训练。"""
|
||||
out = tmp_path / "runs" / "boxes_label_studio"
|
||||
assert run(source, out) == 0
|
||||
meta = yaml.safe_load((out / "data.yaml").read_text(encoding="utf-8"))
|
||||
assert Path(meta["path"]).is_absolute()
|
||||
assert meta["nc"] == 3
|
||||
assert [meta["names"][i] for i in sorted(meta["names"])] == CLASSES
|
||||
# 不划分：两者都指向扁平的 images/
|
||||
assert meta["train"] == meta["val"] == "images"
|
||||
assert (Path(meta["path"]) / meta["train"]).is_dir()
|
||||
assert not (out / "data.yaml").exists()
|
||||
assert not (out / "images" / "train").exists()
|
||||
|
||||
|
||||
def test_default_output_goes_under_runs(source, tmp_path, monkeypatch):
|
||||
"""缺省输出是 runs/<source>_label_studio，不再写到 data/。"""
|
||||
pytest.importorskip("label_studio_sdk.converter.imports.yolo")
|
||||
monkeypatch.chdir(tmp_path)
|
||||
original, sys.argv = sys.argv, ["prepare_label_studio.py", "--source", str(source)]
|
||||
try:
|
||||
assert pls.main() == 0
|
||||
finally:
|
||||
sys.argv = original
|
||||
assert (tmp_path / "runs" / "boxes_label_studio" / "tasks.json").is_file()
|
||||
assert not (tmp_path / "data").exists()
|
||||
|
||||
|
||||
def test_name_collision_after_flattening_is_reported(source, tmp_path, capsys):
|
||||
@@ -175,13 +176,13 @@ def test_name_collision_after_flattening_is_reported(source, tmp_path, capsys):
|
||||
Image.fromarray(np.zeros((32, 32, 3), dtype=np.uint8)).save(duplicate)
|
||||
(source / "labels" / "normal" / "grease_0.txt").write_text("1 0.5 0.5 0.2 0.2\n", encoding="utf-8")
|
||||
|
||||
assert run(source, tmp_path / "data" / "boxes", "--keep-names") != 0
|
||||
assert run(source, tmp_path / "runs" / "boxes_label_studio", "--keep-names") != 0
|
||||
assert "冲突" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_refuses_to_clobber_existing_output(source, tmp_path):
|
||||
"""输出目录非空时要求显式 --overwrite。"""
|
||||
out = tmp_path / "data" / "boxes"
|
||||
out = tmp_path / "runs" / "boxes_label_studio"
|
||||
out.mkdir(parents=True)
|
||||
(out / "keep.txt").write_text("x", encoding="utf-8")
|
||||
assert run(source, out) != 0
|
||||
@@ -197,7 +198,7 @@ def test_missing_label_is_an_error(source, tmp_path):
|
||||
|
||||
|
||||
def test_dry_run_writes_nothing(source, tmp_path):
|
||||
out = tmp_path / "data" / "boxes"
|
||||
out = tmp_path / "runs" / "boxes_label_studio"
|
||||
assert run(source, out, "--dry-run") == 0
|
||||
assert not out.exists()
|
||||
|
||||
@@ -225,13 +226,13 @@ def test_all_output_images_are_png(source, tmp_path):
|
||||
"""BMP / JPEG 来源都要落成真正的 PNG——后缀和文件头都得是。"""
|
||||
_add_sample(source, "grease", "bmp_img", ".bmp", 0, fmt="BMP")
|
||||
_add_sample(source, "normal", "jpg_img", ".jpg", 1, fmt="JPEG")
|
||||
out = tmp_path / "data" / "boxes"
|
||||
out = tmp_path / "runs" / "boxes_label_studio"
|
||||
assert run(source, out) == 0
|
||||
|
||||
images = _all_outputs(out, "images")
|
||||
assert len(images) == 15
|
||||
assert {p.suffix for p in images} == {".png"}
|
||||
assert all(pyd.is_png(p) for p in images)
|
||||
assert all(pls.is_png(p) for p in images)
|
||||
# 标签跟着改名后的图片走，一一对应
|
||||
assert sorted(p.stem for p in images) == sorted(p.stem for p in _all_outputs(out, "labels"))
|
||||
|
||||
@@ -239,7 +240,7 @@ def test_all_output_images_are_png(source, tmp_path):
|
||||
def test_bmp_conversion_is_lossless(source, tmp_path):
|
||||
"""转换不能缩放或有损压缩，否则归一化的框就不再对应原来的像素。"""
|
||||
_, original = _add_sample(source, "grease", "bmp_img", ".bmp", 0, fmt="BMP")
|
||||
out = tmp_path / "data" / "boxes"
|
||||
out = tmp_path / "runs" / "boxes_label_studio"
|
||||
assert run(source, out) == 0
|
||||
|
||||
converted = output_of(out, "bmp_img")
|
||||
@@ -251,7 +252,7 @@ def test_bmp_conversion_is_lossless(source, tmp_path):
|
||||
def test_png_sources_are_still_hard_linked(source, tmp_path):
|
||||
"""只有非 PNG 才转换；PNG 照旧硬链接，不占额外磁盘。"""
|
||||
bmp, _ = _add_sample(source, "grease", "bmp_img", ".bmp", 0, fmt="BMP")
|
||||
out = tmp_path / "data" / "boxes"
|
||||
out = tmp_path / "runs" / "boxes_label_studio"
|
||||
assert run(source, out) == 0
|
||||
|
||||
summary = json.loads((out / "summary.json").read_text(encoding="utf-8"))
|
||||
@@ -264,18 +265,18 @@ def test_png_sources_are_still_hard_linked(source, tmp_path):
|
||||
def test_png_suffix_with_non_png_content_is_converted(source, tmp_path):
|
||||
"""按文件头判断而不是后缀：写着 .png 其实是 JPEG 的也要转。"""
|
||||
fake, _ = _add_sample(source, "grease", "fake", ".png", 0, fmt="JPEG")
|
||||
assert not pyd.is_png(fake)
|
||||
out = tmp_path / "data" / "boxes"
|
||||
assert not pls.is_png(fake)
|
||||
out = tmp_path / "runs" / "boxes_label_studio"
|
||||
assert run(source, out) == 0
|
||||
output = output_of(out, "fake")
|
||||
assert pyd.is_png(output)
|
||||
assert pls.is_png(output)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["L", "RGBA", "CMYK"])
|
||||
def test_conversion_handles_pixel_modes(source, tmp_path, mode):
|
||||
"""灰度 / 带透明度原样保留；PNG 存不下的 CMYK 转成 RGB。"""
|
||||
_add_sample(source, "grease", "moded", ".tif", 0, fmt="TIFF", mode=mode)
|
||||
out = tmp_path / "data" / "boxes"
|
||||
out = tmp_path / "runs" / "boxes_label_studio"
|
||||
assert run(source, out) == 0
|
||||
output = output_of(out, "moded")
|
||||
with Image.open(output) as image:
|
||||
@@ -286,7 +287,7 @@ def test_conversion_handles_pixel_modes(source, tmp_path, mode):
|
||||
def test_same_stem_different_suffix_collides(source, tmp_path, capsys):
|
||||
"""grease_0.png 与另一目录的 grease_0.bmp 统一后缀后会撞成同一个文件，必须报错。"""
|
||||
_add_sample(source, "normal", "grease_0", ".bmp", 1, fmt="BMP")
|
||||
assert run(source, tmp_path / "data" / "boxes", "--keep-names") != 0
|
||||
assert run(source, tmp_path / "runs" / "boxes_label_studio", "--keep-names") != 0
|
||||
assert "冲突" in capsys.readouterr().out
|
||||
|
||||
|
||||
@@ -296,13 +297,13 @@ def test_dry_run_reports_conversion_count(source, tmp_path, capsys):
|
||||
assert "1 张不是 PNG" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_classes_txt_matches_data_yaml(source, tmp_path):
|
||||
"""classes.txt 的行号就是 class_id，必须与 data.yaml 的 names 严格一致。"""
|
||||
out = tmp_path / "data" / "boxes"
|
||||
def test_classes_txt_matches_source_data_yaml(source, tmp_path):
|
||||
"""classes.txt 的行号就是 class_id，必须与来源 data.yaml 的 names 严格一致。"""
|
||||
out = tmp_path / "runs" / "boxes_label_studio"
|
||||
assert run(source, out) == 0
|
||||
|
||||
lines = (out / "classes.txt").read_text(encoding="utf-8").splitlines()
|
||||
meta = yaml.safe_load((out / "data.yaml").read_text(encoding="utf-8"))
|
||||
meta = yaml.safe_load((source / "data.yaml").read_text(encoding="utf-8"))
|
||||
assert lines == CLASSES
|
||||
assert lines == [meta["names"][i] for i in range(meta["nc"])]
|
||||
assert (out / "classes.txt").read_text(encoding="utf-8").endswith("\n")
|
||||
@@ -323,7 +324,7 @@ def _stems(out):
|
||||
|
||||
def test_files_renamed_by_class(source, tmp_path):
|
||||
"""默认按 <类别>_<序号> 重新编号，序号在类内从 1 连续编到该类总数。"""
|
||||
out = tmp_path / "data" / "boxes"
|
||||
out = tmp_path / "runs" / "boxes_label_studio"
|
||||
assert run(source, out) == 0
|
||||
assert _stems(out) == (
|
||||
[f"grease_{i:04d}" for i in range(1, 7)]
|
||||
@@ -335,7 +336,7 @@ def test_files_renamed_by_class(source, tmp_path):
|
||||
|
||||
def test_renamed_file_class_matches_its_label(source, tmp_path):
|
||||
"""文件名里的类别必须与标签第一列一致，否则改名反倒制造了误导。"""
|
||||
out = tmp_path / "data" / "boxes"
|
||||
out = tmp_path / "runs" / "boxes_label_studio"
|
||||
assert run(source, out) == 0
|
||||
for label in _all_outputs(out, "labels"):
|
||||
text = label.read_text(encoding="utf-8").split()
|
||||
@@ -345,7 +346,7 @@ def test_renamed_file_class_matches_its_label(source, tmp_path):
|
||||
|
||||
def test_name_map_traces_back_to_source(source, tmp_path):
|
||||
"""name_map.csv 覆盖每一张输出，且源路径真实存在、内容一致。"""
|
||||
out = tmp_path / "data" / "boxes"
|
||||
out = tmp_path / "runs" / "boxes_label_studio"
|
||||
assert run(source, out) == 0
|
||||
with (out / "name_map.csv").open(encoding="utf-8-sig", newline="") as handle:
|
||||
rows = list(csv.DictReader(handle))
|
||||
@@ -368,7 +369,7 @@ def test_names_stable_across_reruns(source, tmp_path):
|
||||
def test_keep_names_preserves_source_stems(source, tmp_path):
|
||||
"""--keep-names 保留原名，只是后缀统一成 .png。"""
|
||||
_add_sample(source, "grease", "原名_中文", ".bmp", 0, fmt="BMP")
|
||||
out = tmp_path / "data" / "boxes"
|
||||
out = tmp_path / "runs" / "boxes_label_studio"
|
||||
assert run(source, out, "--keep-names") == 0
|
||||
assert "原名_中文" in _stems(out)
|
||||
assert output_of(out, "原名_中文").name == "原名_中文.png"
|
||||
@@ -376,7 +377,7 @@ def test_keep_names_preserves_source_stems(source, tmp_path):
|
||||
|
||||
def test_kept_empty_labels_named_after_source_folder(source, tmp_path):
|
||||
"""空标签没有 class_id，按来源目录名编号；与同类有框的图共用序号，不会撞名。"""
|
||||
out = tmp_path / "data" / "boxes"
|
||||
out = tmp_path / "runs" / "boxes_label_studio"
|
||||
assert run(source, out) == 0
|
||||
assert output_of(out, "normal_0").stem.startswith("normal_")
|
||||
assert len([s for s in _stems(out) if s.startswith("normal_")]) == 5
|
||||
@@ -385,8 +386,8 @@ def test_kept_empty_labels_named_after_source_folder(source, tmp_path):
|
||||
@pytest.fixture
|
||||
def label_studio_out(source, tmp_path):
|
||||
pytest.importorskip("label_studio_sdk.converter.imports.yolo")
|
||||
out = tmp_path / "data" / "boxes"
|
||||
assert run(source, out, "--empty-policy", "normal=keep", "--label-studio") == 0
|
||||
out = tmp_path / "runs" / "boxes_label_studio"
|
||||
assert run(source, out, "--empty-policy", "normal=keep") == 0
|
||||
return out
|
||||
|
||||
|
||||
@@ -394,8 +395,8 @@ def _tasks(out):
|
||||
return json.loads((out / "tasks.json").read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def test_label_studio_files_at_dataset_root(label_studio_out):
|
||||
"""tasks.json 与 labeling config 直接放在数据集根目录，不再有 label_studio/ 子目录。"""
|
||||
def test_label_studio_files_at_output_root(label_studio_out):
|
||||
"""tasks.json 与 labeling config 总是生成，直接放在输出根目录（即 DOCUMENT_ROOT）。"""
|
||||
assert (label_studio_out / "tasks.json").is_file()
|
||||
assert (label_studio_out / "tasks.label_config.xml").is_file()
|
||||
assert not (label_studio_out / "label_studio").exists()
|
||||
@@ -442,8 +443,11 @@ def test_label_studio_config_and_summary(label_studio_out):
|
||||
assert Path(summary["label_studio"]["document_root"]) == label_studio_out.resolve()
|
||||
|
||||
|
||||
def test_label_studio_not_built_by_default(source, tmp_path):
|
||||
out = tmp_path / "data" / "boxes"
|
||||
assert run(source, out) == 0
|
||||
assert not (out / "tasks.json").exists()
|
||||
assert json.loads((out / "summary.json").read_text(encoding="utf-8"))["label_studio"] is None
|
||||
def test_prints_import_steps_with_real_paths(label_studio_out, capsys):
|
||||
"""converter 自带提示漏了环境变量，脚本要给出带实际路径的完整步骤。"""
|
||||
capsys.readouterr()
|
||||
source = label_studio_out.parent / "boxes"
|
||||
assert run(source, label_studio_out, "--overwrite") == 0
|
||||
out = capsys.readouterr().out
|
||||
assert f"LABEL_STUDIO_LOCAL_FILES_DOCUMENT_ROOT={label_studio_out.resolve()}" in out
|
||||
assert "不要点 Sync" in out
|
||||
|
||||
Reference in New Issue
Block a user