From 9b31473b2758144e3922f1e36ed87dd354260a03 Mon Sep 17 00:00:00 2001 From: Shaun Zhou Date: Tue, 8 Sep 2026 15:26:00 +0800 Subject: [PATCH] =?UTF-8?q?feat(export):=20=E6=94=AF=E6=8C=81=E6=8A=8A?= =?UTF-8?q?=E5=8E=9F=E7=88=B6=E7=9B=AE=E5=BD=95=E5=90=8D=E4=BD=9C=E4=B8=BA?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E5=90=8D=E5=89=8D=E7=BC=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 --prefix-parent：复制时把图片原来的父目录名拼到文件名前 （grease/abc.png -> grease_abc.png）。导出后目录只剩簇号，来源信息本来会丢失， 加前缀能留住它，人工核对时一眼看出某张图原本在哪个目录。 - 重名检查改为按**目标文件名**判重：加前缀后原本会撞车的文件可能就不撞了； 不加前缀而撞车时，报错会提示可以试试这个开关 - manifest.json 记下 prefix_parent，导出的 file 字段也用实际写出的文件名 Co-Authored-By: Claude Opus 5 --- README.md | 13 +++++++- scripts/export_clusters.py | 50 ++++++++++++++++++++++------ tests/test_export_clusters.py | 62 +++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 0629073..06f415d 100644 --- a/README.md +++ b/README.md @@ -277,7 +277,18 @@ data/film_r18_cluster_k4/ # <源目录名>_<骨架标签>_cluster_k<簇数> └── manifest.json # 来源 run、特征配置、每张图的原始路径 ``` -**复制真实文件**，不是软链也不改名，所以这个目录可以直接拷走或喂给别的工具。 +**复制真实文件**，不是软链，所以这个目录可以直接拷走或喂给别的工具。 + +默认保持原文件名。加 `--prefix-parent` 会把原父目录名拼到文件名前： + +```bash +uv run python scripts/export_clusters.py --work-dir runs/simclr_r18 --prefix-parent +# back-mosquito-G0/5.14_677mmX316.10m_....png -> back-mosquito-G0_5.14_677mmX316.10m_....png +``` + +导出后目录只剩簇号，原来的来源信息就没了——加前缀能留住它，人工核对时一眼看出 +某张图原本在哪个目录；顺带也化解不同子目录同名文件在同一个簇里撞车的情况 +（撞车时的报错也会提示可以试试这个开关）。 目录名可用 `--tag r18ss` 换标签、`--out` 完全指定。几个保护：源文件缺失、 同簇内出现重名（保持原名会互相覆盖）都会直接报错；目标目录非空时必须显式 `--overwrite`。 diff --git a/scripts/export_clusters.py b/scripts/export_clusters.py index b38f92a..59e2ad0 100755 --- a/scripts/export_clusters.py +++ b/scripts/export_clusters.py @@ -14,7 +14,10 @@ data/film_r18_cluster_k4/ └── manifest.json # 来源 run、每张图的原始路径，便于追溯 ``` -**复制真实文件**，不是软链、也不改名——所以这个目录可以直接拷走或喂给别的工具。 +**复制真实文件**，不是软链——所以这个目录可以直接拷走或喂给别的工具。 +默认保持原文件名；加 `--prefix-parent` 会在文件名前拼上原父目录名 +（`grease/abc.png` -> `grease_abc.png`），便于导出后仍能一眼看出图片来自哪个原始目录， +也能避免不同子目录的同名文件在同一个簇里撞车。 目录名由三段拼成 `<源目录名>_<骨架标签>_cluster_k<簇数>`，取自聚类产出的 summary.json：源目录名来自 assignments.csv 里各路径的公共父目录，骨架标签来自 @@ -89,19 +92,34 @@ def common_root(paths: list[Path]) -> Path: return Path(*common) if common else Path(".") -def check_conflicts(groups: dict[int, list[Path]]) -> None: - """同一个簇里出现同名文件时直接报错——要求保留原文件名，就不能悄悄覆盖。""" +def destination_name(source: Path, prefix_parent: bool = False) -> str: + """目标文件名。 + + `prefix_parent` 时拼上原父目录名（`grease/abc.png` -> `grease_abc.png`）： + 导出后目录只剩簇号，原来的来源信息就没了，加前缀能留住它； + 顺带也化解不同子目录同名文件在同一个簇里撞车的情况。 + """ + return f"{source.parent.name}_{source.name}" if prefix_parent else source.name + + +def check_conflicts(groups: dict[int, list[Path]], prefix_parent: bool = False) -> None: + """同一个簇里出现同名文件时直接报错——不改名就不能悄悄覆盖。 + + 按**目标文件名**判重：加了父目录前缀之后原本会撞车的文件可能就不撞了。 + """ problems = [] for cluster, files in sorted(groups.items()): - names = Counter(f.name for f in files) + names = Counter(destination_name(f, prefix_parent) for f in files) for name, count in names.items(): if count > 1: - sources = [str(f) for f in files if f.name == name][:3] + sources = [ + str(f) for f in files if destination_name(f, prefix_parent) == name + ][:3] problems.append(f" 簇 {cluster} 里 {name!r} 出现 {count} 次: {', '.join(sources)} …") if problems: + hint = "" if prefix_parent else "可以试试 --prefix-parent 用父目录名区分，或" raise ValueError( - "以下文件重名，保持原名会互相覆盖；请先重命名源文件或改用别的导出方式：\n" - + "\n".join(problems[:20]) + f"以下文件重名，同名会互相覆盖；{hint}先重命名源文件：\n" + "\n".join(problems[:20]) ) @@ -118,6 +136,11 @@ def main() -> int: parser.add_argument("--overwrite", action="store_true", help="输出目录已存在时先清空") parser.add_argument("--dry-run", action="store_true", help="只打印计划，不真的复制") parser.add_argument("--no-manifest", action="store_true", help="不写 manifest.json") + parser.add_argument( + "--prefix-parent", + action="store_true", + help="复制时在文件名前拼上原父目录名（grease/abc.png -> grease_abc.png）", + ) args = parser.parse_args() logger = setup_logging() @@ -153,13 +176,18 @@ def main() -> int: logger.info("来源: %s（%d 张，%d 个簇）", assignments_path, len(rows), len(groups)) logger.info("源数据集: %s | 骨架标签: %s", source_root, tag) logger.info("输出目录: %s", out_dir) + if args.prefix_parent: + example = rows[0][0] + logger.info( + "文件名加父目录前缀：%s -> %s", example.name, destination_name(example, True) + ) missing = [p for p, _ in rows if not p.exists()] if missing: logger.error("有 %d 个源文件不存在，例如 %s", len(missing), missing[0]) return 1 try: - check_conflicts(groups) + check_conflicts(groups, args.prefix_parent) except ValueError as exc: logger.error("%s", exc) return 1 @@ -185,11 +213,12 @@ def main() -> int: target.mkdir(parents=True, exist_ok=True) entries = [] for source in sorted(groups[cluster]): - destination = target / source.name # 保持原文件名 + name = destination_name(source, args.prefix_parent) + destination = target / name shutil.copy2(source, destination) # copy2 连同修改时间一起复制 total_bytes += destination.stat().st_size copied += 1 - entries.append({"file": source.name, "source": str(source)}) + entries.append({"file": name, "source": str(source)}) manifest[str(cluster)] = entries logger.info("簇 %d 完成：%d 张", cluster, len(entries)) @@ -200,6 +229,7 @@ def main() -> int: "source_root": str(source_root), "assignments": str(assignments_path.resolve()), "num_clusters": num_clusters, + "prefix_parent": args.prefix_parent, "feature_source": summary.get("feature_source"), "cluster_sizes": {k: len(v) for k, v in sorted(manifest.items())}, "files": manifest, diff --git a/tests/test_export_clusters.py b/tests/test_export_clusters.py index 59511e8..e35cb9e 100644 --- a/tests/test_export_clusters.py +++ b/tests/test_export_clusters.py @@ -4,6 +4,7 @@ import csv import filecmp import json import sys +from pathlib import Path import pytest @@ -78,6 +79,67 @@ def test_exports_real_files_with_original_names(cluster_run, tmp_path, monkeypat assert filecmp.cmp(original, by_name[original.name], shallow=False) +def test_prefix_parent_prepends_source_folder(cluster_run, tmp_path, monkeypatch): + """--prefix-parent 把原父目录名拼到文件名前，导出后仍能看出图来自哪个目录。""" + _, work_dir, files = cluster_run + data_root = tmp_path / "data" + + assert run(["--work-dir", str(work_dir), "--data-root", str(data_root), "--prefix-parent"], + monkeypatch) == 0 + + out = data_root / "film_r18_cluster_k2" + exported = {p.name for p in out.rglob("*.png")} + assert exported == {f"{f.parent.name}_{f.name}" for f in files} + assert all(name.startswith(("grease_", "wrinkle_")) for name in exported) + + # 仍然是真实文件、内容不变 + for path in out.rglob("*.png"): + assert not path.is_symlink() and path.is_file() + + manifest = json.loads((out / "manifest.json").read_text("utf-8")) + assert manifest["prefix_parent"] is True + for entries in manifest["files"].values(): + for entry in entries: + source = Path(entry["source"]) + assert entry["file"] == f"{source.parent.name}_{source.name}" + + +def test_without_prefix_names_are_unchanged(cluster_run, tmp_path, monkeypatch): + _, work_dir, files = cluster_run + data_root = tmp_path / "data" + run(["--work-dir", str(work_dir), "--data-root", str(data_root)], monkeypatch) + + exported = {p.name for p in (data_root / "film_r18_cluster_k2").rglob("*.png")} + assert exported == {f.name for f in files} + + +def test_prefix_parent_resolves_same_name_across_folders(cluster_run, tmp_path, monkeypatch): + """不同子目录的同名文件，不加前缀会撞车报错，加了前缀就能共存。""" + source, work_dir, _ = cluster_run + twin = source / "wrinkle" / "grease_0.png" # 与 grease/grease_0.png 同名 + Image.fromarray(np.zeros((20, 30, 3), dtype=np.uint8)).save(twin) + + rows = list(csv.DictReader((work_dir / "assignments.csv").open(encoding="utf-8"))) + with (work_dir / "assignments.csv").open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + writer.writerow({**rows[0], "path": str(twin), "cluster": rows[0]["cluster"]}) + + data_root = tmp_path / "data" + argv = ["--work-dir", str(work_dir), "--data-root", str(data_root)] + assert run(argv, monkeypatch) == 1 # 不加前缀：撞车 + assert run([*argv, "--prefix-parent"], monkeypatch) == 0 # 加前缀：共存 + + out = data_root / "film_r18_cluster_k2" + assert {"grease_grease_0.png", "wrinkle_grease_0.png"} <= {p.name for p in out.rglob("*.png")} + + +def test_destination_name_helper(): + assert ec.destination_name(Path("data/raw/film/grease/abc.png")) == "abc.png" + assert ec.destination_name(Path("data/raw/film/grease/abc.png"), True) == "grease_abc.png" + + def test_cluster_dirs_are_numbered_from_zero(cluster_run, tmp_path, monkeypatch): _, work_dir, files = cluster_run data_root = tmp_path / "data"