feat(export): 支持把原父目录名作为文件名前缀
新增 --prefix-parent：复制时把图片原来的父目录名拼到文件名前 （grease/abc.png -> grease_abc.png）。导出后目录只剩簇号，来源信息本来会丢失， 加前缀能留住它，人工核对时一眼看出某张图原本在哪个目录。 - 重名检查改为按**目标文件名**判重：加前缀后原本会撞车的文件可能就不撞了； 不加前缀而撞车时，报错会提示可以试试这个开关 - manifest.json 记下 prefix_parent，导出的 file 字段也用实际写出的文件名 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Shaun Zhou
co-authored by
Claude Opus 5
Claude Opus 5
parent
6e543660ce
commit
9b31473b27
@@ -277,7 +277,18 @@ data/film_r18_cluster_k4/ # <源目录名>_<骨架标签>_cluster_k<簇数>
|
||||
└── manifest.json # 来源 run、特征配置、每张图的原始路径
|
||||
```
|
||||
|
||||
**复制真实文件**，不是软链也不改名，所以这个目录可以直接拷走或喂给别的工具。
|
||||
**复制真实文件**，不是软链，所以这个目录可以直接拷走或喂给别的工具。
|
||||
|
||||
默认保持原文件名。加 `--prefix-parent` 会把原父目录名拼到文件名前：
|
||||
|
||||
```bash
|
||||
uv run python scripts/export_clusters.py --work-dir runs/simclr_r18 --prefix-parent
|
||||
# back-mosquito-G0/5.14_677mmX316.10m_....png -> back-mosquito-G0_5.14_677mmX316.10m_....png
|
||||
```
|
||||
|
||||
导出后目录只剩簇号，原来的来源信息就没了——加前缀能留住它，人工核对时一眼看出
|
||||
某张图原本在哪个目录；顺带也化解不同子目录同名文件在同一个簇里撞车的情况
|
||||
（撞车时的报错也会提示可以试试这个开关）。
|
||||
目录名可用 `--tag r18ss` 换标签、`--out` 完全指定。几个保护：源文件缺失、
|
||||
同簇内出现重名（保持原名会互相覆盖）都会直接报错；目标目录非空时必须显式 `--overwrite`。
|
||||
|
||||
|
||||
+40
-10
@@ -14,7 +14,10 @@ data/film_r18_cluster_k4/
|
||||
└── manifest.json # 来源 run、每张图的原始路径，便于追溯
|
||||
```
|
||||
|
||||
**复制真实文件**，不是软链、也不改名——所以这个目录可以直接拷走或喂给别的工具。
|
||||
**复制真实文件**，不是软链——所以这个目录可以直接拷走或喂给别的工具。
|
||||
默认保持原文件名；加 `--prefix-parent` 会在文件名前拼上原父目录名
|
||||
（`grease/abc.png` -> `grease_abc.png`），便于导出后仍能一眼看出图片来自哪个原始目录，
|
||||
也能避免不同子目录的同名文件在同一个簇里撞车。
|
||||
|
||||
目录名由三段拼成 `<源目录名>_<骨架标签>_cluster_k<簇数>`，取自聚类产出的
|
||||
summary.json：源目录名来自 assignments.csv 里各路径的公共父目录，骨架标签来自
|
||||
@@ -89,19 +92,34 @@ def common_root(paths: list[Path]) -> Path:
|
||||
return Path(*common) if common else Path(".")
|
||||
|
||||
|
||||
def check_conflicts(groups: dict[int, list[Path]]) -> None:
|
||||
"""同一个簇里出现同名文件时直接报错——要求保留原文件名，就不能悄悄覆盖。"""
|
||||
def destination_name(source: Path, prefix_parent: bool = False) -> str:
|
||||
"""目标文件名。
|
||||
|
||||
`prefix_parent` 时拼上原父目录名（`grease/abc.png` -> `grease_abc.png`）：
|
||||
导出后目录只剩簇号，原来的来源信息就没了，加前缀能留住它；
|
||||
顺带也化解不同子目录同名文件在同一个簇里撞车的情况。
|
||||
"""
|
||||
return f"{source.parent.name}_{source.name}" if prefix_parent else source.name
|
||||
|
||||
|
||||
def check_conflicts(groups: dict[int, list[Path]], prefix_parent: bool = False) -> None:
|
||||
"""同一个簇里出现同名文件时直接报错——不改名就不能悄悄覆盖。
|
||||
|
||||
按**目标文件名**判重：加了父目录前缀之后原本会撞车的文件可能就不撞了。
|
||||
"""
|
||||
problems = []
|
||||
for cluster, files in sorted(groups.items()):
|
||||
names = Counter(f.name for f in files)
|
||||
names = Counter(destination_name(f, prefix_parent) for f in files)
|
||||
for name, count in names.items():
|
||||
if count > 1:
|
||||
sources = [str(f) for f in files if f.name == name][:3]
|
||||
sources = [
|
||||
str(f) for f in files if destination_name(f, prefix_parent) == name
|
||||
][:3]
|
||||
problems.append(f" 簇 {cluster} 里 {name!r} 出现 {count} 次: {', '.join(sources)} …")
|
||||
if problems:
|
||||
hint = "" if prefix_parent else "可以试试 --prefix-parent 用父目录名区分，或"
|
||||
raise ValueError(
|
||||
"以下文件重名，保持原名会互相覆盖；请先重命名源文件或改用别的导出方式：\n"
|
||||
+ "\n".join(problems[:20])
|
||||
f"以下文件重名，同名会互相覆盖；{hint}先重命名源文件：\n" + "\n".join(problems[:20])
|
||||
)
|
||||
|
||||
|
||||
@@ -118,6 +136,11 @@ def main() -> int:
|
||||
parser.add_argument("--overwrite", action="store_true", help="输出目录已存在时先清空")
|
||||
parser.add_argument("--dry-run", action="store_true", help="只打印计划，不真的复制")
|
||||
parser.add_argument("--no-manifest", action="store_true", help="不写 manifest.json")
|
||||
parser.add_argument(
|
||||
"--prefix-parent",
|
||||
action="store_true",
|
||||
help="复制时在文件名前拼上原父目录名（grease/abc.png -> grease_abc.png）",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
logger = setup_logging()
|
||||
@@ -153,13 +176,18 @@ def main() -> int:
|
||||
logger.info("来源: %s（%d 张，%d 个簇）", assignments_path, len(rows), len(groups))
|
||||
logger.info("源数据集: %s | 骨架标签: %s", source_root, tag)
|
||||
logger.info("输出目录: %s", out_dir)
|
||||
if args.prefix_parent:
|
||||
example = rows[0][0]
|
||||
logger.info(
|
||||
"文件名加父目录前缀：%s -> %s", example.name, destination_name(example, True)
|
||||
)
|
||||
|
||||
missing = [p for p, _ in rows if not p.exists()]
|
||||
if missing:
|
||||
logger.error("有 %d 个源文件不存在，例如 %s", len(missing), missing[0])
|
||||
return 1
|
||||
try:
|
||||
check_conflicts(groups)
|
||||
check_conflicts(groups, args.prefix_parent)
|
||||
except ValueError as exc:
|
||||
logger.error("%s", exc)
|
||||
return 1
|
||||
@@ -185,11 +213,12 @@ def main() -> int:
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
entries = []
|
||||
for source in sorted(groups[cluster]):
|
||||
destination = target / source.name # 保持原文件名
|
||||
name = destination_name(source, args.prefix_parent)
|
||||
destination = target / name
|
||||
shutil.copy2(source, destination) # copy2 连同修改时间一起复制
|
||||
total_bytes += destination.stat().st_size
|
||||
copied += 1
|
||||
entries.append({"file": source.name, "source": str(source)})
|
||||
entries.append({"file": name, "source": str(source)})
|
||||
manifest[str(cluster)] = entries
|
||||
logger.info("簇 %d 完成：%d 张", cluster, len(entries))
|
||||
|
||||
@@ -200,6 +229,7 @@ def main() -> int:
|
||||
"source_root": str(source_root),
|
||||
"assignments": str(assignments_path.resolve()),
|
||||
"num_clusters": num_clusters,
|
||||
"prefix_parent": args.prefix_parent,
|
||||
"feature_source": summary.get("feature_source"),
|
||||
"cluster_sizes": {k: len(v) for k, v in sorted(manifest.items())},
|
||||
"files": manifest,
|
||||
|
||||
@@ -4,6 +4,7 @@ import csv
|
||||
import filecmp
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -78,6 +79,67 @@ def test_exports_real_files_with_original_names(cluster_run, tmp_path, monkeypat
|
||||
assert filecmp.cmp(original, by_name[original.name], shallow=False)
|
||||
|
||||
|
||||
def test_prefix_parent_prepends_source_folder(cluster_run, tmp_path, monkeypatch):
|
||||
"""--prefix-parent 把原父目录名拼到文件名前，导出后仍能看出图来自哪个目录。"""
|
||||
_, work_dir, files = cluster_run
|
||||
data_root = tmp_path / "data"
|
||||
|
||||
assert run(["--work-dir", str(work_dir), "--data-root", str(data_root), "--prefix-parent"],
|
||||
monkeypatch) == 0
|
||||
|
||||
out = data_root / "film_r18_cluster_k2"
|
||||
exported = {p.name for p in out.rglob("*.png")}
|
||||
assert exported == {f"{f.parent.name}_{f.name}" for f in files}
|
||||
assert all(name.startswith(("grease_", "wrinkle_")) for name in exported)
|
||||
|
||||
# 仍然是真实文件、内容不变
|
||||
for path in out.rglob("*.png"):
|
||||
assert not path.is_symlink() and path.is_file()
|
||||
|
||||
manifest = json.loads((out / "manifest.json").read_text("utf-8"))
|
||||
assert manifest["prefix_parent"] is True
|
||||
for entries in manifest["files"].values():
|
||||
for entry in entries:
|
||||
source = Path(entry["source"])
|
||||
assert entry["file"] == f"{source.parent.name}_{source.name}"
|
||||
|
||||
|
||||
def test_without_prefix_names_are_unchanged(cluster_run, tmp_path, monkeypatch):
|
||||
_, work_dir, files = cluster_run
|
||||
data_root = tmp_path / "data"
|
||||
run(["--work-dir", str(work_dir), "--data-root", str(data_root)], monkeypatch)
|
||||
|
||||
exported = {p.name for p in (data_root / "film_r18_cluster_k2").rglob("*.png")}
|
||||
assert exported == {f.name for f in files}
|
||||
|
||||
|
||||
def test_prefix_parent_resolves_same_name_across_folders(cluster_run, tmp_path, monkeypatch):
|
||||
"""不同子目录的同名文件，不加前缀会撞车报错，加了前缀就能共存。"""
|
||||
source, work_dir, _ = cluster_run
|
||||
twin = source / "wrinkle" / "grease_0.png" # 与 grease/grease_0.png 同名
|
||||
Image.fromarray(np.zeros((20, 30, 3), dtype=np.uint8)).save(twin)
|
||||
|
||||
rows = list(csv.DictReader((work_dir / "assignments.csv").open(encoding="utf-8")))
|
||||
with (work_dir / "assignments.csv").open("w", newline="", encoding="utf-8") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=list(rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
writer.writerow({**rows[0], "path": str(twin), "cluster": rows[0]["cluster"]})
|
||||
|
||||
data_root = tmp_path / "data"
|
||||
argv = ["--work-dir", str(work_dir), "--data-root", str(data_root)]
|
||||
assert run(argv, monkeypatch) == 1 # 不加前缀：撞车
|
||||
assert run([*argv, "--prefix-parent"], monkeypatch) == 0 # 加前缀：共存
|
||||
|
||||
out = data_root / "film_r18_cluster_k2"
|
||||
assert {"grease_grease_0.png", "wrinkle_grease_0.png"} <= {p.name for p in out.rglob("*.png")}
|
||||
|
||||
|
||||
def test_destination_name_helper():
|
||||
assert ec.destination_name(Path("data/raw/film/grease/abc.png")) == "abc.png"
|
||||
assert ec.destination_name(Path("data/raw/film/grease/abc.png"), True) == "grease_abc.png"
|
||||
|
||||
|
||||
def test_cluster_dirs_are_numbered_from_zero(cluster_run, tmp_path, monkeypatch):
|
||||
_, work_dir, files = cluster_run
|
||||
data_root = tmp_path / "data"
|
||||
|
||||
Reference in New Issue
Block a user