feat(export): 改为按 clusters/ 目录导出，并重定位失效的软链
原来按 assignments.csv 导出，反映的是聚类当时的划分；而 clusters/ 目录会被人工 整理——挪动分错的图、把 cluster_03 改名成 grease。现在以整理后的目录为准。 - 扫描 clusters/ 下的子目录，目录名即类别名并原样保留（不再是数字编号） - 文件名用原图的名字，去掉 clusters/ 里的 NNNN_ 排序前缀 - 失效软链的重定位：读出链接目标，按其中的 data/ 段落到本机 --source-root 上； 再不行用文件名在源目录里兜底索引（惰性构建，用不到就不扫盘）。 从服务器拷回来的 clusters/ 软链全部指向 /home/deep/... 在本机失效，这是常态 - 解析不到的条目列出并中止，--skip-missing 可忽略 - manifest 记录类别、每张图的原始路径与被跳过的条目 在真实目录上验证：11,212 张、15 个类别，软链 100% 失效但全部重定位成功， 导出后抽查 200 张与原图逐字节一致。跨数据集的数字文件名（blue/113.bmp 与 pearlescent/113.bmp）会被重名检查拦下，--prefix-parent 可化解。 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Shaun Zhou
co-authored by
Claude Opus 5
Claude Opus 5
parent
9b31473b27
commit
f8654f447b
+150
-171
@@ -1,6 +1,5 @@
|
||||
"""按聚类结果导出数据集。"""
|
||||
"""按 clusters/ 目录导出数据集。"""
|
||||
|
||||
import csv
|
||||
import filecmp
|
||||
import json
|
||||
import sys
|
||||
@@ -18,38 +17,37 @@ import export_clusters as ec # noqa: E402
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cluster_run(tmp_path):
|
||||
"""造一份源图片 + assignments.csv + summary.json。"""
|
||||
source = tmp_path / "film"
|
||||
(source / "grease").mkdir(parents=True)
|
||||
(source / "wrinkle").mkdir(parents=True)
|
||||
def curated(tmp_path):
|
||||
"""造出「源图在本机 + clusters/ 里全是失效软链」的现场。
|
||||
|
||||
模拟从服务器拷回来的 clusters/：软链指向 /server/proj/data/raw/...，本机没有该路径，
|
||||
但同样的相对路径在 tmp_path/data/raw/ 下存在。
|
||||
"""
|
||||
rng = np.random.default_rng(0)
|
||||
files = []
|
||||
for folder, count in (("grease", 3), ("wrinkle", 2)):
|
||||
sources = {}
|
||||
for dataset, folder, count in (("film", "grease", 3), ("film", "wrinkle", 2), ("blue", "", 2)):
|
||||
relative = Path("data/raw") / dataset / folder if folder else Path("data/raw") / dataset
|
||||
directory = tmp_path / relative
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
for i in range(count):
|
||||
path = source / folder / f"{folder}_{i}.png"
|
||||
Image.fromarray(rng.integers(0, 255, (20, 30, 3), dtype=np.uint8)).save(path)
|
||||
files.append(path)
|
||||
name = f"{dataset}{folder}{i}.png"
|
||||
path = directory / name
|
||||
Image.fromarray(rng.integers(0, 255, (12, 16, 3), dtype=np.uint8)).save(path)
|
||||
sources[path] = str(Path("/server/proj") / relative / name)
|
||||
|
||||
work_dir = tmp_path / "run"
|
||||
work_dir.mkdir()
|
||||
with (work_dir / "assignments.csv").open("w", newline="", encoding="utf-8") as handle:
|
||||
writer = csv.writer(handle)
|
||||
writer.writerow(["path", "cluster", "distance_to_centroid", "source_dir", "width", "height"])
|
||||
for index, path in enumerate(files):
|
||||
writer.writerow([str(path), index % 2, "0.1", path.parent.name, 30, 20])
|
||||
(work_dir / "summary.json").write_text(
|
||||
json.dumps({
|
||||
"num_clusters": 2,
|
||||
"feature_source": {
|
||||
"method": "simclr",
|
||||
"backbone": {"name": "torchvision", "arch": "resnet18"},
|
||||
},
|
||||
}),
|
||||
encoding="utf-8",
|
||||
clusters = tmp_path / "run" / "clusters"
|
||||
categories = {"grease": [], "normal": []}
|
||||
for index, (local, server) in enumerate(sorted(sources.items())):
|
||||
category = "grease" if index % 2 == 0 else "normal"
|
||||
(clusters / category).mkdir(parents=True, exist_ok=True)
|
||||
link = clusters / category / f"{index:04d}_{local.name}"
|
||||
link.symlink_to(server) # 指向服务器路径，本机必然失效
|
||||
assert not link.exists()
|
||||
categories[category].append(local)
|
||||
(tmp_path / "run" / "summary.json").write_text(
|
||||
json.dumps({"feature_source": {"backbone": {"arch": "resnet101"}}}), encoding="utf-8"
|
||||
)
|
||||
return source, work_dir, files
|
||||
return tmp_path, clusters, categories
|
||||
|
||||
|
||||
def run(argv, monkeypatch):
|
||||
@@ -57,167 +55,148 @@ def run(argv, monkeypatch):
|
||||
return ec.main()
|
||||
|
||||
|
||||
def test_exports_real_files_with_original_names(cluster_run, tmp_path, monkeypatch):
|
||||
source, work_dir, files = cluster_run
|
||||
data_root = tmp_path / "data"
|
||||
def test_broken_symlinks_are_relocated_to_local_source(curated, tmp_path, monkeypatch):
|
||||
"""从服务器拷回来的软链全部失效，仍应按 data/ 段落到本机原图上。"""
|
||||
root, clusters, categories = curated
|
||||
out = tmp_path / "out"
|
||||
|
||||
assert run(["--work-dir", str(work_dir), "--data-root", str(data_root)], monkeypatch) == 0
|
||||
|
||||
out = data_root / "film_r18_cluster_k2" # <源目录名>_<骨架标签>_cluster_k<簇数>
|
||||
assert out.is_dir()
|
||||
assert sorted(p.name for p in out.iterdir() if p.is_dir()) == ["0", "1"]
|
||||
|
||||
exported = sorted(p for p in out.rglob("*.png"))
|
||||
assert len(exported) == len(files)
|
||||
for path in exported:
|
||||
assert not path.is_symlink(), "必须是真实文件，不是软链"
|
||||
assert path.is_file()
|
||||
# 文件名保持不变、内容逐字节一致
|
||||
by_name = {p.name: p for p in exported}
|
||||
for original in files:
|
||||
assert original.name in by_name
|
||||
assert filecmp.cmp(original, by_name[original.name], shallow=False)
|
||||
|
||||
|
||||
def test_prefix_parent_prepends_source_folder(cluster_run, tmp_path, monkeypatch):
|
||||
"""--prefix-parent 把原父目录名拼到文件名前，导出后仍能看出图来自哪个目录。"""
|
||||
_, work_dir, files = cluster_run
|
||||
data_root = tmp_path / "data"
|
||||
|
||||
assert run(["--work-dir", str(work_dir), "--data-root", str(data_root), "--prefix-parent"],
|
||||
assert run(["--clusters", str(clusters), "--out", str(out), "--source-root", str(root)],
|
||||
monkeypatch) == 0
|
||||
|
||||
out = data_root / "film_r18_cluster_k2"
|
||||
exported = {p.name for p in out.rglob("*.png")}
|
||||
assert exported == {f"{f.parent.name}_{f.name}" for f in files}
|
||||
assert all(name.startswith(("grease_", "wrinkle_")) for name in exported)
|
||||
for category, sources in categories.items():
|
||||
exported = sorted((out / category).glob("*.png"))
|
||||
assert len(exported) == len(sources)
|
||||
for path in exported:
|
||||
assert not path.is_symlink(), "必须是真实文件"
|
||||
by_name = {p.name: p for p in exported}
|
||||
for source in sources:
|
||||
assert filecmp.cmp(source, by_name[source.name], shallow=False)
|
||||
|
||||
# 仍然是真实文件、内容不变
|
||||
for path in out.rglob("*.png"):
|
||||
assert not path.is_symlink() and path.is_file()
|
||||
|
||||
def test_category_names_are_preserved(curated, tmp_path, monkeypatch):
|
||||
"""clusters/ 里的子目录名是人工整理出来的类别，导出后要原样保留。"""
|
||||
root, clusters, categories = curated
|
||||
out = tmp_path / "out"
|
||||
run(["--clusters", str(clusters), "--out", str(out), "--source-root", str(root)], monkeypatch)
|
||||
|
||||
assert sorted(p.name for p in out.iterdir() if p.is_dir()) == sorted(categories)
|
||||
|
||||
|
||||
def test_rank_prefix_is_stripped(curated, tmp_path, monkeypatch):
|
||||
"""clusters/ 里的文件名带 0000_ 排序前缀，导出用原图名。"""
|
||||
root, clusters, _ = curated
|
||||
out = tmp_path / "out"
|
||||
run(["--clusters", str(clusters), "--out", str(out), "--source-root", str(root)], monkeypatch)
|
||||
|
||||
assert not any(p.name[:4].isdigit() and p.name[4] == "_" for p in out.rglob("*.png"))
|
||||
|
||||
|
||||
def test_prefix_parent_resolves_cross_dataset_collisions(curated, tmp_path, monkeypatch):
|
||||
"""不同数据集用同样的数字文件名（blue/7.png 与 pearlescent/7.png）时会撞车，
|
||||
加父目录前缀就能共存——这正是真实数据里 blue/uv/pearlescent 的情况。"""
|
||||
root, clusters, _ = curated
|
||||
twins = []
|
||||
for dataset in ("blue", "pearlescent"):
|
||||
directory = root / "data" / "raw" / dataset
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
path = directory / "7.png"
|
||||
Image.fromarray(np.full((12, 16, 3), 7, dtype=np.uint8)).save(path)
|
||||
link = clusters / "normal" / f"05{len(twins)}_7.png"
|
||||
link.symlink_to(f"/server/proj/data/raw/{dataset}/7.png")
|
||||
twins.append(path)
|
||||
|
||||
out = tmp_path / "out"
|
||||
argv = ["--clusters", str(clusters), "--out", str(out), "--source-root", str(root)]
|
||||
|
||||
assert run(argv, monkeypatch) == 1 # 同名撞车
|
||||
assert run([*argv, "--prefix-parent"], monkeypatch) == 0 # 用父目录名区分
|
||||
|
||||
names = {p.name for p in out.rglob("*.png")}
|
||||
assert {"blue_7.png", "pearlescent_7.png"} <= names
|
||||
|
||||
|
||||
def test_real_files_in_clusters_are_used_directly(curated, tmp_path, monkeypatch):
|
||||
"""有人直接把真实图片拷进 clusters/ 时也要能用。"""
|
||||
root, clusters, _ = curated
|
||||
extra = clusters / "normal" / "9999_real.png"
|
||||
Image.fromarray(np.zeros((10, 10, 3), dtype=np.uint8)).save(extra)
|
||||
|
||||
out = tmp_path / "out"
|
||||
assert run(["--clusters", str(clusters), "--out", str(out), "--source-root", str(root),
|
||||
"--prefix-parent"], monkeypatch) == 0
|
||||
assert (out / "normal" / "normal_9999_real.png").is_file()
|
||||
|
||||
|
||||
def test_unresolvable_entries_abort_unless_skipped(curated, tmp_path, monkeypatch):
|
||||
root, clusters, _ = curated
|
||||
orphan = clusters / "normal" / "0099_gone.png"
|
||||
orphan.symlink_to("/server/proj/data/raw/film/grease/does_not_exist.png")
|
||||
|
||||
out = tmp_path / "out"
|
||||
argv = ["--clusters", str(clusters), "--out", str(out), "--source-root", str(root),
|
||||
"--prefix-parent"]
|
||||
assert run(argv, monkeypatch) == 1
|
||||
assert run([*argv, "--skip-missing"], monkeypatch) == 0
|
||||
|
||||
manifest = json.loads((out / "manifest.json").read_text("utf-8"))
|
||||
assert len(manifest["skipped"]) == 1
|
||||
|
||||
|
||||
def test_manifest_records_sources_and_categories(curated, tmp_path, monkeypatch):
|
||||
root, clusters, categories = curated
|
||||
out = tmp_path / "out"
|
||||
run(["--clusters", str(clusters), "--out", str(out), "--source-root", str(root),
|
||||
"--prefix-parent"], monkeypatch)
|
||||
|
||||
manifest = json.loads((out / "manifest.json").read_text("utf-8"))
|
||||
assert sorted(manifest["categories"]) == sorted(categories)
|
||||
assert manifest["prefix_parent"] is True
|
||||
assert manifest["feature_source"]["backbone"]["arch"] == "resnet101"
|
||||
total = sum(manifest["category_sizes"].values())
|
||||
assert total == sum(len(v) for v in categories.values())
|
||||
for entries in manifest["files"].values():
|
||||
for entry in entries:
|
||||
source = Path(entry["source"])
|
||||
assert entry["file"] == f"{source.parent.name}_{source.name}"
|
||||
assert Path(entry["source"]).is_file()
|
||||
|
||||
|
||||
def test_without_prefix_names_are_unchanged(cluster_run, tmp_path, monkeypatch):
|
||||
_, work_dir, files = cluster_run
|
||||
data_root = tmp_path / "data"
|
||||
run(["--work-dir", str(work_dir), "--data-root", str(data_root)], monkeypatch)
|
||||
def test_dry_run_and_overwrite_guard(curated, tmp_path, monkeypatch):
|
||||
root, clusters, _ = curated
|
||||
out = tmp_path / "out"
|
||||
argv = ["--clusters", str(clusters), "--out", str(out), "--source-root", str(root),
|
||||
"--prefix-parent"]
|
||||
|
||||
exported = {p.name for p in (data_root / "film_r18_cluster_k2").rglob("*.png")}
|
||||
assert exported == {f.name for f in files}
|
||||
assert run([*argv, "--dry-run"], monkeypatch) == 0
|
||||
assert not out.exists()
|
||||
|
||||
assert run(argv, monkeypatch) == 0
|
||||
assert run(argv, monkeypatch) == 1 # 已存在且非空
|
||||
assert run([*argv, "--overwrite"], monkeypatch) == 0
|
||||
|
||||
|
||||
def test_prefix_parent_resolves_same_name_across_folders(cluster_run, tmp_path, monkeypatch):
|
||||
"""不同子目录的同名文件，不加前缀会撞车报错，加了前缀就能共存。"""
|
||||
source, work_dir, _ = cluster_run
|
||||
twin = source / "wrinkle" / "grease_0.png" # 与 grease/grease_0.png 同名
|
||||
Image.fromarray(np.zeros((20, 30, 3), dtype=np.uint8)).save(twin)
|
||||
def test_default_out_dir_uses_work_dir_name(curated, tmp_path, monkeypatch):
|
||||
root, clusters, _ = curated
|
||||
data_root = tmp_path / "exported"
|
||||
assert run(["--work-dir", str(clusters.parent), "--data-root", str(data_root),
|
||||
"--source-root", str(root), "--prefix-parent"], monkeypatch) == 0
|
||||
assert (data_root / clusters.parent.name).is_dir()
|
||||
|
||||
rows = list(csv.DictReader((work_dir / "assignments.csv").open(encoding="utf-8")))
|
||||
with (work_dir / "assignments.csv").open("w", newline="", encoding="utf-8") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=list(rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
writer.writerow({**rows[0], "path": str(twin), "cluster": rows[0]["cluster"]})
|
||||
|
||||
data_root = tmp_path / "data"
|
||||
argv = ["--work-dir", str(work_dir), "--data-root", str(data_root)]
|
||||
assert run(argv, monkeypatch) == 1 # 不加前缀：撞车
|
||||
assert run([*argv, "--prefix-parent"], monkeypatch) == 0 # 加前缀：共存
|
||||
def test_resolver_falls_back_to_basename_index(curated, tmp_path):
|
||||
"""链接目标里没有 data/ 段时，用文件名在源目录里兜底查找。"""
|
||||
root, clusters, _ = curated
|
||||
resolver = ec.SourceResolver(root)
|
||||
ec.logger = ec.setup_logging()
|
||||
|
||||
out = data_root / "film_r18_cluster_k2"
|
||||
assert {"grease_grease_0.png", "wrinkle_grease_0.png"} <= {p.name for p in out.rglob("*.png")}
|
||||
odd = clusters / "grease" / "0100_odd.png"
|
||||
target = sorted((root / "data" / "raw" / "film" / "grease").glob("*.png"))[0]
|
||||
odd.symlink_to(f"/elsewhere/{target.name}") # 路径里没有 data/ 段
|
||||
assert resolver.resolve(odd) == target.resolve()
|
||||
|
||||
|
||||
def test_missing_clusters_dir_is_reported(tmp_path, monkeypatch):
|
||||
assert run(["--clusters", str(tmp_path / "nope"), "--out", str(tmp_path / "o")], monkeypatch) == 1
|
||||
|
||||
|
||||
def test_destination_name_helper():
|
||||
assert ec.destination_name(Path("data/raw/film/grease/abc.png")) == "abc.png"
|
||||
assert ec.destination_name(Path("data/raw/film/grease/abc.png"), True) == "grease_abc.png"
|
||||
|
||||
|
||||
def test_cluster_dirs_are_numbered_from_zero(cluster_run, tmp_path, monkeypatch):
|
||||
_, work_dir, files = cluster_run
|
||||
data_root = tmp_path / "data"
|
||||
run(["--work-dir", str(work_dir), "--data-root", str(data_root)], monkeypatch)
|
||||
|
||||
out = data_root / "film_r18_cluster_k2"
|
||||
assert (out / "0").is_dir() and (out / "1").is_dir()
|
||||
assert len(list((out / "0").glob("*.png"))) + len(list((out / "1").glob("*.png"))) == len(files)
|
||||
|
||||
|
||||
def test_manifest_records_provenance(cluster_run, tmp_path, monkeypatch):
|
||||
_, work_dir, files = cluster_run
|
||||
data_root = tmp_path / "data"
|
||||
run(["--work-dir", str(work_dir), "--data-root", str(data_root)], monkeypatch)
|
||||
|
||||
manifest = json.loads((data_root / "film_r18_cluster_k2" / "manifest.json").read_text("utf-8"))
|
||||
assert manifest["num_clusters"] == 2
|
||||
assert manifest["feature_source"]["backbone"]["arch"] == "resnet18"
|
||||
assert sum(manifest["cluster_sizes"].values()) == len(files)
|
||||
# 原图路径可追溯
|
||||
recorded = {e["source"] for entries in manifest["files"].values() for e in entries}
|
||||
assert recorded == {str(f) for f in files}
|
||||
|
||||
|
||||
def test_tag_and_out_overrides(cluster_run, tmp_path, monkeypatch):
|
||||
_, work_dir, _ = cluster_run
|
||||
data_root = tmp_path / "data"
|
||||
|
||||
run(["--work-dir", str(work_dir), "--data-root", str(data_root), "--tag", "r18ss"], monkeypatch)
|
||||
assert (data_root / "film_r18ss_cluster_k2").is_dir()
|
||||
|
||||
explicit = tmp_path / "somewhere" / "my_dataset"
|
||||
run(["--work-dir", str(work_dir), "--out", str(explicit)], monkeypatch)
|
||||
assert explicit.is_dir()
|
||||
|
||||
|
||||
def test_dry_run_writes_nothing(cluster_run, tmp_path, monkeypatch):
|
||||
_, work_dir, _ = cluster_run
|
||||
data_root = tmp_path / "data"
|
||||
assert run(["--work-dir", str(work_dir), "--data-root", str(data_root), "--dry-run"],
|
||||
monkeypatch) == 0
|
||||
assert not data_root.exists()
|
||||
|
||||
|
||||
def test_refuses_to_overwrite_unless_asked(cluster_run, tmp_path, monkeypatch):
|
||||
_, work_dir, _ = cluster_run
|
||||
data_root = tmp_path / "data"
|
||||
argv = ["--work-dir", str(work_dir), "--data-root", str(data_root)]
|
||||
|
||||
assert run(argv, monkeypatch) == 0
|
||||
assert run(argv, monkeypatch) == 1 # 已存在且非空 -> 拒绝
|
||||
assert run([*argv, "--overwrite"], monkeypatch) == 0 # 明确要求才覆盖
|
||||
|
||||
|
||||
def test_duplicate_basenames_in_one_cluster_are_rejected(cluster_run, tmp_path, monkeypatch):
|
||||
"""要求保留原文件名，同簇重名就会互相覆盖，必须报错而不是悄悄丢文件。"""
|
||||
source, work_dir, _ = cluster_run
|
||||
twin = source / "wrinkle" / "grease_0.png" # 与 grease/grease_0.png 同名
|
||||
Image.fromarray(np.zeros((20, 30, 3), dtype=np.uint8)).save(twin)
|
||||
|
||||
rows = list(csv.DictReader((work_dir / "assignments.csv").open(encoding="utf-8")))
|
||||
with (work_dir / "assignments.csv").open("w", newline="", encoding="utf-8") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=list(rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
writer.writerow({**rows[0], "path": str(twin), "cluster": rows[0]["cluster"]})
|
||||
|
||||
assert run(["--work-dir", str(work_dir), "--data-root", str(tmp_path / "data")], monkeypatch) == 1
|
||||
|
||||
|
||||
def test_missing_source_file_is_reported(cluster_run, tmp_path, monkeypatch):
|
||||
_, work_dir, files = cluster_run
|
||||
files[0].unlink()
|
||||
assert run(["--work-dir", str(work_dir), "--data-root", str(tmp_path / "data")], monkeypatch) == 1
|
||||
|
||||
|
||||
def test_backbone_tag_mapping():
|
||||
assert ec.backbone_tag({"feature_source": {"backbone": {"arch": "resnet18"}}}) == "r18"
|
||||
assert ec.backbone_tag({"feature_source": {"backbone": {"name": "tiny_conv"}}}) == "tiny"
|
||||
assert ec.backbone_tag({"feature_source": {"backbone": {"arch": "shufflenet_v2_x1_0"}}}) == "sfv2"
|
||||
assert ec.backbone_tag({}) == "unknown"
|
||||
|
||||
Reference in New Issue
Block a user