feat(cluster): clusters/ 默认改用硬链接,便于直接在 runs/ 下继续分子类
软链接记的是绝对路径,从服务器拷到本机后全部失效,也不便于直接在 runs/ 下整理。 硬链接不占额外磁盘却表现得和真实文件一样:看图软件能开、可以随意挪动、建子目录 继续分子类,整个目录拷到别的机器也不会断链。 - 新增 link_mode: hard(默认)/ sym / copy;跨文件系统时 hard 自动退回软链并警告 - 落盘后打印各方式的条目数,便于确认实际用的是哪种 - export_clusters.py 适配硬链接:条目本身就是原图,但路径在 runs/ 下、名字还带 0125_ 排序前缀。按 inode 反查 assignments.csv 还原原图路径,这样导出的文件名与 --prefix-parent 才能拿到真正的来源目录;查不到则剥掉前缀后用文件名索引兜底 实测:14,579 张跑聚类后 clusters/ 全为硬链接(nlink=2、可直接打开); 把簇改名、建子目录挪图后再导出,排序前缀已剥除、来源目录正确还原为「反面纵黑点」, 120 张内容与原图逐字节一致。 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
f8654f447b
commit
33d28dcc7c
@@ -115,6 +115,40 @@ def test_prefix_parent_resolves_cross_dataset_collisions(curated, tmp_path, monk
|
||||
assert {"blue_7.png", "pearlescent_7.png"} <= names
|
||||
|
||||
|
||||
def test_hard_links_are_traced_back_to_the_original(curated, tmp_path, monkeypatch):
|
||||
"""硬链接条目本身就是原图,但路径在 runs/ 下、名字还带排序前缀。
|
||||
应当按 inode 反查 assignments.csv 还原出原图路径,
|
||||
这样文件名和 --prefix-parent 才能拿到真正的来源目录。"""
|
||||
import csv as csv_mod
|
||||
import os as os_mod
|
||||
|
||||
root, clusters, categories = curated
|
||||
# 新建一张原图(不能复用已在该类别里的,否则和已有条目重名)
|
||||
original = root / "data" / "raw" / "film" / "grease" / "extra.png"
|
||||
Image.fromarray(np.full((12, 16, 3), 3, dtype=np.uint8)).save(original)
|
||||
|
||||
# 造一个硬链接条目 + 对应的 assignments.csv
|
||||
hard = clusters / "grease" / f"0500_{original.name}"
|
||||
os_mod.link(original, hard)
|
||||
with (clusters.parent / "assignments.csv").open("w", newline="", encoding="utf-8") as handle:
|
||||
writer = csv_mod.writer(handle)
|
||||
writer.writerow(["path", "cluster"])
|
||||
for sources in list(categories.values()) + [[original]]:
|
||||
for source in sources:
|
||||
writer.writerow([str(source), 0])
|
||||
|
||||
out = tmp_path / "out"
|
||||
assert run(["--work-dir", str(clusters.parent), "--out", str(out),
|
||||
"--source-root", str(root), "--prefix-parent"], monkeypatch) == 0
|
||||
|
||||
manifest = json.loads((out / "manifest.json").read_text("utf-8"))
|
||||
entries = {e["file"]: e["source"] for e in manifest["files"]["grease"]}
|
||||
expected = f"{original.parent.name}_{original.name}"
|
||||
assert expected in entries, f"应还原成 {expected},实际有 {sorted(entries)}"
|
||||
assert entries[expected] == str(original.resolve()) # 指向原图而不是 runs/ 下的条目
|
||||
assert not any(name[:4].isdigit() and name[4] == "_" for name in entries)
|
||||
|
||||
|
||||
def test_real_files_in_clusters_are_used_directly(curated, tmp_path, monkeypatch):
|
||||
"""有人直接把真实图片拷进 clusters/ 时也要能用。"""
|
||||
root, clusters, _ = curated
|
||||
|
||||
@@ -657,6 +657,68 @@ def test_train_only_conflicts_with_sweep_k(tiny_image_dir, tmp_path, monkeypatch
|
||||
assert cu.main() == 1
|
||||
|
||||
|
||||
def test_clusters_use_hard_links_by_default(tiny_image_dir, tmp_path, monkeypatch):
|
||||
"""clusters/ 里默认放硬链接:表现得像真实文件,拷到别的机器也不会断链。"""
|
||||
sys.path.insert(0, "scripts")
|
||||
import cluster_unlabeled as cu
|
||||
|
||||
work_dir = tmp_path / "run"
|
||||
monkeypatch.setattr(sys, "argv", _cluster_argv(work_dir, tiny_image_dir))
|
||||
assert cu.main() == 0
|
||||
|
||||
entries = [p for p in (work_dir / "clusters").rglob("*") if p.is_file() or p.is_symlink()]
|
||||
assert entries
|
||||
assert not any(p.is_symlink() for p in entries), "默认不该是软链"
|
||||
assert all(p.stat().st_nlink > 1 for p in entries), "应当与原图共享 inode"
|
||||
|
||||
|
||||
def test_link_mode_sym_and_copy(tiny_image_dir, tmp_path, monkeypatch):
|
||||
sys.path.insert(0, "scripts")
|
||||
import cluster_unlabeled as cu
|
||||
|
||||
for mode, expect_symlink in (("sym", True), ("copy", False)):
|
||||
work_dir = tmp_path / f"run_{mode}"
|
||||
monkeypatch.setattr(
|
||||
sys, "argv", _cluster_argv(work_dir, tiny_image_dir, f"link_mode={mode}")
|
||||
)
|
||||
assert cu.main() == 0
|
||||
entries = [p for p in (work_dir / "clusters").rglob("*") if p.is_file() or p.is_symlink()]
|
||||
assert all(p.is_symlink() == expect_symlink for p in entries), mode
|
||||
if mode == "copy":
|
||||
assert all(p.stat().st_nlink == 1 for p in entries), "复制出来的不该共享 inode"
|
||||
|
||||
|
||||
def test_unknown_link_mode_is_rejected(tiny_image_dir, tmp_path, monkeypatch):
|
||||
sys.path.insert(0, "scripts")
|
||||
import cluster_unlabeled as cu
|
||||
|
||||
monkeypatch.setattr(
|
||||
sys, "argv", _cluster_argv(tmp_path / "bad", tiny_image_dir, "link_mode=hardlink")
|
||||
)
|
||||
with pytest.raises(ValueError, match="未知 link_mode"):
|
||||
cu.main()
|
||||
|
||||
|
||||
def test_make_link_falls_back_to_symlink_across_filesystems(tmp_path, monkeypatch):
|
||||
"""跨文件系统时硬链接会失败(EXDEV),应当退回软链而不是崩掉。"""
|
||||
import errno
|
||||
import os as os_mod
|
||||
|
||||
sys.path.insert(0, "scripts")
|
||||
import cluster_unlabeled as cu
|
||||
|
||||
source = tmp_path / "src.png"
|
||||
source.write_bytes(b"data")
|
||||
|
||||
def fake_link(*_args, **_kwargs):
|
||||
raise OSError(errno.EXDEV, "Cross-device link")
|
||||
|
||||
monkeypatch.setattr(os_mod, "link", fake_link)
|
||||
destination = tmp_path / "dst.png"
|
||||
assert cu.make_link(source, destination, "hard") == "sym"
|
||||
assert destination.is_symlink()
|
||||
|
||||
|
||||
def test_scatter_plot_is_written_next_to_previews(tiny_image_dir, tmp_path, monkeypatch):
|
||||
"""cluster_scatter.png 要和 preview 拼图落在同一级目录。"""
|
||||
sys.path.insert(0, "scripts")
|
||||
|
||||
Reference in New Issue
Block a user