feat(cluster): clusters/ 默认改用硬链接,便于直接在 runs/ 下继续分子类
软链接记的是绝对路径,从服务器拷到本机后全部失效,也不便于直接在 runs/ 下整理。 硬链接不占额外磁盘却表现得和真实文件一样:看图软件能开、可以随意挪动、建子目录 继续分子类,整个目录拷到别的机器也不会断链。 - 新增 link_mode: hard(默认)/ sym / copy;跨文件系统时 hard 自动退回软链并警告 - 落盘后打印各方式的条目数,便于确认实际用的是哪种 - export_clusters.py 适配硬链接:条目本身就是原图,但路径在 runs/ 下、名字还带 0125_ 排序前缀。按 inode 反查 assignments.csv 还原原图路径,这样导出的文件名与 --prefix-parent 才能拿到真正的来源目录;查不到则剥掉前缀后用文件名索引兜底 实测:14,579 张跑聚类后 clusters/ 全为硬链接(nlink=2、可直接打开); 把簇改名、建子目录挪图后再导出,排序前缀已剥除、来源目录正确还原为「反面纵黑点」, 120 张内容与原图逐字节一致。 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
f8654f447b
commit
33d28dcc7c
@@ -20,10 +20,12 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import errno
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
@@ -159,10 +161,14 @@ def write_outputs(cfg, work_dir, metas, result, num_clusters, stability):
|
||||
|
||||
link_limit = int(cfg.get("link_limit", 300))
|
||||
preview_count = int(cfg.get("preview_count", 25))
|
||||
link_mode = str(cfg.get("link_mode", "hard")).lower()
|
||||
if link_mode not in LINK_MODES:
|
||||
raise ValueError(f"未知 link_mode: {link_mode},可选 {LINK_MODES}")
|
||||
clusters_dir = work_dir / "clusters"
|
||||
if clusters_dir.exists():
|
||||
shutil.rmtree(clusters_dir)
|
||||
|
||||
used_modes: Counter[str] = Counter()
|
||||
order = sorted(range(len(metas)), key=lambda i: distances[i]) # 离质心近的排前面
|
||||
per_cluster: dict[int, list[int]] = defaultdict(list)
|
||||
for index in order:
|
||||
@@ -177,9 +183,19 @@ def write_outputs(cfg, work_dir, metas, result, num_clusters, stability):
|
||||
source = Path(metas[index]["path"]).resolve()
|
||||
link = target / f"{rank:04d}_{source.stem}{source.suffix}"
|
||||
if not link.exists():
|
||||
link.symlink_to(source)
|
||||
used_modes[make_link(source, link, link_mode)] += 1
|
||||
save_preview(work_dir / f"cluster_{cluster:02d}_preview.png", metas, indices[:preview_count])
|
||||
|
||||
if used_modes:
|
||||
detail = "、".join(f"{mode} {count}" for mode, count in sorted(used_modes.items()))
|
||||
logger.info("clusters/ 下的条目:%s", detail)
|
||||
if link_mode == "hard" and used_modes.get("sym"):
|
||||
logger.warning(
|
||||
"有 %d 个条目跨文件系统、退回了软链——这些在别的机器上会断链,"
|
||||
"可改用 link_mode: copy,或让 runs/ 与数据同盘",
|
||||
used_modes["sym"],
|
||||
)
|
||||
|
||||
if cfg.get("scatter", {}).get("enabled", True) and result.points is not None:
|
||||
scatter_path = work_dir / "cluster_scatter.png"
|
||||
save_scatter(scatter_path, result, cfg)
|
||||
@@ -380,6 +396,38 @@ def warn_misplaced_keys(cfg, method: str) -> None:
|
||||
logger.info("当前 method=%s,`simclr:` 段下的配置不会生效", method)
|
||||
|
||||
|
||||
LINK_MODES = ("hard", "sym", "copy")
|
||||
|
||||
|
||||
def make_link(source: Path, destination: Path, mode: str) -> str:
|
||||
"""在 clusters/ 里放一份指向原图的条目,返回实际用的方式。
|
||||
|
||||
默认硬链接:不占额外磁盘,但表现得和真实文件一模一样——看图软件、文件管理器、
|
||||
以及在 runs/ 下直接继续分子类的工具都能正常处理,把整个目录拷到别的机器上
|
||||
也不会变成断链(软链会,因为它记的是绝对路径)。
|
||||
|
||||
代价是硬链接必须与源文件同一个文件系统;跨盘时自动降级并说明原因。
|
||||
另外硬链接和原图是同一份数据,**在 clusters/ 里改图会改到原图**,只挪动、
|
||||
删除条目则不影响原图。
|
||||
"""
|
||||
if mode == "sym":
|
||||
destination.symlink_to(source)
|
||||
return "sym"
|
||||
if mode == "copy":
|
||||
shutil.copy2(source, destination)
|
||||
return "copy"
|
||||
|
||||
try:
|
||||
os.link(source, destination)
|
||||
return "hard"
|
||||
except OSError as exc:
|
||||
if exc.errno != errno.EXDEV:
|
||||
raise
|
||||
# 跨文件系统只能退回软链;这时拷到别的机器上会断链,用 export_clusters.py 重定位
|
||||
destination.symlink_to(source)
|
||||
return "sym"
|
||||
|
||||
|
||||
def sweep_clusters(embeddings, spec: str, cluster_kwargs: dict) -> None:
|
||||
"""扫一遍簇数,同时看轮廓系数和换种子的稳定性。
|
||||
|
||||
@@ -527,7 +575,7 @@ def main() -> int:
|
||||
torch.save({"embeddings": embeddings, "metas": metas}, work_dir / "features.pt")
|
||||
|
||||
write_outputs(cfg, work_dir, metas, result, num_clusters, stability)
|
||||
logger.info("结果已写入 %s(clusters/ 下是按簇建的软链目录,直接翻图即可核对)", work_dir)
|
||||
logger.info("结果已写入 %s(clusters/ 下按簇分好,直接翻图即可核对与调整)", work_dir)
|
||||
for index, size in enumerate(result.sizes):
|
||||
mean_distance = float(result.distances[result.labels == index].mean()) if size else float("nan")
|
||||
logger.info(" cluster_%02d %6d 张 平均距质心 %.3f", index, size, mean_distance)
|
||||
|
||||
Reference in New Issue
Block a user