feat(cluster): 输出簇分布图，直观展示簇间距离
在 preview 拼图的同级目录生成 cluster_scatter.png： - 左：聚类空间的 PCA 二维投影，散点按簇着色 + 质心标记，标题标注两个主成分的方差占比 - 右：质心两两距离矩阵（在聚类空间里算，不是投影后），带数值标注 投影用 PCA 而不是 t-SNE/UMAP——后两者为了把簇拉开会扭曲簇间距离， 而这张图的目的就是看距离。 - ClusterResult 新增 points 字段，保留 KMeans 实际使用的空间供可视化复用， 质心与散点用同一组主成分变换，避免质心飘到簇外 - summary.json 新增 centroid_distances - 新增 scatter 配置块（enabled / max_points / point_size / alpha / figsize / dpi） - 中文字体按 PingFang / Noto Sans CJK 等顺序探测，找不到自动退回英文标签， 不会画成豆腐块 - 新增依赖 matplotlib Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Shaun Zhou
co-authored by
Claude Opus 5
Claude Opus 5
parent
4e6c8aa2ea
commit
e4bd794025
@@ -34,9 +34,11 @@ from datasets.build import build_dataset # noqa: E402
|
||||
from datasets.unlabeled import unlabeled_collate # noqa: E402
|
||||
from engine.cluster import ( # noqa: E402
|
||||
StageFeatureExtractor,
|
||||
centroid_distances,
|
||||
cluster_features,
|
||||
cluster_stability,
|
||||
extract_features,
|
||||
project_2d,
|
||||
random_fit_index,
|
||||
)
|
||||
from models.registry import BACKBONES # noqa: E402
|
||||
@@ -133,6 +135,7 @@ def write_outputs(cfg, work_dir, metas, result, num_clusters, stability):
|
||||
float(result.distances[result.labels == index].mean()) if size else None
|
||||
for index, size in enumerate(result.sizes)
|
||||
],
|
||||
"centroid_distances": centroid_distances(result.centroids).tolist(),
|
||||
}
|
||||
(work_dir / "summary.json").write_text(
|
||||
json.dumps(summary, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
@@ -161,9 +164,160 @@ def write_outputs(cfg, work_dir, metas, result, num_clusters, stability):
|
||||
link.symlink_to(source)
|
||||
save_preview(work_dir / f"cluster_{cluster:02d}_preview.png", metas, indices[:preview_count])
|
||||
|
||||
if cfg.get("scatter", {}).get("enabled", True) and result.points is not None:
|
||||
scatter_path = work_dir / "cluster_scatter.png"
|
||||
save_scatter(scatter_path, result, cfg)
|
||||
logger.info("簇分布图已保存: %s", scatter_path)
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
#: 按优先级尝试的中文字体（macOS / Linux 常见）。都找不到就退回英文标签，避免画出豆腐块。
|
||||
CJK_FONTS = (
|
||||
"PingFang SC",
|
||||
"Hiragino Sans GB",
|
||||
"Heiti SC",
|
||||
"Noto Sans CJK SC",
|
||||
"Source Han Sans SC",
|
||||
"WenQuanYi Zen Hei",
|
||||
"Microsoft YaHei",
|
||||
"SimHei",
|
||||
"Arial Unicode MS",
|
||||
)
|
||||
|
||||
|
||||
def setup_cjk_font() -> bool:
|
||||
"""给 matplotlib 配一个能显示中文的字体，返回是否配上了。"""
|
||||
import matplotlib
|
||||
from matplotlib import font_manager
|
||||
|
||||
available = {f.name for f in font_manager.fontManager.ttflist}
|
||||
for name in CJK_FONTS:
|
||||
if name in available:
|
||||
matplotlib.rcParams["font.sans-serif"] = [name, *matplotlib.rcParams["font.sans-serif"]]
|
||||
matplotlib.rcParams["axes.unicode_minus"] = False # 中文字体里没有 U+2212
|
||||
return True
|
||||
logger.warning("未找到中文字体（尝试过 %s），图上改用英文标签", "/".join(CJK_FONTS[:3]))
|
||||
return False
|
||||
|
||||
|
||||
def save_scatter(path: Path, result, cfg) -> None:
|
||||
"""把聚类结果画成二维散点 + 质心距离矩阵，放在 preview 拼图的同级目录。
|
||||
|
||||
投影用 PCA 而不是 t-SNE/UMAP：后两者为了把簇拉开会扭曲簇间距离，
|
||||
而这张图的目的就是看簇之间离多远。图上标注了两个主成分的方差占比——
|
||||
占比低说明高维结构被压掉得多，图上的远近只能参考，以右边的距离矩阵为准。
|
||||
"""
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg") # 无显示环境
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
cjk = setup_cjk_font()
|
||||
|
||||
scatter_cfg = dict(cfg.get("scatter", {}))
|
||||
max_points = int(scatter_cfg.get("max_points", 5000))
|
||||
points, centroids, ratio = project_2d(result.points, result.centroids)
|
||||
labels = result.labels
|
||||
|
||||
# 点太多时随机抽样：既避免糊成一团，也控制文件大小
|
||||
if max_points and points.shape[0] > max_points:
|
||||
generator = torch.Generator().manual_seed(int(cfg.get("seed", 42)))
|
||||
index = torch.randperm(points.shape[0], generator=generator)[:max_points]
|
||||
points, labels = points[index], labels[index]
|
||||
subtitle = f"随机抽样 {max_points} / {result.labels.shape[0]} 点"
|
||||
subtitle_en = f"sampled {max_points} / {result.labels.shape[0]}"
|
||||
else:
|
||||
subtitle = f"全部 {points.shape[0]} 点"
|
||||
subtitle_en = f"all {points.shape[0]} points"
|
||||
|
||||
points = points.cpu().numpy()
|
||||
centroids_2d = centroids.cpu().numpy()
|
||||
labels = labels.cpu().numpy()
|
||||
num_clusters = result.centroids.shape[0]
|
||||
colors = plt.get_cmap("tab10")(range(num_clusters))
|
||||
|
||||
figsize = scatter_cfg.get("figsize", [14, 6])
|
||||
fig, (ax, ax_matrix) = plt.subplots(
|
||||
1, 2, figsize=tuple(figsize), gridspec_kw={"width_ratios": [1.6, 1]}
|
||||
)
|
||||
|
||||
for cluster in range(num_clusters):
|
||||
mask = labels == cluster
|
||||
ax.scatter(
|
||||
points[mask, 0],
|
||||
points[mask, 1],
|
||||
s=float(scatter_cfg.get("point_size", 6)),
|
||||
alpha=float(scatter_cfg.get("alpha", 0.35)),
|
||||
color=colors[cluster],
|
||||
linewidths=0,
|
||||
label=f"cluster_{cluster:02d} n={int(result.sizes[cluster])}",
|
||||
)
|
||||
ax.scatter(
|
||||
centroids_2d[:, 0],
|
||||
centroids_2d[:, 1],
|
||||
s=220,
|
||||
marker="X",
|
||||
c=colors,
|
||||
edgecolors="black",
|
||||
linewidths=1.5,
|
||||
zorder=5,
|
||||
)
|
||||
for cluster in range(num_clusters):
|
||||
ax.annotate(
|
||||
f"{cluster:02d}",
|
||||
centroids_2d[cluster],
|
||||
textcoords="offset points",
|
||||
xytext=(0, 12),
|
||||
fontsize=11,
|
||||
fontweight="bold",
|
||||
ha="center",
|
||||
zorder=6,
|
||||
color="black",
|
||||
bbox={"boxstyle": "round,pad=0.2", "fc": "white", "ec": "none", "alpha": 0.8},
|
||||
)
|
||||
|
||||
if cjk:
|
||||
scatter_title = (
|
||||
f"PCA 二维投影（{subtitle}）\n"
|
||||
f"两个主成分方差占比 {ratio[0]:.1%} + {ratio[1]:.1%} = {sum(ratio):.1%}"
|
||||
)
|
||||
matrix_title = "质心两两距离（聚类空间，非投影后）"
|
||||
else:
|
||||
scatter_title = (
|
||||
f"PCA 2D projection ({subtitle_en})\n"
|
||||
f"explained variance {ratio[0]:.1%} + {ratio[1]:.1%} = {sum(ratio):.1%}"
|
||||
)
|
||||
matrix_title = "pairwise centroid distance (in cluster space)"
|
||||
ax.set_title(scatter_title)
|
||||
ax.set_xlabel("PC1")
|
||||
ax.set_ylabel("PC2")
|
||||
ax.legend(loc="best", fontsize=8, framealpha=0.9)
|
||||
ax.grid(alpha=0.2)
|
||||
ax.set_aspect("equal", adjustable="datalim")
|
||||
|
||||
distances = centroid_distances(result.centroids).cpu().numpy()
|
||||
image = ax_matrix.imshow(distances, cmap="viridis")
|
||||
ax_matrix.set_title(matrix_title)
|
||||
ax_matrix.set_xticks(range(num_clusters))
|
||||
ax_matrix.set_yticks(range(num_clusters))
|
||||
ax_matrix.set_xticklabels([f"{i:02d}" for i in range(num_clusters)])
|
||||
ax_matrix.set_yticklabels([f"{i:02d}" for i in range(num_clusters)])
|
||||
for i in range(num_clusters):
|
||||
for j in range(num_clusters):
|
||||
ax_matrix.text(
|
||||
j, i, f"{distances[i, j]:.2f}",
|
||||
ha="center", va="center",
|
||||
color="white" if distances[i, j] < distances.max() * 0.6 else "black",
|
||||
fontsize=9,
|
||||
)
|
||||
fig.colorbar(image, ax=ax_matrix, fraction=0.046)
|
||||
|
||||
fig.tight_layout()
|
||||
fig.savefig(path, dpi=int(scatter_cfg.get("dpi", 150)))
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def save_preview(path: Path, metas: list[dict], indices: list[int], cell: int = 96, columns: int = 5):
|
||||
"""把代表样本拼成一张图，方便一眼看出这个簇是什么缺陷。"""
|
||||
if not indices:
|
||||
|
||||
Reference in New Issue
Block a user