Shaun ZhouandClaude Opus 5
92c88b785c
用户反馈"聚类配置里改不了 lr/weight_decay"，实测是两件事：
1. --set 的键打错字母会凭空造出一个谁也不读的键，训练照跑但改动无效。
实例：simclr.optmizer.weight_decay=0.0（optimizer 少个 i）被静默接受。
现在 apply_overrides 默认只允许覆盖已存在的键，报错时给出同级可用键
与最接近的拼写（"是不是想写 'optimizer'？"）。需要新增键时可 strict=False。
2. lr 其实生效了，但从 2229/2500 续训时 cosine 只剩满值的 3%，
新设的 1e-3 落到 2.87e-5。新增 restart_schedule 开关：按剩余轮数重开曲线，
新 lr 从满值开始；SimCLR 与检测训练都支持。续训日志补上调度进度，
并在 lr 不足 initial_lr 的 10% 时警告并给出两种解法。
顺带：严格校验立刻暴露出 configs/cluster/film.yaml 里根本没有 scatter 段
（代码 cfg.get("scatter", {}) 兜底走默认值，所以功能正常但配置项形同虚设），已补上。
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
391 lines
17 KiB
Python
391 lines
17 KiB
Python
"""SimCLR 自监督预训练：在无标注的薄膜图上学一套特征，供聚类使用。
|
||
|
||
骨架仍然是项目里的纯卷积 backbone（`tiny_conv` 等），预训练完可以直接拿去
|
||
初始化检测模型。支持 cuda / mps / cpu，用 `device: auto` 自动挑最快的后端。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import time
|
||
import warnings
|
||
from collections.abc import Mapping
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import torch
|
||
from torch import nn
|
||
from torch.nn import functional as F
|
||
from torch.utils.data import DataLoader
|
||
|
||
from engine.metrics import AverageMeter
|
||
from engine.optim import build_optimizer, build_scheduler
|
||
from engine.visualizer import SwanLabLogger
|
||
from models.heads.projection import ProjectionHead
|
||
from utils.device import PeakMemory, resolve_amp, synchronize
|
||
from utils.logging import get_logger
|
||
|
||
logger = get_logger("engine.simclr")
|
||
|
||
|
||
def nt_xent_loss(z1: torch.Tensor, z2: torch.Tensor, temperature: float = 0.2) -> torch.Tensor:
|
||
"""NT-Xent：同一张图的两个视图互为正样本，batch 内其余 2N-2 个都是负样本。"""
|
||
batch_size = z1.shape[0]
|
||
z = F.normalize(torch.cat([z1, z2]), dim=1)
|
||
similarity = z @ z.t() / temperature
|
||
similarity.fill_diagonal_(float("-inf")) # 排除自己和自己
|
||
|
||
targets = torch.cat(
|
||
[
|
||
torch.arange(batch_size, 2 * batch_size, device=z.device),
|
||
torch.arange(0, batch_size, device=z.device),
|
||
]
|
||
)
|
||
return F.cross_entropy(similarity, targets)
|
||
|
||
|
||
class SimCLRTrainer:
|
||
"""Args:
|
||
backbone: 输出特征图的卷积骨架（需带 `out_channels`）。
|
||
dataloader: 每个样本产出 2 个视图的 DataLoader（`UnlabeledImageDataset(views=2)`）。
|
||
cfg: `epochs / temperature / optimizer / scheduler / amp / log_interval / save_interval`
|
||
以及 `swanlab`（可视化，字段同 `train.swanlab`）。
|
||
full_config: 记录到 SwanLab 的超参，一般直接传完整 yaml 配置。
|
||
resume: 训练状态文件（`simclr_last.pth`）路径，从上次中断处接着练。
|
||
与 `simclr.checkpoint` 不同：checkpoint 是"有权重就跳过训练直接聚类"，
|
||
resume 是"载入骨架 + 投影头 + 优化器 + 调度器 + 起始 epoch 继续训练"。
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
backbone: nn.Module,
|
||
dataloader: DataLoader,
|
||
cfg: Mapping[str, Any],
|
||
device: torch.device,
|
||
work_dir: str | Path,
|
||
full_config: dict | None = None,
|
||
resume: str | Path | None = None,
|
||
) -> None:
|
||
self.backbone = backbone
|
||
self.dataloader = dataloader
|
||
self.cfg = dict(cfg)
|
||
self.device = device
|
||
self.work_dir = Path(work_dir)
|
||
self.work_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
self.epochs = int(self.cfg.get("epochs", 50))
|
||
self.temperature = float(self.cfg.get("temperature", 0.2))
|
||
self.log_interval = int(self.cfg.get("log_interval", 20))
|
||
# 每隔若干 epoch 落一次盘：自监督动辄跑几十分钟，中途被打断不该把训练成果全丢掉
|
||
self.save_interval = int(self.cfg.get("save_interval", 5))
|
||
|
||
self.projector = ProjectionHead(
|
||
backbone.out_channels, **dict(self.cfg.get("projection", {}))
|
||
).to(device)
|
||
self.model = nn.ModuleDict({"backbone": backbone, "projector": self.projector})
|
||
self.optimizer = build_optimizer(
|
||
self.model, self.cfg.get("optimizer", {"name": "adamw", "lr": 1e-3, "weight_decay": 1e-4})
|
||
)
|
||
self.scheduler = build_scheduler(self.optimizer, self.cfg.get("scheduler"), self.epochs)
|
||
self.amp = resolve_amp(self.cfg.get("amp", "auto"), device)
|
||
self.scaler = torch.amp.GradScaler(device.type, enabled=self.amp)
|
||
self.peak_memory = PeakMemory(device)
|
||
|
||
self.global_step = 0
|
||
self.start_epoch = 0
|
||
# 无论是否续训都打印一次实际生效的超参：配置写错层级时最难发现的就是"改了没反应"
|
||
logger.info(
|
||
"SimCLR 超参：epochs=%d temperature=%.3g optimizer=%s scheduler=%s",
|
||
self.epochs,
|
||
self.temperature,
|
||
dict(self.cfg.get("optimizer", {})) or "(默认 adamw lr=1e-3)",
|
||
(dict(self.cfg.get("scheduler") or {})).get("name") or "(无)",
|
||
)
|
||
if resume:
|
||
self.load_state(resume)
|
||
|
||
self.visualizer = SwanLabLogger(
|
||
work_dir=self.work_dir,
|
||
config=full_config,
|
||
**dict(self.cfg.get("swanlab", {})),
|
||
)
|
||
|
||
def train_one_epoch(self, epoch: int) -> float:
|
||
self.model.train()
|
||
meter = AverageMeter()
|
||
start = time.time()
|
||
|
||
for step, batch in enumerate(self.dataloader):
|
||
view1, view2 = (view.to(self.device, non_blocking=True) for view in batch["views"][:2])
|
||
|
||
with torch.amp.autocast(self.device.type, enabled=self.amp):
|
||
z1 = self.projector(self.backbone(view1))
|
||
z2 = self.projector(self.backbone(view2))
|
||
loss = nt_xent_loss(z1, z2, self.temperature)
|
||
|
||
self.optimizer.zero_grad(set_to_none=True)
|
||
self.scaler.scale(loss).backward()
|
||
self.scaler.step(self.optimizer)
|
||
self.scaler.update()
|
||
|
||
self.peak_memory.update()
|
||
meter.update(float(loss.detach()), view1.shape[0])
|
||
self.global_step += 1
|
||
if step % self.log_interval == 0:
|
||
self.visualizer.log(
|
||
{
|
||
"loss": float(loss.detach()),
|
||
"loss_avg": meter.avg,
|
||
"lr": self.optimizer.param_groups[0]["lr"],
|
||
},
|
||
step=self.global_step,
|
||
prefix="simclr/",
|
||
)
|
||
logger.info(
|
||
"simclr epoch %d [%d/%d] loss=%.4f lr=%.2e",
|
||
epoch,
|
||
step,
|
||
len(self.dataloader),
|
||
meter.avg,
|
||
self.optimizer.param_groups[0]["lr"],
|
||
)
|
||
|
||
synchronize(self.device)
|
||
peak = self.peak_memory.peak_mb
|
||
elapsed = time.time() - start
|
||
self.visualizer.log(
|
||
{
|
||
"epoch": float(epoch),
|
||
"epoch_loss": meter.avg,
|
||
"epoch_seconds": elapsed,
|
||
"gpu_mem_peak_mb": peak,
|
||
},
|
||
step=self.global_step,
|
||
prefix="simclr/",
|
||
)
|
||
logger.info(
|
||
"simclr epoch %d 完成 loss=%.4f 耗时 %.1fs 显存峰值 %.0f MB%s",
|
||
epoch,
|
||
meter.avg,
|
||
elapsed,
|
||
peak,
|
||
"" if self.peak_memory.exact else "（MPS 采样下界）",
|
||
)
|
||
return meter.avg
|
||
|
||
def save_backbone(self, epoch: int) -> Path:
|
||
"""只存骨架权重，不含投影头——投影头训练成对增强不变，会丢掉下游有用的信息。
|
||
|
||
这是给下游用的产物（聚类、初始化 DefectNet）；要接着训练请用 `simclr_last.pth`。
|
||
"""
|
||
path = self.work_dir / "simclr_backbone.pth"
|
||
torch.save({"model": self.backbone.state_dict(), "epochs": epoch + 1}, path)
|
||
return path
|
||
|
||
def save_state(self, epoch: int) -> Path:
|
||
"""完整训练状态，供 `resume` 断点续训。
|
||
|
||
比 `simclr_backbone.pth` 多了投影头、优化器、调度器与 scaler——少了这些，
|
||
续训等于把动量和学习率进度清零，损失会先反弹一段。
|
||
|
||
同时记下当时的优化器/调度器配置，续训时若与新配置不同会打日志说明按新的来。
|
||
"""
|
||
path = self.work_dir / "simclr_last.pth"
|
||
torch.save(
|
||
{
|
||
"backbone": self.backbone.state_dict(),
|
||
"projector": self.projector.state_dict(),
|
||
"optimizer": self.optimizer.state_dict(),
|
||
"scheduler": self.scheduler.state_dict() if self.scheduler else None,
|
||
"scaler": self.scaler.state_dict(),
|
||
"epoch": epoch,
|
||
"global_step": self.global_step,
|
||
"epochs_planned": self.epochs,
|
||
"optimizer_cfg": dict(self.cfg.get("optimizer", {})),
|
||
"scheduler_cfg": dict(self.cfg.get("scheduler") or {}),
|
||
},
|
||
path,
|
||
)
|
||
return path
|
||
|
||
def load_state(self, path: str | Path) -> None:
|
||
"""载入 `save_state` 写出的完整状态，从下一个 epoch 接着练。
|
||
|
||
**配置文件是唯一真相源**：存档只提供"进度"——模型权重、优化器动量缓冲、
|
||
已训到第几个 epoch、global_step；而学习率、weight_decay、总轮数、调度器
|
||
一律以当前 yaml 为准。否则改了配置再续训，改动会被存档静默覆盖掉。
|
||
"""
|
||
path = Path(path)
|
||
if not path.exists():
|
||
raise FileNotFoundError(f"续训状态文件不存在: {path}")
|
||
|
||
state = torch.load(path, map_location=self.device, weights_only=False)
|
||
if "projector" not in state:
|
||
raise ValueError(
|
||
f"{path} 只含骨架权重，无法续训。续训要用 simclr_last.pth；"
|
||
"若只是想跳过训练直接聚类，请改用 simclr.checkpoint。"
|
||
)
|
||
|
||
self.backbone.load_state_dict(state["backbone"])
|
||
self.projector.load_state_dict(state["projector"])
|
||
|
||
# optimizer.load_state_dict 会整体覆盖 param_groups，把 lr/weight_decay 等超参
|
||
# 一并还原成存档时的值。先把配置构造出来的超参抄下来，加载完再按组写回去，
|
||
# 这样只留下动量缓冲（真正的"进度"），超参仍以 yaml 为准。
|
||
# 按组写回也保住了 build_optimizer 的分组结构（BN 与 bias 那组 weight_decay=0）。
|
||
archived_optimizer = (state.get("optimizer_cfg") or {}).get("name")
|
||
current_optimizer = dict(self.cfg.get("optimizer", {})).get("name")
|
||
if archived_optimizer and current_optimizer and archived_optimizer != current_optimizer:
|
||
# 换了优化器种类，动量缓冲的含义完全不同（AdamW 的 exp_avg 对 SGD 毫无意义），
|
||
# 硬塞进去只会白占内存。直接不加载，从零开始积累。
|
||
logger.warning(
|
||
"优化器由 %s 换成了 %s，动量缓冲无法迁移、将从零积累（模型权重与训练进度不受影响）",
|
||
archived_optimizer,
|
||
current_optimizer,
|
||
)
|
||
else:
|
||
intended = [
|
||
{key: value for key, value in group.items() if key != "params"}
|
||
for group in self.optimizer.param_groups
|
||
]
|
||
self.optimizer.load_state_dict(state["optimizer"])
|
||
for group, wanted in zip(self.optimizer.param_groups, intended, strict=True):
|
||
group.update(wanted)
|
||
|
||
if state.get("scaler"):
|
||
self.scaler.load_state_dict(state["scaler"])
|
||
|
||
self.start_epoch = int(state["epoch"]) + 1
|
||
self.global_step = int(state.get("global_step", 0))
|
||
self._restore_scheduler(state)
|
||
self._log_config_changes(state)
|
||
logger.info(
|
||
"已载入训练状态 %s：从 epoch %d 接着练到 %d", path, self.start_epoch, self.epochs
|
||
)
|
||
if self.start_epoch >= self.epochs:
|
||
logger.warning(
|
||
"状态里已训到 epoch %d，不小于计划的 %d 轮——不会再训练。"
|
||
"要继续请调大 simclr.epochs。",
|
||
self.start_epoch,
|
||
self.epochs,
|
||
)
|
||
|
||
def _restore_scheduler(self, state: dict) -> None:
|
||
"""按**当前配置**重建学习率调度，再快进到已训到的 epoch。
|
||
|
||
不恢复存档里的调度器状态：它带着当时的 T_max 和 base_lrs，一旦 yaml 里改了
|
||
总轮数或学习率就会被静默覆盖。原计划 10 轮的 cosine 跑完后 lr 已是 0，
|
||
这时把 epochs 改成 30 接着练，沿用旧 T_max 的话 lr 仍是 0——一步都学不动。
|
||
|
||
cosine / step / multistep 都是 epoch 的确定性函数，快进与恢复状态等价，
|
||
所以统一走重建这一条路，没有第二种行为。
|
||
"""
|
||
del state # 存档里的 scheduler 状态刻意不用
|
||
if self.scheduler is None:
|
||
return
|
||
|
||
# cosine 的链式实现基于当前 lr 递推，必须先复位回 initial_lr，
|
||
# 否则从 0 出发只会一路留在 0。
|
||
for group in self.optimizer.param_groups:
|
||
group["lr"] = group.get("initial_lr", group["lr"])
|
||
|
||
if self.cfg.get("restart_schedule"):
|
||
# 续训到调度末段时，cosine 会把新设的 lr 乘上一个接近 0 的系数
|
||
# （2229/2500 处只剩 2.9%），"调大学习率接着练"就落空了。
|
||
# 打开这个开关：按剩余轮数重开一条曲线，新 lr 从满值开始。
|
||
remaining = max(1, self.epochs - self.start_epoch)
|
||
self.scheduler = build_scheduler(self.optimizer, self.cfg.get("scheduler"), remaining)
|
||
logger.warning(
|
||
"restart_schedule=true：学习率调度从 epoch %d 重新开始，覆盖剩余 %d 轮，"
|
||
"起始 lr=%.2e",
|
||
self.start_epoch,
|
||
remaining,
|
||
self.optimizer.param_groups[0]["lr"],
|
||
)
|
||
return
|
||
self.scheduler = build_scheduler(self.optimizer, self.cfg.get("scheduler"), self.epochs)
|
||
with warnings.catch_warnings():
|
||
warnings.simplefilter("ignore") # 快进阶段的 "step before optimizer.step()" 属预期
|
||
for _ in range(self.start_epoch):
|
||
self.scheduler.step()
|
||
|
||
def _log_config_changes(self, state: dict) -> None:
|
||
"""存档配置与当前配置不一致时说清楚用的是哪一份，避免"改了没生效"的困惑。"""
|
||
changes = []
|
||
planned = int(state.get("epochs_planned", self.epochs))
|
||
if planned != self.epochs:
|
||
changes.append(f"总轮数 {planned} -> {self.epochs}")
|
||
for key, archived in (state.get("optimizer_cfg") or {}).items():
|
||
current = dict(self.cfg.get("optimizer", {})).get(key)
|
||
if current is not None and current != archived:
|
||
changes.append(f"optimizer.{key} {archived} -> {current}")
|
||
archived_scheduler = (state.get("scheduler_cfg") or {}).get("name")
|
||
current_scheduler = (dict(self.cfg.get("scheduler") or {})).get("name")
|
||
# 旧存档没有 scheduler_cfg 字段，那是"不知道"而不是"当时没有调度器"，不能报成差异
|
||
if "scheduler_cfg" in state and archived_scheduler != current_scheduler:
|
||
changes.append(f"scheduler {archived_scheduler} -> {current_scheduler}")
|
||
|
||
if changes:
|
||
logger.warning("配置与存档不一致，按**当前配置**执行：%s", "；".join(changes))
|
||
group = self.optimizer.param_groups[0]
|
||
initial = group.get("initial_lr", float("nan"))
|
||
logger.info(
|
||
"续训超参：lr=%.2e（initial_lr=%.2e，调度进度 %d/%d）weight_decay=%s",
|
||
group["lr"],
|
||
initial,
|
||
self.start_epoch,
|
||
self.epochs,
|
||
[g["weight_decay"] for g in self.optimizer.param_groups],
|
||
)
|
||
if initial and group["lr"] < initial * 0.1 and not self.cfg.get("restart_schedule"):
|
||
logger.warning(
|
||
"当前 lr 只有 initial_lr 的 %.1f%%——调度已接近末尾，此时调大 lr 收效有限。"
|
||
"想让新 lr 真正生效可加 --set simclr.restart_schedule=true（按剩余轮数重开曲线），"
|
||
"或调大 simclr.epochs 拉长整条曲线。",
|
||
group["lr"] / initial * 100,
|
||
)
|
||
|
||
def _checkpoint(self, epoch: int) -> None:
|
||
"""同时落两份：给下游用的骨架，和给续训用的完整状态。"""
|
||
self.save_backbone(epoch)
|
||
self.save_state(epoch)
|
||
|
||
def fit(self) -> Path:
|
||
"""训练并保存骨架权重（不含投影头），返回骨架路径。
|
||
|
||
每 `save_interval` 个 epoch 落一次盘（Ctrl-C 也会先落盘），
|
||
同时写 `simclr_backbone.pth`（下游用）与 `simclr_last.pth`（续训用）。
|
||
"""
|
||
if len(self.dataloader) < 2:
|
||
raise ValueError("SimCLR 需要多个 batch，请调小 batch_size 或增加样本")
|
||
|
||
path = self.work_dir / "simclr_backbone.pth"
|
||
if self.start_epoch >= self.epochs:
|
||
self.save_backbone(self.epochs - 1)
|
||
self.visualizer.finish()
|
||
return path
|
||
|
||
epoch = self.start_epoch
|
||
try:
|
||
for epoch in range(self.start_epoch, self.epochs):
|
||
self.train_one_epoch(epoch)
|
||
if self.scheduler is not None:
|
||
self.scheduler.step()
|
||
if self.save_interval and (epoch + 1) % self.save_interval == 0:
|
||
self._checkpoint(epoch)
|
||
logger.info("已保存中间权重（epoch %d）: %s", epoch, path)
|
||
except KeyboardInterrupt:
|
||
self._checkpoint(epoch)
|
||
logger.warning(
|
||
"训练被中断，已保存 epoch %d 的权重: %s；续训用 --set simclr.resume=%s",
|
||
epoch,
|
||
path,
|
||
self.work_dir / "simclr_last.pth",
|
||
)
|
||
self.visualizer.finish()
|
||
return path
|
||
|
||
self._checkpoint(self.epochs - 1)
|
||
logger.info("自监督骨架已保存: %s", path)
|
||
self.visualizer.finish()
|
||
return path
|