fix(simclr): 换优化器种类时不再加载不兼容的动量缓冲
adamw 换成 sgd 时 load_state_dict 不会报错,SGD 会收下 AdamW 的 exp_avg/exp_avg_sq 却永远用不上——动量实际从零开始,但状态里白占内存,之后存盘还会一路带下去。 改为检测到优化器种类变化就跳过加载并给出日志,模型权重与训练进度不受影响。 README 补一张表,说明续训时哪些配置按 yaml、哪些来自存档、哪些不能改。 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
b7505a743d
commit
004d7a1ce7
@@ -318,6 +318,18 @@ uv run python scripts/cluster_unlabeled.py --config configs/cluster/film.yaml \
|
|||||||
> optimizer.lr 0.001 -> 0.0005`),不会出现"改了没生效"。学习率调度按新的总轮数重建
|
> optimizer.lr 0.001 -> 0.0005`),不会出现"改了没生效"。学习率调度按新的总轮数重建
|
||||||
> 并快进到当前 epoch,不会沿用存档里的 T_max。
|
> 并快进到当前 epoch,不会沿用存档里的 T_max。
|
||||||
|
|
||||||
|
续训时各项配置的行为:
|
||||||
|
|
||||||
|
| 配置项 | 续训时 |
|
||||||
|
| --- | --- |
|
||||||
|
| `epochs` | 按 yaml,且是**总轮数**不是剩余轮数。已训 12 轮、写 40 就再训 28 轮;写成 ≤12 则不再训练并给出提示 |
|
||||||
|
| `optimizer` 的超参(`lr` / `weight_decay` / `betas` / `momentum` …) | 按 yaml |
|
||||||
|
| `scheduler`(类型与参数) | 按 yaml,重建后快进到当前 epoch |
|
||||||
|
| `temperature`、`log_interval`、`save_interval`、`amp`、`batch_size`、增强 | 按 yaml |
|
||||||
|
| 模型权重、投影头、优化器动量缓冲、已训轮数、`global_step` | 来自存档(这些才是"进度") |
|
||||||
|
| `optimizer.name`(adamw ↔ sgd) | 可以改,但动量缓冲无法迁移、会从零积累(有日志提示) |
|
||||||
|
| `backbone` 结构、`projection` 维度 | **不能改**,与存档权重形状不符会直接报错 |
|
||||||
|
|
||||||
> `resume` 与 `checkpoint` 语义不同,二选一(同时给会报错):
|
> `resume` 与 `checkpoint` 语义不同,二选一(同时给会报错):
|
||||||
> **`resume`** = 载入完整状态**接着训练**(要 `simclr_last.pth`);
|
> **`resume`** = 载入完整状态**接着训练**(要 `simclr_last.pth`);
|
||||||
> **`checkpoint`** = 有权重就**跳过训练**直接聚类(要 `simclr_backbone.pth`)。
|
> **`checkpoint`** = 有权重就**跳过训练**直接聚类(要 `simclr_backbone.pth`)。
|
||||||
|
|||||||
@@ -224,6 +224,17 @@ class SimCLRTrainer:
|
|||||||
# 一并还原成存档时的值。先把配置构造出来的超参抄下来,加载完再按组写回去,
|
# 一并还原成存档时的值。先把配置构造出来的超参抄下来,加载完再按组写回去,
|
||||||
# 这样只留下动量缓冲(真正的"进度"),超参仍以 yaml 为准。
|
# 这样只留下动量缓冲(真正的"进度"),超参仍以 yaml 为准。
|
||||||
# 按组写回也保住了 build_optimizer 的分组结构(BN 与 bias 那组 weight_decay=0)。
|
# 按组写回也保住了 build_optimizer 的分组结构(BN 与 bias 那组 weight_decay=0)。
|
||||||
|
archived_optimizer = (state.get("optimizer_cfg") or {}).get("name")
|
||||||
|
current_optimizer = dict(self.cfg.get("optimizer", {})).get("name")
|
||||||
|
if archived_optimizer and current_optimizer and archived_optimizer != current_optimizer:
|
||||||
|
# 换了优化器种类,动量缓冲的含义完全不同(AdamW 的 exp_avg 对 SGD 毫无意义),
|
||||||
|
# 硬塞进去只会白占内存。直接不加载,从零开始积累。
|
||||||
|
logger.warning(
|
||||||
|
"优化器由 %s 换成了 %s,动量缓冲无法迁移、将从零积累(模型权重与训练进度不受影响)",
|
||||||
|
archived_optimizer,
|
||||||
|
current_optimizer,
|
||||||
|
)
|
||||||
|
else:
|
||||||
intended = [
|
intended = [
|
||||||
{key: value for key, value in group.items() if key != "params"}
|
{key: value for key, value in group.items() if key != "params"}
|
||||||
for group in self.optimizer.param_groups
|
for group in self.optimizer.param_groups
|
||||||
|
|||||||
@@ -330,6 +330,33 @@ def test_resume_takes_hyperparams_from_config_not_checkpoint(tiny_image_dir, tmp
|
|||||||
assert resumed.optimizer.state, "动量缓冲应当保留——那才是真正要续的'进度'"
|
assert resumed.optimizer.state, "动量缓冲应当保留——那才是真正要续的'进度'"
|
||||||
|
|
||||||
|
|
||||||
|
def test_resume_across_optimizer_types_drops_incompatible_state(tiny_image_dir, tmp_path):
|
||||||
|
"""换优化器种类时不加载动量缓冲:AdamW 的 exp_avg 对 SGD 没有意义,硬塞只是白占内存。"""
|
||||||
|
work_dir = tmp_path / "run"
|
||||||
|
_make_trainer(
|
||||||
|
tiny_image_dir, work_dir, epochs=2,
|
||||||
|
optimizer={"name": "adamw", "lr": 1e-3, "weight_decay": 1e-4},
|
||||||
|
).fit()
|
||||||
|
|
||||||
|
resumed = SimCLRTrainer(
|
||||||
|
BACKBONES.build({"name": "tiny_conv", "widths": [8, 16], "depths": [1, 1]}),
|
||||||
|
_make_trainer(tiny_image_dir, work_dir, epochs=1).dataloader,
|
||||||
|
{"epochs": 6, "amp": False, "log_interval": 999, "save_interval": 1,
|
||||||
|
"scheduler": {"name": "cosine"},
|
||||||
|
"optimizer": {"name": "sgd", "lr": 0.01, "momentum": 0.9},
|
||||||
|
"projection": {"hidden_channels": 16, "out_channels": 8},
|
||||||
|
"swanlab": {"enabled": False}},
|
||||||
|
torch.device("cpu"),
|
||||||
|
work_dir,
|
||||||
|
resume=work_dir / "simclr_last.pth",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert isinstance(resumed.optimizer, torch.optim.SGD)
|
||||||
|
assert not resumed.optimizer.state, "不该带着 AdamW 的缓冲"
|
||||||
|
assert resumed.start_epoch == 2 # 训练进度仍然接上
|
||||||
|
assert resumed.optimizer.param_groups[0]["momentum"] == 0.9
|
||||||
|
|
||||||
|
|
||||||
def test_resume_with_same_schedule_restores_state(tiny_image_dir, tmp_path):
|
def test_resume_with_same_schedule_restores_state(tiny_image_dir, tmp_path):
|
||||||
"""总轮数不变时,重建 + 快进的结果应与原调度进度一致。"""
|
"""总轮数不变时,重建 + 快进的结果应与原调度进度一致。"""
|
||||||
work_dir = tmp_path / "run"
|
work_dir = tmp_path / "run"
|
||||||
|
|||||||
Reference in New Issue
Block a user