feat: 支持 cuda / mps / cpu 三后端,Mac 上可用 MPS 训练
移除"训练必须有 Nvidia GPU"的硬性守卫,改为按后端能力分派。
- utils/device: resolve_device("auto") 按 cuda > mps > cpu 挑最快的可用后端;
显式指定不可用的后端时直接报错,不会悄悄退回 CPU——训练慢几十倍却不吭声是最坏的情况
- AMP 随后端取默认值:CUDA 开、MPS 关。MPS 上 autocast 数值正常但实测更慢
(tiny_conv @128²:fp32 76 ms/步、fp16 90 ms、bf16 108 ms),需要时 amp: true 可强开
- PeakMemory 统一显存峰值统计:CUDA 读内置计数器,MPS 无峰值 API 改用逐 step 采样
(是下界,日志与产出里已标注)
- profile_inference 支持 MPS,并提示 torch.profiler 没有 MPS 的 ProfilerActivity,
算子明细只反映 CPU 侧下发耗时
- train.py / eval.py 新增 --device,配置默认改为 auto
- check_env 改为列出全部可用后端与 auto 的选择结果
实测:tiny_conv 训练在 MPS 上比 CPU 快约 32 倍。
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
4fc0fa5353
commit
141cb907f2
+49
-10
@@ -13,7 +13,6 @@ from PIL import Image # noqa: E402
|
||||
from engine.simclr import SimCLRTrainer, nt_xent_loss # noqa: E402
|
||||
from models.heads.projection import ProjectionHead # noqa: E402
|
||||
from models.registry import BACKBONES # noqa: E402
|
||||
from utils.device import NoNvidiaGpuError # noqa: E402
|
||||
|
||||
|
||||
def test_nt_xent_is_minimal_when_views_agree():
|
||||
@@ -105,12 +104,46 @@ def test_simclr_trainer_runs_and_saves_backbone_only(tiny_image_dir, tmp_path):
|
||||
backbone.load_state_dict(state["model"]) # 能被干净地加载回去
|
||||
|
||||
|
||||
def test_simclr_saves_intermediate_checkpoints(tiny_image_dir, tmp_path):
|
||||
"""中途落盘:跑到一半被打断也要能拿到权重(此前只在全部 epoch 跑完后才存)。"""
|
||||
from torch.utils.data import DataLoader
|
||||
|
||||
from datasets.unlabeled import UnlabeledImageDataset, unlabeled_collate
|
||||
|
||||
dataset = UnlabeledImageDataset(
|
||||
tiny_image_dir,
|
||||
transforms=[{"name": "Resize", "size": [32, 32]}, {"name": "ToTensor"}],
|
||||
views=2,
|
||||
)
|
||||
loader = DataLoader(dataset, batch_size=4, shuffle=True, drop_last=True, collate_fn=unlabeled_collate)
|
||||
backbone = BACKBONES.build({"name": "tiny_conv", "widths": [8, 16], "depths": [1, 1]})
|
||||
work_dir = tmp_path / "run"
|
||||
|
||||
trainer = SimCLRTrainer(
|
||||
backbone,
|
||||
loader,
|
||||
{"epochs": 4, "amp": False, "log_interval": 100, "save_interval": 2,
|
||||
"projection": {"hidden_channels": 16, "out_channels": 8}},
|
||||
torch.device("cpu"),
|
||||
work_dir,
|
||||
)
|
||||
# 手动走前两个 epoch,模拟"训练中途"
|
||||
trainer.train_one_epoch(0)
|
||||
trainer.train_one_epoch(1)
|
||||
path = trainer.save_backbone(1)
|
||||
|
||||
assert path.exists()
|
||||
assert torch.load(path, map_location="cpu", weights_only=False)["epochs"] == 2
|
||||
|
||||
trainer.fit() # 完整跑完后 epochs 应更新
|
||||
assert torch.load(path, map_location="cpu", weights_only=False)["epochs"] == 4
|
||||
|
||||
|
||||
def test_cluster_script_simclr_branch_end_to_end(tiny_image_dir, tmp_path, monkeypatch):
|
||||
"""method=simclr 全流程:训练 -> 存权重 -> 抽特征 -> 聚类 -> 产出。"""
|
||||
sys.path.insert(0, "scripts")
|
||||
import cluster_unlabeled as cu
|
||||
|
||||
monkeypatch.setattr(cu, "require_cuda", lambda: ["FAKE-GPU"])
|
||||
work_dir = tmp_path / "run"
|
||||
monkeypatch.setattr(
|
||||
sys, "argv",
|
||||
@@ -130,19 +163,25 @@ def test_cluster_script_simclr_branch_end_to_end(tiny_image_dir, tmp_path, monke
|
||||
assert (work_dir / "clusters" / "cluster_00").is_dir()
|
||||
|
||||
|
||||
def test_simclr_branch_is_blocked_without_gpu(tiny_image_dir, tmp_path, monkeypatch):
|
||||
"""无 Nvidia GPU 时 simclr 分支必须被守卫拦下(pretrained 分支不受影响)。"""
|
||||
def test_simclr_runs_on_the_resolved_device(tiny_image_dir, tmp_path, monkeypatch):
|
||||
"""method=simclr 在本机自动选中的后端上(MPS / CUDA / CPU)都应能跑通。"""
|
||||
sys.path.insert(0, "scripts")
|
||||
import cluster_unlabeled as cu
|
||||
|
||||
def refuse():
|
||||
raise NoNvidiaGpuError("没有 Nvidia GPU")
|
||||
from utils.device import describe_device, resolve_device
|
||||
|
||||
monkeypatch.setattr(cu, "require_cuda", refuse)
|
||||
device = resolve_device("auto")
|
||||
work_dir = tmp_path / "auto_run"
|
||||
monkeypatch.setattr(
|
||||
sys, "argv",
|
||||
["cluster_unlabeled.py", "--config", "configs/cluster/film.yaml",
|
||||
"--work-dir", str(tmp_path / "run"), "--root", str(tiny_image_dir),
|
||||
"--set", "device=cpu", "method=simclr", "loader.num_workers=0"],
|
||||
"--work-dir", str(work_dir), "--root", str(tiny_image_dir), "--set",
|
||||
"method=simclr", "loader.num_workers=0", "loader.batch_size=4",
|
||||
"simclr.epochs=1", "simclr.batch_size=4",
|
||||
"simclr.backbone.widths=[8,16]", "simclr.backbone.depths=[1,1]",
|
||||
"simclr.projection.hidden_channels=16", "simclr.projection.out_channels=8",
|
||||
"num_clusters=2", "n_init=2"],
|
||||
)
|
||||
assert cu.main() == 1
|
||||
|
||||
assert cu.main() == 0, f"在 {describe_device(device)} 上跑失败"
|
||||
assert (work_dir / "simclr_backbone.pth").exists()
|
||||
|
||||
Reference in New Issue
Block a user