feat: 支持 cuda / mps / cpu 三后端,Mac 上可用 MPS 训练

移除"训练必须有 Nvidia GPU"的硬性守卫,改为按后端能力分派。

- utils/device: resolve_device("auto") 按 cuda > mps > cpu 挑最快的可用后端;
  显式指定不可用的后端时直接报错,不会悄悄退回 CPU——训练慢几十倍却不吭声是最坏的情况
- AMP 随后端取默认值:CUDA 开、MPS 关。MPS 上 autocast 数值正常但实测更慢
  (tiny_conv @128²:fp32 76 ms/步、fp16 90 ms、bf16 108 ms),需要时 amp: true 可强开
- PeakMemory 统一显存峰值统计:CUDA 读内置计数器,MPS 无峰值 API 改用逐 step 采样
  (是下界,日志与产出里已标注)
- profile_inference 支持 MPS,并提示 torch.profiler 没有 MPS 的 ProfilerActivity,
  算子明细只反映 CPU 侧下发耗时
- train.py / eval.py 新增 --device,配置默认改为 auto
- check_env 改为列出全部可用后端与 auto 的选择结果

实测:tiny_conv 训练在 MPS 上比 CPU 快约 32 倍。

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Shaun Zhou
2026-09-02 21:43:35 +08:00
co-authored by Claude Opus 5
parent 4fc0fa5353
commit 141cb907f2
14 changed files with 475 additions and 148 deletions
+2 -2
View File
@@ -1,8 +1,8 @@
_base_: _base_:
- ../train/baseline.yaml - ../train/baseline.yaml
# 评估允许在 CPU 上跑(部署前的精度回归),训练则必须用 GPU。 # auto 按 cuda > mps > cpu 自动选;部署前做精度回归可以显式 device=cpu。
device: cuda device: auto
checkpoint: runs/baseline/best.pth checkpoint: runs/baseline/best.pth
eval: eval:
+2 -2
View File
@@ -5,11 +5,11 @@ _base_:
work_dir: runs/baseline work_dir: runs/baseline
seed: 42 seed: 42
deterministic: false deterministic: false
device: cuda # 训练只允许 Nvidia GPU,见 utils/device.py device: auto # auto 按 cuda > mps > cpu 挑最快的可用后端;也可写 cuda / mps / cpu
train: train:
epochs: 100 epochs: 100
amp: true amp: auto # auto: CUDA 上开、MPS 上关(实测 MPS autocast 比 fp32 慢);也可写 true / false
clip_grad: 5.0 clip_grad: 5.0
log_interval: 20 log_interval: 20
monitor: f1 # 用于挑选 best.pth 的验证指标 monitor: f1 # 用于挑选 best.pth 的验证指标
+3 -3
View File
@@ -5,11 +5,11 @@ from __future__ import annotations
from collections.abc import Mapping from collections.abc import Mapping
from typing import Any from typing import Any
import torch
from torch.utils.data import DataLoader, Dataset from torch.utils.data import DataLoader, Dataset
from datasets.film_defect import collate_fn from datasets.film_defect import collate_fn
from datasets.registry import DATASETS from datasets.registry import DATASETS
from utils.device import pin_memory_supported, resolve_device
def build_dataset(cfg: Mapping[str, Any], **kwargs: Any) -> Dataset: def build_dataset(cfg: Mapping[str, Any], **kwargs: Any) -> Dataset:
@@ -24,9 +24,9 @@ def build_dataloader(
pin_memory: bool | None = None, pin_memory: bool | None = None,
drop_last: bool = False, drop_last: bool = False,
) -> DataLoader: ) -> DataLoader:
"""`pin_memory` 缺省时只在有 CUDA 设备时开启(CPU/MPS 上开启无意义且会告警)。""" """`pin_memory` 缺省时只在 CUDA 上开启(CPU/MPS 上无收益且会告警)。"""
if pin_memory is None: if pin_memory is None:
pin_memory = torch.cuda.is_available() pin_memory = pin_memory_supported(resolve_device("auto"))
return DataLoader( return DataLoader(
dataset, dataset,
batch_size=batch_size, batch_size=batch_size,
+16 -12
View File
@@ -1,7 +1,4 @@
"""训练循环。 """训练循环。支持 cuda / mps / cpu 三种后端。"""
注意:训练入口必须先通过 `utils.device.require_cuda()` 的 GPU 检查。
"""
from __future__ import annotations from __future__ import annotations
@@ -21,6 +18,7 @@ from engine.losses import DetectionLoss
from engine.metrics import AverageMeter from engine.metrics import AverageMeter
from engine.optim import build_optimizer, build_scheduler from engine.optim import build_optimizer, build_scheduler
from engine.visualizer import SwanLabLogger from engine.visualizer import SwanLabLogger
from utils.device import PeakMemory, resolve_amp
from utils.labels import MULTILABEL from utils.labels import MULTILABEL
from utils.logging import get_logger from utils.logging import get_logger
@@ -66,8 +64,10 @@ class Trainer:
self.optimizer = build_optimizer(model, self.cfg.get("optimizer", {"name": "adamw", "lr": 1e-3})) self.optimizer = build_optimizer(model, self.cfg.get("optimizer", {"name": "adamw", "lr": 1e-3}))
self.scheduler = build_scheduler(self.optimizer, self.cfg.get("scheduler"), self.epochs) self.scheduler = build_scheduler(self.optimizer, self.cfg.get("scheduler"), self.epochs)
self.amp = bool(self.cfg.get("amp", True)) and device.type == "cuda" # amp: auto 时随后端(CUDA 开、MPS 关——MPS 上 autocast 实测比 fp32 慢)
self.scaler = torch.amp.GradScaler("cuda", enabled=self.amp) self.amp = resolve_amp(self.cfg.get("amp", "auto"), device)
self.scaler = torch.amp.GradScaler(device.type, enabled=self.amp)
self.peak_memory = PeakMemory(device)
self.evaluator = ( self.evaluator = (
Evaluator( Evaluator(
@@ -97,7 +97,7 @@ class Trainer:
for step, batch in enumerate(self.train_loader): for step, batch in enumerate(self.train_loader):
batch = _to_device(batch, self.device) batch = _to_device(batch, self.device)
with torch.amp.autocast("cuda", enabled=self.amp): with torch.amp.autocast(self.device.type, enabled=self.amp):
outputs = self.model(batch["images"]) outputs = self.model(batch["images"])
losses = self.criterion(outputs, batch) losses = self.criterion(outputs, batch)
@@ -135,14 +135,18 @@ class Trainer:
return {key: meter.avg for key, meter in meters.items()} return {key: meter.avg for key, meter in meters.items()}
def _gpu_memory(self) -> dict[str, float]: def _gpu_memory(self) -> dict[str, float]:
"""本轮训练的显存峰值(GB);推理端的详细分析见 scripts/profile_inference.py。""" """本轮训练的显存峰值(GB);推理端的详细分析见 scripts/profile_inference.py。
if self.device.type != "cuda":
CUDA 读内置峰值计数器;MPS 没有峰值 API,用逐 step 采样的最大值(是下界);
CPU 不统计。
"""
if self.device.type == "cpu":
return {} return {}
stats = { stats = {
"gpu_mem_allocated_gb": torch.cuda.max_memory_allocated(self.device) / 1024**3, "gpu_mem_allocated_gb": self.peak_memory.peak_mb / 1024,
"gpu_mem_reserved_gb": torch.cuda.max_memory_reserved(self.device) / 1024**3, "gpu_mem_reserved_gb": self.peak_memory.reserved_mb / 1024,
} }
torch.cuda.reset_peak_memory_stats(self.device) self.peak_memory.reset()
return stats return stats
def fit(self) -> float: def fit(self) -> float:
+9 -3
View File
@@ -4,7 +4,7 @@
用法: 用法:
python eval.py --config configs/eval/baseline.yaml --checkpoint runs/baseline/best.pth python eval.py --config configs/eval/baseline.yaml --checkpoint runs/baseline/best.pth
与训练不同,评估允许在 CPU 上运行(`--set device=cpu`),便于部署前回归。 设备同样支持 auto / cuda / mps / cpu;部署前回归可以显式 `--device cpu`。
""" """
from __future__ import annotations from __future__ import annotations
@@ -18,7 +18,7 @@ from datasets import build_dataloader, build_dataset, load_classes
from engine import Evaluator from engine import Evaluator
from models import build_model from models import build_model
from utils import check_label_mode, load_config, setup_logging from utils import check_label_mode, load_config, setup_logging
from utils.device import resolve_device from utils.device import DeviceNotAvailableError, describe_device, resolve_device
def parse_args() -> argparse.Namespace: def parse_args() -> argparse.Namespace:
@@ -27,6 +27,7 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--checkpoint", default=None, help="覆盖配置中的 checkpoint") parser.add_argument("--checkpoint", default=None, help="覆盖配置中的 checkpoint")
parser.add_argument("--split", default=None, help="评估哪个 data 划分,默认取配置的 eval.split") parser.add_argument("--split", default=None, help="评估哪个 data 划分,默认取配置的 eval.split")
parser.add_argument("--set", nargs="*", default=[], metavar="KEY=VALUE") parser.add_argument("--set", nargs="*", default=[], metavar="KEY=VALUE")
parser.add_argument("--device", default=None, help="覆盖配置中的 device:auto / cuda / mps / cpu")
parser.add_argument("--out", default=None, help="把指标写入 json 文件") parser.add_argument("--out", default=None, help="把指标写入 json 文件")
return parser.parse_args() return parser.parse_args()
@@ -45,7 +46,12 @@ def main() -> int:
logger.error("必须通过 --checkpoint 或配置中的 checkpoint 指定权重") logger.error("必须通过 --checkpoint 或配置中的 checkpoint 指定权重")
return 1 return 1
device = resolve_device(str(cfg.get("device", "cuda"))) try:
device = resolve_device(args.device or cfg.get("device", "auto"))
except DeviceNotAvailableError as exc:
logger.error("%s", exc)
return 1
logger.info("评估设备: %s", describe_device(device))
classes = load_classes(cfg["classes"]) classes = load_classes(cfg["classes"])
# label_mode 由 yaml 决定,模型与数据集必须一致,这里统一下发。 # label_mode 由 yaml 决定,模型与数据集必须一致,这里统一下发。
+27 -12
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env python #!/usr/bin/env python
"""环境自检:确认依赖与 Nvidia GPU 是否就绪。 """环境自检:确认依赖与可用的计算后端。
python scripts/check_env.py python scripts/check_env.py
""" """
@@ -11,29 +11,44 @@ from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1])) sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from utils.device import NoNvidiaGpuError, cuda_devices, require_cuda # noqa: E402 from utils.device import ( # noqa: E402
available_backends,
default_amp,
describe_device,
memory_stats,
resolve_device,
)
def main() -> int: def main() -> int:
print(f"Python: {sys.version.split()[0]}") print(f"Python: {sys.version.split()[0]}")
try: try:
import torch import torch
print(f"PyTorch: {torch.__version__} (CUDA build: {torch.version.cuda})")
except ImportError: except ImportError:
print("PyTorch: 未安装 —— 先执行 `uv sync`") print("PyTorch: 未安装 —— 先执行 `uv sync`")
return 1 return 1
devices = cuda_devices() print(f"PyTorch: {torch.__version__} (CUDA 构建: {torch.version.cuda})")
print(f"Nvidia GPU: {devices if devices else '无'}") print(f"可用后端(按优先级): {available_backends()}")
try: print("\n后端明细:")
require_cuda() print(f" cuda: available={torch.cuda.is_available()} device_count={torch.cuda.device_count()}")
except NoNvidiaGpuError as exc: print(
print(f"训练不可用: {exc}") f" mps : available={torch.backends.mps.is_available()} "
print("(评估/导出仍可在 CPU 上进行)") f"built={torch.backends.mps.is_built()}"
)
device = resolve_device("auto")
print(f"\ndevice=auto 会选中: {describe_device(device)}")
print(f" AMP 默认: {'开' if default_amp(device) else '关'}")
stats = memory_stats(device)
if stats:
print(" " + " ".join(f"{k}={v:.0f}" for k, v in stats.items()))
if device.type == "cpu":
print("\n⚠️ 只有 CPU 可用,训练会非常慢(实测比 MPS 慢约 30 倍),建议换有 GPU 的机器。")
return 1 return 1
print("训练环境就绪。") print("\n训练环境就绪。")
return 0 return 0
+31 -17
View File
@@ -31,6 +31,12 @@ from torch.profiler import ProfilerActivity, profile, record_function # noqa: E
from datasets import load_classes # noqa: E402 from datasets import load_classes # noqa: E402
from models import build_model, decode_heatmap # noqa: E402 from models import build_model, decode_heatmap # noqa: E402
from utils import load_config, setup_logging # noqa: E402 from utils import load_config, setup_logging # noqa: E402
from utils.device import ( # noqa: E402
DeviceNotAvailableError,
PeakMemory,
resolve_device,
synchronize,
)
logger = setup_logging() logger = setup_logging()
@@ -45,7 +51,7 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--image-size", type=int, nargs=2, default=None, help="默认取配置的 image_size") parser.add_argument("--image-size", type=int, nargs=2, default=None, help="默认取配置的 image_size")
parser.add_argument("--iters", type=int, default=50, help="计时轮数") parser.add_argument("--iters", type=int, default=50, help="计时轮数")
parser.add_argument("--warmup", type=int, default=10, help="预热轮数(不计入统计)") parser.add_argument("--warmup", type=int, default=10, help="预热轮数(不计入统计)")
parser.add_argument("--device", default=None, help="默认有 GPU 用 cuda,否则 cpu") parser.add_argument("--device", default=None, help="auto / cuda / mps / cpu,默认 auto")
parser.add_argument("--amp", action="store_true", help="用 float16 autocast 推理") parser.add_argument("--amp", action="store_true", help="用 float16 autocast 推理")
parser.add_argument("--postprocess", action="store_true", help="把 heatmap 解码也计入") parser.add_argument("--postprocess", action="store_true", help="把 heatmap 解码也计入")
parser.add_argument("--rows", type=int, default=15, help="算子排行显示条数") parser.add_argument("--rows", type=int, default=15, help="算子排行显示条数")
@@ -73,7 +79,7 @@ def build(args: argparse.Namespace, device: torch.device) -> torch.nn.Module:
@torch.no_grad() @torch.no_grad()
def run_once(model: torch.nn.Module, images: torch.Tensor, args: argparse.Namespace) -> None: def run_once(model: torch.nn.Module, images: torch.Tensor, args: argparse.Namespace) -> None:
device_type = images.device.type device_type = images.device.type
with torch.autocast(device_type, dtype=torch.float16, enabled=args.amp and device_type == "cuda"): with torch.autocast(device_type, dtype=torch.float16, enabled=args.amp and device_type != "cpu"):
with record_function("forward"): with record_function("forward"):
outputs = model(images) outputs = model(images)
if args.postprocess: if args.postprocess:
@@ -89,19 +95,17 @@ def run_once(model: torch.nn.Module, images: torch.Tensor, args: argparse.Namesp
def measure(model: torch.nn.Module, images: torch.Tensor, args: argparse.Namespace) -> dict: def measure(model: torch.nn.Module, images: torch.Tensor, args: argparse.Namespace) -> dict:
"""预热后计时,并统计 CUDA 显存峰值。""" """预热后计时,并统计 CUDA 显存峰值。"""
device = images.device device = images.device
is_cuda = device.type == "cuda" peak = PeakMemory(device)
for _ in range(args.warmup): for _ in range(args.warmup):
run_once(model, images, args) run_once(model, images, args)
if is_cuda: synchronize(device)
torch.cuda.synchronize(device) peak.reset()
torch.cuda.reset_peak_memory_stats(device)
start = time.perf_counter() start = time.perf_counter()
for _ in range(args.iters): for _ in range(args.iters):
run_once(model, images, args) run_once(model, images, args)
if is_cuda: synchronize(device)
torch.cuda.synchronize(device)
elapsed = time.perf_counter() - start elapsed = time.perf_counter() - start
batch = images.shape[0] batch = images.shape[0]
@@ -110,11 +114,12 @@ def measure(model: torch.nn.Module, images: torch.Tensor, args: argparse.Namespa
"latency_ms": elapsed / args.iters * 1000, "latency_ms": elapsed / args.iters * 1000,
"throughput_fps": args.iters * batch / elapsed, "throughput_fps": args.iters * batch / elapsed,
} }
if is_cuda: if device.type != "cpu":
result.update( result.update(
{ {
"peak_allocated_mb": torch.cuda.max_memory_allocated(device) / 1024**2, "peak_allocated_mb": peak.peak_mb,
"peak_reserved_mb": torch.cuda.max_memory_reserved(device) / 1024**2, "peak_reserved_mb": peak.reserved_mb,
"peak_is_exact": peak.exact,
"weights_mb": sum(p.numel() * p.element_size() for p in model.parameters()) / 1024**2, "weights_mb": sum(p.numel() * p.element_size() for p in model.parameters()) / 1024**2,
} }
) )
@@ -124,13 +129,13 @@ def measure(model: torch.nn.Module, images: torch.Tensor, args: argparse.Namespa
def profile_once(model: torch.nn.Module, images: torch.Tensor, args: argparse.Namespace) -> None: def profile_once(model: torch.nn.Module, images: torch.Tensor, args: argparse.Namespace) -> None:
"""算子级 profiling:按显存占用与耗时各排一次序。""" """算子级 profiling:按显存占用与耗时各排一次序。"""
is_cuda = images.device.type == "cuda" is_cuda = images.device.type == "cuda"
# torch.profiler 没有 MPS 的 activity,Metal 侧的 kernel 抓不到,只能看 CPU 侧下发耗时
activities = [ProfilerActivity.CPU] + ([ProfilerActivity.CUDA] if is_cuda else []) activities = [ProfilerActivity.CPU] + ([ProfilerActivity.CUDA] if is_cuda else [])
with profile(activities=activities, profile_memory=True, record_shapes=True) as prof: with profile(activities=activities, profile_memory=True, record_shapes=True) as prof:
for _ in range(max(args.iters // 5, 1)): for _ in range(max(args.iters // 5, 1)):
run_once(model, images, args) run_once(model, images, args)
if is_cuda: synchronize(images.device)
torch.cuda.synchronize(images.device)
memory_key = "self_cuda_memory_usage" if is_cuda else "self_cpu_memory_usage" memory_key = "self_cuda_memory_usage" if is_cuda else "self_cpu_memory_usage"
time_key = "self_cuda_time_total" if is_cuda else "self_cpu_time_total" time_key = "self_cuda_time_total" if is_cuda else "self_cpu_time_total"
@@ -148,9 +153,18 @@ def profile_once(model: torch.nn.Module, images: torch.Tensor, args: argparse.Na
def main() -> int: def main() -> int:
args = parse_args() args = parse_args()
device = torch.device(args.device or ("cuda" if torch.cuda.is_available() else "cpu")) try:
if device.type != "cuda": device = resolve_device(args.device or "auto")
logger.warning("当前在 %s 上运行:没有显存数据,只能看耗时与 CPU 内存。", device) except DeviceNotAvailableError as exc:
logger.error("%s", exc)
return 1
if device.type == "cpu":
logger.warning("当前在 CPU 上运行:没有显存数据,只能看耗时与 CPU 内存。")
elif device.type == "mps":
logger.warning(
"MPS 上 torch.profiler 抓不到 Metal kernel(没有对应的 ProfilerActivity),"
"算子明细只反映 CPU 侧下发耗时;显存用 torch.mps 采样,峰值是下界。"
)
model = build(args, device) model = build(args, device)
cfg = model.cfg cfg = model.cfg
@@ -158,7 +172,7 @@ def main() -> int:
channels = 1 if cfg.get("gray") else 3 channels = 1 if cfg.get("gray") else 3
logger.info( logger.info(
"模型 %s | 参数量 %.2fM | 输出 stride %d | 输入 %dx%dx%d | device %s | amp=%s", "模型 %s | 参数量 %.2fM | 输出 stride %d | 输入 %dx%dx%d | 设备 %s | amp=%s",
cfg["model"]["name"], cfg["model"]["name"],
sum(p.numel() for p in model.parameters()) / 1e6, sum(p.numel() for p in model.parameters()) / 1e6,
model.out_stride, model.out_stride,
+112
View File
@@ -0,0 +1,112 @@
"""设备解析:cuda / mps / cpu 三后端的选择、AMP 默认值与显存统计。"""
import pytest
torch = pytest.importorskip("torch")
from utils.device import ( # noqa: E402
BACKEND_PRIORITY,
DeviceNotAvailableError,
PeakMemory,
amp_supported,
available_backends,
default_amp,
describe_device,
memory_stats,
pin_memory_supported,
resolve_amp,
resolve_device,
synchronize,
)
def test_available_backends_is_ordered_and_always_has_cpu():
backends = available_backends()
assert backends[-1] == "cpu"
assert backends == [b for b in BACKEND_PRIORITY if b in backends]
assert len(set(backends)) == len(backends)
def test_auto_picks_the_highest_priority_available_backend():
device = resolve_device("auto")
assert device.type == available_backends()[0]
assert resolve_device("") == device # 空串等同 auto
def test_cpu_is_always_resolvable():
assert resolve_device("cpu").type == "cpu"
def test_explicit_backend_matches_hardware_or_raises():
"""显式请求不可用的后端必须报错,不能悄悄退回 CPU——那会让训练慢几十倍还不吭声。"""
for backend in ("cuda", "mps"):
if backend in available_backends():
assert resolve_device(backend).type == backend
else:
with pytest.raises(DeviceNotAvailableError, match=backend):
resolve_device(backend)
def test_unknown_device_raises():
with pytest.raises(DeviceNotAvailableError, match="未知设备"):
resolve_device("tpu")
def test_amp_defaults_follow_the_backend():
# CUDA 有 tensor core,默认开;MPS 上 autocast 实测更慢,默认关;CPU 不支持
assert default_amp("cuda") is True
assert default_amp("mps") is False
assert default_amp("cpu") is False
assert amp_supported("cuda") and amp_supported("mps")
assert not amp_supported("cpu")
def test_resolve_amp_handles_auto_and_explicit():
assert resolve_amp("auto", "cuda") is True
assert resolve_amp("auto", "mps") is False
assert resolve_amp(None, "mps") is False
assert resolve_amp(True, "mps") is True # 显式打开,MPS 支持 autocast
assert resolve_amp(True, "cpu") is False # CPU 不支持,显式打开也无效
assert resolve_amp(False, "cuda") is False
def test_pin_memory_only_for_cuda():
assert pin_memory_supported("cuda")
assert not pin_memory_supported("mps")
assert not pin_memory_supported("cpu")
def test_describe_device_is_readable():
assert describe_device("cpu") == "CPU"
assert "MPS" in describe_device("mps")
def test_memory_stats_shape_per_backend():
assert memory_stats("cpu") == {}
stats = memory_stats(resolve_device("auto"))
device_type = resolve_device("auto").type
if device_type == "cuda":
assert {"allocated_mb", "reserved_mb", "peak_allocated_mb"} <= set(stats)
elif device_type == "mps":
assert {"allocated_mb", "driver_allocated_mb"} <= set(stats)
def test_peak_memory_tracks_growth_on_accelerators():
device = resolve_device("auto")
peak = PeakMemory(device)
if device.type == "cpu":
assert peak.peak_mb == 0.0 and peak.reserved_mb == 0.0
return
# MPS 没有峰值 API,靠采样,所以峰值是下界(exact=False)
assert peak.exact == (device.type != "mps")
buffer = torch.empty(8 * 1024 * 1024 // 4, device=device) # 约 8 MB
peak.update()
synchronize(device)
assert peak.peak_mb >= 4.0
del buffer
def test_synchronize_is_a_noop_on_cpu():
synchronize("cpu") # 不应抛异常
-26
View File
@@ -1,26 +0,0 @@
"""GPU 守卫:项目禁止在没有 Nvidia GPU 的设备上训练。"""
import pytest
from utils.device import NoNvidiaGpuError, cuda_devices, require_cuda
def test_require_cuda_matches_hardware():
devices = cuda_devices()
if devices:
assert require_cuda() == devices
else:
with pytest.raises(NoNvidiaGpuError):
require_cuda()
def test_resolve_device_rejects_cuda_without_gpu():
torch = pytest.importorskip("torch")
from utils.device import resolve_device
if torch.cuda.is_available():
assert resolve_device("cuda").type == "cuda"
else:
with pytest.raises(NoNvidiaGpuError):
resolve_device("cuda")
assert resolve_device("cpu").type == "cpu"
+49 -10
View File
@@ -13,7 +13,6 @@ from PIL import Image # noqa: E402
from engine.simclr import SimCLRTrainer, nt_xent_loss # noqa: E402 from engine.simclr import SimCLRTrainer, nt_xent_loss # noqa: E402
from models.heads.projection import ProjectionHead # noqa: E402 from models.heads.projection import ProjectionHead # noqa: E402
from models.registry import BACKBONES # noqa: E402 from models.registry import BACKBONES # noqa: E402
from utils.device import NoNvidiaGpuError # noqa: E402
def test_nt_xent_is_minimal_when_views_agree(): def test_nt_xent_is_minimal_when_views_agree():
@@ -105,12 +104,46 @@ def test_simclr_trainer_runs_and_saves_backbone_only(tiny_image_dir, tmp_path):
backbone.load_state_dict(state["model"]) # 能被干净地加载回去 backbone.load_state_dict(state["model"]) # 能被干净地加载回去
def test_simclr_saves_intermediate_checkpoints(tiny_image_dir, tmp_path):
"""中途落盘:跑到一半被打断也要能拿到权重(此前只在全部 epoch 跑完后才存)。"""
from torch.utils.data import DataLoader
from datasets.unlabeled import UnlabeledImageDataset, unlabeled_collate
dataset = UnlabeledImageDataset(
tiny_image_dir,
transforms=[{"name": "Resize", "size": [32, 32]}, {"name": "ToTensor"}],
views=2,
)
loader = DataLoader(dataset, batch_size=4, shuffle=True, drop_last=True, collate_fn=unlabeled_collate)
backbone = BACKBONES.build({"name": "tiny_conv", "widths": [8, 16], "depths": [1, 1]})
work_dir = tmp_path / "run"
trainer = SimCLRTrainer(
backbone,
loader,
{"epochs": 4, "amp": False, "log_interval": 100, "save_interval": 2,
"projection": {"hidden_channels": 16, "out_channels": 8}},
torch.device("cpu"),
work_dir,
)
# 手动走前两个 epoch,模拟"训练中途"
trainer.train_one_epoch(0)
trainer.train_one_epoch(1)
path = trainer.save_backbone(1)
assert path.exists()
assert torch.load(path, map_location="cpu", weights_only=False)["epochs"] == 2
trainer.fit() # 完整跑完后 epochs 应更新
assert torch.load(path, map_location="cpu", weights_only=False)["epochs"] == 4
def test_cluster_script_simclr_branch_end_to_end(tiny_image_dir, tmp_path, monkeypatch): def test_cluster_script_simclr_branch_end_to_end(tiny_image_dir, tmp_path, monkeypatch):
"""method=simclr 全流程:训练 -> 存权重 -> 抽特征 -> 聚类 -> 产出。""" """method=simclr 全流程:训练 -> 存权重 -> 抽特征 -> 聚类 -> 产出。"""
sys.path.insert(0, "scripts") sys.path.insert(0, "scripts")
import cluster_unlabeled as cu import cluster_unlabeled as cu
monkeypatch.setattr(cu, "require_cuda", lambda: ["FAKE-GPU"])
work_dir = tmp_path / "run" work_dir = tmp_path / "run"
monkeypatch.setattr( monkeypatch.setattr(
sys, "argv", sys, "argv",
@@ -130,19 +163,25 @@ def test_cluster_script_simclr_branch_end_to_end(tiny_image_dir, tmp_path, monke
assert (work_dir / "clusters" / "cluster_00").is_dir() assert (work_dir / "clusters" / "cluster_00").is_dir()
def test_simclr_branch_is_blocked_without_gpu(tiny_image_dir, tmp_path, monkeypatch): def test_simclr_runs_on_the_resolved_device(tiny_image_dir, tmp_path, monkeypatch):
"""无 Nvidia GPU 时 simclr 分支必须被守卫拦下(pretrained 分支不受影响)。""" """method=simclr 在本机自动选中的后端上(MPS / CUDA / CPU)都应能跑通。"""
sys.path.insert(0, "scripts") sys.path.insert(0, "scripts")
import cluster_unlabeled as cu import cluster_unlabeled as cu
def refuse(): from utils.device import describe_device, resolve_device
raise NoNvidiaGpuError("没有 Nvidia GPU")
monkeypatch.setattr(cu, "require_cuda", refuse) device = resolve_device("auto")
work_dir = tmp_path / "auto_run"
monkeypatch.setattr( monkeypatch.setattr(
sys, "argv", sys, "argv",
["cluster_unlabeled.py", "--config", "configs/cluster/film.yaml", ["cluster_unlabeled.py", "--config", "configs/cluster/film.yaml",
"--work-dir", str(tmp_path / "run"), "--root", str(tiny_image_dir), "--work-dir", str(work_dir), "--root", str(tiny_image_dir), "--set",
"--set", "device=cpu", "method=simclr", "loader.num_workers=0"], "method=simclr", "loader.num_workers=0", "loader.batch_size=4",
"simclr.epochs=1", "simclr.batch_size=4",
"simclr.backbone.widths=[8,16]", "simclr.backbone.depths=[1,1]",
"simclr.projection.hidden_channels=16", "simclr.projection.out_channels=8",
"num_clusters=2", "n_init=2"],
) )
assert cu.main() == 1
assert cu.main() == 0, f"在 {describe_device(device)} 上跑失败"
assert (work_dir / "simclr_backbone.pth").exists()
+26 -11
View File
@@ -1,7 +1,7 @@
"""train.py / eval.py 的端到端冒烟:YOLO 数据目录 -> 训练 -> 评估。 """train.py / eval.py 的端到端冒烟:YOLO 数据目录 -> 训练 -> 评估。
覆盖入口脚本的接线(classes 解析、label_mode 下发、GPU 守卫), 覆盖入口脚本的接线:classes 解析、label_mode 下发、设备解析。固定用 CPU 跑,
所以在 CPU 上跑,`require_cuda` 用 monkeypatch 放行。 免得测试受机器上有没有 GPU 影响。
""" """
import sys import sys
@@ -19,7 +19,6 @@ from PIL import Image # noqa: E402
import eval as eval_mod # noqa: E402 import eval as eval_mod # noqa: E402
import train as train_mod # noqa: E402 import train as train_mod # noqa: E402
from datasets import write_data_yaml # noqa: E402 from datasets import write_data_yaml # noqa: E402
from utils.device import NoNvidiaGpuError # noqa: E402
CLASSES = ["scratch", "bubble"] CLASSES = ["scratch", "bubble"]
TRANSFORMS = [ TRANSFORMS = [
@@ -85,7 +84,6 @@ def run(module, argv, monkeypatch):
def test_train_then_eval(project, monkeypatch, tmp_path): def test_train_then_eval(project, monkeypatch, tmp_path):
config_path, root = project config_path, root = project
work_dir = tmp_path / "runs" work_dir = tmp_path / "runs"
monkeypatch.setattr(train_mod, "require_cuda", lambda: ["FAKE-GPU"])
assert run(train_mod, ["train.py", "--config", str(config_path), "--work-dir", str(work_dir)], assert run(train_mod, ["train.py", "--config", str(config_path), "--work-dir", str(work_dir)],
monkeypatch) == 0 monkeypatch) == 0
@@ -115,7 +113,6 @@ def test_classes_from_data_yaml_sizes_the_heads(project, monkeypatch, tmp_path):
return real_build(cfg) return real_build(cfg)
monkeypatch.setattr(train_mod, "build_model", spy) monkeypatch.setattr(train_mod, "build_model", spy)
monkeypatch.setattr(train_mod, "require_cuda", lambda: ["FAKE-GPU"])
run(train_mod, ["train.py", "--config", str(config_path), "--work-dir", str(work_dir)], monkeypatch) run(train_mod, ["train.py", "--config", str(config_path), "--work-dir", str(work_dir)], monkeypatch)
@@ -123,15 +120,33 @@ def test_classes_from_data_yaml_sizes_the_heads(project, monkeypatch, tmp_path):
assert captured["cfg"]["label_mode"] == "multilabel" assert captured["cfg"]["label_mode"] == "multilabel"
def test_training_is_blocked_without_nvidia_gpu(project, monkeypatch, tmp_path): def test_unavailable_device_fails_loudly(project, monkeypatch, tmp_path):
config_path, _ = project """显式请求机器上没有的后端时必须退出,而不是悄悄退回 CPU。"""
from utils.device import available_backends
def refuse(): missing = next((b for b in ("cuda", "mps") if b not in available_backends()), None)
raise NoNvidiaGpuError("没有 Nvidia GPU") if missing is None:
pytest.skip("这台机器 cuda 与 mps 都可用")
monkeypatch.setattr(train_mod, "require_cuda", refuse)
assert run( assert run(
train_mod, train_mod,
["train.py", "--config", str(config_path), "--work-dir", str(tmp_path / "runs3")], ["train.py", "--config", str(config_path_of(project)), "--work-dir", str(tmp_path / "runs3"),
"--device", missing],
monkeypatch, monkeypatch,
) == 1 ) == 1
def test_device_flag_overrides_config(project, monkeypatch, tmp_path):
"""--device 优先于配置里的 device。"""
config_path, _ = project
work_dir = tmp_path / "runs4"
assert run(
train_mod,
["train.py", "--config", str(config_path), "--work-dir", str(work_dir), "--device", "cpu"],
monkeypatch,
) == 0
assert (work_dir / "best.pth").exists()
def config_path_of(project):
return project[0]
+8 -8
View File
@@ -4,8 +4,10 @@
用法: 用法:
python train.py --config configs/train/baseline.yaml python train.py --config configs/train/baseline.yaml
python train.py --config configs/train/baseline.yaml --set train.epochs=10 train.optimizer.lr=5e-4 python train.py --config configs/train/baseline.yaml --set train.epochs=10 train.optimizer.lr=5e-4
python train.py --config configs/train/baseline.yaml --device mps
硬性约束:没有 Nvidia GPU 时直接退出(见 CLAUDE.MD / utils/device.py)。 设备由配置的 `device` 决定,默认 `auto`:按 cuda > mps > cpu 挑最快的可用后端。
显式指定但不可用时直接报错,不会悄悄退回 CPU。
""" """
from __future__ import annotations from __future__ import annotations
@@ -14,13 +16,11 @@ import argparse
import sys import sys
from pathlib import Path from pathlib import Path
import torch
from datasets import build_dataloader, build_dataset, load_classes from datasets import build_dataloader, build_dataset, load_classes
from engine import Trainer from engine import Trainer
from models import build_model from models import build_model
from utils import check_label_mode, load_config, seed_everything, setup_logging from utils import check_label_mode, load_config, seed_everything, setup_logging
from utils.device import NoNvidiaGpuError, require_cuda from utils.device import DeviceNotAvailableError, describe_device, resolve_device
def parse_args() -> argparse.Namespace: def parse_args() -> argparse.Namespace:
@@ -28,6 +28,7 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--config", required=True, help="yaml 配置路径") parser.add_argument("--config", required=True, help="yaml 配置路径")
parser.add_argument("--set", nargs="*", default=[], metavar="KEY=VALUE", help="覆盖配置项") parser.add_argument("--set", nargs="*", default=[], metavar="KEY=VALUE", help="覆盖配置项")
parser.add_argument("--work-dir", default=None, help="覆盖配置中的 work_dir") parser.add_argument("--work-dir", default=None, help="覆盖配置中的 work_dir")
parser.add_argument("--device", default=None, help="覆盖配置中的 device:auto / cuda / mps / cpu")
parser.add_argument("--resume", default=None, help="从检查点恢复训练") parser.add_argument("--resume", default=None, help="从检查点恢复训练")
return parser.parse_args() return parser.parse_args()
@@ -41,14 +42,13 @@ def main() -> int:
logger = setup_logging(work_dir / "train.log") logger = setup_logging(work_dir / "train.log")
try: try:
gpus = require_cuda() device = resolve_device(args.device or cfg.get("device", "auto"))
except NoNvidiaGpuError as exc: except DeviceNotAvailableError as exc:
logger.error("%s", exc) logger.error("%s", exc)
return 1 return 1
logger.info("检测到 GPU: %s", ", ".join(gpus)) logger.info("训练设备: %s", describe_device(device))
seed_everything(int(cfg.get("seed", 42)), bool(cfg.get("deterministic", False))) seed_everything(int(cfg.get("seed", 42)), bool(cfg.get("deterministic", False)))
device = torch.device(cfg.get("device", "cuda"))
classes = load_classes(cfg["classes"]) classes = load_classes(cfg["classes"])
# label_mode 由 yaml 决定,模型与数据集必须一致,这里统一下发。 # label_mode 由 yaml 决定,模型与数据集必须一致,这里统一下发。
+15 -2
View File
@@ -1,7 +1,15 @@
"""通用工具:配置、注册表、日志、随机种子、设备检查。""" """通用工具:配置、注册表、日志、随机种子、设备检查。"""
from utils.config import Config, load_config from utils.config import Config, load_config
from utils.device import require_cuda, resolve_device from utils.device import (
DeviceNotAvailableError,
PeakMemory,
available_backends,
describe_device,
memory_stats,
resolve_amp,
resolve_device,
)
from utils.labels import LABEL_MODES, MULTICLASS, MULTILABEL, check_label_mode from utils.labels import LABEL_MODES, MULTICLASS, MULTILABEL, check_label_mode
from utils.logging import get_logger, setup_logging from utils.logging import get_logger, setup_logging
from utils.registry import Registry from utils.registry import Registry
@@ -10,8 +18,13 @@ from utils.seed import seed_everything
__all__ = [ __all__ = [
"Config", "Config",
"load_config", "load_config",
"require_cuda",
"resolve_device", "resolve_device",
"resolve_amp",
"describe_device",
"available_backends",
"memory_stats",
"PeakMemory",
"DeviceNotAvailableError",
"LABEL_MODES", "LABEL_MODES",
"MULTILABEL", "MULTILABEL",
"MULTICLASS", "MULTICLASS",
+176 -41
View File
@@ -1,54 +1,189 @@
"""设备检查。 """设备解析与能力查询:统一处理 cuda / mps / cpu 三种后端。
项目硬性约定(见 CLAUDE.MD):禁止在没有 Nvidia GPU 的设备上训练。 设计要点:
所有训练入口必须先调用 `require_cuda()`,检查不通过直接抛错退出。 - `resolve_device("auto")` 按 cuda > mps > cpu 的优先级自动挑最快的后端;
- 显式指定但不可用时直接报错,而不是悄悄退回 CPU——训练慢 30 倍却不吭声是最坏的情况;
- AMP 的默认值随后端而变:CUDA 上开(有 tensor core),MPS 上关
(实测 tiny_conv @128² 每步 fp32 76 ms、fp16 90 ms,autocast 只剩转换开销反而更慢);
- 显存统计跨后端统一:CUDA 有内置峰值计数器,MPS 没有,靠采样取最大值。
""" """
from __future__ import annotations from __future__ import annotations
class NoNvidiaGpuError(RuntimeError):
"""当前设备不具备可用的 Nvidia GPU。"""
def cuda_devices() -> list[str]:
"""返回可用的 Nvidia GPU 名称列表;没有 torch 或没有 GPU 时返回空列表。"""
try:
import torch import torch
except ImportError:
return [] #: 自动选择时的优先级
if not torch.cuda.is_available() or torch.version.cuda is None: BACKEND_PRIORITY = ("cuda", "mps", "cpu")
return []
return [torch.cuda.get_device_name(i) for i in range(torch.cuda.device_count())]
def require_cuda() -> list[str]: class DeviceNotAvailableError(RuntimeError):
"""训练前的守卫;无 Nvidia GPU 时抛出 `NoNvidiaGpuError`。""" """请求的设备在当前机器上不可用。"""
try:
import torch
except ImportError as exc:
raise NoNvidiaGpuError("未安装 PyTorch,无法确认 Nvidia GPU,训练已中止。") from exc
if torch.version.cuda is None:
raise NoNvidiaGpuError(
f"当前 PyTorch ({torch.__version__}) 不是 CUDA 构建(可能是 CPU/ROCm/MPS 版本),"
"本项目禁止在非 Nvidia GPU 设备上训练。"
)
if not torch.cuda.is_available() or torch.cuda.device_count() == 0:
raise NoNvidiaGpuError(
"未检测到可用的 Nvidia GPU,本项目禁止在无 GPU 的设备上训练。"
"评估/推理请使用 eval.py 并显式指定 device=cpu。"
)
return cuda_devices()
def resolve_device(device: str = "cuda"): def _cuda_ready() -> bool:
"""把配置中的 device 字符串解析为 `torch.device`。 return torch.cuda.is_available() and torch.cuda.device_count() > 0
仅用于评估/推理等允许 CPU 的场景;训练路径请先走 `require_cuda()`。
def _mps_ready() -> bool:
return torch.backends.mps.is_available() and torch.backends.mps.is_built()
def available_backends() -> list[str]:
"""按优先级返回当前可用的后端,末尾必定有 cpu。"""
backends = []
if _cuda_ready():
backends.append("cuda")
if _mps_ready():
backends.append("mps")
backends.append("cpu")
return backends
def resolve_device(spec: str = "auto") -> torch.device:
"""把配置里的 device 字符串解析成 `torch.device`。
Args:
spec: "auto" 自动挑最优后端;也可写 "cuda" / "cuda:1" / "mps" / "cpu"。
Raises:
DeviceNotAvailableError: 显式请求的后端不可用。
""" """
import torch spec = str(spec).strip().lower()
if spec in ("auto", ""):
return torch.device(available_backends()[0])
if device.startswith("cuda") and not torch.cuda.is_available(): backend = spec.split(":", 1)[0]
raise NoNvidiaGpuError(f"请求了 {device},但当前设备没有可用的 Nvidia GPU。") if backend == "cuda" and not _cuda_ready():
return torch.device(device) raise DeviceNotAvailableError(
f"请求了 {spec},但当前没有可用的 Nvidia GPU"
f"(PyTorch {torch.__version__},CUDA 构建:{torch.version.cuda})。"
f"可用后端:{available_backends()},或用 device=auto 自动选择。"
)
if backend == "mps" and not _mps_ready():
raise DeviceNotAvailableError(
f"请求了 {spec},但当前 PyTorch 不支持 MPS"
f"(available={torch.backends.mps.is_available()}, built={torch.backends.mps.is_built()})。"
f"可用后端:{available_backends()}。"
)
if backend not in ("cuda", "mps", "cpu"):
raise DeviceNotAvailableError(f"未知设备 {spec},支持 cuda / mps / cpu / auto。")
return torch.device(spec)
def describe_device(device: torch.device | str) -> str:
"""给日志用的人类可读描述。"""
device = torch.device(device)
if device.type == "cuda":
index = device.index or 0
return f"{torch.cuda.get_device_name(index)} (cuda:{index})"
if device.type == "mps":
return "Apple Silicon GPU (Metal / MPS)"
return "CPU"
def amp_supported(device: torch.device | str) -> bool:
"""该后端是否支持 autocast 混合精度。"""
return torch.device(device).type in ("cuda", "mps")
def default_amp(device: torch.device | str) -> bool:
"""AMP 的默认开关:只有 CUDA 默认开。
MPS 支持 autocast 且数值正常,但实测比 fp32 更慢,所以默认关闭;
需要省显存时可以在配置里显式 `amp: true` 打开。
"""
return torch.device(device).type == "cuda"
def resolve_amp(setting: bool | str | None, device: torch.device | str) -> bool:
"""解析配置里的 `amp` 字段:`auto`/`None` 随后端,`true`/`false` 显式指定。"""
device = torch.device(device)
if setting is None or (isinstance(setting, str) and setting.lower() == "auto"):
return default_amp(device)
if not bool(setting):
return False
return amp_supported(device)
def pin_memory_supported(device: torch.device | str) -> bool:
"""只有 CUDA 的锁页内存有意义;MPS 上开启会告警且无收益。"""
return torch.device(device).type == "cuda"
def synchronize(device: torch.device | str) -> None:
"""等待该设备上的异步算子完成——计时前后必须调用,否则测到的是下发时间。"""
device = torch.device(device)
if device.type == "cuda":
torch.cuda.synchronize(device)
elif device.type == "mps":
torch.mps.synchronize()
def memory_stats(device: torch.device | str) -> dict[str, float]:
"""统一的显存查询,单位 MB。
CUDA 返回 allocated / reserved / peak;MPS 只有当前分配量与驱动占用
(Metal 没有暴露峰值计数器);CPU 返回空字典。
"""
device = torch.device(device)
mb = 1024.0 * 1024.0
if device.type == "cuda":
return {
"allocated_mb": torch.cuda.memory_allocated(device) / mb,
"reserved_mb": torch.cuda.memory_reserved(device) / mb,
"peak_allocated_mb": torch.cuda.max_memory_allocated(device) / mb,
"peak_reserved_mb": torch.cuda.max_memory_reserved(device) / mb,
}
if device.type == "mps":
return {
"allocated_mb": torch.mps.current_allocated_memory() / mb,
"driver_allocated_mb": torch.mps.driver_allocated_memory() / mb,
}
return {}
class PeakMemory:
"""跨后端的显存峰值统计。
CUDA 有内置峰值计数器,直接读;MPS 没有,只能在 `update()` 时采样取最大值——
因此 MPS 上的峰值是**下界**(采样点之间的瞬时尖峰会被漏掉)。
"""
def __init__(self, device: torch.device | str) -> None:
self.device = torch.device(device)
self._peak = 0.0
self.reset()
def reset(self) -> None:
self._peak = 0.0
if self.device.type == "cuda":
torch.cuda.reset_peak_memory_stats(self.device)
self.update()
def update(self) -> None:
if self.device.type != "mps":
return
self._peak = max(self._peak, torch.mps.current_allocated_memory() / (1024.0 * 1024.0))
@property
def peak_mb(self) -> float:
if self.device.type == "cuda":
return torch.cuda.max_memory_allocated(self.device) / (1024.0 * 1024.0)
if self.device.type == "mps":
self.update()
return self._peak
return 0.0
@property
def reserved_mb(self) -> float:
"""CUDA 是缓存分配器保留量;MPS 是 Metal 驱动占用;CPU 为 0。"""
if self.device.type == "cuda":
return torch.cuda.max_memory_reserved(self.device) / (1024.0 * 1024.0)
if self.device.type == "mps":
return torch.mps.driver_allocated_memory() / (1024.0 * 1024.0)
return 0.0
@property
def exact(self) -> bool:
"""峰值是否精确(MPS 靠采样,只能给下界)。"""
return self.device.type != "mps"