CUDA报错802解决

1、cudaErrorSystemNotReady (802)

前置检查：
lsmod | grep ib_umad   #检查ib模块是否已加载
modprobe ib_uverbs   #如未加载需要手动加载
此报错是Fabric未安装或者未运行
可用 nvidia-smi nvlink -e 查看状态
nvidia-smi -q | grep -A 3 "Fabric" #确认所有 8 张都是 Completed
最后可用 nvidia-smi -r   #强制重置GPU

2、调试cuda是否正常的脚本

python3 -c "
import ctypes
import ctypes.util

# 加载 CUDA Runtime 库
lib_path = ctypes.util.find_library('cudart')
if not lib_path:
    lib_path = '/usr/local/cuda/lib64/libcudart.so'
libcudart = ctypes.CDLL(lib_path)

# 定义函数原型
cudaRuntimeGetVersion = libcudart.cudaRuntimeGetVersion
cudaRuntimeGetVersion.argtypes = [ctypes.POINTER(ctypes.c_int)]
cudaRuntimeGetVersion.restype = ctypes.c_int

# 调用
version = ctypes.c_int()
err = cudaRuntimeGetVersion(ctypes.byref(version))

if err == 0:
    major = version.value // 1000
    minor = (version.value % 1000) // 10
    print(f'CUDA Runtime version: {major}.{minor} (raw: {version.value})')
    print('SUCCESS: CUDA Runtime is ready!')
else:
    print(f'ERROR: cudaRuntimeGetVersion returned code {err}')
"