linux cuda+mkl+magma安装测试

目的：

在linux服务器上安装基于cuda平台的magma计算库

硬件：

nvidia V100 GPU

环境：

ubuntu18.04、cuda tookit 11.4

步骤

1. intel mkl下载安装：

如果不用cpu计算的话，MKL库似乎不是必须的，我没试过，欢迎补充评论
MKL链接
在右边选择 Download stand-alone version，下载选项选择：linux，online&offline(recommanded)，offline（也可以根据自己的情况选择），
右边会出现Command Line Download提示，可以用命令行下载，也可以手动下载（建议迅雷）
下载完成后，随便放到哪个文件夹下，安装提示的命令sudo安装即可，安装后的路径默认为：/opt/intel/oneapi/mkl/latest，头文件和库文件都在这个文件夹下面
修改运行库环境变量（建议写到~/.bashrc中）
export LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH

2. magma下载安装：

magema链接
建议下载使用和我一样的版本：2.5.4
下载完成后解压：tar -zxvf magma-2.5.4.tar.gz
压缩包里包含了README安装指导和make.inc-examples编译配置示例
在解压后的主目录新建一个make.inc文件，文件内容可以从make.inc-examples选择需要的配置文件作为模板填写，作为参考，我的文件内容为：

#//////////////////////////////////////////////////////////////////////////////
#   -- MAGMA (version 2.5.4) --
#      Univ. of Tennessee, Knoxville
#      Univ. of California, Berkeley
#      Univ. of Colorado, Denver
#      @date October 2020
#//////////////////////////////////////////////////////////////////////////////

# GPU_TARGET contains one or more of Fermi, Kepler, Maxwell, Pascal, Volta, Turing, or Ampere
# to specify for which GPUs you want to compile MAGMA:
#     Fermi          - NVIDIA compute capability 2.x cards
#     Kepler         - NVIDIA compute capability 3.x cards
#     Maxwell        - NVIDIA compute capability 5.x cards
#     Pascal         - NVIDIA compute capability 6.x cards
#     Volta/Turing   - NVIDIA compute capability 7.x cards
#     Ampere         - NVIDIA compute capability 8.x cards
# The default is "Kepler Maxwell Pascal".
# Note that NVIDIA no longer supports 1.x cards, as of CUDA 6.5.
# See http://developer.nvidia.com/cuda-gpus
#

GPU_TARGET = Volta

# --------------------
# programs

CC        = gcc
CXX       = g++
NVCC      = nvcc
FORT      = gfortran

ARCH      = ar
ARCHFLAGS = cr
RANLIB    = ranlib


# --------------------
# flags

# Use -fPIC to make shared (.so) and static (.a) library;
# can be commented out if making only static library.
FPIC      = -fPIC

CFLAGS    = -O3 $(FPIC) -fopenmp -DNDEBUG -DADD_ -Wall -Wno-strict-aliasing -Wshadow -DMAGMA_WITH_MKL
FFLAGS    = -O3 $(FPIC)          -DNDEBUG -DADD_ -Wall -Wno-unused-dummy-argument
F90FLAGS  = -O3 $(FPIC)          -DNDEBUG -DADD_ -Wall -Wno-unused-dummy-argument -x f95-cpp-input
NVCCFLAGS = -O3                  -DNDEBUG -DADD_ -Xcompiler "$(FPIC) -Wall -Wno-unused-function -Wno-strict-aliasing" -std=c++11
LDFLAGS   =     $(FPIC) -fopenmp

# C++11 (gcc >= 4.7) is not required, but has benefits like atomic operations
CXXFLAGS := $(CFLAGS) -std=c++11
CFLAGS   += -std=c99


# --------------------
# libraries

# see MKL Link Advisor at http://software.intel.com/sites/products/mkl/
# gcc/gfortran with MKL 10.3, GNU OpenMP threads (use -fopenmp in CFLAGS, LDFLAGS)
LIB       = -lmkl_gf_lp64 -lmkl_gnu_thread -lmkl_core -lpthread -lstdc++ -lm -lgfortran

# Supposedly, gcc can use Intel threads (libiomp5) instead, but be careful that
# libiomp5 and libgomp are NOT BOTH linked. Above, we use gnu threads as a safer option.
# gcc/gfortran with MKL 10.3, Intel OpenMP threads (remove -fopenmp from LDFLAGS above)
#LIB       = -lmkl_gf_lp64 -lmkl_intel_thread -lmkl_core -liomp5 -lpthread -lstdc++ -lm -lgfortran

LIB      += -lcublas -lcusparse -lcudart -lcudadevrt


# --------------------
# directories

# define library directories preferably in your environment, or here.
# for MKL run, e.g.: source /opt/intel/composerxe/mkl/bin/mklvars.sh intel64

MKLROOT = /opt/intel/oneapi/mkl/2022.1.0
CUDADIR = /usr/local/cuda
-include make.check-mkl
-include make.check-cuda

LIBDIR    = -L$(CUDADIR)/lib64 \
            -L$(MKLROOT)/lib/intel64

INC       = -I$(CUDADIR)/include \
            -I$(MKLROOT)/include

编译之前确认一下以下的环境变量

export PATH=/usr/local/cuda/bin/:$PATH
export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH
export CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda
export CUDA_BIN_PATH=/usr/local/cuda
export CUDA_PATH=/usr/local/cuda

然后开始编译：

make lib -j
sudo make install prefix=/usr/local/magma -j
# make clean # 清理不清理随意

编译安装完成后，magma就被安装到了/usr/local/magma目录下，同样，设置环境变量
export LD_LIBRARY_PATH=/usr/local/magma/lib:$LD_LIBRARY_PATH

这样magma库就可以使用了，附带一个svd的例子（例子中的矩阵被我放在了文件里，copy时记得替换掉这部分代码），可以作为验证：
编译命令(有亿点点长，实际使用magma时建议使用makefile、cmake等方式)为：

nvcc ma_svd.cu -o ma_svd -L/usr/local/magma/lib -lmagma_sparse -lmagma -Xcompiler -fopenmp -L/usr/local/cuda/lib64 -L/opt/intel/oneapi/mkl/latest/lib/intel64 -lmkl_gf_lp64 -lmkl_gnu_thread -lmkl_core -lpthread -lstdc++ -lm -lgfortran -lcublas -lcusparse -lcudart -lcudadevrt -I/usr/local/magma/include -DNDEBUG -DADD_ -DMIN_CUDA_ARCH=700 -I/usr/local/cuda/include -I/opt/intel/oneapi/mkl/latest/include -w

代码如下：

#include <iostream>
#include "magma_d.h"
#include "magma.h"
#include <string>
#include <time.h>

using namespace std;

int main(){
    int batch = 1;
    int height = 1024;
    int width = 1024;
    int minmn = height > width ? width : height;
    int shape[3] = {batch, height, width};
    
    double* host_A = (double*)malloc(sizeof(double) * height * width);
    double* host_S = (double*)malloc(sizeof(double) * minmn);
    double* host_U = (double*)malloc(sizeof(double) * height * height);
    double* host_V = (double*)malloc(sizeof(double) * width * width);

    memset(host_S, 0, sizeof(double) * minmn);
    printf("host_S[0]:%lf before\n", host_S[0]);

    string matrix_path1 = "./A_h" + to_string(height) + "_w" + to_string(width)+ ".txt";
    
    // read in host A
    FILE* A_fp = fopen(matrix_path1.data(), "r");
    if(A_fp==NULL){
        printf("open file falied\n");
        return 0;
    }
    for(int i=0; i < height*width; i++){
        fscanf(A_fp, "%lf", &host_A[i]);
    }
    fclose(A_fp);

    double *dev_A;
    cudaMalloc((void **)&dev_A, sizeof(double) * height * width * batch);
    for(int i=0; i<batch; i++){
        cudaMemcpy(dev_A + height*width*i, host_A, sizeof(double) * height * width, cudaMemcpyHostToDevice);
    }

    double *dev_U, *dev_V, *dev_diag;
    cudaMalloc((void **)&dev_diag, sizeof(double) * minmn * batch);
    cudaMalloc((void **)&dev_U, sizeof(double) * height * height * batch);
    cudaMalloc((void **)&dev_V, sizeof(double) * width * width * batch);

    double* workspace = (double*)malloc(sizeof(double) * 1);
    int info = 0;

    magma_init();    

    // int arch = magma_getdevice_arch();
    // printf("arch:%d\n", arch);   // nvidia V100 compute capability = 7.0.0
    
    // query workspace size
    magma_dgesvd(MagmaSomeVec, MagmaSomeVec, height, width, host_A, 1024, host_S, host_U, 1024, host_V, 1024, workspace, -1, &info);
    
    int lwork = (int)workspace[0];
    printf("info:%d, lwork:%d\n", info, lwork);

    free(workspace);
    workspace = (double*)malloc(sizeof(double) * lwork);

    magma_dgesvd(MagmaSomeVec, MagmaSomeVec, height, width, host_A, 1024, host_S, host_U, 1024, host_V, 1024, workspace, lwork, &info);

    printf("info:%d, host_S[0]:%lf after\n", info, host_S[0]);
    return 0;
}