一个cuda profile的原理例子

概要

下面给出一个可编译的最小示例，演示如何用 启动器 + 注入代理库 + collector 的方式在不改动目标可执行的情况下采集 CUPTI activity buffer 并把数据发回采集进程。示例尽量精简以便能在真实环境中编译运行并验证数据流通路。示例仅用于学习与验证，生产环境需做更多健壮性、权限与性能处理。

 

提供的文件

  • launcher.sh 启动脚本，设置环境并启动 collector，然后 exec 目标程序（通过 LD_PRELOAD 注入代理库）。

  • collector.c 简单的 Unix domain socket 服务端，接收代理发来的 activity buffer 并把长度与前几个字节打印出来（示例用途）。

  • libcupti_agent.c 注入代理库（shared object），在库加载时初始化 CUPTI、注册缓冲回调并在 bufferCompleted 中把 buffer 发回 collector。

 

代码 文件 1 launcher.sh

bash
 
#!/bin/bash
# launcher.sh
# Usage: ./launcher.sh /path/to/your_app [args...]
set -e

OUT_DIR=/tmp/cupts_capture
mkdir -p "$OUT_DIR"

# Socket path for IPC between agent and collector
SOCK="$OUT_DIR/cuptsock.sock"
rm -f "$SOCK"

# Path to the agent shared library (adjust as needed)
AGENT_LIB="$(pwd)/libcupti_agent.so"
if [ ! -f "$AGENT_LIB" ]; then
  echo "Agent library not found: $AGENT_LIB"
  exit 1
fi

# Start collector in background
./collector "$SOCK" "$OUT_DIR" &
COLLECTOR_PID=$!

# Export socket path for agent to connect
export CUPTI_AGENT_SOCK="$SOCK"

# Inject agent via LD_PRELOAD and exec target program
export LD_PRELOAD="$AGENT_LIB:$LD_PRELOAD"
# Optionally set CUPTI activity kinds via env (agent reads CUPTI_ACTIVITY_KIND if needed)
export CUPTI_ACTIVITY_KIND="KERNEL,MEMCPY"

# Exec the target program so it inherits LD_PRELOAD
exec "$@"

# Note: collector will exit when launcher/target process ends; you can also kill it manually.
 

代码 文件 2 collector.c

c
 
// collector.c
// Simple unix domain socket collector that receives buffers from agent
// Build: gcc collector.c -o collector

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <stdint.h>
#include <string.h>
#include <errno.h>

int main(int argc, char **argv) {
  if (argc < 3) {
    fprintf(stderr, "Usage: %s <socket_path> <out_dir>\n", argv[0]);
    return 1;
  }
  const char *sockpath = argv[1];
  const char *outdir = argv[2];

  int listen_fd = socket(AF_UNIX, SOCK_STREAM, 0);
  if (listen_fd < 0) { perror("socket"); return 1; }

  struct sockaddr_un addr;
  memset(&addr, 0, sizeof(addr));
  addr.sun_family = AF_UNIX;
  strncpy(addr.sun_path, sockpath, sizeof(addr.sun_path)-1);
  unlink(sockpath);

  if (bind(listen_fd, (struct sockaddr*)&addr, sizeof(addr)) < 0) { perror("bind"); return 1; }
  if (listen(listen_fd, 1) < 0) { perror("listen"); return 1; }

  printf("Collector listening on %s\n", sockpath);
  int conn = accept(listen_fd, NULL, NULL);
  if (conn < 0) { perror("accept"); return 1; }
  printf("Agent connected\n");

  // Receive loop: first 8 bytes length, then data
  while (1) {
    uint64_t len = 0;
    ssize_t r = read(conn, &len, sizeof(len));
    if (r == 0) { printf("Agent closed connection\n"); break; }
    if (r < 0) { perror("read len"); break; }
    if (r != sizeof(len)) { fprintf(stderr, "Partial read len\n"); break; }
    len = (uint64_t)len;
    if (len == 0) continue;
    uint8_t *buf = malloc(len);
    uint64_t got = 0;
    while (got < len) {
      ssize_t n = read(conn, buf + got, len - got);
      if (n <= 0) { perror("read data"); goto cleanup; }
      got += n;
    }
    // For demo, print first few bytes and length
    printf("Received buffer len=%llu first4=0x%02x%02x%02x%02x\n",
           (unsigned long long)len,
           buf[0], buf[1], buf[2], buf[3]);
    // Optionally write raw buffer to file for offline parsing
    static int idx = 0;
    char fname[512];
    snprintf(fname, sizeof(fname), "%s/activity_%04d.bin", outdir, idx++);
    FILE *f = fopen(fname, "wb");
    if (f) {
      fwrite(buf, 1, len, f);
      fclose(f);
    }
    free(buf);
  }

cleanup:
  close(conn);
  close(listen_fd);
  unlink(sockpath);
  return 0;
}
 

代码 文件 3 libcupti_agent.c

c
 
// libcupti_agent.c
// Minimal CUPTI agent library that registers buffer callbacks and sends buffers to collector
// Build: gcc -shared -fPIC libcupti_agent.c -o libcupti_agent.so -I${CUDA_HOME}/extras/CUPTI/include -L${CUDA_HOME}/extras/CUPTI/lib64 -lcupti -ldl -lpthread
// Note: This example omits robust error handling for brevity.

#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <unistd.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <pthread.h>
#include <cupti.h>

static int sockfd = -1;
static pthread_mutex_t sock_lock = PTHREAD_MUTEX_INITIALIZER;

static void open_ipc() {
  const char *sockpath = getenv("CUPTI_AGENT_SOCK");
  if (!sockpath) sockpath = "/tmp/cuptsock.default";
  sockfd = socket(AF_UNIX, SOCK_STREAM, 0);
  if (sockfd < 0) {
    perror("agent socket");
    return;
  }
  struct sockaddr_un addr;
  memset(&addr, 0, sizeof(addr));
  addr.sun_family = AF_UNIX;
  strncpy(addr.sun_path, sockpath, sizeof(addr.sun_path)-1);
  // Try connect; collector should be listening
  if (connect(sockfd, (struct sockaddr*)&addr, sizeof(addr)) < 0) {
    perror("agent connect");
    close(sockfd);
    sockfd = -1;
  }
}

// bufferRequested callback signature
static void bufferRequested(uint8_t **buffer, size_t *size, size_t *maxNumRecords) {
  // allocate a reasonably large buffer
  *size = 8 * 1024 * 1024; // 8MB
  *buffer = (uint8_t*)malloc(*size);
  *maxNumRecords = 0;
}

// bufferCompleted callback signature
static void bufferCompleted(CUcontext ctx, uint32_t streamId, uint8_t *buffer, size_t size, size_t validSize) {
  (void)ctx; (void)streamId; (void)size;
  if (validSize == 0) {
    free(buffer);
    return;
  }
  // Send length then data to collector (synchronously for simplicity)
  pthread_mutex_lock(&sock_lock);
  if (sockfd >= 0) {
    uint64_t len = (uint64_t)validSize;
    ssize_t w = write(sockfd, &len, sizeof(len));
    if (w == sizeof(len)) {
      ssize_t sent = 0;
      while (sent < (ssize_t)len) {
        ssize_t n = write(sockfd, buffer + sent, len - sent);
        if (n <= 0) break;
        sent += n;
      }
    }
  }
  pthread_mutex_unlock(&sock_lock);
  free(buffer);
}

__attribute__((constructor))
static void agent_init(void) {
  // Open IPC to collector
  open_ipc();

  // Initialize CUPTI activity callbacks
  CUptiResult res;
  res = cuptiActivityRegisterCallbacks(bufferRequested, bufferCompleted);
  if (res != CUPTI_SUCCESS) {
    const char *s; cuptiGetResultString(res, &s);
    fprintf(stderr, "cuptiActivityRegisterCallbacks failed: %s\n", s);
    return;
  }

  // Enable a couple of activity kinds
  cuptiActivityEnable(CUPTI_ACTIVITY_KIND_KERNEL);
  cuptiActivityEnable(CUPTI_ACTIVITY_KIND_MEMCPY);

  // Optionally enable other kinds based on env
  // e.g., read CUPTI_ACTIVITY_KIND env and enable accordingly (omitted)
  fprintf(stderr, "CUPTI agent initialized\n");
}

__attribute__((destructor))
static void agent_fini(void) {
  // Flush activity and cleanup
  cuptiActivityFlushAll(0);
  pthread_mutex_lock(&sock_lock);
  if (sockfd >= 0) close(sockfd);
  sockfd = -1;
  pthread_mutex_unlock(&sock_lock);
  fprintf(stderr, "CUPTI agent finalized\n");
}
 

构建命令

在有 CUDA 的机器上，假设 CUDA_HOME 指向 CUDA 安装目录（例如 /usr/local/cuda）：

bash
 
export CUDA_HOME=/usr/local/cuda

# 编译 collector
gcc collector.c -o collector

# 编译 agent shared lib
gcc -shared -fPIC libcupti_agent.c -o libcupti_agent.so \
  -I${CUDA_HOME}/extras/CUPTI/include \
  -L${CUDA_HOME}/extras/CUPTI/lib64 -lcupti -ldl -lpthread

# 使 launcher 可执行
chmod +x launcher.sh
 

运行步骤

  1. 启动目标程序通过 launcher

    bash
     
    ./launcher.sh /path/to/your_app [args...]

    • launcher.sh 会启动 collector 并以 LD_PRELOAD 注入 libcupti_agent.so，然后 exec 目标程序。

  2. 查看 collector 输出

    • collector 会打印收到的 buffer 长度与前几个字节，并把原始 buffer 写入 /tmp/cupts_capture/activity_*.bin。

  3. 验证

    • 在目标程序中执行一些 CUDA 操作（kernel launch、memcpy），collector 应收到 buffer。

    • 若没有数据，检查 stderr 中 agent 初始化输出与 collector 是否成功建立连接，检查 CUPTI 与驱动版本兼容性。

 

重要注意事项与限制

  • 仅作学习示例：示例省略了大量错误处理、重连逻辑、异步上报与高效缓冲管理；生产工具需把 bufferCompleted 的发送改为异步队列并避免阻塞应用线程。

  • 权限与兼容性：LD_PRELOAD 注入对动态链接程序有效；对静态链接或 setuid 程序无效。attach 场景需不同实现且可能需要 root。CUPTI、CUDA Toolkit 与驱动版本必须兼容。

  • 性能影响：同步写 socket 会对应用造成延迟，实际应把数据放入本地环形缓冲并由后台线程批量发送。

  • 时间对齐：示例未做 GPU ticks 到主机时间的映射；实际需要用 CUPTI/驱动提供的映射或多点拟合来对齐时间戳。

  • 安全：不要在不受信任环境下注入库或对生产服务做长时间采集。

 

后续建议

  • 把 bufferCompleted 的发送改为后台线程异步上报并做批量写入以降低对应用的影响。

  • 在 collector 端实现对 CUPTI activity record 的解析（或把原始 buffer 交给离线解析工具），并实现 GPU→主机时间映射与 correlation 关联。

  • 在受控测试上验证 cuptiActivityGetNumDroppedRecords 并根据需要调整 buffer 大小或采集范围。

posted @ 2026-02-24 17:50  huorexiaji  阅读(41)  评论(0)    收藏  举报