GPU & CUDA 粗浅理解(二)（草稿）

#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>

const long long tdelay=1000000LL; //定义了GPU内核中循环等待的时间（以GPU时钟周期为单位）。
const int loops = 10000; //定义了CPU循环执行的次数。
const int hdelay = 1; //定义了CPU在每次循环中调用usleep()时的延迟时间（单位是微秒）。

__global__ void dkern(){ //定义了一个kernel，它通过clock64()函数获取当前的GPU时钟，然后在一个while循环中一直等待，直到经过了tdelay的时间。

  long long start = clock64();
  while(clock64() < start+tdelay);
}

int main(int argc, char *argv[]){

  int my_delay = hdelay;
  if (argc > 1) my_delay = atoi(argv[1]); // 解析参数，允许用户通过命令行设置my_delay的值。
  for (int i = 0; i<loops; i++){ // 使用一个for循环，循环次数为loops，在每次循环中启动dkern内核，然后调用usleep()引入延迟。
    dkern<<<1,1>>>();
    usleep(my_delay);}

  return 0;
}

nvcc -o testUtil testUtil.cu

nsys profile --trace=cuda --stats=true ./your_application

nsys profile --gpu-metrics-device=N 

import sys 
import json 
import pandas as pd 
from nsys_recipe import data_service, log 
from nsys_recipe.data_service import DataService 
 
# To run this script, be sure to add the Nsight Systems package directory to your PYTHONPATH, similar to this: 
# export PYTHONPATH=/opt/nvidia/nsight-systems/2023.4.1/target-linux-x64/python/packages 
 
def compute_utilization(filename, freq=10000): 
    service=DataService(filename) 
    table_column_dict = { 
        "GPU_METRICS": ["typeId", "metricId", "value"], 
        "TARGET_INFO_GPU_METRICS": ["metricName", "metricId"], 
        "META_DATA_CAPTURE": ["name", "value"] 
    } 
    hints={"format":"sqlite"} 
    df_dict = service.read_tables(table_column_dict, hints=hints) 
    df = df_dict.get("GPU_METRICS", None) 
    if df is None: 
        print(f"{filename} does not contain GPU metric data.") 
        return
    tgtinfo_df = df_dict.get("TARGET_INFO_GPU_METRICS", None) 
    if tgtinfo_df is None: 
        print(f"{filename} does not contain TARGET_INFO_GPU_METRICS table.") 
        return
    metadata_df = df_dict.get("META_DATA_CAPTURE", None) 
    if metadata_df is not None: 
        if "GPU_METRICS_OPTIONS:SAMPLING_FREQUENCY" in metadata_df['name'].values: 
            report_freq = metadata_df.loc[ metadata_df['name']=='GPU_METRICS_OPTIONS:SAMPLING_FREQUENCY']['value'].iat[0] 
            if isinstance(report_freq, (int,float)): 
                freq = report_freq 
                print("Setting GPU Metric sample frequency to value in report file. new frequency=",freq)                 
      
    possible_smactive=['SMs Active', 'SM Active', 'SM Active [Throughput %]'] 
    smactive_name_mask = tgtinfo_df['metricName'].isin(possible_smactive) 
    smactive_row = tgtinfo_df[smactive_name_mask] 
    smactive_name = smactive_row['metricName'].iat[0] 
    smactive_id = tgtinfo_df.loc[tgtinfo_df['metricName']==smactive_name,'metricId'].iat[0] 
    smactive_df = df.loc[ df['metricId'] == smactive_id ] 
      
    usage = smactive_df['value'].sum() 
    count = len(smactive_df['value']) 
    count_nonzero = len(smactive_df.loc[smactive_df['value']!=0]) 
    avg_gross_util = usage/count 
    avg_net_util = usage/count_nonzero 
    effective_util = usage/freq/100
      
    print(f"Avg gross GPU utilization:\t%lf %%" % avg_gross_util) 
    print(f"Avg net GPU utilization:\t%lf %%" % avg_net_util) 
    print(f"Effective GPU utilization time:\t%lf s" % effective_util) 
    return metadata_df 
      
if __name__ == '__main__': 
    if len(sys.argv)==2: 
        compute_utilization(sys.argv[1]) 
    elif len(sys.argv)==3: 
        compute_utilization(sys.argv[1], freq=float(sys.argv[2])) 

sudo systemctl start nvidia-dcgm
# 通过 -l 了解有哪些参数可以查看，其中10002、10003分别对应
dcgmi dmon -l
# 查看列表发现1002、1003对应于SM occupation，是我们要的指标
dcgmi dmon -e 1002

# 也可以针对指定程序监控
# 启动监控
dcgmi stats --host 127.0.0.1 -g 0 -e
# 暴露某个程序的信息
dcgmi stats --host 127.0.0.1 -g 0 -p <pid> -v