fix: ship gpu_free_memory correctly
This fixes a typo where gpu_util was shipped as both gpu_util and gpu_free_memory.
@@ -385,6 +385,7 @@ class SysMetricCollectorThread(threading.Thread):
- DiskReadThroughput = Measured in bytes/second
- DiskWriteThroughput = Measured in bytes/second
- GpuUtilization = Measured in percent
+ - GpuFreeMemory = Measured in Gigabytes
"""
FLUSH_INTERVAL = 10 # How often to make API calls
@@ -486,7 +487,7 @@ class SysMetricCollectorThread(threading.Thread):
gpu_memory = gpu_memory_collection.measure(self.current_batch_idx)
for gpu_uuid in gpu_memory.keys():
self.current_batch.add_gpu_measurement(
- SysMetricType.GPU_FREE_MEMORY_METRIC, gpu_uuid, gpu_util[gpu_uuid]
+ SysMetricType.GPU_FREE_MEMORY_METRIC, gpu_uuid, gpu_memory[gpu_uuid]
)
# Check if it is time to flush the batch and start a new batch
@@ -909,7 +910,7 @@ class GpuMemoryCollector:
gpu_uuid = pynvml.nvmlDeviceGetUUID(handle)
try:
info = pynvml.nvmlDeviceGetMemoryInfo(handle)
- measurements[gpu_uuid] = Measurement(timestamp, batch_idx, info.free)
+ measurements[gpu_uuid] = Measurement(timestamp, batch_idx, info.free / GIGA)
except pynvml.NVMLError as e:
logging.info(f"{LOG_NAMESPACE}: failed to sample GPU memory for GPU {i}: {e}")
return measurements
