[{"test_id":"test-01","name":"A100 Idle Baseline","mean_power_w":62.7,"power_range_w":"62.53 - 62.87","utilization_percent":0,"temperature_c":40,"duration_minutes":10,"samples":601,"start_time":"2026/05/09 03:57:48.161","end_time":"2026/05/09 04:07:46.281","status":"complete"},{"test_id":"test-02","name":"A100 Ghost Power Test","duration_seconds":60,"total_samples":66,"ghost_power_events":1,"ghost_power_w":102.14,"ghost_utilization":0,"idle_power_w":63.4,"load_power_range_w":"397 - 413","load_utilization":"94-96%","status":"complete"},{"test_id":"test-03","name":"A100 Sampling Rate Test","sampling_rates":["1s","100ms","10ms"],"ghost_power_detected":true,"ghost_power_w":73.5,"ghost_utilization":0,"conclusion":"Ghost power persists at all sampling rates","status":"complete"},{"test_id":"test-04","name":"A100 Load Ramp Test","ghost_power_detected":true,"ghost_power_w":73.2,"ghost_power_util":0,"severe_lag_detected":true,"severe_lag_power_w":357.7,"severe_lag_util":0,"peak_power_w":357.7,"peak_util":100,"status":"complete"},{"test_id":"test-05","name":"A100 CEI Compute 2048x2048","mean_tflops":14.35,"mean_cei":14350000000000.0,"mean_kernel_time_ms":1.197,"error_percent":0.15,"iterations":30,"status":"complete"},{"test_id":"test-06","name":"A100 CEI Efficiency (Corrected)","mean_cei":17370000000000.0,"mean_tflops":17.37,"mean_power_w":330,"efficiency_gflops_per_w":52.6,"duration_seconds":60,"status":"complete"},{"test_id":"test-07","name":"A100 CEI Compute 4096x4096","mean_cei":15300000000000.0,"mean_tflops":15.3,"duration_minutes":10,"iterations":60,"status":"complete"},{"test_id":"test-08","name":"A100 FP16 Tensor Core","mean_tflops":231.08,"mean_time_ms":0.5948,"iterations":1440905,"duration_minutes":15,"status":"complete"},{"test_id":"test-09","name":"A100 Normality Test","samples":8247,"mean_cei":19000000000000.0,"skewness":-47.1473,"is_normal":false,"status":"complete"},{"test_id":"test-10","name":"A100 Log-Log Scaling Test","peak_cei":17790000000000.0,"peak_tflops":17.79,"peak_matrix_size":4096,"status":"complete"},{"test_id":"test-11","name":"A100 Observability Validation","duration_minutes":5,"idle_power_w":62.74,"burst_power_range_w":"396-406","burst_utilization":100,"ghost_power_events":0,"samples":3044,"status":"complete"},{"test_id":"test-12","name":"A100 8192 Load Test","gpu_model":"NVIDIA A100 (8192 matrix size)","platform":"RunPod","test_date":"2026-05-11","duration_sec":null,"load_minutes":20,"matrix_size":"8192×8192 FP32","power_w_range":"305–342","utilization_percent":100,"status":"complete"},{"test_id":"test-13","name":"A100 Load + Cooldown (Ghost Power Test)","gpu_model":"NVIDIA A100 SXM (RunPod)","platform":"RunPod","test_date":"2026-05-12","duration_sec":660,"load_phase_sec":300,"cooldown_phase_sec":360,"workload":"4096×4096 FP32 matrix multiplication (torch.matmul)","sampling_rate_hz":1,"power_load_max_W":146.66,"power_load_min_W":77.17,"power_load_mean_W":83.2,"power_cooldown_min_W":66.16,"power_cooldown_max_W":78.64,"power_cooldown_mean_W":77.2,"utilization_load_percent":0,"utilization_cooldown_percent":0,"temperature_load_rise_C":2,"temperature_cooldown_drop_C":2,"ghost_power_detected":true,"ghost_power_evidence":"Power remained at ~77-78W for first 60 seconds of cooldown, then slowly settled to ~66-68W after 5+ minutes, never dropping below 66W. This matches P0 state and 1593 MHz memory clock (inferred from prior tests)."},{"test_id":"test-14","name":"Ghost Power – 10 min Load + 10 min Cooldown","gpu":"NVIDIA A100 SXM (RunPod)","date":"2026-05-13","duration_sec":1200,"load_phase_sec":600,"cooldown_phase_sec":600,"workload":"4096×4096 FP32 torch.matmul, continuous","sampling_rate_hz":1,"power_load_max_W":146.66,"power_load_min_W":66.75,"power_load_mean_W":82.3,"power_cooldown_min_W":66.16,"power_cooldown_max_W":78.37,"power_cooldown_mean_W":72.1,"utilization_load_percent":0,"utilization_cooldown_percent":0,"temperature_load_rise_C":2,"temperature_cooldown_drop_C":2,"ghost_power_detected":true,"conclusion":"Power remained elevated (77–78 W) for first minute of cooldown, then slowly declined to ~66 W but never dropped to a true low‑power idle (e.g., 30 W). Reported utilisation 0% throughout, confirming telemetry desynchronisation.","status":"complete"},{"test_id":"test-15","name":"A100 SXM Idle Baseline (15 min)","gpu":"NVIDIA A100 SXM (RunPod)","duration_sec":900,"samples":900,"mean_power_w":67.1,"power_range_w":"66.5 - 68.0","utilization_percent":0,"temperature_c":27,"status":"complete","notes":"Baseline idle power floor for ghost power subtraction."},{"test_id":"test-16","name":"Remediation (Power Management) – Persistence Mode & Power Cap","gpu":"NVIDIA A100 SXM (RunPod)","date":"2026-05-13","commands_attempted":["nvidia-smi -pm 0","nvidia-smi -pl 200"],"results":{"persistence_mode":"Unknown Error – unable to set","power_cap":"Insufficient Permissions – failed to set","current_power_limit_W":500},"root_cause":"RunPod hypervisor blocks low-level GPU power management from containers.","conclusion":"Remediation not possible on standard RunPod instances. Environment limitation.","status":"Blocked by platform"},{"test_id":"test-17","name":"P-State & Memory Clock Retention","gpu":"NVIDIA A100 SXM (RunPod)","duration_minutes":10,"samples":10,"pstate":"P0 (persistent)","memory_clock_mhz":1593,"finding":"GPU remained in P0 with memory clock locked at 1593 MHz throughout the 10‑minute test after load.","status":"complete"},{"test_id":"test-18","name":"Load Ramp – Power vs Matrix Size","gpu":"NVIDIA A100 SXM (RunPod)","matrix_sizes":[512,1024,2048,4096,6144,8192],"peak_power_w":339.1,"peak_power_size":6144,"idle_power_w":67.1,"conclusion":"Power scales with matrix size, peaking at 6144 before dropping due to memory bandwidth saturation.","status":"complete"},{"test_id":"test-19","name":"FP16 Tensor Core – 10min Continuous Matmul (2048×2048)","gpu":"NVIDIA A100 SXM (RunPod)","duration_sec":600,"average_power_w":482.7,"utilization_range":"0-100%","temperature_c":"27-49","iterations":60000,"flops_per_iter":17170000000.0,"total_flops":1030000000000000.0,"cei_flops_per_joule":3560000000.0,"status":"complete"},{"test_id":"test-20","name":"FP16 vs FP32 – 5 Iterations Each (2048×2048 matmul)","gpu":"NVIDIA A100 SXM (RunPod)","date":"2026/05/13","start_time":"2026/05/13 07:05:09.296","end_time":"2026/05/13 07:05:12.925","status":"complete","fp32":{"iterations":60,"avg_time_ms":2.45,"avg_power_w":98.73,"min_power_w":76.06,"max_power_w":137.28,"avg_utilization_pct":0,"temperature_c":44},"fp16":{"iterations":60,"avg_time_ms":0.52,"avg_power_w":77.61,"min_power_w":76.69,"max_power_w":78.72,"avg_utilization_pct":0.4,"temperature_c":45},"findings":{"fp32_power_ramp_detected":true,"ghost_power_detected":true,"ghost_power_note":"FP32 drawing up to 137.28W at 0% utilization","fp16_faster_ratio":"4.7x faster than FP32","fp16_lower_power":true},"conclusion":"FP16 is 4.7x faster and draws significantly less power. FP32 shows ghost power behavior — increasing wattage at 0% reported utilization."},{"test_id":"test-21","name":"FP32 vs FP16 – 60 Iterations Each (2048×2048 matmul)","gpu":"NVIDIA A100 SXM (RunPod)","iterations":60,"workload":"torch.matmul(2048×2048)","precision_comparison":{"FP32":{"mean_kernel_time_ms":1.44,"stddev_kernel_time_ms":0.12,"power_range_W":"76–137","utilization_range":"0–4%","temperature_C":"44–45"},"FP16":{"mean_kernel_time_ms":0.48,"stddev_kernel_time_ms":0.15,"power_range_W":"76–79","utilization_range":"0–4%","temperature_C":"45"},"speedup":3.0,"notes":"First iteration of each precision includes CUDA warm‑up (63ms for FP32, 135ms for FP16) – excluded from means."},"conclusion":"FP16 is approximately 3× faster than FP32 for this matrix size, with similar board power (≈77W) and near‑zero reported utilisation. The power‑utilisation mismatch persists in both precisions.","status":"complete"},{"test_id":"test-22","name":"Persistence Mode Disable Attempt – RunPod Instance 47cdd71daf80","gpu":"NVIDIA A100 SXM","environment":"RunPod (different host)","date":"2026-05-13","command":"nvidia-smi -pm 0","initial_persistence_mode":"Enabled","command_output":"Disabled persistence mode for GPU 00000000:87:00:00.0. All done.","post_check_persistence_mode":"Enabled (unchanged)","power_after_command_W":66.16,"pstate_after":"P0","memory_clock_after_MHz":1593,"conclusion":"The command claimed success but persistence mode remained Enabled. Power and clocks unchanged (still P0, 1593 MHz). Likely the hypervisor ignored the request or the change was transient.","status":"Inconclusive – platform still blocks effective change"},{"test_id":"test-23","name":"A100 SXM Idle Baseline","gpu":"NVIDIA A100 SXM (RunPod)","duration_sec":900,"samples":900,"mean_power_w":67.1,"power_range_w":"66.5 - 68.0","utilization_percent":0,"temperature_c":27,"status":"complete"},{"test_id":"test-24","name":"CEI Validation – 15 min Continuous FP32 Matmul (2048×2048)","gpu":"NVIDIA A100 SXM (RunPod)","duration_sec":900,"iterations":90000,"average_power_w":302.37,"total_flops":1550000000000000.0,"total_energy_j":272130,"cei_flops_per_joule":5680000000.0,"workload":"2048×2048 FP32 torch.matmul, 100 matmuls per second","sampling_rate_hz":1,"status":"complete","notes":"Sustained workload efficiency metric. Lower than per‑kernel approximation due to idle time and power sampling integration."}]