19173aa4370e36cba96ee7049eaaa0dceda5007c,tools/nni_gpu_tool/gpu_metrics_collector.py,,main,#Any#,43

Before Change

    if check_ready_to_run() == False:
        // GPU metrics collector is already running. Exit
        exit(2)
    with open(os.path.join(metrics_output_dir, "gpu_metrics"), "w") as outputFile:
        pass
    os.chmod(os.path.join(metrics_output_dir, "gpu_metrics"), 0o777)
    cmd = "nvidia-smi -q -x"
    while(True):
        try:

After Change

        except Exception:
            traceback.print_exc()
            gen_empty_gpu_metric(metrics_output_dir)
            break
        parse_nvidia_smi_result(smi_output, metrics_output_dir)
        // TODO: change to sleep time configurable via arguments
        time.sleep(5)

Instances