50837ed17dbd9e74af2f01a3255cf3148ead1f4a,sklearn/sklearn-template/template/trainer/utils.py,,read_df_from_gcs,#,79

Before Change

  df_list = []

  // Use "application default credentials"
  storage_client = storage.Client()

  // Parse bucket name and file pattern
  parse_result = urlparse(file_pattern)
  file_pattern = parse_result.path
  bucket_name = parse_result.hostname

  bucket = storage_client.get_bucket(bucket_name)

  // Find all the files match the provided pattern
  blobs = bucket.list_blobs(prefix=file_pattern)

  for blob in blobs:
    file_path = temp_folder + blob.name.split("/")[-1]
    blob.download_to_filename(file_path)
    // Assume there is no header
    df_list.append(pd.read_csv(file_path, header=None))
    // TODO: Can remove after download to save space

  data_df = pd.concat(df_list)

  return data_df

After Change

  // Download the files to local /tmp/ foler
  df_list = []

  for file in gfile.Glob(file_pattern):
    with gfile.Open(file, "r") as f:
      // Assume there is no header
      df_list.append(pd.read_csv(f, header=None))

  data_df = pd.concat(df_list)

  return data_df

Instances