c3031320afca012e143e2df46f9fe4ed3f3d18d3,ludwig/datasets/mixins/process.py,MultifileJoinProcessMixin,process_downloaded_dataset,#MultifileJoinProcessMixin#,54

Before Change

            all_files.append(file_df)

        concat_df = pd.concat(all_files, ignore_index=True)
        if not os.path.exists(self.processed_dataset_path):
            os.makedirs(self.processed_dataset_path)
        concat_df.to_csv(
            os.path.join(self.processed_dataset_path, self.csv_filename),
            index=False)

After Change

            elif split_name == "test_file":
                file_df["split"] = 2
            else:
                raise ValueError(f"Unrecognized split name: {split_name}")
            all_files.append(file_df)

        concat_df = pd.concat(all_files, ignore_index=True)
        os.makedirs(self.processed_dataset_path, exist_ok=True)

Instances