8eee23c8f98660a64fcc65de02c26e01493b8708,torchtext/experimental/datasets/raw/translation.py,,_setup_datasets,#,117

Before Change

            extracted_files.extend(extract_archive(dataset_tar))
    elif isinstance(URLS[dataset_name], str):
        dataset_tar = download_from_url(URLS[dataset_name], root=root, hash_value=MD5[dataset_name], hash_type="md5")
        extracted_files.extend(extract_archive(dataset_tar))
    else:
        raise ValueError(
            "URLS for {} has to be in a form or list or string".format(
                dataset_name))

After Change

        if dataset_name == "IWSLT":
            // IWSLT dataset"s url downloads a multilingual tgz.
            // We need to take an extra step to pick out the specific language pair from it.
            src_language = train_filenames[0].split(".")[-1]
            tgt_language = train_filenames[1].split(".")[-1]
            languages = "-".join([src_language, tgt_language])
            iwslt_tar = ".data/2016-01/texts/{}/{}/{}.tgz"
            iwslt_tar = iwslt_tar.format(
                src_language, tgt_language, languages)
            extracted_dataset_tar = extract_archive(iwslt_tar)
        extracted_files.extend(extracted_dataset_tar)
    else:
        raise ValueError(
            "URLS for {} has to be in a form or list or string".format(
                dataset_name))

Instances