70a99a9759c8fb8d5731270639ebf95dd0e02a04,preprocess.py,,build_save_in_shards_using_shards_size,#Any#Any#Any#Any#Any#,49

Before Change

            tgt_data = ftgt.readlines()

            src_corpus = "".join(src_corpus.split(".")[:-1])
            tgt_corpus = "".join(tgt_corpus.split(".")[:-1])

            num_shards = int(len(src_data) / opt.shard_size)
            for x in range(num_shards):
                f = codecs.open(src_corpus + ".{0}.txt".format(x), "w",

After Change

    with codecs.open(src_corpus, "r", encoding="utf-8") as fsrc:
        with codecs.open(tgt_corpus, "r", encoding="utf-8") as ftgt:
            logger.info("Reading source and target files: %s %s."
                        % (src_corpus, tgt_corpus))
            src_data = fsrc.readlines()
            tgt_data = ftgt.readlines()

            num_shards = int(len(src_data) / opt.shard_size)

Instances