46e35f8597e2a8f161ee6fefe0e23635694c241b,onmt/IO.py,TextDataset,_process_corpus,#TextDataset#,515

Before Change

        // If tgt_path exists, then we need to do the same thing as we did
        // for the source data
        if tgt_path is not None:
            tgt_examples = _read_text_file(tgt_path, tgt_seq_length_trunc,
                                           "tgt")
            (_, tgt_feats), tgt_examples = _peek(tgt_examples)
            tgt_examples = (ex for ex, nfeats in tgt_examples)
            self.n_tgt_feats = tgt_feats
        else:
            tgt_examples = None
            self.n_tgt_feats = 0

        // examples: one for each src line or (src, tgt) line pair.
        // Each element is a dictionary whose keys represent at minimum
        // the src tokens and their indices and potentially also the
        // src and tgt features and alignment information.
        if tgt_examples is not None:
            examples = (_join_dicts(src, tgt)
                        for src, tgt in zip(src_examples, tgt_examples))
        else:
            examples = src_examples

        if dynamic_dict:
            examples = self._dynamic_dict(examples)

        // Peek at the first to see which fields are used.
        ex, examples = _peek(examples)
        keys = ex.keys()

        out_fields = [(k, fields[k]) if k in fields else (k, None)
                      for k in keys]
        example_values = ([ex[k] for k in keys] for ex in examples)
        out_examples = (_construct_example_fromlist(ex_values, out_fields)
                        for ex_values in example_values)

        def filter_pred(example):
            return 0 < len(example.src) <= src_seq_length \

After Change

            _make_example(src_path, src_seq_length_trunc, "src")

        tgt_examples, self.n_tgt_feats = \
            _make_example(tgt_path, tgt_seq_length_trunc, "tgt")

        // examples: one for each src line or (src, tgt) line pair.
        // Each element is a dictionary whose keys represent at minimum
        // the src tokens and their indices and potentially also the
        // src and tgt features and alignment information.
        if tgt_examples is not None:
            examples = (_join_dicts(src, tgt)
                        for src, tgt in zip(src_examples, tgt_examples))
        else:
            examples = src_examples

        if dynamic_dict:
            examples = self._dynamic_dict(examples)

        // Peek at the first to see which fields are used.
        ex, examples = _peek(examples)
        keys = ex.keys()

        out_fields = [(k, fields[k]) if k in fields else (k, None)
                      for k in keys]
        example_values = ([ex[k] for k in keys] for ex in examples)
        out_examples = (_construct_example_fromlist(ex_values, out_fields)
                        for ex_values in example_values)

        def filter_pred(example):
            return 0 < len(example.src) <= src_seq_length \

Instances