6f39f48519a3e4fdc3a1e44b86dba00b3fcbbae3,open_seq2seq/data/text2text/text2text.py,ParallelTextDataLayer,build_graph,#ParallelTextDataLayer#,149

Before Change

      .map(lambda tokens: (tokens, tf.size(tokens)),
           num_parallel_calls=self._map_parallel_calls)

    _src_tgt_dataset = tf.data.Dataset.zip((_sources, _targets)).filter(
      lambda t1, t2: tf.logical_and(tf.less_equal(t1[1], self.max_len),
                                    tf.less_equal(t2[1], self.max_len))
    )

    if self._num_workers > 1:
      _src_tgt_dataset = _src_tgt_dataset\
        .shard(num_shards=self._num_workers, index=self._worker_id)


    if self.params["shuffle"]:
      _src_tgt_dataset = _src_tgt_dataset\
        .shuffle(buffer_size=self.get_size_in_samples())
    else:
      _src_tgt_dataset = _src_tgt_dataset

    if self.params["repeat"]:
      _src_tgt_dataset = _src_tgt_dataset.repeat()

    self.batched_dataset = _src_tgt_dataset.padded_batch(
      self._batch_size,

After Change

      .map(lambda tokens: (tokens, tf.size(tokens)),
           num_parallel_calls=self._map_parallel_calls)

    _src_tgt_dataset = tf.data.Dataset.zip((_sources, _targets)).filter(
      lambda t1, t2: tf.logical_and(tf.less_equal(t1[1], self.max_len),
                                    tf.less_equal(t2[1], self.max_len))
    ).cache()

    if self._num_workers > 1:
      _src_tgt_dataset = _src_tgt_dataset\
        .shard(num_shards=self._num_workers, index=self._worker_id)


    if self.params["shuffle"]:
      _src_tgt_dataset = _src_tgt_dataset\
        .shuffle(buffer_size=self.get_size_in_samples())
    else:
      _src_tgt_dataset = _src_tgt_dataset

    if self.params["repeat"]:
      _src_tgt_dataset = _src_tgt_dataset.repeat()

    self.batched_dataset = _src_tgt_dataset.padded_batch(
      self._batch_size,

Instances