52dd8f17b382dea2ddaf3b4054d7845c8c3b4f72,pycorrector/seq2seq/fce_reader.py,FCEReader,read_samples_by_string,#FCEReader#,26

Before Change


    def read_samples_by_string(self, path):
        for tokens in self.read_tokens(path):
            source = []
            target = []
            for token in tokens:
                target.append(token)
                if self.config.enable_data_dropout:
                    // Random dropout words from the input
                    dropout_token = (token in FCEReader.DROPOUT_TOKENS and random.random() < self.dropout_prob)
                    replace_token = (token in FCEReader.REPLACEMENTS and random.random() < self.replacement_prob)

After Change


            line_src = f.readline()
            line_dst = f.readline()
            if line_src and line_dst:
                source = line_src.lower()[5:].strip().split()
                target = line_dst.lower()[5:].strip().split()
                if self.config.enable_data_dropout:
                    new_source = []
                    for token in source:
                        // Random dropout words from the input
                        dropout_token = (token in FCEReader.DROPOUT_TOKENS and random.random() < self.dropout_prob)
                        replace_token = (token in FCEReader.REPLACEMENTS and random.random() < self.replacement_prob)
                        if replace_token:
                            new_source.append(FCEReader.REPLACEMENTS[source])
                        elif not dropout_token:
                            new_source.append(token)
                    source = new_source
                yield source, target

    def unknown_token(self):
Italian Trulli
In pattern: SUPERPATTERN

Frequency: 3

Non-data size: 6

Instances


Project Name: shibing624/pycorrector
Commit Name: 52dd8f17b382dea2ddaf3b4054d7845c8c3b4f72
Time: 2018-03-29
Author: 507153809@qq.com
File Name: pycorrector/seq2seq/fce_reader.py
Class Name: FCEReader
Method Name: read_samples_by_string


Project Name: arogozhnikov/einops
Commit Name: 29389772364178f76ccf565917870639cad283bb
Time: 2018-09-27
Author: iamfullofspam@gmail.com
File Name: einops.py
Class Name:
Method Name: get_axes_names


Project Name: TheAlgorithms/Python
Commit Name: c359768e257a7bcbfe93e137a0fc1e81b92d6573
Time: 2020-12-08
Author: 58473917+Joe-Sin7h@users.noreply.github.com
File Name: sorts/bitonic_sort.py
Class Name:
Method Name: