52dd8f17b382dea2ddaf3b4054d7845c8c3b4f72,pycorrector/seq2seq/fce_reader.py,FCEReader,read_samples_by_string,#FCEReader#,26

Before Change


    def read_samples_by_string(self, path):
        for tokens in self.read_tokens(path):
            source = []
            target = []
            for token in tokens:
                target.append(token)
                if self.config.enable_data_dropout:
                    // Random dropout words from the input
                    dropout_token = (token in FCEReader.DROPOUT_TOKENS and random.random() < self.dropout_prob)
                    replace_token = (token in FCEReader.REPLACEMENTS and random.random() < self.replacement_prob)

After Change


        self.UNKNOWN_ID = self.token_2_id[FCEReader.UNKNOWN_TOKEN]

    def read_samples_by_string(self, path):
        with open(path, "r", encoding="utf-8") as f:
            line_src = f.readline()
            line_dst = f.readline()
            if line_src and line_dst:
                source = line_src.lower()[5:].strip().split()
                target = line_dst.lower()[5:].strip().split()
                if self.config.enable_data_dropout:
                    new_source = []
                    for token in source:
                        // Random dropout words from the input
                        dropout_token = (token in FCEReader.DROPOUT_TOKENS and random.random() < self.dropout_prob)
                        replace_token = (token in FCEReader.REPLACEMENTS and random.random() < self.replacement_prob)
                        if replace_token:
                            new_source.append(FCEReader.REPLACEMENTS[source])
                        elif not dropout_token:
                            new_source.append(token)
                    source = new_source
                yield source, target

    def unknown_token(self):
        return FCEReader.UNKNOWN_TOKEN

    def read_tokens(self, path):
Italian Trulli
In pattern: SUPERPATTERN

Frequency: 3

Non-data size: 5

Instances


Project Name: shibing624/pycorrector
Commit Name: 52dd8f17b382dea2ddaf3b4054d7845c8c3b4f72
Time: 2018-03-29
Author: 507153809@qq.com
File Name: pycorrector/seq2seq/fce_reader.py
Class Name: FCEReader
Method Name: read_samples_by_string


Project Name: suavecode/SUAVE
Commit Name: b60c2b7edaac23593f3ea59bbe91b7b10a864ec9
Time: 2015-03-07
Author: tmomose@stanford.edu
File Name: trunk/SUAVE/Methods/Aerodynamics/AVL/write_input_deck.py
Class Name:
Method Name: write_input_deck


Project Name: dit/dit
Commit Name: 03c767ba99ab2e46e97dc5c27a133fa2c3830aaa
Time: 2015-03-14
Author: ryangregoryjames@gmail.com
File Name: setup.py
Class Name:
Method Name: main