52dd8f17b382dea2ddaf3b4054d7845c8c3b4f72,pycorrector/seq2seq/fce_reader.py,FCEReader,read_samples_by_string,#FCEReader#,26
Before Change
def read_samples_by_string(self, path):
for tokens in self.read_tokens(path):
source = []
target = []
for token in tokens:
target.append(token)
if self.config.enable_data_dropout:
// Random dropout words from the input
dropout_token = (token in FCEReader.DROPOUT_TOKENS and random.random() < self.dropout_prob)
replace_token = (token in FCEReader.REPLACEMENTS and random.random() < self.replacement_prob)
After Change
line_src = f.readline()
line_dst = f.readline()
if line_src and line_dst:
source = line_src.lower()[5:].strip().split()
target = line_dst.lower()[5:].strip().split()
if self.config.enable_data_dropout:
new_source = []
for token in source:
// Random dropout words from the input
dropout_token = (token in FCEReader.DROPOUT_TOKENS and random.random() < self.dropout_prob)
replace_token = (token in FCEReader.REPLACEMENTS and random.random() < self.replacement_prob)
if replace_token:
new_source.append(FCEReader.REPLACEMENTS[source])
elif not dropout_token:
new_source.append(token)
source = new_source
yield source, target
def unknown_token(self):
In pattern: SUPERPATTERN
Frequency: 3
Non-data size: 6
Instances Project Name: shibing624/pycorrector
Commit Name: 52dd8f17b382dea2ddaf3b4054d7845c8c3b4f72
Time: 2018-03-29
Author: 507153809@qq.com
File Name: pycorrector/seq2seq/fce_reader.py
Class Name: FCEReader
Method Name: read_samples_by_string
Project Name: arogozhnikov/einops
Commit Name: 29389772364178f76ccf565917870639cad283bb
Time: 2018-09-27
Author: iamfullofspam@gmail.com
File Name: einops.py
Class Name:
Method Name: get_axes_names
Project Name: TheAlgorithms/Python
Commit Name: c359768e257a7bcbfe93e137a0fc1e81b92d6573
Time: 2020-12-08
Author: 58473917+Joe-Sin7h@users.noreply.github.com
File Name: sorts/bitonic_sort.py
Class Name:
Method Name: