638810ae894784dec9d65abe156ea2fba11f289e,torchtext/experimental/transforms.py,BasicEnglishNormalize,forward,#BasicEnglishNormalize#Any#,62

Before Change

        

        line = line.lower()
        for regex, replacement_string in self.regex_and_replacement_string_pairs:
            line = regex.Sub(line, replacement_string)
        return line.split()


class RegexTokenizer(nn.Module):

After Change

        Returns:
            List[str]: a list of tokens after normalizing and splitting on whitespace.
        
        return self.regex_tokenizer.forward(line)


class RegexTokenizer(nn.Module):
    rRegex tokenizer for a string sentence that applies all regex replacements defined in patterns_list.

Instances