1d39b5f92b2350460a727f915a4b6f2410d11160,anago/data/preprocess.py,WordPreprocessor,fit,#WordPreprocessor#,201

Before Change

        chars = {}
        tags  = {}

        for w in set(itertools.chain(*X)):
            w = self._lower(w)
            w = self._normalize_num(w)
            if w not in words:
                words[w] = len(words)

After Change

        chars = {UNK: 0}
        tags  = {}

        for w in set(itertools.chain(*X)) | set(self.vocab_init):
            w = self._lower(w)
            w = self._normalize_num(w)
            if w not in words:
                words[w] = len(words)

Instances