bf03e74b9b0858f2e827e6da4b0ca8d960b3b5b6,soynlp/pos/_news_pos.py,NewsPOSExtractor,_extract_compound_nouns,#NewsPOSExtractor#,303

Before Change

        for word, count in words.items():
            tokens = tokenizer.tokenize(word, flatten=False)[0]
            compound_parts = parse_compound(tokens)
            if compound_parts:
                word = "".join(compound_parts)
                nouns_[word] = nouns_.get(word, 0) + count
                if word in words:
                    words[word] = max(0, words.get(word, 0) - count)

        words = {word:count for word, count in words.items()
                 if (not (word in nouns_)) and (count > 0)}

        return nouns_, words

After Change

        tokenizer = MaxScoreTokenizer(scores = {noun:1 for noun in nouns if len(noun) > 1})

        compounds, removals = {}, set()
        for word, count in eojeols.items():
            // format: [(word, begin, end, score, length)]
            tokens = tokenizer.tokenize(word, flatten=False)[0]
            noun = parse_compound(tokens)

Instances