e8852f5ea7b3b5d222f0d6140f9375bd3826d2e4,underthesea/word_sent/model.py,CRFModel,predict,#CRFModel#,19

Before Change

            tokenized_sentence += " "
        format_sentence = ""
        for word in tokenized_sentence.split("_"):
            if word not in self.punctuation:
                format_sentence += word[: -1] + "_"
        tokenized_sentence = format_sentence[:-1]
        return tokenized_sentence

After Change

        tokens = Transformer.transform(sentence)
        tags = self.model.tag(tokens)
        tokens = [item[0] for item in tokens]
        output = []
        for tag, token in zip(tags, tokens):
            if tag == "I_W":
               output[-1] = output[-1] + u" " + token
            else:
                output.append(token)
        if format == "text":
            output = u" ".join([item.replace(" ", "_") for item in output])
        return output

Instances