f4275c0b80197e0f1bbd3a2a1a31cf07d85013b1,scripts/word_embeddings/evaluate_pretrained.py,,,#,173

Before Change

    enforce_max_size(token_embedding_, args_.max_vocab_size)
    known_tokens = set(token_embedding_.idx_to_token)
    // Auto-extend token_embedding with unknown extra eval tokens
    if token_embedding_.unknown_lookup is not None:
        eval_tokens = evaluation.get_tokens_in_evaluation_datasets(args_)
        // pylint: disable=pointless-statement
        token_embedding_[[
            t for t in eval_tokens - known_tokens
            if t in token_embedding_.unknown_lookup
        ]]

        if args_.max_vocab_size is not None and len(
                token_embedding_.idx_to_token) > args_.max_vocab_size:
            logging.warning("Computing embeddings for OOV words that occur "
                            "in the evaluation dataset lead to having "
                            "more words than --max-vocab-size. "
                            "Have %s words (--max-vocab-size %s)",
                            len(token_embedding_.idx_to_token),
                            args_.max_vocab_size)

    similarity_results = evaluation.evaluate_similarity(
        args_, token_embedding_, ctx, logfile=os.path.join(
            args_.logdir, "similarity{}.tsv".format(name)))
    analogy_results = evaluation.evaluate_analogy(

After Change

                              "OOV tokens for analogy"):
            tokens = evaluation.get_analogy_task_tokens(args_)
            if token_embedding_.unknown_token is not None:
                tokens.update(token_embedding_.idx_to_token[1:])
            else:
                tokens.update(token_embedding_.idx_to_token)
        vocab = nlp.Vocab(nlp.data.count_tokens(tokens))

Instances