03b0e7061cf477fbeccb9c128ee76603df582d86,scripts/word_embeddings/evaluate_pretrained.py,,,#,159

Before Change

        token_embedding = load_embedding_from_path(args_)
        name = ""

    if args_.max_vocab_size:
        if args_.embedding_path and ".bin" in args_.embedding_path:
            raise NotImplementedError(
                "Not implemented for binary fastText model.")

        size = min(len(token_embedding._idx_to_token), args_.max_vocab_size)
        token_embedding._idx_to_token = token_embedding._idx_to_token[:size]
        token_embedding._idx_to_vec = token_embedding._idx_to_vec[:size]
        token_embedding._token_to_idx = {
            token: idx
            for idx, token in enumerate(token_embedding._idx_to_token)
        }

    similarity_results = evaluation.evaluate_similarity(
        args_, token_embedding, ctx, logfile=os.path.join(
            args_.logdir, "similarity{}.tsv".format(name)))
    analogy_results = evaluation.evaluate_analogy(

After Change

        name = ""

    enforce_max_size(token_embedding_, args_.max_vocab_size)
    known_tokens = set(token_embedding_.idx_to_token)
    // Auto-extend token_embedding with unknown extra eval tokens
    if token_embedding_.unknown_lookup is not None:
        eval_tokens = evaluation.get_tokens_in_evaluation_datasets(args_)
        // pylint: disable=pointless-statement

Instances