// Load pre-trained embeddings
if not args_.embedding_path:
if args_.embedding_name.lower() == "fasttext":
token_embedding_ = nlp.embedding.create(
args_.embedding_name,
source=args_.embedding_source,
After Change
if args_.similarity_datasets:
with utils.print_time("find relevant tokens for similarity"):
tokens = evaluation.get_similarity_task_tokens(args_)
vocab = nlp.Vocab(nlp.data.count_tokens(tokens))
with utils.print_time("set {} embeddings".format(len(tokens))):
vocab.set_embedding(token_embedding_)
evaluation.evaluate_similarity(
args_, vocab.embedding, ctx, logfile=os.path.join(
args_.logdir, "similarity{}.tsv".format(name)))
if args_.analogy_datasets:
with utils.print_time("extend open vocabulary with "
"OOV tokens for analogy"):
tokens = evaluation.get_analogy_task_tokens(args_)
if token_embedding_.unknown_token is not None:
tokens.update(token_embedding_.idx_to_token[1:])
else:
tokens.update(token_embedding_.idx_to_token)
vocab = nlp.Vocab(nlp.data.count_tokens(tokens))
with utils.print_time("set {} embeddings".format(len(tokens))):
vocab.set_embedding(token_embedding_)
evaluation.evaluate_analogy(
args_, vocab.embedding, ctx, logfile=os.path.join(
