src_vocab = torchtext.vocab.Vocab(Counter(src))
self.src_vocabs.append(src_vocab)
// mapping source tokens to indices in the dynamic dict
src_map = torch.zeros(len(src)).long()
for j, w in enumerate(src):
src_map[j] = src_vocab.stoi[w]
self.src_vocabs.append(src_vocab)
example["src_map"] = src_map
if "tgt" in example:
tgt = example["tgt"]
mask = torch.zeros(len(tgt) + 2).long()
for j, word in enumerate(tgt, 1):
mask[j] = src_vocab.stoi[word]
example["alignment"] = mask
keys = examples[0].keys()
fields = [(k, fields[k]) for k in keys]
After Change
src_vocab = torchtext.vocab.Vocab(Counter(src))
self.src_vocabs.append(src_vocab)
// mapping source tokens to indices in the dynamic dict
src_map = torch.LongTensor([src_vocab.stoi[w] for w in src])
self.src_vocabs.append(src_vocab)
example["src_map"] = src_map
if "tgt" in example:
tgt = example["tgt"]
mask = torch.LongTensor(
[0] + [src_vocab.stoi[w] for w in tgt] + [0])
example["alignment"] = mask
keys = examples[0].keys()
fields = [(k, fields[k]) for k in keys]
