1dc1d25a580f888f02c9449535c16b2d8dbb1549,examples/babi_rnn.py,,,#,143

Before Change

story_maxlen = max(map(len, (x for x, _, _ in train + test)))
query_maxlen = max(map(len, (x for _, x, _ in train + test)))

X, Xq, Y = vectorize_stories(train)
tX, tXq, tY = vectorize_stories(test)

print("vocab = {}".format(vocab))
print("X.shape = {}".format(X.shape))

After Change

challenge = "tasks_1-20_v1-2/en/qa2_two-supporting-facts_{}.txt"
// QA2 with 10,000 samples
// challenge = "tasks_1-20_v1-2/en-10k/qa2_two-supporting-facts_{}.txt"
train = get_stories(tar.extractfile(challenge.format("train")))
test = get_stories(tar.extractfile(challenge.format("test")))

vocab = sorted(reduce(lambda x, y: x | y, (set(story + q + [answer]) for story, q, answer in train + test)))
// Reserve 0 for masking via pad_sequences
vocab_size = len(vocab) + 1
word_idx = dict((c, i + 1) for i, c in enumerate(vocab))
story_maxlen = max(map(len, (x for x, _, _ in train + test)))
query_maxlen = max(map(len, (x for _, x, _ in train + test)))

X, Xq, Y = vectorize_stories(train, word_idx, story_maxlen, query_maxlen)
tX, tXq, tY = vectorize_stories(test, word_idx, story_maxlen, query_maxlen)

print("vocab = {}".format(vocab))
print("X.shape = {}".format(X.shape))
print("Xq.shape = {}".format(Xq.shape))

Instances