基于故事和问题训练两个循环神经网络。
两者的合并向量将用于回答一系列 bAbI 任务。
这些结果与 Weston 等人提供的 LSTM 模型的结果相当：Towards AI-Complete Question Answering: A Set of Prerequisite Toy Tasks。
| Task Number | FB LSTM Baseline | Keras QA |
|---|---|---|
| QA1 - Single Supporting Fact | 50 | 52.1 |
| QA2 - Two Supporting Facts | 20 | 37.0 |
| QA3 - Three Supporting Facts | 20 | 20.5 |
| QA4 - Two Arg. Relations | 61 | 62.9 |
| QA5 - Three Arg. Relations | 70 | 61.9 |
| QA6 - yes/No Questions | 48 | 50.7 |
| QA7 - Counting | 49 | 78.9 |
| QA8 - Lists/Sets | 45 | 77.2 |
| QA9 - Simple Negation | 64 | 64.0 |
| QA10 - Indefinite Knowledge | 44 | 47.7 |
| QA11 - Basic Coreference | 72 | 74.9 |
| QA12 - Conjunction | 74 | 76.4 |
| QA13 - Compound Coreference | 94 | 94.4 |
| QA14 - Time Reasoning | 27 | 34.8 |
| QA15 - Basic Deduction | 21 | 32.4 |
| QA16 - Basic Induction | 23 | 50.6 |
| QA17 - Positional Reasoning | 51 | 49.1 |
| QA18 - Size Reasoning | 52 | 90.8 |
| QA19 - Path Finding | 8 | 9.0 |
| QA20 - Agent's Motivations | 91 | 90.7 |
有关 bAbI 项目的相关资源，请参考: https://research.facebook.com/researchers/1543934539189348
注意
-
使用默认的单词、句子和查询向量尺寸，GRU 模型得到了以下效果：
- 20 轮迭代后，在 QA1 上达到了 52.1% 的测试准确率（在 CPU 上每轮迭代 2 秒）；
- 20 轮迭代后，在 QA2 上达到了 37.0% 的测试准确率（在 CPU 上每轮迭代 16 秒）。
相比之下，Facebook的论文中 LSTM baseline 的准确率分别是 50% 和 20%。
-
这个任务并不是笼统地单独去解析问题。这应该可以提高准确率，且是合并两个 RNN 的一次较好实践。
-
故事和问题的 RNN 之间不共享词向量（词嵌入）。
-
注意观察 1000 个训练样本（en-10k）到 10,000 个的准确度如何变化。使用 1000 是为了与原始论文进行对比。
-
尝试使用 GRU, LSTM 和 JZS1-3，因为它们会产生微妙的不同结果。
-
长度和噪声（即「无用」的故事内容）会影响 LSTM/GRU 提供正确答案的能力。在只提供事实的情况下，这些 RNN可以在许多任务上达到 100% 的准确性。 使用注意力过程的记忆网络和神经网络可以有效地搜索这些噪声以找到相关的语句，从而大大提高性能。这在 QA2 和 QA3 上变得尤为明显，两者都远远显著于 QA1。
from __future__ import print_function
from functools import reduce
import re
import tarfile
import numpy as np
from keras.utils.data_utils import get_file
from keras.layers.embeddings import Embedding
from keras import layers
from keras.layers import recurrent
from keras.models import Model
from keras.preprocessing.sequence import pad_sequences
def tokenize(sent):
'''返回包含标点符号的句子的标记。
>>> tokenize('Bob dropped the apple. Where is the apple?')
['Bob', 'dropped', 'the', 'apple', '.', 'Where', 'is', 'the', 'apple', '?']
'''
return [x.strip() for x in re.split(r'(\W+)', sent) if x.strip()]
def parse_stories(lines, only_supporting=False):
'''解析 bAbi 任务格式中提供的故事
如果 only_supporting 为 true，
则只保留支持答案的句子。
'''
data = []
story = []
for line in lines:
line = line.decode('utf-8').strip()
nid, line = line.split(' ', 1)
nid = int(nid)
if nid == 1:
story = []
if '\t' in line:
q, a, supporting = line.split('\t')
q = tokenize(q)
if only_supporting:
# 只选择相关的子故事
supporting = map(int, supporting.split())
substory = [story[i - 1] for i in supporting]
else:
# 提供所有子故事
substory = [x for x in story if x]
data.append((substory, q, a))
story.append('')
else:
sent = tokenize(line)
story.append(sent)
return data
def get_stories(f, only_supporting=False, max_length=None):
'''给定文件名，读取文件，检索故事，
然后将句子转换为一个独立故事。
如果提供了 max_length,
任何长于 max_length 的故事都将被丢弃。
'''
data = parse_stories(f.readlines(), only_supporting=only_supporting)
flatten = lambda data: reduce(lambda x, y: x + y, data)
data = [(flatten(story), q, answer) for story, q, answer in data
if not max_length or len(flatten(story)) < max_length]
return data
def vectorize_stories(data, word_idx, story_maxlen, query_maxlen):
xs = []
xqs = []
ys = []
for story, query, answer in data:
x = [word_idx[w] for w in story]
xq = [word_idx[w] for w in query]
# 不要忘记索引 0 已被保留
y = np.zeros(len(word_idx) + 1)
y[word_idx[answer]] = 1
xs.append(x)
xqs.append(xq)
ys.append(y)
return (pad_sequences(xs, maxlen=story_maxlen),
pad_sequences(xqs, maxlen=query_maxlen), np.array(ys))
RNN = recurrent.LSTM
EMBED_HIDDEN_SIZE = 50
SENT_HIDDEN_SIZE = 100
QUERY_HIDDEN_SIZE = 100
BATCH_SIZE = 32
EPOCHS = 20
print('RNN / Embed / Sent / Query = {}, {}, {}, {}'.format(RNN,
EMBED_HIDDEN_SIZE,
SENT_HIDDEN_SIZE,
QUERY_HIDDEN_SIZE))
try:
path = get_file('babi-tasks-v1-2.tar.gz',
origin='https://s3.amazonaws.com/text-datasets/'
'babi_tasks_1-20_v1-2.tar.gz')
except:
print('Error downloading dataset, please download it manually:\n'
'$ wget http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2'
'.tar.gz\n'
'$ mv tasks_1-20_v1-2.tar.gz ~/.keras/datasets/babi-tasks-v1-2.tar.gz')
raise
# 默认 QA1 任务，1000 样本
# challenge = 'tasks_1-20_v1-2/en/qa1_single-supporting-fact_{}.txt'
# QA1 任务，10,000 样本
# challenge = 'tasks_1-20_v1-2/en-10k/qa1_single-supporting-fact_{}.txt'
# QA2 任务，1000 样本
challenge = 'tasks_1-20_v1-2/en/qa2_two-supporting-facts_{}.txt'
# QA2 任务，10,000 样本
# challenge = 'tasks_1-20_v1-2/en-10k/qa2_two-supporting-facts_{}.txt'
with tarfile.open(path) as tar:
train = get_stories(tar.extractfile(challenge.format('train')))
test = get_stories(tar.extractfile(challenge.format('test')))
vocab = set()
for story, q, answer in train + test:
vocab |= set(story + q + [answer])
vocab = sorted(vocab)
# 保留 0 以留作 pad_sequences 进行 masking
vocab_size = len(vocab) + 1
word_idx = dict((c, i + 1) for i, c in enumerate(vocab))
story_maxlen = max(map(len, (x for x, _, _ in train + test)))
query_maxlen = max(map(len, (x for _, x, _ in train + test)))
x, xq, y = vectorize_stories(train, word_idx, story_maxlen, query_maxlen)
tx, txq, ty = vectorize_stories(test, word_idx, story_maxlen, query_maxlen)
print('vocab = {}'.format(vocab))
print('x.shape = {}'.format(x.shape))
print('xq.shape = {}'.format(xq.shape))
print('y.shape = {}'.format(y.shape))
print('story_maxlen, query_maxlen = {}, {}'.format(story_maxlen, query_maxlen))
print('Build model...')
sentence = layers.Input(shape=(story_maxlen,), dtype='int32')
encoded_sentence = layers.Embedding(vocab_size, EMBED_HIDDEN_SIZE)(sentence)
encoded_sentence = RNN(SENT_HIDDEN_SIZE)(encoded_sentence)
question = layers.Input(shape=(query_maxlen,), dtype='int32')
encoded_question = layers.Embedding(vocab_size, EMBED_HIDDEN_SIZE)(question)
encoded_question = RNN(QUERY_HIDDEN_SIZE)(encoded_question)
merged = layers.concatenate([encoded_sentence, encoded_question])
preds = layers.Dense(vocab_size, activation='softmax')(merged)
model = Model([sentence, question], preds)
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
print('Training')
model.fit([x, xq], y,
batch_size=BATCH_SIZE,
epochs=EPOCHS,
validation_split=0.05)
print('Evaluation')
loss, acc = model.evaluate([tx, txq], ty,
batch_size=BATCH_SIZE)
print('Test loss / test accuracy = {:.4f} / {:.4f}'.format(loss, acc))