从代码角度理解NNLM（A Neural Probabilistic Language Model）

其框架结构如下所示：

可分为四 个部分：

• 词嵌入部分
• 输入
• 隐含层
• 输出层

我们要明确任务是通过一个文本序列（分词后的序列）去预测下一个字出现的概率，tensorflow代码如下：

参考：https://github.com/pjlintw/NNLM/blob/master/src/nnlm.py

import argparse
import math
import time
import numpy as np
import tensorflow as tf
from datetime import date

from preprocessing import TextLoader


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--data_dir', type=str, default='data/',
                        help='data directory containing input.txt')
    parser.add_argument('--batch_size', type=int, default=120,
                        help='minibatch size')
    parser.add_argument('--win_size', type=int, default=5,
                        help='context sequence length')
    parser.add_argument('--hidden_num', type=int, default=100,
                        help='number of hidden layers')
    parser.add_argument('--word_dim', type=int, default=300,
                        help='number of word embedding')
    parser.add_argument('--num_epochs', type=int, default=3,
                        help='number of epochs')
    parser.add_argument('--grad_clip', type=float, default=10.,
                        help='clip gradients at this value')

    args = parser.parse_args()
    args_msg = '\n'.join([ arg + ': ' + str(getattr(args, arg)) for arg in vars(args)])


    data_loader = TextLoader(args.data_dir, args.batch_size, args.win_size)
    args.vocab_size = data_loader.vocab_size

    graph = tf.Graph()
    with graph.as_default():
        input_data = tf.placeholder(tf.int64, [args.batch_size, args.win_size])
        targets = tf.placeholder(tf.int64, [args.batch_size, 1])

        with tf.variable_scope('nnlm' + 'embedding'):
            embeddings = tf.Variable(tf.random_uniform([args.vocab_size, args.word_dim], -1.0, 1.0))
            embeddings = tf.nn.l2_normalize(embeddings, 1)

        with tf.variable_scope('nnlm' + 'weight'):
            weight_h = tf.Variable(tf.truncated_normal([args.win_size * args.word_dim, args.hidden_num],
                                                       stddev=1.0 / math.sqrt(args.hidden_num)))
            softmax_w = tf.Variable(tf.truncated_normal([args.win_size * args.word_dim, args.vocab_size],
                                                        stddev=1.0 / math.sqrt(args.win_size * args.word_dim)))
            softmax_u = tf.Variable(tf.truncated_normal([args.hidden_num, args.vocab_size],
                                                        stddev=1.0 / math.sqrt(args.hidden_num)))

            b_1 = tf.Variable(tf.random_normal([args.hidden_num]))
            b_2 = tf.Variable(tf.random_normal([args.vocab_size]))

        def infer_output(input_data):
            """
            hidden = tanh(x * H + b_1)
            output = softmax(x * W + hidden * U + b_2)
            """
            input_data_emb = tf.nn.embedding_lookup(embeddings, input_data)
            input_data_emb = tf.reshape(input_data_emb, [-1, args.win_size * args.word_dim])
            hidden = tf.tanh(tf.matmul(input_data_emb, weight_h)) + b_1
            hidden_output = tf.matmul(hidden, softmax_u) + tf.matmul(input_data_emb, softmax_w) + b_2
            output = tf.nn.softmax(hidden_output)
            return output

        outputs = infer_output(input_data)
        one_hot_targets = tf.one_hot(tf.squeeze(targets), args.vocab_size, 1.0, 0.0)
        loss = -tf.reduce_mean(tf.reduce_sum(tf.log(outputs) * one_hot_targets, 1))
        # Clip grad.
        optimizer = tf.train.AdagradOptimizer(0.1)
        gvs = optimizer.compute_gradients(loss)
        capped_gvs = [(tf.clip_by_value(grad, -args.grad_clip, args.grad_clip), var) for grad, var in gvs]
        optimizer = optimizer.apply_gradients(capped_gvs)

        embeddings_norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
        normalized_embeddings = embeddings / embeddings_norm
    processing_message_lst = list()
    with tf.Session(graph=graph) as sess:

        tf.global_variables_initializer().run()
        for e in range(args.num_epochs):
            data_loader.reset_batch_pointer()
            for b in range(data_loader.num_batches):
                start = time.time()
                x, y = data_loader.next_batch()
                feed = {input_data: x, targets: y}
                train_loss, _ = sess.run([loss, optimizer], feed)
                end = time.time()

                processing_message = "{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}".format(
                    b, data_loader.num_batches,
                    e, train_loss, end - start)

                print(processing_message)
                processing_message_lst.append(processing_message)
                # print("{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}".format(
                #     b, data_loader.num_batches,
                #     e, train_loss, end - start))


            np.save('nnlm_word_embeddings.zh', normalized_embeddings.eval())

    # record training processing
    print(start - end)
    local_time = str(time.strftime("%Y-%m-%d_%H:%M:%S", time.localtime()))
    with open("{}.txt".format('casdsa'), 'w', encoding='utf-8') as f:
        f.write(local_time)
        f.write(args_msg)
        f.write('\n'.join(processing_message_lst))


if __name__ == '__main__':
    main()

我们主要关注的是模型的部分：

词嵌入部分：

embeddings = tf.Variable(tf.random_uniform([args.vocab_size, args.word_dim], -1.0, 1.0))

生成一个[V, word_dim]的矩阵，其中V表示词汇表，由所有词构成，word_dim表示每个词表示的维度；

输入部分：

input_data_emb = tf.nn.embedding_lookup(embeddings, input_data)
input_data_emb = tf.reshape(input_data_emb, [-1, args.win_size * args.word_dim])

我们分词后的每一个词是用其在词汇表中对应的索引来表示的，比如：“我 爱 美丽 的 中国”，表示为[8,12,27,112] ，我们对应的标签就是[44]，即我们根据前面4个词来预测最后一个词，此时我们得到的是[batchsize,N,word_dim]，然后将其调整形状为：[batchsize,N*word_dim]；

隐含层部分：

hidden = tf.tanh(tf.matmul(input_data_emb, weight_h)) + b_1

将之前的每个词嵌入拼接起来后做一个映射，再经过一个tanh激活函数；

输出部分：

hidden_output = tf.matmul(hidden, softmax_u) + tf.matmul(input_data_emb, softmax_w) + b_2
output = tf.nn.softmax(hidden_output)

这里由两个部分组成，一个是隐含层的输出，一个是输入层直接经过映射（跳过隐含层）到输出层的输出，需要注意的是输出层的神经元的个数就是词汇表的大小；

最后在计算损失的时候是：

outputs = infer_output(input_data)
one_hot_targets = tf.one_hot(tf.squeeze(targets), args.vocab_size, 1.0, 0.0)
loss = -tf.reduce_mean(tf.reduce_sum(tf.log(outputs) * one_hot_targets, 1))

训练完成之后我们最终需要的是词嵌入，也就是：

embeddings_norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
normalized_embeddings = embeddings / embeddings_norm

下面是pytorch版本的，思路是一样的：

参考：https://github.com/LeeGitaek/NNLM_Paper_Implementation/blob/master/nnlm_model.py

# -*- coding: utf-8 -*-
"""NNLM_paper.ipynb
Automatically generated by Colaboratory.
Original file is located at
    https://colab.research.google.com/drive/1q6tWzcpFLzU_qvzvkdYiDaxSp--y6nFR
"""

import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np
from torch.autograd import Variable

device = 'cuda' if torch.cuda.is_available() else 'cpu'

torch.manual_seed(777)
if device == 'cuda':
  torch.cuda.manual_seed_all(777)

sentences = ['i like dog','i love coffee','i hate milk']

word_list = ' '.join(sentences).split()
word_list = list(set(word_list))
print(word_list)

word_dict = {w: i for i,w in enumerate(word_list)}
print('word dict')
print(word_dict)
number_dict = {i: w for i, w in enumerate(word_list)}
print(number_dict)
n_class = len(word_dict) # number of vocabulary

print(n_class)

#NNLM Parameter
n_step = 2 # n-1 in paper
n_hidden = 2 # h in paper
m = 2       # m in paper
epochs = 5000
learning_rate = 0.001

def make_batch(sentences):
    input_batch = []
    target_batch = []

    for sen in sentences:
        word = sen.split()
        input = [word_dict[n] for n in word[:-1]]
        target = word_dict[word[-1]]

        input_batch.append(input)
        target_batch.append(target)

    return input_batch,target_batch


#model

class NNLM(nn.Module):
    def __init__(self):
        super(NNLM,self).__init__()

        self.C = nn.Embedding(n_class,m)
        self.H = nn.Parameter(torch.randn(n_step * m,n_hidden).type(torch.Tensor))
        self.W = nn.Parameter(torch.randn(n_step * m,n_class).type(torch.Tensor))
        self.d = nn.Parameter(torch.randn(n_hidden).type(torch.Tensor))
        self.U = nn.Parameter(torch.randn(n_hidden,n_class).type(torch.Tensor))
        self.b = nn.Parameter(torch.randn(n_class).type(torch.Tensor))

    def forward(self,x):
        x = self.C(x)
        x = x.view(-1,n_step*m) # batch_size,n_step * n_class
        tanh = torch.tanh(self.d + torch.mm(x,self.H))
        output = self.b + torch.mm(x,self.W)+torch.mm(tanh,self.U)
        return output
model = NNLM()

criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(),lr=learning_rate)

input_batch , target_batch = make_batch(sentences)
print(input_batch)
print('target_batch')
print(target_batch)
input_batch = Variable(torch.LongTensor(input_batch))
target_batch = Variable(torch.LongTensor(target_batch))

for epoch in range(epochs):
    optimizer.zero_grad()
    output = model(input_batch)

    loss = criterion(output,target_batch)
    if (epoch+1)%100 == 0:
        print('epoch : {:.4f} , cost = {:.6f}'.format(epoch+1,loss))

    loss.backward()
    optimizer.step()


predict = model(input_batch).data.max(1,keepdim=True)[1]

print([sen.split()[:2] for sen in sentences],'->',[number_dict[n.item()] for n in predict.squeeze()])