【代码精读】Position-aware Attention and Supervised Data Improve Slot Filling

因为这篇文章采用LSTM+Attention设计了一个简单的关系抽取模型，我们只关注模型部分，其余数据处理与训练测试部分与https://www.cnblogs.com/Harukaze/p/14253446.html这篇文章的代码类似不做过多介绍。

底层LSTM模型以及embedding的设置：

class RelationModel(object):
    """ A wrapper class for the training and evaluation of models.
        用于训练和评估模型的包装类"""
    def __init__(self, opt, emb_matrix=None):
        self.opt = opt
        self.model = PositionAwareRNN(opt, emb_matrix)
        self.criterion = nn.CrossEntropyLoss()
        self.parameters = [p for p in self.model.parameters() if p.requires_grad]
        if opt['cuda']:
            self.model.cuda()
            self.criterion.cuda()
        self.optimizer = torch_utils.get_optimizer(opt['optim'], self.parameters, opt['lr'])
    
    def update(self, batch):
        """ Run a step of forward and backward model update. """
        if self.opt['cuda']:
            inputs = [b.cuda() for b in batch[:7]]
            labels = batch[7].cuda()
        else:
            inputs = [b for b in batch[:7]]
            labels = batch[7]

        # step forward
        self.model.train()
        self.optimizer.zero_grad()
        logits, _ = self.model(inputs)    #logits.shape = [b,42]
        loss = self.criterion(logits, labels)    #labels = [b]
        
        # backward
        loss.backward()
        torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.opt['max_grad_norm'])
        self.optimizer.step()
        loss_val = loss.data.item()
        return loss_val

    def predict(self, batch, unsort=True):
        """ Run forward prediction. If unsort is True, recover the original order of the batch. """
        if self.opt['cuda']:
            inputs = [b.cuda() for b in batch[:7]]
            labels = batch[7].cuda()
        else:
            inputs = [b for b in batch[:7]]
            labels = batch[7]

        orig_idx = batch[8]

        # forward
        self.model.eval()
        logits, _ = self.model(inputs)
        loss = self.criterion(logits, labels)     #logits.shape = [b,42]  labels = [b]
        probs = F.softmax(logits, dim=1).data.cpu().numpy().tolist()
        predictions = np.argmax(logits.data.cpu().numpy(), axis=1).tolist() #predictions.shape = [b]
        if unsort:
            _, predictions, probs = [list(t) for t in zip(*sorted(zip(orig_idx,\
                    predictions, probs)))]
        return predictions, probs, loss.data.item()

    def update_lr(self, new_lr):
        torch_utils.change_lr(self.optimizer, new_lr)

    def save(self, filename, epoch):
        params = {
                'model': self.model.state_dict(),
                'config': self.opt,
                'epoch': epoch
                }
        try:
            torch.save(params, filename)
            print("model saved to {}".format(filename))
        except BaseException:
            print("[Warning: Saving failed... continuing anyway.]")

    def load(self, filename):
        try:
            checkpoint = torch.load(filename)
        except BaseException:
            print("Cannot load model from {}".format(filename))
            exit()
        self.model.load_state_dict(checkpoint['model'])
        self.opt = checkpoint['config']

class PositionAwareRNN(nn.Module):
    """ A sequence model for relation extraction. """

    def __init__(self, opt, emb_matrix=None):
        super(PositionAwareRNN, self).__init__()
        self.drop = nn.Dropout(opt['dropout'])
        self.emb = nn.Embedding(opt['vocab_size'], opt['emb_dim'], padding_idx=constant.PAD_ID)
        if opt['pos_dim'] > 0:
            self.pos_emb = nn.Embedding(len(constant.POS_TO_ID), opt['pos_dim'],
                    padding_idx=constant.PAD_ID)
        if opt['ner_dim'] > 0:
            self.ner_emb = nn.Embedding(len(constant.NER_TO_ID), opt['ner_dim'],
                    padding_idx=constant.PAD_ID)
        
        input_size = opt['emb_dim'] + opt['pos_dim'] + opt['ner_dim']
        self.rnn = nn.LSTM(input_size, opt['hidden_dim'], opt['num_layers'], batch_first=True,\
                dropout=opt['dropout'])
        self.linear = nn.Linear(opt['hidden_dim'], opt['num_class'])

        if opt['attn']:
            self.attn_layer = layers.PositionAwareAttention(opt['hidden_dim'],
                    opt['hidden_dim'], 2*opt['pe_dim'], opt['attn_dim'])
            self.pe_emb = nn.Embedding(constant.MAX_LEN * 2 + 1, opt['pe_dim'])
            #pe_dim = Position encoding dimension = 30

        self.opt = opt
        self.topn = self.opt.get('topn', 1e10)
        self.use_cuda = opt['cuda']
        self.emb_matrix = emb_matrix
        self.init_weights()
    
    def init_weights(self):
        if self.emb_matrix is None:
            self.emb.weight.data[1:,:].uniform_(-1.0, 1.0) # keep padding dimension to be 0
        else:
            self.emb_matrix = torch.from_numpy(self.emb_matrix)
            self.emb.weight.data.copy_(self.emb_matrix)
        if self.opt['pos_dim'] > 0:
            self.pos_emb.weight.data[1:,:].uniform_(-1.0, 1.0)
        if self.opt['ner_dim'] > 0:
            self.ner_emb.weight.data[1:,:].uniform_(-1.0, 1.0)

        self.linear.bias.data.fill_(0)
        init.xavier_uniform_(self.linear.weight, gain=1) # initialize linear layer
        if self.opt['attn']:
            self.pe_emb.weight.data.uniform_(-1.0, 1.0)

        # decide finetuning
        if self.topn <= 0:
            print("Do not finetune word embedding layer.")
            self.emb.weight.requires_grad = False
        elif self.topn < self.opt['vocab_size']:
            print("Finetune top {} word embeddings.".format(self.topn))
            self.emb.weight.register_hook(lambda x: \
                    torch_utils.keep_partial_grad(x, self.topn))
        else:
            print("Finetune all embeddings.")

    def zero_state(self, batch_size): 
        state_shape = (self.opt['num_layers'], batch_size, self.opt['hidden_dim'])
        h0 = c0 = torch.zeros(*state_shape, requires_grad=False)
        if self.use_cuda:
            return h0.cuda(), c0.cuda()
        else:
            return h0, c0
    
    def forward(self, inputs):
        words, masks, pos, ner, deprel, subj_pos, obj_pos = inputs # unpack
        seq_lens = list(masks.data.eq(constant.PAD_ID).long().sum(1).squeeze())
        batch_size = words.size()[0]
        
        # embedding lookup
        word_inputs = self.emb(words)   #word_inputs.shape = [b,maxlen] -> [b,maxlen,300]
        inputs = [word_inputs]
        if self.opt['pos_dim'] > 0:
            inputs += [self.pos_emb(pos)]
        if self.opt['ner_dim'] > 0:
            inputs += [self.ner_emb(ner)]
        inputs = self.drop(torch.cat(inputs, dim=2)) # add dropout to input  #[b,maxlen,300+30+30]
        input_size = inputs.size(2)
        
        # rnn
        h0, c0 = self.zero_state(batch_size)
        inputs = nn.utils.rnn.pack_padded_sequence(inputs, seq_lens, batch_first=True)
        outputs, (ht, ct) = self.rnn(inputs, (h0, c0))   #putput.shape = [b,maxlen,200]
        outputs, output_lens = nn.utils.rnn.pad_packed_sequence(outputs, batch_first=True)
        hidden = self.drop(ht[-1,:,:]) # get the outmost layer h_n    #获得hn的最后一层的值用来dropout
        outputs = self.drop(outputs)
        
        # attention
        if self.opt['attn']:
            # convert all negative PE numbers to positive indices
            # e.g., -2 -1 0 1 will be mapped to 98 99 100 101
            subj_pe_inputs = self.pe_emb(subj_pos + constant.MAX_LEN)
            obj_pe_inputs = self.pe_emb(obj_pos + constant.MAX_LEN)     #我看很多文章此类特征均都转为正数处理
            pe_features = torch.cat((subj_pe_inputs, obj_pe_inputs), dim=2) #pe_features.shape = [b,maxlen,30+30]
            final_hidden = self.attn_layer(outputs, masks, hidden, pe_features)
        else:
            final_hidden = hidden  #final_hidden.shape = [b,200]

        logits = self.linear(final_hidden)  #[1,b,200]->[b,42]
        return logits, final_hidden

从第178行代码可以看到将，双层单向LSTM的输出outputs，mask矩阵，LSTM的$h_n[-1]$，以及处理过的位置向量pre_features输入到Attention层中。

import torch
a = [torch.Tensor([1,2,3]),torch.Tensor([1,2,3]),torch.Tensor([1,2,3])]
print(sum(a)) #tensor([3., 6., 9.])

补充一个sum()实例，便于下面Attention层sum()操作的理解：

class PositionAwareAttention(nn.Module):
    """
    A position-augmented attention layer where the attention weight is
    a = T' . tanh(Ux + Vq + Wf)
    where x is the input, q is the query, and f is additional position features.
    """
    
    def __init__(self, input_size, query_size, feature_size, attn_size):
        super(PositionAwareAttention, self).__init__()
        self.input_size = input_size        #input_size = hidden = 200
        self.query_size = query_size        #input_size = hidden = 200
        self.feature_size = feature_size    #feature_size = 2*30 = 60
        self.attn_size = attn_size          #attn_size = attn_dim =200
        self.ulinear = nn.Linear(input_size, attn_size)             #matrix U.shape = [200,200]
        self.vlinear = nn.Linear(query_size, attn_size, bias=False) #matrix V.shape = [200,200]
        if feature_size > 0:
            self.wlinear = nn.Linear(feature_size, attn_size, bias=False)   #matrix W.shape = [60,200]
        else:
            self.wlinear = None
        self.tlinear = nn.Linear(attn_size, 1)
        self.init_weights()

    def init_weights(self):
        self.ulinear.weight.data.normal_(std=0.001)
        self.vlinear.weight.data.normal_(std=0.001)
        if self.wlinear is not None:
            self.wlinear.weight.data.normal_(std=0.001)
        self.tlinear.weight.data.zero_() # use zero to give uniform attention at the beginning
        #在开始时用零来给予每个位置相同的attention权重
    
    def forward(self, x, x_mask, q, f):
        """
        x : batch_size * seq_len * input_size
        q : batch_size * query_size
        f : batch_size * seq_len * feature_size
        """
        batch_size, seq_len, _ = x.size()

        x_proj = self.ulinear(x.contiguous().view(-1, self.input_size)).view(
            batch_size, seq_len, self.attn_size)
        # x.shape:[b,maxlen,200]->[b*maxlen,200]->[b*maxlen,200]->[b,maxlen,200] 此过程相当于Ux
        q_proj = self.vlinear(q.view(-1, self.query_size)).contiguous().view(
            batch_size, self.attn_size).unsqueeze(1).expand(
                batch_size, seq_len, self.attn_size)
        #q.shape:[b,200]->[b,200]->[b,200]->[b,200]->[b,1,200]->[b,maxlen,200] 此过程相当于 Vq
        #[b,1,200]->[b,maxlen,200]将代表整句话的概要向量summary vector q,复制maxlen，让每个位置单词都能用上
        if self.wlinear is not None:
            f_proj = self.wlinear(f.view(-1, self.feature_size)).contiguous().view(
                batch_size, seq_len, self.attn_size)
            # f.shape:[b,maxlen,60]->[b*maxlen,60]->[b*maxlen,200]->[b,maxlen,200] 此过程相当于Wf
            #[b*maxlen,60]->[b*maxlen,200]相当于把所有batch的单词连在一起与矩阵相乘，得到结果再改变shape

            projs = [x_proj, q_proj, f_proj]
        else:
            projs = [x_proj, q_proj]
        scores = self.tlinear(torch.tanh(sum(projs)).view(-1, self.attn_size)).view(
            batch_size, seq_len)
        '''
        sum(projs)三个矩阵对应位置数值相加 shape:[b,maxlen,200],调整为[b*maxlen,200]
        经过一个激活函数tanh,再乘以一个矩阵T(tlinear),得到[b*maxlen,1]
        最后输出为scores.shape:[b,maxlen]，意为每个batch中每个单词位置的attention权重值
        '''

        # mask padding
        scores.data.masked_fill_(x_mask.data, -float('inf')) #无单词的位置替换为-inf，不分配注意力权重
        weights = F.softmax(scores, dim=1)    #weights.shape:[b,maxlen]
        # weighted average input vectors
        outputs = weights.unsqueeze(1).bmm(x).squeeze(1)
        #weights.shape:[b,maxlen]->[b,1,maxlen]
        #attention矩阵再与输入矩阵x相乘[b,1,maxlen]*[b,maxlen,200] = [b,1,200]->[b,200]
        # z=\sum_{i=1}^n a_ih_i 这里相当于还原了公式得到向量z   maxlen每个单词位置上的权重*每个单词位置上的hidden
        return outputs