BERT基本使用

import torch
from transformers import BertTokenizer, BertModel, BertForMaskedLM
from transformers import logging
logging.set_verbosity_error()

#############################################预处理###################################################
# 加载预训练的模型标记器（词汇表）
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
max_sentence_len = 10

# 标记输入
text = "[CLS] I love you [SEP]"
tokenized_text = tokenizer.tokenize(text)
print(tokenized_text)

# 单词to编号
tokenized_ids = tokenizer.convert_tokens_to_ids(tokenized_text) # 注意是** to ##，别写颠倒了
tokenized_ids += [0] * (max_sentence_len - len(tokenized_ids)) # 句子不够长度，用[PAD]补足（对应id为0）
print(tokenized_ids)

# 定义attention_mask，用来标记哪是有用token(对应1)，哪是[PAD]的token(对应0)
att_mask = [1] * len(tokenized_ids)
att_mask += [0] * (max_sentence_len - len(tokenized_ids))

# 转张量（1*N）
tokenized_ids = torch.tensor(tokenized_ids).reshape(1,-1)
att_mask = torch.tensor(att_mask).reshape(1,-1)

#############################################使用模型##################################################
model = BertModel.from_pretrained('bert-base-uncased', output_hidden_states = False)
hidden_rep, cls_head = model(tokenized_ids, attention_mask=att_mask, return_dict=False) # attention_mask用来标记哪是有用token(对应1)，哪是[PAD]的token(对应0)
print(hidden_rep.shape) # torch.Size([1, 10, 768]) 1:batch(句子个数)  10:token（单词）个数  768:每个token的嵌入表示维度大小
print(cls_head.shape) # torch.Size([1, 768]) 整个句子的聚合表示 1:batch(句子个数)  768:句子嵌入表示维度大小

 

参考文献：

https://blog.csdn.net/yjw123456/article/details/120232707

https://pytorchchina.com/2020/02/29/transformers-quick-start/

https://blog.csdn.net/weixin_46425692/article/details/108890831

 http://wp.iter-cc.com/?p=1378