从零训练一个tokenizer模型

1. 准备训练数据

数据集地址：https://www.modelscope.cn/datasets/gongjy/minimind_dataset/files

2. 准备训练脚本

2.1 初始化tokenizer

定义BPE（Byte-Pair Encoding，字节对编码）模型

# 1. 初始化tokenizer, 定义BPE（Byte-Pair Encoding，字节对编码）模型
tokenizer = Tokenizer(models.BPE())
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)

2.2 定义特殊token

special_tokens = ["<unk>", "<s>", "</s>"]

2.3 定义训练器并添加特殊token

trainer = trainers.BpeTrainer(
            vocab_size=6400,
            special_tokens=special_tokens,  # 确保这三个token被包含
            show_progress=True,
            initial_alphabet=pre_tokenizers.ByteLevel.alphabet()
        )

2.4 读取文本数据

texts = read_texts_from_jsonl(data_path)
data_path = '/root/data/pretrain/pretrain_hq.jsonl'

2.5 训练tokenizer

tokenizer.train_from_iterator(texts, trainer=trainer)

2.5 设置相关参数

# 6. 设置解码器
tokenizer.decoder = decoders.ByteLevel()
# 7. 检查特殊token的索引
assert tokenizer.token_to_id("<unk>") == 0
assert tokenizer.token_to_id("<s>") == 1
assert tokenizer.token_to_id("</s>") == 2

3. 保存tokenizer模型

tokenizer_dir = "/root/model/tokenizer"
tokenizer.save(os.path.join(tokenizer_dir, "tokenizer.json"))
tokenizer.model.save(tokenizer_dir)

# 保存config文件
config = {
    "add_bos_token": False,
    "add_eos_token": False,
    "add_prefix_space": False,
    "added_tokens_decoder": {
        "0": {
            "content": "<unk>",
            "lstrip": False,
            "normalized": False,
            "rstrip": False,
            "single_word": False,
            "special": True
        },
        "1": {
            "content": "<s>",
            "lstrip": False,
            "normalized": False,
            "rstrip": False,
            "single_word": False,
            "special": True
        },
        "2": {
            "content": "</s>",
            "lstrip": False,
            "normalized": False,
            "rstrip": False,
            "single_word": False,
            "special": True
        }
    },
    "additional_special_tokens": [],
    "bos_token": "<s>",
    "clean_up_tokenization_spaces": False,
    "eos_token": "</s>",
    "legacy": True,
    "model_max_length": 32768,
    "pad_token": "<unk>",
    "sp_model_kwargs": {},
    "spaces_between_special_tokens": False,
    "tokenizer_class": "PreTrainedTokenizerFast",
    "unk_token": "<unk>",
    "chat_template": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{{ '<s>system\\n' + system_message + '</s>\\n' }}{% else %}{{ '<s>system\\n你是 MiniMind，是一个有用的人工智能助手。</s>\\n' }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<s>user\\n' + content + '</s>\\n<s>assistant\\n' }}{% elif message['role'] == 'assistant' %}{{ content + '</s>' + '\\n' }}{% endif %}{% endfor %}"
}

# 保存配置文件
with open(os.path.join(tokenizer_dir, "tokenizer_config.json"), "w", encoding="utf-8") as config_file:
    json.dump(config, config_file, ensure_ascii=False, indent=4)

4. 模型验证

# 加载模型
tokenizer = AutoTokenizer.from_pretrained("/root/model/tokenizer")
tokenizer

messages = [
    {"role": "system", "content": "你是一个优秀的聊天机器人，总是给我正确的回应！"},
    {"role": "user", "content": '你来自哪里？'},
    {"role": "assistant", "content": '我来自地球'}
]

new_prompt = tokenizer.apply_chat_template(
    messages,
    tokenize=False
)
new_prompt

# 获取实际词汇表长度（包括特殊符号）
actual_vocab_size = len(tokenizer)
print('tokenizer实际词表长度：', actual_vocab_size)

model_inputs = tokenizer(new_prompt)
print('encoder长度：', len(model_inputs['input_ids']))

input_ids = model_inputs['input_ids']
response = tokenizer.decode(input_ids, skip_special_tokens=False)
print('decoder和原始文本是否一致：', response == new_prompt)