thek0tyara/llama-cpp-turboquant
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

llama : improve sep token handling (#14272)

Browse source
This commit is contained in:
Sigbjørn Skjæret 2025-06-20 14:04:09 +02:00 • committed by GitHub
parent e28c1b93fd
commit 88fc854b4b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 161 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/llama-model-saver.cpp
View file

@ -228,6 +228,7 @@ void llama_model_saver::add_kv_from_model() {
// add_kv(LLM_KV_TOKENIZER_MASK_ID, ???);
add_kv(LLM_KV_TOKENIZER_ADD_BOS, vocab.get_add_bos());
add_kv(LLM_KV_TOKENIZER_ADD_EOS, vocab.get_add_eos());
add_kv(LLM_KV_TOKENIZER_ADD_SEP, vocab.get_add_sep());
add_kv(LLM_KV_TOKENIZER_ADD_PREFIX, vocab.get_add_space_prefix());
add_kv(LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, vocab.get_remove_extra_whitespaces());
add_kv(LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, vocab.get_precompiled_charsmap());