model : add Jina Embeddings v5 Nano (partial EuroBERT) support (#19826)
* WIP: Add EuroBERT support with autoformatting changes This commit includes: - EuroBERT model implementation for GGUF conversion - C++ backend support for EuroBERT architecture - Unintended autoformatting changes to Python files Saving before reverting formatting-only changes. * feat: add back eos assert when not last token pooling * feat: removed duplicated code and cleanup * feat: removed not working architectures and unnecessary check * fix: typo * fix: dynamic pooling config * feat: added an example model for eurobert * feat: proper llama-vocab implementation for jina-v5 * fix: removed unnecessary comments
This commit is contained in:
parent
1ca3d1de15
commit
66287bdaac
12 changed files with 214 additions and 4 deletions
|
|
@ -912,7 +912,9 @@ static bool compute_imatrix(llama_context * ctx, const common_params & params, c
|
|||
|
||||
const bool add_bos = llama_vocab_get_add_bos(vocab);
|
||||
|
||||
GGML_ASSERT(!llama_vocab_get_add_eos(vocab));
|
||||
if (llama_pooling_type(ctx) != LLAMA_POOLING_TYPE_LAST) {
|
||||
GGML_ASSERT(!llama_vocab_get_add_eos(vocab));
|
||||
}
|
||||
|
||||
auto tim1 = std::chrono::high_resolution_clock::now();
|
||||
LOG_INF("%s: tokenizing the input ..\n", __func__);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue