thek0tyara/llama-cpp-turboquant
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

mtmd : support home-cooked Mistral Small Omni (#14928)

Browse source
This commit is contained in:
Xuan-Son Nguyen 2025-10-16 19:00:31 +02:00 • committed by GitHub
parent 683fa6ba4e
commit 1bb4f43380
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 17 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

18
tools/mtmd/clip.cpp
View file

@ -2221,15 +2221,27 @@ struct clip_model_loader {
// projector type
std::string proj_type;
{
// default key
get_string(KEY_PROJ_TYPE, proj_type, false);
if (!proj_type.empty()) {
model.proj_type = clip_projector_type_from_string(proj_type);
// for models with mixed modalities
if (proj_type.empty()) {
if (modality == CLIP_MODALITY_VISION) {
get_string(KEY_VISION_PROJ_TYPE, proj_type, false);
} else if (modality == CLIP_MODALITY_AUDIO) {
get_string(KEY_AUDIO_PROJ_TYPE, proj_type, false);
} else {
GGML_ABORT("unknown modality");
}
}
model.proj_type = clip_projector_type_from_string(proj_type);
if (model.proj_type == PROJECTOR_TYPE_UNKNOWN) {
throw std::runtime_error(string_format("%s: unknown projector type: %s\n", __func__, proj_type.c_str()));
}
// correct arch for multimodal models
// correct arch for multimodal models (legacy method)
if (model.proj_type == PROJECTOR_TYPE_QWEN25O) {
model.proj_type = modality == CLIP_MODALITY_VISION
? PROJECTOR_TYPE_QWEN25VL