thek0tyara/llama-cpp-turboquant
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

server : refactor oai_parser_opt, move it to server_chat_params (#18937)

Browse source 
* server_chat_params

* move chat format into CLI

* use meta whenever possible

* clean up, no more chatml fallback
This commit is contained in:
Xuan-Son Nguyen 2026-01-19 23:28:01 +01:00 • committed by GitHub
parent 1706a6d7c6
commit 6df686bee6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 112 additions and 103 deletions
Download patch file Download diff file Expand all files Collapse all files

32
tools/cli/cli.cpp
View file

@ -71,14 +71,16 @@ struct cli_context {
std::string generate_completion(result_timings & out_timings) {
server_response_reader rd = ctx_server.get_response_reader();
auto formatted = format_chat();
{
// TODO: reduce some copies here in the future
server_task task = server_task(SERVER_TASK_TYPE_COMPLETION);
task.id = rd.get_new_id();
task.index = 0;
task.params = defaults; // copy
task.cli_input = messages; // copy
task.cli_files = input_files; // copy
task.id = rd.get_new_id();
task.index = 0;
task.params = defaults; // copy
task.cli_prompt = formatted.prompt; // copy
task.cli_files = input_files; // copy
task.cli = true;
rd.post_task({std::move(task)});
}
@ -156,6 +158,26 @@ struct cli_context {
return content;
}
}
common_chat_params format_chat() {
auto meta = ctx_server.get_meta();
auto & chat_params = meta.chat_params;
common_chat_templates_inputs inputs;
inputs.messages = common_chat_msgs_parse_oaicompat(messages);
inputs.tools = {}; // TODO
inputs.tool_choice = COMMON_CHAT_TOOL_CHOICE_NONE;
inputs.json_schema = ""; // TODO
inputs.grammar = ""; // TODO
inputs.use_jinja = chat_params.use_jinja;
inputs.parallel_tool_calls = false;
inputs.add_generation_prompt = true;
inputs.reasoning_format = chat_params.reasoning_format;
inputs.enable_thinking = chat_params.enable_thinking;
// Apply chat template to the list of messages
return common_chat_templates_apply(chat_params.tmpls.get(), inputs);
}
};
int main(int argc, char ** argv) {