thek0tyara/llama-cpp-turboquant
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

Merge signed commit 'b8018' into sisyphus

Browse source 
Diff-After-Merge: 2 files changed, 6 insertions(+)

# gpg: Signature made Thu Feb 12 18:11:22 2026 MSK
# gpg:                using RSA key B5690EEEBB952194
# gpg: Good signature from "GitHub <noreply@github.com>" [unknown]

# Conflicts:
#	tools/server/public/index.html.gz
This commit is contained in:
Vitaly Chikunov 2026-02-13 05:12:29 +00:00
commit c61b893e63
447 changed files with 42666 additions and 12084 deletions
Download patch file Download diff file Expand all files Collapse all files

48
tools/cli/README.md
View file

@ -45,10 +45,10 @@
| `--rope-freq-base N` | RoPE base frequency, used by NTK-aware scaling (default: loaded from model)<br/>(env: LLAMA_ARG_ROPE_FREQ_BASE) |
| `--rope-freq-scale N` | RoPE frequency scaling factor, expands context by a factor of 1/N<br/>(env: LLAMA_ARG_ROPE_FREQ_SCALE) |
| `--yarn-orig-ctx N` | YaRN: original context size of model (default: 0 = model training context size)<br/>(env: LLAMA_ARG_YARN_ORIG_CTX) |
| `--yarn-ext-factor N` | YaRN: extrapolation mix factor (default: -1.0, 0.0 = full interpolation)<br/>(env: LLAMA_ARG_YARN_EXT_FACTOR) |
| `--yarn-attn-factor N` | YaRN: scale sqrt(t) or attention magnitude (default: -1.0)<br/>(env: LLAMA_ARG_YARN_ATTN_FACTOR) |
| `--yarn-beta-slow N` | YaRN: high correction dim or alpha (default: -1.0)<br/>(env: LLAMA_ARG_YARN_BETA_SLOW) |
| `--yarn-beta-fast N` | YaRN: low correction dim or beta (default: -1.0)<br/>(env: LLAMA_ARG_YARN_BETA_FAST) |
| `--yarn-ext-factor N` | YaRN: extrapolation mix factor (default: -1.00, 0.0 = full interpolation)<br/>(env: LLAMA_ARG_YARN_EXT_FACTOR) |
| `--yarn-attn-factor N` | YaRN: scale sqrt(t) or attention magnitude (default: -1.00)<br/>(env: LLAMA_ARG_YARN_ATTN_FACTOR) |
| `--yarn-beta-slow N` | YaRN: high correction dim or alpha (default: -1.00)<br/>(env: LLAMA_ARG_YARN_BETA_SLOW) |
| `--yarn-beta-fast N` | YaRN: low correction dim or beta (default: -1.00)<br/>(env: LLAMA_ARG_YARN_BETA_FAST) |
| `-kvo, --kv-offload, -nkvo, --no-kv-offload` | whether to enable KV cache offloading (default: enabled)<br/>(env: LLAMA_ARG_KV_OFFLOAD) |
| `--repack, -nr, --no-repack` | whether to enable weight repacking (default: enabled)<br/>(env: LLAMA_ARG_REPACK) |
| `--no-host` | bypass host buffer allowing extra buffers to be used<br/>(env: LLAMA_ARG_NO_HOST) |
@ -109,30 +109,30 @@
| `-s, --seed SEED` | RNG seed (default: -1, use random seed for -1) |
| `--sampler-seq, --sampling-seq SEQUENCE` | simplified sequence for samplers that will be used (default: edskypmxt) |
| `--ignore-eos` | ignore end of stream token and continue generating (implies --logit-bias EOS-inf) |
| `--temp N` | temperature (default: 0.8) |
| `--temp N` | temperature (default: 0.80) |
| `--top-k N` | top-k sampling (default: 40, 0 = disabled)<br/>(env: LLAMA_ARG_TOP_K) |
| `--top-p N` | top-p sampling (default: 0.9, 1.0 = disabled) |
| `--min-p N` | min-p sampling (default: 0.1, 0.0 = disabled) |
| `--adaptive-target N` | adaptive-p: select tokens near this probability (valid range 0.0 to 1.0; negative = disabled) |
| `--adaptive-decay N` | adaptive-p: EMA decay for adaptation; effective history length ≈ 1/(1-decay) tokens (valid range 0.0 - 0.99) |
| `--top-nsigma N` | top-n-sigma sampling (default: -1.0, -1.0 = disabled) |
| `--xtc-probability N` | xtc probability (default: 0.0, 0.0 = disabled) |
| `--xtc-threshold N` | xtc threshold (default: 0.1, 1.0 = disabled) |
| `--typical N` | locally typical sampling, parameter p (default: 1.0, 1.0 = disabled) |
| `--top-p N` | top-p sampling (default: 0.95, 1.0 = disabled) |
| `--min-p N` | min-p sampling (default: 0.05, 0.0 = disabled) |
| `--top-nsigma N` | top-n-sigma sampling (default: -1.00, -1.0 = disabled) |
| `--xtc-probability N` | xtc probability (default: 0.00, 0.0 = disabled) |
| `--xtc-threshold N` | xtc threshold (default: 0.10, 1.0 = disabled) |
| `--typical N` | locally typical sampling, parameter p (default: 1.00, 1.0 = disabled) |
| `--repeat-last-n N` | last n tokens to consider for penalize (default: 64, 0 = disabled, -1 = ctx_size) |
| `--repeat-penalty N` | penalize repeat sequence of tokens (default: 1.0, 1.0 = disabled) |
| `--presence-penalty N` | repeat alpha presence penalty (default: 0.0, 0.0 = disabled) |
| `--frequency-penalty N` | repeat alpha frequency penalty (default: 0.0, 0.0 = disabled) |
| `--dry-multiplier N` | set DRY sampling multiplier (default: 0.0, 0.0 = disabled) |
| `--repeat-penalty N` | penalize repeat sequence of tokens (default: 1.00, 1.0 = disabled) |
| `--presence-penalty N` | repeat alpha presence penalty (default: 0.00, 0.0 = disabled) |
| `--frequency-penalty N` | repeat alpha frequency penalty (default: 0.00, 0.0 = disabled) |
| `--dry-multiplier N` | set DRY sampling multiplier (default: 0.00, 0.0 = disabled) |
| `--dry-base N` | set DRY sampling base value (default: 1.75) |
| `--dry-allowed-length N` | set allowed length for DRY sampling (default: 2) |
| `--dry-penalty-last-n N` | set DRY penalty for the last n tokens (default: -1, 0 = disable, -1 = context size) |
| `--dry-sequence-breaker STRING` | add sequence breaker for DRY sampling, clearing out default breakers ('\n', ':', '"', '*') in the process; use "none" to not use any sequence breakers |
| `--dynatemp-range N` | dynamic temperature range (default: 0.0, 0.0 = disabled) |
| `--dynatemp-exp N` | dynamic temperature exponent (default: 1.0) |
| `--adaptive-target N` | adaptive-p: select tokens near this probability (valid range 0.0 to 1.0; negative = disabled) (default: -1.00)<br/>[(more info)](https://github.com/ggml-org/llama.cpp/pull/17927) |
| `--adaptive-decay N` | adaptive-p: decay rate for target adaptation over time. lower values are more reactive, higher values are more stable.<br/>(valid range 0.0 to 0.99) (default: 0.90) |
| `--dynatemp-range N` | dynamic temperature range (default: 0.00, 0.0 = disabled) |
| `--dynatemp-exp N` | dynamic temperature exponent (default: 1.00) |
| `--mirostat N` | use Mirostat sampling.<br/>Top K, Nucleus and Locally Typical samplers are ignored if used.<br/>(default: 0, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0) |
| `--mirostat-lr N` | Mirostat learning rate, parameter eta (default: 0.1) |
| `--mirostat-ent N` | Mirostat target entropy, parameter tau (default: 5.0) |
| `--mirostat-lr N` | Mirostat learning rate, parameter eta (default: 0.10) |
| `--mirostat-ent N` | Mirostat target entropy, parameter tau (default: 5.00) |
| `-l, --logit-bias TOKEN_ID(+/-)BIAS` | modifies the likelihood of token appearing in the completion,<br/>i.e. `--logit-bias 15043+1` to increase likelihood of token ' Hello',<br/>or `--logit-bias 15043-1` to decrease likelihood of token ' Hello' |
| `--grammar GRAMMAR` | BNF-like grammar to constrain generations (see samples in grammars/ dir) (default: '') |
| `--grammar-file FNAME` | file to read grammar from |
@ -173,12 +173,12 @@
| `--jinja, --no-jinja` | whether to use jinja template engine for chat (default: enabled)<br/>(env: LLAMA_ARG_JINJA) |
| `--reasoning-format FORMAT` | controls whether thought tags are allowed and/or extracted from the response, and in which format they're returned; one of:<br/>- none: leaves thoughts unparsed in `message.content`<br/>- deepseek: puts thoughts in `message.reasoning_content`<br/>- deepseek-legacy: keeps `<think>` tags in `message.content` while also populating `message.reasoning_content`<br/>(default: auto)<br/>(env: LLAMA_ARG_THINK) |
| `--reasoning-budget N` | controls the amount of thinking allowed; currently only one of: -1 for unrestricted thinking budget, or 0 to disable thinking (default: -1)<br/>(env: LLAMA_ARG_THINK_BUDGET) |
| `--chat-template JINJA_TEMPLATE` | set custom jinja chat template (default: template taken from model's metadata)<br/>if suffix/prefix are specified, template will be disabled<br/>only commonly used templates are accepted (unless --jinja is set before this flag):<br/>list of built-in templates:<br/>bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek2, deepseek3, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, grok-2, hunyuan-dense, hunyuan-moe, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr<br/>(env: LLAMA_ARG_CHAT_TEMPLATE) |
| `--chat-template-file JINJA_TEMPLATE_FILE` | set custom jinja chat template file (default: template taken from model's metadata)<br/>if suffix/prefix are specified, template will be disabled<br/>only commonly used templates are accepted (unless --jinja is set before this flag):<br/>list of built-in templates:<br/>bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek2, deepseek3, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, grok-2, hunyuan-dense, hunyuan-moe, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr<br/>(env: LLAMA_ARG_CHAT_TEMPLATE_FILE) |
| `--chat-template JINJA_TEMPLATE` | set custom jinja chat template (default: template taken from model's metadata)<br/>if suffix/prefix are specified, template will be disabled<br/>only commonly used templates are accepted (unless --jinja is set before this flag):<br/>list of built-in templates:<br/>bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, grok-2, hunyuan-dense, hunyuan-moe, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr<br/>(env: LLAMA_ARG_CHAT_TEMPLATE) |
| `--chat-template-file JINJA_TEMPLATE_FILE` | set custom jinja chat template file (default: template taken from model's metadata)<br/>if suffix/prefix are specified, template will be disabled<br/>only commonly used templates are accepted (unless --jinja is set before this flag):<br/>list of built-in templates:<br/>bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, grok-2, hunyuan-dense, hunyuan-moe, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr<br/>(env: LLAMA_ARG_CHAT_TEMPLATE_FILE) |
| `--simple-io` | use basic IO for better compatibility in subprocesses and limited consoles |
| `--draft, --draft-n, --draft-max N` | number of tokens to draft for speculative decoding (default: 16)<br/>(env: LLAMA_ARG_DRAFT_MAX) |
| `--draft-min, --draft-n-min N` | minimum number of draft tokens to use for speculative decoding (default: 0)<br/>(env: LLAMA_ARG_DRAFT_MIN) |
| `--draft-p-min P` | minimum speculative decoding probability (greedy) (default: 0.8)<br/>(env: LLAMA_ARG_DRAFT_P_MIN) |
| `--draft-p-min P` | minimum speculative decoding probability (greedy) (default: 0.75)<br/>(env: LLAMA_ARG_DRAFT_P_MIN) |
| `-cd, --ctx-size-draft N` | size of the prompt context for the draft model (default: 0, 0 = loaded from model)<br/>(env: LLAMA_ARG_CTX_SIZE_DRAFT) |
| `-devd, --device-draft <dev1,dev2,..>` | comma-separated list of devices to use for offloading the draft model (none = don't offload)<br/>use --list-devices to see a list of available devices |
| `-ngld, --gpu-layers-draft, --n-gpu-layers-draft N` | max. number of draft model layers to store in VRAM, either an exact number, 'auto', or 'all' (default: auto)<br/>(env: LLAMA_ARG_N_GPU_LAYERS_DRAFT) |

44
tools/completion/README.md
View file

@ -128,10 +128,10 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1
| `--rope-freq-base N` | RoPE base frequency, used by NTK-aware scaling (default: loaded from model)<br/>(env: LLAMA_ARG_ROPE_FREQ_BASE) |
| `--rope-freq-scale N` | RoPE frequency scaling factor, expands context by a factor of 1/N<br/>(env: LLAMA_ARG_ROPE_FREQ_SCALE) |
| `--yarn-orig-ctx N` | YaRN: original context size of model (default: 0 = model training context size)<br/>(env: LLAMA_ARG_YARN_ORIG_CTX) |
| `--yarn-ext-factor N` | YaRN: extrapolation mix factor (default: -1.0, 0.0 = full interpolation)<br/>(env: LLAMA_ARG_YARN_EXT_FACTOR) |
| `--yarn-attn-factor N` | YaRN: scale sqrt(t) or attention magnitude (default: -1.0)<br/>(env: LLAMA_ARG_YARN_ATTN_FACTOR) |
| `--yarn-beta-slow N` | YaRN: high correction dim or alpha (default: -1.0)<br/>(env: LLAMA_ARG_YARN_BETA_SLOW) |
| `--yarn-beta-fast N` | YaRN: low correction dim or beta (default: -1.0)<br/>(env: LLAMA_ARG_YARN_BETA_FAST) |
| `--yarn-ext-factor N` | YaRN: extrapolation mix factor (default: -1.00, 0.0 = full interpolation)<br/>(env: LLAMA_ARG_YARN_EXT_FACTOR) |
| `--yarn-attn-factor N` | YaRN: scale sqrt(t) or attention magnitude (default: -1.00)<br/>(env: LLAMA_ARG_YARN_ATTN_FACTOR) |
| `--yarn-beta-slow N` | YaRN: high correction dim or alpha (default: -1.00)<br/>(env: LLAMA_ARG_YARN_BETA_SLOW) |
| `--yarn-beta-fast N` | YaRN: low correction dim or beta (default: -1.00)<br/>(env: LLAMA_ARG_YARN_BETA_FAST) |
| `-kvo, --kv-offload, -nkvo, --no-kv-offload` | whether to enable KV cache offloading (default: enabled)<br/>(env: LLAMA_ARG_KV_OFFLOAD) |
| `--repack, -nr, --no-repack` | whether to enable weight repacking (default: enabled)<br/>(env: LLAMA_ARG_REPACK) |
| `--no-host` | bypass host buffer allowing extra buffers to be used<br/>(env: LLAMA_ARG_NO_HOST) |
@ -192,28 +192,30 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1
| `-s, --seed SEED` | RNG seed (default: -1, use random seed for -1) |
| `--sampler-seq, --sampling-seq SEQUENCE` | simplified sequence for samplers that will be used (default: edskypmxt) |
| `--ignore-eos` | ignore end of stream token and continue generating (implies --logit-bias EOS-inf) |
| `--temp N` | temperature (default: 0.8) |
| `--temp N` | temperature (default: 0.80) |
| `--top-k N` | top-k sampling (default: 40, 0 = disabled)<br/>(env: LLAMA_ARG_TOP_K) |
| `--top-p N` | top-p sampling (default: 0.9, 1.0 = disabled) |
| `--min-p N` | min-p sampling (default: 0.1, 0.0 = disabled) |
| `--top-nsigma N` | top-n-sigma sampling (default: -1.0, -1.0 = disabled) |
| `--xtc-probability N` | xtc probability (default: 0.0, 0.0 = disabled) |
| `--xtc-threshold N` | xtc threshold (default: 0.1, 1.0 = disabled) |
| `--typical N` | locally typical sampling, parameter p (default: 1.0, 1.0 = disabled) |
| `--top-p N` | top-p sampling (default: 0.95, 1.0 = disabled) |
| `--min-p N` | min-p sampling (default: 0.05, 0.0 = disabled) |
| `--top-nsigma N` | top-n-sigma sampling (default: -1.00, -1.0 = disabled) |
| `--xtc-probability N` | xtc probability (default: 0.00, 0.0 = disabled) |
| `--xtc-threshold N` | xtc threshold (default: 0.10, 1.0 = disabled) |
| `--typical N` | locally typical sampling, parameter p (default: 1.00, 1.0 = disabled) |
| `--repeat-last-n N` | last n tokens to consider for penalize (default: 64, 0 = disabled, -1 = ctx_size) |
| `--repeat-penalty N` | penalize repeat sequence of tokens (default: 1.0, 1.0 = disabled) |
| `--presence-penalty N` | repeat alpha presence penalty (default: 0.0, 0.0 = disabled) |
| `--frequency-penalty N` | repeat alpha frequency penalty (default: 0.0, 0.0 = disabled) |
| `--dry-multiplier N` | set DRY sampling multiplier (default: 0.0, 0.0 = disabled) |
| `--repeat-penalty N` | penalize repeat sequence of tokens (default: 1.00, 1.0 = disabled) |
| `--presence-penalty N` | repeat alpha presence penalty (default: 0.00, 0.0 = disabled) |
| `--frequency-penalty N` | repeat alpha frequency penalty (default: 0.00, 0.0 = disabled) |
| `--dry-multiplier N` | set DRY sampling multiplier (default: 0.00, 0.0 = disabled) |
| `--dry-base N` | set DRY sampling base value (default: 1.75) |
| `--dry-allowed-length N` | set allowed length for DRY sampling (default: 2) |
| `--dry-penalty-last-n N` | set DRY penalty for the last n tokens (default: -1, 0 = disable, -1 = context size) |
| `--dry-sequence-breaker STRING` | add sequence breaker for DRY sampling, clearing out default breakers ('\n', ':', '"', '*') in the process; use "none" to not use any sequence breakers |
| `--dynatemp-range N` | dynamic temperature range (default: 0.0, 0.0 = disabled) |
| `--dynatemp-exp N` | dynamic temperature exponent (default: 1.0) |
| `--adaptive-target N` | adaptive-p: select tokens near this probability (valid range 0.0 to 1.0; negative = disabled) (default: -1.00)<br/>[(more info)](https://github.com/ggml-org/llama.cpp/pull/17927) |
| `--adaptive-decay N` | adaptive-p: decay rate for target adaptation over time. lower values are more reactive, higher values are more stable.<br/>(valid range 0.0 to 0.99) (default: 0.90) |
| `--dynatemp-range N` | dynamic temperature range (default: 0.00, 0.0 = disabled) |
| `--dynatemp-exp N` | dynamic temperature exponent (default: 1.00) |
| `--mirostat N` | use Mirostat sampling.<br/>Top K, Nucleus and Locally Typical samplers are ignored if used.<br/>(default: 0, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0) |
| `--mirostat-lr N` | Mirostat learning rate, parameter eta (default: 0.1) |
| `--mirostat-ent N` | Mirostat target entropy, parameter tau (default: 5.0) |
| `--mirostat-lr N` | Mirostat learning rate, parameter eta (default: 0.10) |
| `--mirostat-ent N` | Mirostat target entropy, parameter tau (default: 5.00) |
| `-l, --logit-bias TOKEN_ID(+/-)BIAS` | modifies the likelihood of token appearing in the completion,<br/>i.e. `--logit-bias 15043+1` to increase likelihood of token ' Hello',<br/>or `--logit-bias 15043-1` to decrease likelihood of token ' Hello' |
| `--grammar GRAMMAR` | BNF-like grammar to constrain generations (see samples in grammars/ dir) (default: '') |
| `--grammar-file FNAME` | file to read grammar from |
@ -251,8 +253,8 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1
| `--jinja, --no-jinja` | whether to use jinja template engine for chat (default: disabled)<br/>(env: LLAMA_ARG_JINJA) |
| `--reasoning-format FORMAT` | controls whether thought tags are allowed and/or extracted from the response, and in which format they're returned; one of:<br/>- none: leaves thoughts unparsed in `message.content`<br/>- deepseek: puts thoughts in `message.reasoning_content`<br/>- deepseek-legacy: keeps `<think>` tags in `message.content` while also populating `message.reasoning_content`<br/>(default: auto)<br/>(env: LLAMA_ARG_THINK) |
| `--reasoning-budget N` | controls the amount of thinking allowed; currently only one of: -1 for unrestricted thinking budget, or 0 to disable thinking (default: -1)<br/>(env: LLAMA_ARG_THINK_BUDGET) |
| `--chat-template JINJA_TEMPLATE` | set custom jinja chat template (default: template taken from model's metadata)<br/>if suffix/prefix are specified, template will be disabled<br/>only commonly used templates are accepted (unless --jinja is set before this flag):<br/>list of built-in templates:<br/>bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek2, deepseek3, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, grok-2, hunyuan-dense, hunyuan-moe, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr<br/>(env: LLAMA_ARG_CHAT_TEMPLATE) |
| `--chat-template-file JINJA_TEMPLATE_FILE` | set custom jinja chat template file (default: template taken from model's metadata)<br/>if suffix/prefix are specified, template will be disabled<br/>only commonly used templates are accepted (unless --jinja is set before this flag):<br/>list of built-in templates:<br/>bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek2, deepseek3, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, grok-2, hunyuan-dense, hunyuan-moe, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr<br/>(env: LLAMA_ARG_CHAT_TEMPLATE_FILE) |
| `--chat-template JINJA_TEMPLATE` | set custom jinja chat template (default: template taken from model's metadata)<br/>if suffix/prefix are specified, template will be disabled<br/>only commonly used templates are accepted (unless --jinja is set before this flag):<br/>list of built-in templates:<br/>bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, grok-2, hunyuan-dense, hunyuan-moe, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr<br/>(env: LLAMA_ARG_CHAT_TEMPLATE) |
| `--chat-template-file JINJA_TEMPLATE_FILE` | set custom jinja chat template file (default: template taken from model's metadata)<br/>if suffix/prefix are specified, template will be disabled<br/>only commonly used templates are accepted (unless --jinja is set before this flag):<br/>list of built-in templates:<br/>bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, grok-2, hunyuan-dense, hunyuan-moe, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr<br/>(env: LLAMA_ARG_CHAT_TEMPLATE_FILE) |
| `--simple-io` | use basic IO for better compatibility in subprocesses and limited consoles |
<!-- HELP_END -->

101
tools/completion/completion.cpp
View file

@ -342,44 +342,51 @@ int main(int argc, char ** argv) {
return 1;
}
// debug message about similarity of saved session, if applicable
size_t n_matching_session_tokens = 0;
if (!session_tokens.empty()) {
for (llama_token id : session_tokens) {
if (n_matching_session_tokens >= embd_inp.size() || id != embd_inp[n_matching_session_tokens]) {
break;
bool session_do_save = false;
{
size_t n_match = 0;
if (!session_tokens.empty()) {
for (llama_token id : session_tokens) {
if (n_match >= embd_inp.size() || id != embd_inp[n_match]) {
break;
}
n_match++;
}
if (params.prompt.empty() && n_match == embd_inp.size()) {
LOG_INF("%s: using full prompt from session file\n", __func__);
} else if (n_match >= embd_inp.size()) {
LOG_INF("%s: session file has exact match for prompt!\n", __func__);
} else if (n_match < (embd_inp.size() / 2)) {
LOG_WRN("%s: session file has low similarity to prompt (%zu / %zu tokens); will mostly be reevaluated\n",
__func__, n_match, embd_inp.size());
} else {
LOG_INF("%s: session file matches %zu / %zu tokens of prompt\n",
__func__, n_match, embd_inp.size());
}
if (session_tokens.size() == n_match) {
// [TAG_CONTEXT_STATE_LOGITS]
// in this case, we are going to reuse the logits from the session
// if we ever decide to remove the logits from the session, we need to handle this somehow
// ref: https://github.com/ggml-org/llama.cpp/pull/18862#issuecomment-3756330941
}
// remove any "future" tokens that we might have inherited from the previous session
if (session_tokens.size() > n_match) {
if (!llama_memory_seq_rm(mem, -1, n_match, -1)) {
LOG_WRN("%s: unable to resuse common prefix (for example, when the memory is recurrent)\n", __func__);
llama_memory_clear(mem, true);
session_tokens.clear();
n_match = 0;
} else {
session_tokens.resize(n_match);
}
}
n_matching_session_tokens++;
}
if (params.prompt.empty() && n_matching_session_tokens == embd_inp.size()) {
LOG_INF("%s: using full prompt from session file\n", __func__);
} else if (n_matching_session_tokens >= embd_inp.size()) {
LOG_INF("%s: session file has exact match for prompt!\n", __func__);
} else if (n_matching_session_tokens < (embd_inp.size() / 2)) {
LOG_WRN("%s: session file has low similarity to prompt (%zu / %zu tokens); will mostly be reevaluated\n",
__func__, n_matching_session_tokens, embd_inp.size());
} else {
LOG_INF("%s: session file matches %zu / %zu tokens of prompt\n",
__func__, n_matching_session_tokens, embd_inp.size());
}
// remove any "future" tokens that we might have inherited from the previous session
if (!llama_memory_seq_rm(mem, -1, n_matching_session_tokens, -1)) {
LOG_INF("%s: unable to resuse common prefix\n", __func__);
n_matching_session_tokens = 0;
llama_memory_seq_rm(mem, -1, -1, -1);
}
}
LOG_DBG("recalculate the cached logits (check): embd_inp.size() %zu, n_matching_session_tokens %zu, embd_inp.size() %zu, session_tokens.size() %zu\n",
embd_inp.size(), n_matching_session_tokens, embd_inp.size(), session_tokens.size());
// if we will use the cache for the full prompt without reaching the end of the cache, force
// reevaluation of the last token to recalculate the cached logits
if (!embd_inp.empty() && n_matching_session_tokens == embd_inp.size() && session_tokens.size() > embd_inp.size()) {
LOG_DBG("recalculate the cached logits (do): session_tokens.resize( %zu )\n", embd_inp.size() - 1);
session_tokens.resize(embd_inp.size() - 1);
session_do_save = !path_session.empty() && n_match < embd_inp.size() && !params.prompt_cache_ro;
}
// number of tokens to keep when resetting context
@ -521,10 +528,9 @@ int main(int argc, char ** argv) {
is_interacting = params.interactive_first;
}
bool is_antiprompt = false;
bool input_echo = true;
bool display = true;
bool need_to_save_session = !path_session.empty() && n_matching_session_tokens < embd_inp.size();
bool is_antiprompt = false;
bool input_echo = true;
bool display = true;
int n_past = 0;
int n_remain = params.n_predict;
@ -668,15 +674,12 @@ int main(int argc, char ** argv) {
}
}
for (int i = 0; i < (int) embd.size(); i += params.n_batch) {
int n_eval = (int) embd.size() - i;
if (n_eval > params.n_batch) {
n_eval = params.n_batch;
}
if (!embd.empty()) {
int n_eval = (int) embd.size();
LOG_DBG("eval: %s\n", string_from(ctx, embd).c_str());
if (llama_decode(ctx, llama_batch_get_one(&embd[i], n_eval))) {
GGML_ASSERT(n_eval <= params.n_batch);
if (llama_decode(ctx, llama_batch_get_one(embd.data(), n_eval))) {
LOG_ERR("%s : failed to eval\n", __func__);
return 1;
}
@ -700,8 +703,8 @@ int main(int argc, char ** argv) {
if ((int) embd_inp.size() <= n_consumed && !is_interacting) {
// optionally save the session on first sample (for faster prompt loading next time)
if (!path_session.empty() && need_to_save_session && !params.prompt_cache_ro) {
need_to_save_session = false;
if (session_do_save) {
session_do_save = false;
llama_state_save_file(ctx, path_session.c_str(), session_tokens.data(), session_tokens.size());
LOG_DBG("saved session to %s\n", path_session.c_str());
@ -737,7 +740,7 @@ int main(int argc, char ** argv) {
common_sampler_accept(smpl, embd_inp[n_consumed], /* accept_grammar= */ false);
++n_consumed;
if ((int) embd.size() >= params.n_batch) {
if ((int) embd.size() == params.n_batch) {
break;
}
}

2
tools/cvector-generator/pca.hpp
View file

@ -290,7 +290,7 @@ static void power_iteration(
ggml_gallocr_free(allocr);
// TODO @ngxson : The output vector is randomly inverted
// Solution: https://github.com/ggerganov/llama.cpp/pull/8069#issuecomment-2185328171
// Solution: https://github.com/ggml-org/llama.cpp/pull/8069#issuecomment-2185328171
}
static void run_pca(

2
tools/export-lora/export-lora.cpp
View file

@ -190,7 +190,7 @@ struct lora_merge_ctx {
gguf_set_val_u32(ctx_out, "general.file_type", LLAMA_FTYPE_MOSTLY_F16);
// check if all lora adapters have the same tensors
// TODO: remove this when we can support merging subset of adapters. Ref: https://github.com/ggerganov/llama.cpp/pull/8607#discussion_r1686027777
// TODO: remove this when we can support merging subset of adapters. Ref: https://github.com/ggml-org/llama.cpp/pull/8607#discussion_r1686027777
static const char * err_no_subset_adapter = "Input adapters do not have the same list of tensors. This is not yet supported. Please merge the adapter one-by-one instead of merging all at once.";
if (adapters.size() > 1) {
for (size_t i = 1; i < adapters.size(); ++i) {

2
tools/fit-params/fit-params.cpp
View file

@ -36,7 +36,7 @@ int main(int argc, char ** argv) {
LOG_INF("%s: printing fitted CLI arguments to stdout...\n", __func__);
common_log_flush(common_log_main());
printf("-c %" PRIu32 " -ngl %" PRIu32, cparams.n_ctx, mparams.n_gpu_layers);
printf("-c %" PRIu32 " -ngl %" PRIi32, cparams.n_ctx, mparams.n_gpu_layers);
size_t nd = llama_max_devices();
while (nd > 1 && mparams.tensor_split[nd - 1] == 0.0f) {

1
tools/mtmd/CMakeLists.txt
View file

@ -19,6 +19,7 @@ add_library(mtmd
models/glm4v.cpp
models/internvl.cpp
models/kimivl.cpp
models/kimik25.cpp
models/llama4.cpp
models/llava.cpp
models/minicpmv.cpp

4
tools/mtmd/clip-impl.h
View file

@ -36,6 +36,8 @@
// vision-specific
#define KEY_VISION_PROJ_TYPE "clip.vision.projector_type" // for models with mixed modalities
#define KEY_IMAGE_SIZE "clip.vision.image_size"
#define KEY_IMAGE_MIN_PIXELS "clip.vision.image_min_pixels"
#define KEY_IMAGE_MAX_PIXELS "clip.vision.image_max_pixels"
#define KEY_PREPROC_IMAGE_SIZE "clip.vision.preproc_image_size"
#define KEY_PATCH_SIZE "clip.vision.patch_size"
#define KEY_IMAGE_MEAN "clip.vision.image_mean"
@ -233,6 +235,7 @@ enum projector_type {
PROJECTOR_TYPE_LFM2A,
PROJECTOR_TYPE_GLM4V,
PROJECTOR_TYPE_YOUTUVL,
PROJECTOR_TYPE_KIMIK25,
PROJECTOR_TYPE_UNKNOWN,
};
@ -266,6 +269,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_LFM2A, "lfm2a"},
{ PROJECTOR_TYPE_GLM4V, "glm4v"},
{ PROJECTOR_TYPE_YOUTUVL, "youtuvl"},
{ PROJECTOR_TYPE_KIMIK25, "kimik25"},
};
static projector_type clip_projector_type_from_string(const std::string & str) {

231
tools/mtmd/clip.cpp
View file

@ -10,6 +10,7 @@
#include "ggml-backend.h"
#include "gguf.h"
#include <algorithm>
#include <cassert>
#include <cmath>
#include <cstdlib>
@ -672,8 +673,8 @@ ggml_tensor * clip_graph::build_rope_2d(
{
first = ggml_view_3d(ctx0, cur,
n_dim/2, n_head, n_pos,
ggml_row_size(cur->type, n_dim),
ggml_row_size(cur->type, n_dim*n_head),
cur->nb[1],
cur->nb[2],
0);
first = ggml_rope_ext(
ctx0,
@ -691,8 +692,8 @@ ggml_tensor * clip_graph::build_rope_2d(
{
second = ggml_view_3d(ctx0, cur,
n_dim/2, n_head, n_pos,
ggml_row_size(cur->type, n_dim),
ggml_row_size(cur->type, n_dim*n_head),
cur->nb[1],
cur->nb[2],
n_dim/2 * ggml_element_size(cur));
second = ggml_rope_ext(
ctx0,
@ -825,6 +826,10 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32
{
builder = std::make_unique<clip_graph_kimivl>(ctx, img);
} break;
case PROJECTOR_TYPE_KIMIK25:
{
builder = std::make_unique<clip_graph_kimik25>(ctx, img);
} break;
case PROJECTOR_TYPE_COGVLM:
{
builder = std::make_unique<clip_graph_cogvlm>(ctx, img);
@ -1005,6 +1010,8 @@ struct clip_model_loader {
hparams.minicpmv_query_num = 64;
} else if (hparams.minicpmv_version == 6) {
hparams.minicpmv_query_num = 64;
} else if (hparams.minicpmv_version == 100045) {
hparams.minicpmv_query_num = 64;
} else {
hparams.minicpmv_query_num = 96;
}
@ -1114,9 +1121,8 @@ struct clip_model_loader {
case PROJECTOR_TYPE_LFM2:
{
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
// ref: https://huggingface.co/LiquidAI/LFM2-VL-3B/blob/main/preprocessor_config.json
// config above specifies number of tokens after downsampling, while here it is before, relax lowerbound to 64
hparams.set_limit_image_tokens(64, 1024);
// ref: https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B/blob/main/processor_config.json
hparams.set_limit_image_tokens(64, 256);
} break;
case PROJECTOR_TYPE_PIXTRAL:
case PROJECTOR_TYPE_LIGHTONOCR:
@ -1137,6 +1143,22 @@ struct clip_model_loader {
hparams.set_limit_image_tokens(8, 1024);
hparams.set_warmup_n_tokens(256); // avoid OOM on warmup
} break;
case PROJECTOR_TYPE_KIMIK25:
{
hparams.rope_theta = 10000.0f;
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
int min_pixels = 0, max_pixels = 0;
get_u32(KEY_IMAGE_MIN_PIXELS, min_pixels, false);
get_u32(KEY_IMAGE_MAX_PIXELS, max_pixels, false);
if (min_pixels > 0 && max_pixels > 0) {
hparams.image_min_pixels = min_pixels;
hparams.image_max_pixels = max_pixels;
hparams.warmup_image_size = static_cast<int>(std::sqrt(max_pixels));
} else {
hparams.set_limit_image_tokens(2, 4096);
}
} break;
case PROJECTOR_TYPE_GEMMA3:
{
// default value (used by all model sizes in gemma 3 family)
@ -1666,6 +1688,7 @@ struct clip_model_loader {
model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
} break;
case PROJECTOR_TYPE_KIMIVL:
case PROJECTOR_TYPE_KIMIK25:
{
model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM);
model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B);
@ -2805,6 +2828,119 @@ private:
}
};
// ref: https://github.com/huggingface/transformers/blob/v5.1.0/src/transformers/models/lfm2_vl/image_processing_lfm2_vl_fast.py
// some of the logic is similar to llava_uhd, but with different hyperparameters and some logic is unique (e.g. grid layout)
struct lfm2_vl_image_processor {
// ref: https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B/blob/main/processor_config.json
static constexpr int min_tiles = 2;
static constexpr int max_tiles = 10;
static constexpr float max_pixels_tolerance = 2.0f;
static constexpr int tile_size = 512;
static llava_uhd::slice_instructions get_slice_instructions(struct clip_ctx * ctx, const clip_image_size & original_size) {
llava_uhd::slice_instructions inst;
const auto & params = ctx->model.hparams;
const int align_size = params.patch_size * params.n_merge;
inst.interpolation_overview = img_tool::RESIZE_ALGO_BILINEAR;
inst.interpolation_refined = img_tool::RESIZE_ALGO_BILINEAR;
inst.overview_size = img_tool::calc_size_preserved_ratio(original_size, align_size, params.image_min_pixels, params.image_max_pixels);
// tile if either dimension exceeds tile_size with tolerance
const bool needs_tiling = original_size.width > tile_size * max_pixels_tolerance || original_size.height > tile_size * max_pixels_tolerance;
if (!needs_tiling) {
inst.refined_size = clip_image_size{0, 0};
inst.grid_size = clip_image_size{0, 0};
return inst;
}
const clip_image_size grid = get_grid_layout(original_size.height, original_size.width);
inst.grid_size = grid;
inst.refined_size = clip_image_size{tile_size * grid.width, tile_size * grid.height};
LOG_DBG("%s: original size: %d x %d, overview size: %d x %d, refined size: %d x %d, grid size: %d x %d\n",
__func__,
original_size.width, original_size.height,
inst.overview_size.width, inst.overview_size.height,
inst.refined_size.width, inst.refined_size.height,
grid.width, grid.height);
for (int row = 0; row < grid.height; row++) {
for (int col = 0; col < grid.width; col++) {
llava_uhd::slice_coordinates slice;
slice.x = col * tile_size;
slice.y = row * tile_size;
slice.size = clip_image_size{tile_size, tile_size};
inst.slices.push_back(slice);
LOG_DBG("%s: slice %d: x=%d, y=%d, size=%d x %d\n",
__func__, (int)inst.slices.size() - 1,
slice.x, slice.y, slice.size.width, slice.size.height);
}
}
return inst;
}
private:
static clip_image_size find_closest_aspect_ratio(
float aspect_ratio,
const std::vector<clip_image_size> & target_ratios,
int width, int height) {
float best_ratio_diff = std::numeric_limits<float>::max();
clip_image_size best_ratio = {1, 1};
const float area = static_cast<float>(width * height);
for (const auto & ratio : target_ratios) {
const float target_aspect_ratio = static_cast<float>(ratio.width) / ratio.height;
const float ratio_diff = std::abs(aspect_ratio - target_aspect_ratio);
if (ratio_diff < best_ratio_diff) {
best_ratio_diff = ratio_diff;
best_ratio = ratio;
} else if (ratio_diff == best_ratio_diff) {
const float target_area = static_cast<float>(tile_size * tile_size * ratio.width * ratio.height);
if (area > 0.5f * target_area) {
best_ratio = ratio;
}
}
}
return best_ratio;
}
static std::vector<clip_image_size> get_target_ratios() {
std::vector<clip_image_size> ratios;
for (int n = min_tiles; n <= max_tiles; n++) {
for (int w = 1; w <= n; w++) {
for (int h = 1; h <= n; h++) {
if (w * h >= min_tiles && w * h <= max_tiles) {
bool found = false;
for (const auto & r : ratios) {
if (r.width == w && r.height == h) {
found = true;
break;
}
}
if (!found) {
ratios.push_back({w, h});
}
}
}
}
}
std::sort(ratios.begin(), ratios.end(), [](const clip_image_size & a, const clip_image_size & b) {
return a.width * a.height < b.width * b.height;
});
return ratios;
}
static clip_image_size get_grid_layout(int height, int width) {
const float aspect_ratio = static_cast<float>(width) / height;
const auto ratios = get_target_ratios();
return find_closest_aspect_ratio(aspect_ratio, ratios, width, height);
}
};
// returns the normalized float tensor for llava-1.5, for spatial_unpad with anyres processing for llava-1.6 it returns the normalized image patch tensors as a vector
// res_imgs memory is being allocated here, previous allocations will be freed if found
bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, struct clip_image_f32_batch * res_imgs) {
@ -3019,6 +3155,20 @@ bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, str
} break;
case PROJECTOR_TYPE_LFM2:
{
auto const inst = lfm2_vl_image_processor::get_slice_instructions(ctx, original_size);
std::vector<clip_image_u8_ptr> imgs = llava_uhd::slice_image(img, inst);
for (size_t i = 0; i < imgs.size(); ++i) {
clip_image_f32_ptr res(clip_image_f32_init());
normalize_image_u8_to_f32(*imgs[i], *res, params.image_mean, params.image_std);
res_imgs->entries.push_back(std::move(res));
}
res_imgs->grid_x = inst.grid_size.width;
res_imgs->grid_y = inst.grid_size.height;
} break;
case PROJECTOR_TYPE_KIMIVL:
{
GGML_ASSERT(params.image_min_pixels > 0 && params.image_max_pixels > 0);
@ -3030,8 +3180,24 @@ bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, str
const std::array<uint8_t, 3> pad_color = {122, 116, 104};
clip_image_u8 resized_img;
const bool pad = (ctx->proj_type() != PROJECTOR_TYPE_LFM2);
img_tool::resize(*img, resized_img, target_size, img_tool::RESIZE_ALGO_BILINEAR, pad, pad_color);
img_tool::resize(*img, resized_img, target_size, img_tool::RESIZE_ALGO_BILINEAR, true, pad_color);
clip_image_f32_ptr res(clip_image_f32_init());
normalize_image_u8_to_f32(resized_img, *res, params.image_mean, params.image_std);
res_imgs->entries.push_back(std::move(res));
} break;
case PROJECTOR_TYPE_KIMIK25:
{
GGML_ASSERT(params.image_min_pixels > 0 && params.image_max_pixels > 0);
const clip_image_size target_size = img_tool::calc_size_preserved_ratio(
original_size,
params.patch_size * params.n_merge,
params.image_min_pixels,
params.image_max_pixels);
const std::array<uint8_t, 3> pad_color = {0, 0, 0};
clip_image_u8 resized_img;
img_tool::resize(*img, resized_img, target_size, img_tool::RESIZE_ALGO_BICUBIC, true, pad_color);
clip_image_f32_ptr res(clip_image_f32_init());
normalize_image_u8_to_f32(resized_img, *res, params.image_mean, params.image_std);
res_imgs->entries.push_back(std::move(res));
@ -3209,6 +3375,9 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im
} else if (params.minicpmv_version == 6) {
// MiniCPM-V 4.5
n_patches = 64;
} else if (params.minicpmv_version == 100045) {
// MiniCPM-o 4.5
n_patches = 64;
} else {
GGML_ABORT("Unknown minicpmv version");
}
@ -3242,6 +3411,7 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im
} break;
case PROJECTOR_TYPE_LFM2:
case PROJECTOR_TYPE_KIMIVL:
case PROJECTOR_TYPE_KIMIK25:
{
// dynamic size
int out_patch_size = params.patch_size * ctx->model.hparams.n_merge;
@ -3583,6 +3753,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
} break;
case PROJECTOR_TYPE_PIXTRAL:
case PROJECTOR_TYPE_KIMIVL:
case PROJECTOR_TYPE_KIMIK25:
case PROJECTOR_TYPE_LIGHTONOCR:
{
// set the 2D positions
@ -3719,6 +3890,47 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
ggml_backend_tensor_get(embeddings, vec, 0, ggml_nbytes(embeddings));
}
// Debug: dump final embeddings if MTMD_DEBUG_EMBEDDINGS is set
if (std::getenv("MTMD_DEBUG_EMBEDDINGS") != nullptr) {
const int64_t n_embd = embeddings->ne[0];
const int64_t n_tokens = embeddings->ne[1];
std::vector<float> emb_data(n_embd * n_tokens);
ggml_backend_tensor_get(embeddings, emb_data.data(), 0, ggml_nbytes(embeddings));
LOG_INF("\n=== MTMD_DEBUG_EMBEDDINGS ===\n");
LOG_INF("Shape: [%lld, %lld]\n", (long long)n_embd, (long long)n_tokens);
// Print first few values of first token
LOG_INF("Token 0 (first 16 values): ");
for (int i = 0; i < std::min((int64_t)16, n_embd); i++) {
LOG_INF("%.6f ", emb_data[i]);
}
LOG_INF("\n");
// Print last few values of first token
if (n_embd > 16) {
LOG_INF("Token 0 (last 16 values): ");
for (int64_t i = n_embd - 16; i < n_embd; i++) {
LOG_INF("%.6f ", emb_data[i]);
}
LOG_INF("\n");
}
// Compute and print statistics
float sum = 0.0f, sum_sq = 0.0f, min_val = emb_data[0], max_val = emb_data[0];
for (size_t i = 0; i < emb_data.size(); i++) {
sum += emb_data[i];
sum_sq += emb_data[i] * emb_data[i];
min_val = std::min(min_val, emb_data[i]);
max_val = std::max(max_val, emb_data[i]);
}
float mean = sum / emb_data.size();
float variance = (sum_sq / emb_data.size()) - (mean * mean);
LOG_INF("Stats: mean=%.6f, std=%.6f, min=%.6f, max=%.6f, sum=%.6f\n",
mean, sqrtf(variance), min_val, max_val, sum);
LOG_INF("=== END MTMD_DEBUG_EMBEDDINGS ===\n\n");
}
return true;
}
@ -3765,6 +3977,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
return ctx->model.mm_2_w->ne[1];
case PROJECTOR_TYPE_LFM2:
case PROJECTOR_TYPE_KIMIVL:
case PROJECTOR_TYPE_KIMIK25:
return ctx->model.mm_2_w->ne[1];
case PROJECTOR_TYPE_COGVLM:
return ctx->model.mm_4h_to_h_w->ne[1];

9
tools/mtmd/legacy-models/minicpmv-convert-image-encoder-to-gguf.py
View file

@ -501,7 +501,7 @@ default_image_mean = [0.5, 0.5, 0.5]
default_image_std = [0.5, 0.5, 0.5]
ap.add_argument('--image-mean', type=float, nargs='+', help='Mean of the images for normalization (overrides processor) ', default=None)
ap.add_argument('--image-std', type=float, nargs='+', help='Standard deviation of the images for normalization (overrides processor)', default=None)
ap.add_argument('--minicpmv_version', type=int, help='minicpmv_version: MiniCPM-V-2 use 1; MiniCPM-V-2.5 use 2; MiniCPM-V-2.6 use 3; MiniCPM-o-2.6 use 4; MiniCPM-V 4.0 use 5; MiniCPM-o-4.0 use 6', default=2)
ap.add_argument('--minicpmv_version', type=int, help='minicpmv_version: MiniCPM-V-2 use 1; MiniCPM-V-2.5 use 2; MiniCPM-V-2.6 use 3; MiniCPM-o-2.6 use 4; MiniCPM-V 4.0 use 5; MiniCPM-o-4.0 use 6; MiniCPM-o-4.5 use 100045', default=2)
# with proper
args = ap.parse_args()
@ -610,6 +610,9 @@ else:
elif minicpmv_version == 6:
emb_dim = 4096
block_count = 27
elif minicpmv_version == 100045:
emb_dim = 4096
block_count = 27
default_vision_config = {
"hidden_size": 1152,
@ -637,6 +640,10 @@ elif minicpmv_version == 6:
default_vision_config["model_type"] = "siglip_vision_model"
vision_config = SiglipVisionConfig(**default_vision_config)
model = SiglipVisionTransformer(vision_config)
elif minicpmv_version == 100045:
default_vision_config["model_type"] = "siglip_vision_model"
vision_config = SiglipVisionConfig(**default_vision_config)
model = SiglipVisionTransformer(vision_config)
processor = None
# if model.attn_pool is not None:

101
tools/mtmd/models/kimik25.cpp Normal file
View file

@ -0,0 +1,101 @@
#include "models.h"
#include <cstring>
#include <cmath>
// note: this is similar to clip_graph::resize_position_embeddings, major difference is having
// the w/h in ne[1] and ne[2] instead of assuming with sqrt. Could try storing the tensor in 2D instead
// with a w*h? Also the permute is a bit different at (2, 1, 0, 3) instead of (2, 0, 1, 3).
ggml_tensor * clip_graph_kimik25::resize_position_embeddings_3d(uint32_t interpolation_mode) {
ggml_tensor * pos_embd = model.position_embeddings;
const int height = img.ny / patch_size;
const int width = img.nx / patch_size;
const uint32_t mode = interpolation_mode;
GGML_ASSERT(pos_embd);
const int64_t stored_c = pos_embd->ne[0]; // C = 1152
const int64_t orig_w = pos_embd->ne[1]; // W = 64
const int64_t orig_h = pos_embd->ne[2]; // H = 64
GGML_ASSERT(stored_c == n_embd);
if (height == (int)orig_h && width == (int)orig_w) {
// No interpolation needed, just flatten to [C, H*W]
return ggml_cont_2d(ctx0, pos_embd, n_embd, width * height);
}
pos_embd = ggml_permute(ctx0, pos_embd, 2, 1, 0, 3);
pos_embd = ggml_interpolate(ctx0, pos_embd, height, width, n_embd, 1, mode);
pos_embd = ggml_permute(ctx0, pos_embd, 2, 1, 0, 3);
pos_embd = ggml_cont_2d(ctx0, pos_embd, n_embd, width * height);
return pos_embd;
}
ggml_cgraph * clip_graph_kimik25::build() {
ggml_tensor * pos_h = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
ggml_set_name(pos_h, "pos_h");
ggml_set_input(pos_h);
ggml_tensor * pos_w = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
ggml_set_name(pos_w, "pos_w");
ggml_set_input(pos_w);
ggml_tensor * learned_pos_embd = resize_position_embeddings_3d(GGML_SCALE_MODE_BICUBIC);
// Kimi-K2.5 uses interleaved 2D RoPE pattern natively, but
// Q / K are permuted during conversion to use split format.
auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
cur = build_rope_2d(ctx0, cur, pos_w, pos_h, hparams.rope_theta, false);
return cur;
};
ggml_tensor * inp = build_inp();
// I don't know why, but doing this in the build_vit lead to the ggml_add not occurring?
// Doing it manually here does work.
inp = ggml_add(ctx0, inp, learned_pos_embd);
ggml_tensor * cur = build_vit(
inp, n_patches,
NORM_TYPE_NORMAL,
hparams.ffn_op,
nullptr,
add_pos);
cb(cur, "vit_out", -1);
{
// patch_merger
const int scale_factor = model.hparams.n_merge;
cur = build_patch_merge_permute(cur, scale_factor);
// projection norm
int proj_inp_dim = cur->ne[0];
int n_merged_patches = cur->ne[1];
cur = ggml_view_2d(ctx0, cur,
n_embd, n_merged_patches * scale_factor * scale_factor,
ggml_row_size(cur->type, n_embd), 0);
cur = ggml_norm(ctx0, cur, hparams.eps);
cur = ggml_mul(ctx0, cur, model.mm_input_norm_w);
cur = ggml_add(ctx0, cur, model.mm_input_norm_b);
cur = ggml_view_2d(ctx0, cur,
proj_inp_dim, n_merged_patches,
ggml_row_size(cur->type, proj_inp_dim), 0);
cb(cur, "proj_inp_normed", -1);
// projection mlp
cur = build_ffn(cur,
model.mm_1_w, model.mm_1_b,
nullptr, nullptr,
model.mm_2_w, model.mm_2_b,
FFN_GELU,
-1);
cb(cur, "proj_out", -1);
}
// build the graph
ggml_build_forward_expand(gf, cur);
return gf;
}

7
tools/mtmd/models/models.h
View file

@ -109,3 +109,10 @@ struct clip_graph_mobilenetv5 : clip_graph {
ggml_tensor * inp,
const mobilenetv5_block & block);
};
struct clip_graph_kimik25 : clip_graph {
clip_graph_kimik25(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
ggml_tensor * resize_position_embeddings_3d(uint32_t interpolation_mode);
};

4
tools/mtmd/models/qwen3vl.cpp
View file

@ -182,7 +182,9 @@ ggml_cgraph * clip_graph_qwen3vl::build() {
model.mm_1_w, model.mm_1_b,
ffn_op_type::FFN_GELU, -1);
embeddings = ggml_concat(ctx0, embeddings, deepstack_features, 0); // concat along the feature dimension
if (deepstack_features) {
embeddings = ggml_concat(ctx0, embeddings, deepstack_features, 0);
} // concat along the feature dimension
// build the graph
ggml_build_forward_expand(gf, embeddings);

21
tools/mtmd/mtmd.cpp
View file

@ -85,6 +85,7 @@ enum mtmd_slice_tmpl {
MTMD_SLICE_TMPL_MINICPMV_2_6,
MTMD_SLICE_TMPL_LLAMA4,
MTMD_SLICE_TMPL_IDEFICS3,
MTMD_SLICE_TMPL_LFM2,
};
const char * mtmd_default_marker() {
@ -236,7 +237,7 @@ struct mtmd_context {
tok_row_end_trail = false; // no trailing end-of-row token
ov_img_first = true;
} else if (minicpmv_version == 3 || minicpmv_version == 4 || minicpmv_version == 5 || minicpmv_version == 6) {
} else if (minicpmv_version == 3 || minicpmv_version == 4 || minicpmv_version == 5 || minicpmv_version == 6 || minicpmv_version == 100045) {
// minicpmv 2.6 format:
// <image> (overview) </image><slice> (slice) </slice><slice> (slice) </slice>\n ...
slice_tmpl = MTMD_SLICE_TMPL_MINICPMV_2_6;
@ -307,9 +308,19 @@ struct mtmd_context {
img_end = "<|im_end|>";
} else if (proj == PROJECTOR_TYPE_LFM2) {
img_beg = "<|image_start|>";
img_end = "<|image_end|>";
// multi-tile:
// <|image_start|>
// <|img_row_1_col_1|> (tile) <|img_row_1_col_2|> (tile) ...
// <|img_thumbnail|> (thumbnail)
// <|image_end|>
// single-tile:
// <|image_start|> (image) <|image_end|>
img_beg = "<|image_start|>";
img_end = "<|image_end|>";
slice_tmpl = MTMD_SLICE_TMPL_LFM2;
sli_img_start_tmpl = "<|img_row_%d_col_%d|>";
tok_ov_img_start = {lookup_token("<|img_thumbnail|>")};
ov_img_first = false;
} else if (proj == PROJECTOR_TYPE_GLM4V) {
img_beg = "<|begin_of_image|>";
img_end = "<|end_of_image|>";
@ -562,11 +573,13 @@ struct mtmd_tokenizer {
}
// handle llava-uhd style preprocessing
const bool has_tiling_grid = batch_f32.grid_x > 0 && batch_f32.grid_y > 0;
if (
ctx->slice_tmpl == MTMD_SLICE_TMPL_MINICPMV_2_5
|| ctx->slice_tmpl == MTMD_SLICE_TMPL_MINICPMV_2_6
|| ctx->slice_tmpl == MTMD_SLICE_TMPL_LLAMA4
|| ctx->slice_tmpl == MTMD_SLICE_TMPL_IDEFICS3
|| (ctx->slice_tmpl == MTMD_SLICE_TMPL_LFM2 && has_tiling_grid)
) {
const int n_col = batch_f32.grid_x;
const int n_row = batch_f32.grid_y;

2
tools/perplexity/README.md
View file

@ -29,7 +29,7 @@ In addition to the KL divergence the following statistics are calculated with `-
* Mean change in "correct" token probability. Positive values mean the model gets better at prediction, negative values mean it gets worse.
* Pearson correlation coefficient of the "correct" token probabilites between models.
* Percentiles of change in "correct" token probability. Positive values mean the model gets better at prediction, negative values mean it gets worse. Can be used to judge noise vs. quality loss from quantization. If the percentiles are symmetric then the quantization is essentially just adding noise. If the negative values are significantly larger than the positive values then this indicates that the model is actually becoming worse from the quantization.
* The root mean square of the change in token probabilities. If you were to assume that the quantization simply causes Gaussian noise on the token probabilities then this would be the standard deviation of said noise. The uncertainty on the value is calculated that the change in token probabilities follows a Gaussian distribution. Related discussion: https://github.com/ggerganov/llama.cpp/discussions/2875 .
* The root mean square of the change in token probabilities. If you were to assume that the quantization simply causes Gaussian noise on the token probabilities then this would be the standard deviation of said noise. The uncertainty on the value is calculated that the change in token probabilities follows a Gaussian distribution. Related discussion: https://github.com/ggml-org/llama.cpp/discussions/2875 .
* Same top p: Percentage of how often the token was assigned the highest probabilites by both models. The uncertainty is calculated from the Gaussian approximation of the binomial distribution.
## LLaMA 3 8b Scoreboard

79
tools/quantize/quantize.cpp
View file

@ -119,25 +119,48 @@ static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftyp
[[noreturn]]
static void usage(const char * executable) {
printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] [--imatrix] [--include-weights]\n", executable);
printf(" [--exclude-weights] [--output-tensor-type] [--token-embedding-type] [--tensor-type] [--prune-layers] [--keep-split] [--override-kv]\n");
printf(" [--exclude-weights] [--output-tensor-type] [--token-embedding-type] [--tensor-type] [--tensor-type-file]\n");
printf(" [--prune-layers] [--keep-split] [--override-kv]\n");
printf(" model-f32.gguf [model-quant.gguf] type [nthreads]\n\n");
printf(" --allow-requantize: Allows requantizing tensors that have already been quantized. Warning: This can severely reduce quality compared to quantizing from 16bit or 32bit\n");
printf(" --leave-output-tensor: Will leave output.weight un(re)quantized. Increases model size but may also increase quality, especially when requantizing\n");
printf(" --pure: Disable k-quant mixtures and quantize all tensors to the same type\n");
printf(" --imatrix file_name: use data in file_name as importance matrix for quant optimizations\n");
printf(" --include-weights tensor_name: use importance matrix for this/these tensor(s)\n");
printf(" --exclude-weights tensor_name: use importance matrix for this/these tensor(s)\n");
printf(" --output-tensor-type ggml_type: use this ggml_type for the output.weight tensor\n");
printf(" --token-embedding-type ggml_type: use this ggml_type for the token embeddings tensor\n");
printf(" --tensor-type TENSOR=TYPE: quantize this tensor to this ggml_type. example: --tensor-type attn_q=q8_0\n");
printf(" Advanced option to selectively quantize tensors. May be specified multiple times.\n");
printf(" --prune-layers L0,L1,L2...comma-separated list of layer numbers to prune from the model\n");
printf(" Advanced option to remove all tensors from the given layers\n");
printf(" --keep-split: will generate quantized model in the same shards as input\n");
printf(" --allow-requantize\n");
printf(" allow requantizing tensors that have already been quantized\n");
printf(" WARNING: this can severely reduce quality compared to quantizing\n");
printf(" from 16bit or 32bit!\n");
printf(" --leave-output-tensor\n");
printf(" leave output.weight un(re)quantized\n");
printf(" increases model size but may also increase quality, especially when requantizing\n");
printf(" --pure\n");
printf(" disable k-quant mixtures and quantize all tensors to the same type\n");
printf(" --imatrix file_name\n");
printf(" use data in file_name as importance matrix for quant optimizations\n");
printf(" --include-weights tensor_name\n");
printf(" use importance matrix for this/these tensor(s)\n");
printf(" --exclude-weights tensor_name\n");
printf(" do not use importance matrix for this/these tensor(s)\n");
printf(" --output-tensor-type ggml_type\n");
printf(" use this ggml_type for the output.weight tensor\n");
printf(" --token-embedding-type ggml_type\n");
printf(" use this ggml_type for the token embeddings tensor\n");
printf(" --tensor-type tensor_name=ggml_type\n");
printf(" quantize this tensor to this ggml_type\n");
printf(" this is an advanced option to selectively quantize tensors. may be specified multiple times.\n");
printf(" example: --tensor-type attn_q=q8_0\n");
printf(" --tensor-type-file tensor_types.txt\n");
printf(" list of tensors to quantize to a specific ggml_type\n");
printf(" this is an advanced option to selectively quantize a long list of tensors.\n");
printf(" the file should use the same format as above, separated by spaces or newlines.\n");
printf(" --prune-layers L0,L1,L2...\n");
printf(" comma-separated list of layer numbers to prune from the model\n");
printf(" WARNING: this is an advanced option, use with care.\n");
printf(" --keep-split\n");
printf(" generate quantized model in the same shards as input\n");
printf(" --override-kv KEY=TYPE:VALUE\n");
printf(" Advanced option to override model metadata by key in the quantized model. May be specified multiple times.\n");
printf("Note: --include-weights and --exclude-weights cannot be used together\n");
printf("\nAllowed quantization types:\n");
printf(" override model metadata by key in the quantized model. may be specified multiple times.\n");
printf(" WARNING: this is an advanced option, use with care.\n\n");
printf("note: --include-weights and --exclude-weights cannot be used together\n\n");
printf("-----------------------------------------------------------------------------\n");
printf(" allowed quantization types\n");
printf("-----------------------------------------------------------------------------\n\n");
for (const auto & it : QUANT_OPTIONS) {
if (it.name != "COPY") {
printf(" %2d or ", it.ftype);
@ -415,6 +438,23 @@ static bool parse_tensor_type(const char * data, std::vector<tensor_quantization
return true;
}
static bool parse_tensor_type_file(const char * filename, std::vector<tensor_quantization> & tensor_type) {
std::ifstream file(filename);
if (!file) {
printf("\n%s: failed to open file '%s': %s\n\n", __func__, filename, std::strerror(errno));
return false;
}
std::string arg;
while (file >> arg) {
if (!parse_tensor_type(arg.c_str(), tensor_type)) {
return false;
}
}
return true;
}
static bool parse_layer_prune(const char * data, std::vector<int> & prune_layers) {
if (!data) {
printf("\n%s: no layer pruning ids provided\n\n", __func__);
@ -480,6 +520,10 @@ int main(int argc, char ** argv) {
if (arg_idx == argc-1 || !parse_tensor_type(argv[++arg_idx], tensor_types)) {
usage(argv[0]);
}
} else if (strcmp(argv[arg_idx], "--tensor-type-file") == 0) {
if (arg_idx == argc-1 || !parse_tensor_type_file(argv[++arg_idx], tensor_types)) {
usage(argv[0]);
}
} else if (strcmp(argv[arg_idx], "--prune-layers") == 0) {
if (arg_idx == argc-1 || !parse_layer_prune(argv[++arg_idx], prune_layers)) {
usage(argv[0]);
@ -686,3 +730,4 @@ int main(int argc, char ** argv) {
return 0;
}

64
tools/rpc/rpc-server.cpp
View file

@ -1,12 +1,7 @@
#if defined(_MSC_VER)
#define _SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING
#endif
#include "ggml-rpc.h"
#ifdef _WIN32
# define NOMINMAX
# define DIRECTORY_SEPARATOR '\\'
# include <locale>
# include <windows.h>
# include <fcntl.h>
# include <io.h>
@ -15,23 +10,43 @@
# include <unistd.h>
# include <sys/stat.h>
#endif
#include <codecvt>
#include <string>
#include <stdio.h>
#include <vector>
#include <filesystem>
#include <algorithm>
#include <thread>
#include <regex>
namespace fs = std::filesystem;
#if defined(__linux__)
#include <sys/types.h>
#include <pwd.h>
#endif
// NOTE: this is copied from common.cpp to avoid linking with libcommon
#ifdef _WIN32
static std::wstring utf8_to_wstring(const std::string & str) {
if (str.empty()) {
return std::wstring();
}
int size = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), (int)str.size(), NULL, 0);
if (size <= 0) {
return std::wstring();
}
std::wstring wstr(size, 0);
MultiByteToWideChar(CP_UTF8, 0, str.c_str(), (int)str.size(), &wstr[0], size);
return wstr;
}
#endif
// NOTE: this is copied from common.cpp to avoid linking with libcommon
// returns true if successful, false otherwise
static bool fs_create_directory_with_parents(const std::string & path) {
#ifdef _WIN32
std::wstring_convert<std::codecvt_utf8<wchar_t>> converter;
std::wstring wpath = converter.from_bytes(path);
std::wstring wpath = utf8_to_wstring(path);
// if the path already exists, check whether it's a directory
const DWORD attributes = GetFileAttributesW(wpath.c_str());
@ -44,9 +59,16 @@ static bool fs_create_directory_with_parents(const std::string & path) {
// process path from front to back, procedurally creating directories
while ((pos_slash = path.find('\\', pos_slash)) != std::string::npos) {
const std::wstring subpath = wpath.substr(0, pos_slash);
const wchar_t * test = subpath.c_str();
const bool success = CreateDirectoryW(test, NULL);
pos_slash += 1;
// skip the drive letter, in some systems it can return an access denied error
if (subpath.length() == 2 && subpath[1] == ':') {
continue;
}
const bool success = CreateDirectoryW(subpath.c_str(), NULL);
if (!success) {
const DWORD error = GetLastError();
@ -60,8 +82,6 @@ static bool fs_create_directory_with_parents(const std::string & path) {
return false;
}
}
pos_slash += 1;
}
return true;
@ -115,13 +135,27 @@ static std::string fs_get_cache_directory() {
#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || defined(__OpenBSD__)
if (std::getenv("XDG_CACHE_HOME")) {
cache_directory = std::getenv("XDG_CACHE_HOME");
} else {
} else if (std::getenv("HOME")) {
cache_directory = std::getenv("HOME") + std::string("/.cache/");
} else {
#if defined(__linux__)
/* no $HOME is defined, fallback to getpwuid */
struct passwd *pw = getpwuid(getuid());
if ((!pw) || (!pw->pw_dir)) {
throw std::runtime_error("Failed to find $HOME directory");
}
cache_directory = std::string(pw->pw_dir) + std::string("/.cache/");
#else /* defined(__linux__) */
throw std::runtime_error("Failed to find $HOME directory");
#endif /* defined(__linux__) */
}
#elif defined(__APPLE__)
cache_directory = std::getenv("HOME") + std::string("/Library/Caches/");
#elif defined(_WIN32)
cache_directory = std::getenv("LOCALAPPDATA");
#elif defined(__EMSCRIPTEN__)
GGML_ABORT("not implemented on this platform");
#else
# error Unknown architecture
#endif

53
tools/server/README.md
View file

@ -19,7 +19,7 @@ Set of LLM REST APIs and a web UI to interact with llama.cpp.
* Speculative decoding
* Easy-to-use web UI
For the ful list of features, please refer to [server's changelog](https://github.com/ggml-org/llama.cpp/issues/9291)
For the full list of features, please refer to [server's changelog](https://github.com/ggml-org/llama.cpp/issues/9291)
## Usage
@ -63,10 +63,10 @@ For the ful list of features, please refer to [server's changelog](https://githu
| `--rope-freq-base N` | RoPE base frequency, used by NTK-aware scaling (default: loaded from model)<br/>(env: LLAMA_ARG_ROPE_FREQ_BASE) |
| `--rope-freq-scale N` | RoPE frequency scaling factor, expands context by a factor of 1/N<br/>(env: LLAMA_ARG_ROPE_FREQ_SCALE) |
| `--yarn-orig-ctx N` | YaRN: original context size of model (default: 0 = model training context size)<br/>(env: LLAMA_ARG_YARN_ORIG_CTX) |
| `--yarn-ext-factor N` | YaRN: extrapolation mix factor (default: -1.0, 0.0 = full interpolation)<br/>(env: LLAMA_ARG_YARN_EXT_FACTOR) |
| `--yarn-attn-factor N` | YaRN: scale sqrt(t) or attention magnitude (default: -1.0)<br/>(env: LLAMA_ARG_YARN_ATTN_FACTOR) |
| `--yarn-beta-slow N` | YaRN: high correction dim or alpha (default: -1.0)<br/>(env: LLAMA_ARG_YARN_BETA_SLOW) |
| `--yarn-beta-fast N` | YaRN: low correction dim or beta (default: -1.0)<br/>(env: LLAMA_ARG_YARN_BETA_FAST) |
| `--yarn-ext-factor N` | YaRN: extrapolation mix factor (default: -1.00, 0.0 = full interpolation)<br/>(env: LLAMA_ARG_YARN_EXT_FACTOR) |
| `--yarn-attn-factor N` | YaRN: scale sqrt(t) or attention magnitude (default: -1.00)<br/>(env: LLAMA_ARG_YARN_ATTN_FACTOR) |
| `--yarn-beta-slow N` | YaRN: high correction dim or alpha (default: -1.00)<br/>(env: LLAMA_ARG_YARN_BETA_SLOW) |
| `--yarn-beta-fast N` | YaRN: low correction dim or beta (default: -1.00)<br/>(env: LLAMA_ARG_YARN_BETA_FAST) |
| `-kvo, --kv-offload, -nkvo, --no-kv-offload` | whether to enable KV cache offloading (default: enabled)<br/>(env: LLAMA_ARG_KV_OFFLOAD) |
| `--repack, -nr, --no-repack` | whether to enable weight repacking (default: enabled)<br/>(env: LLAMA_ARG_REPACK) |
| `--no-host` | bypass host buffer allowing extra buffers to be used<br/>(env: LLAMA_ARG_NO_HOST) |
@ -126,30 +126,30 @@ For the ful list of features, please refer to [server's changelog](https://githu
| `-s, --seed SEED` | RNG seed (default: -1, use random seed for -1) |
| `--sampler-seq, --sampling-seq SEQUENCE` | simplified sequence for samplers that will be used (default: edskypmxt) |
| `--ignore-eos` | ignore end of stream token and continue generating (implies --logit-bias EOS-inf) |
| `--temp N` | temperature (default: 0.8) |
| `--temp N` | temperature (default: 0.80) |
| `--top-k N` | top-k sampling (default: 40, 0 = disabled)<br/>(env: LLAMA_ARG_TOP_K) |
| `--top-p N` | top-p sampling (default: 0.9, 1.0 = disabled) |
| `--min-p N` | min-p sampling (default: 0.1, 0.0 = disabled) |
| `--adaptive-target N` | adaptive-p: select tokens near this probability (valid range 0.0 to 1.0; negative = disabled) |
| `--adaptive-decay N` | adaptive-p: EMA decay for adaptation; effective history length ≈ 1/(1-decay) tokens (valid range 0.0 - 0.99) |
| `--top-nsigma N` | top-n-sigma sampling (default: -1.0, -1.0 = disabled) |
| `--xtc-probability N` | xtc probability (default: 0.0, 0.0 = disabled) |
| `--xtc-threshold N` | xtc threshold (default: 0.1, 1.0 = disabled) |
| `--typical N` | locally typical sampling, parameter p (default: 1.0, 1.0 = disabled) |
| `--top-p N` | top-p sampling (default: 0.95, 1.0 = disabled) |
| `--min-p N` | min-p sampling (default: 0.05, 0.0 = disabled) |
| `--top-nsigma N` | top-n-sigma sampling (default: -1.00, -1.0 = disabled) |
| `--xtc-probability N` | xtc probability (default: 0.00, 0.0 = disabled) |
| `--xtc-threshold N` | xtc threshold (default: 0.10, 1.0 = disabled) |
| `--typical N` | locally typical sampling, parameter p (default: 1.00, 1.0 = disabled) |
| `--repeat-last-n N` | last n tokens to consider for penalize (default: 64, 0 = disabled, -1 = ctx_size) |
| `--repeat-penalty N` | penalize repeat sequence of tokens (default: 1.0, 1.0 = disabled) |
| `--presence-penalty N` | repeat alpha presence penalty (default: 0.0, 0.0 = disabled) |
| `--frequency-penalty N` | repeat alpha frequency penalty (default: 0.0, 0.0 = disabled) |
| `--dry-multiplier N` | set DRY sampling multiplier (default: 0.0, 0.0 = disabled) |
| `--repeat-penalty N` | penalize repeat sequence of tokens (default: 1.00, 1.0 = disabled) |
| `--presence-penalty N` | repeat alpha presence penalty (default: 0.00, 0.0 = disabled) |
| `--frequency-penalty N` | repeat alpha frequency penalty (default: 0.00, 0.0 = disabled) |
| `--dry-multiplier N` | set DRY sampling multiplier (default: 0.00, 0.0 = disabled) |
| `--dry-base N` | set DRY sampling base value (default: 1.75) |
| `--dry-allowed-length N` | set allowed length for DRY sampling (default: 2) |
| `--dry-penalty-last-n N` | set DRY penalty for the last n tokens (default: -1, 0 = disable, -1 = context size) |
| `--dry-sequence-breaker STRING` | add sequence breaker for DRY sampling, clearing out default breakers ('\n', ':', '"', '*') in the process; use "none" to not use any sequence breakers |
| `--dynatemp-range N` | dynamic temperature range (default: 0.0, 0.0 = disabled) |
| `--dynatemp-exp N` | dynamic temperature exponent (default: 1.0) |
| `--adaptive-target N` | adaptive-p: select tokens near this probability (valid range 0.0 to 1.0; negative = disabled) (default: -1.00)<br/>[(more info)](https://github.com/ggml-org/llama.cpp/pull/17927) |
| `--adaptive-decay N` | adaptive-p: decay rate for target adaptation over time. lower values are more reactive, higher values are more stable.<br/>(valid range 0.0 to 0.99) (default: 0.90) |
| `--dynatemp-range N` | dynamic temperature range (default: 0.00, 0.0 = disabled) |
| `--dynatemp-exp N` | dynamic temperature exponent (default: 1.00) |
| `--mirostat N` | use Mirostat sampling.<br/>Top K, Nucleus and Locally Typical samplers are ignored if used.<br/>(default: 0, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0) |
| `--mirostat-lr N` | Mirostat learning rate, parameter eta (default: 0.1) |
| `--mirostat-ent N` | Mirostat target entropy, parameter tau (default: 5.0) |
| `--mirostat-lr N` | Mirostat learning rate, parameter eta (default: 0.10) |
| `--mirostat-ent N` | Mirostat target entropy, parameter tau (default: 5.00) |
| `-l, --logit-bias TOKEN_ID(+/-)BIAS` | modifies the likelihood of token appearing in the completion,<br/>i.e. `--logit-bias 15043+1` to increase likelihood of token ' Hello',<br/>or `--logit-bias 15043-1` to decrease likelihood of token ' Hello' |
| `--grammar GRAMMAR` | BNF-like grammar to constrain generations (see samples in grammars/ dir) (default: '') |
| `--grammar-file FNAME` | file to read grammar from |
@ -199,7 +199,8 @@ For the ful list of features, please refer to [server's changelog](https://githu
| `--chat-template-kwargs STRING` | sets additional params for the json template parser, must be a valid json object string, e.g. '{"key1":"value1","key2":"value2"}'<br/>(env: LLAMA_CHAT_TEMPLATE_KWARGS) |
| `-to, --timeout N` | server read/write timeout in seconds (default: 600)<br/>(env: LLAMA_ARG_TIMEOUT) |
| `--threads-http N` | number of threads used to process HTTP requests (default: -1)<br/>(env: LLAMA_ARG_THREADS_HTTP) |
| `--cache-reuse N` | min chunk size to attempt reusing from the cache via KV shifting (default: 0)<br/>[(card)](https://ggml.ai/f0.png)<br/>(env: LLAMA_ARG_CACHE_REUSE) |
| `--cache-prompt, --no-cache-prompt` | whether to enable prompt caching (default: enabled)<br/>(env: LLAMA_ARG_CACHE_PROMPT) |
| `--cache-reuse N` | min chunk size to attempt reusing from the cache via KV shifting, requires prompt caching to be enabled (default: 0)<br/>[(card)](https://ggml.ai/f0.png)<br/>(env: LLAMA_ARG_CACHE_REUSE) |
| `--metrics` | enable prometheus compatible metrics endpoint (default: disabled)<br/>(env: LLAMA_ARG_ENDPOINT_METRICS) |
| `--props` | enable changing global properties via POST /props (default: disabled)<br/>(env: LLAMA_ARG_ENDPOINT_PROPS) |
| `--slots, --no-slots` | expose slots monitoring endpoint (default: enabled)<br/>(env: LLAMA_ARG_ENDPOINT_SLOTS) |
@ -212,8 +213,8 @@ For the ful list of features, please refer to [server's changelog](https://githu
| `--jinja, --no-jinja` | whether to use jinja template engine for chat (default: enabled)<br/>(env: LLAMA_ARG_JINJA) |
| `--reasoning-format FORMAT` | controls whether thought tags are allowed and/or extracted from the response, and in which format they're returned; one of:<br/>- none: leaves thoughts unparsed in `message.content`<br/>- deepseek: puts thoughts in `message.reasoning_content`<br/>- deepseek-legacy: keeps `<think>` tags in `message.content` while also populating `message.reasoning_content`<br/>(default: auto)<br/>(env: LLAMA_ARG_THINK) |
| `--reasoning-budget N` | controls the amount of thinking allowed; currently only one of: -1 for unrestricted thinking budget, or 0 to disable thinking (default: -1)<br/>(env: LLAMA_ARG_THINK_BUDGET) |
| `--chat-template JINJA_TEMPLATE` | set custom jinja chat template (default: template taken from model's metadata)<br/>if suffix/prefix are specified, template will be disabled<br/>only commonly used templates are accepted (unless --jinja is set before this flag):<br/>list of built-in templates:<br/>bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek2, deepseek3, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, grok-2, hunyuan-dense, hunyuan-moe, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr<br/>(env: LLAMA_ARG_CHAT_TEMPLATE) |
| `--chat-template-file JINJA_TEMPLATE_FILE` | set custom jinja chat template file (default: template taken from model's metadata)<br/>if suffix/prefix are specified, template will be disabled<br/>only commonly used templates are accepted (unless --jinja is set before this flag):<br/>list of built-in templates:<br/>bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek2, deepseek3, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, grok-2, hunyuan-dense, hunyuan-moe, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr<br/>(env: LLAMA_ARG_CHAT_TEMPLATE_FILE) |
| `--chat-template JINJA_TEMPLATE` | set custom jinja chat template (default: template taken from model's metadata)<br/>if suffix/prefix are specified, template will be disabled<br/>only commonly used templates are accepted (unless --jinja is set before this flag):<br/>list of built-in templates:<br/>bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, grok-2, hunyuan-dense, hunyuan-moe, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr<br/>(env: LLAMA_ARG_CHAT_TEMPLATE) |
| `--chat-template-file JINJA_TEMPLATE_FILE` | set custom jinja chat template file (default: template taken from model's metadata)<br/>if suffix/prefix are specified, template will be disabled<br/>only commonly used templates are accepted (unless --jinja is set before this flag):<br/>list of built-in templates:<br/>bailing, bailing-think, bailing2, chatglm3, chatglm4, chatml, command-r, deepseek, deepseek2, deepseek3, exaone-moe, exaone3, exaone4, falcon3, gemma, gigachat, glmedge, gpt-oss, granite, grok-2, hunyuan-dense, hunyuan-moe, kimi-k2, llama2, llama2-sys, llama2-sys-bos, llama2-sys-strip, llama3, llama4, megrez, minicpm, mistral-v1, mistral-v3, mistral-v3-tekken, mistral-v7, mistral-v7-tekken, monarch, openchat, orion, pangu-embedded, phi3, phi4, rwkv-world, seed_oss, smolvlm, solar-open, vicuna, vicuna-orca, yandex, zephyr<br/>(env: LLAMA_ARG_CHAT_TEMPLATE_FILE) |
| `--prefill-assistant, --no-prefill-assistant` | whether to prefill the assistant's response if the last message is an assistant message (default: prefill enabled)<br/>when this flag is set, if the last message is an assistant message then it will be treated as a full message and not prefilled<br/><br/>(env: LLAMA_ARG_PREFILL_ASSISTANT) |
| `-sps, --slot-prompt-similarity SIMILARITY` | how much the prompt of a request must match the prompt of a slot in order to use that slot (default: 0.10, 0.0 = disabled) |
| `--lora-init-without-apply` | load LoRA adapters without applying them (apply later via POST /lora-adapters) (default: disabled) |
@ -222,7 +223,7 @@ For the ful list of features, please refer to [server's changelog](https://githu
| `-tbd, --threads-batch-draft N` | number of threads to use during batch and prompt processing (default: same as --threads-draft) |
| `--draft, --draft-n, --draft-max N` | number of tokens to draft for speculative decoding (default: 16)<br/>(env: LLAMA_ARG_DRAFT_MAX) |
| `--draft-min, --draft-n-min N` | minimum number of draft tokens to use for speculative decoding (default: 0)<br/>(env: LLAMA_ARG_DRAFT_MIN) |
| `--draft-p-min P` | minimum speculative decoding probability (greedy) (default: 0.8)<br/>(env: LLAMA_ARG_DRAFT_P_MIN) |
| `--draft-p-min P` | minimum speculative decoding probability (greedy) (default: 0.75)<br/>(env: LLAMA_ARG_DRAFT_P_MIN) |
| `-cd, --ctx-size-draft N` | size of the prompt context for the draft model (default: 0, 0 = loaded from model)<br/>(env: LLAMA_ARG_CTX_SIZE_DRAFT) |
| `-devd, --device-draft <dev1,dev2,..>` | comma-separated list of devices to use for offloading the draft model (none = don't offload)<br/>use --list-devices to see a list of available devices |
| `-ngld, --gpu-layers-draft, --n-gpu-layers-draft N` | max. number of draft model layers to store in VRAM, either an exact number, 'auto', or 'all' (default: auto)<br/>(env: LLAMA_ARG_N_GPU_LAYERS_DRAFT) |

2
tools/server/public_legacy/index-new.html
View file

@ -1096,7 +1096,7 @@ return html`
</section>
<footer>
<p><${ModelGenerationInfo} /></p>
<p>Powered By <a href="https://github.com/ggerganov/llama.cpp#readme" target="_blank">llama.cpp</a> and <a href="https://ggml.ai/" target="_blank">ggml.ai</a></p>
<p>Powered By <a href="https://github.com/ggml-org/llama.cpp#readme" target="_blank">llama.cpp</a> and <a href="https://ggml.ai/" target="_blank">ggml.ai</a></p>
</footer>
</div>
`;

2
tools/server/public_legacy/index.html
View file

@ -1281,7 +1281,7 @@
<footer>
<p><${ModelGenerationInfo} /></p>
<p>Powered by <a href="https://github.com/ggerganov/llama.cpp">llama.cpp</a> and <a href="https://ggml.ai">ggml.ai</a>.</p>
<p>Powered by <a href="https://github.com/ggml-org/llama.cpp">llama.cpp</a> and <a href="https://ggml.ai">ggml.ai</a>.</p>
</footer>
</div>
`;

2
tools/server/public_legacy/theme-mangotango.css
View file

@ -1,5 +1,5 @@
/* Author: Yazan Agha-Schrader */
/* Inspiration from llama.cpp logo/banner https://github.com/ggerganov/llama.cpp#readme */
/* Inspiration from llama.cpp logo/banner https://github.com/ggml-org/llama.cpp#readme */
.theme-mangotango {

202
tools/server/server-context.cpp
View file

@ -48,11 +48,8 @@ enum server_state {
struct server_slot {
int id;
llama_batch batch_spec = {};
// TODO: change to unique_ptrs for consistency:
llama_context * ctx = nullptr;
llama_context * ctx_dft = nullptr;
// multimodal
mtmd_context * mctx = nullptr;
@ -158,7 +155,7 @@ struct server_slot {
double t_prompt_processing; // ms
double t_token_generation; // ms
std::function<void(int /* slot_id */)> callback_on_release;
std::function<void(int /* id_slot */)> callback_on_release;
// Speculative decoding stats
int32_t n_draft_total = 0; // Total draft tokens generated
@ -259,7 +256,7 @@ struct server_slot {
}
bool can_speculate() const {
return ctx_dft;
return !!spec;
}
void add_token(const completion_token_output & token) {
@ -295,6 +292,7 @@ struct server_slot {
SLT_DBG(*this, "the max possible draft is too small: %d < %d - skipping speculative decoding\n", n_draft_max, task->params.speculative.n_min);
n_draft_max = 0;
}
return n_draft_max;
}
@ -397,6 +395,8 @@ struct server_slot {
draft_ratio, n_draft_accepted, n_draft_total
);
}
common_speculative_print_stats(spec);
}
json to_json(bool only_metrics = false) const {
@ -553,18 +553,13 @@ private:
// note: keep these alive - they determine the lifetime of the model, context, etc.
common_init_result_ptr llama_init;
common_init_result_ptr llama_init_dft;
llama_context * ctx = nullptr;
bool vocab_dft_compatible = true;
llama_model * model_dft = nullptr;
llama_context_params cparams_dft;
llama_batch batch {};
llama_model_ptr model_dft;
bool add_bos_token = true;
int32_t n_ctx; // total context for all clients / slots
@ -597,13 +592,8 @@ private:
// Clear any sampling context
for (server_slot & slot : slots) {
llama_free(slot.ctx_dft);
slot.ctx_dft = nullptr;
common_speculative_free(slot.spec);
slot.spec = nullptr;
llama_batch_free(slot.batch_spec);
}
llama_batch_free(batch);
@ -648,44 +638,39 @@ private:
add_bos_token = llama_vocab_get_add_bos(vocab);
if (params_base.has_speculative()) {
SRV_INF("loading draft model '%s'\n", params_base.speculative.model.path.c_str());
if (params_base.speculative.has_dft()) {
SRV_INF("loading draft model '%s'\n", params_base.speculative.mparams_dft.path.c_str());
const auto & params_spec = params_base.speculative;
auto params_dft = params_base;
params_dft.devices = params_base.speculative.devices;
params_dft.model = params_base.speculative.model;
params_dft.n_ctx = params_base.speculative.n_ctx == 0 ? llama_n_ctx_seq(ctx) : params_base.speculative.n_ctx;
params_dft.n_gpu_layers = params_base.speculative.n_gpu_layers;
params_dft.n_parallel = 1;
params_dft.cache_type_k = params_base.speculative.cache_type_k;
params_dft.cache_type_v = params_base.speculative.cache_type_v;
params_dft.n_ctx = params_spec.n_ctx == 0 ? llama_n_ctx_seq(ctx) : params_spec.n_ctx;
params_dft.n_batch = llama_n_ctx_seq(ctx);
params_dft.devices = params_spec.devices;
params_dft.model = params_spec.mparams_dft;
params_dft.n_gpu_layers = params_spec.n_gpu_layers;
params_dft.cache_type_k = params_spec.cache_type_k;
params_dft.cache_type_v = params_spec.cache_type_v;
params_dft.cpuparams.n_threads = params_base.speculative.cpuparams.n_threads;
params_dft.cpuparams_batch.n_threads = params_base.speculative.cpuparams_batch.n_threads;
params_dft.tensor_buft_overrides = params_base.speculative.tensor_buft_overrides;
if (params_spec.cpuparams.n_threads > 0) {
params_dft.cpuparams.n_threads = params_spec.cpuparams.n_threads;
params_dft.cpuparams_batch.n_threads = params_spec.cpuparams_batch.n_threads;
}
llama_init_dft = common_init_from_params(params_dft);
params_dft.tensor_buft_overrides = params_spec.tensor_buft_overrides;
model_dft = llama_init_dft->model();
auto mparams_dft = common_model_params_to_llama(params_dft);
model_dft.reset(llama_model_load_from_file(params_dft.model.path.c_str(), mparams_dft));
if (model_dft == nullptr) {
SRV_ERR("failed to load draft model, '%s'\n", params_base.speculative.model.path.c_str());
SRV_ERR("failed to load draft model, '%s'\n", params_dft.model.path.c_str());
return false;
}
vocab_dft_compatible = common_speculative_are_compatible(ctx, llama_init_dft->context());
if (!vocab_dft_compatible) {
SRV_INF("the draft model '%s' is not compatible with the target model '%s'. tokens will be translated between the draft and target models.\n", params_base.speculative.model.path.c_str(), params_base.model.path.c_str());
}
const int n_ctx_dft = llama_n_ctx(llama_init_dft->context());
cparams_dft = common_context_params_to_llama(params_dft);
cparams_dft.n_batch = n_ctx_dft;
// the context is not needed - we will create one for each slot
llama_init_dft->free_context();
params_base.speculative.model_dft = model_dft.get();
params_base.speculative.cparams_dft = common_context_params_to_llama(params_dft);
}
std::string & mmproj_path = params_base.mmproj.path;
@ -695,6 +680,7 @@ private:
}
mtmd_context_params mparams = mtmd_context_params_default();
mparams.use_gpu = params_base.mmproj_use_gpu;
mparams.print_timings = false;
mparams.n_threads = params_base.cpuparams.n_threads;
@ -702,6 +688,7 @@ private:
mparams.warmup = params_base.warmup;
mparams.image_min_tokens = params_base.image_min_tokens;
mparams.image_max_tokens = params_base.image_max_tokens;
mctx = mtmd_init_from_file(mmproj_path.c_str(), model, mparams);
if (mctx == nullptr) {
SRV_ERR("failed to load multimodal model, '%s'\n", mmproj_path.c_str());
@ -719,9 +706,9 @@ private:
SRV_WRN("%s\n", "cache_reuse is not supported by multimodal, it will be disabled");
}
if (params_base.has_speculative()) {
SRV_ERR("%s\n", "err: speculative decode is not supported by multimodal");
return false;
if (params_base.speculative.type != COMMON_SPECULATIVE_TYPE_NONE) {
params_base.speculative.type = COMMON_SPECULATIVE_TYPE_NONE;
SRV_WRN("%s\n", "speculative decoding is not supported by multimodal, it will be disabled");
}
}
@ -753,40 +740,40 @@ private:
slots.clear();
const bool can_spec = common_speculative_is_compat(ctx);
if (!can_spec) {
SRV_WRN("%s", "speculative decoding not supported by this context\n");
}
// initialize slots
for (int i = 0; i < params_base.n_parallel; i++) {
server_slot slot;
slot.id = i;
slot.ctx = ctx;
slot.id = i;
slot.ctx = ctx;
slot.n_ctx = n_ctx_slot;
slot.mctx = mctx;
slot.mctx = mctx;
slot.prompt.tokens.has_mtmd = mctx != nullptr;
if (model_dft) {
slot.batch_spec = llama_batch_init(params_base.speculative.n_max + 1, 0, 1);
// TODO: rework speculative decoding [TAG_SERVER_SPEC_REWORK]
slot.ctx_dft = llama_init_from_model(model_dft, cparams_dft);
if (slot.ctx_dft == nullptr) {
SRV_ERR("%s", "failed to create draft context\n");
return false;
}
slot.spec = common_speculative_init(slot.ctx, slot.ctx_dft);
if (slot.spec == nullptr) {
SRV_ERR("%s", "failed to create speculator\n");
return false;
}
for (auto & pair : params_base.speculative.replacements) {
common_speculative_add_replacement_tgt_dft(slot.spec, pair.first.c_str(), pair.second.c_str());
// try speculative decoding
if (can_spec) {
slot.spec = common_speculative_init(params_base.speculative, slot.ctx);
if (slot.spec) {
if (mctx) {
SRV_ERR("%s\n", "speculative decoding is not supported with multimodal");
return false;
}
SLT_INF(slot, "%s", "speculative decoding context initialized\n");
} else {
SLT_INF(slot, "%s", "speculative decoding context not initialized\n");
}
}
SLT_INF(slot, "new slot, n_ctx = %d\n", slot.n_ctx);
slot.callback_on_release = [this](int slot_id) {
queue_tasks.pop_deferred_task(slot_id);
slot.callback_on_release = [this](int id_slot) {
queue_tasks.pop_deferred_task(id_slot);
};
slot.reset();
@ -914,6 +901,9 @@ private:
}
server_slot * get_slot_by_id(int id_slot) {
// note: allow id_slot to be out of bounds (wrap around)
id_slot = id_slot % slots.size();
for (server_slot & slot : slots) {
if (slot.id == id_slot) {
return &slot;
@ -1059,7 +1049,7 @@ private:
return res;
}
std::vector<common_adapter_lora_info> construct_lora_list(const std::map<int, float> & config) {
std::vector<common_adapter_lora_info> construct_lora_list(const std::map<int, float> & config) const {
std::vector<common_adapter_lora_info> output = params_base.lora_adapters; // copy
for (size_t i = 0; i < output.size(); ++i) {
auto it = config.find(i);
@ -1162,7 +1152,7 @@ private:
backend_sampling &= task.params.sampling.backend_sampling;
// TODO: speculative decoding requires multiple samples per batch - not supported yet
backend_sampling &= !(slot.ctx_dft && task.params.speculative.n_max > 0);
backend_sampling &= !(slot.spec && task.params.speculative.n_max > 0);
// TODO: getting post/pre sampling logits is not yet supported with backend sampling
backend_sampling &= !need_logits;
@ -1179,14 +1169,6 @@ private:
slot.smpl.reset();
}
// initialize draft batch
// TODO: rework speculative decoding [TAG_SERVER_SPEC_REWORK]
if (slot.ctx_dft) {
llama_batch_free(slot.batch_spec);
slot.batch_spec = llama_batch_init(task.params.speculative.n_max + 1, 0, 1);
}
slot.task = std::make_unique<const server_task>(std::move(task));
slot.state = slot.task->is_child()
@ -1791,7 +1773,7 @@ private:
break;
}
int id_slot = task.slot_action.slot_id;
const int id_slot = task.slot_action.id_slot;
server_slot * slot = get_slot_by_id(id_slot);
if (slot == nullptr) {
send_error(task, "Invalid slot ID", ERROR_TYPE_INVALID_REQUEST);
@ -1829,7 +1811,7 @@ private:
case SERVER_TASK_TYPE_SLOT_RESTORE:
{
if (!check_no_mtmd(task.id)) break;
int id_slot = task.slot_action.slot_id;
const int id_slot = task.slot_action.id_slot;
server_slot * slot = get_slot_by_id(id_slot);
if (slot == nullptr) {
send_error(task, "Invalid slot ID", ERROR_TYPE_INVALID_REQUEST);
@ -1878,7 +1860,7 @@ private:
if (!check_no_mtmd(task.id)) {
break;
}
int id_slot = task.slot_action.slot_id;
const int id_slot = task.slot_action.id_slot;
server_slot * slot = get_slot_by_id(id_slot);
if (slot == nullptr) {
send_error(task, "Invalid slot ID", ERROR_TYPE_INVALID_REQUEST);
@ -2059,19 +2041,23 @@ private:
// generate draft tokens in speculative decoding mode
// TODO: rework to have a single draft llama_context shared across all slots [TAG_SERVER_SPEC_REWORK]
// perform the speculative drafting for all sequences at the same time in a single batch
int n_draft_max = slot.get_n_draft_max();
const int n_draft_max = slot.get_n_draft_max();
if (n_draft_max > 0) {
if (mctx) {
// we should never reach this, as speculative is automatically disabled if mmproj is loaded
GGML_ABORT("not supported by multimodal");
}
struct common_speculative_params params_spec;
params_spec.n_draft = n_draft_max;
params_spec.n_reuse = llama_n_ctx(slot.ctx_dft) - slot.task->params.speculative.n_max;
params_spec.p_min = slot.task->params.speculative.p_min;
const llama_tokens & cached_text_tokens = slot.prompt.tokens.get_text_tokens();
llama_tokens draft = common_speculative_gen_draft(slot.spec, params_spec, cached_text_tokens, slot.sampled);
const auto & params_spec = slot.task->params.speculative;
llama_tokens draft = common_speculative_draft(slot.spec, params_spec, cached_text_tokens, slot.sampled);
if (draft.size() > (size_t) n_draft_max) {
SLT_WRN(slot, "draft size %d exceeds max %d, truncating\n", (int) draft.size(), n_draft_max);
draft.resize(n_draft_max);
}
// add the sampled token to the batch
slot.i_batch_dft.push_back(batch.n_tokens);
@ -2521,7 +2507,8 @@ private:
slot.n_prompt_tokens_processed++;
// process the last few tokens of the prompt separately in order to allow for a checkpoint to be created.
if (do_checkpoint && slot.task->n_tokens() - slot.prompt.n_tokens() == 64) {
const int n_last = std::min(n_batch, 512);
if (do_checkpoint && slot.task->n_tokens() == slot.prompt.n_tokens() + n_last) {
break;
}
}
@ -2742,6 +2729,10 @@ private:
// prompt evaluated for next-token prediction
slot.state = SLOT_STATE_GENERATING;
if (slot.can_speculate()) {
common_speculative_begin(slot.spec, slot.prompt.tokens.get_text_tokens());
}
} else if (slot.state != SLOT_STATE_GENERATING) {
continue; // continue loop of slots
}
@ -2813,6 +2804,9 @@ private:
// update how many tokens out of those tested were accepted
slot.n_draft_accepted += ids.size() - 1;
// inform the speculative decoding about the number of accepted tokens
common_speculative_accept(slot.spec, ids.size() - 1);
// rollback to the state before sampling the draft tokens
slot.prompt.tokens.keep_first(slot.prompt.n_tokens() - n_draft);
@ -3332,7 +3326,7 @@ void server_routes::init_routes() {
}
// TODO: get rid of this dynamic_cast
auto res_task = dynamic_cast<server_task_result_metrics*>(result.get());
auto * res_task = dynamic_cast<server_task_result_metrics*>(result.get());
GGML_ASSERT(res_task != nullptr);
// optionally return "fail_on_no_slot" error
@ -3355,8 +3349,8 @@ void server_routes::init_routes() {
}
std::string id_slot_str = req.get_param("id_slot");
int id_slot;
int id_slot;
try {
id_slot = std::stoi(id_slot_str);
} catch (const std::exception &) {
@ -3368,14 +3362,16 @@ void server_routes::init_routes() {
if (action == "save") {
return handle_slots_save(req, id_slot);
} else if (action == "restore") {
return handle_slots_restore(req, id_slot);
} else if (action == "erase") {
return handle_slots_erase(req, id_slot);
} else {
res->error(format_error_response("Invalid action", ERROR_TYPE_INVALID_REQUEST));
return res;
}
if (action == "restore") {
return handle_slots_restore(req, id_slot);
}
if (action == "erase") {
return handle_slots_erase(req, id_slot);
}
res->error(format_error_response("Invalid action", ERROR_TYPE_INVALID_REQUEST));
return res;
};
this->get_props = [this](const server_http_req &) {
@ -3588,6 +3584,8 @@ void server_routes::init_routes() {
auto res = create_response();
std::vector<raw_buffer> files;
json body = convert_responses_to_chatcmpl(json::parse(req.body));
SRV_DBG("%s\n", "Request converted: OpenAI Responses -> OpenAI Chat Completions");
SRV_DBG("converted request: %s\n", body.dump().c_str());
json body_parsed = oaicompat_chat_params_parse(
body,
meta->chat_params,
@ -3604,6 +3602,8 @@ void server_routes::init_routes() {
auto res = create_response();
std::vector<raw_buffer> files;
json body = convert_anthropic_to_oai(json::parse(req.body));
SRV_DBG("%s\n", "Request converted: Anthropic -> OpenAI Chat Completions");
SRV_DBG("converted request: %s\n", body.dump().c_str());
json body_parsed = oaicompat_chat_params_parse(
body,
meta->chat_params,
@ -3620,6 +3620,8 @@ void server_routes::init_routes() {
auto res = create_response();
std::vector<raw_buffer> files;
json body = convert_anthropic_to_oai(json::parse(req.body));
SRV_DBG("%s\n", "Request converted: Anthropic -> OpenAI Chat Completions");
SRV_DBG("converted request: %s\n", body.dump().c_str());
json body_parsed = oaicompat_chat_params_parse(
body,
meta->chat_params,
@ -3918,7 +3920,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_slots_save(const ser
{
server_task task(SERVER_TASK_TYPE_SLOT_SAVE);
task.id = rd.get_new_id();
task.slot_action.slot_id = id_slot;
task.slot_action.id_slot = id_slot;
task.slot_action.filename = filename;
task.slot_action.filepath = filepath;
rd.post_task(std::move(task));
@ -3954,7 +3956,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_slots_restore(const
{
server_task task(SERVER_TASK_TYPE_SLOT_RESTORE);
task.id = rd.get_new_id();
task.slot_action.slot_id = id_slot;
task.slot_action.id_slot = id_slot;
task.slot_action.filename = filename;
task.slot_action.filepath = filepath;
rd.post_task(std::move(task));
@ -3983,7 +3985,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_slots_erase(const se
{
server_task task(SERVER_TASK_TYPE_SLOT_ERASE);
task.id = rd.get_new_id();
task.slot_action.slot_id = id_slot;
task.slot_action.id_slot = id_slot;
rd.post_task(std::move(task));
}

2
tools/server/server-models.cpp
View file

@ -767,7 +767,7 @@ static bool router_validate_model(const std::string & name, server_models & mode
}
auto meta = models.get_meta(name);
if (!meta.has_value()) {
res_err(res, format_error_response("model not found", ERROR_TYPE_INVALID_REQUEST));
res_err(res, format_error_response(string_format("model '%s' not found", name.c_str()), ERROR_TYPE_INVALID_REQUEST));
return false;
}
if (models_autoload) {

19
tools/server/server-task.cpp
View file

@ -5,6 +5,7 @@
#include "llama.h"
#include "chat.h"
#include "sampling.h"
#include "speculative.h"
#include "json-schema-to-grammar.h"
using json = nlohmann::ordered_json;
@ -76,6 +77,10 @@ json task_params::to_json(bool only_metrics) const {
{"speculative.n_max", speculative.n_max},
{"speculative.n_min", speculative.n_min},
{"speculative.p_min", speculative.p_min},
{"speculative.type", common_speculative_type_to_str(speculative.type)},
{"speculative.ngram_size_n", speculative.ngram_size_n},
{"speculative.ngram_size_m", speculative.ngram_size_m},
{"speculative.ngram_m_hits", speculative.ngram_min_hits},
{"timings_per_token", timings_per_token},
{"post_sampling_probs", post_sampling_probs},
{"backend_sampling", sampling.backend_sampling},
@ -135,6 +140,10 @@ json task_params::to_json(bool only_metrics) const {
{"speculative.n_max", speculative.n_max},
{"speculative.n_min", speculative.n_min},
{"speculative.p_min", speculative.p_min},
{"speculative.type", common_speculative_type_to_str(speculative.type)},
{"speculative.ngram_size_n", speculative.ngram_size_n},
{"speculative.ngram_size_m", speculative.ngram_size_m},
{"speculative.ngram_m_hits", speculative.ngram_min_hits},
{"timings_per_token", timings_per_token},
{"post_sampling_probs", post_sampling_probs},
{"backend_sampling", sampling.backend_sampling},
@ -242,6 +251,16 @@ task_params server_task::params_from_json_cmpl(
params.speculative.n_min = std::max(params.speculative.n_min, 0);
params.speculative.n_max = std::max(params.speculative.n_max, 0);
params.speculative.type = common_speculative_type_from_name(json_value(data, "speculative.type", common_speculative_type_to_str(defaults.speculative.type)));
params.speculative.ngram_size_n = json_value(data, "speculative.ngram_size_n", defaults.speculative.ngram_size_n);
params.speculative.ngram_size_m = json_value(data, "speculative.ngram_size_m", defaults.speculative.ngram_size_m);
params.speculative.ngram_min_hits = json_value(data, "speculative.ngram_m_hits", defaults.speculative.ngram_min_hits);
params.speculative.ngram_size_n = std::max(std::min(1, (int) params.speculative.ngram_size_n), 1024);
params.speculative.ngram_size_m = std::max(std::min(1, (int) params.speculative.ngram_size_m), 1024);
params.speculative.ngram_min_hits = std::max(std::min(1, (int) params.speculative.ngram_min_hits), 1024);
// Use OpenAI API logprobs only if n_probs wasn't provided
if (data.contains("logprobs") && params.sampling.n_probs == defaults.sampling.n_probs){
params.sampling.n_probs = json_value(data, "logprobs", defaults.sampling.n_probs);

2
tools/server/server-task.h
View file

@ -153,7 +153,7 @@ struct server_task {
// used by SERVER_TASK_TYPE_SLOT_SAVE, SERVER_TASK_TYPE_SLOT_RESTORE, SERVER_TASK_TYPE_SLOT_ERASE
struct slot_action {
int slot_id;
int id_slot;
std::string filename;
std::string filepath;
};

3
tools/server/tests/unit/test_speculative.py
View file

@ -30,6 +30,7 @@ def test_with_and_without_draft():
"prompt": "I believe the meaning of life is",
"temperature": 0.0,
"top_k": 1,
"n_predict": 16,
})
assert res.status_code == 200
content_no_draft = res.body["content"]
@ -42,6 +43,7 @@ def test_with_and_without_draft():
"prompt": "I believe the meaning of life is",
"temperature": 0.0,
"top_k": 1,
"n_predict": 16,
})
assert res.status_code == 200
content_draft = res.body["content"]
@ -68,6 +70,7 @@ def test_different_draft_min_draft_max():
"prompt": "I believe the meaning of life is",
"temperature": 0.0,
"top_k": 1,
"n_predict": 16,
})
assert res.status_code == 200
if last_content is not None:

2
tools/server/themes/buttons-top/index.html
View file

@ -1032,7 +1032,7 @@
<footer>
<p><${ModelGenerationInfo} /></p>
<p>Powered by <a href="https://github.com/ggerganov/llama.cpp">llama.cpp</a> and <a href="https://ggml.ai">ggml.ai</a>.</p>
<p>Powered by <a href="https://github.com/ggml-org/llama.cpp">llama.cpp</a> and <a href="https://ggml.ai">ggml.ai</a>.</p>
</footer>
</div>
`;

2
tools/server/themes/wild/index.html
View file

@ -1036,7 +1036,7 @@
<footer>
<p><${ModelGenerationInfo} /></p>
<p>Powered by <a href="https://github.com/ggerganov/llama.cpp">llama.cpp</a> and <a href="https://ggml.ai">ggml.ai</a>.</p>
<p>Powered by <a href="https://github.com/ggml-org/llama.cpp">llama.cpp</a> and <a href="https://ggml.ai">ggml.ai</a>.</p>
</footer>
</div>
`;

59
tools/server/webui/package-lock.json generated
View file

@ -61,7 +61,7 @@
"remark-math": "^6.0.0",
"sass": "^1.93.3",
"storybook": "^10.0.7",
"svelte": "^5.0.0",
"svelte": "^5.38.2",
"svelte-check": "^4.0.0",
"tailwind-merge": "^3.3.1",
"tailwind-variants": "^3.2.2",
@ -88,6 +88,7 @@
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz",
"integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@jridgewell/gen-mapping": "^0.3.5",
@ -867,6 +868,7 @@
"integrity": "sha512-oJrXtQiAXLvT9clCf1K4kxp3eKsQhIaZqxEyowkBcsvZDdZkbWrVmnGknxs5flTD0VGsxrxKgBCZty1EzoiMzA==",
"dev": true,
"license": "Apache-2.0",
"peer": true,
"dependencies": {
"@swc/helpers": "^0.5.0"
}
@ -898,7 +900,6 @@
"version": "2.3.5",
"resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz",
"integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@jridgewell/gen-mapping": "^0.3.5",
@ -2031,6 +2032,7 @@
"integrity": "sha512-rO+YQhHucy47Vh67z318pALmd6x+K1Kj30Fb4a6oOEw4xn4zCo9KTmkMWs24c4oduEXD/eJu3badlRmsVXzyfA==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"ts-dedent": "^2.0.0",
"type-fest": "~2.19"
@ -2114,6 +2116,7 @@
"integrity": "sha512-Vp3zX/qlwerQmHMP6x0Ry1oY7eKKRcOWGc2P59srOp4zcqyn+etJyQpELgOi4+ZSUgteX8Y387NuwruLgGXLUQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@standard-schema/spec": "^1.0.0",
"@sveltejs/acorn-typescript": "^1.0.5",
@ -2153,6 +2156,7 @@
"integrity": "sha512-YZs/OSKOQAQCnJvM/P+F1URotNnYNeU3P2s4oIpzm1uFaqUEqRxUB0g5ejMjEb5Gjb9/PiBI5Ktrq4rUUF8UVQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@sveltejs/vite-plugin-svelte-inspector": "^5.0.0",
"debug": "^4.4.1",
@ -2568,6 +2572,7 @@
"integrity": "sha512-pemlzrSESWbdAloYml3bAJMEfNh1Z7EduzqPKprCH5S341frlpYnUEW0H72dLxa6IsYr+mPno20GiSm+h9dEdQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@babel/code-frame": "^7.10.4",
"@babel/runtime": "^7.12.5",
@ -2735,6 +2740,7 @@
"integrity": "sha512-bJFoMATwIGaxxx8VJPeM8TonI8t579oRvgAuT8zFugJsJZgzqv0Fu8Mhp68iecjzG7cnN3mO2dJQ5uUM2EFrgQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"undici-types": "~6.21.0"
}
@ -2802,6 +2808,7 @@
"integrity": "sha512-kVIaQE9vrN9RLCQMQ3iyRlVJpTiDUY6woHGb30JDkfJErqrQEmtdWH3gV0PBAfGZgQXoqzXOO0T3K6ioApbbAA==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@typescript-eslint/scope-manager": "8.37.0",
"@typescript-eslint/types": "8.37.0",
@ -3026,6 +3033,7 @@
"integrity": "sha512-tJxiPrWmzH8a+w9nLKlQMzAKX/7VjFs50MWgcAj7p9XQ7AQ9/35fByFYptgPELyLw+0aixTnC4pUWV+APcZ/kw==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@testing-library/dom": "^10.4.0",
"@testing-library/user-event": "^14.6.1",
@ -3129,6 +3137,7 @@
"integrity": "sha512-oukfKT9Mk41LreEW09vt45f8wx7DordoWUZMYdY/cyAk7w5TWkTRCNZYF7sX7n2wB7jyGAl74OxgwhPgKaqDMQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@vitest/utils": "3.2.4",
"pathe": "^2.0.3",
@ -3186,6 +3195,7 @@
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz",
"integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==",
"license": "MIT",
"peer": true,
"bin": {
"acorn": "bin/acorn"
},
@ -3738,8 +3748,7 @@
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz",
"integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==",
"dev": true,
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/debug": {
"version": "4.4.1",
@ -3840,10 +3849,9 @@
}
},
"node_modules/devalue": {
"version": "5.3.2",
"resolved": "https://registry.npmjs.org/devalue/-/devalue-5.3.2.tgz",
"integrity": "sha512-UDsjUbpQn9kvm68slnrs+mfxwFkIflOhkanmyabZ8zOYk8SMEIbJ3TK+88g70hSIeytu4y18f0z/hYHMTrXIWw==",
"dev": true,
"version": "5.6.2",
"resolved": "https://registry.npmjs.org/devalue/-/devalue-5.6.2.tgz",
"integrity": "sha512-nPRkjWzzDQlsejL1WVifk5rvcFi/y1onBRxjaFMjZeR9mFpqu2gmAZ9xUB9/IEanEP/vBtGeGganC/GO1fmufg==",
"license": "MIT"
},
"node_modules/devlop": {
@ -3973,6 +3981,7 @@
"dev": true,
"hasInstallScript": true,
"license": "MIT",
"peer": true,
"bin": {
"esbuild": "bin/esbuild"
},
@ -4027,6 +4036,7 @@
"integrity": "sha512-QldCVh/ztyKJJZLr4jXNUByx3gR+TDYZCRXEktiZoUR3PGy4qCmSbkxcIle8GEwGpb5JBZazlaJ/CxLidXdEbQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@eslint-community/eslint-utils": "^4.2.0",
"@eslint-community/regexpp": "^4.12.1",
@ -6939,6 +6949,7 @@
}
],
"license": "MIT",
"peer": true,
"dependencies": {
"nanoid": "^3.3.11",
"picocolors": "^1.1.1",
@ -7072,6 +7083,7 @@
"integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==",
"dev": true,
"license": "MIT",
"peer": true,
"bin": {
"prettier": "bin/prettier.cjs"
},
@ -7088,6 +7100,7 @@
"integrity": "sha512-pn1ra/0mPObzqoIQn/vUTR3ZZI6UuZ0sHqMK5x2jMLGrs53h0sXhkVuDcrlssHwIMk7FYrMjHBPoUSyyEEDlBQ==",
"dev": true,
"license": "MIT",
"peer": true,
"peerDependencies": {
"prettier": "^3.0.0",
"svelte": "^3.2.0 || ^4.0.0-next.0 || ^5.0.0-next.0"
@ -7312,6 +7325,7 @@
"integrity": "sha512-FS+XFBNvn3GTAWq26joslQgWNoFu08F4kl0J4CgdNKADkdSGXQyTCnKteIAJy96Br6YbpEU1LSzV5dYtjMkMDg==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=0.10.0"
}
@ -7322,6 +7336,7 @@
"integrity": "sha512-Xs1hdnE+DyKgeHJeJznQmYMIBG3TKIHJJT95Q58nHLSrElKlGQqDTR2HQ9fx5CN/Gk6Vh/kupBTDLU11/nDk/g==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"scheduler": "^0.26.0"
},
@ -7598,6 +7613,7 @@
"integrity": "sha512-4iya7Jb76fVpQyLoiVpzUrsjQ12r3dM7fIVz+4NwoYvZOShknRmiv+iu9CClZml5ZLGb0XMcYLutK6w9tgxHDw==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@types/estree": "1.0.8"
},
@ -7704,6 +7720,7 @@
"integrity": "sha512-elOcIZRTM76dvxNAjqYrucTSI0teAF/L2Lv0s6f6b7FOwcwIuA357bIE871580AjHJuSvLIRUosgV+lIWx6Rgg==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"chokidar": "^4.0.0",
"immutable": "^5.0.2",
@ -7938,6 +7955,7 @@
"integrity": "sha512-7smAu0o+kdm378Q2uIddk32pn0UdIbrtTVU+rXRVtTVTCrK/P2cCui2y4JH+Bl3NgEq1bbBQpCAF/HKrDjk2Qw==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@storybook/global": "^5.0.0",
"@storybook/icons": "^1.6.0",
@ -8079,12 +8097,13 @@
}
},
"node_modules/svelte": {
"version": "5.36.12",
"resolved": "https://registry.npmjs.org/svelte/-/svelte-5.36.12.tgz",
"integrity": "sha512-c3mWT+b0yBLl3gPGSHiy4pdSQCsPNTjLC0tVoOhrGJ6PPfCzD/RQpAmAfJtQZ304CAae2ph+L3C4aqds3R3seQ==",
"version": "5.48.3",
"resolved": "https://registry.npmjs.org/svelte/-/svelte-5.48.3.tgz",
"integrity": "sha512-w7QZ398cdNherTdiQ/v3SYLLGOO4948Jgjh04PYqtTYVohmBvbmFwLmo7pp8gp4/1tceRWfSTjHgjtfpCVNJmQ==",
"license": "MIT",
"peer": true,
"dependencies": {
"@ampproject/remapping": "^2.3.0",
"@jridgewell/remapping": "^2.3.4",
"@jridgewell/sourcemap-codec": "^1.5.0",
"@sveltejs/acorn-typescript": "^1.0.5",
"@types/estree": "^1.0.5",
@ -8092,8 +8111,9 @@
"aria-query": "^5.3.1",
"axobject-query": "^4.1.0",
"clsx": "^2.1.1",
"devalue": "^5.6.2",
"esm-env": "^1.2.1",
"esrap": "^2.1.0",
"esrap": "^2.2.1",
"is-reference": "^3.0.3",
"locate-character": "^3.0.0",
"magic-string": "^0.30.11",
@ -8281,9 +8301,9 @@
}
},
"node_modules/svelte/node_modules/esrap": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/esrap/-/esrap-2.1.0.tgz",
"integrity": "sha512-yzmPNpl7TBbMRC5Lj2JlJZNPml0tzqoqP5B1JXycNUwtqma9AKCO0M2wHrdgsHcy1WRW7S9rJknAMtByg3usgA==",
"version": "2.2.2",
"resolved": "https://registry.npmjs.org/esrap/-/esrap-2.2.2.tgz",
"integrity": "sha512-zA6497ha+qKvoWIK+WM9NAh5ni17sKZKhbS5B3PoYbBvaYHZWoS33zmFybmyqpn07RLUxSmn+RCls2/XF+d0oQ==",
"license": "MIT",
"dependencies": {
"@jridgewell/sourcemap-codec": "^1.4.15"
@ -8326,6 +8346,7 @@
"integrity": "sha512-gBXpgUm/3rp1lMZZrM/w7D8GKqshif0zAymAhbCyIt8KMe+0v9DQ7cdYLR4FHH/cKpdTXb+A/tKKU3eolfsI+g==",
"dev": true,
"license": "MIT",
"peer": true,
"funding": {
"type": "github",
"url": "https://github.com/sponsors/dcastil"
@ -8356,7 +8377,8 @@
"resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.1.11.tgz",
"integrity": "sha512-2E9TBm6MDD/xKYe+dvJZAmg3yxIEDNRc0jwlNyDg/4Fil2QcSLjFKGVff0lAf1jjeaArlG/M75Ey/EYr/OJtBA==",
"dev": true,
"license": "MIT"
"license": "MIT",
"peer": true
},
"node_modules/tapable": {
"version": "2.2.2",
@ -8569,6 +8591,7 @@
"integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==",
"dev": true,
"license": "Apache-2.0",
"peer": true,
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
@ -8934,6 +8957,7 @@
"integrity": "sha512-BxAKBWmIbrDgrokdGZH1IgkIk/5mMHDreLDmCJ0qpyJaAteP8NvMhkwr/ZCQNqNH97bw/dANTE9PDzqwJghfMQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"esbuild": "^0.25.0",
"fdir": "^6.5.0",
@ -9094,6 +9118,7 @@
"integrity": "sha512-LUCP5ev3GURDysTWiP47wRRUpLKMOfPh+yKTx3kVIEiu5KOMeqzpnYNsKyOoVrULivR8tLcks4+lga33Whn90A==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@types/chai": "^5.2.2",
"@vitest/expect": "3.2.4",

2
tools/server/webui/package.json
View file

@ -62,7 +62,7 @@
"remark-math": "^6.0.0",
"sass": "^1.93.3",
"storybook": "^10.0.7",
"svelte": "^5.0.0",
"svelte": "^5.38.2",
"svelte-check": "^4.0.0",
"tailwind-merge": "^3.3.1",
"tailwind-variants": "^3.2.2",

58
tools/server/webui/src/app.css
View file

@ -14,11 +14,11 @@
--popover-foreground: oklch(0.145 0 0);
--primary: oklch(0.205 0 0);
--primary-foreground: oklch(0.985 0 0);
--secondary: oklch(0.97 0 0);
--secondary: oklch(0.95 0 0);
--secondary-foreground: oklch(0.205 0 0);
--muted: oklch(0.97 0 0);
--muted-foreground: oklch(0.556 0 0);
--accent: oklch(0.97 0 0);
--accent: oklch(0.95 0 0);
--accent-foreground: oklch(0.205 0 0);
--destructive: oklch(0.577 0.245 27.325);
--border: oklch(0.875 0 0);
@ -37,7 +37,7 @@
--sidebar-accent-foreground: oklch(0.205 0 0);
--sidebar-border: oklch(0.922 0 0);
--sidebar-ring: oklch(0.708 0 0);
--code-background: oklch(0.975 0 0);
--code-background: oklch(0.985 0 0);
--code-foreground: oklch(0.145 0 0);
--layer-popover: 1000000;
}
@ -51,7 +51,7 @@
--popover-foreground: oklch(0.985 0 0);
--primary: oklch(0.922 0 0);
--primary-foreground: oklch(0.205 0 0);
--secondary: oklch(0.269 0 0);
--secondary: oklch(0.29 0 0);
--secondary-foreground: oklch(0.985 0 0);
--muted: oklch(0.269 0 0);
--muted-foreground: oklch(0.708 0 0);
@ -116,12 +116,62 @@
--color-sidebar-ring: var(--sidebar-ring);
}
:root {
--chat-form-area-height: 8rem;
--chat-form-area-offset: 2rem;
--max-message-height: max(24rem, min(80dvh, calc(100dvh - var(--chat-form-area-height) - 12rem)));
}
@media (min-width: 640px) {
:root {
--chat-form-area-height: 24rem;
--chat-form-area-offset: 12rem;
}
}
@layer base {
* {
@apply border-border outline-ring/50;
}
body {
@apply bg-background text-foreground;
scrollbar-width: thin;
scrollbar-gutter: stable;
}
/* Global scrollbar styling - visible only on hover */
* {
scrollbar-width: thin;
scrollbar-color: transparent transparent;
transition: scrollbar-color 0.2s ease;
}
*:hover {
scrollbar-color: hsl(var(--muted-foreground) / 0.3) transparent;
}
*::-webkit-scrollbar {
width: 6px;
height: 6px;
}
*::-webkit-scrollbar-track {
background: transparent;
}
*::-webkit-scrollbar-thumb {
background: transparent;
border-radius: 3px;
transition: background 0.2s ease;
}
*:hover::-webkit-scrollbar-thumb {
background: hsl(var(--muted-foreground) / 0.3);
}
*::-webkit-scrollbar-thumb:hover {
background: hsl(var(--muted-foreground) / 0.5);
}
}

48
tools/server/webui/src/lib/components/app/actions/ActionIcon.svelte Normal file
View file

@ -0,0 +1,48 @@
<script lang="ts">
import { Button } from '$lib/components/ui/button';
import * as Tooltip from '$lib/components/ui/tooltip';
import type { Component } from 'svelte';
interface Props {
icon: Component;
tooltip: string;
variant?: 'default' | 'destructive' | 'outline' | 'secondary' | 'ghost' | 'link';
size?: 'default' | 'sm' | 'lg' | 'icon';
class?: string;
disabled?: boolean;
onclick: () => void;
'aria-label'?: string;
}
let {
icon,
tooltip,
variant = 'ghost',
size = 'sm',
class: className = '',
disabled = false,
onclick,
'aria-label': ariaLabel
}: Props = $props();
</script>
<Tooltip.Root>
<Tooltip.Trigger>
<Button
{variant}
{size}
{disabled}
{onclick}
class="h-6 w-6 p-0 {className} flex"
aria-label={ariaLabel || tooltip}
>
{@const IconComponent = icon}
<IconComponent class="h-3 w-3" />
</Button>
</Tooltip.Trigger>
<Tooltip.Content>
<p>{tooltip}</p>
</Tooltip.Content>
</Tooltip.Root>

18
tools/server/webui/src/lib/components/app/actions/ActionIconCopyToClipboard.svelte Normal file
View file

@ -0,0 +1,18 @@
<script lang="ts">
import { Copy } from '@lucide/svelte';
import { copyToClipboard } from '$lib/utils';
interface Props {
ariaLabel?: string;
canCopy?: boolean;
text: string;
}
let { ariaLabel = 'Copy to clipboard', canCopy = true, text }: Props = $props();
</script>
<Copy
class="h-3 w-3 flex-shrink-0 cursor-{canCopy ? 'pointer' : 'not-allowed'}"
aria-label={ariaLabel}
onclick={() => canCopy && copyToClipboard(text)}
/>

26
tools/server/webui/src/lib/components/app/actions/ActionIconRemove.svelte Normal file
View file

@ -0,0 +1,26 @@
<script lang="ts">
import { X } from '@lucide/svelte';
import { Button } from '$lib/components/ui/button';
interface Props {
id: string;
onRemove?: (id: string) => void;
class?: string;
}
let { id, onRemove, class: className = '' }: Props = $props();
</script>
<Button
type="button"
variant="ghost"
size="sm"
class="h-6 w-6 bg-white/20 p-0 hover:bg-white/30 {className}"
onclick={(e: MouseEvent) => {
e.stopPropagation();
onRemove?.(id);
}}
aria-label="Remove file"
>
<X class="h-3 w-3" />
</Button>

46
tools/server/webui/src/lib/components/app/actions/ActionIconsCodeBlock.svelte Normal file
View file

@ -0,0 +1,46 @@
<script lang="ts">
import { Eye } from '@lucide/svelte';
import ActionIconCopyToClipboard from '$lib/components/app/actions/ActionIconCopyToClipboard.svelte';
import { FileTypeText } from '$lib/enums';
interface Props {
code: string;
language: string;
disabled?: boolean;
onPreview?: (code: string, language: string) => void;
}
let { code, language, disabled = false, onPreview }: Props = $props();
const showPreview = $derived(language?.toLowerCase() === FileTypeText.HTML);
function handlePreview() {
if (disabled) return;
onPreview?.(code, language);
}
</script>
<div class="code-block-actions">
<div class="copy-code-btn" class:opacity-50={disabled} class:!cursor-not-allowed={disabled}>
<ActionIconCopyToClipboard
text={code}
canCopy={!disabled}
ariaLabel={disabled ? 'Code incomplete' : 'Copy code'}
/>
</div>
{#if showPreview}
<button
class="preview-code-btn"
class:opacity-50={disabled}
class:!cursor-not-allowed={disabled}
title={disabled ? 'Code incomplete' : 'Preview code'}
aria-label="Preview code"
aria-disabled={disabled}
type="button"
onclick={handlePreview}
>
<Eye size={16} />
</button>
{/if}
</div>

19
tools/server/webui/src/lib/components/app/actions/index.ts Normal file
View file

@ -0,0 +1,19 @@
/**
*
* ACTIONS
*
* Small interactive components for user actions.
*
*/
/** Styled icon button for action triggers with tooltip. */
export { default as ActionIcon } from './ActionIcon.svelte';
/** Code block actions component (copy, preview). */
export { default as ActionIconsCodeBlock } from './ActionIconsCodeBlock.svelte';
/** Copy-to-clipboard icon button with click handler. */
export { default as ActionIconCopyToClipboard } from './ActionIconCopyToClipboard.svelte';
/** Remove/delete icon button with X icon. */
export { default as ActionIconRemove } from './ActionIconRemove.svelte';

44
tools/server/webui/src/lib/components/app/badges/BadgeChatStatistic.svelte Normal file
View file

@ -0,0 +1,44 @@
<script lang="ts">
import { BadgeInfo } from '$lib/components/app';
import * as Tooltip from '$lib/components/ui/tooltip';
import { copyToClipboard } from '$lib/utils';
import type { Component } from 'svelte';
interface Props {
class?: string;
icon: Component;
value: string | number;
tooltipLabel?: string;
}
let { class: className = '', icon: Icon, value, tooltipLabel }: Props = $props();
function handleClick() {
void copyToClipboard(String(value));
}
</script>
{#if tooltipLabel}
<Tooltip.Root>
<Tooltip.Trigger>
<BadgeInfo class={className} onclick={handleClick}>
{#snippet icon()}
<Icon class="h-3 w-3" />
{/snippet}
{value}
</BadgeInfo>
</Tooltip.Trigger>
<Tooltip.Content>
<p>{tooltipLabel}</p>
</Tooltip.Content>
</Tooltip.Root>
{:else}
<BadgeInfo class={className} onclick={handleClick}>
{#snippet icon()}
<Icon class="h-3 w-3" />
{/snippet}
{value}
</BadgeInfo>
{/if}

27
tools/server/webui/src/lib/components/app/badges/BadgeInfo.svelte Normal file
View file

@ -0,0 +1,27 @@
<script lang="ts">
import { cn } from '$lib/components/ui/utils';
import type { Snippet } from 'svelte';
interface Props {
children: Snippet;
class?: string;
icon?: Snippet;
onclick?: () => void;
}
let { children, class: className = '', icon, onclick }: Props = $props();
</script>
<button
class={cn(
'inline-flex cursor-pointer items-center gap-1 rounded-sm bg-muted-foreground/15 px-1.5 py-0.75',
className
)}
{onclick}
>
{#if icon}
{@render icon()}
{/if}
{@render children()}
</button>

39
tools/server/webui/src/lib/components/app/badges/BadgeModality.svelte Normal file
View file

@ -0,0 +1,39 @@
<script lang="ts">
import { ModelModality } from '$lib/enums';
import { MODALITY_ICONS, MODALITY_LABELS } from '$lib/constants/icons';
import { cn } from '$lib/components/ui/utils';
type DisplayableModality = ModelModality.VISION | ModelModality.AUDIO;
interface Props {
modalities: ModelModality[];
class?: string;
}
let { modalities, class: className = '' }: Props = $props();
// Filter to only modalities that have icons (VISION, AUDIO)
const displayableModalities = $derived(
modalities.filter(
(m): m is DisplayableModality => m === ModelModality.VISION || m === ModelModality.AUDIO
)
);
</script>
{#each displayableModalities as modality, index (index)}
{@const IconComponent = MODALITY_ICONS[modality]}
{@const label = MODALITY_LABELS[modality]}
<span
class={cn(
'inline-flex items-center gap-1 rounded-md bg-muted px-2 py-1 text-xs font-medium',
className
)}
>
{#if IconComponent}
<IconComponent class="h-3 w-3" />
{/if}
{label}
</span>
{/each}

16
tools/server/webui/src/lib/components/app/badges/index.ts Normal file
View file

@ -0,0 +1,16 @@
/**
*
* BADGES & INDICATORS
*
* Small visual indicators for status and metadata.
*
*/
/** Badge displaying chat statistics (tokens, timing). */
export { default as BadgeChatStatistic } from './BadgeChatStatistic.svelte';
/** Generic info badge with optional tooltip and click handler. */
export { default as BadgeInfo } from './BadgeInfo.svelte';
/** Badge indicating model modality (vision, audio, tools). */
export { default as BadgeModality } from './BadgeModality.svelte';

20
tools/server/webui/src/lib/components/app/chat/ChatForm/ChatForm.svelte
View file

@ -27,11 +27,13 @@
interface Props {
class?: string;
disabled?: boolean;
initialMessage?: string;
isLoading?: boolean;
onFileRemove?: (fileId: string) => void;
onFileUpload?: (files: File[]) => void;
onSend?: (message: string, files?: ChatUploadedFile[]) => Promise<boolean>;
onStop?: () => void;
onSystemPromptAdd?: (draft: { message: string; files: ChatUploadedFile[] }) => void;
showHelperText?: boolean;
uploadedFiles?: ChatUploadedFile[];
}
@ -39,11 +41,13 @@
let {
class: className,
disabled = false,
initialMessage = '',
isLoading = false,
onFileRemove,
onFileUpload,
onSend,
onStop,
onSystemPromptAdd,
showHelperText = true,
uploadedFiles = $bindable([])
}: Props = $props();
@ -53,15 +57,28 @@
let currentConfig = $derived(config());
let fileInputRef: ChatFormFileInputInvisible | undefined = $state(undefined);
let isRecording = $state(false);
let message = $state('');
let message = $state(initialMessage);
let pasteLongTextToFileLength = $derived.by(() => {
const n = Number(currentConfig.pasteLongTextToFileLen);
return Number.isNaN(n) ? Number(SETTING_CONFIG_DEFAULT.pasteLongTextToFileLen) : n;
});
let previousIsLoading = $state(isLoading);
let previousInitialMessage = $state(initialMessage);
let recordingSupported = $state(false);
let textareaRef: ChatFormTextarea | undefined = $state(undefined);
// Sync message when initialMessage prop changes (e.g., after draft restoration)
$effect(() => {
if (initialMessage !== previousInitialMessage) {
message = initialMessage;
previousInitialMessage = initialMessage;
}
});
function handleSystemPromptClick() {
onSystemPromptAdd?.({ message, files: uploadedFiles });
}
// Check if model is selected (in ROUTER mode)
let conversationModel = $derived(
chatStore.getConversationModel(activeMessages() as DatabaseMessage[])
@ -308,6 +325,7 @@
onFileUpload={handleFileUpload}
onMicClick={handleMicClick}
onStop={handleStop}
onSystemPromptClick={handleSystemPromptClick}
/>
</div>
</form>

22
tools/server/webui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionFileAttachments.svelte
View file

@ -1,5 +1,6 @@
<script lang="ts">
import { Paperclip } from '@lucide/svelte';
import { MessageSquare } from '@lucide/svelte';
import { Button } from '$lib/components/ui/button';
import * as DropdownMenu from '$lib/components/ui/dropdown-menu';
import * as Tooltip from '$lib/components/ui/tooltip';
@ -11,6 +12,7 @@
hasAudioModality?: boolean;
hasVisionModality?: boolean;
onFileUpload?: () => void;
onSystemPromptClick?: () => void;
}
let {
@ -18,7 +20,8 @@
disabled = false,
hasAudioModality = false,
hasVisionModality = false,
onFileUpload
onFileUpload,
onSystemPromptClick
}: Props = $props();
const fileUploadTooltipText = $derived.by(() => {
@ -118,6 +121,23 @@
</Tooltip.Content>
{/if}
</Tooltip.Root>
<DropdownMenu.Separator />
<Tooltip.Root>
<Tooltip.Trigger class="w-full">
<DropdownMenu.Item
class="flex cursor-pointer items-center gap-2"
onclick={() => onSystemPromptClick?.()}
>
<MessageSquare class="h-4 w-4" />
<span>System Prompt</span>
</DropdownMenu.Item>
</Tooltip.Trigger>
<Tooltip.Content>
<p>Add a custom system message for this conversation</p>
</Tooltip.Content>
</Tooltip.Root>
</DropdownMenu.Content>
</DropdownMenu.Root>
</div>

5
tools/server/webui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte
View file

@ -27,6 +27,7 @@
onFileUpload?: () => void;
onMicClick?: () => void;
onStop?: () => void;
onSystemPromptClick?: () => void;
}
let {
@ -39,7 +40,8 @@
uploadedFiles = [],
onFileUpload,
onMicClick,
onStop
onStop,
onSystemPromptClick
}: Props = $props();
let currentConfig = $derived(config());
@ -170,6 +172,7 @@
{hasAudioModality}
{hasVisionModality}
{onFileUpload}
{onSystemPromptClick}
/>
<ModelsSelector

76
tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessage.svelte
View file

@ -1,6 +1,15 @@
<script lang="ts">
import { chatStore } from '$lib/stores/chat.svelte';
import { goto } from '$app/navigation';
import {
chatStore,
pendingEditMessageId,
clearPendingEditMessageId,
removeSystemPromptPlaceholder
} from '$lib/stores/chat.svelte';
import { conversationsStore } from '$lib/stores/conversations.svelte';
import { DatabaseService } from '$lib/services';
import { config } from '$lib/stores/settings.svelte';
import { SYSTEM_MESSAGE_PLACEHOLDER } from '$lib/constants/ui';
import { copyToClipboard, isIMEComposing, formatMessageForClipboard } from '$lib/utils';
import ChatMessageAssistant from './ChatMessageAssistant.svelte';
import ChatMessageUser from './ChatMessageUser.svelte';
@ -92,8 +101,30 @@
return null;
});
function handleCancelEdit() {
// Auto-start edit mode if this message is the pending edit target
$effect(() => {
const pendingId = pendingEditMessageId();
if (pendingId && pendingId === message.id && !isEditing) {
handleEdit();
clearPendingEditMessageId();
}
});
async function handleCancelEdit() {
isEditing = false;
// If canceling a new system message with placeholder content, remove it without deleting children
if (message.role === 'system') {
const conversationDeleted = await removeSystemPromptPlaceholder(message.id);
if (conversationDeleted) {
goto('/');
}
return;
}
editedContent = message.content;
editedExtras = message.extra ? [...message.extra] : [];
editedUploadedFiles = [];
@ -114,8 +145,17 @@
onCopy?.(message);
}
function handleConfirmDelete() {
onDelete?.(message);
async function handleConfirmDelete() {
if (message.role === 'system') {
const conversationDeleted = await removeSystemPromptPlaceholder(message.id);
if (conversationDeleted) {
goto('/');
}
} else {
onDelete?.(message);
}
showDeleteDialog = false;
}
@ -126,7 +166,12 @@
function handleEdit() {
isEditing = true;
editedContent = message.content;
// Clear placeholder content for system messages
editedContent =
message.role === 'system' && message.content === SYSTEM_MESSAGE_PLACEHOLDER
? ''
: message.content;
textareaElement?.focus();
editedExtras = message.extra ? [...message.extra] : [];
editedUploadedFiles = [];
@ -166,7 +211,26 @@
}
async function handleSaveEdit() {
if (message.role === 'user' || message.role === 'system') {
if (message.role === 'system') {
// System messages: update in place without branching
const newContent = editedContent.trim();
// If content is empty or still the placeholder, remove without deleting children
if (!newContent) {
const conversationDeleted = await removeSystemPromptPlaceholder(message.id);
isEditing = false;
if (conversationDeleted) {
goto('/');
}
return;
}
await DatabaseService.updateMessage(message.id, { content: newContent });
const index = conversationsStore.findMessageIndex(message.id);
if (index !== -1) {
conversationsStore.updateMessageAtIndex(index, { content: newContent });
}
} else if (message.role === 'user') {
const finalExtras = await getMergedExtras();
onEditWithBranching?.(message, editedContent.trim(), finalExtras);
} else {

2
tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessageSystem.svelte
View file

@ -116,7 +116,7 @@
<Button class="h-8 px-3" onclick={onSaveEdit} disabled={!editedContent.trim()} size="sm">
<Check class="mr-1 h-3 w-3" />
Send
Save
</Button>
</div>
</div>

20
tools/server/webui/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte
View file

@ -71,6 +71,8 @@
let emptyFileNames = $state<string[]>([]);
let initialMessage = $state('');
let isEmpty = $derived(
showCenteredEmpty && !activeConversation() && activeMessages().length === 0 && !isLoading()
);
@ -221,6 +223,14 @@
}
}
async function handleSystemPromptAdd(draft: { message: string; files: ChatUploadedFile[] }) {
if (draft.message || draft.files.length > 0) {
chatStore.savePendingDraft(draft.message, draft.files);
}
await chatStore.addSystemPrompt();
}
function handleScroll() {
if (disableAutoScroll || !chatScrollContainer) return;
@ -343,6 +353,12 @@
if (!disableAutoScroll) {
setTimeout(() => scrollChatToBottom('instant'), INITIAL_SCROLL_DELAY);
}
const pendingDraft = chatStore.consumePendingDraft();
if (pendingDraft) {
initialMessage = pendingDraft.message;
uploadedFiles = pendingDraft.files;
}
});
$effect(() => {
@ -428,11 +444,13 @@
<div class="conversation-chat-form pointer-events-auto rounded-t-3xl pb-4">
<ChatForm
disabled={hasPropsError || isEditing()}
{initialMessage}
isLoading={isCurrentConversationLoading}
onFileRemove={handleFileRemove}
onFileUpload={handleFileUpload}
onSend={handleSendMessage}
onStop={() => chatStore.stopGeneration()}
onSystemPromptAdd={handleSystemPromptAdd}
showHelperText={false}
bind:uploadedFiles
/>
@ -486,11 +504,13 @@
<div in:fly={{ y: 10, duration: 250, delay: hasPropsError ? 0 : 300 }}>
<ChatForm
disabled={hasPropsError}
{initialMessage}
isLoading={isCurrentConversationLoading}
onFileRemove={handleFileRemove}
onFileUpload={handleFileUpload}
onSend={handleSendMessage}
onStop={() => chatStore.stopGeneration()}
onSystemPromptAdd={handleSystemPromptAdd}
showHelperText={true}
bind:uploadedFiles
/>

97
tools/server/webui/src/lib/components/app/content/CollapsibleContentBlock.svelte Normal file
View file

@ -0,0 +1,97 @@
<script lang="ts">
import ChevronsUpDownIcon from '@lucide/svelte/icons/chevrons-up-down';
import * as Collapsible from '$lib/components/ui/collapsible/index.js';
import { buttonVariants } from '$lib/components/ui/button/index.js';
import { Card } from '$lib/components/ui/card';
import { createAutoScrollController } from '$lib/hooks/use-auto-scroll.svelte';
import type { Snippet } from 'svelte';
import type { Component } from 'svelte';
interface Props {
open?: boolean;
class?: string;
icon?: Component;
iconClass?: string;
title: string;
subtitle?: string;
isStreaming?: boolean;
onToggle?: () => void;
children: Snippet;
}
let {
open = $bindable(false),
class: className = '',
icon: Icon,
iconClass = 'h-4 w-4',
title,
subtitle,
isStreaming = false,
onToggle,
children
}: Props = $props();
let contentContainer: HTMLDivElement | undefined = $state();
const autoScroll = createAutoScrollController();
$effect(() => {
autoScroll.setContainer(contentContainer);
});
$effect(() => {
// Only auto-scroll when open and streaming
autoScroll.updateInterval(open && isStreaming);
});
function handleScroll() {
autoScroll.handleScroll();
}
</script>
<Collapsible.Root
{open}
onOpenChange={(value) => {
open = value;
onToggle?.();
}}
class={className}
>
<Card class="gap-0 border-muted bg-muted/30 py-0">
<Collapsible.Trigger class="flex w-full cursor-pointer items-center justify-between p-3">
<div class="flex items-center gap-2 text-muted-foreground">
{#if Icon}
<Icon class={iconClass} />
{/if}
<span class="font-mono text-sm font-medium">{title}</span>
{#if subtitle}
<span class="text-xs italic">{subtitle}</span>
{/if}
</div>
<div
class={buttonVariants({
variant: 'ghost',
size: 'sm',
class: 'h-6 w-6 p-0 text-muted-foreground hover:text-foreground'
})}
>
<ChevronsUpDownIcon class="h-4 w-4" />
<span class="sr-only">Toggle content</span>
</div>
</Collapsible.Trigger>
<Collapsible.Content>
<div
bind:this={contentContainer}
class="overflow-y-auto border-t border-muted px-3 pb-3"
onscroll={handleScroll}
style="min-height: var(--min-message-height); max-height: var(--max-message-height);"
>
{@render children()}
</div>
</Collapsible.Content>
</Card>
</Collapsible.Root>

1201
tools/server/webui/src/lib/components/app/content/MarkdownContent.svelte Normal file
View file

File diff suppressed because it is too large Load diff

95
tools/server/webui/src/lib/components/app/content/SyntaxHighlightedCode.svelte Normal file
View file

@ -0,0 +1,95 @@
<script lang="ts">
import hljs from 'highlight.js';
import { browser } from '$app/environment';
import { mode } from 'mode-watcher';
import githubDarkCss from 'highlight.js/styles/github-dark.css?inline';
import githubLightCss from 'highlight.js/styles/github.css?inline';
interface Props {
code: string;
language?: string;
class?: string;
maxHeight?: string;
maxWidth?: string;
}
let {
code,
language = 'text',
class: className = '',
maxHeight = '60vh',
maxWidth = ''
}: Props = $props();
let highlightedHtml = $state('');
function loadHighlightTheme(isDark: boolean) {
if (!browser) return;
const existingThemes = document.querySelectorAll('style[data-highlight-theme-preview]');
existingThemes.forEach((style) => style.remove());
const style = document.createElement('style');
style.setAttribute('data-highlight-theme-preview', 'true');
style.textContent = isDark ? githubDarkCss : githubLightCss;
document.head.appendChild(style);
}
$effect(() => {
const currentMode = mode.current;
const isDark = currentMode === 'dark';
loadHighlightTheme(isDark);
});
$effect(() => {
if (!code) {
highlightedHtml = '';
return;
}
try {
// Check if the language is supported
const lang = language.toLowerCase();
const isSupported = hljs.getLanguage(lang);
if (isSupported) {
const result = hljs.highlight(code, { language: lang });
highlightedHtml = result.value;
} else {
// Try auto-detection or fallback to plain text
const result = hljs.highlightAuto(code);
highlightedHtml = result.value;
}
} catch {
// Fallback to escaped plain text
highlightedHtml = code.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
}
});
</script>
<div
class="code-preview-wrapper rounded-lg border border-border bg-muted {className}"
style="max-height: {maxHeight}; max-width: {maxWidth};"
>
<!-- Needs to be formatted as single line for proper rendering -->
<pre class="m-0"><code class="hljs text-sm leading-relaxed">{@html highlightedHtml}</code></pre>
</div>
<style>
.code-preview-wrapper {
font-family:
ui-monospace, SFMono-Regular, 'SF Mono', Monaco, 'Cascadia Code', 'Roboto Mono', Consolas,
'Liberation Mono', Menlo, monospace;
}
.code-preview-wrapper pre {
background: transparent;
}
.code-preview-wrapper code {
background: transparent;
}
</style>

79
tools/server/webui/src/lib/components/app/content/index.ts Normal file
View file

@ -0,0 +1,79 @@
/**
*
* CONTENT RENDERING
*
* Components for rendering rich content: markdown, code, and previews.
*
*/
/**
* **MarkdownContent** - Rich markdown renderer
*
* Renders markdown content with syntax highlighting, LaTeX math,
* tables, links, and code blocks. Optimized for streaming with
* incremental block-based rendering.
*
* **Features:**
* - GFM (GitHub Flavored Markdown): tables, task lists, strikethrough
* - LaTeX math via KaTeX (`$inline$` and `$$block$$`)
* - Syntax highlighting (highlight.js) with language detection
* - Code copy buttons with click feedback
* - External links open in new tab with security attrs
* - Image attachment resolution from message extras
* - Dark/light theme support (auto-switching)
* - Streaming-optimized incremental rendering
* - Code preview dialog for large blocks
*
* @example
* ```svelte
* <MarkdownContent content={message.content} attachments={message.extra} />
* ```
*/
export { default as MarkdownContent } from './MarkdownContent.svelte';
/**
* **SyntaxHighlightedCode** - Code syntax highlighting
*
* Renders code with syntax highlighting using highlight.js.
* Supports theme switching and scrollable containers.
*
* **Features:**
* - Auto language detection with fallback
* - Dark/light theme auto-switching
* - Scrollable container with configurable max dimensions
* - Monospace font styling
* - Preserves whitespace and formatting
*
* @example
* ```svelte
* <SyntaxHighlightedCode code={jsonString} language="json" />
* ```
*/
export { default as SyntaxHighlightedCode } from './SyntaxHighlightedCode.svelte';
/**
* **CollapsibleContentBlock** - Expandable content card
*
* Reusable collapsible card with header, icon, and auto-scroll.
* Used for tool calls and reasoning blocks in chat messages.
*
* **Features:**
* - Collapsible content with smooth animation
* - Custom icon and title display
* - Optional subtitle/status text
* - Auto-scroll during streaming (pauses on user scroll)
* - Configurable max height with overflow scroll
*
* @example
* ```svelte
* <CollapsibleContentBlock
* bind:open
* icon={BrainIcon}
* title="Thinking..."
* isStreaming={true}
* >
* {reasoningContent}
* </CollapsibleContentBlock>
* ```
*/
export { default as CollapsibleContentBlock } from './CollapsibleContentBlock.svelte';

10
tools/server/webui/src/lib/components/app/misc/ConversationSelection.svelte
View file

@ -17,9 +17,13 @@
let { conversations, messageCountMap = new Map(), mode, onCancel, onConfirm }: Props = $props();
let searchQuery = $state('');
let selectedIds = $state.raw<SvelteSet<string>>(new SvelteSet(conversations.map((c) => c.id)));
let selectedIds = $state.raw<SvelteSet<string>>(getInitialSelectedIds());
let lastClickedId = $state<string | null>(null);
function getInitialSelectedIds(): SvelteSet<string> {
return new SvelteSet(conversations.map((c) => c.id));
}
let filteredConversations = $derived(
conversations.filter((conv) => {
const name = conv.name || 'Untitled conversation';
@ -92,7 +96,7 @@
}
function handleCancel() {
selectedIds = new SvelteSet(conversations.map((c) => c.id));
selectedIds = getInitialSelectedIds();
searchQuery = '';
lastClickedId = null;
@ -100,7 +104,7 @@
}
export function reset() {
selectedIds = new SvelteSet(conversations.map((c) => c.id));
selectedIds = getInitialSelectedIds();
searchQuery = '';
lastClickedId = null;
}

93
tools/server/webui/src/lib/components/app/misc/HorizontalScrollCarousel.svelte Normal file
View file

@ -0,0 +1,93 @@
<script lang="ts">
import { ChevronLeft, ChevronRight } from '@lucide/svelte';
interface Props {
class?: string;
children?: import('svelte').Snippet;
gapSize?: string;
onScrollableChange?: (isScrollable: boolean) => void;
}
let { class: className = '', children, gapSize = '3', onScrollableChange }: Props = $props();
let canScrollLeft = $state(false);
let canScrollRight = $state(false);
let scrollContainer: HTMLDivElement | undefined = $state();
function scrollLeft(event?: MouseEvent) {
event?.stopPropagation();
event?.preventDefault();
if (!scrollContainer) return;
scrollContainer.scrollBy({ left: scrollContainer.clientWidth * -0.67, behavior: 'smooth' });
}
function scrollRight(event?: MouseEvent) {
event?.stopPropagation();
event?.preventDefault();
if (!scrollContainer) return;
scrollContainer.scrollBy({ left: scrollContainer.clientWidth * 0.67, behavior: 'smooth' });
}
function updateScrollButtons() {
if (!scrollContainer) return;
const { scrollLeft, scrollWidth, clientWidth } = scrollContainer;
canScrollLeft = scrollLeft > 0;
canScrollRight = scrollLeft < scrollWidth - clientWidth - 1;
const isScrollable = scrollWidth > clientWidth;
onScrollableChange?.(isScrollable);
}
export function resetScroll() {
if (scrollContainer) {
scrollContainer.scrollLeft = 0;
setTimeout(() => {
updateScrollButtons();
}, 0);
}
}
$effect(() => {
if (scrollContainer) {
setTimeout(() => {
updateScrollButtons();
}, 0);
}
});
</script>
<div class="relative {className}">
<button
class="absolute top-1/2 left-4 z-10 flex h-6 w-6 -translate-y-1/2 items-center justify-center rounded-full bg-foreground/15 shadow-md backdrop-blur-xs transition-opacity hover:bg-foreground/35 {canScrollLeft
? 'opacity-100'
: 'pointer-events-none opacity-0'}"
onclick={scrollLeft}
aria-label="Scroll left"
>
<ChevronLeft class="h-4 w-4" />
</button>
<div
class="scrollbar-hide flex items-start gap-{gapSize} overflow-x-auto"
bind:this={scrollContainer}
onscroll={updateScrollButtons}
>
{@render children?.()}
</div>
<button
class="absolute top-1/2 right-4 z-10 flex h-6 w-6 -translate-y-1/2 items-center justify-center rounded-full bg-foreground/15 shadow-md backdrop-blur-xs transition-opacity hover:bg-foreground/35 {canScrollRight
? 'opacity-100'
: 'pointer-events-none opacity-0'}"
onclick={scrollRight}
aria-label="Scroll right"
>
<ChevronRight class="h-4 w-4" />
</button>
</div>

4
tools/server/webui/src/lib/components/app/misc/KeyboardShortcutInfo.svelte
View file

@ -11,7 +11,9 @@
let baseClasses =
'px-1 pointer-events-none inline-flex select-none items-center gap-0.5 font-sans text-md font-medium opacity-0 transition-opacity -my-1';
let variantClasses = variant === 'destructive' ? 'text-destructive' : 'text-muted-foreground';
let variantClasses = $derived(
variant === 'destructive' ? 'text-destructive' : 'text-muted-foreground'
);
</script>
<kbd class="{baseClasses} {variantClasses} {className}">

48
tools/server/webui/src/lib/components/app/misc/TruncatedText.svelte Normal file
View file

@ -0,0 +1,48 @@
<script lang="ts">
import * as Tooltip from '$lib/components/ui/tooltip';
interface Props {
text: string;
class?: string;
}
let { text, class: className = '' }: Props = $props();
let textElement: HTMLSpanElement | undefined = $state();
let isTruncated = $state(false);
function checkTruncation() {
if (textElement) {
isTruncated = textElement.scrollWidth > textElement.clientWidth;
}
}
$effect(() => {
if (textElement) {
checkTruncation();
const observer = new ResizeObserver(checkTruncation);
observer.observe(textElement);
return () => observer.disconnect();
}
});
</script>
{#if isTruncated}
<Tooltip.Root>
<Tooltip.Trigger class={className}>
<span bind:this={textElement} class="block truncate">
{text}
</span>
</Tooltip.Trigger>
<Tooltip.Content class="z-[9999]">
<p>{text}</p>
</Tooltip.Content>
</Tooltip.Root>
{:else}
<span bind:this={textElement} class="{className} block truncate">
{text}
</span>
{/if}

45
tools/server/webui/src/lib/components/app/misc/index.ts Normal file
View file

@ -0,0 +1,45 @@
/**
*
* MISC
*
* Miscellaneous utility components.
*
*/
/**
* **ConversationSelection** - Multi-select conversation picker
*
* List of conversations with checkboxes for multi-selection.
* Used in import/export dialogs for selecting conversations.
*
* **Features:**
* - Search/filter conversations by name
* - Select all / deselect all controls
* - Shift-click for range selection
* - Message count display per conversation
* - Mode-specific UI (export vs import)
*/
export { default as ConversationSelection } from './ConversationSelection.svelte';
/**
* Horizontal scrollable carousel with navigation arrows.
* Used for displaying items in a horizontally scrollable container
* with left/right navigation buttons that appear on hover.
*/
export { default as HorizontalScrollCarousel } from './HorizontalScrollCarousel.svelte';
/**
* **TruncatedText** - Text with ellipsis and tooltip
*
* Displays text with automatic truncation and full content in tooltip.
* Useful for long names or paths in constrained spaces.
*/
export { default as TruncatedText } from './TruncatedText.svelte';
/**
* **KeyboardShortcutInfo** - Keyboard shortcut hint display
*
* Displays keyboard shortcut hints (e.g., "⌘ + Enter").
* Supports special keys like shift, cmd, and custom text.
*/
export { default as KeyboardShortcutInfo } from './KeyboardShortcutInfo.svelte';

86
tools/server/webui/src/lib/components/app/navigation/DropdownMenuActions.svelte Normal file
View file

@ -0,0 +1,86 @@
<script lang="ts">
import * as DropdownMenu from '$lib/components/ui/dropdown-menu';
import * as Tooltip from '$lib/components/ui/tooltip';
import { KeyboardShortcutInfo } from '$lib/components/app';
import type { Component } from 'svelte';
interface ActionItem {
icon: Component;
label: string;
onclick: (event: Event) => void;
variant?: 'default' | 'destructive';
disabled?: boolean;
shortcut?: string[];
separator?: boolean;
}
interface Props {
triggerIcon: Component;
triggerTooltip?: string;
triggerClass?: string;
actions: ActionItem[];
align?: 'start' | 'center' | 'end';
open?: boolean;
}
let {
triggerIcon,
triggerTooltip,
triggerClass = '',
actions,
align = 'end',
open = $bindable(false)
}: Props = $props();
</script>
<DropdownMenu.Root bind:open>
<DropdownMenu.Trigger
class="flex h-6 w-6 cursor-pointer items-center justify-center rounded-md p-0 text-sm font-medium transition-colors hover:bg-accent hover:text-accent-foreground focus:bg-accent focus:text-accent-foreground focus:outline-none disabled:pointer-events-none disabled:opacity-50 data-[state=open]:bg-accent data-[state=open]:text-accent-foreground {triggerClass}"
onclick={(e) => e.stopPropagation()}
>
{#if triggerTooltip}
<Tooltip.Root>
<Tooltip.Trigger>
{@render iconComponent(triggerIcon, 'h-3 w-3')}
<span class="sr-only">{triggerTooltip}</span>
</Tooltip.Trigger>
<Tooltip.Content>
<p>{triggerTooltip}</p>
</Tooltip.Content>
</Tooltip.Root>
{:else}
{@render iconComponent(triggerIcon, 'h-3 w-3')}
{/if}
</DropdownMenu.Trigger>
<DropdownMenu.Content {align} class="z-[999999] w-48">
{#each actions as action, index (action.label)}
{#if action.separator && index > 0}
<DropdownMenu.Separator />
{/if}
<DropdownMenu.Item
onclick={action.onclick}
variant={action.variant}
disabled={action.disabled}
class="flex items-center justify-between hover:[&>kbd]:opacity-100"
>
<div class="flex items-center gap-2">
{@render iconComponent(
action.icon,
`h-4 w-4 ${action.variant === 'destructive' ? 'text-destructive' : ''}`
)}
{action.label}
</div>
{#if action.shortcut}
<KeyboardShortcutInfo keys={action.shortcut} variant={action.variant} />
{/if}
</DropdownMenu.Item>
{/each}
</DropdownMenu.Content>
</DropdownMenu.Root>
{#snippet iconComponent(IconComponent: Component, className: string)}
<IconComponent class={className} />
{/snippet}

50
tools/server/webui/src/lib/components/app/navigation/DropdownMenuSearchable.svelte Normal file
View file

@ -0,0 +1,50 @@
<script lang="ts">
import type { Snippet } from 'svelte';
import * as DropdownMenu from '$lib/components/ui/dropdown-menu';
import { SearchInput } from '$lib/components/app';
interface Props {
placeholder?: string;
searchValue?: string;
onSearchChange?: (value: string) => void;
onSearchKeyDown?: (event: KeyboardEvent) => void;
emptyMessage?: string;
isEmpty?: boolean;
children: Snippet;
footer?: Snippet;
}
let {
placeholder = 'Search...',
searchValue = $bindable(''),
onSearchChange,
onSearchKeyDown,
emptyMessage = 'No items found',
isEmpty = false,
children,
footer
}: Props = $props();
</script>
<div class="sticky top-0 z-10 mb-2 bg-popover p-1 pt-2">
<SearchInput
{placeholder}
bind:value={searchValue}
onInput={onSearchChange}
onKeyDown={onSearchKeyDown}
/>
</div>
<div class="overflow-y-auto">
{@render children()}
{#if isEmpty}
<div class="px-2 py-3 text-center text-sm text-muted-foreground">{emptyMessage}</div>
{/if}
</div>
{#if footer}
<DropdownMenu.Separator />
{@render footer()}
{/if}

65
tools/server/webui/src/lib/components/app/navigation/index.ts Normal file
View file

@ -0,0 +1,65 @@
/**
*
* NAVIGATION & MENUS
*
* Components for dropdown menus and action selection.
*
*/
/**
* **DropdownMenuSearchable** - Searchable content for dropdown menus
*
* Renders a search input with filtered content area, empty state, and optional footer.
* Designed to be injected into any dropdown container (DropdownMenu.Content,
* DropdownMenu.SubContent, etc.) without providing its own Root.
*
* **Features:**
* - Search/filter input
* - Keyboard navigation support
* - Custom content and footer via snippets
* - Empty state message
*
* @example
* ```svelte
* <DropdownMenu.Root>
* <DropdownMenu.Trigger>...</DropdownMenu.Trigger>
* <DropdownMenu.Content class="pt-0">
* <DropdownMenuSearchable
* bind:searchValue
* placeholder="Search..."
* isEmpty={filteredItems.length === 0}
* >
* {#each items as item}<Item {item} />{/each}
* </DropdownMenuSearchable>
* </DropdownMenu.Content>
* </DropdownMenu.Root>
* ```
*/
export { default as DropdownMenuSearchable } from './DropdownMenuSearchable.svelte';
/**
* **DropdownMenuActions** - Multi-action dropdown menu
*
* Dropdown menu for multiple action options with icons and shortcuts.
* Supports destructive variants and keyboard shortcut hints.
*
* **Features:**
* - Configurable trigger icon with tooltip
* - Action items with icons and labels
* - Destructive variant styling
* - Keyboard shortcut display
* - Separator support between groups
*
* @example
* ```svelte
* <DropdownMenuActions
* triggerIcon={MoreHorizontal}
* triggerTooltip="More actions"
* actions={[
* { icon: Edit, label: 'Edit', onclick: handleEdit },
* { icon: Trash, label: 'Delete', onclick: handleDelete, variant: 'destructive' }
* ]}
* />
* ```
*/
export { default as DropdownMenuActions } from './DropdownMenuActions.svelte';

3
tools/server/webui/src/lib/components/app/server/ServerErrorSplash.svelte
View file

@ -8,6 +8,7 @@
import { serverStore, serverLoading } from '$lib/stores/server.svelte';
import { config, settingsStore } from '$lib/stores/settings.svelte';
import { fade, fly, scale } from 'svelte/transition';
import { KeyboardKey } from '$lib/enums/keyboard';
interface Props {
class?: string;
@ -117,7 +118,7 @@
}
function handleApiKeyKeydown(event: KeyboardEvent) {
if (event.key === 'Enter') {
if (event.key === KeyboardKey.ENTER) {
handleSaveApiKey();
}
}

2
tools/server/webui/src/lib/components/app/server/ServerStatus.svelte
View file

@ -48,7 +48,7 @@
{model || 'Unknown Model'}
</Badge>
{#if serverData.default_generation_settings.n_ctx}
{#if serverData?.default_generation_settings?.n_ctx}
<Badge variant="secondary" class="text-xs">
ctx: {serverData.default_generation_settings.n_ctx.toLocaleString()}
</Badge>

80
tools/server/webui/src/lib/components/app/server/index.ts Normal file
View file

@ -0,0 +1,80 @@
/**
*
* SERVER
*
* Components for displaying server connection state and handling
* connection errors. Integrates with serverStore for state management.
*
*/
/**
* **ServerStatus** - Server connection status indicator
*
* Compact status display showing connection state, model name,
* and context size. Used in headers and loading screens.
*
* **Architecture:**
* - Reads state from serverStore (props, loading, error)
* - Displays model name from modelsStore
*
* **Features:**
* - Status dot: green (connected), yellow (connecting), red (error), gray (unknown)
* - Status text label
* - Model name badge with icon
* - Context size badge
* - Optional error action button
*
* @example
* ```svelte
* <ServerStatus showActions />
* ```
*/
export { default as ServerStatus } from './ServerStatus.svelte';
/**
* **ServerErrorSplash** - Full-screen connection error display
*
* Blocking error screen shown when server connection fails.
* Provides retry options and API key input for authentication errors.
*
* **Architecture:**
* - Detects access denied errors for API key flow
* - Validates API key against server before saving
* - Integrates with settingsStore for API key persistence
*
* **Features:**
* - Error message display with icon
* - Retry connection button with loading state
* - API key input for authentication errors
* - API key validation with success/error feedback
* - Troubleshooting section with server start commands
* - Animated transitions for UI elements
*
* @example
* ```svelte
* <ServerErrorSplash
* error={serverError}
* onRetry={handleRetry}
* showTroubleshooting
* />
* ```
*/
export { default as ServerErrorSplash } from './ServerErrorSplash.svelte';
/**
* **ServerLoadingSplash** - Full-screen loading display
*
* Shown during initial server connection. Displays loading animation
* with ServerStatus component for real-time connection state.
*
* **Features:**
* - Animated server icon
* - Customizable loading message
* - Embedded ServerStatus for live updates
*
* @example
* ```svelte
* <ServerLoadingSplash message="Connecting to server..." />
* ```
*/
export { default as ServerLoadingSplash } from './ServerLoadingSplash.svelte';

2
tools/server/webui/src/lib/components/ui/badge/badge.svelte
View file

@ -42,7 +42,7 @@
bind:this={ref}
data-slot="badge"
{href}
class={cn(badgeVariants({ variant }), className)}
class={cn(badgeVariants({ variant }), className, 'backdrop-blur-sm')}
{...restProps}
>
{@render children?.()}

5
tools/server/webui/src/lib/components/ui/button/button.svelte
View file

@ -12,8 +12,9 @@
'bg-destructive shadow-xs hover:bg-destructive/90 focus-visible:ring-destructive/20 dark:focus-visible:ring-destructive/40 dark:bg-destructive/60 text-white',
outline:
'bg-background shadow-xs hover:bg-accent hover:text-accent-foreground dark:bg-input/30 dark:border-input dark:hover:bg-input/50 border',
secondary: 'bg-secondary text-secondary-foreground shadow-xs hover:bg-secondary/80',
ghost: 'hover:bg-accent hover:text-accent-foreground dark:hover:bg-accent/50',
secondary:
'dark:bg-secondary dark:text-secondary-foreground bg-background shadow-sm text-foreground hover:bg-muted-foreground/20',
ghost: 'hover:text-accent-foreground hover:bg-muted-foreground/10',
link: 'text-primary underline-offset-4 hover:underline'
},
size: {

4
tools/server/webui/src/lib/components/ui/card/card.svelte
View file

@ -1,6 +1,7 @@
<script lang="ts">
import type { HTMLAttributes } from 'svelte/elements';
import { cn, type WithElementRef } from '$lib/components/ui/utils';
import { BOX_BORDER } from '$lib/constants/css-classes';
let {
ref = $bindable(null),
@ -14,7 +15,8 @@
bind:this={ref}
data-slot="card"
class={cn(
'flex flex-col gap-6 rounded-xl border bg-card py-6 text-card-foreground shadow-sm',
'flex flex-col gap-6 rounded-xl bg-card py-6 text-card-foreground shadow-sm',
BOX_BORDER,
className
)}
{...restProps}

2
tools/server/webui/src/lib/components/ui/dropdown-menu/dropdown-menu-content.svelte
View file

@ -19,7 +19,7 @@
data-slot="dropdown-menu-content"
{sideOffset}
class={cn(
'z-50 max-h-(--bits-dropdown-menu-content-available-height) min-w-[8rem] origin-(--bits-dropdown-menu-content-transform-origin) overflow-x-hidden overflow-y-auto rounded-md border border-border bg-popover p-1 text-popover-foreground shadow-md outline-none data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:zoom-out-95 data-[state=open]:animate-in data-[state=open]:fade-in-0 data-[state=open]:zoom-in-95 dark:border-border/20',
'z-50 max-h-(--bits-dropdown-menu-content-available-height) min-w-[8rem] origin-(--bits-dropdown-menu-content-transform-origin) overflow-x-hidden overflow-y-auto rounded-md border border-border bg-popover p-1.5 text-popover-foreground shadow-md outline-none data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:zoom-out-95 data-[state=open]:animate-in data-[state=open]:fade-in-0 data-[state=open]:zoom-in-95 dark:border-border/20',
className
)}
{...restProps}

1
tools/server/webui/src/lib/components/ui/input/input.svelte
View file

@ -44,6 +44,7 @@
'aria-invalid:border-destructive aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40',
className
)}
style="backdrop-filter: blur(0.5rem);"
{type}
bind:value
{...restProps}

3
tools/server/webui/src/lib/components/ui/sidebar/sidebar-trigger.svelte
View file

@ -1,6 +1,5 @@
<script lang="ts">
import { Button } from '$lib/components/ui/button/index.js';
import { cn } from '$lib/components/ui/utils.js';
import PanelLeftIcon from '@lucide/svelte/icons/panel-left';
import type { ComponentProps } from 'svelte';
import { useSidebar } from './context.svelte.js';
@ -22,7 +21,7 @@
data-slot="sidebar-trigger"
variant="ghost"
size="icon"
class={cn('size-7', className)}
class="rounded-full backdrop-blur-lg {className} h-9! w-9!"
type="button"
onclick={(e) => {
onclick?.(e);

2
tools/server/webui/src/lib/components/ui/switch/switch.svelte
View file

@ -15,7 +15,7 @@
bind:checked
data-slot="switch"
class={cn(
'peer inline-flex h-[1.15rem] w-8 shrink-0 items-center rounded-full border border-transparent shadow-xs transition-all outline-none focus-visible:border-ring focus-visible:ring-[3px] focus-visible:ring-ring/50 disabled:cursor-not-allowed disabled:opacity-50 data-[state=checked]:bg-primary data-[state=unchecked]:bg-input dark:data-[state=unchecked]:bg-input/80',
'peer inline-flex h-[1.15rem] w-8 shrink-0 cursor-pointer items-center rounded-full border border-transparent shadow-xs transition-all outline-none focus-visible:border-ring focus-visible:ring-[3px] focus-visible:ring-ring/50 disabled:cursor-not-allowed disabled:opacity-50 data-[state=checked]:bg-primary data-[state=unchecked]:bg-input dark:data-[state=unchecked]:bg-input/80',
className
)}
{...restProps}

26
tools/server/webui/src/lib/components/ui/tooltip/tooltip-content.svelte
View file

@ -9,22 +9,28 @@
side = 'top',
children,
arrowClasses,
noPortal = false,
...restProps
}: TooltipPrimitive.ContentProps & {
arrowClasses?: string;
noPortal?: boolean;
} = $props();
const contentClass = $derived(
cn(
'z-50 w-fit origin-(--bits-tooltip-content-transform-origin) animate-in rounded-md bg-primary px-3 py-1.5 text-xs text-balance text-primary-foreground fade-in-0 zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:zoom-out-95',
className
)
);
</script>
<TooltipPrimitive.Portal>
{#snippet tooltipContent()}
<TooltipPrimitive.Content
bind:ref
data-slot="tooltip-content"
{sideOffset}
{side}
class={cn(
'z-50 w-fit origin-(--bits-tooltip-content-transform-origin) animate-in rounded-md bg-primary px-3 py-1.5 text-xs text-balance text-primary-foreground fade-in-0 zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:zoom-out-95',
className
)}
class={contentClass}
{...restProps}
>
{@render children?.()}
@ -44,4 +50,12 @@
{/snippet}
</TooltipPrimitive.Arrow>
</TooltipPrimitive.Content>
</TooltipPrimitive.Portal>
{/snippet}
{#if noPortal}
{@render tooltipContent()}
{:else}
<TooltipPrimitive.Portal>
{@render tooltipContent()}
</TooltipPrimitive.Portal>
{/if}

3
tools/server/webui/src/lib/constants/binary-detection.ts
View file

@ -1,9 +1,6 @@
export interface BinaryDetectionOptions {
/** Number of characters to check from the beginning of the file */
prefixLength: number;
/** Maximum ratio of suspicious characters allowed (0.0 to 1.0) */
suspiciousCharThresholdRatio: number;
/** Maximum absolute number of null bytes allowed */
maxAbsoluteNullBytes: number;
}

3
tools/server/webui/src/lib/constants/chat-form.ts Normal file
View file

@ -0,0 +1,3 @@
export const INITIAL_FILE_SIZE = 0;
export const PROMPT_CONTENT_SEPARATOR = '\n\n';
export const CLIPBOARD_CONTENT_QUOTE_PREFIX = '"';

8
tools/server/webui/src/lib/constants/code-blocks.ts Normal file
View file

@ -0,0 +1,8 @@
export const CODE_BLOCK_SCROLL_CONTAINER_CLASS = 'code-block-scroll-container';
export const CODE_BLOCK_WRAPPER_CLASS = 'code-block-wrapper';
export const CODE_BLOCK_HEADER_CLASS = 'code-block-header';
export const CODE_BLOCK_ACTIONS_CLASS = 'code-block-actions';
export const CODE_LANGUAGE_CLASS = 'code-language';
export const COPY_CODE_BTN_CLASS = 'copy-code-btn';
export const PREVIEW_CODE_BTN_CLASS = 'preview-code-btn';
export const RELATIVE_CLASS = 'relative';

7
tools/server/webui/src/lib/constants/code.ts Normal file
View file

@ -0,0 +1,7 @@
export const NEWLINE = '\n';
export const DEFAULT_LANGUAGE = 'text';
export const LANG_PATTERN = /^(\w*)\n?/;
export const AMPERSAND_REGEX = /&/g;
export const LT_REGEX = /</g;
export const GT_REGEX = />/g;
export const FENCE_PATTERN = /^```|\n```/g;

10
tools/server/webui/src/lib/constants/css-classes.ts Normal file
View file

@ -0,0 +1,10 @@
export const BOX_BORDER =
'border border-border/30 focus-within:border-border dark:border-border/20 dark:focus-within:border-border';
export const INPUT_CLASSES = `
bg-muted/60 dark:bg-muted/75
${BOX_BORDER}
shadow-sm
outline-none
text-foreground
`;

8
tools/server/webui/src/lib/constants/formatters.ts Normal file
View file

@ -0,0 +1,8 @@
export const MS_PER_SECOND = 1000;
export const SECONDS_PER_MINUTE = 60;
export const SECONDS_PER_HOUR = 3600;
export const SHORT_DURATION_THRESHOLD = 1;
export const MEDIUM_DURATION_THRESHOLD = 10;
/** Default display value when no performance time is available */
export const DEFAULT_PERFORMANCE_TIME = '0s';

4
tools/server/webui/src/lib/constants/markdown.ts Normal file
View file

@ -0,0 +1,4 @@
export const IMAGE_NOT_ERROR_BOUND_SELECTOR = 'img:not([data-error-bound])';
export const DATA_ERROR_BOUND_ATTR = 'errorBound';
export const DATA_ERROR_HANDLED_ATTR = 'errorHandled';
export const BOOL_TRUE_STRING = 'true';

7
tools/server/webui/src/lib/constants/processing-info.ts
View file

@ -1 +1,8 @@
export const PROCESSING_INFO_TIMEOUT = 2000;
/**
* Statistics units labels
*/
export const STATS_UNITS = {
TOKENS_PER_SECOND: 't/s'
} as const;

33
tools/server/webui/src/lib/constants/settings-fields.ts Normal file
View file

@ -0,0 +1,33 @@
/**
* List of all numeric fields in settings configuration.
* These fields will be converted from strings to numbers during save.
*/
export const NUMERIC_FIELDS = [
'temperature',
'top_k',
'top_p',
'min_p',
'max_tokens',
'pasteLongTextToFileLen',
'dynatemp_range',
'dynatemp_exponent',
'typ_p',
'xtc_probability',
'xtc_threshold',
'repeat_last_n',
'repeat_penalty',
'presence_penalty',
'frequency_penalty',
'dry_multiplier',
'dry_base',
'dry_allowed_length',
'dry_penalty_last_n',
'agenticMaxTurns',
'agenticMaxToolPreviewLines'
] as const;
/**
* Fields that must be positive integers (>= 1).
* These will be clamped to minimum 1 and rounded during save.
*/
export const POSITIVE_INTEGER_FIELDS = ['agenticMaxTurns', 'agenticMaxToolPreviewLines'] as const;

2
tools/server/webui/src/lib/constants/tooltip-config.ts
View file

@ -1 +1 @@
export const TOOLTIP_DELAY_DURATION = 100;
export const TOOLTIP_DELAY_DURATION = 500;

1
tools/server/webui/src/lib/constants/ui.ts Normal file
View file

@ -0,0 +1 @@
export const SYSTEM_MESSAGE_PLACEHOLDER = 'System message';

34
tools/server/webui/src/lib/contexts/chat-actions.context.ts Normal file
View file

@ -0,0 +1,34 @@
import { getContext, setContext } from 'svelte';
export interface ChatActionsContext {
copy: (message: DatabaseMessage) => void;
delete: (message: DatabaseMessage) => void;
navigateToSibling: (siblingId: string) => void;
editWithBranching: (
message: DatabaseMessage,
newContent: string,
newExtras?: DatabaseMessageExtra[]
) => void;
editWithReplacement: (
message: DatabaseMessage,
newContent: string,
shouldBranch: boolean
) => void;
editUserMessagePreserveResponses: (
message: DatabaseMessage,
newContent: string,
newExtras?: DatabaseMessageExtra[]
) => void;
regenerateWithBranching: (message: DatabaseMessage, modelOverride?: string) => void;
continueAssistantMessage: (message: DatabaseMessage) => void;
}
const CHAT_ACTIONS_KEY = Symbol.for('chat-actions');
export function setChatActionsContext(ctx: ChatActionsContext): ChatActionsContext {
return setContext(CHAT_ACTIONS_KEY, ctx);
}
export function getChatActionsContext(): ChatActionsContext {
return getContext(CHAT_ACTIONS_KEY);
}

13
tools/server/webui/src/lib/contexts/index.ts Normal file
View file

@ -0,0 +1,13 @@
export {
getMessageEditContext,
setMessageEditContext,
type MessageEditContext,
type MessageEditState,
type MessageEditActions
} from './message-edit.context';
export {
getChatActionsContext,
setChatActionsContext,
type ChatActionsContext
} from './chat-actions.context';

39
tools/server/webui/src/lib/contexts/message-edit.context.ts Normal file
View file

@ -0,0 +1,39 @@
import { getContext, setContext } from 'svelte';
export interface MessageEditState {
readonly isEditing: boolean;
readonly editedContent: string;
readonly editedExtras: DatabaseMessageExtra[];
readonly editedUploadedFiles: ChatUploadedFile[];
readonly originalContent: string;
readonly originalExtras: DatabaseMessageExtra[];
readonly showSaveOnlyOption: boolean;
}
export interface MessageEditActions {
setContent: (content: string) => void;
setExtras: (extras: DatabaseMessageExtra[]) => void;
setUploadedFiles: (files: ChatUploadedFile[]) => void;
save: () => void;
saveOnly: () => void;
cancel: () => void;
startEdit: () => void;
}
export type MessageEditContext = MessageEditState & MessageEditActions;
const MESSAGE_EDIT_KEY = Symbol.for('chat-message-edit');
/**
* Sets the message edit context. Call this in the parent component (ChatMessage.svelte).
*/
export function setMessageEditContext(ctx: MessageEditContext): MessageEditContext {
return setContext(MESSAGE_EDIT_KEY, ctx);
}
/**
* Gets the message edit context. Call this in child components.
*/
export function getMessageEditContext(): MessageEditContext {
return getContext(MESSAGE_EDIT_KEY);
}

49
tools/server/webui/src/lib/enums/chat.ts
View file

@ -1,4 +1,51 @@
export enum ChatMessageStatsView {
GENERATION = 'generation',
READING = 'reading'
READING = 'reading',
TOOLS = 'tools',
SUMMARY = 'summary'
}
/**
* Reasoning format options for API requests.
*/
export enum ReasoningFormat {
NONE = 'none',
AUTO = 'auto'
}
/**
* Message roles for chat messages.
*/
export enum MessageRole {
USER = 'user',
ASSISTANT = 'assistant',
SYSTEM = 'system',
TOOL = 'tool'
}
/**
* Message types for different content kinds.
*/
export enum MessageType {
ROOT = 'root',
TEXT = 'text',
THINK = 'think',
SYSTEM = 'system'
}
/**
* Content part types for API chat message content.
*/
export enum ContentPartType {
TEXT = 'text',
IMAGE_URL = 'image_url',
INPUT_AUDIO = 'input_audio'
}
/**
* Error dialog types for displaying server/timeout errors.
*/
export enum ErrorDialogType {
TIMEOUT = 'timeout',
SERVER = 'server'
}

15
tools/server/webui/src/lib/enums/keyboard.ts Normal file
View file

@ -0,0 +1,15 @@
/**
* Keyboard key names for event handling
*/
export enum KeyboardKey {
ENTER = 'Enter',
ESCAPE = 'Escape',
ARROW_UP = 'ArrowUp',
ARROW_DOWN = 'ArrowDown',
TAB = 'Tab',
D_LOWER = 'd',
D_UPPER = 'D',
E_UPPER = 'E',
K_LOWER = 'k',
O_UPPER = 'O'
}

26
tools/server/webui/src/lib/enums/settings.ts Normal file
View file

@ -0,0 +1,26 @@
/**
* Parameter source - indicates whether a parameter uses default or custom value
*/
export enum ParameterSource {
DEFAULT = 'default',
CUSTOM = 'custom'
}
/**
* Syncable parameter type - data types for parameters that can be synced with server
*/
export enum SyncableParameterType {
NUMBER = 'number',
STRING = 'string',
BOOLEAN = 'boolean'
}
/**
* Settings field type - defines the input type for settings fields
*/
export enum SettingsFieldType {
INPUT = 'input',
TEXTAREA = 'textarea',
CHECKBOX = 'checkbox',
SELECT = 'select'
}

165
tools/server/webui/src/lib/hooks/use-auto-scroll.svelte.ts Normal file
View file

@ -0,0 +1,165 @@
import { AUTO_SCROLL_AT_BOTTOM_THRESHOLD, AUTO_SCROLL_INTERVAL } from '$lib/constants/auto-scroll';
export interface AutoScrollOptions {
/** Whether auto-scroll is disabled globally (e.g., from settings) */
disabled?: boolean;
}
/**
* Creates an auto-scroll controller for a scrollable container.
*
* Features:
* - Auto-scrolls to bottom during streaming/loading
* - Stops auto-scroll when user manually scrolls up
* - Resumes auto-scroll when user scrolls back to bottom
*/
export class AutoScrollController {
private _autoScrollEnabled = $state(true);
private _userScrolledUp = $state(false);
private _lastScrollTop = $state(0);
private _scrollInterval: ReturnType<typeof setInterval> | undefined;
private _scrollTimeout: ReturnType<typeof setTimeout> | undefined;
private _container: HTMLElement | undefined;
private _disabled: boolean;
constructor(options: AutoScrollOptions = {}) {
this._disabled = options.disabled ?? false;
}
get autoScrollEnabled(): boolean {
return this._autoScrollEnabled;
}
get userScrolledUp(): boolean {
return this._userScrolledUp;
}
/**
* Binds the controller to a scrollable container element.
*/
setContainer(container: HTMLElement | undefined): void {
this._container = container;
}
/**
* Updates the disabled state.
*/
setDisabled(disabled: boolean): void {
this._disabled = disabled;
if (disabled) {
this._autoScrollEnabled = false;
this.stopInterval();
}
}
/**
* Handles scroll events to detect user scroll direction and toggle auto-scroll.
*/
handleScroll(): void {
if (this._disabled || !this._container) return;
const { scrollTop, scrollHeight, clientHeight } = this._container;
const distanceFromBottom = scrollHeight - scrollTop - clientHeight;
const isAtBottom = distanceFromBottom < AUTO_SCROLL_AT_BOTTOM_THRESHOLD;
if (scrollTop < this._lastScrollTop && !isAtBottom) {
this._userScrolledUp = true;
this._autoScrollEnabled = false;
} else if (isAtBottom && this._userScrolledUp) {
this._userScrolledUp = false;
this._autoScrollEnabled = true;
}
if (this._scrollTimeout) {
clearTimeout(this._scrollTimeout);
}
this._scrollTimeout = setTimeout(() => {
if (isAtBottom) {
this._userScrolledUp = false;
this._autoScrollEnabled = true;
}
}, AUTO_SCROLL_INTERVAL);
this._lastScrollTop = scrollTop;
}
/**
* Scrolls the container to the bottom.
*/
scrollToBottom(behavior: ScrollBehavior = 'smooth'): void {
if (this._disabled || !this._container) return;
this._container.scrollTo({
top: this._container.scrollHeight,
behavior
});
}
/**
* Enables auto-scroll (e.g., when user sends a message).
*/
enable(): void {
if (this._disabled) return;
this._userScrolledUp = false;
this._autoScrollEnabled = true;
}
/**
* Starts the auto-scroll interval for continuous scrolling during streaming.
*/
startInterval(): void {
if (this._disabled || this._scrollInterval) return;
this._scrollInterval = setInterval(() => {
this.scrollToBottom();
}, AUTO_SCROLL_INTERVAL);
}
/**
* Stops the auto-scroll interval.
*/
stopInterval(): void {
if (this._scrollInterval) {
clearInterval(this._scrollInterval);
this._scrollInterval = undefined;
}
}
/**
* Updates the auto-scroll interval based on streaming state.
* Call this in a $effect to automatically manage the interval.
*/
updateInterval(isStreaming: boolean): void {
if (this._disabled) {
this.stopInterval();
return;
}
if (isStreaming && this._autoScrollEnabled) {
if (!this._scrollInterval) {
this.startInterval();
}
} else {
this.stopInterval();
}
}
/**
* Cleans up resources. Call this in onDestroy or when the component unmounts.
*/
destroy(): void {
this.stopInterval();
if (this._scrollTimeout) {
clearTimeout(this._scrollTimeout);
this._scrollTimeout = undefined;
}
}
}
/**
* Creates a new AutoScrollController instance.
*/
export function createAutoScrollController(options: AutoScrollOptions = {}): AutoScrollController {
return new AutoScrollController(options);
}

85
tools/server/webui/src/lib/hooks/use-processing-state.svelte.ts
View file

@ -1,7 +1,9 @@
import { activeProcessingState } from '$lib/stores/chat.svelte';
import { config } from '$lib/stores/settings.svelte';
import { STATS_UNITS } from '$lib/constants/processing-info';
import type { ApiProcessingState } from '$lib/types';
export interface LiveProcessingStats {
interface LiveProcessingStats {
tokensProcessed: number;
totalTokens: number;
timeMs: number;
@ -9,7 +11,7 @@ export interface LiveProcessingStats {
etaSecs?: number;
}
export interface LiveGenerationStats {
interface LiveGenerationStats {
tokensGenerated: number;
timeMs: number;
tokensPerSecond: number;
@ -18,6 +20,7 @@ export interface LiveGenerationStats {
export interface UseProcessingStateReturn {
readonly processingState: ApiProcessingState | null;
getProcessingDetails(): string[];
getTechnicalDetails(): string[];
getProcessingMessage(): string;
getPromptProgressText(): string | null;
getLiveProcessingStats(): LiveProcessingStats | null;
@ -138,8 +141,31 @@ export function useProcessingState(): UseProcessingStateReturn {
const details: string[] = [];
// Show prompt processing progress with ETA during preparation phase
if (stateToUse.promptProgress) {
const { processed, total, time_ms, cache } = stateToUse.promptProgress;
const actualProcessed = processed - cache;
const actualTotal = total - cache;
if (actualProcessed < actualTotal && actualProcessed > 0) {
const percent = Math.round((actualProcessed / actualTotal) * 100);
const eta = getETASecs(actualProcessed, actualTotal, time_ms);
if (eta !== undefined) {
const etaSecs = Math.ceil(eta);
details.push(`Processing ${percent}% (ETA: ${etaSecs}s)`);
} else {
details.push(`Processing ${percent}%`);
}
}
}
// Always show context info when we have valid data
if (stateToUse.contextUsed >= 0 && stateToUse.contextTotal > 0) {
if (
typeof stateToUse.contextTotal === 'number' &&
stateToUse.contextUsed >= 0 &&
stateToUse.contextTotal > 0
) {
const contextPercent = Math.round((stateToUse.contextUsed / stateToUse.contextTotal) * 100);
details.push(
@ -163,7 +189,57 @@ export function useProcessingState(): UseProcessingStateReturn {
}
if (stateToUse.tokensPerSecond && stateToUse.tokensPerSecond > 0) {
details.push(`${stateToUse.tokensPerSecond.toFixed(1)} tokens/sec`);
details.push(`${stateToUse.tokensPerSecond.toFixed(1)} ${STATS_UNITS.TOKENS_PER_SECOND}`);
}
if (stateToUse.speculative) {
details.push('Speculative decoding enabled');
}
return details;
}
/**
* Returns technical details without the progress message (for bottom bar)
*/
function getTechnicalDetails(): string[] {
const stateToUse = processingState || lastKnownState;
if (!stateToUse) {
return [];
}
const details: string[] = [];
// Always show context info when we have valid data
if (
typeof stateToUse.contextTotal === 'number' &&
stateToUse.contextUsed >= 0 &&
stateToUse.contextTotal > 0
) {
const contextPercent = Math.round((stateToUse.contextUsed / stateToUse.contextTotal) * 100);
details.push(
`Context: ${stateToUse.contextUsed}/${stateToUse.contextTotal} (${contextPercent}%)`
);
}
if (stateToUse.outputTokensUsed > 0) {
// Handle infinite max_tokens (-1) case
if (stateToUse.outputTokensMax <= 0) {
details.push(`Output: ${stateToUse.outputTokensUsed}/∞`);
} else {
const outputPercent = Math.round(
(stateToUse.outputTokensUsed / stateToUse.outputTokensMax) * 100
);
details.push(
`Output: ${stateToUse.outputTokensUsed}/${stateToUse.outputTokensMax} (${outputPercent}%)`
);
}
}
if (stateToUse.tokensPerSecond && stateToUse.tokensPerSecond > 0) {
details.push(`${stateToUse.tokensPerSecond.toFixed(1)} ${STATS_UNITS.TOKENS_PER_SECOND}`);
}
if (stateToUse.speculative) {
@ -251,6 +327,7 @@ export function useProcessingState(): UseProcessingStateReturn {
return processingState;
},
getProcessingDetails,
getTechnicalDetails,
getProcessingMessage,
getPromptProgressText,
getLiveProcessingStats,

33
tools/server/webui/src/lib/markdown/enhance-code-blocks.ts
View file

@ -13,6 +13,16 @@
import type { Plugin } from 'unified';
import type { Root, Element, ElementContent } from 'hast';
import { visit } from 'unist-util-visit';
import {
CODE_BLOCK_SCROLL_CONTAINER_CLASS,
CODE_BLOCK_WRAPPER_CLASS,
CODE_BLOCK_HEADER_CLASS,
CODE_BLOCK_ACTIONS_CLASS,
CODE_LANGUAGE_CLASS,
COPY_CODE_BTN_CLASS,
PREVIEW_CODE_BTN_CLASS,
RELATIVE_CLASS
} from '$lib/constants/code-blocks';
declare global {
interface Window {
@ -42,7 +52,7 @@ function createCopyButton(codeId: string): Element {
type: 'element',
tagName: 'button',
properties: {
className: ['copy-code-btn'],
className: [COPY_CODE_BTN_CLASS],
'data-code-id': codeId,
title: 'Copy code',
type: 'button'
@ -56,7 +66,7 @@ function createPreviewButton(codeId: string): Element {
type: 'element',
tagName: 'button',
properties: {
className: ['preview-code-btn'],
className: [PREVIEW_CODE_BTN_CLASS],
'data-code-id': codeId,
title: 'Preview code',
type: 'button'
@ -75,30 +85,39 @@ function createHeader(language: string, codeId: string): Element {
return {
type: 'element',
tagName: 'div',
properties: { className: ['code-block-header'] },
properties: { className: [CODE_BLOCK_HEADER_CLASS] },
children: [
{
type: 'element',
tagName: 'span',
properties: { className: ['code-language'] },
properties: { className: [CODE_LANGUAGE_CLASS] },
children: [{ type: 'text', value: language }]
},
{
type: 'element',
tagName: 'div',
properties: { className: ['code-block-actions'] },
properties: { className: [CODE_BLOCK_ACTIONS_CLASS] },
children: actions
}
]
};
}
function createScrollContainer(preElement: Element): Element {
return {
type: 'element',
tagName: 'div',
properties: { className: [CODE_BLOCK_SCROLL_CONTAINER_CLASS] },
children: [preElement]
};
}
function createWrapper(header: Element, preElement: Element): Element {
return {
type: 'element',
tagName: 'div',
properties: { className: ['code-block-wrapper'] },
children: [header, preElement]
properties: { className: [CODE_BLOCK_WRAPPER_CLASS, RELATIVE_CLASS] },
children: [header, createScrollContainer(preElement)]
};
}

368
tools/server/webui/src/lib/services/database.service.ts Normal file
View file

@ -0,0 +1,368 @@
import Dexie, { type EntityTable } from 'dexie';
import { findDescendantMessages } from '$lib/utils';
class LlamacppDatabase extends Dexie {
conversations!: EntityTable<DatabaseConversation, string>;
messages!: EntityTable<DatabaseMessage, string>;
constructor() {
super('LlamacppWebui');
this.version(1).stores({
conversations: 'id, lastModified, currNode, name',
messages: 'id, convId, type, role, timestamp, parent, children'
});
}
}
const db = new LlamacppDatabase();
import { v4 as uuid } from 'uuid';
import { MessageRole } from '$lib/enums/chat';
export class DatabaseService {
/**
*
*
* Conversations
*
*
*/
/**
* Creates a new conversation.
*
* @param name - Name of the conversation
* @returns The created conversation
*/
static async createConversation(name: string): Promise<DatabaseConversation> {
const conversation: DatabaseConversation = {
id: uuid(),
name,
lastModified: Date.now(),
currNode: ''
};
await db.conversations.add(conversation);
return conversation;
}
/**
*
*
* Messages
*
*
*/
/**
* Creates a new message branch by adding a message and updating parent/child relationships.
* Also updates the conversation's currNode to point to the new message.
*
* @param message - Message to add (without id)
* @param parentId - Parent message ID to attach to
* @returns The created message
*/
static async createMessageBranch(
message: Omit<DatabaseMessage, 'id'>,
parentId: string | null
): Promise<DatabaseMessage> {
return await db.transaction('rw', [db.conversations, db.messages], async () => {
// Handle null parent (root message case)
if (parentId !== null) {
const parentMessage = await db.messages.get(parentId);
if (!parentMessage) {
throw new Error(`Parent message ${parentId} not found`);
}
}
const newMessage: DatabaseMessage = {
...message,
id: uuid(),
parent: parentId,
toolCalls: message.toolCalls ?? '',
children: []
};
await db.messages.add(newMessage);
// Update parent's children array if parent exists
if (parentId !== null) {
const parentMessage = await db.messages.get(parentId);
if (parentMessage) {
await db.messages.update(parentId, {
children: [...parentMessage.children, newMessage.id]
});
}
}
await this.updateConversation(message.convId, {
currNode: newMessage.id
});
return newMessage;
});
}
/**
* Creates a root message for a new conversation.
* Root messages are not displayed but serve as the tree root for branching.
*
* @param convId - Conversation ID
* @returns The created root message
*/
static async createRootMessage(convId: string): Promise<string> {
const rootMessage: DatabaseMessage = {
id: uuid(),
convId,
type: 'root',
timestamp: Date.now(),
role: MessageRole.SYSTEM,
content: '',
parent: null,
toolCalls: '',
children: []
};
await db.messages.add(rootMessage);
return rootMessage.id;
}
/**
* Creates a system prompt message for a conversation.
*
* @param convId - Conversation ID
* @param systemPrompt - The system prompt content (must be non-empty)
* @param parentId - Parent message ID (typically the root message)
* @returns The created system message
* @throws Error if systemPrompt is empty
*/
static async createSystemMessage(
convId: string,
systemPrompt: string,
parentId: string
): Promise<DatabaseMessage> {
const trimmedPrompt = systemPrompt.trim();
if (!trimmedPrompt) {
throw new Error('Cannot create system message with empty content');
}
const systemMessage: DatabaseMessage = {
id: uuid(),
convId,
type: MessageRole.SYSTEM,
timestamp: Date.now(),
role: MessageRole.SYSTEM,
content: trimmedPrompt,
parent: parentId,
children: []
};
await db.messages.add(systemMessage);
const parentMessage = await db.messages.get(parentId);
if (parentMessage) {
await db.messages.update(parentId, {
children: [...parentMessage.children, systemMessage.id]
});
}
return systemMessage;
}
/**
* Deletes a conversation and all its messages.
*
* @param id - Conversation ID
*/
static async deleteConversation(id: string): Promise<void> {
await db.transaction('rw', [db.conversations, db.messages], async () => {
await db.conversations.delete(id);
await db.messages.where('convId').equals(id).delete();
});
}
/**
* Deletes a message and removes it from its parent's children array.
*
* @param messageId - ID of the message to delete
*/
static async deleteMessage(messageId: string): Promise<void> {
await db.transaction('rw', db.messages, async () => {
const message = await db.messages.get(messageId);
if (!message) return;
// Remove this message from its parent's children array
if (message.parent) {
const parent = await db.messages.get(message.parent);
if (parent) {
parent.children = parent.children.filter((childId: string) => childId !== messageId);
await db.messages.put(parent);
}
}
// Delete the message
await db.messages.delete(messageId);
});
}
/**
* Deletes a message and all its descendant messages (cascading deletion).
* This removes the entire branch starting from the specified message.
*
* @param conversationId - ID of the conversation containing the message
* @param messageId - ID of the root message to delete (along with all descendants)
* @returns Array of all deleted message IDs
*/
static async deleteMessageCascading(
conversationId: string,
messageId: string
): Promise<string[]> {
return await db.transaction('rw', db.messages, async () => {
// Get all messages in the conversation to find descendants
const allMessages = await db.messages.where('convId').equals(conversationId).toArray();
// Find all descendant messages
const descendants = findDescendantMessages(allMessages, messageId);
const allToDelete = [messageId, ...descendants];
// Get the message to delete for parent cleanup
const message = await db.messages.get(messageId);
if (message && message.parent) {
const parent = await db.messages.get(message.parent);
if (parent) {
parent.children = parent.children.filter((childId: string) => childId !== messageId);
await db.messages.put(parent);
}
}
// Delete all messages in the branch
await db.messages.bulkDelete(allToDelete);
return allToDelete;
});
}
/**
* Gets all conversations, sorted by last modified time (newest first).
*
* @returns Array of conversations
*/
static async getAllConversations(): Promise<DatabaseConversation[]> {
return await db.conversations.orderBy('lastModified').reverse().toArray();
}
/**
* Gets a conversation by ID.
*
* @param id - Conversation ID
* @returns The conversation if found, otherwise undefined
*/
static async getConversation(id: string): Promise<DatabaseConversation | undefined> {
return await db.conversations.get(id);
}
/**
* Gets all messages in a conversation, sorted by timestamp (oldest first).
*
* @param convId - Conversation ID
* @returns Array of messages in the conversation
*/
static async getConversationMessages(convId: string): Promise<DatabaseMessage[]> {
return await db.messages.where('convId').equals(convId).sortBy('timestamp');
}
/**
* Updates a conversation.
*
* @param id - Conversation ID
* @param updates - Partial updates to apply
* @returns Promise that resolves when the conversation is updated
*/
static async updateConversation(
id: string,
updates: Partial<Omit<DatabaseConversation, 'id'>>
): Promise<void> {
await db.conversations.update(id, {
...updates,
lastModified: Date.now()
});
}
/**
*
*
* Navigation
*
*
*/
/**
* Updates the conversation's current node (active branch).
* This determines which conversation path is currently being viewed.
*
* @param convId - Conversation ID
* @param nodeId - Message ID to set as current node
*/
static async updateCurrentNode(convId: string, nodeId: string): Promise<void> {
await this.updateConversation(convId, {
currNode: nodeId
});
}
/**
* Updates a message.
*
* @param id - Message ID
* @param updates - Partial updates to apply
* @returns Promise that resolves when the message is updated
*/
static async updateMessage(
id: string,
updates: Partial<Omit<DatabaseMessage, 'id'>>
): Promise<void> {
await db.messages.update(id, updates);
}
/**
*
*
* Import
*
*
*/
/**
* Imports multiple conversations and their messages.
* Skips conversations that already exist.
*
* @param data - Array of { conv, messages } objects
*/
static async importConversations(
data: { conv: DatabaseConversation; messages: DatabaseMessage[] }[]
): Promise<{ imported: number; skipped: number }> {
let importedCount = 0;
let skippedCount = 0;
return await db.transaction('rw', [db.conversations, db.messages], async () => {
for (const item of data) {
const { conv, messages } = item;
const existing = await db.conversations.get(conv.id);
if (existing) {
console.warn(`Conversation "${conv.name}" already exists, skipping...`);
skippedCount++;
continue;
}
await db.conversations.add(conv);
for (const msg of messages) {
await db.messages.put(msg);
}
importedCount++;
}
return { imported: importedCount, skipped: skippedCount };
});
}
}

99
tools/server/webui/src/lib/services/models.service.ts Normal file
View file

@ -0,0 +1,99 @@
import { ServerModelStatus } from '$lib/enums';
import { apiFetch, apiPost } from '$lib/utils/api-fetch';
export class ModelsService {
/**
*
*
* Listing
*
*
*/
/**
* Fetch list of models from OpenAI-compatible endpoint.
* Works in both MODEL and ROUTER modes.
*
* @returns List of available models with basic metadata
*/
static async list(): Promise<ApiModelListResponse> {
return apiFetch<ApiModelListResponse>('/v1/models');
}
/**
* Fetch list of all models with detailed metadata (ROUTER mode).
* Returns models with load status, paths, and other metadata
* beyond what the OpenAI-compatible endpoint provides.
*
* @returns List of models with detailed status and configuration info
*/
static async listRouter(): Promise<ApiRouterModelsListResponse> {
return apiFetch<ApiRouterModelsListResponse>('/v1/models');
}
/**
*
*
* Load/Unload
*
*
*/
/**
* Load a model (ROUTER mode only).
* Sends POST request to `/models/load`. Note: the endpoint returns success
* before loading completes — use polling to await actual load status.
*
* @param modelId - Model identifier to load
* @param extraArgs - Optional additional arguments to pass to the model instance
* @returns Load response from the server
*/
static async load(modelId: string, extraArgs?: string[]): Promise<ApiRouterModelsLoadResponse> {
const payload: { model: string; extra_args?: string[] } = { model: modelId };
if (extraArgs && extraArgs.length > 0) {
payload.extra_args = extraArgs;
}
return apiPost<ApiRouterModelsLoadResponse>('/models/load', payload);
}
/**
* Unload a model (ROUTER mode only).
* Sends POST request to `/models/unload`. Note: the endpoint returns success
* before unloading completes — use polling to await actual unload status.
*
* @param modelId - Model identifier to unload
* @returns Unload response from the server
*/
static async unload(modelId: string): Promise<ApiRouterModelsUnloadResponse> {
return apiPost<ApiRouterModelsUnloadResponse>('/models/unload', { model: modelId });
}
/**
*
*
* Status
*
*
*/
/**
* Check if a model is loaded based on its metadata.
*
* @param model - Model data entry from the API response
* @returns True if the model status is LOADED
*/
static isModelLoaded(model: ApiModelDataEntry): boolean {
return model.status.value === ServerModelStatus.LOADED;
}
/**
* Check if a model is currently loading.
*
* @param model - Model data entry from the API response
* @returns True if the model status is LOADING
*/
static isModelLoading(model: ApiModelDataEntry): boolean {
return model.status.value === ServerModelStatus.LOADING;
}
}

148
tools/server/webui/src/lib/services/parameter-sync.service.spec.ts Normal file
View file

@ -0,0 +1,148 @@
import { describe, it, expect } from 'vitest';
import { ParameterSyncService } from './parameter-sync.service';
describe('ParameterSyncService', () => {
describe('roundFloatingPoint', () => {
it('should fix JavaScript floating-point precision issues', () => {
// Test the specific values from the screenshot
const mockServerParams = {
top_p: 0.949999988079071,
min_p: 0.009999999776482582,
temperature: 0.800000011920929,
top_k: 40,
samplers: ['top_k', 'typ_p', 'top_p', 'min_p', 'temperature']
};
const result = ParameterSyncService.extractServerDefaults({
...mockServerParams,
// Add other required fields to match the API type
n_predict: 512,
seed: -1,
dynatemp_range: 0.0,
dynatemp_exponent: 1.0,
xtc_probability: 0.0,
xtc_threshold: 0.1,
typ_p: 1.0,
repeat_last_n: 64,
repeat_penalty: 1.0,
presence_penalty: 0.0,
frequency_penalty: 0.0,
dry_multiplier: 0.0,
dry_base: 1.75,
dry_allowed_length: 2,
dry_penalty_last_n: -1,
mirostat: 0,
mirostat_tau: 5.0,
mirostat_eta: 0.1,
stop: [],
max_tokens: -1,
n_keep: 0,
n_discard: 0,
ignore_eos: false,
stream: true,
logit_bias: [],
n_probs: 0,
min_keep: 0,
grammar: '',
grammar_lazy: false,
grammar_triggers: [],
preserved_tokens: [],
chat_format: '',
reasoning_format: '',
reasoning_in_content: false,
thinking_forced_open: false,
'speculative.n_max': 0,
'speculative.n_min': 0,
'speculative.p_min': 0.0,
timings_per_token: false,
post_sampling_probs: false,
lora: [],
top_n_sigma: 0.0,
dry_sequence_breakers: []
} as ApiLlamaCppServerProps['default_generation_settings']['params']);
// Check that the problematic floating-point values are rounded correctly
expect(result.top_p).toBe(0.95);
expect(result.min_p).toBe(0.01);
expect(result.temperature).toBe(0.8);
expect(result.top_k).toBe(40); // Integer should remain unchanged
expect(result.samplers).toBe('top_k;typ_p;top_p;min_p;temperature');
});
it('should preserve non-numeric values', () => {
const mockServerParams = {
samplers: ['top_k', 'temperature'],
max_tokens: -1,
temperature: 0.7
};
const result = ParameterSyncService.extractServerDefaults({
...mockServerParams,
// Minimal required fields
n_predict: 512,
seed: -1,
dynatemp_range: 0.0,
dynatemp_exponent: 1.0,
top_k: 40,
top_p: 0.95,
min_p: 0.05,
xtc_probability: 0.0,
xtc_threshold: 0.1,
typ_p: 1.0,
repeat_last_n: 64,
repeat_penalty: 1.0,
presence_penalty: 0.0,
frequency_penalty: 0.0,
dry_multiplier: 0.0,
dry_base: 1.75,
dry_allowed_length: 2,
dry_penalty_last_n: -1,
mirostat: 0,
mirostat_tau: 5.0,
mirostat_eta: 0.1,
stop: [],
n_keep: 0,
n_discard: 0,
ignore_eos: false,
stream: true,
logit_bias: [],
n_probs: 0,
min_keep: 0,
grammar: '',
grammar_lazy: false,
grammar_triggers: [],
preserved_tokens: [],
chat_format: '',
reasoning_format: '',
reasoning_in_content: false,
thinking_forced_open: false,
'speculative.n_max': 0,
'speculative.n_min': 0,
'speculative.p_min': 0.0,
timings_per_token: false,
post_sampling_probs: false,
lora: [],
top_n_sigma: 0.0,
dry_sequence_breakers: []
} as ApiLlamaCppServerProps['default_generation_settings']['params']);
expect(result.samplers).toBe('top_k;temperature');
expect(result.max_tokens).toBe(-1);
expect(result.temperature).toBe(0.7);
});
it('should merge webui settings from props when provided', () => {
const result = ParameterSyncService.extractServerDefaults(null, {
pasteLongTextToFileLen: 0,
pdfAsImage: true,
renderUserContentAsMarkdown: false,
theme: 'dark'
});
expect(result.pasteLongTextToFileLen).toBe(0);
expect(result.pdfAsImage).toBe(true);
expect(result.renderUserContentAsMarkdown).toBe(false);
expect(result.theme).toBeUndefined();
});
});
});

400
tools/server/webui/src/lib/services/parameter-sync.service.ts Normal file
View file

@ -0,0 +1,400 @@
import { normalizeFloatingPoint } from '$lib/utils';
import { SyncableParameterType, ParameterSource } from '$lib/enums/settings';
type ParameterValue = string | number | boolean;
type ParameterRecord = Record<string, ParameterValue>;
interface ParameterInfo {
value: string | number | boolean;
source: ParameterSource;
serverDefault?: string | number | boolean;
userOverride?: string | number | boolean;
}
interface SyncableParameter {
key: string;
serverKey: string;
type: SyncableParameterType;
canSync: boolean;
}
/**
* Mapping of webui setting keys to server parameter keys.
* Only parameters listed here can be synced from the server `/props` endpoint.
* Each entry defines the webui key, corresponding server key, value type,
* and whether sync is enabled.
*/
export const SYNCABLE_PARAMETERS: SyncableParameter[] = [
{
key: 'temperature',
serverKey: 'temperature',
type: SyncableParameterType.NUMBER,
canSync: true
},
{ key: 'top_k', serverKey: 'top_k', type: SyncableParameterType.NUMBER, canSync: true },
{ key: 'top_p', serverKey: 'top_p', type: SyncableParameterType.NUMBER, canSync: true },
{ key: 'min_p', serverKey: 'min_p', type: SyncableParameterType.NUMBER, canSync: true },
{
key: 'dynatemp_range',
serverKey: 'dynatemp_range',
type: SyncableParameterType.NUMBER,
canSync: true
},
{
key: 'dynatemp_exponent',
serverKey: 'dynatemp_exponent',
type: SyncableParameterType.NUMBER,
canSync: true
},
{
key: 'xtc_probability',
serverKey: 'xtc_probability',
type: SyncableParameterType.NUMBER,
canSync: true
},
{
key: 'xtc_threshold',
serverKey: 'xtc_threshold',
type: SyncableParameterType.NUMBER,
canSync: true
},
{ key: 'typ_p', serverKey: 'typ_p', type: SyncableParameterType.NUMBER, canSync: true },
{
key: 'repeat_last_n',
serverKey: 'repeat_last_n',
type: SyncableParameterType.NUMBER,
canSync: true
},
{
key: 'repeat_penalty',
serverKey: 'repeat_penalty',
type: SyncableParameterType.NUMBER,
canSync: true
},
{
key: 'presence_penalty',
serverKey: 'presence_penalty',
type: SyncableParameterType.NUMBER,
canSync: true
},
{
key: 'frequency_penalty',
serverKey: 'frequency_penalty',
type: SyncableParameterType.NUMBER,
canSync: true
},
{
key: 'dry_multiplier',
serverKey: 'dry_multiplier',
type: SyncableParameterType.NUMBER,
canSync: true
},
{ key: 'dry_base', serverKey: 'dry_base', type: SyncableParameterType.NUMBER, canSync: true },
{
key: 'dry_allowed_length',
serverKey: 'dry_allowed_length',
type: SyncableParameterType.NUMBER,
canSync: true
},
{
key: 'dry_penalty_last_n',
serverKey: 'dry_penalty_last_n',
type: SyncableParameterType.NUMBER,
canSync: true
},
{ key: 'max_tokens', serverKey: 'max_tokens', type: SyncableParameterType.NUMBER, canSync: true },
{ key: 'samplers', serverKey: 'samplers', type: SyncableParameterType.STRING, canSync: true },
{
key: 'pasteLongTextToFileLen',
serverKey: 'pasteLongTextToFileLen',
type: SyncableParameterType.NUMBER,
canSync: true
},
{
key: 'pdfAsImage',
serverKey: 'pdfAsImage',
type: SyncableParameterType.BOOLEAN,
canSync: true
},
{
key: 'showThoughtInProgress',
serverKey: 'showThoughtInProgress',
type: SyncableParameterType.BOOLEAN,
canSync: true
},
{
key: 'keepStatsVisible',
serverKey: 'keepStatsVisible',
type: SyncableParameterType.BOOLEAN,
canSync: true
},
{
key: 'showMessageStats',
serverKey: 'showMessageStats',
type: SyncableParameterType.BOOLEAN,
canSync: true
},
{
key: 'askForTitleConfirmation',
serverKey: 'askForTitleConfirmation',
type: SyncableParameterType.BOOLEAN,
canSync: true
},
{
key: 'disableAutoScroll',
serverKey: 'disableAutoScroll',
type: SyncableParameterType.BOOLEAN,
canSync: true
},
{
key: 'renderUserContentAsMarkdown',
serverKey: 'renderUserContentAsMarkdown',
type: SyncableParameterType.BOOLEAN,
canSync: true
},
{
key: 'autoMicOnEmpty',
serverKey: 'autoMicOnEmpty',
type: SyncableParameterType.BOOLEAN,
canSync: true
},
{
key: 'pyInterpreterEnabled',
serverKey: 'pyInterpreterEnabled',
type: SyncableParameterType.BOOLEAN,
canSync: true
},
{
key: 'enableContinueGeneration',
serverKey: 'enableContinueGeneration',
type: SyncableParameterType.BOOLEAN,
canSync: true
}
];
export class ParameterSyncService {
/**
*
*
* Extraction
*
*
*/
/**
* Round floating-point numbers to avoid JavaScript precision issues.
* E.g., 0.1 + 0.2 = 0.30000000000000004 → 0.3
*
* @param value - Parameter value to normalize
* @returns Precision-normalized value
*/
private static roundFloatingPoint(value: ParameterValue): ParameterValue {
return normalizeFloatingPoint(value) as ParameterValue;
}
/**
* Extract server default parameters that can be synced from `/props` response.
* Handles both generation settings parameters and webui-specific settings.
* Converts samplers array to semicolon-delimited string for UI display.
*
* @param serverParams - Raw generation settings from server `/props` endpoint
* @param webuiSettings - Optional webui-specific settings from server
* @returns Record of extracted parameter key-value pairs with normalized precision
*/
static extractServerDefaults(
serverParams: ApiLlamaCppServerProps['default_generation_settings']['params'] | null,
webuiSettings?: Record<string, string | number | boolean>
): ParameterRecord {
const extracted: ParameterRecord = {};
if (serverParams) {
for (const param of SYNCABLE_PARAMETERS) {
if (param.canSync && param.serverKey in serverParams) {
const value = (serverParams as unknown as Record<string, ParameterValue>)[
param.serverKey
];
if (value !== undefined) {
// Apply precision rounding to avoid JavaScript floating-point issues
extracted[param.key] = this.roundFloatingPoint(value);
}
}
}
// Handle samplers array conversion to string
if (serverParams.samplers && Array.isArray(serverParams.samplers)) {
extracted.samplers = serverParams.samplers.join(';');
}
}
if (webuiSettings) {
for (const param of SYNCABLE_PARAMETERS) {
if (param.canSync && param.serverKey in webuiSettings) {
const value = webuiSettings[param.serverKey];
if (value !== undefined) {
extracted[param.key] = this.roundFloatingPoint(value);
}
}
}
}
return extracted;
}
/**
*
*
* Merging
*
*
*/
/**
* Merge server defaults with current user settings.
* User overrides always take priority — only parameters not in `userOverrides`
* set will be updated from server defaults.
*
* @param currentSettings - Current parameter values in the settings store
* @param serverDefaults - Default values extracted from server props
* @param userOverrides - Set of parameter keys explicitly overridden by the user
* @returns Merged parameter record with user overrides preserved
*/
static mergeWithServerDefaults(
currentSettings: ParameterRecord,
serverDefaults: ParameterRecord,
userOverrides: Set<string> = new Set()
): ParameterRecord {
const merged = { ...currentSettings };
for (const [key, serverValue] of Object.entries(serverDefaults)) {
// Only update if user hasn't explicitly overridden this parameter
if (!userOverrides.has(key)) {
merged[key] = this.roundFloatingPoint(serverValue);
}
}
return merged;
}
/**
*
*
* Info
*
*
*/
/**
* Get parameter information including source and values.
* Used by ChatSettingsParameterSourceIndicator to display the correct badge
* (Custom vs Default) for each parameter in the settings UI.
*
* @param key - The parameter key to get info for
* @param currentValue - The current value of the parameter
* @param propsDefaults - Server default values from `/props`
* @param userOverrides - Set of parameter keys explicitly overridden by the user
* @returns Parameter info with source, server default, and user override values
*/
static getParameterInfo(
key: string,
currentValue: ParameterValue,
propsDefaults: ParameterRecord,
userOverrides: Set<string>
): ParameterInfo {
const hasPropsDefault = propsDefaults[key] !== undefined;
const isUserOverride = userOverrides.has(key);
// Simple logic: either using default (from props) or custom (user override)
const source = isUserOverride ? ParameterSource.CUSTOM : ParameterSource.DEFAULT;
return {
value: currentValue,
source,
serverDefault: hasPropsDefault ? propsDefaults[key] : undefined, // Keep same field name for compatibility
userOverride: isUserOverride ? currentValue : undefined
};
}
/**
* Check if a parameter can be synced from server.
*
* @param key - The parameter key to check
* @returns True if the parameter is in the syncable parameters list
*/
static canSyncParameter(key: string): boolean {
return SYNCABLE_PARAMETERS.some((param) => param.key === key && param.canSync);
}
/**
* Get all syncable parameter keys.
*
* @returns Array of parameter keys that can be synced from server
*/
static getSyncableParameterKeys(): string[] {
return SYNCABLE_PARAMETERS.filter((param) => param.canSync).map((param) => param.key);
}
/**
* Validate a server parameter value against its expected type.
*
* @param key - The parameter key to validate
* @param value - The value to validate
* @returns True if value matches the expected type for this parameter
*/
static validateServerParameter(key: string, value: ParameterValue): boolean {
const param = SYNCABLE_PARAMETERS.find((p) => p.key === key);
if (!param) return false;
switch (param.type) {
case SyncableParameterType.NUMBER:
return typeof value === 'number' && !isNaN(value);
case SyncableParameterType.STRING:
return typeof value === 'string';
case SyncableParameterType.BOOLEAN:
return typeof value === 'boolean';
default:
return false;
}
}
/**
*
*
* Diff
*
*
*/
/**
* Create a diff between current settings and server defaults.
* Shows which parameters differ from server values, useful for debugging
* and for the "Reset to defaults" functionality.
*
* @param currentSettings - Current parameter values in the settings store
* @param serverDefaults - Default values extracted from server props
* @returns Record of parameter diffs with current value, server value, and whether they differ
*/
static createParameterDiff(
currentSettings: ParameterRecord,
serverDefaults: ParameterRecord
): Record<string, { current: ParameterValue; server: ParameterValue; differs: boolean }> {
const diff: Record<
string,
{ current: ParameterValue; server: ParameterValue; differs: boolean }
> = {};
for (const key of this.getSyncableParameterKeys()) {
const currentValue = currentSettings[key];
const serverValue = serverDefaults[key];
if (serverValue !== undefined) {
diff[key] = {
current: currentValue,
server: serverValue,
differs: currentValue !== serverValue
};
}
}
return diff;
}
}

47
tools/server/webui/src/lib/services/props.service.ts Normal file
View file

@ -0,0 +1,47 @@
import { apiFetchWithParams } from '$lib/utils/api-fetch';
export class PropsService {
/**
*
*
* Fetching
*
*
*/
/**
* Fetches global server properties from the `/props` endpoint.
* In MODEL mode, returns modalities for the single loaded model.
* In ROUTER mode, returns server-wide settings without model-specific modalities.
*
* @param autoload - If false, prevents automatic model loading (default: false)
* @returns Server properties including default generation settings and capabilities
* @throws {Error} If the request fails or returns invalid data
*/
static async fetch(autoload = false): Promise<ApiLlamaCppServerProps> {
const params: Record<string, string> = {};
if (!autoload) {
params.autoload = 'false';
}
return apiFetchWithParams<ApiLlamaCppServerProps>('./props', params, { authOnly: true });
}
/**
* Fetches server properties for a specific model (ROUTER mode only).
* Required in ROUTER mode because global `/props` does not include per-model modalities.
*
* @param modelId - The model ID to fetch properties for
* @param autoload - If false, prevents automatic model loading (default: false)
* @returns Server properties specific to the requested model
* @throws {Error} If the request fails, model not found, or model not loaded
*/
static async fetchForModel(modelId: string, autoload = false): Promise<ApiLlamaCppServerProps> {
const params: Record<string, string> = { model: modelId };
if (!autoload) {
params.autoload = 'false';
}
return apiFetchWithParams<ApiLlamaCppServerProps>('./props', params, { authOnly: true });
}
}

216
tools/server/webui/src/lib/stores/chat.svelte.ts
View file

@ -15,6 +15,7 @@ import {
} from '$lib/utils';
import { SvelteMap } from 'svelte/reactivity';
import { DEFAULT_CONTEXT } from '$lib/constants/default-context';
import { SYSTEM_MESSAGE_PLACEHOLDER } from '$lib/constants/ui';
/**
* chatStore - Active AI interaction and streaming state management
@ -76,6 +77,10 @@ class ChatStore {
private isStreamingActive = $state(false);
private isEditModeActive = $state(false);
private addFilesHandler: ((files: File[]) => void) | null = $state(null);
pendingEditMessageId = $state<string | null>(null);
// Draft preservation for navigation (e.g., when adding system prompt from welcome page)
private _pendingDraftMessage = $state<string>('');
private _pendingDraftFiles = $state<ChatUploadedFile[]>([]);
// ─────────────────────────────────────────────────────────────────────────────
// Loading State
@ -455,6 +460,166 @@ class ChatStore {
}
}
/**
* Adds a system message at the top of a conversation and triggers edit mode.
* The system message is inserted between root and the first message of the active branch.
* Creates a new conversation if one doesn't exist.
*/
async addSystemPrompt(): Promise<void> {
let activeConv = conversationsStore.activeConversation;
// Create conversation if needed
if (!activeConv) {
await conversationsStore.createConversation();
activeConv = conversationsStore.activeConversation;
}
if (!activeConv) return;
try {
// Get all messages to find the root
const allMessages = await conversationsStore.getConversationMessages(activeConv.id);
const rootMessage = allMessages.find((m) => m.type === 'root' && m.parent === null);
let rootId: string;
// Create root message if it doesn't exist
if (!rootMessage) {
rootId = await DatabaseService.createRootMessage(activeConv.id);
} else {
rootId = rootMessage.id;
}
// Check if there's already a system message as root's child
const existingSystemMessage = allMessages.find(
(m) => m.role === 'system' && m.parent === rootId
);
if (existingSystemMessage) {
// If system message exists, just trigger edit mode on it
this.pendingEditMessageId = existingSystemMessage.id;
// Make sure it's in active messages at the beginning
if (!conversationsStore.activeMessages.some((m) => m.id === existingSystemMessage.id)) {
conversationsStore.activeMessages.unshift(existingSystemMessage);
}
return;
}
// Find the first message of the active branch (child of root that's in activeMessages)
const activeMessages = conversationsStore.activeMessages;
const firstActiveMessage = activeMessages.find((m) => m.parent === rootId);
// Create new system message with placeholder content (will be edited by user)
const systemMessage = await DatabaseService.createSystemMessage(
activeConv.id,
SYSTEM_MESSAGE_PLACEHOLDER,
rootId
);
// If there's a first message in the active branch, re-parent it to the system message
if (firstActiveMessage) {
// Update the first message's parent to be the system message
await DatabaseService.updateMessage(firstActiveMessage.id, {
parent: systemMessage.id
});
// Update the system message's children to include the first message
await DatabaseService.updateMessage(systemMessage.id, {
children: [firstActiveMessage.id]
});
// Remove first message from root's children
const updatedRootChildren = rootMessage
? rootMessage.children.filter((id: string) => id !== firstActiveMessage.id)
: [];
// Note: system message was already added to root's children by createSystemMessage
await DatabaseService.updateMessage(rootId, {
children: [
...updatedRootChildren.filter((id: string) => id !== systemMessage.id),
systemMessage.id
]
});
// Update local state
const firstMsgIndex = conversationsStore.findMessageIndex(firstActiveMessage.id);
if (firstMsgIndex !== -1) {
conversationsStore.updateMessageAtIndex(firstMsgIndex, { parent: systemMessage.id });
}
}
// Add system message to active messages at the beginning
conversationsStore.activeMessages.unshift(systemMessage);
// Set pending edit message ID to trigger edit mode
this.pendingEditMessageId = systemMessage.id;
conversationsStore.updateConversationTimestamp();
} catch (error) {
console.error('Failed to add system prompt:', error);
}
}
/**
* Removes a system message placeholder without deleting its children.
* Re-parents children back to the root message.
* If this is a new empty conversation (only root + system placeholder), deletes the entire conversation.
* @returns true if the entire conversation was deleted, false otherwise
*/
async removeSystemPromptPlaceholder(messageId: string): Promise<boolean> {
const activeConv = conversationsStore.activeConversation;
if (!activeConv) return false;
try {
const allMessages = await conversationsStore.getConversationMessages(activeConv.id);
const systemMessage = allMessages.find((m) => m.id === messageId);
if (!systemMessage || systemMessage.role !== 'system') return false;
const rootMessage = allMessages.find((m) => m.type === 'root' && m.parent === null);
if (!rootMessage) return false;
// Check if this is a new empty conversation (only root + system placeholder)
const isEmptyConversation = allMessages.length === 2 && systemMessage.children.length === 0;
if (isEmptyConversation) {
// Delete the entire conversation
await conversationsStore.deleteConversation(activeConv.id);
return true;
}
// Re-parent system message's children to root
for (const childId of systemMessage.children) {
await DatabaseService.updateMessage(childId, { parent: rootMessage.id });
// Update local state
const childIndex = conversationsStore.findMessageIndex(childId);
if (childIndex !== -1) {
conversationsStore.updateMessageAtIndex(childIndex, { parent: rootMessage.id });
}
}
// Update root's children: remove system message, add system's children
const newRootChildren = [
...rootMessage.children.filter((id: string) => id !== messageId),
...systemMessage.children
];
await DatabaseService.updateMessage(rootMessage.id, { children: newRootChildren });
// Delete the system message (without cascade)
await DatabaseService.deleteMessage(messageId);
// Remove from active messages
const systemIndex = conversationsStore.findMessageIndex(messageId);
if (systemIndex !== -1) {
conversationsStore.activeMessages.splice(systemIndex, 1);
}
conversationsStore.updateConversationTimestamp();
return false;
} catch (error) {
console.error('Failed to remove system prompt placeholder:', error);
return false;
}
}
private async createAssistantMessage(parentId?: string): Promise<DatabaseMessage | null> {
const activeConv = conversationsStore.activeConversation;
if (!activeConv) return null;
@ -916,6 +1081,28 @@ class ChatStore {
if (!activeConv)
return { totalCount: 0, userMessages: 0, assistantMessages: 0, messageTypes: [] };
const allMessages = await conversationsStore.getConversationMessages(activeConv.id);
const messageToDelete = allMessages.find((m) => m.id === messageId);
// For system messages, don't count descendants as they will be preserved (reparented to root)
if (messageToDelete?.role === 'system') {
const messagesToDelete = allMessages.filter((m) => m.id === messageId);
let userMessages = 0,
assistantMessages = 0;
const messageTypes: string[] = [];
for (const msg of messagesToDelete) {
if (msg.role === 'user') {
userMessages++;
if (!messageTypes.includes('user message')) messageTypes.push('user message');
} else if (msg.role === 'assistant') {
assistantMessages++;
if (!messageTypes.includes('assistant response')) messageTypes.push('assistant response');
}
}
return { totalCount: 1, userMessages, assistantMessages, messageTypes };
}
const descendants = findDescendantMessages(allMessages, messageId);
const allToDelete = [messageId, ...descendants];
const messagesToDelete = allMessages.filter((m) => allToDelete.includes(m.id));
@ -1381,6 +1568,31 @@ class ChatStore {
return this.addFilesHandler;
}
savePendingDraft(message: string, files: ChatUploadedFile[]): void {
this._pendingDraftMessage = message;
this._pendingDraftFiles = [...files];
}
consumePendingDraft(): { message: string; files: ChatUploadedFile[] } | null {
if (!this._pendingDraftMessage && this._pendingDraftFiles.length === 0) {
return null;
}
const draft = {
message: this._pendingDraftMessage,
files: [...this._pendingDraftFiles]
};
this._pendingDraftMessage = '';
this._pendingDraftFiles = [];
return draft;
}
hasPendingDraft(): boolean {
return Boolean(this._pendingDraftMessage) || this._pendingDraftFiles.length > 0;
}
public getAllLoadingChats(): string[] {
return Array.from(this.chatLoadingStates.keys());
}
@ -1485,3 +1697,7 @@ export const isEditing = () => chatStore.isEditing();
export const isLoading = () => chatStore.isLoading;
export const setEditModeActive = (handler: (files: File[]) => void) =>
chatStore.setEditModeActive(handler);
export const pendingEditMessageId = () => chatStore.pendingEditMessageId;
export const clearPendingEditMessageId = () => (chatStore.pendingEditMessageId = null);
export const removeSystemPromptPlaceholder = (messageId: string) =>
chatStore.removeSystemPromptPlaceholder(messageId);

26
tools/server/webui/src/lib/types/api.d.ts vendored
View file

@ -1,8 +1,19 @@
import type { ServerModelStatus, ServerRole } from '$lib/enums';
import type { ChatMessagePromptProgress } from './chat';
import type { ContentPartType, ServerModelStatus, ServerRole } from '$lib/enums';
import type { ChatMessagePromptProgress, ChatRole } from './chat';
export interface ApiChatCompletionToolFunction {
name: string;
description?: string;
parameters: Record<string, unknown>;
}
export interface ApiChatCompletionTool {
type: 'function';
function: ApiChatCompletionToolFunction;
}
export interface ApiChatMessageContentPart {
type: 'text' | 'image_url' | 'input_audio';
type: ContentPartType;
text?: string;
image_url?: {
url: string;
@ -34,6 +45,8 @@ export interface ApiErrorResponse {
export interface ApiChatMessageData {
role: ChatRole;
content: string | ApiChatMessageContentPart[];
tool_calls?: ApiChatCompletionToolCall[];
tool_call_id?: string;
timestamp?: number;
}
@ -188,6 +201,7 @@ export interface ApiChatCompletionRequest {
stream?: boolean;
model?: string;
return_progress?: boolean;
tools?: ApiChatCompletionTool[];
// Reasoning parameters
reasoning_format?: string;
// Generation parameters
@ -247,6 +261,7 @@ export interface ApiChatCompletionStreamChunk {
model?: string;
tool_calls?: ApiChatCompletionToolCallDelta[];
};
finish_reason?: string | null;
}>;
timings?: {
prompt_n?: number;
@ -267,8 +282,9 @@ export interface ApiChatCompletionResponse {
content: string;
reasoning_content?: string;
model?: string;
tool_calls?: ApiChatCompletionToolCallDelta[];
tool_calls?: ApiChatCompletionToolCall[];
};
finish_reason?: string | null;
}>;
}
@ -335,7 +351,7 @@ export interface ApiProcessingState {
tokensDecoded: number;
tokensRemaining: number;
contextUsed: number;
contextTotal: number;
contextTotal: number | null;
outputTokensUsed: number; // Total output tokens (thinking + regular content)
outputTokensMax: number; // Max output tokens allowed
temperature: number;

12
tools/server/webui/src/lib/types/models.d.ts vendored
View file

@ -1,8 +1,5 @@
import type { ApiModelDataEntry, ApiModelDetails } from '$lib/types/api';
/**
* Model modalities - vision and audio capabilities
*/
export interface ModelModalities {
vision: boolean;
audio: boolean;
@ -14,8 +11,15 @@ export interface ModelOption {
model: string;
description?: string;
capabilities: string[];
/** Model modalities from /props endpoint */
modalities?: ModelModalities;
details?: ApiModelDetails['details'];
meta?: ApiModelDataEntry['meta'];
}
/**
* Modality capabilities for file validation
*/
export interface ModalityCapabilities {
hasVision: boolean;
hasAudio: boolean;
}

151
tools/server/webui/src/lib/utils/abort.ts Normal file
View file

@ -0,0 +1,151 @@
/**
* Abort Signal Utilities
*
* Provides utilities for consistent AbortSignal propagation across the application.
* These utilities help ensure that async operations can be properly cancelled
* when needed (e.g., user stops generation, navigates away, etc.).
*/
/**
* Throws an AbortError if the signal is aborted.
* Use this at the start of async operations to fail fast.
*
* @param signal - Optional AbortSignal to check
* @throws DOMException with name 'AbortError' if signal is aborted
*
* @example
* ```ts
* async function fetchData(signal?: AbortSignal) {
* throwIfAborted(signal);
* // ... proceed with operation
* }
* ```
*/
export function throwIfAborted(signal?: AbortSignal): void {
if (signal?.aborted) {
throw new DOMException('Operation was aborted', 'AbortError');
}
}
/**
* Checks if an error is an AbortError.
* Use this to distinguish between user-initiated cancellation and actual errors.
*
* @param error - Error to check
* @returns true if the error is an AbortError
*
* @example
* ```ts
* try {
* await fetchData(signal);
* } catch (error) {
* if (isAbortError(error)) {
* // User cancelled - no error dialog needed
* return;
* }
* // Handle actual error
* }
* ```
*/
export function isAbortError(error: unknown): boolean {
if (error instanceof DOMException && error.name === 'AbortError') {
return true;
}
if (error instanceof Error && error.name === 'AbortError') {
return true;
}
return false;
}
/**
* Creates a new AbortController that is linked to one or more parent signals.
* When any parent signal aborts, the returned controller also aborts.
*
* Useful for creating child operations that should be cancelled when
* either the parent operation or their own timeout/condition triggers.
*
* @param signals - Parent signals to link to (undefined signals are ignored)
* @returns A new AbortController linked to all provided signals
*
* @example
* ```ts
* // Link to user's abort signal and add a timeout
* const linked = createLinkedController(userSignal, timeoutSignal);
* await fetch(url, { signal: linked.signal });
* ```
*/
export function createLinkedController(...signals: (AbortSignal | undefined)[]): AbortController {
const controller = new AbortController();
for (const signal of signals) {
if (!signal) continue;
// If already aborted, abort immediately
if (signal.aborted) {
controller.abort(signal.reason);
return controller;
}
// Link to parent signal
signal.addEventListener('abort', () => controller.abort(signal.reason), { once: true });
}
return controller;
}
/**
* Creates an AbortSignal that times out after the specified duration.
*
* @param ms - Timeout duration in milliseconds
* @returns AbortSignal that will abort after the timeout
*
* @example
* ```ts
* const signal = createTimeoutSignal(5000); // 5 second timeout
* await fetch(url, { signal });
* ```
*/
export function createTimeoutSignal(ms: number): AbortSignal {
return AbortSignal.timeout(ms);
}
/**
* Wraps a promise to reject if the signal is aborted.
* Useful for making non-abortable promises respect an AbortSignal.
*
* @param promise - Promise to wrap
* @param signal - AbortSignal to respect
* @returns Promise that rejects with AbortError if signal aborts
*
* @example
* ```ts
* // Make a non-abortable operation respect abort signal
* const result = await withAbortSignal(
* someNonAbortableOperation(),
* signal
* );
* ```
*/
export async function withAbortSignal<T>(promise: Promise<T>, signal?: AbortSignal): Promise<T> {
if (!signal) return promise;
throwIfAborted(signal);
return new Promise<T>((resolve, reject) => {
const abortHandler = () => {
reject(new DOMException('Operation was aborted', 'AbortError'));
};
signal.addEventListener('abort', abortHandler, { once: true });
promise
.then((value) => {
signal.removeEventListener('abort', abortHandler);
resolve(value);
})
.catch((error) => {
signal.removeEventListener('abort', abortHandler);
reject(error);
});
});
}

154
tools/server/webui/src/lib/utils/api-fetch.ts Normal file
View file

@ -0,0 +1,154 @@
import { base } from '$app/paths';
import { getJsonHeaders, getAuthHeaders } from './api-headers';
/**
* API Fetch Utilities
*
* Provides common fetch patterns used across services:
* - Automatic JSON headers
* - Error handling with proper error messages
* - Base path resolution
*/
export interface ApiFetchOptions extends Omit<RequestInit, 'headers'> {
/**
* Use auth-only headers (no Content-Type).
* Default: false (uses JSON headers with Content-Type: application/json)
*/
authOnly?: boolean;
/**
* Additional headers to merge with default headers.
*/
headers?: Record<string, string>;
}
/**
* Fetch JSON data from an API endpoint with standard headers and error handling.
*
* @param path - API path (will be prefixed with base path)
* @param options - Fetch options with additional authOnly flag
* @returns Parsed JSON response
* @throws Error with formatted message on failure
*
* @example
* ```typescript
* // GET request
* const models = await apiFetch<ApiModelListResponse>('/v1/models');
*
* // POST request
* const result = await apiFetch<ApiResponse>('/models/load', {
* method: 'POST',
* body: JSON.stringify({ model: 'gpt-4' })
* });
* ```
*/
export async function apiFetch<T>(path: string, options: ApiFetchOptions = {}): Promise<T> {
const { authOnly = false, headers: customHeaders, ...fetchOptions } = options;
const baseHeaders = authOnly ? getAuthHeaders() : getJsonHeaders();
const headers = { ...baseHeaders, ...customHeaders };
const url = path.startsWith('http://') || path.startsWith('https://') ? path : `${base}${path}`;
const response = await fetch(url, {
...fetchOptions,
headers
});
if (!response.ok) {
const errorMessage = await parseErrorMessage(response);
throw new Error(errorMessage);
}
return response.json() as Promise<T>;
}
/**
* Fetch with URL constructed from base URL and query parameters.
*
* @param basePath - Base API path
* @param params - Query parameters to append
* @param options - Fetch options
* @returns Parsed JSON response
*
* @example
* ```typescript
* const props = await apiFetchWithParams<ApiProps>('./props', {
* model: 'gpt-4',
* autoload: 'false'
* });
* ```
*/
export async function apiFetchWithParams<T>(
basePath: string,
params: Record<string, string>,
options: ApiFetchOptions = {}
): Promise<T> {
const url = new URL(basePath, window.location.href);
for (const [key, value] of Object.entries(params)) {
if (value !== undefined && value !== null) {
url.searchParams.set(key, value);
}
}
const { authOnly = false, headers: customHeaders, ...fetchOptions } = options;
const baseHeaders = authOnly ? getAuthHeaders() : getJsonHeaders();
const headers = { ...baseHeaders, ...customHeaders };
const response = await fetch(url.toString(), {
...fetchOptions,
headers
});
if (!response.ok) {
const errorMessage = await parseErrorMessage(response);
throw new Error(errorMessage);
}
return response.json() as Promise<T>;
}
/**
* POST JSON data to an API endpoint.
*
* @param path - API path
* @param body - Request body (will be JSON stringified)
* @param options - Additional fetch options
* @returns Parsed JSON response
*/
export async function apiPost<T, B = unknown>(
path: string,
body: B,
options: ApiFetchOptions = {}
): Promise<T> {
return apiFetch<T>(path, {
method: 'POST',
body: JSON.stringify(body),
...options
});
}
/**
* Parse error message from a failed response.
* Tries to extract error message from JSON body, falls back to status text.
*/
async function parseErrorMessage(response: Response): Promise<string> {
try {
const errorData = await response.json();
if (errorData?.error?.message) {
return errorData.error.message;
}
if (errorData?.error && typeof errorData.error === 'string') {
return errorData.error;
}
if (errorData?.message) {
return errorData.message;
}
} catch {
// JSON parsing failed, use status text
}
return `Request failed: ${response.status} ${response.statusText}`;
}

11
tools/server/webui/src/lib/utils/branching.ts
View file

@ -15,6 +15,8 @@
* └── message 5 (assistant)
*/
import { MessageRole } from '$lib/enums/chat';
/**
* Filters messages to get the conversation path from root to a specific leaf node.
* If the leafNodeId doesn't exist, returns the path with the latest timestamp.
@ -65,8 +67,13 @@ export function filterByLeafNodeId(
currentNode = nodeMap.get(currentNode.parent);
}
// Sort by timestamp to get chronological order (root to leaf)
result.sort((a, b) => a.timestamp - b.timestamp);
// Sort: system messages first, then by timestamp
result.sort((a, b) => {
if (a.role === MessageRole.SYSTEM && b.role !== MessageRole.SYSTEM) return -1;
if (a.role !== MessageRole.SYSTEM && b.role === MessageRole.SYSTEM) return 1;
return a.timestamp - b.timestamp;
});
return result;
}

Some files were not shown because too many files have changed in this diff Show more