thek0tyara/llama-cpp-turboquant
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

Merge signed commit 'b8192' into sisyphus

Browse source 
Diff-After-Merge: 2 files changed, 6 insertions(+)

# gpg: Signature made Tue Mar  3 12:40:26 2026 MSK
# gpg:                using RSA key B5690EEEBB952194
# gpg: Good signature from "GitHub <noreply@github.com>" [unknown]

# Conflicts:
#	tools/server/public/index.html.gz
This commit is contained in:
Vitaly Chikunov 2026-03-03 22:15:48 +00:00
commit c532b4731a
380 changed files with 27663 additions and 17630 deletions
Download patch file Download diff file Expand all files Collapse all files

10
tools/cli/README.md
View file

@ -57,8 +57,8 @@
| `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)<br/>(env: LLAMA_ARG_DEFRAG_THOLD) |
| `-np, --parallel N` | number of parallel sequences to decode (default: 1)<br/>(env: LLAMA_ARG_N_PARALLEL) |
| `--mlock` | force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) |
| `--mmap, --no-mmap` | whether to memory-map model. Explicitly enabling mmap disables direct-io. (if mmap disabled, slower load but may reduce pageouts if not using mlock) (default: enabled)<br/>(env: LLAMA_ARG_MMAP) |
| `-dio, --direct-io, -ndio, --no-direct-io` | use DirectIO if available. Takes precedence over --mmap (default: enabled)<br/>(env: LLAMA_ARG_DIO) |
| `--mmap, --no-mmap` | whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock) (default: enabled)<br/>(env: LLAMA_ARG_MMAP) |
| `-dio, --direct-io, -ndio, --no-direct-io` | use DirectIO if available. (default: disabled)<br/>(env: LLAMA_ARG_DIO) |
| `--numa TYPE` | attempt optimizations that help on some NUMA systems<br/>- distribute: spread execution evenly over all nodes<br/>- isolate: only spawn threads on CPUs on the node that execution started on<br/>- numactl: use the CPU map provided by numactl<br/>if run without this previously, it is recommended to drop the system page cache before using this<br/>see https://github.com/ggml-org/llama.cpp/issues/1437<br/>(env: LLAMA_ARG_NUMA) |
| `-dev, --device <dev1,dev2,..>` | comma-separated list of devices to use for offloading (none = don't offload)<br/>use --list-devices to see a list of available devices<br/>(env: LLAMA_ARG_DEVICE) |
| `--list-devices` | print list of available devices and exit |
@ -109,14 +109,14 @@
| `-s, --seed SEED` | RNG seed (default: -1, use random seed for -1) |
| `--sampler-seq, --sampling-seq SEQUENCE` | simplified sequence for samplers that will be used (default: edskypmxt) |
| `--ignore-eos` | ignore end of stream token and continue generating (implies --logit-bias EOS-inf) |
| `--temp N` | temperature (default: 0.80) |
| `--temp, --temperature N` | temperature (default: 0.80) |
| `--top-k N` | top-k sampling (default: 40, 0 = disabled)<br/>(env: LLAMA_ARG_TOP_K) |
| `--top-p N` | top-p sampling (default: 0.95, 1.0 = disabled) |
| `--min-p N` | min-p sampling (default: 0.05, 0.0 = disabled) |
| `--top-nsigma N` | top-n-sigma sampling (default: -1.00, -1.0 = disabled) |
| `--top-nsigma, --top-n-sigma N` | top-n-sigma sampling (default: -1.00, -1.0 = disabled) |
| `--xtc-probability N` | xtc probability (default: 0.00, 0.0 = disabled) |
| `--xtc-threshold N` | xtc threshold (default: 0.10, 1.0 = disabled) |
| `--typical N` | locally typical sampling, parameter p (default: 1.00, 1.0 = disabled) |
| `--typical, --typical-p N` | locally typical sampling, parameter p (default: 1.00, 1.0 = disabled) |
| `--repeat-last-n N` | last n tokens to consider for penalize (default: 64, 0 = disabled, -1 = ctx_size) |
| `--repeat-penalty N` | penalize repeat sequence of tokens (default: 1.00, 1.0 = disabled) |
| `--presence-penalty N` | repeat alpha presence penalty (default: 0.00, 0.0 = disabled) |

18
tools/cli/cli.cpp
View file

@ -52,6 +52,7 @@ struct cli_context {
json messages = json::array();
std::vector<raw_buffer> input_files;
task_params defaults;
bool verbose_prompt;
// thread for showing "loading" animation
std::atomic<bool> loading_show;
@ -66,6 +67,8 @@ struct cli_context {
defaults.stream = true; // make sure we always use streaming mode
defaults.timings_per_token = true; // in order to get timings even when we cancel mid-way
// defaults.return_progress = true; // TODO: show progress
verbose_prompt = params.verbose_prompt;
}
std::string generate_completion(result_timings & out_timings) {
@ -91,6 +94,12 @@ struct cli_context {
rd.post_task({std::move(task)});
}
if (verbose_prompt) {
console::set_display(DISPLAY_TYPE_PROMPT);
console::log("%s\n\n", chat_params.prompt.c_str());
console::set_display(DISPLAY_TYPE_RESET);
}
// wait for first result
console::spinner::start();
server_task_result_ptr result = rd.next(should_stop);
@ -371,6 +380,15 @@ int main(int argc, char ** argv) {
console::error("file does not exist or cannot be opened: '%s'\n", fname.c_str());
continue;
}
if (inf.fim_sep_token != LLAMA_TOKEN_NULL) {
cur_msg += common_token_to_piece(ctx_cli.ctx_server.get_llama_context(), inf.fim_sep_token, true);
cur_msg += fname;
cur_msg.push_back('\n');
} else {
cur_msg += "--- File: ";
cur_msg += fname;
cur_msg += " ---\n";
}
cur_msg += marker;
console::log("Loaded text from '%s'\n", fname.c_str());
continue;

10
tools/completion/README.md
View file

@ -140,8 +140,8 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1
| `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)<br/>(env: LLAMA_ARG_DEFRAG_THOLD) |
| `-np, --parallel N` | number of parallel sequences to decode (default: 1)<br/>(env: LLAMA_ARG_N_PARALLEL) |
| `--mlock` | force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) |
| `--mmap, --no-mmap` | whether to memory-map model. Explicitly enabling mmap disables direct-io. (if mmap disabled, slower load but may reduce pageouts if not using mlock) (default: enabled)<br/>(env: LLAMA_ARG_MMAP) |
| `-dio, --direct-io, -ndio, --no-direct-io` | use DirectIO if available. Takes precedence over --mmap (default: enabled)<br/>(env: LLAMA_ARG_DIO) |
| `--mmap, --no-mmap` | whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock) (default: enabled)<br/>(env: LLAMA_ARG_MMAP) |
| `-dio, --direct-io, -ndio, --no-direct-io` | use DirectIO if available. (default: disabled)<br/>(env: LLAMA_ARG_DIO) |
| `--numa TYPE` | attempt optimizations that help on some NUMA systems<br/>- distribute: spread execution evenly over all nodes<br/>- isolate: only spawn threads on CPUs on the node that execution started on<br/>- numactl: use the CPU map provided by numactl<br/>if run without this previously, it is recommended to drop the system page cache before using this<br/>see https://github.com/ggml-org/llama.cpp/issues/1437<br/>(env: LLAMA_ARG_NUMA) |
| `-dev, --device <dev1,dev2,..>` | comma-separated list of devices to use for offloading (none = don't offload)<br/>use --list-devices to see a list of available devices<br/>(env: LLAMA_ARG_DEVICE) |
| `--list-devices` | print list of available devices and exit |
@ -192,14 +192,14 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1
| `-s, --seed SEED` | RNG seed (default: -1, use random seed for -1) |
| `--sampler-seq, --sampling-seq SEQUENCE` | simplified sequence for samplers that will be used (default: edskypmxt) |
| `--ignore-eos` | ignore end of stream token and continue generating (implies --logit-bias EOS-inf) |
| `--temp N` | temperature (default: 0.80) |
| `--temp, --temperature N` | temperature (default: 0.80) |
| `--top-k N` | top-k sampling (default: 40, 0 = disabled)<br/>(env: LLAMA_ARG_TOP_K) |
| `--top-p N` | top-p sampling (default: 0.95, 1.0 = disabled) |
| `--min-p N` | min-p sampling (default: 0.05, 0.0 = disabled) |
| `--top-nsigma N` | top-n-sigma sampling (default: -1.00, -1.0 = disabled) |
| `--top-nsigma, --top-n-sigma N` | top-n-sigma sampling (default: -1.00, -1.0 = disabled) |
| `--xtc-probability N` | xtc probability (default: 0.00, 0.0 = disabled) |
| `--xtc-threshold N` | xtc threshold (default: 0.10, 1.0 = disabled) |
| `--typical N` | locally typical sampling, parameter p (default: 1.00, 1.0 = disabled) |
| `--typical, --typical-p N` | locally typical sampling, parameter p (default: 1.00, 1.0 = disabled) |
| `--repeat-last-n N` | last n tokens to consider for penalize (default: 64, 0 = disabled, -1 = ctx_size) |
| `--repeat-penalty N` | penalize repeat sequence of tokens (default: 1.00, 1.0 = disabled) |
| `--presence-penalty N` | repeat alpha presence penalty (default: 0.00, 0.0 = disabled) |

38
tools/completion/completion.cpp
View file

@ -387,6 +387,17 @@ int main(int argc, char ** argv) {
}
session_do_save = !path_session.empty() && n_match < embd_inp.size() && !params.prompt_cache_ro;
// Logits are not stored as part of the session state so we need to
// "replay" the last token to get logits for sampling.
if (!session_tokens.empty() && n_match > 0 && n_match == session_tokens.size()) {
if (!common_replay_last_token(ctx, session_tokens.back(), n_match)) {
return 1;
}
session_do_save = false;
LOG_INF("%s: replayed last token from session\n", __func__);
}
}
// number of tokens to keep when resetting context
@ -675,40 +686,27 @@ int main(int argc, char ** argv) {
}
if (!embd.empty()) {
int n_eval = (int) embd.size();
LOG_DBG("eval: %s\n", string_from(ctx, embd).c_str());
GGML_ASSERT(n_eval <= params.n_batch);
if (llama_decode(ctx, llama_batch_get_one(embd.data(), n_eval))) {
LOG_ERR("%s : failed to eval\n", __func__);
const bool is_last_batch = (n_consumed >= (int) embd_inp.size());
const bool save_now = session_do_save && is_last_batch;
if (!common_prompt_batch_decode(ctx, embd, n_past, params.n_batch, path_session, save_now)) {
return 1;
}
n_past += n_eval;
session_tokens.insert(session_tokens.end(), embd.begin(), embd.begin());
n_session_consumed = session_tokens.size();
session_do_save = false;
LOG_DBG("n_past = %d\n", n_past);
// Display total tokens alongside total time
if (params.n_print > 0 && n_past % params.n_print == 0) {
LOG_DBG("\n\033[31mTokens consumed so far = %d / %d \033[0m\n", n_past, n_ctx);
}
}
if (!embd.empty() && !path_session.empty()) {
session_tokens.insert(session_tokens.end(), embd.begin(), embd.end());
n_session_consumed = session_tokens.size();
}
}
embd.clear();
if ((int) embd_inp.size() <= n_consumed && !is_interacting) {
// optionally save the session on first sample (for faster prompt loading next time)
if (session_do_save) {
session_do_save = false;
llama_state_save_file(ctx, path_session.c_str(), session_tokens.data(), session_tokens.size());
LOG_DBG("saved session to %s\n", path_session.c_str());
}
const llama_token id = common_sampler_sample(smpl, ctx, -1);

4
tools/imatrix/imatrix.cpp
View file

@ -912,7 +912,9 @@ static bool compute_imatrix(llama_context * ctx, const common_params & params, c
const bool add_bos = llama_vocab_get_add_bos(vocab);
GGML_ASSERT(!llama_vocab_get_add_eos(vocab));
if (llama_pooling_type(ctx) != LLAMA_POOLING_TYPE_LAST) {
GGML_ASSERT(!llama_vocab_get_add_eos(vocab));
}
auto tim1 = std::chrono::high_resolution_clock::now();
LOG_INF("%s: tokenizing the input ..\n", __func__);

2
tools/mtmd/CMakeLists.txt
View file

@ -20,9 +20,11 @@ add_library(mtmd
models/internvl.cpp
models/kimivl.cpp
models/kimik25.cpp
models/nemotron-v2-vl.cpp
models/llama4.cpp
models/llava.cpp
models/minicpmv.cpp
models/paddleocr.cpp
models/pixtral.cpp
models/qwen2vl.cpp
models/qwen3vl.cpp

4
tools/mtmd/clip-impl.h
View file

@ -229,6 +229,7 @@ enum projector_type {
PROJECTOR_TYPE_MUSIC_FLAMINGO,
PROJECTOR_TYPE_LFM2,
PROJECTOR_TYPE_KIMIVL,
PROJECTOR_TYPE_PADDLEOCR,
PROJECTOR_TYPE_LIGHTONOCR,
PROJECTOR_TYPE_COGVLM,
PROJECTOR_TYPE_JANUS_PRO,
@ -236,6 +237,7 @@ enum projector_type {
PROJECTOR_TYPE_GLM4V,
PROJECTOR_TYPE_YOUTUVL,
PROJECTOR_TYPE_KIMIK25,
PROJECTOR_TYPE_NEMOTRON_V2_VL,
PROJECTOR_TYPE_UNKNOWN,
};
@ -263,6 +265,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_MUSIC_FLAMINGO, "musicflamingo"},
{ PROJECTOR_TYPE_LFM2, "lfm2"},
{ PROJECTOR_TYPE_KIMIVL, "kimivl"},
{ PROJECTOR_TYPE_PADDLEOCR, "paddleocr"},
{ PROJECTOR_TYPE_LIGHTONOCR,"lightonocr"},
{ PROJECTOR_TYPE_COGVLM, "cogvlm"},
{ PROJECTOR_TYPE_JANUS_PRO, "janus_pro"},
@ -270,6 +273,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_GLM4V, "glm4v"},
{ PROJECTOR_TYPE_YOUTUVL, "youtuvl"},
{ PROJECTOR_TYPE_KIMIK25, "kimik25"},
{ PROJECTOR_TYPE_NEMOTRON_V2_VL, "nemotron_v2_vl"},
};
static projector_type clip_projector_type_from_string(const std::string & str) {

1
tools/mtmd/clip-model.h
View file

@ -15,6 +15,7 @@ enum ffn_op_type {
FFN_GELU_ERF,
FFN_SILU,
FFN_GELU_QUICK,
FFN_RELU_SQR,
};
enum norm_type {

88
tools/mtmd/clip.cpp
View file

@ -342,9 +342,17 @@ ggml_tensor * clip_graph::build_vit(
/* nb2 */ cur->nb[1],
/* offset */ ggml_row_size(cur->type, 2 * n_embd));
// TODO: q/k norm requires row size == n_embd, while here it's d_head
// we can add support in the future if needed
GGML_ASSERT(layer.q_norm == nullptr && layer.k_norm == nullptr);
if (layer.q_norm) {
GGML_ASSERT(layer.q_norm->ne[0] == Qcur->ne[0]);
Qcur = build_norm(Qcur, layer.q_norm, NULL, norm_t, eps, il);
cb(Qcur, "Qcur_norm", il);
}
if (layer.k_norm) {
GGML_ASSERT(layer.k_norm->ne[0] == Kcur->ne[0]);
Kcur = build_norm(Kcur, layer.k_norm, NULL, norm_t, eps, il);
cb(Kcur, "Kcur_norm", il);
}
} else {
// separate q, k, v
@ -559,6 +567,12 @@ ggml_tensor * clip_graph::build_ffn(
cur = ggml_gelu_quick(ctx0, cur);
cb(cur, "ffn_gelu_quick", il);
} break;
case FFN_RELU_SQR:
{
cur = ggml_relu(ctx0, cur);
cur = ggml_sqr(ctx0, cur);
cb(cur, "ffn_relu_sqr", il);
} break;
}
if (down) {
@ -614,9 +628,6 @@ ggml_tensor * clip_graph::build_attn(
ggml_tensor * v = ggml_permute(ctx0, v_cur, 1, 2, 0, 3);
v = ggml_cont(ctx0, v);
const auto n_tokens = q->ne[1];
const auto n_head = q->ne[2];
ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);
// F32 may not needed for vision encoders?
// ggml_mul_mat_set_prec(kq, GGML_PREC_F32);
@ -625,7 +636,7 @@ ggml_tensor * clip_graph::build_attn(
ggml_tensor * kqv = ggml_mul_mat(ctx0, v, kq);
cur = ggml_permute(ctx0, kqv, 0, 2, 1, 3);
cur = ggml_cont_2d(ctx0, cur, cur->ne[0]*n_head, n_tokens);
cur = ggml_cont_2d(ctx0, cur, cur->ne[0] * cur->ne[1], cur->ne[2] * cur->ne[3]);
}
cb(cur, "kqv_out", il);
@ -810,6 +821,10 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32
{
builder = std::make_unique<clip_graph_internvl>(ctx, img);
} break;
case PROJECTOR_TYPE_NEMOTRON_V2_VL:
{
builder = std::make_unique<clip_graph_nemotron_v2_vl>(ctx, img);
} break;
case PROJECTOR_TYPE_LLAMA4:
{
builder = std::make_unique<clip_graph_llama4>(ctx, img);
@ -826,6 +841,10 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32
{
builder = std::make_unique<clip_graph_kimivl>(ctx, img);
} break;
case PROJECTOR_TYPE_PADDLEOCR:
{
builder = std::make_unique<clip_graph_paddleocr>(ctx, img);
} break;
case PROJECTOR_TYPE_KIMIK25:
{
builder = std::make_unique<clip_graph_kimik25>(ctx, img);
@ -1110,6 +1129,7 @@ struct clip_model_loader {
}
} break;
case PROJECTOR_TYPE_INTERNVL:
case PROJECTOR_TYPE_NEMOTRON_V2_VL:
{
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
} break;
@ -1240,6 +1260,14 @@ struct clip_model_loader {
hparams.audio_window_len = 400;
hparams.audio_hop_len = 160;
} break;
case PROJECTOR_TYPE_PADDLEOCR:
{
hparams.n_merge = 2;
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
hparams.set_warmup_n_tokens(28*28); // avoid OOM on warmup
} break;
case PROJECTOR_TYPE_LFM2A:
{
// audio preprocessing params
@ -1688,6 +1716,7 @@ struct clip_model_loader {
model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
} break;
case PROJECTOR_TYPE_KIMIVL:
case PROJECTOR_TYPE_PADDLEOCR:
case PROJECTOR_TYPE_KIMIK25:
{
model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM);
@ -1767,6 +1796,12 @@ struct clip_model_loader {
model.mm_3_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "weight"));
model.mm_3_b = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "bias"));
} break;
case PROJECTOR_TYPE_NEMOTRON_V2_VL:
{
model.mm_0_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "weight"));
model.mm_1_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight"));
model.mm_3_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "weight"));
} break;
case PROJECTOR_TYPE_GLMA:
{
model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight"));
@ -2968,6 +3003,7 @@ bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, str
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_QWEN3VL:
case PROJECTOR_TYPE_GLM4V:
case PROJECTOR_TYPE_PADDLEOCR:
{
GGML_ASSERT(params.image_min_pixels > 0 && params.image_max_pixels > 0);
clip_image_u8 resized;
@ -3088,6 +3124,7 @@ bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, str
case PROJECTOR_TYPE_GLM_EDGE:
case PROJECTOR_TYPE_GEMMA3:
case PROJECTOR_TYPE_INTERNVL: // TODO @ngxson : support dynamic resolution
case PROJECTOR_TYPE_NEMOTRON_V2_VL:
{
clip_image_u8 resized_image;
int sz = params.image_size;
@ -3307,6 +3344,7 @@ int clip_n_output_tokens_x(const struct clip_ctx * ctx, struct clip_image_f32 *
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_QWEN3VL:
case PROJECTOR_TYPE_GLM4V:
case PROJECTOR_TYPE_PADDLEOCR:
case PROJECTOR_TYPE_YOUTUVL:
return (img->nx / params.patch_size) / 2;
default:
@ -3323,6 +3361,7 @@ int clip_n_output_tokens_y(const struct clip_ctx * ctx, struct clip_image_f32 *
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_QWEN3VL:
case PROJECTOR_TYPE_GLM4V:
case PROJECTOR_TYPE_PADDLEOCR:
case PROJECTOR_TYPE_YOUTUVL:
return (img->ny / params.patch_size) / 2;
default:
@ -3397,6 +3436,7 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im
case PROJECTOR_TYPE_GEMMA3:
case PROJECTOR_TYPE_IDEFICS3:
case PROJECTOR_TYPE_INTERNVL:
case PROJECTOR_TYPE_NEMOTRON_V2_VL:
case PROJECTOR_TYPE_LLAMA4:
{
// both X and Y are downscaled by the scale factor
@ -3419,6 +3459,13 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im
int y_patch = CLIP_ALIGN(img->ny, out_patch_size) / out_patch_size;
n_patches = x_patch * y_patch;
} break;
case PROJECTOR_TYPE_PADDLEOCR:
{
// dynamic size
int n_merge = ctx->model.hparams.n_merge;
int stride = n_merge * n_merge;
n_patches = CLIP_ALIGN(n_patches, stride) / stride;
} break;
case PROJECTOR_TYPE_PIXTRAL:
case PROJECTOR_TYPE_LIGHTONOCR:
{
@ -3666,6 +3713,30 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
}
}
set_input_i32("positions", positions);
} break;
case PROJECTOR_TYPE_PADDLEOCR:
{
const int merge_ratio = hparams.n_merge;
const int pw = image_size_width / patch_size;
const int ph = image_size_height / patch_size;
std::vector<int> positions(n_pos * 4);
int ptr = 0;
// NOTE: same as Qwen-VL, but x and y are swapped
for (int y = 0; y < ph; y += merge_ratio) {
for (int dy = 0; dy < 2; dy++) {
for (int x = 0; x < pw; x += merge_ratio) {
for (int dx = 0; dx < 2; dx++) {
positions[ ptr] = y + dy;
positions[ num_patches + ptr] = x + dx;
positions[2 * num_patches + ptr] = y + dy;
positions[3 * num_patches + ptr] = x + dx;
ptr++;
}
}
}
}
set_input_i32("positions", positions);
} break;
case PROJECTOR_TYPE_QWEN25VL:
@ -3805,6 +3876,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
case PROJECTOR_TYPE_GEMMA3NV:
case PROJECTOR_TYPE_IDEFICS3:
case PROJECTOR_TYPE_INTERNVL:
case PROJECTOR_TYPE_NEMOTRON_V2_VL:
case PROJECTOR_TYPE_QWEN2A:
case PROJECTOR_TYPE_GLMA:
case PROJECTOR_TYPE_ULTRAVOX:
@ -3968,6 +4040,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
case PROJECTOR_TYPE_MUSIC_FLAMINGO:
return ctx->model.mm_2_w->ne[1];
case PROJECTOR_TYPE_INTERNVL:
case PROJECTOR_TYPE_NEMOTRON_V2_VL:
return ctx->model.mm_3_w->ne[1];
case PROJECTOR_TYPE_LLAMA4:
return ctx->model.mm_model_proj->ne[1];
@ -3977,6 +4050,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
return ctx->model.mm_2_w->ne[1];
case PROJECTOR_TYPE_LFM2:
case PROJECTOR_TYPE_KIMIVL:
case PROJECTOR_TYPE_PADDLEOCR:
case PROJECTOR_TYPE_KIMIK25:
return ctx->model.mm_2_w->ne[1];
case PROJECTOR_TYPE_COGVLM:

30
tools/mtmd/models/glm4v.cpp
View file

@ -2,7 +2,6 @@
ggml_cgraph * clip_graph_glm4v::build() {
GGML_ASSERT(model.patch_bias != nullptr);
GGML_ASSERT(model.position_embeddings != nullptr);
GGML_ASSERT(model.class_embedding == nullptr);
const int batch_size = 1;
@ -45,19 +44,22 @@ ggml_cgraph * clip_graph_glm4v::build() {
// pos-conv norm
inp = build_norm(inp, model.norm_embd_w, model.norm_embd_b, norm_t, eps, -1);
// calculate absolute position embedding and apply
ggml_tensor * learned_pos_embd = resize_position_embeddings(GGML_SCALE_MODE_BICUBIC);
learned_pos_embd = ggml_cont_4d(
ctx0, learned_pos_embd,
n_embd * 2, n_patches_x / 2, n_patches_y, batch_size);
learned_pos_embd = ggml_reshape_4d(
ctx0, learned_pos_embd,
n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2));
learned_pos_embd = ggml_permute(ctx0, learned_pos_embd, 0, 2, 1, 3);
learned_pos_embd = ggml_cont_3d(
ctx0, learned_pos_embd,
n_embd, n_patches_x * n_patches_y, batch_size);
cb(learned_pos_embd, "learned_pos_embd", -1);
ggml_tensor * learned_pos_embd = nullptr;
// Note: GLM-OCR does not have learned position embeddings
if (model.position_embeddings != nullptr) {
learned_pos_embd = resize_position_embeddings(GGML_SCALE_MODE_BICUBIC);
learned_pos_embd = ggml_cont_4d(
ctx0, learned_pos_embd,
n_embd * 2, n_patches_x / 2, n_patches_y, batch_size);
learned_pos_embd = ggml_reshape_4d(
ctx0, learned_pos_embd,
n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2));
learned_pos_embd = ggml_permute(ctx0, learned_pos_embd, 0, 2, 1, 3);
learned_pos_embd = ggml_cont_3d(
ctx0, learned_pos_embd,
n_embd, n_patches_x * n_patches_y, batch_size);
cb(learned_pos_embd, "learned_pos_embd", -1);
}
auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
return ggml_rope_multi(

10
tools/mtmd/models/models.h
View file

@ -42,6 +42,11 @@ struct clip_graph_internvl : clip_graph {
ggml_cgraph * build() override;
};
struct clip_graph_nemotron_v2_vl : clip_graph {
clip_graph_nemotron_v2_vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct clip_graph_llama4 : clip_graph {
clip_graph_llama4(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
@ -52,6 +57,11 @@ struct clip_graph_kimivl : clip_graph {
ggml_cgraph * build() override;
};
struct clip_graph_paddleocr : clip_graph {
clip_graph_paddleocr(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct clip_graph_cogvlm : clip_graph {
clip_graph_cogvlm(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;

35
tools/mtmd/models/nemotron-v2-vl.cpp Normal file
View file

@ -0,0 +1,35 @@
#include "models.h"
ggml_cgraph * clip_graph_nemotron_v2_vl::build() {
GGML_ASSERT(model.class_embedding != nullptr);
GGML_ASSERT(model.position_embeddings != nullptr);
const int n_registers = model.class_embedding->ne[1];
const int n_pos = n_patches + n_registers;
ggml_tensor * inp = build_inp();
// add position embeddings (pre-downsampled during GGUF conversion for fixed 512x512 input)
inp = ggml_add(ctx0, inp, model.position_embeddings);
cb(inp, "inp_pos", -1);
inp = ggml_concat(ctx0, model.class_embedding, inp, 1);
ggml_tensor * cur = build_vit(inp, n_pos, NORM_TYPE_NORMAL, hparams.ffn_op, nullptr, nullptr);
cur = ggml_view_2d(ctx0, cur,
n_embd, n_patches,
ggml_row_size(cur->type, n_embd),
n_registers * ggml_row_size(cur->type, n_embd));
cur = build_patch_merge_permute(cur, model.hparams.n_merge);
{
cur = build_norm(cur, model.mm_0_w, nullptr, NORM_TYPE_RMS, 1e-6, -1);
cur = build_ffn(cur, model.mm_1_w, nullptr, nullptr, nullptr, model.mm_3_w, nullptr, FFN_RELU_SQR, -1);
}
ggml_build_forward_expand(gf, cur);
return gf;
}

52
tools/mtmd/models/paddleocr.cpp Normal file
View file

@ -0,0 +1,52 @@
#include "models.h"
ggml_cgraph * clip_graph_paddleocr::build() {
const int n_pos = n_patches;
const int num_position_ids = n_pos * 4; // m-rope requires 4 dim per position
int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4};
ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_position_ids);
ggml_set_name(positions, "positions");
ggml_set_input(positions);
auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
return ggml_rope_multi(
ctx0, cur, positions, nullptr,
d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION,
32768, 10000, 1, 0, 1, 32, 1);
};
ggml_tensor * learned_pos_embd = resize_position_embeddings();
ggml_tensor * inp = build_inp();
ggml_tensor * cur = build_vit(
inp, n_patches,
NORM_TYPE_NORMAL,
hparams.ffn_op,
learned_pos_embd,
add_pos);
cb(cur, "vit_out", -1);
{
// mlp_AR paddleocr projector
float proj_norm_eps = 1e-5;
cur = build_norm(cur,
model.mm_input_norm_w, model.mm_input_norm_b,
NORM_TYPE_NORMAL, proj_norm_eps, -1);
const int scale_factor = model.hparams.n_merge;
cur = build_patch_merge_permute(cur, scale_factor);
cur = build_ffn(cur,
model.mm_1_w, model.mm_1_b,
nullptr, nullptr,
model.mm_2_w, model.mm_2_b,
hparams.ffn_op, -1);
cb(cur, "mlp_out", -1);
}
// build the graph
ggml_build_forward_expand(gf, cur);
return gf;
}

2
tools/mtmd/mtmd-helper.cpp
View file

@ -248,7 +248,7 @@ int32_t mtmd_helper_decode_image_chunk(
int32_t n_tokens = mtmd_input_chunk_get_n_tokens(chunk);
int32_t i_batch = 0;
int32_t n_img_batches = GGML_PAD(n_tokens, n_batch) / n_batch;
int32_t n_img_batches = (n_tokens + n_batch - 1) / n_batch;
decode_embd_batch batch_embd(encoded_embd, n_tokens, n_pos_per_embd, n_mmproj_embd);
if (mtmd_decode_use_mrope(ctx)) {

7
tools/mtmd/mtmd.cpp
View file

@ -175,7 +175,7 @@ struct mtmd_context {
clip_context_params ctx_clip_params {
/* use_gpu */ ctx_params.use_gpu,
/* flash_attn_type */ CLIP_FLASH_ATTN_TYPE_AUTO,
/* flash_attn_type */ mtmd_get_clip_flash_attn_type(ctx_params.flash_attn_type),
/* image_min_tokens */ ctx_params.image_min_tokens,
/* image_max_tokens */ ctx_params.image_max_tokens,
/* warmup */ ctx_params.warmup,
@ -325,6 +325,10 @@ struct mtmd_context {
img_beg = "<|begin_of_image|>";
img_end = "<|end_of_image|>";
} else if (proj == PROJECTOR_TYPE_PADDLEOCR) {
// <|IMAGE_START|> ... (image embeddings) ... <|IMAGE_END|>
img_beg = "<|IMAGE_START|>";
img_end = "<|IMAGE_END|>";
}
}
@ -890,6 +894,7 @@ bool mtmd_decode_use_mrope(mtmd_context * ctx) {
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_QWEN3VL:
case PROJECTOR_TYPE_GLM4V:
case PROJECTOR_TYPE_PADDLEOCR:
return true;
default:
return false;

9
tools/mtmd/tests.sh
View file

@ -28,6 +28,14 @@ if [ "${1:-}" = "huge" ]; then
echo "Include BIG and HUGE models..."
fi
# Check if the second argument is "flash", then enable flash attention
# This is useful to test if flash attention off works correctly
FLASH_ATTN="on"
if [ "${2:-}" = "flash_off" ] || [ "${1:-}" = "flash_off" ]; then
FLASH_ATTN="off"
echo "Flash attention disabled..."
fi
###############
arr_prefix=()
@ -143,6 +151,7 @@ for i in "${!arr_hf[@]}"; do
-hf $(printf %q "$hf") \
--image $(printf %q "$SCRIPT_DIR/$inp_file") \
--temp 0 -n 128 \
--flash-attn $(printf %q "$FLASH_ATTN") \
${extra_args}"
# if extra_args does not contain -p, we add a default prompt

154
tools/perplexity/perplexity.cpp
View file

@ -347,7 +347,8 @@ static results_perplexity perplexity_v2(llama_context * ctx, const common_params
int count = 0;
double nll = 0.0;
LOG_INF("%s: calculating perplexity over %d chunks, batch_size=%d\n", __func__, n_chunk, n_batch);
const int n_seq = std::max(1, n_batch / n_ctx);
LOG_INF("%s: computing over %d chunks, n_ctx=%d, batch_size=%d, n_seq=%d\n", __func__, n_chunk, n_ctx, n_batch, n_seq);
for (int i = 0; i < n_chunk; ++i) {
const int start = i * params.ppl_stride;
@ -1737,11 +1738,21 @@ static void kl_divergence(llama_context * ctx, const common_params & params) {
}
const int n_batch = params.n_batch;
const int num_batches = (n_ctx + n_batch - 1)/n_batch;
const int num_batches = (static_cast<int>(n_ctx) + n_batch - 1) / n_batch;
// Calculate n_seq based on the logits file's n_ctx, but cap it at what the context supports
const int n_seq_max = llama_n_seq_max(ctx);
int n_seq = std::max(1, n_batch / static_cast<int>(n_ctx));
if (n_seq > n_seq_max) {
LOG_WRN("%s: calculated n_seq=%d exceeds context's n_seq_max=%d, capping at %d\n",
__func__, n_seq, n_seq_max, n_seq_max);
n_seq = n_seq_max;
}
const int nv = 2*((n_vocab + 1)/2) + 4;
const bool add_bos = llama_vocab_get_add_bos(vocab);
GGML_ASSERT(!llama_vocab_get_add_eos(vocab));
llama_batch batch = llama_batch_init(std::min(n_batch, static_cast<int>(n_ctx)*n_seq), 0, 1);
std::vector<uint16_t> log_probs_uint16(size_t(n_ctx - 1 - n_ctx/2) * nv);
std::vector<float> kld_values(size_t(n_ctx - 1 - n_ctx/2)*n_chunk);
std::vector<float> p_diff_values(size_t(n_ctx - 1 - n_ctx/2)*n_chunk);
@ -1750,6 +1761,8 @@ static void kl_divergence(llama_context * ctx, const common_params & params) {
logits.reserve(size_t(n_ctx) * n_vocab);
}
LOG_INF("%s: computing over %d chunks, n_ctx=%u, batch_size=%d, n_seq=%d\n", __func__, n_chunk, n_ctx, n_batch, n_seq);
std::vector<std::thread> workers(std::thread::hardware_concurrency() - 1);
auto mean_and_uncertainty = [] (double sum, double sum2, size_t count) {
@ -1774,107 +1787,122 @@ static void kl_divergence(llama_context * ctx, const common_params & params) {
auto kld_ptr = kld_values.data();
auto p_diff_ptr = p_diff_values.data();
for (int i = 0; i < n_chunk; ++i) {
const int first = n_ctx/2;
for (int i = 0; i < n_chunk; i += n_seq) {
const int start = i * n_ctx;
const int end = start + n_ctx;
const auto t_start = std::chrono::high_resolution_clock::now();
const int n_seq_batch = std::min(n_seq, n_chunk - i);
if (in.read((char *)log_probs_uint16.data(), log_probs_uint16.size()*sizeof(uint16_t)).fail()) {
LOG_ERR("%s: failed reading log-probs for chunk %d\n", __func__, i);
return;
}
const auto t_start = std::chrono::high_resolution_clock::now();
// clear the KV cache
llama_memory_clear(llama_get_memory(ctx), true);
llama_batch batch = llama_batch_init(n_batch, 0, 1);
for (int j = 0; j < num_batches; ++j) {
const int batch_start = start + j * n_batch;
const int batch_size = std::min(end - batch_start, n_batch);
// save original token and restore it after eval
const auto token_org = tokens[batch_start];
// add BOS token for the first batch of each chunk
if (add_bos && j == 0) {
tokens[batch_start] = llama_vocab_bos(vocab);
}
int n_outputs = 0;
common_batch_clear(batch);
for (int i = 0; i < batch_size; i++) {
common_batch_add(batch, tokens[batch_start + i], j*n_batch + i, {0}, true);
for (int seq = 0; seq < n_seq_batch; seq++) {
int seq_start = batch_start + seq*n_ctx;
// save original token and restore it after eval
const auto token_org = tokens[seq_start];
// add BOS token for the first batch of each chunk
if (add_bos && j == 0) {
tokens[seq_start] = llama_vocab_bos(vocab);
}
for (int k = 0; k < batch_size; ++k) {
const int pos = j*n_batch + k;
const bool need_logits = pos >= first;
common_batch_add(batch, tokens[seq_start + k], pos, { seq }, need_logits);
n_outputs += need_logits;
}
// restore the original token in case it was set to BOS
tokens[seq_start] = token_org;
}
if (llama_decode(ctx, batch)) {
LOG_ERR("%s : failed to eval\n", __func__);
LOG_ERR("%s : failed to decode\n", __func__);
llama_batch_free(batch);
return;
}
// restore the original token in case it was set to BOS
tokens[batch_start] = token_org;
if (num_batches > 1) {
if (num_batches > 1 && n_outputs > 0) {
const auto * batch_logits = llama_get_logits(ctx);
logits.insert(logits.end(), batch_logits, batch_logits + size_t(batch_size) * n_vocab);
logits.insert(logits.end(), batch_logits, batch_logits + size_t(n_outputs) * n_vocab);
}
}
llama_batch_free(batch);
const auto t_end = std::chrono::high_resolution_clock::now();
if (i == 0) {
llama_synchronize(ctx);
const auto t_end = std::chrono::high_resolution_clock::now();
const float t_total = std::chrono::duration<float>(t_end - t_start).count();
LOG_INF("%s: %.2f seconds per pass - ETA ", __func__, t_total);
int total_seconds = (int)(t_total * n_chunk);
int total_seconds = (int)(t_total * n_chunk / n_seq);
if (total_seconds >= 60*60) {
LOG("%d hours ", total_seconds / (60*60));
total_seconds = total_seconds % (60*60);
}
LOG("%.2f minutes\n", total_seconds / 60.0);
LOG("\n");
LOG("chunk PPL ln(PPL(Q)/PPL(base)) KL Divergence Δp RMS Same top p\n");
}
LOG("\n");
LOG("chunk PPL ln(PPL(Q)/PPL(base)) KL Divergence Δp RMS Same top p\n");
const int first = n_ctx/2;
const float * all_logits = num_batches > 1 ? logits.data() : llama_get_logits(ctx);
process_logits(n_vocab, all_logits + size_t(first)*n_vocab, tokens.data() + start + first, n_ctx - 1 - first,
workers, log_probs_uint16, kld, kld_ptr, p_diff_ptr);
p_diff_ptr += n_ctx - 1 - first;
kld_ptr += n_ctx - 1 - first;
// Read log probs for each sequence in the batch
for (int seq = 0; seq < n_seq_batch; seq++) {
if (in.read((char *)log_probs_uint16.data(), log_probs_uint16.size()*sizeof(uint16_t)).fail()) {
LOG_ERR("%s: failed reading log-probs for chunk %d\n", __func__, i + seq);
llama_batch_free(batch);
return;
}
LOG("%4d", i+1);
const float * all_logits = num_batches > 1 ? logits.data() : llama_get_logits_ith(ctx, seq*n_ctx + first);
auto log_ppl = mean_and_uncertainty(kld.sum_nll, kld.sum_nll2, kld.count);
const double ppl_val = exp(log_ppl.first);
const double ppl_unc = ppl_val * log_ppl.second; // ppl_unc = sqrt( (dexp(x) / dx) ** 2 * log_ppl.second ** 2 )
LOG(" %9.4lf ± %9.4lf", ppl_val, ppl_unc);
process_logits(n_vocab, all_logits, tokens.data() + start + seq*n_ctx + first, n_ctx - 1 - first,
workers, log_probs_uint16, kld, kld_ptr, p_diff_ptr);
p_diff_ptr += n_ctx - 1 - first;
kld_ptr += n_ctx - 1 - first;
auto log_ppl_base = mean_and_uncertainty(kld.sum_nll_base, kld.sum_nll_base2, kld.count);
const double log_ppl_cov = covariance(kld.sum_nll, kld.sum_nll_base, kld.sum_nll_nll_base, kld.count);
const double log_ppl_ratio_val = log_ppl.first - log_ppl_base.first;
const double log_ppl_ratio_unc = sqrt(log_ppl.second*log_ppl.second + log_ppl_base.second*log_ppl_base.second - 2.0*log_ppl_cov);
LOG(" %10.5lf ± %10.5lf", log_ppl_ratio_val, log_ppl_ratio_unc);
LOG("%4d", i + seq + 1);
auto kl_div = mean_and_uncertainty(kld.sum_kld, kld.sum_kld2, kld.count);
LOG(" %10.5lf ± %10.5lf", kl_div.first, kl_div.second);
auto log_ppl = mean_and_uncertainty(kld.sum_nll, kld.sum_nll2, kld.count);
const double ppl_val = exp(log_ppl.first);
const double ppl_unc = ppl_val * log_ppl.second;
LOG(" %9.4lf ± %9.4lf", ppl_val, ppl_unc);
auto p_diff_mse = mean_and_uncertainty(kld.sum_p_diff2, kld.sum_p_diff4, kld.count);
const double p_diff_rms_val = sqrt(p_diff_mse.first);
const double p_diff_rms_unc = 0.5/p_diff_rms_val * p_diff_mse.second;
LOG(" %6.3lf ± %6.3lf %%", 100.0*p_diff_rms_val, 100.0*p_diff_rms_unc);
auto log_ppl_base = mean_and_uncertainty(kld.sum_nll_base, kld.sum_nll_base2, kld.count);
const double log_ppl_cov = covariance(kld.sum_nll, kld.sum_nll_base, kld.sum_nll_nll_base, kld.count);
const double log_ppl_ratio_val = log_ppl.first - log_ppl_base.first;
const double log_ppl_ratio_unc = sqrt(log_ppl.second*log_ppl.second + log_ppl_base.second*log_ppl_base.second - 2.0*log_ppl_cov);
LOG(" %10.5lf ± %10.5lf", log_ppl_ratio_val, log_ppl_ratio_unc);
double p_top_val = 1.*kld.n_same_top/kld.count;
double p_top_unc = sqrt(p_top_val*(1 - p_top_val)/(kld.count - 1));
LOG(" %6.3lf ± %6.3lf %%", 100.0*p_top_val, 100.0*p_top_unc);
auto kl_div = mean_and_uncertainty(kld.sum_kld, kld.sum_kld2, kld.count);
LOG(" %10.5lf ± %10.5lf", kl_div.first, kl_div.second);
LOG("\n");
auto p_diff_mse = mean_and_uncertainty(kld.sum_p_diff2, kld.sum_p_diff4, kld.count);
const double p_diff_rms_val = sqrt(p_diff_mse.first);
const double p_diff_rms_unc = 0.5/p_diff_rms_val * p_diff_mse.second;
LOG(" %6.3lf ± %6.3lf %%", 100.0*p_diff_rms_val, 100.0*p_diff_rms_unc);
double p_top_val = 1.*kld.n_same_top/kld.count;
double p_top_unc = sqrt(p_top_val*(1 - p_top_val)/(kld.count - 1));
LOG(" %6.3lf ± %6.3lf %%", 100.0*p_top_val, 100.0*p_top_unc);
LOG("\n");
}
logits.clear();
}
llama_batch_free(batch);
LOG("\n");
if (kld.count < 100) return; // we do not wish to do statistics on so few values
@ -1996,7 +2024,7 @@ int main(int argc, char ** argv) {
const bool ppl = !params.hellaswag && !params.winogrande && !params.multiple_choice && !params.kl_divergence;
if (ppl) {
if (ppl || params.kl_divergence) {
const int32_t n_seq = std::max(1, params.n_batch / n_ctx);
const int32_t n_kv = n_seq * n_ctx;
@ -2006,12 +2034,8 @@ int main(int argc, char ** argv) {
params.n_batch = std::min(params.n_batch, n_kv);
} else {
params.n_batch = std::min(params.n_batch, params.n_ctx);
if (params.kl_divergence) {
params.n_parallel = 1;
} else {
// ensure there's at least enough seq_ids for HellaSwag
params.n_parallel = std::max(4, params.n_parallel);
}
// ensure there's at least enough seq_ids for HellaSwag
params.n_parallel = std::max(4, params.n_parallel);
}
if (params.ppl_stride > 0) {

57
tools/quantize/quantize.cpp
View file

@ -120,7 +120,7 @@ static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftyp
static void usage(const char * executable) {
printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] [--imatrix] [--include-weights]\n", executable);
printf(" [--exclude-weights] [--output-tensor-type] [--token-embedding-type] [--tensor-type] [--tensor-type-file]\n");
printf(" [--prune-layers] [--keep-split] [--override-kv]\n");
printf(" [--prune-layers] [--keep-split] [--override-kv] [--dry-run]\n");
printf(" model-f32.gguf [model-quant.gguf] type [nthreads]\n\n");
printf(" --allow-requantize\n");
printf(" allow requantizing tensors that have already been quantized\n");
@ -156,7 +156,10 @@ static void usage(const char * executable) {
printf(" generate quantized model in the same shards as input\n");
printf(" --override-kv KEY=TYPE:VALUE\n");
printf(" override model metadata by key in the quantized model. may be specified multiple times.\n");
printf(" WARNING: this is an advanced option, use with care.\n\n");
printf(" WARNING: this is an advanced option, use with care.\n");
printf(" --dry-run\n");
printf(" calculate and show the final quantization size without performing quantization\n");
printf(" example: llama-quantize --dry-run model-f32.gguf Q4_K\n\n");
printf("note: --include-weights and --exclude-weights cannot be used together\n\n");
printf("-----------------------------------------------------------------------------\n");
printf(" allowed quantization types\n");
@ -532,6 +535,8 @@ int main(int argc, char ** argv) {
if (arg_idx == argc-1 || !string_parse_kv_override(argv[++arg_idx], kv_overrides)) {
usage(argv[0]);
}
} else if (strcmp(argv[arg_idx], "--dry-run") == 0) {
params.dry_run = true;
} else if (strcmp(argv[arg_idx], "--allow-requantize") == 0) {
params.allow_requantize = true;
} else if (strcmp(argv[arg_idx], "--pure") == 0) {
@ -630,22 +635,26 @@ int main(int argc, char ** argv) {
std::string ftype_str;
std::string suffix = ".gguf";
if (try_parse_ftype(argv[arg_idx], params.ftype, ftype_str)) {
std::string fpath;
const size_t pos = fname_inp.find_last_of("/\\");
if (pos != std::string::npos) {
fpath = fname_inp.substr(0, pos + 1);
}
// argv[arg_idx] is the ftype directly: <input> <ftype>
if (!params.dry_run) {
std::string fpath;
const size_t pos = fname_inp.find_last_of("/\\");
if (pos != std::string::npos) {
fpath = fname_inp.substr(0, pos + 1);
}
// export as [inp path]/ggml-model-[ftype]. Only add extension if there is no splitting
fname_out = fpath + "ggml-model-" + ftype_str;
if (!params.keep_split) {
fname_out += suffix;
// export as [inp path]/ggml-model-[ftype]. Only add extension if there is no splitting
fname_out = fpath + "ggml-model-" + ftype_str;
if (!params.keep_split) {
fname_out += suffix;
}
}
arg_idx++;
if (ftype_str == "COPY") {
params.only_copy = true;
}
} else {
// argv[arg_idx] is not a valid ftype, so treat it as output path: <input> <output> <ftype>
fname_out = argv[arg_idx];
if (params.keep_split && fname_out.find(suffix) != std::string::npos) {
fname_out = fname_out.substr(0, fname_out.length() - suffix.length());
@ -677,25 +686,33 @@ int main(int argc, char ** argv) {
}
}
if ((params.ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || params.ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS ||
params.ftype == LLAMA_FTYPE_MOSTLY_IQ2_S ||
params.ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S ||
params.ftype == LLAMA_FTYPE_MOSTLY_IQ1_S ||
params.ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) && imatrix_data.empty()) {
if (!params.dry_run &&
(
params.ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || params.ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS ||
params.ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || params.ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S ||
params.ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || params.ftype == LLAMA_FTYPE_MOSTLY_IQ1_M
) && imatrix_data.empty()) {
fprintf(stderr, "\n==========================================================================================================\n");
fprintf(stderr, "Please do not use IQ1_S, IQ1_M, IQ2_S, IQ2_XXS, IQ2_XS or Q2_K_S quantization without an importance matrix\n");
fprintf(stderr, "==========================================================================================================\n\n\n");
return 1;
}
if (std::error_code ec; std::filesystem::equivalent(fname_inp, fname_out, ec)) {
fprintf(stderr, "%s: error: input and output files are the same: '%s'\n", __func__, fname_inp.c_str());
return 1;
if (!params.dry_run) {
if (std::error_code ec; std::filesystem::equivalent(fname_inp, fname_out, ec)) {
fprintf(stderr, "%s: error: input and output files are the same: '%s'\n", __func__, fname_inp.c_str());
return 1;
}
}
print_build_info();
fprintf(stderr, "%s: quantizing '%s' to '%s' as %s", __func__, fname_inp.c_str(), fname_out.c_str(), ftype_str.c_str());
if (params.dry_run) {
fprintf(stderr, "%s: calculating quantization size for '%s' as %s", __func__, fname_inp.c_str(), ftype_str.c_str());
} else {
fprintf(stderr, "%s: quantizing '%s' to '%s' as %s", __func__, fname_inp.c_str(), fname_out.c_str(), ftype_str.c_str());
}
if (params.nthread > 0) {
fprintf(stderr, " using %d threads", params.nthread);
}

3
tools/rpc/rpc-server.cpp
View file

@ -132,7 +132,8 @@ static std::string fs_get_cache_directory() {
if (getenv("LLAMA_CACHE")) {
cache_directory = std::getenv("LLAMA_CACHE");
} else {
#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || defined(__OpenBSD__)
#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || \
defined(__OpenBSD__) || defined(__NetBSD__)
if (std::getenv("XDG_CACHE_HOME")) {
cache_directory = std::getenv("XDG_CACHE_HOME");
} else if (std::getenv("HOME")) {

8
tools/server/CMakeLists.txt
View file

@ -28,10 +28,6 @@ target_link_libraries(${TARGET} PUBLIC common mtmd ${CMAKE_THREAD_LIBS_INIT})
set(TARGET llama-server)
if (NOT LLAMA_HTTPLIB)
message(FATAL_ERROR "LLAMA_HTTPLIB is OFF, cannot build llama-server. Hint: to skip building server, set -DLLAMA_BUILD_SERVER=OFF")
endif()
set(TARGET_SRCS
server.cpp
server-http.cpp
@ -63,8 +59,4 @@ target_include_directories(${TARGET} PRIVATE ../mtmd)
target_include_directories(${TARGET} PRIVATE ${CMAKE_SOURCE_DIR})
target_link_libraries(${TARGET} PRIVATE server-context PUBLIC common cpp-httplib ${CMAKE_THREAD_LIBS_INIT})
if (WIN32)
TARGET_LINK_LIBRARIES(${TARGET} PRIVATE ws2_32)
endif()
target_compile_features(${TARGET} PRIVATE cxx_std_17)

23
tools/server/README.md
View file

@ -74,8 +74,8 @@ For the full list of features, please refer to [server's changelog](https://gith
| `-ctv, --cache-type-v TYPE` | KV cache data type for V<br/>allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1<br/>(default: f16)<br/>(env: LLAMA_ARG_CACHE_TYPE_V) |
| `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)<br/>(env: LLAMA_ARG_DEFRAG_THOLD) |
| `--mlock` | force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) |
| `--mmap, --no-mmap` | whether to memory-map model. Explicitly enabling mmap disables direct-io. (if mmap disabled, slower load but may reduce pageouts if not using mlock) (default: enabled)<br/>(env: LLAMA_ARG_MMAP) |
| `-dio, --direct-io, -ndio, --no-direct-io` | use DirectIO if available. Takes precedence over --mmap (default: enabled)<br/>(env: LLAMA_ARG_DIO) |
| `--mmap, --no-mmap` | whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock) (default: enabled)<br/>(env: LLAMA_ARG_MMAP) |
| `-dio, --direct-io, -ndio, --no-direct-io` | use DirectIO if available. (default: disabled)<br/>(env: LLAMA_ARG_DIO) |
| `--numa TYPE` | attempt optimizations that help on some NUMA systems<br/>- distribute: spread execution evenly over all nodes<br/>- isolate: only spawn threads on CPUs on the node that execution started on<br/>- numactl: use the CPU map provided by numactl<br/>if run without this previously, it is recommended to drop the system page cache before using this<br/>see https://github.com/ggml-org/llama.cpp/issues/1437<br/>(env: LLAMA_ARG_NUMA) |
| `-dev, --device <dev1,dev2,..>` | comma-separated list of devices to use for offloading (none = don't offload)<br/>use --list-devices to see a list of available devices<br/>(env: LLAMA_ARG_DEVICE) |
| `--list-devices` | print list of available devices and exit |
@ -126,14 +126,14 @@ For the full list of features, please refer to [server's changelog](https://gith
| `-s, --seed SEED` | RNG seed (default: -1, use random seed for -1) |
| `--sampler-seq, --sampling-seq SEQUENCE` | simplified sequence for samplers that will be used (default: edskypmxt) |
| `--ignore-eos` | ignore end of stream token and continue generating (implies --logit-bias EOS-inf) |
| `--temp N` | temperature (default: 0.80) |
| `--temp, --temperature N` | temperature (default: 0.80) |
| `--top-k N` | top-k sampling (default: 40, 0 = disabled)<br/>(env: LLAMA_ARG_TOP_K) |
| `--top-p N` | top-p sampling (default: 0.95, 1.0 = disabled) |
| `--min-p N` | min-p sampling (default: 0.05, 0.0 = disabled) |
| `--top-nsigma N` | top-n-sigma sampling (default: -1.00, -1.0 = disabled) |
| `--top-nsigma, --top-n-sigma N` | top-n-sigma sampling (default: -1.00, -1.0 = disabled) |
| `--xtc-probability N` | xtc probability (default: 0.00, 0.0 = disabled) |
| `--xtc-threshold N` | xtc threshold (default: 0.10, 1.0 = disabled) |
| `--typical N` | locally typical sampling, parameter p (default: 1.00, 1.0 = disabled) |
| `--typical, --typical-p N` | locally typical sampling, parameter p (default: 1.00, 1.0 = disabled) |
| `--repeat-last-n N` | last n tokens to consider for penalize (default: 64, 0 = disabled, -1 = ctx_size) |
| `--repeat-penalty N` | penalize repeat sequence of tokens (default: 1.00, 1.0 = disabled) |
| `--presence-penalty N` | repeat alpha presence penalty (default: 0.00, 0.0 = disabled) |
@ -162,9 +162,11 @@ For the full list of features, please refer to [server's changelog](https://gith
| Argument | Explanation |
| -------- | ----------- |
| `-lcs, --lookup-cache-static FNAME` | path to static lookup cache to use for lookup decoding (not updated by generation) |
| `-lcd, --lookup-cache-dynamic FNAME` | path to dynamic lookup cache to use for lookup decoding (updated by generation) |
| `--ctx-checkpoints, --swa-checkpoints N` | max number of context checkpoints to create per slot (default: 8)[(more info)](https://github.com/ggml-org/llama.cpp/pull/15293)<br/>(env: LLAMA_ARG_CTX_CHECKPOINTS) |
| `-cram, --cache-ram N` | set the maximum cache size in MiB (default: 8192, -1 - no limit, 0 - disable)[(more info)](https://github.com/ggml-org/llama.cpp/pull/16391)<br/>(env: LLAMA_ARG_CACHE_RAM) |
| `-kvu, --kv-unified` | use single unified KV buffer shared across all sequences (default: enabled if number of slots is auto)<br/>(env: LLAMA_ARG_KV_UNIFIED) |
| `-kvu, --kv-unified, -no-kvu, --no-kv-unified` | use single unified KV buffer shared across all sequences (default: enabled if number of slots is auto)<br/>(env: LLAMA_ARG_KV_UNIFIED) |
| `--context-shift, --no-context-shift` | whether to use context shift on infinite text generation (default: disabled)<br/>(env: LLAMA_ARG_CONTEXT_SHIFT) |
| `-r, --reverse-prompt PROMPT` | halt generation at PROMPT, return control in interactive mode |
| `-sp, --special` | special tokens output enabled (default: false) |
@ -182,7 +184,8 @@ For the full list of features, please refer to [server's changelog](https://gith
| `-otd, --override-tensor-draft <tensor name pattern>=<buffer type>,...` | override tensor buffer type for draft model |
| `-cmoed, --cpu-moe-draft` | keep all Mixture of Experts (MoE) weights in the CPU for the draft model<br/>(env: LLAMA_ARG_CPU_MOE_DRAFT) |
| `-ncmoed, --n-cpu-moe-draft N` | keep the Mixture of Experts (MoE) weights of the first N layers in the CPU for the draft model<br/>(env: LLAMA_ARG_N_CPU_MOE_DRAFT) |
| `-a, --alias STRING` | set alias for model name (to be used by REST API)<br/>(env: LLAMA_ARG_ALIAS) |
| `-a, --alias STRING` | set model name aliases, comma-separated (to be used by API)<br/>(env: LLAMA_ARG_ALIAS) |
| `--tags STRING` | set model tags, comma-separated (informational, not used for routing)<br/>(env: LLAMA_ARG_TAGS) |
| `--host HOST` | ip address to listen, or bind to an UNIX socket if the address ends with .sock (default: 127.0.0.1)<br/>(env: LLAMA_ARG_HOST) |
| `--port PORT` | port to listen (default: 8080)<br/>(env: LLAMA_ARG_PORT) |
| `--path PATH` | path to serve static files from (default: )<br/>(env: LLAMA_ARG_STATIC_PATH) |
@ -229,6 +232,10 @@ For the full list of features, please refer to [server's changelog](https://gith
| `-ngld, --gpu-layers-draft, --n-gpu-layers-draft N` | max. number of draft model layers to store in VRAM, either an exact number, 'auto', or 'all' (default: auto)<br/>(env: LLAMA_ARG_N_GPU_LAYERS_DRAFT) |
| `-md, --model-draft FNAME` | draft model for speculative decoding (default: unused)<br/>(env: LLAMA_ARG_MODEL_DRAFT) |
| `--spec-replace TARGET DRAFT` | translate the string in TARGET into DRAFT if the draft model and main model are not compatible |
| `--spec-type [none\|ngram-cache\|ngram-simple\|ngram-map-k\|ngram-map-k4v\|ngram-mod]` | type of speculative decoding to use when no draft model is provided (default: none) |
| `--spec-ngram-size-n N` | ngram size N for ngram-simple/ngram-map speculative decoding, length of lookup n-gram (default: 12) |
| `--spec-ngram-size-m N` | ngram size M for ngram-simple/ngram-map speculative decoding, length of draft m-gram (default: 48) |
| `--spec-ngram-min-hits N` | minimum hits for ngram-map speculative decoding (default: 1) |
| `-mv, --model-vocoder FNAME` | vocoder model for audio generation (default: unused) |
| `--tts-use-guide-tokens` | Use guide tokens to improve TTS word recall |
| `--embd-gemma-default` | use default EmbeddingGemma model (note: can download weights from the internet) |
@ -1510,7 +1517,7 @@ version = 1
; If the same key is defined in a specific preset, it will override the value in this global section.
[*]
c = 8192
n-gpu-layer = 8
n-gpu-layers = 8
; If the key corresponds to an existing model on the server,
; this will be used as the default config for that model

162
tools/server/server-common.cpp
View file

@ -231,19 +231,77 @@ server_tokens::server_tokens(mtmd::input_chunks & mtmd_chunks, bool has_mtmd) :
server_tokens::server_tokens(const llama_tokens & tokens, bool has_mtmd) : has_mtmd(has_mtmd), tokens(tokens) {
}
llama_pos server_tokens::pos_next() const {
llama_pos server_tokens::pos_next(int64_t n_tokens) const {
if (!has_mtmd) {
return tokens.size();
if (n_tokens < 0) {
return tokens.size();
}
return n_tokens;
}
llama_pos res = tokens.size();
if (n_tokens < 0) {
llama_pos res = tokens.size();
for (auto it = map_idx_to_media.begin(); it != map_idx_to_media.end(); ++it) {
const auto & chunk = it->second;
res += mtmd_input_chunk_get_n_pos(chunk.get()) - mtmd_input_chunk_get_n_tokens(chunk.get());
for (auto it = map_idx_to_media.begin(); it != map_idx_to_media.end(); ++it) {
const auto & chunk = it->second;
res += mtmd_input_chunk_get_n_pos(chunk.get()) - mtmd_input_chunk_get_n_tokens(chunk.get());
}
return res;
}
return res;
int64_t idx = 0;
llama_pos pos = 0;
GGML_ASSERT(n_tokens <= (int64_t)tokens.size());
while (idx < n_tokens) {
const auto media_it = map_idx_to_media.find(idx);
if (media_it != map_idx_to_media.end()) {
const auto & chunk = media_it->second;
const llama_pos n_pos = mtmd_input_chunk_get_n_pos(chunk.get());
const size_t n_tok = mtmd_input_chunk_get_n_tokens(chunk.get());
pos += n_pos;
idx += n_tok;
} else {
pos++;
idx++;
}
}
return pos;
}
size_t server_tokens::size_up_to_pos(llama_pos max_pos) const {
if (!has_mtmd) {
return std::min((size_t)(max_pos + 1), tokens.size());
}
size_t idx = 0;
llama_pos pos = 0;
while (idx < tokens.size()) {
const auto media_it = map_idx_to_media.find(idx);
if (media_it != map_idx_to_media.end()) {
const auto & chunk = media_it->second;
const llama_pos n_pos = mtmd_input_chunk_get_n_pos(chunk.get());
const size_t n_tok = mtmd_input_chunk_get_n_tokens(chunk.get());
pos += n_pos;
idx += n_tok;
} else {
pos++;
idx++;
}
if (pos > max_pos) {
break;
}
}
return idx;
}
std::string server_tokens::str() const {
@ -916,8 +974,7 @@ json oaicompat_chat_params_parse(
json image_url = json_value(p, "image_url", json::object());
handle_media(out_files, image_url, opt.media_path);
// replace this chunk with a marker
p["type"] = "text";
p["type"] = "media_marker";
p["text"] = mtmd_default_marker();
p.erase("image_url");
@ -938,8 +995,7 @@ json oaicompat_chat_params_parse(
// TODO: add audio_url support by reusing handle_media()
// replace this chunk with a marker
p["type"] = "text";
p["type"] = "media_marker";
p["text"] = mtmd_default_marker();
p.erase("input_audio");
@ -1107,6 +1163,8 @@ json convert_responses_to_chatcmpl(const json & response_body) {
};
for (json item : input_value) {
bool merge_prev = !chatcmpl_messages.empty() && chatcmpl_messages.back().value("role", "") == "assistant";
if (exists_and_is_string(item, "content")) {
// #responses_create-input-input_item_list-input_message-content-text_input
// Only "Input message" contains item["content"]::string
@ -1195,7 +1253,7 @@ json convert_responses_to_chatcmpl(const json & response_body) {
item.at("type") == "message"
) {
// #responses_create-input-input_item_list-item-output_message
std::vector<json> chatcmpl_content;
auto chatcmpl_content = json::array();
for (const auto & output_text : item.at("content")) {
const std::string type = json_value(output_text, "type", std::string());
@ -1212,10 +1270,19 @@ json convert_responses_to_chatcmpl(const json & response_body) {
});
}
item.erase("status");
item.erase("type");
item["content"] = chatcmpl_content;
chatcmpl_messages.push_back(item);
if (merge_prev) {
auto & prev_msg = chatcmpl_messages.back();
if (!exists_and_is_array(prev_msg, "content")) {
prev_msg["content"] = json::array();
}
auto & prev_content = prev_msg["content"];
prev_content.insert(prev_content.end(), chatcmpl_content.begin(), chatcmpl_content.end());
} else {
item.erase("status");
item.erase("type");
item["content"] = chatcmpl_content;
chatcmpl_messages.push_back(item);
}
} else if (exists_and_is_string(item, "arguments") &&
exists_and_is_string(item, "call_id") &&
exists_and_is_string(item, "name") &&
@ -1223,24 +1290,27 @@ json convert_responses_to_chatcmpl(const json & response_body) {
item.at("type") == "function_call"
) {
// #responses_create-input-input_item_list-item-function_tool_call
json msg = json {
{"role", "assistant"},
{"tool_calls", json::array({ json {
{"function", json {
{"arguments", item.at("arguments")},
{"name", item.at("name")},
}},
{"id", item.at("call_id")},
{"type", "function"},
}})},
json tool_call = {
{"function", json {
{"arguments", item.at("arguments")},
{"name", item.at("name")},
}},
{"id", item.at("call_id")},
{"type", "function"},
};
if (!chatcmpl_messages.empty() && chatcmpl_messages.back().contains("reasoning_content")) {
// Move reasoning content from dummy message to tool call message
msg["reasoning_content"] = chatcmpl_messages.back().at("reasoning_content");
chatcmpl_messages.pop_back();
if (merge_prev) {
auto & prev_msg = chatcmpl_messages.back();
if (!exists_and_is_array(prev_msg, "tool_calls")) {
prev_msg["tool_calls"] = json::array();
}
prev_msg["tool_calls"].push_back(tool_call);
} else {
chatcmpl_messages.push_back(json {
{"role", "assistant"},
{"tool_calls", json::array({tool_call})}
});
}
chatcmpl_messages.push_back(msg);
} else if (exists_and_is_string(item, "call_id") &&
(exists_and_is_string(item, "output") || exists_and_is_array(item, "output")) &&
exists_and_is_string(item, "type") &&
@ -1284,12 +1354,16 @@ json convert_responses_to_chatcmpl(const json & response_body) {
throw std::invalid_argument("item['content']['text'] is not a string");
}
// Pack reasoning content in dummy message
chatcmpl_messages.push_back(json {
{"role", "assistant"},
{"content", json::array()},
{"reasoning_content", item.at("content")[0].at("text")},
});
if (merge_prev) {
auto & prev_msg = chatcmpl_messages.back();
prev_msg["reasoning_content"] = item.at("content")[0].at("text");
} else {
chatcmpl_messages.push_back(json {
{"role", "assistant"},
{"content", json::array()},
{"reasoning_content", item.at("content")[0].at("text")},
});
}
} else {
throw std::invalid_argument("Cannot determine type of 'item'");
}
@ -1298,20 +1372,6 @@ json convert_responses_to_chatcmpl(const json & response_body) {
throw std::invalid_argument("'input' must be a string or array of objects");
}
// Remove unused dummy message which contains
// reasoning content not followed by tool call
chatcmpl_messages.erase(std::remove_if(
chatcmpl_messages.begin(),
chatcmpl_messages.end(),
[](const json & x){ return x.contains("role") &&
x.at("role") == "assistant" &&
x.contains("content") &&
x.at("content") == json::array() &&
x.contains("reasoning_content");
}),
chatcmpl_messages.end()
);
chatcmpl_body["messages"] = chatcmpl_messages;
if (response_body.contains("tools")) {

7
tools/server/server-common.h
View file

@ -167,7 +167,12 @@ public:
// for debugging
std::string str() const;
llama_pos pos_next() const;
// the next position after n_tokens. if n_tokens < 0, return the next position after all tokens.
llama_pos pos_next(int64_t n_tokens = -1) const;
// number of tokens with position <= max_pos
size_t size_up_to_pos(llama_pos max_pos) const;
const mtmd::input_chunk_ptr & find_chunk(size_t idx) const;
void push_back(llama_token tok);

81
tools/server/server-context.cpp
View file

@ -77,6 +77,7 @@ struct server_slot {
size_t last_nl_pos = 0;
std::string generated_text;
std::string debug_generated_text;
llama_tokens generated_tokens;
// idx of draft tokens in the main batch
@ -425,7 +426,7 @@ struct server_slot {
if (!only_metrics) {
res["prompt"] = ptask->tokens.detokenize(ctx, true);
res["generated"] = generated_text;
res["generated"] = generated_text.empty() ? debug_generated_text : generated_text;
}
}
@ -579,6 +580,8 @@ private:
float slot_prompt_similarity = 0.0f;
std::string model_name; // name of the loaded model, to be used by API
std::set<std::string> model_aliases; // additional names for the model
std::set<std::string> model_tags; // informational tags
bool sleeping = false;
@ -812,10 +815,9 @@ private:
SRV_WRN("%s", "for more info see https://github.com/ggml-org/llama.cpp/pull/16391\n");
if (!params_base.model_alias.empty()) {
// user explicitly specified model name
model_name = params_base.model_alias;
// backward compat: use first alias as model name
model_name = *params_base.model_alias.begin();
} else if (!params_base.model.name.empty()) {
// use model name in registry format (for models in cache)
model_name = params_base.model.name;
} else {
// fallback: derive model name from file name
@ -823,6 +825,9 @@ private:
model_name = model_path.filename().string();
}
model_aliases = params_base.model_alias;
model_tags = params_base.model_tags;
if (!is_resume) {
return init();
}
@ -994,9 +999,6 @@ private:
// don't update the cache if the slot's context is empty
update_cache = update_cache && tokens.size() > 0;
// TODO: mtmd does not support prompt cache
update_cache = update_cache && (ret->mctx == nullptr);
if (update_cache) {
SRV_WRN("%s", "updating prompt cache\n");
@ -1441,7 +1443,13 @@ private:
res->id = slot.task->id;
res->id_slot = slot.id;
res->index = slot.task->index;
res->index = slot.task->index;
// keep copy of last generated text for debugging purposes
if (slots_debug) {
slot.debug_generated_text = slot.generated_text;
}
// in stream mode, content and tokens are already in last partial chunk
if (slot.task->params.stream) {
res->content = "";
@ -2275,15 +2283,15 @@ private:
n_past = 0;
}
llama_pos pos_next = slot.prompt.tokens.pos_next(n_past);
// note: when n_swa == 0, the model does not use SWA, which is equivalent to a window of 1
const auto n_swa = std::max(1, llama_model_n_swa(model));
// the largest pos_min required for a checkpoint to be useful
const auto pos_min_thold = std::max(0, n_past - n_swa);
const auto pos_min_thold = std::max(0, pos_next - n_swa);
// note: disallow with mtmd contexts for now
// https://github.com/ggml-org/llama.cpp/issues/17043
if (!mctx && n_past > 0 && n_past < slot.prompt.n_tokens()) {
if (n_past > 0 && n_past < slot.prompt.n_tokens()) {
const auto pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx), slot.id);
if (pos_min == -1) {
SLT_ERR(slot, "n_past = %d, slot.prompt.tokens.size() = %d, seq_id = %d, pos_min = %d\n", n_past, (int) slot.prompt.tokens.size(), slot.id, pos_min);
@ -2334,9 +2342,6 @@ private:
}
if (pos_min > pos_min_thold) {
// TODO: support can be added in the future when corresponding vision models get released
GGML_ASSERT(!slot.prompt.tokens.has_mtmd);
SLT_WRN(slot, "n_past = %d, slot.prompt.tokens.size() = %d, seq_id = %d, pos_min = %d, n_swa = %d\n", n_past, (int) slot.prompt.tokens.size(), slot.id, pos_min, n_swa);
// search for a context checkpoint
@ -2357,18 +2362,20 @@ private:
const size_t n = llama_state_seq_set_data_ext(ctx, it->data.data(), checkpoint_size, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
if (n != checkpoint_size) {
SLT_ERR(slot, "failed to restore context checkpoint (pos_min = %d, pos_max = %d, size = %.3f MiB)\n", it->pos_min, it->pos_max, (float) checkpoint_size / 1024 / 1024);
SLT_ERR(slot, "failed to restore context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n", it->pos_min, it->pos_max, it->n_tokens, (float) checkpoint_size / 1024 / 1024);
do_reset = true;
//printf("[DEBUG] `do_reset` was set to `true` after failing to restore a checkpoint");
} else {
n_past = std::min(n_past, std::max(it->pos_min + 1, it->pos_max));
SLT_WRN(slot, "restored context checkpoint (pos_min = %d, pos_max = %d, size = %.3f MiB)\n", it->pos_min, it->pos_max, (float) checkpoint_size / 1024 / 1024);
pos_next = std::min(pos_next, std::max(it->pos_min + 1, it->pos_max));
n_past = std::min(slot.prompt.tokens.size_up_to_pos(pos_next), (size_t) it->n_tokens);
SLT_WRN(slot, "restored context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n", it->pos_min, it->pos_max, it->n_tokens, (float) checkpoint_size / 1024 / 1024);
}
}
if (do_reset) {
SLT_WRN(slot, "forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see %s)\n",
"https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055");
pos_next = 0;
n_past = 0;
}
}
@ -2379,7 +2386,7 @@ private:
for (auto it = slot.prompt.checkpoints.begin(); it != slot.prompt.checkpoints.end();) {
const auto & cur = *it;
if (cur.pos_min > pos_min_thold) {
SLT_WRN(slot, "erased invalidated context checkpoint (pos_min = %d, pos_max = %d, n_swa = %d, size = %.3f MiB)\n", cur.pos_min, cur.pos_max, n_swa, (float) cur.data.size() / 1024 / 1024);
SLT_WRN(slot, "erased invalidated context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", n_swa = %d, size = %.3f MiB)\n", cur.pos_min, cur.pos_max, cur.n_tokens, n_swa, (float) cur.data.size() / 1024 / 1024);
it = slot.prompt.checkpoints.erase(it);
} else {
++it;
@ -2395,7 +2402,7 @@ private:
SLT_WRN(slot, "n_past was set to %d\n", n_past);
}
slot.n_prompt_tokens_cache = n_past;
slot.n_prompt_tokens_cache = n_past;
slot.n_prompt_tokens_processed = 0;
slot.prompt.tokens.keep_first(n_past);
@ -2513,10 +2520,6 @@ private:
}
}
// SLT_INF(slot, "new slot.prompt.tokens: %s\n", slot.slot.prompt.tokens.str().c_str());
SLT_INF(slot, "prompt processing progress, n_tokens = %d, batch.n_tokens = %d, progress = %f\n", slot.prompt.n_tokens(), batch.n_tokens, (float) slot.prompt.n_tokens() / slot.task->n_tokens());
// entire prompt has been processed
if (slot.prompt.n_tokens() == slot.task->n_tokens()) {
slot.state = SLOT_STATE_DONE_PROMPT;
@ -2529,8 +2532,6 @@ private:
slot.n_decoded = 0;
slot.i_batch = batch.n_tokens - 1;
SLT_INF(slot, "prompt done, n_tokens = %d, batch.n_tokens = %d\n", slot.prompt.n_tokens(), batch.n_tokens);
slot.init_sampler();
const auto pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx), slot.id);
@ -2542,13 +2543,15 @@ private:
// no need to create checkpoints that are too close together
do_checkpoint = do_checkpoint && (slot.prompt.checkpoints.empty() || pos_max > slot.prompt.checkpoints.back().pos_max + 64);
// note: we create the checkpoint before calling llama_decode(), so the current batch is not
// yet processed and therefore it is not part of the checkpoint.
if (do_checkpoint) {
while (slot.prompt.checkpoints.size() >= (size_t) params_base.n_ctx_checkpoints) {
// make room for the new checkpoint, if needed
const auto & cur = slot.prompt.checkpoints.front();
SLT_WRN(slot, "erasing old context checkpoint (pos_min = %d, pos_max = %d, size = %.3f MiB)\n",
cur.pos_min, cur.pos_max, (float) cur.data.size() / 1024 / 1024);
SLT_WRN(slot, "erasing old context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n",
cur.pos_min, cur.pos_max, cur.n_tokens, (float) cur.data.size() / 1024 / 1024);
slot.prompt.checkpoints.erase(slot.prompt.checkpoints.begin());
}
@ -2556,16 +2559,21 @@ private:
const size_t checkpoint_size = llama_state_seq_get_size_ext(ctx, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
auto & cur = slot.prompt.checkpoints.emplace_back(server_prompt_checkpoint{
/*.pos_min = */ pos_min,
/*.pos_max = */ pos_max,
/*.data = */ std::vector<uint8_t>(checkpoint_size),
/*.pos_min = */ pos_min,
/*.pos_max = */ pos_max,
/*.n_tokens = */ slot.prompt.n_tokens() - batch.n_tokens,
/*.data = */ std::vector<uint8_t>(checkpoint_size),
});
llama_state_seq_get_data_ext(ctx, cur.data.data(), checkpoint_size, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
SLT_WRN(slot, "created context checkpoint %d of %d (pos_min = %d, pos_max = %d, size = %.3f MiB)\n",
(int) slot.prompt.checkpoints.size(), params_base.n_ctx_checkpoints, cur.pos_min, cur.pos_max, (float) cur.data.size() / 1024 / 1024);
SLT_WRN(slot, "created context checkpoint %d of %d (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n",
(int) slot.prompt.checkpoints.size(), params_base.n_ctx_checkpoints, cur.pos_min, cur.pos_max, cur.n_tokens, (float) cur.data.size() / 1024 / 1024);
}
SLT_INF(slot, "prompt processing done, n_tokens = %d, batch.n_tokens = %d\n", slot.prompt.n_tokens(), batch.n_tokens);
} else {
SLT_INF(slot, "prompt processing progress, n_tokens = %d, batch.n_tokens = %d, progress = %f\n", slot.prompt.n_tokens(), batch.n_tokens, (float) slot.prompt.n_tokens() / slot.task->n_tokens());
}
}
@ -2888,6 +2896,8 @@ server_context_meta server_context::get_meta() const {
return server_context_meta {
/* build_info */ build_info,
/* model_name */ impl->model_name,
/* model_aliases */ impl->model_aliases,
/* model_tags */ impl->model_tags,
/* model_path */ impl->params_base.model.path,
/* has_mtmd */ impl->mctx != nullptr,
/* has_inp_image */ impl->chat_params.allow_image,
@ -2904,6 +2914,9 @@ server_context_meta server_context::get_meta() const {
/* fim_pre_token */ llama_vocab_fim_pre(impl->vocab),
/* fim_sub_token */ llama_vocab_fim_suf(impl->vocab),
/* fim_mid_token */ llama_vocab_fim_mid(impl->vocab),
/* fim_pad_token */ llama_vocab_fim_pad(impl->vocab),
/* fim_rep_token */ llama_vocab_fim_rep(impl->vocab),
/* fim_sep_token */ llama_vocab_fim_sep(impl->vocab),
/* model_vocab_type */ llama_vocab_type(impl->vocab),
/* model_vocab_n_tokens */ llama_vocab_n_tokens(impl->vocab),
@ -3681,6 +3694,8 @@ void server_routes::init_routes() {
{"data", {
{
{"id", meta->model_name},
{"aliases", meta->model_aliases},
{"tags", meta->model_tags},
{"object", "model"},
{"created", std::time(0)},
{"owned_by", "llamacpp"},

8
tools/server/server-context.h
View file

@ -1,3 +1,5 @@
#pragma once
#include "server-http.h"
#include "server-task.h"
#include "server-queue.h"
@ -6,12 +8,15 @@
#include <cstddef>
#include <memory>
#include <set>
struct server_context_impl; // private implementation
struct server_context_meta {
std::string build_info;
std::string model_name;
std::set<std::string> model_aliases;
std::set<std::string> model_tags;
std::string model_path;
bool has_mtmd;
bool has_inp_image;
@ -30,6 +35,9 @@ struct server_context_meta {
llama_token fim_pre_token;
llama_token fim_sub_token;
llama_token fim_mid_token;
llama_token fim_pad_token;
llama_token fim_rep_token;
llama_token fim_sep_token;
// model meta
enum llama_vocab_type model_vocab_type;

13
tools/server/server-http.cpp
View file

@ -339,6 +339,17 @@ static std::map<std::string, std::string> get_headers(const httplib::Request & r
return headers;
}
static std::string build_query_string(const httplib::Request & req) {
std::string qs;
for (const auto & [key, value] : req.params) {
if (!qs.empty()) {
qs += '&';
}
qs += httplib::encode_query_component(key) + "=" + httplib::encode_query_component(value);
}
return qs;
}
// using unique_ptr for request to allow safe capturing in lambdas
using server_http_req_ptr = std::unique_ptr<server_http_req>;
@ -382,6 +393,7 @@ void server_http_context::get(const std::string & path, const server_http_contex
get_params(req),
get_headers(req),
req.path,
build_query_string(req),
req.body,
req.is_connection_closed
});
@ -396,6 +408,7 @@ void server_http_context::post(const std::string & path, const server_http_conte
get_params(req),
get_headers(req),
req.path,
build_query_string(req),
req.body,
req.is_connection_closed
});

3
tools/server/server-http.h
View file

@ -36,7 +36,8 @@ using server_http_res_ptr = std::unique_ptr<server_http_res>;
struct server_http_req {
std::map<std::string, std::string> params; // path_params + query_params
std::map<std::string, std::string> headers; // reserved for future use
std::string path; // reserved for future use
std::string path;
std::string query_string; // query parameters string (e.g. "action=save")
std::string body;
const std::function<bool()> & should_stop;

109
tools/server/server-models.cpp
View file

@ -184,6 +184,51 @@ void server_models::add_model(server_model_meta && meta) {
if (mapping.find(meta.name) != mapping.end()) {
throw std::runtime_error(string_format("model '%s' appears multiple times", meta.name.c_str()));
}
// check model name does not conflict with existing aliases
for (const auto & [key, inst] : mapping) {
if (inst.meta.aliases.count(meta.name)) {
throw std::runtime_error(string_format("model name '%s' conflicts with alias of model '%s'",
meta.name.c_str(), key.c_str()));
}
}
// parse aliases from preset's --alias option (comma-separated)
std::string alias_str;
if (meta.preset.get_option("LLAMA_ARG_ALIAS", alias_str) && !alias_str.empty()) {
for (auto & alias : string_split<std::string>(alias_str, ',')) {
alias = string_strip(alias);
if (!alias.empty()) {
meta.aliases.insert(alias);
}
}
}
// parse tags from preset's --tags option (comma-separated)
std::string tags_str;
if (meta.preset.get_option("LLAMA_ARG_TAGS", tags_str) && !tags_str.empty()) {
for (auto & tag : string_split<std::string>(tags_str, ',')) {
tag = string_strip(tag);
if (!tag.empty()) {
meta.tags.insert(tag);
}
}
}
// validate aliases do not conflict with existing names or aliases
for (const auto & alias : meta.aliases) {
if (mapping.find(alias) != mapping.end()) {
throw std::runtime_error(string_format("alias '%s' for model '%s' conflicts with existing model name",
alias.c_str(), meta.name.c_str()));
}
for (const auto & [key, inst] : mapping) {
if (inst.meta.aliases.count(alias)) {
throw std::runtime_error(string_format("alias '%s' for model '%s' conflicts with alias of model '%s'",
alias.c_str(), meta.name.c_str(), key.c_str()));
}
}
}
meta.update_args(ctx_preset, bin_path); // render args
std::string name = meta.name;
mapping[name] = instance_t{
@ -249,6 +294,8 @@ void server_models::load_models() {
server_model_meta meta{
/* preset */ preset.second,
/* name */ preset.first,
/* aliases */ {},
/* tags */ {},
/* port */ 0,
/* status */ SERVER_MODEL_STATUS_UNLOADED,
/* last_used */ 0,
@ -265,10 +312,28 @@ void server_models::load_models() {
for (const auto & [name, preset] : custom_presets) {
custom_names.insert(name);
}
auto join_set = [](const std::set<std::string> & s) {
std::string result;
for (const auto & v : s) {
if (!result.empty()) {
result += ", ";
}
result += v;
}
return result;
};
SRV_INF("Available models (%zu) (*: custom preset)\n", mapping.size());
for (const auto & [name, inst] : mapping) {
bool has_custom = custom_names.find(name) != custom_names.end();
SRV_INF(" %c %s\n", has_custom ? '*' : ' ', name.c_str());
std::string info;
if (!inst.meta.aliases.empty()) {
info += " (aliases: " + join_set(inst.meta.aliases) + ")";
}
if (!inst.meta.tags.empty()) {
info += " [tags: " + join_set(inst.meta.tags) + "]";
}
SRV_INF(" %c %s%s\n", has_custom ? '*' : ' ', name.c_str(), info.c_str());
}
}
@ -291,7 +356,9 @@ void server_models::load_models() {
for (const auto & [name, inst] : mapping) {
std::string val;
if (inst.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val)) {
models_to_load.push_back(name);
if (common_arg_utils::is_truthy(val)) {
models_to_load.push_back(name);
}
}
}
if ((int)models_to_load.size() > base_params.models_max) {
@ -318,7 +385,15 @@ void server_models::update_meta(const std::string & name, const server_model_met
bool server_models::has_model(const std::string & name) {
std::lock_guard<std::mutex> lk(mutex);
return mapping.find(name) != mapping.end();
if (mapping.find(name) != mapping.end()) {
return true;
}
for (const auto & [key, inst] : mapping) {
if (inst.meta.aliases.count(name)) {
return true;
}
}
return false;
}
std::optional<server_model_meta> server_models::get_meta(const std::string & name) {
@ -327,6 +402,11 @@ std::optional<server_model_meta> server_models::get_meta(const std::string & nam
if (it != mapping.end()) {
return it->second.meta;
}
for (const auto & [key, inst] : mapping) {
if (inst.meta.aliases.count(name)) {
return inst.meta;
}
}
return std::nullopt;
}
@ -697,11 +777,15 @@ server_http_res_ptr server_models::proxy_request(const server_http_req & req, co
mapping[name].meta.last_used = ggml_time_ms();
}
SRV_INF("proxying request to model %s on port %d\n", name.c_str(), meta->port);
std::string proxy_path = req.path;
if (!req.query_string.empty()) {
proxy_path += '?' + req.query_string;
}
auto proxy = std::make_unique<server_http_proxy>(
method,
CHILD_ADDR,
meta->port,
req.path,
proxy_path,
req.headers,
req.body,
req.should_stop,
@ -760,7 +844,7 @@ static void res_err(std::unique_ptr<server_http_res> & res, const json & error_d
res->data = safe_json_to_str({{ "error", error_data }});
}
static bool router_validate_model(const std::string & name, server_models & models, bool models_autoload, std::unique_ptr<server_http_res> & res) {
static bool router_validate_model(std::string & name, server_models & models, bool models_autoload, std::unique_ptr<server_http_res> & res) {
if (name.empty()) {
res_err(res, format_error_response("model name is missing from the request", ERROR_TYPE_INVALID_REQUEST));
return false;
@ -770,6 +854,8 @@ static bool router_validate_model(const std::string & name, server_models & mode
res_err(res, format_error_response(string_format("model '%s' not found", name.c_str()), ERROR_TYPE_INVALID_REQUEST));
return false;
}
// resolve alias to canonical model name
name = meta->name;
if (models_autoload) {
models.ensure_model_loaded(name);
} else {
@ -841,16 +927,16 @@ void server_models_routes::init_routes() {
auto res = std::make_unique<server_http_res>();
json body = json::parse(req.body);
std::string name = json_value(body, "model", std::string());
auto model = models.get_meta(name);
if (!model.has_value()) {
auto meta = models.get_meta(name);
if (!meta.has_value()) {
res_err(res, format_error_response("model is not found", ERROR_TYPE_NOT_FOUND));
return res;
}
if (model->status == SERVER_MODEL_STATUS_LOADED) {
if (meta->status == SERVER_MODEL_STATUS_LOADED) {
res_err(res, format_error_response("model is already loaded", ERROR_TYPE_INVALID_REQUEST));
return res;
}
models.load(name);
models.load(meta->name);
res_ok(res, {{"success", true}});
return res;
};
@ -871,6 +957,7 @@ void server_models_routes::init_routes() {
preset_copy.unset_option("LLAMA_ARG_HOST");
preset_copy.unset_option("LLAMA_ARG_PORT");
preset_copy.unset_option("LLAMA_ARG_ALIAS");
preset_copy.unset_option("LLAMA_ARG_TAGS");
status["preset"] = preset_copy.to_ini();
}
if (meta.is_failed()) {
@ -879,6 +966,8 @@ void server_models_routes::init_routes() {
}
models_json.push_back(json {
{"id", meta.name},
{"aliases", meta.aliases},
{"tags", meta.tags},
{"object", "model"}, // for OAI-compat
{"owned_by", "llamacpp"}, // for OAI-compat
{"created", t}, // for OAI-compat
@ -906,7 +995,7 @@ void server_models_routes::init_routes() {
res_err(res, format_error_response("model is not loaded", ERROR_TYPE_INVALID_REQUEST));
return res;
}
models.unload(name);
models.unload(model->name);
res_ok(res, {{"success", true}});
return res;
};

2
tools/server/server-models.h
View file

@ -52,6 +52,8 @@ static std::string server_model_status_to_string(server_model_status status) {
struct server_model_meta {
common_preset preset;
std::string name;
std::set<std::string> aliases; // additional names that resolve to this model
std::set<std::string> tags; // informational tags, not used for routing
int port = 0;
server_model_status status = SERVER_MODEL_STATUS_UNLOADED;
int64_t last_used = 0; // for LRU unloading

6
tools/server/server-task.cpp
View file

@ -204,7 +204,8 @@ task_params server_task::params_from_json_cmpl(
params.cache_prompt = json_value(data, "cache_prompt", defaults.cache_prompt);
params.return_tokens = json_value(data, "return_tokens", false);
params.return_progress = json_value(data, "return_progress", false);
params.n_predict = json_value(data, "n_predict", json_value(data, "max_tokens", defaults.n_predict));
auto max_tokens = json_value(data, "max_tokens", defaults.n_predict);
params.n_predict = json_value(data, "n_predict", json_value(data, "max_completion_tokens", max_tokens));
params.n_indent = json_value(data, "n_indent", defaults.n_indent);
params.n_keep = json_value(data, "n_keep", defaults.n_keep);
params.n_discard = json_value(data, "n_discard", defaults.n_discard);
@ -1899,10 +1900,9 @@ server_prompt * server_prompt_cache::alloc(const server_prompt & prompt, size_t
return nullptr;
}
// TODO: for some reason we can't copy server_tokens, so we have to do this workaround
auto & cur = states.emplace_back();
cur = {
/*.tokens =*/ server_tokens(prompt.tokens.get_text_tokens(), false),
/*.tokens =*/ prompt.tokens.clone(),
/*.data =*/ std::move(state_data),
/*.checkpoints =*/ prompt.checkpoints,
};

2
tools/server/server-task.h
View file

@ -557,6 +557,8 @@ struct server_prompt_checkpoint {
llama_pos pos_min;
llama_pos pos_max;
int64_t n_tokens;
std::vector<uint8_t> data;
size_t size() const {

3
tools/server/server.cpp
View file

@ -92,7 +92,7 @@ int main(int argc, char ** argv) {
// for consistency between server router mode and single-model mode, we set the same model name as alias
if (params.model_alias.empty() && !params.model.name.empty()) {
params.model_alias = params.model.name;
params.model_alias.insert(params.model.name);
}
common_init();
@ -178,6 +178,7 @@ int main(int argc, char ** argv) {
ctx_http.post("/v1/chat/completions", ex_wrapper(routes.post_chat_completions));
ctx_http.post("/api/chat", ex_wrapper(routes.post_chat_completions)); // ollama specific endpoint
ctx_http.post("/v1/responses", ex_wrapper(routes.post_responses_oai));
ctx_http.post("/responses", ex_wrapper(routes.post_responses_oai));
ctx_http.post("/v1/messages", ex_wrapper(routes.post_anthropic_messages)); // anthropic messages API
ctx_http.post("/v1/messages/count_tokens", ex_wrapper(routes.post_anthropic_count_tokens)); // anthropic token counting
ctx_http.post("/infill", ex_wrapper(routes.post_infill));

17
tools/server/tests/unit/test_basic.py
View file

@ -94,3 +94,20 @@ def test_no_webui():
server.start()
res = requests.get(url)
assert res.status_code == 404
def test_server_model_aliases_and_tags():
global server
server.model_alias = "tinyllama-2,fim,code"
server.model_tags = "chat,fim,small"
server.start()
res = server.make_request("GET", "/models")
assert res.status_code == 200
assert len(res.body["data"]) == 1
model = res.body["data"][0]
# aliases field must contain all aliases
assert set(model["aliases"]) == {"tinyllama-2", "fim", "code"}
# tags field must contain all tags
assert set(model["tags"]) == {"chat", "fim", "small"}
# id is derived from first alias (alphabetical order from std::set)
assert model["id"] == "code"

3
tools/server/tests/utils.py
View file

@ -56,6 +56,7 @@ class ServerProcess:
# custom options
model_alias: str | None = None
model_tags: str | None = None
model_url: str | None = None
model_file: str | None = None
model_draft: str | None = None
@ -180,6 +181,8 @@ class ServerProcess:
server_args.extend(["--pooling", self.pooling])
if self.model_alias:
server_args.extend(["--alias", self.model_alias])
if self.model_tags:
server_args.extend(["--tags", self.model_tags])
if self.n_ctx:
server_args.extend(["--ctx-size", self.n_ctx])
if self.n_slots:

17
tools/server/webui/.storybook/main.ts
View file

@ -1,17 +1,24 @@
import type { StorybookConfig } from '@storybook/sveltekit';
import { dirname, resolve } from 'path';
import { fileURLToPath } from 'url';
const __dirname = dirname(fileURLToPath(import.meta.url));
const config: StorybookConfig = {
stories: ['../tests/stories/**/*.mdx', '../tests/stories/**/*.stories.@(js|ts|svelte)'],
addons: [
'@storybook/addon-svelte-csf',
'@chromatic-com/storybook',
'@storybook/addon-docs',
'@storybook/addon-vitest',
'@storybook/addon-a11y',
'@storybook/addon-vitest'
'@storybook/addon-docs'
],
framework: {
name: '@storybook/sveltekit',
options: {}
framework: '@storybook/sveltekit',
viteFinal: async (config) => {
config.server = config.server || {};
config.server.fs = config.server.fs || {};
config.server.fs.allow = [...(config.server.fs.allow || []), resolve(__dirname, '../tests')];
return config;
}
};
export default config;

2
tools/server/webui/.storybook/preview.ts
View file

@ -13,7 +13,7 @@ const preview: Preview = {
},
backgrounds: {
disable: true
disabled: true
},
a11y: {

2
tools/server/webui/README.md
View file

@ -101,7 +101,7 @@ In a separate terminal, start the backend server:
./llama-server -m model.gguf
# Multi-model (ROUTER mode)
./llama-server --model-store /path/to/models
./llama-server --models-dir /path/to/models
```
### 3. Start Development Servers

40
tools/server/webui/docs/flows/settings-flow.md
View file

@ -49,14 +49,20 @@ sequenceDiagram
settingsStore->>serverStore: defaultParams
serverStore-->>settingsStore: {temperature, top_p, top_k, ...}
settingsStore->>ParamSvc: extractServerDefaults(defaultParams)
ParamSvc-->>settingsStore: Record<string, value>
loop each SYNCABLE_PARAMETER
alt key NOT in userOverrides
settingsStore->>settingsStore: config[key] = serverDefault[key]
Note right of settingsStore: Non-overridden params adopt server default
else key in userOverrides
Note right of settingsStore: Keep user value, skip server default
end
end
settingsStore->>ParamSvc: mergeWithServerDefaults(config, serverDefaults)
Note right of ParamSvc: For each syncable parameter:<br/>- If NOT in userOverrides → use server default<br/>- If in userOverrides → keep user value
ParamSvc-->>settingsStore: mergedConfig
alt serverStore.props has webuiSettings
settingsStore->>settingsStore: Apply webuiSettings from server
Note right of settingsStore: Server-provided UI settings<br/>(e.g. showRawOutputSwitch)
end
settingsStore->>settingsStore: config = mergedConfig
settingsStore->>settingsStore: saveConfig()
deactivate settingsStore
@ -67,11 +73,18 @@ sequenceDiagram
UI->>settingsStore: updateConfig(key, value)
activate settingsStore
settingsStore->>settingsStore: config[key] = value
settingsStore->>settingsStore: userOverrides.add(key)
Note right of settingsStore: Mark as user-modified (won't be overwritten by server)
alt value matches server default for key
settingsStore->>settingsStore: userOverrides.delete(key)
Note right of settingsStore: Matches server default, remove override
else value differs from server default
settingsStore->>settingsStore: userOverrides.add(key)
Note right of settingsStore: Mark as user-modified (won't be overwritten)
end
settingsStore->>settingsStore: saveConfig()
settingsStore->>LS: set("llama-config", config)
settingsStore->>LS: set("llama-userOverrides", [...userOverrides])
settingsStore->>LS: set(CONFIG_LOCALSTORAGE_KEY, config)
settingsStore->>LS: set(USER_OVERRIDES_LOCALSTORAGE_KEY, [...userOverrides])
deactivate settingsStore
UI->>settingsStore: updateMultipleConfig({key1: val1, key2: val2})
@ -88,10 +101,9 @@ sequenceDiagram
UI->>settingsStore: resetConfig()
activate settingsStore
settingsStore->>settingsStore: config = SETTING_CONFIG_DEFAULT
settingsStore->>settingsStore: config = {...SETTING_CONFIG_DEFAULT}
settingsStore->>settingsStore: userOverrides.clear()
settingsStore->>settingsStore: syncWithServerDefaults()
Note right of settingsStore: Apply server defaults for syncable params
Note right of settingsStore: All params reset to defaults<br/>Next syncWithServerDefaults will adopt server values
settingsStore->>settingsStore: saveConfig()
deactivate settingsStore
@ -139,6 +151,6 @@ sequenceDiagram
Note over settingsStore: UI-only (not synced):
rect rgb(255, 240, 240)
Note over settingsStore: systemMessage, custom (JSON)<br/>showStatistics, enableContinueGeneration<br/>autoMicOnEmpty, disableAutoScroll<br/>apiKey, pdfAsImage, disableReasoningFormat
Note over settingsStore: systemMessage, custom (JSON)<br/>showStatistics, enableContinueGeneration<br/>autoMicOnEmpty, disableAutoScroll<br/>apiKey, pdfAsImage, disableReasoningParsing, showRawOutputSwitch
end
```

4
tools/server/webui/eslint.config.js
View file

@ -27,7 +27,9 @@ export default ts.config(
// typescript-eslint strongly recommend that you do not use the no-undef lint rule on TypeScript projects.
// see: https://typescript-eslint.io/troubleshooting/faqs/eslint/#i-get-errors-from-the-no-undef-rule-about-global-variables-not-being-defined-even-though-there-are-no-typescript-errors
'no-undef': 'off',
'svelte/no-at-html-tags': 'off'
'svelte/no-at-html-tags': 'off',
// This app uses hash-based routing (#/) where resolve() from $app/paths does not apply
'svelte/no-navigation-without-resolve': 'off'
}
},
{

1540
tools/server/webui/package-lock.json generated
View file

File diff suppressed because it is too large Load diff

17
tools/server/webui/package.json
View file

@ -23,31 +23,32 @@
"cleanup": "rm -rf .svelte-kit build node_modules test-results"
},
"devDependencies": {
"@chromatic-com/storybook": "^4.1.2",
"@chromatic-com/storybook": "^5.0.0",
"@eslint/compat": "^1.2.5",
"@eslint/js": "^9.18.0",
"@internationalized/date": "^3.10.1",
"@lucide/svelte": "^0.515.0",
"@playwright/test": "^1.49.1",
"@storybook/addon-a11y": "^10.0.7",
"@storybook/addon-docs": "^10.0.7",
"@storybook/addon-a11y": "^10.2.4",
"@storybook/addon-docs": "^10.2.4",
"@storybook/addon-svelte-csf": "^5.0.10",
"@storybook/addon-vitest": "^10.0.7",
"@storybook/sveltekit": "^10.0.7",
"@storybook/addon-vitest": "^10.2.4",
"@storybook/sveltekit": "^10.2.4",
"@sveltejs/adapter-static": "^3.0.10",
"@sveltejs/kit": "^2.48.4",
"@sveltejs/vite-plugin-svelte": "^6.2.1",
"@tailwindcss/forms": "^0.5.9",
"@tailwindcss/typography": "^0.5.15",
"@tailwindcss/vite": "^4.0.0",
"@types/node": "^22",
"@types/node": "^24",
"@vitest/browser": "^3.2.3",
"@vitest/coverage-v8": "^3.2.3",
"bits-ui": "^2.14.4",
"clsx": "^2.1.1",
"dexie": "^4.0.11",
"eslint": "^9.18.0",
"eslint-config-prettier": "^10.0.1",
"eslint-plugin-storybook": "^10.0.7",
"eslint-plugin-storybook": "^10.2.4",
"eslint-plugin-svelte": "^3.0.0",
"fflate": "^0.8.2",
"globals": "^16.0.0",
@ -61,7 +62,7 @@
"rehype-katex": "^7.0.1",
"remark-math": "^6.0.0",
"sass": "^1.93.3",
"storybook": "^10.0.7",
"storybook": "^10.2.4",
"svelte": "^5.38.2",
"svelte-check": "^4.0.0",
"tailwind-merge": "^3.3.1",

2
tools/server/webui/src/lib/components/app/actions/ActionIconsCodeBlock.svelte
View file

@ -1,6 +1,6 @@
<script lang="ts">
import { Eye } from '@lucide/svelte';
import ActionIconCopyToClipboard from '$lib/components/app/actions/ActionIconCopyToClipboard.svelte';
import { ActionIconCopyToClipboard } from '$lib/components/app';
import { FileTypeText } from '$lib/enums';
interface Props {

5
tools/server/webui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentPreview.svelte
View file

@ -8,7 +8,8 @@
isImageFile,
isPdfFile,
isAudioFile,
getLanguageFromFilename
getLanguageFromFilename,
createBase64DataUrl
} from '$lib/utils';
import { convertPDFToImage } from '$lib/utils/browser-only';
import { modelsStore } from '$lib/stores/models.svelte';
@ -255,7 +256,7 @@
<audio
controls
class="mb-4 w-full"
src={`data:${attachment.mimeType};base64,${attachment.base64Data}`}
src={createBase64DataUrl(attachment.mimeType, attachment.base64Data)}
>
Your browser does not support the audio element.
</audio>

6
tools/server/webui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentThumbnailFile.svelte
View file

@ -1,5 +1,5 @@
<script lang="ts">
import { RemoveButton } from '$lib/components/app';
import { ActionIconRemove } from '$lib/components/app';
import { formatFileSize, getFileTypeLabel, getPreviewText, isTextFile } from '$lib/utils';
import { AttachmentType } from '$lib/enums';
@ -104,7 +104,7 @@
onclick={onClick}
>
<div class="absolute top-2 right-2 opacity-0 transition-opacity group-hover:opacity-100">
<RemoveButton {id} {onRemove} />
<ActionIconRemove {id} {onRemove} />
</div>
<div class="pr-8">
@ -158,7 +158,7 @@
{#if !readonly}
<div class="absolute top-2 right-2 opacity-0 transition-opacity group-hover:opacity-100">
<RemoveButton {id} {onRemove} />
<ActionIconRemove {id} {onRemove} />
</div>
{/if}
</button>

4
tools/server/webui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentThumbnailImage.svelte
View file

@ -1,5 +1,5 @@
<script lang="ts">
import { RemoveButton } from '$lib/components/app';
import { ActionIconRemove } from '$lib/components/app';
interface Props {
id: string;
@ -58,7 +58,7 @@
<div
class="absolute top-1 right-1 flex items-center justify-center opacity-0 transition-opacity group-hover:opacity-100"
>
<RemoveButton {id} {onRemove} class="text-white" />
<ActionIconRemove {id} {onRemove} class="text-white" />
</div>
{/if}
</div>

145
tools/server/webui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList.svelte
View file

@ -1,8 +1,12 @@
<script lang="ts">
import { ChatAttachmentThumbnailImage, ChatAttachmentThumbnailFile } from '$lib/components/app';
import {
ChatAttachmentThumbnailImage,
ChatAttachmentThumbnailFile,
HorizontalScrollCarousel,
DialogChatAttachmentPreview,
DialogChatAttachmentsViewAll
} from '$lib/components/app';
import { Button } from '$lib/components/ui/button';
import { ChevronLeft, ChevronRight } from '@lucide/svelte';
import { DialogChatAttachmentPreview, DialogChatAttachmentsViewAll } from '$lib/components/app';
import { getAttachmentDisplayItems } from '$lib/utils';
interface Props {
@ -41,12 +45,10 @@
let displayItems = $derived(getAttachmentDisplayItems({ uploadedFiles, attachments }));
let canScrollLeft = $state(false);
let canScrollRight = $state(false);
let carouselRef: HorizontalScrollCarousel | undefined = $state();
let isScrollable = $state(false);
let previewDialogOpen = $state(false);
let previewItem = $state<ChatAttachmentPreviewItem | null>(null);
let scrollContainer: HTMLDivElement | undefined = $state();
let showViewAll = $derived(limitToSingleRow && displayItems.length > 0 && isScrollable);
let viewAllDialogOpen = $state(false);
@ -65,41 +67,9 @@
previewDialogOpen = true;
}
function scrollLeft(event?: MouseEvent) {
event?.stopPropagation();
event?.preventDefault();
if (!scrollContainer) return;
scrollContainer.scrollBy({ left: scrollContainer.clientWidth * -0.67, behavior: 'smooth' });
}
function scrollRight(event?: MouseEvent) {
event?.stopPropagation();
event?.preventDefault();
if (!scrollContainer) return;
scrollContainer.scrollBy({ left: scrollContainer.clientWidth * 0.67, behavior: 'smooth' });
}
function updateScrollButtons() {
if (!scrollContainer) return;
const { scrollLeft, scrollWidth, clientWidth } = scrollContainer;
canScrollLeft = scrollLeft > 0;
canScrollRight = scrollLeft < scrollWidth - clientWidth - 1;
isScrollable = scrollWidth > clientWidth;
}
$effect(() => {
if (scrollContainer && displayItems.length) {
scrollContainer.scrollLeft = 0;
setTimeout(() => {
updateScrollButtons();
}, 0);
if (carouselRef && displayItems.length) {
carouselRef.resetScroll();
}
});
</script>
@ -107,67 +77,40 @@
{#if displayItems.length > 0}
<div class={className} {style}>
{#if limitToSingleRow}
<div class="relative">
<button
class="absolute top-1/2 left-4 z-10 flex h-6 w-6 -translate-y-1/2 items-center justify-center rounded-full bg-foreground/15 shadow-md backdrop-blur-xs transition-opacity hover:bg-foreground/35 {canScrollLeft
? 'opacity-100'
: 'pointer-events-none opacity-0'}"
onclick={scrollLeft}
aria-label="Scroll left"
>
<ChevronLeft class="h-4 w-4" />
</button>
<div
class="scrollbar-hide flex items-start gap-3 overflow-x-auto"
bind:this={scrollContainer}
onscroll={updateScrollButtons}
>
{#each displayItems as item (item.id)}
{#if item.isImage && item.preview}
<ChatAttachmentThumbnailImage
class="flex-shrink-0 cursor-pointer {limitToSingleRow
? 'first:ml-4 last:mr-4'
: ''}"
id={item.id}
name={item.name}
preview={item.preview}
{readonly}
onRemove={onFileRemove}
height={imageHeight}
width={imageWidth}
{imageClass}
onClick={(event) => openPreview(item, event)}
/>
{:else}
<ChatAttachmentThumbnailFile
class="flex-shrink-0 cursor-pointer {limitToSingleRow
? 'first:ml-4 last:mr-4'
: ''}"
id={item.id}
name={item.name}
size={item.size}
{readonly}
onRemove={onFileRemove}
textContent={item.textContent}
attachment={item.attachment}
uploadedFile={item.uploadedFile}
onClick={(event) => openPreview(item, event)}
/>
{/if}
{/each}
</div>
<button
class="absolute top-1/2 right-4 z-10 flex h-6 w-6 -translate-y-1/2 items-center justify-center rounded-full bg-foreground/15 shadow-md backdrop-blur-xs transition-opacity hover:bg-foreground/35 {canScrollRight
? 'opacity-100'
: 'pointer-events-none opacity-0'}"
onclick={scrollRight}
aria-label="Scroll right"
>
<ChevronRight class="h-4 w-4" />
</button>
</div>
<HorizontalScrollCarousel
bind:this={carouselRef}
onScrollableChange={(scrollable) => (isScrollable = scrollable)}
>
{#each displayItems as item (item.id)}
{#if item.isImage && item.preview}
<ChatAttachmentThumbnailImage
class="flex-shrink-0 cursor-pointer {limitToSingleRow ? 'first:ml-4 last:mr-4' : ''}"
id={item.id}
name={item.name}
preview={item.preview}
{readonly}
onRemove={onFileRemove}
height={imageHeight}
width={imageWidth}
{imageClass}
onClick={(event) => openPreview(item, event)}
/>
{:else}
<ChatAttachmentThumbnailFile
class="flex-shrink-0 cursor-pointer {limitToSingleRow ? 'first:ml-4 last:mr-4' : ''}"
id={item.id}
name={item.name}
size={item.size}
{readonly}
onRemove={onFileRemove}
textContent={item.textContent}
attachment={item.attachment}
uploadedFile={item.uploadedFile}
onClick={(event) => openPreview(item, event)}
/>
{/if}
{/each}
</HorizontalScrollCarousel>
{#if showViewAll}
<div class="mt-2 -mr-2 flex justify-end px-4">

370
tools/server/webui/src/lib/components/app/chat/ChatForm/ChatForm.svelte
View file

@ -1,20 +1,19 @@
<script lang="ts">
import { afterNavigate } from '$app/navigation';
import {
ChatAttachmentsList,
ChatFormActions,
ChatFormFileInputInvisible,
ChatFormHelperText,
ChatFormTextarea
} from '$lib/components/app';
import { INPUT_CLASSES } from '$lib/constants/input-classes';
import { INPUT_CLASSES } from '$lib/constants/css-classes';
import { SETTING_CONFIG_DEFAULT } from '$lib/constants/settings-config';
import { CLIPBOARD_CONTENT_QUOTE_PREFIX } from '$lib/constants/chat-form';
import { KeyboardKey, MimeTypeText } from '$lib/enums';
import { config } from '$lib/stores/settings.svelte';
import { modelOptions, selectedModelId } from '$lib/stores/models.svelte';
import { isRouterMode } from '$lib/stores/server.svelte';
import { chatStore } from '$lib/stores/chat.svelte';
import { activeMessages } from '$lib/stores/conversations.svelte';
import { MimeTypeText } from '$lib/enums';
import { isIMEComposing, parseClipboardContent } from '$lib/utils';
import {
AudioRecorder,
@ -25,68 +24,82 @@
import { onMount } from 'svelte';
interface Props {
// Data
attachments?: DatabaseMessageExtra[];
uploadedFiles?: ChatUploadedFile[];
value?: string;
// UI State
class?: string;
disabled?: boolean;
initialMessage?: string;
isLoading?: boolean;
onFileRemove?: (fileId: string) => void;
onFileUpload?: (files: File[]) => void;
onSend?: (message: string, files?: ChatUploadedFile[]) => Promise<boolean>;
placeholder?: string;
// Event Handlers
onAttachmentRemove?: (index: number) => void;
onFilesAdd?: (files: File[]) => void;
onStop?: () => void;
onSystemPromptAdd?: (draft: { message: string; files: ChatUploadedFile[] }) => void;
showHelperText?: boolean;
uploadedFiles?: ChatUploadedFile[];
onSubmit?: () => void;
onSystemPromptClick?: (draft: { message: string; files: ChatUploadedFile[] }) => void;
onUploadedFileRemove?: (fileId: string) => void;
onValueChange?: (value: string) => void;
}
let {
class: className,
attachments = [],
class: className = '',
disabled = false,
initialMessage = '',
isLoading = false,
onFileRemove,
onFileUpload,
onSend,
placeholder = 'Type a message...',
uploadedFiles = $bindable([]),
value = $bindable(''),
onAttachmentRemove,
onFilesAdd,
onStop,
onSystemPromptAdd,
showHelperText = true,
uploadedFiles = $bindable([])
onSubmit,
onSystemPromptClick,
onUploadedFileRemove,
onValueChange
}: Props = $props();
/**
*
*
* STATE
*
*
*/
// Component References
let audioRecorder: AudioRecorder | undefined;
let chatFormActionsRef: ChatFormActions | undefined = $state(undefined);
let currentConfig = $derived(config());
let fileInputRef: ChatFormFileInputInvisible | undefined = $state(undefined);
let textareaRef: ChatFormTextarea | undefined = $state(undefined);
// Audio Recording State
let isRecording = $state(false);
let message = $state(initialMessage);
let recordingSupported = $state(false);
/**
*
*
* DERIVED STATE
*
*
*/
// Configuration
let currentConfig = $derived(config());
let pasteLongTextToFileLength = $derived.by(() => {
const n = Number(currentConfig.pasteLongTextToFileLen);
return Number.isNaN(n) ? Number(SETTING_CONFIG_DEFAULT.pasteLongTextToFileLen) : n;
});
let previousIsLoading = $state(isLoading);
let previousInitialMessage = $state(initialMessage);
let recordingSupported = $state(false);
let textareaRef: ChatFormTextarea | undefined = $state(undefined);
// Sync message when initialMessage prop changes (e.g., after draft restoration)
$effect(() => {
if (initialMessage !== previousInitialMessage) {
message = initialMessage;
previousInitialMessage = initialMessage;
}
});
function handleSystemPromptClick() {
onSystemPromptAdd?.({ message, files: uploadedFiles });
}
// Check if model is selected (in ROUTER mode)
// Model Selection Logic
let isRouter = $derived(isRouterMode());
let conversationModel = $derived(
chatStore.getConversationModel(activeMessages() as DatabaseMessage[])
);
let isRouter = $derived(isRouterMode());
let hasModelSelected = $derived(!isRouter || !!conversationModel || !!selectedModelId());
// Get active model ID for capability detection
let activeModelId = $derived.by(() => {
const options = modelOptions();
@ -94,14 +107,12 @@
return options.length > 0 ? options[0].model : null;
}
// First try user-selected model
const selectedId = selectedModelId();
if (selectedId) {
const model = options.find((m) => m.id === selectedId);
if (model) return model.model;
}
// Fallback to conversation model
if (conversationModel) {
const model = options.find((m) => m.model === conversationModel);
if (model) return model.model;
@ -110,46 +121,101 @@
return null;
});
function checkModelSelected(): boolean {
// Form Validation State
let hasModelSelected = $derived(!isRouter || !!conversationModel || !!selectedModelId());
let hasLoadingAttachments = $derived(uploadedFiles.some((f) => f.isLoading));
let hasAttachments = $derived(
(attachments && attachments.length > 0) || (uploadedFiles && uploadedFiles.length > 0)
);
let canSubmit = $derived(value.trim().length > 0 || hasAttachments);
/**
*
*
* LIFECYCLE
*
*
*/
onMount(() => {
recordingSupported = isAudioRecordingSupported();
audioRecorder = new AudioRecorder();
});
/**
*
*
* PUBLIC API
*
*
*/
export function focus() {
textareaRef?.focus();
}
export function resetTextareaHeight() {
textareaRef?.resetHeight();
}
export function openModelSelector() {
chatFormActionsRef?.openModelSelector();
}
/**
* Check if a model is selected, open selector if not
* @returns true if model is selected, false otherwise
*/
export function checkModelSelected(): boolean {
if (!hasModelSelected) {
// Open the model selector
chatFormActionsRef?.openModelSelector();
return false;
}
return true;
}
/**
*
*
* EVENT HANDLERS - File Management
*
*
*/
function handleFileSelect(files: File[]) {
onFileUpload?.(files);
onFilesAdd?.(files);
}
function handleFileUpload() {
fileInputRef?.click();
}
async function handleKeydown(event: KeyboardEvent) {
if (event.key === 'Enter' && !event.shiftKey && !isIMEComposing(event)) {
function handleFileRemove(fileId: string) {
if (fileId.startsWith('attachment-')) {
const index = parseInt(fileId.replace('attachment-', ''), 10);
if (!isNaN(index) && index >= 0 && index < attachments.length) {
onAttachmentRemove?.(index);
}
} else {
onUploadedFileRemove?.(fileId);
}
}
/**
*
*
* EVENT HANDLERS - Input & Keyboard
*
*
*/
function handleKeydown(event: KeyboardEvent) {
if (event.key === KeyboardKey.ENTER && !event.shiftKey && !isIMEComposing(event)) {
event.preventDefault();
if ((!message.trim() && uploadedFiles.length === 0) || disabled || isLoading) return;
if (!canSubmit || disabled || isLoading || hasLoadingAttachments) return;
if (!checkModelSelected()) return;
const messageToSend = message.trim();
const filesToSend = [...uploadedFiles];
message = '';
uploadedFiles = [];
textareaRef?.resetHeight();
const success = await onSend?.(messageToSend, filesToSend);
if (!success) {
message = messageToSend;
uploadedFiles = filesToSend;
}
onSubmit?.();
}
}
@ -163,29 +229,30 @@
if (files.length > 0) {
event.preventDefault();
onFileUpload?.(files);
onFilesAdd?.(files);
return;
}
const text = event.clipboardData.getData(MimeTypeText.PLAIN);
if (text.startsWith('"')) {
if (text.startsWith(CLIPBOARD_CONTENT_QUOTE_PREFIX)) {
const parsed = parseClipboardContent(text);
if (parsed.textAttachments.length > 0) {
event.preventDefault();
value = parsed.message;
onValueChange?.(parsed.message);
message = parsed.message;
const attachmentFiles = parsed.textAttachments.map(
(att) =>
new File([att.content], att.name, {
type: MimeTypeText.PLAIN
})
);
onFileUpload?.(attachmentFiles);
// Handle text attachments as files
if (parsed.textAttachments.length > 0) {
const attachmentFiles = parsed.textAttachments.map(
(att) =>
new File([att.content], att.name, {
type: MimeTypeText.PLAIN
})
);
onFilesAdd?.(attachmentFiles);
}
setTimeout(() => {
textareaRef?.focus();
@ -206,14 +273,21 @@
type: MimeTypeText.PLAIN
});
onFileUpload?.([textFile]);
onFilesAdd?.([textFile]);
}
}
/**
*
*
* EVENT HANDLERS - Audio Recording
*
*
*/
async function handleMicClick() {
if (!audioRecorder || !recordingSupported) {
console.warn('Audio recording not supported');
return;
}
@ -223,7 +297,7 @@
const wavBlob = await convertToWav(audioBlob);
const audioFile = createAudioFile(wavBlob);
onFileUpload?.([audioFile]);
onFilesAdd?.([audioFile]);
isRecording = false;
} catch (error) {
console.error('Failed to stop recording:', error);
@ -238,96 +312,64 @@
}
}
}
function handleStop() {
onStop?.();
}
async function handleSubmit(event: SubmitEvent) {
event.preventDefault();
if ((!message.trim() && uploadedFiles.length === 0) || disabled || isLoading) return;
// Check if model is selected first
if (!checkModelSelected()) return;
const messageToSend = message.trim();
const filesToSend = [...uploadedFiles];
message = '';
uploadedFiles = [];
textareaRef?.resetHeight();
const success = await onSend?.(messageToSend, filesToSend);
if (!success) {
message = messageToSend;
uploadedFiles = filesToSend;
}
}
onMount(() => {
setTimeout(() => textareaRef?.focus(), 10);
recordingSupported = isAudioRecordingSupported();
audioRecorder = new AudioRecorder();
});
afterNavigate(() => {
setTimeout(() => textareaRef?.focus(), 10);
});
$effect(() => {
if (previousIsLoading && !isLoading) {
setTimeout(() => textareaRef?.focus(), 10);
}
previousIsLoading = isLoading;
});
</script>
<ChatFormFileInputInvisible bind:this={fileInputRef} onFileSelect={handleFileSelect} />
<form
onsubmit={handleSubmit}
class="{INPUT_CLASSES} border-radius-bottom-none mx-auto max-w-[48rem] overflow-hidden rounded-3xl backdrop-blur-md {disabled
? 'cursor-not-allowed opacity-60'
: ''} {className}"
data-slot="chat-form"
class="relative {className}"
onsubmit={(e) => {
e.preventDefault();
if (!canSubmit || disabled || isLoading || hasLoadingAttachments) return;
onSubmit?.();
}}
>
<ChatAttachmentsList
bind:uploadedFiles
{onFileRemove}
limitToSingleRow
class="py-5"
style="scroll-padding: 1rem;"
activeModelId={activeModelId ?? undefined}
/>
<div
class="flex-column relative min-h-[48px] items-center rounded-3xl px-5 py-3 shadow-sm transition-all focus-within:shadow-md"
onpaste={handlePaste}
class="{INPUT_CLASSES} overflow-hidden rounded-3xl backdrop-blur-md {disabled
? 'cursor-not-allowed opacity-60'
: ''}"
data-slot="input-area"
>
<ChatFormTextarea
bind:this={textareaRef}
bind:value={message}
onKeydown={handleKeydown}
{disabled}
<ChatAttachmentsList
{attachments}
bind:uploadedFiles
onFileRemove={handleFileRemove}
limitToSingleRow
class="py-5"
style="scroll-padding: 1rem;"
activeModelId={activeModelId ?? undefined}
/>
<ChatFormActions
bind:this={chatFormActionsRef}
canSend={message.trim().length > 0 || uploadedFiles.length > 0}
hasText={message.trim().length > 0}
{disabled}
{isLoading}
{isRecording}
{uploadedFiles}
onFileUpload={handleFileUpload}
onMicClick={handleMicClick}
onStop={handleStop}
onSystemPromptClick={handleSystemPromptClick}
/>
<div
class="flex-column relative min-h-[48px] items-center rounded-3xl py-2 pb-2.25 shadow-sm transition-all focus-within:shadow-md md:!py-3"
onpaste={handlePaste}
>
<ChatFormTextarea
class="px-5 py-1.5 md:pt-0"
bind:this={textareaRef}
bind:value
onKeydown={handleKeydown}
onInput={() => {
onValueChange?.(value);
}}
{disabled}
{placeholder}
/>
<ChatFormActions
class="px-3"
bind:this={chatFormActionsRef}
canSend={canSubmit}
hasText={value.trim().length > 0}
{disabled}
{isLoading}
{isRecording}
{uploadedFiles}
onFileUpload={handleFileUpload}
onMicClick={handleMicClick}
{onStop}
onSystemPromptClick={() => onSystemPromptClick?.({ message: value, files: uploadedFiles })}
/>
</div>
</div>
</form>
<ChatFormHelperText show={showHelperText} />

173
tools/server/webui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAttachmentsDropdown.svelte Normal file
View file

@ -0,0 +1,173 @@
<script lang="ts">
import { page } from '$app/state';
import { Plus, MessageSquare } from '@lucide/svelte';
import { Button } from '$lib/components/ui/button';
import * as DropdownMenu from '$lib/components/ui/dropdown-menu';
import * as Tooltip from '$lib/components/ui/tooltip';
import { FILE_TYPE_ICONS } from '$lib/constants/icons';
import { TOOLTIP_DELAY_DURATION } from '$lib/constants/tooltip-config';
interface Props {
class?: string;
disabled?: boolean;
hasAudioModality?: boolean;
hasVisionModality?: boolean;
onFileUpload?: () => void;
onSystemPromptClick?: () => void;
}
let {
class: className = '',
disabled = false,
hasAudioModality = false,
hasVisionModality = false,
onFileUpload,
onSystemPromptClick
}: Props = $props();
let isNewChat = $derived(!page.params.id);
let systemMessageTooltip = $derived(
isNewChat
? 'Add custom system message for a new conversation'
: 'Inject custom system message at the beginning of the conversation'
);
let dropdownOpen = $state(false);
const fileUploadTooltipText = 'Add files, system prompt or MCP Servers';
</script>
<div class="flex items-center gap-1 {className}">
<DropdownMenu.Root bind:open={dropdownOpen}>
<DropdownMenu.Trigger name="Attach files" {disabled}>
<Tooltip.Root>
<Tooltip.Trigger class="w-full">
<Button
class="file-upload-button h-8 w-8 rounded-full p-0"
{disabled}
variant="secondary"
type="button"
>
<span class="sr-only">{fileUploadTooltipText}</span>
<Plus class="h-4 w-4" />
</Button>
</Tooltip.Trigger>
<Tooltip.Content>
<p>{fileUploadTooltipText}</p>
</Tooltip.Content>
</Tooltip.Root>
</DropdownMenu.Trigger>
<DropdownMenu.Content align="start" class="w-48">
{#if hasVisionModality}
<DropdownMenu.Item
class="images-button flex cursor-pointer items-center gap-2"
onclick={() => onFileUpload?.()}
>
<FILE_TYPE_ICONS.image class="h-4 w-4" />
<span>Images</span>
</DropdownMenu.Item>
{:else}
<Tooltip.Root delayDuration={TOOLTIP_DELAY_DURATION}>
<Tooltip.Trigger class="w-full">
<DropdownMenu.Item
class="images-button flex cursor-pointer items-center gap-2"
disabled
>
<FILE_TYPE_ICONS.image class="h-4 w-4" />
<span>Images</span>
</DropdownMenu.Item>
</Tooltip.Trigger>
<Tooltip.Content side="right">
<p>Images require vision models to be processed</p>
</Tooltip.Content>
</Tooltip.Root>
{/if}
{#if hasAudioModality}
<DropdownMenu.Item
class="audio-button flex cursor-pointer items-center gap-2"
onclick={() => onFileUpload?.()}
>
<FILE_TYPE_ICONS.audio class="h-4 w-4" />
<span>Audio Files</span>
</DropdownMenu.Item>
{:else}
<Tooltip.Root delayDuration={TOOLTIP_DELAY_DURATION}>
<Tooltip.Trigger class="w-full">
<DropdownMenu.Item class="audio-button flex cursor-pointer items-center gap-2" disabled>
<FILE_TYPE_ICONS.audio class="h-4 w-4" />
<span>Audio Files</span>
</DropdownMenu.Item>
</Tooltip.Trigger>
<Tooltip.Content side="right">
<p>Audio files require audio models to be processed</p>
</Tooltip.Content>
</Tooltip.Root>
{/if}
<DropdownMenu.Item
class="flex cursor-pointer items-center gap-2"
onclick={() => onFileUpload?.()}
>
<FILE_TYPE_ICONS.text class="h-4 w-4" />
<span>Text Files</span>
</DropdownMenu.Item>
{#if hasVisionModality}
<DropdownMenu.Item
class="flex cursor-pointer items-center gap-2"
onclick={() => onFileUpload?.()}
>
<FILE_TYPE_ICONS.pdf class="h-4 w-4" />
<span>PDF Files</span>
</DropdownMenu.Item>
{:else}
<Tooltip.Root delayDuration={TOOLTIP_DELAY_DURATION}>
<Tooltip.Trigger class="w-full">
<DropdownMenu.Item
class="flex cursor-pointer items-center gap-2"
onclick={() => onFileUpload?.()}
>
<FILE_TYPE_ICONS.pdf class="h-4 w-4" />
<span>PDF Files</span>
</DropdownMenu.Item>
</Tooltip.Trigger>
<Tooltip.Content side="right">
<p>PDFs will be converted to text. Image-based PDFs may not work properly.</p>
</Tooltip.Content>
</Tooltip.Root>
{/if}
<Tooltip.Root delayDuration={TOOLTIP_DELAY_DURATION}>
<Tooltip.Trigger class="w-full">
<DropdownMenu.Item
class="flex cursor-pointer items-center gap-2"
onclick={() => onSystemPromptClick?.()}
>
<MessageSquare class="h-4 w-4" />
<span>System Message</span>
</DropdownMenu.Item>
</Tooltip.Trigger>
<Tooltip.Content side="right">
<p>{systemMessageTooltip}</p>
</Tooltip.Content>
</Tooltip.Root>
</DropdownMenu.Content>
</DropdownMenu.Root>
</div>

143
tools/server/webui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionFileAttachments.svelte
View file

@ -1,143 +0,0 @@
<script lang="ts">
import { Paperclip } from '@lucide/svelte';
import { MessageSquare } from '@lucide/svelte';
import { Button } from '$lib/components/ui/button';
import * as DropdownMenu from '$lib/components/ui/dropdown-menu';
import * as Tooltip from '$lib/components/ui/tooltip';
import { FILE_TYPE_ICONS } from '$lib/constants/icons';
interface Props {
class?: string;
disabled?: boolean;
hasAudioModality?: boolean;
hasVisionModality?: boolean;
onFileUpload?: () => void;
onSystemPromptClick?: () => void;
}
let {
class: className = '',
disabled = false,
hasAudioModality = false,
hasVisionModality = false,
onFileUpload,
onSystemPromptClick
}: Props = $props();
const fileUploadTooltipText = $derived.by(() => {
return !hasVisionModality
? 'Text files and PDFs supported. Images, audio, and video require vision models.'
: 'Attach files';
});
</script>
<div class="flex items-center gap-1 {className}">
<DropdownMenu.Root>
<DropdownMenu.Trigger name="Attach files" {disabled}>
<Tooltip.Root>
<Tooltip.Trigger>
<Button
class="file-upload-button h-8 w-8 rounded-full bg-transparent p-0 text-muted-foreground hover:bg-foreground/10 hover:text-foreground"
{disabled}
type="button"
>
<span class="sr-only">Attach files</span>
<Paperclip class="h-4 w-4" />
</Button>
</Tooltip.Trigger>
<Tooltip.Content>
<p>{fileUploadTooltipText}</p>
</Tooltip.Content>
</Tooltip.Root>
</DropdownMenu.Trigger>
<DropdownMenu.Content align="start" class="w-48">
<Tooltip.Root>
<Tooltip.Trigger class="w-full">
<DropdownMenu.Item
class="images-button flex cursor-pointer items-center gap-2"
disabled={!hasVisionModality}
onclick={() => onFileUpload?.()}
>
<FILE_TYPE_ICONS.image class="h-4 w-4" />
<span>Images</span>
</DropdownMenu.Item>
</Tooltip.Trigger>
{#if !hasVisionModality}
<Tooltip.Content>
<p>Images require vision models to be processed</p>
</Tooltip.Content>
{/if}
</Tooltip.Root>
<Tooltip.Root>
<Tooltip.Trigger class="w-full">
<DropdownMenu.Item
class="audio-button flex cursor-pointer items-center gap-2"
disabled={!hasAudioModality}
onclick={() => onFileUpload?.()}
>
<FILE_TYPE_ICONS.audio class="h-4 w-4" />
<span>Audio Files</span>
</DropdownMenu.Item>
</Tooltip.Trigger>
{#if !hasAudioModality}
<Tooltip.Content>
<p>Audio files require audio models to be processed</p>
</Tooltip.Content>
{/if}
</Tooltip.Root>
<DropdownMenu.Item
class="flex cursor-pointer items-center gap-2"
onclick={() => onFileUpload?.()}
>
<FILE_TYPE_ICONS.text class="h-4 w-4" />
<span>Text Files</span>
</DropdownMenu.Item>
<Tooltip.Root>
<Tooltip.Trigger class="w-full">
<DropdownMenu.Item
class="flex cursor-pointer items-center gap-2"
onclick={() => onFileUpload?.()}
>
<FILE_TYPE_ICONS.pdf class="h-4 w-4" />
<span>PDF Files</span>
</DropdownMenu.Item>
</Tooltip.Trigger>
{#if !hasVisionModality}
<Tooltip.Content>
<p>PDFs will be converted to text. Image-based PDFs may not work properly.</p>
</Tooltip.Content>
{/if}
</Tooltip.Root>
<DropdownMenu.Separator />
<Tooltip.Root>
<Tooltip.Trigger class="w-full">
<DropdownMenu.Item
class="flex cursor-pointer items-center gap-2"
onclick={() => onSystemPromptClick?.()}
>
<MessageSquare class="h-4 w-4" />
<span>System Prompt</span>
</DropdownMenu.Item>
</Tooltip.Trigger>
<Tooltip.Content>
<p>Add a custom system message for this conversation</p>
</Tooltip.Content>
</Tooltip.Root>
</DropdownMenu.Content>
</DropdownMenu.Root>
</div>

56
tools/server/webui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte
View file

@ -2,7 +2,7 @@
import { Square } from '@lucide/svelte';
import { Button } from '$lib/components/ui/button';
import {
ChatFormActionFileAttachments,
ChatFormActionAttachmentsDropdown,
ChatFormActionRecord,
ChatFormActionSubmit,
ModelsSelector
@ -13,8 +13,7 @@
import { modelsStore, modelOptions, selectedModelId } from '$lib/stores/models.svelte';
import { isRouterMode } from '$lib/stores/server.svelte';
import { chatStore } from '$lib/stores/chat.svelte';
import { activeMessages, usedModalities } from '$lib/stores/conversations.svelte';
import { useModelChangeValidation } from '$lib/hooks/use-model-change-validation.svelte';
import { activeMessages } from '$lib/stores/conversations.svelte';
interface Props {
canSend?: boolean;
@ -154,44 +153,41 @@
export function openModelSelector() {
selectorModelRef?.open();
}
const { handleModelChange } = useModelChangeValidation({
getRequiredModalities: () => usedModalities(),
onValidationFailure: async (previousModelId) => {
if (previousModelId) {
await modelsStore.selectModelById(previousModelId);
}
}
});
</script>
<div class="flex w-full items-center gap-3 {className}" style="container-type: inline-size">
<ChatFormActionFileAttachments
class="mr-auto"
{disabled}
{hasAudioModality}
{hasVisionModality}
{onFileUpload}
{onSystemPromptClick}
/>
<div class="mr-auto flex items-center gap-2">
<ChatFormActionAttachmentsDropdown
{disabled}
{hasAudioModality}
{hasVisionModality}
{onFileUpload}
{onSystemPromptClick}
/>
</div>
<ModelsSelector
{disabled}
bind:this={selectorModelRef}
currentModel={conversationModel}
forceForegroundText={true}
useGlobalSelection={true}
onModelChange={handleModelChange}
/>
<div class="ml-auto flex items-center gap-1.5">
<ModelsSelector
{disabled}
bind:this={selectorModelRef}
currentModel={conversationModel}
forceForegroundText={true}
useGlobalSelection={true}
/>
</div>
{#if isLoading}
<Button
type="button"
variant="secondary"
onclick={onStop}
class="h-8 w-8 bg-transparent p-0 hover:bg-destructive/20"
class="group h-8 w-8 rounded-full p-0 hover:bg-destructive/10!"
>
<span class="sr-only">Stop</span>
<Square class="h-8 w-8 fill-destructive stroke-destructive" />
<Square
class="h-8 w-8 fill-muted-foreground stroke-muted-foreground group-hover:fill-destructive group-hover:stroke-destructive hover:fill-destructive hover:stroke-destructive"
/>
</Button>
{:else if shouldShowRecordButton}
<ChatFormActionRecord {disabled} {hasAudioModality} {isLoading} {isRecording} {onMicClick} />

7
tools/server/webui/src/lib/components/app/chat/ChatForm/ChatFormTextarea.svelte
View file

@ -5,6 +5,7 @@
interface Props {
class?: string;
disabled?: boolean;
onInput?: () => void;
onKeydown?: (event: KeyboardEvent) => void;
onPaste?: (event: ClipboardEvent) => void;
placeholder?: string;
@ -14,6 +15,7 @@
let {
class: className = '',
disabled = false,
onInput,
onKeydown,
onPaste,
placeholder = 'Ask anything...',
@ -52,7 +54,10 @@
class:cursor-not-allowed={disabled}
{disabled}
onkeydown={onKeydown}
oninput={(event) => autoResizeTextarea(event.currentTarget)}
oninput={(event) => {
autoResizeTextarea(event.currentTarget);
onInput?.();
}}
onpaste={onPaste}
{placeholder}
></textarea>

243
tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessage.svelte
View file

@ -1,113 +1,94 @@
<script lang="ts">
import { goto } from '$app/navigation';
import {
chatStore,
pendingEditMessageId,
clearPendingEditMessageId,
removeSystemPromptPlaceholder
} from '$lib/stores/chat.svelte';
import { base } from '$app/paths';
import { getChatActionsContext, setMessageEditContext } from '$lib/contexts';
import { chatStore, pendingEditMessageId } from '$lib/stores/chat.svelte';
import { conversationsStore } from '$lib/stores/conversations.svelte';
import { DatabaseService } from '$lib/services';
import { config } from '$lib/stores/settings.svelte';
import { SYSTEM_MESSAGE_PLACEHOLDER } from '$lib/constants/ui';
import { copyToClipboard, isIMEComposing, formatMessageForClipboard } from '$lib/utils';
import ChatMessageAssistant from './ChatMessageAssistant.svelte';
import ChatMessageUser from './ChatMessageUser.svelte';
import ChatMessageSystem from './ChatMessageSystem.svelte';
import { MessageRole } from '$lib/enums';
import {
ChatMessageAssistant,
ChatMessageUser,
ChatMessageSystem
} from '$lib/components/app/chat';
import { parseFilesToMessageExtras } from '$lib/utils/browser-only';
interface Props {
class?: string;
message: DatabaseMessage;
onCopy?: (message: DatabaseMessage) => void;
onContinueAssistantMessage?: (message: DatabaseMessage) => void;
onDelete?: (message: DatabaseMessage) => void;
onEditWithBranching?: (
message: DatabaseMessage,
newContent: string,
newExtras?: DatabaseMessageExtra[]
) => void;
onEditWithReplacement?: (
message: DatabaseMessage,
newContent: string,
shouldBranch: boolean
) => void;
onEditUserMessagePreserveResponses?: (
message: DatabaseMessage,
newContent: string,
newExtras?: DatabaseMessageExtra[]
) => void;
onNavigateToSibling?: (siblingId: string) => void;
onRegenerateWithBranching?: (message: DatabaseMessage, modelOverride?: string) => void;
isLastAssistantMessage?: boolean;
siblingInfo?: ChatMessageSiblingInfo | null;
}
let {
class: className = '',
message,
onCopy,
onContinueAssistantMessage,
onDelete,
onEditWithBranching,
onEditWithReplacement,
onEditUserMessagePreserveResponses,
onNavigateToSibling,
onRegenerateWithBranching,
isLastAssistantMessage = false,
siblingInfo = null
}: Props = $props();
const chatActions = getChatActionsContext();
let deletionInfo = $state<{
totalCount: number;
userMessages: number;
assistantMessages: number;
messageTypes: string[];
} | null>(null);
let editedContent = $state(message.content);
let editedExtras = $state<DatabaseMessageExtra[]>(message.extra ? [...message.extra] : []);
let editedContent = $derived(message.content);
let editedExtras = $derived<DatabaseMessageExtra[]>(message.extra ? [...message.extra] : []);
let editedUploadedFiles = $state<ChatUploadedFile[]>([]);
let isEditing = $state(false);
let showDeleteDialog = $state(false);
let shouldBranchAfterEdit = $state(false);
let textareaElement: HTMLTextAreaElement | undefined = $state();
let thinkingContent = $derived.by(() => {
if (message.role === 'assistant') {
const trimmedThinking = message.thinking?.trim();
let showSaveOnlyOption = $derived(message.role === MessageRole.USER);
return trimmedThinking ? trimmedThinking : null;
}
return null;
setMessageEditContext({
get isEditing() {
return isEditing;
},
get editedContent() {
return editedContent;
},
get editedExtras() {
return editedExtras;
},
get editedUploadedFiles() {
return editedUploadedFiles;
},
get originalContent() {
return message.content;
},
get originalExtras() {
return message.extra || [];
},
get showSaveOnlyOption() {
return showSaveOnlyOption;
},
setContent: (content: string) => {
editedContent = content;
},
setExtras: (extras: DatabaseMessageExtra[]) => {
editedExtras = extras;
},
setUploadedFiles: (files: ChatUploadedFile[]) => {
editedUploadedFiles = files;
},
save: handleSaveEdit,
saveOnly: handleSaveEditOnly,
cancel: handleCancelEdit,
startEdit: handleEdit
});
let toolCallContent = $derived.by((): ApiChatCompletionToolCall[] | string | null => {
if (message.role === 'assistant') {
const trimmedToolCalls = message.toolCalls?.trim();
if (!trimmedToolCalls) {
return null;
}
try {
const parsed = JSON.parse(trimmedToolCalls);
if (Array.isArray(parsed)) {
return parsed as ApiChatCompletionToolCall[];
}
} catch {
// Harmony-only path: fall back to the raw string so issues surface visibly.
}
return trimmedToolCalls;
}
return null;
});
// Auto-start edit mode if this message is the pending edit target
$effect(() => {
const pendingId = pendingEditMessageId();
if (pendingId && pendingId === message.id && !isEditing) {
handleEdit();
clearPendingEditMessageId();
chatStore.clearPendingEditMessageId();
}
});
@ -115,11 +96,11 @@
isEditing = false;
// If canceling a new system message with placeholder content, remove it without deleting children
if (message.role === 'system') {
const conversationDeleted = await removeSystemPromptPlaceholder(message.id);
if (message.role === MessageRole.SYSTEM) {
const conversationDeleted = await chatStore.removeSystemPromptPlaceholder(message.id);
if (conversationDeleted) {
goto('/');
goto(`${base}/`);
}
return;
@ -130,30 +111,19 @@
editedUploadedFiles = [];
}
function handleEditedExtrasChange(extras: DatabaseMessageExtra[]) {
editedExtras = extras;
}
function handleEditedUploadedFilesChange(files: ChatUploadedFile[]) {
editedUploadedFiles = files;
}
async function handleCopy() {
const asPlainText = Boolean(config().copyTextAttachmentsAsPlainText);
const clipboardContent = formatMessageForClipboard(message.content, message.extra, asPlainText);
await copyToClipboard(clipboardContent, 'Message copied to clipboard');
onCopy?.(message);
function handleCopy() {
chatActions.copy(message);
}
async function handleConfirmDelete() {
if (message.role === 'system') {
const conversationDeleted = await removeSystemPromptPlaceholder(message.id);
if (message.role === MessageRole.SYSTEM) {
const conversationDeleted = await chatStore.removeSystemPromptPlaceholder(message.id);
if (conversationDeleted) {
goto('/');
goto(`${base}/`);
}
} else {
onDelete?.(message);
chatActions.delete(message);
}
showDeleteDialog = false;
@ -166,9 +136,9 @@
function handleEdit() {
isEditing = true;
// Clear placeholder content for system messages
// Clear temporary placeholder content for system messages
editedContent =
message.role === 'system' && message.content === SYSTEM_MESSAGE_PLACEHOLDER
message.role === MessageRole.SYSTEM && message.content === SYSTEM_MESSAGE_PLACEHOLDER
? ''
: message.content;
textareaElement?.focus();
@ -186,41 +156,29 @@
}, 0);
}
function handleEditedContentChange(content: string) {
editedContent = content;
}
function handleEditKeydown(event: KeyboardEvent) {
// Check for IME composition using isComposing property and keyCode 229 (specifically for IME composition on Safari)
// This prevents saving edit when confirming IME word selection (e.g., Japanese/Chinese input)
if (event.key === 'Enter' && !event.shiftKey && !isIMEComposing(event)) {
event.preventDefault();
handleSaveEdit();
} else if (event.key === 'Escape') {
event.preventDefault();
handleCancelEdit();
}
}
function handleRegenerate(modelOverride?: string) {
onRegenerateWithBranching?.(message, modelOverride);
chatActions.regenerateWithBranching(message, modelOverride);
}
function handleContinue() {
onContinueAssistantMessage?.(message);
chatActions.continueAssistantMessage(message);
}
function handleNavigateToSibling(siblingId: string) {
chatActions.navigateToSibling(siblingId);
}
async function handleSaveEdit() {
if (message.role === 'system') {
if (message.role === MessageRole.SYSTEM) {
// System messages: update in place without branching
const newContent = editedContent.trim();
// If content is empty or still the placeholder, remove without deleting children
// If content is empty, remove without deleting children
if (!newContent) {
const conversationDeleted = await removeSystemPromptPlaceholder(message.id);
const conversationDeleted = await chatStore.removeSystemPromptPlaceholder(message.id);
isEditing = false;
if (conversationDeleted) {
goto('/');
goto(`${base}/`);
}
return;
}
@ -230,13 +188,13 @@
if (index !== -1) {
conversationsStore.updateMessageAtIndex(index, { content: newContent });
}
} else if (message.role === 'user') {
} else if (message.role === MessageRole.USER) {
const finalExtras = await getMergedExtras();
onEditWithBranching?.(message, editedContent.trim(), finalExtras);
chatActions.editWithBranching(message, editedContent.trim(), finalExtras);
} else {
// For assistant messages, preserve exact content including trailing whitespace
// This is important for the Continue feature to work properly
onEditWithReplacement?.(message, editedContent, shouldBranchAfterEdit);
chatActions.editWithReplacement(message, editedContent, shouldBranchAfterEdit);
}
isEditing = false;
@ -245,10 +203,10 @@
}
async function handleSaveEditOnly() {
if (message.role === 'user') {
if (message.role === MessageRole.USER) {
// For user messages, trim to avoid accidental whitespace
const finalExtras = await getMergedExtras();
onEditUserMessagePreserveResponses?.(message, editedContent.trim(), finalExtras);
chatActions.editUserMessagePreserveResponses(message, editedContent.trim(), finalExtras);
}
isEditing = false;
@ -260,8 +218,8 @@
return editedExtras;
}
const { parseFilesToMessageExtras } = await import('$lib/utils/browser-only');
const result = await parseFilesToMessageExtras(editedUploadedFiles);
const plainFiles = $state.snapshot(editedUploadedFiles);
const result = await parseFilesToMessageExtras(plainFiles);
const newExtras = result?.extras || [];
return [...editedExtras, ...newExtras];
@ -272,49 +230,31 @@
}
</script>
{#if message.role === 'system'}
{#if message.role === MessageRole.SYSTEM}
<ChatMessageSystem
bind:textareaElement
class={className}
{deletionInfo}
{editedContent}
{isEditing}
{message}
onCancelEdit={handleCancelEdit}
onConfirmDelete={handleConfirmDelete}
onCopy={handleCopy}
onDelete={handleDelete}
onEdit={handleEdit}
onEditKeydown={handleEditKeydown}
onEditedContentChange={handleEditedContentChange}
{onNavigateToSibling}
onSaveEdit={handleSaveEdit}
onNavigateToSibling={handleNavigateToSibling}
onShowDeleteDialogChange={handleShowDeleteDialogChange}
{showDeleteDialog}
{siblingInfo}
/>
{:else if message.role === 'user'}
{:else if message.role === MessageRole.USER}
<ChatMessageUser
bind:textareaElement
class={className}
{deletionInfo}
{editedContent}
{editedExtras}
{editedUploadedFiles}
{isEditing}
{message}
onCancelEdit={handleCancelEdit}
onConfirmDelete={handleConfirmDelete}
onCopy={handleCopy}
onDelete={handleDelete}
onEdit={handleEdit}
onEditKeydown={handleEditKeydown}
onEditedContentChange={handleEditedContentChange}
onEditedExtrasChange={handleEditedExtrasChange}
onEditedUploadedFilesChange={handleEditedUploadedFilesChange}
{onNavigateToSibling}
onSaveEdit={handleSaveEdit}
onSaveEditOnly={handleSaveEditOnly}
onNavigateToSibling={handleNavigateToSibling}
onShowDeleteDialogChange={handleShowDeleteDialogChange}
{showDeleteDialog}
{siblingInfo}
@ -324,27 +264,18 @@
bind:textareaElement
class={className}
{deletionInfo}
{editedContent}
{isEditing}
{isLastAssistantMessage}
{message}
messageContent={message.content}
onCancelEdit={handleCancelEdit}
onConfirmDelete={handleConfirmDelete}
onContinue={handleContinue}
onCopy={handleCopy}
onDelete={handleDelete}
onEdit={handleEdit}
onEditKeydown={handleEditKeydown}
onEditedContentChange={handleEditedContentChange}
{onNavigateToSibling}
onNavigateToSibling={handleNavigateToSibling}
onRegenerate={handleRegenerate}
onSaveEdit={handleSaveEdit}
onShowDeleteDialogChange={handleShowDeleteDialogChange}
{shouldBranchAfterEdit}
onShouldBranchAfterEditChange={(value) => (shouldBranchAfterEdit = value)}
{showDeleteDialog}
{siblingInfo}
{thinkingContent}
{toolCallContent}
/>
{/if}

42
tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessageActions.svelte
View file

@ -1,13 +1,15 @@
<script lang="ts">
import { Edit, Copy, RefreshCw, Trash2, ArrowRight } from '@lucide/svelte';
import {
ActionButton,
ActionIcon,
ChatMessageBranchingControls,
DialogConfirmation
} from '$lib/components/app';
import { Switch } from '$lib/components/ui/switch';
import { MessageRole } from '$lib/enums';
interface Props {
role: 'user' | 'assistant';
role: MessageRole.USER | MessageRole.ASSISTANT;
justify: 'start' | 'end';
actionsPosition: 'left' | 'right';
siblingInfo?: ChatMessageSiblingInfo | null;
@ -26,6 +28,9 @@
onConfirmDelete: () => void;
onNavigateToSibling?: (siblingId: string) => void;
onShowDeleteDialogChange: (show: boolean) => void;
showRawOutputSwitch?: boolean;
rawOutputEnabled?: boolean;
onRawOutputToggle?: (enabled: boolean) => void;
}
let {
@ -42,7 +47,10 @@
onRegenerate,
role,
siblingInfo = null,
showDeleteDialog
showDeleteDialog,
showRawOutputSwitch = false,
rawOutputEnabled = false,
onRawOutputToggle
}: Props = $props();
function handleConfirmDelete() {
@ -51,9 +59,9 @@
}
</script>
<div class="relative {justify === 'start' ? 'mt-2' : ''} flex h-6 items-center justify-{justify}">
<div class="relative {justify === 'start' ? 'mt-2' : ''} flex h-6 items-center justify-between">
<div
class="absolute top-0 {actionsPosition === 'left'
class="{actionsPosition === 'left'
? 'left-0'
: 'right-0'} flex items-center gap-2 opacity-100 transition-opacity"
>
@ -64,23 +72,33 @@
<div
class="pointer-events-auto inset-0 flex items-center gap-1 opacity-100 transition-all duration-150"
>
<ActionButton icon={Copy} tooltip="Copy" onclick={onCopy} />
<ActionIcon icon={Copy} tooltip="Copy" onclick={onCopy} />
{#if onEdit}
<ActionButton icon={Edit} tooltip="Edit" onclick={onEdit} />
<ActionIcon icon={Edit} tooltip="Edit" onclick={onEdit} />
{/if}
{#if role === 'assistant' && onRegenerate}
<ActionButton icon={RefreshCw} tooltip="Regenerate" onclick={() => onRegenerate()} />
{#if role === MessageRole.ASSISTANT && onRegenerate}
<ActionIcon icon={RefreshCw} tooltip="Regenerate" onclick={() => onRegenerate()} />
{/if}
{#if role === 'assistant' && onContinue}
<ActionButton icon={ArrowRight} tooltip="Continue" onclick={onContinue} />
{#if role === MessageRole.ASSISTANT && onContinue}
<ActionIcon icon={ArrowRight} tooltip="Continue" onclick={onContinue} />
{/if}
<ActionButton icon={Trash2} tooltip="Delete" onclick={onDelete} />
<ActionIcon icon={Trash2} tooltip="Delete" onclick={onDelete} />
</div>
</div>
{#if showRawOutputSwitch}
<div class="flex items-center gap-2">
<span class="text-xs text-muted-foreground">Show raw output</span>
<Switch
checked={rawOutputEnabled}
onCheckedChange={(checked) => onRawOutputToggle?.(checked)}
/>
</div>
{/if}
</div>
<DialogConfirmation

386
tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessageAssistant.svelte
View file

@ -1,26 +1,29 @@
<script lang="ts">
import {
ModelBadge,
ChatMessageActions,
ChatMessageStatistics,
ChatMessageThinkingBlock,
CopyToClipboardIcon,
MarkdownContent,
ModelBadge,
ModelsSelector
} from '$lib/components/app';
import ChatMessageThinkingBlock from './ChatMessageThinkingBlock.svelte';
import { getMessageEditContext } from '$lib/contexts';
import { useProcessingState } from '$lib/hooks/use-processing-state.svelte';
import { useModelChangeValidation } from '$lib/hooks/use-model-change-validation.svelte';
import { isLoading } from '$lib/stores/chat.svelte';
import { autoResizeTextarea, copyToClipboard } from '$lib/utils';
import { isLoading, isChatStreaming } from '$lib/stores/chat.svelte';
import { autoResizeTextarea, copyToClipboard, isIMEComposing } from '$lib/utils';
import { tick } from 'svelte';
import { fade } from 'svelte/transition';
import { Check, X, Wrench } from '@lucide/svelte';
import { Check, X } from '@lucide/svelte';
import { Button } from '$lib/components/ui/button';
import { Checkbox } from '$lib/components/ui/checkbox';
import { INPUT_CLASSES } from '$lib/constants/input-classes';
import { INPUT_CLASSES } from '$lib/constants/css-classes';
import { MessageRole, KeyboardKey } from '$lib/enums';
import Label from '$lib/components/ui/label/label.svelte';
import { config } from '$lib/stores/settings.svelte';
import { conversationsStore } from '$lib/stores/conversations.svelte';
import { isRouterMode } from '$lib/stores/server.svelte';
import { modelsStore } from '$lib/stores/models.svelte';
import { ServerModelStatus } from '$lib/enums';
import { REASONING_TAGS } from '$lib/constants/agentic';
interface Props {
class?: string;
@ -30,150 +33,198 @@
assistantMessages: number;
messageTypes: string[];
} | null;
editedContent?: string;
isEditing?: boolean;
isLastAssistantMessage?: boolean;
message: DatabaseMessage;
messageContent: string | undefined;
onCancelEdit?: () => void;
onCopy: () => void;
onConfirmDelete: () => void;
onContinue?: () => void;
onDelete: () => void;
onEdit?: () => void;
onEditKeydown?: (event: KeyboardEvent) => void;
onEditedContentChange?: (content: string) => void;
onNavigateToSibling?: (siblingId: string) => void;
onRegenerate: (modelOverride?: string) => void;
onSaveEdit?: () => void;
onShowDeleteDialogChange: (show: boolean) => void;
onShouldBranchAfterEditChange?: (value: boolean) => void;
showDeleteDialog: boolean;
shouldBranchAfterEdit?: boolean;
siblingInfo?: ChatMessageSiblingInfo | null;
textareaElement?: HTMLTextAreaElement;
thinkingContent: string | null;
toolCallContent: ApiChatCompletionToolCall[] | string | null;
}
interface ParsedReasoningContent {
content: string;
reasoningContent: string | null;
hasReasoningMarkers: boolean;
}
function parseReasoningContent(content: string | undefined): ParsedReasoningContent {
if (!content) {
return {
content: '',
reasoningContent: null,
hasReasoningMarkers: false
};
}
const plainParts: string[] = [];
const reasoningParts: string[] = [];
const { START, END } = REASONING_TAGS;
let cursor = 0;
let hasReasoningMarkers = false;
while (cursor < content.length) {
const startIndex = content.indexOf(START, cursor);
if (startIndex === -1) {
plainParts.push(content.slice(cursor));
break;
}
hasReasoningMarkers = true;
plainParts.push(content.slice(cursor, startIndex));
const reasoningStart = startIndex + START.length;
const endIndex = content.indexOf(END, reasoningStart);
if (endIndex === -1) {
reasoningParts.push(content.slice(reasoningStart));
cursor = content.length;
break;
}
reasoningParts.push(content.slice(reasoningStart, endIndex));
cursor = endIndex + END.length;
}
return {
content: plainParts.join(''),
reasoningContent: reasoningParts.length > 0 ? reasoningParts.join('\n\n') : null,
hasReasoningMarkers
};
}
let {
class: className = '',
deletionInfo,
editedContent = '',
isEditing = false,
isLastAssistantMessage = false,
message,
messageContent,
onCancelEdit,
onConfirmDelete,
onContinue,
onCopy,
onDelete,
onEdit,
onEditKeydown,
onEditedContentChange,
onNavigateToSibling,
onRegenerate,
onSaveEdit,
onShowDeleteDialogChange,
onShouldBranchAfterEditChange,
showDeleteDialog,
shouldBranchAfterEdit = false,
siblingInfo = null,
textareaElement = $bindable(),
thinkingContent,
toolCallContent = null
textareaElement = $bindable()
}: Props = $props();
const toolCalls = $derived(
Array.isArray(toolCallContent) ? (toolCallContent as ApiChatCompletionToolCall[]) : null
);
const fallbackToolCalls = $derived(typeof toolCallContent === 'string' ? toolCallContent : null);
// Get edit context
const editCtx = getMessageEditContext();
// Local state for assistant-specific editing
let shouldBranchAfterEdit = $state(false);
function handleEditKeydown(event: KeyboardEvent) {
if (event.key === KeyboardKey.ENTER && !event.shiftKey && !isIMEComposing(event)) {
event.preventDefault();
editCtx.save();
} else if (event.key === KeyboardKey.ESCAPE) {
event.preventDefault();
editCtx.cancel();
}
}
const parsedMessageContent = $derived.by(() => parseReasoningContent(messageContent));
const visibleMessageContent = $derived(parsedMessageContent.content);
const thinkingContent = $derived(parsedMessageContent.reasoningContent);
const hasReasoningMarkers = $derived(parsedMessageContent.hasReasoningMarkers);
const processingState = useProcessingState();
let currentConfig = $derived(config());
let isRouter = $derived(isRouterMode());
let displayedModel = $derived((): string | null => {
if (message.model) {
return message.model;
let showRawOutput = $state(false);
let statsContainerEl: HTMLDivElement | undefined = $state();
function getScrollParent(el: HTMLElement): HTMLElement | null {
let parent = el.parentElement;
while (parent) {
const style = getComputedStyle(parent);
if (/(auto|scroll)/.test(style.overflowY)) {
return parent;
}
parent = parent.parentElement;
}
return null;
}
async function handleStatsViewChange() {
const el = statsContainerEl;
if (!el) {
return;
}
return null;
});
const scrollParent = getScrollParent(el);
if (!scrollParent) {
return;
}
const { handleModelChange } = useModelChangeValidation({
getRequiredModalities: () => conversationsStore.getModalitiesUpToMessage(message.id),
onSuccess: (modelName) => onRegenerate(modelName)
});
const yBefore = el.getBoundingClientRect().top;
await tick();
const delta = el.getBoundingClientRect().top - yBefore;
if (delta !== 0) {
scrollParent.scrollTop += delta;
}
// Correct any drift after browser paint
requestAnimationFrame(() => {
const drift = el.getBoundingClientRect().top - yBefore;
if (Math.abs(drift) > 1) {
scrollParent.scrollTop += drift;
}
});
}
let displayedModel = $derived(message.model ?? null);
let isCurrentlyLoading = $derived(isLoading());
let isStreaming = $derived(isChatStreaming());
let hasNoContent = $derived(!visibleMessageContent?.trim());
let isActivelyProcessing = $derived(isCurrentlyLoading || isStreaming);
let showProcessingInfoTop = $derived(
message?.role === MessageRole.ASSISTANT &&
isActivelyProcessing &&
hasNoContent &&
isLastAssistantMessage
);
let showProcessingInfoBottom = $derived(
message?.role === MessageRole.ASSISTANT &&
isActivelyProcessing &&
!hasNoContent &&
isLastAssistantMessage
);
function handleCopyModel() {
const model = displayedModel();
void copyToClipboard(model ?? '');
void copyToClipboard(displayedModel ?? '');
}
$effect(() => {
if (isEditing && textareaElement) {
if (editCtx.isEditing && textareaElement) {
autoResizeTextarea(textareaElement);
}
});
$effect(() => {
if (isLoading() && !message?.content?.trim()) {
if (showProcessingInfoTop || showProcessingInfoBottom) {
processingState.startMonitoring();
}
});
function formatToolCallBadge(toolCall: ApiChatCompletionToolCall, index: number) {
const callNumber = index + 1;
const functionName = toolCall.function?.name?.trim();
const label = functionName || `Call #${callNumber}`;
const payload: Record<string, unknown> = {};
const id = toolCall.id?.trim();
if (id) {
payload.id = id;
}
const type = toolCall.type?.trim();
if (type) {
payload.type = type;
}
if (toolCall.function) {
const fnPayload: Record<string, unknown> = {};
const name = toolCall.function.name?.trim();
if (name) {
fnPayload.name = name;
}
const rawArguments = toolCall.function.arguments?.trim();
if (rawArguments) {
try {
fnPayload.arguments = JSON.parse(rawArguments);
} catch {
fnPayload.arguments = rawArguments;
}
}
if (Object.keys(fnPayload).length > 0) {
payload.function = fnPayload;
}
}
const formattedPayload = JSON.stringify(payload, null, 2);
return {
label,
tooltip: formattedPayload,
copyValue: formattedPayload
};
}
function handleCopyToolCall(payload: string) {
void copyToClipboard(payload, 'Tool call copied to clipboard');
}
</script>
<div
@ -181,34 +232,36 @@
role="group"
aria-label="Assistant message with actions"
>
{#if thinkingContent}
{#if !editCtx.isEditing && thinkingContent}
<ChatMessageThinkingBlock
reasoningContent={thinkingContent}
isStreaming={!message.timestamp}
hasRegularContent={!!messageContent?.trim()}
hasRegularContent={!!visibleMessageContent?.trim()}
/>
{/if}
{#if message?.role === 'assistant' && isLoading() && !message?.content?.trim()}
{#if showProcessingInfoTop}
<div class="mt-6 w-full max-w-[48rem]" in:fade>
<div class="processing-container">
<span class="processing-text">
{processingState.getPromptProgressText() ?? processingState.getProcessingMessage()}
{processingState.getPromptProgressText() ??
processingState.getProcessingMessage() ??
'Processing...'}
</span>
</div>
</div>
{/if}
{#if isEditing}
{#if editCtx.isEditing}
<div class="w-full">
<textarea
bind:this={textareaElement}
bind:value={editedContent}
value={editCtx.editedContent}
class="min-h-[50vh] w-full resize-y rounded-2xl px-3 py-2 text-sm {INPUT_CLASSES}"
onkeydown={onEditKeydown}
onkeydown={handleEditKeydown}
oninput={(e) => {
autoResizeTextarea(e.currentTarget);
onEditedContentChange?.(e.currentTarget.value);
editCtx.setContent(e.currentTarget.value);
}}
placeholder="Edit assistant message..."
></textarea>
@ -218,30 +271,35 @@
<Checkbox
id="branch-after-edit"
bind:checked={shouldBranchAfterEdit}
onCheckedChange={(checked) => onShouldBranchAfterEditChange?.(checked === true)}
onCheckedChange={(checked) => (shouldBranchAfterEdit = checked === true)}
/>
<Label for="branch-after-edit" class="cursor-pointer text-sm text-muted-foreground">
Branch conversation after edit
</Label>
</div>
<div class="flex gap-2">
<Button class="h-8 px-3" onclick={onCancelEdit} size="sm" variant="outline">
<Button class="h-8 px-3" onclick={editCtx.cancel} size="sm" variant="outline">
<X class="mr-1 h-3 w-3" />
Cancel
</Button>
<Button class="h-8 px-3" onclick={onSaveEdit} disabled={!editedContent?.trim()} size="sm">
<Button
class="h-8 px-3"
onclick={editCtx.save}
disabled={!editCtx.editedContent?.trim()}
size="sm"
>
<Check class="mr-1 h-3 w-3" />
Save
</Button>
</div>
</div>
</div>
{:else if message.role === 'assistant'}
{#if config().disableReasoningFormat}
{:else if message.role === MessageRole.ASSISTANT}
{#if showRawOutput}
<pre class="raw-output">{messageContent || ''}</pre>
{:else}
<MarkdownContent content={messageContent || ''} />
<MarkdownContent content={visibleMessageContent || ''} attachments={message.extra} />
{/if}
{:else}
<div class="text-sm whitespace-pre-wrap">
@ -249,18 +307,41 @@
</div>
{/if}
{#if showProcessingInfoBottom}
<div class="mt-4 w-full max-w-[48rem]" in:fade>
<div class="processing-container">
<span class="processing-text">
{processingState.getPromptProgressText() ??
processingState.getProcessingMessage() ??
'Processing...'}
</span>
</div>
</div>
{/if}
<div class="info my-6 grid gap-4 tabular-nums">
{#if displayedModel()}
<div class="inline-flex flex-wrap items-start gap-2 text-xs text-muted-foreground">
{#if displayedModel}
<div
bind:this={statsContainerEl}
class="inline-flex flex-wrap items-start gap-2 text-xs text-muted-foreground"
>
{#if isRouter}
<ModelsSelector
currentModel={displayedModel()}
onModelChange={handleModelChange}
currentModel={displayedModel}
disabled={isLoading()}
upToMessageId={message.id}
onModelChange={async (modelId, modelName) => {
const status = modelsStore.getModelStatus(modelId);
if (status !== ServerModelStatus.LOADED) {
await modelsStore.loadModel(modelId);
}
onRegenerate(modelName);
return true;
}}
/>
{:else}
<ModelBadge model={displayedModel() || undefined} onclick={handleCopyModel} />
<ModelBadge model={displayedModel || undefined} onclick={handleCopyModel} />
{/if}
{#if currentConfig.showMessageStats && message.timings && message.timings.predicted_n && message.timings.predicted_ms}
@ -269,6 +350,7 @@
promptMs={message.timings.prompt_ms}
predictedTokens={message.timings.predicted_n}
predictedMs={message.timings.predicted_ms}
onActiveViewChange={handleStatsViewChange}
/>
{:else if isLoading() && currentConfig.showMessageStats}
{@const liveStats = processingState.getLiveProcessingStats()}
@ -290,53 +372,11 @@
{/if}
</div>
{/if}
{#if config().showToolCalls}
{#if (toolCalls && toolCalls.length > 0) || fallbackToolCalls}
<span class="inline-flex flex-wrap items-center gap-2 text-xs text-muted-foreground">
<span class="inline-flex items-center gap-1">
<Wrench class="h-3.5 w-3.5" />
<span>Tool calls:</span>
</span>
{#if toolCalls && toolCalls.length > 0}
{#each toolCalls as toolCall, index (toolCall.id ?? `${index}`)}
{@const badge = formatToolCallBadge(toolCall, index)}
<button
type="button"
class="tool-call-badge inline-flex cursor-pointer items-center gap-1 rounded-sm bg-muted-foreground/15 px-1.5 py-0.75"
title={badge.tooltip}
aria-label={`Copy tool call ${badge.label}`}
onclick={() => handleCopyToolCall(badge.copyValue)}
>
{badge.label}
<CopyToClipboardIcon
text={badge.copyValue}
ariaLabel={`Copy tool call ${badge.label}`}
/>
</button>
{/each}
{:else if fallbackToolCalls}
<button
type="button"
class="tool-call-badge tool-call-badge--fallback inline-flex cursor-pointer items-center gap-1 rounded-sm bg-muted-foreground/15 px-1.5 py-0.75"
title={fallbackToolCalls}
aria-label="Copy tool call payload"
onclick={() => handleCopyToolCall(fallbackToolCalls)}
>
{fallbackToolCalls}
<CopyToClipboardIcon text={fallbackToolCalls} ariaLabel="Copy tool call payload" />
</button>
{/if}
</span>
{/if}
{/if}
</div>
{#if message.timestamp && !isEditing}
{#if message.timestamp && !editCtx.isEditing}
<ChatMessageActions
role="assistant"
role={MessageRole.ASSISTANT}
justify="start"
actionsPosition="left"
{siblingInfo}
@ -345,13 +385,16 @@
{onCopy}
{onEdit}
{onRegenerate}
onContinue={currentConfig.enableContinueGeneration && !thinkingContent
onContinue={currentConfig.enableContinueGeneration && !hasReasoningMarkers
? onContinue
: undefined}
{onDelete}
{onConfirmDelete}
{onNavigateToSibling}
{onShowDeleteDialogChange}
showRawOutputSwitch={currentConfig.showRawOutputSwitch}
rawOutputEnabled={showRawOutput}
onRawOutputToggle={(enabled) => (showRawOutput = enabled)}
/>
{/if}
</div>
@ -402,17 +445,4 @@
white-space: pre-wrap;
word-break: break-word;
}
.tool-call-badge {
max-width: 12rem;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.tool-call-badge--fallback {
max-width: 20rem;
white-space: normal;
word-break: break-word;
}
</style>

351
tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessageEditForm.svelte
View file

@ -1,79 +1,26 @@
<script lang="ts">
import { X, ArrowUp, Paperclip, AlertTriangle } from '@lucide/svelte';
import { X, AlertTriangle } from '@lucide/svelte';
import { Button } from '$lib/components/ui/button';
import { Switch } from '$lib/components/ui/switch';
import { ChatAttachmentsList, DialogConfirmation, ModelsSelector } from '$lib/components/app';
import { INPUT_CLASSES } from '$lib/constants/input-classes';
import { SETTING_CONFIG_DEFAULT } from '$lib/constants/settings-config';
import { AttachmentType, FileTypeCategory, MimeTypeText } from '$lib/enums';
import { config } from '$lib/stores/settings.svelte';
import { useModelChangeValidation } from '$lib/hooks/use-model-change-validation.svelte';
import { setEditModeActive, clearEditMode } from '$lib/stores/chat.svelte';
import { conversationsStore } from '$lib/stores/conversations.svelte';
import { modelsStore } from '$lib/stores/models.svelte';
import { isRouterMode } from '$lib/stores/server.svelte';
import {
autoResizeTextarea,
getFileTypeCategory,
getFileTypeCategoryByExtension,
parseClipboardContent
} from '$lib/utils';
import { ChatForm, DialogConfirmation } from '$lib/components/app';
import { getMessageEditContext } from '$lib/contexts';
import { KeyboardKey } from '$lib/enums';
import { chatStore } from '$lib/stores/chat.svelte';
import { processFilesToChatUploaded } from '$lib/utils/browser-only';
interface Props {
messageId: string;
editedContent: string;
editedExtras?: DatabaseMessageExtra[];
editedUploadedFiles?: ChatUploadedFile[];
originalContent: string;
originalExtras?: DatabaseMessageExtra[];
showSaveOnlyOption?: boolean;
onCancelEdit: () => void;
onSaveEdit: () => void;
onSaveEditOnly?: () => void;
onEditKeydown: (event: KeyboardEvent) => void;
onEditedContentChange: (content: string) => void;
onEditedExtrasChange?: (extras: DatabaseMessageExtra[]) => void;
onEditedUploadedFilesChange?: (files: ChatUploadedFile[]) => void;
textareaElement?: HTMLTextAreaElement;
}
const editCtx = getMessageEditContext();
let {
messageId,
editedContent,
editedExtras = [],
editedUploadedFiles = [],
originalContent,
originalExtras = [],
showSaveOnlyOption = false,
onCancelEdit,
onSaveEdit,
onSaveEditOnly,
onEditKeydown,
onEditedContentChange,
onEditedExtrasChange,
onEditedUploadedFilesChange,
textareaElement = $bindable()
}: Props = $props();
let fileInputElement: HTMLInputElement | undefined = $state();
let inputAreaRef: ChatForm | undefined = $state(undefined);
let saveWithoutRegenerate = $state(false);
let showDiscardDialog = $state(false);
let isRouter = $derived(isRouterMode());
let currentConfig = $derived(config());
let pasteLongTextToFileLength = $derived.by(() => {
const n = Number(currentConfig.pasteLongTextToFileLen);
return Number.isNaN(n) ? Number(SETTING_CONFIG_DEFAULT.pasteLongTextToFileLen) : n;
});
let hasUnsavedChanges = $derived.by(() => {
if (editedContent !== originalContent) return true;
if (editedUploadedFiles.length > 0) return true;
if (editCtx.editedContent !== editCtx.originalContent) return true;
if (editCtx.editedUploadedFiles.length > 0) return true;
const extrasChanged =
editedExtras.length !== originalExtras.length ||
editedExtras.some((extra, i) => extra !== originalExtras[i]);
editCtx.editedExtras.length !== editCtx.originalExtras.length ||
editCtx.editedExtras.some((extra, i) => extra !== editCtx.originalExtras[i]);
if (extrasChanged) return true;
@ -81,77 +28,14 @@
});
let hasAttachments = $derived(
(editedExtras && editedExtras.length > 0) ||
(editedUploadedFiles && editedUploadedFiles.length > 0)
(editCtx.editedExtras && editCtx.editedExtras.length > 0) ||
(editCtx.editedUploadedFiles && editCtx.editedUploadedFiles.length > 0)
);
let canSubmit = $derived(editedContent.trim().length > 0 || hasAttachments);
function getEditedAttachmentsModalities(): ModelModalities {
const modalities: ModelModalities = { vision: false, audio: false };
for (const extra of editedExtras) {
if (extra.type === AttachmentType.IMAGE) {
modalities.vision = true;
}
if (
extra.type === AttachmentType.PDF &&
'processedAsImages' in extra &&
extra.processedAsImages
) {
modalities.vision = true;
}
if (extra.type === AttachmentType.AUDIO) {
modalities.audio = true;
}
}
for (const file of editedUploadedFiles) {
const category = getFileTypeCategory(file.type) || getFileTypeCategoryByExtension(file.name);
if (category === FileTypeCategory.IMAGE) {
modalities.vision = true;
}
if (category === FileTypeCategory.AUDIO) {
modalities.audio = true;
}
}
return modalities;
}
function getRequiredModalities(): ModelModalities {
const beforeModalities = conversationsStore.getModalitiesUpToMessage(messageId);
const editedModalities = getEditedAttachmentsModalities();
return {
vision: beforeModalities.vision || editedModalities.vision,
audio: beforeModalities.audio || editedModalities.audio
};
}
const { handleModelChange } = useModelChangeValidation({
getRequiredModalities,
onValidationFailure: async (previousModelId) => {
if (previousModelId) {
await modelsStore.selectModelById(previousModelId);
}
}
});
function handleFileInputChange(event: Event) {
const input = event.target as HTMLInputElement;
if (!input.files || input.files.length === 0) return;
const files = Array.from(input.files);
processNewFiles(files);
input.value = '';
}
let canSubmit = $derived(editCtx.editedContent.trim().length > 0 || hasAttachments);
function handleGlobalKeydown(event: KeyboardEvent) {
if (event.key === 'Escape') {
if (event.key === KeyboardKey.ESCAPE) {
event.preventDefault();
attemptCancel();
}
@ -161,205 +45,66 @@
if (hasUnsavedChanges) {
showDiscardDialog = true;
} else {
onCancelEdit();
editCtx.cancel();
}
}
function handleRemoveExistingAttachment(index: number) {
if (!onEditedExtrasChange) return;
const newExtras = [...editedExtras];
newExtras.splice(index, 1);
onEditedExtrasChange(newExtras);
}
function handleRemoveUploadedFile(fileId: string) {
if (!onEditedUploadedFilesChange) return;
const newFiles = editedUploadedFiles.filter((f) => f.id !== fileId);
onEditedUploadedFilesChange(newFiles);
}
function handleSubmit() {
if (!canSubmit) return;
if (saveWithoutRegenerate && onSaveEditOnly) {
onSaveEditOnly();
if (saveWithoutRegenerate && editCtx.showSaveOnlyOption) {
editCtx.saveOnly();
} else {
onSaveEdit();
editCtx.save();
}
saveWithoutRegenerate = false;
}
async function processNewFiles(files: File[]) {
if (!onEditedUploadedFilesChange) return;
function handleAttachmentRemove(index: number) {
const newExtras = [...editCtx.editedExtras];
newExtras.splice(index, 1);
editCtx.setExtras(newExtras);
}
const { processFilesToChatUploaded } = await import('$lib/utils/browser-only');
function handleUploadedFileRemove(fileId: string) {
const newFiles = editCtx.editedUploadedFiles.filter((f) => f.id !== fileId);
editCtx.setUploadedFiles(newFiles);
}
async function handleFilesAdd(files: File[]) {
const processed = await processFilesToChatUploaded(files);
onEditedUploadedFilesChange([...editedUploadedFiles, ...processed]);
}
function handlePaste(event: ClipboardEvent) {
if (!event.clipboardData) return;
const files = Array.from(event.clipboardData.items)
.filter((item) => item.kind === 'file')
.map((item) => item.getAsFile())
.filter((file): file is File => file !== null);
if (files.length > 0) {
event.preventDefault();
processNewFiles(files);
return;
}
const text = event.clipboardData.getData(MimeTypeText.PLAIN);
if (text.startsWith('"')) {
const parsed = parseClipboardContent(text);
if (parsed.textAttachments.length > 0) {
event.preventDefault();
onEditedContentChange(parsed.message);
const attachmentFiles = parsed.textAttachments.map(
(att) =>
new File([att.content], att.name, {
type: MimeTypeText.PLAIN
})
);
processNewFiles(attachmentFiles);
setTimeout(() => {
textareaElement?.focus();
}, 10);
return;
}
}
if (
text.length > 0 &&
pasteLongTextToFileLength > 0 &&
text.length > pasteLongTextToFileLength
) {
event.preventDefault();
const textFile = new File([text], 'Pasted', {
type: MimeTypeText.PLAIN
});
processNewFiles([textFile]);
}
editCtx.setUploadedFiles([...editCtx.editedUploadedFiles, ...processed]);
}
$effect(() => {
if (textareaElement) {
autoResizeTextarea(textareaElement);
}
});
$effect(() => {
setEditModeActive(processNewFiles);
chatStore.setEditModeActive(handleFilesAdd);
return () => {
clearEditMode();
chatStore.clearEditMode();
};
});
</script>
<svelte:window onkeydown={handleGlobalKeydown} />
<input
bind:this={fileInputElement}
type="file"
multiple
class="hidden"
onchange={handleFileInputChange}
/>
<div
class="{INPUT_CLASSES} w-full max-w-[80%] overflow-hidden rounded-3xl backdrop-blur-md"
data-slot="edit-form"
>
<ChatAttachmentsList
attachments={editedExtras}
uploadedFiles={editedUploadedFiles}
readonly={false}
onFileRemove={(fileId) => {
if (fileId.startsWith('attachment-')) {
const index = parseInt(fileId.replace('attachment-', ''), 10);
if (!isNaN(index) && index >= 0 && index < editedExtras.length) {
handleRemoveExistingAttachment(index);
}
} else {
handleRemoveUploadedFile(fileId);
}
}}
limitToSingleRow
class="py-5"
style="scroll-padding: 1rem;"
<div class="relative w-full max-w-[80%]">
<ChatForm
bind:this={inputAreaRef}
value={editCtx.editedContent}
attachments={editCtx.editedExtras}
uploadedFiles={editCtx.editedUploadedFiles}
placeholder="Edit your message..."
onValueChange={editCtx.setContent}
onAttachmentRemove={handleAttachmentRemove}
onUploadedFileRemove={handleUploadedFileRemove}
onFilesAdd={handleFilesAdd}
onSubmit={handleSubmit}
/>
<div class="relative min-h-[48px] px-5 py-3">
<textarea
bind:this={textareaElement}
bind:value={editedContent}
class="field-sizing-content max-h-80 min-h-10 w-full resize-none bg-transparent text-sm outline-none"
onkeydown={onEditKeydown}
oninput={(e) => {
autoResizeTextarea(e.currentTarget);
onEditedContentChange(e.currentTarget.value);
}}
onpaste={handlePaste}
placeholder="Edit your message..."
></textarea>
<div class="flex w-full items-center gap-3" style="container-type: inline-size">
<Button
class="h-8 w-8 shrink-0 rounded-full bg-transparent p-0 text-muted-foreground hover:bg-foreground/10 hover:text-foreground"
onclick={() => fileInputElement?.click()}
type="button"
title="Add attachment"
>
<span class="sr-only">Attach files</span>
<Paperclip class="h-4 w-4" />
</Button>
<div class="flex-1"></div>
{#if isRouter}
<ModelsSelector
forceForegroundText={true}
useGlobalSelection={true}
onModelChange={handleModelChange}
/>
{/if}
<Button
class="h-8 w-8 shrink-0 rounded-full p-0"
onclick={handleSubmit}
disabled={!canSubmit}
type="button"
title={saveWithoutRegenerate ? 'Save changes' : 'Send and regenerate'}
>
<span class="sr-only">{saveWithoutRegenerate ? 'Save' : 'Send'}</span>
<ArrowUp class="h-5 w-5" />
</Button>
</div>
</div>
</div>
<div class="mt-2 flex w-full max-w-[80%] items-center justify-between">
{#if showSaveOnlyOption && onSaveEditOnly}
{#if editCtx.showSaveOnlyOption}
<div class="flex items-center gap-2">
<Switch id="save-only-switch" bind:checked={saveWithoutRegenerate} class="scale-75" />
@ -386,6 +131,6 @@
cancelText="Keep editing"
variant="destructive"
icon={AlertTriangle}
onConfirm={onCancelEdit}
onConfirm={editCtx.cancel}
onCancel={() => (showDiscardDialog = false)}
/>

45
tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessageStatistics.svelte
View file

@ -3,18 +3,18 @@
import { BadgeChatStatistic } from '$lib/components/app';
import * as Tooltip from '$lib/components/ui/tooltip';
import { ChatMessageStatsView } from '$lib/enums';
import { formatPerformanceTime } from '$lib/utils';
import { MS_PER_SECOND, DEFAULT_PERFORMANCE_TIME } from '$lib/constants/formatters';
interface Props {
predictedTokens?: number;
predictedMs?: number;
promptTokens?: number;
promptMs?: number;
// Live mode: when true, shows stats during streaming
isLive?: boolean;
// Whether prompt processing is still in progress
isProcessingPrompt?: boolean;
// Initial view to show (defaults to READING in live mode)
initialView?: ChatMessageStatsView;
onActiveViewChange?: (view: ChatMessageStatsView) => void;
}
let {
@ -24,12 +24,17 @@
promptMs,
isLive = false,
isProcessingPrompt = false,
initialView = ChatMessageStatsView.GENERATION
initialView = ChatMessageStatsView.GENERATION,
onActiveViewChange
}: Props = $props();
let activeView: ChatMessageStatsView = $state(initialView);
let activeView: ChatMessageStatsView = $derived(initialView);
let hasAutoSwitchedToGeneration = $state(false);
$effect(() => {
onActiveViewChange?.(activeView);
});
// In live mode: auto-switch to GENERATION tab when prompt processing completes
$effect(() => {
if (isLive) {
@ -56,26 +61,28 @@
predictedMs > 0
);
let tokensPerSecond = $derived(hasGenerationStats ? (predictedTokens! / predictedMs!) * 1000 : 0);
let timeInSeconds = $derived(
predictedMs !== undefined ? (predictedMs / 1000).toFixed(2) : '0.00'
let tokensPerSecond = $derived(
hasGenerationStats ? (predictedTokens! / predictedMs!) * MS_PER_SECOND : 0
);
let formattedTime = $derived(
predictedMs !== undefined ? formatPerformanceTime(predictedMs) : DEFAULT_PERFORMANCE_TIME
);
let promptTokensPerSecond = $derived(
promptTokens !== undefined && promptMs !== undefined && promptMs > 0
? (promptTokens / promptMs) * 1000
? (promptTokens / promptMs) * MS_PER_SECOND
: undefined
);
let promptTimeInSeconds = $derived(
promptMs !== undefined ? (promptMs / 1000).toFixed(2) : undefined
let formattedPromptTime = $derived(
promptMs !== undefined ? formatPerformanceTime(promptMs) : undefined
);
let hasPromptStats = $derived(
promptTokens !== undefined &&
promptMs !== undefined &&
promptTokensPerSecond !== undefined &&
promptTimeInSeconds !== undefined
formattedPromptTime !== undefined
);
// In live mode, generation tab is disabled until we have generation stats
@ -96,9 +103,11 @@
onclick={() => (activeView = ChatMessageStatsView.READING)}
>
<BookOpenText class="h-3 w-3" />
<span class="sr-only">Reading</span>
</button>
</Tooltip.Trigger>
<Tooltip.Content>
<p>Reading (prompt processing)</p>
</Tooltip.Content>
@ -118,9 +127,11 @@
disabled={isGenerationDisabled}
>
<Sparkles class="h-3 w-3" />
<span class="sr-only">Generation</span>
</button>
</Tooltip.Trigger>
<Tooltip.Content>
<p>
{isGenerationDisabled
@ -139,16 +150,18 @@
value="{predictedTokens?.toLocaleString()} tokens"
tooltipLabel="Generated tokens"
/>
<BadgeChatStatistic
class="bg-transparent"
icon={Clock}
value="{timeInSeconds}s"
value={formattedTime}
tooltipLabel="Generation time"
/>
<BadgeChatStatistic
class="bg-transparent"
icon={Gauge}
value="{tokensPerSecond.toFixed(2)} tokens/s"
value="{tokensPerSecond.toFixed(2)} t/s"
tooltipLabel="Generation speed"
/>
{:else if hasPromptStats}
@ -158,12 +171,14 @@
value="{promptTokens} tokens"
tooltipLabel="Prompt tokens"
/>
<BadgeChatStatistic
class="bg-transparent"
icon={Clock}
value="{promptTimeInSeconds}s"
value={formattedPromptTime ?? '0s'}
tooltipLabel="Prompt processing time"
/>
<BadgeChatStatistic
class="bg-transparent"
icon={Gauge}

65
tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessageSystem.svelte
View file

@ -3,15 +3,16 @@
import { Card } from '$lib/components/ui/card';
import { Button } from '$lib/components/ui/button';
import { MarkdownContent } from '$lib/components/app';
import { INPUT_CLASSES } from '$lib/constants/input-classes';
import { getMessageEditContext } from '$lib/contexts';
import { INPUT_CLASSES } from '$lib/constants/css-classes';
import { config } from '$lib/stores/settings.svelte';
import { isIMEComposing } from '$lib/utils';
import ChatMessageActions from './ChatMessageActions.svelte';
import { KeyboardKey, MessageRole } from '$lib/enums';
interface Props {
class?: string;
message: DatabaseMessage;
isEditing: boolean;
editedContent: string;
siblingInfo?: ChatMessageSiblingInfo | null;
showDeleteDialog: boolean;
deletionInfo: {
@ -20,10 +21,6 @@
assistantMessages: number;
messageTypes: string[];
} | null;
onCancelEdit: () => void;
onSaveEdit: () => void;
onEditKeydown: (event: KeyboardEvent) => void;
onEditedContentChange: (content: string) => void;
onCopy: () => void;
onEdit: () => void;
onDelete: () => void;
@ -36,15 +33,9 @@
let {
class: className = '',
message,
isEditing,
editedContent,
siblingInfo = null,
showDeleteDialog,
deletionInfo,
onCancelEdit,
onSaveEdit,
onEditKeydown,
onEditedContentChange,
onCopy,
onEdit,
onDelete,
@ -54,10 +45,25 @@
textareaElement = $bindable()
}: Props = $props();
const editCtx = getMessageEditContext();
function handleEditKeydown(event: KeyboardEvent) {
if (event.key === KeyboardKey.ENTER && !event.shiftKey && !isIMEComposing(event)) {
event.preventDefault();
editCtx.save();
} else if (event.key === KeyboardKey.ESCAPE) {
event.preventDefault();
editCtx.cancel();
}
}
let isMultiline = $state(false);
let messageElement: HTMLElement | undefined = $state();
let isExpanded = $state(false);
let contentHeight = $state(0);
const MAX_HEIGHT = 200; // pixels
const currentConfig = config();
@ -97,25 +103,32 @@
class="group flex flex-col items-end gap-3 md:gap-2 {className}"
role="group"
>
{#if isEditing}
{#if editCtx.isEditing}
<div class="w-full max-w-[80%]">
<textarea
bind:this={textareaElement}
bind:value={editedContent}
value={editCtx.editedContent}
class="min-h-[60px] w-full resize-none rounded-2xl px-3 py-2 text-sm {INPUT_CLASSES}"
onkeydown={onEditKeydown}
oninput={(e) => onEditedContentChange(e.currentTarget.value)}
onkeydown={handleEditKeydown}
oninput={(e) => editCtx.setContent(e.currentTarget.value)}
placeholder="Edit system message..."
></textarea>
<div class="mt-2 flex justify-end gap-2">
<Button class="h-8 px-3" onclick={onCancelEdit} size="sm" variant="outline">
<Button class="h-8 px-3" onclick={editCtx.cancel} size="sm" variant="outline">
<X class="mr-1 h-3 w-3" />
Cancel
</Button>
<Button class="h-8 px-3" onclick={onSaveEdit} disabled={!editedContent.trim()} size="sm">
<Button
class="h-8 px-3"
onclick={editCtx.save}
disabled={!editCtx.editedContent.trim()}
size="sm"
>
<Check class="mr-1 h-3 w-3" />
Save
</Button>
</div>
@ -131,12 +144,12 @@
type="button"
>
<Card
class="rounded-[1.125rem] !border-2 !border-dashed !border-border/50 bg-muted px-3.75 py-1.5 data-[multiline]:py-2.5"
class="overflow-y-auto rounded-[1.125rem] !border-2 !border-dashed !border-border/50 bg-muted px-3.75 py-1.5 data-[multiline]:py-2.5"
data-multiline={isMultiline ? '' : undefined}
style="border: 2px dashed hsl(var(--border));"
style="border: 2px dashed hsl(var(--border)); max-height: var(--max-message-height);"
>
<div
class="relative overflow-hidden transition-all duration-300 {isExpanded
class="relative transition-all duration-300 {isExpanded
? 'cursor-text select-text'
: 'select-none'}"
style={!isExpanded && showExpandButton
@ -145,7 +158,10 @@
>
{#if currentConfig.renderUserContentAsMarkdown}
<div bind:this={messageElement} class="text-md {isExpanded ? 'cursor-text' : ''}">
<MarkdownContent class="markdown-system-content" content={message.content} />
<MarkdownContent
class="markdown-system-content overflow-auto"
content={message.content}
/>
</div>
{:else}
<span
@ -160,6 +176,7 @@
<div
class="pointer-events-none absolute right-0 bottom-0 left-0 h-48 bg-gradient-to-t from-muted to-transparent"
></div>
<div
class="pointer-events-none absolute right-0 bottom-4 left-0 flex justify-center opacity-0 transition-opacity group-hover/expand:opacity-100"
>
@ -208,7 +225,7 @@
{onShowDeleteDialogChange}
{siblingInfo}
{showDeleteDialog}
role="user"
role={MessageRole.USER}
/>
</div>
{/if}

73
tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessageUser.svelte
View file

@ -1,67 +1,48 @@
<script lang="ts">
import { Card } from '$lib/components/ui/card';
import { ChatAttachmentsList, MarkdownContent } from '$lib/components/app';
import { getMessageEditContext } from '$lib/contexts';
import { config } from '$lib/stores/settings.svelte';
import ChatMessageActions from './ChatMessageActions.svelte';
import ChatMessageEditForm from './ChatMessageEditForm.svelte';
import { MessageRole } from '$lib/enums';
interface Props {
class?: string;
message: DatabaseMessage;
isEditing: boolean;
editedContent: string;
editedExtras?: DatabaseMessageExtra[];
editedUploadedFiles?: ChatUploadedFile[];
siblingInfo?: ChatMessageSiblingInfo | null;
showDeleteDialog: boolean;
deletionInfo: {
totalCount: number;
userMessages: number;
assistantMessages: number;
messageTypes: string[];
} | null;
onCancelEdit: () => void;
onSaveEdit: () => void;
onSaveEditOnly?: () => void;
onEditKeydown: (event: KeyboardEvent) => void;
onEditedContentChange: (content: string) => void;
onEditedExtrasChange?: (extras: DatabaseMessageExtra[]) => void;
onEditedUploadedFilesChange?: (files: ChatUploadedFile[]) => void;
onCopy: () => void;
showDeleteDialog: boolean;
onEdit: () => void;
onDelete: () => void;
onConfirmDelete: () => void;
onNavigateToSibling?: (siblingId: string) => void;
onShowDeleteDialogChange: (show: boolean) => void;
textareaElement?: HTMLTextAreaElement;
onNavigateToSibling?: (siblingId: string) => void;
onCopy: () => void;
}
let {
class: className = '',
message,
isEditing,
editedContent,
editedExtras = [],
editedUploadedFiles = [],
siblingInfo = null,
showDeleteDialog,
deletionInfo,
onCancelEdit,
onSaveEdit,
onSaveEditOnly,
onEditKeydown,
onEditedContentChange,
onEditedExtrasChange,
onEditedUploadedFilesChange,
onCopy,
showDeleteDialog,
onEdit,
onDelete,
onConfirmDelete,
onNavigateToSibling,
onShowDeleteDialogChange,
textareaElement = $bindable()
onNavigateToSibling,
onCopy
}: Props = $props();
// Get contexts
const editCtx = getMessageEditContext();
let isMultiline = $state(false);
let messageElement: HTMLElement | undefined = $state();
const currentConfig = config();
@ -96,24 +77,8 @@
class="group flex flex-col items-end gap-3 md:gap-2 {className}"
role="group"
>
{#if isEditing}
<ChatMessageEditForm
bind:textareaElement
messageId={message.id}
{editedContent}
{editedExtras}
{editedUploadedFiles}
originalContent={message.content}
originalExtras={message.extra}
showSaveOnlyOption={!!onSaveEditOnly}
{onCancelEdit}
{onSaveEdit}
{onSaveEditOnly}
{onEditKeydown}
{onEditedContentChange}
{onEditedExtrasChange}
{onEditedUploadedFilesChange}
/>
{#if editCtx.isEditing}
<ChatMessageEditForm />
{:else}
{#if message.extra && message.extra.length > 0}
<div class="mb-2 max-w-[80%]">
@ -123,15 +88,13 @@
{#if message.content.trim()}
<Card
class="max-w-[80%] rounded-[1.125rem] border-none bg-primary px-3.75 py-1.5 text-primary-foreground data-[multiline]:py-2.5"
class="max-w-[80%] overflow-y-auto rounded-[1.125rem] border-none bg-primary/5 px-3.75 py-1.5 text-foreground backdrop-blur-md data-[multiline]:py-2.5 dark:bg-primary/15"
data-multiline={isMultiline ? '' : undefined}
style="max-height: var(--max-message-height); overflow-wrap: anywhere; word-break: break-word;"
>
{#if currentConfig.renderUserContentAsMarkdown}
<div bind:this={messageElement} class="text-md">
<MarkdownContent
class="markdown-user-content text-primary-foreground"
content={message.content}
/>
<div bind:this={messageElement}>
<MarkdownContent class="markdown-user-content -my-4" content={message.content} />
</div>
{:else}
<span bind:this={messageElement} class="text-md whitespace-pre-wrap">
@ -155,7 +118,7 @@
{onShowDeleteDialogChange}
{siblingInfo}
{showDeleteDialog}
role="user"
role={MessageRole.USER}
/>
</div>
{/if}

159
tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessages.svelte
View file

@ -1,9 +1,11 @@
<script lang="ts">
import { ChatMessage } from '$lib/components/app';
import { setChatActionsContext } from '$lib/contexts';
import { MessageRole } from '$lib/enums';
import { chatStore } from '$lib/stores/chat.svelte';
import { conversationsStore, activeConversation } from '$lib/stores/conversations.svelte';
import { config } from '$lib/stores/settings.svelte';
import { getMessageSiblings } from '$lib/utils';
import { copyToClipboard, formatMessageForClipboard, getMessageSiblings } from '$lib/utils';
interface Props {
class?: string;
@ -16,6 +18,69 @@
let allConversationMessages = $state<DatabaseMessage[]>([]);
const currentConfig = config();
setChatActionsContext({
copy: async (message: DatabaseMessage) => {
const asPlainText = Boolean(currentConfig.copyTextAttachmentsAsPlainText);
const clipboardContent = formatMessageForClipboard(
message.content,
message.extra,
asPlainText
);
await copyToClipboard(clipboardContent, 'Message copied to clipboard');
},
delete: async (message: DatabaseMessage) => {
await chatStore.deleteMessage(message.id);
refreshAllMessages();
},
navigateToSibling: async (siblingId: string) => {
await conversationsStore.navigateToSibling(siblingId);
},
editWithBranching: async (
message: DatabaseMessage,
newContent: string,
newExtras?: DatabaseMessageExtra[]
) => {
onUserAction?.();
await chatStore.editMessageWithBranching(message.id, newContent, newExtras);
refreshAllMessages();
},
editWithReplacement: async (
message: DatabaseMessage,
newContent: string,
shouldBranch: boolean
) => {
onUserAction?.();
await chatStore.editAssistantMessage(message.id, newContent, shouldBranch);
refreshAllMessages();
},
editUserMessagePreserveResponses: async (
message: DatabaseMessage,
newContent: string,
newExtras?: DatabaseMessageExtra[]
) => {
onUserAction?.();
await chatStore.editUserMessagePreserveResponses(message.id, newContent, newExtras);
refreshAllMessages();
},
regenerateWithBranching: async (message: DatabaseMessage, modelOverride?: string) => {
onUserAction?.();
await chatStore.regenerateMessageWithBranching(message.id, modelOverride);
refreshAllMessages();
},
continueAssistantMessage: async (message: DatabaseMessage) => {
onUserAction?.();
await chatStore.continueAssistantMessage(message.id);
refreshAllMessages();
}
});
function refreshAllMessages() {
const conversation = activeConversation();
@ -42,16 +107,28 @@
return [];
}
// Filter out system messages if showSystemMessage is false
const filteredMessages = currentConfig.showSystemMessage
? messages
: messages.filter((msg) => msg.type !== 'system');
: messages.filter((msg) => msg.type !== MessageRole.SYSTEM);
return filteredMessages.map((message) => {
let lastAssistantIndex = -1;
for (let i = filteredMessages.length - 1; i >= 0; i--) {
if (filteredMessages[i].role === MessageRole.ASSISTANT) {
lastAssistantIndex = i;
break;
}
}
return filteredMessages.map((message, index) => {
const siblingInfo = getMessageSiblings(allConversationMessages, message.id);
const isLastAssistantMessage =
message.role === MessageRole.ASSISTANT && index === lastAssistantIndex;
return {
message,
isLastAssistantMessage,
siblingInfo: siblingInfo || {
message,
siblingIds: [message.id],
@ -61,83 +138,15 @@
};
});
});
async function handleNavigateToSibling(siblingId: string) {
await conversationsStore.navigateToSibling(siblingId);
}
async function handleEditWithBranching(
message: DatabaseMessage,
newContent: string,
newExtras?: DatabaseMessageExtra[]
) {
onUserAction?.();
await chatStore.editMessageWithBranching(message.id, newContent, newExtras);
refreshAllMessages();
}
async function handleEditWithReplacement(
message: DatabaseMessage,
newContent: string,
shouldBranch: boolean
) {
onUserAction?.();
await chatStore.editAssistantMessage(message.id, newContent, shouldBranch);
refreshAllMessages();
}
async function handleRegenerateWithBranching(message: DatabaseMessage, modelOverride?: string) {
onUserAction?.();
await chatStore.regenerateMessageWithBranching(message.id, modelOverride);
refreshAllMessages();
}
async function handleContinueAssistantMessage(message: DatabaseMessage) {
onUserAction?.();
await chatStore.continueAssistantMessage(message.id);
refreshAllMessages();
}
async function handleEditUserMessagePreserveResponses(
message: DatabaseMessage,
newContent: string,
newExtras?: DatabaseMessageExtra[]
) {
onUserAction?.();
await chatStore.editUserMessagePreserveResponses(message.id, newContent, newExtras);
refreshAllMessages();
}
async function handleDeleteMessage(message: DatabaseMessage) {
await chatStore.deleteMessage(message.id);
refreshAllMessages();
}
</script>
<div class="flex h-full flex-col space-y-10 pt-16 md:pt-24 {className}" style="height: auto; ">
{#each displayMessages as { message, siblingInfo } (message.id)}
<div class="flex h-full flex-col space-y-10 pt-24 {className}" style="height: auto; ">
{#each displayMessages as { message, isLastAssistantMessage, siblingInfo } (message.id)}
<ChatMessage
class="mx-auto w-full max-w-[48rem]"
{message}
{isLastAssistantMessage}
{siblingInfo}
onDelete={handleDeleteMessage}
onNavigateToSibling={handleNavigateToSibling}
onEditWithBranching={handleEditWithBranching}
onEditWithReplacement={handleEditWithReplacement}
onEditUserMessagePreserveResponses={handleEditUserMessagePreserveResponses}
onRegenerateWithBranching={handleRegenerateWithBranching}
onContinueAssistantMessage={handleContinueAssistantMessage}
/>
{/each}
</div>

116
tools/server/webui/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte
View file

@ -1,7 +1,7 @@
<script lang="ts">
import { afterNavigate } from '$app/navigation';
import {
ChatForm,
ChatScreenForm,
ChatScreenHeader,
ChatMessages,
ChatScreenProcessingInfo,
@ -12,15 +12,14 @@
} from '$lib/components/app';
import * as Alert from '$lib/components/ui/alert';
import * as AlertDialog from '$lib/components/ui/alert-dialog';
import {
AUTO_SCROLL_AT_BOTTOM_THRESHOLD,
AUTO_SCROLL_INTERVAL,
INITIAL_SCROLL_DELAY
} from '$lib/constants/auto-scroll';
import { INITIAL_SCROLL_DELAY } from '$lib/constants/auto-scroll';
import { KeyboardKey } from '$lib/enums';
import { createAutoScrollController } from '$lib/hooks/use-auto-scroll.svelte';
import {
chatStore,
errorDialog,
isLoading,
isChatStreaming,
isEditing,
getAddFilesHandler
} from '$lib/stores/chat.svelte';
@ -34,6 +33,7 @@
import { modelsStore, modelOptions, selectedModelId } from '$lib/stores/models.svelte';
import { isFileTypeSupported, filterFilesByModalities } from '$lib/utils';
import { parseFilesToMessageExtras, processFilesToChatUploaded } from '$lib/utils/browser-only';
import { ErrorDialogType } from '$lib/enums';
import { onMount } from 'svelte';
import { fade, fly, slide } from 'svelte/transition';
import { Trash2, AlertTriangle, RefreshCw } from '@lucide/svelte';
@ -42,16 +42,13 @@
let { showCenteredEmpty = false } = $props();
let disableAutoScroll = $derived(Boolean(config().disableAutoScroll));
let autoScrollEnabled = $state(true);
let chatScrollContainer: HTMLDivElement | undefined = $state();
let dragCounter = $state(0);
let isDragOver = $state(false);
let lastScrollTop = $state(0);
let scrollInterval: ReturnType<typeof setInterval> | undefined;
let scrollTimeout: ReturnType<typeof setTimeout> | undefined;
let showFileErrorDialog = $state(false);
let uploadedFiles = $state<ChatUploadedFile[]>([]);
let userScrolledUp = $state(false);
const autoScroll = createAutoScrollController();
let fileErrorData = $state<{
generallyUnsupported: File[];
@ -81,7 +78,7 @@
let isServerLoading = $derived(serverLoading());
let hasPropsError = $derived(!!serverError());
let isCurrentConversationLoading = $derived(isLoading());
let isCurrentConversationLoading = $derived(isLoading() || isChatStreaming());
let isRouter = $derived(isRouterMode());
@ -215,7 +212,11 @@
function handleKeydown(event: KeyboardEvent) {
const isCtrlOrCmd = event.ctrlKey || event.metaKey;
if (isCtrlOrCmd && event.shiftKey && (event.key === 'd' || event.key === 'D')) {
if (
isCtrlOrCmd &&
event.shiftKey &&
(event.key === KeyboardKey.D_LOWER || event.key === KeyboardKey.D_UPPER)
) {
event.preventDefault();
if (activeConversation()) {
showDeleteDialog = true;
@ -232,37 +233,13 @@
}
function handleScroll() {
if (disableAutoScroll || !chatScrollContainer) return;
const { scrollTop, scrollHeight, clientHeight } = chatScrollContainer;
const distanceFromBottom = scrollHeight - scrollTop - clientHeight;
const isAtBottom = distanceFromBottom < AUTO_SCROLL_AT_BOTTOM_THRESHOLD;
if (scrollTop < lastScrollTop && !isAtBottom) {
userScrolledUp = true;
autoScrollEnabled = false;
} else if (isAtBottom && userScrolledUp) {
userScrolledUp = false;
autoScrollEnabled = true;
}
if (scrollTimeout) {
clearTimeout(scrollTimeout);
}
scrollTimeout = setTimeout(() => {
if (isAtBottom) {
userScrolledUp = false;
autoScrollEnabled = true;
}
}, AUTO_SCROLL_INTERVAL);
lastScrollTop = scrollTop;
autoScroll.handleScroll();
}
async function handleSendMessage(message: string, files?: ChatUploadedFile[]): Promise<boolean> {
const result = files
? await parseFilesToMessageExtras(files, activeModelId ?? undefined)
const plainFiles = files ? $state.snapshot(files) : undefined;
const result = plainFiles
? await parseFilesToMessageExtras(plainFiles, activeModelId ?? undefined)
: undefined;
if (result?.emptyFiles && result.emptyFiles.length > 0) {
@ -279,12 +256,9 @@
const extras = result?.extras;
// Enable autoscroll for user-initiated message sending
if (!disableAutoScroll) {
userScrolledUp = false;
autoScrollEnabled = true;
}
autoScroll.enable();
await chatStore.sendMessage(message, extras);
scrollChatToBottom();
autoScroll.scrollToBottom();
return true;
}
@ -334,24 +308,15 @@
}
}
function scrollChatToBottom(behavior: ScrollBehavior = 'smooth') {
if (disableAutoScroll) return;
chatScrollContainer?.scrollTo({
top: chatScrollContainer?.scrollHeight,
behavior
});
}
afterNavigate(() => {
if (!disableAutoScroll) {
setTimeout(() => scrollChatToBottom('instant'), INITIAL_SCROLL_DELAY);
setTimeout(() => autoScroll.scrollToBottom('instant'), INITIAL_SCROLL_DELAY);
}
});
onMount(() => {
if (!disableAutoScroll) {
setTimeout(() => scrollChatToBottom('instant'), INITIAL_SCROLL_DELAY);
setTimeout(() => autoScroll.scrollToBottom('instant'), INITIAL_SCROLL_DELAY);
}
const pendingDraft = chatStore.consumePendingDraft();
@ -362,21 +327,15 @@
});
$effect(() => {
if (disableAutoScroll) {
autoScrollEnabled = false;
if (scrollInterval) {
clearInterval(scrollInterval);
scrollInterval = undefined;
}
return;
}
autoScroll.setContainer(chatScrollContainer);
});
if (isCurrentConversationLoading && autoScrollEnabled) {
scrollInterval = setInterval(scrollChatToBottom, AUTO_SCROLL_INTERVAL);
} else if (scrollInterval) {
clearInterval(scrollInterval);
scrollInterval = undefined;
}
$effect(() => {
autoScroll.setDisabled(disableAutoScroll);
});
$effect(() => {
autoScroll.updateInterval(isCurrentConversationLoading);
});
</script>
@ -404,11 +363,8 @@
class="mb-16 md:mb-24"
messages={activeMessages()}
onUserAction={() => {
if (!disableAutoScroll) {
userScrolledUp = false;
autoScrollEnabled = true;
scrollChatToBottom();
}
autoScroll.enable();
autoScroll.scrollToBottom();
}}
/>
@ -442,7 +398,7 @@
{/if}
<div class="conversation-chat-form pointer-events-auto rounded-t-3xl pb-4">
<ChatForm
<ChatScreenForm
disabled={hasPropsError || isEditing()}
{initialMessage}
isLoading={isCurrentConversationLoading}
@ -472,7 +428,7 @@
>
<div class="w-full max-w-[48rem] px-4">
<div class="mb-10 text-center" in:fade={{ duration: 300 }}>
<h1 class="mb-4 text-3xl font-semibold tracking-tight">llama.cpp</h1>
<h1 class="mb-2 text-3xl font-semibold tracking-tight">llama.cpp</h1>
<p class="text-lg text-muted-foreground">
{serverStore.props?.modalities?.audio
@ -502,7 +458,7 @@
{/if}
<div in:fly={{ y: 10, duration: 250, delay: hasPropsError ? 0 : 300 }}>
<ChatForm
<ChatScreenForm
disabled={hasPropsError}
{initialMessage}
isLoading={isCurrentConversationLoading}
@ -615,7 +571,7 @@
contextInfo={activeErrorDialog?.contextInfo}
onOpenChange={handleErrorDialogOpenChange}
open={Boolean(activeErrorDialog)}
type={activeErrorDialog?.type ?? 'server'}
type={activeErrorDialog?.type ?? ErrorDialogType.SERVER}
/>
<style>

121
tools/server/webui/src/lib/components/app/chat/ChatScreen/ChatScreenForm.svelte Normal file
View file

@ -0,0 +1,121 @@
<script lang="ts">
import { afterNavigate } from '$app/navigation';
import { ChatFormHelperText, ChatForm } from '$lib/components/app';
import { onMount } from 'svelte';
interface Props {
class?: string;
disabled?: boolean;
initialMessage?: string;
isLoading?: boolean;
onFileRemove?: (fileId: string) => void;
onFileUpload?: (files: File[]) => void;
onSend?: (message: string, files?: ChatUploadedFile[]) => Promise<boolean>;
onStop?: () => void;
onSystemPromptAdd?: (draft: { message: string; files: ChatUploadedFile[] }) => void;
showHelperText?: boolean;
uploadedFiles?: ChatUploadedFile[];
}
let {
class: className,
disabled = false,
initialMessage = '',
isLoading = false,
onFileRemove,
onFileUpload,
onSend,
onStop,
onSystemPromptAdd,
showHelperText = true,
uploadedFiles = $bindable([])
}: Props = $props();
let chatFormRef: ChatForm | undefined = $state(undefined);
let message = $derived(initialMessage);
let previousIsLoading = $derived(isLoading);
let previousInitialMessage = $derived(initialMessage);
// Sync message when initialMessage prop changes (e.g., after draft restoration)
$effect(() => {
if (initialMessage !== previousInitialMessage) {
message = initialMessage;
previousInitialMessage = initialMessage;
}
});
function handleSystemPromptClick() {
onSystemPromptAdd?.({ message, files: uploadedFiles });
}
let hasLoadingAttachments = $derived(uploadedFiles.some((f) => f.isLoading));
async function handleSubmit() {
if (
(!message.trim() && uploadedFiles.length === 0) ||
disabled ||
isLoading ||
hasLoadingAttachments
)
return;
if (!chatFormRef?.checkModelSelected()) return;
const messageToSend = message.trim();
const filesToSend = [...uploadedFiles];
message = '';
uploadedFiles = [];
chatFormRef?.resetTextareaHeight();
const success = await onSend?.(messageToSend, filesToSend);
if (!success) {
message = messageToSend;
uploadedFiles = filesToSend;
}
}
function handleFilesAdd(files: File[]) {
onFileUpload?.(files);
}
function handleUploadedFileRemove(fileId: string) {
onFileRemove?.(fileId);
}
onMount(() => {
setTimeout(() => chatFormRef?.focus(), 10);
});
afterNavigate(() => {
setTimeout(() => chatFormRef?.focus(), 10);
});
$effect(() => {
if (previousIsLoading && !isLoading) {
setTimeout(() => chatFormRef?.focus(), 10);
}
previousIsLoading = isLoading;
});
</script>
<div class="relative mx-auto max-w-[48rem]">
<ChatForm
bind:this={chatFormRef}
bind:value={message}
bind:uploadedFiles
class={className}
{disabled}
{isLoading}
onFilesAdd={handleFilesAdd}
{onStop}
onSubmit={handleSubmit}
onSystemPromptClick={handleSystemPromptClick}
onUploadedFileRemove={handleUploadedFileRemove}
/>
</div>
<ChatFormHelperText show={showHelperText} />

9
tools/server/webui/src/lib/components/app/chat/ChatScreen/ChatScreenHeader.svelte
View file

@ -14,12 +14,17 @@
</script>
<header
class="md:background-transparent pointer-events-none fixed top-0 right-0 left-0 z-50 flex items-center justify-end bg-background/40 p-4 backdrop-blur-xl duration-200 ease-linear {sidebar.open
class="pointer-events-none fixed top-0 right-0 left-0 z-50 flex items-center justify-end p-4 duration-200 ease-linear {sidebar.open
? 'md:left-[var(--sidebar-width)]'
: ''}"
>
<div class="pointer-events-auto flex items-center space-x-2">
<Button variant="ghost" size="sm" onclick={toggleSettings}>
<Button
variant="ghost"
size="icon"
onclick={toggleSettings}
class="rounded-full backdrop-blur-lg"
>
<Settings class="h-4 w-4" />
</Button>
</div>

7
tools/server/webui/src/lib/components/app/chat/ChatScreen/ChatScreenProcessingInfo.svelte
View file

@ -11,7 +11,7 @@
let isCurrentConversationLoading = $derived(isLoading());
let isStreaming = $derived(isChatStreaming());
let hasProcessingData = $derived(processingState.processingState !== null);
let processingDetails = $derived(processingState.getProcessingDetails());
let processingDetails = $derived(processingState.getTechnicalDetails());
let showProcessingInfo = $derived(
isCurrentConversationLoading || isStreaming || config().keepStatsVisible || hasProcessingData
@ -63,7 +63,7 @@
<div class="chat-processing-info-container pointer-events-none" class:visible={showProcessingInfo}>
<div class="chat-processing-info-content">
{#each processingDetails as detail (detail)}
<span class="chat-processing-info-detail pointer-events-auto">{detail}</span>
<span class="chat-processing-info-detail pointer-events-auto backdrop-blur-sm">{detail}</span>
{/each}
</div>
</div>
@ -73,7 +73,7 @@
position: sticky;
top: 0;
z-index: 10;
padding: 1.5rem 1rem;
padding: 0 1rem 0.75rem;
opacity: 0;
transform: translateY(50%);
transition:
@ -100,7 +100,6 @@
color: var(--muted-foreground);
font-size: 0.75rem;
padding: 0.25rem 0.75rem;
background: var(--muted);
border-radius: 0.375rem;
font-family:
ui-monospace, SFMono-Regular, 'SF Mono', Consolas, 'Liberation Mono', Menlo, monospace;

242
tools/server/webui/src/lib/components/app/chat/ChatSettings/ChatSettings.svelte
View file

@ -5,8 +5,6 @@
AlertTriangle,
Code,
Monitor,
Sun,
Moon,
ChevronLeft,
ChevronRight,
Database
@ -18,250 +16,261 @@
} from '$lib/components/app';
import { ScrollArea } from '$lib/components/ui/scroll-area';
import { config, settingsStore } from '$lib/stores/settings.svelte';
import {
SETTINGS_SECTION_TITLES,
type SettingsSectionTitle
} from '$lib/constants/settings-sections';
import { setMode } from 'mode-watcher';
import { ColorMode } from '$lib/enums/ui';
import { SettingsFieldType } from '$lib/enums/settings';
import type { Component } from 'svelte';
import { NUMERIC_FIELDS, POSITIVE_INTEGER_FIELDS } from '$lib/constants/settings-fields';
import { SETTINGS_COLOR_MODES_CONFIG } from '$lib/constants/settings-config';
import { SETTINGS_KEYS } from '$lib/constants/settings-keys';
interface Props {
onSave?: () => void;
initialSection?: SettingsSectionTitle;
}
let { onSave }: Props = $props();
let { onSave, initialSection }: Props = $props();
const settingSections: Array<{
fields: SettingsFieldConfig[];
icon: Component;
title: string;
title: SettingsSectionTitle;
}> = [
{
title: 'General',
title: SETTINGS_SECTION_TITLES.GENERAL,
icon: Settings,
fields: [
{
key: 'theme',
key: SETTINGS_KEYS.THEME,
label: 'Theme',
type: 'select',
options: [
{ value: 'system', label: 'System', icon: Monitor },
{ value: 'light', label: 'Light', icon: Sun },
{ value: 'dark', label: 'Dark', icon: Moon }
]
type: SettingsFieldType.SELECT,
options: SETTINGS_COLOR_MODES_CONFIG
},
{ key: 'apiKey', label: 'API Key', type: 'input' },
{ key: SETTINGS_KEYS.API_KEY, label: 'API Key', type: SettingsFieldType.INPUT },
{
key: 'systemMessage',
key: SETTINGS_KEYS.SYSTEM_MESSAGE,
label: 'System Message',
type: 'textarea'
type: SettingsFieldType.TEXTAREA
},
{
key: 'pasteLongTextToFileLen',
key: SETTINGS_KEYS.PASTE_LONG_TEXT_TO_FILE_LEN,
label: 'Paste long text to file length',
type: 'input'
type: SettingsFieldType.INPUT
},
{
key: 'copyTextAttachmentsAsPlainText',
key: SETTINGS_KEYS.COPY_TEXT_ATTACHMENTS_AS_PLAIN_TEXT,
label: 'Copy text attachments as plain text',
type: 'checkbox'
type: SettingsFieldType.CHECKBOX
},
{
key: 'enableContinueGeneration',
key: SETTINGS_KEYS.ENABLE_CONTINUE_GENERATION,
label: 'Enable "Continue" button',
type: 'checkbox',
type: SettingsFieldType.CHECKBOX,
isExperimental: true
},
{
key: 'pdfAsImage',
key: SETTINGS_KEYS.PDF_AS_IMAGE,
label: 'Parse PDF as image',
type: 'checkbox'
type: SettingsFieldType.CHECKBOX
},
{
key: 'askForTitleConfirmation',
key: SETTINGS_KEYS.ASK_FOR_TITLE_CONFIRMATION,
label: 'Ask for confirmation before changing conversation title',
type: 'checkbox'
type: SettingsFieldType.CHECKBOX
}
]
},
{
title: 'Display',
title: SETTINGS_SECTION_TITLES.DISPLAY,
icon: Monitor,
fields: [
{
key: 'showMessageStats',
key: SETTINGS_KEYS.SHOW_MESSAGE_STATS,
label: 'Show message generation statistics',
type: 'checkbox'
type: SettingsFieldType.CHECKBOX
},
{
key: 'showThoughtInProgress',
key: SETTINGS_KEYS.SHOW_THOUGHT_IN_PROGRESS,
label: 'Show thought in progress',
type: 'checkbox'
type: SettingsFieldType.CHECKBOX
},
{
key: 'keepStatsVisible',
key: SETTINGS_KEYS.KEEP_STATS_VISIBLE,
label: 'Keep stats visible after generation',
type: 'checkbox'
type: SettingsFieldType.CHECKBOX
},
{
key: 'autoMicOnEmpty',
key: SETTINGS_KEYS.AUTO_MIC_ON_EMPTY,
label: 'Show microphone on empty input',
type: 'checkbox',
type: SettingsFieldType.CHECKBOX,
isExperimental: true
},
{
key: 'renderUserContentAsMarkdown',
key: SETTINGS_KEYS.RENDER_USER_CONTENT_AS_MARKDOWN,
label: 'Render user content as Markdown',
type: 'checkbox'
type: SettingsFieldType.CHECKBOX
},
{
key: 'disableAutoScroll',
key: SETTINGS_KEYS.FULL_HEIGHT_CODE_BLOCKS,
label: 'Use full height code blocks',
type: SettingsFieldType.CHECKBOX
},
{
key: SETTINGS_KEYS.DISABLE_AUTO_SCROLL,
label: 'Disable automatic scroll',
type: 'checkbox'
type: SettingsFieldType.CHECKBOX
},
{
key: 'alwaysShowSidebarOnDesktop',
key: SETTINGS_KEYS.ALWAYS_SHOW_SIDEBAR_ON_DESKTOP,
label: 'Always show sidebar on desktop',
type: 'checkbox'
type: SettingsFieldType.CHECKBOX
},
{
key: 'autoShowSidebarOnNewChat',
key: SETTINGS_KEYS.AUTO_SHOW_SIDEBAR_ON_NEW_CHAT,
label: 'Auto-show sidebar on new chat',
type: 'checkbox'
type: SettingsFieldType.CHECKBOX
}
]
},
{
title: 'Sampling',
title: SETTINGS_SECTION_TITLES.SAMPLING,
icon: Funnel,
fields: [
{
key: 'temperature',
key: SETTINGS_KEYS.TEMPERATURE,
label: 'Temperature',
type: 'input'
type: SettingsFieldType.INPUT
},
{
key: 'dynatemp_range',
key: SETTINGS_KEYS.DYNATEMP_RANGE,
label: 'Dynamic temperature range',
type: 'input'
type: SettingsFieldType.INPUT
},
{
key: 'dynatemp_exponent',
key: SETTINGS_KEYS.DYNATEMP_EXPONENT,
label: 'Dynamic temperature exponent',
type: 'input'
type: SettingsFieldType.INPUT
},
{
key: 'top_k',
key: SETTINGS_KEYS.TOP_K,
label: 'Top K',
type: 'input'
type: SettingsFieldType.INPUT
},
{
key: 'top_p',
key: SETTINGS_KEYS.TOP_P,
label: 'Top P',
type: 'input'
type: SettingsFieldType.INPUT
},
{
key: 'min_p',
key: SETTINGS_KEYS.MIN_P,
label: 'Min P',
type: 'input'
type: SettingsFieldType.INPUT
},
{
key: 'xtc_probability',
key: SETTINGS_KEYS.XTC_PROBABILITY,
label: 'XTC probability',
type: 'input'
type: SettingsFieldType.INPUT
},
{
key: 'xtc_threshold',
key: SETTINGS_KEYS.XTC_THRESHOLD,
label: 'XTC threshold',
type: 'input'
type: SettingsFieldType.INPUT
},
{
key: 'typ_p',
key: SETTINGS_KEYS.TYP_P,
label: 'Typical P',
type: 'input'
type: SettingsFieldType.INPUT
},
{
key: 'max_tokens',
key: SETTINGS_KEYS.MAX_TOKENS,
label: 'Max tokens',
type: 'input'
type: SettingsFieldType.INPUT
},
{
key: 'samplers',
key: SETTINGS_KEYS.SAMPLERS,
label: 'Samplers',
type: 'input'
type: SettingsFieldType.INPUT
},
{
key: 'backend_sampling',
key: SETTINGS_KEYS.BACKEND_SAMPLING,
label: 'Backend sampling',
type: 'checkbox'
type: SettingsFieldType.CHECKBOX
}
]
},
{
title: 'Penalties',
title: SETTINGS_SECTION_TITLES.PENALTIES,
icon: AlertTriangle,
fields: [
{
key: 'repeat_last_n',
key: SETTINGS_KEYS.REPEAT_LAST_N,
label: 'Repeat last N',
type: 'input'
type: SettingsFieldType.INPUT
},
{
key: 'repeat_penalty',
key: SETTINGS_KEYS.REPEAT_PENALTY,
label: 'Repeat penalty',
type: 'input'
type: SettingsFieldType.INPUT
},
{
key: 'presence_penalty',
key: SETTINGS_KEYS.PRESENCE_PENALTY,
label: 'Presence penalty',
type: 'input'
type: SettingsFieldType.INPUT
},
{
key: 'frequency_penalty',
key: SETTINGS_KEYS.FREQUENCY_PENALTY,
label: 'Frequency penalty',
type: 'input'
type: SettingsFieldType.INPUT
},
{
key: 'dry_multiplier',
key: SETTINGS_KEYS.DRY_MULTIPLIER,
label: 'DRY multiplier',
type: 'input'
type: SettingsFieldType.INPUT
},
{
key: 'dry_base',
key: SETTINGS_KEYS.DRY_BASE,
label: 'DRY base',
type: 'input'
type: SettingsFieldType.INPUT
},
{
key: 'dry_allowed_length',
key: SETTINGS_KEYS.DRY_ALLOWED_LENGTH,
label: 'DRY allowed length',
type: 'input'
type: SettingsFieldType.INPUT
},
{
key: 'dry_penalty_last_n',
key: SETTINGS_KEYS.DRY_PENALTY_LAST_N,
label: 'DRY penalty last N',
type: 'input'
type: SettingsFieldType.INPUT
}
]
},
{
title: 'Import/Export',
title: SETTINGS_SECTION_TITLES.IMPORT_EXPORT,
icon: Database,
fields: []
},
{
title: 'Developer',
title: SETTINGS_SECTION_TITLES.DEVELOPER,
icon: Code,
fields: [
{
key: 'showToolCalls',
label: 'Show tool call labels',
type: 'checkbox'
key: SETTINGS_KEYS.DISABLE_REASONING_PARSING,
label: 'Disable reasoning content parsing',
type: SettingsFieldType.CHECKBOX
},
{
key: 'disableReasoningFormat',
label: 'Show raw LLM output',
type: 'checkbox'
key: SETTINGS_KEYS.SHOW_RAW_OUTPUT_SWITCH,
label: 'Enable raw output toggle',
type: SettingsFieldType.CHECKBOX
},
{
key: 'custom',
key: SETTINGS_KEYS.CUSTOM,
label: 'Custom JSON',
type: 'textarea'
type: SettingsFieldType.TEXTAREA
}
]
}
@ -280,7 +289,9 @@
// }
];
let activeSection = $state('General');
let activeSection = $derived<SettingsSectionTitle>(
initialSection ?? SETTINGS_SECTION_TITLES.GENERAL
);
let currentSection = $derived(
settingSections.find((section) => section.title === activeSection) || settingSections[0]
);
@ -290,10 +301,16 @@
let canScrollRight = $state(false);
let scrollContainer: HTMLDivElement | undefined = $state();
$effect(() => {
if (initialSection) {
activeSection = initialSection;
}
});
function handleThemeChange(newTheme: string) {
localConfig.theme = newTheme;
setMode(newTheme as 'light' | 'dark' | 'system');
setMode(newTheme as ColorMode);
}
function handleConfigChange(key: string, value: string | boolean) {
@ -303,7 +320,7 @@
function handleReset() {
localConfig = { ...config() };
setMode(localConfig.theme as 'light' | 'dark' | 'system');
setMode(localConfig.theme as ColorMode);
}
function handleSave() {
@ -319,33 +336,16 @@
// Convert numeric strings to numbers for numeric fields
const processedConfig = { ...localConfig };
const numericFields = [
'temperature',
'top_k',
'top_p',
'min_p',
'max_tokens',
'pasteLongTextToFileLen',
'dynatemp_range',
'dynatemp_exponent',
'typ_p',
'xtc_probability',
'xtc_threshold',
'repeat_last_n',
'repeat_penalty',
'presence_penalty',
'frequency_penalty',
'dry_multiplier',
'dry_base',
'dry_allowed_length',
'dry_penalty_last_n'
];
for (const field of numericFields) {
for (const field of NUMERIC_FIELDS) {
if (processedConfig[field] !== undefined && processedConfig[field] !== '') {
const numValue = Number(processedConfig[field]);
if (!isNaN(numValue)) {
processedConfig[field] = numValue;
if ((POSITIVE_INTEGER_FIELDS as readonly string[]).includes(field)) {
processedConfig[field] = Math.max(1, Math.round(numValue));
} else {
processedConfig[field] = numValue;
}
} else {
alert(`Invalid numeric value for ${field}. Please enter a valid number.`);
return;
@ -484,7 +484,7 @@
<h3 class="text-lg font-semibold">{currentSection.title}</h3>
</div>
{#if currentSection.title === 'Import/Export'}
{#if currentSection.title === SETTINGS_SECTION_TITLES.IMPORT_EXPORT}
<ChatSettingsImportExportTab />
{:else}
<div class="space-y-6">

14
tools/server/webui/src/lib/components/app/chat/ChatSettings/ChatSettingsFields.svelte
View file

@ -6,6 +6,8 @@
import * as Select from '$lib/components/ui/select';
import { Textarea } from '$lib/components/ui/textarea';
import { SETTING_CONFIG_DEFAULT, SETTING_CONFIG_INFO } from '$lib/constants/settings-config';
import { SETTINGS_KEYS } from '$lib/constants/settings-keys';
import { SettingsFieldType } from '$lib/enums/settings';
import { settingsStore } from '$lib/stores/settings.svelte';
import { ChatSettingsParameterSourceIndicator } from '$lib/components/app';
import type { Component } from 'svelte';
@ -31,7 +33,7 @@
{#each fields as field (field.key)}
<div class="space-y-2">
{#if field.type === 'input'}
{#if field.type === SettingsFieldType.INPUT}
{@const paramInfo = getParameterSourceInfo(field.key)}
{@const currentValue = String(localConfig[field.key] ?? '')}
{@const propsDefault = paramInfo?.serverDefault}
@ -98,7 +100,7 @@
{@html field.help || SETTING_CONFIG_INFO[field.key]}
</p>
{/if}
{:else if field.type === 'textarea'}
{:else if field.type === SettingsFieldType.TEXTAREA}
<Label for={field.key} class="block flex items-center gap-1.5 text-sm font-medium">
{field.label}
@ -121,7 +123,7 @@
</p>
{/if}
{#if field.key === 'systemMessage'}
{#if field.key === SETTINGS_KEYS.SYSTEM_MESSAGE}
<div class="mt-3 flex items-center gap-2">
<Checkbox
id="showSystemMessage"
@ -134,7 +136,7 @@
</Label>
</div>
{/if}
{:else if field.type === 'select'}
{:else if field.type === SettingsFieldType.SELECT}
{@const selectedOption = field.options?.find(
(opt: { value: string; label: string; icon?: Component }) =>
opt.value === localConfig[field.key]
@ -166,7 +168,7 @@
type="single"
value={currentValue}
onValueChange={(value) => {
if (field.key === 'theme' && value && onThemeChange) {
if (field.key === SETTINGS_KEYS.THEME && value && onThemeChange) {
onThemeChange(value);
} else {
onConfigChange(field.key, value);
@ -222,7 +224,7 @@
{field.help || SETTING_CONFIG_INFO[field.key]}
</p>
{/if}
{:else if field.type === 'checkbox'}
{:else if field.type === SettingsFieldType.CHECKBOX}
<div class="flex items-start space-x-3">
<Checkbox
id={field.key}

3
tools/server/webui/src/lib/components/app/chat/ChatSettings/ChatSettingsImportExportTab.svelte
View file

@ -1,11 +1,10 @@
<script lang="ts">
import { Download, Upload, Trash2 } from '@lucide/svelte';
import { Button } from '$lib/components/ui/button';
import { DialogConversationSelection } from '$lib/components/app';
import { DialogConversationSelection, DialogConfirmation } from '$lib/components/app';
import { createMessageCountMap } from '$lib/utils';
import { conversationsStore, conversations } from '$lib/stores/conversations.svelte';
import { toast } from 'svelte-sonner';
import DialogConfirmation from '$lib/components/app/dialogs/DialogConfirmation.svelte';
let exportedConversations = $state<DatabaseConversation[]>([]);
let importedConversations = $state<DatabaseConversation[]>([]);

2
tools/server/webui/src/lib/components/app/chat/ChatSidebar/ChatSidebar.svelte
View file

@ -9,7 +9,7 @@
import Input from '$lib/components/ui/input/input.svelte';
import { conversationsStore, conversations } from '$lib/stores/conversations.svelte';
import { chatStore } from '$lib/stores/chat.svelte';
import { getPreviewText } from '$lib/utils/text';
import { getPreviewText } from '$lib/utils';
import ChatSidebarActions from './ChatSidebarActions.svelte';
const sidebar = Sidebar.useSidebar();

6
tools/server/webui/src/lib/components/app/chat/ChatSidebar/ChatSidebarConversationItem.svelte
View file

@ -1,6 +1,6 @@
<script lang="ts">
import { Trash2, Pencil, MoreHorizontal, Download, Loader2, Square } from '@lucide/svelte';
import { ActionDropdown } from '$lib/components/app';
import { DropdownMenuActions } from '$lib/components/app';
import * as Tooltip from '$lib/components/ui/tooltip';
import { getAllLoadingChats } from '$lib/stores/chat.svelte';
import { conversationsStore } from '$lib/stores/conversations.svelte';
@ -128,7 +128,7 @@
{#if renderActionsDropdown}
<div class="actions flex items-center">
<ActionDropdown
<DropdownMenuActions
triggerIcon={MoreHorizontal}
triggerTooltip="More actions"
bind:open={dropdownOpen}
@ -142,7 +142,7 @@
{
icon: Download,
label: 'Export',
onclick: (e) => {
onclick: (e: Event) => {
e.stopPropagation();
conversationsStore.downloadConversation(conversation.id);
},

597
tools/server/webui/src/lib/components/app/chat/index.ts Normal file
View file

@ -0,0 +1,597 @@
/**
*
* ATTACHMENTS
*
* Components for displaying and managing different attachment types in chat messages.
* Supports two operational modes:
* - **Readonly mode**: For displaying stored attachments in sent messages (DatabaseMessageExtra[])
* - **Editable mode**: For managing pending uploads in the input form (ChatUploadedFile[])
*
* The attachment system uses `getAttachmentDisplayItems()` utility to normalize both
* data sources into a unified display format, enabling consistent rendering regardless
* of the attachment origin.
*
*/
/**
* **ChatAttachmentsList** - Unified display for file attachments in chat
*
* Central component for rendering file attachments in both ChatMessage (readonly)
* and ChatForm (editable) contexts.
*
* **Architecture:**
* - Delegates rendering to specialized thumbnail components based on attachment type
* - Manages scroll state and navigation arrows for horizontal overflow
* - Integrates with DialogChatAttachmentPreview for full-size viewing
* - Validates vision modality support via `activeModelId` prop
*
* **Features:**
* - Horizontal scroll with smooth navigation arrows
* - Image thumbnails with lazy loading and error fallback
* - File type icons for non-image files (PDF, text, audio, etc.)
* - Click-to-preview with full-size dialog and download option
* - "View All" button when `limitToSingleRow` is enabled and content overflows
* - Vision modality validation to warn about unsupported image uploads
* - Customizable thumbnail dimensions via `imageHeight`/`imageWidth` props
*
* @example
* ```svelte
* <!-- Readonly mode (in ChatMessage) -->
* <ChatAttachmentsList attachments={message.extra} readonly />
*
* <!-- Editable mode (in ChatForm) -->
* <ChatAttachmentsList
* bind:uploadedFiles
* onFileRemove={(id) => removeFile(id)}
* limitToSingleRow
* activeModelId={selectedModel}
* />
* ```
*/
export { default as ChatAttachmentsList } from './ChatAttachments/ChatAttachmentsList.svelte';
/**
* Thumbnail for non-image file attachments. Displays file type icon based on extension,
* file name (truncated), and file size.
* Handles text files, PDFs, audio, and other document types.
*/
export { default as ChatAttachmentThumbnailFile } from './ChatAttachments/ChatAttachmentThumbnailFile.svelte';
/**
* Thumbnail for image attachments with lazy loading and error fallback.
* Displays image preview with configurable dimensions. Falls back to placeholder
* on load error.
*/
export { default as ChatAttachmentThumbnailImage } from './ChatAttachments/ChatAttachmentThumbnailImage.svelte';
/**
* Grid view of all attachments for "View All" dialog. Displays all attachments
* in a responsive grid layout when there are too many to show inline.
* Triggered by "+X more" button in ChatAttachmentsList.
*/
export { default as ChatAttachmentsViewAll } from './ChatAttachments/ChatAttachmentsViewAll.svelte';
/**
* Full-size preview dialog for attachments. Opens when clicking on any attachment
* thumbnail. Shows the attachment in full size with options to download or close.
* Handles both image and non-image attachments with appropriate rendering.
*/
export { default as ChatAttachmentPreview } from './ChatAttachments/ChatAttachmentPreview.svelte';
/**
*
* FORM
*
* Components for the chat input area. The form handles user input, file attachments,
* audio recording. It integrates with multiple stores:
* - `chatStore` for message submission and generation control
* - `modelsStore` for model selection and validation
*
* The form exposes a public API for programmatic control from parent components
* (focus, height reset, model selector, validation).
*
*/
/**
* **ChatForm** - Main chat input component with rich features
*
* The primary input interface for composing and sending chat messages.
* Orchestrates text input, file attachments, audio recording.
* Used by ChatScreenForm and ChatMessageEditForm for both new conversations and message editing.
*
* **Architecture:**
* - Composes ChatFormTextarea, ChatFormActions, and ChatFormPromptPicker
* - Manages file upload state via `uploadedFiles` bindable prop
* - Integrates with ModelsSelector for model selection in router mode
* - Communicates with parent via callbacks (onSubmit, onFilesAdd, onStop, etc.)
*
* **Input Handling:**
* - IME-safe Enter key handling (waits for composition end)
* - Shift+Enter for newline, Enter for submit
* - Paste handler for files and long text (> {pasteLongTextToFileLen} chars → file conversion)
*
* **Features:**
* - Auto-resizing textarea with placeholder
* - File upload via button dropdown (images/text/PDF), drag-drop, or paste
* - Audio recording with WAV conversion (when model supports audio)
* - Model selector integration (router mode)
* - Loading state with stop button, disabled state for errors
*
* **Exported API:**
* - `focus()` - Focus the textarea programmatically
* - `resetTextareaHeight()` - Reset textarea to default height after submit
* - `openModelSelector()` - Open model selection dropdown
* - `checkModelSelected(): boolean` - Validate model selection, show error if none
*
* @example
* ```svelte
* <ChatForm
* bind:this={chatFormRef}
* bind:value={message}
* bind:uploadedFiles
* {isLoading}
* onSubmit={handleSubmit}
* onFilesAdd={processFiles}
* onStop={handleStop}
* />
* ```
*/
export { default as ChatForm } from './ChatForm/ChatForm.svelte';
/**
* Dropdown button for file attachment selection. Opens a menu with options for
* Images, Text Files, and PDF Files. Each option filters the file picker to
* appropriate types. Images option is disabled when model lacks vision modality.
*/
export { default as ChatFormActionAttachmentsDropdown } from './ChatForm/ChatFormActions/ChatFormActionAttachmentsDropdown.svelte';
/**
* Audio recording button with real-time recording indicator. Records audio
* and converts to WAV format for upload. Only visible when the active model
* supports audio modality and setting for automatic audio input is enabled. Shows recording duration while active.
*/
export { default as ChatFormActionRecord } from './ChatForm/ChatFormActions/ChatFormActionRecord.svelte';
/**
* Container for chat form action buttons. Arranges file attachment, audio record,
* and submit/stop buttons in a horizontal layout. Handles conditional visibility
* based on model capabilities and loading state.
*/
export { default as ChatFormActions } from './ChatForm/ChatFormActions/ChatFormActions.svelte';
/**
* Submit/stop button with loading state. Shows send icon normally, transforms
* to stop icon during generation. Disabled when input is empty or form is disabled.
* Triggers onSubmit or onStop callbacks based on current state.
*/
export { default as ChatFormActionSubmit } from './ChatForm/ChatFormActions/ChatFormActionSubmit.svelte';
/**
* Hidden file input element for programmatic file selection.
*/
export { default as ChatFormFileInputInvisible } from './ChatForm/ChatFormFileInputInvisible.svelte';
/**
* Helper text display below chat.
*/
export { default as ChatFormHelperText } from './ChatForm/ChatFormHelperText.svelte';
/**
* Auto-resizing textarea with IME composition support. Automatically adjusts
* height based on content. Handles IME input correctly (waits for composition
* end before processing Enter key). Exposes focus() and resetHeight() methods.
*/
export { default as ChatFormTextarea } from './ChatForm/ChatFormTextarea.svelte';
/**
*
* MESSAGES
*
* Components for displaying chat messages. The message system supports:
* - **Conversation branching**: Messages can have siblings (alternative versions)
* created by editing or regenerating. Users can navigate between branches.
* - **Role-based rendering**: Different layouts for user, assistant, and system messages
* - **Streaming support**: Real-time display of assistant responses as they generate
* - **Agentic workflows**: Special rendering for tool calls and reasoning blocks
*
* The branching system uses `getMessageSiblings()` utility to compute sibling info
* for each message based on the full conversation tree stored in the database.
*
*/
/**
* **ChatMessages** - Message list container with branching support
*
* Container component that renders the list of messages in a conversation.
* Computes sibling information for each message to enable branch navigation.
* Integrates with conversationsStore for message operations.
*
* **Architecture:**
* - Fetches all conversation messages to compute sibling relationships
* - Filters system messages based on user config (`showSystemMessage`)
* - Delegates rendering to ChatMessage for each message
* - Propagates all message operations to chatStore via callbacks
*
* **Branching Logic:**
* - Uses `getMessageSiblings()` to find all messages with same parent
* - Computes `siblingInfo: { currentIndex, totalSiblings, siblingIds }`
* - Enables navigation between alternative message versions
*
* **Message Operations (delegated to chatStore):**
* - Edit with branching: Creates new message branch, preserves original
* - Edit with replacement: Modifies message in place
* - Regenerate: Creates new assistant response as sibling
* - Delete: Removes message and all descendants (cascade)
* - Continue: Appends to incomplete assistant message
*
* @example
* ```svelte
* <ChatMessages
* messages={activeMessages()}
* onUserAction={resetAutoScroll}
* />
* ```
*/
export { default as ChatMessages } from './ChatMessages/ChatMessages.svelte';
/**
* **ChatMessage** - Single message display with actions
*
* Renders a single chat message with role-specific styling and full action
* support. Delegates to specialized components based on message role:
* ChatMessageUser, ChatMessageAssistant, or ChatMessageSystem.
*
* **Architecture:**
* - Routes to role-specific component based on `message.type`
* - Manages edit mode state and inline editing UI
* - Handles action callbacks (copy, edit, delete, regenerate)
* - Displays branching controls when message has siblings
*
* **User Messages:**
* - Shows attachments via ChatAttachmentsList
* - Edit creates new branch or preserves responses
*
* **Assistant Messages:**
* - Renders content via MarkdownContent or ChatMessageAgenticContent
* - Shows model info badge (when enabled)
* - Regenerate creates sibling with optional model override
* - Continue action for incomplete responses
*
* **Features:**
* - Inline editing with file attachments support
* - Copy formatted content to clipboard
* - Delete with confirmation (shows cascade delete count)
* - Branching controls for sibling navigation
* - Statistics display (tokens, timing)
*
* @example
* ```svelte
* <ChatMessage
* {message}
* {siblingInfo}
* onEditWithBranching={handleEdit}
* onRegenerateWithBranching={handleRegenerate}
* onNavigateToSibling={handleNavigate}
* />
* ```
*/
export { default as ChatMessage } from './ChatMessages/ChatMessage.svelte';
/**
* Action buttons toolbar for messages. Displays copy, edit, delete, and regenerate
* buttons based on message role. Includes branching controls when message has siblings.
* Shows delete confirmation dialog with cascade delete count. Handles raw output toggle
* for assistant messages.
*/
export { default as ChatMessageActions } from './ChatMessages/ChatMessageActions.svelte';
/**
* Navigation controls for message siblings (conversation branches). Displays
* prev/next arrows with current position counter (e.g., "2/5"). Enables users
* to navigate between alternative versions of a message created by editing
* or regenerating. Uses `conversationsStore.navigateToSibling()` for navigation.
*/
export { default as ChatMessageBranchingControls } from './ChatMessages/ChatMessageBranchingControls.svelte';
/**
* Statistics display for assistant messages. Shows token counts (prompt/completion),
* generation timing, tokens per second, and model name (when enabled in settings).
* Data sourced from message.timings stored during generation.
*/
export { default as ChatMessageStatistics } from './ChatMessages/ChatMessageStatistics.svelte';
/**
* System message display component. Renders system messages with distinct styling.
* Visibility controlled by `showSystemMessage` config setting.
*/
export { default as ChatMessageSystem } from './ChatMessages/ChatMessageSystem.svelte';
/**
* User message display component. Renders user messages with right-aligned bubble styling.
* Shows message content, attachments via ChatAttachmentsList.
* Supports inline editing mode with ChatMessageEditForm integration.
*/
export { default as ChatMessageUser } from './ChatMessages/ChatMessageUser.svelte';
/**
* Assistant message display component. Renders assistant responses with left-aligned styling.
* Supports both plain markdown content (via MarkdownContent) and agentic content with tool calls
* (via ChatMessageAgenticContent). Shows model info badge, statistics, and action buttons.
* Handles streaming state with real-time content updates.
*/
export { default as ChatMessageAssistant } from './ChatMessages/ChatMessageAssistant.svelte';
/**
* Inline message editing form. Provides textarea for editing message content with
* attachment management. Shows save/cancel buttons and optional "Save only" button
* for editing without regenerating responses. Used within ChatMessage components
* when user enters edit mode.
*/
export { default as ChatMessageEditForm } from './ChatMessages/ChatMessageEditForm.svelte';
/**
*
* SCREEN
*
* Top-level chat interface components. ChatScreen is the main container that
* orchestrates all chat functionality. It integrates with multiple stores:
* - `chatStore` for message operations and generation control
* - `conversationsStore` for conversation management
* - `serverStore` for server connection state
* - `modelsStore` for model capabilities (vision, audio modalities)
*
* The screen handles the complete chat lifecycle from empty state to active
* conversation with streaming responses.
*
*/
/**
* **ChatScreen** - Main chat interface container
*
* Top-level component that orchestrates the entire chat interface. Manages
* messages display, input form, file handling, auto-scroll, error dialogs,
* and server state. Used as the main content area in chat routes.
*
* **Architecture:**
* - Composes ChatMessages, ChatScreenForm, ChatScreenHeader, and dialogs
* - Manages auto-scroll via `createAutoScrollController()` hook
* - Handles file upload pipeline (validation → processing → state update)
* - Integrates with serverStore for loading/error/warning states
* - Tracks active model for modality validation (vision, audio)
*
* **File Upload Pipeline:**
* 1. Files received via drag-drop, paste, or file picker
* 2. Validated against supported types (`isFileTypeSupported()`)
* 3. Filtered by model modalities (`filterFilesByModalities()`)
* 4. Empty files detected and reported via DialogEmptyFileAlert
* 5. Valid files processed to ChatUploadedFile[] format
* 6. Unsupported files shown in error dialog with reasons
*
* **State Management:**
* - `isEmpty`: Shows centered welcome UI when no conversation active
* - `isCurrentConversationLoading`: Tracks generation state for current chat
* - `activeModelId`: Determines available modalities for file validation
* - `uploadedFiles`: Pending file attachments for next message
*
* **Features:**
* - Messages display with smart auto-scroll (pauses on user scroll up)
* - File drag-drop with visual overlay indicator
* - File validation with detailed error messages
* - Error dialog management (chat errors, model unavailable)
* - Server loading/error/warning states with appropriate UI
* - Conversation deletion with confirmation dialog
* - Processing info display (tokens/sec, timing) during generation
* - Keyboard shortcuts (Ctrl+Shift+Backspace to delete conversation)
*
* @example
* ```svelte
* <!-- In chat route -->
* <ChatScreen showCenteredEmpty={true} />
*
* <!-- In conversation route -->
* <ChatScreen showCenteredEmpty={false} />
* ```
*/
export { default as ChatScreen } from './ChatScreen/ChatScreen.svelte';
/**
* Visual overlay displayed when user drags files over the chat screen.
* Shows drop zone indicator to guide users where to release files.
* Integrated with ChatScreen's drag-drop file upload handling.
*/
export { default as ChatScreenDragOverlay } from './ChatScreen/ChatScreenDragOverlay.svelte';
/**
* Chat form wrapper within ChatScreen. Positions the ChatForm component at the
* bottom of the screen with proper padding and max-width constraints. Handles
* the visual container styling for the input area.
*/
export { default as ChatScreenForm } from './ChatScreen/ChatScreenForm.svelte';
/**
* Header bar for chat screen. Displays conversation title (or "New Chat"),
* model selector (in router mode), and action buttons (delete conversation).
* Sticky positioned at the top of the chat area.
*/
export { default as ChatScreenHeader } from './ChatScreen/ChatScreenHeader.svelte';
/**
* Processing info display during generation. Shows real-time statistics:
* tokens per second, prompt/completion token counts, and elapsed time.
* Data sourced from slotsService polling during active generation.
* Only visible when `isCurrentConversationLoading` is true.
*/
export { default as ChatScreenProcessingInfo } from './ChatScreen/ChatScreenProcessingInfo.svelte';
/**
*
* SETTINGS
*
* Application settings components. Settings are persisted to localStorage via
* the config store and synchronized with server `/props` endpoint for sampling
* parameters. The settings panel uses a tabbed interface with mobile-responsive
* horizontal scrolling tabs.
*
* **Parameter Sync System:**
* Sampling parameters (temperature, top_p, etc.) can come from three sources:
* 1. **Server Props**: Default values from `/props` endpoint
* 2. **User Custom**: Values explicitly set by user (overrides server)
* 3. **App Default**: Fallback when server props unavailable
*
* The `ChatSettingsParameterSourceIndicator` badge shows which source is active.
*
*/
/**
* **ChatSettings** - Application settings panel
*
* Comprehensive settings interface with categorized sections. Manages all
* user preferences and sampling parameters. Integrates with config store
* for persistence and ParameterSyncService for server synchronization.
*
* **Architecture:**
* - Uses tabbed navigation with category sections
* - Maintains local form state, commits on save
* - Tracks user overrides vs server defaults for sampling params
* - Exposes reset() method for dialog close without save
*
* **Categories:**
* - **General**: API key, system message, show system messages toggle
* - **Display**: Theme selection, message actions visibility, model info badge
* - **Sampling**: Temperature, top_p, top_k, min_p, repeat_penalty, etc.
* - **Penalties**: Frequency penalty, presence penalty, repeat last N
* - **Import/Export**: Conversation backup and restore
* - **Developer**: Debug options, disable auto-scroll
*
* **Parameter Sync:**
* - Fetches defaults from server `/props` endpoint
* - Shows source indicator badge (Custom/Server Props/Default)
* - Real-time badge updates as user types
* - Tracks which parameters user has explicitly overridden
*
* **Features:**
* - Mobile-responsive layout with horizontal scrolling tabs
* - Form validation with error messages
* - Secure API key storage (masked input)
* - Import/export conversations as JSON
* - Reset to defaults option per parameter
*
* **Exported API:**
* - `reset()` - Reset form fields to currently saved values (for cancel action)
*
* @example
* ```svelte
* <ChatSettings
* bind:this={settingsRef}
* onSave={() => dialogOpen = false}
* onCancel={() => { settingsRef.reset(); dialogOpen = false; }}
* />
* ```
*/
export { default as ChatSettings } from './ChatSettings/ChatSettings.svelte';
/**
* Footer with save/cancel buttons for settings panel. Positioned at bottom
* of settings dialog. Save button commits form state to config store,
* cancel button triggers reset and close.
*/
export { default as ChatSettingsFooter } from './ChatSettings/ChatSettingsFooter.svelte';
/**
* Form fields renderer for individual settings. Generates appropriate input
* components based on field type (text, number, select, checkbox, textarea).
* Handles validation, help text display, and parameter source indicators.
*/
export { default as ChatSettingsFields } from './ChatSettings/ChatSettingsFields.svelte';
/**
* Import/export tab content for conversation data management. Provides buttons
* to export all conversations as JSON file and import from JSON file.
* Handles file download/upload and data validation.
*/
export { default as ChatSettingsImportExportTab } from './ChatSettings/ChatSettingsImportExportTab.svelte';
/**
* Badge indicating parameter source for sampling settings. Shows one of:
* - **Custom**: User has explicitly set this value (orange badge)
* - **Server Props**: Using default from `/props` endpoint (blue badge)
* - **Default**: Using app default, server props unavailable (gray badge)
* Updates in real-time as user types to show immediate feedback.
*/
export { default as ChatSettingsParameterSourceIndicator } from './ChatSettings/ChatSettingsParameterSourceIndicator.svelte';
/**
*
* SIDEBAR
*
* The sidebar integrates with ShadCN's sidebar component system
* for consistent styling and mobile responsiveness.
* Conversations are loaded from conversationsStore and displayed in reverse
* chronological order (most recent first).
*
*/
/**
* **ChatSidebar** - Chat Sidebar with actions menu and conversation list
*
* Collapsible sidebar displaying conversation history with search and
* management actions. Integrates with ShadCN sidebar component for
* consistent styling and mobile responsiveness.
*
* **Architecture:**
* - Uses ShadCN Sidebar.* components for structure
* - Fetches conversations from conversationsStore
* - Manages search state and filtered results locally
* - Handles conversation CRUD operations via conversationsStore
*
* **Navigation:**
* - Click conversation to navigate to `/chat/[id]`
* - New chat button navigates to `/` (root)
* - Active conversation highlighted based on route params
*
* **Conversation Management:**
* - Right-click or menu button for context menu
* - Rename: Opens inline edit dialog
* - Delete: Shows confirmation with conversation preview
* - Delete All: Removes all conversations with confirmation
*
* **Features:**
* - Search/filter conversations by title
* - Conversation list with message previews (first message truncated)
* - Active conversation highlighting
* - Mobile-responsive collapse/expand via ShadCN sidebar
* - New chat button in header
* - Settings button opens DialogChatSettings
*
* **Exported API:**
* - `handleMobileSidebarItemClick()` - Close sidebar on mobile after item selection
* - `activateSearchMode()` - Focus search input programmatically
* - `editActiveConversation()` - Open rename dialog for current conversation
*
* @example
* ```svelte
* <ChatSidebar bind:this={sidebarRef} />
* ```
*/
export { default as ChatSidebar } from './ChatSidebar/ChatSidebar.svelte';
/**
* Action buttons for sidebar header. Contains new chat button, settings button,
* and delete all conversations button. Manages dialog states for settings and
* delete confirmation.
*/
export { default as ChatSidebarActions } from './ChatSidebar/ChatSidebarActions.svelte';
/**
* Single conversation item in sidebar. Displays conversation title (truncated),
* last message preview, and timestamp. Shows context menu on right-click with
* rename and delete options. Highlights when active (matches current route).
* Handles click to navigate and keyboard accessibility.
*/
export { default as ChatSidebarConversationItem } from './ChatSidebar/ChatSidebarConversationItem.svelte';
/**
* Search input for filtering conversations in sidebar. Filters conversation
* list by title as user types. Shows clear button when query is not empty.
* Integrated into sidebar header with proper styling.
*/
export { default as ChatSidebarSearch } from './ChatSidebar/ChatSidebarSearch.svelte';

32
tools/server/webui/src/lib/components/app/content/MarkdownContent.svelte
View file

@ -15,6 +15,7 @@
import { rehypeRestoreTableHtml } from '$lib/markdown/table-html-restorer';
import { rehypeEnhanceLinks } from '$lib/markdown/enhance-links';
import { rehypeEnhanceCodeBlocks } from '$lib/markdown/enhance-code-blocks';
import { rehypeResolveAttachmentImages } from '$lib/markdown/resolve-attachment-images';
import { remarkLiteralHtml } from '$lib/markdown/literal-html';
import { copyCodeToClipboard, preprocessLaTeX, getImageErrorFallbackHtml } from '$lib/utils';
import {
@ -23,6 +24,7 @@
DATA_ERROR_HANDLED_ATTR,
BOOL_TRUE_STRING
} from '$lib/constants/markdown';
import { UrlPrefix } from '$lib/enums';
import { FileTypeText } from '$lib/enums/files';
import {
highlightCode,
@ -33,10 +35,11 @@
import githubDarkCss from 'highlight.js/styles/github-dark.css?inline';
import githubLightCss from 'highlight.js/styles/github.css?inline';
import { mode } from 'mode-watcher';
import ActionIconsCodeBlock from '$lib/components/app/actions/ActionIconsCodeBlock.svelte';
import DialogCodePreview from '$lib/components/app/misc/CodePreviewDialog.svelte';
import { ActionIconsCodeBlock, DialogCodePreview } from '$lib/components/app';
import { createAutoScrollController } from '$lib/hooks/use-auto-scroll.svelte';
import type { DatabaseMessageExtra } from '$lib/types/database';
import { config } from '$lib/stores/settings.svelte';
import { SETTINGS_KEYS } from '$lib/constants/settings-keys';
interface Props {
attachments?: DatabaseMessageExtra[];
@ -100,6 +103,7 @@
.use(rehypeRestoreTableHtml) // Restore limited HTML (e.g., <br>, <ul>) inside Markdown tables
.use(rehypeEnhanceLinks) // Add target="_blank" to links
.use(rehypeEnhanceCodeBlocks) // Wrap code blocks with header and actions
.use(rehypeResolveAttachmentImages, { attachments })
.use(rehypeStringify, { allowDangerousHtml: true }); // Convert to HTML string
});
@ -500,7 +504,10 @@
if (!img || !img.src) return;
// Don't handle data URLs or already-handled images
if (img.src.startsWith('data:') || img.dataset[DATA_ERROR_HANDLED_ATTR] === BOOL_TRUE_STRING)
if (
img.src.startsWith(UrlPrefix.DATA) ||
img.dataset[DATA_ERROR_HANDLED_ATTR] === BOOL_TRUE_STRING
)
return;
img.dataset[DATA_ERROR_HANDLED_ATTR] = BOOL_TRUE_STRING;
@ -588,7 +595,12 @@
});
</script>
<div bind:this={containerRef} class={className}>
<div
bind:this={containerRef}
class="{className}{config()[SETTINGS_KEYS.FULL_HEIGHT_CODE_BLOCKS]
? ' full-height-code-blocks'
: ''}"
>
{#each renderedBlocks as block (block.id)}
<div class="markdown-block" data-block-id={block.id}>
<!-- eslint-disable-next-line no-at-html-tags -->
@ -611,7 +623,7 @@
code={incompleteCodeBlock.code}
language={incompleteCodeBlock.language || 'text'}
disabled={true}
onPreview={(code: string, lang: string) => {
onPreview={(code, lang) => {
previewCode = code;
previewLanguage = lang;
previewDialogOpen = true;
@ -909,6 +921,16 @@
line-height: 1.3;
}
.full-height-code-blocks :global(.code-block-wrapper) {
max-height: none;
}
.full-height-code-blocks :global(.code-block-scroll-container),
.full-height-code-blocks .streaming-code-scroll-container {
max-height: none;
overflow-y: visible;
}
div :global(.code-block-header) {
display: flex;
justify-content: space-between;

12
tools/server/webui/src/lib/components/app/dialogs/DialogChatError.svelte
View file

@ -1,10 +1,11 @@
<script lang="ts">
import * as AlertDialog from '$lib/components/ui/alert-dialog';
import { AlertTriangle, TimerOff } from '@lucide/svelte';
import { ErrorDialogType } from '$lib/enums';
interface Props {
open: boolean;
type: 'timeout' | 'server';
type: ErrorDialogType;
message: string;
contextInfo?: { n_prompt_tokens: number; n_ctx: number };
onOpenChange?: (open: boolean) => void;
@ -12,7 +13,7 @@
let { open = $bindable(), type, message, contextInfo, onOpenChange }: Props = $props();
const isTimeout = $derived(type === 'timeout');
const isTimeout = $derived(type === ErrorDialogType.TIMEOUT);
const title = $derived(isTimeout ? 'TCP Timeout' : 'Server Error');
const description = $derived(
isTimeout
@ -58,7 +59,12 @@
<span class="font-medium">Prompt tokens:</span>
{contextInfo.n_prompt_tokens.toLocaleString()}
</p>
<p><span class="font-medium">Context size:</span> {contextInfo.n_ctx.toLocaleString()}</p>
{#if contextInfo.n_ctx}
<p>
<span class="font-medium">Context size:</span>
{contextInfo.n_ctx.toLocaleString()}
</p>
{/if}
</div>
{/if}
</div>

11
tools/server/webui/src/lib/components/app/dialogs/DialogChatSettings.svelte
View file

@ -1,13 +1,15 @@
<script lang="ts">
import * as Dialog from '$lib/components/ui/dialog';
import { ChatSettings } from '$lib/components/app';
import type { SettingsSectionTitle } from '$lib/constants/settings-sections';
interface Props {
onOpenChange?: (open: boolean) => void;
open?: boolean;
initialSection?: SettingsSectionTitle;
}
let { onOpenChange, open = false }: Props = $props();
let { onOpenChange, open = false, initialSection }: Props = $props();
let chatSettingsRef: ChatSettings | undefined = $state();
@ -28,10 +30,9 @@
<Dialog.Root {open} onOpenChange={handleClose}>
<Dialog.Content
class="z-999999 flex h-[100dvh] max-h-[100dvh] min-h-[100dvh] flex-col gap-0 rounded-none p-0
md:h-[64vh] md:max-h-[64vh] md:min-h-0 md:rounded-lg"
style="max-width: 48rem;"
class="z-999999 flex h-[100dvh] max-h-[100dvh] min-h-[100dvh] max-w-4xl! flex-col gap-0 rounded-none
p-0 md:h-[64vh] md:max-h-[64vh] md:min-h-0 md:rounded-lg"
>
<ChatSettings bind:this={chatSettingsRef} onSave={handleSave} />
<ChatSettings bind:this={chatSettingsRef} onSave={handleSave} {initialSection} />
</Dialog.Content>
</Dialog.Root>

2
tools/server/webui/src/lib/components/app/misc/CodePreviewDialog.svelte → tools/server/webui/src/lib/components/app/dialogs/DialogCodePreview.svelte
View file

@ -37,7 +37,7 @@
<iframe
bind:this={iframeRef}
title="Preview {language}"
sandbox="allow-scripts"
sandbox="allow-scripts allow-same-origin"
class="code-preview-iframe"
></iframe>

3
tools/server/webui/src/lib/components/app/dialogs/DialogConfirmation.svelte
View file

@ -1,6 +1,7 @@
<script lang="ts">
import * as AlertDialog from '$lib/components/ui/alert-dialog';
import type { Component } from 'svelte';
import { KeyboardKey } from '$lib/enums';
interface Props {
open: boolean;
@ -29,7 +30,7 @@
}: Props = $props();
function handleKeydown(event: KeyboardEvent) {
if (event.key === 'Enter') {
if (event.key === KeyboardKey.ENTER) {
event.preventDefault();
onConfirm();
}

39
tools/server/webui/src/lib/components/app/dialogs/DialogModelInformation.svelte
View file

@ -1,7 +1,7 @@
<script lang="ts">
import * as Dialog from '$lib/components/ui/dialog';
import * as Table from '$lib/components/ui/table';
import { BadgeModality, CopyToClipboardIcon } from '$lib/components/app';
import { BadgeModality, ActionIconCopyToClipboard } from '$lib/components/app';
import { serverStore } from '$lib/stores/server.svelte';
import { modelsStore, modelOptions, modelsLoading } from '$lib/stores/models.svelte';
import { formatFileSize, formatParameters, formatNumber } from '$lib/utils';
@ -47,6 +47,7 @@
<Dialog.Header>
<Dialog.Title>Model Information</Dialog.Title>
<Dialog.Description>Current model details and capabilities</Dialog.Description>
</Dialog.Header>
@ -73,7 +74,7 @@
{modelName}
</span>
<CopyToClipboardIcon
<ActionIconCopyToClipboard
text={modelName || ''}
canCopy={!!modelName}
ariaLabel="Copy model name to clipboard"
@ -97,7 +98,7 @@
{serverProps.model_path}
</span>
<CopyToClipboardIcon
<ActionIconCopyToClipboard
text={serverProps.model_path}
ariaLabel="Copy model path to clipboard"
/>
@ -105,17 +106,29 @@
</Table.Row>
<!-- Context Size -->
<Table.Row>
<Table.Cell class="h-10 align-middle font-medium">Context Size</Table.Cell>
<Table.Cell
>{formatNumber(serverProps.default_generation_settings.n_ctx)} tokens</Table.Cell
>
</Table.Row>
{#if serverProps?.default_generation_settings?.n_ctx}
<Table.Row>
<Table.Cell class="h-10 align-middle font-medium">Context Size</Table.Cell>
<Table.Cell
>{formatNumber(serverProps.default_generation_settings.n_ctx)} tokens</Table.Cell
>
</Table.Row>
{:else}
<Table.Row>
<Table.Cell class="h-10 align-middle font-medium text-red-500"
>Context Size</Table.Cell
>
<Table.Cell class="text-red-500">Not available</Table.Cell>
</Table.Row>
{/if}
<!-- Training Context -->
{#if modelMeta?.n_ctx_train}
<Table.Row>
<Table.Cell class="h-10 align-middle font-medium">Training Context</Table.Cell>
<Table.Cell>{formatNumber(modelMeta.n_ctx_train)} tokens</Table.Cell>
</Table.Row>
{/if}
@ -124,6 +137,7 @@
{#if modelMeta?.size}
<Table.Row>
<Table.Cell class="h-10 align-middle font-medium">Model Size</Table.Cell>
<Table.Cell>{formatFileSize(modelMeta.size)}</Table.Cell>
</Table.Row>
{/if}
@ -132,6 +146,7 @@
{#if modelMeta?.n_params}
<Table.Row>
<Table.Cell class="h-10 align-middle font-medium">Parameters</Table.Cell>
<Table.Cell>{formatParameters(modelMeta.n_params)}</Table.Cell>
</Table.Row>
{/if}
@ -140,6 +155,7 @@
{#if modelMeta?.n_embd}
<Table.Row>
<Table.Cell class="align-middle font-medium">Embedding Size</Table.Cell>
<Table.Cell>{formatNumber(modelMeta.n_embd)}</Table.Cell>
</Table.Row>
{/if}
@ -148,6 +164,7 @@
{#if modelMeta?.n_vocab}
<Table.Row>
<Table.Cell class="align-middle font-medium">Vocabulary Size</Table.Cell>
<Table.Cell>{formatNumber(modelMeta.n_vocab)} tokens</Table.Cell>
</Table.Row>
{/if}
@ -163,6 +180,7 @@
<!-- Total Slots -->
<Table.Row>
<Table.Cell class="align-middle font-medium">Parallel Slots</Table.Cell>
<Table.Cell>{serverProps.total_slots}</Table.Cell>
</Table.Row>
@ -170,6 +188,7 @@
{#if modalities.length > 0}
<Table.Row>
<Table.Cell class="align-middle font-medium">Modalities</Table.Cell>
<Table.Cell>
<div class="flex flex-wrap gap-1">
<BadgeModality {modalities} />
@ -181,6 +200,7 @@
<!-- Build Info -->
<Table.Row>
<Table.Cell class="align-middle font-medium">Build Info</Table.Cell>
<Table.Cell class="align-middle font-mono text-xs"
>{serverProps.build_info}</Table.Cell
>
@ -190,6 +210,7 @@
{#if serverProps.chat_template}
<Table.Row>
<Table.Cell class="align-middle font-medium">Chat Template</Table.Cell>
<Table.Cell class="py-10">
<div class="max-h-120 overflow-y-auto rounded-md bg-muted p-4">
<pre

416
tools/server/webui/src/lib/components/app/dialogs/index.ts Normal file
View file

@ -0,0 +1,416 @@
/**
*
* DIALOGS
*
* Modal dialog components for the chat application.
*
* All dialogs use ShadCN Dialog or AlertDialog components for consistent
* styling, accessibility, and animation. They integrate with application
* stores for state management and data access.
*
*/
/**
*
* SETTINGS DIALOGS
*
* Dialogs for application and server configuration.
*
*/
/**
* **DialogChatSettings** - Settings dialog wrapper
*
* Modal dialog containing ChatSettings component with proper
* open/close state management and automatic form reset on open.
*
* **Architecture:**
* - Wraps ChatSettings component in ShadCN Dialog
* - Manages open/close state via bindable `open` prop
* - Resets form state when dialog opens to discard unsaved changes
*
* @example
* ```svelte
* <DialogChatSettings bind:open={showSettings} />
* ```
*/
export { default as DialogChatSettings } from './DialogChatSettings.svelte';
/**
*
* CONFIRMATION DIALOGS
*
* Dialogs for user action confirmations. Use AlertDialog for blocking
* confirmations that require explicit user decision before proceeding.
*
*/
/**
* **DialogConfirmation** - Generic confirmation dialog
*
* Reusable confirmation dialog with customizable title, description,
* and action buttons. Supports destructive action styling and custom icons.
* Used for delete confirmations, irreversible actions, and important decisions.
*
* **Architecture:**
* - Uses ShadCN AlertDialog
* - Supports variant styling (default, destructive)
* - Customizable button labels and callbacks
*
* **Features:**
* - Customizable title and description text
* - Destructive variant with red styling for dangerous actions
* - Custom icon support in header
* - Cancel and confirm button callbacks
* - Keyboard accessible (Escape to cancel, Enter to confirm)
*
* @example
* ```svelte
* <DialogConfirmation
* bind:open={showDelete}
* title="Delete conversation?"
* description="This action cannot be undone."
* variant="destructive"
* onConfirm={handleDelete}
* onCancel={() => showDelete = false}
* />
* ```
*/
export { default as DialogConfirmation } from './DialogConfirmation.svelte';
/**
* **DialogConversationTitleUpdate** - Conversation rename confirmation
*
* Confirmation dialog shown when editing the first user message in a conversation.
* Asks user whether to update the conversation title to match the new message content.
*
* **Architecture:**
* - Uses ShadCN AlertDialog
* - Shows current vs proposed title comparison
* - Triggered by ChatMessages when first message is edited
*
* **Features:**
* - Side-by-side display of current and new title
* - "Keep Current Title" and "Update Title" action buttons
* - Styled title previews in muted background boxes
*
* @example
* ```svelte
* <DialogConversationTitleUpdate
* bind:open={showTitleUpdate}
* currentTitle={conversation.name}
* newTitle={truncatedMessageContent}
* onConfirm={updateTitle}
* onCancel={() => showTitleUpdate = false}
* />
* ```
*/
export { default as DialogConversationTitleUpdate } from './DialogConversationTitleUpdate.svelte';
/**
*
* CONTENT PREVIEW DIALOGS
*
* Dialogs for previewing and displaying content in full-screen or modal views.
*
*/
/**
* **DialogCodePreview** - Full-screen code/HTML preview
*
* Full-screen dialog for previewing HTML or code in an isolated iframe.
* Used by MarkdownContent component for previewing rendered HTML blocks
* from code blocks in chat messages.
*
* **Architecture:**
* - Uses ShadCN Dialog with full viewport layout
* - Sandboxed iframe execution (allow-scripts only)
* - Clears content when closed for security
*
* **Features:**
* - Full viewport iframe preview
* - Sandboxed execution environment
* - Close button with mix-blend-difference for visibility over any content
* - Automatic content cleanup on close
* - Supports HTML preview with proper isolation
*
* @example
* ```svelte
* <DialogCodePreview
* bind:open={showPreview}
* code={htmlContent}
* language="html"
* />
* ```
*/
export { default as DialogCodePreview } from './DialogCodePreview.svelte';
/**
*
* ATTACHMENT DIALOGS
*
* Dialogs for viewing and managing file attachments. Support both
* uploaded files (pending) and stored attachments (in messages).
*
*/
/**
* **DialogChatAttachmentPreview** - Full-size attachment preview
*
* Modal dialog for viewing file attachments at full size. Supports different
* file types with appropriate preview modes: images, text files, PDFs, and audio.
*
* **Architecture:**
* - Wraps ChatAttachmentPreview component in ShadCN Dialog
* - Accepts either uploaded file or stored attachment as data source
* - Resets preview state when dialog opens
*
* **Features:**
* - Full-size image display with proper scaling
* - Text file content with syntax highlighting
* - PDF preview with text/image view toggle
* - Audio file placeholder with download option
* - File name and size display in header
* - Download button for all file types
* - Vision modality check for image attachments
*
* @example
* ```svelte
* <!-- Preview uploaded file -->
* <DialogChatAttachmentPreview
* bind:open={showPreview}
* uploadedFile={selectedFile}
* activeModelId={currentModel}
* />
*
* <!-- Preview stored attachment -->
* <DialogChatAttachmentPreview
* bind:open={showPreview}
* attachment={selectedAttachment}
* />
* ```
*/
export { default as DialogChatAttachmentPreview } from './DialogChatAttachmentPreview.svelte';
/**
* **DialogChatAttachmentsViewAll** - Grid view of all attachments
*
* Dialog showing all attachments in a responsive grid layout. Triggered by
* "+X more" button in ChatAttachmentsList when there are too many attachments
* to display inline.
*
* **Architecture:**
* - Wraps ChatAttachmentsViewAll component in ShadCN Dialog
* - Supports both readonly (message view) and editable (form) modes
* - Displays total attachment count in header
*
* **Features:**
* - Responsive grid layout for all attachments
* - Thumbnail previews with click-to-expand
* - Remove button in editable mode
* - Configurable thumbnail dimensions
* - Vision modality validation for images
*
* @example
* ```svelte
* <DialogChatAttachmentsViewAll
* bind:open={showAllAttachments}
* attachments={message.extra}
* readonly
* />
* ```
*/
export { default as DialogChatAttachmentsViewAll } from './DialogChatAttachmentsViewAll.svelte';
/**
*
* ERROR & ALERT DIALOGS
*
* Dialogs for displaying errors, warnings, and alerts to users.
* Provide context about what went wrong and recovery options.
*
*/
/**
* **DialogChatError** - Chat/generation error display
*
* Alert dialog for displaying chat and generation errors with context
* information. Supports different error types with appropriate styling
* and messaging.
*
* **Architecture:**
* - Uses ShadCN AlertDialog for modal display
* - Differentiates between timeout and server errors
* - Shows context info when available (token counts)
*
* **Error Types:**
* - **timeout**: TCP timeout with timer icon, red destructive styling
* - **server**: Server error with warning icon, amber warning styling
*
* **Features:**
* - Type-specific icons (TimerOff for timeout, AlertTriangle for server)
* - Error message display in styled badge
* - Context info showing prompt tokens and context size
* - Close button to dismiss
*
* @example
* ```svelte
* <DialogChatError
* bind:open={showError}
* type="server"
* message={errorMessage}
* contextInfo={{ n_prompt_tokens: 1024, n_ctx: 4096 }}
* />
* ```
*/
export { default as DialogChatError } from './DialogChatError.svelte';
/**
* **DialogEmptyFileAlert** - Empty file upload warning
*
* Alert dialog shown when user attempts to upload empty files. Lists the
* empty files that were detected and removed from attachments, with
* explanation of why empty files cannot be processed.
*
* **Architecture:**
* - Uses ShadCN AlertDialog for modal display
* - Receives list of empty file names from ChatScreen
* - Triggered during file upload validation
*
* **Features:**
* - FileX icon indicating file error
* - List of empty file names in monospace font
* - Explanation of what happened and why
* - Single "Got it" dismiss button
*
* @example
* ```svelte
* <DialogEmptyFileAlert
* bind:open={showEmptyAlert}
* emptyFiles={['empty.txt', 'blank.md']}
* />
* ```
*/
export { default as DialogEmptyFileAlert } from './DialogEmptyFileAlert.svelte';
/**
* **DialogModelNotAvailable** - Model unavailable error
*
* Alert dialog shown when the requested model (from URL params or selection)
* is not available on the server. Displays the requested model name and
* offers selection from available models.
*
* **Architecture:**
* - Uses ShadCN AlertDialog for modal display
* - Integrates with SvelteKit navigation for model switching
* - Receives available models list from modelsStore
*
* **Features:**
* - Warning icon with amber styling
* - Requested model name display in styled badge
* - Scrollable list of available models
* - Click model to navigate with updated URL params
* - Cancel button to dismiss without selection
*
* @example
* ```svelte
* <DialogModelNotAvailable
* bind:open={showModelError}
* modelName={requestedModel}
* availableModels={modelsList}
* />
* ```
*/
export { default as DialogModelNotAvailable } from './DialogModelNotAvailable.svelte';
/**
*
* DATA MANAGEMENT DIALOGS
*
* Dialogs for managing conversation data, including import/export
* and selection operations.
*
*/
/**
* **DialogConversationSelection** - Conversation picker for import/export
*
* Dialog for selecting conversations during import or export operations.
* Displays list of conversations with checkboxes for multi-selection.
* Used by ChatSettingsImportExportTab for data management.
*
* **Architecture:**
* - Wraps ConversationSelection component in ShadCN Dialog
* - Supports export mode (select from local) and import mode (select from file)
* - Resets selection state when dialog opens
* - High z-index to appear above settings dialog
*
* **Features:**
* - Multi-select with checkboxes
* - Conversation title and message count display
* - Select all / deselect all controls
* - Mode-specific descriptions (export vs import)
* - Cancel and confirm callbacks with selected conversations
*
* @example
* ```svelte
* <DialogConversationSelection
* bind:open={showExportSelection}
* conversations={allConversations}
* messageCountMap={messageCounts}
* mode="export"
* onConfirm={handleExport}
* onCancel={() => showExportSelection = false}
* />
* ```
*/
export { default as DialogConversationSelection } from './DialogConversationSelection.svelte';
/**
*
* MODEL INFORMATION DIALOGS
*
* Dialogs for displaying model and server information.
*
*/
/**
* **DialogModelInformation** - Model details display
*
* Dialog showing comprehensive information about the currently loaded model
* and server configuration. Displays model metadata, capabilities, and
* server settings in a structured table format.
*
* **Architecture:**
* - Uses ShadCN Dialog with wide layout for table display
* - Fetches data from serverStore (props) and modelsStore (metadata)
* - Auto-fetches models when dialog opens if not loaded
*
* **Information Displayed:**
* - **Model**: Name with copy button
* - **File Path**: Full path to model file with copy button
* - **Context Size**: Current context window size
* - **Training Context**: Original training context (if available)
* - **Model Size**: File size in human-readable format
* - **Parameters**: Parameter count (e.g., "7B", "70B")
* - **Embedding Size**: Embedding dimension
* - **Vocabulary Size**: Token vocabulary size
* - **Vocabulary Type**: Tokenizer type (BPE, etc.)
* - **Parallel Slots**: Number of concurrent request slots
* - **Modalities**: Supported input types (text, vision, audio)
* - **Build Info**: Server build information
* - **Chat Template**: Full Jinja template in scrollable code block
*
* **Features:**
* - Copy buttons for model name and path
* - Modality badges with icons
* - Responsive table layout with container queries
* - Loading state while fetching model info
* - Scrollable chat template display
*
* @example
* ```svelte
* <DialogModelInformation bind:open={showModelInfo} />
* ```
*/
export { default as DialogModelInformation } from './DialogModelInformation.svelte';

110
tools/server/webui/src/lib/components/app/forms/KeyValuePairs.svelte Normal file
View file

@ -0,0 +1,110 @@
<script lang="ts">
import { Plus, Trash2 } from '@lucide/svelte';
import { Input } from '$lib/components/ui/input';
import { autoResizeTextarea } from '$lib/utils';
import type { KeyValuePair } from '$lib/types';
interface Props {
class?: string;
pairs: KeyValuePair[];
onPairsChange: (pairs: KeyValuePair[]) => void;
keyPlaceholder?: string;
valuePlaceholder?: string;
addButtonLabel?: string;
emptyMessage?: string;
sectionLabel?: string;
sectionLabelOptional?: boolean;
}
let {
class: className = '',
pairs,
onPairsChange,
keyPlaceholder = 'Key',
valuePlaceholder = 'Value',
addButtonLabel = 'Add',
emptyMessage = 'No items configured.',
sectionLabel,
sectionLabelOptional = true
}: Props = $props();
function addPair() {
onPairsChange([...pairs, { key: '', value: '' }]);
}
function removePair(index: number) {
onPairsChange(pairs.filter((_, i) => i !== index));
}
function updatePairKey(index: number, key: string) {
const newPairs = [...pairs];
newPairs[index] = { ...newPairs[index], key };
onPairsChange(newPairs);
}
function updatePairValue(index: number, value: string) {
const newPairs = [...pairs];
newPairs[index] = { ...newPairs[index], value };
onPairsChange(newPairs);
}
</script>
<div class={className}>
<div class="mb-2 flex items-center justify-between">
{#if sectionLabel}
<span class="text-xs font-medium">
{sectionLabel}
{#if sectionLabelOptional}
<span class="text-muted-foreground">(optional)</span>
{/if}
</span>
{/if}
<button
type="button"
class="inline-flex cursor-pointer items-center gap-1 rounded-md px-1.5 py-1 text-xs text-muted-foreground hover:bg-muted hover:text-foreground"
onclick={addPair}
>
<Plus class="h-3 w-3" />
{addButtonLabel}
</button>
</div>
{#if pairs.length > 0}
<div class="space-y-3">
{#each pairs as pair, index (index)}
<div class="flex items-start gap-2">
<Input
type="text"
placeholder={keyPlaceholder}
value={pair.key}
oninput={(e) => updatePairKey(index, e.currentTarget.value)}
class="flex-1"
/>
<textarea
use:autoResizeTextarea
placeholder={valuePlaceholder}
value={pair.value}
oninput={(e) => {
updatePairValue(index, e.currentTarget.value);
autoResizeTextarea(e.currentTarget);
}}
class="flex-1 resize-none rounded-md border border-input bg-transparent px-3 py-2 text-sm leading-5 placeholder:text-muted-foreground focus-visible:ring-1 focus-visible:ring-ring focus-visible:outline-none"
rows="1"
></textarea>
<button
type="button"
class="mt-1.5 shrink-0 cursor-pointer rounded-md p-1 text-muted-foreground hover:bg-destructive/10 hover:text-destructive"
onclick={() => removePair(index)}
aria-label="Remove item"
>
<Trash2 class="h-3.5 w-3.5" />
</button>
</div>
{/each}
</div>
{:else}
<p class="text-xs text-muted-foreground">{emptyMessage}</p>
{/if}
</div>

2
tools/server/webui/src/lib/components/app/misc/SearchInput.svelte → tools/server/webui/src/lib/components/app/forms/SearchInput.svelte
View file

@ -46,7 +46,7 @@
<div class="relative {className}">
<Search
class="absolute top-1/2 left-3 h-4 w-4 -translate-y-1/2 transform text-muted-foreground"
class="absolute top-1/2 left-3 z-10 h-4 w-4 -translate-y-1/2 transform text-muted-foreground"
/>
<Input

30
tools/server/webui/src/lib/components/app/forms/index.ts Normal file
View file

@ -0,0 +1,30 @@
/**
*
* FORMS & INPUTS
*
* Form-related utility components.
*
*/
/**
* **SearchInput** - Search field with clear button
*
* Input field optimized for search with clear button and keyboard handling.
* Supports placeholder, autofocus, and change callbacks.
*/
export { default as SearchInput } from './SearchInput.svelte';
/**
* **KeyValuePairs** - Editable key-value list
*
* Dynamic list of key-value pairs with add/remove functionality.
* Used for HTTP headers, metadata, and configuration.
*
* **Features:**
* - Add new pairs with button
* - Remove individual pairs
* - Customizable placeholders and labels
* - Empty state message
* - Auto-resize value textarea
*/
export { default as KeyValuePairs } from './KeyValuePairs.svelte';

85
tools/server/webui/src/lib/components/app/index.ts
View file

@ -1,75 +1,10 @@
// Chat
export { default as ChatAttachmentPreview } from './chat/ChatAttachments/ChatAttachmentPreview.svelte';
export { default as ChatAttachmentThumbnailFile } from './chat/ChatAttachments/ChatAttachmentThumbnailFile.svelte';
export { default as ChatAttachmentThumbnailImage } from './chat/ChatAttachments/ChatAttachmentThumbnailImage.svelte';
export { default as ChatAttachmentsList } from './chat/ChatAttachments/ChatAttachmentsList.svelte';
export { default as ChatAttachmentsViewAll } from './chat/ChatAttachments/ChatAttachmentsViewAll.svelte';
export { default as ChatForm } from './chat/ChatForm/ChatForm.svelte';
export { default as ChatFormActionFileAttachments } from './chat/ChatForm/ChatFormActions/ChatFormActionFileAttachments.svelte';
export { default as ChatFormActionRecord } from './chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte';
export { default as ChatFormActions } from './chat/ChatForm/ChatFormActions/ChatFormActions.svelte';
export { default as ChatFormActionSubmit } from './chat/ChatForm/ChatFormActions/ChatFormActionSubmit.svelte';
export { default as ChatFormFileInputInvisible } from './chat/ChatForm/ChatFormFileInputInvisible.svelte';
export { default as ChatFormHelperText } from './chat/ChatForm/ChatFormHelperText.svelte';
export { default as ChatFormTextarea } from './chat/ChatForm/ChatFormTextarea.svelte';
export { default as ChatMessage } from './chat/ChatMessages/ChatMessage.svelte';
export { default as ChatMessageActions } from './chat/ChatMessages/ChatMessageActions.svelte';
export { default as ChatMessageBranchingControls } from './chat/ChatMessages/ChatMessageBranchingControls.svelte';
export { default as ChatMessageStatistics } from './chat/ChatMessages/ChatMessageStatistics.svelte';
export { default as ChatMessageSystem } from './chat/ChatMessages/ChatMessageSystem.svelte';
export { default as ChatMessageThinkingBlock } from './chat/ChatMessages/ChatMessageThinkingBlock.svelte';
export { default as ChatMessages } from './chat/ChatMessages/ChatMessages.svelte';
export { default as MessageBranchingControls } from './chat/ChatMessages/ChatMessageBranchingControls.svelte';
export { default as ChatScreen } from './chat/ChatScreen/ChatScreen.svelte';
export { default as ChatScreenHeader } from './chat/ChatScreen/ChatScreenHeader.svelte';
export { default as ChatScreenProcessingInfo } from './chat/ChatScreen/ChatScreenProcessingInfo.svelte';
export { default as ChatSettings } from './chat/ChatSettings/ChatSettings.svelte';
export { default as ChatSettingsFooter } from './chat/ChatSettings/ChatSettingsFooter.svelte';
export { default as ChatSettingsFields } from './chat/ChatSettings/ChatSettingsFields.svelte';
export { default as ChatSettingsImportExportTab } from './chat/ChatSettings/ChatSettingsImportExportTab.svelte';
export { default as ChatSettingsParameterSourceIndicator } from './chat/ChatSettings/ChatSettingsParameterSourceIndicator.svelte';
export { default as ChatSidebar } from './chat/ChatSidebar/ChatSidebar.svelte';
export { default as ChatSidebarConversationItem } from './chat/ChatSidebar/ChatSidebarConversationItem.svelte';
export { default as ChatSidebarSearch } from './chat/ChatSidebar/ChatSidebarSearch.svelte';
// Dialogs
export { default as DialogChatAttachmentPreview } from './dialogs/DialogChatAttachmentPreview.svelte';
export { default as DialogChatAttachmentsViewAll } from './dialogs/DialogChatAttachmentsViewAll.svelte';
export { default as DialogChatError } from './dialogs/DialogChatError.svelte';
export { default as DialogChatSettings } from './dialogs/DialogChatSettings.svelte';
export { default as DialogConfirmation } from './dialogs/DialogConfirmation.svelte';
export { default as DialogConversationSelection } from './dialogs/DialogConversationSelection.svelte';
export { default as DialogConversationTitleUpdate } from './dialogs/DialogConversationTitleUpdate.svelte';
export { default as DialogEmptyFileAlert } from './dialogs/DialogEmptyFileAlert.svelte';
export { default as DialogModelInformation } from './dialogs/DialogModelInformation.svelte';
export { default as DialogModelNotAvailable } from './dialogs/DialogModelNotAvailable.svelte';
// Miscellanous
export { default as ActionButton } from './misc/ActionButton.svelte';
export { default as ActionDropdown } from './misc/ActionDropdown.svelte';
export { default as BadgeChatStatistic } from './misc/BadgeChatStatistic.svelte';
export { default as BadgeInfo } from './misc/BadgeInfo.svelte';
export { default as ModelBadge } from './models/ModelBadge.svelte';
export { default as BadgeModality } from './misc/BadgeModality.svelte';
export { default as ConversationSelection } from './misc/ConversationSelection.svelte';
export { default as CopyToClipboardIcon } from './misc/CopyToClipboardIcon.svelte';
export { default as KeyboardShortcutInfo } from './misc/KeyboardShortcutInfo.svelte';
export { default as MarkdownContent } from './misc/MarkdownContent.svelte';
export { default as RemoveButton } from './misc/RemoveButton.svelte';
export { default as SearchInput } from './misc/SearchInput.svelte';
export { default as SyntaxHighlightedCode } from './misc/SyntaxHighlightedCode.svelte';
export { default as ModelsSelector } from './models/ModelsSelector.svelte';
// Server
export { default as ServerStatus } from './server/ServerStatus.svelte';
export { default as ServerErrorSplash } from './server/ServerErrorSplash.svelte';
export { default as ServerLoadingSplash } from './server/ServerLoadingSplash.svelte';
export * from './actions';
export * from './badges';
export * from './chat';
export * from './content';
export * from './dialogs';
export * from './forms';
export * from './misc';
export * from './models';
export * from './navigation';
export * from './server';

47
tools/server/webui/src/lib/components/app/misc/ActionButton.svelte
View file

@ -1,47 +0,0 @@
<script lang="ts">
import { Button } from '$lib/components/ui/button';
import * as Tooltip from '$lib/components/ui/tooltip';
import type { Component } from 'svelte';
interface Props {
icon: Component;
tooltip: string;
variant?: 'default' | 'destructive' | 'outline' | 'secondary' | 'ghost' | 'link';
size?: 'default' | 'sm' | 'lg' | 'icon';
class?: string;
disabled?: boolean;
onclick: () => void;
'aria-label'?: string;
}
let {
icon,
tooltip,
variant = 'ghost',
size = 'sm',
class: className = '',
disabled = false,
onclick,
'aria-label': ariaLabel
}: Props = $props();
</script>
<Tooltip.Root>
<Tooltip.Trigger>
<Button
{variant}
{size}
{disabled}
{onclick}
class="h-6 w-6 p-0 {className} flex"
aria-label={ariaLabel || tooltip}
>
{@const IconComponent = icon}
<IconComponent class="h-3 w-3" />
</Button>
</Tooltip.Trigger>
<Tooltip.Content>
<p>{tooltip}</p>
</Tooltip.Content>
</Tooltip.Root>

86
tools/server/webui/src/lib/components/app/misc/ActionDropdown.svelte
View file

@ -1,86 +0,0 @@
<script lang="ts">
import * as DropdownMenu from '$lib/components/ui/dropdown-menu';
import * as Tooltip from '$lib/components/ui/tooltip';
import { KeyboardShortcutInfo } from '$lib/components/app';
import type { Component } from 'svelte';
interface ActionItem {
icon: Component;
label: string;
onclick: (event: Event) => void;
variant?: 'default' | 'destructive';
disabled?: boolean;
shortcut?: string[];
separator?: boolean;
}
interface Props {
triggerIcon: Component;
triggerTooltip?: string;
triggerClass?: string;
actions: ActionItem[];
align?: 'start' | 'center' | 'end';
open?: boolean;
}
let {
triggerIcon,
triggerTooltip,
triggerClass = '',
actions,
align = 'end',
open = $bindable(false)
}: Props = $props();
</script>
<DropdownMenu.Root bind:open>
<DropdownMenu.Trigger
class="flex h-6 w-6 cursor-pointer items-center justify-center rounded-md p-0 text-sm font-medium transition-colors hover:bg-accent hover:text-accent-foreground focus:bg-accent focus:text-accent-foreground focus:outline-none disabled:pointer-events-none disabled:opacity-50 data-[state=open]:bg-accent data-[state=open]:text-accent-foreground {triggerClass}"
onclick={(e) => e.stopPropagation()}
>
{#if triggerTooltip}
<Tooltip.Root>
<Tooltip.Trigger>
{@render iconComponent(triggerIcon, 'h-3 w-3')}
<span class="sr-only">{triggerTooltip}</span>
</Tooltip.Trigger>
<Tooltip.Content>
<p>{triggerTooltip}</p>
</Tooltip.Content>
</Tooltip.Root>
{:else}
{@render iconComponent(triggerIcon, 'h-3 w-3')}
{/if}
</DropdownMenu.Trigger>
<DropdownMenu.Content {align} class="z-[999999] w-48">
{#each actions as action, index (action.label)}
{#if action.separator && index > 0}
<DropdownMenu.Separator />
{/if}
<DropdownMenu.Item
onclick={action.onclick}
variant={action.variant}
disabled={action.disabled}
class="flex items-center justify-between hover:[&>kbd]:opacity-100"
>
<div class="flex items-center gap-2">
{@render iconComponent(
action.icon,
`h-4 w-4 ${action.variant === 'destructive' ? 'text-destructive' : ''}`
)}
{action.label}
</div>
{#if action.shortcut}
<KeyboardShortcutInfo keys={action.shortcut} variant={action.variant} />
{/if}
</DropdownMenu.Item>
{/each}
</DropdownMenu.Content>
</DropdownMenu.Root>
{#snippet iconComponent(IconComponent: Component, className: string)}
<IconComponent class={className} />
{/snippet}

44
tools/server/webui/src/lib/components/app/misc/BadgeChatStatistic.svelte
View file

@ -1,44 +0,0 @@
<script lang="ts">
import { BadgeInfo } from '$lib/components/app';
import * as Tooltip from '$lib/components/ui/tooltip';
import { copyToClipboard } from '$lib/utils';
import type { Component } from 'svelte';
interface Props {
class?: string;
icon: Component;
value: string | number;
tooltipLabel?: string;
}
let { class: className = '', icon: Icon, value, tooltipLabel }: Props = $props();
function handleClick() {
void copyToClipboard(String(value));
}
</script>
{#if tooltipLabel}
<Tooltip.Root>
<Tooltip.Trigger>
<BadgeInfo class={className} onclick={handleClick}>
{#snippet icon()}
<Icon class="h-3 w-3" />
{/snippet}
{value}
</BadgeInfo>
</Tooltip.Trigger>
<Tooltip.Content>
<p>{tooltipLabel}</p>
</Tooltip.Content>
</Tooltip.Root>
{:else}
<BadgeInfo class={className} onclick={handleClick}>
{#snippet icon()}
<Icon class="h-3 w-3" />
{/snippet}
{value}
</BadgeInfo>
{/if}

27
tools/server/webui/src/lib/components/app/misc/BadgeInfo.svelte
View file

@ -1,27 +0,0 @@
<script lang="ts">
import { cn } from '$lib/components/ui/utils';
import type { Snippet } from 'svelte';
interface Props {
children: Snippet;
class?: string;
icon?: Snippet;
onclick?: () => void;
}
let { children, class: className = '', icon, onclick }: Props = $props();
</script>
<button
class={cn(
'inline-flex cursor-pointer items-center gap-1 rounded-sm bg-muted-foreground/15 px-1.5 py-0.75',
className
)}
{onclick}
>
{#if icon}
{@render icon()}
{/if}
{@render children()}
</button>

39
tools/server/webui/src/lib/components/app/misc/BadgeModality.svelte
View file

@ -1,39 +0,0 @@
<script lang="ts">
import { ModelModality } from '$lib/enums';
import { MODALITY_ICONS, MODALITY_LABELS } from '$lib/constants/icons';
import { cn } from '$lib/components/ui/utils';
type DisplayableModality = ModelModality.VISION | ModelModality.AUDIO;
interface Props {
modalities: ModelModality[];
class?: string;
}
let { modalities, class: className = '' }: Props = $props();
// Filter to only modalities that have icons (VISION, AUDIO)
const displayableModalities = $derived(
modalities.filter(
(m): m is DisplayableModality => m === ModelModality.VISION || m === ModelModality.AUDIO
)
);
</script>
{#each displayableModalities as modality, index (index)}
{@const IconComponent = MODALITY_ICONS[modality]}
{@const label = MODALITY_LABELS[modality]}
<span
class={cn(
'inline-flex items-center gap-1 rounded-md bg-muted px-2 py-1 text-xs font-medium',
className
)}
>
{#if IconComponent}
<IconComponent class="h-3 w-3" />
{/if}
{label}
</span>
{/each}

18
tools/server/webui/src/lib/components/app/misc/CopyToClipboardIcon.svelte
View file

@ -1,18 +0,0 @@
<script lang="ts">
import { Copy } from '@lucide/svelte';
import { copyToClipboard } from '$lib/utils';
interface Props {
ariaLabel?: string;
canCopy?: boolean;
text: string;
}
let { ariaLabel = 'Copy to clipboard', canCopy = true, text }: Props = $props();
</script>
<Copy
class="h-3 w-3 flex-shrink-0 cursor-{canCopy ? 'pointer' : 'not-allowed'}"
aria-label={ariaLabel}
onclick={() => canCopy && copyToClipboard(text)}
/>

870
tools/server/webui/src/lib/components/app/misc/MarkdownContent.svelte
View file

@ -1,870 +0,0 @@
<script lang="ts">
import { remark } from 'remark';
import remarkBreaks from 'remark-breaks';
import remarkGfm from 'remark-gfm';
import remarkMath from 'remark-math';
import rehypeHighlight from 'rehype-highlight';
import remarkRehype from 'remark-rehype';
import rehypeKatex from 'rehype-katex';
import rehypeStringify from 'rehype-stringify';
import type { Root as HastRoot, RootContent as HastRootContent } from 'hast';
import type { Root as MdastRoot } from 'mdast';
import { browser } from '$app/environment';
import { onDestroy, tick } from 'svelte';
import { rehypeRestoreTableHtml } from '$lib/markdown/table-html-restorer';
import { rehypeEnhanceLinks } from '$lib/markdown/enhance-links';
import { rehypeEnhanceCodeBlocks } from '$lib/markdown/enhance-code-blocks';
import { remarkLiteralHtml } from '$lib/markdown/literal-html';
import { copyCodeToClipboard, preprocessLaTeX } from '$lib/utils';
import '$styles/katex-custom.scss';
import githubDarkCss from 'highlight.js/styles/github-dark.css?inline';
import githubLightCss from 'highlight.js/styles/github.css?inline';
import { mode } from 'mode-watcher';
import CodePreviewDialog from './CodePreviewDialog.svelte';
interface Props {
content: string;
class?: string;
}
interface MarkdownBlock {
id: string;
html: string;
}
let { content, class: className = '' }: Props = $props();
let containerRef = $state<HTMLDivElement>();
let renderedBlocks = $state<MarkdownBlock[]>([]);
let unstableBlockHtml = $state('');
let previewDialogOpen = $state(false);
let previewCode = $state('');
let previewLanguage = $state('text');
let pendingMarkdown: string | null = null;
let isProcessing = false;
const themeStyleId = `highlight-theme-${(window.idxThemeStyle = (window.idxThemeStyle ?? 0) + 1)}`;
let processor = $derived(() => {
return remark()
.use(remarkGfm) // GitHub Flavored Markdown
.use(remarkMath) // Parse $inline$ and $$block$$ math
.use(remarkBreaks) // Convert line breaks to <br>
.use(remarkLiteralHtml) // Treat raw HTML as literal text with preserved indentation
.use(remarkRehype) // Convert Markdown AST to rehype
.use(rehypeKatex) // Render math using KaTeX
.use(rehypeHighlight) // Add syntax highlighting
.use(rehypeRestoreTableHtml) // Restore limited HTML (e.g., <br>, <ul>) inside Markdown tables
.use(rehypeEnhanceLinks) // Add target="_blank" to links
.use(rehypeEnhanceCodeBlocks) // Wrap code blocks with header and actions
.use(rehypeStringify, { allowDangerousHtml: true }); // Convert to HTML string
});
/**
* Removes click event listeners from copy and preview buttons.
* Called on component destroy.
*/
function cleanupEventListeners() {
if (!containerRef) return;
const copyButtons = containerRef.querySelectorAll<HTMLButtonElement>('.copy-code-btn');
const previewButtons = containerRef.querySelectorAll<HTMLButtonElement>('.preview-code-btn');
for (const button of copyButtons) {
button.removeEventListener('click', handleCopyClick);
}
for (const button of previewButtons) {
button.removeEventListener('click', handlePreviewClick);
}
}
/**
* Removes this component's highlight.js theme style from the document head.
* Called on component destroy to clean up injected styles.
*/
function cleanupHighlightTheme() {
if (!browser) return;
const existingTheme = document.getElementById(themeStyleId);
existingTheme?.remove();
}
/**
* Loads the appropriate highlight.js theme based on dark/light mode.
* Injects a scoped style element into the document head.
* @param isDark - Whether to load the dark theme (true) or light theme (false)
*/
function loadHighlightTheme(isDark: boolean) {
if (!browser) return;
const existingTheme = document.getElementById(themeStyleId);
existingTheme?.remove();
const style = document.createElement('style');
style.id = themeStyleId;
style.textContent = isDark ? githubDarkCss : githubLightCss;
document.head.appendChild(style);
}
/**
* Extracts code information from a button click target within a code block.
* @param target - The clicked button element
* @returns Object with rawCode and language, or null if extraction fails
*/
function getCodeInfoFromTarget(target: HTMLElement) {
const wrapper = target.closest('.code-block-wrapper');
if (!wrapper) {
console.error('No wrapper found');
return null;
}
const codeElement = wrapper.querySelector<HTMLElement>('code[data-code-id]');
if (!codeElement) {
console.error('No code element found in wrapper');
return null;
}
const rawCode = codeElement.textContent ?? '';
const languageLabel = wrapper.querySelector<HTMLElement>('.code-language');
const language = languageLabel?.textContent?.trim() || 'text';
return { rawCode, language };
}
/**
* Generates a unique identifier for a HAST node based on its position.
* Used for stable block identification during incremental rendering.
* @param node - The HAST root content node
* @param indexFallback - Fallback index if position is unavailable
* @returns Unique string identifier for the node
*/
function getHastNodeId(node: HastRootContent, indexFallback: number): string {
const position = node.position;
if (position?.start?.offset != null && position?.end?.offset != null) {
return `hast-${position.start.offset}-${position.end.offset}`;
}
return `${node.type}-${indexFallback}`;
}
/**
* Handles click events on copy buttons within code blocks.
* Copies the raw code content to the clipboard.
* @param event - The click event from the copy button
*/
async function handleCopyClick(event: Event) {
event.preventDefault();
event.stopPropagation();
const target = event.currentTarget as HTMLButtonElement | null;
if (!target) {
return;
}
const info = getCodeInfoFromTarget(target);
if (!info) {
return;
}
try {
await copyCodeToClipboard(info.rawCode);
} catch (error) {
console.error('Failed to copy code:', error);
}
}
/**
* Handles preview dialog open state changes.
* Clears preview content when dialog is closed.
* @param open - Whether the dialog is being opened or closed
*/
function handlePreviewDialogOpenChange(open: boolean) {
previewDialogOpen = open;
if (!open) {
previewCode = '';
previewLanguage = 'text';
}
}
/**
* Handles click events on preview buttons within HTML code blocks.
* Opens a preview dialog with the rendered HTML content.
* @param event - The click event from the preview button
*/
function handlePreviewClick(event: Event) {
event.preventDefault();
event.stopPropagation();
const target = event.currentTarget as HTMLButtonElement | null;
if (!target) {
return;
}
const info = getCodeInfoFromTarget(target);
if (!info) {
return;
}
previewCode = info.rawCode;
previewLanguage = info.language;
previewDialogOpen = true;
}
/**
* Processes markdown content into stable and unstable HTML blocks.
* Uses incremental rendering: stable blocks are cached, unstable block is re-rendered.
* @param markdown - The raw markdown string to process
*/
async function processMarkdown(markdown: string) {
if (!markdown) {
renderedBlocks = [];
unstableBlockHtml = '';
return;
}
const normalized = preprocessLaTeX(markdown);
const processorInstance = processor();
const ast = processorInstance.parse(normalized) as MdastRoot;
const processedRoot = (await processorInstance.run(ast)) as HastRoot;
const processedChildren = processedRoot.children ?? [];
const stableCount = Math.max(processedChildren.length - 1, 0);
const nextBlocks: MarkdownBlock[] = [];
for (let index = 0; index < stableCount; index++) {
const hastChild = processedChildren[index];
const id = getHastNodeId(hastChild, index);
const existing = renderedBlocks[index];
if (existing && existing.id === id) {
nextBlocks.push(existing);
continue;
}
const html = stringifyProcessedNode(
processorInstance,
processedRoot,
processedChildren[index]
);
nextBlocks.push({ id, html });
}
let unstableHtml = '';
if (processedChildren.length > stableCount) {
const unstableChild = processedChildren[stableCount];
unstableHtml = stringifyProcessedNode(processorInstance, processedRoot, unstableChild);
}
renderedBlocks = nextBlocks;
await tick(); // Force DOM sync before updating unstable HTML block
unstableBlockHtml = unstableHtml;
}
/**
* Attaches click event listeners to copy and preview buttons in code blocks.
* Uses data-listener-bound attribute to prevent duplicate bindings.
*/
function setupCodeBlockActions() {
if (!containerRef) return;
const wrappers = containerRef.querySelectorAll<HTMLElement>('.code-block-wrapper');
for (const wrapper of wrappers) {
const copyButton = wrapper.querySelector<HTMLButtonElement>('.copy-code-btn');
const previewButton = wrapper.querySelector<HTMLButtonElement>('.preview-code-btn');
if (copyButton && copyButton.dataset.listenerBound !== 'true') {
copyButton.dataset.listenerBound = 'true';
copyButton.addEventListener('click', handleCopyClick);
}
if (previewButton && previewButton.dataset.listenerBound !== 'true') {
previewButton.dataset.listenerBound = 'true';
previewButton.addEventListener('click', handlePreviewClick);
}
}
}
/**
* Converts a single HAST node to an enhanced HTML string.
* Applies link and code block enhancements to the output.
* @param processorInstance - The remark/rehype processor instance
* @param processedRoot - The full processed HAST root (for context)
* @param child - The specific HAST child node to stringify
* @returns Enhanced HTML string representation of the node
*/
function stringifyProcessedNode(
processorInstance: ReturnType<typeof processor>,
processedRoot: HastRoot,
child: unknown
) {
const root: HastRoot = {
...(processedRoot as HastRoot),
children: [child as never]
};
return processorInstance.stringify(root);
}
/**
* Queues markdown for processing with coalescing support.
* Only processes the latest markdown when multiple updates arrive quickly.
* @param markdown - The markdown content to render
*/
async function updateRenderedBlocks(markdown: string) {
pendingMarkdown = markdown;
if (isProcessing) {
return;
}
isProcessing = true;
try {
while (pendingMarkdown !== null) {
const nextMarkdown = pendingMarkdown;
pendingMarkdown = null;
await processMarkdown(nextMarkdown);
}
} catch (error) {
console.error('Failed to process markdown:', error);
renderedBlocks = [];
unstableBlockHtml = markdown.replace(/\n/g, '<br>');
} finally {
isProcessing = false;
}
}
$effect(() => {
const currentMode = mode.current;
const isDark = currentMode === 'dark';
loadHighlightTheme(isDark);
});
$effect(() => {
updateRenderedBlocks(content);
});
$effect(() => {
const hasRenderedBlocks = renderedBlocks.length > 0;
const hasUnstableBlock = Boolean(unstableBlockHtml);
if ((hasRenderedBlocks || hasUnstableBlock) && containerRef) {
setupCodeBlockActions();
}
});
onDestroy(() => {
cleanupEventListeners();
cleanupHighlightTheme();
});
</script>
<div bind:this={containerRef} class={className}>
{#each renderedBlocks as block (block.id)}
<div class="markdown-block" data-block-id={block.id}>
<!-- eslint-disable-next-line no-at-html-tags -->
{@html block.html}
</div>
{/each}
{#if unstableBlockHtml}
<div class="markdown-block markdown-block--unstable" data-block-id="unstable">
<!-- eslint-disable-next-line no-at-html-tags -->
{@html unstableBlockHtml}
</div>
{/if}
</div>
<CodePreviewDialog
open={previewDialogOpen}
code={previewCode}
language={previewLanguage}
onOpenChange={handlePreviewDialogOpenChange}
/>
<style>
.markdown-block,
.markdown-block--unstable {
display: contents;
}
/* Base typography styles */
div :global(p:not(:last-child)) {
margin-bottom: 1rem;
line-height: 1.75;
}
div :global(:is(h1, h2, h3, h4, h5, h6):first-child) {
margin-top: 0;
}
/* Headers with consistent spacing */
div :global(h1) {
font-size: 1.875rem;
font-weight: 700;
line-height: 1.2;
margin: 1.5rem 0 0.75rem 0;
}
div :global(h2) {
font-size: 1.5rem;
font-weight: 600;
line-height: 1.3;
margin: 1.25rem 0 0.5rem 0;
}
div :global(h3) {
font-size: 1.25rem;
font-weight: 600;
margin: 1.5rem 0 0.5rem 0;
line-height: 1.4;
}
div :global(h4) {
font-size: 1.125rem;
font-weight: 600;
margin: 0.75rem 0 0.25rem 0;
}
div :global(h5) {
font-size: 1rem;
font-weight: 600;
margin: 0.5rem 0 0.25rem 0;
}
div :global(h6) {
font-size: 0.875rem;
font-weight: 600;
margin: 0.5rem 0 0.25rem 0;
}
/* Text formatting */
div :global(strong) {
font-weight: 600;
}
div :global(em) {
font-style: italic;
}
div :global(del) {
text-decoration: line-through;
opacity: 0.7;
}
/* Inline code */
div :global(code:not(pre code)) {
background: var(--muted);
color: var(--muted-foreground);
padding: 0.125rem 0.375rem;
border-radius: 0.375rem;
font-size: 0.875rem;
font-family:
ui-monospace, SFMono-Regular, 'SF Mono', Monaco, 'Cascadia Code', 'Roboto Mono', Consolas,
'Liberation Mono', Menlo, monospace;
}
/* Links */
div :global(a) {
color: var(--primary);
text-decoration: underline;
text-underline-offset: 2px;
transition: color 0.2s ease;
}
div :global(a:hover) {
color: var(--primary);
}
/* Lists */
div :global(ul) {
list-style-type: disc;
margin-left: 1.5rem;
margin-bottom: 1rem;
}
div :global(ol) {
list-style-type: decimal;
margin-left: 1.5rem;
margin-bottom: 1rem;
}
div :global(li) {
margin-bottom: 0.25rem;
padding-left: 0.5rem;
}
div :global(li::marker) {
color: var(--muted-foreground);
}
/* Nested lists */
div :global(ul ul) {
list-style-type: circle;
margin-top: 0.25rem;
margin-bottom: 0.25rem;
}
div :global(ol ol) {
list-style-type: lower-alpha;
margin-top: 0.25rem;
margin-bottom: 0.25rem;
}
/* Task lists */
div :global(.task-list-item) {
list-style: none;
margin-left: 0;
padding-left: 0;
}
div :global(.task-list-item-checkbox) {
margin-right: 0.5rem;
margin-top: 0.125rem;
}
/* Blockquotes */
div :global(blockquote) {
border-left: 4px solid var(--border);
padding: 0.5rem 1rem;
margin: 1.5rem 0;
font-style: italic;
color: var(--muted-foreground);
background: var(--muted);
border-radius: 0 0.375rem 0.375rem 0;
}
/* Tables */
div :global(table) {
width: 100%;
margin: 1.5rem 0;
border-collapse: collapse;
border: 1px solid var(--border);
border-radius: 0.375rem;
overflow: hidden;
}
div :global(th) {
background: hsl(var(--muted) / 0.3);
border: 1px solid var(--border);
padding: 0.5rem 0.75rem;
text-align: left;
font-weight: 600;
}
div :global(td) {
border: 1px solid var(--border);
padding: 0.5rem 0.75rem;
}
div :global(tr:nth-child(even)) {
background: hsl(var(--muted) / 0.1);
}
/* User message markdown should keep table borders visible on light primary backgrounds */
div.markdown-user-content :global(table),
div.markdown-user-content :global(th),
div.markdown-user-content :global(td),
div.markdown-user-content :global(.table-wrapper) {
border-color: currentColor;
}
/* Horizontal rules */
div :global(hr) {
border: none;
border-top: 1px solid var(--border);
margin: 1.5rem 0;
}
/* Images */
div :global(img) {
border-radius: 0.5rem;
box-shadow:
0 1px 3px 0 rgb(0 0 0 / 0.1),
0 1px 2px -1px rgb(0 0 0 / 0.1);
margin: 1.5rem 0;
max-width: 100%;
height: auto;
}
/* Code blocks */
div :global(.code-block-wrapper) {
margin: 1.5rem 0;
border-radius: 0.75rem;
overflow: hidden;
border: 1px solid var(--border);
background: var(--code-background);
}
div :global(.code-block-header) {
display: flex;
justify-content: space-between;
align-items: center;
padding: 0.5rem 1rem;
background: hsl(var(--muted) / 0.5);
border-bottom: 1px solid var(--border);
font-size: 0.875rem;
}
div :global(.code-language) {
color: var(--code-foreground);
font-weight: 500;
font-family:
ui-monospace, SFMono-Regular, 'SF Mono', Monaco, 'Cascadia Code', 'Roboto Mono', Consolas,
'Liberation Mono', Menlo, monospace;
text-transform: uppercase;
font-size: 0.75rem;
letter-spacing: 0.05em;
}
div :global(.code-block-actions) {
display: flex;
align-items: center;
gap: 0.5rem;
}
div :global(.copy-code-btn),
div :global(.preview-code-btn) {
display: flex;
align-items: center;
justify-content: center;
padding: 0;
background: transparent;
color: var(--code-foreground);
cursor: pointer;
transition: all 0.2s ease;
}
div :global(.copy-code-btn:hover),
div :global(.preview-code-btn:hover) {
transform: scale(1.05);
}
div :global(.copy-code-btn:active),
div :global(.preview-code-btn:active) {
transform: scale(0.95);
}
div :global(.code-block-wrapper pre) {
background: transparent;
padding: 1rem;
margin: 0;
overflow-x: auto;
border-radius: 0;
border: none;
font-size: 0.875rem;
line-height: 1.5;
}
div :global(pre) {
background: var(--muted);
margin: 1.5rem 0;
overflow-x: auto;
border-radius: 1rem;
border: none;
}
div :global(code) {
background: transparent;
color: var(--code-foreground);
}
/* Mentions and hashtags */
div :global(.mention) {
color: hsl(var(--primary));
font-weight: 500;
text-decoration: none;
}
div :global(.mention:hover) {
text-decoration: underline;
}
div :global(.hashtag) {
color: hsl(var(--primary));
font-weight: 500;
text-decoration: none;
}
div :global(.hashtag:hover) {
text-decoration: underline;
}
/* Advanced table enhancements */
div :global(table) {
transition: all 0.2s ease;
}
div :global(table:hover) {
box-shadow:
0 4px 6px -1px rgb(0 0 0 / 0.1),
0 2px 4px -2px rgb(0 0 0 / 0.1);
}
div :global(th:hover),
div :global(td:hover) {
background: var(--muted);
}
/* Disable hover effects when rendering user messages */
.markdown-user-content :global(a),
.markdown-user-content :global(a:hover) {
color: var(--primary-foreground);
}
.markdown-user-content :global(table:hover) {
box-shadow: none;
}
.markdown-user-content :global(th:hover),
.markdown-user-content :global(td:hover) {
background: inherit;
}
/* Enhanced blockquotes */
div :global(blockquote) {
transition: all 0.2s ease;
position: relative;
}
div :global(blockquote:hover) {
border-left-width: 6px;
background: var(--muted);
transform: translateX(2px);
}
div :global(blockquote::before) {
content: '"';
position: absolute;
top: -0.5rem;
left: 0.5rem;
font-size: 3rem;
color: var(--muted-foreground);
font-family: serif;
line-height: 1;
}
/* Enhanced images */
div :global(img) {
transition: all 0.3s ease;
cursor: pointer;
}
div :global(img:hover) {
transform: scale(1.02);
box-shadow:
0 10px 15px -3px rgb(0 0 0 / 0.1),
0 4px 6px -4px rgb(0 0 0 / 0.1);
}
/* Image zoom overlay */
div :global(.image-zoom-overlay) {
position: fixed;
top: 0;
left: 0;
right: 0;
bottom: 0;
background: rgba(0, 0, 0, 0.8);
display: flex;
align-items: center;
justify-content: center;
z-index: 1000;
cursor: pointer;
}
div :global(.image-zoom-overlay img) {
max-width: 90vw;
max-height: 90vh;
border-radius: 0.5rem;
box-shadow: 0 25px 50px -12px rgb(0 0 0 / 0.25);
}
/* Enhanced horizontal rules */
div :global(hr) {
border: none;
height: 2px;
background: linear-gradient(to right, transparent, var(--border), transparent);
margin: 2rem 0;
position: relative;
}
div :global(hr::after) {
content: '';
position: absolute;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
width: 1rem;
height: 1rem;
background: var(--border);
border-radius: 50%;
}
/* Scrollable tables */
div :global(.table-wrapper) {
overflow-x: auto;
margin: 1.5rem 0;
border-radius: 0.5rem;
border: 1px solid var(--border);
}
div :global(.table-wrapper table) {
margin: 0;
border: none;
}
/* Responsive adjustments */
@media (max-width: 640px) {
div :global(h1) {
font-size: 1.5rem;
}
div :global(h2) {
font-size: 1.25rem;
}
div :global(h3) {
font-size: 1.125rem;
}
div :global(table) {
font-size: 0.875rem;
}
div :global(th),
div :global(td) {
padding: 0.375rem 0.5rem;
}
div :global(.table-wrapper) {
margin: 0.5rem -1rem;
border-radius: 0;
border-left: none;
border-right: none;
}
}
/* Dark mode adjustments */
@media (prefers-color-scheme: dark) {
div :global(blockquote:hover) {
background: var(--muted);
}
}
</style>

26
tools/server/webui/src/lib/components/app/misc/RemoveButton.svelte
View file

@ -1,26 +0,0 @@
<script lang="ts">
import { X } from '@lucide/svelte';
import { Button } from '$lib/components/ui/button';
interface Props {
id: string;
onRemove?: (id: string) => void;
class?: string;
}
let { id, onRemove, class: className = '' }: Props = $props();
</script>
<Button
type="button"
variant="ghost"
size="sm"
class="h-6 w-6 bg-white/20 p-0 hover:bg-white/30 {className}"
onclick={(e) => {
e.stopPropagation();
onRemove?.(id);
}}
aria-label="Remove file"
>
<X class="h-3 w-3" />
</Button>

97
tools/server/webui/src/lib/components/app/misc/SyntaxHighlightedCode.svelte
View file

@ -1,97 +0,0 @@
<script lang="ts">
import hljs from 'highlight.js';
import { browser } from '$app/environment';
import { mode } from 'mode-watcher';
import githubDarkCss from 'highlight.js/styles/github-dark.css?inline';
import githubLightCss from 'highlight.js/styles/github.css?inline';
interface Props {
code: string;
language?: string;
class?: string;
maxHeight?: string;
maxWidth?: string;
}
let {
code,
language = 'text',
class: className = '',
maxHeight = '60vh',
maxWidth = ''
}: Props = $props();
let highlightedHtml = $state('');
function loadHighlightTheme(isDark: boolean) {
if (!browser) return;
const existingThemes = document.querySelectorAll('style[data-highlight-theme-preview]');
existingThemes.forEach((style) => style.remove());
const style = document.createElement('style');
style.setAttribute('data-highlight-theme-preview', 'true');
style.textContent = isDark ? githubDarkCss : githubLightCss;
document.head.appendChild(style);
}
$effect(() => {
const currentMode = mode.current;
const isDark = currentMode === 'dark';
loadHighlightTheme(isDark);
});
$effect(() => {
if (!code) {
highlightedHtml = '';
return;
}
try {
// Check if the language is supported
const lang = language.toLowerCase();
const isSupported = hljs.getLanguage(lang);
if (isSupported) {
const result = hljs.highlight(code, { language: lang });
highlightedHtml = result.value;
} else {
// Try auto-detection or fallback to plain text
const result = hljs.highlightAuto(code);
highlightedHtml = result.value;
}
} catch {
// Fallback to escaped plain text
highlightedHtml = code.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
}
});
</script>
<div
class="code-preview-wrapper overflow-auto rounded-lg border border-border bg-muted {className}"
style="max-height: {maxHeight}; max-width: {maxWidth};"
>
<!-- Needs to be formatted as single line for proper rendering -->
<pre class="m-0 overflow-x-auto p-4"><code class="hljs text-sm leading-relaxed"
>{@html highlightedHtml}</code
></pre>
</div>
<style>
.code-preview-wrapper {
font-family:
ui-monospace, SFMono-Regular, 'SF Mono', Monaco, 'Cascadia Code', 'Roboto Mono', Consolas,
'Liberation Mono', Menlo, monospace;
}
.code-preview-wrapper pre {
background: transparent;
}
.code-preview-wrapper code {
background: transparent;
}
</style>

4
tools/server/webui/src/lib/components/app/models/ModelBadge.svelte
View file

@ -1,6 +1,6 @@
<script lang="ts">
import { Package } from '@lucide/svelte';
import { BadgeInfo, CopyToClipboardIcon } from '$lib/components/app';
import { BadgeInfo, ActionIconCopyToClipboard } from '$lib/components/app';
import { modelsStore } from '$lib/stores/models.svelte';
import { serverStore } from '$lib/stores/server.svelte';
import * as Tooltip from '$lib/components/ui/tooltip';
@ -34,7 +34,7 @@
{model}
{#if showCopyIcon}
<CopyToClipboardIcon text={model || ''} ariaLabel="Copy model name" />
<ActionIconCopyToClipboard text={model || ''} ariaLabel="Copy model name" />
{/if}
</BadgeInfo>
{/snippet}

373
tools/server/webui/src/lib/components/app/models/ModelsSelector.svelte
View file

@ -1,8 +1,8 @@
<script lang="ts">
import { onMount, tick } from 'svelte';
import { ChevronDown, EyeOff, Loader2, MicOff, Package, Power } from '@lucide/svelte';
import { onMount } from 'svelte';
import { ChevronDown, Loader2, Package, Power } from '@lucide/svelte';
import * as DropdownMenu from '$lib/components/ui/dropdown-menu';
import * as Tooltip from '$lib/components/ui/tooltip';
import * as Popover from '$lib/components/ui/popover';
import { cn } from '$lib/components/ui/utils';
import {
modelsStore,
@ -11,13 +11,15 @@
modelsUpdating,
selectedModelId,
routerModels,
propsCacheVersion,
singleModelName
} from '$lib/stores/models.svelte';
import { usedModalities, conversationsStore } from '$lib/stores/conversations.svelte';
import { ServerModelStatus } from '$lib/enums';
import { KeyboardKey, ServerModelStatus } from '$lib/enums';
import { isRouterMode } from '$lib/stores/server.svelte';
import { DialogModelInformation, SearchInput } from '$lib/components/app';
import {
DialogModelInformation,
DropdownMenuSearchable,
TruncatedText
} from '$lib/components/app';
import type { ModelOption } from '$lib/types/models';
interface Props {
@ -29,12 +31,6 @@
forceForegroundText?: boolean;
/** When true, user's global selection takes priority over currentModel (for form selector) */
useGlobalSelection?: boolean;
/**
* When provided, only consider modalities from messages BEFORE this message.
* Used for regeneration - allows selecting models that don't support modalities
* used in later messages.
*/
upToMessageId?: string;
}
let {
@ -43,11 +39,16 @@
onModelChange,
disabled = false,
forceForegroundText = false,
useGlobalSelection = false,
upToMessageId
useGlobalSelection = false
}: Props = $props();
let options = $derived(modelOptions());
let options = $derived(
modelOptions().filter((option) => {
const modelProps = modelsStore.getModelProps(option.model);
return modelProps?.webui !== false;
})
);
let loading = $derived(modelsLoading());
let updating = $derived(modelsUpdating());
let activeId = $derived(selectedModelId());
@ -57,74 +58,11 @@
// Reactive router models state - needed for proper reactivity of status checks
let currentRouterModels = $derived(routerModels());
let requiredModalities = $derived(
upToMessageId ? conversationsStore.getModalitiesUpToMessage(upToMessageId) : usedModalities()
);
function getModelStatus(modelId: string): ServerModelStatus | null {
const model = currentRouterModels.find((m) => m.id === modelId);
return (model?.status?.value as ServerModelStatus) ?? null;
}
/**
* Checks if a model supports all modalities used in the conversation.
* Returns true if the model can be selected, false if it should be disabled.
*/
function isModelCompatible(option: ModelOption): boolean {
void propsCacheVersion();
const modelModalities = modelsStore.getModelModalities(option.model);
if (!modelModalities) {
const status = getModelStatus(option.model);
if (status === ServerModelStatus.LOADED) {
if (requiredModalities.vision || requiredModalities.audio) return false;
}
return true;
}
if (requiredModalities.vision && !modelModalities.vision) return false;
if (requiredModalities.audio && !modelModalities.audio) return false;
return true;
}
/**
* Gets missing modalities for a model.
* Returns object with vision/audio booleans indicating what's missing.
*/
function getMissingModalities(option: ModelOption): { vision: boolean; audio: boolean } | null {
void propsCacheVersion();
const modelModalities = modelsStore.getModelModalities(option.model);
if (!modelModalities) {
const status = getModelStatus(option.model);
if (status === ServerModelStatus.LOADED) {
const missing = {
vision: requiredModalities.vision,
audio: requiredModalities.audio
};
if (missing.vision || missing.audio) return missing;
}
return null;
}
const missing = {
vision: requiredModalities.vision && !modelModalities.vision,
audio: requiredModalities.audio && !modelModalities.audio
};
if (!missing.vision && !missing.audio) return null;
return missing;
}
let isHighlightedCurrentModelActive = $derived(
!isRouter || !currentModel
? false
@ -142,7 +80,6 @@
});
let searchTerm = $state('');
let searchInputRef = $state<HTMLInputElement | null>(null);
let highlightedIndex = $state<number>(-1);
let filteredOptions: ModelOption[] = $derived(
@ -157,13 +94,6 @@
})()
);
// Get indices of compatible options for keyboard navigation
let compatibleIndices = $derived(
filteredOptions
.map((option, index) => (isModelCompatible(option) ? index : -1))
.filter((i) => i !== -1)
);
// Reset highlighted index when search term changes
$effect(() => {
void searchTerm;
@ -179,7 +109,7 @@
});
});
// Handle changes to the model selector pop-down or the model dialog, depending on if the server is in
// Handle changes to the model selector dropdown or the model dialog, depending on if the server is in
// router mode or not.
function handleOpenChange(open: boolean) {
if (loading || updating) return;
@ -190,11 +120,6 @@
searchTerm = '';
highlightedIndex = -1;
// Focus search input after popover opens
tick().then(() => {
requestAnimationFrame(() => searchInputRef?.focus());
});
modelsStore.fetchRouterModels().then(() => {
modelsStore.fetchModalitiesForLoadedModels();
});
@ -215,36 +140,32 @@
function handleSearchKeyDown(event: KeyboardEvent) {
if (event.isComposing) return;
if (event.key === 'ArrowDown') {
if (event.key === KeyboardKey.ARROW_DOWN) {
event.preventDefault();
if (compatibleIndices.length === 0) return;
if (filteredOptions.length === 0) return;
const currentPos = compatibleIndices.indexOf(highlightedIndex);
if (currentPos === -1 || currentPos === compatibleIndices.length - 1) {
highlightedIndex = compatibleIndices[0];
if (highlightedIndex === -1 || highlightedIndex === filteredOptions.length - 1) {
highlightedIndex = 0;
} else {
highlightedIndex = compatibleIndices[currentPos + 1];
highlightedIndex += 1;
}
} else if (event.key === 'ArrowUp') {
} else if (event.key === KeyboardKey.ARROW_UP) {
event.preventDefault();
if (compatibleIndices.length === 0) return;
if (filteredOptions.length === 0) return;
const currentPos = compatibleIndices.indexOf(highlightedIndex);
if (currentPos === -1 || currentPos === 0) {
highlightedIndex = compatibleIndices[compatibleIndices.length - 1];
if (highlightedIndex === -1 || highlightedIndex === 0) {
highlightedIndex = filteredOptions.length - 1;
} else {
highlightedIndex = compatibleIndices[currentPos - 1];
highlightedIndex -= 1;
}
} else if (event.key === 'Enter') {
} else if (event.key === KeyboardKey.ENTER) {
event.preventDefault();
if (highlightedIndex >= 0 && highlightedIndex < filteredOptions.length) {
const option = filteredOptions[highlightedIndex];
if (isModelCompatible(option)) {
handleSelect(option.id);
}
} else if (compatibleIndices.length > 0) {
// No selection - highlight first compatible option
highlightedIndex = compatibleIndices[0];
handleSelect(option.id);
} else if (filteredOptions.length > 0) {
// No selection - highlight first option
highlightedIndex = 0;
}
}
}
@ -347,68 +268,72 @@
{@const selectedOption = getDisplayOption()}
{#if isRouter}
<Popover.Root bind:open={isOpen} onOpenChange={handleOpenChange}>
<Popover.Trigger
class={cn(
`inline-flex cursor-pointer items-center gap-1.5 rounded-sm bg-muted-foreground/10 px-1.5 py-1 text-xs transition hover:text-foreground focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-60`,
!isCurrentModelInCache()
? 'bg-red-400/10 !text-red-400 hover:bg-red-400/20 hover:text-red-400'
: forceForegroundText
? 'text-foreground'
: isHighlightedCurrentModelActive
? 'text-foreground'
: 'text-muted-foreground',
isOpen ? 'text-foreground' : ''
)}
style="max-width: min(calc(100cqw - 6.5rem), 32rem)"
<DropdownMenu.Root bind:open={isOpen} onOpenChange={handleOpenChange}>
<DropdownMenu.Trigger
disabled={disabled || updating}
onclick={(e) => {
e.preventDefault();
e.stopPropagation();
}}
>
<Package class="h-3.5 w-3.5" />
<button
type="button"
class={cn(
`inline-grid cursor-pointer grid-cols-[1fr_auto_1fr] items-center gap-1.5 rounded-sm bg-muted-foreground/10 px-1.5 py-1 text-xs transition hover:text-foreground focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-60`,
!isCurrentModelInCache()
? 'bg-red-400/10 !text-red-400 hover:bg-red-400/20 hover:text-red-400'
: forceForegroundText
? 'text-foreground'
: isHighlightedCurrentModelActive
? 'text-foreground'
: 'text-muted-foreground',
isOpen ? 'text-foreground' : ''
)}
style="max-width: min(calc(100cqw - 9rem), 20rem)"
disabled={disabled || updating}
>
<Package class="h-3.5 w-3.5" />
<span class="truncate font-medium">
{selectedOption?.model || 'Select model'}
</span>
<TruncatedText
text={selectedOption?.model || 'Select model'}
class="min-w-0 font-medium"
/>
{#if updating}
<Loader2 class="h-3 w-3.5 animate-spin" />
{:else}
<ChevronDown class="h-3 w-3.5" />
{/if}
</Popover.Trigger>
{#if updating}
<Loader2 class="h-3 w-3.5 animate-spin" />
{:else}
<ChevronDown class="h-3 w-3.5" />
{/if}
</button>
</DropdownMenu.Trigger>
<Popover.Content
class="group/popover-content w-96 max-w-[calc(100vw-2rem)] p-0"
<DropdownMenu.Content
align="end"
sideOffset={8}
collisionPadding={16}
class="w-full max-w-[100vw] pt-0 sm:w-max sm:max-w-[calc(100vw-2rem)]"
>
<div class="flex max-h-[50dvh] flex-col overflow-hidden">
<div
class="order-1 shrink-0 border-b p-4 group-data-[side=top]/popover-content:order-2 group-data-[side=top]/popover-content:border-t group-data-[side=top]/popover-content:border-b-0"
>
<SearchInput
id="model-search"
placeholder="Search models..."
bind:value={searchTerm}
bind:ref={searchInputRef}
onClose={() => handleOpenChange(false)}
onKeyDown={handleSearchKeyDown}
/>
</div>
<div
class="models-list order-2 min-h-0 flex-1 overflow-y-auto group-data-[side=top]/popover-content:order-1"
>
<DropdownMenuSearchable
bind:searchValue={searchTerm}
placeholder="Search models..."
onSearchKeyDown={handleSearchKeyDown}
emptyMessage="No models found."
isEmpty={filteredOptions.length === 0 && isCurrentModelInCache()}
>
<div class="models-list">
{#if !isCurrentModelInCache() && currentModel}
<!-- Show unavailable model as first option (disabled) -->
<button
type="button"
class="flex w-full cursor-not-allowed items-center bg-red-400/10 px-4 py-2 text-left text-sm text-red-400"
class="flex w-full cursor-not-allowed items-center bg-red-400/10 p-2 text-left text-sm text-red-400"
role="option"
aria-selected="true"
aria-disabled="true"
disabled
>
<span class="truncate">{selectedOption?.name || currentModel}</span>
<span
class="min-w-0 flex-1 truncate text-left sm:overflow-visible sm:text-clip sm:whitespace-nowrap"
>
{selectedOption?.name || currentModel}
</span>
<span class="ml-2 text-xs whitespace-nowrap opacity-70">(not available)</span>
</button>
<div class="my-1 h-px bg-border"></div>
@ -421,104 +346,78 @@
{@const isLoaded = status === ServerModelStatus.LOADED}
{@const isLoading = status === ServerModelStatus.LOADING}
{@const isSelected = currentModel === option.model || activeId === option.id}
{@const isCompatible = isModelCompatible(option)}
{@const isHighlighted = index === highlightedIndex}
{@const missingModalities = getMissingModalities(option)}
<div
class={cn(
'group flex w-full items-center gap-2 px-4 py-2 text-left text-sm transition focus:outline-none',
isCompatible
? 'cursor-pointer hover:bg-muted focus:bg-muted'
: 'cursor-not-allowed opacity-50',
'group flex w-full items-center gap-2 rounded-sm p-2 text-left text-sm transition focus:outline-none',
'cursor-pointer hover:bg-muted focus:bg-muted',
isSelected || isHighlighted
? 'bg-accent text-accent-foreground'
: isCompatible
? 'hover:bg-accent hover:text-accent-foreground'
: '',
: 'hover:bg-accent hover:text-accent-foreground',
isLoaded ? 'text-popover-foreground' : 'text-muted-foreground'
)}
role="option"
aria-selected={isSelected || isHighlighted}
aria-disabled={!isCompatible}
tabindex={isCompatible ? 0 : -1}
onclick={() => isCompatible && handleSelect(option.id)}
tabindex="0"
onclick={() => handleSelect(option.id)}
onmouseenter={() => (highlightedIndex = index)}
onkeydown={(e) => {
if (isCompatible && (e.key === 'Enter' || e.key === ' ')) {
if (e.key === 'Enter' || e.key === ' ') {
e.preventDefault();
handleSelect(option.id);
}
}}
>
<span class="min-w-0 flex-1 truncate">{option.model}</span>
<span
class="min-w-0 flex-1 truncate text-left sm:overflow-visible sm:pr-2 sm:text-clip sm:whitespace-nowrap"
>
{option.model}
</span>
{#if missingModalities}
<span class="flex shrink-0 items-center gap-1 text-muted-foreground/70">
{#if missingModalities.vision}
<Tooltip.Root>
<Tooltip.Trigger>
<EyeOff class="h-3.5 w-3.5" />
</Tooltip.Trigger>
<Tooltip.Content class="z-[9999]">
<p>No vision support</p>
</Tooltip.Content>
</Tooltip.Root>
{/if}
{#if missingModalities.audio}
<Tooltip.Root>
<Tooltip.Trigger>
<MicOff class="h-3.5 w-3.5" />
</Tooltip.Trigger>
<Tooltip.Content class="z-[9999]">
<p>No audio support</p>
</Tooltip.Content>
</Tooltip.Root>
{/if}
</span>
{/if}
{#if isLoading}
<Tooltip.Root>
<Tooltip.Trigger>
<Loader2 class="h-4 w-4 shrink-0 animate-spin text-muted-foreground" />
</Tooltip.Trigger>
<Tooltip.Content class="z-[9999]">
<p>Loading model...</p>
</Tooltip.Content>
</Tooltip.Root>
{:else if isLoaded}
<Tooltip.Root>
<Tooltip.Trigger>
<button
type="button"
class="relative ml-2 flex h-4 w-4 shrink-0 items-center justify-center"
onclick={(e) => {
e.stopPropagation();
modelsStore.unloadModel(option.model);
}}
>
<span
class="mr-2 h-2 w-2 rounded-full bg-green-500 transition-opacity group-hover:opacity-0"
></span>
<Power
class="absolute mr-2 h-4 w-4 text-red-500 opacity-0 transition-opacity group-hover:opacity-100 hover:text-red-600"
/>
</button>
</Tooltip.Trigger>
<Tooltip.Content class="z-[9999]">
<p>Unload model</p>
</Tooltip.Content>
</Tooltip.Root>
{:else}
<span class="mx-2 h-2 w-2 rounded-full bg-muted-foreground/50"></span>
{/if}
<div class="flex w-6 shrink-0 justify-center">
{#if isLoading}
<Tooltip.Root>
<Tooltip.Trigger>
<Loader2 class="h-4 w-4 animate-spin text-muted-foreground" />
</Tooltip.Trigger>
<Tooltip.Content class="z-[9999]">
<p>Loading model...</p>
</Tooltip.Content>
</Tooltip.Root>
{:else if isLoaded}
<Tooltip.Root>
<Tooltip.Trigger>
<button
type="button"
class="relative flex h-4 w-4 items-center justify-center"
onclick={(e) => {
e.stopPropagation();
modelsStore.unloadModel(option.model);
}}
>
<span
class="h-2 w-2 rounded-full bg-green-500 transition-opacity group-hover:opacity-0"
></span>
<Power
class="absolute h-4 w-4 text-red-500 opacity-0 transition-opacity group-hover:opacity-100 hover:text-red-600"
/>
</button>
</Tooltip.Trigger>
<Tooltip.Content class="z-[9999]">
<p>Unload model</p>
</Tooltip.Content>
</Tooltip.Root>
{:else}
<span class="h-2 w-2 rounded-full bg-muted-foreground/50"></span>
{/if}
</div>
</div>
{/each}
</div>
</div>
</Popover.Content>
</Popover.Root>
</DropdownMenuSearchable>
</DropdownMenu.Content>
</DropdownMenu.Root>
{:else}
<button
class={cn(
@ -538,9 +437,7 @@
>
<Package class="h-3.5 w-3.5" />
<span class="truncate font-medium">
{selectedOption?.model}
</span>
<TruncatedText text={selectedOption?.model || ''} class="min-w-0 font-medium" />
{#if updating}
<Loader2 class="h-3 w-3.5 animate-spin" />

73
tools/server/webui/src/lib/components/app/models/index.ts Normal file
View file

@ -0,0 +1,73 @@
/**
*
* MODELS
*
* Components for model selection and display. Supports two server modes:
* - **Single model mode**: Server runs with one model, selector shows model info
* - **Router mode**: Server runs with multiple models, selector enables switching
*
* Integrates with modelsStore for model data and serverStore for mode detection.
*
*/
/**
* **ModelsSelector** - Model selection dropdown
*
* Dropdown for selecting AI models with status indicators,
* search, and model information display. Adapts UI based on server mode.
*
* **Architecture:**
* - Uses DropdownMenuSearchable for model list
* - Integrates with modelsStore for model options and selection
* - Detects router vs single mode from serverStore
* - Opens DialogModelInformation for model details
*
* **Features:**
* - Searchable model list with keyboard navigation
* - Model status indicators (loading/ready/error/updating)
* - Model capabilities badges (vision, tools, etc.)
* - Current/active model highlighting
* - Model information dialog on info button click
* - Router mode: shows all available models with status
* - Single mode: shows current model name only
* - Loading/updating skeleton states
* - Global selection support for form integration
*
* @example
* ```svelte
* <ModelsSelector
* currentModel={conversation.modelId}
* onModelChange={(id, name) => updateModel(id)}
* useGlobalSelection
* />
* ```
*/
export { default as ModelsSelector } from './ModelsSelector.svelte';
/**
* **ModelBadge** - Model name display badge
*
* Compact badge showing current model name with package icon.
* Only visible in single model mode. Supports tooltip and copy functionality.
*
* **Architecture:**
* - Reads model name from modelsStore or prop
* - Checks server mode from serverStore
* - Uses BadgeInfo for consistent styling
*
* **Features:**
* - Optional copy to clipboard button
* - Optional tooltip with model details
* - Click handler for model info dialog
* - Only renders in model mode (not router)
*
* @example
* ```svelte
* <ModelBadge
* onclick={() => showModelInfo = true}
* showTooltip
* showCopyIcon
* />
* ```
*/
export { default as ModelBadge } from './ModelBadge.svelte';

2
tools/server/webui/src/lib/components/app/server/ServerErrorSplash.svelte
View file

@ -8,7 +8,7 @@
import { serverStore, serverLoading } from '$lib/stores/server.svelte';
import { config, settingsStore } from '$lib/stores/settings.svelte';
import { fade, fly, scale } from 'svelte/transition';
import { KeyboardKey } from '$lib/enums/keyboard';
import { KeyboardKey } from '$lib/enums';
interface Props {
class?: string;

37
tools/server/webui/src/lib/constants/agentic.ts Normal file
View file

@ -0,0 +1,37 @@
// Agentic tool call tag markers
export const AGENTIC_TAGS = {
TOOL_CALL_START: '<<<AGENTIC_TOOL_CALL_START>>>',
TOOL_CALL_END: '<<<AGENTIC_TOOL_CALL_END>>>',
TOOL_NAME_PREFIX: '<<<TOOL_NAME:',
TOOL_ARGS_START: '<<<TOOL_ARGS_START>>>',
TOOL_ARGS_END: '<<<TOOL_ARGS_END>>>',
TAG_SUFFIX: '>>>'
} as const;
export const REASONING_TAGS = {
START: '<<<reasoning_content_start>>>',
END: '<<<reasoning_content_end>>>'
} as const;
// Regex patterns for parsing agentic content
export const AGENTIC_REGEX = {
// Matches completed tool calls (with END marker)
COMPLETED_TOOL_CALL:
/<<<AGENTIC_TOOL_CALL_START>>>\n<<<TOOL_NAME:(.+?)>>>\n<<<TOOL_ARGS_START>>>([\s\S]*?)<<<TOOL_ARGS_END>>>([\s\S]*?)<<<AGENTIC_TOOL_CALL_END>>>/g,
// Matches pending tool call (has NAME and ARGS but no END)
PENDING_TOOL_CALL:
/<<<AGENTIC_TOOL_CALL_START>>>\n<<<TOOL_NAME:(.+?)>>>\n<<<TOOL_ARGS_START>>>([\s\S]*?)<<<TOOL_ARGS_END>>>([\s\S]*)$/,
// Matches partial tool call (has START and NAME, ARGS still streaming)
PARTIAL_WITH_NAME:
/<<<AGENTIC_TOOL_CALL_START>>>\n<<<TOOL_NAME:(.+?)>>>\n<<<TOOL_ARGS_START>>>([\s\S]*)$/,
// Matches early tool call (just START marker)
EARLY_MATCH: /<<<AGENTIC_TOOL_CALL_START>>>([\s\S]*)$/,
// Matches partial marker at end of content
PARTIAL_MARKER: /<<<[A-Za-z_]*$/,
// Matches reasoning content blocks (including tags)
REASONING_BLOCK: /<<<reasoning_content_start>>>[\s\S]*?<<<reasoning_content_end>>>/g,
// Matches an opening reasoning tag and any remaining content (unterminated)
REASONING_OPEN: /<<<reasoning_content_start>>>[\s\S]*$/,
// Matches tool name inside content
TOOL_NAME_EXTRACT: /<<<TOOL_NAME:([^>]+)>>>/
} as const;

2
tools/server/webui/src/lib/constants/attachment-labels.ts Normal file
View file

@ -0,0 +1,2 @@
export const ATTACHMENT_LABEL_FILE = 'File';
export const ATTACHMENT_LABEL_PDF_FILE = 'PDF File';

6
tools/server/webui/src/lib/constants/binary-detection.ts
View file

@ -1,8 +1,4 @@
export interface BinaryDetectionOptions {
prefixLength: number;
suspiciousCharThresholdRatio: number;
maxAbsoluteNullBytes: number;
}
import type { BinaryDetectionOptions } from '$lib/types';
export const DEFAULT_BINARY_DETECTION_OPTIONS: BinaryDetectionOptions = {
prefixLength: 1024 * 10, // Check the first 10KB of the string

42
tools/server/webui/src/lib/constants/cache.ts Normal file
View file

@ -0,0 +1,42 @@
/**
* Cache configuration constants
*/
/**
* Default TTL (Time-To-Live) for cache entries in milliseconds
* @default 5 minutes
*/
export const DEFAULT_CACHE_TTL_MS = 5 * 60 * 1000;
/**
* Default maximum number of entries in a cache
* @default 100
*/
export const DEFAULT_CACHE_MAX_ENTRIES = 100;
/**
* TTL for model props cache in milliseconds
* Props don't change frequently, so we can cache them longer
* @default 10 minutes
*/
export const MODEL_PROPS_CACHE_TTL_MS = 10 * 60 * 1000;
/**
* Maximum number of model props to cache
* @default 50
*/
export const MODEL_PROPS_CACHE_MAX_ENTRIES = 50;
/**
* Maximum number of inactive conversation states to keep in memory
* States for conversations beyond this limit will be cleaned up
* @default 10
*/
export const MAX_INACTIVE_CONVERSATION_STATES = 10;
/**
* Maximum age (in ms) for inactive conversation states before cleanup
* States older than this will be removed during cleanup
* @default 30 minutes
*/
export const INACTIVE_CONVERSATION_STATE_MAX_AGE_MS = 30 * 60 * 1000;

1
tools/server/webui/src/lib/constants/default-context.ts
View file

@ -1 +0,0 @@
export const DEFAULT_CONTEXT = 4096;

6
tools/server/webui/src/lib/constants/input-classes.ts
View file

@ -1,6 +0,0 @@
export const INPUT_CLASSES = `
bg-muted/70 dark:bg-muted/85
border border-border/30 focus-within:border-border dark:border-border/20 dark:focus-within:border-border
outline-none
text-foreground
`;

26
tools/server/webui/src/lib/constants/settings-config.ts
View file

@ -1,13 +1,16 @@
import { ColorMode } from '$lib/enums/ui';
import { Monitor, Moon, Sun } from '@lucide/svelte';
export const SETTING_CONFIG_DEFAULT: Record<string, string | number | boolean> = {
// Note: in order not to introduce breaking changes, please keep the same data type (number, string, etc) if you want to change the default value. Do not use null or undefined for default value.
// Do not use nested objects, keep it single level. Prefix the key if you need to group them.
apiKey: '',
systemMessage: '',
showSystemMessage: true,
theme: 'system',
theme: ColorMode.SYSTEM,
showThoughtInProgress: false,
showToolCalls: false,
disableReasoningFormat: false,
disableReasoningParsing: false,
showRawOutputSwitch: false,
keepStatsVisible: false,
showMessageStats: true,
askForTitleConfirmation: false,
@ -19,6 +22,7 @@ export const SETTING_CONFIG_DEFAULT: Record<string, string | number | boolean> =
alwaysShowSidebarOnDesktop: false,
autoShowSidebarOnNewChat: true,
autoMicOnEmpty: false,
fullHeightCodeBlocks: false,
// make sure these default values are in sync with `common.h`
samplers: 'top_k;typ_p;top_p;min_p;temperature',
backend_sampling: false,
@ -90,10 +94,10 @@ export const SETTING_CONFIG_INFO: Record<string, string> = {
max_tokens: 'The maximum number of token per output. Use -1 for infinite (no limit).',
custom: 'Custom JSON parameters to send to the API. Must be valid JSON format.',
showThoughtInProgress: 'Expand thought process by default when generating messages.',
showToolCalls:
'Display tool call labels and payloads from Harmony-compatible delta.tool_calls data below assistant messages.',
disableReasoningFormat:
'Show raw LLM output without backend parsing and frontend Markdown rendering to inspect streaming across different models.',
disableReasoningParsing:
'Send reasoning_format=none to prevent server-side extraction of reasoning tokens into separate field',
showRawOutputSwitch:
'Show toggle button to display messages as plain text instead of Markdown-formatted content',
keepStatsVisible: 'Keep processing statistics visible after generation finishes.',
showMessageStats:
'Display generation statistics (tokens/second, token count, duration) below each assistant message.',
@ -110,8 +114,16 @@ export const SETTING_CONFIG_INFO: Record<string, string> = {
'Automatically show sidebar when starting a new chat. Disable to keep the sidebar hidden until you click on it.',
autoMicOnEmpty:
'Automatically show microphone button instead of send button when textarea is empty for models with audio modality support.',
fullHeightCodeBlocks:
'Always display code blocks at their full natural height, overriding any height limits.',
pyInterpreterEnabled:
'Enable Python interpreter using Pyodide. Allows running Python code in markdown code blocks.',
enableContinueGeneration:
'Enable "Continue" button for assistant messages. Currently works only with non-reasoning models.'
};
export const SETTINGS_COLOR_MODES_CONFIG = [
{ value: ColorMode.SYSTEM, label: 'System', icon: Monitor },
{ value: ColorMode.LIGHT, label: 'Light', icon: Sun },
{ value: ColorMode.DARK, label: 'Dark', icon: Moon }
];

Some files were not shown because too many files have changed in this diff Show more