thek0tyara/llama-cpp-turboquant
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

llama: use FA + max. GPU layers by default (#15434)

Browse source 
* llama: use max. GPU layers by default, auto -fa

* ggml-backend: abort instead of segfault
This commit is contained in:
Johannes Gäßler 2025-08-30 16:32:10 +02:00 • committed by GitHub
parent 38ad381f9f
commit e81b8e4b7f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 235 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

6
scripts/server-bench.py
View file

@ -151,12 +151,6 @@ def benchmark(
if os.environ.get("LLAMA_ARG_N_PARALLEL") is None:
logger.info("LLAMA_ARG_N_PARALLEL not explicitly set, using 32")
os.environ["LLAMA_ARG_N_PARALLEL"] = "32"
if not external_server and os.environ.get("LLAMA_ARG_N_GPU_LAYERS") is None:
logger.info("LLAMA_ARG_N_GPU_LAYERS not explicitly set, using 999")
os.environ["LLAMA_ARG_N_GPU_LAYERS"] = "999"
if not external_server and os.environ.get("LLAMA_ARG_FLASH_ATTN") is None:
logger.info("LLAMA_ARG_FLASH_ATTN not explicitly set, using 'true'")
os.environ["LLAMA_ARG_FLASH_ATTN"] = "true"
parallel: int = int(os.environ.get("LLAMA_ARG_N_PARALLEL")) # type: ignore
prompts: Union[None, list[str], list[list[int]]] = get_prompts_text(prompt_source, n_prompts)

2
scripts/tool_bench.py
View file

@ -323,7 +323,7 @@ def run(
server.jinja = True
server.ctk = ctk
server.ctv = ctv
server.fa = fa
server.fa = "on" if fa else "off"
server.n_predict = n_predict
server.model_hf_repo = hf
server.model_hf_file = None