thek0tyara/llama-cpp-turboquant
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

Autoparser - complete refactoring of parser architecture (#18675)

Browse source 
* Autoparser - full single commit squish

* Final pre-merge changes: minor fixes, Kimi 2.5 model parser
This commit is contained in:
Piotr Wilkin (ilintar) 2026-03-06 21:01:00 +01:00 • committed by GitHub
parent 34df42f7be
commit 566059a26b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
63 changed files with 12967 additions and 10071 deletions
Download patch file Download diff file Expand all files Collapse all files

9
common/unicode.h
View file

@ -2,6 +2,8 @@
#include <cstdint>
#include <string_view>
#include <vector>
#include <string>
// UTF-8 parsing utilities for streaming-aware unicode support
@ -16,7 +18,10 @@ struct utf8_parse_result {
// Determine the expected length of a UTF-8 sequence from its first byte
// Returns 0 for invalid first bytes
size_t utf8_sequence_length(unsigned char first_byte);
size_t common_utf8_sequence_length(unsigned char first_byte);
// Parse a single UTF-8 codepoint from input
utf8_parse_result parse_utf8_codepoint(std::string_view input, size_t offset);
utf8_parse_result common_parse_utf8_codepoint(std::string_view input, size_t offset);
std::string common_unicode_cpts_to_utf8(const std::vector<uint32_t> & cps);
std::string common_unicode_cpt_to_utf8(uint32_t cpt);