thek0tyara/llama-cpp-turboquant
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

common : gracefully handle incomplete output (#20191)

Browse source 
* common : handle incomplete UTF-8 at end of input in PEG parser

* cont : if reached end prematurely, emit needs_more_input to propagate partial output

* cont: refactor peg parse context to add lenient flag

* cont : remove partial flag, keep lenient flag
This commit is contained in:
Aldehir Rojas 2026-03-08 11:17:02 -05:00 • committed by GitHub
parent 9b24886f78
commit 451ef08432
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 139 additions and 125 deletions
Download patch file Download diff file Expand all files Collapse all files

6
tests/peg-parser/test-python-dict-parser.cpp
View file

@ -85,7 +85,7 @@ void test_python_dict_parser(testing &t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
std::string input = "{'name': 'test', 'value': ";
common_peg_parse_context ctx(input, true);
common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
@ -97,7 +97,7 @@ void test_python_dict_parser(testing &t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
std::string input = "{'name': 'test";
common_peg_parse_context ctx(input, true);
common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
@ -229,7 +229,7 @@ void test_python_dict_parser(testing &t) {
t.test("incomplete string", [&](testing &t) {
std::string input = "'hello";
common_peg_parse_context ctx(input, true);
common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_true("need_more_input", result.need_more_input());