thek0tyara/llama-cpp-turboquant
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

common: consolidate PEG string parsers (#20263)

Browse source 
* common : consolidate PEG string parsers
* cont : fix json_string_content()
This commit is contained in:
Aldehir Rojas 2026-03-09 18:29:21 -05:00 • committed by GitHub
parent 0842b9b465
commit c96f608d98
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 142 additions and 159 deletions
Download patch file Download diff file Expand all files Collapse all files

2
tests/peg-parser/test-python-dict-parser.cpp
View file

@ -197,7 +197,7 @@ void test_python_dict_parser(testing &t) {
// Test single-quoted string content parser directly
t.test("single-quoted string content parser", [](testing &t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
return p.sequence({ p.literal("'"), p.single_quoted_string_content(), p.literal("'"), p.space() });
return p.sequence({ p.literal("'"), p.string_content('\''), p.literal("'"), p.space() });
});
t.test("simple string", [&](testing &t) {

8
tests/peg-parser/test-unicode.cpp
View file

@ -327,7 +327,7 @@ void test_unicode(testing &t) {
t.test(test_name, [&](testing &t) {
auto parser = build_peg_parser([](common_peg_parser_builder& p) {
return p.sequence({p.json_string_content(), p.literal("\"")});
return p.sequence({p.string_content('"'), p.literal("\"")});
});
common_peg_parse_context ctx(tc.input);
@ -364,7 +364,7 @@ void test_unicode(testing &t) {
t.test(test_name, [&](testing &t) {
auto parser = build_peg_parser([](common_peg_parser_builder& p) {
return p.json_string_content();
return p.string_content('"');
});
common_peg_parse_context ctx(tc.input, COMMON_PEG_PARSE_FLAG_LENIENT);
@ -398,7 +398,7 @@ void test_unicode(testing &t) {
t.test(test_name, [&](testing &t) {
auto parser = build_peg_parser([](common_peg_parser_builder& p) {
return p.json_string_content();
return p.string_content('"');
});
common_peg_parse_context ctx(tc.input);
@ -427,7 +427,7 @@ void test_unicode(testing &t) {
t.test(test_name, [&](testing &t) {
auto parser = build_peg_parser([](common_peg_parser_builder& p) {
return p.sequence({p.json_string_content(), p.literal("\"")});
return p.sequence({p.string_content('"'), p.literal("\"")});
});
common_peg_parse_context ctx(tc.input);

4
tests/test-chat-peg-parser.cpp
View file

@ -597,9 +597,9 @@ void test_command7_parser_compare(testing & t) {
auto response = "<|START_RESPONSE|>" << p.content(p.until("<|END_RESPONSE|>")) << "<|END_RESPONSE|>";
auto tool_call_id = p.atomic("\"tool_call_id\"" << (":" << ("\"" + p.tool_id(p.json_string_content()) + "\"")));
auto tool_call_id = p.atomic("\"tool_call_id\"" << (":" << ("\"" + p.tool_id(p.string_content('"')) + "\"")));
auto tool_call_name =
p.atomic("\"tool_name\"" << (":" << ("\"" + p.tool_name(p.json_string_content()) + "\"")));
p.atomic("\"tool_name\"" << (":" << ("\"" + p.tool_name(p.string_content('"')) + "\"")));
auto tool_call_args = "\"parameters\"" << (":" << p.tool_args(p.json()));
auto tool_call_fields = p.rule("tool-call-fields", tool_call_id | tool_call_name | tool_call_args);