mirrors/llama.cpp
0
0 Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-10-28 17:56:13 +08:00
Code Activity
Files
rpc_tensor_loading
llama.cpp/common/jinja/value.cpp
T
Xuan-Son NguyenAlde RojasPascal
 48c9f08b7b common, grammar: add recursive depth limit, make it a requirement (#31153) 
* common, grammar: add recursive depth limit, make it a requirement

* catch more recursive functions

* common : tidy up peg parser changes

* grammar : reject candidates in a loop, without a depth limit

The depth limit in llama_grammar_reject_candidates_for_stack counted
the code points of a vocab token, so any token of 256 chars or more
was always rejected under a grammar, such as the 512-space token of
GLM-4.7-Flash or the long space tokens of Laguna-S-2.1.

The rejection now walks the code points with its own stack of levels
instead of recursing: there is no limit and no stack risk, whatever
the length of a token from the GGUF, and the masks match master.

* grammar : track rejected candidates as indices into the input

At a given code point a candidate only differs from its input by the
position of its code point pointer, so the levels now carry indices
into the input candidates instead of copies. The first level needs no
special case and the rejects map back to the input directly.

Grammar sampling gets faster than master on GLM-4.7, Laguna, Qwen3.6
and gemma-4 with the json, list and spaces grammars, same masks.

* nit : shorten the comment of llama_grammar_reject_step

* common : add depth_guard for schema walks

* grammar : move GRAMMAR_RECURSIVE_LIMIT into the .cpp

* common : reduce recursion limit for peg parser

* improve

---------

Co-authored-by: Alde Rojas <hello@alde.dev>
Co-authored-by: Pascal <admin@serveurperso.com>
2026-10-25 23:53:08 +01:00

1752 lines
71 KiB
C++
Raw Permalink Blame History

#include "runtime.h"
#include "unicode.h"
#include "value.h"
// for converting from JSON to jinja values
#include "json.h"
#include <sstream>
#include <stdexcept>
#include <string>
#include <cctype>
#include <cfloat>
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <vector>
#include <optional>
#include <algorithm>
#define FILENAME "jinja-value"
namespace jinja {
// func_args method implementations
value func_args::get_kwarg(const std::string & key, value default_val) const {
for (const auto & arg : args) {
if (is_val<value_kwarg>(arg)) {
auto * kwarg = cast_val<value_kwarg>(arg);
if (kwarg->key == key) {
return kwarg->val;
}
}
}
return default_val;
}
value func_args::get_kwarg_or_pos(const std::string & key, size_t pos) const {
value val = get_kwarg(key, mk_val<value_undefined>());
if (val->is_undefined() && pos < count() && !is_val<value_kwarg>(args[pos])) {
return args[pos];
}
return val;
}
value func_args::get_pos(size_t pos) const {
if (count() > pos) {
return args[pos];
}
throw raised_exception("Function '" + func_name + "' expected at least " + std::to_string(pos + 1) + " arguments, got " + std::to_string(count()));
}
value func_args::get_pos(size_t pos, value default_val) const {
if (count() > pos) {
return args[pos];
}
return default_val;
}
void func_args::push_back(const value & val) {
args.push_back(val);
}
void func_args::push_front(const value & val) {
args.insert(args.begin(), val);
}
const std::vector<value> & func_args::get_args() const {
return args;
}
/**
* Function that mimics Python's array slicing.
*/
template<typename T>
static T slice(const T & array, int64_t start, int64_t stop, int64_t step = 1) {
int64_t len = static_cast<int64_t>(array.size());
int64_t direction = (step > 0) ? 1 : ((step < 0) ? -1 : 0);
int64_t start_val = 0;
int64_t stop_val = 0;
if (direction >= 0) {
start_val = start;
if (start_val < 0) {
start_val = std::max(len + start_val, (int64_t)0);
} else {
start_val = std::min(start_val, len);
}
stop_val = stop;
if (stop_val < 0) {
stop_val = std::max(len + stop_val, (int64_t)0);
} else {
stop_val = std::min(stop_val, len);
}
} else {
start_val = start;
if (start_val < 0) {
start_val = std::max(len + start_val, (int64_t)0);
} else {
start_val = std::min(start_val, len - 1);
}
stop_val = stop;
if (stop_val < -1) {
stop_val = std::max(len + stop_val, (int64_t)-1);
} else {
stop_val = std::min(stop_val, len - 1);
}
}
T result;
if (direction == 0) {
return result;
}
for (int64_t i = start_val; direction * i < direction * stop_val; i += step) {
if (i >= 0 && i < len) {
result.push_back(array[static_cast<size_t>(i)]);
}
}
return result;
}
template<typename T>
static value empty_value_fn(const func_args &) {
if constexpr (std::is_same_v<T, value_int>) {
return mk_val<T>(0);
} else if constexpr (std::is_same_v<T, value_float>) {
return mk_val<T>(0.0);
} else if constexpr (std::is_same_v<T, value_bool>) {
return mk_val<T>(false);
} else {
return mk_val<T>();
}
}
template<typename T>
static value test_type_fn(const func_args & args) {
args.ensure_count(1);
bool is_type = is_val<T>(args.get_pos(0));
JJ_DEBUG("test_type_fn: type=%s result=%d", typeid(T).name(), is_type ? 1 : 0);
return mk_val<value_bool>(is_type);
}
template<typename T, typename U>
static value test_type_fn(const func_args & args) {
args.ensure_count(1);
bool is_type = is_val<T>(args.get_pos(0)) || is_val<U>(args.get_pos(0));
JJ_DEBUG("test_type_fn: type=%s or %s result=%d", typeid(T).name(), typeid(U).name(), is_type ? 1 : 0);
return mk_val<value_bool>(is_type);
}
template<typename T, typename U, typename V>
static value test_type_fn(const func_args & args) {
args.ensure_count(1);
bool is_type = is_val<T>(args.get_pos(0)) || is_val<U>(args.get_pos(0)) || is_val<V>(args.get_pos(0));
JJ_DEBUG("test_type_fn: type=%s, %s or %s result=%d", typeid(T).name(), typeid(U).name(), typeid(V).name(), is_type ? 1 : 0);
return mk_val<value_bool>(is_type);
}
template<typename T, typename U, typename V, typename W>
static value test_type_fn(const func_args & args) {
args.ensure_count(1);
bool is_type = is_val<T>(args.get_pos(0)) || is_val<U>(args.get_pos(0)) || is_val<V>(args.get_pos(0)) || is_val<W>(args.get_pos(0));
JJ_DEBUG("test_type_fn: type=%s, %s, %s or %s result=%d", typeid(T).name(), typeid(U).name(), typeid(V).name(), typeid(W).name(), is_type ? 1 : 0);
return mk_val<value_bool>(is_type);
}
template<value_compare_op op>
static value test_compare_fn(const func_args & args) {
args.ensure_count(2, 2);
return mk_val<value_bool>(value_compare(args.get_pos(0), args.get_pos(1), op));
}
static void append_codepoint_as_ascii_json_escape(std::string & out, uint32_t codepoint) {
auto append_u16 = [&out](uint32_t value) {
char buf[8];
snprintf(buf, sizeof(buf), "\\u%04x", static_cast<unsigned int>(value));
out += buf;
};
if (codepoint <= 0xFFFF) {
append_u16(codepoint);
return;
}
codepoint -= 0x10000;
append_u16(0xD800 + ((codepoint >> 10) & 0x3FF));
append_u16(0xDC00 + (codepoint & 0x3FF));
}
static std::string json_ensure_ascii_preserving_format(const std::string & json_str) {
std::string output;
output.reserve(json_str.size());
bool in_string = false;
bool escaped = false;
for (size_t pos = 0; pos < json_str.size();) {
const char ch = json_str[pos];
if (!in_string) {
output.push_back(ch);
if (ch == '"') {
in_string = true;
}
++pos;
continue;
}
if (escaped) {
output.push_back(ch);
escaped = false;
++pos;
continue;
}
if (ch == '\\') {
output.push_back(ch);
escaped = true;
++pos;
continue;
}
if (ch == '"') {
output.push_back(ch);
in_string = false;
++pos;
continue;
}
const unsigned char uch = static_cast<unsigned char>(ch);
if (uch < 0x80) {
output.push_back(ch);
++pos;
continue;
}
auto parsed = common_parse_utf8_codepoint(json_str, pos);
if (parsed.status != utf8_parse_result::SUCCESS) {
output += "\\ufffd";
++pos;
continue;
}
append_codepoint_as_ascii_json_escape(output, parsed.codepoint);
pos += parsed.bytes_consumed;
}
return output;
}
static std::string float_repr(double val);
static value tojson(const func_args & args) {
args.ensure_count(1, 5);
value val_ascii = args.get_kwarg_or_pos("ensure_ascii", 1);
value val_indent = args.get_kwarg_or_pos("indent", 2);
value val_separators = args.get_kwarg_or_pos("separators", 3);
value val_sort = args.get_kwarg_or_pos("sort_keys", 4);
int indent = -1;
if (args.ctx.is_get_stats) {
// mark as used (recursively) for stats
auto val_input = args.get_pos(0);
value_t::stats_t::mark_used(const_cast<value&>(val_input), true);
}
if (is_val<value_int>(val_indent)) {
indent = static_cast<int>(val_indent->as_int());
}
if (val_sort->as_bool()) { // undefined == false
throw not_implemented_exception("tojson sort_keys=true not implemented");
}
const bool ensure_ascii = val_ascii->as_bool(); // undefined == false
auto separators = (is_val<value_array>(val_separators) ? val_separators : mk_val<value_array>())->as_array();
std::string item_sep = separators.size() > 0 ? separators[0]->as_string().str() : (indent < 0 ? ", " : ",");
std::string key_sep = separators.size() > 1 ? separators[1]->as_string().str() : ": ";
std::string json_str = value_to_json(args.get_pos(0), indent, item_sep, key_sep);
if (ensure_ascii) {
json_str = json_ensure_ascii_preserving_format(json_str);
}
return mk_val<value_string>(json_str);
}
static value & get_attribute(const value & val, const value & attr, value & default_val) {
if (!attr->is_undefined()) {
if (is_val<value_array>(val)) {
value idx = attr;
if (is_val<value_string>(attr)) {
const std::string s = attr->as_string().str();
if (!s.empty() && std::all_of(s.begin(), s.end(), [](unsigned char c) { return std::isdigit(c); })) {
try {
idx = mk_val<value_int>(std::stoll(s));
} catch (...) {
idx = mk_val<value_undefined>();
}
}
}
return val->at(idx, default_val);
} else if (is_val<value_object>(val)) {
return val->at(attr, default_val);
}
}
return default_val;
}
template<bool is_reject, bool lookup_attr>
static value select_or_reject(const func_args & args) {
// example: array | select("odd") -> test_is_odd(item)
// example: array | selectattr("status", "equalto", "active") -> test_is_equalto(item.status, "active")
constexpr size_t test_pos = lookup_attr ? 2 : 1;
args.ensure_count(test_pos);
args.ensure_val<value_array>(args.get_pos(0));
value attr;
if constexpr (lookup_attr) {
attr = args.get_pos(1);
args.ensure_val<value_string>(attr);
}
auto & builtins = global_builtins();
auto test_it = builtins.end();
if (args.count() > test_pos) {
args.ensure_val<value_string>(args.get_pos(test_pos));
std::string test_name = args.get_pos(test_pos)->as_string().str();
test_it = builtins.find("test_is_" + test_name);
if (test_it == builtins.end()) {
throw raised_exception(args.func_name + ": unknown test '" + test_name + "'");
}
}
auto arr = args.get_pos(0)->as_array();
auto out = mk_val<value_array>();
value val_default = mk_val<value_undefined>();
for (const auto & item : arr) {
value val = item;
if constexpr (lookup_attr) {
val = get_attribute(item, attr, val_default);
}
bool is_selected;
if (test_it == builtins.end()) {
is_selected = val->as_bool();
} else {
func_args test_args(args.ctx);
test_args.push_back(val);
for (size_t i = test_pos + 1; i < args.count(); i++) {
test_args.push_back(args.get_pos(i));
}
is_selected = test_it->second(test_args)->as_bool();
}
if constexpr (is_reject) is_selected = !is_selected;
if (is_selected) out->push_back(item);
}
return out;
}
static value default_value(const func_args & args) {
args.ensure_count(2, 3);
value val_check = args.get_kwarg_or_pos("boolean", 2);
bool check_bool = val_check->as_bool(); // undefined == false
bool no_value = check_bool
? (!args.get_pos(0)->as_bool())
: (args.get_pos(0)->is_undefined() || args.get_pos(0)->is_none());
return no_value ? args.get_pos(1) : args.get_pos(0);
}
static value toobject(const func_args & args) {
auto out = mk_val<value_object>();
value iter = args.get_pos(0, mk_val<value_undefined>());
bool iter_first = false;
if (is_val<value_array>(iter)) {
iter_first = true;
for (const auto & it : iter->as_array()) {
if (is_val<value_array>(it) && it->as_array().size() == 2) {
auto tuple = it->as_array();
auto key = tuple[0];
auto val = tuple[1];
JJ_DEBUG("namespace/dict: adding key '%s'", key->as_string().str().c_str());
out->insert(key, val);
} else {
throw raised_exception("namespace/dict() iterable argument must consist of tuples, not " + it->type());
}
}
} else if (is_val<value_object>(iter)) {
iter_first = true;
for (const auto & pair : iter->as_ordered_object()) {
JJ_DEBUG("namespace/dict: adding key '%s'", pair.first->as_string().str().c_str());
out->insert(pair.first, pair.second);
}
}
for (const auto & arg : args.get_args()) {
if (is_val<value_kwarg>(arg)) {
auto kwarg = cast_val<value_kwarg>(arg);
JJ_DEBUG("namespace/dict: adding key '%s'", kwarg->key.c_str());
out->insert(kwarg->key, kwarg->val);
} else if (!iter_first) {
throw raised_exception("namespace/dict() arguments must be kwargs, dict and/or iterable of tuples, not " + arg->type());
}
iter_first = false;
}
return out;
}
const func_builtins & global_builtins() {
static const func_builtins builtins = {
{"raise_exception", [](const func_args & args) -> value {
args.ensure_vals<value_string>();
std::string msg = args.get_pos(0)->as_string().str();
throw raised_exception("Jinja Exception: " + msg);
}},
{"dict", toobject},
{"namespace", [](const func_args & args) -> value {
// unlike a dict, a namespace has no methods: ns.items is the attribute, as in jinja2
value out = toobject(args);
cast_val<value_object>(out)->has_builtins = false;
return out;
}},
{"strftime_now", [](const func_args & args) -> value {
args.ensure_vals<value_string>();
std::string format = args.get_pos(0)->as_string().str();
// get current time
// TODO: make sure this is the same behavior as Python's strftime
char buf[100];
if (std::strftime(buf, sizeof(buf), format.c_str(), std::localtime(&args.ctx.current_time))) {
return mk_val<value_string>(std::string(buf));
} else {
throw raised_exception("strftime_now: failed to format time");
}
}},
{"range", [](const func_args & args) -> value {
args.ensure_count(1, 3);
args.ensure_vals<value_int, value_int, value_int>(true, false, false);
auto arg0 = args.get_pos(0);
auto arg1 = args.get_pos(1, mk_val<value_undefined>());
auto arg2 = args.get_pos(2, mk_val<value_undefined>());
int64_t start, stop, step;
if (args.count() == 1) {
start = 0;
stop = arg0->as_int();
step = 1;
} else if (args.count() == 2) {
start = arg0->as_int();
stop = arg1->as_int();
step = 1;
} else {
start = arg0->as_int();
stop = arg1->as_int();
step = arg2->as_int();
}
auto out = mk_val<value_array>();
if (step == 0) {
throw raised_exception("range() step argument must not be zero");
}
if (step > 0) {
for (int64_t i = start; i < stop; i += step) {
out->push_back(mk_val<value_int>(i));
}
} else {
for (int64_t i = start; i > stop; i += step) {
out->push_back(mk_val<value_int>(i));
}
}
return out;
}},
{"tojson", tojson},
// tests
{"test_is_boolean", test_type_fn<value_bool>},
{"test_is_callable", test_type_fn<value_func>},
{"test_is_odd", [](const func_args & args) -> value {
args.ensure_vals<value_int>();
int64_t val = args.get_pos(0)->as_int();
return mk_val<value_bool>(val % 2 != 0);
}},
{"test_is_even", [](const func_args & args) -> value {
args.ensure_vals<value_int>();
int64_t val = args.get_pos(0)->as_int();
return mk_val<value_bool>(val % 2 == 0);
}},
{"test_is_false", [](const func_args & args) -> value {
args.ensure_count(1);
bool val = is_val<value_bool>(args.get_pos(0)) && !args.get_pos(0)->as_bool();
return mk_val<value_bool>(val);
}},
{"test_is_true", [](const func_args & args) -> value {
args.ensure_count(1);
bool val = is_val<value_bool>(args.get_pos(0)) && args.get_pos(0)->as_bool();
return mk_val<value_bool>(val);
}},
{"test_is_divisibleby", [](const func_args & args) -> value {
args.ensure_vals<value_int, value_int>();
bool res = args.get_pos(0)->val_int % args.get_pos(1)->val_int == 0;
return mk_val<value_bool>(res);
}},
{"test_is_string", test_type_fn<value_string>},
{"test_is_integer", test_type_fn<value_int>},
{"test_is_float", test_type_fn<value_float>},
{"test_is_number", test_type_fn<value_int, value_float>},
{"test_is_iterable", test_type_fn<value_object, value_array, value_string, value_undefined>},
{"test_is_sequence", test_type_fn<value_object, value_array, value_string, value_undefined>},
{"test_is_mapping", test_type_fn<value_object>},
{"test_is_lower", [](const func_args & args) -> value {
args.ensure_vals<value_string>();
return mk_val<value_bool>(args.get_pos(0)->val_str.is_lowercase());
}},
{"test_is_upper", [](const func_args & args) -> value {
args.ensure_vals<value_string>();
return mk_val<value_bool>(args.get_pos(0)->val_str.is_uppercase());
}},
{"test_is_none", test_type_fn<value_none>},
{"test_is_defined", [](const func_args & args) -> value {
args.ensure_count(1);
bool res = !args.get_pos(0)->is_undefined();
JJ_DEBUG("test_is_defined: result=%d", res ? 1 : 0);
return mk_val<value_bool>(res);
}},
{"test_is_undefined", test_type_fn<value_undefined>},
{"test_is_eq", test_compare_fn<value_compare_op::eq>},
{"test_is_equalto", test_compare_fn<value_compare_op::eq>},
{"test_is_ge", test_compare_fn<value_compare_op::ge>},
{"test_is_gt", test_compare_fn<value_compare_op::gt>},
{"test_is_greaterthan", test_compare_fn<value_compare_op::gt>},
{"test_is_lt", test_compare_fn<value_compare_op::lt>},
{"test_is_lessthan", test_compare_fn<value_compare_op::lt>},
{"test_is_ne", test_compare_fn<value_compare_op::ne>},
{"test_is_in", [](const func_args & args) -> value {
args.ensure_count(2);
auto needle = args.get_pos(0);
auto haystack = args.get_pos(1);
if (is_val<value_undefined>(haystack)) {
return mk_val<value_bool>(false);
}
if (is_val<value_array>(haystack)) {
for (const auto & item : haystack->as_array()) {
if (*needle == *item) {
return mk_val<value_bool>(true);
}
}
return mk_val<value_bool>(false);
}
if (is_val<value_string>(haystack)) {
if (!is_val<value_string>(needle)) {
throw raised_exception("'in' test expects args[1] as string when args[0] is string, got args[1] as " + needle->type());
}
return mk_val<value_bool>(
haystack->as_string().str().find(needle->as_string().str()) != std::string::npos);
}
if (is_val<value_object>(haystack)) {
return mk_val<value_bool>(haystack->has_key(needle));
}
throw raised_exception("'in' test expects iterable as first argument, got " + haystack->type());
}},
{"test_is_test", [](const func_args & args) -> value {
args.ensure_vals<value_string>();
auto & builtins = global_builtins();
std::string test_name = args.get_pos(0)->val_str.str();
auto it = builtins.find("test_is_" + test_name);
bool res = it != builtins.end();
return mk_val<value_bool>(res);
}},
{"test_is_sameas", [](const func_args & args) -> value {
// Check if an object points to the same memory address as another object
args.ensure_count(2);
auto a = args.get_pos(0);
auto b = args.get_pos(1);
bool res = false;
if (!is_val<value_undefined>(a) && !is_val<value_undefined>(b)) {
if (is_val<value_none>(a) && is_val<value_none>(b)) {
res = true;
} else if (is_val<value_bool>(a) && is_val<value_bool>(b)) {
if (a->as_bool() == b->as_bool()) {
res = true;
}
} else if (is_val<value_int>(a) && is_val<value_int>(b)) {
const int64_t x = a->as_int();
// Allow comparison within small-int cache range
if (x >= -5 && x <= 256 && x == b->as_int()) {
res = true;
}
} else if (a == b) {
res = true;
}
}
return mk_val<value_bool>(res);
}},
{"test_is_escaped", [](const func_args & args) -> value {
(void)args;
throw not_implemented_exception("escaped test not implemented");
}},
{"test_is_filter", [](const func_args & args) -> value {
(void)args;
throw not_implemented_exception("filter test not implemented");
}},
};
return builtins;
}
const func_builtins & value_int_t::get_builtins() const {
static const func_builtins builtins = {
{"default", default_value},
{"abs", [](const func_args & args) -> value {
args.ensure_vals<value_int>();
int64_t val = args.get_pos(0)->as_int();
return mk_val<value_int>(val < 0 ? -val : val);
}},
{"int", [](const func_args & args) -> value {
args.ensure_vals<value_int>();
return mk_val<value_int>(args.get_pos(0)->as_int());
}},
{"float", [](const func_args & args) -> value {
args.ensure_vals<value_int>();
double val = static_cast<double>(args.get_pos(0)->as_int());
return mk_val<value_float>(val);
}},
{"safe", tojson},
{"string", tojson},
{"tojson", tojson},
};
return builtins;
}
const func_builtins & value_float_t::get_builtins() const {
// Python's str() of a float, nan and inf stay as they are (tojson writes NaN and Infinity)
static const func_handler tostring = [](const func_args & args) -> value {
args.ensure_vals<value_float>();
return mk_val<value_string>(float_repr(args.get_pos(0)->as_float()));
};
static const func_builtins builtins = {
{"default", default_value},
{"abs", [](const func_args & args) -> value {
args.ensure_vals<value_float>();
double val = args.get_pos(0)->as_float();
return mk_val<value_float>(val < 0.0 ? -val : val);
}},
{"int", [](const func_args & args) -> value {
args.ensure_vals<value_float>();
int64_t val = static_cast<int64_t>(args.get_pos(0)->as_float());
return mk_val<value_int>(val);
}},
{"float", [](const func_args & args) -> value {
args.ensure_vals<value_float>();
return mk_val<value_float>(args.get_pos(0)->as_float());
}},
{"safe", tostring},
{"string", tostring},
{"tojson", tojson},
};
return builtins;
}
static bool string_startswith(const std::string & str, const std::string & prefix) {
if (str.length() < prefix.length()) return false;
return str.compare(0, prefix.length(), prefix) == 0;
}
static bool string_endswith(const std::string & str, const std::string & suffix) {
if (str.length() < suffix.length()) return false;
return str.compare(str.length() - suffix.length(), suffix.length(), suffix) == 0;
}
[[noreturn]] static value string_join_not_implemented(const func_args &) {
throw not_implemented_exception("String join builtin not implemented");
}
const func_builtins & value_string_t::get_builtins() const {
static const func_builtins builtins = {
{"default", default_value},
{"upper", [](const func_args & args) -> value {
args.ensure_vals<value_string>();
jinja::string str = args.get_pos(0)->as_string().uppercase();
return mk_val<value_string>(str);
}},
{"lower", [](const func_args & args) -> value {
args.ensure_vals<value_string>();
jinja::string str = args.get_pos(0)->as_string().lowercase();
return mk_val<value_string>(str);
}},
{"strip", [](const func_args & args) -> value {
value val_input = args.get_pos(0);
if (!is_val<value_string>(val_input)) {
throw raised_exception("strip() first argument must be a string");
}
value val_chars = args.get_kwarg_or_pos("chars", 1);
if (val_chars->is_undefined()) {
return mk_val<value_string>(args.get_pos(0)->as_string().strip(true, true));
} else {
return mk_val<value_string>(args.get_pos(0)->as_string().strip(true, true, val_chars->as_string().str()));
}
}},
{"rstrip", [](const func_args & args) -> value {
args.ensure_vals<value_string>();
value val_chars = args.get_kwarg_or_pos("chars", 1);
if (val_chars->is_undefined()) {
return mk_val<value_string>(args.get_pos(0)->as_string().strip(false, true));
} else {
return mk_val<value_string>(args.get_pos(0)->as_string().strip(false, true, val_chars->as_string().str()));
}
}},
{"lstrip", [](const func_args & args) -> value {
args.ensure_vals<value_string>();
value val_chars = args.get_kwarg_or_pos("chars", 1);
if (val_chars->is_undefined()) {
return mk_val<value_string>(args.get_pos(0)->as_string().strip(true, false));
} else {
return mk_val<value_string>(args.get_pos(0)->as_string().strip(true, false, val_chars->as_string().str()));
}
}},
{"title", [](const func_args & args) -> value {
args.ensure_vals<value_string>();
jinja::string str = args.get_pos(0)->as_string().titlecase();
return mk_val<value_string>(str);
}},
{"capitalize", [](const func_args & args) -> value {
args.ensure_vals<value_string>();
jinja::string str = args.get_pos(0)->as_string().capitalize();
return mk_val<value_string>(str);
}},
{"length", [](const func_args & args) -> value {
args.ensure_vals<value_string>();
jinja::string str = args.get_pos(0)->as_string();
return mk_val<value_int>(str.length());
}},
{"startswith", [](const func_args & args) -> value {
args.ensure_vals<value_string, value_string>();
std::string str = args.get_pos(0)->as_string().str();
std::string prefix = args.get_pos(1)->as_string().str();
return mk_val<value_bool>(string_startswith(str, prefix));
}},
{"endswith", [](const func_args & args) -> value {
args.ensure_vals<value_string, value_string>();
std::string str = args.get_pos(0)->as_string().str();
std::string suffix = args.get_pos(1)->as_string().str();
return mk_val<value_bool>(string_endswith(str, suffix));
}},
{"split", [](const func_args & args) -> value {
args.ensure_count(1, 3);
value val_input = args.get_pos(0);
if (!is_val<value_string>(val_input)) {
throw raised_exception("split() first argument must be a string");
}
std::string str = val_input->as_string().str();
// FIXME: Support non-specified delimiter (split on consecutive (no leading or trailing) whitespace)
std::string delim = (args.count() > 1) ? args.get_pos(1)->as_string().str() : " ";
if (delim.empty()) {
throw raised_exception("empty separator");
}
int64_t maxsplit = (args.count() > 2) ? args.get_pos(2)->as_int() : -1;
auto result = mk_val<value_array>();
size_t pos = 0;
std::string token;
while ((pos = str.find(delim)) != std::string::npos && maxsplit != 0) {
token = str.substr(0, pos);
result->push_back(mk_val<value_string>(token));
str.erase(0, pos + delim.length());
--maxsplit;
}
auto res = mk_val<value_string>(str);
res->val_str.mark_input_based_on(args.get_pos(0)->val_str);
result->push_back(std::move(res));
return result;
}},
{"rsplit", [](const func_args & args) -> value {
args.ensure_count(1, 3);
value val_input = args.get_pos(0);
if (!is_val<value_string>(val_input)) {
throw raised_exception("rsplit() first argument must be a string");
}
std::string str = val_input->as_string().str();
// FIXME: Support non-specified delimiter (split on consecutive (no leading or trailing) whitespace)
std::string delim = (args.count() > 1) ? args.get_pos(1)->as_string().str() : " ";
if (delim.empty()) {
throw raised_exception("empty separator");
}
int64_t maxsplit = (args.count() > 2) ? args.get_pos(2)->as_int() : -1;
auto result = mk_val<value_array>();
size_t pos = 0;
std::string token;
while ((pos = str.rfind(delim)) != std::string::npos && maxsplit != 0) {
token = str.substr(pos + delim.length());
result->push_back(mk_val<value_string>(token));
str.erase(pos);
--maxsplit;
}
auto res = mk_val<value_string>(str);
res->val_str.mark_input_based_on(args.get_pos(0)->val_str);
result->push_back(std::move(res));
result->reverse();
return result;
}},
{"replace", [](const func_args & args) -> value {
args.ensure_vals<value_string, value_string, value_string, value_int>(true, true, true, false);
std::string str = args.get_pos(0)->as_string().str();
std::string old_str = args.get_pos(1)->as_string().str();
std::string new_str = args.get_pos(2)->as_string().str();
int64_t count = args.count() > 3 ? args.get_pos(3)->as_int() : -1;
if (count > 0) {
throw not_implemented_exception("String replace with count argument not implemented");
}
if (old_str != new_str) {
size_t pos = 0;
if (old_str.empty()) {
std::string new_res;
new_res.reserve(str.length() + new_str.length() * (str.length() + 1));
new_res += new_str;
for (const char c : str) {
new_res.push_back(c);
new_res += new_str;
}
str = new_res;
} else {
while ((pos = str.find(old_str, pos)) != std::string::npos) {
str.replace(pos, old_str.length(), new_str);
pos += new_str.length();
}
}
}
auto res = mk_val<value_string>(str);
res->val_str.mark_input_based_on(args.get_pos(0)->val_str);
return res;
}},
{"format", [](const func_args & args) -> value {
value val_input = args.get_pos(0);
if (!is_val<value_string>(val_input)) {
throw raised_exception("format() first argument must be a string");
}
const jinja::string & fmt = val_input->as_string();
const bool fmt_is_input = fmt.all_parts_are_input();
const std::string str = fmt.str();
jinja::string result;
std::string literal;
auto flush_literal = [&]() {
if (!literal.empty()) {
result.parts.push_back({fmt_is_input, literal});
literal.clear();
}
};
size_t arg_idx = 1; // positional args follow the format string
for (size_t i = 0; i < str.size(); ++i) {
if (str[i] != '{') {
literal += str[i];
continue;
}
if (i + 1 >= str.size() || str[i + 1] != '}') {
throw not_implemented_exception("format() only supports simple '{}' placeholders");
}
++i;
flush_literal();
const jinja::string arg_str = args.get_pos(arg_idx++)->as_string();
result.parts.insert(result.parts.end(), arg_str.parts.begin(), arg_str.parts.end());
}
flush_literal();
return mk_val<value_string>(result);
}},
{"int", [](const func_args & args) -> value {
value val_input = args.get_pos(0);
value val_default = args.get_kwarg_or_pos("default", 1);
value val_base = args.get_kwarg_or_pos("base", 2);
const int base = val_base->is_undefined() ? 10 : val_base->as_int();
if (base != 0 && (base < 2 || base > 36)) {
// an out-of-range base makes std::stoi fail fast on the MSVC CRT instead of throwing
throw raised_exception("int() base must be 0 or between 2 and 36");
}
if (is_val<value_string>(val_input) == false) {
throw raised_exception("int() first argument must be a string");
}
std::string str = val_input->as_string().str();
try {
return mk_val<value_int>(std::stoi(str, nullptr, base));
} catch (...) {
return mk_val<value_int>(val_default->is_undefined() ? 0 : val_default->as_int());
}
}},
{"float", [](const func_args & args) -> value {
args.ensure_vals<value_string>();
value val_default = args.get_kwarg_or_pos("default", 1);
std::string str = args.get_pos(0)->as_string().str();
try {
return mk_val<value_float>(std::stod(str));
} catch (...) {
return mk_val<value_float>(val_default->is_undefined() ? 0.0 : val_default->as_float());
}
}},
{"string", [](const func_args & args) -> value {
// no-op
args.ensure_vals<value_string>();
return mk_val<value_string>(args.get_pos(0)->as_string());
}},
{"default", [](const func_args & args) -> value {
value input = args.get_pos(0);
if (!is_val<value_string>(input)) {
throw raised_exception("default() first argument must be a string");
}
value default_val = mk_val<value_string>("");
if (args.count() > 1 && !args.get_pos(1)->is_undefined()) {
default_val = args.get_pos(1);
}
value boolean_val = args.get_kwarg_or_pos("boolean", 2); // undefined == false
if (input->is_undefined() || (boolean_val->as_bool() && !input->as_bool())) {
return default_val;
} else {
return input;
}
}},
{"slice", [](const func_args & args) -> value {
args.ensure_count(1, 4);
args.ensure_vals<value_string, value_int, value_int, value_int>(true, true, false, false);
auto arg0 = args.get_pos(1);
auto arg1 = args.get_pos(2, mk_val<value_undefined>());
auto arg2 = args.get_pos(3, mk_val<value_undefined>());
int64_t start, stop, step;
if (args.count() == 1) {
start = 0;
stop = arg0->as_int();
step = 1;
} else if (args.count() == 2) {
start = arg0->as_int();
stop = arg1->as_int();
step = 1;
} else {
start = arg0->as_int();
stop = arg1->as_int();
step = arg2->as_int();
}
if (step == 0) {
throw raised_exception("slice step cannot be zero");
}
auto input = args.get_pos(0);
auto sliced = slice(input->as_string().str(), start, stop, step);
auto res = mk_val<value_string>(sliced);
res->val_str.mark_input_based_on(input->as_string());
return res;
}},
{"safe", [](const func_args & args) -> value {
// no-op for now
args.ensure_vals<value_string>();
return args.get_pos(0);
}},
{"tojson", tojson},
{"indent", [](const func_args &args) -> value {
args.ensure_count(1, 4);
value val_input = args.get_pos(0);
value val_width = args.get_kwarg_or_pos("width", 1);
const bool first = args.get_kwarg_or_pos("first", 2)->as_bool(); // undefined == false
const bool blank = args.get_kwarg_or_pos("blank", 3)->as_bool(); // undefined == false
if (!is_val<value_string>(val_input)) {
throw raised_exception("indent() first argument must be a string");
}
std::string indent;
if (is_val<value_int>(val_width)) {
indent.assign(val_width->as_int(), ' ');
} else if (is_val<value_string>(val_width)) {
indent = val_width->as_string().str();
} else {
indent = " ";
}
std::string indented;
std::string input = val_input->as_string().str();
std::istringstream iss = std::istringstream(input);
std::string line;
while (std::getline(iss, line)) {
if (!indented.empty()) {
indented.push_back('\n');
}
if ((indented.empty() ? first : (!line.empty() || blank))) {
indented += indent;
}
indented += line;
}
if (!input.empty() && input.back() == '\n') {
indented.push_back('\n');
if (blank) {
indented += indent;
}
}
auto res = mk_val<value_string>(indented);
res->val_str.mark_input_based_on(val_input->as_string());
return res;
}},
{"join", string_join_not_implemented},
};
return builtins;
}
const func_builtins & value_bool_t::get_builtins() const {
static const func_handler tostring = [](const func_args & args) -> value {
args.ensure_vals<value_bool>();
bool val = args.get_pos(0)->as_bool();
return mk_val<value_string>(val ? "True" : "False");
};
static const func_builtins builtins = {
{"default", default_value},
{"int", [](const func_args & args) -> value {
args.ensure_vals<value_bool>();
bool val = args.get_pos(0)->as_bool();
return mk_val<value_int>(val ? 1 : 0);
}},
{"float", [](const func_args & args) -> value {
args.ensure_vals<value_bool>();
bool val = args.get_pos(0)->as_bool();
return mk_val<value_float>(val ? 1.0 : 0.0);
}},
{"safe", tostring},
{"string", tostring},
{"tojson", tojson},
};
return builtins;
}
const func_builtins & value_array_t::get_builtins() const {
static const func_builtins builtins = {
{"default", default_value},
{"list", [](const func_args & args) -> value {
args.ensure_vals<value_array>();
const auto & arr = args.get_pos(0)->as_array();
auto result = mk_val<value_array>();
for (const auto& v : arr) {
result->push_back(v);
}
return result;
}},
{"first", [](const func_args & args) -> value {
args.ensure_vals<value_array>();
const auto & arr = args.get_pos(0)->as_array();
if (arr.empty()) {
return mk_val<value_undefined>();
}
return arr[0];
}},
{"last", [](const func_args & args) -> value {
args.ensure_vals<value_array>();
const auto & arr = args.get_pos(0)->as_array();
if (arr.empty()) {
return mk_val<value_undefined>();
}
return arr[arr.size() - 1];
}},
{"length", [](const func_args & args) -> value {
args.ensure_vals<value_array>();
const auto & arr = args.get_pos(0)->as_array();
return mk_val<value_int>(static_cast<int64_t>(arr.size()));
}},
{"slice", [](const func_args & args) -> value {
args.ensure_count(1, 4);
args.ensure_vals<value_array, value_int, value_int, value_int>(true, true, false, false);
auto val = args.get_pos(0);
auto arg0 = args.get_pos(1);
auto arg1 = args.get_pos(2, mk_val<value_undefined>());
auto arg2 = args.get_pos(3, mk_val<value_undefined>());
int64_t start, stop, step;
if (args.count() == 1) {
start = 0;
stop = arg0->as_int();
step = 1;
} else if (args.count() == 2) {
start = arg0->as_int();
stop = arg1->as_int();
step = 1;
} else {
start = arg0->as_int();
stop = arg1->as_int();
step = arg2->as_int();
}
if (step == 0) {
throw raised_exception("slice step cannot be zero");
}
auto arr = slice(val->as_array(), start, stop, step);
return is_val<value_tuple>(val) ? mk_val<value_tuple>(std::move(arr)) : mk_val<value_array>(std::move(arr));
}},
{"selectattr", select_or_reject<false, true>},
{"select", select_or_reject<false, false>},
{"rejectattr", select_or_reject<true, true>},
{"reject", select_or_reject<true, false>},
{"join", [](const func_args & args) -> value {
args.ensure_count(1, 3);
if (!is_val<value_array>(args.get_pos(0))) {
throw raised_exception("join() first argument must be an array");
}
value val_delim = args.get_kwarg_or_pos("d", 1);
value attribute = args.get_kwarg_or_pos("attribute", 2);
value undef = mk_val<value_undefined>();
const auto & arr = args.get_pos(0)->as_array();
const std::string delim = val_delim->is_undefined() ? "" : val_delim->as_string().str();
std::string result;
for (size_t i = 0; i < arr.size(); ++i) {
value val_arr = arr[i];
if (!attribute->is_undefined()) {
val_arr = get_attribute(val_arr, attribute, undef);
}
if (!is_val<value_string>(val_arr) && !is_val<value_int>(val_arr) && !is_val<value_float>(val_arr)) {
throw raised_exception("join() can only join arrays of strings or numerics");
}
result += val_arr->as_string().str();
if (i < arr.size() - 1) {
result += delim;
}
}
return mk_val<value_string>(result);
}},
{"string", [](const func_args & args) -> value {
args.ensure_vals<value_array>();
if (args.ctx.is_get_stats) {
// mark as used (recursively) for stats
auto val_input = args.get_pos(0);
value_t::stats_t::mark_used(const_cast<value&>(val_input), true);
}
return mk_val<value_string>(args.get_pos(0)->as_string());
}},
{"tojson", tojson},
{"map", [](const func_args & args) -> value {
args.ensure_count(2);
if (!is_val<value_array>(args.get_pos(0))) {
throw raised_exception("map: first argument must be an array");
}
if (!is_val<value_kwarg>(args.get_args().at(1))) {
throw not_implemented_exception("map: filter-mapping not implemented");
}
value val = args.get_pos(0);
value attribute = args.get_kwarg_or_pos("attribute", 1);
value default_val = args.get_kwarg("default", mk_val<value_undefined>());
auto out = mk_val<value_array>();
auto arr = val->as_array();
for (const auto & item : arr) {
value attr_val = get_attribute(item, attribute, default_val);
out->push_back(attr_val);
}
return is_val<value_tuple>(val) ? mk_val<value_tuple>(std::move(out->as_array())) : out;
}},
{"append", [](const func_args & args) -> value {
args.ensure_count(2);
if (!is_val<value_array>(args.get_pos(0))) {
throw raised_exception("append: first argument must be an array");
}
const value_array_t * arr = cast_val<value_array>(args.get_pos(0));
// need to use const_cast here to modify the array
value_array_t * arr_editable = const_cast<value_array_t *>(arr);
arr_editable->push_back(args.get_pos(1));
return args.get_pos(0);
}},
{"pop", [](const func_args & args) -> value {
args.ensure_count(1, 2);
args.ensure_vals<value_array, value_int>(true, false);
int64_t index = args.count() == 2 ? args.get_pos(1)->as_int() : -1;
const value_array_t * arr = cast_val<value_array>(args.get_pos(0));
// need to use const_cast here to modify the array
value_array_t * arr_editable = const_cast<value_array_t *>(arr);
return arr_editable->pop_at(index);
}},
{"sort", [](const func_args & args) -> value {
args.ensure_count(1, 4);
if (!is_val<value_array>(args.get_pos(0))) {
throw raised_exception("sort: first argument must be an array");
}
value val = args.get_pos(0);
value val_reverse = args.get_kwarg_or_pos("reverse", 1);
value val_case = args.get_kwarg_or_pos("case_sensitive", 2);
value attribute = args.get_kwarg_or_pos("attribute", 3);
// FIXME: sorting is currently always case sensitive
//const bool case_sensitive = val_case->as_bool(); // undefined == false
const bool reverse = val_reverse->as_bool(); // undefined == false
value undef = mk_val<value_undefined>();
std::vector<value> arr = val->as_array(); // copy
std::sort(arr.begin(), arr.end(),[&](const value & a, const value & b) {
value val_a = a;
value val_b = b;
if (!attribute->is_undefined()) {
val_a = get_attribute(a, attribute, undef);
val_b = get_attribute(b, attribute, undef);
}
return value_compare(val_a, val_b, reverse ? value_compare_op::gt : value_compare_op::lt);
});
return is_val<value_tuple>(val) ? mk_val<value_tuple>(std::move(arr)) : mk_val<value_array>(std::move(arr));
}},
{"reverse", [](const func_args & args) -> value {
args.ensure_vals<value_array>();
value val = args.get_pos(0);
std::vector<value> arr = val->as_array(); // copy
std::reverse(arr.begin(), arr.end());
return is_val<value_tuple>(val) ? mk_val<value_tuple>(std::move(arr)) : mk_val<value_array>(std::move(arr));
}},
{"min", [](const func_args & args) -> value {
args.ensure_count(1, 4);
args.ensure_vals<value_array>();
value val_case = args.get_kwarg_or_pos("case_sensitive", 1);
value attribute = args.get_kwarg_or_pos("attribute", 2);
// FIXME: min is currently always case sensitive
(void) val_case;
value undef = mk_val<value_undefined>();
const auto & arr = args.get_pos(0)->as_array();
if (arr.empty()) {
return undef;
}
value result = arr[0];
for (const auto & item : arr) {
value val_arr = item;
value val_cmp = result;
if (!attribute->is_undefined()) {
val_arr = get_attribute(val_arr, attribute, undef);
val_cmp = get_attribute(val_cmp, attribute, undef);
}
if (value_compare(val_arr, val_cmp, value_compare_op::lt)) {
result = item;
}
}
return result;
}},
{"max", [](const func_args & args) -> value {
args.ensure_count(1, 4);
args.ensure_vals<value_array>();
value val_case = args.get_kwarg_or_pos("case_sensitive", 1);
value attribute = args.get_kwarg_or_pos("attribute", 2);
// FIXME: max is currently always case sensitive
(void) val_case;
value undef = mk_val<value_undefined>();
const auto & arr = args.get_pos(0)->as_array();
if (arr.empty()) {
return undef;
}
value result = arr[0];
for (const auto & item : arr) {
value val_arr = item;
value val_cmp = result;
if (!attribute->is_undefined()) {
val_arr = get_attribute(val_arr, attribute, undef);
val_cmp = get_attribute(val_cmp, attribute, undef);
}
if (value_compare(val_arr, val_cmp, value_compare_op::gt)) {
result = item;
}
}
return result;
}},
{"unique", [](const func_args & args) -> value {
args.ensure_count(1, 3);
args.ensure_vals<value_array>();
value val = args.get_pos(0);
value val_case = args.get_kwarg_or_pos("case_sensitive", 1);
value attribute = args.get_kwarg_or_pos("attribute", 2);
// FIXME: unique is currently always case sensitive
//const bool case_sensitive = val_case->as_bool(); // undefined == false
value undef = mk_val<value_undefined>();
std::vector<value> result;
std::vector<value> seen;
for (const auto & item : val->as_array()) {
value key = item;
if (!attribute->is_undefined()) {
key = get_attribute(item, attribute, undef);
}
bool dup = false;
for (const auto & s : seen) {
if (value_compare(s, key, value_compare_op::eq)) {
dup = true;
break;
}
}
if (!dup) {
seen.push_back(key);
result.push_back(item);
}
}
return mk_val<value_array>(std::move(result));
}},
};
return builtins;
}
[[noreturn]] static value object_join_not_implemented(const func_args &) {
throw not_implemented_exception("object join not implemented");
}
const func_builtins & value_object_t::get_builtins() const {
if (!has_builtins) {
static const func_builtins no_builtins = {};
return no_builtins;
}
static const func_builtins builtins = {
// {"default", default_value}, // cause issue with gpt-oss
{"get", [](const func_args & args) -> value {
args.ensure_count(2, 3);
if (!is_val<value_object>(args.get_pos(0))) {
throw raised_exception("get: first argument must be an object");
}
if (!is_val<value_string>(args.get_pos(1))) {
throw raised_exception("get: second argument must be a string (key)");
}
value default_val = mk_val<value_none>();
if (args.count() == 3) {
default_val = args.get_pos(2);
}
const value obj = args.get_pos(0);
const value key = args.get_pos(1);
return obj->at(key, default_val);
}},
{"keys", [](const func_args & args) -> value {
args.ensure_vals<value_object>();
const auto & obj = args.get_pos(0)->as_ordered_object();
auto result = mk_val<value_array>();
for (const auto & pair : obj) {
result->push_back(pair.first);
}
return result;
}},
{"list", [](const func_args & args) -> value {
args.ensure_vals<value_object>();
const auto & obj = args.get_pos(0)->as_ordered_object();
auto result = mk_val<value_array>();
for (const auto & pair : obj) {
result->push_back(pair.first);
}
return result;
}},
{"values", [](const func_args & args) -> value {
args.ensure_vals<value_object>();
const auto & obj = args.get_pos(0)->as_ordered_object();
auto result = mk_val<value_array>();
for (const auto & pair : obj) {
result->push_back(pair.second);
}
return result;
}},
{"items", [](const func_args & args) -> value {
args.ensure_vals<value_object>();
const auto & obj = args.get_pos(0)->as_ordered_object();
auto result = mk_val<value_array>();
for (const auto & pair : obj) {
auto item = mk_val<value_tuple>(pair);
result->push_back(std::move(item));
}
return result;
}},
{"tojson", tojson},
{"string", [](const func_args & args) -> value {
args.ensure_vals<value_object>();
if (args.ctx.is_get_stats) {
// mark as used (recursively) for stats
auto val_input = args.get_pos(0);
value_t::stats_t::mark_used(const_cast<value&>(val_input), true);
}
return mk_val<value_string>(args.get_pos(0)->as_string());
}},
{"length", [](const func_args & args) -> value {
args.ensure_vals<value_object>();
const auto & obj = args.get_pos(0)->as_ordered_object();
return mk_val<value_int>(static_cast<int64_t>(obj.size()));
}},
{"tojson", [](const func_args & args) -> value {
args.ensure_vals<value_object>();
// use global to_json
return global_builtins().at("tojson")(args);
}},
{"dictsort", [](const func_args & args) -> value {
value val_input = args.get_pos(0);
value val_case = args.get_kwarg_or_pos("case_sensitive", 1);
value val_by = args.get_kwarg_or_pos("by", 2);
value val_reverse = args.get_kwarg_or_pos("reverse", 3);
// FIXME: sorting is currently always case sensitive
//const bool case_sensitive = val_case->as_bool(); // undefined == false
const bool reverse = val_reverse->as_bool(); // undefined == false
const bool by_value = is_val<value_string>(val_by) && val_by->as_string().str() == "value" ? true : false;
auto result = mk_val<value_object>(val_input); // copy
std::sort(result->val_obj.begin(), result->val_obj.end(), [&](const auto & a, const auto & b) {
if (by_value) {
return value_compare(a.second, b.second, reverse ? value_compare_op::gt : value_compare_op::lt);
} else {
return value_compare(a.first, b.first, reverse ? value_compare_op::gt : value_compare_op::lt);
}
});
return result;
}},
{"join", object_join_not_implemented},
};
return builtins;
}
const func_builtins & value_none_t::get_builtins() const {
static const func_handler tostring = [](const func_args &) -> value {
return mk_val<value_string>("None");
};
static const func_builtins builtins = {
{"default", default_value},
{"tojson", tojson},
{"string", tostring},
{"safe", tostring},
{"items", empty_value_fn<value_array>},
{"map", empty_value_fn<value_array>},
{"reject", empty_value_fn<value_array>},
{"rejectattr", empty_value_fn<value_array>},
{"select", empty_value_fn<value_array>},
{"selectattr", empty_value_fn<value_array>},
{"unique", empty_value_fn<value_array>},
};
return builtins;
}
const func_builtins & value_undefined_t::get_builtins() const {
static const func_builtins builtins = {
{"default", default_value},
{"capitalize", empty_value_fn<value_string>},
{"first", empty_value_fn<value_undefined>},
{"items", empty_value_fn<value_array>},
{"join", empty_value_fn<value_string>},
{"last", empty_value_fn<value_undefined>},
{"length", empty_value_fn<value_int>},
{"list", empty_value_fn<value_array>},
{"lower", empty_value_fn<value_string>},
{"map", empty_value_fn<value_array>},
{"max", empty_value_fn<value_undefined>},
{"min", empty_value_fn<value_undefined>},
{"reject", empty_value_fn<value_array>},
{"rejectattr", empty_value_fn<value_array>},
{"replace", empty_value_fn<value_string>},
{"reverse", empty_value_fn<value_array>},
{"safe", empty_value_fn<value_string>},
{"select", empty_value_fn<value_array>},
{"selectattr", empty_value_fn<value_array>},
{"sort", empty_value_fn<value_array>},
{"string", empty_value_fn<value_string>},
{"strip", empty_value_fn<value_string>},
{"sum", empty_value_fn<value_int>},
{"title", empty_value_fn<value_string>},
{"truncate", empty_value_fn<value_string>},
{"unique", empty_value_fn<value_array>},
{"upper", empty_value_fn<value_string>},
{"wordcount", empty_value_fn<value_int>},
};
return builtins;
}
//////////////////////////////////
// the runtime walks values with recursion, a deeper input overflows the stack
static value from_json(const common_json & j, bool mark_input, int depth = 0) {
if (j.is_null()) {
return mk_val<value_none>();
} else if (j.is_boolean()) {
return mk_val<value_bool>(j.get<bool>());
} else if (j.is_number_integer()) {
return mk_val<value_int>(j.get<int64_t>());
} else if (j.is_number_float()) {
return mk_val<value_float>(j.get<double>());
} else if (j.is_string()) {
auto str = mk_val<value_string>(j.get<std::string>());
if (mark_input) {
str->mark_input();
}
return str;
} else if ((j.is_array() || j.is_object()) && depth >= JINJA_RECURSIVE_LIMIT) {
throw std::invalid_argument("JSON is nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep");
} else if (j.is_array()) {
auto arr = mk_val<value_array>();
for (const auto & item : j) {
arr->push_back(from_json(item, mark_input, depth + 1));
}
return arr;
} else if (j.is_object()) {
auto obj = mk_val<value_object>();
for (auto it = j.begin(); it != j.end(); ++it) {
obj->insert(it.key(), from_json(it.value(), mark_input, depth + 1));
}
return obj;
} else {
throw std::runtime_error("Unsupported JSON value type");
}
}
// compare operator for value_t
bool value_compare(const value & a, const value & b, value_compare_op op) {
auto cmp = [&]() {
// compare numeric types
if ((is_val<value_int>(a) || is_val<value_float>(a)) &&
(is_val<value_int>(b) || is_val<value_float>(b))){
try {
if (op == value_compare_op::eq) {
return a->as_float() == b->as_float();
} else if (op == value_compare_op::ge) {
return a->as_float() >= b->as_float();
} else if (op == value_compare_op::gt) {
return a->as_float() > b->as_float();
} else if (op == value_compare_op::lt) {
return a->as_float() < b->as_float();
} else if (op == value_compare_op::ne) {
return a->as_float() != b->as_float();
} else {
throw std::runtime_error("Unsupported comparison operator for numeric types");
}
} catch (...) {}
}
// compare string and number
// TODO: not sure if this is the right behavior
if ((is_val<value_string>(b) && (is_val<value_int>(a) || is_val<value_float>(a))) ||
(is_val<value_string>(a) && (is_val<value_int>(b) || is_val<value_float>(b))) ||
(is_val<value_string>(a) && is_val<value_string>(b))) {
try {
if (op == value_compare_op::eq) {
return a->as_string().str() == b->as_string().str();
} else if (op == value_compare_op::ge) {
return a->as_string().str() >= b->as_string().str();
} else if (op == value_compare_op::gt) {
return a->as_string().str() > b->as_string().str();
} else if (op == value_compare_op::lt) {
return a->as_string().str() < b->as_string().str();
} else if (op == value_compare_op::ne) {
return a->as_string().str() != b->as_string().str();
} else {
throw std::runtime_error("Unsupported comparison operator for string/number types");
}
} catch (...) {}
}
// compare boolean simple
if (is_val<value_bool>(a) && is_val<value_bool>(b)) {
if (op == value_compare_op::eq) {
return a->as_bool() == b->as_bool();
} else if (op == value_compare_op::ne) {
return a->as_bool() != b->as_bool();
} else {
throw std::runtime_error("Unsupported comparison operator for bool type");
}
}
// compare by type
if (a->type() != b->type()) {
return false;
}
return false;
};
auto result = cmp();
JJ_DEBUG("Comparing types: %s and %s result=%d", a->type().c_str(), b->type().c_str(), result);
return result;
}
template<>
void global_from_json(context & ctx, const common_json & json_obj, bool mark_input) {
// printf("global_from_json: %s\n" , json_obj.dump(2).c_str());
if (json_obj.is_null() || !json_obj.is_object()) {
throw std::runtime_error("global_from_json: input JSON value must be an object");
}
for (auto it = json_obj.begin(); it != json_obj.end(); ++it) {
JJ_DEBUG("global_from_json: setting key '%s'", it.key().c_str());
ctx.set_val(it.key(), from_json(it.value(), mark_input));
}
}
// a float as Python's repr() and str() write it: the shortest digits that read back as the same value
static std::string float_repr(double val) {
if (!std::isfinite(val)) {
return std::isnan(val) ? "nan" : val < 0 ? "-inf" : "inf";
}
const double mag = std::fabs(val);
// at a power of 2 the gap to the value below is half the gap above, the nearest digits can miss while the next ones up read back
int exp2 = 0;
const bool is_pow2 = std::frexp(mag, &exp2) == 0.5;
// a normal double has at most one decimal of up to DBL_DIG digits that reads back as it, the search can start at DBL_DIG digits
// 17 digits always read back
std::string digits;
int exp10 = 0;
for (int prec = mag < DBL_MIN ? 0 : DBL_DIG - 1; digits.empty(); prec++) {
char buf[32];
snprintf(buf, sizeof(buf), "%.*e", prec, mag);
const char * e = std::strchr(buf, 'e');
// only the digits, the decimal point depends on the locale
std::string d(buf, e - buf);
d.erase(std::remove_if(d.begin(), d.end(), [](char c) { return c < '0' || c > '9'; }), d.end());
exp10 = std::atoi(e + 1);
const auto reads_back = [&](const std::string & cand) {
return std::strtod((cand + "e" + std::to_string(exp10 - prec)).c_str(), nullptr) == mag;
};
if (prec == 16 || reads_back(d)) {
digits = d;
} else if (is_pow2 && d.back() != '9') {
// a last digit 9 is not bumped (it would carry): no power of 2, from 2^-1074 to 2^1023, needs it
d.back()++;
if (reads_back(d)) {
digits = d;
}
}
}
while (digits.size() > 1 && digits.back() == '0') {
digits.pop_back();
}
// fixed notation for 1e-4 <= |val| < 1e16, else d.ddde+XX
const int n = static_cast<int>(digits.size());
const int point = exp10 + 1; // number of digits before the decimal point
std::string out = std::signbit(val) ? "-" : "";
if (point > -4 && point <= 16) {
if (point <= 0) {
out += "0." + std::string(-point, '0') + digits;
} else if (point >= n) {
out += digits + std::string(point - n, '0') + ".0";
} else {
out += digits.substr(0, point) + "." + digits.substr(point);
}
} else {
char buf[16];
snprintf(buf, sizeof(buf), "e%c%02d", exp10 < 0 ? '-' : '+', std::abs(exp10));
out += digits.substr(0, 1) + (n > 1 ? "." + digits.substr(1) : "") + buf;
}
return out;
}
// a float as json.dumps writes it: nan and inf as NaN and Infinity, which are not valid JSON
static std::string float_json(double val) {
if (std::isnan(val)) {
return "NaN";
}
if (std::isinf(val)) {
return val < 0 ? "-Infinity" : "Infinity";
}
return float_repr(val);
}
// recursively convert value to JSON string
// TODO: avoid circular references
static void value_to_json_internal(std::ostringstream & oss, const value & val, int curr_lvl, int indent, const std::string_view item_sep, const std::string_view key_sep) {
auto indent_str = [indent, curr_lvl]() -> std::string {
return (indent > 0) ? std::string(curr_lvl * indent, ' ') : "";
};
auto newline = [indent]() -> std::string {
return (indent >= 0) ? "\n" : "";
};
if (is_val<value_none>(val) || val->is_undefined()) {
oss << "null";
} else if (is_val<value_bool>(val)) {
oss << (val->as_bool() ? "true" : "false");
} else if (is_val<value_int>(val)) {
oss << val->as_int();
} else if (is_val<value_float>(val)) {
oss << float_json(val->as_float());
} else if (is_val<value_string>(val)) {
oss << "\"";
for (char c : val->as_string().str()) {
switch (c) {
case '"': oss << "\\\""; break;
case '\\': oss << "\\\\"; break;
case '\b': oss << "\\b"; break;
case '\f': oss << "\\f"; break;
case '\n': oss << "\\n"; break;
case '\r': oss << "\\r"; break;
case '\t': oss << "\\t"; break;
default:
if (static_cast<unsigned char>(c) < 0x20) {
char buf[7];
snprintf(buf, sizeof(buf), "\\u%04x", static_cast<unsigned char>(c));
oss << buf;
} else {
oss << c;
}
}
}
oss << "\"";
} else if ((is_val<value_array>(val) || is_val<value_object>(val)) && curr_lvl >= JINJA_RECURSIVE_LIMIT) {
throw std::runtime_error("value is nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep");
} else if (is_val<value_array>(val)) {
const auto & arr = val->as_array();
oss << "[";
if (!arr.empty()) {
oss << newline();
for (size_t i = 0; i < arr.size(); ++i) {
oss << indent_str() << (indent > 0 ? std::string(indent, ' ') : "");
value_to_json_internal(oss, arr[i], curr_lvl + 1, indent, item_sep, key_sep);
if (i < arr.size() - 1) {
oss << item_sep;
}
oss << newline();
}
oss << indent_str();
}
oss << "]";
} else if (is_val<value_object>(val)) {
const auto & obj = val->as_ordered_object(); // IMPORTANT: need to keep exact order
oss << "{";
if (!obj.empty()) {
oss << newline();
size_t i = 0;
for (const auto & pair : obj) {
oss << indent_str() << (indent > 0 ? std::string(indent, ' ') : "");
const std::string key = is_val<value_float>(pair.first) ? float_json(pair.first->as_float()) : pair.first->as_string().str();
value_to_json_internal(oss, mk_val<value_string>(key), curr_lvl + 1, indent, item_sep, key_sep);
oss << key_sep;
value_to_json_internal(oss, pair.second, curr_lvl + 1, indent, item_sep, key_sep);
if (i < obj.size() - 1) {
oss << item_sep;
}
oss << newline();
++i;
}
oss << indent_str();
}
oss << "}";
} else {
oss << "null";
}
}
std::string value_to_json(const value & val, int indent, const std::string_view item_sep, const std::string_view key_sep) {
std::ostringstream oss;
value_to_json_internal(oss, val, 0, indent, item_sep, key_sep);
JJ_DEBUG("value_to_json: result=%s", oss.str().c_str());
return oss.str();
}
// TODO: avoid circular references
std::string value_to_string_repr(const value & val) {
if (is_val<value_string>(val)) {
const std::string val_str = val->as_string().str();
if (val_str.find('\'') != std::string::npos) {
return value_to_json(val);
} else {
return "'" + val_str + "'";
}
} else {
return val->as_repr();
}
}
// stats utility
void value_t::stats_t::mark_used(value & val, bool deep, int depth) {
val->stats.used = true;
if (deep && (is_val<value_array>(val) || is_val<value_object>(val)) && depth >= JINJA_RECURSIVE_LIMIT) {
throw std::runtime_error("value is nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep");
}
if (deep) {
if (is_val<value_array>(val)) {
for (auto & item : val->val_arr) {
mark_used(item, deep, depth + 1);
}
} else if (is_val<value_object>(val)) {
for (auto & pair : val->val_obj) {
mark_used(pair.first, deep, depth + 1);
mark_used(pair.second, deep, depth + 1);
}
}
}
}
} // namespace jinja
View Git Blame Copy Permalink