labels now allow underscored; bug fix

README.md

@@ -66,7 +66,7 @@ tinytemplate_compile(const char *src, size_t len,
 First you need to compile the template into a program and then you can evaluate it by executing the program.
 
 ### Compilation
-The compilation function expects a source string of the template through `src` and with length in bytes `len`. The string doesn't need to be zero-terminated. The program will be compilated by writing its instructions in the `program` buffer. The `program` array is assumed to have space for a maximum of `max_instr` instructions. If more space would be required to memorize the program, the compilation fails returning the status code `TINYTEMPLATE_STATUS_EMEMORY`. At that point the caller program can either abort or try again with a bigger buffer. If compilation succeded, then the number of instruction is written to the variable `num_instr`. A status code of the compilation is returned through the main return value. The status `TINYTEMPLATE_STATUS_DONE` means all went well, while all other status codes refer to a specific error that occurred. If the compilation failed, an error message is written to the caller-provided buffer `errmsg` whith size `errmax`.
+The compilation function expects a source string of the template through `src` and with length in bytes `len`. The string doesn't need to be zero-terminated. The program will be compilated by writing its instructions in the `prog` buffer. The `prog` array is assumed to have space for a maximum of `max_instr` instructions. If more space would be required to memorize the program, the compilation fails returning the status code `TINYTEMPLATE_STATUS_EMEMORY`. At that point the caller program can either abort or try again with a bigger buffer. If compilation succeded, then the number of instruction is written to the variable `num_instr`. A status code of the compilation is returned through the main return value. The status `TINYTEMPLATE_STATUS_DONE` means all went well, while all other status codes refer to a specific error that occurred. If the compilation failed, an error message is written to the caller-provided buffer `errmsg` whith size `errmax`.
 
 Here's an example of how you would use this function:
 ```c

python/pytinytemplate.c

@@ -0,0 +1,143 @@
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+#include "tinytemplate.h"
+
+typedef struct {
+    PyObject_HEAD
+    PyObject *src;
+    tinytemplate_instr_t *program;
+} Template;
+
+static void
+TinyTemplate_dealloc(CustomObject *self)
+{
+    Py_XDECREF(self->src);
+    Py_TYPE(self)->tp_free((PyObject *) self);
+}
+
+static PyObject *
+TinyTemplate_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+    Template *self;
+    self = (Template *) type->tp_alloc(type, 0);
+    if (self != NULL) {
+        self->src = PyUnicode_FromString("");
+        if (self->src == NULL) {
+            Py_DECREF(self);
+            return NULL;
+        }
+    }
+    return (PyObject *) self;
+}
+
+static int
+TinyTemplate_init(Template *self, PyObject *args, PyObject *kwds)
+{
+    static char *kwlist[] = {"src", NULL};
+    PyObject *src = NULL;
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwlist, &src))
+        return -1;
+
+    if (src) {
+        PyObject *tmp = self->src;
+        Py_INCREF(src);
+        self->src = src;
+        Py_XDECREF(tmp);
+    }
+    return 0;
+}
+
+static PyObject *
+TinyTemplate_eval(Template *self, PyObject *args)
+{
+    PyObject *src = self->src;
+
+    if (src == NULL) {
+        PyErr_SetString(PyExc_AttributeError, "src");
+        return NULL;
+    }
+    if (!PyUnicode_Check(src))
+        return NULL; // Raise exception?
+
+    PyObject *bytes = PyUnicode_AsEncodedString(src, "UTF-8", "strict"); // Owned reference
+    if (bytes == NULL) {
+        ..
+        return NULL;
+    }
+
+    char *cstr = PyBytes_AS_STRING(bytes);
+
+    ..
+
+}
+
+static PyMemberDef TinyTemplate_members[] = {
+    {"src", T_OBJECT_EX, offsetof(Template, src), 0, "source string"},
+    {NULL}  /* Sentinel */
+};
+
+static PyMethodDef TinyTemplate_methods[] = {
+    {"eval", (PyCFunction) TinyTemplate_eval, METH_NOARGS, "Evaluate the template" },
+    {NULL}  /* Sentinel */
+};
+
+static PyTypeObject TemplateType = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    .tp_name = "tinytemplate.Template",
+    .tp_doc = PyDoc_STR("A template"),
+    .tp_basicsize = sizeof(Template),
+    .tp_itemsize = 0,
+    .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+    .tp_new = TinyTemplate_new,
+    .tp_init = (initproc) TinyTemplate_init,
+    .tp_dealloc = (destructor) TinyTemplate_dealloc,
+    .tp_members = TinyTemplate_members,
+    .tp_methods = TinyTemplate_methods,
+};
+
+static PyObject*
+compile(PyObject *self, PyObject *args)
+{
+    PyObject *src;
+    if (!PyArg_ParseTuple(args, "o", &src))
+        return NULL;
+
+    TinyTemplate_instr_t *prog = malloc();
+    
+    return PyObject_Call((PyObject*) &TemplateType, src);
+}
+
+static PyMethodDef methods[] = {
+    {"compile", compile, METH_VARARGS, "Compile a template"},
+    {NULL, NULL, 0, NULL}, /* Sentinel */
+};
+
+static struct PyModuleDef module = {
+    PyModuleDef_HEAD_INIT,
+    "tinytemplate",   /* name of module */
+    spam_doc, /* module documentation, may be NULL */
+    0,       /* size of per-interpreter state of the module,
+                 or -1 if the module keeps state in global variables. */
+    methods,
+};
+
+PyMODINIT_FUNC
+PyInit_tinytemplate(void)
+{
+    if (PyType_Ready(&TemplateType) < 0)
+        return NULL;
+
+    PyObject *m = PyModule_Create(&module);
+    if (m == NULL)
+        return NULL;
+
+    Py_INCREF(&TemplateType);
+    if (PyModule_AddObject(m, "Template", (PyObject *) &TemplateType) < 0) {
+        Py_DECREF(&TemplateType);
+        Py_DECREF(m);
+        return NULL;
+    }
+
+    return m;
+}

python/setup.py

@@ -0,0 +1,3 @@
+from distutils.core import setup, Extension
+setup(name="tinytemplate", version="1.0",
+      ext_modules=[Extension("tinytemplate", ["pytinytemplate.c"])])

src/tinytemplate.c

@@ -178,6 +178,7 @@ append_instr(compile_state_t *state,
         case OPCODE_ITER:
         case OPCODE_CHLD:
         case OPCODE_IDX:
+        instr->operands[0].as_size = va_arg(operands, size_t); 
         break;
 
         case OPCODE_NEXT:
@@ -388,12 +389,12 @@ next_token_kword_or_ident(scanner_t *scanner, slice_t *slice,
 {
     (void) payload;
 
-    assert(follows_alpha(scanner));
+    assert(follows_alpha(scanner) || follows_char(scanner, '_'));
 
     size_t offset = scanner->cur;
     do
         scanner->cur++;
-    while (follows_alpha(scanner));
+    while (follows_alpha(scanner) || follows_char(scanner, '_'));
     size_t length = scanner->cur - offset;
 
     if (slice) {
@@ -553,7 +554,7 @@ parse_primary(scanner_t *scanner,
 
                     // Check if the label matches the iteration label
                     if (slice.length == scope->for_child_label.length && !memcmp(scanner->src + slice.offset, scanner->src + scope->for_child_label.offset, slice.length)) {
-                        append_instr(state, OPCODE_CHLD, slice.offset, slice.length);
+                        append_instr(state, OPCODE_CHLD, j);
                         found = true;
                         break;
                     }
@@ -563,7 +564,7 @@ parse_primary(scanner_t *scanner,
                     // its length will be 0 and the expression's
                     // label won't match.
                     if (slice.length == scope->for_index_label.length && !memcmp(scanner->src + slice.offset, scanner->src + scope->for_index_label.offset, slice.length)) {
-                        append_instr(state, OPCODE_IDX, slice.offset, slice.length);
+                        append_instr(state, OPCODE_IDX, j);
                         found = true;
                         break;
                     }
@@ -1001,7 +1002,7 @@ tinytemplate_compile(const char *src, size_t len,
         size_t raw_off = scanner.cur; // Start offset of the raw block
         while (scanner.cur < scanner.len) {
             // Look for a "{" or the end
-            while (follows_char(&scanner, '{'))
+            while (scanner.cur < scanner.len && !follows_char(&scanner, '{'))
                 scanner.cur++;
 
             // If the end wasn't reached (a "{" was found)

src/tinytemplate.h

@@ -42,8 +42,7 @@ typedef struct {
 } tinytemplate_array_t;
 
 typedef struct {
-    const char *str;
+    const char *str; size_t len;
-    size_t len;
 } tinytemplate_string_t;
 
 typedef union {

tests/fuzz.c

@@ -0,0 +1,59 @@
+#include <stdio.h>
+#include <stddef.h>
+#include "../src/tinytemplate.h"
+
+/* this lets the source compile without afl-clang-fast/lto */
+#ifndef __AFL_FUZZ_TESTCASE_LEN
+
+ssize_t       fuzz_len;
+unsigned char fuzz_buf[1024000];
+
+#define __AFL_FUZZ_TESTCASE_LEN fuzz_len
+#define __AFL_FUZZ_TESTCASE_BUF fuzz_buf
+//#define __AFL_FUZZ_INIT() void sync(void);
+//#define __AFL_LOOP(x) ((fuzz_len = read(0, fuzz_buf, sizeof(fuzz_buf))) > 0 ? 1 : 0)
+//#define __AFL_INIT() sync()
+
+#endif
+
+__AFL_FUZZ_INIT();
+
+//#pragma clang optimize off
+//#pragma GCC            optimize("O0")
+
+static void callback(void *userp, const char *label, size_t label_len,
+                     const char *str, size_t len)
+{
+    (void) userp;
+    (void) label;
+    (void) label_len;
+    (void) str;
+    (void) len;
+}
+
+int main(void) 
+{
+
+#ifdef __AFL_HAVE_MANUAL_CONTROL
+    __AFL_INIT();
+#endif
+
+    unsigned char *buf = __AFL_FUZZ_TESTCASE_BUF;   // must be after __AFL_INIT
+                                                    // and before __AFL_LOOP!
+    tinytemplate_instr_t program[1024];
+    size_t max_instr = sizeof(program)/sizeof(program[0]);
+
+    while (__AFL_LOOP(10000)) {
+
+        int len = __AFL_FUZZ_TESTCASE_LEN;  // don't use the macro directly in a
+                                            // call!
+
+        tinytemplate_status_t status;
+
+        status = tinytemplate_compile((char*) buf, len, program, max_instr, NULL, NULL, 0);
+        if (status == TINYTEMPLATE_STATUS_DONE)
+            status = tinytemplate_eval((char*) buf, program, NULL, NULL, callback, NULL, 0);
+    }
+
+    return 0;
+}

tests/samples/000.tt

@@ -0,0 +1 @@
+{{1+2}}

tests/samples/001.tt

@@ -0,0 +1,3 @@
+{% if 1 + 2 %}
+text
+{% end %}

tests/samples/002.tt

@@ -0,0 +1,5 @@
+{% if 1 + 2 %}
+text
+{% else %}
+hoyo
+{% end %}

tests/samples/003.tt

@@ -0,0 +1,3 @@
+{% for item in items %}
+    {{item}}
+{% end %}

tests/test.c

@@ -0,0 +1,77 @@
+
+struct {
+    const char *src;
+    const char *exp;
+    const char *params;
+} cases[] = {
+
+};
+
+typedef enum {
+    SECTION_INPUT,
+    SECTION_OUTPUT,
+    SECTION_PARAMS,
+} Section;
+
+static void runTest(const char *src, size_t len)
+{
+    size_t params_offset;
+    size_t params_length;
+    size_t input_offset;
+    size_t input_length;
+    size_t output_offset;
+    size_t output_length;
+
+    size_t i = 0;
+    while (1) {
+        while (i < len && src[i] != '@')
+            i++;
+
+        if (i < len && isalpha(src[i])) {
+
+            size_t kword_off = i;
+            do
+                i++;
+            while (i < len && isalpha(src[i]));
+            size_t kword_len = i - kword_off;
+            
+            bool is_kword = true;
+            Section section;
+            if (kword_len == 6 && !strncmp("params", src + kword_off, 6)) {
+                section = SECTION_PARAMS;
+            } else if (kword_len == 5 && !strncmp("input", src + kword_off, 5)) {
+                section = SECTION_INPUT;
+            } else if (kword_len == 6 && !strncmp("output", src + kword_off, 6)) {
+                section = SECTION_OUTPUT;
+            } else
+                is_kword = false;
+
+            if (is_kword) {
+                // Get to the end of the line
+                while (i < len && (src[i] == ' ' || src[i] == '\t' || src[i] == '\r'))
+                    i++;
+                if (i < len && src[i] != '\n') {
+                    
+                }
+                while (i < len && src[i] != '\n')
+                    i++;
+                if (i < len) {
+                    assert(src[i] == '\n');
+                    i++;
+                }
+                switch (section) {
+                    case SECTION_INPUT:   input_offset = i; break;
+                    case SECTION_OUTPUT: output_offset = i; break;
+                    case SECTION_PARAMS: params_offset = i; break;
+                }
+            }
+        }
+        
+    }
+}
+
+int main(void)
+{
+
+    return 0;
+}