new built-in keysof and rewrote the json parser

This commit is contained in:
cozis
2022-12-06 22:43:13 +01:00
parent 7e3eb6364a
commit 380b8a8917
11 changed files with 521 additions and 219 deletions
+282 -202
View File
@@ -1,240 +1,320 @@
Scanner = {src: String, i: int};
fun newScanner(src: String) {
scan = {src: src, i: 0};
assert(istypeof(Scanner, scan));
return scan;
}
fun isDigit(char: ?String) {
if char == none:
return false;
return string.ord(char) >= string.ord('0')
and string.ord(char) <= string.ord('9');
}
fun isSpace(c: String)
return c == " "
or c == "\t"
or c == "\n";
return c == ' '
or c == '\t'
or c == '\n';
fun isDigit(c: String) {
d = string.ord(c)
- string.ord("0");
return d >= 0
and d <= 9;
fun min(x, y) {
if x < y:
return x;
return y;
}
ParsingContext = {str: String, cur: int};
fun skipSpaces(context: ParsingContext) {
# Skip zero or more whitespace characters.
while context.cur < count(context.str)
and isSpace(context.str[context.cur]):
context.cur = context.cur + 1;
}
fun parseNull(context: ParsingContext) {
assert(context.cur < count(context.str)
and context.str[context.cur] == 'n');
if context.cur+3 > count(context.str)
or context.str[context.cur+1] != 'u'
or context.str[context.cur+2] != 'l'
or context.str[context.cur+3] != 'l':
return none, "Unexpected token starting with \"n\"";
context.cur = context.cur + count("null");
fun hint(scan: Scanner, n: int = 1) {
k = scan.i + n;
if k < count(scan.src):
return scan.src[k];
return none;
}
fun parseTrue(context: ParsingContext) {
fun current(scan: Scanner)
return hint(scan, 0);
assert(context.cur < count(context.str)
and context.str[context.cur] == 't');
if context.cur+3 > count(context.str)
or context.str[context.cur+1] != 'r'
or context.str[context.cur+2] != 'u'
or context.str[context.cur+3] != 'e':
return none, "Unexpected token starting with \"t\"";
context.cur = context.cur + count("true");
return true;
fun consume(scan: Scanner, n: int = 1) {
assert(n > 0);
limit = count(scan.src);
scan.i = min(scan.i + n, limit);
}
fun parseFalse(context: ParsingContext) {
fun consumeSpaces(scan: Scanner)
while isSpace(current(scan)):
consume(scan);
assert(context.cur < count(context.str)
and context.str[context.cur] == 'f');
fun parseString(scan: Scanner) {
if context.cur+3 > count(context.str)
or context.str[context.cur+1] != 'a'
or context.str[context.cur+2] != 'l'
or context.str[context.cur+3] != 's'
or context.str[context.cur+4] != 'e':
return none, "Unexpected token starting with \"f\"";
assert(current(scan) == '"');
consume(scan); # Consume the opening "
context.cur = context.cur + count("false");
return false;
}
fun dummy() {}
Func = type(dummy);
fun skip(context: ParsingContext, test_callback: Func) {
k = context.cur;
while k < count(context.str)
and test_callback(context.str[k]):
k = k + 1;
return k;
}
fun parseInteger(context: ParsingContext) {
assert(context.cur < count(context.str) and isDigit(context.str[context.cur]));
buffer = 0;
do {
c = context.str[context.cur];
d = string.ord(c) - string.ord('0');
buffer = buffer * 10 + d;
context.cur = context.cur + 1;
} while context.cur < count(context.str)
and isDigit(context.str[context.cur]);
return buffer;
}
fun parseFloating(context: ParsingContext) {
assert(context.cur < count(context.str)
and isDigit(context.str[context.cur]));
# It's ensured by the caller (parseIntegerOrFloating)
# that now the strings contains two sequences of
# digits with a dot in the middle.
buffer = 0.0;
# Scan the integer part.
do {
c = context.str[context.cur];
d = string.ord(c) - string.ord('0');
buffer = buffer * 10 + d;
context.cur = context.cur + 1;
} while context.str[context.cur] != ".";
assert(isDigit(context.str[context.cur+1]));
# Scan the decimal part.
q = 1;
do {
c = context.str[context.cur];
d = string.ord(c) - string.ord('0');
q = q / 10;
buffer = buffer + q * d;
context.cur = context.cur + 1;
} while context.cur < count(context.str)
and isDigit(context.str[context.cur+1]);
return buffer;
}
fun parseIntegerOrFloating(context: ParsingContext) {
assert(context.cur < count(context.str)
and isDigit(context.str[context.cur]));
# Is the first digit sequence followed by a dot
# and then one or more digits?
k = skip(context, isDigit);
if k+1 < count(context.str) and context.str[k] == "." and isDigit(context.str[k+1]):
# It's a floating point value, then!
return parseFloating(context);
else
# It's good old integer!
return parseInteger(context);
}
fun parseString(context: ParsingContext) {
assert(context.cur < count(context.str)
and context.str[context.cur] == '"');
context.cur = context.cur + 1;
buffer = "";
while context.cur < count(context.str)
and context.str[context.cur] != '"': {
# TODO: Support special characters such as \n, \t ecc
buffer = string.cat(buffer, context.str[context.cur]);
context.cur = context.cur + 1;
temp = "";
char = current(scan);
while char != '"' and char != none: {
temp = string.cat(temp, char);
consume(scan);
char = current(scan);
}
if context.cur == count(context.str):
return none, "Source string ended inside a string literal";
if char == none: {
val = none;
err = "Source ended inside a string literal";
} else {
assert(char == '"');
consume(scan);
val = temp;
err = none;
}
context.cur = context.cur + 1;
return buffer;
return val, err;
}
fun parseArray(context: ParsingContext) {
assert(context.cur < count(context.str)
and context.str[context.cur] == '[');
fun parseObject(scan: Scanner) {
context.cur = context.cur + 1;
assert(current(scan) == '{');
consume(scan);
skipSpaces(context);
obj = {};
consumeSpaces(scan);
if current(scan) != '}': do {
key, err = parseValue(scan);
if err != none:
return none, err;
if type(key) != String:
return none, "Object key isn't a string";
if context.cur == count(context.str):
return none, "Source string ended inside an array";
consumeSpaces(scan);
if current(scan) != ':':
return none, "Missing ':' separator after key";
consume(scan);
if context.str[context.cur] == ']':
return [];
list = [];
while true: {
item, err = parseValue(context);
val, err = parseValue(scan);
if err != none:
return none, err;
list[count(list)] = item;
if obj[key] != none:
return none, "Duplicate key in object";
obj[key] = val;
skipSpaces(context);
if context.cur == count(context.str):
return none, "Source string ended inside an array";
if context.str[context.cur] == ']':
break;
if context.str[context.cur] != ',':
return none, "Unexpected token inside an array, where either ',' or ']' were expected";
context.cur = context.cur + 1;
skipSpaces(context);
}
assert(context.cur < count(context.str)
and context.str[context.cur] == ']');
context.cur = context.cur + 1;
return list;
consumeSpaces(scan);
if current(scan) == '}':
done = true;
else {
if current(scan) != ',':
return none, "Invalid character where either ',' or '}' were expected";
consume(scan); # Consume the ","
done = false;
}
} while not done;
assert(current(scan) == '}');
consume(scan);
return obj;
}
fun parseValue(context: ParsingContext) {
fun parseArray(scan: Scanner) {
if context.cur == count(context.str):
return none, "Source ended where a value was expected";
assert(current(scan) == '[');
consume(scan); # Consume the [
c = context.str[context.cur];
if c == "{": val, err = parseObject(context);
else if c == "[": val, err = parseArray(context);
else if c == '"': val, err = parseString(context);
else if isDigit(c): val, err = parseIntegerOrFloating(context);
else if c == 'n': val, err = parseNull(context);
else if c == 't': val, err = parseTrue(context);
else if c == 'f': val, err = parseFalse(context);
else {
array = [];
consumeSpaces(scan);
if current(scan) != ']': do {
val, err = parseValue(scan);
if err != none:
return none, err;
array[count(array)] = val;
consumeSpaces(scan);
if current(scan) == ']':
done = true;
else {
if current(scan) != ',':
return none, "Invalid character where either ',' or ']' were expected";
consume(scan); # Consume the ","
done = false;
}
} while not done;
assert(current(scan) == ']');
consume(scan);
return array;
}
fun parseFalse(scan: Scanner) {
assert(current(scan) == 'f');
consume(scan);
if current(scan) != 'a':
return none, "Invalid character after 'f'";
consume(scan);
if current(scan) != 'l':
return none, "Invalid character after 'fa'";
consume(scan);
if current(scan) != 's':
return none, "Invalid character after 'fal'";
consume(scan);
if current(scan) != 'e':
return none, "Invalid character after 'fals'";
consume(scan);
return false;
}
fun parseTrue(scan: Scanner) {
assert(current(scan) == 't');
consume(scan);
if current(scan) != 'r':
return none, "Invalid character after 't'";
consume(scan);
if current(scan) != 'u':
return none, "Invalid character after 'tr'";
consume(scan);
if current(scan) != 'e':
return none, "Invalid character after 'tru'";
consume(scan);
return true;
}
fun parseNull(scan: Scanner) {
assert(current(scan) == 'n');
consume(scan);
if current(scan) != 'u':
return none, "Invalid character after 'n'";
consume(scan);
if current(scan) != 'l':
return none, "Invalid character after 'nu'";
consume(scan);
if current(scan) != 'l':
return none, "Invalid character after 'nul'";
consume(scan);
return none;
}
fun integerFromDigit(char: String) {
assert(count(char) == 1);
return string.ord(char)
- string.ord('0');
}
fun parseNumber(scan: Scanner) {
char = current(scan);
sign = 1;
if char == '+' or char == '-': {
sign = {'+': 1, '-': -1}[char];
consume(scan);
char = current(scan);
if not isDigit(char):
return none, "Invalid character after +/-";
}
temp = 0.0;
do {
assert(isDigit(char));
n = integerFromDigit(char);
temp = temp * 10 + n;
consume(scan);
char = current(scan);
} while isDigit(char);
if char == '.' and isDigit(hint(scan, 1)): {
consume(scan); # Consume the dot
char = current(scan);
q = 1.0;
do {
q = q / 10;
n = integerFromDigit(char);
temp = temp + q * n;
consume(scan);
char = current(scan);
} while isDigit(char);
}
return sign * temp;
}
fun parseValue(scan: Scanner) {
table = {
'"': parseString,
'{': parseObject,
'[': parseArray,
'f': parseFalse,
't': parseTrue,
'n': parseNull,
'+': parseNumber,
'-': parseNumber,
'0': parseNumber,
'1': parseNumber,
'2': parseNumber,
'3': parseNumber,
'4': parseNumber,
'5': parseNumber,
'6': parseNumber,
'7': parseNumber,
'8': parseNumber,
'9': parseNumber
};
consumeSpaces(scan);
char = current(scan);
if char == none: {
val = none;
err = string.cat("Unexepected character \"", c, "\" where a value was expected");
err = "Source ended where a value was expected";
} else {
callback = table[char];
if callback == none: {
val = none;
err = "Invalid character where a value was expected";
} else
(val, err) = callback(scan);
}
return (val, err);
}
fun parse(src: String) {
scan = newScanner(src);
val, err = parseValue(scan);
return val, err;
}
fun parse(str: String) {
context = {str: str, cur: 0};
skipSpaces(context);
val, err = parseValue(context);
fun parseFile(file: String | File) {
if type(file) == String: {
file, err = files.open(file, files.READ);
if err != none:
return none, err;
}
src, err = files.read(file);
if err != none:
return none, err;
assert(type(src) == String);
val, err = parse(src);
return val, err;
}
+120 -9
View File
@@ -1,11 +1,122 @@
json = import("json.noja");
utils = import("../utils.noja");
json = import("json.noja");
assert(json.parse("null") == none);
assert(json.parse("true") == true);
assert(json.parse("false") == false);
assert(json.parse('"Hello, world!"') == "Hello, world!");
assert(utils.compareAny(json.parse('[]'), []));
assert(utils.compareAny(json.parse('[1, 2, 3]'), [1, 2, 3]));
Error = String;
JSONValueType = (bool | float | String | Map | List | None);
TestCase = [String, ?JSONValueType, ?Error];
print("No assertions fired!\n");
fun compareLists(a: List, b: List) {
len = count(a);
if len != count(b):
return false;
i = 0;
while i < len: {
if a[i] != b[i]:
return false;
i = i+1;
}
return true;
}
fun compareMaps(a: Map, b: Map) {
keys = keysof(a);
len = count(keys);
if len != count(b):
return false;
i = 0;
while i < len: {
k = keys[i];
if a[k] != b[k]:
return false;
i = i+1;
}
return true;
}
fun compareAny(a, b) {
Ta = type(a);
Tb = type(b);
if Ta != Tb:
return false;
if Ta == List:
return compareLists(a, b);
if Ta == Map:
return compareMaps(a, b);
return a == b;
}
fun runTestCase(test_case: TestCase) {
res, err = json.parse(test_case[0]);
return compareAny(res, test_case[1])
and compareAny(err, test_case[2]);
}
fun runAllTestCases(test_cases: List) {
total = count(test_cases);
passed = 0;
evaluated = 0;
while evaluated < total: {
test_case = test_cases[evaluated];
input = test_case[0];
expres = test_case[1]; # Expected result
experr = test_case[2]; # Expected error
res, err = json.parse(input);
#print("res=", res, ", err=", err, "\n");
#print("expres=", expres, ", experr=", experr, "\n");
result = compareAny(res, expres)
and compareAny(err, experr);
if result: {
print("Test ", evaluated, ": PASSED\n");
passed = passed+1;
} else {
print("Test ", evaluated, ": FAILED\n");
print("\tInput:\n");
print("\t\t", input, "\n");
if experr == none: {
if err == none:
print("\tGot result ", res, " but was expected ", expres, "\n");
else
print("\tFailed unexpectedly [", err, "]\n");
} else if err == none:
print("\tSucceded unexpectedly. Was expected error [", experr, "]\n");
else
print("\tWas expected error [", experr, "], but got [", err, "]\n");
}
evaluated = evaluated+1;
}
return passed, (total-passed), total;
}
fun newPassingTestCase(src: String, res: JSONValueType)
return [src, res, none];
fun newFailingTestCase(src: String, err: String)
return [src, none, err];
test_cases = [
newPassingTestCase("null", none),
newPassingTestCase("true", true),
newPassingTestCase("false", false),
newPassingTestCase("\"Hello, world!\"", "Hello, world!"),
newPassingTestCase("1", 1.0),
newPassingTestCase("2.3", 2.3),
newPassingTestCase("[]", []),
newPassingTestCase("[1]", [1.0]),
newPassingTestCase("[1, 2]", [1.0, 2.0]),
newPassingTestCase("[1, 2, 3]", [1.0, 2.0, 3.0]),
newPassingTestCase("{}", {}),
newPassingTestCase("{\"k1\": true}", {k1: true}),
newPassingTestCase("{\"k1\": true, \"k2\": false}", {k1: true, k2: false})
];
passed, failed, total = runAllTestCases(test_cases);
print("passed: ", passed, "\n");
print("failed: ", failed, "\n");
print(" total: ", total, "\n");
+15
View File
@@ -317,6 +317,20 @@ static int bin_istypeof(Runtime *runtime, Object **argv, unsigned int argc, Obje
return 1;
}
static int bin_keysof(Runtime *runtime, Object **argv, unsigned int argc, Object *rets[static MAX_RETS], Error *error)
{
UNUSED(argc);
UNUSED(rets);
UNUSED(runtime);
ASSERT(argc == 1);
Heap *heap = Runtime_GetHeap(runtime);
Object *keys = Object_KeysOf(argv[0], heap, error);
if (keys == NULL)
return -1;
rets[0] = keys;
return 1;
}
void bins_basic_init(StaticMapSlot slots[])
{
slots[0].as_type = Object_GetTypeType();
@@ -361,5 +375,6 @@ StaticMapSlot bins_basic[] = {
{ "error", SM_FUNCT, .as_funct = bin_error, .argc = 1 },
{ "assert", SM_FUNCT, .as_funct = bin_assert, .argc = -1 },
{ "istypeof", SM_FUNCT, .as_funct = bin_istypeof, .argc = 2, },
{ "keysof", SM_FUNCT, .as_funct = bin_keysof, .argc = 1, },
{ NULL, SM_END, {}, {} },
};
+10
View File
@@ -275,3 +275,13 @@ static void buffer_print(Object *self, FILE *fp)
BufferObject *buffer = (BufferObject*) self;
print_bytes(fp, buffer->payload->body + buffer->offset, buffer->length);
}
static Object*
keysof(Object *self,
Heap *heap,
Error *error)
{
BufferObject *buffer = (BufferObject*) self;
int count = buffer->length;
return Object_NewListOfConsecutiveIntegers(0, count-1, heap, error);
}
+41
View File
@@ -48,6 +48,7 @@ static void walkexts(Object *self, void (*callback)(void **referer, unsigne
static Object *copy(Object *self, Heap *heap, Error *err);
static int hash(Object *self);
static bool istypeof(Object *self, Object *other, Heap *heap, Error *error);
static Object* keysof(Object *self, Heap *heap, Error *error);
static TypeObject t_list = {
.base = (Object) { .type = &t_type, .flags = Object_STATIC },
@@ -59,6 +60,7 @@ static TypeObject t_list = {
.insert = insert,
.count = count,
.print = print,
.keysof = keysof,
.istypeof = istypeof,
.walk = walk,
.walkexts = walkexts,
@@ -92,6 +94,16 @@ istypeof(Object *self,
return true;
}
static Object*
keysof(Object *self,
Heap *heap,
Error *error)
{
ListObject *list = (ListObject*) self;
int count = list->count;
return Object_NewListOfConsecutiveIntegers(0, count-1, heap, error);
}
static int hash(Object *self)
{
ASSERT(self != NULL);
@@ -304,3 +316,32 @@ static void print(Object *self, FILE *fp)
}
fprintf(fp, "]");
}
Object *Object_NewListOfConsecutiveIntegers(int first, int last, Heap *heap, Error *error)
{
int count = last - first + 1;
Object *list = Object_NewList(count, heap, error);
if (list == NULL)
return NULL;
for (int i = 0; i < count; i++) {
Object *key = Object_FromInt(i, heap, error);
if (key == NULL)
return NULL;
Object *val;
if (first == 0)
val = key;
else {
val = Object_FromInt(first+i, heap, error);
if (val == NULL)
return NULL;
}
if (!Object_Insert(list, key, val, heap, error))
return NULL;
}
return list;
}
+12 -1
View File
@@ -48,7 +48,8 @@ static void walk(Object *self, void (*callback)(Object **referer, void *userp),
static void walkexts(Object *self, void (*callback)(void **referer, unsigned int size, void *userp), void *userp);
static Object *copy(Object *self, Heap *heap, Error *err);
static int hash(Object *self);
static bool istypeof(Object *self, Object *other, Heap *heap, Error *error);
static bool istypeof(Object *self, Object *other, Heap *heap, Error *error);
static Object *keysof(Object *self, Heap *heap, Error *error);
static TypeObject t_map = {
.base = (Object) { .type = &t_type, .flags = Object_STATIC },
@@ -60,11 +61,21 @@ static TypeObject t_map = {
.insert = insert,
.count = count,
.print = print,
.keysof = keysof,
.istypeof = istypeof,
.walk = walk,
.walkexts = walkexts,
};
static Object*
keysof(Object *self,
Heap *heap,
Error *error)
{
MapObject *map = (MapObject*) self;
return Object_NewList2(map->count, map->keys, heap, error);
}
static bool
istypeof(Object *self,
Object *other,
+12
View File
@@ -49,6 +49,7 @@ static void print(Object *obj, FILE *fp);
static _Bool op_eql(Object *self, Object *other);
static void walkexts(Object *self, void (*callback)(void **referer, unsigned int size, void *userp), void *userp);
static Object *select_(Object *self, Object *key, Heap *heap, Error *error);
static Object *keysof(Object *self, Heap *heap, Error *error);
static TypeObject t_string = {
.base = (Object) { .type = &t_type, .flags = Object_STATIC },
@@ -60,6 +61,7 @@ static TypeObject t_string = {
.print = print,
.select = select_,
.op_eql = op_eql,
.keysof = keysof,
.walkexts = walkexts,
};
@@ -232,3 +234,13 @@ static void walkexts(Object *self, void (*callback)(void **referer, unsigned int
callback((void**) &str->body, str->bytes+1, userp);
}
static Object*
keysof(Object *self,
Heap *heap,
Error *error)
{
StringObject *str = (StringObject*) self;
int count = str->count;
return Object_NewListOfConsecutiveIntegers(0, count-1, heap, error);
}
+10 -1
View File
@@ -2,6 +2,7 @@
#include "objects.h"
static void walk(Object *self, void (*callback)(Object **referer, void *userp), void *userp);
static void print(Object *self, FILE *fp);
static bool istypeof(Object *self, Object *other, Heap *heap, Error *error);
typedef struct {
@@ -18,7 +19,7 @@ static TypeObject t_sum = {
.hash = NULL,
.copy = NULL,
.call = NULL,
.print = NULL,
.print = print,
.select = NULL,
.delete = NULL,
@@ -64,3 +65,11 @@ static void walk(Object *self, void (*callback)(Object **referer, void *userp),
callback(sum->items + 0, userp);
callback(sum->items + 1, userp);
}
static void print(Object *self, FILE *fp)
{
SumObject *sum = (SumObject*) self;
Object_Print(sum->items[0], fp);
fprintf(fp, " | ");
Object_Print(sum->items[1], fp);
}
+10
View File
@@ -88,6 +88,16 @@ const TypeObject *Object_GetType(const Object *obj)
return obj->type;
}
Object*
Object_KeysOf(Object *self,
Heap *heap,
Error *error)
{
if (self->type->keysof == NULL)
return Object_NewNone(heap, error);
return self->type->keysof(self, heap, error);
}
bool Object_IsTypeOf(Object *typ, Object *obj, Heap *heap, Error *error)
{
if (typ->type->istypeof == NULL)
+3
View File
@@ -71,6 +71,7 @@ struct TypeObject {
Object *(*delete)(Object *self, Object *key, Heap *heap, Error *err);
bool (*insert)(Object *self, Object *key, Object *val, Heap *heap, Error *err);
int (*count)(Object *self);
Object *(*keysof)(Object *self, Heap *heap, Error *error);
// Types.
bool (*istypeof)(Object *self, Object *other, Heap *heap, Error *error);
@@ -104,6 +105,7 @@ int Object_Hash (Object *obj);
Object* Object_Copy (Object *obj, Heap *heap, Error *err);
int Object_Call (Object *obj, Object **argv, unsigned int argc, Object *rets[static MAX_RETS], Heap *heap, Error *err);
void Object_Print(Object *obj, FILE *fp);
Object* Object_KeysOf(Object *self, Heap *heap, Error *error);
bool Object_IsTypeOf(Object *typ, Object *obj, Heap *heap, Error *error);
Object* Object_Select(Object *coll, Object *key, Heap *heap, Error *err);
Object* Object_Delete(Object *coll, Object *key, Heap *heap, Error *err);
@@ -115,6 +117,7 @@ void Object_WalkExtensions(Object *parent, void (*callback)(void **referer,
Object* Object_NewMap(int num, Heap *heap, Error *error);
Object* Object_NewList(int capacity, Heap *heap, Error *error);
Object* Object_NewList2(int num, Object **items, Heap *heap, Error *error);
Object* Object_NewListOfConsecutiveIntegers(int first, int last, Heap *heap, Error *error);
Object* Object_NewNone(Heap *heap, Error *error);
Object* Object_NewBuffer(size_t size, Heap *heap, Error *error);
Object* Object_NewClosure(Object *parent, Object *new_map, Heap *heap, Error *error);