added utf-8 string support

build.sh

@@ -14,6 +14,7 @@ done
 mkdir temp
 
 mkdir temp/utils
+gcc -c src/utils/utf8.c       -o temp/utils/utf8.o       $FLAGS
 gcc -c src/utils/hash.c       -o temp/utils/hash.o       $FLAGS
 gcc -c src/utils/stack.c      -o temp/utils/stack.o      $FLAGS
 gcc -c src/utils/error.c      -o temp/utils/error.o      $FLAGS
@@ -81,6 +82,7 @@ ar rcs build/libnoja-runtime.a \
 	build/libnoja-objects.a
 
 gcc src/main.c \
+	temp/utils/utf8.o        \
 	temp/utils/hash.o        \
 	temp/utils/stack.o       \
 	temp/utils/source.o      \

docs/language.md

@@ -35,9 +35,7 @@ The most basic type of statement is an expression. They work similarly to other
      #   Like integers, they are represented using 64 bits.
 
 "hello!"; # Strings:
-'hello!'; #   They represent text. They are encoding-agnostic, because it treats 
+'hello!'; #   They represent UTF-8 encoded text.
-          #   them as sequences of bytes. This may not be the most ergonomic choice, 
-          #   but it was the simplest one. 
 
 true;  # Booleans:
 false; #   Nothing new here. They represent two values that have the property of being

samples/list_files.noja

@@ -1,6 +1,6 @@
 
 dirname = '.';
-dir = file.openDir(dirname);
+dir = files.openDir(dirname);
 
-while (filename = file.nextDirItem(dir)) != none:
+while (filename = files.nextDirItem(dir)) != none:
 	print(filename, '\n');

samples/read_file.noja

@@ -2,6 +2,6 @@ buff = newBuffer(1024);
 name = 'samples/bubble_sort.noja';
 hdle = files.openFile(name, files.READ);
 n = files.read(hdle, buff);
-resl = sliceBuffer(buff, 0, n);
+resl = bufferToString(sliceBuffer(buff, 0, n));
 print('Read ', n, ' bytes.\n');
 print(resl);

src/builtins/basic.c

@@ -128,6 +128,21 @@ static Object *bin_sliceBuffer(Runtime *runtime, Object **argv, unsigned int arg
 	return Object_SliceBuffer(argv[0], offset, length, Runtime_GetHeap(runtime), error);
 }
 
+static Object *bin_bufferToString(Runtime *runtime, Object **argv, unsigned int argc, Error *error)
+{
+	assert(argc == 1);
+
+	void *buffaddr;
+	int   buffsize;
+
+	buffaddr = Object_GetBufferAddrAndSize(argv[0], &buffsize, error);
+
+	if(error->occurred)
+		return NULL;
+
+	return Object_FromString(buffaddr, buffsize, Runtime_GetHeap(runtime), error);
+}
+
 const StaticMapSlot bins_basic[] = {
 	{ "math",  SM_SMAP, .as_smap = bins_math, },
 	{ "files", SM_SMAP, .as_smap = bins_files, },
@@ -135,6 +150,7 @@ const StaticMapSlot bins_basic[] = {
 
 	{ "newBuffer",   SM_FUNCT, .as_funct = bin_newBuffer, .argc = 1 },
 	{ "sliceBuffer", SM_FUNCT, .as_funct = bin_sliceBuffer, .argc = 3 },
+	{ "bufferToString", SM_FUNCT, .as_funct = bin_bufferToString, .argc = 1 },
 
 	{ "strcat", SM_FUNCT, .as_funct = bin_strcat, .argc = -1 },
 

src/objects/o_string.c

@@ -2,11 +2,13 @@
 #include <assert.h>
 #include "../utils/defs.h"
 #include "../utils/hash.h"
+#include "../utils/utf8.h"
 #include "objects.h"
 
 typedef struct {
 	Object  base;
-	int     size;
+	int     count;
+	int     bytes;
 	char   *body;
 } StringObject;
 
@@ -43,7 +45,7 @@ static char *to_string(Object *self, int *size, Heap *heap, Error *err)
 	StringObject *s = (StringObject*) self;
 
 	if(size)
-		*size = s->size;
+		*size = s->bytes;
 
 	return s->body;
 }
@@ -57,13 +59,22 @@ Object *Object_FromString(const char *str, int len, Heap *heap, Error *error)
 	if(len < 0)
 		len = strlen(str);
 
+	int count = utf8_strlen(str, len);
+
+	if(count < 0)
+		{
+			Error_Report(error, 0, "Invalid UTF-8 sequence");
+			return NULL;
+		}
+
 	StringObject *strobj = Heap_Malloc(heap, &t_string, error);
 
 	if(strobj == NULL)
 		return NULL;
 
 	strobj->body = Heap_RawMalloc(heap, len+1, error);
-	strobj->size = len;
+	strobj->bytes = len;
+	strobj->count = count;
 
 	if(strobj->body == NULL)
 		return NULL;
@@ -82,7 +93,7 @@ static int count(Object *self)
 
 	StringObject *strobj = (StringObject*) self;
 
-	return strobj->size;
+	return strobj->count;
 }
 
 static int hash(Object *self)
@@ -92,7 +103,7 @@ static int hash(Object *self)
 
 	StringObject *strobj = (StringObject*) self;
 
-	return hashbytes((unsigned char*) strobj->body, strobj->size);
+	return hashbytes((unsigned char*) strobj->body, strobj->count);
 }
 
 static Object *copy(Object *self, Heap *heap, Error *err)
@@ -115,7 +126,7 @@ static _Bool op_eql(Object *self, Object *other)
 	StringObject *s1 = (StringObject*) self;
 	StringObject *s2 = (StringObject*) other;
 
-	_Bool match = s1->size == s2->size && !strncmp(s1->body, s2->body, s1->size);
+	_Bool match = s1->bytes == s2->bytes && !strncmp(s1->body, s2->body, s1->bytes);
 
 	return match;
 }
@@ -128,12 +139,12 @@ static void print(Object *obj, FILE *fp)
 
 	StringObject *str = (StringObject*) obj;
 
-	fprintf(fp, "%.*s", str->size, str->body);
+	fprintf(fp, "%.*s", str->bytes, str->body);
 }
 
 static void walkexts(Object *self, void (*callback)(void **referer, unsigned int size, void *userp), void *userp)
 {
 	StringObject *str = (StringObject*) self;
 	
-	callback((void**) &str->body, str->size+1, userp);
+	callback((void**) &str->body, str->bytes+1, userp);
 }

src/utils/utf8.c

@@ -0,0 +1,423 @@
+#include <assert.h>
+#include <stddef.h> // NULL
+#include "utf8.h"
+
+// If this is turned on, this library will assume that the 
+// UTF-8 strings will mainly contain ASCII characters.
+#define ASSUME_ASCII 1
+
+/* SYMBOL
+**   utf8_sequence_from_utf32_codepoint
+**
+** DESCRIPTION
+**   Transform a UTF-32 encoded codepoint to a UTF-8 encoded byte sequence.
+**
+** ARGUMENTS
+**   The [utf8_data] pointer refers to the location where the UTF-8 sequence
+**   will be stored.
+**
+**   The [nbytes] argument specifies the maximum number of bytes that can
+**   be written to [utf8_data]. It can't be negative.
+**
+**   The [utf32_code] argument is the UTF-32 code that will be converted.
+**
+** RETURN
+**   If [utf32_code] is valid UTF-32 and the provided buffer is big enough, 
+**   the UTF-8 equivalent sequence is stored in [utf8_data]. No more than
+**   [nbytes] are ever written. If one of those conitions isn't true, -1 is
+**   returned.
+*/
+int utf8_sequence_from_utf32_codepoint(char *utf8_data, int nbytes, uint32_t utf32_code)
+{
+    if(utf32_code < 128)
+        {
+            if(nbytes < 1)
+                return -1;
+
+            utf8_data[0] = utf32_code;
+            return 1;
+        }
+
+    if(utf32_code < 2048)
+        {
+            if(nbytes < 2)
+                return -1;
+
+            utf8_data[0] = 0xc0 | (utf32_code >> 6);
+            utf8_data[1] = 0x80 | (utf32_code & 0x3f);
+            return 2;
+        }
+
+    if(utf32_code < 65536)
+        {
+            if(nbytes < 3)
+                return -1;
+
+            utf8_data[0] = 0xe0 | (utf32_code >> 12);
+            utf8_data[1] = 0x80 | ((utf32_code >> 6) & 0x3f);
+            utf8_data[2] = 0x80 | (utf32_code & 0x3f);
+            return 3;
+        }
+
+    if(utf32_code <= 0x10ffff)
+        {
+            if(nbytes < 4)
+                return -1;
+
+            utf8_data[0] = 0xf0 | (utf32_code >> 18);
+            utf8_data[1] = 0x80 | ((utf32_code >> 12) & 0x3f);
+            utf8_data[2] = 0x80 | ((utf32_code >>  6) & 0x3f);
+            utf8_data[3] = 0x80 | (utf32_code & 0x3f);
+            return 4;
+        }
+
+    // Code is out of range for UTF-8.
+    return -1;
+}
+
+/* SYMBOL
+**   utf8_sequence_to_utf32_codepoint
+**
+** DESCRIPTION
+**   Transform a UTF-8 encoded byte sequence pointed by `utf8_data`
+**   into a UTF-32 encoded codepoint.
+**
+** ARGUMENTS
+**   The [utf8_data] pointer refers to the location of the UTF-8 sequence.
+**
+**   The [nbytes] argument specifies the maximum number of bytes that can
+**   be read after [utf8_data]. It can't be negative. 
+**
+**   NOTE: The [nbytes] argument has no relation to the UTF-8 byte count sequence. 
+**         You may think about this argument as the "raw" string length (the one 
+**         [strlen] whould return if [utf8_data] were zero-terminated). 
+**
+**   The [utf32_code] argument is the location where the encoded UTF-32 code
+**   will be stored. It may be NULL, in which case the value is evaluated and then
+**   thrown away.
+**
+** RETURN
+**   The codepoint is returned through the output parameter `utf32_code`.
+**   The returned value is the number of bytes of the UTF-8 sequence that
+**   were scanned to encode the UTF-32 code, or -1 if the UTF-8 sequence
+**   is invalid.
+**
+** NOTE: By calling this function with a NULL [utf32_code], you can check the
+**       validity of a UTF-8 sequence.
+*/
+int utf8_sequence_to_utf32_codepoint(const char *utf8_data, int nbytes, uint32_t *utf32_code)
+{
+    assert(utf8_data != NULL);
+    assert(nbytes >= 0);
+
+    uint32_t dummy;
+    if(utf32_code == NULL)
+        utf32_code = &dummy;
+
+    if(nbytes == 0)
+        return -1;
+
+    if(utf8_data[0] & 0x80)
+        {
+            // May be UTF-8.
+            
+            if((unsigned char) utf8_data[0] >= 0xF0)
+                {
+                    // 4 bytes.
+                    // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+
+                    if(nbytes < 4)
+                        return -1;
+                    
+                    uint32_t temp 
+                        = (((uint32_t) utf8_data[0] & 0x07) << 18) 
+                        | (((uint32_t) utf8_data[1] & 0x3f) << 12)
+                        | (((uint32_t) utf8_data[2] & 0x3f) <<  6)
+                        | (((uint32_t) utf8_data[3] & 0x3f));
+
+                    if(temp > 0x10ffff)
+                        return -1;
+
+                    *utf32_code = temp;
+                    return 4;
+                }
+            
+            if((unsigned char) utf8_data[0] >= 0xE0)
+                {
+                    // 3 bytes.
+                    // 1110xxxx 10xxxxxx 10xxxxxx
+
+                    if(nbytes < 3)
+                        return -1;
+
+                    uint32_t temp
+                        = (((uint32_t) utf8_data[0] & 0x0f) << 12)
+                        | (((uint32_t) utf8_data[1] & 0x3f) <<  6)
+                        | (((uint32_t) utf8_data[2] & 0x3f));
+                    
+                    if(temp > 0x10ffff)
+                        return -1;
+
+                    *utf32_code = temp;
+                    return 3;
+                }
+            
+            if((unsigned char) utf8_data[0] >= 0xC0)
+                {
+                    // 2 bytes.
+                    // 110xxxxx 10xxxxxx
+
+                    if(nbytes < 2)
+                        return -1;
+
+                    *utf32_code 
+                        = (((uint32_t) utf8_data[0] & 0x1f) << 6)
+                        | (((uint32_t) utf8_data[1] & 0x3f));
+                    
+                    assert(*utf32_code <= 0x10ffff);
+                    return 2;
+                }
+            
+            // 1 byte
+            // 10xxxxxx
+            *utf32_code = (uint32_t) utf8_data[0] & 0x3f;
+            return 1;
+        }
+
+    // It's ASCII
+    // 0xxxxxxx
+
+    *utf32_code = (uint32_t) utf8_data[0];
+    return 1;
+}
+
+/* SYMBOL
+**   utf8_strlen
+**
+** DESCRIPTION
+**   Count the number of characters of a UTF-8 string. 
+**
+**   NOTE: By "character" we mean a valid UTF-8 sequence. 
+**
+** ARGUMENTS
+**   The [utf8_data] pointer refers to the location of the UTF-8 string.
+**
+**   The [nbytes] argument specifies the byte count of the string referred
+**   by [utf8_data]. It can't be negative. 
+**
+** RETURN
+**   Returns the number of characters encoded by [utf8_data], or -1 if
+**   the string is not valid UTF-8.
+**
+** NOTE: By calling this function on an ASCII-only string, the return
+**       value is equal to [nbytes].
+**
+** NOTE: You can check the validity of a UTF-8 string
+**       by calling this function and checking that it's
+**       return value is not negative.
+*/
+int utf8_strlen(const char *utf8_data, int nbytes)
+{
+    assert(utf8_data != NULL);
+    assert(nbytes >= 0);
+
+    int len = 0;
+
+    int i = 0;
+    while(i < nbytes)
+        {
+
+#if ASSUME_ASCII
+            {
+                int ASCII_start = i;
+
+                // Skip through ASCII
+                while(i < nbytes && (utf8_data[i] & 0x80) == 0)
+                    i += 1;
+
+                int ASCII_end = i;
+
+                len += (ASCII_end - ASCII_start);
+
+                // Either we scanned through all of the
+                // string, or we encountered some unicode.
+
+                if(i == nbytes)
+                    // String ended.
+                    break;
+            }
+#endif
+
+            // Found unicode.
+            {
+                int n = utf8_sequence_to_utf32_codepoint(utf8_data + i, nbytes - i, NULL);
+
+                if(n < 1)
+                    return -1;
+
+                i += n;
+                len += 1;
+            }
+        }
+    return len;
+}
+
+/* SYMBOL
+**   utf8_prev
+**
+** DESCRIPTION
+**   Get the UTF-8 sequence that comes before a given byte index
+**   inside a given string.
+**   
+**   NOTE: This is what you use when you want to iterate over a
+**         UTF-8 string backwards.
+**
+** ARGUMENTS
+**   The [utf8_data] pointer refers to the location of the UTF-8 string.
+**
+**   The [nbytes] argument is the raw size of the [utf8_data] string.
+**   It can't be negative. 
+**
+**   The [idx] argument is the index of the byte that follows the UTF-8 
+**   sequence to be decoded.
+**
+**   The [utf32_code] argument, if not NULL, is used to return the UTF-32
+**   version of the decoded UTF-8 sequence.
+**
+** RETURN
+**   Returns the index of the first byte of the decoded UTF-8 sequence,
+**   or -1 is the sequence wasn't valid UTF-8.
+**
+**   NOTE: If the function didn't fail, by subtracting the returned value
+**         from [idx], you'll get the number of bytes of the decoded 
+**         sequence.
+*/
+int utf8_prev(const char *utf8_data, int nbytes, int idx, uint32_t *utf32_code)
+{
+    assert(idx >= 0);
+    assert(idx <= nbytes);
+
+    // [idx] currently refers to the head byte
+    // of a UTF-8 sequence. We need to first
+    // get to the last byte of the previous
+    // sequence.
+    idx -= 1;
+
+    if(idx == -1)
+        // There was no previous sequence!
+        return 0; // Return the same index that was provided.
+
+    int tail = idx;
+
+#if ASSUME_ASCII
+    {
+        // This block isn't necessary for
+        // this function to work but it
+        // makes strings that are mainly ascii
+        // to go faster.
+
+        if((utf8_data[tail] & 0x80) == 0)
+            {
+                if(utf32_code)
+                    *utf32_code = utf8_data[tail];
+                return tail;
+            }
+    }
+#endif
+
+    // Skip all of the auxiliary bytes in the
+    // form '10xxxxxx'.
+
+    while(idx > -1 && (utf8_data[idx] & 0xc0) == 0x80)
+        idx -= 1;
+
+    if(idx == -1)
+        {
+            // No head sequence byte was found,
+            // so this isn't valid UTF-8.
+            return -1;
+        }
+
+    // The index of the head byte.
+    int head = idx;
+
+    // The number of auxiliary bytes is given
+    // by the difference
+    int aux = tail - head;
+
+    // The total number of bytes of the
+    // sequence is [aux + 1].
+
+    int n = utf8_sequence_to_utf32_codepoint(utf8_data + head, aux + 1, utf32_code);
+
+    if(n < 1)
+        // The sequence wasn't valid UTF-8.
+        return -1;
+
+    assert(n > 0);
+
+    if(n < aux + 1)
+        // Not all of the auxiliary bytes were considered while parsing.
+        return -1;
+
+    assert(n == aux + 1);
+
+    return head;
+}
+
+/* SYMBOL
+**   utf8_next
+**
+** DESCRIPTION
+**   Get the UTF-8 sequence from a UTF-8 string that starts AFTER the
+**   sequence that starts at a given byte index.
+**   
+**   NOTE: This is what you use when you want to iterate over a
+**         UTF-8 string.
+**
+** ARGUMENTS
+**   The [utf8_data] pointer refers to the location of the UTF-8 string.
+**
+**   The [nbytes] argument is the raw size of the [utf8_data] string.
+**   It can't be negative. 
+**
+**   The [idx] argument is the index of the first byte of the sequence 
+**   that comes before the sequence to be decoded.
+**
+**   The [utf32_code] argument, if not NULL, is used to return the UTF-32
+**   version of the decoded UTF-8 sequence.
+**
+** RETURN
+**   Returns the index of the first byte of the decoded UTF-8 sequence,
+**   or -1 is the sequence wasn't valid UTF-8.
+**
+**   NOTE: If the function didn't fail, by subtracting [idx] from the 
+**         returned value, you'll get the number of bytes of the decoded 
+**         sequence.
+*/
+int utf8_next(const char *utf8_data, int nbytes, int idx, uint32_t *utf32_code)
+{
+    // Get the byte count of the current sequence.
+    int n = utf8_sequence_to_utf32_codepoint(utf8_data + idx, nbytes, NULL);
+    
+    if(n < 1)
+        return -1;
+
+    // Now get the codepoint of the next sequence.
+    int k = utf8_sequence_to_utf32_codepoint(utf8_data + idx + n, nbytes, utf32_code);
+
+    if(k < 1)
+        return -1;
+
+    return idx + n;
+}
+
+int utf8_curr(const char *utf8_data, int nbytes, int idx, uint32_t *utf32_code)
+{
+    assert(idx >= 0);
+    assert(idx < nbytes);
+
+    int n = utf8_sequence_to_utf32_codepoint(utf8_data + idx, nbytes - idx, utf32_code);
+
+    return n > 0;
+}

src/utils/utf8.h

@@ -0,0 +1,10 @@
+#ifndef XUTF8_H
+#define XUTF8_H
+#include <stdint.h> // uint32_t
+int utf8_sequence_from_utf32_codepoint(char *utf8_data, int nbytes, uint32_t utf32_code);
+int utf8_sequence_to_utf32_codepoint(const char *utf8_data, int nbytes, uint32_t *utf32_code);
+int utf8_strlen(const char *utf8_data, int nbytes);
+int utf8_prev(const char *utf8_data, int nbytes, int idx, uint32_t *utf32_code);
+int utf8_next(const char *utf8_data, int nbytes, int idx, uint32_t *utf32_code);
+int utf8_curr(const char *utf8_data, int nbytes, int idx, uint32_t *utf32_code);
+#endif // #ifndef UTF8_H