diff --git a/build.sh b/build.sh index 627de43..df8837f 100755 --- a/build.sh +++ b/build.sh @@ -14,6 +14,7 @@ done mkdir temp mkdir temp/utils +gcc -c src/utils/utf8.c -o temp/utils/utf8.o $FLAGS gcc -c src/utils/hash.c -o temp/utils/hash.o $FLAGS gcc -c src/utils/stack.c -o temp/utils/stack.o $FLAGS gcc -c src/utils/error.c -o temp/utils/error.o $FLAGS @@ -81,6 +82,7 @@ ar rcs build/libnoja-runtime.a \ build/libnoja-objects.a gcc src/main.c \ + temp/utils/utf8.o \ temp/utils/hash.o \ temp/utils/stack.o \ temp/utils/source.o \ diff --git a/docs/language.md b/docs/language.md index 7fbbfea..a39fc4c 100644 --- a/docs/language.md +++ b/docs/language.md @@ -35,9 +35,7 @@ The most basic type of statement is an expression. They work similarly to other # Like integers, they are represented using 64 bits. "hello!"; # Strings: -'hello!'; # They represent text. They are encoding-agnostic, because it treats - # them as sequences of bytes. This may not be the most ergonomic choice, - # but it was the simplest one. +'hello!'; # They represent UTF-8 encoded text. true; # Booleans: false; # Nothing new here. They represent two values that have the property of being diff --git a/samples/list_files.noja b/samples/list_files.noja index 6f75871..15be59d 100644 --- a/samples/list_files.noja +++ b/samples/list_files.noja @@ -1,6 +1,6 @@ dirname = '.'; -dir = file.openDir(dirname); +dir = files.openDir(dirname); -while (filename = file.nextDirItem(dir)) != none: - print(filename, '\n'); +while (filename = files.nextDirItem(dir)) != none: + print(filename, '\n'); \ No newline at end of file diff --git a/samples/read_file.noja b/samples/read_file.noja index 86de4d5..d8c1a60 100644 --- a/samples/read_file.noja +++ b/samples/read_file.noja @@ -2,6 +2,6 @@ buff = newBuffer(1024); name = 'samples/bubble_sort.noja'; hdle = files.openFile(name, files.READ); n = files.read(hdle, buff); -resl = sliceBuffer(buff, 0, n); +resl = bufferToString(sliceBuffer(buff, 0, n)); print('Read ', n, ' bytes.\n'); print(resl); \ No newline at end of file diff --git a/src/builtins/basic.c b/src/builtins/basic.c index f890a8e..767d795 100644 --- a/src/builtins/basic.c +++ b/src/builtins/basic.c @@ -128,6 +128,21 @@ static Object *bin_sliceBuffer(Runtime *runtime, Object **argv, unsigned int arg return Object_SliceBuffer(argv[0], offset, length, Runtime_GetHeap(runtime), error); } +static Object *bin_bufferToString(Runtime *runtime, Object **argv, unsigned int argc, Error *error) +{ + assert(argc == 1); + + void *buffaddr; + int buffsize; + + buffaddr = Object_GetBufferAddrAndSize(argv[0], &buffsize, error); + + if(error->occurred) + return NULL; + + return Object_FromString(buffaddr, buffsize, Runtime_GetHeap(runtime), error); +} + const StaticMapSlot bins_basic[] = { { "math", SM_SMAP, .as_smap = bins_math, }, { "files", SM_SMAP, .as_smap = bins_files, }, @@ -135,6 +150,7 @@ const StaticMapSlot bins_basic[] = { { "newBuffer", SM_FUNCT, .as_funct = bin_newBuffer, .argc = 1 }, { "sliceBuffer", SM_FUNCT, .as_funct = bin_sliceBuffer, .argc = 3 }, + { "bufferToString", SM_FUNCT, .as_funct = bin_bufferToString, .argc = 1 }, { "strcat", SM_FUNCT, .as_funct = bin_strcat, .argc = -1 }, diff --git a/src/main.c b/src/main.c index 3f104a1..bbd8e95 100644 --- a/src/main.c +++ b/src/main.c @@ -334,4 +334,4 @@ int main(int argc, char **argv) fprintf(stderr, "Error: Incorrect usage.\n\n"); fprintf(stderr, usage); return -1; -} +} \ No newline at end of file diff --git a/src/objects/o_string.c b/src/objects/o_string.c index d6fea11..df284cc 100644 --- a/src/objects/o_string.c +++ b/src/objects/o_string.c @@ -2,11 +2,13 @@ #include #include "../utils/defs.h" #include "../utils/hash.h" +#include "../utils/utf8.h" #include "objects.h" typedef struct { Object base; - int size; + int count; + int bytes; char *body; } StringObject; @@ -43,7 +45,7 @@ static char *to_string(Object *self, int *size, Heap *heap, Error *err) StringObject *s = (StringObject*) self; if(size) - *size = s->size; + *size = s->bytes; return s->body; } @@ -57,13 +59,22 @@ Object *Object_FromString(const char *str, int len, Heap *heap, Error *error) if(len < 0) len = strlen(str); + int count = utf8_strlen(str, len); + + if(count < 0) + { + Error_Report(error, 0, "Invalid UTF-8 sequence"); + return NULL; + } + StringObject *strobj = Heap_Malloc(heap, &t_string, error); if(strobj == NULL) return NULL; strobj->body = Heap_RawMalloc(heap, len+1, error); - strobj->size = len; + strobj->bytes = len; + strobj->count = count; if(strobj->body == NULL) return NULL; @@ -82,7 +93,7 @@ static int count(Object *self) StringObject *strobj = (StringObject*) self; - return strobj->size; + return strobj->count; } static int hash(Object *self) @@ -92,7 +103,7 @@ static int hash(Object *self) StringObject *strobj = (StringObject*) self; - return hashbytes((unsigned char*) strobj->body, strobj->size); + return hashbytes((unsigned char*) strobj->body, strobj->count); } static Object *copy(Object *self, Heap *heap, Error *err) @@ -115,7 +126,7 @@ static _Bool op_eql(Object *self, Object *other) StringObject *s1 = (StringObject*) self; StringObject *s2 = (StringObject*) other; - _Bool match = s1->size == s2->size && !strncmp(s1->body, s2->body, s1->size); + _Bool match = s1->bytes == s2->bytes && !strncmp(s1->body, s2->body, s1->bytes); return match; } @@ -128,12 +139,12 @@ static void print(Object *obj, FILE *fp) StringObject *str = (StringObject*) obj; - fprintf(fp, "%.*s", str->size, str->body); + fprintf(fp, "%.*s", str->bytes, str->body); } static void walkexts(Object *self, void (*callback)(void **referer, unsigned int size, void *userp), void *userp) { StringObject *str = (StringObject*) self; - callback((void**) &str->body, str->size+1, userp); + callback((void**) &str->body, str->bytes+1, userp); } \ No newline at end of file diff --git a/src/utils/utf8.c b/src/utils/utf8.c new file mode 100644 index 0000000..c48f09f --- /dev/null +++ b/src/utils/utf8.c @@ -0,0 +1,423 @@ +#include +#include // NULL +#include "utf8.h" + +// If this is turned on, this library will assume that the +// UTF-8 strings will mainly contain ASCII characters. +#define ASSUME_ASCII 1 + +/* SYMBOL +** utf8_sequence_from_utf32_codepoint +** +** DESCRIPTION +** Transform a UTF-32 encoded codepoint to a UTF-8 encoded byte sequence. +** +** ARGUMENTS +** The [utf8_data] pointer refers to the location where the UTF-8 sequence +** will be stored. +** +** The [nbytes] argument specifies the maximum number of bytes that can +** be written to [utf8_data]. It can't be negative. +** +** The [utf32_code] argument is the UTF-32 code that will be converted. +** +** RETURN +** If [utf32_code] is valid UTF-32 and the provided buffer is big enough, +** the UTF-8 equivalent sequence is stored in [utf8_data]. No more than +** [nbytes] are ever written. If one of those conitions isn't true, -1 is +** returned. +*/ +int utf8_sequence_from_utf32_codepoint(char *utf8_data, int nbytes, uint32_t utf32_code) +{ + if(utf32_code < 128) + { + if(nbytes < 1) + return -1; + + utf8_data[0] = utf32_code; + return 1; + } + + if(utf32_code < 2048) + { + if(nbytes < 2) + return -1; + + utf8_data[0] = 0xc0 | (utf32_code >> 6); + utf8_data[1] = 0x80 | (utf32_code & 0x3f); + return 2; + } + + if(utf32_code < 65536) + { + if(nbytes < 3) + return -1; + + utf8_data[0] = 0xe0 | (utf32_code >> 12); + utf8_data[1] = 0x80 | ((utf32_code >> 6) & 0x3f); + utf8_data[2] = 0x80 | (utf32_code & 0x3f); + return 3; + } + + if(utf32_code <= 0x10ffff) + { + if(nbytes < 4) + return -1; + + utf8_data[0] = 0xf0 | (utf32_code >> 18); + utf8_data[1] = 0x80 | ((utf32_code >> 12) & 0x3f); + utf8_data[2] = 0x80 | ((utf32_code >> 6) & 0x3f); + utf8_data[3] = 0x80 | (utf32_code & 0x3f); + return 4; + } + + // Code is out of range for UTF-8. + return -1; +} + +/* SYMBOL +** utf8_sequence_to_utf32_codepoint +** +** DESCRIPTION +** Transform a UTF-8 encoded byte sequence pointed by `utf8_data` +** into a UTF-32 encoded codepoint. +** +** ARGUMENTS +** The [utf8_data] pointer refers to the location of the UTF-8 sequence. +** +** The [nbytes] argument specifies the maximum number of bytes that can +** be read after [utf8_data]. It can't be negative. +** +** NOTE: The [nbytes] argument has no relation to the UTF-8 byte count sequence. +** You may think about this argument as the "raw" string length (the one +** [strlen] whould return if [utf8_data] were zero-terminated). +** +** The [utf32_code] argument is the location where the encoded UTF-32 code +** will be stored. It may be NULL, in which case the value is evaluated and then +** thrown away. +** +** RETURN +** The codepoint is returned through the output parameter `utf32_code`. +** The returned value is the number of bytes of the UTF-8 sequence that +** were scanned to encode the UTF-32 code, or -1 if the UTF-8 sequence +** is invalid. +** +** NOTE: By calling this function with a NULL [utf32_code], you can check the +** validity of a UTF-8 sequence. +*/ +int utf8_sequence_to_utf32_codepoint(const char *utf8_data, int nbytes, uint32_t *utf32_code) +{ + assert(utf8_data != NULL); + assert(nbytes >= 0); + + uint32_t dummy; + if(utf32_code == NULL) + utf32_code = &dummy; + + if(nbytes == 0) + return -1; + + if(utf8_data[0] & 0x80) + { + // May be UTF-8. + + if((unsigned char) utf8_data[0] >= 0xF0) + { + // 4 bytes. + // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + + if(nbytes < 4) + return -1; + + uint32_t temp + = (((uint32_t) utf8_data[0] & 0x07) << 18) + | (((uint32_t) utf8_data[1] & 0x3f) << 12) + | (((uint32_t) utf8_data[2] & 0x3f) << 6) + | (((uint32_t) utf8_data[3] & 0x3f)); + + if(temp > 0x10ffff) + return -1; + + *utf32_code = temp; + return 4; + } + + if((unsigned char) utf8_data[0] >= 0xE0) + { + // 3 bytes. + // 1110xxxx 10xxxxxx 10xxxxxx + + if(nbytes < 3) + return -1; + + uint32_t temp + = (((uint32_t) utf8_data[0] & 0x0f) << 12) + | (((uint32_t) utf8_data[1] & 0x3f) << 6) + | (((uint32_t) utf8_data[2] & 0x3f)); + + if(temp > 0x10ffff) + return -1; + + *utf32_code = temp; + return 3; + } + + if((unsigned char) utf8_data[0] >= 0xC0) + { + // 2 bytes. + // 110xxxxx 10xxxxxx + + if(nbytes < 2) + return -1; + + *utf32_code + = (((uint32_t) utf8_data[0] & 0x1f) << 6) + | (((uint32_t) utf8_data[1] & 0x3f)); + + assert(*utf32_code <= 0x10ffff); + return 2; + } + + // 1 byte + // 10xxxxxx + *utf32_code = (uint32_t) utf8_data[0] & 0x3f; + return 1; + } + + // It's ASCII + // 0xxxxxxx + + *utf32_code = (uint32_t) utf8_data[0]; + return 1; +} + +/* SYMBOL +** utf8_strlen +** +** DESCRIPTION +** Count the number of characters of a UTF-8 string. +** +** NOTE: By "character" we mean a valid UTF-8 sequence. +** +** ARGUMENTS +** The [utf8_data] pointer refers to the location of the UTF-8 string. +** +** The [nbytes] argument specifies the byte count of the string referred +** by [utf8_data]. It can't be negative. +** +** RETURN +** Returns the number of characters encoded by [utf8_data], or -1 if +** the string is not valid UTF-8. +** +** NOTE: By calling this function on an ASCII-only string, the return +** value is equal to [nbytes]. +** +** NOTE: You can check the validity of a UTF-8 string +** by calling this function and checking that it's +** return value is not negative. +*/ +int utf8_strlen(const char *utf8_data, int nbytes) +{ + assert(utf8_data != NULL); + assert(nbytes >= 0); + + int len = 0; + + int i = 0; + while(i < nbytes) + { + +#if ASSUME_ASCII + { + int ASCII_start = i; + + // Skip through ASCII + while(i < nbytes && (utf8_data[i] & 0x80) == 0) + i += 1; + + int ASCII_end = i; + + len += (ASCII_end - ASCII_start); + + // Either we scanned through all of the + // string, or we encountered some unicode. + + if(i == nbytes) + // String ended. + break; + } +#endif + + // Found unicode. + { + int n = utf8_sequence_to_utf32_codepoint(utf8_data + i, nbytes - i, NULL); + + if(n < 1) + return -1; + + i += n; + len += 1; + } + } + return len; +} + +/* SYMBOL +** utf8_prev +** +** DESCRIPTION +** Get the UTF-8 sequence that comes before a given byte index +** inside a given string. +** +** NOTE: This is what you use when you want to iterate over a +** UTF-8 string backwards. +** +** ARGUMENTS +** The [utf8_data] pointer refers to the location of the UTF-8 string. +** +** The [nbytes] argument is the raw size of the [utf8_data] string. +** It can't be negative. +** +** The [idx] argument is the index of the byte that follows the UTF-8 +** sequence to be decoded. +** +** The [utf32_code] argument, if not NULL, is used to return the UTF-32 +** version of the decoded UTF-8 sequence. +** +** RETURN +** Returns the index of the first byte of the decoded UTF-8 sequence, +** or -1 is the sequence wasn't valid UTF-8. +** +** NOTE: If the function didn't fail, by subtracting the returned value +** from [idx], you'll get the number of bytes of the decoded +** sequence. +*/ +int utf8_prev(const char *utf8_data, int nbytes, int idx, uint32_t *utf32_code) +{ + assert(idx >= 0); + assert(idx <= nbytes); + + // [idx] currently refers to the head byte + // of a UTF-8 sequence. We need to first + // get to the last byte of the previous + // sequence. + idx -= 1; + + if(idx == -1) + // There was no previous sequence! + return 0; // Return the same index that was provided. + + int tail = idx; + +#if ASSUME_ASCII + { + // This block isn't necessary for + // this function to work but it + // makes strings that are mainly ascii + // to go faster. + + if((utf8_data[tail] & 0x80) == 0) + { + if(utf32_code) + *utf32_code = utf8_data[tail]; + return tail; + } + } +#endif + + // Skip all of the auxiliary bytes in the + // form '10xxxxxx'. + + while(idx > -1 && (utf8_data[idx] & 0xc0) == 0x80) + idx -= 1; + + if(idx == -1) + { + // No head sequence byte was found, + // so this isn't valid UTF-8. + return -1; + } + + // The index of the head byte. + int head = idx; + + // The number of auxiliary bytes is given + // by the difference + int aux = tail - head; + + // The total number of bytes of the + // sequence is [aux + 1]. + + int n = utf8_sequence_to_utf32_codepoint(utf8_data + head, aux + 1, utf32_code); + + if(n < 1) + // The sequence wasn't valid UTF-8. + return -1; + + assert(n > 0); + + if(n < aux + 1) + // Not all of the auxiliary bytes were considered while parsing. + return -1; + + assert(n == aux + 1); + + return head; +} + +/* SYMBOL +** utf8_next +** +** DESCRIPTION +** Get the UTF-8 sequence from a UTF-8 string that starts AFTER the +** sequence that starts at a given byte index. +** +** NOTE: This is what you use when you want to iterate over a +** UTF-8 string. +** +** ARGUMENTS +** The [utf8_data] pointer refers to the location of the UTF-8 string. +** +** The [nbytes] argument is the raw size of the [utf8_data] string. +** It can't be negative. +** +** The [idx] argument is the index of the first byte of the sequence +** that comes before the sequence to be decoded. +** +** The [utf32_code] argument, if not NULL, is used to return the UTF-32 +** version of the decoded UTF-8 sequence. +** +** RETURN +** Returns the index of the first byte of the decoded UTF-8 sequence, +** or -1 is the sequence wasn't valid UTF-8. +** +** NOTE: If the function didn't fail, by subtracting [idx] from the +** returned value, you'll get the number of bytes of the decoded +** sequence. +*/ +int utf8_next(const char *utf8_data, int nbytes, int idx, uint32_t *utf32_code) +{ + // Get the byte count of the current sequence. + int n = utf8_sequence_to_utf32_codepoint(utf8_data + idx, nbytes, NULL); + + if(n < 1) + return -1; + + // Now get the codepoint of the next sequence. + int k = utf8_sequence_to_utf32_codepoint(utf8_data + idx + n, nbytes, utf32_code); + + if(k < 1) + return -1; + + return idx + n; +} + +int utf8_curr(const char *utf8_data, int nbytes, int idx, uint32_t *utf32_code) +{ + assert(idx >= 0); + assert(idx < nbytes); + + int n = utf8_sequence_to_utf32_codepoint(utf8_data + idx, nbytes - idx, utf32_code); + + return n > 0; +} \ No newline at end of file diff --git a/src/utils/utf8.h b/src/utils/utf8.h new file mode 100644 index 0000000..bd55920 --- /dev/null +++ b/src/utils/utf8.h @@ -0,0 +1,10 @@ +#ifndef XUTF8_H +#define XUTF8_H +#include // uint32_t +int utf8_sequence_from_utf32_codepoint(char *utf8_data, int nbytes, uint32_t utf32_code); +int utf8_sequence_to_utf32_codepoint(const char *utf8_data, int nbytes, uint32_t *utf32_code); +int utf8_strlen(const char *utf8_data, int nbytes); +int utf8_prev(const char *utf8_data, int nbytes, int idx, uint32_t *utf32_code); +int utf8_next(const char *utf8_data, int nbytes, int idx, uint32_t *utf32_code); +int utf8_curr(const char *utf8_data, int nbytes, int idx, uint32_t *utf32_code); +#endif // #ifndef UTF8_H \ No newline at end of file