cozis/Noja
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added utf-8 string support

Browse Source
This commit is contained in:
cozis
2022-03-13 00:40:10 +01:00
parent 6017f99b07
commit ee68905f92
9 changed files with 476 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build.sh
+2
View File
@@ -14,6 +14,7 @@ done
mkdir temp
mkdir temp/utils
gcc -c src/utils/utf8.c -o temp/utils/utf8.o $FLAGS
gcc -c src/utils/hash.c -o temp/utils/hash.o $FLAGS
gcc -c src/utils/stack.c -o temp/utils/stack.o $FLAGS
gcc -c src/utils/error.c -o temp/utils/error.o $FLAGS
@@ -81,6 +82,7 @@ ar rcs build/libnoja-runtime.a \
build/libnoja-objects.a
gcc src/main.c \
temp/utils/utf8.o \
temp/utils/hash.o \
temp/utils/stack.o \
temp/utils/source.o \
docs/language.md
+1 -3
View File
@@ -35,9 +35,7 @@ The most basic type of statement is an expression. They work similarly to other
# Like integers, they are represented using 64 bits.
"hello!"; # Strings:
'hello!'; # They represent text. They are encoding-agnostic, because it treats
# them as sequences of bytes. This may not be the most ergonomic choice,
# but it was the simplest one.
'hello!'; # They represent UTF-8 encoded text.
true; # Booleans:
false; # Nothing new here. They represent two values that have the property of being
samples/list_files.noja
+3 -3
View File
@@ -1,6 +1,6 @@
dirname = '.';
dir = file.openDir(dirname);
dir = files.openDir(dirname);
while (filename = file.nextDirItem(dir)) != none:
print(filename, '\n');
while (filename = files.nextDirItem(dir)) != none:
print(filename, '\n');
samples/read_file.noja
+1 -1
View File
@@ -2,6 +2,6 @@ buff = newBuffer(1024);
name = 'samples/bubble_sort.noja';
hdle = files.openFile(name, files.READ);
n = files.read(hdle, buff);
resl = sliceBuffer(buff, 0, n);
resl = bufferToString(sliceBuffer(buff, 0, n));
print('Read ', n, ' bytes.\n');
print(resl);
src/builtins/basic.c
+16
View File
@@ -128,6 +128,21 @@ static Object *bin_sliceBuffer(Runtime *runtime, Object **argv, unsigned int arg
return Object_SliceBuffer(argv[0], offset, length, Runtime_GetHeap(runtime), error);
}
static Object *bin_bufferToString(Runtime *runtime, Object **argv, unsigned int argc, Error *error)
{
assert(argc == 1);
void *buffaddr;
int buffsize;
buffaddr = Object_GetBufferAddrAndSize(argv[0], &buffsize, error);
if(error->occurred)
return NULL;
return Object_FromString(buffaddr, buffsize, Runtime_GetHeap(runtime), error);
}
const StaticMapSlot bins_basic[] = {
{ "math", SM_SMAP, .as_smap = bins_math, },
{ "files", SM_SMAP, .as_smap = bins_files, },
@@ -135,6 +150,7 @@ const StaticMapSlot bins_basic[] = {
{ "newBuffer", SM_FUNCT, .as_funct = bin_newBuffer, .argc = 1 },
{ "sliceBuffer", SM_FUNCT, .as_funct = bin_sliceBuffer, .argc = 3 },
{ "bufferToString", SM_FUNCT, .as_funct = bin_bufferToString, .argc = 1 },
{ "strcat", SM_FUNCT, .as_funct = bin_strcat, .argc = -1 },
src/main.c
+1 -1
View File
@@ -334,4 +334,4 @@ int main(int argc, char **argv)
fprintf(stderr, "Error: Incorrect usage.\n\n");
fprintf(stderr, usage);
return -1;
}
}
src/objects/o_string.c
+19 -8
View File
@@ -2,11 +2,13 @@
#include <assert.h>
#include "../utils/defs.h"
#include "../utils/hash.h"
#include "../utils/utf8.h"
#include "objects.h"
typedef struct {
Object base;
int size;
int count;
int bytes;
char *body;
} StringObject;
@@ -43,7 +45,7 @@ static char *to_string(Object *self, int *size, Heap *heap, Error *err)
StringObject *s = (StringObject*) self;
if(size)
*size = s->size;
*size = s->bytes;
return s->body;
}
@@ -57,13 +59,22 @@ Object *Object_FromString(const char *str, int len, Heap *heap, Error *error)
if(len < 0)
len = strlen(str);
int count = utf8_strlen(str, len);
if(count < 0)
{
Error_Report(error, 0, "Invalid UTF-8 sequence");
return NULL;
}
StringObject *strobj = Heap_Malloc(heap, &t_string, error);
if(strobj == NULL)
return NULL;
strobj->body = Heap_RawMalloc(heap, len+1, error);
strobj->size = len;
strobj->bytes = len;
strobj->count = count;
if(strobj->body == NULL)
return NULL;
@@ -82,7 +93,7 @@ static int count(Object *self)
StringObject *strobj = (StringObject*) self;
return strobj->size;
return strobj->count;
}
static int hash(Object *self)
@@ -92,7 +103,7 @@ static int hash(Object *self)
StringObject *strobj = (StringObject*) self;
return hashbytes((unsigned char*) strobj->body, strobj->size);
return hashbytes((unsigned char*) strobj->body, strobj->count);
}
static Object *copy(Object *self, Heap *heap, Error *err)
@@ -115,7 +126,7 @@ static _Bool op_eql(Object *self, Object *other)
StringObject *s1 = (StringObject*) self;
StringObject *s2 = (StringObject*) other;
_Bool match = s1->size == s2->size && !strncmp(s1->body, s2->body, s1->size);
_Bool match = s1->bytes == s2->bytes && !strncmp(s1->body, s2->body, s1->bytes);
return match;
}
@@ -128,12 +139,12 @@ static void print(Object *obj, FILE *fp)
StringObject *str = (StringObject*) obj;
fprintf(fp, "%.*s", str->size, str->body);
fprintf(fp, "%.*s", str->bytes, str->body);
}
static void walkexts(Object *self, void (*callback)(void **referer, unsigned int size, void *userp), void *userp)
{
StringObject *str = (StringObject*) self;
callback((void**) &str->body, str->size+1, userp);
callback((void**) &str->body, str->bytes+1, userp);
}
src/utils/utf8.c
+423
View File
@@ -0,0 +1,423 @@
#include <assert.h>
#include <stddef.h> // NULL
#include "utf8.h"
// If this is turned on, this library will assume that the
// UTF-8 strings will mainly contain ASCII characters.
#define ASSUME_ASCII 1
/* SYMBOL
** utf8_sequence_from_utf32_codepoint
**
** DESCRIPTION
** Transform a UTF-32 encoded codepoint to a UTF-8 encoded byte sequence.
**
** ARGUMENTS
** The [utf8_data] pointer refers to the location where the UTF-8 sequence
** will be stored.
**
** The [nbytes] argument specifies the maximum number of bytes that can
** be written to [utf8_data]. It can't be negative.
**
** The [utf32_code] argument is the UTF-32 code that will be converted.
**
** RETURN
** If [utf32_code] is valid UTF-32 and the provided buffer is big enough,
** the UTF-8 equivalent sequence is stored in [utf8_data]. No more than
** [nbytes] are ever written. If one of those conitions isn't true, -1 is
** returned.
*/
int utf8_sequence_from_utf32_codepoint(char *utf8_data, int nbytes, uint32_t utf32_code)
{
if(utf32_code < 128)
{
if(nbytes < 1)
return -1;
utf8_data[0] = utf32_code;
return 1;
}
if(utf32_code < 2048)
{
if(nbytes < 2)
return -1;
utf8_data[0] = 0xc0 | (utf32_code >> 6);
utf8_data[1] = 0x80 | (utf32_code & 0x3f);
return 2;
}
if(utf32_code < 65536)
{
if(nbytes < 3)
return -1;
utf8_data[0] = 0xe0 | (utf32_code >> 12);
utf8_data[1] = 0x80 | ((utf32_code >> 6) & 0x3f);
utf8_data[2] = 0x80 | (utf32_code & 0x3f);
return 3;
}
if(utf32_code <= 0x10ffff)
{
if(nbytes < 4)
return -1;
utf8_data[0] = 0xf0 | (utf32_code >> 18);
utf8_data[1] = 0x80 | ((utf32_code >> 12) & 0x3f);
utf8_data[2] = 0x80 | ((utf32_code >> 6) & 0x3f);
utf8_data[3] = 0x80 | (utf32_code & 0x3f);
return 4;
}
// Code is out of range for UTF-8.
return -1;
}
/* SYMBOL
** utf8_sequence_to_utf32_codepoint
**
** DESCRIPTION
** Transform a UTF-8 encoded byte sequence pointed by `utf8_data`
** into a UTF-32 encoded codepoint.
**
** ARGUMENTS
** The [utf8_data] pointer refers to the location of the UTF-8 sequence.
**
** The [nbytes] argument specifies the maximum number of bytes that can
** be read after [utf8_data]. It can't be negative.
**
** NOTE: The [nbytes] argument has no relation to the UTF-8 byte count sequence.
** You may think about this argument as the "raw" string length (the one
** [strlen] whould return if [utf8_data] were zero-terminated).
**
** The [utf32_code] argument is the location where the encoded UTF-32 code
** will be stored. It may be NULL, in which case the value is evaluated and then
** thrown away.
**
** RETURN
** The codepoint is returned through the output parameter `utf32_code`.
** The returned value is the number of bytes of the UTF-8 sequence that
** were scanned to encode the UTF-32 code, or -1 if the UTF-8 sequence
** is invalid.
**
** NOTE: By calling this function with a NULL [utf32_code], you can check the
** validity of a UTF-8 sequence.
*/
int utf8_sequence_to_utf32_codepoint(const char *utf8_data, int nbytes, uint32_t *utf32_code)
{
assert(utf8_data != NULL);
assert(nbytes >= 0);
uint32_t dummy;
if(utf32_code == NULL)
utf32_code = &dummy;
if(nbytes == 0)
return -1;
if(utf8_data[0] & 0x80)
{
// May be UTF-8.
if((unsigned char) utf8_data[0] >= 0xF0)
{
// 4 bytes.
// 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
if(nbytes < 4)
return -1;
uint32_t temp
= (((uint32_t) utf8_data[0] & 0x07) << 18)
| (((uint32_t) utf8_data[1] & 0x3f) << 12)
| (((uint32_t) utf8_data[2] & 0x3f) << 6)
| (((uint32_t) utf8_data[3] & 0x3f));
if(temp > 0x10ffff)
return -1;
*utf32_code = temp;
return 4;
}
if((unsigned char) utf8_data[0] >= 0xE0)
{
// 3 bytes.
// 1110xxxx 10xxxxxx 10xxxxxx
if(nbytes < 3)
return -1;
uint32_t temp
= (((uint32_t) utf8_data[0] & 0x0f) << 12)
| (((uint32_t) utf8_data[1] & 0x3f) << 6)
| (((uint32_t) utf8_data[2] & 0x3f));
if(temp > 0x10ffff)
return -1;
*utf32_code = temp;
return 3;
}
if((unsigned char) utf8_data[0] >= 0xC0)
{
// 2 bytes.
// 110xxxxx 10xxxxxx
if(nbytes < 2)
return -1;
*utf32_code
= (((uint32_t) utf8_data[0] & 0x1f) << 6)
| (((uint32_t) utf8_data[1] & 0x3f));
assert(*utf32_code <= 0x10ffff);
return 2;
}
// 1 byte
// 10xxxxxx
*utf32_code = (uint32_t) utf8_data[0] & 0x3f;
return 1;
}
// It's ASCII
// 0xxxxxxx
*utf32_code = (uint32_t) utf8_data[0];
return 1;
}
/* SYMBOL
** utf8_strlen
**
** DESCRIPTION
** Count the number of characters of a UTF-8 string.
**
** NOTE: By "character" we mean a valid UTF-8 sequence.
**
** ARGUMENTS
** The [utf8_data] pointer refers to the location of the UTF-8 string.
**
** The [nbytes] argument specifies the byte count of the string referred
** by [utf8_data]. It can't be negative.
**
** RETURN
** Returns the number of characters encoded by [utf8_data], or -1 if
** the string is not valid UTF-8.
**
** NOTE: By calling this function on an ASCII-only string, the return
** value is equal to [nbytes].
**
** NOTE: You can check the validity of a UTF-8 string
** by calling this function and checking that it's
** return value is not negative.
*/
int utf8_strlen(const char *utf8_data, int nbytes)
{
assert(utf8_data != NULL);
assert(nbytes >= 0);
int len = 0;
int i = 0;
while(i < nbytes)
{
#if ASSUME_ASCII
{
int ASCII_start = i;
// Skip through ASCII
while(i < nbytes && (utf8_data[i] & 0x80) == 0)
i += 1;
int ASCII_end = i;
len += (ASCII_end - ASCII_start);
// Either we scanned through all of the
// string, or we encountered some unicode.
if(i == nbytes)
// String ended.
break;
}
#endif
// Found unicode.
{
int n = utf8_sequence_to_utf32_codepoint(utf8_data + i, nbytes - i, NULL);
if(n < 1)
return -1;
i += n;
len += 1;
}
}
return len;
}
/* SYMBOL
** utf8_prev
**
** DESCRIPTION
** Get the UTF-8 sequence that comes before a given byte index
** inside a given string.
**
** NOTE: This is what you use when you want to iterate over a
** UTF-8 string backwards.
**
** ARGUMENTS
** The [utf8_data] pointer refers to the location of the UTF-8 string.
**
** The [nbytes] argument is the raw size of the [utf8_data] string.
** It can't be negative.
**
** The [idx] argument is the index of the byte that follows the UTF-8
** sequence to be decoded.
**
** The [utf32_code] argument, if not NULL, is used to return the UTF-32
** version of the decoded UTF-8 sequence.
**
** RETURN
** Returns the index of the first byte of the decoded UTF-8 sequence,
** or -1 is the sequence wasn't valid UTF-8.
**
** NOTE: If the function didn't fail, by subtracting the returned value
** from [idx], you'll get the number of bytes of the decoded
** sequence.
*/
int utf8_prev(const char *utf8_data, int nbytes, int idx, uint32_t *utf32_code)
{
assert(idx >= 0);
assert(idx <= nbytes);
// [idx] currently refers to the head byte
// of a UTF-8 sequence. We need to first
// get to the last byte of the previous
// sequence.
idx -= 1;
if(idx == -1)
// There was no previous sequence!
return 0; // Return the same index that was provided.
int tail = idx;
#if ASSUME_ASCII
{
// This block isn't necessary for
// this function to work but it
// makes strings that are mainly ascii
// to go faster.
if((utf8_data[tail] & 0x80) == 0)
{
if(utf32_code)
*utf32_code = utf8_data[tail];
return tail;
}
}
#endif
// Skip all of the auxiliary bytes in the
// form '10xxxxxx'.
while(idx > -1 && (utf8_data[idx] & 0xc0) == 0x80)
idx -= 1;
if(idx == -1)
{
// No head sequence byte was found,
// so this isn't valid UTF-8.
return -1;
}
// The index of the head byte.
int head = idx;
// The number of auxiliary bytes is given
// by the difference
int aux = tail - head;
// The total number of bytes of the
// sequence is [aux + 1].
int n = utf8_sequence_to_utf32_codepoint(utf8_data + head, aux + 1, utf32_code);
if(n < 1)
// The sequence wasn't valid UTF-8.
return -1;
assert(n > 0);
if(n < aux + 1)
// Not all of the auxiliary bytes were considered while parsing.
return -1;
assert(n == aux + 1);
return head;
}
/* SYMBOL
** utf8_next
**
** DESCRIPTION
** Get the UTF-8 sequence from a UTF-8 string that starts AFTER the
** sequence that starts at a given byte index.
**
** NOTE: This is what you use when you want to iterate over a
** UTF-8 string.
**
** ARGUMENTS
** The [utf8_data] pointer refers to the location of the UTF-8 string.
**
** The [nbytes] argument is the raw size of the [utf8_data] string.
** It can't be negative.
**
** The [idx] argument is the index of the first byte of the sequence
** that comes before the sequence to be decoded.
**
** The [utf32_code] argument, if not NULL, is used to return the UTF-32
** version of the decoded UTF-8 sequence.
**
** RETURN
** Returns the index of the first byte of the decoded UTF-8 sequence,
** or -1 is the sequence wasn't valid UTF-8.
**
** NOTE: If the function didn't fail, by subtracting [idx] from the
** returned value, you'll get the number of bytes of the decoded
** sequence.
*/
int utf8_next(const char *utf8_data, int nbytes, int idx, uint32_t *utf32_code)
{
// Get the byte count of the current sequence.
int n = utf8_sequence_to_utf32_codepoint(utf8_data + idx, nbytes, NULL);
if(n < 1)
return -1;
// Now get the codepoint of the next sequence.
int k = utf8_sequence_to_utf32_codepoint(utf8_data + idx + n, nbytes, utf32_code);
if(k < 1)
return -1;
return idx + n;
}
int utf8_curr(const char *utf8_data, int nbytes, int idx, uint32_t *utf32_code)
{
assert(idx >= 0);
assert(idx < nbytes);
int n = utf8_sequence_to_utf32_codepoint(utf8_data + idx, nbytes - idx, utf32_code);
return n > 0;
}
src/utils/utf8.h
+10
View File
@@ -0,0 +1,10 @@
#ifndef XUTF8_H
#define XUTF8_H
#include <stdint.h> // uint32_t
int utf8_sequence_from_utf32_codepoint(char *utf8_data, int nbytes, uint32_t utf32_code);
int utf8_sequence_to_utf32_codepoint(const char *utf8_data, int nbytes, uint32_t *utf32_code);
int utf8_strlen(const char *utf8_data, int nbytes);
int utf8_prev(const char *utf8_data, int nbytes, int idx, uint32_t *utf32_code);
int utf8_next(const char *utf8_data, int nbytes, int idx, uint32_t *utf32_code);
int utf8_curr(const char *utf8_data, int nbytes, int idx, uint32_t *utf32_code);
#endif // #ifndef UTF8_H