First version of the URL builder

README.md

@@ -1,5 +1,5 @@
 # url.c
-This is a small library to parse and manipulate URLs in conformance to RFC 3986 and (most of) the WHATWG specification.
+This is a small library to parse and manipulate URLs in conformance to RFC 3986 and WHATWG.
 
 It features
 * No allocations
@@ -15,4 +15,6 @@ You just copy paste `url.h` and `url.c` in your source tree and compile them as
 Some example programs are provided in the `examples/` folder. To compile them, run `build.bat` on Windows or `build.sh` on Linux.
 
 # Testing
-The parser and reference resolver is tested against the URL section of the [web platform tests](https://github.com/web-platform-tests/wpt), which validates implementations that adhere to the WHATWG specification (what the major browsers actually do). To run the test suite, `cd` into `tests/` and compile the test runner using `build.sh` (Linux) or `build.bat` (Windows). Then, run `./test.out` (Linux) or `./test.exe` (Windows). Note that not all tests pass, which is to be expected as adherence to the WHATWG specification is a work-in-progress.
+The parser and reference resolver is tested against the URL section of the [web platform tests](https://github.com/web-platform-tests/wpt). Note that not all tests pass. This is due to the parser not doing any normalization and not supporting more niche features such as converting `\` characters to `/` or supporting drive letters like `C:` in the URL's path.
+
+To run the test suite, `cd` into `tests/` and compile the test runner using `build.sh` (Linux) or `build.bat` (Windows). Then, run `./test.out` (Linux) or `./test.exe` (Windows).

build.bat

@@ -2,3 +2,4 @@ gcc -o 000_parse.exe         examples/000_parse.c         url.c -I. -Wall -Wextr
 gcc -o 001_serialize.exe     examples/001_serialize.c     url.c -I. -Wall -Wextra -ggdb
 gcc -o 002_partial_parse.exe examples/002_partial_parse.c url.c -I. -Wall -Wextra -ggdb
 gcc -o 003_references.exe    examples/003_references.c    url.c -I. -Wall -Wextra -ggdb
+gcc -o 004_build.exe         examples/004_build.c         url.c -I. -Wall -Wextra -ggdb

build.sh

@@ -2,3 +2,4 @@ gcc -o 000_parse.out         examples/000_parse.c         url.c -I. -Wall -Wextr
 gcc -o 001_serialize.out     examples/001_serialize.c     url.c -I. -Wall -Wextra -ggdb
 gcc -o 002_partial_parse.out examples/002_partial_parse.c url.c -I. -Wall -Wextra -ggdb
 gcc -o 003_references.out    examples/003_references.c    url.c -I. -Wall -Wextra -ggdb
+gcc -o 004_build.out         examples/004_build.c         url.c -I. -Wall -Wextra -ggdb

examples/004_build.c

@@ -0,0 +1,33 @@
+#include <url.h>
+#include <stdio.h>
+
+int main(void)
+{
+    URL_Builder builder;
+    url_builder_init(&builder);
+
+    url_builder_set_scheme(&builder, "http", -1);
+
+    url_builder_set_username(&builder, "cozis", -1);
+
+    url_builder_set_password(&builder, "my_secret", -1);
+
+    url_builder_set_host_name(&builder, "www.ex@mple.com", -1);
+
+    url_builder_set_path(&builder, "/../../index.html", -1);
+
+    char url[1<<9];
+    int len = url_builder_finalize(&builder, url, sizeof(url));
+    if (len < 0) {
+        printf("Couldn't build URL\n");
+        return -1;
+    }
+    if (len >= (int) sizeof(url)) {
+        printf("URL buffer is too small\n");
+        return -1;
+    }
+    url[len] = '\0';
+
+    printf("Built URL: %s\n", url);
+    return 0;
+}

tests/test.c

@@ -31,10 +31,6 @@ void test_printf(char *fmt, ...)
         sizeof(test_output_buffer) - test_output_buffer_used,
         fmt, args);
     va_end(args);
-
-    assert(len > -1);
-    if (len > (int) sizeof(test_output_buffer) - test_output_buffer_used)
-        len = sizeof(test_output_buffer) - test_output_buffer_used;
     test_output_buffer_used += len;
 }
 
@@ -202,21 +198,15 @@ static int json_to_url(cJSON *json, URL *url)
         url->fragment.len--;
     }
 
-    if (port.len == 0) {
+    char tmp[8];
-        url->no_port = 1;
+    if (port.len >= (int) sizeof(tmp)) {
-        url->port = 0;
+        test_printf(" JSON object's port field is longer than expected\n");
-    } else {
+        return -1;
-        char tmp[8];
-        if (port.len >= (int) sizeof(tmp)) {
-            test_printf(" JSON object's port field is longer than expected\n");
-            return -1;
-        }
-        memcpy(tmp, port.ptr, port.len);
-        tmp[port.len] = 0;
-
-        url->no_port = 0;
-        url->port = atoi(tmp);
     }
+    memcpy(tmp, port.ptr, port.len);
+    tmp[port.len] = 0;
+
+    url->port = atoi(tmp);
     return 0;
 }
 
@@ -227,103 +217,73 @@ static bool streq(URL_String a, URL_String b)
     return !memcmp(a.ptr, b.ptr, a.len);
 }
 
-static int compare_urls(URL parsed, URL expected)
+static int compare_urls(URL url_1, URL url_2, char *label_1, char *label_2)
 {
     #define UNPACK(s) (s).len, (s).ptr
 
     bool ok = true;
 
-    if (!streq(parsed.scheme, expected.scheme)) {
+    if (!streq(url_1.scheme, url_2.scheme)) {
         if (ok) test_printf("\n");
         test_printf("  scheme mismatch\n");
-        test_printf("    parsed   [%.*s]\n", UNPACK(parsed.scheme));
+        test_printf("    %s [%.*s]\n", label_1, UNPACK(url_1.scheme));
-        test_printf("    expected [%.*s]\n", UNPACK(expected.scheme));
+        test_printf("    %s [%.*s]\n", label_2, UNPACK(url_2.scheme));
         ok = false;
     }
 
-    char tmp[1<<9];
+    if (!streq(url_1.username, url_2.username)) {
-    int ret = url_percent_decode(parsed.username, tmp, sizeof(tmp));
-    assert(ret > -1 && ret < (int) sizeof(tmp));
-
-    if (!streq((URL_String) { tmp, ret }, expected.username)) {
         if (ok) test_printf("\n");
         test_printf("  username mismatch\n");
-        test_printf("    parsed   [%.*s]\n", ret, tmp);
+        test_printf("    %s [%.*s]\n", label_1, UNPACK(url_1.username));
-        test_printf("    expected [%.*s]\n", UNPACK(expected.username));
+        test_printf("    %s [%.*s]\n", label_2, UNPACK(url_2.username));
         ok = false;
     }
 
-    ret = url_percent_decode(parsed.password, tmp, sizeof(tmp));
+    if (!streq(url_1.password, url_2.password)) {
-    assert(ret > -1 && ret < (int) sizeof(tmp));
-
-    if (!streq((URL_String) { tmp, ret }, expected.password)) {
         if (ok) test_printf("\n");
         test_printf("  password mismatch\n");
-        test_printf("    parsed   [%.*s]\n", ret, tmp);
+        test_printf("    %s [%.*s]\n", label_1, UNPACK(url_1.password));
-        test_printf("    expected [%.*s]\n", UNPACK(expected.password));
+        test_printf("    %s [%.*s]\n", label_2, UNPACK(url_2.password));
         ok = false;
     }
 
-    ret = url_percent_decode(parsed.host_text, tmp, sizeof(tmp));
+    if (!streq(url_1.host_text, url_2.host_text)) {
-    assert(ret > -1 && ret < (int) sizeof(tmp));
-
-    if (!streq((URL_String) { tmp, ret }, expected.host_text)) {
         if (ok) test_printf("\n");
         test_printf("  host mismatch\n");
-        test_printf("    parsed   [%.*s]\n", ret, tmp);
+        test_printf("    %s [%.*s]\n", label_1, UNPACK(url_1.host_text));
-        test_printf("    expected [%.*s]\n", UNPACK(expected.host_text));
+        test_printf("    %s [%.*s]\n", label_2, UNPACK(url_2.host_text));
         ok = false;
     }
 
-    if (parsed.no_port != expected.no_port) {
+    if (url_1.port != url_2.port) {
-        if (ok) test_printf("\n");
-        if (parsed.no_port) {
-            test_printf("  port mismatch\n");
-            test_printf("    parsed   (empty)\n");
-            test_printf("    expected [%d]\n", expected.port);
-        } else {
-            test_printf("  port mismatch\n");
-            test_printf("    parsed   [%d]\n", parsed.port);
-            test_printf("    expected (empty)\n");
-        }
-        ok = false;
-    }
-
-    if (parsed.port != expected.port) {
         if (ok) test_printf("\n");
         test_printf("  port mismatch\n");
-        test_printf("    parsed   [%d]\n", parsed.port);
+        test_printf("    %s [%d]\n", label_1, url_1.port);
-        test_printf("    expected [%d]\n", expected.port);
+        test_printf("    %s [%d]\n", label_2, url_2.port);
         ok = false;
     }
 
-    if (!streq(parsed.path, expected.path)) {
+    if (!streq(url_1.path, url_2.path)) {
         if (ok) test_printf("\n");
         test_printf("  path mismatch\n");
-        test_printf("    parsed   [%.*s]\n", UNPACK(parsed.path));
+        test_printf("    %s [%.*s]\n", label_1, UNPACK(url_1.path));
-        test_printf("    expected [%.*s]\n", UNPACK(expected.path));
+        test_printf("    %s [%.*s]\n", label_2, UNPACK(url_2.path));
         ok = false;
     }
 
-    ret = url_percent_decode(parsed.query, tmp, sizeof(tmp));
+    if (!streq(url_1.query, url_2.query)) {
-    assert(ret > -1 && ret < (int) sizeof(tmp));
-
-    if (!streq((URL_String) { tmp, ret }, expected.query)) {
         if (ok) test_printf("\n");
         test_printf("  query mismatch\n");
-        test_printf("    parsed   [%.*s]\n", ret, tmp);
+        test_printf("    %s [%.*s]\n", label_1, UNPACK(url_1.query));
-        test_printf("    expected [%.*s]\n", UNPACK(expected.query));
+        test_printf("    %s [%.*s]\n", label_2, UNPACK(url_2.query));
         ok = false;
     }
 
-    ret = url_percent_decode(parsed.fragment, tmp, sizeof(tmp));
+    if (!streq(url_1.fragment, url_2.fragment)) {
-    assert(ret > -1 && ret < (int) sizeof(tmp));
-
-    if (!streq((URL_String) { tmp, ret }, expected.fragment)) {
         if (ok) test_printf("\n");
         test_printf("  fragment mismatch\n");
-        test_printf("    parsed   [%.*s]\n", ret, tmp);
+        test_printf("    %s [%.*s]\n", label_1, UNPACK(url_1.fragment));
-        test_printf("    expected [%.*s]\n", UNPACK(expected.fragment));
+        test_printf("    %s [%.*s]\n", label_2, UNPACK(url_2.fragment));
         ok = false;
     }
 
@@ -434,7 +394,7 @@ static int run_test(cJSON *json)
         if (json_to_url(json, &expected) < 0)
             return -1;
 
-        if (!compare_urls(parsed, expected)) {
+        if (!compare_urls(parsed, expected, "parsed", "expected")) {
             test_printf("  input: ");
             test_print_str(src);
             test_printf("\n");
@@ -512,7 +472,6 @@ int main(void)
 
     if (!cJSON_IsArray(root)) {
         fprintf(stderr, "Error: %s doesn't contain a JSON array as expected", path);
-        cJSON_Delete(root);
         free(buf);
         return -1;
     }
@@ -565,7 +524,6 @@ int main(void)
     printf("  failed : %d\n", num_failed);
     printf("  aborted: %d\n", num_aborted);
 
-    cJSON_Delete(root);
     free(buf);
     return 0;
 }

url.c

@@ -35,7 +35,7 @@
 #define UINT16_MAX (URL_u16) ((1 << 16)-1)
 
 #define NULL ((void*) 0)
-#define S(X) ((URL_String) { (X), sizeof(X)-1 })
+#define S(X) (URL_String) { (X), sizeof(X)-1 }
 #define EMPTY (URL_String) { NULL, 0 }
 #define SLICE(src, off, end) (URL_String) { src + off, end - off }
 
@@ -49,6 +49,14 @@ static void memcpy_(char *dst, char *src, int len)
 }
 #endif
 
+static int strlen_(char *s)
+{
+    int n = 0;
+    while (s[n] != '\0')
+        n++;
+    return n;
+}
+
 static URL_b32 streq(URL_String a, URL_String b)
 {
     if (a.len != b.len)
@@ -524,20 +532,9 @@ int url_parse(char *src, int len, int *pcur, URL *out, int flags)
             // First, try the IPv4 rule
             if (cur < len && is_digit(src[cur])) {
                 if (url_parse_ipv4(src, len, &cur, &out->host_ipv4) == 0) {
-                    // If we managed to parse an IPv4 address but the
+                    out->host_type = URL_HOST_IPV4;
-                    // following character si a valid character for a
+                    is_ipv4 = 1;
-                    // registered name, then it wasn't an IPv4 after
-                    // all. For instance this should not be parsed as
-                    // an IPv4:
-                    //     127.0.0.1.com
-                    if (cur == len || !is_reg_name(src[cur])) {
-                        out->host_type = URL_HOST_IPV4;
-                        is_ipv4 = 1;
-                    }
                 }
-
-                if (!is_ipv4)
-                    cur = host_off;
             }
 
             if (!is_ipv4) {
@@ -580,26 +577,25 @@ int url_parse(char *src, int len, int *pcur, URL *out, int flags)
         out->no_port = 1;
         out->port    = 0;
         if (cur < len && src[cur] == ':') {
+
+            // The WHATWG standard forbids port numbers with the
+            // "file" protocol.
+            if ((flags & URL_FLAG_RFC3986) == 0) {
+                if (streqcase(out->scheme, S("file")))
+                    return -1;
+            }
+
             cur++; // Consume the ':'
+            out->no_port = 0;
 
-            if (cur < len && is_digit(src[cur])) {
+            while (cur < len && is_digit(src[cur])) {
 
-                // The WHATWG standard forbids port numbers with the
+                int n = src[cur] - '0';
-                // "file" protocol.
+                cur++;
-                if ((flags & URL_FLAG_RFC3986) == 0) {
-                    if (streqcase(out->scheme, S("file")))
-                        return -1;
-                }
 
-                out->no_port = 0;
+                if (out->port > (UINT16_MAX - n) / 10)
-                do {
+                    return -1; // Overflow
-                    int n = src[cur] - '0';
+                out->port = out->port * 10 + n;
-                    cur++;
-
-                    if (out->port > (UINT16_MAX - n) / 10)
-                        return -1; // Overflow
-                    out->port = out->port * 10 + n;
-                } while (cur < len && is_digit(src[cur]));
             }
         }
 
@@ -796,6 +792,13 @@ static void append(Builder *b, URL_String s)
     b->len += s.len;
 }
 
+static void appendc(Builder *b, char c)
+{
+    if (b->len < b->cap)
+        b->dst[b->len] = c;
+    b->len++;
+}
+
 static void append_port(Builder *b, URL_u16 port)
 {
     char buf[sizeof("65536")-1];
@@ -857,8 +860,6 @@ static void append_fragment(Builder *b, URL_String fragment)
 
 int url_serialize(URL url, URL *base, char *dst, int cap)
 {
-    ASSERT(cap == 0 || dst != NULL);
-
     if (base != NULL && base->scheme.len == 0)
         return -1; // Base is not an absolute URL
 
@@ -962,8 +963,6 @@ static URL_b32 is_white_space(char c)
 
 int url_remove_white_space(char *src, int len, char *dst, int cap)
 {
-    ASSERT(cap == 0 || dst != NULL);
-
     while (len > 0 && is_white_space(src[0])) {
         src++;
         len--;
@@ -986,8 +985,6 @@ int url_remove_white_space(char *src, int len, char *dst, int cap)
 
 int url_percent_decode(URL_String str, char *dst, int cap)
 {
-    ASSERT(cap == 0 || dst != NULL);
-
     char *src = str.ptr;
     int   len = str.len;
     int   rd  = 0;
@@ -1016,3 +1013,162 @@ int url_percent_decode(URL_String str, char *dst, int cap)
     }
     return wr;
 }
+
+static void append_percent_encoded(Builder *b, URL_String str, URL_b32 (*testfn)(char c))
+{
+    for (int i = 0; i < str.len; i++) {
+        char c = str.ptr[i];
+        if (testfn(c))
+            appendc(b, str.ptr[i]);
+        else {
+            static const char table[] = "0123456789abcdef";
+            appendc(b, '%');
+            appendc(b, table[(URL_u8) c >> 4]);
+            appendc(b, table[(URL_u8) c & 0xF]);
+        }
+    }
+}
+
+void url_builder_init(URL_Builder *ub)
+{
+    ub->scheme    = EMPTY;
+    ub->username  = EMPTY;
+    ub->password  = EMPTY;
+    ub->host_type = URL_HOST_EMPTY;
+    ub->no_port   = 1;
+    ub->port      = 0;
+    ub->path      = EMPTY;
+    ub->query     = EMPTY;
+    ub->fragment  = EMPTY;
+}
+
+void url_builder_set_scheme(URL_Builder *ub, char *str, int len)
+{
+    if (len < 0) len = str ? strlen_(str) : 0;
+    ub->scheme = (URL_String) { str, len };
+}
+
+void url_builder_set_username(URL_Builder *ub, char *str, int len)
+{
+    if (len < 0) len = str ? strlen_(str) : 0;
+    ub->username = (URL_String) { str, len };
+}
+
+void url_builder_set_password(URL_Builder *ub, char *str, int len)
+{
+    if (len < 0) len = str ? strlen_(str) : 0;
+    ub->password = (URL_String) { str, len };
+}
+
+void url_builder_set_host_name(URL_Builder *ub, char *str, int len)
+{
+    if (len < 0) len = str ? strlen_(str) : 0;
+    ub->host_type = URL_HOST_NAME;
+    ub->host_name = (URL_String) { str, len };
+}
+
+void url_builder_set_host_ipv4(URL_Builder *ub, URL_IPv4 ipv4)
+{
+    ub->host_type = URL_HOST_IPV4;
+    ub->host_ipv4 = ipv4;
+}
+
+void url_builder_set_host_ipv6(URL_Builder *ub, URL_IPv6 ipv6)
+{
+    ub->host_type = URL_HOST_IPV6;
+    ub->host_ipv6 = ipv6;
+}
+
+void url_builder_set_port(URL_Builder *ub, URL_u16 port)
+{
+    ub->no_port = 0;
+    ub->port = port;
+}
+
+void url_builder_set_path(URL_Builder *ub, char *str, int len)
+{
+    if (len < 0) len = str ? strlen_(str) : 0;
+    ub->path = (URL_String) { str, len };
+}
+
+void url_builder_set_query(URL_Builder *ub, char *str, int len)
+{
+    if (len < 0) len = str ? strlen_(str) : 0;
+    ub->query = (URL_String) { str, len };
+}
+
+void url_builder_set_fragment(URL_Builder *ub, char *str, int len)
+{
+    if (len < 0) len = str ? strlen_(str) : 0;
+    ub->fragment = (URL_String) { str, len };
+}
+
+int url_builder_finalize(URL_Builder *ub, char *dst, int cap)
+{
+    Builder b = { dst, cap, 0 };
+
+    // Validate and append the schema
+    {
+        if (ub->scheme.len == 0 || !is_scheme_first(ub->scheme.ptr[0]))
+            return -1;
+        for (int i = 1; i < ub->scheme.len; i++)
+            if (!is_scheme(ub->scheme.ptr[i]))
+                return -1;
+        append(&b, ub->scheme);
+    }
+
+    URL_b32 authority = 0;
+    if (ub->username.len > 0 || ub->password.len > 0 || ub->host_type != URL_HOST_EMPTY || ub->no_port == 0) {
+        authority = 1;
+        appendc(&b, '/');
+        appendc(&b, '/');
+        if (ub->username.len > 0 || ub->password.len > 0) {
+            append_percent_encoded(&b, ub->username, is_userinfo);
+            appendc(&b, ':');
+            append_percent_encoded(&b, ub->password, is_userinfo);
+            appendc(&b, '@');
+        }
+        switch (ub->host_type) {
+        case URL_HOST_EMPTY:
+            // Do nothing
+            break;
+        case URL_HOST_NAME:
+            append_percent_encoded(&b, ub->host_name, is_reg_name);
+            break;
+        case URL_HOST_IPV4:
+            ASSERT(0); // TODO
+            break;
+        case URL_HOST_IPV6:
+            appendc(&b, '[');
+            ASSERT(0);
+            appendc(&b, ']');
+            break;
+        }
+        if (ub->no_port == 0) {
+            appendc(&b, ':');
+            append_port(&b, ub->port);
+        }
+    }
+
+    if (ub->path.len > 0) {
+
+        // If an authority is present, the path can't be relative
+        if (authority && ub->path.ptr[0] != '/')
+            return -1;
+
+        PathComps comps = {0};
+        if (resolve_dots_and_append_comps(&comps, ub->path) < 0)
+                return -1;
+        for (int i = 0; i < comps.count; i++) {
+            if (i > 0 || comps.first_slash)
+                append(&b, S("/"));
+            append_percent_encoded(&b, comps.stack[i], is_pchar);
+        }
+        if (comps.trailing_slash)
+            append(&b, S("/"));
+    }
+
+    append_query(&b, ub->query);
+    append_fragment(&b, ub->fragment);
+    return b.len;
+}

url.h

@@ -45,6 +45,7 @@ typedef enum {
     URL_HOST_NAME,
 } URL_HostType;
 
+// This struct's fields are strictly read-only.
 typedef struct {
 
     // Flags used to parse this URL
@@ -174,4 +175,39 @@ int url_percent_decode(URL_String str, char *dst, int cap);
 // though it's just a best-effor normalization for now.
 int url_serialize(URL url, URL *base, char *dst, int cap);
 
+typedef struct {
+
+    URL_String scheme;
+
+    URL_String username;
+    URL_String password;
+
+    URL_HostType host_type;
+    union {
+        URL_String host_name;
+        URL_IPv4   host_ipv4;
+        URL_IPv6   host_ipv6;
+    };
+
+    URL_b32    no_port;
+    URL_u16    port;
+
+    URL_String path;
+    URL_String query;
+    URL_String fragment;
+} URL_Builder;
+
+void url_builder_init(URL_Builder *ub);
+void url_builder_set_scheme(URL_Builder *ub, char *str, int len);
+void url_builder_set_username(URL_Builder *ub, char *str, int len);
+void url_builder_set_password(URL_Builder *ub, char *str, int len);
+void url_builder_set_host_name(URL_Builder *ub, char *str, int len);
+void url_builder_set_host_ipv4(URL_Builder *ub, URL_IPv4 ipv4);
+void url_builder_set_host_ipv6(URL_Builder *ub, URL_IPv6 ipv6);
+void url_builder_set_port(URL_Builder *ub, URL_u16 port);
+void url_builder_set_path(URL_Builder *ub, char *str, int len);
+void url_builder_set_query(URL_Builder *ub, char *str, int len);
+void url_builder_set_fragment(URL_Builder *ub, char *str, int len);
+int  url_builder_finalize(URL_Builder *ub, char *dst, int cap);
+
 #endif // URL_INCLUDED