Merge pull request #1922 from tursodatabase/glauber/sqlean

bundle SQLean extensions

Author: penberg

Cargo.lock

@@ -16,6 +16,7 @@ libsql-wasmtime-bindings = { version = "0.2.1", optional = true }
 [build-dependencies]
 bindgen = "0.66.1"
 cc = "1.0"
+glob = "0.3"
 
 [features]
 session = []
@@ -28,3 +29,17 @@ wasm32-wasi-vfs = []
 unlock_notify = []
 preupdate_hook = []
 sqlcipher = []
+sqlean-extension-uuid = []
+sqlean-extension-crypto = []
+sqlean-extension-fuzzy = []
+sqlean-extension-math = []
+sqlean-extension-stats = []
+sqlean-extension-text = []
+sqlean-extensions = [
+    "sqlean-extension-uuid",
+    "sqlean-extension-crypto",
+    "sqlean-extension-fuzzy",
+    "sqlean-extension-math",
+    "sqlean-extension-stats",
+    "sqlean-extension-text"
+]

libsql-ffi/Cargo.toml

@@ -1,3 +1,4 @@
+use glob::glob;
 use std::env;
 use std::ffi::OsString;
 use std::fs::{self, OpenOptions};
@@ -130,6 +131,57 @@ fn make_amalgamation() {
     .unwrap();
 }
 
+fn generate_sqlean(enabled_extensions: &[&str], output_path: &Path) -> io::Result<()> {
+    let mut content = String::from(
+        r#"// Generated by build.rs
+
+
+#include "sqlite3.c"
+SQLITE_EXTENSION_INIT1
+
+"#,
+    );
+
+    for ext in enabled_extensions {
+        content.push_str(&format!("#include \"{}/extension.h\"\n", ext));
+    }
+
+    content.push_str(
+        r#"
+#include "sqlean.h"
+
+static void sqlean_version(sqlite3_context* context, int argc, sqlite3_value** argv) {
+    sqlite3_result_text(context, SQLEAN_VERSION, -1, SQLITE_STATIC);
+}
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_sqlean_init(sqlite3* db, char** errmsg_ptr, const sqlite3_api_routines* api) {
+    (void)errmsg_ptr;
+    SQLITE_EXTENSION_INIT2(api);
+    static const int flags = SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC;
+    sqlite3_create_function(db, "sqlean_version", 0, flags, 0, sqlean_version, 0, 0);
+"#,
+    );
+
+    for ext in enabled_extensions {
+        content.push_str(&format!("    {}_init(db);\n", ext));
+    }
+
+    content.push_str(
+        r#"    return SQLITE_OK;
+}
+
+int core_init(const char* dummy) {
+    return sqlite3_auto_extension((void*)sqlite3_sqlean_init);
+}
+"#,
+    );
+
+    std::fs::write(output_path, content)
+}
+
 pub fn build_bundled(out_dir: &str, out_path: &Path) {
     let bindgen_rs_path = if cfg!(feature = "session") {
         "bundled/bindings/session_bindgen.rs"
@@ -146,8 +198,7 @@ pub fn build_bundled(out_dir: &str, out_path: &Path) {
     std::fs::copy(format!("{dir}/{bindgen_rs_path}"), out_path).unwrap();
 
     let mut cfg = cc::Build::new();
-    cfg.file(format!("{BUNDLED_DIR}/src/sqlite3.c"))
-        .flag("-std=c11")
+    cfg.flag("-std=c11")
         .flag("-DSQLITE_CORE")
         .flag("-DSQLITE_DEFAULT_FOREIGN_KEYS=1")
         .flag("-DSQLITE_ENABLE_API_ARMOR")
@@ -169,6 +220,62 @@ pub fn build_bundled(out_dir: &str, out_path: &Path) {
         .flag("-D_POSIX_THREAD_SAFE_FUNCTIONS") // cross compile with MinGW
         .warnings(false);
 
+    let mut sqlean_patterns = vec![];
+    let mut enabled_extensions = Vec::new();
+
+    if cfg!(feature = "sqlean-extension-crypto") {
+        enabled_extensions.push("crypto");
+        sqlean_patterns.push("crypto/*.c");
+    }
+
+    if cfg!(feature = "sqlean-extension-fuzzy") {
+        enabled_extensions.push("fuzzy");
+        sqlean_patterns.push("fuzzy/*.c");
+    }
+
+    if cfg!(feature = "sqlean-extension-math") {
+        enabled_extensions.push("math");
+        sqlean_patterns.push("math/*.c");
+    }
+
+    if cfg!(feature = "sqlean-extension-stats") {
+        enabled_extensions.push("stats");
+        sqlean_patterns.push("stats/*.c");
+    }
+
+    if cfg!(feature = "sqlean-extension-text") {
+        enabled_extensions.push("text");
+        sqlean_patterns.push("text/*.c");
+        sqlean_patterns.push("text/*/*.c");
+    }
+
+    if cfg!(feature = "sqlean-extension-uuid") {
+        enabled_extensions.push("uuid");
+        sqlean_patterns.push("uuid/*.c");
+    }
+
+    if sqlean_patterns.is_empty() {
+        cfg.file(format!("{BUNDLED_DIR}/src/sqlite3.c"));
+    } else {
+        cfg.flag("-DSQLITE_EXTRA_INIT=core_init");
+
+        let mut sqlean_sources = Vec::new();
+        for pattern in sqlean_patterns {
+            let full_pattern = format!("{BUNDLED_DIR}/sqlean/{}", pattern);
+            sqlean_sources.extend(glob(&full_pattern).unwrap().filter_map(Result::ok));
+        }
+
+        cfg.files(sqlean_sources);
+
+        let sqlean = Path::new(BUNDLED_DIR)
+            .join("src")
+            .join("sqlite3-sqlean-generated.c");
+        generate_sqlean(&enabled_extensions, &sqlean).unwrap();
+        cfg.file(&sqlean);
+
+        cfg.include(format!("{BUNDLED_DIR}/sqlean/"));
+    }
+
     if cfg!(feature = "wasmtime-bindings") {
         cfg.flag("-DLIBSQL_ENABLE_WASM_RUNTIME=1");
     }

libsql-ffi/build.rs

@@ -0,0 +1,82 @@
+// Copyright (c) 2023 Anton Zhiyanov, MIT License
+// https://github.com/nalgeon/sqlean
+
+// Base32 encoding/decoding (RFC 4648)
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+static const char base32_chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
+
+uint8_t* base32_encode(const uint8_t* src, size_t len, size_t* out_len) {
+    *out_len = ((len + 4) / 5) * 8;
+    uint8_t* encoded = malloc(*out_len + 1);
+    if (encoded == NULL) {
+        *out_len = 0;
+        return NULL;
+    }
+
+    for (size_t i = 0, j = 0; i < len;) {
+        uint32_t octet0 = i < len ? src[i++] : 0;
+        uint32_t octet1 = i < len ? src[i++] : 0;
+        uint32_t octet2 = i < len ? src[i++] : 0;
+        uint32_t octet3 = i < len ? src[i++] : 0;
+        uint32_t octet4 = i < len ? src[i++] : 0;
+
+        encoded[j++] = base32_chars[octet0 >> 3];
+        encoded[j++] = base32_chars[((octet0 & 0x07) << 2) | (octet1 >> 6)];
+        encoded[j++] = base32_chars[(octet1 >> 1) & 0x1F];
+        encoded[j++] = base32_chars[((octet1 & 0x01) << 4) | (octet2 >> 4)];
+        encoded[j++] = base32_chars[((octet2 & 0x0F) << 1) | (octet3 >> 7)];
+        encoded[j++] = base32_chars[(octet3 >> 2) & 0x1F];
+        encoded[j++] = base32_chars[((octet3 & 0x03) << 3) | (octet4 >> 5)];
+        encoded[j++] = base32_chars[octet4 & 0x1F];
+    }
+
+    if (len % 5 != 0) {
+        size_t padding = 7 - (len % 5) * 8 / 5;
+        for (size_t i = 0; i < padding; i++) {
+            encoded[*out_len - padding + i] = '=';
+        }
+    }
+
+    encoded[*out_len] = '\0';
+    return encoded;
+}
+
+uint8_t* base32_decode(const uint8_t* src, size_t len, size_t* out_len) {
+    while (len > 0 && src[len - 1] == '=') {
+        len--;
+    }
+    *out_len = len * 5 / 8;
+    uint8_t* decoded = malloc(*out_len);
+    if (decoded == NULL) {
+        *out_len = 0;
+        return NULL;
+    }
+
+    size_t bits = 0, value = 0, count = 0;
+    for (size_t i = 0; i < len; i++) {
+        uint8_t c = src[i];
+        if (c >= 'A' && c <= 'Z') {
+            c -= 'A';
+        } else if (c >= '2' && c <= '7') {
+            c -= '2' - 26;
+        } else {
+            continue;
+        }
+        value = (value << 5) | c;
+        bits += 5;
+        if (bits >= 8) {
+            decoded[count++] = (uint8_t)(value >> (bits - 8));
+            bits -= 8;
+        }
+    }
+    if (bits >= 5 || (value & ((1 << bits) - 1)) != 0) {
+        free(decoded);
+        return NULL;
+    }
+    *out_len = count;
+    return decoded;
+}

libsql-ffi/bundled/sqlean/crypto/base32.c

@@ -0,0 +1,14 @@
+// Copyright (c) 2023 Anton Zhiyanov, MIT License
+// https://github.com/nalgeon/sqlean
+
+// Base32 encoding/decoding (RFC 4648)
+
+#ifndef _BASE32_H_
+#define _BASE32_H_
+
+#include <stdint.h>
+
+uint8_t* base32_encode(const uint8_t* src, size_t len, size_t* out_len);
+uint8_t* base32_decode(const uint8_t* src, size_t len, size_t* out_len);
+
+#endif /* _BASE32_H_ */

libsql-ffi/bundled/sqlean/crypto/base32.h

@@ -0,0 +1,103 @@
+// Copyright (c) 2023 Anton Zhiyanov, MIT License
+// https://github.com/nalgeon/sqlean
+
+// Base64 encoding/decoding (RFC 4648)
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+static const char base64_chars[] =
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+uint8_t* base64_encode(const uint8_t* src, size_t len, size_t* out_len) {
+    uint8_t* encoded = NULL;
+    size_t i, j;
+    uint32_t octets;
+
+    *out_len = ((len + 2) / 3) * 4;
+    encoded = malloc(*out_len + 1);
+    if (encoded == NULL) {
+        *out_len = 0;
+        return NULL;
+    }
+
+    for (i = 0, j = 0; i < len; i += 3, j += 4) {
+        octets =
+            (src[i] << 16) | ((i + 1 < len ? src[i + 1] : 0) << 8) | (i + 2 < len ? src[i + 2] : 0);
+        encoded[j] = base64_chars[(octets >> 18) & 0x3f];
+        encoded[j + 1] = base64_chars[(octets >> 12) & 0x3f];
+        encoded[j + 2] = base64_chars[(octets >> 6) & 0x3f];
+        encoded[j + 3] = base64_chars[octets & 0x3f];
+    }
+
+    if (len % 3 == 1) {
+        encoded[*out_len - 1] = '=';
+        encoded[*out_len - 2] = '=';
+    } else if (len % 3 == 2) {
+        encoded[*out_len - 1] = '=';
+    }
+
+    encoded[*out_len] = '\0';
+    return encoded;
+}
+
+static const uint8_t base64_table[] = {
+    // Map base64 characters to their corresponding values
+    ['A'] = 0,  ['B'] = 1,  ['C'] = 2,  ['D'] = 3,  ['E'] = 4,  ['F'] = 5,  ['G'] = 6,  ['H'] = 7,
+    ['I'] = 8,  ['J'] = 9,  ['K'] = 10, ['L'] = 11, ['M'] = 12, ['N'] = 13, ['O'] = 14, ['P'] = 15,
+    ['Q'] = 16, ['R'] = 17, ['S'] = 18, ['T'] = 19, ['U'] = 20, ['V'] = 21, ['W'] = 22, ['X'] = 23,
+    ['Y'] = 24, ['Z'] = 25, ['a'] = 26, ['b'] = 27, ['c'] = 28, ['d'] = 29, ['e'] = 30, ['f'] = 31,
+    ['g'] = 32, ['h'] = 33, ['i'] = 34, ['j'] = 35, ['k'] = 36, ['l'] = 37, ['m'] = 38, ['n'] = 39,
+    ['o'] = 40, ['p'] = 41, ['q'] = 42, ['r'] = 43, ['s'] = 44, ['t'] = 45, ['u'] = 46, ['v'] = 47,
+    ['w'] = 48, ['x'] = 49, ['y'] = 50, ['z'] = 51, ['0'] = 52, ['1'] = 53, ['2'] = 54, ['3'] = 55,
+    ['4'] = 56, ['5'] = 57, ['6'] = 58, ['7'] = 59, ['8'] = 60, ['9'] = 61, ['+'] = 62, ['/'] = 63,
+};
+
+uint8_t* base64_decode(const uint8_t* src, size_t len, size_t* out_len) {
+    if (len % 4 != 0) {
+        return NULL;
+    }
+
+    size_t padding = 0;
+    if (src[len - 1] == '=') {
+        padding++;
+    }
+    if (src[len - 2] == '=') {
+        padding++;
+    }
+
+    *out_len = (len / 4) * 3 - padding;
+    uint8_t* decoded = malloc(*out_len);
+    if (decoded == NULL) {
+        *out_len = 0;
+        return NULL;
+    }
+
+    for (size_t i = 0, j = 0; i < len; i += 4, j += 3) {
+        uint32_t block = 0;
+        for (size_t k = 0; k < 4; k++) {
+            block <<= 6;
+            if (src[i + k] == '=') {
+                padding--;
+            } else {
+                uint8_t index = base64_table[src[i + k]];
+                if (index == 0 && src[i + k] != 'A') {
+                    free(decoded);
+                    return NULL;
+                }
+                block |= index;
+            }
+        }
+
+        decoded[j] = (block >> 16) & 0xFF;
+        if (j + 1 < *out_len) {
+            decoded[j + 1] = (block >> 8) & 0xFF;
+        }
+        if (j + 2 < *out_len) {
+            decoded[j + 2] = block & 0xFF;
+        }
+    }
+
+    return decoded;
+}

libsql-ffi/bundled/sqlean/crypto/base64.c

@@ -0,0 +1,15 @@
+// Copyright (c) 2023 Anton Zhiyanov, MIT License
+// https://github.com/nalgeon/sqlean
+
+// Base64 encoding/decoding (RFC 4648)
+
+#ifndef BASE64_H
+#define BASE64_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+uint8_t* base64_encode(const uint8_t* src, size_t len, size_t* out_len);
+uint8_t* base64_decode(const uint8_t* src, size_t len, size_t* out_len);
+
+#endif /* BASE64_H */