Merge pull request #1679 from tursodatabase/vector-search-float16

vector search: add float16 support

Author: sivukhin

libsql-ffi/bundled/SQLite3MultipleCiphers/src/sqlite3.c

@@ -62,3 +62,5 @@ libsql
 /crates/target/
 /has_tclsh*
 /libsql.wasm
+test_libsql_f16_table.h
+test_libsql_f16

libsql-ffi/bundled/src/sqlite3.c

@@ -195,7 +195,7 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
          sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \
          table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
          update.lo userauth.lo upsert.lo util.lo vacuum.lo \
-         vector.lo vectorfloat32.lo vectorfloat64.lo vectorfloat1bit.lo vectorfloat8.lo \
+         vector.lo vectorfloat32.lo vectorfloat64.lo vectorfloat1bit.lo vectorfloat8.lo vectorfloat16.lo \
          vectorIndex.lo vectordiskann.lo vectorvtab.lo \
          vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
          vdbetrace.lo vdbevtab.lo \
@@ -304,6 +304,7 @@ SRC = \
   $(TOP)/src/vector.c \
   $(TOP)/src/vectorInt.h \
   $(TOP)/src/vectorfloat1bit.c \
+  $(TOP)/src/vectorfloat16.c \
   $(TOP)/src/vectorfloat32.c \
   $(TOP)/src/vectorfloat64.c \
   $(TOP)/src/vectorfloat8.c \
@@ -1143,6 +1144,9 @@ vector.lo:	$(TOP)/src/vector.c $(HDR)
 vectorfloat1bit.lo:	$(TOP)/src/vectorfloat1bit.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/vectorfloat1bit.c
 
+vectorfloat16.lo:	$(TOP)/src/vectorfloat16.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/vectorfloat16.c
+
 vectorfloat32.lo:	$(TOP)/src/vectorfloat32.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/vectorfloat32.c
 

libsql-sqlite3/.gitignore

@@ -0,0 +1,29 @@
+/*
+ * GENERATE: python3 test_libsql_f16.py > test_libsql_f16_table.h
+ * BUILD: cc test_libsql_f16.c -I ../ -L ../.libs -llibsql -lm -o test_libsql_f16
+ * RUN:   LD_LIBRARY_PATH=../.libs ./test_libsql_diskann
+*/
+
+#include "assert.h"
+#include "stdbool.h"
+#include "stdarg.h"
+#include "stddef.h"
+#include "vectorfloat16.c"
+#include "test_libsql_f16_table.h"
+
+#define eprintf(...) fprintf(stderr, __VA_ARGS__)
+#define ensure(condition, ...) { if (!(condition)) { eprintf(__VA_ARGS__); exit(1); } }
+
+int main() {
+  for(int i = 0; i < 65536; i++){
+    u32 expected = F16ToF32[i];
+    float actual = vectorF16ToFloat(i);
+    u32 actual_u32 = *((u32*)&actual);
+    ensure(expected == actual_u32, "conversion from %x failed: %f != %f (%x != %x)", i, *(float*)&expected, *(float*)&actual_u32, expected, actual_u32);
+  }
+  for(int i = 0; i < 65536; i++){
+    u16 expected = F32ToF16[i];
+    u16 actual = vectorF16FromFloat(*(float*)&F32[i]);
+    ensure(expected == actual, "conversion from %x (%f, it=%d) failed: %x != %x", F32[i], *(float*)&F32[i], i, expected, actual);
+  }
+}

libsql-sqlite3/Makefile.in

@@ -0,0 +1,42 @@
+import random
+import struct
+import numpy as np
+
+u32_list = [random.randint(0, 2**32) for _ in range(65536)]
+
+print("""
+u32 F32[65536] = {
+""")
+for i, x in enumerate(u32_list):
+    if i % 8 == 0: print("    ", end='');
+    print('{:>10}u, '.format(x), end='')
+    if i % 8 == 7: print()
+print("};")
+
+
+print("""
+u16 F32ToF16[65536] = {
+""")
+for i, x in enumerate(u32_list):
+    if i % 8 == 0: print("    ", end='');
+    u32_bytes = struct.pack('<I', x)
+    f32 = np.float16(struct.unpack('<f', u32_bytes)[0])
+    f16_bytes = struct.pack('<e', f32)
+    u16 = struct.unpack('<H', f16_bytes)[0]
+    print('{:>10}, '.format(u16), end='')
+    if i % 8 == 7: print()
+print("};")
+
+print("""
+u32 F16ToF32[65536] = {
+""")
+
+for x in range(65536):
+    if x % 8 == 0: print("    ", end='');
+    u16_bytes = struct.pack('<H', x)
+    f16 = struct.unpack('<e', u16_bytes)[0]
+    f32_bytes = struct.pack('<f', f16)
+    u32 = struct.unpack('<I', f32_bytes)[0]
+    print('{:>10}u, '.format(u32), end='')
+    if x % 8 == 7: print()
+print("};")