fix behaviour of VACUUM for vector indices to make rowid consistent

between shadow tables and base table

Author: sivukhin

libsql-sqlite3/src/vacuum.c

@@ -17,6 +17,10 @@
 #include "sqliteInt.h"
 #include "vdbeInt.h"
 
+#ifndef SQLITE_OMIT_VECTOR
+#include "vectorIndexInt.h"
+#endif
+
 #if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH)
 
 /*
@@ -294,6 +298,27 @@ SQLITE_NOINLINE int sqlite3RunVacuum(
   if( rc!=SQLITE_OK ) goto end_of_vacuum;
   db->init.iDb = 0;
 
+#ifndef SQLITE_OMIT_VECTOR
+  // shadow tables for vector index will be populated automatically during CREATE INDEX command
+  // so we must skip them at this step
+  if( sqlite3FindTable(db, VECTOR_INDEX_GLOBAL_META_TABLE, zDbMain) != NULL ){
+    rc = execSqlF(db, pzErrMsg,
+        "SELECT'INSERT INTO vacuum_db.'||quote(name)"
+        "||' SELECT*FROM\"%w\".'||quote(name)"
+        "FROM vacuum_db.sqlite_schema "
+        "WHERE type='table'AND coalesce(rootpage,1)>0 AND name NOT IN (SELECT name||'_shadow' FROM " VECTOR_INDEX_GLOBAL_META_TABLE ")",
+        zDbMain
+    );
+  }else{
+    rc = execSqlF(db, pzErrMsg,
+        "SELECT'INSERT INTO vacuum_db.'||quote(name)"
+        "||' SELECT*FROM\"%w\".'||quote(name)"
+        "FROM vacuum_db.sqlite_schema "
+        "WHERE type='table'AND coalesce(rootpage,1)>0 AND name",
+        zDbMain
+    );
+  }
+#else
   /* Loop through the tables in the main database. For each, do
   ** an "INSERT INTO vacuum_db.xxx SELECT * FROM main.xxx;" to copy
   ** the contents to the temporary database.
@@ -305,6 +330,7 @@ SQLITE_NOINLINE int sqlite3RunVacuum(
       "WHERE type='table'AND coalesce(rootpage,1)>0",
       zDbMain
   );
+#endif
   assert( (db->mDbFlags & DBFLAG_Vacuum)!=0 );
   db->mDbFlags &= ~DBFLAG_Vacuum;
   if( rc!=SQLITE_OK ) goto end_of_vacuum;

libsql-sqlite3/src/vectorIndex.c

@@ -49,11 +49,6 @@
 ** VectorIdxParams utilities
 ****************************************************************************/
 
-// VACUUM creates tables and indices first and only then populate data
-// we need to ignore inserts from 'INSERT INTO vacuum.t SELECT * FROM t' statements because
-// all shadow tables will be populated by VACUUM process during regular process of table copy
-#define IsVacuum(db) ((db->mDbFlags&DBFLAG_Vacuum)!=0)
-
 void vectorIdxParamsInit(VectorIdxParams *pParams, u8 *pBinBuf, int nBinSize) {
   assert( nBinSize <= VECTOR_INDEX_PARAMS_BUF_SIZE );
 
@@ -772,10 +767,6 @@ int vectorIndexDrop(sqlite3 *db, const char *zDbSName, const char *zIdxName) {
   // this is done to prevent unrecoverable situations where index were dropped but index parameters deletion failed and second attempt will fail on first step
   int rcIdx, rcParams;
 
-  if( IsVacuum(db) ){
-    return SQLITE_OK;
-  }
-
   assert( zDbSName != NULL );
 
   rcIdx = diskAnnDropIndex(db, zDbSName, zIdxName);
@@ -786,10 +777,6 @@ int vectorIndexDrop(sqlite3 *db, const char *zDbSName, const char *zIdxName) {
 int vectorIndexClear(sqlite3 *db, const char *zDbSName, const char *zIdxName) {
   assert( zDbSName != NULL );
 
-  if( IsVacuum(db) ){
-    return SQLITE_OK;
-  }
-
   return diskAnnClearIndex(db, zDbSName, zIdxName);
 }
 
@@ -799,7 +786,7 @@ int vectorIndexClear(sqlite3 *db, const char *zDbSName, const char *zIdxName) {
  * this made intentionally in order to natively support upload of SQLite dumps
  *
  * dump populates tables first and create indices after
- * so we must omit them because shadow tables already filled
+ * so we must omit index refill setp because shadow tables already filled
  *
  * 1. in case of any error                                        :-1 returned (and pParse errMsg is populated with some error message)
  * 2. if vector index must not be created                         : 0 returned
@@ -817,10 +804,6 @@ int vectorIndexCreate(Parse *pParse, const Index *pIdx, const char *zDbSName, co
   int hasLibsqlVectorIdxFn = 0, hasCollation = 0;
   const char *pzErrMsg;
 
-  if( IsVacuum(pParse->db) ){
-    return CREATE_IGNORE;
-  }
-
   assert( zDbSName != NULL );
 
   sqlite3 *db = pParse->db;
@@ -970,7 +953,6 @@ int vectorIndexSearch(
   VectorIdxParams idxParams;
   vectorIdxParamsInit(&idxParams, NULL, 0);
 
-  assert( !IsVacuum(db) );
   assert( zDbSName != NULL );
 
   if( argc != 3 ){
@@ -1055,10 +1037,6 @@ int vectorIndexInsert(
   int rc;
   VectorInRow vectorInRow;
 
-  if( IsVacuum(pCur->db) ){
-    return SQLITE_OK;
-  }
-
   rc = vectorInRowAlloc(pCur->db, pRecord, &vectorInRow, pzErrMsg);
   if( rc != SQLITE_OK ){
     return rc;
@@ -1078,10 +1056,6 @@ int vectorIndexDelete(
 ){
   VectorInRow payload;
 
-  if( IsVacuum(pCur->db) ){
-    return SQLITE_OK;
-  }
-
   payload.pVector = NULL;
   payload.nKeys = r->nField - 1;
   payload.pKeyValues = r->aMem + 1;

libsql-sqlite3/test/libsql_vector_index.test

@@ -236,12 +236,17 @@ do_execsql_test vector-attach {
 
 do_execsql_test vector-vacuum {
   CREATE TABLE t_vacuum ( emb FLOAT32(2) );
-  INSERT INTO t_vacuum VALUES (vector('[1,2]')), (vector('[3,4]'));
+  INSERT INTO t_vacuum VALUES (vector('[1,2]')), (vector('[3,4]')), (vector('[5,6]'));
   CREATE INDEX t_vacuum_idx ON t_vacuum(libsql_vector_idx(emb));
   VACUUM;
   SELECT COUNT(*) FROM t_vacuum;
   SELECT COUNT(*) FROM t_vacuum_idx_shadow;
-} {2 2}
+  DELETE FROM t_vacuum WHERE rowid = 2;
+  VACUUM;
+  SELECT * FROM vector_top_k('t_vacuum_idx', vector('[1,2]'), 3);
+  SELECT * FROM vector_top_k('t_vacuum_idx', vector('[5,6]'), 3);
+  SELECT * FROM vector_top_k('t_vacuum_idx', vector('[3,4]'), 3);
+} {3 3 1 2 2 1 2 1}
 
 do_execsql_test vector-many-columns {
   CREATE TABLE t_many ( i INTEGER PRIMARY KEY, e1 FLOAT32(2), e2 FLOAT32(2) );