Skip to content

Commit 9d15ebf

Browse files
sivukhinsivukhin
authored
Merge pull request #1641 from tursodatabase/vector-search-fix-vacuum
fix behaviour of VACUUM for vector indices to make rowid consistent between shadow tables and base table
2 parents 95be43c + 2115277 commit 9d15ebf

5 files changed

Lines changed: 90 additions & 83 deletions

File tree

‎libsql-ffi/bundled/SQLite3MultipleCiphers/src/sqlite3.c‎

Lines changed: 28 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@
6969
** src/test2.c
7070
** src/test3.c
7171
** src/test8.c
72+
** src/vacuum.c
7273
** src/vdbe.c
7374
** src/vdbeInt.h
7475
** src/vdbeapi.c
@@ -155952,6 +155953,10 @@ SQLITE_PRIVATE void sqlite3UpsertDoUpdate(
155952155953
/* #include "sqliteInt.h" */
155953155954
/* #include "vdbeInt.h" */
155954155955

155956+
#ifndef SQLITE_OMIT_VECTOR
155957+
/* #include "vectorIndexInt.h" */
155958+
#endif
155959+
155955155960
#if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH)
155956155961

155957155962
/*
@@ -156229,6 +156234,27 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum(
156229156234
if( rc!=SQLITE_OK ) goto end_of_vacuum;
156230156235
db->init.iDb = 0;
156231156236

156237+
#ifndef SQLITE_OMIT_VECTOR
156238+
// shadow tables for vector index will be populated automatically during CREATE INDEX command
156239+
// so we must skip them at this step
156240+
if( sqlite3FindTable(db, VECTOR_INDEX_GLOBAL_META_TABLE, zDbMain) != NULL ){
156241+
rc = execSqlF(db, pzErrMsg,
156242+
"SELECT'INSERT INTO vacuum_db.'||quote(name)"
156243+
"||' SELECT*FROM\"%w\".'||quote(name)"
156244+
"FROM vacuum_db.sqlite_schema "
156245+
"WHERE type='table'AND coalesce(rootpage,1)>0 AND name NOT IN (SELECT name||'_shadow' FROM " VECTOR_INDEX_GLOBAL_META_TABLE ")",
156246+
zDbMain
156247+
);
156248+
}else{
156249+
rc = execSqlF(db, pzErrMsg,
156250+
"SELECT'INSERT INTO vacuum_db.'||quote(name)"
156251+
"||' SELECT*FROM\"%w\".'||quote(name)"
156252+
"FROM vacuum_db.sqlite_schema "
156253+
"WHERE type='table'AND coalesce(rootpage,1)>0",
156254+
zDbMain
156255+
);
156256+
}
156257+
#else
156232156258
/* Loop through the tables in the main database. For each, do
156233156259
** an "INSERT INTO vacuum_db.xxx SELECT * FROM main.xxx;" to copy
156234156260
** the contents to the temporary database.
@@ -156240,6 +156266,7 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum(
156240156266
"WHERE type='table'AND coalesce(rootpage,1)>0",
156241156267
zDbMain
156242156268
);
156269+
#endif
156243156270
assert( (db->mDbFlags & DBFLAG_Vacuum)!=0 );
156244156271
db->mDbFlags &= ~DBFLAG_Vacuum;
156245156272
if( rc!=SQLITE_OK ) goto end_of_vacuum;
@@ -213658,11 +213685,6 @@ int vectorF64ParseSqliteBlob(
213658213685
** VectorIdxParams utilities
213659213686
****************************************************************************/
213660213687

213661-
// VACUUM creates tables and indices first and only then populate data
213662-
// we need to ignore inserts from 'INSERT INTO vacuum.t SELECT * FROM t' statements because
213663-
// all shadow tables will be populated by VACUUM process during regular process of table copy
213664-
#define IsVacuum(db) ((db->mDbFlags&DBFLAG_Vacuum)!=0)
213665-
213666213688
void vectorIdxParamsInit(VectorIdxParams *pParams, u8 *pBinBuf, int nBinSize) {
213667213689
assert( nBinSize <= VECTOR_INDEX_PARAMS_BUF_SIZE );
213668213690

@@ -214381,10 +214403,6 @@ int vectorIndexDrop(sqlite3 *db, const char *zDbSName, const char *zIdxName) {
214381214403
// this is done to prevent unrecoverable situations where index were dropped but index parameters deletion failed and second attempt will fail on first step
214382214404
int rcIdx, rcParams;
214383214405

214384-
if( IsVacuum(db) ){
214385-
return SQLITE_OK;
214386-
}
214387-
214388214406
assert( zDbSName != NULL );
214389214407

214390214408
rcIdx = diskAnnDropIndex(db, zDbSName, zIdxName);
@@ -214395,10 +214413,6 @@ int vectorIndexDrop(sqlite3 *db, const char *zDbSName, const char *zIdxName) {
214395214413
int vectorIndexClear(sqlite3 *db, const char *zDbSName, const char *zIdxName) {
214396214414
assert( zDbSName != NULL );
214397214415

214398-
if( IsVacuum(db) ){
214399-
return SQLITE_OK;
214400-
}
214401-
214402214416
return diskAnnClearIndex(db, zDbSName, zIdxName);
214403214417
}
214404214418

@@ -214408,7 +214422,7 @@ int vectorIndexClear(sqlite3 *db, const char *zDbSName, const char *zIdxName) {
214408214422
* this made intentionally in order to natively support upload of SQLite dumps
214409214423
*
214410214424
* dump populates tables first and create indices after
214411-
* so we must omit them because shadow tables already filled
214425+
* so we must omit index refill setp because shadow tables already filled
214412214426
*
214413214427
* 1. in case of any error :-1 returned (and pParse errMsg is populated with some error message)
214414214428
* 2. if vector index must not be created : 0 returned
@@ -214426,10 +214440,6 @@ int vectorIndexCreate(Parse *pParse, const Index *pIdx, const char *zDbSName, co
214426214440
int hasLibsqlVectorIdxFn = 0, hasCollation = 0;
214427214441
const char *pzErrMsg;
214428214442

214429-
if( IsVacuum(pParse->db) ){
214430-
return CREATE_IGNORE;
214431-
}
214432-
214433214443
assert( zDbSName != NULL );
214434214444

214435214445
sqlite3 *db = pParse->db;
@@ -214582,7 +214592,6 @@ int vectorIndexSearch(
214582214592
VectorIdxParams idxParams;
214583214593
vectorIdxParamsInit(&idxParams, NULL, 0);
214584214594

214585-
assert( !IsVacuum(db) );
214586214595
assert( zDbSName != NULL );
214587214596

214588214597
if( argc != 3 ){
@@ -214667,10 +214676,6 @@ int vectorIndexInsert(
214667214676
int rc;
214668214677
VectorInRow vectorInRow;
214669214678

214670-
if( IsVacuum(pCur->db) ){
214671-
return SQLITE_OK;
214672-
}
214673-
214674214679
rc = vectorInRowAlloc(pCur->db, pRecord, &vectorInRow, pzErrMsg);
214675214680
if( rc != SQLITE_OK ){
214676214681
return rc;
@@ -214690,10 +214695,6 @@ int vectorIndexDelete(
214690214695
){
214691214696
VectorInRow payload;
214692214697

214693-
if( IsVacuum(pCur->db) ){
214694-
return SQLITE_OK;
214695-
}
214696-
214697214698
payload.pVector = NULL;
214698214699
payload.nKeys = r->nField - 1;
214699214700
payload.pKeyValues = r->aMem + 1;

‎libsql-ffi/bundled/src/sqlite3.c‎

Lines changed: 28 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@
6969
** src/test2.c
7070
** src/test3.c
7171
** src/test8.c
72+
** src/vacuum.c
7273
** src/vdbe.c
7374
** src/vdbeInt.h
7475
** src/vdbeapi.c
@@ -155952,6 +155953,10 @@ SQLITE_PRIVATE void sqlite3UpsertDoUpdate(
155952155953
/* #include "sqliteInt.h" */
155953155954
/* #include "vdbeInt.h" */
155954155955

155956+
#ifndef SQLITE_OMIT_VECTOR
155957+
/* #include "vectorIndexInt.h" */
155958+
#endif
155959+
155955155960
#if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH)
155956155961

155957155962
/*
@@ -156229,6 +156234,27 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum(
156229156234
if( rc!=SQLITE_OK ) goto end_of_vacuum;
156230156235
db->init.iDb = 0;
156231156236

156237+
#ifndef SQLITE_OMIT_VECTOR
156238+
// shadow tables for vector index will be populated automatically during CREATE INDEX command
156239+
// so we must skip them at this step
156240+
if( sqlite3FindTable(db, VECTOR_INDEX_GLOBAL_META_TABLE, zDbMain) != NULL ){
156241+
rc = execSqlF(db, pzErrMsg,
156242+
"SELECT'INSERT INTO vacuum_db.'||quote(name)"
156243+
"||' SELECT*FROM\"%w\".'||quote(name)"
156244+
"FROM vacuum_db.sqlite_schema "
156245+
"WHERE type='table'AND coalesce(rootpage,1)>0 AND name NOT IN (SELECT name||'_shadow' FROM " VECTOR_INDEX_GLOBAL_META_TABLE ")",
156246+
zDbMain
156247+
);
156248+
}else{
156249+
rc = execSqlF(db, pzErrMsg,
156250+
"SELECT'INSERT INTO vacuum_db.'||quote(name)"
156251+
"||' SELECT*FROM\"%w\".'||quote(name)"
156252+
"FROM vacuum_db.sqlite_schema "
156253+
"WHERE type='table'AND coalesce(rootpage,1)>0",
156254+
zDbMain
156255+
);
156256+
}
156257+
#else
156232156258
/* Loop through the tables in the main database. For each, do
156233156259
** an "INSERT INTO vacuum_db.xxx SELECT * FROM main.xxx;" to copy
156234156260
** the contents to the temporary database.
@@ -156240,6 +156266,7 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum(
156240156266
"WHERE type='table'AND coalesce(rootpage,1)>0",
156241156267
zDbMain
156242156268
);
156269+
#endif
156243156270
assert( (db->mDbFlags & DBFLAG_Vacuum)!=0 );
156244156271
db->mDbFlags &= ~DBFLAG_Vacuum;
156245156272
if( rc!=SQLITE_OK ) goto end_of_vacuum;
@@ -213658,11 +213685,6 @@ int vectorF64ParseSqliteBlob(
213658213685
** VectorIdxParams utilities
213659213686
****************************************************************************/
213660213687

213661-
// VACUUM creates tables and indices first and only then populate data
213662-
// we need to ignore inserts from 'INSERT INTO vacuum.t SELECT * FROM t' statements because
213663-
// all shadow tables will be populated by VACUUM process during regular process of table copy
213664-
#define IsVacuum(db) ((db->mDbFlags&DBFLAG_Vacuum)!=0)
213665-
213666213688
void vectorIdxParamsInit(VectorIdxParams *pParams, u8 *pBinBuf, int nBinSize) {
213667213689
assert( nBinSize <= VECTOR_INDEX_PARAMS_BUF_SIZE );
213668213690

@@ -214381,10 +214403,6 @@ int vectorIndexDrop(sqlite3 *db, const char *zDbSName, const char *zIdxName) {
214381214403
// this is done to prevent unrecoverable situations where index were dropped but index parameters deletion failed and second attempt will fail on first step
214382214404
int rcIdx, rcParams;
214383214405

214384-
if( IsVacuum(db) ){
214385-
return SQLITE_OK;
214386-
}
214387-
214388214406
assert( zDbSName != NULL );
214389214407

214390214408
rcIdx = diskAnnDropIndex(db, zDbSName, zIdxName);
@@ -214395,10 +214413,6 @@ int vectorIndexDrop(sqlite3 *db, const char *zDbSName, const char *zIdxName) {
214395214413
int vectorIndexClear(sqlite3 *db, const char *zDbSName, const char *zIdxName) {
214396214414
assert( zDbSName != NULL );
214397214415

214398-
if( IsVacuum(db) ){
214399-
return SQLITE_OK;
214400-
}
214401-
214402214416
return diskAnnClearIndex(db, zDbSName, zIdxName);
214403214417
}
214404214418

@@ -214408,7 +214422,7 @@ int vectorIndexClear(sqlite3 *db, const char *zDbSName, const char *zIdxName) {
214408214422
* this made intentionally in order to natively support upload of SQLite dumps
214409214423
*
214410214424
* dump populates tables first and create indices after
214411-
* so we must omit them because shadow tables already filled
214425+
* so we must omit index refill setp because shadow tables already filled
214412214426
*
214413214427
* 1. in case of any error :-1 returned (and pParse errMsg is populated with some error message)
214414214428
* 2. if vector index must not be created : 0 returned
@@ -214426,10 +214440,6 @@ int vectorIndexCreate(Parse *pParse, const Index *pIdx, const char *zDbSName, co
214426214440
int hasLibsqlVectorIdxFn = 0, hasCollation = 0;
214427214441
const char *pzErrMsg;
214428214442

214429-
if( IsVacuum(pParse->db) ){
214430-
return CREATE_IGNORE;
214431-
}
214432-
214433214443
assert( zDbSName != NULL );
214434214444

214435214445
sqlite3 *db = pParse->db;
@@ -214582,7 +214592,6 @@ int vectorIndexSearch(
214582214592
VectorIdxParams idxParams;
214583214593
vectorIdxParamsInit(&idxParams, NULL, 0);
214584214594

214585-
assert( !IsVacuum(db) );
214586214595
assert( zDbSName != NULL );
214587214596

214588214597
if( argc != 3 ){
@@ -214667,10 +214676,6 @@ int vectorIndexInsert(
214667214676
int rc;
214668214677
VectorInRow vectorInRow;
214669214678

214670-
if( IsVacuum(pCur->db) ){
214671-
return SQLITE_OK;
214672-
}
214673-
214674214679
rc = vectorInRowAlloc(pCur->db, pRecord, &vectorInRow, pzErrMsg);
214675214680
if( rc != SQLITE_OK ){
214676214681
return rc;
@@ -214690,10 +214695,6 @@ int vectorIndexDelete(
214690214695
){
214691214696
VectorInRow payload;
214692214697

214693-
if( IsVacuum(pCur->db) ){
214694-
return SQLITE_OK;
214695-
}
214696-
214697214698
payload.pVector = NULL;
214698214699
payload.nKeys = r->nField - 1;
214699214700
payload.pKeyValues = r->aMem + 1;

‎libsql-sqlite3/src/vacuum.c‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,10 @@
1717
#include "sqliteInt.h"
1818
#include "vdbeInt.h"
1919

20+
#ifndef SQLITE_OMIT_VECTOR
21+
#include "vectorIndexInt.h"
22+
#endif
23+
2024
#if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH)
2125

2226
/*
@@ -294,6 +298,27 @@ SQLITE_NOINLINE int sqlite3RunVacuum(
294298
if( rc!=SQLITE_OK ) goto end_of_vacuum;
295299
db->init.iDb = 0;
296300

301+
#ifndef SQLITE_OMIT_VECTOR
302+
// shadow tables for vector index will be populated automatically during CREATE INDEX command
303+
// so we must skip them at this step
304+
if( sqlite3FindTable(db, VECTOR_INDEX_GLOBAL_META_TABLE, zDbMain) != NULL ){
305+
rc = execSqlF(db, pzErrMsg,
306+
"SELECT'INSERT INTO vacuum_db.'||quote(name)"
307+
"||' SELECT*FROM\"%w\".'||quote(name)"
308+
"FROM vacuum_db.sqlite_schema "
309+
"WHERE type='table'AND coalesce(rootpage,1)>0 AND name NOT IN (SELECT name||'_shadow' FROM " VECTOR_INDEX_GLOBAL_META_TABLE ")",
310+
zDbMain
311+
);
312+
}else{
313+
rc = execSqlF(db, pzErrMsg,
314+
"SELECT'INSERT INTO vacuum_db.'||quote(name)"
315+
"||' SELECT*FROM\"%w\".'||quote(name)"
316+
"FROM vacuum_db.sqlite_schema "
317+
"WHERE type='table'AND coalesce(rootpage,1)>0",
318+
zDbMain
319+
);
320+
}
321+
#else
297322
/* Loop through the tables in the main database. For each, do
298323
** an "INSERT INTO vacuum_db.xxx SELECT * FROM main.xxx;" to copy
299324
** the contents to the temporary database.
@@ -305,6 +330,7 @@ SQLITE_NOINLINE int sqlite3RunVacuum(
305330
"WHERE type='table'AND coalesce(rootpage,1)>0",
306331
zDbMain
307332
);
333+
#endif
308334
assert( (db->mDbFlags & DBFLAG_Vacuum)!=0 );
309335
db->mDbFlags &= ~DBFLAG_Vacuum;
310336
if( rc!=SQLITE_OK ) goto end_of_vacuum;

0 commit comments

Comments
 (0)