Skip to content

Commit 2f15a0e

Browse files
sivukhinsivukhin
authored
Merge pull request #1678 from tursodatabase/hide-vector-index-for-sqlite-planner
vector search: hide vector index for sqlite planner
2 parents f76bc0a + 18e09dd commit 2f15a0e

7 files changed

Lines changed: 51 additions & 29 deletions

File tree

‎libsql-ffi/bundled/SQLite3MultipleCiphers/src/sqlite3.c‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@
7878
** src/vtab.c
7979
** src/wal.c
8080
** src/wal.h
81+
** src/where.c
8182
** src/wherecode.c
8283
** test/all.test
8384
** test/permutations.test
@@ -126770,11 +126771,6 @@ SQLITE_PRIVATE void sqlite3CreateIndex(
126770126771
goto exit_create_index;
126771126772
}
126772126773
if( vectorIdxRc >= 1 ){
126773-
/*
126774-
* SQLite can use B-Tree indices in some optimizations (like SELECT COUNT(*) can use any full B-Tree index instead of PK index)
126775-
* But, SQLite pretty conservative about usage of unordered indices - that's what we need here
126776-
*/
126777-
pIndex->bUnordered = 1;
126778126774
pIndex->idxIsVector = 1;
126779126775
}
126780126776
if( vectorIdxRc == 1 ){
@@ -152469,6 +152465,7 @@ SQLITE_PRIVATE int sqlite3Select(
152469152465
if( pIdx->bUnordered==0
152470152466
&& pIdx->szIdxRow<pTab->szTabRow
152471152467
&& pIdx->pPartIdxWhere==0
152468+
&& pIdx->idxIsVector==0
152472152469
&& (!pBest || pIdx->szIdxRow<pBest->szIdxRow)
152473152470
){
152474152471
pBest = pIdx;
@@ -166093,9 +166090,10 @@ static int whereLoopAddBtreeIndex(
166093166090
assert( pNew->u.btree.nBtm==0 );
166094166091
opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS;
166095166092
}
166096-
if( pProbe->bUnordered || pProbe->bLowQual ){
166093+
if( pProbe->bUnordered || pProbe->bLowQual || pProbe->idxIsVector ){
166097166094
if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE);
166098166095
if( pProbe->bLowQual ) opMask &= ~(WO_EQ|WO_IN|WO_IS);
166096+
if( pProbe->idxIsVector ) opMask = 0;
166099166097
}
166100166098

166101166099
assert( pNew->u.btree.nEq<pProbe->nColumn );
@@ -166477,7 +166475,7 @@ static int indexMightHelpWithOrderBy(
166477166475
ExprList *aColExpr;
166478166476
int ii, jj;
166479166477

166480-
if( pIndex->bUnordered ) return 0;
166478+
if( pIndex->bUnordered || pIndex->idxIsVector ) return 0;
166481166479
if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0;
166482166480
for(ii=0; ii<pOB->nExpr; ii++){
166483166481
Expr *pExpr = sqlite3ExprSkipCollateAndLikely(pOB->a[ii].pExpr);
@@ -166646,6 +166644,9 @@ static SQLITE_NOINLINE u32 whereIsCoveringIndex(
166646166644
** if pIdx is covering. Assume it is not. */
166647166645
return 0;
166648166646
}
166647+
if( pIdx->idxIsVector==1 ){
166648+
return 0;
166649+
}
166649166650
if( pIdx->bHasExpr==0 ){
166650166651
for(i=0; i<pIdx->nColumn; i++){
166651166652
if( pIdx->aiColumn[i]>=BMS-1 ) break;
@@ -166934,6 +166935,9 @@ static int whereLoopAddBtree(
166934166935
testcase( pNew->iTab!=pSrc->iCursor ); /* See ticket [98d973b8f5] */
166935166936
continue; /* Partial index inappropriate for this query */
166936166937
}
166938+
if( pProbe->idxIsVector!=0 ){
166939+
continue; /* Vector index inappropriate for this query */
166940+
}
166937166941
if( pProbe->bNoQuery ) continue;
166938166942
rSize = pProbe->aiRowLogEst[0];
166939166943
pNew->u.btree.nEq = 0;
@@ -167937,7 +167941,7 @@ static i8 wherePathSatisfiesOrderBy(
167937167941
pIndex = 0;
167938167942
nKeyCol = 0;
167939167943
nColumn = 1;
167940-
}else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){
167944+
}else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered || pIndex->idxIsVector ){
167941167945
return 0;
167942167946
}else{
167943167947
nKeyCol = pIndex->nKeyCol;
@@ -216046,7 +216050,6 @@ int vectorIndexSearch(
216046216050
rc = SQLITE_ERROR;
216047216051
goto out;
216048216052
}
216049-
assert( type == VECTOR_TYPE_FLOAT32 || type == VECTOR_TYPE_FLOAT64 || type == VECTOR_TYPE_FLOAT1BIT );
216050216053

216051216054
pVector = vectorAlloc(type, dims);
216052216055
if( pVector == NULL ){

‎libsql-ffi/bundled/src/sqlite3.c‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@
7878
** src/vtab.c
7979
** src/wal.c
8080
** src/wal.h
81+
** src/where.c
8182
** src/wherecode.c
8283
** test/all.test
8384
** test/permutations.test
@@ -126770,11 +126771,6 @@ SQLITE_PRIVATE void sqlite3CreateIndex(
126770126771
goto exit_create_index;
126771126772
}
126772126773
if( vectorIdxRc >= 1 ){
126773-
/*
126774-
* SQLite can use B-Tree indices in some optimizations (like SELECT COUNT(*) can use any full B-Tree index instead of PK index)
126775-
* But, SQLite pretty conservative about usage of unordered indices - that's what we need here
126776-
*/
126777-
pIndex->bUnordered = 1;
126778126774
pIndex->idxIsVector = 1;
126779126775
}
126780126776
if( vectorIdxRc == 1 ){
@@ -152469,6 +152465,7 @@ SQLITE_PRIVATE int sqlite3Select(
152469152465
if( pIdx->bUnordered==0
152470152466
&& pIdx->szIdxRow<pTab->szTabRow
152471152467
&& pIdx->pPartIdxWhere==0
152468+
&& pIdx->idxIsVector==0
152472152469
&& (!pBest || pIdx->szIdxRow<pBest->szIdxRow)
152473152470
){
152474152471
pBest = pIdx;
@@ -166093,9 +166090,10 @@ static int whereLoopAddBtreeIndex(
166093166090
assert( pNew->u.btree.nBtm==0 );
166094166091
opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS;
166095166092
}
166096-
if( pProbe->bUnordered || pProbe->bLowQual ){
166093+
if( pProbe->bUnordered || pProbe->bLowQual || pProbe->idxIsVector ){
166097166094
if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE);
166098166095
if( pProbe->bLowQual ) opMask &= ~(WO_EQ|WO_IN|WO_IS);
166096+
if( pProbe->idxIsVector ) opMask = 0;
166099166097
}
166100166098

166101166099
assert( pNew->u.btree.nEq<pProbe->nColumn );
@@ -166477,7 +166475,7 @@ static int indexMightHelpWithOrderBy(
166477166475
ExprList *aColExpr;
166478166476
int ii, jj;
166479166477

166480-
if( pIndex->bUnordered ) return 0;
166478+
if( pIndex->bUnordered || pIndex->idxIsVector ) return 0;
166481166479
if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0;
166482166480
for(ii=0; ii<pOB->nExpr; ii++){
166483166481
Expr *pExpr = sqlite3ExprSkipCollateAndLikely(pOB->a[ii].pExpr);
@@ -166646,6 +166644,9 @@ static SQLITE_NOINLINE u32 whereIsCoveringIndex(
166646166644
** if pIdx is covering. Assume it is not. */
166647166645
return 0;
166648166646
}
166647+
if( pIdx->idxIsVector==1 ){
166648+
return 0;
166649+
}
166649166650
if( pIdx->bHasExpr==0 ){
166650166651
for(i=0; i<pIdx->nColumn; i++){
166651166652
if( pIdx->aiColumn[i]>=BMS-1 ) break;
@@ -166934,6 +166935,9 @@ static int whereLoopAddBtree(
166934166935
testcase( pNew->iTab!=pSrc->iCursor ); /* See ticket [98d973b8f5] */
166935166936
continue; /* Partial index inappropriate for this query */
166936166937
}
166938+
if( pProbe->idxIsVector!=0 ){
166939+
continue; /* Vector index inappropriate for this query */
166940+
}
166937166941
if( pProbe->bNoQuery ) continue;
166938166942
rSize = pProbe->aiRowLogEst[0];
166939166943
pNew->u.btree.nEq = 0;
@@ -167937,7 +167941,7 @@ static i8 wherePathSatisfiesOrderBy(
167937167941
pIndex = 0;
167938167942
nKeyCol = 0;
167939167943
nColumn = 1;
167940-
}else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){
167944+
}else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered || pIndex->idxIsVector ){
167941167945
return 0;
167942167946
}else{
167943167947
nKeyCol = pIndex->nKeyCol;
@@ -216046,7 +216050,6 @@ int vectorIndexSearch(
216046216050
rc = SQLITE_ERROR;
216047216051
goto out;
216048216052
}
216049-
assert( type == VECTOR_TYPE_FLOAT32 || type == VECTOR_TYPE_FLOAT64 || type == VECTOR_TYPE_FLOAT1BIT );
216050216053

216051216054
pVector = vectorAlloc(type, dims);
216052216055
if( pVector == NULL ){

‎libsql-sqlite3/src/build.c‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4345,11 +4345,6 @@ void sqlite3CreateIndex(
43454345
goto exit_create_index;
43464346
}
43474347
if( vectorIdxRc >= 1 ){
4348-
/*
4349-
* SQLite can use B-Tree indices in some optimizations (like SELECT COUNT(*) can use any full B-Tree index instead of PK index)
4350-
* But, SQLite pretty conservative about usage of unordered indices - that's what we need here
4351-
*/
4352-
pIndex->bUnordered = 1;
43534348
pIndex->idxIsVector = 1;
43544349
}
43554350
if( vectorIdxRc == 1 ){

‎libsql-sqlite3/src/select.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8423,6 +8423,7 @@ int sqlite3Select(
84238423
if( pIdx->bUnordered==0
84248424
&& pIdx->szIdxRow<pTab->szTabRow
84258425
&& pIdx->pPartIdxWhere==0
8426+
&& pIdx->idxIsVector==0
84268427
&& (!pBest || pIdx->szIdxRow<pBest->szIdxRow)
84278428
){
84288429
pBest = pIdx;

‎libsql-sqlite3/src/vectorIndex.c‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -971,7 +971,6 @@ int vectorIndexSearch(
971971
rc = SQLITE_ERROR;
972972
goto out;
973973
}
974-
assert( type == VECTOR_TYPE_FLOAT32 || type == VECTOR_TYPE_FLOAT64 || type == VECTOR_TYPE_FLOAT1BIT );
975974

976975
pVector = vectorAlloc(type, dims);
977976
if( pVector == NULL ){

‎libsql-sqlite3/src/where.c‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2973,9 +2973,10 @@ static int whereLoopAddBtreeIndex(
29732973
assert( pNew->u.btree.nBtm==0 );
29742974
opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS;
29752975
}
2976-
if( pProbe->bUnordered || pProbe->bLowQual ){
2976+
if( pProbe->bUnordered || pProbe->bLowQual || pProbe->idxIsVector ){
29772977
if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE);
29782978
if( pProbe->bLowQual ) opMask &= ~(WO_EQ|WO_IN|WO_IS);
2979+
if( pProbe->idxIsVector ) opMask = 0;
29792980
}
29802981

29812982
assert( pNew->u.btree.nEq<pProbe->nColumn );
@@ -3357,7 +3358,7 @@ static int indexMightHelpWithOrderBy(
33573358
ExprList *aColExpr;
33583359
int ii, jj;
33593360

3360-
if( pIndex->bUnordered ) return 0;
3361+
if( pIndex->bUnordered || pIndex->idxIsVector ) return 0;
33613362
if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0;
33623363
for(ii=0; ii<pOB->nExpr; ii++){
33633364
Expr *pExpr = sqlite3ExprSkipCollateAndLikely(pOB->a[ii].pExpr);
@@ -3526,6 +3527,9 @@ static SQLITE_NOINLINE u32 whereIsCoveringIndex(
35263527
** if pIdx is covering. Assume it is not. */
35273528
return 0;
35283529
}
3530+
if( pIdx->idxIsVector==1 ){
3531+
return 0;
3532+
}
35293533
if( pIdx->bHasExpr==0 ){
35303534
for(i=0; i<pIdx->nColumn; i++){
35313535
if( pIdx->aiColumn[i]>=BMS-1 ) break;
@@ -3814,6 +3818,9 @@ static int whereLoopAddBtree(
38143818
testcase( pNew->iTab!=pSrc->iCursor ); /* See ticket [98d973b8f5] */
38153819
continue; /* Partial index inappropriate for this query */
38163820
}
3821+
if( pProbe->idxIsVector!=0 ){
3822+
continue; /* Vector index inappropriate for this query */
3823+
}
38173824
if( pProbe->bNoQuery ) continue;
38183825
rSize = pProbe->aiRowLogEst[0];
38193826
pNew->u.btree.nEq = 0;
@@ -4817,7 +4824,7 @@ static i8 wherePathSatisfiesOrderBy(
48174824
pIndex = 0;
48184825
nKeyCol = 0;
48194826
nColumn = 1;
4820-
}else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){
4827+
}else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered || pIndex->idxIsVector ){
48214828
return 0;
48224829
}else{
48234830
nKeyCol = pIndex->nKeyCol;

‎libsql-sqlite3/test/libsql_vector_index.test‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,20 @@ do_execsql_test vector-index-dont-affect-sql {
225225
SELECT rowid FROM t_vector_other_sql WHERE emb = vector('[7,8]');
226226
} {4 1 1 2 3 4}
227227

228+
do_execsql_test vector-partial-index-dont-affect-sql {
229+
CREATE TABLE t_vector_other_sql_part ( emb FLOAT32(2), t TEXT );
230+
INSERT INTO t_vector_other_sql_part VALUES (vector('[1,2]'), 'paper'), (vector('[3,4]'), 'journal');
231+
CREATE INDEX t_vector_other_sql_part_idx ON t_vector_other_sql_part(libsql_vector_idx(emb));
232+
INSERT INTO t_vector_other_sql_part VALUES (vector('[5,6]'), 'journal'), (vector('[7,8]'), 'paper');
233+
SELECT COUNT(*) FROM t_vector_other_sql_part;
234+
SELECT COUNT(*) FROM t_vector_other_sql_part WHERE emb = vector('[1,2]');
235+
SELECT COUNT(*) FROM t_vector_other_sql_part WHERE t = 'paper';
236+
SELECT rowid FROM t_vector_other_sql_part WHERE emb = vector('[1,2]');
237+
SELECT rowid FROM t_vector_other_sql_part WHERE emb = vector('[3,4]');
238+
SELECT rowid FROM t_vector_other_sql_part WHERE emb = vector('[5,6]');
239+
SELECT rowid FROM t_vector_other_sql_part WHERE emb = vector('[7,8]');
240+
} {4 1 2 1 2 3 4}
241+
228242
do_execsql_test vector-index-dont-affect-sql-pk {
229243
CREATE TABLE t_vector_other_sql_pk ( name TEXT PRIMARY KEY, emb FLOAT32(2) );
230244
INSERT INTO t_vector_other_sql_pk VALUES ('a', vector('[1,2]')), ('b', vector('[3,4]'));
@@ -307,6 +321,8 @@ do_execsql_test vector-f64-index {
307321
SELECT * FROM vector_top_k('t_f64_idx', vector64('[1,2]'), 2);
308322
} {1 2}
309323

324+
reset_db
325+
310326
do_execsql_test vector-partial {
311327
CREATE TABLE t_partial( name TEXT, type INT, v FLOAT32(3));
312328
INSERT INTO t_partial VALUES ( 'a', 0, vector('[1,2,3]') );
@@ -448,8 +464,6 @@ proc error_messages {sql} {
448464
set ret [sqlite3_errmsg db]
449465
}
450466

451-
reset_db
452-
453467
do_test vector-errors {
454468
set ret [list]
455469
lappend ret [error_messages {CREATE INDEX t_no_idx ON t_no( libsql_vector_idx(v) )}]

0 commit comments

Comments
 (0)