+ case 99: /* onconf ::= */
+{yygotominor.yy328 = OE_Default;}
+ break;
+ case 101: /* orconf ::= */
+{yygotominor.yy186 = OE_Default;}
+ break;
+ case 102: /* orconf ::= OR resolvetype */
+{yygotominor.yy186 = (u8)yymsp[0].minor.yy328;}
+ break;
+ case 104: /* resolvetype ::= IGNORE */
+{yygotominor.yy328 = OE_Ignore;}
+ break;
+ case 105: /* resolvetype ::= REPLACE */
+{yygotominor.yy328 = OE_Replace;}
+ break;
+ case 106: /* cmd ::= DROP TABLE ifexists fullname */
+{
+ sqlite3DropTable(pParse, yymsp[0].minor.yy65, 0, yymsp[-1].minor.yy328);
+}
+ break;
+ case 109: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */
+{
+ sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy14, yymsp[0].minor.yy3, yymsp[-7].minor.yy328, yymsp[-5].minor.yy328);
+}
+ break;
+ case 110: /* cmd ::= DROP VIEW ifexists fullname */
+{
+ sqlite3DropTable(pParse, yymsp[0].minor.yy65, 1, yymsp[-1].minor.yy328);
+}
+ break;
+ case 111: /* cmd ::= select */
+{
+ SelectDest dest = {SRT_Output, 0, 0, 0, 0, 0};
+ sqlite3Select(pParse, yymsp[0].minor.yy3, &dest);
+ sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy3);
+}
+ break;
+ case 112: /* select ::= with selectnowith */
+{
+ Select *p = yymsp[0].minor.yy3;
+ if( p ){
+ p->pWith = yymsp[-1].minor.yy59;
+ parserDoubleLinkSelect(pParse, p);
+ }else{
+ sqlite3WithDelete(pParse->db, yymsp[-1].minor.yy59);
+ }
+ yygotominor.yy3 = p;
+}
+ break;
+ case 113: /* selectnowith ::= oneselect */
+ case 119: /* oneselect ::= values */ yytestcase(yyruleno==119);
+{yygotominor.yy3 = yymsp[0].minor.yy3;}
+ break;
+ case 114: /* selectnowith ::= selectnowith multiselect_op oneselect */
+{
+ Select *pRhs = yymsp[0].minor.yy3;
+ Select *pLhs = yymsp[-2].minor.yy3;
+ if( pRhs && pRhs->pPrior ){
+ SrcList *pFrom;
+ Token x;
+ x.n = 0;
+ parserDoubleLinkSelect(pParse, pRhs);
+ pFrom = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&x,pRhs,0,0);
+ pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0,0);
+ }
+ if( pRhs ){
+ pRhs->op = (u8)yymsp[-1].minor.yy328;
+ pRhs->pPrior = pLhs;
+ if( ALWAYS(pLhs) ) pLhs->selFlags &= ~SF_MultiValue;
+ pRhs->selFlags &= ~SF_MultiValue;
+ if( yymsp[-1].minor.yy328!=TK_ALL ) pParse->hasCompound = 1;
+ }else{
+ sqlite3SelectDelete(pParse->db, pLhs);
+ }
+ yygotominor.yy3 = pRhs;
+}
+ break;
+ case 116: /* multiselect_op ::= UNION ALL */
+{yygotominor.yy328 = TK_ALL;}
+ break;
+ case 118: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */
+{
+ yygotominor.yy3 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy14,yymsp[-5].minor.yy65,yymsp[-4].minor.yy132,yymsp[-3].minor.yy14,yymsp[-2].minor.yy132,yymsp[-1].minor.yy14,yymsp[-7].minor.yy381,yymsp[0].minor.yy476.pLimit,yymsp[0].minor.yy476.pOffset);
+#if SELECTTRACE_ENABLED
+ /* Populate the Select.zSelName[] string that is used to help with
+ ** query planner debugging, to differentiate between multiple Select
+ ** objects in a complex query.
+ **
+ ** If the SELECT keyword is immediately followed by a C-style comment
+ ** then extract the first few alphanumeric characters from within that
+ ** comment to be the zSelName value. Otherwise, the label is #N where
+ ** is an integer that is incremented with each SELECT statement seen.
+ */
+ if( yygotominor.yy3!=0 ){
+ const char *z = yymsp[-8].minor.yy0.z+6;
+ int i;
+ sqlite3_snprintf(sizeof(yygotominor.yy3->zSelName), yygotominor.yy3->zSelName, "#%d",
+ ++pParse->nSelect);
+ while( z[0]==' ' ) z++;
+ if( z[0]=='/' && z[1]=='*' ){
+ z += 2;
+ while( z[0]==' ' ) z++;
+ for(i=0; sqlite3Isalnum(z[i]); i++){}
+ sqlite3_snprintf(sizeof(yygotominor.yy3->zSelName), yygotominor.yy3->zSelName, "%.*s", i, z);
+ }
+ }
+#endif /* SELECTRACE_ENABLED */
+}
+ break;
+ case 120: /* values ::= VALUES LP nexprlist RP */
+{
+ yygotominor.yy3 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy14,0,0,0,0,0,SF_Values,0,0);
+}
+ break;
+ case 121: /* values ::= values COMMA LP exprlist RP */
+{
+ Select *pRight, *pLeft = yymsp[-4].minor.yy3;
+ pRight = sqlite3SelectNew(pParse,yymsp[-1].minor.yy14,0,0,0,0,0,SF_Values|SF_MultiValue,0,0);
+ if( ALWAYS(pLeft) ) pLeft->selFlags &= ~SF_MultiValue;
+ if( pRight ){
+ pRight->op = TK_ALL;
+ pLeft = yymsp[-4].minor.yy3;
+ pRight->pPrior = pLeft;
+ yygotominor.yy3 = pRight;
+ }else{
+ yygotominor.yy3 = pLeft;
+ }
+}
+ break;
+ case 122: /* distinct ::= DISTINCT */
+{yygotominor.yy381 = SF_Distinct;}
+ break;
+ case 123: /* distinct ::= ALL */
+{yygotominor.yy381 = SF_All;}
+ break;
+ case 124: /* distinct ::= */
+{yygotominor.yy381 = 0;}
+ break;
+ case 125: /* sclp ::= selcollist COMMA */
+ case 244: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==244);
+{yygotominor.yy14 = yymsp[-1].minor.yy14;}
+ break;
+ case 126: /* sclp ::= */
+ case 155: /* orderby_opt ::= */ yytestcase(yyruleno==155);
+ case 162: /* groupby_opt ::= */ yytestcase(yyruleno==162);
+ case 237: /* exprlist ::= */ yytestcase(yyruleno==237);
+ case 243: /* eidlist_opt ::= */ yytestcase(yyruleno==243);
+{yygotominor.yy14 = 0;}
+ break;
+ case 127: /* selcollist ::= sclp expr as */
+{
+ yygotominor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy14, yymsp[-1].minor.yy346.pExpr);
+ if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[0].minor.yy0, 1);
+ sqlite3ExprListSetSpan(pParse,yygotominor.yy14,&yymsp[-1].minor.yy346);
+}
+ break;
+ case 128: /* selcollist ::= sclp STAR */
+{
+ Expr *p = sqlite3Expr(pParse->db, TK_ALL, 0);
+ yygotominor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-1].minor.yy14, p);
+}
+ break;
+ case 129: /* selcollist ::= sclp nm DOT STAR */
+{
+ Expr *pRight = sqlite3PExpr(pParse, TK_ALL, 0, 0, &yymsp[0].minor.yy0);
+ Expr *pLeft = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0);
+ Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0);
+ yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy14, pDot);
+}
+ break;
+ case 132: /* as ::= */
+{yygotominor.yy0.n = 0;}
+ break;
+ case 133: /* from ::= */
+{yygotominor.yy65 = sqlite3DbMallocZero(pParse->db, sizeof(*yygotominor.yy65));}
+ break;
+ case 134: /* from ::= FROM seltablist */
+{
+ yygotominor.yy65 = yymsp[0].minor.yy65;
+ sqlite3SrcListShiftJoinType(yygotominor.yy65);
+}
+ break;
+ case 135: /* stl_prefix ::= seltablist joinop */
+{
+ yygotominor.yy65 = yymsp[-1].minor.yy65;
+ if( ALWAYS(yygotominor.yy65 && yygotominor.yy65->nSrc>0) ) yygotominor.yy65->a[yygotominor.yy65->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy328;
+}
+ break;
+ case 136: /* stl_prefix ::= */
+{yygotominor.yy65 = 0;}
+ break;
+ case 137: /* seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */
+{
+ yygotominor.yy65 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy65,&yymsp[-5].minor.yy0,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,0,yymsp[-1].minor.yy132,yymsp[0].minor.yy408);
+ sqlite3SrcListIndexedBy(pParse, yygotominor.yy65, &yymsp[-2].minor.yy0);
+}
+ break;
+ case 138: /* seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt */
+{
+ yygotominor.yy65 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-8].minor.yy65,&yymsp[-7].minor.yy0,&yymsp[-6].minor.yy0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy132,yymsp[0].minor.yy408);
+ sqlite3SrcListFuncArgs(pParse, yygotominor.yy65, yymsp[-4].minor.yy14);
+}
+ break;
+ case 139: /* seltablist ::= stl_prefix LP select RP as on_opt using_opt */
+{
+ yygotominor.yy65 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy65,0,0,&yymsp[-2].minor.yy0,yymsp[-4].minor.yy3,yymsp[-1].minor.yy132,yymsp[0].minor.yy408);
+ }
+ break;
+ case 140: /* seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */
+{
+ if( yymsp[-6].minor.yy65==0 && yymsp[-2].minor.yy0.n==0 && yymsp[-1].minor.yy132==0 && yymsp[0].minor.yy408==0 ){
+ yygotominor.yy65 = yymsp[-4].minor.yy65;
+ }else if( yymsp[-4].minor.yy65->nSrc==1 ){
+ yygotominor.yy65 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy65,0,0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy132,yymsp[0].minor.yy408);
+ if( yygotominor.yy65 ){
+ struct SrcList_item *pNew = &yygotominor.yy65->a[yygotominor.yy65->nSrc-1];
+ struct SrcList_item *pOld = yymsp[-4].minor.yy65->a;
+ pNew->zName = pOld->zName;
+ pNew->zDatabase = pOld->zDatabase;
+ pNew->pSelect = pOld->pSelect;
+ pOld->zName = pOld->zDatabase = 0;
+ pOld->pSelect = 0;
+ }
+ sqlite3SrcListDelete(pParse->db, yymsp[-4].minor.yy65);
+ }else{
+ Select *pSubquery;
+ sqlite3SrcListShiftJoinType(yymsp[-4].minor.yy65);
+ pSubquery = sqlite3SelectNew(pParse,0,yymsp[-4].minor.yy65,0,0,0,0,SF_NestedFrom,0,0);
+ yygotominor.yy65 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy65,0,0,&yymsp[-2].minor.yy0,pSubquery,yymsp[-1].minor.yy132,yymsp[0].minor.yy408);
+ }
+ }
+ break;
+ case 141: /* dbnm ::= */
+ case 150: /* indexed_opt ::= */ yytestcase(yyruleno==150);
+{yygotominor.yy0.z=0; yygotominor.yy0.n=0;}
+ break;
+ case 143: /* fullname ::= nm dbnm */
+{yygotominor.yy65 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0);}
+ break;
+ case 144: /* joinop ::= COMMA|JOIN */
+{ yygotominor.yy328 = JT_INNER; }
+ break;
+ case 145: /* joinop ::= JOIN_KW JOIN */
+{ yygotominor.yy328 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); }
+ break;
+ case 146: /* joinop ::= JOIN_KW nm JOIN */
+{ yygotominor.yy328 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); }
+ break;
+ case 147: /* joinop ::= JOIN_KW nm nm JOIN */
+{ yygotominor.yy328 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); }
+ break;
+ case 148: /* on_opt ::= ON expr */
+ case 165: /* having_opt ::= HAVING expr */ yytestcase(yyruleno==165);
+ case 172: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==172);
+ case 232: /* case_else ::= ELSE expr */ yytestcase(yyruleno==232);
+ case 234: /* case_operand ::= expr */ yytestcase(yyruleno==234);
+{yygotominor.yy132 = yymsp[0].minor.yy346.pExpr;}
+ break;
+ case 149: /* on_opt ::= */
+ case 164: /* having_opt ::= */ yytestcase(yyruleno==164);
+ case 171: /* where_opt ::= */ yytestcase(yyruleno==171);
+ case 233: /* case_else ::= */ yytestcase(yyruleno==233);
+ case 235: /* case_operand ::= */ yytestcase(yyruleno==235);
+{yygotominor.yy132 = 0;}
+ break;
+ case 152: /* indexed_opt ::= NOT INDEXED */
+{yygotominor.yy0.z=0; yygotominor.yy0.n=1;}
+ break;
+ case 153: /* using_opt ::= USING LP idlist RP */
+ case 181: /* idlist_opt ::= LP idlist RP */ yytestcase(yyruleno==181);
+{yygotominor.yy408 = yymsp[-1].minor.yy408;}
+ break;
+ case 154: /* using_opt ::= */
+ case 180: /* idlist_opt ::= */ yytestcase(yyruleno==180);
+{yygotominor.yy408 = 0;}
+ break;
+ case 156: /* orderby_opt ::= ORDER BY sortlist */
+ case 163: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==163);
+ case 236: /* exprlist ::= nexprlist */ yytestcase(yyruleno==236);
+{yygotominor.yy14 = yymsp[0].minor.yy14;}
+ break;
+ case 157: /* sortlist ::= sortlist COMMA expr sortorder */
+{
+ yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy14,yymsp[-1].minor.yy346.pExpr);
+ sqlite3ExprListSetSortOrder(yygotominor.yy14,yymsp[0].minor.yy328);
+}
+ break;
+ case 158: /* sortlist ::= expr sortorder */
+{
+ yygotominor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[-1].minor.yy346.pExpr);
+ sqlite3ExprListSetSortOrder(yygotominor.yy14,yymsp[0].minor.yy328);
+}
+ break;
+ case 159: /* sortorder ::= ASC */
+{yygotominor.yy328 = SQLITE_SO_ASC;}
+ break;
+ case 160: /* sortorder ::= DESC */
+{yygotominor.yy328 = SQLITE_SO_DESC;}
+ break;
+ case 161: /* sortorder ::= */
+{yygotominor.yy328 = SQLITE_SO_UNDEFINED;}
+ break;
+ case 166: /* limit_opt ::= */
+{yygotominor.yy476.pLimit = 0; yygotominor.yy476.pOffset = 0;}
+ break;
+ case 167: /* limit_opt ::= LIMIT expr */
+{yygotominor.yy476.pLimit = yymsp[0].minor.yy346.pExpr; yygotominor.yy476.pOffset = 0;}
+ break;
+ case 168: /* limit_opt ::= LIMIT expr OFFSET expr */
+{yygotominor.yy476.pLimit = yymsp[-2].minor.yy346.pExpr; yygotominor.yy476.pOffset = yymsp[0].minor.yy346.pExpr;}
+ break;
+ case 169: /* limit_opt ::= LIMIT expr COMMA expr */
+{yygotominor.yy476.pOffset = yymsp[-2].minor.yy346.pExpr; yygotominor.yy476.pLimit = yymsp[0].minor.yy346.pExpr;}
+ break;
+ case 170: /* cmd ::= with DELETE FROM fullname indexed_opt where_opt */
+{
+ sqlite3WithPush(pParse, yymsp[-5].minor.yy59, 1);
+ sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy65, &yymsp[-1].minor.yy0);
+ sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy65,yymsp[0].minor.yy132);
+}
+ break;
+ case 173: /* cmd ::= with UPDATE orconf fullname indexed_opt SET setlist where_opt */
+{
+ sqlite3WithPush(pParse, yymsp[-7].minor.yy59, 1);
+ sqlite3SrcListIndexedBy(pParse, yymsp[-4].minor.yy65, &yymsp[-3].minor.yy0);
+ sqlite3ExprListCheckLength(pParse,yymsp[-1].minor.yy14,"set list");
+ sqlite3Update(pParse,yymsp[-4].minor.yy65,yymsp[-1].minor.yy14,yymsp[0].minor.yy132,yymsp[-5].minor.yy186);
+}
+ break;
+ case 174: /* setlist ::= setlist COMMA nm EQ expr */
+{
+ yygotominor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy14, yymsp[0].minor.yy346.pExpr);
+ sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[-2].minor.yy0, 1);
+}
+ break;
+ case 175: /* setlist ::= nm EQ expr */
+{
+ yygotominor.yy14 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy346.pExpr);
+ sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[-2].minor.yy0, 1);
+}
+ break;
+ case 176: /* cmd ::= with insert_cmd INTO fullname idlist_opt select */
+{
+ sqlite3WithPush(pParse, yymsp[-5].minor.yy59, 1);
+ sqlite3Insert(pParse, yymsp[-2].minor.yy65, yymsp[0].minor.yy3, yymsp[-1].minor.yy408, yymsp[-4].minor.yy186);
+}
+ break;
+ case 177: /* cmd ::= with insert_cmd INTO fullname idlist_opt DEFAULT VALUES */
+{
+ sqlite3WithPush(pParse, yymsp[-6].minor.yy59, 1);
+ sqlite3Insert(pParse, yymsp[-3].minor.yy65, 0, yymsp[-2].minor.yy408, yymsp[-5].minor.yy186);
+}
+ break;
+ case 178: /* insert_cmd ::= INSERT orconf */
+{yygotominor.yy186 = yymsp[0].minor.yy186;}
+ break;
+ case 179: /* insert_cmd ::= REPLACE */
+{yygotominor.yy186 = OE_Replace;}
+ break;
+ case 182: /* idlist ::= idlist COMMA nm */
+{yygotominor.yy408 = sqlite3IdListAppend(pParse->db,yymsp[-2].minor.yy408,&yymsp[0].minor.yy0);}
+ break;
+ case 183: /* idlist ::= nm */
+{yygotominor.yy408 = sqlite3IdListAppend(pParse->db,0,&yymsp[0].minor.yy0);}
+ break;
+ case 184: /* expr ::= term */
+{yygotominor.yy346 = yymsp[0].minor.yy346;}
+ break;
+ case 185: /* expr ::= LP expr RP */
+{yygotominor.yy346.pExpr = yymsp[-1].minor.yy346.pExpr; spanSet(&yygotominor.yy346,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0);}
+ break;
+ case 186: /* term ::= NULL */
+ case 191: /* term ::= INTEGER|FLOAT|BLOB */ yytestcase(yyruleno==191);
+ case 192: /* term ::= STRING */ yytestcase(yyruleno==192);
+{spanExpr(&yygotominor.yy346, pParse, yymsp[0].major, &yymsp[0].minor.yy0);}
+ break;
+ case 187: /* expr ::= ID|INDEXED */
+ case 188: /* expr ::= JOIN_KW */ yytestcase(yyruleno==188);
+{spanExpr(&yygotominor.yy346, pParse, TK_ID, &yymsp[0].minor.yy0);}
+ break;
+ case 189: /* expr ::= nm DOT nm */
+{
+ Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0);
+ Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[0].minor.yy0);
+ yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp2, 0);
+ spanSet(&yygotominor.yy346,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0);
+}
+ break;
+ case 190: /* expr ::= nm DOT nm DOT nm */
+{
+ Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-4].minor.yy0);
+ Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0);
+ Expr *temp3 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[0].minor.yy0);
+ Expr *temp4 = sqlite3PExpr(pParse, TK_DOT, temp2, temp3, 0);
+ yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp4, 0);
+ spanSet(&yygotominor.yy346,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0);
+}
+ break;
+ case 193: /* expr ::= VARIABLE */
+{
+ if( yymsp[0].minor.yy0.n>=2 && yymsp[0].minor.yy0.z[0]=='#' && sqlite3Isdigit(yymsp[0].minor.yy0.z[1]) ){
+ /* When doing a nested parse, one can include terms in an expression
+ ** that look like this: #1 #2 ... These terms refer to registers
+ ** in the virtual machine. #N is the N-th register. */
+ if( pParse->nested==0 ){
+ sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &yymsp[0].minor.yy0);
+ yygotominor.yy346.pExpr = 0;
+ }else{
+ yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0, &yymsp[0].minor.yy0);
+ if( yygotominor.yy346.pExpr ) sqlite3GetInt32(&yymsp[0].minor.yy0.z[1], &yygotominor.yy346.pExpr->iTable);
+ }
+ }else{
+ spanExpr(&yygotominor.yy346, pParse, TK_VARIABLE, &yymsp[0].minor.yy0);
+ sqlite3ExprAssignVarNumber(pParse, yygotominor.yy346.pExpr);
+ }
+ spanSet(&yygotominor.yy346, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0);
+}
+ break;
+ case 194: /* expr ::= expr COLLATE ID|STRING */
+{
+ yygotominor.yy346.pExpr = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy346.pExpr, &yymsp[0].minor.yy0, 1);
+ yygotominor.yy346.zStart = yymsp[-2].minor.yy346.zStart;
+ yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n];
+}
+ break;
+ case 195: /* expr ::= CAST LP expr AS typetoken RP */
+{
+ yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_CAST, yymsp[-3].minor.yy346.pExpr, 0, &yymsp[-1].minor.yy0);
+ spanSet(&yygotominor.yy346,&yymsp[-5].minor.yy0,&yymsp[0].minor.yy0);
+}
+ break;
+ case 196: /* expr ::= ID|INDEXED LP distinct exprlist RP */
+{
+ if( yymsp[-1].minor.yy14 && yymsp[-1].minor.yy14->nExpr>pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){
+ sqlite3ErrorMsg(pParse, "too many arguments on function %T", &yymsp[-4].minor.yy0);
+ }
+ yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy14, &yymsp[-4].minor.yy0);
+ spanSet(&yygotominor.yy346,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0);
+ if( yymsp[-2].minor.yy381==SF_Distinct && yygotominor.yy346.pExpr ){
+ yygotominor.yy346.pExpr->flags |= EP_Distinct;
+ }
+}
+ break;
+ case 197: /* expr ::= ID|INDEXED LP STAR RP */
+{
+ yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0);
+ spanSet(&yygotominor.yy346,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0);
+}
+ break;
+ case 198: /* term ::= CTIME_KW */
+{
+ yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0);
+ spanSet(&yygotominor.yy346, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0);
+}
+ break;
+ case 199: /* expr ::= expr AND expr */
+ case 200: /* expr ::= expr OR expr */ yytestcase(yyruleno==200);
+ case 201: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==201);
+ case 202: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==202);
+ case 203: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==203);
+ case 204: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==204);
+ case 205: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==205);
+ case 206: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==206);
+{spanBinaryExpr(&yygotominor.yy346,pParse,yymsp[-1].major,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy346);}
+ break;
+ case 207: /* likeop ::= LIKE_KW|MATCH */
+{yygotominor.yy96.eOperator = yymsp[0].minor.yy0; yygotominor.yy96.bNot = 0;}
+ break;
+ case 208: /* likeop ::= NOT LIKE_KW|MATCH */
+{yygotominor.yy96.eOperator = yymsp[0].minor.yy0; yygotominor.yy96.bNot = 1;}
+ break;
+ case 209: /* expr ::= expr likeop expr */
+{
+ ExprList *pList;
+ pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy346.pExpr);
+ pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy346.pExpr);
+ yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy96.eOperator);
+ if( yymsp[-1].minor.yy96.bNot ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0);
+ yygotominor.yy346.zStart = yymsp[-2].minor.yy346.zStart;
+ yygotominor.yy346.zEnd = yymsp[0].minor.yy346.zEnd;
+ if( yygotominor.yy346.pExpr ) yygotominor.yy346.pExpr->flags |= EP_InfixFunc;
+}
+ break;
+ case 210: /* expr ::= expr likeop expr ESCAPE expr */
+{
+ ExprList *pList;
+ pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy346.pExpr);
+ pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy346.pExpr);
+ pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy346.pExpr);
+ yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy96.eOperator);
+ if( yymsp[-3].minor.yy96.bNot ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0);
+ yygotominor.yy346.zStart = yymsp[-4].minor.yy346.zStart;
+ yygotominor.yy346.zEnd = yymsp[0].minor.yy346.zEnd;
+ if( yygotominor.yy346.pExpr ) yygotominor.yy346.pExpr->flags |= EP_InfixFunc;
+}
+ break;
+ case 211: /* expr ::= expr ISNULL|NOTNULL */
+{spanUnaryPostfix(&yygotominor.yy346,pParse,yymsp[0].major,&yymsp[-1].minor.yy346,&yymsp[0].minor.yy0);}
+ break;
+ case 212: /* expr ::= expr NOT NULL */
+{spanUnaryPostfix(&yygotominor.yy346,pParse,TK_NOTNULL,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy0);}
+ break;
+ case 213: /* expr ::= expr IS expr */
+{
+ spanBinaryExpr(&yygotominor.yy346,pParse,TK_IS,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy346);
+ binaryToUnaryIfNull(pParse, yymsp[0].minor.yy346.pExpr, yygotominor.yy346.pExpr, TK_ISNULL);
+}
+ break;
+ case 214: /* expr ::= expr IS NOT expr */
+{
+ spanBinaryExpr(&yygotominor.yy346,pParse,TK_ISNOT,&yymsp[-3].minor.yy346,&yymsp[0].minor.yy346);
+ binaryToUnaryIfNull(pParse, yymsp[0].minor.yy346.pExpr, yygotominor.yy346.pExpr, TK_NOTNULL);
+}
+ break;
+ case 215: /* expr ::= NOT expr */
+ case 216: /* expr ::= BITNOT expr */ yytestcase(yyruleno==216);
+{spanUnaryPrefix(&yygotominor.yy346,pParse,yymsp[-1].major,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);}
+ break;
+ case 217: /* expr ::= MINUS expr */
+{spanUnaryPrefix(&yygotominor.yy346,pParse,TK_UMINUS,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);}
+ break;
+ case 218: /* expr ::= PLUS expr */
+{spanUnaryPrefix(&yygotominor.yy346,pParse,TK_UPLUS,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);}
+ break;
+ case 221: /* expr ::= expr between_op expr AND expr */
+{
+ ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy346.pExpr);
+ pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy346.pExpr);
+ yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy346.pExpr, 0, 0);
+ if( yygotominor.yy346.pExpr ){
+ yygotominor.yy346.pExpr->x.pList = pList;
+ }else{
+ sqlite3ExprListDelete(pParse->db, pList);
+ }
+ if( yymsp[-3].minor.yy328 ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0);
+ yygotominor.yy346.zStart = yymsp[-4].minor.yy346.zStart;
+ yygotominor.yy346.zEnd = yymsp[0].minor.yy346.zEnd;
+}
+ break;
+ case 224: /* expr ::= expr in_op LP exprlist RP */
+{
+ if( yymsp[-1].minor.yy14==0 ){
+ /* Expressions of the form
+ **
+ ** expr1 IN ()
+ ** expr1 NOT IN ()
+ **
+ ** simplify to constants 0 (false) and 1 (true), respectively,
+ ** regardless of the value of expr1.
+ */
+ yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, &sqlite3IntTokens[yymsp[-3].minor.yy328]);
+ sqlite3ExprDelete(pParse->db, yymsp[-4].minor.yy346.pExpr);
+ }else if( yymsp[-1].minor.yy14->nExpr==1 ){
+ /* Expressions of the form:
+ **
+ ** expr1 IN (?1)
+ ** expr1 NOT IN (?2)
+ **
+ ** with exactly one value on the RHS can be simplified to something
+ ** like this:
+ **
+ ** expr1 == ?1
+ ** expr1 <> ?2
+ **
+ ** But, the RHS of the == or <> is marked with the EP_Generic flag
+ ** so that it may not contribute to the computation of comparison
+ ** affinity or the collating sequence to use for comparison. Otherwise,
+ ** the semantics would be subtly different from IN or NOT IN.
+ */
+ Expr *pRHS = yymsp[-1].minor.yy14->a[0].pExpr;
+ yymsp[-1].minor.yy14->a[0].pExpr = 0;
+ sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14);
+ /* pRHS cannot be NULL because a malloc error would have been detected
+ ** before now and control would have never reached this point */
+ if( ALWAYS(pRHS) ){
+ pRHS->flags &= ~EP_Collate;
+ pRHS->flags |= EP_Generic;
+ }
+ yygotominor.yy346.pExpr = sqlite3PExpr(pParse, yymsp[-3].minor.yy328 ? TK_NE : TK_EQ, yymsp[-4].minor.yy346.pExpr, pRHS, 0);
+ }else{
+ yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy346.pExpr, 0, 0);
+ if( yygotominor.yy346.pExpr ){
+ yygotominor.yy346.pExpr->x.pList = yymsp[-1].minor.yy14;
+ sqlite3ExprSetHeightAndFlags(pParse, yygotominor.yy346.pExpr);
+ }else{
+ sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14);
+ }
+ if( yymsp[-3].minor.yy328 ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0);
+ }
+ yygotominor.yy346.zStart = yymsp[-4].minor.yy346.zStart;
+ yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n];
+ }
+ break;
+ case 225: /* expr ::= LP select RP */
+{
+ yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_SELECT, 0, 0, 0);
+ if( yygotominor.yy346.pExpr ){
+ yygotominor.yy346.pExpr->x.pSelect = yymsp[-1].minor.yy3;
+ ExprSetProperty(yygotominor.yy346.pExpr, EP_xIsSelect|EP_Subquery);
+ sqlite3ExprSetHeightAndFlags(pParse, yygotominor.yy346.pExpr);
+ }else{
+ sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy3);
+ }
+ yygotominor.yy346.zStart = yymsp[-2].minor.yy0.z;
+ yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n];
+ }
+ break;
+ case 226: /* expr ::= expr in_op LP select RP */
+{
+ yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy346.pExpr, 0, 0);
+ if( yygotominor.yy346.pExpr ){
+ yygotominor.yy346.pExpr->x.pSelect = yymsp[-1].minor.yy3;
+ ExprSetProperty(yygotominor.yy346.pExpr, EP_xIsSelect|EP_Subquery);
+ sqlite3ExprSetHeightAndFlags(pParse, yygotominor.yy346.pExpr);
+ }else{
+ sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy3);
+ }
+ if( yymsp[-3].minor.yy328 ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0);
+ yygotominor.yy346.zStart = yymsp[-4].minor.yy346.zStart;
+ yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n];
+ }
+ break;
+ case 227: /* expr ::= expr in_op nm dbnm */
+{
+ SrcList *pSrc = sqlite3SrcListAppend(pParse->db, 0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0);
+ yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-3].minor.yy346.pExpr, 0, 0);
+ if( yygotominor.yy346.pExpr ){
+ yygotominor.yy346.pExpr->x.pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0,0);
+ ExprSetProperty(yygotominor.yy346.pExpr, EP_xIsSelect|EP_Subquery);
+ sqlite3ExprSetHeightAndFlags(pParse, yygotominor.yy346.pExpr);
+ }else{
+ sqlite3SrcListDelete(pParse->db, pSrc);
+ }
+ if( yymsp[-2].minor.yy328 ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0);
+ yygotominor.yy346.zStart = yymsp[-3].minor.yy346.zStart;
+ yygotominor.yy346.zEnd = yymsp[0].minor.yy0.z ? &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] : &yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n];
+ }
+ break;
+ case 228: /* expr ::= EXISTS LP select RP */
+{
+ Expr *p = yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_EXISTS, 0, 0, 0);
+ if( p ){
+ p->x.pSelect = yymsp[-1].minor.yy3;
+ ExprSetProperty(p, EP_xIsSelect|EP_Subquery);
+ sqlite3ExprSetHeightAndFlags(pParse, p);
+ }else{
+ sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy3);
+ }
+ yygotominor.yy346.zStart = yymsp[-3].minor.yy0.z;
+ yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n];
+ }
+ break;
+ case 229: /* expr ::= CASE case_operand case_exprlist case_else END */
+{
+ yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy132, 0, 0);
+ if( yygotominor.yy346.pExpr ){
+ yygotominor.yy346.pExpr->x.pList = yymsp[-1].minor.yy132 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy14,yymsp[-1].minor.yy132) : yymsp[-2].minor.yy14;
+ sqlite3ExprSetHeightAndFlags(pParse, yygotominor.yy346.pExpr);
+ }else{
+ sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy14);
+ sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy132);
+ }
+ yygotominor.yy346.zStart = yymsp[-4].minor.yy0.z;
+ yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n];
+}
+ break;
+ case 230: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */
+{
+ yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, yymsp[-2].minor.yy346.pExpr);
+ yygotominor.yy14 = sqlite3ExprListAppend(pParse,yygotominor.yy14, yymsp[0].minor.yy346.pExpr);
+}
+ break;
+ case 231: /* case_exprlist ::= WHEN expr THEN expr */
+{
+ yygotominor.yy14 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy346.pExpr);
+ yygotominor.yy14 = sqlite3ExprListAppend(pParse,yygotominor.yy14, yymsp[0].minor.yy346.pExpr);
+}
+ break;
+ case 238: /* nexprlist ::= nexprlist COMMA expr */
+{yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy14,yymsp[0].minor.yy346.pExpr);}
+ break;
+ case 239: /* nexprlist ::= expr */
+{yygotominor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy346.pExpr);}
+ break;
+ case 240: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */
+{
+ sqlite3CreateIndex(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0,
+ sqlite3SrcListAppend(pParse->db,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy14, yymsp[-10].minor.yy328,
+ &yymsp[-11].minor.yy0, yymsp[0].minor.yy132, SQLITE_SO_ASC, yymsp[-8].minor.yy328);
+}
+ break;
+ case 241: /* uniqueflag ::= UNIQUE */
+ case 292: /* raisetype ::= ABORT */ yytestcase(yyruleno==292);
+{yygotominor.yy328 = OE_Abort;}
+ break;
+ case 242: /* uniqueflag ::= */
+{yygotominor.yy328 = OE_None;}
+ break;
+ case 245: /* eidlist ::= eidlist COMMA nm collate sortorder */
+{
+ yygotominor.yy14 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy14, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy328, yymsp[0].minor.yy328);
+}
+ break;
+ case 246: /* eidlist ::= nm collate sortorder */
+{
+ yygotominor.yy14 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy328, yymsp[0].minor.yy328);
+}
+ break;
+ case 249: /* cmd ::= DROP INDEX ifexists fullname */
+{sqlite3DropIndex(pParse, yymsp[0].minor.yy65, yymsp[-1].minor.yy328);}
+ break;
+ case 250: /* cmd ::= VACUUM */
+ case 251: /* cmd ::= VACUUM nm */ yytestcase(yyruleno==251);
+{sqlite3Vacuum(pParse);}
+ break;
+ case 252: /* cmd ::= PRAGMA nm dbnm */
+{sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);}
+ break;
+ case 253: /* cmd ::= PRAGMA nm dbnm EQ nmnum */
+{sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,0);}
+ break;
+ case 254: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */
+{sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,0);}
+ break;
+ case 255: /* cmd ::= PRAGMA nm dbnm EQ minus_num */
+{sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,1);}
+ break;
+ case 256: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */
+{sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,1);}
+ break;
+ case 265: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */
+{
+ Token all;
+ all.z = yymsp[-3].minor.yy0.z;
+ all.n = (int)(yymsp[0].minor.yy0.z - yymsp[-3].minor.yy0.z) + yymsp[0].minor.yy0.n;
+ sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy473, &all);
+}
+ break;
+ case 266: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */
+{
+ sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy328, yymsp[-4].minor.yy378.a, yymsp[-4].minor.yy378.b, yymsp[-2].minor.yy65, yymsp[0].minor.yy132, yymsp[-10].minor.yy328, yymsp[-8].minor.yy328);
+ yygotominor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0);
+}
+ break;
+ case 267: /* trigger_time ::= BEFORE */
+ case 270: /* trigger_time ::= */ yytestcase(yyruleno==270);
+{ yygotominor.yy328 = TK_BEFORE; }
+ break;
+ case 268: /* trigger_time ::= AFTER */
+{ yygotominor.yy328 = TK_AFTER; }
+ break;
+ case 269: /* trigger_time ::= INSTEAD OF */
+{ yygotominor.yy328 = TK_INSTEAD;}
+ break;
+ case 271: /* trigger_event ::= DELETE|INSERT */
+ case 272: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==272);
+{yygotominor.yy378.a = yymsp[0].major; yygotominor.yy378.b = 0;}
+ break;
+ case 273: /* trigger_event ::= UPDATE OF idlist */
+{yygotominor.yy378.a = TK_UPDATE; yygotominor.yy378.b = yymsp[0].minor.yy408;}
+ break;
+ case 276: /* when_clause ::= */
+ case 297: /* key_opt ::= */ yytestcase(yyruleno==297);
+{ yygotominor.yy132 = 0; }
+ break;
+ case 277: /* when_clause ::= WHEN expr */
+ case 298: /* key_opt ::= KEY expr */ yytestcase(yyruleno==298);
+{ yygotominor.yy132 = yymsp[0].minor.yy346.pExpr; }
+ break;
+ case 278: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */
+{
+ assert( yymsp[-2].minor.yy473!=0 );
+ yymsp[-2].minor.yy473->pLast->pNext = yymsp[-1].minor.yy473;
+ yymsp[-2].minor.yy473->pLast = yymsp[-1].minor.yy473;
+ yygotominor.yy473 = yymsp[-2].minor.yy473;
+}
+ break;
+ case 279: /* trigger_cmd_list ::= trigger_cmd SEMI */
+{
+ assert( yymsp[-1].minor.yy473!=0 );
+ yymsp[-1].minor.yy473->pLast = yymsp[-1].minor.yy473;
+ yygotominor.yy473 = yymsp[-1].minor.yy473;
+}
+ break;
+ case 281: /* trnm ::= nm DOT nm */
+{
+ yygotominor.yy0 = yymsp[0].minor.yy0;
+ sqlite3ErrorMsg(pParse,
+ "qualified table names are not allowed on INSERT, UPDATE, and DELETE "
+ "statements within triggers");
+}
+ break;
+ case 283: /* tridxby ::= INDEXED BY nm */
+{
+ sqlite3ErrorMsg(pParse,
+ "the INDEXED BY clause is not allowed on UPDATE or DELETE statements "
+ "within triggers");
+}
+ break;
+ case 284: /* tridxby ::= NOT INDEXED */
+{
+ sqlite3ErrorMsg(pParse,
+ "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements "
+ "within triggers");
+}
+ break;
+ case 285: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt */
+{ yygotominor.yy473 = sqlite3TriggerUpdateStep(pParse->db, &yymsp[-4].minor.yy0, yymsp[-1].minor.yy14, yymsp[0].minor.yy132, yymsp[-5].minor.yy186); }
+ break;
+ case 286: /* trigger_cmd ::= insert_cmd INTO trnm idlist_opt select */
+{yygotominor.yy473 = sqlite3TriggerInsertStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy408, yymsp[0].minor.yy3, yymsp[-4].minor.yy186);}
+ break;
+ case 287: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt */
+{yygotominor.yy473 = sqlite3TriggerDeleteStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[0].minor.yy132);}
+ break;
+ case 288: /* trigger_cmd ::= select */
+{yygotominor.yy473 = sqlite3TriggerSelectStep(pParse->db, yymsp[0].minor.yy3); }
+ break;
+ case 289: /* expr ::= RAISE LP IGNORE RP */
+{
+ yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, 0);
+ if( yygotominor.yy346.pExpr ){
+ yygotominor.yy346.pExpr->affinity = OE_Ignore;
+ }
+ yygotominor.yy346.zStart = yymsp[-3].minor.yy0.z;
+ yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n];
+}
+ break;
+ case 290: /* expr ::= RAISE LP raisetype COMMA nm RP */
+{
+ yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, &yymsp[-1].minor.yy0);
+ if( yygotominor.yy346.pExpr ) {
+ yygotominor.yy346.pExpr->affinity = (char)yymsp[-3].minor.yy328;
+ }
+ yygotominor.yy346.zStart = yymsp[-5].minor.yy0.z;
+ yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n];
+}
+ break;
+ case 291: /* raisetype ::= ROLLBACK */
+{yygotominor.yy328 = OE_Rollback;}
+ break;
+ case 293: /* raisetype ::= FAIL */
+{yygotominor.yy328 = OE_Fail;}
+ break;
+ case 294: /* cmd ::= DROP TRIGGER ifexists fullname */
+{
+ sqlite3DropTrigger(pParse,yymsp[0].minor.yy65,yymsp[-1].minor.yy328);
+}
+ break;
+ case 295: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */
+{
+ sqlite3Attach(pParse, yymsp[-3].minor.yy346.pExpr, yymsp[-1].minor.yy346.pExpr, yymsp[0].minor.yy132);
+}
+ break;
+ case 296: /* cmd ::= DETACH database_kw_opt expr */
+{
+ sqlite3Detach(pParse, yymsp[0].minor.yy346.pExpr);
+}
+ break;
+ case 301: /* cmd ::= REINDEX */
+{sqlite3Reindex(pParse, 0, 0);}
+ break;
+ case 302: /* cmd ::= REINDEX nm dbnm */
+{sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);}
+ break;
+ case 303: /* cmd ::= ANALYZE */
+{sqlite3Analyze(pParse, 0, 0);}
+ break;
+ case 304: /* cmd ::= ANALYZE nm dbnm */
+{sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);}
+ break;
+ case 305: /* cmd ::= ALTER TABLE fullname RENAME TO nm */
+{
+ sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy65,&yymsp[0].minor.yy0);
+}
+ break;
+ case 306: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column */
+{
+ sqlite3AlterFinishAddColumn(pParse, &yymsp[0].minor.yy0);
+}
+ break;
+ case 307: /* add_column_fullname ::= fullname */
+{
+ pParse->db->lookaside.bEnabled = 0;
+ sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy65);
+}
+ break;
+ case 310: /* cmd ::= create_vtab */
+{sqlite3VtabFinishParse(pParse,0);}
+ break;
+ case 311: /* cmd ::= create_vtab LP vtabarglist RP */
+{sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);}
+ break;
+ case 312: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */
+{
+ sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy328);
+}
+ break;
+ case 315: /* vtabarg ::= */
+{sqlite3VtabArgInit(pParse);}
+ break;
+ case 317: /* vtabargtoken ::= ANY */
+ case 318: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==318);
+ case 319: /* lp ::= LP */ yytestcase(yyruleno==319);
+{sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);}
+ break;
+ case 323: /* with ::= */
+{yygotominor.yy59 = 0;}
+ break;
+ case 324: /* with ::= WITH wqlist */
+ case 325: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==325);
+{ yygotominor.yy59 = yymsp[0].minor.yy59; }
+ break;
+ case 326: /* wqlist ::= nm eidlist_opt AS LP select RP */
+{
+ yygotominor.yy59 = sqlite3WithAdd(pParse, 0, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy14, yymsp[-1].minor.yy3);
+}
+ break;
+ case 327: /* wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP */
+{
+ yygotominor.yy59 = sqlite3WithAdd(pParse, yymsp[-7].minor.yy59, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy14, yymsp[-1].minor.yy3);
+}
+ break;
+ default:
+ /* (0) input ::= cmdlist */ yytestcase(yyruleno==0);
+ /* (1) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==1);
+ /* (2) cmdlist ::= ecmd */ yytestcase(yyruleno==2);
+ /* (3) ecmd ::= SEMI */ yytestcase(yyruleno==3);
+ /* (4) ecmd ::= explain cmdx SEMI */ yytestcase(yyruleno==4);
+ /* (10) trans_opt ::= */ yytestcase(yyruleno==10);
+ /* (11) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==11);
+ /* (12) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==12);
+ /* (20) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==20);
+ /* (21) savepoint_opt ::= */ yytestcase(yyruleno==21);
+ /* (25) cmd ::= create_table create_table_args */ yytestcase(yyruleno==25);
+ /* (36) columnlist ::= columnlist COMMA column */ yytestcase(yyruleno==36);
+ /* (37) columnlist ::= column */ yytestcase(yyruleno==37);
+ /* (43) type ::= */ yytestcase(yyruleno==43);
+ /* (50) signed ::= plus_num */ yytestcase(yyruleno==50);
+ /* (51) signed ::= minus_num */ yytestcase(yyruleno==51);
+ /* (52) carglist ::= carglist ccons */ yytestcase(yyruleno==52);
+ /* (53) carglist ::= */ yytestcase(yyruleno==53);
+ /* (60) ccons ::= NULL onconf */ yytestcase(yyruleno==60);
+ /* (88) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==88);
+ /* (89) conslist ::= tcons */ yytestcase(yyruleno==89);
+ /* (91) tconscomma ::= */ yytestcase(yyruleno==91);
+ /* (274) foreach_clause ::= */ yytestcase(yyruleno==274);
+ /* (275) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==275);
+ /* (282) tridxby ::= */ yytestcase(yyruleno==282);
+ /* (299) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==299);
+ /* (300) database_kw_opt ::= */ yytestcase(yyruleno==300);
+ /* (308) kwcolumn_opt ::= */ yytestcase(yyruleno==308);
+ /* (309) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==309);
+ /* (313) vtabarglist ::= vtabarg */ yytestcase(yyruleno==313);
+ /* (314) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==314);
+ /* (316) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==316);
+ /* (320) anylist ::= */ yytestcase(yyruleno==320);
+ /* (321) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==321);
+ /* (322) anylist ::= anylist ANY */ yytestcase(yyruleno==322);
+ break;
+ };
+ assert( yyruleno>=0 && yyruleno<sizeof(yyRuleInfo)/sizeof(yyRuleInfo[0]) );
+ yygoto = yyRuleInfo[yyruleno].lhs;
+ yysize = yyRuleInfo[yyruleno].nrhs;
+ yypParser->yyidx -= yysize;
+ yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto);
+ if( yyact <= YY_MAX_SHIFTREDUCE ){
+ if( yyact>YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
+ /* If the reduce action popped at least
+ ** one element off the stack, then we can push the new element back
+ ** onto the stack here, and skip the stack overflow test in yy_shift().
+ ** That gives a significant speed improvement. */
+ if( yysize ){
+ yypParser->yyidx++;
+ yymsp -= yysize-1;
+ yymsp->stateno = (YYACTIONTYPE)yyact;
+ yymsp->major = (YYCODETYPE)yygoto;
+ yymsp->minor = yygotominor;
+ yyTraceShift(yypParser, yyact);
+ }else{
+ yy_shift(yypParser,yyact,yygoto,&yygotominor);
+ }
+ }else{
+ assert( yyact == YY_ACCEPT_ACTION );
+ yy_accept(yypParser);
+ }
+}
+
+/*
+** The following code executes when the parse fails
+*/
+#ifndef YYNOERRORRECOVERY
+static void yy_parse_failed(
+ yyParser *yypParser /* The parser */
+){
+ sqlite3ParserARG_FETCH;
+#ifndef NDEBUG
+ if( yyTraceFILE ){
+ fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt);
+ }
+#endif
+ while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser);
+ /* Here code is inserted which will be executed whenever the
+ ** parser fails */
+ sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */
+}
+#endif /* YYNOERRORRECOVERY */
+
+/*
+** The following code executes when a syntax error first occurs.
+*/
+static void yy_syntax_error(
+ yyParser *yypParser, /* The parser */
+ int yymajor, /* The major type of the error token */
+ YYMINORTYPE yyminor /* The minor type of the error token */
+){
+ sqlite3ParserARG_FETCH;
+#define TOKEN (yyminor.yy0)
+
+ UNUSED_PARAMETER(yymajor); /* Silence some compiler warnings */
+ assert( TOKEN.z[0] ); /* The tokenizer always gives us a token */
+ sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN);
+ sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */
+}
+
+/*
+** The following is executed when the parser accepts
+*/
+static void yy_accept(
+ yyParser *yypParser /* The parser */
+){
+ sqlite3ParserARG_FETCH;
+#ifndef NDEBUG
+ if( yyTraceFILE ){
+ fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt);
+ }
+#endif
+ while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser);
+ /* Here code is inserted which will be executed whenever the
+ ** parser accepts */
+ sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */
+}
+
+/* The main parser program.
+** The first argument is a pointer to a structure obtained from
+** "sqlite3ParserAlloc" which describes the current state of the parser.
+** The second argument is the major token number. The third is
+** the minor token. The fourth optional argument is whatever the
+** user wants (and specified in the grammar) and is available for
+** use by the action routines.
+**
+** Inputs:
+** <ul>
+** <li> A pointer to the parser (an opaque structure.)
+** <li> The major token number.
+** <li> The minor token number.
+** <li> An option argument of a grammar-specified type.
+** </ul>
+**
+** Outputs:
+** None.
+*/
+SQLITE_PRIVATE void sqlite3Parser(
+ void *yyp, /* The parser */
+ int yymajor, /* The major token code number */
+ sqlite3ParserTOKENTYPE yyminor /* The value for the token */
+ sqlite3ParserARG_PDECL /* Optional %extra_argument parameter */
+){
+ YYMINORTYPE yyminorunion;
+ int yyact; /* The parser action. */
+#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY)
+ int yyendofinput; /* True if we are at the end of input */
+#endif
+#ifdef YYERRORSYMBOL
+ int yyerrorhit = 0; /* True if yymajor has invoked an error */
+#endif
+ yyParser *yypParser; /* The parser */
+
+ /* (re)initialize the parser, if necessary */
+ yypParser = (yyParser*)yyp;
+ if( yypParser->yyidx<0 ){
+#if YYSTACKDEPTH<=0
+ if( yypParser->yystksz <=0 ){
+ /*memset(&yyminorunion, 0, sizeof(yyminorunion));*/
+ yyminorunion = yyzerominor;
+ yyStackOverflow(yypParser, &yyminorunion);
+ return;
+ }
+#endif
+ yypParser->yyidx = 0;
+ yypParser->yyerrcnt = -1;
+ yypParser->yystack[0].stateno = 0;
+ yypParser->yystack[0].major = 0;
+ }
+ yyminorunion.yy0 = yyminor;
+#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY)
+ yyendofinput = (yymajor==0);
+#endif
+ sqlite3ParserARG_STORE;
+
+#ifndef NDEBUG
+ if( yyTraceFILE ){
+ fprintf(yyTraceFILE,"%sInput %s\n",yyTracePrompt,yyTokenName[yymajor]);
+ }
+#endif
+
+ do{
+ yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor);
+ if( yyact <= YY_MAX_SHIFTREDUCE ){
+ if( yyact > YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
+ yy_shift(yypParser,yyact,yymajor,&yyminorunion);
+ yypParser->yyerrcnt--;
+ yymajor = YYNOCODE;
+ }else if( yyact <= YY_MAX_REDUCE ){
+ yy_reduce(yypParser,yyact-YY_MIN_REDUCE);
+ }else{
+ assert( yyact == YY_ERROR_ACTION );
+#ifdef YYERRORSYMBOL
+ int yymx;
+#endif
+#ifndef NDEBUG
+ if( yyTraceFILE ){
+ fprintf(yyTraceFILE,"%sSyntax Error!\n",yyTracePrompt);
+ }
+#endif
+#ifdef YYERRORSYMBOL
+ /* A syntax error has occurred.
+ ** The response to an error depends upon whether or not the
+ ** grammar defines an error token "ERROR".
+ **
+ ** This is what we do if the grammar does define ERROR:
+ **
+ ** * Call the %syntax_error function.
+ **
+ ** * Begin popping the stack until we enter a state where
+ ** it is legal to shift the error symbol, then shift
+ ** the error symbol.
+ **
+ ** * Set the error count to three.
+ **
+ ** * Begin accepting and shifting new tokens. No new error
+ ** processing will occur until three tokens have been
+ ** shifted successfully.
+ **
+ */
+ if( yypParser->yyerrcnt<0 ){
+ yy_syntax_error(yypParser,yymajor,yyminorunion);
+ }
+ yymx = yypParser->yystack[yypParser->yyidx].major;
+ if( yymx==YYERRORSYMBOL || yyerrorhit ){
+#ifndef NDEBUG
+ if( yyTraceFILE ){
+ fprintf(yyTraceFILE,"%sDiscard input token %s\n",
+ yyTracePrompt,yyTokenName[yymajor]);
+ }
+#endif
+ yy_destructor(yypParser, (YYCODETYPE)yymajor,&yyminorunion);
+ yymajor = YYNOCODE;
+ }else{
+ while(
+ yypParser->yyidx >= 0 &&
+ yymx != YYERRORSYMBOL &&
+ (yyact = yy_find_reduce_action(
+ yypParser->yystack[yypParser->yyidx].stateno,
+ YYERRORSYMBOL)) >= YY_MIN_REDUCE
+ ){
+ yy_pop_parser_stack(yypParser);
+ }
+ if( yypParser->yyidx < 0 || yymajor==0 ){
+ yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion);
+ yy_parse_failed(yypParser);
+ yymajor = YYNOCODE;
+ }else if( yymx!=YYERRORSYMBOL ){
+ YYMINORTYPE u2;
+ u2.YYERRSYMDT = 0;
+ yy_shift(yypParser,yyact,YYERRORSYMBOL,&u2);
+ }
+ }
+ yypParser->yyerrcnt = 3;
+ yyerrorhit = 1;
+#elif defined(YYNOERRORRECOVERY)
+ /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to
+ ** do any kind of error recovery. Instead, simply invoke the syntax
+ ** error routine and continue going as if nothing had happened.
+ **
+ ** Applications can set this macro (for example inside %include) if
+ ** they intend to abandon the parse upon the first syntax error seen.
+ */
+ yy_syntax_error(yypParser,yymajor,yyminorunion);
+ yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion);
+ yymajor = YYNOCODE;
+
+#else /* YYERRORSYMBOL is not defined */
+ /* This is what we do if the grammar does not define ERROR:
+ **
+ ** * Report an error message, and throw away the input token.
+ **
+ ** * If the input token is $, then fail the parse.
+ **
+ ** As before, subsequent error messages are suppressed until
+ ** three input tokens have been successfully shifted.
+ */
+ if( yypParser->yyerrcnt<=0 ){
+ yy_syntax_error(yypParser,yymajor,yyminorunion);
+ }
+ yypParser->yyerrcnt = 3;
+ yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion);
+ if( yyendofinput ){
+ yy_parse_failed(yypParser);
+ }
+ yymajor = YYNOCODE;
+#endif
+ }
+ }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 );
+#ifndef NDEBUG
+ if( yyTraceFILE ){
+ fprintf(yyTraceFILE,"%sReturn\n",yyTracePrompt);
+ }
+#endif
+ return;
+}
+
+/************** End of parse.c ***********************************************/
+/************** Begin file tokenize.c ****************************************/
+/*
+** 2001 September 15
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+*************************************************************************
+** An tokenizer for SQL
+**
+** This file contains C code that splits an SQL input string up into
+** individual tokens and sends those tokens one-by-one over to the
+** parser for analysis.
+*/
+/* #include "sqliteInt.h" */
+/* #include <stdlib.h> */
+
+/*
+** The charMap() macro maps alphabetic characters into their
+** lower-case ASCII equivalent. On ASCII machines, this is just
+** an upper-to-lower case map. On EBCDIC machines we also need
+** to adjust the encoding. Only alphabetic characters and underscores
+** need to be translated.
+*/
+#ifdef SQLITE_ASCII
+# define charMap(X) sqlite3UpperToLower[(unsigned char)X]
+#endif
+#ifdef SQLITE_EBCDIC
+# define charMap(X) ebcdicToAscii[(unsigned char)X]
+const unsigned char ebcdicToAscii[] = {
+/* 0 1 2 3 4 5 6 7 8 9 A B C D E F */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 3x */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4x */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 5x */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, /* 6x */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 7x */
+ 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* 8x */
+ 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* 9x */
+ 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ax */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */
+ 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* Cx */
+ 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* Dx */
+ 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ex */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Fx */
+};
+#endif
+
+/*
+** The sqlite3KeywordCode function looks up an identifier to determine if
+** it is a keyword. If it is a keyword, the token code of that keyword is
+** returned. If the input is not a keyword, TK_ID is returned.
+**
+** The implementation of this routine was generated by a program,
+** mkkeywordhash.h, located in the tool subdirectory of the distribution.
+** The output of the mkkeywordhash.c program is written into a file
+** named keywordhash.h and then included into this source file by
+** the #include below.
+*/
+/************** Include keywordhash.h in the middle of tokenize.c ************/
+/************** Begin file keywordhash.h *************************************/
+/***** This file contains automatically generated code ******
+**
+** The code in this file has been automatically generated by
+**
+** sqlite/tool/mkkeywordhash.c
+**
+** The code in this file implements a function that determines whether
+** or not a given identifier is really an SQL keyword. The same thing
+** might be implemented more directly using a hand-written hash table.
+** But by using this automatically generated code, the size of the code
+** is substantially reduced. This is important for embedded applications
+** on platforms with limited memory.
+*/
+/* Hash score: 182 */
+static int keywordCode(const char *z, int n){
+ /* zText[] encodes 834 bytes of keywords in 554 bytes */
+ /* REINDEXEDESCAPEACHECKEYBEFOREIGNOREGEXPLAINSTEADDATABASELECT */
+ /* ABLEFTHENDEFERRABLELSEXCEPTRANSACTIONATURALTERAISEXCLUSIVE */
+ /* XISTSAVEPOINTERSECTRIGGEREFERENCESCONSTRAINTOFFSETEMPORARY */
+ /* UNIQUERYWITHOUTERELEASEATTACHAVINGROUPDATEBEGINNERECURSIVE */
+ /* BETWEENOTNULLIKECASCADELETECASECOLLATECREATECURRENT_DATEDETACH */
+ /* IMMEDIATEJOINSERTMATCHPLANALYZEPRAGMABORTVALUESVIRTUALIMITWHEN */
+ /* WHERENAMEAFTEREPLACEANDEFAULTAUTOINCREMENTCASTCOLUMNCOMMIT */
+ /* CONFLICTCROSSCURRENT_TIMESTAMPRIMARYDEFERREDISTINCTDROPFAIL */
+ /* FROMFULLGLOBYIFISNULLORDERESTRICTRIGHTROLLBACKROWUNIONUSING */
+ /* VACUUMVIEWINITIALLY */
+ static const char zText[553] = {
+ 'R','E','I','N','D','E','X','E','D','E','S','C','A','P','E','A','C','H',
+ 'E','C','K','E','Y','B','E','F','O','R','E','I','G','N','O','R','E','G',
+ 'E','X','P','L','A','I','N','S','T','E','A','D','D','A','T','A','B','A',
+ 'S','E','L','E','C','T','A','B','L','E','F','T','H','E','N','D','E','F',
+ 'E','R','R','A','B','L','E','L','S','E','X','C','E','P','T','R','A','N',
+ 'S','A','C','T','I','O','N','A','T','U','R','A','L','T','E','R','A','I',
+ 'S','E','X','C','L','U','S','I','V','E','X','I','S','T','S','A','V','E',
+ 'P','O','I','N','T','E','R','S','E','C','T','R','I','G','G','E','R','E',
+ 'F','E','R','E','N','C','E','S','C','O','N','S','T','R','A','I','N','T',
+ 'O','F','F','S','E','T','E','M','P','O','R','A','R','Y','U','N','I','Q',
+ 'U','E','R','Y','W','I','T','H','O','U','T','E','R','E','L','E','A','S',
+ 'E','A','T','T','A','C','H','A','V','I','N','G','R','O','U','P','D','A',
+ 'T','E','B','E','G','I','N','N','E','R','E','C','U','R','S','I','V','E',
+ 'B','E','T','W','E','E','N','O','T','N','U','L','L','I','K','E','C','A',
+ 'S','C','A','D','E','L','E','T','E','C','A','S','E','C','O','L','L','A',
+ 'T','E','C','R','E','A','T','E','C','U','R','R','E','N','T','_','D','A',
+ 'T','E','D','E','T','A','C','H','I','M','M','E','D','I','A','T','E','J',
+ 'O','I','N','S','E','R','T','M','A','T','C','H','P','L','A','N','A','L',
+ 'Y','Z','E','P','R','A','G','M','A','B','O','R','T','V','A','L','U','E',
+ 'S','V','I','R','T','U','A','L','I','M','I','T','W','H','E','N','W','H',
+ 'E','R','E','N','A','M','E','A','F','T','E','R','E','P','L','A','C','E',
+ 'A','N','D','E','F','A','U','L','T','A','U','T','O','I','N','C','R','E',
+ 'M','E','N','T','C','A','S','T','C','O','L','U','M','N','C','O','M','M',
+ 'I','T','C','O','N','F','L','I','C','T','C','R','O','S','S','C','U','R',
+ 'R','E','N','T','_','T','I','M','E','S','T','A','M','P','R','I','M','A',
+ 'R','Y','D','E','F','E','R','R','E','D','I','S','T','I','N','C','T','D',
+ 'R','O','P','F','A','I','L','F','R','O','M','F','U','L','L','G','L','O',
+ 'B','Y','I','F','I','S','N','U','L','L','O','R','D','E','R','E','S','T',
+ 'R','I','C','T','R','I','G','H','T','R','O','L','L','B','A','C','K','R',
+ 'O','W','U','N','I','O','N','U','S','I','N','G','V','A','C','U','U','M',
+ 'V','I','E','W','I','N','I','T','I','A','L','L','Y',
+ };
+ static const unsigned char aHash[127] = {
+ 76, 105, 117, 74, 0, 45, 0, 0, 82, 0, 77, 0, 0,
+ 42, 12, 78, 15, 0, 116, 85, 54, 112, 0, 19, 0, 0,
+ 121, 0, 119, 115, 0, 22, 93, 0, 9, 0, 0, 70, 71,
+ 0, 69, 6, 0, 48, 90, 102, 0, 118, 101, 0, 0, 44,
+ 0, 103, 24, 0, 17, 0, 122, 53, 23, 0, 5, 110, 25,
+ 96, 0, 0, 124, 106, 60, 123, 57, 28, 55, 0, 91, 0,
+ 100, 26, 0, 99, 0, 0, 0, 95, 92, 97, 88, 109, 14,
+ 39, 108, 0, 81, 0, 18, 89, 111, 32, 0, 120, 80, 113,
+ 62, 46, 84, 0, 0, 94, 40, 59, 114, 0, 36, 0, 0,
+ 29, 0, 86, 63, 64, 0, 20, 61, 0, 56,
+ };
+ static const unsigned char aNext[124] = {
+ 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 2, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0,
+ 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 33, 0, 21, 0, 0, 0, 0, 0, 50,
+ 0, 43, 3, 47, 0, 0, 0, 0, 30, 0, 58, 0, 38,
+ 0, 0, 0, 1, 66, 0, 0, 67, 0, 41, 0, 0, 0,
+ 0, 0, 0, 49, 65, 0, 0, 0, 0, 31, 52, 16, 34,
+ 10, 0, 0, 0, 0, 0, 0, 0, 11, 72, 79, 0, 8,
+ 0, 104, 98, 0, 107, 0, 87, 0, 75, 51, 0, 27, 37,
+ 73, 83, 0, 35, 68, 0, 0,
+ };
+ static const unsigned char aLen[124] = {
+ 7, 7, 5, 4, 6, 4, 5, 3, 6, 7, 3, 6, 6,
+ 7, 7, 3, 8, 2, 6, 5, 4, 4, 3, 10, 4, 6,
+ 11, 6, 2, 7, 5, 5, 9, 6, 9, 9, 7, 10, 10,
+ 4, 6, 2, 3, 9, 4, 2, 6, 5, 7, 4, 5, 7,
+ 6, 6, 5, 6, 5, 5, 9, 7, 7, 3, 2, 4, 4,
+ 7, 3, 6, 4, 7, 6, 12, 6, 9, 4, 6, 5, 4,
+ 7, 6, 5, 6, 7, 5, 4, 5, 6, 5, 7, 3, 7,
+ 13, 2, 2, 4, 6, 6, 8, 5, 17, 12, 7, 8, 8,
+ 2, 4, 4, 4, 4, 4, 2, 2, 6, 5, 8, 5, 8,
+ 3, 5, 5, 6, 4, 9, 3,
+ };
+ static const unsigned short int aOffset[124] = {
+ 0, 2, 2, 8, 9, 14, 16, 20, 23, 25, 25, 29, 33,
+ 36, 41, 46, 48, 53, 54, 59, 62, 65, 67, 69, 78, 81,
+ 86, 91, 95, 96, 101, 105, 109, 117, 122, 128, 136, 142, 152,
+ 159, 162, 162, 165, 167, 167, 171, 176, 179, 184, 184, 188, 192,
+ 199, 204, 209, 212, 218, 221, 225, 234, 240, 240, 240, 243, 246,
+ 250, 251, 255, 261, 265, 272, 278, 290, 296, 305, 307, 313, 318,
+ 320, 327, 332, 337, 343, 349, 354, 358, 361, 367, 371, 378, 380,
+ 387, 389, 391, 400, 404, 410, 416, 424, 429, 429, 445, 452, 459,
+ 460, 467, 471, 475, 479, 483, 486, 488, 490, 496, 500, 508, 513,
+ 521, 524, 529, 534, 540, 544, 549,
+ };
+ static const unsigned char aCode[124] = {
+ TK_REINDEX, TK_INDEXED, TK_INDEX, TK_DESC, TK_ESCAPE,
+ TK_EACH, TK_CHECK, TK_KEY, TK_BEFORE, TK_FOREIGN,
+ TK_FOR, TK_IGNORE, TK_LIKE_KW, TK_EXPLAIN, TK_INSTEAD,
+ TK_ADD, TK_DATABASE, TK_AS, TK_SELECT, TK_TABLE,
+ TK_JOIN_KW, TK_THEN, TK_END, TK_DEFERRABLE, TK_ELSE,
+ TK_EXCEPT, TK_TRANSACTION,TK_ACTION, TK_ON, TK_JOIN_KW,
+ TK_ALTER, TK_RAISE, TK_EXCLUSIVE, TK_EXISTS, TK_SAVEPOINT,
+ TK_INTERSECT, TK_TRIGGER, TK_REFERENCES, TK_CONSTRAINT, TK_INTO,
+ TK_OFFSET, TK_OF, TK_SET, TK_TEMP, TK_TEMP,
+ TK_OR, TK_UNIQUE, TK_QUERY, TK_WITHOUT, TK_WITH,
+ TK_JOIN_KW, TK_RELEASE, TK_ATTACH, TK_HAVING, TK_GROUP,
+ TK_UPDATE, TK_BEGIN, TK_JOIN_KW, TK_RECURSIVE, TK_BETWEEN,
+ TK_NOTNULL, TK_NOT, TK_NO, TK_NULL, TK_LIKE_KW,
+ TK_CASCADE, TK_ASC, TK_DELETE, TK_CASE, TK_COLLATE,
+ TK_CREATE, TK_CTIME_KW, TK_DETACH, TK_IMMEDIATE, TK_JOIN,
+ TK_INSERT, TK_MATCH, TK_PLAN, TK_ANALYZE, TK_PRAGMA,
+ TK_ABORT, TK_VALUES, TK_VIRTUAL, TK_LIMIT, TK_WHEN,
+ TK_WHERE, TK_RENAME, TK_AFTER, TK_REPLACE, TK_AND,
+ TK_DEFAULT, TK_AUTOINCR, TK_TO, TK_IN, TK_CAST,
+ TK_COLUMNKW, TK_COMMIT, TK_CONFLICT, TK_JOIN_KW, TK_CTIME_KW,
+ TK_CTIME_KW, TK_PRIMARY, TK_DEFERRED, TK_DISTINCT, TK_IS,
+ TK_DROP, TK_FAIL, TK_FROM, TK_JOIN_KW, TK_LIKE_KW,
+ TK_BY, TK_IF, TK_ISNULL, TK_ORDER, TK_RESTRICT,
+ TK_JOIN_KW, TK_ROLLBACK, TK_ROW, TK_UNION, TK_USING,
+ TK_VACUUM, TK_VIEW, TK_INITIALLY, TK_ALL,
+ };
+ int h, i;
+ if( n<2 ) return TK_ID;
+ h = ((charMap(z[0])*4) ^
+ (charMap(z[n-1])*3) ^
+ n) % 127;
+ for(i=((int)aHash[h])-1; i>=0; i=((int)aNext[i])-1){
+ if( aLen[i]==n && sqlite3StrNICmp(&zText[aOffset[i]],z,n)==0 ){
+ testcase( i==0 ); /* REINDEX */
+ testcase( i==1 ); /* INDEXED */
+ testcase( i==2 ); /* INDEX */
+ testcase( i==3 ); /* DESC */
+ testcase( i==4 ); /* ESCAPE */
+ testcase( i==5 ); /* EACH */
+ testcase( i==6 ); /* CHECK */
+ testcase( i==7 ); /* KEY */
+ testcase( i==8 ); /* BEFORE */
+ testcase( i==9 ); /* FOREIGN */
+ testcase( i==10 ); /* FOR */
+ testcase( i==11 ); /* IGNORE */
+ testcase( i==12 ); /* REGEXP */
+ testcase( i==13 ); /* EXPLAIN */
+ testcase( i==14 ); /* INSTEAD */
+ testcase( i==15 ); /* ADD */
+ testcase( i==16 ); /* DATABASE */
+ testcase( i==17 ); /* AS */
+ testcase( i==18 ); /* SELECT */
+ testcase( i==19 ); /* TABLE */
+ testcase( i==20 ); /* LEFT */
+ testcase( i==21 ); /* THEN */
+ testcase( i==22 ); /* END */
+ testcase( i==23 ); /* DEFERRABLE */
+ testcase( i==24 ); /* ELSE */
+ testcase( i==25 ); /* EXCEPT */
+ testcase( i==26 ); /* TRANSACTION */
+ testcase( i==27 ); /* ACTION */
+ testcase( i==28 ); /* ON */
+ testcase( i==29 ); /* NATURAL */
+ testcase( i==30 ); /* ALTER */
+ testcase( i==31 ); /* RAISE */
+ testcase( i==32 ); /* EXCLUSIVE */
+ testcase( i==33 ); /* EXISTS */
+ testcase( i==34 ); /* SAVEPOINT */
+ testcase( i==35 ); /* INTERSECT */
+ testcase( i==36 ); /* TRIGGER */
+ testcase( i==37 ); /* REFERENCES */
+ testcase( i==38 ); /* CONSTRAINT */
+ testcase( i==39 ); /* INTO */
+ testcase( i==40 ); /* OFFSET */
+ testcase( i==41 ); /* OF */
+ testcase( i==42 ); /* SET */
+ testcase( i==43 ); /* TEMPORARY */
+ testcase( i==44 ); /* TEMP */
+ testcase( i==45 ); /* OR */
+ testcase( i==46 ); /* UNIQUE */
+ testcase( i==47 ); /* QUERY */
+ testcase( i==48 ); /* WITHOUT */
+ testcase( i==49 ); /* WITH */
+ testcase( i==50 ); /* OUTER */
+ testcase( i==51 ); /* RELEASE */
+ testcase( i==52 ); /* ATTACH */
+ testcase( i==53 ); /* HAVING */
+ testcase( i==54 ); /* GROUP */
+ testcase( i==55 ); /* UPDATE */
+ testcase( i==56 ); /* BEGIN */
+ testcase( i==57 ); /* INNER */
+ testcase( i==58 ); /* RECURSIVE */
+ testcase( i==59 ); /* BETWEEN */
+ testcase( i==60 ); /* NOTNULL */
+ testcase( i==61 ); /* NOT */
+ testcase( i==62 ); /* NO */
+ testcase( i==63 ); /* NULL */
+ testcase( i==64 ); /* LIKE */
+ testcase( i==65 ); /* CASCADE */
+ testcase( i==66 ); /* ASC */
+ testcase( i==67 ); /* DELETE */
+ testcase( i==68 ); /* CASE */
+ testcase( i==69 ); /* COLLATE */
+ testcase( i==70 ); /* CREATE */
+ testcase( i==71 ); /* CURRENT_DATE */
+ testcase( i==72 ); /* DETACH */
+ testcase( i==73 ); /* IMMEDIATE */
+ testcase( i==74 ); /* JOIN */
+ testcase( i==75 ); /* INSERT */
+ testcase( i==76 ); /* MATCH */
+ testcase( i==77 ); /* PLAN */
+ testcase( i==78 ); /* ANALYZE */
+ testcase( i==79 ); /* PRAGMA */
+ testcase( i==80 ); /* ABORT */
+ testcase( i==81 ); /* VALUES */
+ testcase( i==82 ); /* VIRTUAL */
+ testcase( i==83 ); /* LIMIT */
+ testcase( i==84 ); /* WHEN */
+ testcase( i==85 ); /* WHERE */
+ testcase( i==86 ); /* RENAME */
+ testcase( i==87 ); /* AFTER */
+ testcase( i==88 ); /* REPLACE */
+ testcase( i==89 ); /* AND */
+ testcase( i==90 ); /* DEFAULT */
+ testcase( i==91 ); /* AUTOINCREMENT */
+ testcase( i==92 ); /* TO */
+ testcase( i==93 ); /* IN */
+ testcase( i==94 ); /* CAST */
+ testcase( i==95 ); /* COLUMN */
+ testcase( i==96 ); /* COMMIT */
+ testcase( i==97 ); /* CONFLICT */
+ testcase( i==98 ); /* CROSS */
+ testcase( i==99 ); /* CURRENT_TIMESTAMP */
+ testcase( i==100 ); /* CURRENT_TIME */
+ testcase( i==101 ); /* PRIMARY */
+ testcase( i==102 ); /* DEFERRED */
+ testcase( i==103 ); /* DISTINCT */
+ testcase( i==104 ); /* IS */
+ testcase( i==105 ); /* DROP */
+ testcase( i==106 ); /* FAIL */
+ testcase( i==107 ); /* FROM */
+ testcase( i==108 ); /* FULL */
+ testcase( i==109 ); /* GLOB */
+ testcase( i==110 ); /* BY */
+ testcase( i==111 ); /* IF */
+ testcase( i==112 ); /* ISNULL */
+ testcase( i==113 ); /* ORDER */
+ testcase( i==114 ); /* RESTRICT */
+ testcase( i==115 ); /* RIGHT */
+ testcase( i==116 ); /* ROLLBACK */
+ testcase( i==117 ); /* ROW */
+ testcase( i==118 ); /* UNION */
+ testcase( i==119 ); /* USING */
+ testcase( i==120 ); /* VACUUM */
+ testcase( i==121 ); /* VIEW */
+ testcase( i==122 ); /* INITIALLY */
+ testcase( i==123 ); /* ALL */
+ return aCode[i];
+ }
+ }
+ return TK_ID;
+}
+SQLITE_PRIVATE int sqlite3KeywordCode(const unsigned char *z, int n){
+ return keywordCode((char*)z, n);
+}
+#define SQLITE_N_KEYWORD 124
+
+/************** End of keywordhash.h *****************************************/
+/************** Continuing where we left off in tokenize.c *******************/
+
+
+/*
+** If X is a character that can be used in an identifier then
+** IdChar(X) will be true. Otherwise it is false.
+**
+** For ASCII, any character with the high-order bit set is
+** allowed in an identifier. For 7-bit characters,
+** sqlite3IsIdChar[X] must be 1.
+**
+** For EBCDIC, the rules are more complex but have the same
+** end result.
+**
+** Ticket #1066. the SQL standard does not allow '$' in the
+** middle of identifiers. But many SQL implementations do.
+** SQLite will allow '$' in identifiers for compatibility.
+** But the feature is undocumented.
+*/
+#ifdef SQLITE_ASCII
+#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0)
+#endif
+#ifdef SQLITE_EBCDIC
+SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[] = {
+/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
+ 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 4x */
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, /* 5x */
+ 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, /* 6x */
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, /* 7x */
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, /* 8x */
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, /* 9x */
+ 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, /* Ax */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Cx */
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Dx */
+ 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Ex */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, /* Fx */
+};
+#define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40]))
+#endif
+
+/* Make the IdChar function accessible from ctime.c */
+#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS
+SQLITE_PRIVATE int sqlite3IsIdChar(u8 c){ return IdChar(c); }
+#endif
+
+
+/*
+** Return the length of the token that begins at z[0].
+** Store the token type in *tokenType before returning.
+*/
+SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){
+ int i, c;
+ switch( *z ){
+ case ' ': case '\t': case '\n': case '\f': case '\r': {
+ testcase( z[0]==' ' );
+ testcase( z[0]=='\t' );
+ testcase( z[0]=='\n' );
+ testcase( z[0]=='\f' );
+ testcase( z[0]=='\r' );
+ for(i=1; sqlite3Isspace(z[i]); i++){}
+ *tokenType = TK_SPACE;
+ return i;
+ }
+ case '-': {
+ if( z[1]=='-' ){
+ for(i=2; (c=z[i])!=0 && c!='\n'; i++){}
+ *tokenType = TK_SPACE; /* IMP: R-22934-25134 */
+ return i;
+ }
+ *tokenType = TK_MINUS;
+ return 1;
+ }
+ case '(': {
+ *tokenType = TK_LP;
+ return 1;
+ }
+ case ')': {
+ *tokenType = TK_RP;
+ return 1;
+ }
+ case ';': {
+ *tokenType = TK_SEMI;
+ return 1;
+ }
+ case '+': {
+ *tokenType = TK_PLUS;
+ return 1;
+ }
+ case '*': {
+ *tokenType = TK_STAR;
+ return 1;
+ }
+ case '/': {
+ if( z[1]!='*' || z[2]==0 ){
+ *tokenType = TK_SLASH;
+ return 1;
+ }
+ for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){}
+ if( c ) i++;
+ *tokenType = TK_SPACE; /* IMP: R-22934-25134 */
+ return i;
+ }
+ case '%': {
+ *tokenType = TK_REM;
+ return 1;
+ }
+ case '=': {
+ *tokenType = TK_EQ;
+ return 1 + (z[1]=='=');
+ }
+ case '<': {
+ if( (c=z[1])=='=' ){
+ *tokenType = TK_LE;
+ return 2;
+ }else if( c=='>' ){
+ *tokenType = TK_NE;
+ return 2;
+ }else if( c=='<' ){
+ *tokenType = TK_LSHIFT;
+ return 2;
+ }else{
+ *tokenType = TK_LT;
+ return 1;
+ }
+ }
+ case '>': {
+ if( (c=z[1])=='=' ){
+ *tokenType = TK_GE;
+ return 2;
+ }else if( c=='>' ){
+ *tokenType = TK_RSHIFT;
+ return 2;
+ }else{
+ *tokenType = TK_GT;
+ return 1;
+ }
+ }
+ case '!': {
+ if( z[1]!='=' ){
+ *tokenType = TK_ILLEGAL;
+ return 2;
+ }else{
+ *tokenType = TK_NE;
+ return 2;
+ }
+ }
+ case '|': {
+ if( z[1]!='|' ){
+ *tokenType = TK_BITOR;
+ return 1;
+ }else{
+ *tokenType = TK_CONCAT;
+ return 2;
+ }
+ }
+ case ',': {
+ *tokenType = TK_COMMA;
+ return 1;
+ }
+ case '&': {
+ *tokenType = TK_BITAND;
+ return 1;
+ }
+ case '~': {
+ *tokenType = TK_BITNOT;
+ return 1;
+ }
+ case '`':
+ case '\'':
+ case '"': {
+ int delim = z[0];
+ testcase( delim=='`' );
+ testcase( delim=='\'' );
+ testcase( delim=='"' );
+ for(i=1; (c=z[i])!=0; i++){
+ if( c==delim ){
+ if( z[i+1]==delim ){
+ i++;
+ }else{
+ break;
+ }
+ }
+ }
+ if( c=='\'' ){
+ *tokenType = TK_STRING;
+ return i+1;
+ }else if( c!=0 ){
+ *tokenType = TK_ID;
+ return i+1;
+ }else{
+ *tokenType = TK_ILLEGAL;
+ return i;
+ }
+ }
+ case '.': {
+#ifndef SQLITE_OMIT_FLOATING_POINT
+ if( !sqlite3Isdigit(z[1]) )
+#endif
+ {
+ *tokenType = TK_DOT;
+ return 1;
+ }
+ /* If the next character is a digit, this is a floating point
+ ** number that begins with ".". Fall thru into the next case */
+ }
+ case '0': case '1': case '2': case '3': case '4':
+ case '5': case '6': case '7': case '8': case '9': {
+ testcase( z[0]=='0' ); testcase( z[0]=='1' ); testcase( z[0]=='2' );
+ testcase( z[0]=='3' ); testcase( z[0]=='4' ); testcase( z[0]=='5' );
+ testcase( z[0]=='6' ); testcase( z[0]=='7' ); testcase( z[0]=='8' );
+ testcase( z[0]=='9' );
+ *tokenType = TK_INTEGER;
+#ifndef SQLITE_OMIT_HEX_INTEGER
+ if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){
+ for(i=3; sqlite3Isxdigit(z[i]); i++){}
+ return i;
+ }
+#endif
+ for(i=0; sqlite3Isdigit(z[i]); i++){}
+#ifndef SQLITE_OMIT_FLOATING_POINT
+ if( z[i]=='.' ){
+ i++;
+ while( sqlite3Isdigit(z[i]) ){ i++; }
+ *tokenType = TK_FLOAT;
+ }
+ if( (z[i]=='e' || z[i]=='E') &&
+ ( sqlite3Isdigit(z[i+1])
+ || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2]))
+ )
+ ){
+ i += 2;
+ while( sqlite3Isdigit(z[i]) ){ i++; }
+ *tokenType = TK_FLOAT;
+ }
+#endif
+ while( IdChar(z[i]) ){
+ *tokenType = TK_ILLEGAL;
+ i++;
+ }
+ return i;
+ }
+ case '[': {
+ for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){}
+ *tokenType = c==']' ? TK_ID : TK_ILLEGAL;
+ return i;
+ }
+ case '?': {
+ *tokenType = TK_VARIABLE;
+ for(i=1; sqlite3Isdigit(z[i]); i++){}
+ return i;
+ }
+#ifndef SQLITE_OMIT_TCL_VARIABLE
+ case '$':
+#endif
+ case '@': /* For compatibility with MS SQL Server */
+ case '#':
+ case ':': {
+ int n = 0;
+ testcase( z[0]=='$' ); testcase( z[0]=='@' );
+ testcase( z[0]==':' ); testcase( z[0]=='#' );
+ *tokenType = TK_VARIABLE;
+ for(i=1; (c=z[i])!=0; i++){
+ if( IdChar(c) ){
+ n++;
+#ifndef SQLITE_OMIT_TCL_VARIABLE
+ }else if( c=='(' && n>0 ){
+ do{
+ i++;
+ }while( (c=z[i])!=0 && !sqlite3Isspace(c) && c!=')' );
+ if( c==')' ){
+ i++;
+ }else{
+ *tokenType = TK_ILLEGAL;
+ }
+ break;
+ }else if( c==':' && z[i+1]==':' ){
+ i++;
+#endif
+ }else{
+ break;
+ }
+ }
+ if( n==0 ) *tokenType = TK_ILLEGAL;
+ return i;
+ }
+#ifndef SQLITE_OMIT_BLOB_LITERAL
+ case 'x': case 'X': {
+ testcase( z[0]=='x' ); testcase( z[0]=='X' );
+ if( z[1]=='\'' ){
+ *tokenType = TK_BLOB;
+ for(i=2; sqlite3Isxdigit(z[i]); i++){}
+ if( z[i]!='\'' || i%2 ){
+ *tokenType = TK_ILLEGAL;
+ while( z[i] && z[i]!='\'' ){ i++; }
+ }
+ if( z[i] ) i++;
+ return i;
+ }
+ /* Otherwise fall through to the next case */
+ }
+#endif
+ default: {
+ if( !IdChar(*z) ){
+ break;
+ }
+ for(i=1; IdChar(z[i]); i++){}
+ *tokenType = keywordCode((char*)z, i);
+ return i;
+ }
+ }
+ *tokenType = TK_ILLEGAL;
+ return 1;
+}
+
+/*
+** Run the parser on the given SQL string. The parser structure is
+** passed in. An SQLITE_ status code is returned. If an error occurs
+** then an and attempt is made to write an error message into
+** memory obtained from sqlite3_malloc() and to make *pzErrMsg point to that
+** error message.
+*/
+SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
+ int nErr = 0; /* Number of errors encountered */
+ int i; /* Loop counter */
+ void *pEngine; /* The LEMON-generated LALR(1) parser */
+ int tokenType; /* type of the next token */
+ int lastTokenParsed = -1; /* type of the previous token */
+ u8 enableLookaside; /* Saved value of db->lookaside.bEnabled */
+ sqlite3 *db = pParse->db; /* The database connection */
+ int mxSqlLen; /* Max length of an SQL string */
+
+ assert( zSql!=0 );
+ mxSqlLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH];
+ if( db->nVdbeActive==0 ){
+ db->u1.isInterrupted = 0;
+ }
+ pParse->rc = SQLITE_OK;
+ pParse->zTail = zSql;
+ i = 0;
+ assert( pzErrMsg!=0 );
+ /* sqlite3ParserTrace(stdout, "parser: "); */
+ pEngine = sqlite3ParserAlloc(sqlite3Malloc);
+ if( pEngine==0 ){
+ db->mallocFailed = 1;
+ return SQLITE_NOMEM;
+ }
+ assert( pParse->pNewTable==0 );
+ assert( pParse->pNewTrigger==0 );
+ assert( pParse->nVar==0 );
+ assert( pParse->nzVar==0 );
+ assert( pParse->azVar==0 );
+ enableLookaside = db->lookaside.bEnabled;
+ if( db->lookaside.pStart ) db->lookaside.bEnabled = 1;
+ while( !db->mallocFailed && zSql[i]!=0 ){
+ assert( i>=0 );
+ pParse->sLastToken.z = &zSql[i];
+ pParse->sLastToken.n = sqlite3GetToken((unsigned char*)&zSql[i],&tokenType);
+ i += pParse->sLastToken.n;
+ if( i>mxSqlLen ){
+ pParse->rc = SQLITE_TOOBIG;
+ break;
+ }
+ switch( tokenType ){
+ case TK_SPACE: {
+ if( db->u1.isInterrupted ){
+ sqlite3ErrorMsg(pParse, "interrupt");
+ pParse->rc = SQLITE_INTERRUPT;
+ goto abort_parse;
+ }
+ break;
+ }
+ case TK_ILLEGAL: {
+ sqlite3ErrorMsg(pParse, "unrecognized token: \"%T\"",
+ &pParse->sLastToken);
+ goto abort_parse;
+ }
+ case TK_SEMI: {
+ pParse->zTail = &zSql[i];
+ /* Fall thru into the default case */
+ }
+ default: {
+ sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse);
+ lastTokenParsed = tokenType;
+ if( pParse->rc!=SQLITE_OK ){
+ goto abort_parse;
+ }
+ break;
+ }
+ }
+ }
+abort_parse:
+ assert( nErr==0 );
+ if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){
+ assert( zSql[i]==0 );
+ if( lastTokenParsed!=TK_SEMI ){
+ sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse);
+ pParse->zTail = &zSql[i];
+ }
+ if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){
+ sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse);
+ }
+ }
+#ifdef YYTRACKMAXSTACKDEPTH
+ sqlite3_mutex_enter(sqlite3MallocMutex());
+ sqlite3StatusSet(SQLITE_STATUS_PARSER_STACK,
+ sqlite3ParserStackPeak(pEngine)
+ );
+ sqlite3_mutex_leave(sqlite3MallocMutex());
+#endif /* YYDEBUG */
+ sqlite3ParserFree(pEngine, sqlite3_free);
+ db->lookaside.bEnabled = enableLookaside;
+ if( db->mallocFailed ){
+ pParse->rc = SQLITE_NOMEM;
+ }
+ if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){
+ pParse->zErrMsg = sqlite3MPrintf(db, "%s", sqlite3ErrStr(pParse->rc));
+ }
+ assert( pzErrMsg!=0 );
+ if( pParse->zErrMsg ){
+ *pzErrMsg = pParse->zErrMsg;
+ sqlite3_log(pParse->rc, "%s", *pzErrMsg);
+ pParse->zErrMsg = 0;
+ nErr++;
+ }
+ if( pParse->pVdbe && pParse->nErr>0 && pParse->nested==0 ){
+ sqlite3VdbeDelete(pParse->pVdbe);
+ pParse->pVdbe = 0;
+ }
+#ifndef SQLITE_OMIT_SHARED_CACHE
+ if( pParse->nested==0 ){
+ sqlite3DbFree(db, pParse->aTableLock);
+ pParse->aTableLock = 0;
+ pParse->nTableLock = 0;
+ }
+#endif
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+ sqlite3_free(pParse->apVtabLock);
+#endif
+
+ if( !IN_DECLARE_VTAB ){
+ /* If the pParse->declareVtab flag is set, do not delete any table
+ ** structure built up in pParse->pNewTable. The calling code (see vtab.c)
+ ** will take responsibility for freeing the Table structure.
+ */
+ sqlite3DeleteTable(db, pParse->pNewTable);
+ }
+
+ if( pParse->bFreeWith ) sqlite3WithDelete(db, pParse->pWith);
+ sqlite3DeleteTrigger(db, pParse->pNewTrigger);
+ for(i=pParse->nzVar-1; i>=0; i--) sqlite3DbFree(db, pParse->azVar[i]);
+ sqlite3DbFree(db, pParse->azVar);
+ while( pParse->pAinc ){
+ AutoincInfo *p = pParse->pAinc;
+ pParse->pAinc = p->pNext;
+ sqlite3DbFree(db, p);
+ }
+ while( pParse->pZombieTab ){
+ Table *p = pParse->pZombieTab;
+ pParse->pZombieTab = p->pNextZombie;
+ sqlite3DeleteTable(db, p);
+ }
+ assert( nErr==0 || pParse->rc!=SQLITE_OK );
+ return nErr;
+}
+
+/************** End of tokenize.c ********************************************/
+/************** Begin file complete.c ****************************************/
+/*
+** 2001 September 15
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+*************************************************************************
+** An tokenizer for SQL
+**
+** This file contains C code that implements the sqlite3_complete() API.
+** This code used to be part of the tokenizer.c source file. But by
+** separating it out, the code will be automatically omitted from
+** static links that do not use it.
+*/
+/* #include "sqliteInt.h" */
+#ifndef SQLITE_OMIT_COMPLETE
+
+/*
+** This is defined in tokenize.c. We just have to import the definition.
+*/
+#ifndef SQLITE_AMALGAMATION
+#ifdef SQLITE_ASCII
+#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0)
+#endif
+#ifdef SQLITE_EBCDIC
+SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[];
+#define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40]))
+#endif
+#endif /* SQLITE_AMALGAMATION */
+
+
+/*
+** Token types used by the sqlite3_complete() routine. See the header
+** comments on that procedure for additional information.
+*/
+#define tkSEMI 0
+#define tkWS 1
+#define tkOTHER 2
+#ifndef SQLITE_OMIT_TRIGGER
+#define tkEXPLAIN 3
+#define tkCREATE 4
+#define tkTEMP 5
+#define tkTRIGGER 6
+#define tkEND 7
+#endif
+
+/*
+** Return TRUE if the given SQL string ends in a semicolon.
+**
+** Special handling is require for CREATE TRIGGER statements.
+** Whenever the CREATE TRIGGER keywords are seen, the statement
+** must end with ";END;".
+**
+** This implementation uses a state machine with 8 states:
+**
+** (0) INVALID We have not yet seen a non-whitespace character.
+**
+** (1) START At the beginning or end of an SQL statement. This routine
+** returns 1 if it ends in the START state and 0 if it ends
+** in any other state.
+**
+** (2) NORMAL We are in the middle of statement which ends with a single
+** semicolon.
+**
+** (3) EXPLAIN The keyword EXPLAIN has been seen at the beginning of
+** a statement.
+**
+** (4) CREATE The keyword CREATE has been seen at the beginning of a
+** statement, possibly preceded by EXPLAIN and/or followed by
+** TEMP or TEMPORARY
+**
+** (5) TRIGGER We are in the middle of a trigger definition that must be
+** ended by a semicolon, the keyword END, and another semicolon.
+**
+** (6) SEMI We've seen the first semicolon in the ";END;" that occurs at
+** the end of a trigger definition.
+**
+** (7) END We've seen the ";END" of the ";END;" that occurs at the end
+** of a trigger definition.
+**
+** Transitions between states above are determined by tokens extracted
+** from the input. The following tokens are significant:
+**
+** (0) tkSEMI A semicolon.
+** (1) tkWS Whitespace.
+** (2) tkOTHER Any other SQL token.
+** (3) tkEXPLAIN The "explain" keyword.
+** (4) tkCREATE The "create" keyword.
+** (5) tkTEMP The "temp" or "temporary" keyword.
+** (6) tkTRIGGER The "trigger" keyword.
+** (7) tkEND The "end" keyword.
+**
+** Whitespace never causes a state transition and is always ignored.
+** This means that a SQL string of all whitespace is invalid.
+**
+** If we compile with SQLITE_OMIT_TRIGGER, all of the computation needed
+** to recognize the end of a trigger can be omitted. All we have to do
+** is look for a semicolon that is not part of an string or comment.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_complete(const char *zSql){
+ u8 state = 0; /* Current state, using numbers defined in header comment */
+ u8 token; /* Value of the next token */
+
+#ifndef SQLITE_OMIT_TRIGGER
+ /* A complex statement machine used to detect the end of a CREATE TRIGGER
+ ** statement. This is the normal case.
+ */
+ static const u8 trans[8][8] = {
+ /* Token: */
+ /* State: ** SEMI WS OTHER EXPLAIN CREATE TEMP TRIGGER END */
+ /* 0 INVALID: */ { 1, 0, 2, 3, 4, 2, 2, 2, },
+ /* 1 START: */ { 1, 1, 2, 3, 4, 2, 2, 2, },
+ /* 2 NORMAL: */ { 1, 2, 2, 2, 2, 2, 2, 2, },
+ /* 3 EXPLAIN: */ { 1, 3, 3, 2, 4, 2, 2, 2, },
+ /* 4 CREATE: */ { 1, 4, 2, 2, 2, 4, 5, 2, },
+ /* 5 TRIGGER: */ { 6, 5, 5, 5, 5, 5, 5, 5, },
+ /* 6 SEMI: */ { 6, 6, 5, 5, 5, 5, 5, 7, },
+ /* 7 END: */ { 1, 7, 5, 5, 5, 5, 5, 5, },
+ };
+#else
+ /* If triggers are not supported by this compile then the statement machine
+ ** used to detect the end of a statement is much simpler
+ */
+ static const u8 trans[3][3] = {
+ /* Token: */
+ /* State: ** SEMI WS OTHER */
+ /* 0 INVALID: */ { 1, 0, 2, },
+ /* 1 START: */ { 1, 1, 2, },
+ /* 2 NORMAL: */ { 1, 2, 2, },
+ };
+#endif /* SQLITE_OMIT_TRIGGER */
+
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( zSql==0 ){
+ (void)SQLITE_MISUSE_BKPT;
+ return 0;
+ }
+#endif
+
+ while( *zSql ){
+ switch( *zSql ){
+ case ';': { /* A semicolon */
+ token = tkSEMI;
+ break;
+ }
+ case ' ':
+ case '\r':
+ case '\t':
+ case '\n':
+ case '\f': { /* White space is ignored */
+ token = tkWS;
+ break;
+ }
+ case '/': { /* C-style comments */
+ if( zSql[1]!='*' ){
+ token = tkOTHER;
+ break;
+ }
+ zSql += 2;
+ while( zSql[0] && (zSql[0]!='*' || zSql[1]!='/') ){ zSql++; }
+ if( zSql[0]==0 ) return 0;
+ zSql++;
+ token = tkWS;
+ break;
+ }
+ case '-': { /* SQL-style comments from "--" to end of line */
+ if( zSql[1]!='-' ){
+ token = tkOTHER;
+ break;
+ }
+ while( *zSql && *zSql!='\n' ){ zSql++; }
+ if( *zSql==0 ) return state==1;
+ token = tkWS;
+ break;
+ }
+ case '[': { /* Microsoft-style identifiers in [...] */
+ zSql++;
+ while( *zSql && *zSql!=']' ){ zSql++; }
+ if( *zSql==0 ) return 0;
+ token = tkOTHER;
+ break;
+ }
+ case '`': /* Grave-accent quoted symbols used by MySQL */
+ case '"': /* single- and double-quoted strings */
+ case '\'': {
+ int c = *zSql;
+ zSql++;
+ while( *zSql && *zSql!=c ){ zSql++; }
+ if( *zSql==0 ) return 0;
+ token = tkOTHER;
+ break;
+ }
+ default: {
+#ifdef SQLITE_EBCDIC
+ unsigned char c;
+#endif
+ if( IdChar((u8)*zSql) ){
+ /* Keywords and unquoted identifiers */
+ int nId;
+ for(nId=1; IdChar(zSql[nId]); nId++){}
+#ifdef SQLITE_OMIT_TRIGGER
+ token = tkOTHER;
+#else
+ switch( *zSql ){
+ case 'c': case 'C': {
+ if( nId==6 && sqlite3StrNICmp(zSql, "create", 6)==0 ){
+ token = tkCREATE;
+ }else{
+ token = tkOTHER;
+ }
+ break;
+ }
+ case 't': case 'T': {
+ if( nId==7 && sqlite3StrNICmp(zSql, "trigger", 7)==0 ){
+ token = tkTRIGGER;
+ }else if( nId==4 && sqlite3StrNICmp(zSql, "temp", 4)==0 ){
+ token = tkTEMP;
+ }else if( nId==9 && sqlite3StrNICmp(zSql, "temporary", 9)==0 ){
+ token = tkTEMP;
+ }else{
+ token = tkOTHER;
+ }
+ break;
+ }
+ case 'e': case 'E': {
+ if( nId==3 && sqlite3StrNICmp(zSql, "end", 3)==0 ){
+ token = tkEND;
+ }else
+#ifndef SQLITE_OMIT_EXPLAIN
+ if( nId==7 && sqlite3StrNICmp(zSql, "explain", 7)==0 ){
+ token = tkEXPLAIN;
+ }else
+#endif
+ {
+ token = tkOTHER;
+ }
+ break;
+ }
+ default: {
+ token = tkOTHER;
+ break;
+ }
+ }
+#endif /* SQLITE_OMIT_TRIGGER */
+ zSql += nId-1;
+ }else{
+ /* Operators and special symbols */
+ token = tkOTHER;
+ }
+ break;
+ }
+ }
+ state = trans[state][token];
+ zSql++;
+ }
+ return state==1;
+}
+
+#ifndef SQLITE_OMIT_UTF16
+/*
+** This routine is the same as the sqlite3_complete() routine described
+** above, except that the parameter is required to be UTF-16 encoded, not
+** UTF-8.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_complete16(const void *zSql){
+ sqlite3_value *pVal;
+ char const *zSql8;
+ int rc;
+
+#ifndef SQLITE_OMIT_AUTOINIT
+ rc = sqlite3_initialize();
+ if( rc ) return rc;
+#endif
+ pVal = sqlite3ValueNew(0);
+ sqlite3ValueSetStr(pVal, -1, zSql, SQLITE_UTF16NATIVE, SQLITE_STATIC);
+ zSql8 = sqlite3ValueText(pVal, SQLITE_UTF8);
+ if( zSql8 ){
+ rc = sqlite3_complete(zSql8);
+ }else{
+ rc = SQLITE_NOMEM;
+ }
+ sqlite3ValueFree(pVal);
+ return rc & 0xff;
+}
+#endif /* SQLITE_OMIT_UTF16 */
+#endif /* SQLITE_OMIT_COMPLETE */
+
+/************** End of complete.c ********************************************/
+/************** Begin file main.c ********************************************/
+/*
+** 2001 September 15
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+*************************************************************************
+** Main file for the SQLite library. The routines in this file
+** implement the programmer interface to the library. Routines in
+** other files are for internal use by SQLite and should not be
+** accessed by users of the library.
+*/
+/* #include "sqliteInt.h" */
+
+#ifdef SQLITE_ENABLE_FTS3
+/************** Include fts3.h in the middle of main.c ***********************/
+/************** Begin file fts3.h ********************************************/
+/*
+** 2006 Oct 10
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This header file is used by programs that want to link against the
+** FTS3 library. All it does is declare the sqlite3Fts3Init() interface.
+*/
+/* #include "sqlite3.h" */
+
+#if 0
+extern "C" {
+#endif /* __cplusplus */
+
+SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db);
+
+#if 0
+} /* extern "C" */
+#endif /* __cplusplus */
+
+/************** End of fts3.h ************************************************/
+/************** Continuing where we left off in main.c ***********************/
+#endif
+#ifdef SQLITE_ENABLE_RTREE
+/************** Include rtree.h in the middle of main.c **********************/
+/************** Begin file rtree.h *******************************************/
+/*
+** 2008 May 26
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This header file is used by programs that want to link against the
+** RTREE library. All it does is declare the sqlite3RtreeInit() interface.
+*/
+/* #include "sqlite3.h" */
+
+#if 0
+extern "C" {
+#endif /* __cplusplus */
+
+SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db);
+
+#if 0
+} /* extern "C" */
+#endif /* __cplusplus */
+
+/************** End of rtree.h ***********************************************/
+/************** Continuing where we left off in main.c ***********************/
+#endif
+#ifdef SQLITE_ENABLE_ICU
+/************** Include sqliteicu.h in the middle of main.c ******************/
+/************** Begin file sqliteicu.h ***************************************/
+/*
+** 2008 May 26
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This header file is used by programs that want to link against the
+** ICU extension. All it does is declare the sqlite3IcuInit() interface.
+*/
+/* #include "sqlite3.h" */
+
+#if 0
+extern "C" {
+#endif /* __cplusplus */
+
+SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db);
+
+#if 0
+} /* extern "C" */
+#endif /* __cplusplus */
+
+
+/************** End of sqliteicu.h *******************************************/
+/************** Continuing where we left off in main.c ***********************/
+#endif
+#ifdef SQLITE_ENABLE_JSON1
+SQLITE_PRIVATE int sqlite3Json1Init(sqlite3*);
+#endif
+#ifdef SQLITE_ENABLE_FTS5
+SQLITE_PRIVATE int sqlite3Fts5Init(sqlite3*);
+#endif
+
+#ifndef SQLITE_AMALGAMATION
+/* IMPLEMENTATION-OF: R-46656-45156 The sqlite3_version[] string constant
+** contains the text of SQLITE_VERSION macro.
+*/
+SQLITE_API const char sqlite3_version[] = SQLITE_VERSION;
+#endif
+
+/* IMPLEMENTATION-OF: R-53536-42575 The sqlite3_libversion() function returns
+** a pointer to the to the sqlite3_version[] string constant.
+*/
+SQLITE_API const char *SQLITE_STDCALL sqlite3_libversion(void){ return sqlite3_version; }
+
+/* IMPLEMENTATION-OF: R-63124-39300 The sqlite3_sourceid() function returns a
+** pointer to a string constant whose value is the same as the
+** SQLITE_SOURCE_ID C preprocessor macro.
+*/
+SQLITE_API const char *SQLITE_STDCALL sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; }
+
+/* IMPLEMENTATION-OF: R-35210-63508 The sqlite3_libversion_number() function
+** returns an integer equal to SQLITE_VERSION_NUMBER.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_libversion_number(void){ return SQLITE_VERSION_NUMBER; }
+
+/* IMPLEMENTATION-OF: R-20790-14025 The sqlite3_threadsafe() function returns
+** zero if and only if SQLite was compiled with mutexing code omitted due to
+** the SQLITE_THREADSAFE compile-time option being set to 0.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_threadsafe(void){ return SQLITE_THREADSAFE; }
+
+/*
+** When compiling the test fixture or with debugging enabled (on Win32),
+** this variable being set to non-zero will cause OSTRACE macros to emit
+** extra diagnostic information.
+*/
+#ifdef SQLITE_HAVE_OS_TRACE
+# ifndef SQLITE_DEBUG_OS_TRACE
+# define SQLITE_DEBUG_OS_TRACE 0
+# endif
+ int sqlite3OSTrace = SQLITE_DEBUG_OS_TRACE;
+#endif
+
+#if !defined(SQLITE_OMIT_TRACE) && defined(SQLITE_ENABLE_IOTRACE)
+/*
+** If the following function pointer is not NULL and if
+** SQLITE_ENABLE_IOTRACE is enabled, then messages describing
+** I/O active are written using this function. These messages
+** are intended for debugging activity only.
+*/
+SQLITE_API void (SQLITE_CDECL *sqlite3IoTrace)(const char*, ...) = 0;
+#endif
+
+/*
+** If the following global variable points to a string which is the
+** name of a directory, then that directory will be used to store
+** temporary files.
+**
+** See also the "PRAGMA temp_store_directory" SQL command.
+*/
+SQLITE_API char *sqlite3_temp_directory = 0;
+
+/*
+** If the following global variable points to a string which is the
+** name of a directory, then that directory will be used to store
+** all database files specified with a relative pathname.
+**
+** See also the "PRAGMA data_store_directory" SQL command.
+*/
+SQLITE_API char *sqlite3_data_directory = 0;
+
+/*
+** Initialize SQLite.
+**
+** This routine must be called to initialize the memory allocation,
+** VFS, and mutex subsystems prior to doing any serious work with
+** SQLite. But as long as you do not compile with SQLITE_OMIT_AUTOINIT
+** this routine will be called automatically by key routines such as
+** sqlite3_open().
+**
+** This routine is a no-op except on its very first call for the process,
+** or for the first call after a call to sqlite3_shutdown.
+**
+** The first thread to call this routine runs the initialization to
+** completion. If subsequent threads call this routine before the first
+** thread has finished the initialization process, then the subsequent
+** threads must block until the first thread finishes with the initialization.
+**
+** The first thread might call this routine recursively. Recursive
+** calls to this routine should not block, of course. Otherwise the
+** initialization process would never complete.
+**
+** Let X be the first thread to enter this routine. Let Y be some other
+** thread. Then while the initial invocation of this routine by X is
+** incomplete, it is required that:
+**
+** * Calls to this routine from Y must block until the outer-most
+** call by X completes.
+**
+** * Recursive calls to this routine from thread X return immediately
+** without blocking.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_initialize(void){
+ MUTEX_LOGIC( sqlite3_mutex *pMaster; ) /* The main static mutex */
+ int rc; /* Result code */
+#ifdef SQLITE_EXTRA_INIT
+ int bRunExtraInit = 0; /* Extra initialization needed */
+#endif
+
+#ifdef SQLITE_OMIT_WSD
+ rc = sqlite3_wsd_init(4096, 24);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+#endif
+
+ /* If the following assert() fails on some obscure processor/compiler
+ ** combination, the work-around is to set the correct pointer
+ ** size at compile-time using -DSQLITE_PTRSIZE=n compile-time option */
+ assert( SQLITE_PTRSIZE==sizeof(char*) );
+
+ /* If SQLite is already completely initialized, then this call
+ ** to sqlite3_initialize() should be a no-op. But the initialization
+ ** must be complete. So isInit must not be set until the very end
+ ** of this routine.
+ */
+ if( sqlite3GlobalConfig.isInit ) return SQLITE_OK;
+
+ /* Make sure the mutex subsystem is initialized. If unable to
+ ** initialize the mutex subsystem, return early with the error.
+ ** If the system is so sick that we are unable to allocate a mutex,
+ ** there is not much SQLite is going to be able to do.
+ **
+ ** The mutex subsystem must take care of serializing its own
+ ** initialization.
+ */
+ rc = sqlite3MutexInit();
+ if( rc ) return rc;
+
+ /* Initialize the malloc() system and the recursive pInitMutex mutex.
+ ** This operation is protected by the STATIC_MASTER mutex. Note that
+ ** MutexAlloc() is called for a static mutex prior to initializing the
+ ** malloc subsystem - this implies that the allocation of a static
+ ** mutex must not require support from the malloc subsystem.
+ */
+ MUTEX_LOGIC( pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); )
+ sqlite3_mutex_enter(pMaster);
+ sqlite3GlobalConfig.isMutexInit = 1;
+ if( !sqlite3GlobalConfig.isMallocInit ){
+ rc = sqlite3MallocInit();
+ }
+ if( rc==SQLITE_OK ){
+ sqlite3GlobalConfig.isMallocInit = 1;
+ if( !sqlite3GlobalConfig.pInitMutex ){
+ sqlite3GlobalConfig.pInitMutex =
+ sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE);
+ if( sqlite3GlobalConfig.bCoreMutex && !sqlite3GlobalConfig.pInitMutex ){
+ rc = SQLITE_NOMEM;
+ }
+ }
+ }
+ if( rc==SQLITE_OK ){
+ sqlite3GlobalConfig.nRefInitMutex++;
+ }
+ sqlite3_mutex_leave(pMaster);
+
+ /* If rc is not SQLITE_OK at this point, then either the malloc
+ ** subsystem could not be initialized or the system failed to allocate
+ ** the pInitMutex mutex. Return an error in either case. */
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+
+ /* Do the rest of the initialization under the recursive mutex so
+ ** that we will be able to handle recursive calls into
+ ** sqlite3_initialize(). The recursive calls normally come through
+ ** sqlite3_os_init() when it invokes sqlite3_vfs_register(), but other
+ ** recursive calls might also be possible.
+ **
+ ** IMPLEMENTATION-OF: R-00140-37445 SQLite automatically serializes calls
+ ** to the xInit method, so the xInit method need not be threadsafe.
+ **
+ ** The following mutex is what serializes access to the appdef pcache xInit
+ ** methods. The sqlite3_pcache_methods.xInit() all is embedded in the
+ ** call to sqlite3PcacheInitialize().
+ */
+ sqlite3_mutex_enter(sqlite3GlobalConfig.pInitMutex);
+ if( sqlite3GlobalConfig.isInit==0 && sqlite3GlobalConfig.inProgress==0 ){
+ FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions);
+ sqlite3GlobalConfig.inProgress = 1;
+ memset(pHash, 0, sizeof(sqlite3GlobalFunctions));
+ sqlite3RegisterGlobalFunctions();
+ if( sqlite3GlobalConfig.isPCacheInit==0 ){
+ rc = sqlite3PcacheInitialize();
+ }
+ if( rc==SQLITE_OK ){
+ sqlite3GlobalConfig.isPCacheInit = 1;
+ rc = sqlite3OsInit();
+ }
+ if( rc==SQLITE_OK ){
+ sqlite3PCacheBufferSetup( sqlite3GlobalConfig.pPage,
+ sqlite3GlobalConfig.szPage, sqlite3GlobalConfig.nPage);
+ sqlite3GlobalConfig.isInit = 1;
+#ifdef SQLITE_EXTRA_INIT
+ bRunExtraInit = 1;
+#endif
+ }
+ sqlite3GlobalConfig.inProgress = 0;
+ }
+ sqlite3_mutex_leave(sqlite3GlobalConfig.pInitMutex);
+
+ /* Go back under the static mutex and clean up the recursive
+ ** mutex to prevent a resource leak.
+ */
+ sqlite3_mutex_enter(pMaster);
+ sqlite3GlobalConfig.nRefInitMutex--;
+ if( sqlite3GlobalConfig.nRefInitMutex<=0 ){
+ assert( sqlite3GlobalConfig.nRefInitMutex==0 );
+ sqlite3_mutex_free(sqlite3GlobalConfig.pInitMutex);
+ sqlite3GlobalConfig.pInitMutex = 0;
+ }
+ sqlite3_mutex_leave(pMaster);
+
+ /* The following is just a sanity check to make sure SQLite has
+ ** been compiled correctly. It is important to run this code, but
+ ** we don't want to run it too often and soak up CPU cycles for no
+ ** reason. So we run it once during initialization.
+ */
+#ifndef NDEBUG
+#ifndef SQLITE_OMIT_FLOATING_POINT
+ /* This section of code's only "output" is via assert() statements. */
+ if ( rc==SQLITE_OK ){
+ u64 x = (((u64)1)<<63)-1;
+ double y;
+ assert(sizeof(x)==8);
+ assert(sizeof(x)==sizeof(y));
+ memcpy(&y, &x, 8);
+ assert( sqlite3IsNaN(y) );
+ }
+#endif
+#endif
+
+ /* Do extra initialization steps requested by the SQLITE_EXTRA_INIT
+ ** compile-time option.
+ */
+#ifdef SQLITE_EXTRA_INIT
+ if( bRunExtraInit ){
+ int SQLITE_EXTRA_INIT(const char*);
+ rc = SQLITE_EXTRA_INIT(0);
+ }
+#endif
+
+ return rc;
+}
+
+/*
+** Undo the effects of sqlite3_initialize(). Must not be called while
+** there are outstanding database connections or memory allocations or
+** while any part of SQLite is otherwise in use in any thread. This
+** routine is not threadsafe. But it is safe to invoke this routine
+** on when SQLite is already shut down. If SQLite is already shut down
+** when this routine is invoked, then this routine is a harmless no-op.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_shutdown(void){
+#ifdef SQLITE_OMIT_WSD
+ int rc = sqlite3_wsd_init(4096, 24);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+#endif
+
+ if( sqlite3GlobalConfig.isInit ){
+#ifdef SQLITE_EXTRA_SHUTDOWN
+ void SQLITE_EXTRA_SHUTDOWN(void);
+ SQLITE_EXTRA_SHUTDOWN();
+#endif
+ sqlite3_os_end();
+ sqlite3_reset_auto_extension();
+ sqlite3GlobalConfig.isInit = 0;
+ }
+ if( sqlite3GlobalConfig.isPCacheInit ){
+ sqlite3PcacheShutdown();
+ sqlite3GlobalConfig.isPCacheInit = 0;
+ }
+ if( sqlite3GlobalConfig.isMallocInit ){
+ sqlite3MallocEnd();
+ sqlite3GlobalConfig.isMallocInit = 0;
+
+#ifndef SQLITE_OMIT_SHUTDOWN_DIRECTORIES
+ /* The heap subsystem has now been shutdown and these values are supposed
+ ** to be NULL or point to memory that was obtained from sqlite3_malloc(),
+ ** which would rely on that heap subsystem; therefore, make sure these
+ ** values cannot refer to heap memory that was just invalidated when the
+ ** heap subsystem was shutdown. This is only done if the current call to
+ ** this function resulted in the heap subsystem actually being shutdown.
+ */
+ sqlite3_data_directory = 0;
+ sqlite3_temp_directory = 0;
+#endif
+ }
+ if( sqlite3GlobalConfig.isMutexInit ){
+ sqlite3MutexEnd();
+ sqlite3GlobalConfig.isMutexInit = 0;
+ }
+
+ return SQLITE_OK;
+}
+
+/*
+** This API allows applications to modify the global configuration of
+** the SQLite library at run-time.
+**
+** This routine should only be called when there are no outstanding
+** database connections or memory allocations. This routine is not
+** threadsafe. Failure to heed these warnings can lead to unpredictable
+** behavior.
+*/
+SQLITE_API int SQLITE_CDECL sqlite3_config(int op, ...){
+ va_list ap;
+ int rc = SQLITE_OK;
+
+ /* sqlite3_config() shall return SQLITE_MISUSE if it is invoked while
+ ** the SQLite library is in use. */
+ if( sqlite3GlobalConfig.isInit ) return SQLITE_MISUSE_BKPT;
+
+ va_start(ap, op);
+ switch( op ){
+
+ /* Mutex configuration options are only available in a threadsafe
+ ** compile.
+ */
+#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-54466-46756 */
+ case SQLITE_CONFIG_SINGLETHREAD: {
+ /* EVIDENCE-OF: R-02748-19096 This option sets the threading mode to
+ ** Single-thread. */
+ sqlite3GlobalConfig.bCoreMutex = 0; /* Disable mutex on core */
+ sqlite3GlobalConfig.bFullMutex = 0; /* Disable mutex on connections */
+ break;
+ }
+#endif
+#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-20520-54086 */
+ case SQLITE_CONFIG_MULTITHREAD: {
+ /* EVIDENCE-OF: R-14374-42468 This option sets the threading mode to
+ ** Multi-thread. */
+ sqlite3GlobalConfig.bCoreMutex = 1; /* Enable mutex on core */
+ sqlite3GlobalConfig.bFullMutex = 0; /* Disable mutex on connections */
+ break;
+ }
+#endif
+#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-59593-21810 */
+ case SQLITE_CONFIG_SERIALIZED: {
+ /* EVIDENCE-OF: R-41220-51800 This option sets the threading mode to
+ ** Serialized. */
+ sqlite3GlobalConfig.bCoreMutex = 1; /* Enable mutex on core */
+ sqlite3GlobalConfig.bFullMutex = 1; /* Enable mutex on connections */
+ break;
+ }
+#endif
+#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-63666-48755 */
+ case SQLITE_CONFIG_MUTEX: {
+ /* Specify an alternative mutex implementation */
+ sqlite3GlobalConfig.mutex = *va_arg(ap, sqlite3_mutex_methods*);
+ break;
+ }
+#endif
+#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-14450-37597 */
+ case SQLITE_CONFIG_GETMUTEX: {
+ /* Retrieve the current mutex implementation */
+ *va_arg(ap, sqlite3_mutex_methods*) = sqlite3GlobalConfig.mutex;
+ break;
+ }
+#endif
+
+ case SQLITE_CONFIG_MALLOC: {
+ /* EVIDENCE-OF: R-55594-21030 The SQLITE_CONFIG_MALLOC option takes a
+ ** single argument which is a pointer to an instance of the
+ ** sqlite3_mem_methods structure. The argument specifies alternative
+ ** low-level memory allocation routines to be used in place of the memory
+ ** allocation routines built into SQLite. */
+ sqlite3GlobalConfig.m = *va_arg(ap, sqlite3_mem_methods*);
+ break;
+ }
+ case SQLITE_CONFIG_GETMALLOC: {
+ /* EVIDENCE-OF: R-51213-46414 The SQLITE_CONFIG_GETMALLOC option takes a
+ ** single argument which is a pointer to an instance of the
+ ** sqlite3_mem_methods structure. The sqlite3_mem_methods structure is
+ ** filled with the currently defined memory allocation routines. */
+ if( sqlite3GlobalConfig.m.xMalloc==0 ) sqlite3MemSetDefault();
+ *va_arg(ap, sqlite3_mem_methods*) = sqlite3GlobalConfig.m;
+ break;
+ }
+ case SQLITE_CONFIG_MEMSTATUS: {
+ /* EVIDENCE-OF: R-61275-35157 The SQLITE_CONFIG_MEMSTATUS option takes
+ ** single argument of type int, interpreted as a boolean, which enables
+ ** or disables the collection of memory allocation statistics. */
+ sqlite3GlobalConfig.bMemstat = va_arg(ap, int);
+ break;
+ }
+ case SQLITE_CONFIG_SCRATCH: {
+ /* EVIDENCE-OF: R-08404-60887 There are three arguments to
+ ** SQLITE_CONFIG_SCRATCH: A pointer an 8-byte aligned memory buffer from
+ ** which the scratch allocations will be drawn, the size of each scratch
+ ** allocation (sz), and the maximum number of scratch allocations (N). */
+ sqlite3GlobalConfig.pScratch = va_arg(ap, void*);
+ sqlite3GlobalConfig.szScratch = va_arg(ap, int);
+ sqlite3GlobalConfig.nScratch = va_arg(ap, int);
+ break;
+ }
+ case SQLITE_CONFIG_PAGECACHE: {
+ /* EVIDENCE-OF: R-31408-40510 There are three arguments to
+ ** SQLITE_CONFIG_PAGECACHE: A pointer to 8-byte aligned memory, the size
+ ** of each page buffer (sz), and the number of pages (N). */
+ sqlite3GlobalConfig.pPage = va_arg(ap, void*);
+ sqlite3GlobalConfig.szPage = va_arg(ap, int);
+ sqlite3GlobalConfig.nPage = va_arg(ap, int);
+ break;
+ }
+ case SQLITE_CONFIG_PCACHE_HDRSZ: {
+ /* EVIDENCE-OF: R-39100-27317 The SQLITE_CONFIG_PCACHE_HDRSZ option takes
+ ** a single parameter which is a pointer to an integer and writes into
+ ** that integer the number of extra bytes per page required for each page
+ ** in SQLITE_CONFIG_PAGECACHE. */
+ *va_arg(ap, int*) =
+ sqlite3HeaderSizeBtree() +
+ sqlite3HeaderSizePcache() +
+ sqlite3HeaderSizePcache1();
+ break;
+ }
+
+ case SQLITE_CONFIG_PCACHE: {
+ /* no-op */
+ break;
+ }
+ case SQLITE_CONFIG_GETPCACHE: {
+ /* now an error */
+ rc = SQLITE_ERROR;
+ break;
+ }
+
+ case SQLITE_CONFIG_PCACHE2: {
+ /* EVIDENCE-OF: R-63325-48378 The SQLITE_CONFIG_PCACHE2 option takes a
+ ** single argument which is a pointer to an sqlite3_pcache_methods2
+ ** object. This object specifies the interface to a custom page cache
+ ** implementation. */
+ sqlite3GlobalConfig.pcache2 = *va_arg(ap, sqlite3_pcache_methods2*);
+ break;
+ }
+ case SQLITE_CONFIG_GETPCACHE2: {
+ /* EVIDENCE-OF: R-22035-46182 The SQLITE_CONFIG_GETPCACHE2 option takes a
+ ** single argument which is a pointer to an sqlite3_pcache_methods2
+ ** object. SQLite copies of the current page cache implementation into
+ ** that object. */
+ if( sqlite3GlobalConfig.pcache2.xInit==0 ){
+ sqlite3PCacheSetDefault();
+ }
+ *va_arg(ap, sqlite3_pcache_methods2*) = sqlite3GlobalConfig.pcache2;
+ break;
+ }
+
+/* EVIDENCE-OF: R-06626-12911 The SQLITE_CONFIG_HEAP option is only
+** available if SQLite is compiled with either SQLITE_ENABLE_MEMSYS3 or
+** SQLITE_ENABLE_MEMSYS5 and returns SQLITE_ERROR if invoked otherwise. */
+#if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5)
+ case SQLITE_CONFIG_HEAP: {
+ /* EVIDENCE-OF: R-19854-42126 There are three arguments to
+ ** SQLITE_CONFIG_HEAP: An 8-byte aligned pointer to the memory, the
+ ** number of bytes in the memory buffer, and the minimum allocation size.
+ */
+ sqlite3GlobalConfig.pHeap = va_arg(ap, void*);
+ sqlite3GlobalConfig.nHeap = va_arg(ap, int);
+ sqlite3GlobalConfig.mnReq = va_arg(ap, int);
+
+ if( sqlite3GlobalConfig.mnReq<1 ){
+ sqlite3GlobalConfig.mnReq = 1;
+ }else if( sqlite3GlobalConfig.mnReq>(1<<12) ){
+ /* cap min request size at 2^12 */
+ sqlite3GlobalConfig.mnReq = (1<<12);
+ }
+
+ if( sqlite3GlobalConfig.pHeap==0 ){
+ /* EVIDENCE-OF: R-49920-60189 If the first pointer (the memory pointer)
+ ** is NULL, then SQLite reverts to using its default memory allocator
+ ** (the system malloc() implementation), undoing any prior invocation of
+ ** SQLITE_CONFIG_MALLOC.
+ **
+ ** Setting sqlite3GlobalConfig.m to all zeros will cause malloc to
+ ** revert to its default implementation when sqlite3_initialize() is run
+ */
+ memset(&sqlite3GlobalConfig.m, 0, sizeof(sqlite3GlobalConfig.m));
+ }else{
+ /* EVIDENCE-OF: R-61006-08918 If the memory pointer is not NULL then the
+ ** alternative memory allocator is engaged to handle all of SQLites
+ ** memory allocation needs. */
+#ifdef SQLITE_ENABLE_MEMSYS3
+ sqlite3GlobalConfig.m = *sqlite3MemGetMemsys3();
+#endif
+#ifdef SQLITE_ENABLE_MEMSYS5
+ sqlite3GlobalConfig.m = *sqlite3MemGetMemsys5();
+#endif
+ }
+ break;
+ }
+#endif
+
+ case SQLITE_CONFIG_LOOKASIDE: {
+ sqlite3GlobalConfig.szLookaside = va_arg(ap, int);
+ sqlite3GlobalConfig.nLookaside = va_arg(ap, int);
+ break;
+ }
+
+ /* Record a pointer to the logger function and its first argument.
+ ** The default is NULL. Logging is disabled if the function pointer is
+ ** NULL.
+ */
+ case SQLITE_CONFIG_LOG: {
+ /* MSVC is picky about pulling func ptrs from va lists.
+ ** http://support.microsoft.com/kb/47961
+ ** sqlite3GlobalConfig.xLog = va_arg(ap, void(*)(void*,int,const char*));
+ */
+ typedef void(*LOGFUNC_t)(void*,int,const char*);
+ sqlite3GlobalConfig.xLog = va_arg(ap, LOGFUNC_t);
+ sqlite3GlobalConfig.pLogArg = va_arg(ap, void*);
+ break;
+ }
+
+ /* EVIDENCE-OF: R-55548-33817 The compile-time setting for URI filenames
+ ** can be changed at start-time using the
+ ** sqlite3_config(SQLITE_CONFIG_URI,1) or
+ ** sqlite3_config(SQLITE_CONFIG_URI,0) configuration calls.
+ */
+ case SQLITE_CONFIG_URI: {
+ /* EVIDENCE-OF: R-25451-61125 The SQLITE_CONFIG_URI option takes a single
+ ** argument of type int. If non-zero, then URI handling is globally
+ ** enabled. If the parameter is zero, then URI handling is globally
+ ** disabled. */
+ sqlite3GlobalConfig.bOpenUri = va_arg(ap, int);
+ break;
+ }
+
+ case SQLITE_CONFIG_COVERING_INDEX_SCAN: {
+ /* EVIDENCE-OF: R-36592-02772 The SQLITE_CONFIG_COVERING_INDEX_SCAN
+ ** option takes a single integer argument which is interpreted as a
+ ** boolean in order to enable or disable the use of covering indices for
+ ** full table scans in the query optimizer. */
+ sqlite3GlobalConfig.bUseCis = va_arg(ap, int);
+ break;
+ }
+
+#ifdef SQLITE_ENABLE_SQLLOG
+ case SQLITE_CONFIG_SQLLOG: {
+ typedef void(*SQLLOGFUNC_t)(void*, sqlite3*, const char*, int);
+ sqlite3GlobalConfig.xSqllog = va_arg(ap, SQLLOGFUNC_t);
+ sqlite3GlobalConfig.pSqllogArg = va_arg(ap, void *);
+ break;
+ }
+#endif
+
+ case SQLITE_CONFIG_MMAP_SIZE: {
+ /* EVIDENCE-OF: R-58063-38258 SQLITE_CONFIG_MMAP_SIZE takes two 64-bit
+ ** integer (sqlite3_int64) values that are the default mmap size limit
+ ** (the default setting for PRAGMA mmap_size) and the maximum allowed
+ ** mmap size limit. */
+ sqlite3_int64 szMmap = va_arg(ap, sqlite3_int64);
+ sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64);
+ /* EVIDENCE-OF: R-53367-43190 If either argument to this option is
+ ** negative, then that argument is changed to its compile-time default.
+ **
+ ** EVIDENCE-OF: R-34993-45031 The maximum allowed mmap size will be
+ ** silently truncated if necessary so that it does not exceed the
+ ** compile-time maximum mmap size set by the SQLITE_MAX_MMAP_SIZE
+ ** compile-time option.
+ */
+ if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ){
+ mxMmap = SQLITE_MAX_MMAP_SIZE;
+ }
+ if( szMmap<0 ) szMmap = SQLITE_DEFAULT_MMAP_SIZE;
+ if( szMmap>mxMmap) szMmap = mxMmap;
+ sqlite3GlobalConfig.mxMmap = mxMmap;
+ sqlite3GlobalConfig.szMmap = szMmap;
+ break;
+ }
+
+#if SQLITE_OS_WIN && defined(SQLITE_WIN32_MALLOC) /* IMP: R-04780-55815 */
+ case SQLITE_CONFIG_WIN32_HEAPSIZE: {
+ /* EVIDENCE-OF: R-34926-03360 SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit
+ ** unsigned integer value that specifies the maximum size of the created
+ ** heap. */
+ sqlite3GlobalConfig.nHeap = va_arg(ap, int);
+ break;
+ }
+#endif
+
+ case SQLITE_CONFIG_PMASZ: {
+ sqlite3GlobalConfig.szPma = va_arg(ap, unsigned int);
+ break;
+ }
+
+ default: {
+ rc = SQLITE_ERROR;
+ break;
+ }
+ }
+ va_end(ap);
+ return rc;
+}
+
+/*
+** Set up the lookaside buffers for a database connection.
+** Return SQLITE_OK on success.
+** If lookaside is already active, return SQLITE_BUSY.
+**
+** The sz parameter is the number of bytes in each lookaside slot.
+** The cnt parameter is the number of slots. If pStart is NULL the
+** space for the lookaside memory is obtained from sqlite3_malloc().
+** If pStart is not NULL then it is sz*cnt bytes of memory to use for
+** the lookaside memory.
+*/
+static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){
+#ifndef SQLITE_OMIT_LOOKASIDE
+ void *pStart;
+ if( db->lookaside.nOut ){
+ return SQLITE_BUSY;
+ }
+ /* Free any existing lookaside buffer for this handle before
+ ** allocating a new one so we don't have to have space for
+ ** both at the same time.
+ */
+ if( db->lookaside.bMalloced ){
+ sqlite3_free(db->lookaside.pStart);
+ }
+ /* The size of a lookaside slot after ROUNDDOWN8 needs to be larger
+ ** than a pointer to be useful.
+ */
+ sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */
+ if( sz<=(int)sizeof(LookasideSlot*) ) sz = 0;
+ if( cnt<0 ) cnt = 0;
+ if( sz==0 || cnt==0 ){
+ sz = 0;
+ pStart = 0;
+ }else if( pBuf==0 ){
+ sqlite3BeginBenignMalloc();
+ pStart = sqlite3Malloc( sz*cnt ); /* IMP: R-61949-35727 */
+ sqlite3EndBenignMalloc();
+ if( pStart ) cnt = sqlite3MallocSize(pStart)/sz;
+ }else{
+ pStart = pBuf;
+ }
+ db->lookaside.pStart = pStart;
+ db->lookaside.pFree = 0;
+ db->lookaside.sz = (u16)sz;
+ if( pStart ){
+ int i;
+ LookasideSlot *p;
+ assert( sz > (int)sizeof(LookasideSlot*) );
+ p = (LookasideSlot*)pStart;
+ for(i=cnt-1; i>=0; i--){
+ p->pNext = db->lookaside.pFree;
+ db->lookaside.pFree = p;
+ p = (LookasideSlot*)&((u8*)p)[sz];
+ }
+ db->lookaside.pEnd = p;
+ db->lookaside.bEnabled = 1;
+ db->lookaside.bMalloced = pBuf==0 ?1:0;
+ }else{
+ db->lookaside.pStart = db;
+ db->lookaside.pEnd = db;
+ db->lookaside.bEnabled = 0;
+ db->lookaside.bMalloced = 0;
+ }
+#endif /* SQLITE_OMIT_LOOKASIDE */
+ return SQLITE_OK;
+}
+
+/*
+** Return the mutex associated with a database connection.
+*/
+SQLITE_API sqlite3_mutex *SQLITE_STDCALL sqlite3_db_mutex(sqlite3 *db){
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) ){
+ (void)SQLITE_MISUSE_BKPT;
+ return 0;
+ }
+#endif
+ return db->mutex;
+}
+
+/*
+** Free up as much memory as we can from the given database
+** connection.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_db_release_memory(sqlite3 *db){
+ int i;
+
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
+#endif
+ sqlite3_mutex_enter(db->mutex);
+ sqlite3BtreeEnterAll(db);
+ for(i=0; i<db->nDb; i++){
+ Btree *pBt = db->aDb[i].pBt;
+ if( pBt ){
+ Pager *pPager = sqlite3BtreePager(pBt);
+ sqlite3PagerShrink(pPager);
+ }
+ }
+ sqlite3BtreeLeaveAll(db);
+ sqlite3_mutex_leave(db->mutex);
+ return SQLITE_OK;
+}
+
+/*
+** Configuration settings for an individual database connection
+*/
+SQLITE_API int SQLITE_CDECL sqlite3_db_config(sqlite3 *db, int op, ...){
+ va_list ap;
+ int rc;
+ va_start(ap, op);
+ switch( op ){
+ case SQLITE_DBCONFIG_LOOKASIDE: {
+ void *pBuf = va_arg(ap, void*); /* IMP: R-26835-10964 */
+ int sz = va_arg(ap, int); /* IMP: R-47871-25994 */
+ int cnt = va_arg(ap, int); /* IMP: R-04460-53386 */
+ rc = setupLookaside(db, pBuf, sz, cnt);
+ break;
+ }
+ default: {
+ static const struct {
+ int op; /* The opcode */
+ u32 mask; /* Mask of the bit in sqlite3.flags to set/clear */
+ } aFlagOp[] = {
+ { SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_ForeignKeys },
+ { SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_EnableTrigger },
+ };
+ unsigned int i;
+ rc = SQLITE_ERROR; /* IMP: R-42790-23372 */
+ for(i=0; i<ArraySize(aFlagOp); i++){
+ if( aFlagOp[i].op==op ){
+ int onoff = va_arg(ap, int);
+ int *pRes = va_arg(ap, int*);
+ int oldFlags = db->flags;
+ if( onoff>0 ){
+ db->flags |= aFlagOp[i].mask;
+ }else if( onoff==0 ){
+ db->flags &= ~aFlagOp[i].mask;
+ }
+ if( oldFlags!=db->flags ){
+ sqlite3ExpirePreparedStatements(db);
+ }
+ if( pRes ){
+ *pRes = (db->flags & aFlagOp[i].mask)!=0;
+ }
+ rc = SQLITE_OK;
+ break;
+ }
+ }
+ break;
+ }
+ }
+ va_end(ap);
+ return rc;
+}
+
+
+/*
+** Return true if the buffer z[0..n-1] contains all spaces.
+*/
+static int allSpaces(const char *z, int n){
+ while( n>0 && z[n-1]==' ' ){ n--; }
+ return n==0;
+}
+
+/*
+** This is the default collating function named "BINARY" which is always
+** available.
+**
+** If the padFlag argument is not NULL then space padding at the end
+** of strings is ignored. This implements the RTRIM collation.
+*/
+static int binCollFunc(
+ void *padFlag,
+ int nKey1, const void *pKey1,
+ int nKey2, const void *pKey2
+){
+ int rc, n;
+ n = nKey1<nKey2 ? nKey1 : nKey2;
+ /* EVIDENCE-OF: R-65033-28449 The built-in BINARY collation compares
+ ** strings byte by byte using the memcmp() function from the standard C
+ ** library. */
+ rc = memcmp(pKey1, pKey2, n);
+ if( rc==0 ){
+ if( padFlag
+ && allSpaces(((char*)pKey1)+n, nKey1-n)
+ && allSpaces(((char*)pKey2)+n, nKey2-n)
+ ){
+ /* EVIDENCE-OF: R-31624-24737 RTRIM is like BINARY except that extra
+ ** spaces at the end of either string do not change the result. In other
+ ** words, strings will compare equal to one another as long as they
+ ** differ only in the number of spaces at the end.
+ */
+ }else{
+ rc = nKey1 - nKey2;
+ }
+ }
+ return rc;
+}
+
+/*
+** Another built-in collating sequence: NOCASE.
+**
+** This collating sequence is intended to be used for "case independent
+** comparison". SQLite's knowledge of upper and lower case equivalents
+** extends only to the 26 characters used in the English language.
+**
+** At the moment there is only a UTF-8 implementation.
+*/
+static int nocaseCollatingFunc(
+ void *NotUsed,
+ int nKey1, const void *pKey1,
+ int nKey2, const void *pKey2
+){
+ int r = sqlite3StrNICmp(
+ (const char *)pKey1, (const char *)pKey2, (nKey1<nKey2)?nKey1:nKey2);
+ UNUSED_PARAMETER(NotUsed);
+ if( 0==r ){
+ r = nKey1-nKey2;
+ }
+ return r;
+}
+
+/*
+** Return the ROWID of the most recent insert
+*/
+SQLITE_API sqlite_int64 SQLITE_STDCALL sqlite3_last_insert_rowid(sqlite3 *db){
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) ){
+ (void)SQLITE_MISUSE_BKPT;
+ return 0;
+ }
+#endif
+ return db->lastRowid;
+}
+
+/*
+** Return the number of changes in the most recent call to sqlite3_exec().
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_changes(sqlite3 *db){
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) ){
+ (void)SQLITE_MISUSE_BKPT;
+ return 0;
+ }
+#endif
+ return db->nChange;
+}
+
+/*
+** Return the number of changes since the database handle was opened.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_total_changes(sqlite3 *db){
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) ){
+ (void)SQLITE_MISUSE_BKPT;
+ return 0;
+ }
+#endif
+ return db->nTotalChange;
+}
+
+/*
+** Close all open savepoints. This function only manipulates fields of the
+** database handle object, it does not close any savepoints that may be open
+** at the b-tree/pager level.
+*/
+SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *db){
+ while( db->pSavepoint ){
+ Savepoint *pTmp = db->pSavepoint;
+ db->pSavepoint = pTmp->pNext;
+ sqlite3DbFree(db, pTmp);
+ }
+ db->nSavepoint = 0;
+ db->nStatement = 0;
+ db->isTransactionSavepoint = 0;
+}
+
+/*
+** Invoke the destructor function associated with FuncDef p, if any. Except,
+** if this is not the last copy of the function, do not invoke it. Multiple
+** copies of a single function are created when create_function() is called
+** with SQLITE_ANY as the encoding.
+*/
+static void functionDestroy(sqlite3 *db, FuncDef *p){
+ FuncDestructor *pDestructor = p->pDestructor;
+ if( pDestructor ){
+ pDestructor->nRef--;
+ if( pDestructor->nRef==0 ){
+ pDestructor->xDestroy(pDestructor->pUserData);
+ sqlite3DbFree(db, pDestructor);
+ }
+ }
+}
+
+/*
+** Disconnect all sqlite3_vtab objects that belong to database connection
+** db. This is called when db is being closed.
+*/
+static void disconnectAllVtab(sqlite3 *db){
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+ int i;
+ HashElem *p;
+ sqlite3BtreeEnterAll(db);
+ for(i=0; i<db->nDb; i++){
+ Schema *pSchema = db->aDb[i].pSchema;
+ if( db->aDb[i].pSchema ){
+ for(p=sqliteHashFirst(&pSchema->tblHash); p; p=sqliteHashNext(p)){
+ Table *pTab = (Table *)sqliteHashData(p);
+ if( IsVirtual(pTab) ) sqlite3VtabDisconnect(db, pTab);
+ }
+ }
+ }
+ for(p=sqliteHashFirst(&db->aModule); p; p=sqliteHashNext(p)){
+ Module *pMod = (Module *)sqliteHashData(p);
+ if( pMod->pEpoTab ){
+ sqlite3VtabDisconnect(db, pMod->pEpoTab);
+ }
+ }
+ sqlite3VtabUnlockList(db);
+ sqlite3BtreeLeaveAll(db);
+#else
+ UNUSED_PARAMETER(db);
+#endif
+}
+
+/*
+** Return TRUE if database connection db has unfinalized prepared
+** statements or unfinished sqlite3_backup objects.
+*/
+static int connectionIsBusy(sqlite3 *db){
+ int j;
+ assert( sqlite3_mutex_held(db->mutex) );
+ if( db->pVdbe ) return 1;
+ for(j=0; j<db->nDb; j++){
+ Btree *pBt = db->aDb[j].pBt;
+ if( pBt && sqlite3BtreeIsInBackup(pBt) ) return 1;
+ }
+ return 0;
+}
+
+/*
+** Close an existing SQLite database
+*/
+static int sqlite3Close(sqlite3 *db, int forceZombie){
+ if( !db ){
+ /* EVIDENCE-OF: R-63257-11740 Calling sqlite3_close() or
+ ** sqlite3_close_v2() with a NULL pointer argument is a harmless no-op. */
+ return SQLITE_OK;
+ }
+ if( !sqlite3SafetyCheckSickOrOk(db) ){
+ return SQLITE_MISUSE_BKPT;
+ }
+ sqlite3_mutex_enter(db->mutex);
+
+ /* Force xDisconnect calls on all virtual tables */
+ disconnectAllVtab(db);
+
+ /* If a transaction is open, the disconnectAllVtab() call above
+ ** will not have called the xDisconnect() method on any virtual
+ ** tables in the db->aVTrans[] array. The following sqlite3VtabRollback()
+ ** call will do so. We need to do this before the check for active
+ ** SQL statements below, as the v-table implementation may be storing
+ ** some prepared statements internally.
+ */
+ sqlite3VtabRollback(db);
+
+ /* Legacy behavior (sqlite3_close() behavior) is to return
+ ** SQLITE_BUSY if the connection can not be closed immediately.
+ */
+ if( !forceZombie && connectionIsBusy(db) ){
+ sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to close due to unfinalized "
+ "statements or unfinished backups");
+ sqlite3_mutex_leave(db->mutex);
+ return SQLITE_BUSY;
+ }
+
+#ifdef SQLITE_ENABLE_SQLLOG
+ if( sqlite3GlobalConfig.xSqllog ){
+ /* Closing the handle. Fourth parameter is passed the value 2. */
+ sqlite3GlobalConfig.xSqllog(sqlite3GlobalConfig.pSqllogArg, db, 0, 2);
+ }
+#endif
+
+ /* Convert the connection into a zombie and then close it.
+ */
+ db->magic = SQLITE_MAGIC_ZOMBIE;
+ sqlite3LeaveMutexAndCloseZombie(db);
+ return SQLITE_OK;
+}
+
+/*
+** Two variations on the public interface for closing a database
+** connection. The sqlite3_close() version returns SQLITE_BUSY and
+** leaves the connection option if there are unfinalized prepared
+** statements or unfinished sqlite3_backups. The sqlite3_close_v2()
+** version forces the connection to become a zombie if there are
+** unclosed resources, and arranges for deallocation when the last
+** prepare statement or sqlite3_backup closes.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_close(sqlite3 *db){ return sqlite3Close(db,0); }
+SQLITE_API int SQLITE_STDCALL sqlite3_close_v2(sqlite3 *db){ return sqlite3Close(db,1); }
+
+
+/*
+** Close the mutex on database connection db.
+**
+** Furthermore, if database connection db is a zombie (meaning that there
+** has been a prior call to sqlite3_close(db) or sqlite3_close_v2(db)) and
+** every sqlite3_stmt has now been finalized and every sqlite3_backup has
+** finished, then free all resources.
+*/
+SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){
+ HashElem *i; /* Hash table iterator */
+ int j;
+
+ /* If there are outstanding sqlite3_stmt or sqlite3_backup objects
+ ** or if the connection has not yet been closed by sqlite3_close_v2(),
+ ** then just leave the mutex and return.
+ */
+ if( db->magic!=SQLITE_MAGIC_ZOMBIE || connectionIsBusy(db) ){
+ sqlite3_mutex_leave(db->mutex);
+ return;
+ }
+
+ /* If we reach this point, it means that the database connection has
+ ** closed all sqlite3_stmt and sqlite3_backup objects and has been
+ ** passed to sqlite3_close (meaning that it is a zombie). Therefore,
+ ** go ahead and free all resources.
+ */
+
+ /* If a transaction is open, roll it back. This also ensures that if
+ ** any database schemas have been modified by an uncommitted transaction
+ ** they are reset. And that the required b-tree mutex is held to make
+ ** the pager rollback and schema reset an atomic operation. */
+ sqlite3RollbackAll(db, SQLITE_OK);
+
+ /* Free any outstanding Savepoint structures. */
+ sqlite3CloseSavepoints(db);
+
+ /* Close all database connections */
+ for(j=0; j<db->nDb; j++){
+ struct Db *pDb = &db->aDb[j];
+ if( pDb->pBt ){
+ sqlite3BtreeClose(pDb->pBt);
+ pDb->pBt = 0;
+ if( j!=1 ){
+ pDb->pSchema = 0;
+ }
+ }
+ }
+ /* Clear the TEMP schema separately and last */
+ if( db->aDb[1].pSchema ){
+ sqlite3SchemaClear(db->aDb[1].pSchema);
+ }
+ sqlite3VtabUnlockList(db);
+
+ /* Free up the array of auxiliary databases */
+ sqlite3CollapseDatabaseArray(db);
+ assert( db->nDb<=2 );
+ assert( db->aDb==db->aDbStatic );
+
+ /* Tell the code in notify.c that the connection no longer holds any
+ ** locks and does not require any further unlock-notify callbacks.
+ */
+ sqlite3ConnectionClosed(db);
+
+ for(j=0; j<ArraySize(db->aFunc.a); j++){
+ FuncDef *pNext, *pHash, *p;
+ for(p=db->aFunc.a[j]; p; p=pHash){
+ pHash = p->pHash;
+ while( p ){
+ functionDestroy(db, p);
+ pNext = p->pNext;
+ sqlite3DbFree(db, p);
+ p = pNext;
+ }
+ }
+ }
+ for(i=sqliteHashFirst(&db->aCollSeq); i; i=sqliteHashNext(i)){
+ CollSeq *pColl = (CollSeq *)sqliteHashData(i);
+ /* Invoke any destructors registered for collation sequence user data. */
+ for(j=0; j<3; j++){
+ if( pColl[j].xDel ){
+ pColl[j].xDel(pColl[j].pUser);
+ }
+ }
+ sqlite3DbFree(db, pColl);
+ }
+ sqlite3HashClear(&db->aCollSeq);
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+ for(i=sqliteHashFirst(&db->aModule); i; i=sqliteHashNext(i)){
+ Module *pMod = (Module *)sqliteHashData(i);
+ if( pMod->xDestroy ){
+ pMod->xDestroy(pMod->pAux);
+ }
+ sqlite3VtabEponymousTableClear(db, pMod);
+ sqlite3DbFree(db, pMod);
+ }
+ sqlite3HashClear(&db->aModule);
+#endif
+
+ sqlite3Error(db, SQLITE_OK); /* Deallocates any cached error strings. */
+ sqlite3ValueFree(db->pErr);
+ sqlite3CloseExtensions(db);
+#if SQLITE_USER_AUTHENTICATION
+ sqlite3_free(db->auth.zAuthUser);
+ sqlite3_free(db->auth.zAuthPW);
+#endif
+
+ db->magic = SQLITE_MAGIC_ERROR;
+
+ /* The temp-database schema is allocated differently from the other schema
+ ** objects (using sqliteMalloc() directly, instead of sqlite3BtreeSchema()).
+ ** So it needs to be freed here. Todo: Why not roll the temp schema into
+ ** the same sqliteMalloc() as the one that allocates the database
+ ** structure?
+ */
+ sqlite3DbFree(db, db->aDb[1].pSchema);
+ sqlite3_mutex_leave(db->mutex);
+ db->magic = SQLITE_MAGIC_CLOSED;
+ sqlite3_mutex_free(db->mutex);
+ assert( db->lookaside.nOut==0 ); /* Fails on a lookaside memory leak */
+ if( db->lookaside.bMalloced ){
+ sqlite3_free(db->lookaside.pStart);
+ }
+ sqlite3_free(db);
+}
+
+/*
+** Rollback all database files. If tripCode is not SQLITE_OK, then
+** any write cursors are invalidated ("tripped" - as in "tripping a circuit
+** breaker") and made to return tripCode if there are any further
+** attempts to use that cursor. Read cursors remain open and valid
+** but are "saved" in case the table pages are moved around.
+*/
+SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){
+ int i;
+ int inTrans = 0;
+ int schemaChange;
+ assert( sqlite3_mutex_held(db->mutex) );
+ sqlite3BeginBenignMalloc();
+
+ /* Obtain all b-tree mutexes before making any calls to BtreeRollback().
+ ** This is important in case the transaction being rolled back has
+ ** modified the database schema. If the b-tree mutexes are not taken
+ ** here, then another shared-cache connection might sneak in between
+ ** the database rollback and schema reset, which can cause false
+ ** corruption reports in some cases. */
+ sqlite3BtreeEnterAll(db);
+ schemaChange = (db->flags & SQLITE_InternChanges)!=0 && db->init.busy==0;
+
+ for(i=0; i<db->nDb; i++){
+ Btree *p = db->aDb[i].pBt;
+ if( p ){
+ if( sqlite3BtreeIsInTrans(p) ){
+ inTrans = 1;
+ }
+ sqlite3BtreeRollback(p, tripCode, !schemaChange);
+ }
+ }
+ sqlite3VtabRollback(db);
+ sqlite3EndBenignMalloc();
+
+ if( (db->flags&SQLITE_InternChanges)!=0 && db->init.busy==0 ){
+ sqlite3ExpirePreparedStatements(db);
+ sqlite3ResetAllSchemasOfConnection(db);
+ }
+ sqlite3BtreeLeaveAll(db);
+
+ /* Any deferred constraint violations have now been resolved. */
+ db->nDeferredCons = 0;
+ db->nDeferredImmCons = 0;
+ db->flags &= ~SQLITE_DeferFKs;
+
+ /* If one has been configured, invoke the rollback-hook callback */
+ if( db->xRollbackCallback && (inTrans || !db->autoCommit) ){
+ db->xRollbackCallback(db->pRollbackArg);
+ }
+}
+
+/*
+** Return a static string containing the name corresponding to the error code
+** specified in the argument.
+*/
+#if defined(SQLITE_NEED_ERR_NAME)
+SQLITE_PRIVATE const char *sqlite3ErrName(int rc){
+ const char *zName = 0;
+ int i, origRc = rc;
+ for(i=0; i<2 && zName==0; i++, rc &= 0xff){
+ switch( rc ){
+ case SQLITE_OK: zName = "SQLITE_OK"; break;
+ case SQLITE_ERROR: zName = "SQLITE_ERROR"; break;
+ case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break;
+ case SQLITE_PERM: zName = "SQLITE_PERM"; break;
+ case SQLITE_ABORT: zName = "SQLITE_ABORT"; break;
+ case SQLITE_ABORT_ROLLBACK: zName = "SQLITE_ABORT_ROLLBACK"; break;
+ case SQLITE_BUSY: zName = "SQLITE_BUSY"; break;
+ case SQLITE_BUSY_RECOVERY: zName = "SQLITE_BUSY_RECOVERY"; break;
+ case SQLITE_BUSY_SNAPSHOT: zName = "SQLITE_BUSY_SNAPSHOT"; break;
+ case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break;
+ case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break;
+ case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break;
+ case SQLITE_READONLY: zName = "SQLITE_READONLY"; break;
+ case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break;
+ case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break;
+ case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break;
+ case SQLITE_READONLY_DBMOVED: zName = "SQLITE_READONLY_DBMOVED"; break;
+ case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break;
+ case SQLITE_IOERR: zName = "SQLITE_IOERR"; break;
+ case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break;
+ case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break;
+ case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break;
+ case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break;
+ case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break;
+ case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break;
+ case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break;
+ case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break;
+ case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break;
+ case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break;
+ case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break;
+ case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break;
+ case SQLITE_IOERR_CHECKRESERVEDLOCK:
+ zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break;
+ case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break;
+ case SQLITE_IOERR_CLOSE: zName = "SQLITE_IOERR_CLOSE"; break;
+ case SQLITE_IOERR_DIR_CLOSE: zName = "SQLITE_IOERR_DIR_CLOSE"; break;
+ case SQLITE_IOERR_SHMOPEN: zName = "SQLITE_IOERR_SHMOPEN"; break;
+ case SQLITE_IOERR_SHMSIZE: zName = "SQLITE_IOERR_SHMSIZE"; break;
+ case SQLITE_IOERR_SHMLOCK: zName = "SQLITE_IOERR_SHMLOCK"; break;
+ case SQLITE_IOERR_SHMMAP: zName = "SQLITE_IOERR_SHMMAP"; break;
+ case SQLITE_IOERR_SEEK: zName = "SQLITE_IOERR_SEEK"; break;
+ case SQLITE_IOERR_DELETE_NOENT: zName = "SQLITE_IOERR_DELETE_NOENT";break;
+ case SQLITE_IOERR_MMAP: zName = "SQLITE_IOERR_MMAP"; break;
+ case SQLITE_IOERR_GETTEMPPATH: zName = "SQLITE_IOERR_GETTEMPPATH"; break;
+ case SQLITE_IOERR_CONVPATH: zName = "SQLITE_IOERR_CONVPATH"; break;
+ case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break;
+ case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break;
+ case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break;
+ case SQLITE_FULL: zName = "SQLITE_FULL"; break;
+ case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break;
+ case SQLITE_CANTOPEN_NOTEMPDIR: zName = "SQLITE_CANTOPEN_NOTEMPDIR";break;
+ case SQLITE_CANTOPEN_ISDIR: zName = "SQLITE_CANTOPEN_ISDIR"; break;
+ case SQLITE_CANTOPEN_FULLPATH: zName = "SQLITE_CANTOPEN_FULLPATH"; break;
+ case SQLITE_CANTOPEN_CONVPATH: zName = "SQLITE_CANTOPEN_CONVPATH"; break;
+ case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break;
+ case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break;
+ case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break;
+ case SQLITE_TOOBIG: zName = "SQLITE_TOOBIG"; break;
+ case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break;
+ case SQLITE_CONSTRAINT_UNIQUE: zName = "SQLITE_CONSTRAINT_UNIQUE"; break;
+ case SQLITE_CONSTRAINT_TRIGGER: zName = "SQLITE_CONSTRAINT_TRIGGER";break;
+ case SQLITE_CONSTRAINT_FOREIGNKEY:
+ zName = "SQLITE_CONSTRAINT_FOREIGNKEY"; break;
+ case SQLITE_CONSTRAINT_CHECK: zName = "SQLITE_CONSTRAINT_CHECK"; break;
+ case SQLITE_CONSTRAINT_PRIMARYKEY:
+ zName = "SQLITE_CONSTRAINT_PRIMARYKEY"; break;
+ case SQLITE_CONSTRAINT_NOTNULL: zName = "SQLITE_CONSTRAINT_NOTNULL";break;
+ case SQLITE_CONSTRAINT_COMMITHOOK:
+ zName = "SQLITE_CONSTRAINT_COMMITHOOK"; break;
+ case SQLITE_CONSTRAINT_VTAB: zName = "SQLITE_CONSTRAINT_VTAB"; break;
+ case SQLITE_CONSTRAINT_FUNCTION:
+ zName = "SQLITE_CONSTRAINT_FUNCTION"; break;
+ case SQLITE_CONSTRAINT_ROWID: zName = "SQLITE_CONSTRAINT_ROWID"; break;
+ case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break;
+ case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break;
+ case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break;
+ case SQLITE_AUTH: zName = "SQLITE_AUTH"; break;
+ case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break;
+ case SQLITE_RANGE: zName = "SQLITE_RANGE"; break;
+ case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break;
+ case SQLITE_ROW: zName = "SQLITE_ROW"; break;
+ case SQLITE_NOTICE: zName = "SQLITE_NOTICE"; break;
+ case SQLITE_NOTICE_RECOVER_WAL: zName = "SQLITE_NOTICE_RECOVER_WAL";break;
+ case SQLITE_NOTICE_RECOVER_ROLLBACK:
+ zName = "SQLITE_NOTICE_RECOVER_ROLLBACK"; break;
+ case SQLITE_WARNING: zName = "SQLITE_WARNING"; break;
+ case SQLITE_WARNING_AUTOINDEX: zName = "SQLITE_WARNING_AUTOINDEX"; break;
+ case SQLITE_DONE: zName = "SQLITE_DONE"; break;
+ }
+ }
+ if( zName==0 ){
+ static char zBuf[50];
+ sqlite3_snprintf(sizeof(zBuf), zBuf, "SQLITE_UNKNOWN(%d)", origRc);
+ zName = zBuf;
+ }
+ return zName;
+}
+#endif
+
+/*
+** Return a static string that describes the kind of error specified in the
+** argument.
+*/
+SQLITE_PRIVATE const char *sqlite3ErrStr(int rc){
+ static const char* const aMsg[] = {
+ /* SQLITE_OK */ "not an error",
+ /* SQLITE_ERROR */ "SQL logic error or missing database",
+ /* SQLITE_INTERNAL */ 0,
+ /* SQLITE_PERM */ "access permission denied",
+ /* SQLITE_ABORT */ "callback requested query abort",
+ /* SQLITE_BUSY */ "database is locked",
+ /* SQLITE_LOCKED */ "database table is locked",
+ /* SQLITE_NOMEM */ "out of memory",
+ /* SQLITE_READONLY */ "attempt to write a readonly database",
+ /* SQLITE_INTERRUPT */ "interrupted",
+ /* SQLITE_IOERR */ "disk I/O error",
+ /* SQLITE_CORRUPT */ "database disk image is malformed",
+ /* SQLITE_NOTFOUND */ "unknown operation",
+ /* SQLITE_FULL */ "database or disk is full",
+ /* SQLITE_CANTOPEN */ "unable to open database file",
+ /* SQLITE_PROTOCOL */ "locking protocol",
+ /* SQLITE_EMPTY */ "table contains no data",
+ /* SQLITE_SCHEMA */ "database schema has changed",
+ /* SQLITE_TOOBIG */ "string or blob too big",
+ /* SQLITE_CONSTRAINT */ "constraint failed",
+ /* SQLITE_MISMATCH */ "datatype mismatch",
+ /* SQLITE_MISUSE */ "library routine called out of sequence",
+ /* SQLITE_NOLFS */ "large file support is disabled",
+ /* SQLITE_AUTH */ "authorization denied",
+ /* SQLITE_FORMAT */ "auxiliary database format error",
+ /* SQLITE_RANGE */ "bind or column index out of range",
+ /* SQLITE_NOTADB */ "file is encrypted or is not a database",
+ };
+ const char *zErr = "unknown error";
+ switch( rc ){
+ case SQLITE_ABORT_ROLLBACK: {
+ zErr = "abort due to ROLLBACK";
+ break;
+ }
+ default: {
+ rc &= 0xff;
+ if( ALWAYS(rc>=0) && rc<ArraySize(aMsg) && aMsg[rc]!=0 ){
+ zErr = aMsg[rc];
+ }
+ break;
+ }
+ }
+ return zErr;
+}
+
+/*
+** This routine implements a busy callback that sleeps and tries
+** again until a timeout value is reached. The timeout value is
+** an integer number of milliseconds passed in as the first
+** argument.
+*/
+static int sqliteDefaultBusyCallback(
+ void *ptr, /* Database connection */
+ int count /* Number of times table has been busy */
+){
+#if SQLITE_OS_WIN || HAVE_USLEEP
+ static const u8 delays[] =
+ { 1, 2, 5, 10, 15, 20, 25, 25, 25, 50, 50, 100 };
+ static const u8 totals[] =
+ { 0, 1, 3, 8, 18, 33, 53, 78, 103, 128, 178, 228 };
+# define NDELAY ArraySize(delays)
+ sqlite3 *db = (sqlite3 *)ptr;
+ int timeout = db->busyTimeout;
+ int delay, prior;
+
+ assert( count>=0 );
+ if( count < NDELAY ){
+ delay = delays[count];
+ prior = totals[count];
+ }else{
+ delay = delays[NDELAY-1];
+ prior = totals[NDELAY-1] + delay*(count-(NDELAY-1));
+ }
+ if( prior + delay > timeout ){
+ delay = timeout - prior;
+ if( delay<=0 ) return 0;
+ }
+ sqlite3OsSleep(db->pVfs, delay*1000);
+ return 1;
+#else
+ sqlite3 *db = (sqlite3 *)ptr;
+ int timeout = ((sqlite3 *)ptr)->busyTimeout;
+ if( (count+1)*1000 > timeout ){
+ return 0;
+ }
+ sqlite3OsSleep(db->pVfs, 1000000);
+ return 1;
+#endif
+}
+
+/*
+** Invoke the given busy handler.
+**
+** This routine is called when an operation failed with a lock.
+** If this routine returns non-zero, the lock is retried. If it
+** returns 0, the operation aborts with an SQLITE_BUSY error.
+*/
+SQLITE_PRIVATE int sqlite3InvokeBusyHandler(BusyHandler *p){
+ int rc;
+ if( NEVER(p==0) || p->xFunc==0 || p->nBusy<0 ) return 0;
+ rc = p->xFunc(p->pArg, p->nBusy);
+ if( rc==0 ){
+ p->nBusy = -1;
+ }else{
+ p->nBusy++;
+ }
+ return rc;
+}
+
+/*
+** This routine sets the busy callback for an Sqlite database to the
+** given callback function with the given argument.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_busy_handler(
+ sqlite3 *db,
+ int (*xBusy)(void*,int),
+ void *pArg
+){
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
+#endif
+ sqlite3_mutex_enter(db->mutex);
+ db->busyHandler.xFunc = xBusy;
+ db->busyHandler.pArg = pArg;
+ db->busyHandler.nBusy = 0;
+ db->busyTimeout = 0;
+ sqlite3_mutex_leave(db->mutex);
+ return SQLITE_OK;
+}
+
+#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
+/*
+** This routine sets the progress callback for an Sqlite database to the
+** given callback function with the given argument. The progress callback will
+** be invoked every nOps opcodes.
+*/
+SQLITE_API void SQLITE_STDCALL sqlite3_progress_handler(
+ sqlite3 *db,
+ int nOps,
+ int (*xProgress)(void*),
+ void *pArg
+){
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) ){
+ (void)SQLITE_MISUSE_BKPT;
+ return;
+ }
+#endif
+ sqlite3_mutex_enter(db->mutex);
+ if( nOps>0 ){
+ db->xProgress = xProgress;
+ db->nProgressOps = (unsigned)nOps;
+ db->pProgressArg = pArg;
+ }else{
+ db->xProgress = 0;
+ db->nProgressOps = 0;
+ db->pProgressArg = 0;
+ }
+ sqlite3_mutex_leave(db->mutex);
+}
+#endif
+
+
+/*
+** This routine installs a default busy handler that waits for the
+** specified number of milliseconds before returning 0.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_busy_timeout(sqlite3 *db, int ms){
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
+#endif
+ if( ms>0 ){
+ sqlite3_busy_handler(db, sqliteDefaultBusyCallback, (void*)db);
+ db->busyTimeout = ms;
+ }else{
+ sqlite3_busy_handler(db, 0, 0);
+ }
+ return SQLITE_OK;
+}
+
+/*
+** Cause any pending operation to stop at its earliest opportunity.
+*/
+SQLITE_API void SQLITE_STDCALL sqlite3_interrupt(sqlite3 *db){
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) ){
+ (void)SQLITE_MISUSE_BKPT;
+ return;
+ }
+#endif
+ db->u1.isInterrupted = 1;
+}
+
+
+/*
+** This function is exactly the same as sqlite3_create_function(), except
+** that it is designed to be called by internal code. The difference is
+** that if a malloc() fails in sqlite3_create_function(), an error code
+** is returned and the mallocFailed flag cleared.
+*/
+SQLITE_PRIVATE int sqlite3CreateFunc(
+ sqlite3 *db,
+ const char *zFunctionName,
+ int nArg,
+ int enc,
+ void *pUserData,
+ void (*xFunc)(sqlite3_context*,int,sqlite3_value **),
+ void (*xStep)(sqlite3_context*,int,sqlite3_value **),
+ void (*xFinal)(sqlite3_context*),
+ FuncDestructor *pDestructor
+){
+ FuncDef *p;
+ int nName;
+ int extraFlags;
+
+ assert( sqlite3_mutex_held(db->mutex) );
+ if( zFunctionName==0 ||
+ (xFunc && (xFinal || xStep)) ||
+ (!xFunc && (xFinal && !xStep)) ||
+ (!xFunc && (!xFinal && xStep)) ||
+ (nArg<-1 || nArg>SQLITE_MAX_FUNCTION_ARG) ||
+ (255<(nName = sqlite3Strlen30( zFunctionName))) ){
+ return SQLITE_MISUSE_BKPT;
+ }
+
+ assert( SQLITE_FUNC_CONSTANT==SQLITE_DETERMINISTIC );
+ extraFlags = enc & SQLITE_DETERMINISTIC;
+ enc &= (SQLITE_FUNC_ENCMASK|SQLITE_ANY);
+
+#ifndef SQLITE_OMIT_UTF16
+ /* If SQLITE_UTF16 is specified as the encoding type, transform this
+ ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the
+ ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally.
+ **
+ ** If SQLITE_ANY is specified, add three versions of the function
+ ** to the hash table.
+ */
+ if( enc==SQLITE_UTF16 ){
+ enc = SQLITE_UTF16NATIVE;
+ }else if( enc==SQLITE_ANY ){
+ int rc;
+ rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF8|extraFlags,
+ pUserData, xFunc, xStep, xFinal, pDestructor);
+ if( rc==SQLITE_OK ){
+ rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF16LE|extraFlags,
+ pUserData, xFunc, xStep, xFinal, pDestructor);
+ }
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+ enc = SQLITE_UTF16BE;
+ }
+#else
+ enc = SQLITE_UTF8;
+#endif
+
+ /* Check if an existing function is being overridden or deleted. If so,
+ ** and there are active VMs, then return SQLITE_BUSY. If a function
+ ** is being overridden/deleted but there are no active VMs, allow the
+ ** operation to continue but invalidate all precompiled statements.
+ */
+ p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 0);
+ if( p && (p->funcFlags & SQLITE_FUNC_ENCMASK)==enc && p->nArg==nArg ){
+ if( db->nVdbeActive ){
+ sqlite3ErrorWithMsg(db, SQLITE_BUSY,
+ "unable to delete/modify user-function due to active statements");
+ assert( !db->mallocFailed );
+ return SQLITE_BUSY;
+ }else{
+ sqlite3ExpirePreparedStatements(db);
+ }
+ }
+
+ p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 1);
+ assert(p || db->mallocFailed);
+ if( !p ){
+ return SQLITE_NOMEM;
+ }
+
+ /* If an older version of the function with a configured destructor is
+ ** being replaced invoke the destructor function here. */
+ functionDestroy(db, p);
+
+ if( pDestructor ){
+ pDestructor->nRef++;
+ }
+ p->pDestructor = pDestructor;
+ p->funcFlags = (p->funcFlags & SQLITE_FUNC_ENCMASK) | extraFlags;
+ testcase( p->funcFlags & SQLITE_DETERMINISTIC );
+ p->xFunc = xFunc;
+ p->xStep = xStep;
+ p->xFinalize = xFinal;
+ p->pUserData = pUserData;
+ p->nArg = (u16)nArg;
+ return SQLITE_OK;
+}
+
+/*
+** Create new user functions.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_create_function(
+ sqlite3 *db,
+ const char *zFunc,
+ int nArg,
+ int enc,
+ void *p,
+ void (*xFunc)(sqlite3_context*,int,sqlite3_value **),
+ void (*xStep)(sqlite3_context*,int,sqlite3_value **),
+ void (*xFinal)(sqlite3_context*)
+){
+ return sqlite3_create_function_v2(db, zFunc, nArg, enc, p, xFunc, xStep,
+ xFinal, 0);
+}
+
+SQLITE_API int SQLITE_STDCALL sqlite3_create_function_v2(
+ sqlite3 *db,
+ const char *zFunc,
+ int nArg,
+ int enc,
+ void *p,
+ void (*xFunc)(sqlite3_context*,int,sqlite3_value **),
+ void (*xStep)(sqlite3_context*,int,sqlite3_value **),
+ void (*xFinal)(sqlite3_context*),
+ void (*xDestroy)(void *)
+){
+ int rc = SQLITE_ERROR;
+ FuncDestructor *pArg = 0;
+
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) ){
+ return SQLITE_MISUSE_BKPT;
+ }
+#endif
+ sqlite3_mutex_enter(db->mutex);
+ if( xDestroy ){
+ pArg = (FuncDestructor *)sqlite3DbMallocZero(db, sizeof(FuncDestructor));
+ if( !pArg ){
+ xDestroy(p);
+ goto out;
+ }
+ pArg->xDestroy = xDestroy;
+ pArg->pUserData = p;
+ }
+ rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, xFunc, xStep, xFinal, pArg);
+ if( pArg && pArg->nRef==0 ){
+ assert( rc!=SQLITE_OK );
+ xDestroy(p);
+ sqlite3DbFree(db, pArg);
+ }
+
+ out:
+ rc = sqlite3ApiExit(db, rc);
+ sqlite3_mutex_leave(db->mutex);
+ return rc;
+}
+
+#ifndef SQLITE_OMIT_UTF16
+SQLITE_API int SQLITE_STDCALL sqlite3_create_function16(
+ sqlite3 *db,
+ const void *zFunctionName,
+ int nArg,
+ int eTextRep,
+ void *p,
+ void (*xFunc)(sqlite3_context*,int,sqlite3_value**),
+ void (*xStep)(sqlite3_context*,int,sqlite3_value**),
+ void (*xFinal)(sqlite3_context*)
+){
+ int rc;
+ char *zFunc8;
+
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) || zFunctionName==0 ) return SQLITE_MISUSE_BKPT;
+#endif
+ sqlite3_mutex_enter(db->mutex);
+ assert( !db->mallocFailed );
+ zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE);
+ rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xFunc, xStep, xFinal,0);
+ sqlite3DbFree(db, zFunc8);
+ rc = sqlite3ApiExit(db, rc);
+ sqlite3_mutex_leave(db->mutex);
+ return rc;
+}
+#endif
+
+
+/*
+** Declare that a function has been overloaded by a virtual table.
+**
+** If the function already exists as a regular global function, then
+** this routine is a no-op. If the function does not exist, then create
+** a new one that always throws a run-time error.
+**
+** When virtual tables intend to provide an overloaded function, they
+** should call this routine to make sure the global function exists.
+** A global function must exist in order for name resolution to work
+** properly.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_overload_function(
+ sqlite3 *db,
+ const char *zName,
+ int nArg
+){
+ int nName = sqlite3Strlen30(zName);
+ int rc = SQLITE_OK;
+
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) || zName==0 || nArg<-2 ){
+ return SQLITE_MISUSE_BKPT;
+ }
+#endif
+ sqlite3_mutex_enter(db->mutex);
+ if( sqlite3FindFunction(db, zName, nName, nArg, SQLITE_UTF8, 0)==0 ){
+ rc = sqlite3CreateFunc(db, zName, nArg, SQLITE_UTF8,
+ 0, sqlite3InvalidFunction, 0, 0, 0);
+ }
+ rc = sqlite3ApiExit(db, rc);
+ sqlite3_mutex_leave(db->mutex);
+ return rc;
+}
+
+#ifndef SQLITE_OMIT_TRACE
+/*
+** Register a trace function. The pArg from the previously registered trace
+** is returned.
+**
+** A NULL trace function means that no tracing is executes. A non-NULL
+** trace is a pointer to a function that is invoked at the start of each
+** SQL statement.
+*/
+SQLITE_API void *SQLITE_STDCALL sqlite3_trace(sqlite3 *db, void (*xTrace)(void*,const char*), void *pArg){
+ void *pOld;
+
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) ){
+ (void)SQLITE_MISUSE_BKPT;
+ return 0;
+ }
+#endif
+ sqlite3_mutex_enter(db->mutex);
+ pOld = db->pTraceArg;
+ db->xTrace = xTrace;
+ db->pTraceArg = pArg;
+ sqlite3_mutex_leave(db->mutex);
+ return pOld;
+}
+/*
+** Register a profile function. The pArg from the previously registered
+** profile function is returned.
+**
+** A NULL profile function means that no profiling is executes. A non-NULL
+** profile is a pointer to a function that is invoked at the conclusion of
+** each SQL statement that is run.
+*/
+SQLITE_API void *SQLITE_STDCALL sqlite3_profile(
+ sqlite3 *db,
+ void (*xProfile)(void*,const char*,sqlite_uint64),
+ void *pArg
+){
+ void *pOld;
+
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) ){
+ (void)SQLITE_MISUSE_BKPT;
+ return 0;
+ }
+#endif
+ sqlite3_mutex_enter(db->mutex);
+ pOld = db->pProfileArg;
+ db->xProfile = xProfile;
+ db->pProfileArg = pArg;
+ sqlite3_mutex_leave(db->mutex);
+ return pOld;
+}
+#endif /* SQLITE_OMIT_TRACE */
+
+/*
+** Register a function to be invoked when a transaction commits.
+** If the invoked function returns non-zero, then the commit becomes a
+** rollback.
+*/
+SQLITE_API void *SQLITE_STDCALL sqlite3_commit_hook(
+ sqlite3 *db, /* Attach the hook to this database */
+ int (*xCallback)(void*), /* Function to invoke on each commit */
+ void *pArg /* Argument to the function */
+){
+ void *pOld;
+
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) ){
+ (void)SQLITE_MISUSE_BKPT;
+ return 0;
+ }
+#endif
+ sqlite3_mutex_enter(db->mutex);
+ pOld = db->pCommitArg;
+ db->xCommitCallback = xCallback;
+ db->pCommitArg = pArg;
+ sqlite3_mutex_leave(db->mutex);
+ return pOld;
+}
+
+/*
+** Register a callback to be invoked each time a row is updated,
+** inserted or deleted using this database connection.
+*/
+SQLITE_API void *SQLITE_STDCALL sqlite3_update_hook(
+ sqlite3 *db, /* Attach the hook to this database */
+ void (*xCallback)(void*,int,char const *,char const *,sqlite_int64),
+ void *pArg /* Argument to the function */
+){
+ void *pRet;
+
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) ){
+ (void)SQLITE_MISUSE_BKPT;
+ return 0;
+ }
+#endif
+ sqlite3_mutex_enter(db->mutex);
+ pRet = db->pUpdateArg;
+ db->xUpdateCallback = xCallback;
+ db->pUpdateArg = pArg;
+ sqlite3_mutex_leave(db->mutex);
+ return pRet;
+}
+
+/*
+** Register a callback to be invoked each time a transaction is rolled
+** back by this database connection.
+*/
+SQLITE_API void *SQLITE_STDCALL sqlite3_rollback_hook(
+ sqlite3 *db, /* Attach the hook to this database */
+ void (*xCallback)(void*), /* Callback function */
+ void *pArg /* Argument to the function */
+){
+ void *pRet;
+
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) ){
+ (void)SQLITE_MISUSE_BKPT;
+ return 0;
+ }
+#endif
+ sqlite3_mutex_enter(db->mutex);
+ pRet = db->pRollbackArg;
+ db->xRollbackCallback = xCallback;
+ db->pRollbackArg = pArg;
+ sqlite3_mutex_leave(db->mutex);
+ return pRet;
+}
+
+#ifndef SQLITE_OMIT_WAL
+/*
+** The sqlite3_wal_hook() callback registered by sqlite3_wal_autocheckpoint().
+** Invoke sqlite3_wal_checkpoint if the number of frames in the log file
+** is greater than sqlite3.pWalArg cast to an integer (the value configured by
+** wal_autocheckpoint()).
+*/
+SQLITE_PRIVATE int sqlite3WalDefaultHook(
+ void *pClientData, /* Argument */
+ sqlite3 *db, /* Connection */
+ const char *zDb, /* Database */
+ int nFrame /* Size of WAL */
+){
+ if( nFrame>=SQLITE_PTR_TO_INT(pClientData) ){
+ sqlite3BeginBenignMalloc();
+ sqlite3_wal_checkpoint(db, zDb);
+ sqlite3EndBenignMalloc();
+ }
+ return SQLITE_OK;
+}
+#endif /* SQLITE_OMIT_WAL */
+
+/*
+** Configure an sqlite3_wal_hook() callback to automatically checkpoint
+** a database after committing a transaction if there are nFrame or
+** more frames in the log file. Passing zero or a negative value as the
+** nFrame parameter disables automatic checkpoints entirely.
+**
+** The callback registered by this function replaces any existing callback
+** registered using sqlite3_wal_hook(). Likewise, registering a callback
+** using sqlite3_wal_hook() disables the automatic checkpoint mechanism
+** configured by this function.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){
+#ifdef SQLITE_OMIT_WAL
+ UNUSED_PARAMETER(db);
+ UNUSED_PARAMETER(nFrame);
+#else
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
+#endif
+ if( nFrame>0 ){
+ sqlite3_wal_hook(db, sqlite3WalDefaultHook, SQLITE_INT_TO_PTR(nFrame));
+ }else{
+ sqlite3_wal_hook(db, 0, 0);
+ }
+#endif
+ return SQLITE_OK;
+}
+
+/*
+** Register a callback to be invoked each time a transaction is written
+** into the write-ahead-log by this database connection.
+*/
+SQLITE_API void *SQLITE_STDCALL sqlite3_wal_hook(
+ sqlite3 *db, /* Attach the hook to this db handle */
+ int(*xCallback)(void *, sqlite3*, const char*, int),
+ void *pArg /* First argument passed to xCallback() */
+){
+#ifndef SQLITE_OMIT_WAL
+ void *pRet;
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) ){
+ (void)SQLITE_MISUSE_BKPT;
+ return 0;
+ }
+#endif
+ sqlite3_mutex_enter(db->mutex);
+ pRet = db->pWalArg;
+ db->xWalCallback = xCallback;
+ db->pWalArg = pArg;
+ sqlite3_mutex_leave(db->mutex);
+ return pRet;
+#else
+ return 0;
+#endif
+}
+
+/*
+** Checkpoint database zDb.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint_v2(
+ sqlite3 *db, /* Database handle */
+ const char *zDb, /* Name of attached database (or NULL) */
+ int eMode, /* SQLITE_CHECKPOINT_* value */
+ int *pnLog, /* OUT: Size of WAL log in frames */
+ int *pnCkpt /* OUT: Total number of frames checkpointed */
+){
+#ifdef SQLITE_OMIT_WAL
+ return SQLITE_OK;
+#else
+ int rc; /* Return code */
+ int iDb = SQLITE_MAX_ATTACHED; /* sqlite3.aDb[] index of db to checkpoint */
+
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
+#endif
+
+ /* Initialize the output variables to -1 in case an error occurs. */
+ if( pnLog ) *pnLog = -1;
+ if( pnCkpt ) *pnCkpt = -1;
+
+ assert( SQLITE_CHECKPOINT_PASSIVE==0 );
+ assert( SQLITE_CHECKPOINT_FULL==1 );
+ assert( SQLITE_CHECKPOINT_RESTART==2 );
+ assert( SQLITE_CHECKPOINT_TRUNCATE==3 );
+ if( eMode<SQLITE_CHECKPOINT_PASSIVE || eMode>SQLITE_CHECKPOINT_TRUNCATE ){
+ /* EVIDENCE-OF: R-03996-12088 The M parameter must be a valid checkpoint
+ ** mode: */
+ return SQLITE_MISUSE;
+ }
+
+ sqlite3_mutex_enter(db->mutex);
+ if( zDb && zDb[0] ){
+ iDb = sqlite3FindDbName(db, zDb);
+ }
+ if( iDb<0 ){
+ rc = SQLITE_ERROR;
+ sqlite3ErrorWithMsg(db, SQLITE_ERROR, "unknown database: %s", zDb);
+ }else{
+ db->busyHandler.nBusy = 0;
+ rc = sqlite3Checkpoint(db, iDb, eMode, pnLog, pnCkpt);
+ sqlite3Error(db, rc);
+ }
+ rc = sqlite3ApiExit(db, rc);
+ sqlite3_mutex_leave(db->mutex);
+ return rc;
+#endif
+}
+
+
+/*
+** Checkpoint database zDb. If zDb is NULL, or if the buffer zDb points
+** to contains a zero-length string, all attached databases are
+** checkpointed.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){
+ /* EVIDENCE-OF: R-41613-20553 The sqlite3_wal_checkpoint(D,X) is equivalent to
+ ** sqlite3_wal_checkpoint_v2(D,X,SQLITE_CHECKPOINT_PASSIVE,0,0). */
+ return sqlite3_wal_checkpoint_v2(db,zDb,SQLITE_CHECKPOINT_PASSIVE,0,0);
+}
+
+#ifndef SQLITE_OMIT_WAL
+/*
+** Run a checkpoint on database iDb. This is a no-op if database iDb is
+** not currently open in WAL mode.
+**
+** If a transaction is open on the database being checkpointed, this
+** function returns SQLITE_LOCKED and a checkpoint is not attempted. If
+** an error occurs while running the checkpoint, an SQLite error code is
+** returned (i.e. SQLITE_IOERR). Otherwise, SQLITE_OK.
+**
+** The mutex on database handle db should be held by the caller. The mutex
+** associated with the specific b-tree being checkpointed is taken by
+** this function while the checkpoint is running.
+**
+** If iDb is passed SQLITE_MAX_ATTACHED, then all attached databases are
+** checkpointed. If an error is encountered it is returned immediately -
+** no attempt is made to checkpoint any remaining databases.
+**
+** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART.
+*/
+SQLITE_PRIVATE int sqlite3Checkpoint(sqlite3 *db, int iDb, int eMode, int *pnLog, int *pnCkpt){
+ int rc = SQLITE_OK; /* Return code */
+ int i; /* Used to iterate through attached dbs */
+ int bBusy = 0; /* True if SQLITE_BUSY has been encountered */
+
+ assert( sqlite3_mutex_held(db->mutex) );
+ assert( !pnLog || *pnLog==-1 );
+ assert( !pnCkpt || *pnCkpt==-1 );
+
+ for(i=0; i<db->nDb && rc==SQLITE_OK; i++){
+ if( i==iDb || iDb==SQLITE_MAX_ATTACHED ){
+ rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt, eMode, pnLog, pnCkpt);
+ pnLog = 0;
+ pnCkpt = 0;
+ if( rc==SQLITE_BUSY ){
+ bBusy = 1;
+ rc = SQLITE_OK;
+ }
+ }
+ }
+
+ return (rc==SQLITE_OK && bBusy) ? SQLITE_BUSY : rc;
+}
+#endif /* SQLITE_OMIT_WAL */
+
+/*
+** This function returns true if main-memory should be used instead of
+** a temporary file for transient pager files and statement journals.
+** The value returned depends on the value of db->temp_store (runtime
+** parameter) and the compile time value of SQLITE_TEMP_STORE. The
+** following table describes the relationship between these two values
+** and this functions return value.
+**
+** SQLITE_TEMP_STORE db->temp_store Location of temporary database
+** ----------------- -------------- ------------------------------
+** 0 any file (return 0)
+** 1 1 file (return 0)
+** 1 2 memory (return 1)
+** 1 0 file (return 0)
+** 2 1 file (return 0)
+** 2 2 memory (return 1)
+** 2 0 memory (return 1)
+** 3 any memory (return 1)
+*/
+SQLITE_PRIVATE int sqlite3TempInMemory(const sqlite3 *db){
+#if SQLITE_TEMP_STORE==1
+ return ( db->temp_store==2 );
+#endif
+#if SQLITE_TEMP_STORE==2
+ return ( db->temp_store!=1 );
+#endif
+#if SQLITE_TEMP_STORE==3
+ UNUSED_PARAMETER(db);
+ return 1;
+#endif
+#if SQLITE_TEMP_STORE<1 || SQLITE_TEMP_STORE>3
+ UNUSED_PARAMETER(db);
+ return 0;
+#endif
+}
+
+/*
+** Return UTF-8 encoded English language explanation of the most recent
+** error.
+*/
+SQLITE_API const char *SQLITE_STDCALL sqlite3_errmsg(sqlite3 *db){
+ const char *z;
+ if( !db ){
+ return sqlite3ErrStr(SQLITE_NOMEM);
+ }
+ if( !sqlite3SafetyCheckSickOrOk(db) ){
+ return sqlite3ErrStr(SQLITE_MISUSE_BKPT);
+ }
+ sqlite3_mutex_enter(db->mutex);
+ if( db->mallocFailed ){
+ z = sqlite3ErrStr(SQLITE_NOMEM);
+ }else{
+ testcase( db->pErr==0 );
+ z = (char*)sqlite3_value_text(db->pErr);
+ assert( !db->mallocFailed );
+ if( z==0 ){
+ z = sqlite3ErrStr(db->errCode);
+ }
+ }
+ sqlite3_mutex_leave(db->mutex);
+ return z;
+}
+
+#ifndef SQLITE_OMIT_UTF16
+/*
+** Return UTF-16 encoded English language explanation of the most recent
+** error.
+*/
+SQLITE_API const void *SQLITE_STDCALL sqlite3_errmsg16(sqlite3 *db){
+ static const u16 outOfMem[] = {
+ 'o', 'u', 't', ' ', 'o', 'f', ' ', 'm', 'e', 'm', 'o', 'r', 'y', 0
+ };
+ static const u16 misuse[] = {
+ 'l', 'i', 'b', 'r', 'a', 'r', 'y', ' ',
+ 'r', 'o', 'u', 't', 'i', 'n', 'e', ' ',
+ 'c', 'a', 'l', 'l', 'e', 'd', ' ',
+ 'o', 'u', 't', ' ',
+ 'o', 'f', ' ',
+ 's', 'e', 'q', 'u', 'e', 'n', 'c', 'e', 0
+ };
+
+ const void *z;
+ if( !db ){
+ return (void *)outOfMem;
+ }
+ if( !sqlite3SafetyCheckSickOrOk(db) ){
+ return (void *)misuse;
+ }
+ sqlite3_mutex_enter(db->mutex);
+ if( db->mallocFailed ){
+ z = (void *)outOfMem;
+ }else{
+ z = sqlite3_value_text16(db->pErr);
+ if( z==0 ){
+ sqlite3ErrorWithMsg(db, db->errCode, sqlite3ErrStr(db->errCode));
+ z = sqlite3_value_text16(db->pErr);
+ }
+ /* A malloc() may have failed within the call to sqlite3_value_text16()
+ ** above. If this is the case, then the db->mallocFailed flag needs to
+ ** be cleared before returning. Do this directly, instead of via
+ ** sqlite3ApiExit(), to avoid setting the database handle error message.
+ */
+ db->mallocFailed = 0;
+ }
+ sqlite3_mutex_leave(db->mutex);
+ return z;
+}
+#endif /* SQLITE_OMIT_UTF16 */
+
+/*
+** Return the most recent error code generated by an SQLite routine. If NULL is
+** passed to this function, we assume a malloc() failed during sqlite3_open().
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_errcode(sqlite3 *db){
+ if( db && !sqlite3SafetyCheckSickOrOk(db) ){
+ return SQLITE_MISUSE_BKPT;
+ }
+ if( !db || db->mallocFailed ){
+ return SQLITE_NOMEM;
+ }
+ return db->errCode & db->errMask;
+}
+SQLITE_API int SQLITE_STDCALL sqlite3_extended_errcode(sqlite3 *db){
+ if( db && !sqlite3SafetyCheckSickOrOk(db) ){
+ return SQLITE_MISUSE_BKPT;
+ }
+ if( !db || db->mallocFailed ){
+ return SQLITE_NOMEM;
+ }
+ return db->errCode;
+}
+
+/*
+** Return a string that describes the kind of error specified in the
+** argument. For now, this simply calls the internal sqlite3ErrStr()
+** function.
+*/
+SQLITE_API const char *SQLITE_STDCALL sqlite3_errstr(int rc){
+ return sqlite3ErrStr(rc);
+}
+
+/*
+** Create a new collating function for database "db". The name is zName
+** and the encoding is enc.
+*/
+static int createCollation(
+ sqlite3* db,
+ const char *zName,
+ u8 enc,
+ void* pCtx,
+ int(*xCompare)(void*,int,const void*,int,const void*),
+ void(*xDel)(void*)
+){
+ CollSeq *pColl;
+ int enc2;
+
+ assert( sqlite3_mutex_held(db->mutex) );
+
+ /* If SQLITE_UTF16 is specified as the encoding type, transform this
+ ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the
+ ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally.
+ */
+ enc2 = enc;
+ testcase( enc2==SQLITE_UTF16 );
+ testcase( enc2==SQLITE_UTF16_ALIGNED );
+ if( enc2==SQLITE_UTF16 || enc2==SQLITE_UTF16_ALIGNED ){
+ enc2 = SQLITE_UTF16NATIVE;
+ }
+ if( enc2<SQLITE_UTF8 || enc2>SQLITE_UTF16BE ){
+ return SQLITE_MISUSE_BKPT;
+ }
+
+ /* Check if this call is removing or replacing an existing collation
+ ** sequence. If so, and there are active VMs, return busy. If there
+ ** are no active VMs, invalidate any pre-compiled statements.
+ */
+ pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 0);
+ if( pColl && pColl->xCmp ){
+ if( db->nVdbeActive ){
+ sqlite3ErrorWithMsg(db, SQLITE_BUSY,
+ "unable to delete/modify collation sequence due to active statements");
+ return SQLITE_BUSY;
+ }
+ sqlite3ExpirePreparedStatements(db);
+
+ /* If collation sequence pColl was created directly by a call to
+ ** sqlite3_create_collation, and not generated by synthCollSeq(),
+ ** then any copies made by synthCollSeq() need to be invalidated.
+ ** Also, collation destructor - CollSeq.xDel() - function may need
+ ** to be called.
+ */
+ if( (pColl->enc & ~SQLITE_UTF16_ALIGNED)==enc2 ){
+ CollSeq *aColl = sqlite3HashFind(&db->aCollSeq, zName);
+ int j;
+ for(j=0; j<3; j++){
+ CollSeq *p = &aColl[j];
+ if( p->enc==pColl->enc ){
+ if( p->xDel ){
+ p->xDel(p->pUser);
+ }
+ p->xCmp = 0;
+ }
+ }
+ }
+ }
+
+ pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 1);
+ if( pColl==0 ) return SQLITE_NOMEM;
+ pColl->xCmp = xCompare;
+ pColl->pUser = pCtx;
+ pColl->xDel = xDel;
+ pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED));
+ sqlite3Error(db, SQLITE_OK);
+ return SQLITE_OK;
+}
+
+
+/*
+** This array defines hard upper bounds on limit values. The
+** initializer must be kept in sync with the SQLITE_LIMIT_*
+** #defines in sqlite3.h.
+*/
+static const int aHardLimit[] = {
+ SQLITE_MAX_LENGTH,
+ SQLITE_MAX_SQL_LENGTH,
+ SQLITE_MAX_COLUMN,
+ SQLITE_MAX_EXPR_DEPTH,
+ SQLITE_MAX_COMPOUND_SELECT,
+ SQLITE_MAX_VDBE_OP,
+ SQLITE_MAX_FUNCTION_ARG,
+ SQLITE_MAX_ATTACHED,
+ SQLITE_MAX_LIKE_PATTERN_LENGTH,
+ SQLITE_MAX_VARIABLE_NUMBER, /* IMP: R-38091-32352 */
+ SQLITE_MAX_TRIGGER_DEPTH,
+ SQLITE_MAX_WORKER_THREADS,
+};
+
+/*
+** Make sure the hard limits are set to reasonable values
+*/
+#if SQLITE_MAX_LENGTH<100
+# error SQLITE_MAX_LENGTH must be at least 100
+#endif
+#if SQLITE_MAX_SQL_LENGTH<100
+# error SQLITE_MAX_SQL_LENGTH must be at least 100
+#endif
+#if SQLITE_MAX_SQL_LENGTH>SQLITE_MAX_LENGTH
+# error SQLITE_MAX_SQL_LENGTH must not be greater than SQLITE_MAX_LENGTH
+#endif
+#if SQLITE_MAX_COMPOUND_SELECT<2
+# error SQLITE_MAX_COMPOUND_SELECT must be at least 2
+#endif
+#if SQLITE_MAX_VDBE_OP<40
+# error SQLITE_MAX_VDBE_OP must be at least 40
+#endif
+#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>1000
+# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 1000
+#endif
+#if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>125
+# error SQLITE_MAX_ATTACHED must be between 0 and 125
+#endif
+#if SQLITE_MAX_LIKE_PATTERN_LENGTH<1
+# error SQLITE_MAX_LIKE_PATTERN_LENGTH must be at least 1
+#endif
+#if SQLITE_MAX_COLUMN>32767
+# error SQLITE_MAX_COLUMN must not exceed 32767
+#endif
+#if SQLITE_MAX_TRIGGER_DEPTH<1
+# error SQLITE_MAX_TRIGGER_DEPTH must be at least 1
+#endif
+#if SQLITE_MAX_WORKER_THREADS<0 || SQLITE_MAX_WORKER_THREADS>50
+# error SQLITE_MAX_WORKER_THREADS must be between 0 and 50
+#endif
+
+
+/*
+** Change the value of a limit. Report the old value.
+** If an invalid limit index is supplied, report -1.
+** Make no changes but still report the old value if the
+** new limit is negative.
+**
+** A new lower limit does not shrink existing constructs.
+** It merely prevents new constructs that exceed the limit
+** from forming.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_limit(sqlite3 *db, int limitId, int newLimit){
+ int oldLimit;
+
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) ){
+ (void)SQLITE_MISUSE_BKPT;
+ return -1;
+ }
+#endif
+
+ /* EVIDENCE-OF: R-30189-54097 For each limit category SQLITE_LIMIT_NAME
+ ** there is a hard upper bound set at compile-time by a C preprocessor
+ ** macro called SQLITE_MAX_NAME. (The "_LIMIT_" in the name is changed to
+ ** "_MAX_".)
+ */
+ assert( aHardLimit[SQLITE_LIMIT_LENGTH]==SQLITE_MAX_LENGTH );
+ assert( aHardLimit[SQLITE_LIMIT_SQL_LENGTH]==SQLITE_MAX_SQL_LENGTH );
+ assert( aHardLimit[SQLITE_LIMIT_COLUMN]==SQLITE_MAX_COLUMN );
+ assert( aHardLimit[SQLITE_LIMIT_EXPR_DEPTH]==SQLITE_MAX_EXPR_DEPTH );
+ assert( aHardLimit[SQLITE_LIMIT_COMPOUND_SELECT]==SQLITE_MAX_COMPOUND_SELECT);
+ assert( aHardLimit[SQLITE_LIMIT_VDBE_OP]==SQLITE_MAX_VDBE_OP );
+ assert( aHardLimit[SQLITE_LIMIT_FUNCTION_ARG]==SQLITE_MAX_FUNCTION_ARG );
+ assert( aHardLimit[SQLITE_LIMIT_ATTACHED]==SQLITE_MAX_ATTACHED );
+ assert( aHardLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]==
+ SQLITE_MAX_LIKE_PATTERN_LENGTH );
+ assert( aHardLimit[SQLITE_LIMIT_VARIABLE_NUMBER]==SQLITE_MAX_VARIABLE_NUMBER);
+ assert( aHardLimit[SQLITE_LIMIT_TRIGGER_DEPTH]==SQLITE_MAX_TRIGGER_DEPTH );
+ assert( aHardLimit[SQLITE_LIMIT_WORKER_THREADS]==SQLITE_MAX_WORKER_THREADS );
+ assert( SQLITE_LIMIT_WORKER_THREADS==(SQLITE_N_LIMIT-1) );
+
+
+ if( limitId<0 || limitId>=SQLITE_N_LIMIT ){
+ return -1;
+ }
+ oldLimit = db->aLimit[limitId];
+ if( newLimit>=0 ){ /* IMP: R-52476-28732 */
+ if( newLimit>aHardLimit[limitId] ){
+ newLimit = aHardLimit[limitId]; /* IMP: R-51463-25634 */
+ }
+ db->aLimit[limitId] = newLimit;
+ }
+ return oldLimit; /* IMP: R-53341-35419 */
+}
+
+/*
+** This function is used to parse both URIs and non-URI filenames passed by the
+** user to API functions sqlite3_open() or sqlite3_open_v2(), and for database
+** URIs specified as part of ATTACH statements.
+**
+** The first argument to this function is the name of the VFS to use (or
+** a NULL to signify the default VFS) if the URI does not contain a "vfs=xxx"
+** query parameter. The second argument contains the URI (or non-URI filename)
+** itself. When this function is called the *pFlags variable should contain
+** the default flags to open the database handle with. The value stored in
+** *pFlags may be updated before returning if the URI filename contains
+** "cache=xxx" or "mode=xxx" query parameters.
+**
+** If successful, SQLITE_OK is returned. In this case *ppVfs is set to point to
+** the VFS that should be used to open the database file. *pzFile is set to
+** point to a buffer containing the name of the file to open. It is the
+** responsibility of the caller to eventually call sqlite3_free() to release
+** this buffer.
+**
+** If an error occurs, then an SQLite error code is returned and *pzErrMsg
+** may be set to point to a buffer containing an English language error
+** message. It is the responsibility of the caller to eventually release
+** this buffer by calling sqlite3_free().
+*/
+SQLITE_PRIVATE int sqlite3ParseUri(
+ const char *zDefaultVfs, /* VFS to use if no "vfs=xxx" query option */
+ const char *zUri, /* Nul-terminated URI to parse */
+ unsigned int *pFlags, /* IN/OUT: SQLITE_OPEN_XXX flags */
+ sqlite3_vfs **ppVfs, /* OUT: VFS to use */
+ char **pzFile, /* OUT: Filename component of URI */
+ char **pzErrMsg /* OUT: Error message (if rc!=SQLITE_OK) */
+){
+ int rc = SQLITE_OK;
+ unsigned int flags = *pFlags;
+ const char *zVfs = zDefaultVfs;
+ char *zFile;
+ char c;
+ int nUri = sqlite3Strlen30(zUri);
+
+ assert( *pzErrMsg==0 );
+
+ if( ((flags & SQLITE_OPEN_URI) /* IMP: R-48725-32206 */
+ || sqlite3GlobalConfig.bOpenUri) /* IMP: R-51689-46548 */
+ && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */
+ ){
+ char *zOpt;
+ int eState; /* Parser state when parsing URI */
+ int iIn; /* Input character index */
+ int iOut = 0; /* Output character index */
+ u64 nByte = nUri+2; /* Bytes of space to allocate */
+
+ /* Make sure the SQLITE_OPEN_URI flag is set to indicate to the VFS xOpen
+ ** method that there may be extra parameters following the file-name. */
+ flags |= SQLITE_OPEN_URI;
+
+ for(iIn=0; iIn<nUri; iIn++) nByte += (zUri[iIn]=='&');
+ zFile = sqlite3_malloc64(nByte);
+ if( !zFile ) return SQLITE_NOMEM;
+
+ iIn = 5;
+#ifdef SQLITE_ALLOW_URI_AUTHORITY
+ if( strncmp(zUri+5, "///", 3)==0 ){
+ iIn = 7;
+ /* The following condition causes URIs with five leading / characters
+ ** like file://///host/path to be converted into UNCs like //host/path.
+ ** The correct URI for that UNC has only two or four leading / characters
+ ** file://host/path or file:////host/path. But 5 leading slashes is a
+ ** common error, we are told, so we handle it as a special case. */
+ if( strncmp(zUri+7, "///", 3)==0 ){ iIn++; }
+ }else if( strncmp(zUri+5, "//localhost/", 12)==0 ){
+ iIn = 16;
+ }
+#else
+ /* Discard the scheme and authority segments of the URI. */
+ if( zUri[5]=='/' && zUri[6]=='/' ){
+ iIn = 7;
+ while( zUri[iIn] && zUri[iIn]!='/' ) iIn++;
+ if( iIn!=7 && (iIn!=16 || memcmp("localhost", &zUri[7], 9)) ){
+ *pzErrMsg = sqlite3_mprintf("invalid uri authority: %.*s",
+ iIn-7, &zUri[7]);
+ rc = SQLITE_ERROR;
+ goto parse_uri_out;
+ }
+ }
+#endif
+
+ /* Copy the filename and any query parameters into the zFile buffer.
+ ** Decode %HH escape codes along the way.
+ **
+ ** Within this loop, variable eState may be set to 0, 1 or 2, depending
+ ** on the parsing context. As follows:
+ **
+ ** 0: Parsing file-name.
+ ** 1: Parsing name section of a name=value query parameter.
+ ** 2: Parsing value section of a name=value query parameter.
+ */
+ eState = 0;
+ while( (c = zUri[iIn])!=0 && c!='#' ){
+ iIn++;
+ if( c=='%'
+ && sqlite3Isxdigit(zUri[iIn])
+ && sqlite3Isxdigit(zUri[iIn+1])
+ ){
+ int octet = (sqlite3HexToInt(zUri[iIn++]) << 4);
+ octet += sqlite3HexToInt(zUri[iIn++]);
+
+ assert( octet>=0 && octet<256 );
+ if( octet==0 ){
+ /* This branch is taken when "%00" appears within the URI. In this
+ ** case we ignore all text in the remainder of the path, name or
+ ** value currently being parsed. So ignore the current character
+ ** and skip to the next "?", "=" or "&", as appropriate. */
+ while( (c = zUri[iIn])!=0 && c!='#'
+ && (eState!=0 || c!='?')
+ && (eState!=1 || (c!='=' && c!='&'))
+ && (eState!=2 || c!='&')
+ ){
+ iIn++;
+ }
+ continue;
+ }
+ c = octet;
+ }else if( eState==1 && (c=='&' || c=='=') ){
+ if( zFile[iOut-1]==0 ){
+ /* An empty option name. Ignore this option altogether. */
+ while( zUri[iIn] && zUri[iIn]!='#' && zUri[iIn-1]!='&' ) iIn++;
+ continue;
+ }
+ if( c=='&' ){
+ zFile[iOut++] = '\0';
+ }else{
+ eState = 2;
+ }
+ c = 0;
+ }else if( (eState==0 && c=='?') || (eState==2 && c=='&') ){
+ c = 0;
+ eState = 1;
+ }
+ zFile[iOut++] = c;
+ }
+ if( eState==1 ) zFile[iOut++] = '\0';
+ zFile[iOut++] = '\0';
+ zFile[iOut++] = '\0';
+
+ /* Check if there were any options specified that should be interpreted
+ ** here. Options that are interpreted here include "vfs" and those that
+ ** correspond to flags that may be passed to the sqlite3_open_v2()
+ ** method. */
+ zOpt = &zFile[sqlite3Strlen30(zFile)+1];
+ while( zOpt[0] ){
+ int nOpt = sqlite3Strlen30(zOpt);
+ char *zVal = &zOpt[nOpt+1];
+ int nVal = sqlite3Strlen30(zVal);
+
+ if( nOpt==3 && memcmp("vfs", zOpt, 3)==0 ){
+ zVfs = zVal;
+ }else{
+ struct OpenMode {
+ const char *z;
+ int mode;
+ } *aMode = 0;
+ char *zModeType = 0;
+ int mask = 0;
+ int limit = 0;
+
+ if( nOpt==5 && memcmp("cache", zOpt, 5)==0 ){
+ static struct OpenMode aCacheMode[] = {
+ { "shared", SQLITE_OPEN_SHAREDCACHE },
+ { "private", SQLITE_OPEN_PRIVATECACHE },
+ { 0, 0 }
+ };
+
+ mask = SQLITE_OPEN_SHAREDCACHE|SQLITE_OPEN_PRIVATECACHE;
+ aMode = aCacheMode;
+ limit = mask;
+ zModeType = "cache";
+ }
+ if( nOpt==4 && memcmp("mode", zOpt, 4)==0 ){
+ static struct OpenMode aOpenMode[] = {
+ { "ro", SQLITE_OPEN_READONLY },
+ { "rw", SQLITE_OPEN_READWRITE },
+ { "rwc", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE },
+ { "memory", SQLITE_OPEN_MEMORY },
+ { 0, 0 }
+ };
+
+ mask = SQLITE_OPEN_READONLY | SQLITE_OPEN_READWRITE
+ | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY;
+ aMode = aOpenMode;
+ limit = mask & flags;
+ zModeType = "access";
+ }
+
+ if( aMode ){
+ int i;
+ int mode = 0;
+ for(i=0; aMode[i].z; i++){
+ const char *z = aMode[i].z;
+ if( nVal==sqlite3Strlen30(z) && 0==memcmp(zVal, z, nVal) ){
+ mode = aMode[i].mode;
+ break;
+ }
+ }
+ if( mode==0 ){
+ *pzErrMsg = sqlite3_mprintf("no such %s mode: %s", zModeType, zVal);
+ rc = SQLITE_ERROR;
+ goto parse_uri_out;
+ }
+ if( (mode & ~SQLITE_OPEN_MEMORY)>limit ){
+ *pzErrMsg = sqlite3_mprintf("%s mode not allowed: %s",
+ zModeType, zVal);
+ rc = SQLITE_PERM;
+ goto parse_uri_out;
+ }
+ flags = (flags & ~mask) | mode;
+ }
+ }
+
+ zOpt = &zVal[nVal+1];
+ }
+
+ }else{
+ zFile = sqlite3_malloc64(nUri+2);
+ if( !zFile ) return SQLITE_NOMEM;
+ memcpy(zFile, zUri, nUri);
+ zFile[nUri] = '\0';
+ zFile[nUri+1] = '\0';
+ flags &= ~SQLITE_OPEN_URI;
+ }
+
+ *ppVfs = sqlite3_vfs_find(zVfs);
+ if( *ppVfs==0 ){
+ *pzErrMsg = sqlite3_mprintf("no such vfs: %s", zVfs);
+ rc = SQLITE_ERROR;
+ }
+ parse_uri_out:
+ if( rc!=SQLITE_OK ){
+ sqlite3_free(zFile);
+ zFile = 0;
+ }
+ *pFlags = flags;
+ *pzFile = zFile;
+ return rc;
+}
+
+
+/*
+** This routine does the work of opening a database on behalf of
+** sqlite3_open() and sqlite3_open16(). The database filename "zFilename"
+** is UTF-8 encoded.
+*/
+static int openDatabase(
+ const char *zFilename, /* Database filename UTF-8 encoded */
+ sqlite3 **ppDb, /* OUT: Returned database handle */
+ unsigned int flags, /* Operational flags */
+ const char *zVfs /* Name of the VFS to use */
+){
+ sqlite3 *db; /* Store allocated handle here */
+ int rc; /* Return code */
+ int isThreadsafe; /* True for threadsafe connections */
+ char *zOpen = 0; /* Filename argument to pass to BtreeOpen() */
+ char *zErrMsg = 0; /* Error message from sqlite3ParseUri() */
+
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( ppDb==0 ) return SQLITE_MISUSE_BKPT;
+#endif
+ *ppDb = 0;
+#ifndef SQLITE_OMIT_AUTOINIT
+ rc = sqlite3_initialize();
+ if( rc ) return rc;
+#endif
+
+ /* Only allow sensible combinations of bits in the flags argument.
+ ** Throw an error if any non-sense combination is used. If we
+ ** do not block illegal combinations here, it could trigger
+ ** assert() statements in deeper layers. Sensible combinations
+ ** are:
+ **
+ ** 1: SQLITE_OPEN_READONLY
+ ** 2: SQLITE_OPEN_READWRITE
+ ** 6: SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE
+ */
+ assert( SQLITE_OPEN_READONLY == 0x01 );
+ assert( SQLITE_OPEN_READWRITE == 0x02 );
+ assert( SQLITE_OPEN_CREATE == 0x04 );
+ testcase( (1<<(flags&7))==0x02 ); /* READONLY */
+ testcase( (1<<(flags&7))==0x04 ); /* READWRITE */
+ testcase( (1<<(flags&7))==0x40 ); /* READWRITE | CREATE */
+ if( ((1<<(flags&7)) & 0x46)==0 ){
+ return SQLITE_MISUSE_BKPT; /* IMP: R-65497-44594 */
+ }
+
+ if( sqlite3GlobalConfig.bCoreMutex==0 ){
+ isThreadsafe = 0;
+ }else if( flags & SQLITE_OPEN_NOMUTEX ){
+ isThreadsafe = 0;
+ }else if( flags & SQLITE_OPEN_FULLMUTEX ){
+ isThreadsafe = 1;
+ }else{
+ isThreadsafe = sqlite3GlobalConfig.bFullMutex;
+ }
+ if( flags & SQLITE_OPEN_PRIVATECACHE ){
+ flags &= ~SQLITE_OPEN_SHAREDCACHE;
+ }else if( sqlite3GlobalConfig.sharedCacheEnabled ){
+ flags |= SQLITE_OPEN_SHAREDCACHE;
+ }
+
+ /* Remove harmful bits from the flags parameter
+ **
+ ** The SQLITE_OPEN_NOMUTEX and SQLITE_OPEN_FULLMUTEX flags were
+ ** dealt with in the previous code block. Besides these, the only
+ ** valid input flags for sqlite3_open_v2() are SQLITE_OPEN_READONLY,
+ ** SQLITE_OPEN_READWRITE, SQLITE_OPEN_CREATE, SQLITE_OPEN_SHAREDCACHE,
+ ** SQLITE_OPEN_PRIVATECACHE, and some reserved bits. Silently mask
+ ** off all other flags.
+ */
+ flags &= ~( SQLITE_OPEN_DELETEONCLOSE |
+ SQLITE_OPEN_EXCLUSIVE |
+ SQLITE_OPEN_MAIN_DB |
+ SQLITE_OPEN_TEMP_DB |
+ SQLITE_OPEN_TRANSIENT_DB |
+ SQLITE_OPEN_MAIN_JOURNAL |
+ SQLITE_OPEN_TEMP_JOURNAL |
+ SQLITE_OPEN_SUBJOURNAL |
+ SQLITE_OPEN_MASTER_JOURNAL |
+ SQLITE_OPEN_NOMUTEX |
+ SQLITE_OPEN_FULLMUTEX |
+ SQLITE_OPEN_WAL
+ );
+
+ /* Allocate the sqlite data structure */
+ db = sqlite3MallocZero( sizeof(sqlite3) );
+ if( db==0 ) goto opendb_out;
+ if( isThreadsafe ){
+ db->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE);
+ if( db->mutex==0 ){
+ sqlite3_free(db);
+ db = 0;
+ goto opendb_out;
+ }
+ }
+ sqlite3_mutex_enter(db->mutex);
+ db->errMask = 0xff;
+ db->nDb = 2;
+ db->magic = SQLITE_MAGIC_BUSY;
+ db->aDb = db->aDbStatic;
+
+ assert( sizeof(db->aLimit)==sizeof(aHardLimit) );
+ memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit));
+ db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = SQLITE_DEFAULT_WORKER_THREADS;
+ db->autoCommit = 1;
+ db->nextAutovac = -1;
+ db->szMmap = sqlite3GlobalConfig.szMmap;
+ db->nextPagesize = 0;
+ db->nMaxSorterMmap = 0x7FFFFFFF;
+ db->flags |= SQLITE_ShortColNames | SQLITE_EnableTrigger | SQLITE_CacheSpill
+#if !defined(SQLITE_DEFAULT_AUTOMATIC_INDEX) || SQLITE_DEFAULT_AUTOMATIC_INDEX
+ | SQLITE_AutoIndex
+#endif
+#if SQLITE_DEFAULT_CKPTFULLFSYNC
+ | SQLITE_CkptFullFSync
+#endif
+#if SQLITE_DEFAULT_FILE_FORMAT<4
+ | SQLITE_LegacyFileFmt
+#endif
+#ifdef SQLITE_ENABLE_LOAD_EXTENSION
+ | SQLITE_LoadExtension
+#endif
+#if SQLITE_DEFAULT_RECURSIVE_TRIGGERS
+ | SQLITE_RecTriggers
+#endif
+#if defined(SQLITE_DEFAULT_FOREIGN_KEYS) && SQLITE_DEFAULT_FOREIGN_KEYS
+ | SQLITE_ForeignKeys
+#endif
+#if defined(SQLITE_REVERSE_UNORDERED_SELECTS)
+ | SQLITE_ReverseOrder
+#endif
+#if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK)
+ | SQLITE_CellSizeCk
+#endif
+ ;
+ sqlite3HashInit(&db->aCollSeq);
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+ sqlite3HashInit(&db->aModule);
+#endif
+
+ /* Add the default collation sequence BINARY. BINARY works for both UTF-8
+ ** and UTF-16, so add a version for each to avoid any unnecessary
+ ** conversions. The only error that can occur here is a malloc() failure.
+ **
+ ** EVIDENCE-OF: R-52786-44878 SQLite defines three built-in collating
+ ** functions:
+ */
+ createCollation(db, "BINARY", SQLITE_UTF8, 0, binCollFunc, 0);
+ createCollation(db, "BINARY", SQLITE_UTF16BE, 0, binCollFunc, 0);
+ createCollation(db, "BINARY", SQLITE_UTF16LE, 0, binCollFunc, 0);
+ createCollation(db, "NOCASE", SQLITE_UTF8, 0, nocaseCollatingFunc, 0);
+ createCollation(db, "RTRIM", SQLITE_UTF8, (void*)1, binCollFunc, 0);
+ if( db->mallocFailed ){
+ goto opendb_out;
+ }
+ /* EVIDENCE-OF: R-08308-17224 The default collating function for all
+ ** strings is BINARY.
+ */
+ db->pDfltColl = sqlite3FindCollSeq(db, SQLITE_UTF8, "BINARY", 0);
+ assert( db->pDfltColl!=0 );
+
+ /* Parse the filename/URI argument. */
+ db->openFlags = flags;
+ rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg);
+ if( rc!=SQLITE_OK ){
+ if( rc==SQLITE_NOMEM ) db->mallocFailed = 1;
+ sqlite3ErrorWithMsg(db, rc, zErrMsg ? "%s" : 0, zErrMsg);
+ sqlite3_free(zErrMsg);
+ goto opendb_out;
+ }
+
+ /* Open the backend database driver */
+ rc = sqlite3BtreeOpen(db->pVfs, zOpen, db, &db->aDb[0].pBt, 0,
+ flags | SQLITE_OPEN_MAIN_DB);
+ if( rc!=SQLITE_OK ){
+ if( rc==SQLITE_IOERR_NOMEM ){
+ rc = SQLITE_NOMEM;
+ }
+ sqlite3Error(db, rc);
+ goto opendb_out;
+ }
+ sqlite3BtreeEnter(db->aDb[0].pBt);
+ db->aDb[0].pSchema = sqlite3SchemaGet(db, db->aDb[0].pBt);
+ if( !db->mallocFailed ) ENC(db) = SCHEMA_ENC(db);
+ sqlite3BtreeLeave(db->aDb[0].pBt);
+ db->aDb[1].pSchema = sqlite3SchemaGet(db, 0);
+
+ /* The default safety_level for the main database is 'full'; for the temp
+ ** database it is 'NONE'. This matches the pager layer defaults.
+ */
+ db->aDb[0].zName = "main";
+ db->aDb[0].safety_level = 3;
+ db->aDb[1].zName = "temp";
+ db->aDb[1].safety_level = 1;
+
+ db->magic = SQLITE_MAGIC_OPEN;
+ if( db->mallocFailed ){
+ goto opendb_out;
+ }
+
+ /* Register all built-in functions, but do not attempt to read the
+ ** database schema yet. This is delayed until the first time the database
+ ** is accessed.
+ */
+ sqlite3Error(db, SQLITE_OK);
+ sqlite3RegisterBuiltinFunctions(db);
+
+ /* Load automatic extensions - extensions that have been registered
+ ** using the sqlite3_automatic_extension() API.
+ */
+ rc = sqlite3_errcode(db);
+ if( rc==SQLITE_OK ){
+ sqlite3AutoLoadExtensions(db);
+ rc = sqlite3_errcode(db);
+ if( rc!=SQLITE_OK ){
+ goto opendb_out;
+ }
+ }
+
+#ifdef SQLITE_ENABLE_FTS1
+ if( !db->mallocFailed ){
+ extern int sqlite3Fts1Init(sqlite3*);
+ rc = sqlite3Fts1Init(db);
+ }
+#endif
+
+#ifdef SQLITE_ENABLE_FTS2
+ if( !db->mallocFailed && rc==SQLITE_OK ){
+ extern int sqlite3Fts2Init(sqlite3*);
+ rc = sqlite3Fts2Init(db);
+ }
+#endif
+
+#ifdef SQLITE_ENABLE_FTS3 /* automatically defined by SQLITE_ENABLE_FTS4 */
+ if( !db->mallocFailed && rc==SQLITE_OK ){
+ rc = sqlite3Fts3Init(db);
+ }
+#endif
+
+#ifdef SQLITE_ENABLE_FTS5
+ if( !db->mallocFailed && rc==SQLITE_OK ){
+ rc = sqlite3Fts5Init(db);
+ }
+#endif
+
+#ifdef SQLITE_ENABLE_ICU
+ if( !db->mallocFailed && rc==SQLITE_OK ){
+ rc = sqlite3IcuInit(db);
+ }
+#endif
+
+#ifdef SQLITE_ENABLE_RTREE
+ if( !db->mallocFailed && rc==SQLITE_OK){
+ rc = sqlite3RtreeInit(db);
+ }
+#endif
+
+#ifdef SQLITE_ENABLE_DBSTAT_VTAB
+ if( !db->mallocFailed && rc==SQLITE_OK){
+ rc = sqlite3DbstatRegister(db);
+ }
+#endif
+
+#ifdef SQLITE_ENABLE_JSON1
+ if( !db->mallocFailed && rc==SQLITE_OK){
+ rc = sqlite3Json1Init(db);
+ }
+#endif
+
+ /* -DSQLITE_DEFAULT_LOCKING_MODE=1 makes EXCLUSIVE the default locking
+ ** mode. -DSQLITE_DEFAULT_LOCKING_MODE=0 make NORMAL the default locking
+ ** mode. Doing nothing at all also makes NORMAL the default.
+ */
+#ifdef SQLITE_DEFAULT_LOCKING_MODE
+ db->dfltLockMode = SQLITE_DEFAULT_LOCKING_MODE;
+ sqlite3PagerLockingMode(sqlite3BtreePager(db->aDb[0].pBt),
+ SQLITE_DEFAULT_LOCKING_MODE);
+#endif
+
+ if( rc ) sqlite3Error(db, rc);
+
+ /* Enable the lookaside-malloc subsystem */
+ setupLookaside(db, 0, sqlite3GlobalConfig.szLookaside,
+ sqlite3GlobalConfig.nLookaside);
+
+ sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_WAL_AUTOCHECKPOINT);
+
+opendb_out:
+ sqlite3_free(zOpen);
+ if( db ){
+ assert( db->mutex!=0 || isThreadsafe==0
+ || sqlite3GlobalConfig.bFullMutex==0 );
+ sqlite3_mutex_leave(db->mutex);
+ }
+ rc = sqlite3_errcode(db);
+ assert( db!=0 || rc==SQLITE_NOMEM );
+ if( rc==SQLITE_NOMEM ){
+ sqlite3_close(db);
+ db = 0;
+ }else if( rc!=SQLITE_OK ){
+ db->magic = SQLITE_MAGIC_SICK;
+ }
+ *ppDb = db;
+#ifdef SQLITE_ENABLE_SQLLOG
+ if( sqlite3GlobalConfig.xSqllog ){
+ /* Opening a db handle. Fourth parameter is passed 0. */
+ void *pArg = sqlite3GlobalConfig.pSqllogArg;
+ sqlite3GlobalConfig.xSqllog(pArg, db, zFilename, 0);
+ }
+#endif
+ return rc & 0xff;
+}
+
+/*
+** Open a new database handle.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_open(
+ const char *zFilename,
+ sqlite3 **ppDb
+){
+ return openDatabase(zFilename, ppDb,
+ SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0);
+}
+SQLITE_API int SQLITE_STDCALL sqlite3_open_v2(
+ const char *filename, /* Database filename (UTF-8) */
+ sqlite3 **ppDb, /* OUT: SQLite db handle */
+ int flags, /* Flags */
+ const char *zVfs /* Name of VFS module to use */
+){
+ return openDatabase(filename, ppDb, (unsigned int)flags, zVfs);
+}
+
+#ifndef SQLITE_OMIT_UTF16
+/*
+** Open a new database handle.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_open16(
+ const void *zFilename,
+ sqlite3 **ppDb
+){
+ char const *zFilename8; /* zFilename encoded in UTF-8 instead of UTF-16 */
+ sqlite3_value *pVal;
+ int rc;
+
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( ppDb==0 ) return SQLITE_MISUSE_BKPT;
+#endif
+ *ppDb = 0;
+#ifndef SQLITE_OMIT_AUTOINIT
+ rc = sqlite3_initialize();
+ if( rc ) return rc;
+#endif
+ if( zFilename==0 ) zFilename = "\000\000";
+ pVal = sqlite3ValueNew(0);
+ sqlite3ValueSetStr(pVal, -1, zFilename, SQLITE_UTF16NATIVE, SQLITE_STATIC);
+ zFilename8 = sqlite3ValueText(pVal, SQLITE_UTF8);
+ if( zFilename8 ){
+ rc = openDatabase(zFilename8, ppDb,
+ SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0);
+ assert( *ppDb || rc==SQLITE_NOMEM );
+ if( rc==SQLITE_OK && !DbHasProperty(*ppDb, 0, DB_SchemaLoaded) ){
+ SCHEMA_ENC(*ppDb) = ENC(*ppDb) = SQLITE_UTF16NATIVE;
+ }
+ }else{
+ rc = SQLITE_NOMEM;
+ }
+ sqlite3ValueFree(pVal);
+
+ return rc & 0xff;
+}
+#endif /* SQLITE_OMIT_UTF16 */
+
+/*
+** Register a new collation sequence with the database handle db.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_create_collation(
+ sqlite3* db,
+ const char *zName,
+ int enc,
+ void* pCtx,
+ int(*xCompare)(void*,int,const void*,int,const void*)
+){
+ return sqlite3_create_collation_v2(db, zName, enc, pCtx, xCompare, 0);
+}
+
+/*
+** Register a new collation sequence with the database handle db.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_create_collation_v2(
+ sqlite3* db,
+ const char *zName,
+ int enc,
+ void* pCtx,
+ int(*xCompare)(void*,int,const void*,int,const void*),
+ void(*xDel)(void*)
+){
+ int rc;
+
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT;
+#endif
+ sqlite3_mutex_enter(db->mutex);
+ assert( !db->mallocFailed );
+ rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, xDel);
+ rc = sqlite3ApiExit(db, rc);
+ sqlite3_mutex_leave(db->mutex);
+ return rc;
+}
+
+#ifndef SQLITE_OMIT_UTF16
+/*
+** Register a new collation sequence with the database handle db.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_create_collation16(
+ sqlite3* db,
+ const void *zName,
+ int enc,
+ void* pCtx,
+ int(*xCompare)(void*,int,const void*,int,const void*)
+){
+ int rc = SQLITE_OK;
+ char *zName8;
+
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT;
+#endif
+ sqlite3_mutex_enter(db->mutex);
+ assert( !db->mallocFailed );
+ zName8 = sqlite3Utf16to8(db, zName, -1, SQLITE_UTF16NATIVE);
+ if( zName8 ){
+ rc = createCollation(db, zName8, (u8)enc, pCtx, xCompare, 0);
+ sqlite3DbFree(db, zName8);
+ }
+ rc = sqlite3ApiExit(db, rc);
+ sqlite3_mutex_leave(db->mutex);
+ return rc;
+}
+#endif /* SQLITE_OMIT_UTF16 */
+
+/*
+** Register a collation sequence factory callback with the database handle
+** db. Replace any previously installed collation sequence factory.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_collation_needed(
+ sqlite3 *db,
+ void *pCollNeededArg,
+ void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*)
+){
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
+#endif
+ sqlite3_mutex_enter(db->mutex);
+ db->xCollNeeded = xCollNeeded;
+ db->xCollNeeded16 = 0;
+ db->pCollNeededArg = pCollNeededArg;
+ sqlite3_mutex_leave(db->mutex);
+ return SQLITE_OK;
+}
+
+#ifndef SQLITE_OMIT_UTF16
+/*
+** Register a collation sequence factory callback with the database handle
+** db. Replace any previously installed collation sequence factory.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_collation_needed16(
+ sqlite3 *db,
+ void *pCollNeededArg,
+ void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*)
+){
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
+#endif
+ sqlite3_mutex_enter(db->mutex);
+ db->xCollNeeded = 0;
+ db->xCollNeeded16 = xCollNeeded16;
+ db->pCollNeededArg = pCollNeededArg;
+ sqlite3_mutex_leave(db->mutex);
+ return SQLITE_OK;
+}
+#endif /* SQLITE_OMIT_UTF16 */
+
+#ifndef SQLITE_OMIT_DEPRECATED
+/*
+** This function is now an anachronism. It used to be used to recover from a
+** malloc() failure, but SQLite now does this automatically.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_global_recover(void){
+ return SQLITE_OK;
+}
+#endif
+
+/*
+** Test to see whether or not the database connection is in autocommit
+** mode. Return TRUE if it is and FALSE if not. Autocommit mode is on
+** by default. Autocommit is disabled by a BEGIN statement and reenabled
+** by the next COMMIT or ROLLBACK.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_get_autocommit(sqlite3 *db){
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) ){
+ (void)SQLITE_MISUSE_BKPT;
+ return 0;
+ }
+#endif
+ return db->autoCommit;
+}
+
+/*
+** The following routines are substitutes for constants SQLITE_CORRUPT,
+** SQLITE_MISUSE, SQLITE_CANTOPEN, SQLITE_IOERR and possibly other error
+** constants. They serve two purposes:
+**
+** 1. Serve as a convenient place to set a breakpoint in a debugger
+** to detect when version error conditions occurs.
+**
+** 2. Invoke sqlite3_log() to provide the source code location where
+** a low-level error is first detected.
+*/
+SQLITE_PRIVATE int sqlite3CorruptError(int lineno){
+ testcase( sqlite3GlobalConfig.xLog!=0 );
+ sqlite3_log(SQLITE_CORRUPT,
+ "database corruption at line %d of [%.10s]",
+ lineno, 20+sqlite3_sourceid());
+ return SQLITE_CORRUPT;
+}
+SQLITE_PRIVATE int sqlite3MisuseError(int lineno){
+ testcase( sqlite3GlobalConfig.xLog!=0 );
+ sqlite3_log(SQLITE_MISUSE,
+ "misuse at line %d of [%.10s]",
+ lineno, 20+sqlite3_sourceid());
+ return SQLITE_MISUSE;
+}
+SQLITE_PRIVATE int sqlite3CantopenError(int lineno){
+ testcase( sqlite3GlobalConfig.xLog!=0 );
+ sqlite3_log(SQLITE_CANTOPEN,
+ "cannot open file at line %d of [%.10s]",
+ lineno, 20+sqlite3_sourceid());
+ return SQLITE_CANTOPEN;
+}
+
+
+#ifndef SQLITE_OMIT_DEPRECATED
+/*
+** This is a convenience routine that makes sure that all thread-specific
+** data for this thread has been deallocated.
+**
+** SQLite no longer uses thread-specific data so this routine is now a
+** no-op. It is retained for historical compatibility.
+*/
+SQLITE_API void SQLITE_STDCALL sqlite3_thread_cleanup(void){
+}
+#endif
+
+/*
+** Return meta information about a specific column of a database table.
+** See comment in sqlite3.h (sqlite.h.in) for details.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_table_column_metadata(
+ sqlite3 *db, /* Connection handle */
+ const char *zDbName, /* Database name or NULL */
+ const char *zTableName, /* Table name */
+ const char *zColumnName, /* Column name */
+ char const **pzDataType, /* OUTPUT: Declared data type */
+ char const **pzCollSeq, /* OUTPUT: Collation sequence name */
+ int *pNotNull, /* OUTPUT: True if NOT NULL constraint exists */
+ int *pPrimaryKey, /* OUTPUT: True if column part of PK */
+ int *pAutoinc /* OUTPUT: True if column is auto-increment */
+){
+ int rc;
+ char *zErrMsg = 0;
+ Table *pTab = 0;
+ Column *pCol = 0;
+ int iCol = 0;
+ char const *zDataType = 0;
+ char const *zCollSeq = 0;
+ int notnull = 0;
+ int primarykey = 0;
+ int autoinc = 0;
+
+
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) || zTableName==0 ){
+ return SQLITE_MISUSE_BKPT;
+ }
+#endif
+
+ /* Ensure the database schema has been loaded */
+ sqlite3_mutex_enter(db->mutex);
+ sqlite3BtreeEnterAll(db);
+ rc = sqlite3Init(db, &zErrMsg);
+ if( SQLITE_OK!=rc ){
+ goto error_out;
+ }
+
+ /* Locate the table in question */
+ pTab = sqlite3FindTable(db, zTableName, zDbName);
+ if( !pTab || pTab->pSelect ){
+ pTab = 0;
+ goto error_out;
+ }
+
+ /* Find the column for which info is requested */
+ if( zColumnName==0 ){
+ /* Query for existance of table only */
+ }else{
+ for(iCol=0; iCol<pTab->nCol; iCol++){
+ pCol = &pTab->aCol[iCol];
+ if( 0==sqlite3StrICmp(pCol->zName, zColumnName) ){
+ break;
+ }
+ }
+ if( iCol==pTab->nCol ){
+ if( HasRowid(pTab) && sqlite3IsRowid(zColumnName) ){
+ iCol = pTab->iPKey;
+ pCol = iCol>=0 ? &pTab->aCol[iCol] : 0;
+ }else{
+ pTab = 0;
+ goto error_out;
+ }
+ }
+ }
+
+ /* The following block stores the meta information that will be returned
+ ** to the caller in local variables zDataType, zCollSeq, notnull, primarykey
+ ** and autoinc. At this point there are two possibilities:
+ **
+ ** 1. The specified column name was rowid", "oid" or "_rowid_"
+ ** and there is no explicitly declared IPK column.
+ **
+ ** 2. The table is not a view and the column name identified an
+ ** explicitly declared column. Copy meta information from *pCol.
+ */
+ if( pCol ){
+ zDataType = pCol->zType;
+ zCollSeq = pCol->zColl;
+ notnull = pCol->notNull!=0;
+ primarykey = (pCol->colFlags & COLFLAG_PRIMKEY)!=0;
+ autoinc = pTab->iPKey==iCol && (pTab->tabFlags & TF_Autoincrement)!=0;
+ }else{
+ zDataType = "INTEGER";
+ primarykey = 1;
+ }
+ if( !zCollSeq ){
+ zCollSeq = "BINARY";
+ }
+
+error_out:
+ sqlite3BtreeLeaveAll(db);
+
+ /* Whether the function call succeeded or failed, set the output parameters
+ ** to whatever their local counterparts contain. If an error did occur,
+ ** this has the effect of zeroing all output parameters.
+ */
+ if( pzDataType ) *pzDataType = zDataType;
+ if( pzCollSeq ) *pzCollSeq = zCollSeq;
+ if( pNotNull ) *pNotNull = notnull;
+ if( pPrimaryKey ) *pPrimaryKey = primarykey;
+ if( pAutoinc ) *pAutoinc = autoinc;
+
+ if( SQLITE_OK==rc && !pTab ){
+ sqlite3DbFree(db, zErrMsg);
+ zErrMsg = sqlite3MPrintf(db, "no such table column: %s.%s", zTableName,
+ zColumnName);
+ rc = SQLITE_ERROR;
+ }
+ sqlite3ErrorWithMsg(db, rc, (zErrMsg?"%s":0), zErrMsg);
+ sqlite3DbFree(db, zErrMsg);
+ rc = sqlite3ApiExit(db, rc);
+ sqlite3_mutex_leave(db->mutex);
+ return rc;
+}
+
+/*
+** Sleep for a little while. Return the amount of time slept.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_sleep(int ms){
+ sqlite3_vfs *pVfs;
+ int rc;
+ pVfs = sqlite3_vfs_find(0);
+ if( pVfs==0 ) return 0;
+
+ /* This function works in milliseconds, but the underlying OsSleep()
+ ** API uses microseconds. Hence the 1000's.
+ */
+ rc = (sqlite3OsSleep(pVfs, 1000*ms)/1000);
+ return rc;
+}
+
+/*
+** Enable or disable the extended result codes.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_extended_result_codes(sqlite3 *db, int onoff){
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
+#endif
+ sqlite3_mutex_enter(db->mutex);
+ db->errMask = onoff ? 0xffffffff : 0xff;
+ sqlite3_mutex_leave(db->mutex);
+ return SQLITE_OK;
+}
+
+/*
+** Invoke the xFileControl method on a particular database.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){
+ int rc = SQLITE_ERROR;
+ Btree *pBtree;
+
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
+#endif
+ sqlite3_mutex_enter(db->mutex);
+ pBtree = sqlite3DbNameToBtree(db, zDbName);
+ if( pBtree ){
+ Pager *pPager;
+ sqlite3_file *fd;
+ sqlite3BtreeEnter(pBtree);
+ pPager = sqlite3BtreePager(pBtree);
+ assert( pPager!=0 );
+ fd = sqlite3PagerFile(pPager);
+ assert( fd!=0 );
+ if( op==SQLITE_FCNTL_FILE_POINTER ){
+ *(sqlite3_file**)pArg = fd;
+ rc = SQLITE_OK;
+ }else if( fd->pMethods ){
+ rc = sqlite3OsFileControl(fd, op, pArg);
+ }else{
+ rc = SQLITE_NOTFOUND;
+ }
+ sqlite3BtreeLeave(pBtree);
+ }
+ sqlite3_mutex_leave(db->mutex);
+ return rc;
+}
+
+/*
+** Interface to the testing logic.
+*/
+SQLITE_API int SQLITE_CDECL sqlite3_test_control(int op, ...){
+ int rc = 0;
+#ifdef SQLITE_OMIT_BUILTIN_TEST
+ UNUSED_PARAMETER(op);
+#else
+ va_list ap;
+ va_start(ap, op);
+ switch( op ){
+
+ /*
+ ** Save the current state of the PRNG.
+ */
+ case SQLITE_TESTCTRL_PRNG_SAVE: {
+ sqlite3PrngSaveState();
+ break;
+ }
+
+ /*
+ ** Restore the state of the PRNG to the last state saved using
+ ** PRNG_SAVE. If PRNG_SAVE has never before been called, then
+ ** this verb acts like PRNG_RESET.
+ */
+ case SQLITE_TESTCTRL_PRNG_RESTORE: {
+ sqlite3PrngRestoreState();
+ break;
+ }
+
+ /*
+ ** Reset the PRNG back to its uninitialized state. The next call
+ ** to sqlite3_randomness() will reseed the PRNG using a single call
+ ** to the xRandomness method of the default VFS.
+ */
+ case SQLITE_TESTCTRL_PRNG_RESET: {
+ sqlite3_randomness(0,0);
+ break;
+ }
+
+ /*
+ ** sqlite3_test_control(BITVEC_TEST, size, program)
+ **
+ ** Run a test against a Bitvec object of size. The program argument
+ ** is an array of integers that defines the test. Return -1 on a
+ ** memory allocation error, 0 on success, or non-zero for an error.
+ ** See the sqlite3BitvecBuiltinTest() for additional information.
+ */
+ case SQLITE_TESTCTRL_BITVEC_TEST: {
+ int sz = va_arg(ap, int);
+ int *aProg = va_arg(ap, int*);
+ rc = sqlite3BitvecBuiltinTest(sz, aProg);
+ break;
+ }
+
+ /*
+ ** sqlite3_test_control(FAULT_INSTALL, xCallback)
+ **
+ ** Arrange to invoke xCallback() whenever sqlite3FaultSim() is called,
+ ** if xCallback is not NULL.
+ **
+ ** As a test of the fault simulator mechanism itself, sqlite3FaultSim(0)
+ ** is called immediately after installing the new callback and the return
+ ** value from sqlite3FaultSim(0) becomes the return from
+ ** sqlite3_test_control().
+ */
+ case SQLITE_TESTCTRL_FAULT_INSTALL: {
+ /* MSVC is picky about pulling func ptrs from va lists.
+ ** http://support.microsoft.com/kb/47961
+ ** sqlite3GlobalConfig.xTestCallback = va_arg(ap, int(*)(int));
+ */
+ typedef int(*TESTCALLBACKFUNC_t)(int);
+ sqlite3GlobalConfig.xTestCallback = va_arg(ap, TESTCALLBACKFUNC_t);
+ rc = sqlite3FaultSim(0);
+ break;
+ }
+
+ /*
+ ** sqlite3_test_control(BENIGN_MALLOC_HOOKS, xBegin, xEnd)
+ **
+ ** Register hooks to call to indicate which malloc() failures
+ ** are benign.
+ */
+ case SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS: {
+ typedef void (*void_function)(void);
+ void_function xBenignBegin;
+ void_function xBenignEnd;
+ xBenignBegin = va_arg(ap, void_function);
+ xBenignEnd = va_arg(ap, void_function);
+ sqlite3BenignMallocHooks(xBenignBegin, xBenignEnd);
+ break;
+ }
+
+ /*
+ ** sqlite3_test_control(SQLITE_TESTCTRL_PENDING_BYTE, unsigned int X)
+ **
+ ** Set the PENDING byte to the value in the argument, if X>0.
+ ** Make no changes if X==0. Return the value of the pending byte
+ ** as it existing before this routine was called.
+ **
+ ** IMPORTANT: Changing the PENDING byte from 0x40000000 results in
+ ** an incompatible database file format. Changing the PENDING byte
+ ** while any database connection is open results in undefined and
+ ** deleterious behavior.
+ */
+ case SQLITE_TESTCTRL_PENDING_BYTE: {
+ rc = PENDING_BYTE;
+#ifndef SQLITE_OMIT_WSD
+ {
+ unsigned int newVal = va_arg(ap, unsigned int);
+ if( newVal ) sqlite3PendingByte = newVal;
+ }
+#endif
+ break;
+ }
+
+ /*
+ ** sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, int X)
+ **
+ ** This action provides a run-time test to see whether or not
+ ** assert() was enabled at compile-time. If X is true and assert()
+ ** is enabled, then the return value is true. If X is true and
+ ** assert() is disabled, then the return value is zero. If X is
+ ** false and assert() is enabled, then the assertion fires and the
+ ** process aborts. If X is false and assert() is disabled, then the
+ ** return value is zero.
+ */
+ case SQLITE_TESTCTRL_ASSERT: {
+ volatile int x = 0;
+ assert( (x = va_arg(ap,int))!=0 );
+ rc = x;
+ break;
+ }
+
+
+ /*
+ ** sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, int X)
+ **
+ ** This action provides a run-time test to see how the ALWAYS and
+ ** NEVER macros were defined at compile-time.
+ **
+ ** The return value is ALWAYS(X).
+ **
+ ** The recommended test is X==2. If the return value is 2, that means
+ ** ALWAYS() and NEVER() are both no-op pass-through macros, which is the
+ ** default setting. If the return value is 1, then ALWAYS() is either
+ ** hard-coded to true or else it asserts if its argument is false.
+ ** The first behavior (hard-coded to true) is the case if
+ ** SQLITE_TESTCTRL_ASSERT shows that assert() is disabled and the second
+ ** behavior (assert if the argument to ALWAYS() is false) is the case if
+ ** SQLITE_TESTCTRL_ASSERT shows that assert() is enabled.
+ **
+ ** The run-time test procedure might look something like this:
+ **
+ ** if( sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, 2)==2 ){
+ ** // ALWAYS() and NEVER() are no-op pass-through macros
+ ** }else if( sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, 1) ){
+ ** // ALWAYS(x) asserts that x is true. NEVER(x) asserts x is false.
+ ** }else{
+ ** // ALWAYS(x) is a constant 1. NEVER(x) is a constant 0.
+ ** }
+ */
+ case SQLITE_TESTCTRL_ALWAYS: {
+ int x = va_arg(ap,int);
+ rc = ALWAYS(x);
+ break;
+ }
+
+ /*
+ ** sqlite3_test_control(SQLITE_TESTCTRL_BYTEORDER);
+ **
+ ** The integer returned reveals the byte-order of the computer on which
+ ** SQLite is running:
+ **
+ ** 1 big-endian, determined at run-time
+ ** 10 little-endian, determined at run-time
+ ** 432101 big-endian, determined at compile-time
+ ** 123410 little-endian, determined at compile-time
+ */
+ case SQLITE_TESTCTRL_BYTEORDER: {
+ rc = SQLITE_BYTEORDER*100 + SQLITE_LITTLEENDIAN*10 + SQLITE_BIGENDIAN;
+ break;
+ }
+
+ /* sqlite3_test_control(SQLITE_TESTCTRL_RESERVE, sqlite3 *db, int N)
+ **
+ ** Set the nReserve size to N for the main database on the database
+ ** connection db.
+ */
+ case SQLITE_TESTCTRL_RESERVE: {
+ sqlite3 *db = va_arg(ap, sqlite3*);
+ int x = va_arg(ap,int);
+ sqlite3_mutex_enter(db->mutex);
+ sqlite3BtreeSetPageSize(db->aDb[0].pBt, 0, x, 0);
+ sqlite3_mutex_leave(db->mutex);
+ break;
+ }
+
+ /* sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, sqlite3 *db, int N)
+ **
+ ** Enable or disable various optimizations for testing purposes. The
+ ** argument N is a bitmask of optimizations to be disabled. For normal
+ ** operation N should be 0. The idea is that a test program (like the
+ ** SQL Logic Test or SLT test module) can run the same SQL multiple times
+ ** with various optimizations disabled to verify that the same answer
+ ** is obtained in every case.
+ */
+ case SQLITE_TESTCTRL_OPTIMIZATIONS: {
+ sqlite3 *db = va_arg(ap, sqlite3*);
+ db->dbOptFlags = (u16)(va_arg(ap, int) & 0xffff);
+ break;
+ }
+
+#ifdef SQLITE_N_KEYWORD
+ /* sqlite3_test_control(SQLITE_TESTCTRL_ISKEYWORD, const char *zWord)
+ **
+ ** If zWord is a keyword recognized by the parser, then return the
+ ** number of keywords. Or if zWord is not a keyword, return 0.
+ **
+ ** This test feature is only available in the amalgamation since
+ ** the SQLITE_N_KEYWORD macro is not defined in this file if SQLite
+ ** is built using separate source files.
+ */
+ case SQLITE_TESTCTRL_ISKEYWORD: {
+ const char *zWord = va_arg(ap, const char*);
+ int n = sqlite3Strlen30(zWord);
+ rc = (sqlite3KeywordCode((u8*)zWord, n)!=TK_ID) ? SQLITE_N_KEYWORD : 0;
+ break;
+ }
+#endif
+
+ /* sqlite3_test_control(SQLITE_TESTCTRL_SCRATCHMALLOC, sz, &pNew, pFree);
+ **
+ ** Pass pFree into sqlite3ScratchFree().
+ ** If sz>0 then allocate a scratch buffer into pNew.
+ */
+ case SQLITE_TESTCTRL_SCRATCHMALLOC: {
+ void *pFree, **ppNew;
+ int sz;
+ sz = va_arg(ap, int);
+ ppNew = va_arg(ap, void**);
+ pFree = va_arg(ap, void*);
+ if( sz ) *ppNew = sqlite3ScratchMalloc(sz);
+ sqlite3ScratchFree(pFree);
+ break;
+ }
+
+ /* sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, int onoff);
+ **
+ ** If parameter onoff is non-zero, configure the wrappers so that all
+ ** subsequent calls to localtime() and variants fail. If onoff is zero,
+ ** undo this setting.
+ */
+ case SQLITE_TESTCTRL_LOCALTIME_FAULT: {
+ sqlite3GlobalConfig.bLocaltimeFault = va_arg(ap, int);
+ break;
+ }
+
+ /* sqlite3_test_control(SQLITE_TESTCTRL_NEVER_CORRUPT, int);
+ **
+ ** Set or clear a flag that indicates that the database file is always well-
+ ** formed and never corrupt. This flag is clear by default, indicating that
+ ** database files might have arbitrary corruption. Setting the flag during
+ ** testing causes certain assert() statements in the code to be activated
+ ** that demonstrat invariants on well-formed database files.
+ */
+ case SQLITE_TESTCTRL_NEVER_CORRUPT: {
+ sqlite3GlobalConfig.neverCorrupt = va_arg(ap, int);
+ break;
+ }
+
+
+ /* sqlite3_test_control(SQLITE_TESTCTRL_VDBE_COVERAGE, xCallback, ptr);
+ **
+ ** Set the VDBE coverage callback function to xCallback with context
+ ** pointer ptr.
+ */
+ case SQLITE_TESTCTRL_VDBE_COVERAGE: {
+#ifdef SQLITE_VDBE_COVERAGE
+ typedef void (*branch_callback)(void*,int,u8,u8);
+ sqlite3GlobalConfig.xVdbeBranch = va_arg(ap,branch_callback);
+ sqlite3GlobalConfig.pVdbeBranchArg = va_arg(ap,void*);
+#endif
+ break;
+ }
+
+ /* sqlite3_test_control(SQLITE_TESTCTRL_SORTER_MMAP, db, nMax); */
+ case SQLITE_TESTCTRL_SORTER_MMAP: {
+ sqlite3 *db = va_arg(ap, sqlite3*);
+ db->nMaxSorterMmap = va_arg(ap, int);
+ break;
+ }
+
+ /* sqlite3_test_control(SQLITE_TESTCTRL_ISINIT);
+ **
+ ** Return SQLITE_OK if SQLite has been initialized and SQLITE_ERROR if
+ ** not.
+ */
+ case SQLITE_TESTCTRL_ISINIT: {
+ if( sqlite3GlobalConfig.isInit==0 ) rc = SQLITE_ERROR;
+ break;
+ }
+
+ /* sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, dbName, onOff, tnum);
+ **
+ ** This test control is used to create imposter tables. "db" is a pointer
+ ** to the database connection. dbName is the database name (ex: "main" or
+ ** "temp") which will receive the imposter. "onOff" turns imposter mode on
+ ** or off. "tnum" is the root page of the b-tree to which the imposter
+ ** table should connect.
+ **
+ ** Enable imposter mode only when the schema has already been parsed. Then
+ ** run a single CREATE TABLE statement to construct the imposter table in
+ ** the parsed schema. Then turn imposter mode back off again.
+ **
+ ** If onOff==0 and tnum>0 then reset the schema for all databases, causing
+ ** the schema to be reparsed the next time it is needed. This has the
+ ** effect of erasing all imposter tables.
+ */
+ case SQLITE_TESTCTRL_IMPOSTER: {
+ sqlite3 *db = va_arg(ap, sqlite3*);
+ sqlite3_mutex_enter(db->mutex);
+ db->init.iDb = sqlite3FindDbName(db, va_arg(ap,const char*));
+ db->init.busy = db->init.imposterTable = va_arg(ap,int);
+ db->init.newTnum = va_arg(ap,int);
+ if( db->init.busy==0 && db->init.newTnum>0 ){
+ sqlite3ResetAllSchemasOfConnection(db);
+ }
+ sqlite3_mutex_leave(db->mutex);
+ break;
+ }
+ }
+ va_end(ap);
+#endif /* SQLITE_OMIT_BUILTIN_TEST */
+ return rc;
+}
+
+/*
+** This is a utility routine, useful to VFS implementations, that checks
+** to see if a database file was a URI that contained a specific query
+** parameter, and if so obtains the value of the query parameter.
+**
+** The zFilename argument is the filename pointer passed into the xOpen()
+** method of a VFS implementation. The zParam argument is the name of the
+** query parameter we seek. This routine returns the value of the zParam
+** parameter if it exists. If the parameter does not exist, this routine
+** returns a NULL pointer.
+*/
+SQLITE_API const char *SQLITE_STDCALL sqlite3_uri_parameter(const char *zFilename, const char *zParam){
+ if( zFilename==0 || zParam==0 ) return 0;
+ zFilename += sqlite3Strlen30(zFilename) + 1;
+ while( zFilename[0] ){
+ int x = strcmp(zFilename, zParam);
+ zFilename += sqlite3Strlen30(zFilename) + 1;
+ if( x==0 ) return zFilename;
+ zFilename += sqlite3Strlen30(zFilename) + 1;
+ }
+ return 0;
+}
+
+/*
+** Return a boolean value for a query parameter.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_uri_boolean(const char *zFilename, const char *zParam, int bDflt){
+ const char *z = sqlite3_uri_parameter(zFilename, zParam);
+ bDflt = bDflt!=0;
+ return z ? sqlite3GetBoolean(z, bDflt) : bDflt;
+}
+
+/*
+** Return a 64-bit integer value for a query parameter.
+*/
+SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_uri_int64(
+ const char *zFilename, /* Filename as passed to xOpen */
+ const char *zParam, /* URI parameter sought */
+ sqlite3_int64 bDflt /* return if parameter is missing */
+){
+ const char *z = sqlite3_uri_parameter(zFilename, zParam);
+ sqlite3_int64 v;
+ if( z && sqlite3DecOrHexToI64(z, &v)==SQLITE_OK ){
+ bDflt = v;
+ }
+ return bDflt;
+}
+
+/*
+** Return the Btree pointer identified by zDbName. Return NULL if not found.
+*/
+SQLITE_PRIVATE Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){
+ int i;
+ for(i=0; i<db->nDb; i++){
+ if( db->aDb[i].pBt
+ && (zDbName==0 || sqlite3StrICmp(zDbName, db->aDb[i].zName)==0)
+ ){
+ return db->aDb[i].pBt;
+ }
+ }
+ return 0;
+}
+
+/*
+** Return the filename of the database associated with a database
+** connection.
+*/
+SQLITE_API const char *SQLITE_STDCALL sqlite3_db_filename(sqlite3 *db, const char *zDbName){
+ Btree *pBt;
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) ){
+ (void)SQLITE_MISUSE_BKPT;
+ return 0;
+ }
+#endif
+ pBt = sqlite3DbNameToBtree(db, zDbName);
+ return pBt ? sqlite3BtreeGetFilename(pBt) : 0;
+}
+
+/*
+** Return 1 if database is read-only or 0 if read/write. Return -1 if
+** no such database exists.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_db_readonly(sqlite3 *db, const char *zDbName){
+ Btree *pBt;
+#ifdef SQLITE_ENABLE_API_ARMOR
+ if( !sqlite3SafetyCheckOk(db) ){
+ (void)SQLITE_MISUSE_BKPT;
+ return -1;
+ }
+#endif
+ pBt = sqlite3DbNameToBtree(db, zDbName);
+ return pBt ? sqlite3BtreeIsReadonly(pBt) : -1;
+}
+
+/************** End of main.c ************************************************/
+/************** Begin file notify.c ******************************************/
+/*
+** 2009 March 3
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This file contains the implementation of the sqlite3_unlock_notify()
+** API method and its associated functionality.
+*/
+/* #include "sqliteInt.h" */
+/* #include "btreeInt.h" */
+
+/* Omit this entire file if SQLITE_ENABLE_UNLOCK_NOTIFY is not defined. */
+#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY
+
+/*
+** Public interfaces:
+**
+** sqlite3ConnectionBlocked()
+** sqlite3ConnectionUnlocked()
+** sqlite3ConnectionClosed()
+** sqlite3_unlock_notify()
+*/
+
+#define assertMutexHeld() \
+ assert( sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)) )
+
+/*
+** Head of a linked list of all sqlite3 objects created by this process
+** for which either sqlite3.pBlockingConnection or sqlite3.pUnlockConnection
+** is not NULL. This variable may only accessed while the STATIC_MASTER
+** mutex is held.
+*/
+static sqlite3 *SQLITE_WSD sqlite3BlockedList = 0;
+
+#ifndef NDEBUG
+/*
+** This function is a complex assert() that verifies the following
+** properties of the blocked connections list:
+**
+** 1) Each entry in the list has a non-NULL value for either
+** pUnlockConnection or pBlockingConnection, or both.
+**
+** 2) All entries in the list that share a common value for
+** xUnlockNotify are grouped together.
+**
+** 3) If the argument db is not NULL, then none of the entries in the
+** blocked connections list have pUnlockConnection or pBlockingConnection
+** set to db. This is used when closing connection db.
+*/
+static void checkListProperties(sqlite3 *db){
+ sqlite3 *p;
+ for(p=sqlite3BlockedList; p; p=p->pNextBlocked){
+ int seen = 0;
+ sqlite3 *p2;
+
+ /* Verify property (1) */
+ assert( p->pUnlockConnection || p->pBlockingConnection );
+
+ /* Verify property (2) */
+ for(p2=sqlite3BlockedList; p2!=p; p2=p2->pNextBlocked){
+ if( p2->xUnlockNotify==p->xUnlockNotify ) seen = 1;
+ assert( p2->xUnlockNotify==p->xUnlockNotify || !seen );
+ assert( db==0 || p->pUnlockConnection!=db );
+ assert( db==0 || p->pBlockingConnection!=db );
+ }
+ }
+}
+#else
+# define checkListProperties(x)
+#endif
+
+/*
+** Remove connection db from the blocked connections list. If connection
+** db is not currently a part of the list, this function is a no-op.
+*/
+static void removeFromBlockedList(sqlite3 *db){
+ sqlite3 **pp;
+ assertMutexHeld();
+ for(pp=&sqlite3BlockedList; *pp; pp = &(*pp)->pNextBlocked){
+ if( *pp==db ){
+ *pp = (*pp)->pNextBlocked;
+ break;
+ }
+ }
+}
+
+/*
+** Add connection db to the blocked connections list. It is assumed
+** that it is not already a part of the list.
+*/
+static void addToBlockedList(sqlite3 *db){
+ sqlite3 **pp;
+ assertMutexHeld();
+ for(
+ pp=&sqlite3BlockedList;
+ *pp && (*pp)->xUnlockNotify!=db->xUnlockNotify;
+ pp=&(*pp)->pNextBlocked
+ );
+ db->pNextBlocked = *pp;
+ *pp = db;
+}
+
+/*
+** Obtain the STATIC_MASTER mutex.
+*/
+static void enterMutex(void){
+ sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER));
+ checkListProperties(0);
+}
+
+/*
+** Release the STATIC_MASTER mutex.
+*/
+static void leaveMutex(void){
+ assertMutexHeld();
+ checkListProperties(0);
+ sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER));
+}
+
+/*
+** Register an unlock-notify callback.
+**
+** This is called after connection "db" has attempted some operation
+** but has received an SQLITE_LOCKED error because another connection
+** (call it pOther) in the same process was busy using the same shared
+** cache. pOther is found by looking at db->pBlockingConnection.
+**
+** If there is no blocking connection, the callback is invoked immediately,
+** before this routine returns.
+**
+** If pOther is already blocked on db, then report SQLITE_LOCKED, to indicate
+** a deadlock.
+**
+** Otherwise, make arrangements to invoke xNotify when pOther drops
+** its locks.
+**
+** Each call to this routine overrides any prior callbacks registered
+** on the same "db". If xNotify==0 then any prior callbacks are immediately
+** cancelled.
+*/
+SQLITE_API int SQLITE_STDCALL sqlite3_unlock_notify(
+ sqlite3 *db,
+ void (*xNotify)(void **, int),
+ void *pArg
+){
+ int rc = SQLITE_OK;
+
+ sqlite3_mutex_enter(db->mutex);
+ enterMutex();
+
+ if( xNotify==0 ){
+ removeFromBlockedList(db);
+ db->pBlockingConnection = 0;
+ db->pUnlockConnection = 0;
+ db->xUnlockNotify = 0;
+ db->pUnlockArg = 0;
+ }else if( 0==db->pBlockingConnection ){
+ /* The blocking transaction has been concluded. Or there never was a
+ ** blocking transaction. In either case, invoke the notify callback
+ ** immediately.
+ */
+ xNotify(&pArg, 1);
+ }else{
+ sqlite3 *p;
+
+ for(p=db->pBlockingConnection; p && p!=db; p=p->pUnlockConnection){}
+ if( p ){
+ rc = SQLITE_LOCKED; /* Deadlock detected. */
+ }else{
+ db->pUnlockConnection = db->pBlockingConnection;
+ db->xUnlockNotify = xNotify;
+ db->pUnlockArg = pArg;
+ removeFromBlockedList(db);
+ addToBlockedList(db);
+ }
+ }
+
+ leaveMutex();
+ assert( !db->mallocFailed );
+ sqlite3ErrorWithMsg(db, rc, (rc?"database is deadlocked":0));
+ sqlite3_mutex_leave(db->mutex);
+ return rc;
+}
+
+/*
+** This function is called while stepping or preparing a statement
+** associated with connection db. The operation will return SQLITE_LOCKED
+** to the user because it requires a lock that will not be available
+** until connection pBlocker concludes its current transaction.
+*/
+SQLITE_PRIVATE void sqlite3ConnectionBlocked(sqlite3 *db, sqlite3 *pBlocker){
+ enterMutex();
+ if( db->pBlockingConnection==0 && db->pUnlockConnection==0 ){
+ addToBlockedList(db);
+ }
+ db->pBlockingConnection = pBlocker;
+ leaveMutex();
+}
+
+/*
+** This function is called when
+** the transaction opened by database db has just finished. Locks held
+** by database connection db have been released.
+**
+** This function loops through each entry in the blocked connections
+** list and does the following:
+**
+** 1) If the sqlite3.pBlockingConnection member of a list entry is
+** set to db, then set pBlockingConnection=0.
+**
+** 2) If the sqlite3.pUnlockConnection member of a list entry is
+** set to db, then invoke the configured unlock-notify callback and
+** set pUnlockConnection=0.
+**
+** 3) If the two steps above mean that pBlockingConnection==0 and
+** pUnlockConnection==0, remove the entry from the blocked connections
+** list.
+*/
+SQLITE_PRIVATE void sqlite3ConnectionUnlocked(sqlite3 *db){
+ void (*xUnlockNotify)(void **, int) = 0; /* Unlock-notify cb to invoke */
+ int nArg = 0; /* Number of entries in aArg[] */
+ sqlite3 **pp; /* Iterator variable */
+ void **aArg; /* Arguments to the unlock callback */
+ void **aDyn = 0; /* Dynamically allocated space for aArg[] */
+ void *aStatic[16]; /* Starter space for aArg[]. No malloc required */
+
+ aArg = aStatic;
+ enterMutex(); /* Enter STATIC_MASTER mutex */
+
+ /* This loop runs once for each entry in the blocked-connections list. */
+ for(pp=&sqlite3BlockedList; *pp; /* no-op */ ){
+ sqlite3 *p = *pp;
+
+ /* Step 1. */
+ if( p->pBlockingConnection==db ){
+ p->pBlockingConnection = 0;
+ }
+
+ /* Step 2. */
+ if( p->pUnlockConnection==db ){
+ assert( p->xUnlockNotify );
+ if( p->xUnlockNotify!=xUnlockNotify && nArg!=0 ){
+ xUnlockNotify(aArg, nArg);
+ nArg = 0;
+ }
+
+ sqlite3BeginBenignMalloc();
+ assert( aArg==aDyn || (aDyn==0 && aArg==aStatic) );
+ assert( nArg<=(int)ArraySize(aStatic) || aArg==aDyn );
+ if( (!aDyn && nArg==(int)ArraySize(aStatic))
+ || (aDyn && nArg==(int)(sqlite3MallocSize(aDyn)/sizeof(void*)))
+ ){
+ /* The aArg[] array needs to grow. */
+ void **pNew = (void **)sqlite3Malloc(nArg*sizeof(void *)*2);
+ if( pNew ){
+ memcpy(pNew, aArg, nArg*sizeof(void *));
+ sqlite3_free(aDyn);
+ aDyn = aArg = pNew;
+ }else{
+ /* This occurs when the array of context pointers that need to
+ ** be passed to the unlock-notify callback is larger than the
+ ** aStatic[] array allocated on the stack and the attempt to
+ ** allocate a larger array from the heap has failed.
+ **
+ ** This is a difficult situation to handle. Returning an error
+ ** code to the caller is insufficient, as even if an error code
+ ** is returned the transaction on connection db will still be
+ ** closed and the unlock-notify callbacks on blocked connections
+ ** will go unissued. This might cause the application to wait
+ ** indefinitely for an unlock-notify callback that will never
+ ** arrive.
+ **
+ ** Instead, invoke the unlock-notify callback with the context
+ ** array already accumulated. We can then clear the array and
+ ** begin accumulating any further context pointers without
+ ** requiring any dynamic allocation. This is sub-optimal because
+ ** it means that instead of one callback with a large array of
+ ** context pointers the application will receive two or more
+ ** callbacks with smaller arrays of context pointers, which will
+ ** reduce the applications ability to prioritize multiple
+ ** connections. But it is the best that can be done under the
+ ** circumstances.
+ */
+ xUnlockNotify(aArg, nArg);
+ nArg = 0;
+ }
+ }
+ sqlite3EndBenignMalloc();
+
+ aArg[nArg++] = p->pUnlockArg;
+ xUnlockNotify = p->xUnlockNotify;
+ p->pUnlockConnection = 0;
+ p->xUnlockNotify = 0;
+ p->pUnlockArg = 0;
+ }
+
+ /* Step 3. */
+ if( p->pBlockingConnection==0 && p->pUnlockConnection==0 ){
+ /* Remove connection p from the blocked connections list. */
+ *pp = p->pNextBlocked;
+ p->pNextBlocked = 0;
+ }else{
+ pp = &p->pNextBlocked;
+ }
+ }
+
+ if( nArg!=0 ){
+ xUnlockNotify(aArg, nArg);
+ }
+ sqlite3_free(aDyn);
+ leaveMutex(); /* Leave STATIC_MASTER mutex */
+}
+
+/*
+** This is called when the database connection passed as an argument is
+** being closed. The connection is removed from the blocked list.
+*/
+SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){
+ sqlite3ConnectionUnlocked(db);
+ enterMutex();
+ removeFromBlockedList(db);
+ checkListProperties(db);
+ leaveMutex();
+}
+#endif
+
+/************** End of notify.c **********************************************/
+/************** Begin file fts3.c ********************************************/
+/*
+** 2006 Oct 10
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This is an SQLite module implementing full-text search.
+*/
+
+/*
+** The code in this file is only compiled if:
+**
+** * The FTS3 module is being built as an extension
+** (in which case SQLITE_CORE is not defined), or
+**
+** * The FTS3 module is being built into the core of
+** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
+*/
+
+/* The full-text index is stored in a series of b+tree (-like)
+** structures called segments which map terms to doclists. The
+** structures are like b+trees in layout, but are constructed from the
+** bottom up in optimal fashion and are not updatable. Since trees
+** are built from the bottom up, things will be described from the
+** bottom up.
+**
+**
+**** Varints ****
+** The basic unit of encoding is a variable-length integer called a
+** varint. We encode variable-length integers in little-endian order
+** using seven bits * per byte as follows:
+**
+** KEY:
+** A = 0xxxxxxx 7 bits of data and one flag bit
+** B = 1xxxxxxx 7 bits of data and one flag bit
+**
+** 7 bits - A
+** 14 bits - BA
+** 21 bits - BBA
+** and so on.
+**
+** This is similar in concept to how sqlite encodes "varints" but
+** the encoding is not the same. SQLite varints are big-endian
+** are are limited to 9 bytes in length whereas FTS3 varints are
+** little-endian and can be up to 10 bytes in length (in theory).
+**
+** Example encodings:
+**
+** 1: 0x01
+** 127: 0x7f
+** 128: 0x81 0x00
+**
+**
+**** Document lists ****
+** A doclist (document list) holds a docid-sorted list of hits for a
+** given term. Doclists hold docids and associated token positions.
+** A docid is the unique integer identifier for a single document.
+** A position is the index of a word within the document. The first
+** word of the document has a position of 0.
+**
+** FTS3 used to optionally store character offsets using a compile-time
+** option. But that functionality is no longer supported.
+**
+** A doclist is stored like this:
+**
+** array {
+** varint docid; (delta from previous doclist)
+** array { (position list for column 0)
+** varint position; (2 more than the delta from previous position)
+** }
+** array {
+** varint POS_COLUMN; (marks start of position list for new column)
+** varint column; (index of new column)
+** array {
+** varint position; (2 more than the delta from previous position)
+** }
+** }
+** varint POS_END; (marks end of positions for this document.
+** }
+**
+** Here, array { X } means zero or more occurrences of X, adjacent in
+** memory. A "position" is an index of a token in the token stream
+** generated by the tokenizer. Note that POS_END and POS_COLUMN occur
+** in the same logical place as the position element, and act as sentinals
+** ending a position list array. POS_END is 0. POS_COLUMN is 1.
+** The positions numbers are not stored literally but rather as two more
+** than the difference from the prior position, or the just the position plus
+** 2 for the first position. Example:
+**
+** label: A B C D E F G H I J K
+** value: 123 5 9 1 1 14 35 0 234 72 0
+**
+** The 123 value is the first docid. For column zero in this document
+** there are two matches at positions 3 and 10 (5-2 and 9-2+3). The 1
+** at D signals the start of a new column; the 1 at E indicates that the
+** new column is column number 1. There are two positions at 12 and 45
+** (14-2 and 35-2+12). The 0 at H indicate the end-of-document. The
+** 234 at I is the delta to next docid (357). It has one position 70
+** (72-2) and then terminates with the 0 at K.
+**
+** A "position-list" is the list of positions for multiple columns for
+** a single docid. A "column-list" is the set of positions for a single
+** column. Hence, a position-list consists of one or more column-lists,
+** a document record consists of a docid followed by a position-list and
+** a doclist consists of one or more document records.
+**
+** A bare doclist omits the position information, becoming an
+** array of varint-encoded docids.
+**
+**** Segment leaf nodes ****
+** Segment leaf nodes store terms and doclists, ordered by term. Leaf
+** nodes are written using LeafWriter, and read using LeafReader (to
+** iterate through a single leaf node's data) and LeavesReader (to
+** iterate through a segment's entire leaf layer). Leaf nodes have
+** the format:
+**
+** varint iHeight; (height from leaf level, always 0)
+** varint nTerm; (length of first term)
+** char pTerm[nTerm]; (content of first term)
+** varint nDoclist; (length of term's associated doclist)
+** char pDoclist[nDoclist]; (content of doclist)
+** array {
+** (further terms are delta-encoded)
+** varint nPrefix; (length of prefix shared with previous term)
+** varint nSuffix; (length of unshared suffix)
+** char pTermSuffix[nSuffix];(unshared suffix of next term)
+** varint nDoclist; (length of term's associated doclist)
+** char pDoclist[nDoclist]; (content of doclist)
+** }
+**
+** Here, array { X } means zero or more occurrences of X, adjacent in
+** memory.
+**
+** Leaf nodes are broken into blocks which are stored contiguously in
+** the %_segments table in sorted order. This means that when the end
+** of a node is reached, the next term is in the node with the next
+** greater node id.
+**
+** New data is spilled to a new leaf node when the current node
+** exceeds LEAF_MAX bytes (default 2048). New data which itself is
+** larger than STANDALONE_MIN (default 1024) is placed in a standalone
+** node (a leaf node with a single term and doclist). The goal of
+** these settings is to pack together groups of small doclists while
+** making it efficient to directly access large doclists. The
+** assumption is that large doclists represent terms which are more
+** likely to be query targets.
+**
+** TODO(shess) It may be useful for blocking decisions to be more
+** dynamic. For instance, it may make more sense to have a 2.5k leaf
+** node rather than splitting into 2k and .5k nodes. My intuition is
+** that this might extend through 2x or 4x the pagesize.
+**
+**
+**** Segment interior nodes ****
+** Segment interior nodes store blockids for subtree nodes and terms
+** to describe what data is stored by the each subtree. Interior
+** nodes are written using InteriorWriter, and read using
+** InteriorReader. InteriorWriters are created as needed when
+** SegmentWriter creates new leaf nodes, or when an interior node
+** itself grows too big and must be split. The format of interior
+** nodes:
+**
+** varint iHeight; (height from leaf level, always >0)
+** varint iBlockid; (block id of node's leftmost subtree)
+** optional {
+** varint nTerm; (length of first term)
+** char pTerm[nTerm]; (content of first term)
+** array {
+** (further terms are delta-encoded)
+** varint nPrefix; (length of shared prefix with previous term)
+** varint nSuffix; (length of unshared suffix)
+** char pTermSuffix[nSuffix]; (unshared suffix of next term)
+** }
+** }
+**
+** Here, optional { X } means an optional element, while array { X }
+** means zero or more occurrences of X, adjacent in memory.
+**
+** An interior node encodes n terms separating n+1 subtrees. The
+** subtree blocks are contiguous, so only the first subtree's blockid
+** is encoded. The subtree at iBlockid will contain all terms less
+** than the first term encoded (or all terms if no term is encoded).
+** Otherwise, for terms greater than or equal to pTerm[i] but less
+** than pTerm[i+1], the subtree for that term will be rooted at
+** iBlockid+i. Interior nodes only store enough term data to
+** distinguish adjacent children (if the rightmost term of the left
+** child is "something", and the leftmost term of the right child is
+** "wicked", only "w" is stored).
+**
+** New data is spilled to a new interior node at the same height when
+** the current node exceeds INTERIOR_MAX bytes (default 2048).
+** INTERIOR_MIN_TERMS (default 7) keeps large terms from monopolizing
+** interior nodes and making the tree too skinny. The interior nodes
+** at a given height are naturally tracked by interior nodes at
+** height+1, and so on.
+**
+**
+**** Segment directory ****
+** The segment directory in table %_segdir stores meta-information for
+** merging and deleting segments, and also the root node of the
+** segment's tree.
+**
+** The root node is the top node of the segment's tree after encoding
+** the entire segment, restricted to ROOT_MAX bytes (default 1024).
+** This could be either a leaf node or an interior node. If the top
+** node requires more than ROOT_MAX bytes, it is flushed to %_segments
+** and a new root interior node is generated (which should always fit
+** within ROOT_MAX because it only needs space for 2 varints, the
+** height and the blockid of the previous root).
+**
+** The meta-information in the segment directory is:
+** level - segment level (see below)
+** idx - index within level
+** - (level,idx uniquely identify a segment)
+** start_block - first leaf node
+** leaves_end_block - last leaf node
+** end_block - last block (including interior nodes)
+** root - contents of root node
+**
+** If the root node is a leaf node, then start_block,
+** leaves_end_block, and end_block are all 0.
+**
+**
+**** Segment merging ****
+** To amortize update costs, segments are grouped into levels and
+** merged in batches. Each increase in level represents exponentially
+** more documents.
+**
+** New documents (actually, document updates) are tokenized and
+** written individually (using LeafWriter) to a level 0 segment, with
+** incrementing idx. When idx reaches MERGE_COUNT (default 16), all
+** level 0 segments are merged into a single level 1 segment. Level 1
+** is populated like level 0, and eventually MERGE_COUNT level 1
+** segments are merged to a single level 2 segment (representing
+** MERGE_COUNT^2 updates), and so on.
+**
+** A segment merge traverses all segments at a given level in
+** parallel, performing a straightforward sorted merge. Since segment
+** leaf nodes are written in to the %_segments table in order, this
+** merge traverses the underlying sqlite disk structures efficiently.
+** After the merge, all segment blocks from the merged level are
+** deleted.
+**
+** MERGE_COUNT controls how often we merge segments. 16 seems to be
+** somewhat of a sweet spot for insertion performance. 32 and 64 show
+** very similar performance numbers to 16 on insertion, though they're
+** a tiny bit slower (perhaps due to more overhead in merge-time
+** sorting). 8 is about 20% slower than 16, 4 about 50% slower than
+** 16, 2 about 66% slower than 16.
+**
+** At query time, high MERGE_COUNT increases the number of segments
+** which need to be scanned and merged. For instance, with 100k docs
+** inserted:
+**
+** MERGE_COUNT segments
+** 16 25
+** 8 12
+** 4 10
+** 2 6
+**
+** This appears to have only a moderate impact on queries for very
+** frequent terms (which are somewhat dominated by segment merge
+** costs), and infrequent and non-existent terms still seem to be fast
+** even with many segments.
+**
+** TODO(shess) That said, it would be nice to have a better query-side
+** argument for MERGE_COUNT of 16. Also, it is possible/likely that
+** optimizations to things like doclist merging will swing the sweet
+** spot around.
+**
+**
+**
+**** Handling of deletions and updates ****
+** Since we're using a segmented structure, with no docid-oriented
+** index into the term index, we clearly cannot simply update the term
+** index when a document is deleted or updated. For deletions, we
+** write an empty doclist (varint(docid) varint(POS_END)), for updates
+** we simply write the new doclist. Segment merges overwrite older
+** data for a particular docid with newer data, so deletes or updates
+** will eventually overtake the earlier data and knock it out. The
+** query logic likewise merges doclists so that newer data knocks out
+** older data.
+*/
+
+/************** Include fts3Int.h in the middle of fts3.c ********************/
+/************** Begin file fts3Int.h *****************************************/
+/*
+** 2009 Nov 12
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+*/
+#ifndef _FTSINT_H
+#define _FTSINT_H
+
+#if !defined(NDEBUG) && !defined(SQLITE_DEBUG)
+# define NDEBUG 1
+#endif
+
+/*
+** FTS4 is really an extension for FTS3. It is enabled using the
+** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all
+** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3.
+*/
+#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3)
+# define SQLITE_ENABLE_FTS3
+#endif
+
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+/* If not building as part of the core, include sqlite3ext.h. */
+#ifndef SQLITE_CORE
+/* # include "sqlite3ext.h" */
+SQLITE_EXTENSION_INIT3
+#endif
+
+/* #include "sqlite3.h" */
+/************** Include fts3_tokenizer.h in the middle of fts3Int.h **********/
+/************** Begin file fts3_tokenizer.h **********************************/
+/*
+** 2006 July 10
+**
+** The author disclaims copyright to this source code.
+**
+*************************************************************************
+** Defines the interface to tokenizers used by fulltext-search. There
+** are three basic components:
+**
+** sqlite3_tokenizer_module is a singleton defining the tokenizer
+** interface functions. This is essentially the class structure for
+** tokenizers.
+**
+** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
+** including customization information defined at creation time.
+**
+** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
+** tokens from a particular input.
+*/
+#ifndef _FTS3_TOKENIZER_H_
+#define _FTS3_TOKENIZER_H_
+
+/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
+** If tokenizers are to be allowed to call sqlite3_*() functions, then
+** we will need a way to register the API consistently.
+*/
+/* #include "sqlite3.h" */
+
+/*
+** Structures used by the tokenizer interface. When a new tokenizer
+** implementation is registered, the caller provides a pointer to
+** an sqlite3_tokenizer_module containing pointers to the callback
+** functions that make up an implementation.
+**
+** When an fts3 table is created, it passes any arguments passed to
+** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the
+** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer
+** implementation. The xCreate() function in turn returns an
+** sqlite3_tokenizer structure representing the specific tokenizer to
+** be used for the fts3 table (customized by the tokenizer clause arguments).
+**
+** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen()
+** method is called. It returns an sqlite3_tokenizer_cursor object
+** that may be used to tokenize a specific input buffer based on
+** the tokenization rules supplied by a specific sqlite3_tokenizer
+** object.
+*/
+typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
+typedef struct sqlite3_tokenizer sqlite3_tokenizer;
+typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
+
+struct sqlite3_tokenizer_module {
+
+ /*
+ ** Structure version. Should always be set to 0 or 1.
+ */
+ int iVersion;
+
+ /*
+ ** Create a new tokenizer. The values in the argv[] array are the
+ ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL
+ ** TABLE statement that created the fts3 table. For example, if
+ ** the following SQL is executed:
+ **
+ ** CREATE .. USING fts3( ... , tokenizer <tokenizer-name> arg1 arg2)
+ **
+ ** then argc is set to 2, and the argv[] array contains pointers
+ ** to the strings "arg1" and "arg2".
+ **
+ ** This method should return either SQLITE_OK (0), or an SQLite error
+ ** code. If SQLITE_OK is returned, then *ppTokenizer should be set
+ ** to point at the newly created tokenizer structure. The generic
+ ** sqlite3_tokenizer.pModule variable should not be initialized by
+ ** this callback. The caller will do so.
+ */
+ int (*xCreate)(
+ int argc, /* Size of argv array */
+ const char *const*argv, /* Tokenizer argument strings */
+ sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */
+ );
+
+ /*
+ ** Destroy an existing tokenizer. The fts3 module calls this method
+ ** exactly once for each successful call to xCreate().
+ */
+ int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
+
+ /*
+ ** Create a tokenizer cursor to tokenize an input buffer. The caller
+ ** is responsible for ensuring that the input buffer remains valid
+ ** until the cursor is closed (using the xClose() method).
+ */
+ int (*xOpen)(
+ sqlite3_tokenizer *pTokenizer, /* Tokenizer object */
+ const char *pInput, int nBytes, /* Input buffer */
+ sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */
+ );
+
+ /*
+ ** Destroy an existing tokenizer cursor. The fts3 module calls this
+ ** method exactly once for each successful call to xOpen().
+ */
+ int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
+
+ /*
+ ** Retrieve the next token from the tokenizer cursor pCursor. This
+ ** method should either return SQLITE_OK and set the values of the
+ ** "OUT" variables identified below, or SQLITE_DONE to indicate that
+ ** the end of the buffer has been reached, or an SQLite error code.
+ **
+ ** *ppToken should be set to point at a buffer containing the
+ ** normalized version of the token (i.e. after any case-folding and/or
+ ** stemming has been performed). *pnBytes should be set to the length
+ ** of this buffer in bytes. The input text that generated the token is
+ ** identified by the byte offsets returned in *piStartOffset and
+ ** *piEndOffset. *piStartOffset should be set to the index of the first
+ ** byte of the token in the input buffer. *piEndOffset should be set
+ ** to the index of the first byte just past the end of the token in
+ ** the input buffer.
+ **
+ ** The buffer *ppToken is set to point at is managed by the tokenizer
+ ** implementation. It is only required to be valid until the next call
+ ** to xNext() or xClose().
+ */
+ /* TODO(shess) current implementation requires pInput to be
+ ** nul-terminated. This should either be fixed, or pInput/nBytes
+ ** should be converted to zInput.
+ */
+ int (*xNext)(
+ sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */
+ const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */
+ int *piStartOffset, /* OUT: Byte offset of token in input buffer */
+ int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */
+ int *piPosition /* OUT: Number of tokens returned before this one */
+ );
+
+ /***********************************************************************
+ ** Methods below this point are only available if iVersion>=1.
+ */
+
+ /*
+ ** Configure the language id of a tokenizer cursor.
+ */
+ int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid);
+};
+
+struct sqlite3_tokenizer {
+ const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */
+ /* Tokenizer implementations will typically add additional fields */
+};
+
+struct sqlite3_tokenizer_cursor {
+ sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */
+ /* Tokenizer implementations will typically add additional fields */
+};
+
+int fts3_global_term_cnt(int iTerm, int iCol);
+int fts3_term_cnt(int iTerm, int iCol);
+
+
+#endif /* _FTS3_TOKENIZER_H_ */
+
+/************** End of fts3_tokenizer.h **************************************/
+/************** Continuing where we left off in fts3Int.h ********************/
+/************** Include fts3_hash.h in the middle of fts3Int.h ***************/
+/************** Begin file fts3_hash.h ***************************************/
+/*
+** 2001 September 22
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This is the header file for the generic hash-table implementation
+** used in SQLite. We've modified it slightly to serve as a standalone
+** hash table implementation for the full-text indexing module.
+**
+*/
+#ifndef _FTS3_HASH_H_
+#define _FTS3_HASH_H_
+
+/* Forward declarations of structures. */
+typedef struct Fts3Hash Fts3Hash;
+typedef struct Fts3HashElem Fts3HashElem;
+
+/* A complete hash table is an instance of the following structure.
+** The internals of this structure are intended to be opaque -- client
+** code should not attempt to access or modify the fields of this structure
+** directly. Change this structure only by using the routines below.
+** However, many of the "procedures" and "functions" for modifying and
+** accessing this structure are really macros, so we can't really make
+** this structure opaque.
+*/
+struct Fts3Hash {
+ char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */
+ char copyKey; /* True if copy of key made on insert */
+ int count; /* Number of entries in this table */
+ Fts3HashElem *first; /* The first element of the array */
+ int htsize; /* Number of buckets in the hash table */
+ struct _fts3ht { /* the hash table */
+ int count; /* Number of entries with this hash */
+ Fts3HashElem *chain; /* Pointer to first entry with this hash */
+ } *ht;
+};
+
+/* Each element in the hash table is an instance of the following
+** structure. All elements are stored on a single doubly-linked list.
+**
+** Again, this structure is intended to be opaque, but it can't really
+** be opaque because it is used by macros.
+*/
+struct Fts3HashElem {
+ Fts3HashElem *next, *prev; /* Next and previous elements in the table */
+ void *data; /* Data associated with this element */
+ void *pKey; int nKey; /* Key associated with this element */
+};
+
+/*
+** There are 2 different modes of operation for a hash table:
+**
+** FTS3_HASH_STRING pKey points to a string that is nKey bytes long
+** (including the null-terminator, if any). Case
+** is respected in comparisons.
+**
+** FTS3_HASH_BINARY pKey points to binary data nKey bytes long.
+** memcmp() is used to compare keys.
+**
+** A copy of the key is made if the copyKey parameter to fts3HashInit is 1.
+*/
+#define FTS3_HASH_STRING 1
+#define FTS3_HASH_BINARY 2
+
+/*
+** Access routines. To delete, insert a NULL pointer.
+*/
+SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey);
+SQLITE_PRIVATE void *sqlite3Fts3HashInsert(Fts3Hash*, const void *pKey, int nKey, void *pData);
+SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash*, const void *pKey, int nKey);
+SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash*);
+SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const void *, int);
+
+/*
+** Shorthand for the functions above
+*/
+#define fts3HashInit sqlite3Fts3HashInit
+#define fts3HashInsert sqlite3Fts3HashInsert
+#define fts3HashFind sqlite3Fts3HashFind
+#define fts3HashClear sqlite3Fts3HashClear
+#define fts3HashFindElem sqlite3Fts3HashFindElem
+
+/*
+** Macros for looping over all elements of a hash table. The idiom is
+** like this:
+**
+** Fts3Hash h;
+** Fts3HashElem *p;
+** ...
+** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){
+** SomeStructure *pData = fts3HashData(p);
+** // do something with pData
+** }
+*/
+#define fts3HashFirst(H) ((H)->first)
+#define fts3HashNext(E) ((E)->next)
+#define fts3HashData(E) ((E)->data)
+#define fts3HashKey(E) ((E)->pKey)
+#define fts3HashKeysize(E) ((E)->nKey)
+
+/*
+** Number of entries in a hash table
+*/
+#define fts3HashCount(H) ((H)->count)
+
+#endif /* _FTS3_HASH_H_ */
+
+/************** End of fts3_hash.h *******************************************/
+/************** Continuing where we left off in fts3Int.h ********************/
+
+/*
+** This constant determines the maximum depth of an FTS expression tree
+** that the library will create and use. FTS uses recursion to perform
+** various operations on the query tree, so the disadvantage of a large
+** limit is that it may allow very large queries to use large amounts
+** of stack space (perhaps causing a stack overflow).
+*/
+#ifndef SQLITE_FTS3_MAX_EXPR_DEPTH
+# define SQLITE_FTS3_MAX_EXPR_DEPTH 12
+#endif
+
+
+/*
+** This constant controls how often segments are merged. Once there are
+** FTS3_MERGE_COUNT segments of level N, they are merged into a single
+** segment of level N+1.
+*/
+#define FTS3_MERGE_COUNT 16
+
+/*
+** This is the maximum amount of data (in bytes) to store in the
+** Fts3Table.pendingTerms hash table. Normally, the hash table is
+** populated as documents are inserted/updated/deleted in a transaction
+** and used to create a new segment when the transaction is committed.
+** However if this limit is reached midway through a transaction, a new
+** segment is created and the hash table cleared immediately.
+*/
+#define FTS3_MAX_PENDING_DATA (1*1024*1024)
+
+/*
+** Macro to return the number of elements in an array. SQLite has a
+** similar macro called ArraySize(). Use a different name to avoid
+** a collision when building an amalgamation with built-in FTS3.
+*/
+#define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0])))
+
+
+#ifndef MIN
+# define MIN(x,y) ((x)<(y)?(x):(y))
+#endif
+#ifndef MAX
+# define MAX(x,y) ((x)>(y)?(x):(y))
+#endif
+
+/*
+** Maximum length of a varint encoded integer. The varint format is different
+** from that used by SQLite, so the maximum length is 10, not 9.
+*/
+#define FTS3_VARINT_MAX 10
+
+/*
+** FTS4 virtual tables may maintain multiple indexes - one index of all terms
+** in the document set and zero or more prefix indexes. All indexes are stored
+** as one or more b+-trees in the %_segments and %_segdir tables.
+**
+** It is possible to determine which index a b+-tree belongs to based on the
+** value stored in the "%_segdir.level" column. Given this value L, the index
+** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with
+** level values between 0 and 1023 (inclusive) belong to index 0, all levels
+** between 1024 and 2047 to index 1, and so on.
+**
+** It is considered impossible for an index to use more than 1024 levels. In
+** theory though this may happen, but only after at least
+** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables.
+*/
+#define FTS3_SEGDIR_MAXLEVEL 1024
+#define FTS3_SEGDIR_MAXLEVEL_STR "1024"
+
+/*
+** The testcase() macro is only used by the amalgamation. If undefined,
+** make it a no-op.
+*/
+#ifndef testcase
+# define testcase(X)
+#endif
+
+/*
+** Terminator values for position-lists and column-lists.
+*/
+#define POS_COLUMN (1) /* Column-list terminator */
+#define POS_END (0) /* Position-list terminator */
+
+/*
+** This section provides definitions to allow the
+** FTS3 extension to be compiled outside of the
+** amalgamation.
+*/
+#ifndef SQLITE_AMALGAMATION
+/*
+** Macros indicating that conditional expressions are always true or
+** false.
+*/
+#ifdef SQLITE_COVERAGE_TEST
+# define ALWAYS(x) (1)
+# define NEVER(X) (0)
+#elif defined(SQLITE_DEBUG)
+# define ALWAYS(x) sqlite3Fts3Always((x)!=0)
+# define NEVER(x) sqlite3Fts3Never((x)!=0)
+SQLITE_PRIVATE int sqlite3Fts3Always(int b);
+SQLITE_PRIVATE int sqlite3Fts3Never(int b);
+#else
+# define ALWAYS(x) (x)
+# define NEVER(x) (x)
+#endif
+
+/*
+** Internal types used by SQLite.
+*/
+typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */
+typedef short int i16; /* 2-byte (or larger) signed integer */
+typedef unsigned int u32; /* 4-byte unsigned integer */
+typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */
+typedef sqlite3_int64 i64; /* 8-byte signed integer */
+
+/*
+** Macro used to suppress compiler warnings for unused parameters.
+*/
+#define UNUSED_PARAMETER(x) (void)(x)
+
+/*
+** Activate assert() only if SQLITE_TEST is enabled.
+*/
+#if !defined(NDEBUG) && !defined(SQLITE_DEBUG)
+# define NDEBUG 1
+#endif
+
+/*
+** The TESTONLY macro is used to enclose variable declarations or
+** other bits of code that are needed to support the arguments
+** within testcase() and assert() macros.
+*/
+#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST)
+# define TESTONLY(X) X
+#else
+# define TESTONLY(X)
+#endif
+
+#endif /* SQLITE_AMALGAMATION */
+
+#ifdef SQLITE_DEBUG
+SQLITE_PRIVATE int sqlite3Fts3Corrupt(void);
+# define FTS_CORRUPT_VTAB sqlite3Fts3Corrupt()
+#else
+# define FTS_CORRUPT_VTAB SQLITE_CORRUPT_VTAB
+#endif
+
+typedef struct Fts3Table Fts3Table;
+typedef struct Fts3Cursor Fts3Cursor;
+typedef struct Fts3Expr Fts3Expr;
+typedef struct Fts3Phrase Fts3Phrase;
+typedef struct Fts3PhraseToken Fts3PhraseToken;
+
+typedef struct Fts3Doclist Fts3Doclist;
+typedef struct Fts3SegFilter Fts3SegFilter;
+typedef struct Fts3DeferredToken Fts3DeferredToken;
+typedef struct Fts3SegReader Fts3SegReader;
+typedef struct Fts3MultiSegReader Fts3MultiSegReader;
+
+typedef struct MatchinfoBuffer MatchinfoBuffer;
+
+/*
+** A connection to a fulltext index is an instance of the following
+** structure. The xCreate and xConnect methods create an instance
+** of this structure and xDestroy and xDisconnect free that instance.
+** All other methods receive a pointer to the structure as one of their
+** arguments.
+*/
+struct Fts3Table {
+ sqlite3_vtab base; /* Base class used by SQLite core */
+ sqlite3 *db; /* The database connection */
+ const char *zDb; /* logical database name */
+ const char *zName; /* virtual table name */
+ int nColumn; /* number of named columns in virtual table */
+ char **azColumn; /* column names. malloced */
+ u8 *abNotindexed; /* True for 'notindexed' columns */
+ sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */
+ char *zContentTbl; /* content=xxx option, or NULL */
+ char *zLanguageid; /* languageid=xxx option, or NULL */
+ int nAutoincrmerge; /* Value configured by 'automerge' */
+ u32 nLeafAdd; /* Number of leaf blocks added this trans */
+
+ /* Precompiled statements used by the implementation. Each of these
+ ** statements is run and reset within a single virtual table API call.
+ */
+ sqlite3_stmt *aStmt[40];
+
+ char *zReadExprlist;
+ char *zWriteExprlist;
+
+ int nNodeSize; /* Soft limit for node size */
+ u8 bFts4; /* True for FTS4, false for FTS3 */
+ u8 bHasStat; /* True if %_stat table exists (2==unknown) */
+ u8 bHasDocsize; /* True if %_docsize table exists */
+ u8 bDescIdx; /* True if doclists are in reverse order */
+ u8 bIgnoreSavepoint; /* True to ignore xSavepoint invocations */
+ int nPgsz; /* Page size for host database */
+ char *zSegmentsTbl; /* Name of %_segments table */
+ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */
+
+ /*
+ ** The following array of hash tables is used to buffer pending index
+ ** updates during transactions. All pending updates buffered at any one
+ ** time must share a common language-id (see the FTS4 langid= feature).
+ ** The current language id is stored in variable iPrevLangid.
+ **
+ ** A single FTS4 table may have multiple full-text indexes. For each index
+ ** there is an entry in the aIndex[] array. Index 0 is an index of all the
+ ** terms that appear in the document set. Each subsequent index in aIndex[]
+ ** is an index of prefixes of a specific length.
+ **
+ ** Variable nPendingData contains an estimate the memory consumed by the
+ ** pending data structures, including hash table overhead, but not including
+ ** malloc overhead. When nPendingData exceeds nMaxPendingData, all hash
+ ** tables are flushed to disk. Variable iPrevDocid is the docid of the most
+ ** recently inserted record.
+ */
+ int nIndex; /* Size of aIndex[] */
+ struct Fts3Index {
+ int nPrefix; /* Prefix length (0 for main terms index) */
+ Fts3Hash hPending; /* Pending terms table for this index */
+ } *aIndex;
+ int nMaxPendingData; /* Max pending data before flush to disk */
+ int nPendingData; /* Current bytes of pending data */
+ sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */
+ int iPrevLangid; /* Langid of recently inserted document */
+ int bPrevDelete; /* True if last operation was a delete */
+
+#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST)
+ /* State variables used for validating that the transaction control
+ ** methods of the virtual table are called at appropriate times. These
+ ** values do not contribute to FTS functionality; they are used for
+ ** verifying the operation of the SQLite core.
+ */
+ int inTransaction; /* True after xBegin but before xCommit/xRollback */
+ int mxSavepoint; /* Largest valid xSavepoint integer */
+#endif
+
+#ifdef SQLITE_TEST
+ /* True to disable the incremental doclist optimization. This is controled
+ ** by special insert command 'test-no-incr-doclist'. */
+ int bNoIncrDoclist;
+#endif
+};
+
+/*
+** When the core wants to read from the virtual table, it creates a
+** virtual table cursor (an instance of the following structure) using
+** the xOpen method. Cursors are destroyed using the xClose method.
+*/
+struct Fts3Cursor {
+ sqlite3_vtab_cursor base; /* Base class used by SQLite core */
+ i16 eSearch; /* Search strategy (see below) */
+ u8 isEof; /* True if at End Of Results */
+ u8 isRequireSeek; /* True if must seek pStmt to %_content row */
+ sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */
+ Fts3Expr *pExpr; /* Parsed MATCH query string */
+ int iLangid; /* Language being queried for */
+ int nPhrase; /* Number of matchable phrases in query */
+ Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */
+ sqlite3_int64 iPrevId; /* Previous id read from aDoclist */
+ char *pNextId; /* Pointer into the body of aDoclist */
+ char *aDoclist; /* List of docids for full-text queries */
+ int nDoclist; /* Size of buffer at aDoclist */
+ u8 bDesc; /* True to sort in descending order */
+ int eEvalmode; /* An FTS3_EVAL_XX constant */
+ int nRowAvg; /* Average size of database rows, in pages */
+ sqlite3_int64 nDoc; /* Documents in table */
+ i64 iMinDocid; /* Minimum docid to return */
+ i64 iMaxDocid; /* Maximum docid to return */
+ int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */
+ MatchinfoBuffer *pMIBuffer; /* Buffer for matchinfo data */
+};
+
+#define FTS3_EVAL_FILTER 0
+#define FTS3_EVAL_NEXT 1
+#define FTS3_EVAL_MATCHINFO 2
+
+/*
+** The Fts3Cursor.eSearch member is always set to one of the following.
+** Actualy, Fts3Cursor.eSearch can be greater than or equal to
+** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index
+** of the column to be searched. For example, in
+**
+** CREATE VIRTUAL TABLE ex1 USING fts3(a,b,c,d);
+** SELECT docid FROM ex1 WHERE b MATCH 'one two three';
+**
+** Because the LHS of the MATCH operator is 2nd column "b",
+** Fts3Cursor.eSearch will be set to FTS3_FULLTEXT_SEARCH+1. (+0 for a,
+** +1 for b, +2 for c, +3 for d.) If the LHS of MATCH were "ex1"
+** indicating that all columns should be searched,
+** then eSearch would be set to FTS3_FULLTEXT_SEARCH+4.
+*/
+#define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */
+#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */
+#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */
+
+/*
+** The lower 16-bits of the sqlite3_index_info.idxNum value set by
+** the xBestIndex() method contains the Fts3Cursor.eSearch value described
+** above. The upper 16-bits contain a combination of the following
+** bits, used to describe extra constraints on full-text searches.
+*/
+#define FTS3_HAVE_LANGID 0x00010000 /* languageid=? */
+#define FTS3_HAVE_DOCID_GE 0x00020000 /* docid>=? */
+#define FTS3_HAVE_DOCID_LE 0x00040000 /* docid<=? */
+
+struct Fts3Doclist {
+ char *aAll; /* Array containing doclist (or NULL) */
+ int nAll; /* Size of a[] in bytes */
+ char *pNextDocid; /* Pointer to next docid */
+
+ sqlite3_int64 iDocid; /* Current docid (if pList!=0) */
+ int bFreeList; /* True if pList should be sqlite3_free()d */
+ char *pList; /* Pointer to position list following iDocid */
+ int nList; /* Length of position list */
+};
+
+/*
+** A "phrase" is a sequence of one or more tokens that must match in
+** sequence. A single token is the base case and the most common case.
+** For a sequence of tokens contained in double-quotes (i.e. "one two three")
+** nToken will be the number of tokens in the string.
+*/
+struct Fts3PhraseToken {
+ char *z; /* Text of the token */
+ int n; /* Number of bytes in buffer z */
+ int isPrefix; /* True if token ends with a "*" character */
+ int bFirst; /* True if token must appear at position 0 */
+
+ /* Variables above this point are populated when the expression is
+ ** parsed (by code in fts3_expr.c). Below this point the variables are
+ ** used when evaluating the expression. */
+ Fts3DeferredToken *pDeferred; /* Deferred token object for this token */
+ Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */
+};
+
+struct Fts3Phrase {
+ /* Cache of doclist for this phrase. */
+ Fts3Doclist doclist;
+ int bIncr; /* True if doclist is loaded incrementally */
+ int iDoclistToken;
+
+ /* Used by sqlite3Fts3EvalPhrasePoslist() if this is a descendent of an
+ ** OR condition. */
+ char *pOrPoslist;
+ i64 iOrDocid;
+
+ /* Variables below this point are populated by fts3_expr.c when parsing
+ ** a MATCH expression. Everything above is part of the evaluation phase.
+ */
+ int nToken; /* Number of tokens in the phrase */
+ int iColumn; /* Index of column this phrase must match */
+ Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */
+};
+
+/*
+** A tree of these objects forms the RHS of a MATCH operator.
+**
+** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist
+** points to a malloced buffer, size nDoclist bytes, containing the results
+** of this phrase query in FTS3 doclist format. As usual, the initial
+** "Length" field found in doclists stored on disk is omitted from this
+** buffer.
+**
+** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global
+** matchinfo data. If it is not NULL, it points to an array of size nCol*3,
+** where nCol is the number of columns in the queried FTS table. The array
+** is populated as follows:
+**
+** aMI[iCol*3 + 0] = Undefined
+** aMI[iCol*3 + 1] = Number of occurrences
+** aMI[iCol*3 + 2] = Number of rows containing at least one instance
+**
+** The aMI array is allocated using sqlite3_malloc(). It should be freed
+** when the expression node is.
+*/
+struct Fts3Expr {
+ int eType; /* One of the FTSQUERY_XXX values defined below */
+ int nNear; /* Valid if eType==FTSQUERY_NEAR */
+ Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */
+ Fts3Expr *pLeft; /* Left operand */
+ Fts3Expr *pRight; /* Right operand */
+ Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */
+
+ /* The following are used by the fts3_eval.c module. */
+ sqlite3_int64 iDocid; /* Current docid */
+ u8 bEof; /* True this expression is at EOF already */
+ u8 bStart; /* True if iDocid is valid */
+ u8 bDeferred; /* True if this expression is entirely deferred */
+
+ /* The following are used by the fts3_snippet.c module. */
+ int iPhrase; /* Index of this phrase in matchinfo() results */
+ u32 *aMI; /* See above */
+};
+
+/*
+** Candidate values for Fts3Query.eType. Note that the order of the first
+** four values is in order of precedence when parsing expressions. For
+** example, the following:
+**
+** "a OR b AND c NOT d NEAR e"
+**
+** is equivalent to:
+**
+** "a OR (b AND (c NOT (d NEAR e)))"
+*/
+#define FTSQUERY_NEAR 1
+#define FTSQUERY_NOT 2
+#define FTSQUERY_AND 3
+#define FTSQUERY_OR 4
+#define FTSQUERY_PHRASE 5
+
+
+/* fts3_write.c */
+SQLITE_PRIVATE int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*);
+SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *);
+SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *);
+SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *);
+SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(int, int, sqlite3_int64,
+ sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
+SQLITE_PRIVATE int sqlite3Fts3SegReaderPending(
+ Fts3Table*,int,const char*,int,int,Fts3SegReader**);
+SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *);
+SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, int, sqlite3_stmt **);
+SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*);
+
+SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **);
+SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **);
+
+#ifndef SQLITE_DISABLE_FTS4_DEFERRED
+SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *);
+SQLITE_PRIVATE int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int);
+SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *);
+SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *);
+SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *);
+#else
+# define sqlite3Fts3FreeDeferredTokens(x)
+# define sqlite3Fts3DeferToken(x,y,z) SQLITE_OK
+# define sqlite3Fts3CacheDeferredDoclists(x) SQLITE_OK
+# define sqlite3Fts3FreeDeferredDoclists(x)
+# define sqlite3Fts3DeferredTokenList(x,y,z) SQLITE_OK
+#endif
+
+SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *);
+SQLITE_PRIVATE int sqlite3Fts3MaxLevel(Fts3Table *, int *);
+
+/* Special values interpreted by sqlite3SegReaderCursor() */
+#define FTS3_SEGCURSOR_PENDING -1
+#define FTS3_SEGCURSOR_ALL -2
+
+SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*);
+SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *);
+SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *);
+
+SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor(Fts3Table *,
+ int, int, int, const char *, int, int, int, Fts3MultiSegReader *);
+
+/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */
+#define FTS3_SEGMENT_REQUIRE_POS 0x00000001
+#define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002
+#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004
+#define FTS3_SEGMENT_PREFIX 0x00000008
+#define FTS3_SEGMENT_SCAN 0x00000010
+#define FTS3_SEGMENT_FIRST 0x00000020
+
+/* Type passed as 4th argument to SegmentReaderIterate() */
+struct Fts3SegFilter {
+ const char *zTerm;
+ int nTerm;
+ int iCol;
+ int flags;
+};
+
+struct Fts3MultiSegReader {
+ /* Used internally by sqlite3Fts3SegReaderXXX() calls */
+ Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */
+ int nSegment; /* Size of apSegment array */
+ int nAdvance; /* How many seg-readers to advance */
+ Fts3SegFilter *pFilter; /* Pointer to filter object */
+ char *aBuffer; /* Buffer to merge doclists in */
+ int nBuffer; /* Allocated size of aBuffer[] in bytes */
+
+ int iColFilter; /* If >=0, filter for this column */
+ int bRestart;
+
+ /* Used by fts3.c only. */
+ int nCost; /* Cost of running iterator */
+ int bLookup; /* True if a lookup of a single entry. */
+
+ /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */
+ char *zTerm; /* Pointer to term buffer */
+ int nTerm; /* Size of zTerm in bytes */
+ char *aDoclist; /* Pointer to doclist buffer */
+ int nDoclist; /* Size of aDoclist[] in bytes */
+};
+
+SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table*,int,int);
+
+#define fts3GetVarint32(p, piVal) ( \
+ (*(u8*)(p)&0x80) ? sqlite3Fts3GetVarint32(p, piVal) : (*piVal=*(u8*)(p), 1) \
+)
+
+/* fts3.c */
+SQLITE_PRIVATE void sqlite3Fts3ErrMsg(char**,const char*,...);
+SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *, sqlite3_int64);
+SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
+SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *, int *);
+SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64);
+SQLITE_PRIVATE void sqlite3Fts3Dequote(char *);
+SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*);
+SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *);
+SQLITE_PRIVATE int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *);
+SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int*, Fts3Table*);
+SQLITE_PRIVATE int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc);
+
+/* fts3_tokenizer.c */
+SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *);
+SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *);
+SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *,
+ sqlite3_tokenizer **, char **
+);
+SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char);
+
+/* fts3_snippet.c */
+SQLITE_PRIVATE void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*);
+SQLITE_PRIVATE void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *,
+ const char *, const char *, int, int
+);
+SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *);
+SQLITE_PRIVATE void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p);
+
+/* fts3_expr.c */
+SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int,
+ char **, int, int, int, const char *, int, Fts3Expr **, char **
+);
+SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *);
+#ifdef SQLITE_TEST
+SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
+SQLITE_PRIVATE int sqlite3Fts3InitTerm(sqlite3 *db);
+#endif
+
+SQLITE_PRIVATE int sqlite3Fts3OpenTokenizer(sqlite3_tokenizer *, int, const char *, int,
+ sqlite3_tokenizer_cursor **
+);
+
+/* fts3_aux.c */
+SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db);
+
+SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *);
+
+SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart(
+ Fts3Table*, Fts3MultiSegReader*, int, const char*, int);
+SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext(
+ Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *);
+SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **);
+SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *);
+SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr);
+
+/* fts3_tokenize_vtab.c */
+SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *);
+
+/* fts3_unicode2.c (functions generated by parsing unicode text files) */
+#ifndef SQLITE_DISABLE_FTS3_UNICODE
+SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int, int);
+SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int);
+SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int);
+#endif
+
+#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */
+#endif /* _FTSINT_H */
+
+/************** End of fts3Int.h *********************************************/
+/************** Continuing where we left off in fts3.c ***********************/
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE)
+# define SQLITE_CORE 1
+#endif
+
+/* #include <assert.h> */
+/* #include <stdlib.h> */
+/* #include <stddef.h> */
+/* #include <stdio.h> */
+/* #include <string.h> */
+/* #include <stdarg.h> */
+
+/* #include "fts3.h" */
+#ifndef SQLITE_CORE
+/* # include "sqlite3ext.h" */
+ SQLITE_EXTENSION_INIT1
+#endif
+
+static int fts3EvalNext(Fts3Cursor *pCsr);
+static int fts3EvalStart(Fts3Cursor *pCsr);
+static int fts3TermSegReaderCursor(
+ Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **);
+
+#ifndef SQLITE_AMALGAMATION
+# if defined(SQLITE_DEBUG)
+SQLITE_PRIVATE int sqlite3Fts3Always(int b) { assert( b ); return b; }
+SQLITE_PRIVATE int sqlite3Fts3Never(int b) { assert( !b ); return b; }
+# endif
+#endif
+
+/*
+** Write a 64-bit variable-length integer to memory starting at p[0].
+** The length of data written will be between 1 and FTS3_VARINT_MAX bytes.
+** The number of bytes written is returned.
+*/
+SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){
+ unsigned char *q = (unsigned char *) p;
+ sqlite_uint64 vu = v;
+ do{
+ *q++ = (unsigned char) ((vu & 0x7f) | 0x80);
+ vu >>= 7;
+ }while( vu!=0 );
+ q[-1] &= 0x7f; /* turn off high bit in final byte */
+ assert( q - (unsigned char *)p <= FTS3_VARINT_MAX );
+ return (int) (q - (unsigned char *)p);
+}
+
+#define GETVARINT_STEP(v, ptr, shift, mask1, mask2, var, ret) \
+ v = (v & mask1) | ( (*ptr++) << shift ); \
+ if( (v & mask2)==0 ){ var = v; return ret; }
+#define GETVARINT_INIT(v, ptr, shift, mask1, mask2, var, ret) \
+ v = (*ptr++); \
+ if( (v & mask2)==0 ){ var = v; return ret; }
+
+/*
+** Read a 64-bit variable-length integer from memory starting at p[0].
+** Return the number of bytes read, or 0 on error.
+** The value is stored in *v.
+*/
+SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){
+ const char *pStart = p;
+ u32 a;
+ u64 b;
+ int shift;
+
+ GETVARINT_INIT(a, p, 0, 0x00, 0x80, *v, 1);
+ GETVARINT_STEP(a, p, 7, 0x7F, 0x4000, *v, 2);
+ GETVARINT_STEP(a, p, 14, 0x3FFF, 0x200000, *v, 3);
+ GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *v, 4);
+ b = (a & 0x0FFFFFFF );
+
+ for(shift=28; shift<=63; shift+=7){
+ u64 c = *p++;
+ b += (c&0x7F) << shift;
+ if( (c & 0x80)==0 ) break;
+ }
+ *v = b;
+ return (int)(p - pStart);
+}
+
+/*
+** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to a
+** 32-bit integer before it is returned.
+*/
+SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *p, int *pi){
+ u32 a;
+
+#ifndef fts3GetVarint32
+ GETVARINT_INIT(a, p, 0, 0x00, 0x80, *pi, 1);
+#else
+ a = (*p++);
+ assert( a & 0x80 );
+#endif
+
+ GETVARINT_STEP(a, p, 7, 0x7F, 0x4000, *pi, 2);
+ GETVARINT_STEP(a, p, 14, 0x3FFF, 0x200000, *pi, 3);
+ GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *pi, 4);
+ a = (a & 0x0FFFFFFF );
+ *pi = (int)(a | ((u32)(*p & 0x0F) << 28));
+ return 5;
+}
+
+/*
+** Return the number of bytes required to encode v as a varint
+*/
+SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64 v){
+ int i = 0;
+ do{
+ i++;
+ v >>= 7;
+ }while( v!=0 );
+ return i;
+}
+
+/*
+** Convert an SQL-style quoted string into a normal string by removing
+** the quote characters. The conversion is done in-place. If the
+** input does not begin with a quote character, then this routine
+** is a no-op.
+**
+** Examples:
+**
+** "abc" becomes abc
+** 'xyz' becomes xyz
+** [pqr] becomes pqr
+** `mno` becomes mno
+**
+*/
+SQLITE_PRIVATE void sqlite3Fts3Dequote(char *z){
+ char quote; /* Quote character (if any ) */
+
+ quote = z[0];
+ if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){
+ int iIn = 1; /* Index of next byte to read from input */
+ int iOut = 0; /* Index of next byte to write to output */
+
+ /* If the first byte was a '[', then the close-quote character is a ']' */
+ if( quote=='[' ) quote = ']';
+
+ while( z[iIn] ){
+ if( z[iIn]==quote ){
+ if( z[iIn+1]!=quote ) break;
+ z[iOut++] = quote;
+ iIn += 2;
+ }else{
+ z[iOut++] = z[iIn++];
+ }
+ }
+ z[iOut] = '\0';
+ }
+}
+
+/*
+** Read a single varint from the doclist at *pp and advance *pp to point
+** to the first byte past the end of the varint. Add the value of the varint
+** to *pVal.
+*/
+static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){
+ sqlite3_int64 iVal;
+ *pp += sqlite3Fts3GetVarint(*pp, &iVal);
+ *pVal += iVal;
+}
+
+/*
+** When this function is called, *pp points to the first byte following a
+** varint that is part of a doclist (or position-list, or any other list
+** of varints). This function moves *pp to point to the start of that varint,
+** and sets *pVal by the varint value.
+**
+** Argument pStart points to the first byte of the doclist that the
+** varint is part of.
+*/
+static void fts3GetReverseVarint(
+ char **pp,
+ char *pStart,
+ sqlite3_int64 *pVal
+){
+ sqlite3_int64 iVal;
+ char *p;
+
+ /* Pointer p now points at the first byte past the varint we are
+ ** interested in. So, unless the doclist is corrupt, the 0x80 bit is
+ ** clear on character p[-1]. */
+ for(p = (*pp)-2; p>=pStart && *p&0x80; p--);
+ p++;
+ *pp = p;
+
+ sqlite3Fts3GetVarint(p, &iVal);
+ *pVal = iVal;
+}
+
+/*
+** The xDisconnect() virtual table method.
+*/
+static int fts3DisconnectMethod(sqlite3_vtab *pVtab){
+ Fts3Table *p = (Fts3Table *)pVtab;
+ int i;
+
+ assert( p->nPendingData==0 );
+ assert( p->pSegments==0 );
+
+ /* Free any prepared statements held */
+ for(i=0; i<SizeofArray(p->aStmt); i++){
+ sqlite3_finalize(p->aStmt[i]);
+ }
+ sqlite3_free(p->zSegmentsTbl);
+ sqlite3_free(p->zReadExprlist);
+ sqlite3_free(p->zWriteExprlist);
+ sqlite3_free(p->zContentTbl);
+ sqlite3_free(p->zLanguageid);
+
+ /* Invoke the tokenizer destructor to free the tokenizer. */
+ p->pTokenizer->pModule->xDestroy(p->pTokenizer);
+
+ sqlite3_free(p);
+ return SQLITE_OK;
+}
+
+/*
+** Write an error message into *pzErr
+*/
+SQLITE_PRIVATE void sqlite3Fts3ErrMsg(char **pzErr, const char *zFormat, ...){
+ va_list ap;
+ sqlite3_free(*pzErr);
+ va_start(ap, zFormat);
+ *pzErr = sqlite3_vmprintf(zFormat, ap);
+ va_end(ap);
+}
+
+/*
+** Construct one or more SQL statements from the format string given
+** and then evaluate those statements. The success code is written
+** into *pRc.
+**
+** If *pRc is initially non-zero then this routine is a no-op.
+*/
+static void fts3DbExec(
+ int *pRc, /* Success code */
+ sqlite3 *db, /* Database in which to run SQL */
+ const char *zFormat, /* Format string for SQL */
+ ... /* Arguments to the format string */
+){
+ va_list ap;
+ char *zSql;
+ if( *pRc ) return;
+ va_start(ap, zFormat);
+ zSql = sqlite3_vmprintf(zFormat, ap);
+ va_end(ap);
+ if( zSql==0 ){
+ *pRc = SQLITE_NOMEM;
+ }else{
+ *pRc = sqlite3_exec(db, zSql, 0, 0, 0);
+ sqlite3_free(zSql);
+ }
+}
+
+/*
+** The xDestroy() virtual table method.
+*/
+static int fts3DestroyMethod(sqlite3_vtab *pVtab){
+ Fts3Table *p = (Fts3Table *)pVtab;
+ int rc = SQLITE_OK; /* Return code */
+ const char *zDb = p->zDb; /* Name of database (e.g. "main", "temp") */
+ sqlite3 *db = p->db; /* Database handle */
+
+ /* Drop the shadow tables */
+ if( p->zContentTbl==0 ){
+ fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", zDb, p->zName);
+ }
+ fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", zDb,p->zName);
+ fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", zDb, p->zName);
+ fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", zDb, p->zName);
+ fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", zDb, p->zName);
+
+ /* If everything has worked, invoke fts3DisconnectMethod() to free the
+ ** memory associated with the Fts3Table structure and return SQLITE_OK.
+ ** Otherwise, return an SQLite error code.
+ */
+ return (rc==SQLITE_OK ? fts3DisconnectMethod(pVtab) : rc);
+}
+
+
+/*
+** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table
+** passed as the first argument. This is done as part of the xConnect()
+** and xCreate() methods.
+**
+** If *pRc is non-zero when this function is called, it is a no-op.
+** Otherwise, if an error occurs, an SQLite error code is stored in *pRc
+** before returning.
+*/
+static void fts3DeclareVtab(int *pRc, Fts3Table *p){
+ if( *pRc==SQLITE_OK ){
+ int i; /* Iterator variable */
+ int rc; /* Return code */
+ char *zSql; /* SQL statement passed to declare_vtab() */
+ char *zCols; /* List of user defined columns */
+ const char *zLanguageid;
+
+ zLanguageid = (p->zLanguageid ? p->zLanguageid : "__langid");
+ sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1);
+
+ /* Create a list of user columns for the virtual table */
+ zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]);
+ for(i=1; zCols && i<p->nColumn; i++){
+ zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]);
+ }
+
+ /* Create the whole "CREATE TABLE" statement to pass to SQLite */
+ zSql = sqlite3_mprintf(
+ "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN, %Q HIDDEN)",
+ zCols, p->zName, zLanguageid
+ );
+ if( !zCols || !zSql ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_declare_vtab(p->db, zSql);
+ }
+
+ sqlite3_free(zSql);
+ sqlite3_free(zCols);
+ *pRc = rc;
+ }
+}
+
+/*
+** Create the %_stat table if it does not already exist.
+*/
+SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int *pRc, Fts3Table *p){
+ fts3DbExec(pRc, p->db,
+ "CREATE TABLE IF NOT EXISTS %Q.'%q_stat'"
+ "(id INTEGER PRIMARY KEY, value BLOB);",
+ p->zDb, p->zName
+ );
+ if( (*pRc)==SQLITE_OK ) p->bHasStat = 1;
+}
+
+/*
+** Create the backing store tables (%_content, %_segments and %_segdir)
+** required by the FTS3 table passed as the only argument. This is done
+** as part of the vtab xCreate() method.
+**
+** If the p->bHasDocsize boolean is true (indicating that this is an
+** FTS4 table, not an FTS3 table) then also create the %_docsize and
+** %_stat tables required by FTS4.
+*/
+static int fts3CreateTables(Fts3Table *p){
+ int rc = SQLITE_OK; /* Return code */
+ int i; /* Iterator variable */
+ sqlite3 *db = p->db; /* The database connection */
+
+ if( p->zContentTbl==0 ){
+ const char *zLanguageid = p->zLanguageid;
+ char *zContentCols; /* Columns of %_content table */
+
+ /* Create a list of user columns for the content table */
+ zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY");
+ for(i=0; zContentCols && i<p->nColumn; i++){
+ char *z = p->azColumn[i];
+ zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z);
+ }
+ if( zLanguageid && zContentCols ){
+ zContentCols = sqlite3_mprintf("%z, langid", zContentCols, zLanguageid);
+ }
+ if( zContentCols==0 ) rc = SQLITE_NOMEM;
+
+ /* Create the content table */
+ fts3DbExec(&rc, db,
+ "CREATE TABLE %Q.'%q_content'(%s)",
+ p->zDb, p->zName, zContentCols
+ );
+ sqlite3_free(zContentCols);
+ }
+
+ /* Create other tables */
+ fts3DbExec(&rc, db,
+ "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);",
+ p->zDb, p->zName
+ );
+ fts3DbExec(&rc, db,
+ "CREATE TABLE %Q.'%q_segdir'("
+ "level INTEGER,"
+ "idx INTEGER,"
+ "start_block INTEGER,"
+ "leaves_end_block INTEGER,"
+ "end_block INTEGER,"
+ "root BLOB,"
+ "PRIMARY KEY(level, idx)"
+ ");",
+ p->zDb, p->zName
+ );
+ if( p->bHasDocsize ){
+ fts3DbExec(&rc, db,
+ "CREATE TABLE %Q.'%q_docsize'(docid INTEGER PRIMARY KEY, size BLOB);",
+ p->zDb, p->zName
+ );
+ }
+ assert( p->bHasStat==p->bFts4 );
+ if( p->bHasStat ){
+ sqlite3Fts3CreateStatTable(&rc, p);
+ }
+ return rc;
+}
+
+/*
+** Store the current database page-size in bytes in p->nPgsz.
+**
+** If *pRc is non-zero when this function is called, it is a no-op.
+** Otherwise, if an error occurs, an SQLite error code is stored in *pRc
+** before returning.
+*/
+static void fts3DatabasePageSize(int *pRc, Fts3Table *p){
+ if( *pRc==SQLITE_OK ){
+ int rc; /* Return code */
+ char *zSql; /* SQL text "PRAGMA %Q.page_size" */
+ sqlite3_stmt *pStmt; /* Compiled "PRAGMA %Q.page_size" statement */
+
+ zSql = sqlite3_mprintf("PRAGMA %Q.page_size", p->zDb);
+ if( !zSql ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_step(pStmt);
+ p->nPgsz = sqlite3_column_int(pStmt, 0);
+ rc = sqlite3_finalize(pStmt);
+ }else if( rc==SQLITE_AUTH ){
+ p->nPgsz = 1024;
+ rc = SQLITE_OK;
+ }
+ }
+ assert( p->nPgsz>0 || rc!=SQLITE_OK );
+ sqlite3_free(zSql);
+ *pRc = rc;
+ }
+}
+
+/*
+** "Special" FTS4 arguments are column specifications of the following form:
+**
+** <key> = <value>
+**
+** There may not be whitespace surrounding the "=" character. The <value>
+** term may be quoted, but the <key> may not.
+*/
+static int fts3IsSpecialColumn(
+ const char *z,
+ int *pnKey,
+ char **pzValue
+){
+ char *zValue;
+ const char *zCsr = z;
+
+ while( *zCsr!='=' ){
+ if( *zCsr=='\0' ) return 0;
+ zCsr++;
+ }
+
+ *pnKey = (int)(zCsr-z);
+ zValue = sqlite3_mprintf("%s", &zCsr[1]);
+ if( zValue ){
+ sqlite3Fts3Dequote(zValue);
+ }
+ *pzValue = zValue;
+ return 1;
+}
+
+/*
+** Append the output of a printf() style formatting to an existing string.
+*/
+static void fts3Appendf(
+ int *pRc, /* IN/OUT: Error code */
+ char **pz, /* IN/OUT: Pointer to string buffer */
+ const char *zFormat, /* Printf format string to append */
+ ... /* Arguments for printf format string */
+){
+ if( *pRc==SQLITE_OK ){
+ va_list ap;
+ char *z;
+ va_start(ap, zFormat);
+ z = sqlite3_vmprintf(zFormat, ap);
+ va_end(ap);
+ if( z && *pz ){
+ char *z2 = sqlite3_mprintf("%s%s", *pz, z);
+ sqlite3_free(z);
+ z = z2;
+ }
+ if( z==0 ) *pRc = SQLITE_NOMEM;
+ sqlite3_free(*pz);
+ *pz = z;
+ }
+}
+
+/*
+** Return a copy of input string zInput enclosed in double-quotes (") and
+** with all double quote characters escaped. For example:
+**
+** fts3QuoteId("un \"zip\"") -> "un \"\"zip\"\""
+**
+** The pointer returned points to memory obtained from sqlite3_malloc(). It
+** is the callers responsibility to call sqlite3_free() to release this
+** memory.
+*/
+static char *fts3QuoteId(char const *zInput){
+ int nRet;
+ char *zRet;
+ nRet = 2 + (int)strlen(zInput)*2 + 1;
+ zRet = sqlite3_malloc(nRet);
+ if( zRet ){
+ int i;
+ char *z = zRet;
+ *(z++) = '"';
+ for(i=0; zInput[i]; i++){
+ if( zInput[i]=='"' ) *(z++) = '"';
+ *(z++) = zInput[i];
+ }
+ *(z++) = '"';
+ *(z++) = '\0';
+ }
+ return zRet;
+}
+
+/*
+** Return a list of comma separated SQL expressions and a FROM clause that
+** could be used in a SELECT statement such as the following:
+**
+** SELECT <list of expressions> FROM %_content AS x ...
+**
+** to return the docid, followed by each column of text data in order
+** from left to write. If parameter zFunc is not NULL, then instead of
+** being returned directly each column of text data is passed to an SQL
+** function named zFunc first. For example, if zFunc is "unzip" and the
+** table has the three user-defined columns "a", "b", and "c", the following
+** string is returned:
+**
+** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c') FROM %_content AS x"
+**
+** The pointer returned points to a buffer allocated by sqlite3_malloc(). It
+** is the responsibility of the caller to eventually free it.
+**
+** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and
+** a NULL pointer is returned). Otherwise, if an OOM error is encountered
+** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If
+** no error occurs, *pRc is left unmodified.
+*/
+static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){
+ char *zRet = 0;
+ char *zFree = 0;
+ char *zFunction;
+ int i;
+
+ if( p->zContentTbl==0 ){
+ if( !zFunc ){
+ zFunction = "";
+ }else{
+ zFree = zFunction = fts3QuoteId(zFunc);
+ }
+ fts3Appendf(pRc, &zRet, "docid");
+ for(i=0; i<p->nColumn; i++){
+ fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]);
+ }
+ if( p->zLanguageid ){
+ fts3Appendf(pRc, &zRet, ", x.%Q", "langid");
+ }
+ sqlite3_free(zFree);
+ }else{
+ fts3Appendf(pRc, &zRet, "rowid");
+ for(i=0; i<p->nColumn; i++){
+ fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]);
+ }
+ if( p->zLanguageid ){
+ fts3Appendf(pRc, &zRet, ", x.%Q", p->zLanguageid);
+ }
+ }
+ fts3Appendf(pRc, &zRet, " FROM '%q'.'%q%s' AS x",
+ p->zDb,
+ (p->zContentTbl ? p->zContentTbl : p->zName),
+ (p->zContentTbl ? "" : "_content")
+ );
+ return zRet;
+}
+
+/*
+** Return a list of N comma separated question marks, where N is the number
+** of columns in the %_content table (one for the docid plus one for each
+** user-defined text column).
+**
+** If argument zFunc is not NULL, then all but the first question mark
+** is preceded by zFunc and an open bracket, and followed by a closed
+** bracket. For example, if zFunc is "zip" and the FTS3 table has three
+** user-defined text columns, the following string is returned:
+**
+** "?, zip(?), zip(?), zip(?)"
+**
+** The pointer returned points to a buffer allocated by sqlite3_malloc(). It
+** is the responsibility of the caller to eventually free it.
+**
+** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and
+** a NULL pointer is returned). Otherwise, if an OOM error is encountered
+** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If
+** no error occurs, *pRc is left unmodified.
+*/
+static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){
+ char *zRet = 0;
+ char *zFree = 0;
+ char *zFunction;
+ int i;
+
+ if( !zFunc ){
+ zFunction = "";
+ }else{
+ zFree = zFunction = fts3QuoteId(zFunc);
+ }
+ fts3Appendf(pRc, &zRet, "?");
+ for(i=0; i<p->nColumn; i++){
+ fts3Appendf(pRc, &zRet, ",%s(?)", zFunction);
+ }
+ if( p->zLanguageid ){
+ fts3Appendf(pRc, &zRet, ", ?");
+ }
+ sqlite3_free(zFree);
+ return zRet;
+}
+
+/*
+** This function interprets the string at (*pp) as a non-negative integer
+** value. It reads the integer and sets *pnOut to the value read, then
+** sets *pp to point to the byte immediately following the last byte of
+** the integer value.
+**
+** Only decimal digits ('0'..'9') may be part of an integer value.
+**
+** If *pp does not being with a decimal digit SQLITE_ERROR is returned and
+** the output value undefined. Otherwise SQLITE_OK is returned.
+**
+** This function is used when parsing the "prefix=" FTS4 parameter.
+*/
+static int fts3GobbleInt(const char **pp, int *pnOut){
+ const int MAX_NPREFIX = 10000000;
+ const char *p; /* Iterator pointer */
+ int nInt = 0; /* Output value */
+
+ for(p=*pp; p[0]>='0' && p[0]<='9'; p++){
+ nInt = nInt * 10 + (p[0] - '0');
+ if( nInt>MAX_NPREFIX ){
+ nInt = 0;
+ break;
+ }
+ }
+ if( p==*pp ) return SQLITE_ERROR;
+ *pnOut = nInt;
+ *pp = p;
+ return SQLITE_OK;
+}
+
+/*
+** This function is called to allocate an array of Fts3Index structures
+** representing the indexes maintained by the current FTS table. FTS tables
+** always maintain the main "terms" index, but may also maintain one or
+** more "prefix" indexes, depending on the value of the "prefix=" parameter
+** (if any) specified as part of the CREATE VIRTUAL TABLE statement.
+**
+** Argument zParam is passed the value of the "prefix=" option if one was
+** specified, or NULL otherwise.
+**
+** If no error occurs, SQLITE_OK is returned and *apIndex set to point to
+** the allocated array. *pnIndex is set to the number of elements in the
+** array. If an error does occur, an SQLite error code is returned.
+**
+** Regardless of whether or not an error is returned, it is the responsibility
+** of the caller to call sqlite3_free() on the output array to free it.
+*/
+static int fts3PrefixParameter(
+ const char *zParam, /* ABC in prefix=ABC parameter to parse */
+ int *pnIndex, /* OUT: size of *apIndex[] array */
+ struct Fts3Index **apIndex /* OUT: Array of indexes for this table */
+){
+ struct Fts3Index *aIndex; /* Allocated array */
+ int nIndex = 1; /* Number of entries in array */
+
+ if( zParam && zParam[0] ){
+ const char *p;
+ nIndex++;
+ for(p=zParam; *p; p++){
+ if( *p==',' ) nIndex++;
+ }
+ }
+
+ aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex);
+ *apIndex = aIndex;
+ if( !aIndex ){
+ return SQLITE_NOMEM;
+ }
+
+ memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex);
+ if( zParam ){
+ const char *p = zParam;
+ int i;
+ for(i=1; i<nIndex; i++){
+ int nPrefix = 0;
+ if( fts3GobbleInt(&p, &nPrefix) ) return SQLITE_ERROR;
+ assert( nPrefix>=0 );
+ if( nPrefix==0 ){
+ nIndex--;
+ i--;
+ }else{
+ aIndex[i].nPrefix = nPrefix;
+ }
+ p++;
+ }
+ }
+
+ *pnIndex = nIndex;
+ return SQLITE_OK;
+}
+
+/*
+** This function is called when initializing an FTS4 table that uses the
+** content=xxx option. It determines the number of and names of the columns
+** of the new FTS4 table.
+**
+** The third argument passed to this function is the value passed to the
+** config=xxx option (i.e. "xxx"). This function queries the database for
+** a table of that name. If found, the output variables are populated
+** as follows:
+**
+** *pnCol: Set to the number of columns table xxx has,
+**
+** *pnStr: Set to the total amount of space required to store a copy
+** of each columns name, including the nul-terminator.
+**
+** *pazCol: Set to point to an array of *pnCol strings. Each string is
+** the name of the corresponding column in table xxx. The array
+** and its contents are allocated using a single allocation. It
+** is the responsibility of the caller to free this allocation
+** by eventually passing the *pazCol value to sqlite3_free().
+**
+** If the table cannot be found, an error code is returned and the output
+** variables are undefined. Or, if an OOM is encountered, SQLITE_NOMEM is
+** returned (and the output variables are undefined).
+*/
+static int fts3ContentColumns(
+ sqlite3 *db, /* Database handle */
+ const char *zDb, /* Name of db (i.e. "main", "temp" etc.) */
+ const char *zTbl, /* Name of content table */
+ const char ***pazCol, /* OUT: Malloc'd array of column names */
+ int *pnCol, /* OUT: Size of array *pazCol */
+ int *pnStr, /* OUT: Bytes of string content */
+ char **pzErr /* OUT: error message */
+){
+ int rc = SQLITE_OK; /* Return code */
+ char *zSql; /* "SELECT *" statement on zTbl */
+ sqlite3_stmt *pStmt = 0; /* Compiled version of zSql */
+
+ zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", zDb, zTbl);
+ if( !zSql ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0);
+ if( rc!=SQLITE_OK ){
+ sqlite3Fts3ErrMsg(pzErr, "%s", sqlite3_errmsg(db));
+ }
+ }
+ sqlite3_free(zSql);
+
+ if( rc==SQLITE_OK ){
+ const char **azCol; /* Output array */
+ int nStr = 0; /* Size of all column names (incl. 0x00) */
+ int nCol; /* Number of table columns */
+ int i; /* Used to iterate through columns */
+
+ /* Loop through the returned columns. Set nStr to the number of bytes of
+ ** space required to store a copy of each column name, including the
+ ** nul-terminator byte. */
+ nCol = sqlite3_column_count(pStmt);
+ for(i=0; i<nCol; i++){
+ const char *zCol = sqlite3_column_name(pStmt, i);
+ nStr += (int)strlen(zCol) + 1;
+ }
+
+ /* Allocate and populate the array to return. */
+ azCol = (const char **)sqlite3_malloc(sizeof(char *) * nCol + nStr);
+ if( azCol==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ char *p = (char *)&azCol[nCol];
+ for(i=0; i<nCol; i++){
+ const char *zCol = sqlite3_column_name(pStmt, i);
+ int n = (int)strlen(zCol)+1;
+ memcpy(p, zCol, n);
+ azCol[i] = p;
+ p += n;
+ }
+ }
+ sqlite3_finalize(pStmt);
+
+ /* Set the output variables. */
+ *pnCol = nCol;
+ *pnStr = nStr;
+ *pazCol = azCol;
+ }
+
+ return rc;
+}
+
+/*
+** This function is the implementation of both the xConnect and xCreate
+** methods of the FTS3 virtual table.
+**
+** The argv[] array contains the following:
+**
+** argv[0] -> module name ("fts3" or "fts4")
+** argv[1] -> database name
+** argv[2] -> table name
+** argv[...] -> "column name" and other module argument fields.
+*/
+static int fts3InitVtab(
+ int isCreate, /* True for xCreate, false for xConnect */
+ sqlite3 *db, /* The SQLite database connection */
+ void *pAux, /* Hash table containing tokenizers */
+ int argc, /* Number of elements in argv array */
+ const char * const *argv, /* xCreate/xConnect argument array */
+ sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */
+ char **pzErr /* Write any error message here */
+){
+ Fts3Hash *pHash = (Fts3Hash *)pAux;
+ Fts3Table *p = 0; /* Pointer to allocated vtab */
+ int rc = SQLITE_OK; /* Return code */
+ int i; /* Iterator variable */
+ int nByte; /* Size of allocation used for *p */
+ int iCol; /* Column index */
+ int nString = 0; /* Bytes required to hold all column names */
+ int nCol = 0; /* Number of columns in the FTS table */
+ char *zCsr; /* Space for holding column names */
+ int nDb; /* Bytes required to hold database name */
+ int nName; /* Bytes required to hold table name */
+ int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */
+ const char **aCol; /* Array of column names */
+ sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */
+
+ int nIndex = 0; /* Size of aIndex[] array */
+ struct Fts3Index *aIndex = 0; /* Array of indexes for this table */
+
+ /* The results of parsing supported FTS4 key=value options: */
+ int bNoDocsize = 0; /* True to omit %_docsize table */
+ int bDescIdx = 0; /* True to store descending indexes */
+ char *zPrefix = 0; /* Prefix parameter value (or NULL) */
+ char *zCompress = 0; /* compress=? parameter (or NULL) */
+ char *zUncompress = 0; /* uncompress=? parameter (or NULL) */
+ char *zContent = 0; /* content=? parameter (or NULL) */
+ char *zLanguageid = 0; /* languageid=? parameter (or NULL) */
+ char **azNotindexed = 0; /* The set of notindexed= columns */
+ int nNotindexed = 0; /* Size of azNotindexed[] array */
+
+ assert( strlen(argv[0])==4 );
+ assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4)
+ || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4)
+ );
+
+ nDb = (int)strlen(argv[1]) + 1;
+ nName = (int)strlen(argv[2]) + 1;
+
+ nByte = sizeof(const char *) * (argc-2);
+ aCol = (const char **)sqlite3_malloc(nByte);
+ if( aCol ){
+ memset((void*)aCol, 0, nByte);
+ azNotindexed = (char **)sqlite3_malloc(nByte);
+ }
+ if( azNotindexed ){
+ memset(azNotindexed, 0, nByte);
+ }
+ if( !aCol || !azNotindexed ){
+ rc = SQLITE_NOMEM;
+ goto fts3_init_out;
+ }
+
+ /* Loop through all of the arguments passed by the user to the FTS3/4
+ ** module (i.e. all the column names and special arguments). This loop
+ ** does the following:
+ **
+ ** + Figures out the number of columns the FTSX table will have, and
+ ** the number of bytes of space that must be allocated to store copies
+ ** of the column names.
+ **
+ ** + If there is a tokenizer specification included in the arguments,
+ ** initializes the tokenizer pTokenizer.
+ */
+ for(i=3; rc==SQLITE_OK && i<argc; i++){
+ char const *z = argv[i];
+ int nKey;
+ char *zVal;
+
+ /* Check if this is a tokenizer specification */
+ if( !pTokenizer
+ && strlen(z)>8
+ && 0==sqlite3_strnicmp(z, "tokenize", 8)
+ && 0==sqlite3Fts3IsIdChar(z[8])
+ ){
+ rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr);
+ }
+
+ /* Check if it is an FTS4 special argument. */
+ else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){
+ struct Fts4Option {
+ const char *zOpt;
+ int nOpt;
+ } aFts4Opt[] = {
+ { "matchinfo", 9 }, /* 0 -> MATCHINFO */
+ { "prefix", 6 }, /* 1 -> PREFIX */
+ { "compress", 8 }, /* 2 -> COMPRESS */
+ { "uncompress", 10 }, /* 3 -> UNCOMPRESS */
+ { "order", 5 }, /* 4 -> ORDER */
+ { "content", 7 }, /* 5 -> CONTENT */
+ { "languageid", 10 }, /* 6 -> LANGUAGEID */
+ { "notindexed", 10 } /* 7 -> NOTINDEXED */
+ };
+
+ int iOpt;
+ if( !zVal ){
+ rc = SQLITE_NOMEM;
+ }else{
+ for(iOpt=0; iOpt<SizeofArray(aFts4Opt); iOpt++){
+ struct Fts4Option *pOp = &aFts4Opt[iOpt];
+ if( nKey==pOp->nOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){
+ break;
+ }
+ }
+ if( iOpt==SizeofArray(aFts4Opt) ){
+ sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z);
+ rc = SQLITE_ERROR;
+ }else{
+ switch( iOpt ){
+ case 0: /* MATCHINFO */
+ if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){
+ sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal);
+ rc = SQLITE_ERROR;
+ }
+ bNoDocsize = 1;
+ break;
+
+ case 1: /* PREFIX */
+ sqlite3_free(zPrefix);
+ zPrefix = zVal;
+ zVal = 0;
+ break;
+
+ case 2: /* COMPRESS */
+ sqlite3_free(zCompress);
+ zCompress = zVal;
+ zVal = 0;
+ break;
+
+ case 3: /* UNCOMPRESS */
+ sqlite3_free(zUncompress);
+ zUncompress = zVal;
+ zVal = 0;
+ break;
+
+ case 4: /* ORDER */
+ if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3))
+ && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4))
+ ){
+ sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal);
+ rc = SQLITE_ERROR;
+ }
+ bDescIdx = (zVal[0]=='d' || zVal[0]=='D');
+ break;
+
+ case 5: /* CONTENT */
+ sqlite3_free(zContent);
+ zContent = zVal;
+ zVal = 0;
+ break;
+
+ case 6: /* LANGUAGEID */
+ assert( iOpt==6 );
+ sqlite3_free(zLanguageid);
+ zLanguageid = zVal;
+ zVal = 0;
+ break;
+
+ case 7: /* NOTINDEXED */
+ azNotindexed[nNotindexed++] = zVal;
+ zVal = 0;
+ break;
+ }
+ }
+ sqlite3_free(zVal);
+ }
+ }
+
+ /* Otherwise, the argument is a column name. */
+ else {
+ nString += (int)(strlen(z) + 1);
+ aCol[nCol++] = z;
+ }
+ }
+
+ /* If a content=xxx option was specified, the following:
+ **
+ ** 1. Ignore any compress= and uncompress= options.
+ **
+ ** 2. If no column names were specified as part of the CREATE VIRTUAL
+ ** TABLE statement, use all columns from the content table.
+ */
+ if( rc==SQLITE_OK && zContent ){
+ sqlite3_free(zCompress);
+ sqlite3_free(zUncompress);
+ zCompress = 0;
+ zUncompress = 0;
+ if( nCol==0 ){
+ sqlite3_free((void*)aCol);
+ aCol = 0;
+ rc = fts3ContentColumns(db, argv[1], zContent,&aCol,&nCol,&nString,pzErr);
+
+ /* If a languageid= option was specified, remove the language id
+ ** column from the aCol[] array. */
+ if( rc==SQLITE_OK && zLanguageid ){
+ int j;
+ for(j=0; j<nCol; j++){
+ if( sqlite3_stricmp(zLanguageid, aCol[j])==0 ){
+ int k;
+ for(k=j; k<nCol; k++) aCol[k] = aCol[k+1];
+ nCol--;
+ break;
+ }
+ }
+ }
+ }
+ }
+ if( rc!=SQLITE_OK ) goto fts3_init_out;
+
+ if( nCol==0 ){
+ assert( nString==0 );
+ aCol[0] = "content";
+ nString = 8;
+ nCol = 1;
+ }
+
+ if( pTokenizer==0 ){
+ rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr);
+ if( rc!=SQLITE_OK ) goto fts3_init_out;
+ }
+ assert( pTokenizer );
+
+ rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex);
+ if( rc==SQLITE_ERROR ){
+ assert( zPrefix );
+ sqlite3Fts3ErrMsg(pzErr, "error parsing prefix parameter: %s", zPrefix);
+ }
+ if( rc!=SQLITE_OK ) goto fts3_init_out;
+
+ /* Allocate and populate the Fts3Table structure. */
+ nByte = sizeof(Fts3Table) + /* Fts3Table */
+ nCol * sizeof(char *) + /* azColumn */
+ nIndex * sizeof(struct Fts3Index) + /* aIndex */
+ nCol * sizeof(u8) + /* abNotindexed */
+ nName + /* zName */
+ nDb + /* zDb */
+ nString; /* Space for azColumn strings */
+ p = (Fts3Table*)sqlite3_malloc(nByte);
+ if( p==0 ){
+ rc = SQLITE_NOMEM;
+ goto fts3_init_out;
+ }
+ memset(p, 0, nByte);
+ p->db = db;
+ p->nColumn = nCol;
+ p->nPendingData = 0;
+ p->azColumn = (char **)&p[1];
+ p->pTokenizer = pTokenizer;
+ p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
+ p->bHasDocsize = (isFts4 && bNoDocsize==0);
+ p->bHasStat = isFts4;
+ p->bFts4 = isFts4;
+ p->bDescIdx = bDescIdx;
+ p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */
+ p->zContentTbl = zContent;
+ p->zLanguageid = zLanguageid;
+ zContent = 0;
+ zLanguageid = 0;
+ TESTONLY( p->inTransaction = -1 );
+ TESTONLY( p->mxSavepoint = -1 );
+
+ p->aIndex = (struct Fts3Index *)&p->azColumn[nCol];
+ memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex);
+ p->nIndex = nIndex;
+ for(i=0; i<nIndex; i++){
+ fts3HashInit(&p->aIndex[i].hPending, FTS3_HASH_STRING, 1);
+ }
+ p->abNotindexed = (u8 *)&p->aIndex[nIndex];
+
+ /* Fill in the zName and zDb fields of the vtab structure. */
+ zCsr = (char *)&p->abNotindexed[nCol];
+ p->zName = zCsr;
+ memcpy(zCsr, argv[2], nName);
+ zCsr += nName;
+ p->zDb = zCsr;
+ memcpy(zCsr, argv[1], nDb);
+ zCsr += nDb;
+
+ /* Fill in the azColumn array */
+ for(iCol=0; iCol<nCol; iCol++){
+ char *z;
+ int n = 0;
+ z = (char *)sqlite3Fts3NextToken(aCol[iCol], &n);
+ memcpy(zCsr, z, n);
+ zCsr[n] = '\0';
+ sqlite3Fts3Dequote(zCsr);
+ p->azColumn[iCol] = zCsr;
+ zCsr += n+1;
+ assert( zCsr <= &((char *)p)[nByte] );
+ }
+
+ /* Fill in the abNotindexed array */
+ for(iCol=0; iCol<nCol; iCol++){
+ int n = (int)strlen(p->azColumn[iCol]);
+ for(i=0; i<nNotindexed; i++){
+ char *zNot = azNotindexed[i];
+ if( zNot && n==(int)strlen(zNot)
+ && 0==sqlite3_strnicmp(p->azColumn[iCol], zNot, n)
+ ){
+ p->abNotindexed[iCol] = 1;
+ sqlite3_free(zNot);
+ azNotindexed[i] = 0;
+ }
+ }
+ }
+ for(i=0; i<nNotindexed; i++){
+ if( azNotindexed[i] ){
+ sqlite3Fts3ErrMsg(pzErr, "no such column: %s", azNotindexed[i]);
+ rc = SQLITE_ERROR;
+ }
+ }
+
+ if( rc==SQLITE_OK && (zCompress==0)!=(zUncompress==0) ){
+ char const *zMiss = (zCompress==0 ? "compress" : "uncompress");
+ rc = SQLITE_ERROR;
+ sqlite3Fts3ErrMsg(pzErr, "missing %s parameter in fts4 constructor", zMiss);
+ }
+ p->zReadExprlist = fts3ReadExprList(p, zUncompress, &rc);
+ p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc);
+ if( rc!=SQLITE_OK ) goto fts3_init_out;
+
+ /* If this is an xCreate call, create the underlying tables in the
+ ** database. TODO: For xConnect(), it could verify that said tables exist.
+ */
+ if( isCreate ){
+ rc = fts3CreateTables(p);
+ }
+
+ /* Check to see if a legacy fts3 table has been "upgraded" by the
+ ** addition of a %_stat table so that it can use incremental merge.
+ */
+ if( !isFts4 && !isCreate ){
+ p->bHasStat = 2;
+ }
+
+ /* Figure out the page-size for the database. This is required in order to
+ ** estimate the cost of loading large doclists from the database. */
+ fts3DatabasePageSize(&rc, p);
+ p->nNodeSize = p->nPgsz-35;
+
+ /* Declare the table schema to SQLite. */
+ fts3DeclareVtab(&rc, p);
+
+fts3_init_out:
+ sqlite3_free(zPrefix);
+ sqlite3_free(aIndex);
+ sqlite3_free(zCompress);
+ sqlite3_free(zUncompress);
+ sqlite3_free(zContent);
+ sqlite3_free(zLanguageid);
+ for(i=0; i<nNotindexed; i++) sqlite3_free(azNotindexed[i]);
+ sqlite3_free((void *)aCol);
+ sqlite3_free((void *)azNotindexed);
+ if( rc!=SQLITE_OK ){
+ if( p ){
+ fts3DisconnectMethod((sqlite3_vtab *)p);
+ }else if( pTokenizer ){
+ pTokenizer->pModule->xDestroy(pTokenizer);
+ }
+ }else{
+ assert( p->pSegments==0 );
+ *ppVTab = &p->base;
+ }
+ return rc;
+}
+
+/*
+** The xConnect() and xCreate() methods for the virtual table. All the
+** work is done in function fts3InitVtab().
+*/
+static int fts3ConnectMethod(
+ sqlite3 *db, /* Database connection */
+ void *pAux, /* Pointer to tokenizer hash table */
+ int argc, /* Number of elements in argv array */
+ const char * const *argv, /* xCreate/xConnect argument array */
+ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
+ char **pzErr /* OUT: sqlite3_malloc'd error message */
+){
+ return fts3InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr);
+}
+static int fts3CreateMethod(
+ sqlite3 *db, /* Database connection */
+ void *pAux, /* Pointer to tokenizer hash table */
+ int argc, /* Number of elements in argv array */
+ const char * const *argv, /* xCreate/xConnect argument array */
+ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
+ char **pzErr /* OUT: sqlite3_malloc'd error message */
+){
+ return fts3InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr);
+}
+
+/*
+** Set the pIdxInfo->estimatedRows variable to nRow. Unless this
+** extension is currently being used by a version of SQLite too old to
+** support estimatedRows. In that case this function is a no-op.
+*/
+static void fts3SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){
+#if SQLITE_VERSION_NUMBER>=3008002
+ if( sqlite3_libversion_number()>=3008002 ){
+ pIdxInfo->estimatedRows = nRow;
+ }
+#endif
+}
+
+/*
+** Set the SQLITE_INDEX_SCAN_UNIQUE flag in pIdxInfo->flags. Unless this
+** extension is currently being used by a version of SQLite too old to
+** support index-info flags. In that case this function is a no-op.
+*/
+static void fts3SetUniqueFlag(sqlite3_index_info *pIdxInfo){
+#if SQLITE_VERSION_NUMBER>=3008012
+ if( sqlite3_libversion_number()>=3008012 ){
+ pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE;
+ }
+#endif
+}
+
+/*
+** Implementation of the xBestIndex method for FTS3 tables. There
+** are three possible strategies, in order of preference:
+**
+** 1. Direct lookup by rowid or docid.
+** 2. Full-text search using a MATCH operator on a non-docid column.
+** 3. Linear scan of %_content table.
+*/
+static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
+ Fts3Table *p = (Fts3Table *)pVTab;
+ int i; /* Iterator variable */
+ int iCons = -1; /* Index of constraint to use */
+
+ int iLangidCons = -1; /* Index of langid=x constraint, if present */
+ int iDocidGe = -1; /* Index of docid>=x constraint, if present */
+ int iDocidLe = -1; /* Index of docid<=x constraint, if present */
+ int iIdx;
+
+ /* By default use a full table scan. This is an expensive option,
+ ** so search through the constraints to see if a more efficient
+ ** strategy is possible.
+ */
+ pInfo->idxNum = FTS3_FULLSCAN_SEARCH;
+ pInfo->estimatedCost = 5000000;
+ for(i=0; i<pInfo->nConstraint; i++){
+ int bDocid; /* True if this constraint is on docid */
+ struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i];
+ if( pCons->usable==0 ){
+ if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH ){
+ /* There exists an unusable MATCH constraint. This means that if
+ ** the planner does elect to use the results of this call as part
+ ** of the overall query plan the user will see an "unable to use
+ ** function MATCH in the requested context" error. To discourage
+ ** this, return a very high cost here. */
+ pInfo->idxNum = FTS3_FULLSCAN_SEARCH;
+ pInfo->estimatedCost = 1e50;
+ fts3SetEstimatedRows(pInfo, ((sqlite3_int64)1) << 50);
+ return SQLITE_OK;
+ }
+ continue;
+ }
+
+ bDocid = (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1);
+
+ /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */
+ if( iCons<0 && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ && bDocid ){
+ pInfo->idxNum = FTS3_DOCID_SEARCH;
+ pInfo->estimatedCost = 1.0;
+ iCons = i;
+ }
+
+ /* A MATCH constraint. Use a full-text search.
+ **
+ ** If there is more than one MATCH constraint available, use the first
+ ** one encountered. If there is both a MATCH constraint and a direct
+ ** rowid/docid lookup, prefer the MATCH strategy. This is done even
+ ** though the rowid/docid lookup is faster than a MATCH query, selecting
+ ** it would lead to an "unable to use function MATCH in the requested
+ ** context" error.
+ */
+ if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH
+ && pCons->iColumn>=0 && pCons->iColumn<=p->nColumn
+ ){
+ pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn;
+ pInfo->estimatedCost = 2.0;
+ iCons = i;
+ }
+
+ /* Equality constraint on the langid column */
+ if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ
+ && pCons->iColumn==p->nColumn + 2
+ ){
+ iLangidCons = i;
+ }
+
+ if( bDocid ){
+ switch( pCons->op ){
+ case SQLITE_INDEX_CONSTRAINT_GE:
+ case SQLITE_INDEX_CONSTRAINT_GT:
+ iDocidGe = i;
+ break;
+
+ case SQLITE_INDEX_CONSTRAINT_LE:
+ case SQLITE_INDEX_CONSTRAINT_LT:
+ iDocidLe = i;
+ break;
+ }
+ }
+ }
+
+ /* If using a docid=? or rowid=? strategy, set the UNIQUE flag. */
+ if( pInfo->idxNum==FTS3_DOCID_SEARCH ) fts3SetUniqueFlag(pInfo);
+
+ iIdx = 1;
+ if( iCons>=0 ){
+ pInfo->aConstraintUsage[iCons].argvIndex = iIdx++;
+ pInfo->aConstraintUsage[iCons].omit = 1;
+ }
+ if( iLangidCons>=0 ){
+ pInfo->idxNum |= FTS3_HAVE_LANGID;
+ pInfo->aConstraintUsage[iLangidCons].argvIndex = iIdx++;
+ }
+ if( iDocidGe>=0 ){
+ pInfo->idxNum |= FTS3_HAVE_DOCID_GE;
+ pInfo->aConstraintUsage[iDocidGe].argvIndex = iIdx++;
+ }
+ if( iDocidLe>=0 ){
+ pInfo->idxNum |= FTS3_HAVE_DOCID_LE;
+ pInfo->aConstraintUsage[iDocidLe].argvIndex = iIdx++;
+ }
+
+ /* Regardless of the strategy selected, FTS can deliver rows in rowid (or
+ ** docid) order. Both ascending and descending are possible.
+ */
+ if( pInfo->nOrderBy==1 ){
+ struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0];
+ if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){
+ if( pOrder->desc ){
+ pInfo->idxStr = "DESC";
+ }else{
+ pInfo->idxStr = "ASC";
+ }
+ pInfo->orderByConsumed = 1;
+ }
+ }
+
+ assert( p->pSegments==0 );
+ return SQLITE_OK;
+}
+
+/*
+** Implementation of xOpen method.
+*/
+static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
+ sqlite3_vtab_cursor *pCsr; /* Allocated cursor */
+
+ UNUSED_PARAMETER(pVTab);
+
+ /* Allocate a buffer large enough for an Fts3Cursor structure. If the
+ ** allocation succeeds, zero it and return SQLITE_OK. Otherwise,
+ ** if the allocation fails, return SQLITE_NOMEM.
+ */
+ *ppCsr = pCsr = (sqlite3_vtab_cursor *)sqlite3_malloc(sizeof(Fts3Cursor));
+ if( !pCsr ){
+ return SQLITE_NOMEM;
+ }
+ memset(pCsr, 0, sizeof(Fts3Cursor));
+ return SQLITE_OK;
+}
+
+/*
+** Close the cursor. For additional information see the documentation
+** on the xClose method of the virtual table interface.
+*/
+static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){
+ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
+ assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
+ sqlite3_finalize(pCsr->pStmt);
+ sqlite3Fts3ExprFree(pCsr->pExpr);
+ sqlite3Fts3FreeDeferredTokens(pCsr);
+ sqlite3_free(pCsr->aDoclist);
+ sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
+ assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
+ sqlite3_free(pCsr);
+ return SQLITE_OK;
+}
+
+/*
+** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then
+** compose and prepare an SQL statement of the form:
+**
+** "SELECT <columns> FROM %_content WHERE rowid = ?"
+**
+** (or the equivalent for a content=xxx table) and set pCsr->pStmt to
+** it. If an error occurs, return an SQLite error code.
+**
+** Otherwise, set *ppStmt to point to pCsr->pStmt and return SQLITE_OK.
+*/
+static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){
+ int rc = SQLITE_OK;
+ if( pCsr->pStmt==0 ){
+ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
+ char *zSql;
+ zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist);
+ if( !zSql ) return SQLITE_NOMEM;
+ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
+ sqlite3_free(zSql);
+ }
+ *ppStmt = pCsr->pStmt;
+ return rc;
+}
+
+/*
+** Position the pCsr->pStmt statement so that it is on the row
+** of the %_content table that contains the last match. Return
+** SQLITE_OK on success.
+*/
+static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){
+ int rc = SQLITE_OK;
+ if( pCsr->isRequireSeek ){
+ sqlite3_stmt *pStmt = 0;
+
+ rc = fts3CursorSeekStmt(pCsr, &pStmt);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId);
+ pCsr->isRequireSeek = 0;
+ if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){
+ return SQLITE_OK;
+ }else{
+ rc = sqlite3_reset(pCsr->pStmt);
+ if( rc==SQLITE_OK && ((Fts3Table *)pCsr->base.pVtab)->zContentTbl==0 ){
+ /* If no row was found and no error has occurred, then the %_content
+ ** table is missing a row that is present in the full-text index.
+ ** The data structures are corrupt. */
+ rc = FTS_CORRUPT_VTAB;
+ pCsr->isEof = 1;
+ }
+ }
+ }
+ }
+
+ if( rc!=SQLITE_OK && pContext ){
+ sqlite3_result_error_code(pContext, rc);
+ }
+ return rc;
+}
+
+/*
+** This function is used to process a single interior node when searching
+** a b-tree for a term or term prefix. The node data is passed to this
+** function via the zNode/nNode parameters. The term to search for is
+** passed in zTerm/nTerm.
+**
+** If piFirst is not NULL, then this function sets *piFirst to the blockid
+** of the child node that heads the sub-tree that may contain the term.
+**
+** If piLast is not NULL, then *piLast is set to the right-most child node
+** that heads a sub-tree that may contain a term for which zTerm/nTerm is
+** a prefix.
+**
+** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK.
+*/
+static int fts3ScanInteriorNode(
+ const char *zTerm, /* Term to select leaves for */
+ int nTerm, /* Size of term zTerm in bytes */
+ const char *zNode, /* Buffer containing segment interior node */
+ int nNode, /* Size of buffer at zNode */
+ sqlite3_int64 *piFirst, /* OUT: Selected child node */
+ sqlite3_int64 *piLast /* OUT: Selected child node */
+){
+ int rc = SQLITE_OK; /* Return code */
+ const char *zCsr = zNode; /* Cursor to iterate through node */
+ const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
+ char *zBuffer = 0; /* Buffer to load terms into */
+ int nAlloc = 0; /* Size of allocated buffer */
+ int isFirstTerm = 1; /* True when processing first term on page */
+ sqlite3_int64 iChild; /* Block id of child node to descend to */
+
+ /* Skip over the 'height' varint that occurs at the start of every
+ ** interior node. Then load the blockid of the left-child of the b-tree
+ ** node into variable iChild.
+ **
+ ** Even if the data structure on disk is corrupted, this (reading two
+ ** varints from the buffer) does not risk an overread. If zNode is a
+ ** root node, then the buffer comes from a SELECT statement. SQLite does
+ ** not make this guarantee explicitly, but in practice there are always
+ ** either more than 20 bytes of allocated space following the nNode bytes of
+ ** contents, or two zero bytes. Or, if the node is read from the %_segments
+ ** table, then there are always 20 bytes of zeroed padding following the
+ ** nNode bytes of content (see sqlite3Fts3ReadBlock() for details).
+ */
+ zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
+ zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
+ if( zCsr>zEnd ){
+ return FTS_CORRUPT_VTAB;
+ }
+
+ while( zCsr<zEnd && (piFirst || piLast) ){
+ int cmp; /* memcmp() result */
+ int nSuffix; /* Size of term suffix */
+ int nPrefix = 0; /* Size of term prefix */
+ int nBuffer; /* Total term size */
+
+ /* Load the next term on the node into zBuffer. Use realloc() to expand
+ ** the size of zBuffer if required. */
+ if( !isFirstTerm ){
+ zCsr += fts3GetVarint32(zCsr, &nPrefix);
+ }
+ isFirstTerm = 0;
+ zCsr += fts3GetVarint32(zCsr, &nSuffix);
+
+ if( nPrefix<0 || nSuffix<0 || &zCsr[nSuffix]>zEnd ){
+ rc = FTS_CORRUPT_VTAB;
+ goto finish_scan;
+ }
+ if( nPrefix+nSuffix>nAlloc ){
+ char *zNew;
+ nAlloc = (nPrefix+nSuffix) * 2;
+ zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
+ if( !zNew ){
+ rc = SQLITE_NOMEM;
+ goto finish_scan;
+ }
+ zBuffer = zNew;
+ }
+ assert( zBuffer );
+ memcpy(&zBuffer[nPrefix], zCsr, nSuffix);
+ nBuffer = nPrefix + nSuffix;
+ zCsr += nSuffix;
+
+ /* Compare the term we are searching for with the term just loaded from
+ ** the interior node. If the specified term is greater than or equal
+ ** to the term from the interior node, then all terms on the sub-tree
+ ** headed by node iChild are smaller than zTerm. No need to search
+ ** iChild.
+ **
+ ** If the interior node term is larger than the specified term, then
+ ** the tree headed by iChild may contain the specified term.
+ */
+ cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer));
+ if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){
+ *piFirst = iChild;
+ piFirst = 0;
+ }
+
+ if( piLast && cmp<0 ){
+ *piLast = iChild;
+ piLast = 0;
+ }
+
+ iChild++;
+ };
+
+ if( piFirst ) *piFirst = iChild;
+ if( piLast ) *piLast = iChild;
+
+ finish_scan:
+ sqlite3_free(zBuffer);
+ return rc;
+}
+
+
+/*
+** The buffer pointed to by argument zNode (size nNode bytes) contains an
+** interior node of a b-tree segment. The zTerm buffer (size nTerm bytes)
+** contains a term. This function searches the sub-tree headed by the zNode
+** node for the range of leaf nodes that may contain the specified term
+** or terms for which the specified term is a prefix.
+**
+** If piLeaf is not NULL, then *piLeaf is set to the blockid of the
+** left-most leaf node in the tree that may contain the specified term.
+** If piLeaf2 is not NULL, then *piLeaf2 is set to the blockid of the
+** right-most leaf node that may contain a term for which the specified
+** term is a prefix.
+**
+** It is possible that the range of returned leaf nodes does not contain
+** the specified term or any terms for which it is a prefix. However, if the
+** segment does contain any such terms, they are stored within the identified
+** range. Because this function only inspects interior segment nodes (and
+** never loads leaf nodes into memory), it is not possible to be sure.
+**
+** If an error occurs, an error code other than SQLITE_OK is returned.
+*/
+static int fts3SelectLeaf(
+ Fts3Table *p, /* Virtual table handle */
+ const char *zTerm, /* Term to select leaves for */
+ int nTerm, /* Size of term zTerm in bytes */
+ const char *zNode, /* Buffer containing segment interior node */
+ int nNode, /* Size of buffer at zNode */
+ sqlite3_int64 *piLeaf, /* Selected leaf node */
+ sqlite3_int64 *piLeaf2 /* Selected leaf node */
+){
+ int rc = SQLITE_OK; /* Return code */
+ int iHeight; /* Height of this node in tree */
+
+ assert( piLeaf || piLeaf2 );
+
+ fts3GetVarint32(zNode, &iHeight);
+ rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2);
+ assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) );
+
+ if( rc==SQLITE_OK && iHeight>1 ){
+ char *zBlob = 0; /* Blob read from %_segments table */
+ int nBlob = 0; /* Size of zBlob in bytes */
+
+ if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){
+ rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0);
+ if( rc==SQLITE_OK ){
+ rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0);
+ }
+ sqlite3_free(zBlob);
+ piLeaf = 0;
+ zBlob = 0;
+ }
+
+ if( rc==SQLITE_OK ){
+ rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0);
+ }
+ if( rc==SQLITE_OK ){
+ rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2);
+ }
+ sqlite3_free(zBlob);
+ }
+
+ return rc;
+}
+
+/*
+** This function is used to create delta-encoded serialized lists of FTS3
+** varints. Each call to this function appends a single varint to a list.
+*/
+static void fts3PutDeltaVarint(
+ char **pp, /* IN/OUT: Output pointer */
+ sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */
+ sqlite3_int64 iVal /* Write this value to the list */
+){
+ assert( iVal-*piPrev > 0 || (*piPrev==0 && iVal==0) );
+ *pp += sqlite3Fts3PutVarint(*pp, iVal-*piPrev);
+ *piPrev = iVal;
+}
+
+/*
+** When this function is called, *ppPoslist is assumed to point to the
+** start of a position-list. After it returns, *ppPoslist points to the
+** first byte after the position-list.
+**
+** A position list is list of positions (delta encoded) and columns for
+** a single document record of a doclist. So, in other words, this
+** routine advances *ppPoslist so that it points to the next docid in
+** the doclist, or to the first byte past the end of the doclist.
+**
+** If pp is not NULL, then the contents of the position list are copied
+** to *pp. *pp is set to point to the first byte past the last byte copied
+** before this function returns.
+*/
+static void fts3PoslistCopy(char **pp, char **ppPoslist){
+ char *pEnd = *ppPoslist;
+ char c = 0;
+
+ /* The end of a position list is marked by a zero encoded as an FTS3
+ ** varint. A single POS_END (0) byte. Except, if the 0 byte is preceded by
+ ** a byte with the 0x80 bit set, then it is not a varint 0, but the tail
+ ** of some other, multi-byte, value.
+ **
+ ** The following while-loop moves pEnd to point to the first byte that is not
+ ** immediately preceded by a byte with the 0x80 bit set. Then increments
+ ** pEnd once more so that it points to the byte immediately following the
+ ** last byte in the position-list.
+ */
+ while( *pEnd | c ){
+ c = *pEnd++ & 0x80;
+ testcase( c!=0 && (*pEnd)==0 );
+ }
+ pEnd++; /* Advance past the POS_END terminator byte */
+
+ if( pp ){
+ int n = (int)(pEnd - *ppPoslist);
+ char *p = *pp;
+ memcpy(p, *ppPoslist, n);
+ p += n;
+ *pp = p;
+ }
+ *ppPoslist = pEnd;
+}
+
+/*
+** When this function is called, *ppPoslist is assumed to point to the
+** start of a column-list. After it returns, *ppPoslist points to the
+** to the terminator (POS_COLUMN or POS_END) byte of the column-list.
+**
+** A column-list is list of delta-encoded positions for a single column
+** within a single document within a doclist.
+**
+** The column-list is terminated either by a POS_COLUMN varint (1) or
+** a POS_END varint (0). This routine leaves *ppPoslist pointing to
+** the POS_COLUMN or POS_END that terminates the column-list.
+**
+** If pp is not NULL, then the contents of the column-list are copied
+** to *pp. *pp is set to point to the first byte past the last byte copied
+** before this function returns. The POS_COLUMN or POS_END terminator
+** is not copied into *pp.
+*/
+static void fts3ColumnlistCopy(char **pp, char **ppPoslist){
+ char *pEnd = *ppPoslist;
+ char c = 0;
+
+ /* A column-list is terminated by either a 0x01 or 0x00 byte that is
+ ** not part of a multi-byte varint.
+ */
+ while( 0xFE & (*pEnd | c) ){
+ c = *pEnd++ & 0x80;
+ testcase( c!=0 && ((*pEnd)&0xfe)==0 );
+ }
+ if( pp ){
+ int n = (int)(pEnd - *ppPoslist);
+ char *p = *pp;
+ memcpy(p, *ppPoslist, n);
+ p += n;
+ *pp = p;
+ }
+ *ppPoslist = pEnd;
+}
+
+/*
+** Value used to signify the end of an position-list. This is safe because
+** it is not possible to have a document with 2^31 terms.
+*/
+#define POSITION_LIST_END 0x7fffffff
+
+/*
+** This function is used to help parse position-lists. When this function is
+** called, *pp may point to the start of the next varint in the position-list
+** being parsed, or it may point to 1 byte past the end of the position-list
+** (in which case **pp will be a terminator bytes POS_END (0) or
+** (1)).
+**
+** If *pp points past the end of the current position-list, set *pi to
+** POSITION_LIST_END and return. Otherwise, read the next varint from *pp,
+** increment the current value of *pi by the value read, and set *pp to
+** point to the next value before returning.
+**
+** Before calling this routine *pi must be initialized to the value of
+** the previous position, or zero if we are reading the first position
+** in the position-list. Because positions are delta-encoded, the value
+** of the previous position is needed in order to compute the value of
+** the next position.
+*/
+static void fts3ReadNextPos(
+ char **pp, /* IN/OUT: Pointer into position-list buffer */
+ sqlite3_int64 *pi /* IN/OUT: Value read from position-list */
+){
+ if( (**pp)&0xFE ){
+ fts3GetDeltaVarint(pp, pi);
+ *pi -= 2;
+ }else{
+ *pi = POSITION_LIST_END;
+ }
+}
+
+/*
+** If parameter iCol is not 0, write an POS_COLUMN (1) byte followed by
+** the value of iCol encoded as a varint to *pp. This will start a new
+** column list.
+**
+** Set *pp to point to the byte just after the last byte written before
+** returning (do not modify it if iCol==0). Return the total number of bytes
+** written (0 if iCol==0).
+*/
+static int fts3PutColNumber(char **pp, int iCol){
+ int n = 0; /* Number of bytes written */
+ if( iCol ){
+ char *p = *pp; /* Output pointer */
+ n = 1 + sqlite3Fts3PutVarint(&p[1], iCol);
+ *p = 0x01;
+ *pp = &p[n];
+ }
+ return n;
+}
+
+/*
+** Compute the union of two position lists. The output written
+** into *pp contains all positions of both *pp1 and *pp2 in sorted
+** order and with any duplicates removed. All pointers are
+** updated appropriately. The caller is responsible for insuring
+** that there is enough space in *pp to hold the complete output.
+*/
+static void fts3PoslistMerge(
+ char **pp, /* Output buffer */
+ char **pp1, /* Left input list */
+ char **pp2 /* Right input list */
+){
+ char *p = *pp;
+ char *p1 = *pp1;
+ char *p2 = *pp2;
+
+ while( *p1 || *p2 ){
+ int iCol1; /* The current column index in pp1 */
+ int iCol2; /* The current column index in pp2 */
+
+ if( *p1==POS_COLUMN ) fts3GetVarint32(&p1[1], &iCol1);
+ else if( *p1==POS_END ) iCol1 = POSITION_LIST_END;
+ else iCol1 = 0;
+
+ if( *p2==POS_COLUMN ) fts3GetVarint32(&p2[1], &iCol2);
+ else if( *p2==POS_END ) iCol2 = POSITION_LIST_END;
+ else iCol2 = 0;
+
+ if( iCol1==iCol2 ){
+ sqlite3_int64 i1 = 0; /* Last position from pp1 */
+ sqlite3_int64 i2 = 0; /* Last position from pp2 */
+ sqlite3_int64 iPrev = 0;
+ int n = fts3PutColNumber(&p, iCol1);
+ p1 += n;
+ p2 += n;
+
+ /* At this point, both p1 and p2 point to the start of column-lists
+ ** for the same column (the column with index iCol1 and iCol2).
+ ** A column-list is a list of non-negative delta-encoded varints, each
+ ** incremented by 2 before being stored. Each list is terminated by a
+ ** POS_END (0) or POS_COLUMN (1). The following block merges the two lists
+ ** and writes the results to buffer p. p is left pointing to the byte
+ ** after the list written. No terminator (POS_END or POS_COLUMN) is
+ ** written to the output.
+ */
+ fts3GetDeltaVarint(&p1, &i1);
+ fts3GetDeltaVarint(&p2, &i2);
+ do {
+ fts3PutDeltaVarint(&p, &iPrev, (i1<i2) ? i1 : i2);
+ iPrev -= 2;
+ if( i1==i2 ){
+ fts3ReadNextPos(&p1, &i1);
+ fts3ReadNextPos(&p2, &i2);
+ }else if( i1<i2 ){
+ fts3ReadNextPos(&p1, &i1);
+ }else{
+ fts3ReadNextPos(&p2, &i2);
+ }
+ }while( i1!=POSITION_LIST_END || i2!=POSITION_LIST_END );
+ }else if( iCol1<iCol2 ){
+ p1 += fts3PutColNumber(&p, iCol1);
+ fts3ColumnlistCopy(&p, &p1);
+ }else{
+ p2 += fts3PutColNumber(&p, iCol2);
+ fts3ColumnlistCopy(&p, &p2);
+ }
+ }
+
+ *p++ = POS_END;
+ *pp = p;
+ *pp1 = p1 + 1;
+ *pp2 = p2 + 1;
+}
+
+/*
+** This function is used to merge two position lists into one. When it is
+** called, *pp1 and *pp2 must both point to position lists. A position-list is
+** the part of a doclist that follows each document id. For example, if a row
+** contains:
+**
+** 'a b c'|'x y z'|'a b b a'
+**
+** Then the position list for this row for token 'b' would consist of:
+**
+** 0x02 0x01 0x02 0x03 0x03 0x00
+**
+** When this function returns, both *pp1 and *pp2 are left pointing to the
+** byte following the 0x00 terminator of their respective position lists.
+**
+** If isSaveLeft is 0, an entry is added to the output position list for
+** each position in *pp2 for which there exists one or more positions in
+** *pp1 so that (pos(*pp2)>pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e.
+** when the *pp1 token appears before the *pp2 token, but not more than nToken
+** slots before it.
+**
+** e.g. nToken==1 searches for adjacent positions.
+*/
+static int fts3PoslistPhraseMerge(
+ char **pp, /* IN/OUT: Preallocated output buffer */
+ int nToken, /* Maximum difference in token positions */
+ int isSaveLeft, /* Save the left position */
+ int isExact, /* If *pp1 is exactly nTokens before *pp2 */
+ char **pp1, /* IN/OUT: Left input list */
+ char **pp2 /* IN/OUT: Right input list */
+){
+ char *p = *pp;
+ char *p1 = *pp1;
+ char *p2 = *pp2;
+ int iCol1 = 0;
+ int iCol2 = 0;
+
+ /* Never set both isSaveLeft and isExact for the same invocation. */
+ assert( isSaveLeft==0 || isExact==0 );
+
+ assert( p!=0 && *p1!=0 && *p2!=0 );
+ if( *p1==POS_COLUMN ){
+ p1++;
+ p1 += fts3GetVarint32(p1, &iCol1);
+ }
+ if( *p2==POS_COLUMN ){
+ p2++;
+ p2 += fts3GetVarint32(p2, &iCol2);
+ }
+
+ while( 1 ){
+ if( iCol1==iCol2 ){
+ char *pSave = p;
+ sqlite3_int64 iPrev = 0;
+ sqlite3_int64 iPos1 = 0;
+ sqlite3_int64 iPos2 = 0;
+
+ if( iCol1 ){
+ *p++ = POS_COLUMN;
+ p += sqlite3Fts3PutVarint(p, iCol1);
+ }
+
+ assert( *p1!=POS_END && *p1!=POS_COLUMN );
+ assert( *p2!=POS_END && *p2!=POS_COLUMN );
+ fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2;
+ fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2;
+
+ while( 1 ){
+ if( iPos2==iPos1+nToken
+ || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken)
+ ){
+ sqlite3_int64 iSave;
+ iSave = isSaveLeft ? iPos1 : iPos2;
+ fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2;
+ pSave = 0;
+ assert( p );
+ }
+ if( (!isSaveLeft && iPos2<=(iPos1+nToken)) || iPos2<=iPos1 ){
+ if( (*p2&0xFE)==0 ) break;
+ fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2;
+ }else{
+ if( (*p1&0xFE)==0 ) break;
+ fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2;
+ }
+ }
+
+ if( pSave ){
+ assert( pp && p );
+ p = pSave;
+ }
+
+ fts3ColumnlistCopy(0, &p1);
+ fts3ColumnlistCopy(0, &p2);
+ assert( (*p1&0xFE)==0 && (*p2&0xFE)==0 );
+ if( 0==*p1 || 0==*p2 ) break;
+
+ p1++;
+ p1 += fts3GetVarint32(p1, &iCol1);
+ p2++;
+ p2 += fts3GetVarint32(p2, &iCol2);
+ }
+
+ /* Advance pointer p1 or p2 (whichever corresponds to the smaller of
+ ** iCol1 and iCol2) so that it points to either the 0x00 that marks the
+ ** end of the position list, or the 0x01 that precedes the next
+ ** column-number in the position list.
+ */
+ else if( iCol1<iCol2 ){
+ fts3ColumnlistCopy(0, &p1);
+ if( 0==*p1 ) break;
+ p1++;
+ p1 += fts3GetVarint32(p1, &iCol1);
+ }else{
+ fts3ColumnlistCopy(0, &p2);
+ if( 0==*p2 ) break;
+ p2++;
+ p2 += fts3GetVarint32(p2, &iCol2);
+ }
+ }
+
+ fts3PoslistCopy(0, &p2);
+ fts3PoslistCopy(0, &p1);
+ *pp1 = p1;
+ *pp2 = p2;
+ if( *pp==p ){
+ return 0;
+ }
+ *p++ = 0x00;
+ *pp = p;
+ return 1;
+}
+
+/*
+** Merge two position-lists as required by the NEAR operator. The argument
+** position lists correspond to the left and right phrases of an expression
+** like:
+**
+** "phrase 1" NEAR "phrase number 2"
+**
+** Position list *pp1 corresponds to the left-hand side of the NEAR
+** expression and *pp2 to the right. As usual, the indexes in the position
+** lists are the offsets of the last token in each phrase (tokens "1" and "2"
+** in the example above).
+**
+** The output position list - written to *pp - is a copy of *pp2 with those
+** entries that are not sufficiently NEAR entries in *pp1 removed.
+*/
+static int fts3PoslistNearMerge(
+ char **pp, /* Output buffer */
+ char *aTmp, /* Temporary buffer space */
+ int nRight, /* Maximum difference in token positions */
+ int nLeft, /* Maximum difference in token positions */
+ char **pp1, /* IN/OUT: Left input list */
+ char **pp2 /* IN/OUT: Right input list */
+){
+ char *p1 = *pp1;
+ char *p2 = *pp2;
+
+ char *pTmp1 = aTmp;
+ char *pTmp2;
+ char *aTmp2;
+ int res = 1;
+
+ fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2);
+ aTmp2 = pTmp2 = pTmp1;
+ *pp1 = p1;
+ *pp2 = p2;
+ fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1);
+ if( pTmp1!=aTmp && pTmp2!=aTmp2 ){
+ fts3PoslistMerge(pp, &aTmp, &aTmp2);
+ }else if( pTmp1!=aTmp ){
+ fts3PoslistCopy(pp, &aTmp);
+ }else if( pTmp2!=aTmp2 ){
+ fts3PoslistCopy(pp, &aTmp2);
+ }else{
+ res = 0;
+ }
+
+ return res;
+}
+
+/*
+** An instance of this function is used to merge together the (potentially
+** large number of) doclists for each term that matches a prefix query.
+** See function fts3TermSelectMerge() for details.
+*/
+typedef struct TermSelect TermSelect;
+struct TermSelect {
+ char *aaOutput[16]; /* Malloc'd output buffers */
+ int anOutput[16]; /* Size each output buffer in bytes */
+};
+
+/*
+** This function is used to read a single varint from a buffer. Parameter
+** pEnd points 1 byte past the end of the buffer. When this function is
+** called, if *pp points to pEnd or greater, then the end of the buffer
+** has been reached. In this case *pp is set to 0 and the function returns.
+**
+** If *pp does not point to or past pEnd, then a single varint is read
+** from *pp. *pp is then set to point 1 byte past the end of the read varint.
+**
+** If bDescIdx is false, the value read is added to *pVal before returning.
+** If it is true, the value read is subtracted from *pVal before this
+** function returns.
+*/
+static void fts3GetDeltaVarint3(
+ char **pp, /* IN/OUT: Point to read varint from */
+ char *pEnd, /* End of buffer */
+ int bDescIdx, /* True if docids are descending */
+ sqlite3_int64 *pVal /* IN/OUT: Integer value */
+){
+ if( *pp>=pEnd ){
+ *pp = 0;
+ }else{
+ sqlite3_int64 iVal;
+ *pp += sqlite3Fts3GetVarint(*pp, &iVal);
+ if( bDescIdx ){
+ *pVal -= iVal;
+ }else{
+ *pVal += iVal;
+ }
+ }
+}
+
+/*
+** This function is used to write a single varint to a buffer. The varint
+** is written to *pp. Before returning, *pp is set to point 1 byte past the
+** end of the value written.
+**
+** If *pbFirst is zero when this function is called, the value written to
+** the buffer is that of parameter iVal.
+**
+** If *pbFirst is non-zero when this function is called, then the value
+** written is either (iVal-*piPrev) (if bDescIdx is zero) or (*piPrev-iVal)
+** (if bDescIdx is non-zero).
+**
+** Before returning, this function always sets *pbFirst to 1 and *piPrev
+** to the value of parameter iVal.
+*/
+static void fts3PutDeltaVarint3(
+ char **pp, /* IN/OUT: Output pointer */
+ int bDescIdx, /* True for descending docids */
+ sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */
+ int *pbFirst, /* IN/OUT: True after first int written */
+ sqlite3_int64 iVal /* Write this value to the list */
+){
+ sqlite3_int64 iWrite;
+ if( bDescIdx==0 || *pbFirst==0 ){
+ iWrite = iVal - *piPrev;
+ }else{
+ iWrite = *piPrev - iVal;
+ }
+ assert( *pbFirst || *piPrev==0 );
+ assert( *pbFirst==0 || iWrite>0 );
+ *pp += sqlite3Fts3PutVarint(*pp, iWrite);
+ *piPrev = iVal;
+ *pbFirst = 1;
+}
+
+
+/*
+** This macro is used by various functions that merge doclists. The two
+** arguments are 64-bit docid values. If the value of the stack variable
+** bDescDoclist is 0 when this macro is invoked, then it returns (i1-i2).
+** Otherwise, (i2-i1).
+**
+** Using this makes it easier to write code that can merge doclists that are
+** sorted in either ascending or descending order.
+*/
+#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1-i2))
+
+/*
+** This function does an "OR" merge of two doclists (output contains all
+** positions contained in either argument doclist). If the docids in the
+** input doclists are sorted in ascending order, parameter bDescDoclist
+** should be false. If they are sorted in ascending order, it should be
+** passed a non-zero value.
+**
+** If no error occurs, *paOut is set to point at an sqlite3_malloc'd buffer
+** containing the output doclist and SQLITE_OK is returned. In this case
+** *pnOut is set to the number of bytes in the output doclist.
+**
+** If an error occurs, an SQLite error code is returned. The output values
+** are undefined in this case.
+*/
+static int fts3DoclistOrMerge(
+ int bDescDoclist, /* True if arguments are desc */
+ char *a1, int n1, /* First doclist */
+ char *a2, int n2, /* Second doclist */
+ char **paOut, int *pnOut /* OUT: Malloc'd doclist */
+){
+ sqlite3_int64 i1 = 0;
+ sqlite3_int64 i2 = 0;
+ sqlite3_int64 iPrev = 0;
+ char *pEnd1 = &a1[n1];
+ char *pEnd2 = &a2[n2];
+ char *p1 = a1;
+ char *p2 = a2;
+ char *p;
+ char *aOut;
+ int bFirstOut = 0;
+
+ *paOut = 0;
+ *pnOut = 0;
+
+ /* Allocate space for the output. Both the input and output doclists
+ ** are delta encoded. If they are in ascending order (bDescDoclist==0),
+ ** then the first docid in each list is simply encoded as a varint. For
+ ** each subsequent docid, the varint stored is the difference between the
+ ** current and previous docid (a positive number - since the list is in
+ ** ascending order).
+ **
+ ** The first docid written to the output is therefore encoded using the
+ ** same number of bytes as it is in whichever of the input lists it is
+ ** read from. And each subsequent docid read from the same input list
+ ** consumes either the same or less bytes as it did in the input (since
+ ** the difference between it and the previous value in the output must
+ ** be a positive value less than or equal to the delta value read from
+ ** the input list). The same argument applies to all but the first docid
+ ** read from the 'other' list. And to the contents of all position lists
+ ** that will be copied and merged from the input to the output.
+ **
+ ** However, if the first docid copied to the output is a negative number,
+ ** then the encoding of the first docid from the 'other' input list may
+ ** be larger in the output than it was in the input (since the delta value
+ ** may be a larger positive integer than the actual docid).
+ **
+ ** The space required to store the output is therefore the sum of the
+ ** sizes of the two inputs, plus enough space for exactly one of the input
+ ** docids to grow.
+ **
+ ** A symetric argument may be made if the doclists are in descending
+ ** order.
+ */
+ aOut = sqlite3_malloc(n1+n2+FTS3_VARINT_MAX-1);
+ if( !aOut ) return SQLITE_NOMEM;
+
+ p = aOut;
+ fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
+ fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);
+ while( p1 || p2 ){
+ sqlite3_int64 iDiff = DOCID_CMP(i1, i2);
+
+ if( p2 && p1 && iDiff==0 ){
+ fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1);
+ fts3PoslistMerge(&p, &p1, &p2);
+ fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1);
+ fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
+ }else if( !p2 || (p1 && iDiff<0) ){
+ fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1);
+ fts3PoslistCopy(&p, &p1);
+ fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1);
+ }else{
+ fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i2);
+ fts3PoslistCopy(&p, &p2);
+ fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
+ }
+ }
+
+ *paOut = aOut;
+ *pnOut = (int)(p-aOut);
+ assert( *pnOut<=n1+n2+FTS3_VARINT_MAX-1 );
+ return SQLITE_OK;
+}
+
+/*
+** This function does a "phrase" merge of two doclists. In a phrase merge,
+** the output contains a copy of each position from the right-hand input
+** doclist for which there is a position in the left-hand input doclist
+** exactly nDist tokens before it.
+**
+** If the docids in the input doclists are sorted in ascending order,
+** parameter bDescDoclist should be false. If they are sorted in ascending
+** order, it should be passed a non-zero value.
+**
+** The right-hand input doclist is overwritten by this function.
+*/
+static int fts3DoclistPhraseMerge(
+ int bDescDoclist, /* True if arguments are desc */
+ int nDist, /* Distance from left to right (1=adjacent) */
+ char *aLeft, int nLeft, /* Left doclist */
+ char **paRight, int *pnRight /* IN/OUT: Right/output doclist */
+){
+ sqlite3_int64 i1 = 0;
+ sqlite3_int64 i2 = 0;
+ sqlite3_int64 iPrev = 0;
+ char *aRight = *paRight;
+ char *pEnd1 = &aLeft[nLeft];
+ char *pEnd2 = &aRight[*pnRight];
+ char *p1 = aLeft;
+ char *p2 = aRight;
+ char *p;
+ int bFirstOut = 0;
+ char *aOut;
+
+ assert( nDist>0 );
+ if( bDescDoclist ){
+ aOut = sqlite3_malloc(*pnRight + FTS3_VARINT_MAX);
+ if( aOut==0 ) return SQLITE_NOMEM;
+ }else{
+ aOut = aRight;
+ }
+ p = aOut;
+
+ fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
+ fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);
+
+ while( p1 && p2 ){
+ sqlite3_int64 iDiff = DOCID_CMP(i1, i2);
+ if( iDiff==0 ){
+ char *pSave = p;
+ sqlite3_int64 iPrevSave = iPrev;
+ int bFirstOutSave = bFirstOut;
+
+ fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1);
+ if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){
+ p = pSave;
+ iPrev = iPrevSave;
+ bFirstOut = bFirstOutSave;
+ }
+ fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1);
+ fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
+ }else if( iDiff<0 ){
+ fts3PoslistCopy(0, &p1);
+ fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1);
+ }else{
+ fts3PoslistCopy(0, &p2);
+ fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
+ }
+ }
+
+ *pnRight = (int)(p - aOut);
+ if( bDescDoclist ){
+ sqlite3_free(aRight);
+ *paRight = aOut;
+ }
+
+ return SQLITE_OK;
+}
+
+/*
+** Argument pList points to a position list nList bytes in size. This
+** function checks to see if the position list contains any entries for
+** a token in position 0 (of any column). If so, it writes argument iDelta
+** to the output buffer pOut, followed by a position list consisting only
+** of the entries from pList at position 0, and terminated by an 0x00 byte.
+** The value returned is the number of bytes written to pOut (if any).
+*/
+SQLITE_PRIVATE int sqlite3Fts3FirstFilter(
+ sqlite3_int64 iDelta, /* Varint that may be written to pOut */
+ char *pList, /* Position list (no 0x00 term) */
+ int nList, /* Size of pList in bytes */
+ char *pOut /* Write output here */
+){
+ int nOut = 0;
+ int bWritten = 0; /* True once iDelta has been written */
+ char *p = pList;
+ char *pEnd = &pList[nList];
+
+ if( *p!=0x01 ){
+ if( *p==0x02 ){
+ nOut += sqlite3Fts3PutVarint(&pOut[nOut], iDelta);
+ pOut[nOut++] = 0x02;
+ bWritten = 1;
+ }
+ fts3ColumnlistCopy(0, &p);
+ }
+
+ while( p<pEnd && *p==0x01 ){
+ sqlite3_int64 iCol;
+ p++;
+ p += sqlite3Fts3GetVarint(p, &iCol);
+ if( *p==0x02 ){
+ if( bWritten==0 ){
+ nOut += sqlite3Fts3PutVarint(&pOut[nOut], iDelta);
+ bWritten = 1;
+ }
+ pOut[nOut++] = 0x01;
+ nOut += sqlite3Fts3PutVarint(&pOut[nOut], iCol);
+ pOut[nOut++] = 0x02;
+ }
+ fts3ColumnlistCopy(0, &p);
+ }
+ if( bWritten ){
+ pOut[nOut++] = 0x00;
+ }
+
+ return nOut;
+}
+
+
+/*
+** Merge all doclists in the TermSelect.aaOutput[] array into a single
+** doclist stored in TermSelect.aaOutput[0]. If successful, delete all
+** other doclists (except the aaOutput[0] one) and return SQLITE_OK.
+**
+** If an OOM error occurs, return SQLITE_NOMEM. In this case it is
+** the responsibility of the caller to free any doclists left in the
+** TermSelect.aaOutput[] array.
+*/
+static int fts3TermSelectFinishMerge(Fts3Table *p, TermSelect *pTS){
+ char *aOut = 0;
+ int nOut = 0;
+ int i;
+
+ /* Loop through the doclists in the aaOutput[] array. Merge them all
+ ** into a single doclist.
+ */
+ for(i=0; i<SizeofArray(pTS->aaOutput); i++){
+ if( pTS->aaOutput[i] ){
+ if( !aOut ){
+ aOut = pTS->aaOutput[i];
+ nOut = pTS->anOutput[i];
+ pTS->aaOutput[i] = 0;
+ }else{
+ int nNew;
+ char *aNew;
+
+ int rc = fts3DoclistOrMerge(p->bDescIdx,
+ pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew
+ );
+ if( rc!=SQLITE_OK ){
+ sqlite3_free(aOut);
+ return rc;
+ }
+
+ sqlite3_free(pTS->aaOutput[i]);
+ sqlite3_free(aOut);
+ pTS->aaOutput[i] = 0;
+ aOut = aNew;
+ nOut = nNew;
+ }
+ }
+ }
+
+ pTS->aaOutput[0] = aOut;
+ pTS->anOutput[0] = nOut;
+ return SQLITE_OK;
+}
+
+/*
+** Merge the doclist aDoclist/nDoclist into the TermSelect object passed
+** as the first argument. The merge is an "OR" merge (see function
+** fts3DoclistOrMerge() for details).
+**
+** This function is called with the doclist for each term that matches
+** a queried prefix. It merges all these doclists into one, the doclist
+** for the specified prefix. Since there can be a very large number of
+** doclists to merge, the merging is done pair-wise using the TermSelect
+** object.
+**
+** This function returns SQLITE_OK if the merge is successful, or an
+** SQLite error code (SQLITE_NOMEM) if an error occurs.
+*/
+static int fts3TermSelectMerge(
+ Fts3Table *p, /* FTS table handle */
+ TermSelect *pTS, /* TermSelect object to merge into */
+ char *aDoclist, /* Pointer to doclist */
+ int nDoclist /* Size of aDoclist in bytes */
+){
+ if( pTS->aaOutput[0]==0 ){
+ /* If this is the first term selected, copy the doclist to the output
+ ** buffer using memcpy().
+ **
+ ** Add FTS3_VARINT_MAX bytes of unused space to the end of the
+ ** allocation. This is so as to ensure that the buffer is big enough
+ ** to hold the current doclist AND'd with any other doclist. If the
+ ** doclists are stored in order=ASC order, this padding would not be
+ ** required (since the size of [doclistA AND doclistB] is always less
+ ** than or equal to the size of [doclistA] in that case). But this is
+ ** not true for order=DESC. For example, a doclist containing (1, -1)
+ ** may be smaller than (-1), as in the first example the -1 may be stored
+ ** as a single-byte delta, whereas in the second it must be stored as a
+ ** FTS3_VARINT_MAX byte varint.
+ **
+ ** Similar padding is added in the fts3DoclistOrMerge() function.
+ */
+ pTS->aaOutput[0] = sqlite3_malloc(nDoclist + FTS3_VARINT_MAX + 1);
+ pTS->anOutput[0] = nDoclist;
+ if( pTS->aaOutput[0] ){
+ memcpy(pTS->aaOutput[0], aDoclist, nDoclist);
+ }else{
+ return SQLITE_NOMEM;
+ }
+ }else{
+ char *aMerge = aDoclist;
+ int nMerge = nDoclist;
+ int iOut;
+
+ for(iOut=0; iOut<SizeofArray(pTS->aaOutput); iOut++){
+ if( pTS->aaOutput[iOut]==0 ){
+ assert( iOut>0 );
+ pTS->aaOutput[iOut] = aMerge;
+ pTS->anOutput[iOut] = nMerge;
+ break;
+ }else{
+ char *aNew;
+ int nNew;
+
+ int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge,
+ pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew
+ );
+ if( rc!=SQLITE_OK ){
+ if( aMerge!=aDoclist ) sqlite3_free(aMerge);
+ return rc;
+ }
+
+ if( aMerge!=aDoclist ) sqlite3_free(aMerge);
+ sqlite3_free(pTS->aaOutput[iOut]);
+ pTS->aaOutput[iOut] = 0;
+
+ aMerge = aNew;
+ nMerge = nNew;
+ if( (iOut+1)==SizeofArray(pTS->aaOutput) ){
+ pTS->aaOutput[iOut] = aMerge;
+ pTS->anOutput[iOut] = nMerge;
+ }
+ }
+ }
+ }
+ return SQLITE_OK;
+}
+
+/*
+** Append SegReader object pNew to the end of the pCsr->apSegment[] array.
+*/
+static int fts3SegReaderCursorAppend(
+ Fts3MultiSegReader *pCsr,
+ Fts3SegReader *pNew
+){
+ if( (pCsr->nSegment%16)==0 ){
+ Fts3SegReader **apNew;
+ int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*);
+ apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte);
+ if( !apNew ){
+ sqlite3Fts3SegReaderFree(pNew);
+ return SQLITE_NOMEM;
+ }
+ pCsr->apSegment = apNew;
+ }
+ pCsr->apSegment[pCsr->nSegment++] = pNew;
+ return SQLITE_OK;
+}
+
+/*
+** Add seg-reader objects to the Fts3MultiSegReader object passed as the
+** 8th argument.
+**
+** This function returns SQLITE_OK if successful, or an SQLite error code
+** otherwise.
+*/
+static int fts3SegReaderCursor(
+ Fts3Table *p, /* FTS3 table handle */
+ int iLangid, /* Language id */
+ int iIndex, /* Index to search (from 0 to p->nIndex-1) */
+ int iLevel, /* Level of segments to scan */
+ const char *zTerm, /* Term to query for */
+ int nTerm, /* Size of zTerm in bytes */
+ int isPrefix, /* True for a prefix search */
+ int isScan, /* True to scan from zTerm to EOF */
+ Fts3MultiSegReader *pCsr /* Cursor object to populate */
+){
+ int rc = SQLITE_OK; /* Error code */
+ sqlite3_stmt *pStmt = 0; /* Statement to iterate through segments */
+ int rc2; /* Result of sqlite3_reset() */
+
+ /* If iLevel is less than 0 and this is not a scan, include a seg-reader
+ ** for the pending-terms. If this is a scan, then this call must be being
+ ** made by an fts4aux module, not an FTS table. In this case calling
+ ** Fts3SegReaderPending might segfault, as the data structures used by
+ ** fts4aux are not completely populated. So it's easiest to filter these
+ ** calls out here. */
+ if( iLevel<0 && p->aIndex ){
+ Fts3SegReader *pSeg = 0;
+ rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix||isScan, &pSeg);
+ if( rc==SQLITE_OK && pSeg ){
+ rc = fts3SegReaderCursorAppend(pCsr, pSeg);
+ }
+ }
+
+ if( iLevel!=FTS3_SEGCURSOR_PENDING ){
+ if( rc==SQLITE_OK ){
+ rc = sqlite3Fts3AllSegdirs(p, iLangid, iIndex, iLevel, &pStmt);
+ }
+
+ while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
+ Fts3SegReader *pSeg = 0;
+
+ /* Read the values returned by the SELECT into local variables. */
+ sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1);
+ sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2);
+ sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3);
+ int nRoot = sqlite3_column_bytes(pStmt, 4);
+ char const *zRoot = sqlite3_column_blob(pStmt, 4);
+
+ /* If zTerm is not NULL, and this segment is not stored entirely on its
+ ** root node, the range of leaves scanned can be reduced. Do this. */
+ if( iStartBlock && zTerm ){
+ sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0);
+ rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi);
+ if( rc!=SQLITE_OK ) goto finished;
+ if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock;
+ }
+
+ rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1,
+ (isPrefix==0 && isScan==0),
+ iStartBlock, iLeavesEndBlock,
+ iEndBlock, zRoot, nRoot, &pSeg
+ );
+ if( rc!=SQLITE_OK ) goto finished;
+ rc = fts3SegReaderCursorAppend(pCsr, pSeg);
+ }
+ }
+
+ finished:
+ rc2 = sqlite3_reset(pStmt);
+ if( rc==SQLITE_DONE ) rc = rc2;
+
+ return rc;
+}
+
+/*
+** Set up a cursor object for iterating through a full-text index or a
+** single level therein.
+*/
+SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor(
+ Fts3Table *p, /* FTS3 table handle */
+ int iLangid, /* Language-id to search */
+ int iIndex, /* Index to search (from 0 to p->nIndex-1) */
+ int iLevel, /* Level of segments to scan */
+ const char *zTerm, /* Term to query for */
+ int nTerm, /* Size of zTerm in bytes */
+ int isPrefix, /* True for a prefix search */
+ int isScan, /* True to scan from zTerm to EOF */
+ Fts3MultiSegReader *pCsr /* Cursor object to populate */
+){
+ assert( iIndex>=0 && iIndex<p->nIndex );
+ assert( iLevel==FTS3_SEGCURSOR_ALL
+ || iLevel==FTS3_SEGCURSOR_PENDING
+ || iLevel>=0
+ );
+ assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
+ assert( FTS3_SEGCURSOR_ALL<0 && FTS3_SEGCURSOR_PENDING<0 );
+ assert( isPrefix==0 || isScan==0 );
+
+ memset(pCsr, 0, sizeof(Fts3MultiSegReader));
+ return fts3SegReaderCursor(
+ p, iLangid, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr
+ );
+}
+
+/*
+** In addition to its current configuration, have the Fts3MultiSegReader
+** passed as the 4th argument also scan the doclist for term zTerm/nTerm.
+**
+** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
+*/
+static int fts3SegReaderCursorAddZero(
+ Fts3Table *p, /* FTS virtual table handle */
+ int iLangid,
+ const char *zTerm, /* Term to scan doclist of */
+ int nTerm, /* Number of bytes in zTerm */
+ Fts3MultiSegReader *pCsr /* Fts3MultiSegReader to modify */
+){
+ return fts3SegReaderCursor(p,
+ iLangid, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr
+ );
+}
+
+/*
+** Open an Fts3MultiSegReader to scan the doclist for term zTerm/nTerm. Or,
+** if isPrefix is true, to scan the doclist for all terms for which
+** zTerm/nTerm is a prefix. If successful, return SQLITE_OK and write
+** a pointer to the new Fts3MultiSegReader to *ppSegcsr. Otherwise, return
+** an SQLite error code.
+**
+** It is the responsibility of the caller to free this object by eventually
+** passing it to fts3SegReaderCursorFree()
+**
+** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
+** Output parameter *ppSegcsr is set to 0 if an error occurs.
+*/
+static int fts3TermSegReaderCursor(
+ Fts3Cursor *pCsr, /* Virtual table cursor handle */
+ const char *zTerm, /* Term to query for */
+ int nTerm, /* Size of zTerm in bytes */
+ int isPrefix, /* True for a prefix search */
+ Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */
+){
+ Fts3MultiSegReader *pSegcsr; /* Object to allocate and return */
+ int rc = SQLITE_NOMEM; /* Return code */
+
+ pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader));
+ if( pSegcsr ){
+ int i;
+ int bFound = 0; /* True once an index has been found */
+ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
+
+ if( isPrefix ){
+ for(i=1; bFound==0 && i<p->nIndex; i++){
+ if( p->aIndex[i].nPrefix==nTerm ){
+ bFound = 1;
+ rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid,
+ i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr
+ );
+ pSegcsr->bLookup = 1;
+ }
+ }
+
+ for(i=1; bFound==0 && i<p->nIndex; i++){
+ if( p->aIndex[i].nPrefix==nTerm+1 ){
+ bFound = 1;
+ rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid,
+ i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr
+ );
+ if( rc==SQLITE_OK ){
+ rc = fts3SegReaderCursorAddZero(
+ p, pCsr->iLangid, zTerm, nTerm, pSegcsr
+ );
+ }
+ }
+ }
+ }
+
+ if( bFound==0 ){
+ rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid,
+ 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr
+ );
+ pSegcsr->bLookup = !isPrefix;
+ }
+ }
+
+ *ppSegcsr = pSegcsr;
+ return rc;
+}
+
+/*
+** Free an Fts3MultiSegReader allocated by fts3TermSegReaderCursor().
+*/
+static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){
+ sqlite3Fts3SegReaderFinish(pSegcsr);
+ sqlite3_free(pSegcsr);
+}
+
+/*
+** This function retrieves the doclist for the specified term (or term
+** prefix) from the database.
+*/
+static int fts3TermSelect(
+ Fts3Table *p, /* Virtual table handle */
+ Fts3PhraseToken *pTok, /* Token to query for */
+ int iColumn, /* Column to query (or -ve for all columns) */
+ int *pnOut, /* OUT: Size of buffer at *ppOut */
+ char **ppOut /* OUT: Malloced result buffer */
+){
+ int rc; /* Return code */
+ Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */
+ TermSelect tsc; /* Object for pair-wise doclist merging */
+ Fts3SegFilter filter; /* Segment term filter configuration */
+
+ pSegcsr = pTok->pSegcsr;
+ memset(&tsc, 0, sizeof(TermSelect));
+
+ filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | FTS3_SEGMENT_REQUIRE_POS
+ | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0)
+ | (pTok->bFirst ? FTS3_SEGMENT_FIRST : 0)
+ | (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0);
+ filter.iCol = iColumn;
+ filter.zTerm = pTok->z;
+ filter.nTerm = pTok->n;
+
+ rc = sqlite3Fts3SegReaderStart(p, pSegcsr, &filter);
+ while( SQLITE_OK==rc
+ && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr))
+ ){
+ rc = fts3TermSelectMerge(p, &tsc, pSegcsr->aDoclist, pSegcsr->nDoclist);
+ }
+
+ if( rc==SQLITE_OK ){
+ rc = fts3TermSelectFinishMerge(p, &tsc);
+ }
+ if( rc==SQLITE_OK ){
+ *ppOut = tsc.aaOutput[0];
+ *pnOut = tsc.anOutput[0];
+ }else{
+ int i;
+ for(i=0; i<SizeofArray(tsc.aaOutput); i++){
+ sqlite3_free(tsc.aaOutput[i]);
+ }
+ }
+
+ fts3SegReaderCursorFree(pSegcsr);
+ pTok->pSegcsr = 0;
+ return rc;
+}
+
+/*
+** This function counts the total number of docids in the doclist stored
+** in buffer aList[], size nList bytes.
+**
+** If the isPoslist argument is true, then it is assumed that the doclist
+** contains a position-list following each docid. Otherwise, it is assumed
+** that the doclist is simply a list of docids stored as delta encoded
+** varints.
+*/
+static int fts3DoclistCountDocids(char *aList, int nList){
+ int nDoc = 0; /* Return value */
+ if( aList ){
+ char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */
+ char *p = aList; /* Cursor */
+ while( p<aEnd ){
+ nDoc++;
+ while( (*p++)&0x80 ); /* Skip docid varint */
+ fts3PoslistCopy(0, &p); /* Skip over position list */
+ }
+ }
+
+ return nDoc;
+}
+
+/*
+** Advance the cursor to the next row in the %_content table that
+** matches the search criteria. For a MATCH search, this will be
+** the next row that matches. For a full-table scan, this will be
+** simply the next row in the %_content table. For a docid lookup,
+** this routine simply sets the EOF flag.
+**
+** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned
+** even if we reach end-of-file. The fts3EofMethod() will be called
+** subsequently to determine whether or not an EOF was hit.
+*/
+static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
+ int rc;
+ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
+ if( pCsr->eSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){
+ if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
+ pCsr->isEof = 1;
+ rc = sqlite3_reset(pCsr->pStmt);
+ }else{
+ pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0);
+ rc = SQLITE_OK;
+ }
+ }else{
+ rc = fts3EvalNext((Fts3Cursor *)pCursor);
+ }
+ assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
+ return rc;
+}
+
+/*
+** The following are copied from sqliteInt.h.
+**
+** Constants for the largest and smallest possible 64-bit signed integers.
+** These macros are designed to work correctly on both 32-bit and 64-bit
+** compilers.
+*/
+#ifndef SQLITE_AMALGAMATION
+# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32))
+# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64)
+#endif
+
+/*
+** If the numeric type of argument pVal is "integer", then return it
+** converted to a 64-bit signed integer. Otherwise, return a copy of
+** the second parameter, iDefault.
+*/
+static sqlite3_int64 fts3DocidRange(sqlite3_value *pVal, i64 iDefault){
+ if( pVal ){
+ int eType = sqlite3_value_numeric_type(pVal);
+ if( eType==SQLITE_INTEGER ){
+ return sqlite3_value_int64(pVal);
+ }
+ }
+ return iDefault;
+}
+
+/*
+** This is the xFilter interface for the virtual table. See
+** the virtual table xFilter method documentation for additional
+** information.
+**
+** If idxNum==FTS3_FULLSCAN_SEARCH then do a full table scan against
+** the %_content table.
+**
+** If idxNum==FTS3_DOCID_SEARCH then do a docid lookup for a single entry
+** in the %_content table.
+**
+** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index. The
+** column on the left-hand side of the MATCH operator is column
+** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand
+** side of the MATCH operator.
+*/
+static int fts3FilterMethod(
+ sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
+ int idxNum, /* Strategy index */
+ const char *idxStr, /* Unused */
+ int nVal, /* Number of elements in apVal */
+ sqlite3_value **apVal /* Arguments for the indexing scheme */
+){
+ int rc = SQLITE_OK;
+ char *zSql; /* SQL statement used to access %_content */
+ int eSearch;
+ Fts3Table *p = (Fts3Table *)pCursor->pVtab;
+ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
+
+ sqlite3_value *pCons = 0; /* The MATCH or rowid constraint, if any */
+ sqlite3_value *pLangid = 0; /* The "langid = ?" constraint, if any */
+ sqlite3_value *pDocidGe = 0; /* The "docid >= ?" constraint, if any */
+ sqlite3_value *pDocidLe = 0; /* The "docid <= ?" constraint, if any */
+ int iIdx;
+
+ UNUSED_PARAMETER(idxStr);
+ UNUSED_PARAMETER(nVal);
+
+ eSearch = (idxNum & 0x0000FFFF);
+ assert( eSearch>=0 && eSearch<=(FTS3_FULLTEXT_SEARCH+p->nColumn) );
+ assert( p->pSegments==0 );
+
+ /* Collect arguments into local variables */
+ iIdx = 0;
+ if( eSearch!=FTS3_FULLSCAN_SEARCH ) pCons = apVal[iIdx++];
+ if( idxNum & FTS3_HAVE_LANGID ) pLangid = apVal[iIdx++];
+ if( idxNum & FTS3_HAVE_DOCID_GE ) pDocidGe = apVal[iIdx++];
+ if( idxNum & FTS3_HAVE_DOCID_LE ) pDocidLe = apVal[iIdx++];
+ assert( iIdx==nVal );
+
+ /* In case the cursor has been used before, clear it now. */
+ sqlite3_finalize(pCsr->pStmt);
+ sqlite3_free(pCsr->aDoclist);
+ sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
+ sqlite3Fts3ExprFree(pCsr->pExpr);
+ memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
+
+ /* Set the lower and upper bounds on docids to return */
+ pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64);
+ pCsr->iMaxDocid = fts3DocidRange(pDocidLe, LARGEST_INT64);
+
+ if( idxStr ){
+ pCsr->bDesc = (idxStr[0]=='D');
+ }else{
+ pCsr->bDesc = p->bDescIdx;
+ }
+ pCsr->eSearch = (i16)eSearch;
+
+ if( eSearch!=FTS3_DOCID_SEARCH && eSearch!=FTS3_FULLSCAN_SEARCH ){
+ int iCol = eSearch-FTS3_FULLTEXT_SEARCH;
+ const char *zQuery = (const char *)sqlite3_value_text(pCons);
+
+ if( zQuery==0 && sqlite3_value_type(pCons)!=SQLITE_NULL ){
+ return SQLITE_NOMEM;
+ }
+
+ pCsr->iLangid = 0;
+ if( pLangid ) pCsr->iLangid = sqlite3_value_int(pLangid);
+
+ assert( p->base.zErrMsg==0 );
+ rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid,
+ p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr,
+ &p->base.zErrMsg
+ );
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+
+ rc = fts3EvalStart(pCsr);
+ sqlite3Fts3SegmentsClose(p);
+ if( rc!=SQLITE_OK ) return rc;
+ pCsr->pNextId = pCsr->aDoclist;
+ pCsr->iPrevId = 0;
+ }
+
+ /* Compile a SELECT statement for this cursor. For a full-table-scan, the
+ ** statement loops through all rows of the %_content table. For a
+ ** full-text query or docid lookup, the statement retrieves a single
+ ** row by docid.
+ */
+ if( eSearch==FTS3_FULLSCAN_SEARCH ){
+ if( pDocidGe || pDocidLe ){
+ zSql = sqlite3_mprintf(
+ "SELECT %s WHERE rowid BETWEEN %lld AND %lld ORDER BY rowid %s",
+ p->zReadExprlist, pCsr->iMinDocid, pCsr->iMaxDocid,
+ (pCsr->bDesc ? "DESC" : "ASC")
+ );
+ }else{
+ zSql = sqlite3_mprintf("SELECT %s ORDER BY rowid %s",
+ p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC")
+ );
+ }
+ if( zSql ){
+ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
+ sqlite3_free(zSql);
+ }else{
+ rc = SQLITE_NOMEM;
+ }
+ }else if( eSearch==FTS3_DOCID_SEARCH ){
+ rc = fts3CursorSeekStmt(pCsr, &pCsr->pStmt);
+ if( rc==SQLITE_OK ){
+ rc = sqlite3_bind_value(pCsr->pStmt, 1, pCons);
+ }
+ }
+ if( rc!=SQLITE_OK ) return rc;
+
+ return fts3NextMethod(pCursor);
+}
+
+/*
+** This is the xEof method of the virtual table. SQLite calls this
+** routine to find out if it has reached the end of a result set.
+*/
+static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){
+ return ((Fts3Cursor *)pCursor)->isEof;
+}
+
+/*
+** This is the xRowid method. The SQLite core calls this routine to
+** retrieve the rowid for the current row of the result set. fts3
+** exposes %_content.docid as the rowid for the virtual table. The
+** rowid should be written to *pRowid.
+*/
+static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
+ Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
+ *pRowid = pCsr->iPrevId;
+ return SQLITE_OK;
+}
+
+/*
+** This is the xColumn method, called by SQLite to request a value from
+** the row that the supplied cursor currently points to.
+**
+** If:
+**
+** (iCol < p->nColumn) -> The value of the iCol'th user column.
+** (iCol == p->nColumn) -> Magic column with the same name as the table.
+** (iCol == p->nColumn+1) -> Docid column
+** (iCol == p->nColumn+2) -> Langid column
+*/
+static int fts3ColumnMethod(
+ sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
+ sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */
+ int iCol /* Index of column to read value from */
+){
+ int rc = SQLITE_OK; /* Return Code */
+ Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
+ Fts3Table *p = (Fts3Table *)pCursor->pVtab;
+
+ /* The column value supplied by SQLite must be in range. */
+ assert( iCol>=0 && iCol<=p->nColumn+2 );
+
+ if( iCol==p->nColumn+1 ){
+ /* This call is a request for the "docid" column. Since "docid" is an
+ ** alias for "rowid", use the xRowid() method to obtain the value.
+ */
+ sqlite3_result_int64(pCtx, pCsr->iPrevId);
+ }else if( iCol==p->nColumn ){
+ /* The extra column whose name is the same as the table.
+ ** Return a blob which is a pointer to the cursor. */
+ sqlite3_result_blob(pCtx, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
+ }else if( iCol==p->nColumn+2 && pCsr->pExpr ){
+ sqlite3_result_int64(pCtx, pCsr->iLangid);
+ }else{
+ /* The requested column is either a user column (one that contains
+ ** indexed data), or the language-id column. */
+ rc = fts3CursorSeek(0, pCsr);
+
+ if( rc==SQLITE_OK ){
+ if( iCol==p->nColumn+2 ){
+ int iLangid = 0;
+ if( p->zLanguageid ){
+ iLangid = sqlite3_column_int(pCsr->pStmt, p->nColumn+1);
+ }
+ sqlite3_result_int(pCtx, iLangid);
+ }else if( sqlite3_data_count(pCsr->pStmt)>(iCol+1) ){
+ sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1));
+ }
+ }
+ }
+
+ assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
+ return rc;
+}
+
+/*
+** This function is the implementation of the xUpdate callback used by
+** FTS3 virtual tables. It is invoked by SQLite each time a row is to be
+** inserted, updated or deleted.
+*/
+static int fts3UpdateMethod(
+ sqlite3_vtab *pVtab, /* Virtual table handle */
+ int nArg, /* Size of argument array */
+ sqlite3_value **apVal, /* Array of arguments */
+ sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */
+){
+ return sqlite3Fts3UpdateMethod(pVtab, nArg, apVal, pRowid);
+}
+
+/*
+** Implementation of xSync() method. Flush the contents of the pending-terms
+** hash-table to the database.
+*/
+static int fts3SyncMethod(sqlite3_vtab *pVtab){
+
+ /* Following an incremental-merge operation, assuming that the input
+ ** segments are not completely consumed (the usual case), they are updated
+ ** in place to remove the entries that have already been merged. This
+ ** involves updating the leaf block that contains the smallest unmerged
+ ** entry and each block (if any) between the leaf and the root node. So
+ ** if the height of the input segment b-trees is N, and input segments
+ ** are merged eight at a time, updating the input segments at the end
+ ** of an incremental-merge requires writing (8*(1+N)) blocks. N is usually
+ ** small - often between 0 and 2. So the overhead of the incremental
+ ** merge is somewhere between 8 and 24 blocks. To avoid this overhead
+ ** dwarfing the actual productive work accomplished, the incremental merge
+ ** is only attempted if it will write at least 64 leaf blocks. Hence
+ ** nMinMerge.
+ **
+ ** Of course, updating the input segments also involves deleting a bunch
+ ** of blocks from the segments table. But this is not considered overhead
+ ** as it would also be required by a crisis-merge that used the same input
+ ** segments.
+ */
+ const u32 nMinMerge = 64; /* Minimum amount of incr-merge work to do */
+
+ Fts3Table *p = (Fts3Table*)pVtab;
+ int rc = sqlite3Fts3PendingTermsFlush(p);
+
+ if( rc==SQLITE_OK
+ && p->nLeafAdd>(nMinMerge/16)
+ && p->nAutoincrmerge && p->nAutoincrmerge!=0xff
+ ){
+ int mxLevel = 0; /* Maximum relative level value in db */
+ int A; /* Incr-merge parameter A */
+
+ rc = sqlite3Fts3MaxLevel(p, &mxLevel);
+ assert( rc==SQLITE_OK || mxLevel==0 );
+ A = p->nLeafAdd * mxLevel;
+ A += (A/2);
+ if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge);
+ }
+ sqlite3Fts3SegmentsClose(p);
+ return rc;
+}
+
+/*
+** If it is currently unknown whether or not the FTS table has an %_stat
+** table (if p->bHasStat==2), attempt to determine this (set p->bHasStat
+** to 0 or 1). Return SQLITE_OK if successful, or an SQLite error code
+** if an error occurs.
+*/
+static int fts3SetHasStat(Fts3Table *p){
+ int rc = SQLITE_OK;
+ if( p->bHasStat==2 ){
+ const char *zFmt ="SELECT 1 FROM %Q.sqlite_master WHERE tbl_name='%q_stat'";
+ char *zSql = sqlite3_mprintf(zFmt, p->zDb, p->zName);
+ if( zSql ){
+ sqlite3_stmt *pStmt = 0;
+ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
+ if( rc==SQLITE_OK ){
+ int bHasStat = (sqlite3_step(pStmt)==SQLITE_ROW);
+ rc = sqlite3_finalize(pStmt);
+ if( rc==SQLITE_OK ) p->bHasStat = bHasStat;
+ }
+ sqlite3_free(zSql);
+ }else{
+ rc = SQLITE_NOMEM;
+ }
+ }
+ return rc;
+}
+
+/*
+** Implementation of xBegin() method.
+*/
+static int fts3BeginMethod(sqlite3_vtab *pVtab){
+ Fts3Table *p = (Fts3Table*)pVtab;
+ UNUSED_PARAMETER(pVtab);
+ assert( p->pSegments==0 );
+ assert( p->nPendingData==0 );
+ assert( p->inTransaction!=1 );
+ TESTONLY( p->inTransaction = 1 );
+ TESTONLY( p->mxSavepoint = -1; );
+ p->nLeafAdd = 0;
+ return fts3SetHasStat(p);
+}
+
+/*
+** Implementation of xCommit() method. This is a no-op. The contents of
+** the pending-terms hash-table have already been flushed into the database
+** by fts3SyncMethod().
+*/
+static int fts3CommitMethod(sqlite3_vtab *pVtab){
+ TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
+ UNUSED_PARAMETER(pVtab);
+ assert( p->nPendingData==0 );
+ assert( p->inTransaction!=0 );
+ assert( p->pSegments==0 );
+ TESTONLY( p->inTransaction = 0 );
+ TESTONLY( p->mxSavepoint = -1; );
+ return SQLITE_OK;
+}
+
+/*
+** Implementation of xRollback(). Discard the contents of the pending-terms
+** hash-table. Any changes made to the database are reverted by SQLite.
+*/
+static int fts3RollbackMethod(sqlite3_vtab *pVtab){
+ Fts3Table *p = (Fts3Table*)pVtab;
+ sqlite3Fts3PendingTermsClear(p);
+ assert( p->inTransaction!=0 );
+ TESTONLY( p->inTransaction = 0 );
+ TESTONLY( p->mxSavepoint = -1; );
+ return SQLITE_OK;
+}
+
+/*
+** When called, *ppPoslist must point to the byte immediately following the
+** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function
+** moves *ppPoslist so that it instead points to the first byte of the
+** same position list.
+*/
+static void fts3ReversePoslist(char *pStart, char **ppPoslist){
+ char *p = &(*ppPoslist)[-2];
+ char c = 0;
+
+ /* Skip backwards passed any trailing 0x00 bytes added by NearTrim() */
+ while( p>pStart && (c=*p--)==0 );
+
+ /* Search backwards for a varint with value zero (the end of the previous
+ ** poslist). This is an 0x00 byte preceded by some byte that does not
+ ** have the 0x80 bit set. */
+ while( p>pStart && (*p & 0x80) | c ){
+ c = *p--;
+ }
+ assert( p==pStart || c==0 );
+
+ /* At this point p points to that preceding byte without the 0x80 bit
+ ** set. So to find the start of the poslist, skip forward 2 bytes then
+ ** over a varint.
+ **
+ ** Normally. The other case is that p==pStart and the poslist to return
+ ** is the first in the doclist. In this case do not skip forward 2 bytes.
+ ** The second part of the if condition (c==0 && *ppPoslist>&p[2])
+ ** is required for cases where the first byte of a doclist and the
+ ** doclist is empty. For example, if the first docid is 10, a doclist
+ ** that begins with:
+ **
+ ** 0x0A 0x00 <next docid delta varint>
+ */
+ if( p>pStart || (c==0 && *ppPoslist>&p[2]) ){ p = &p[2]; }
+ while( *p++&0x80 );
+ *ppPoslist = p;
+}
+
+/*
+** Helper function used by the implementation of the overloaded snippet(),
+** offsets() and optimize() SQL functions.
+**
+** If the value passed as the third argument is a blob of size
+** sizeof(Fts3Cursor*), then the blob contents are copied to the
+** output variable *ppCsr and SQLITE_OK is returned. Otherwise, an error
+** message is written to context pContext and SQLITE_ERROR returned. The
+** string passed via zFunc is used as part of the error message.
+*/
+static int fts3FunctionArg(
+ sqlite3_context *pContext, /* SQL function call context */
+ const char *zFunc, /* Function name */
+ sqlite3_value *pVal, /* argv[0] passed to function */
+ Fts3Cursor **ppCsr /* OUT: Store cursor handle here */
+){
+ Fts3Cursor *pRet;
+ if( sqlite3_value_type(pVal)!=SQLITE_BLOB
+ || sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *)
+ ){
+ char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc);
+ sqlite3_result_error(pContext, zErr, -1);
+ sqlite3_free(zErr);
+ return SQLITE_ERROR;
+ }
+ memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *));
+ *ppCsr = pRet;
+ return SQLITE_OK;
+}
+
+/*
+** Implementation of the snippet() function for FTS3
+*/
+static void fts3SnippetFunc(
+ sqlite3_context *pContext, /* SQLite function call context */
+ int nVal, /* Size of apVal[] array */
+ sqlite3_value **apVal /* Array of arguments */
+){
+ Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */
+ const char *zStart = "<b>";
+ const char *zEnd = "</b>";
+ const char *zEllipsis = "<b>...</b>";
+ int iCol = -1;
+ int nToken = 15; /* Default number of tokens in snippet */
+
+ /* There must be at least one argument passed to this function (otherwise
+ ** the non-overloaded version would have been called instead of this one).
+ */
+ assert( nVal>=1 );
+
+ if( nVal>6 ){
+ sqlite3_result_error(pContext,
+ "wrong number of arguments to function snippet()", -1);
+ return;
+ }
+ if( fts3FunctionArg(pContext, "snippet", apVal[0], &pCsr) ) return;
+
+ switch( nVal ){
+ case 6: nToken = sqlite3_value_int(apVal[5]);
+ case 5: iCol = sqlite3_value_int(apVal[4]);
+ case 4: zEllipsis = (const char*)sqlite3_value_text(apVal[3]);
+ case 3: zEnd = (const char*)sqlite3_value_text(apVal[2]);
+ case 2: zStart = (const char*)sqlite3_value_text(apVal[1]);
+ }
+ if( !zEllipsis || !zEnd || !zStart ){
+ sqlite3_result_error_nomem(pContext);
+ }else if( nToken==0 ){
+ sqlite3_result_text(pContext, "", -1, SQLITE_STATIC);
+ }else if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){
+ sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis, iCol, nToken);
+ }
+}
+
+/*
+** Implementation of the offsets() function for FTS3
+*/
+static void fts3OffsetsFunc(
+ sqlite3_context *pContext, /* SQLite function call context */
+ int nVal, /* Size of argument array */
+ sqlite3_value **apVal /* Array of arguments */
+){
+ Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */
+
+ UNUSED_PARAMETER(nVal);
+
+ assert( nVal==1 );
+ if( fts3FunctionArg(pContext, "offsets", apVal[0], &pCsr) ) return;
+ assert( pCsr );
+ if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){
+ sqlite3Fts3Offsets(pContext, pCsr);
+ }
+}
+
+/*
+** Implementation of the special optimize() function for FTS3. This
+** function merges all segments in the database to a single segment.
+** Example usage is:
+**
+** SELECT optimize(t) FROM t LIMIT 1;
+**
+** where 't' is the name of an FTS3 table.
+*/
+static void fts3OptimizeFunc(
+ sqlite3_context *pContext, /* SQLite function call context */
+ int nVal, /* Size of argument array */
+ sqlite3_value **apVal /* Array of arguments */
+){
+ int rc; /* Return code */
+ Fts3Table *p; /* Virtual table handle */
+ Fts3Cursor *pCursor; /* Cursor handle passed through apVal[0] */
+
+ UNUSED_PARAMETER(nVal);
+
+ assert( nVal==1 );
+ if( fts3FunctionArg(pContext, "optimize", apVal[0], &pCursor) ) return;
+ p = (Fts3Table *)pCursor->base.pVtab;
+ assert( p );
+
+ rc = sqlite3Fts3Optimize(p);
+
+ switch( rc ){
+ case SQLITE_OK:
+ sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC);
+ break;
+ case SQLITE_DONE:
+ sqlite3_result_text(pContext, "Index already optimal", -1, SQLITE_STATIC);
+ break;
+ default:
+ sqlite3_result_error_code(pContext, rc);
+ break;
+ }
+}
+
+/*
+** Implementation of the matchinfo() function for FTS3
+*/
+static void fts3MatchinfoFunc(
+ sqlite3_context *pContext, /* SQLite function call context */
+ int nVal, /* Size of argument array */
+ sqlite3_value **apVal /* Array of arguments */
+){
+ Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */
+ assert( nVal==1 || nVal==2 );
+ if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){
+ const char *zArg = 0;
+ if( nVal>1 ){
+ zArg = (const char *)sqlite3_value_text(apVal[1]);
+ }
+ sqlite3Fts3Matchinfo(pContext, pCsr, zArg);
+ }
+}
+
+/*
+** This routine implements the xFindFunction method for the FTS3
+** virtual table.
+*/
+static int fts3FindFunctionMethod(
+ sqlite3_vtab *pVtab, /* Virtual table handle */
+ int nArg, /* Number of SQL function arguments */
+ const char *zName, /* Name of SQL function */
+ void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */
+ void **ppArg /* Unused */
+){
+ struct Overloaded {
+ const char *zName;
+ void (*xFunc)(sqlite3_context*,int,sqlite3_value**);
+ } aOverload[] = {
+ { "snippet", fts3SnippetFunc },
+ { "offsets", fts3OffsetsFunc },
+ { "optimize", fts3OptimizeFunc },
+ { "matchinfo", fts3MatchinfoFunc },
+ };
+ int i; /* Iterator variable */
+
+ UNUSED_PARAMETER(pVtab);
+ UNUSED_PARAMETER(nArg);
+ UNUSED_PARAMETER(ppArg);
+
+ for(i=0; i<SizeofArray(aOverload); i++){
+ if( strcmp(zName, aOverload[i].zName)==0 ){
+ *pxFunc = aOverload[i].xFunc;
+ return 1;
+ }
+ }
+
+ /* No function of the specified name was found. Return 0. */
+ return 0;
+}
+
+/*
+** Implementation of FTS3 xRename method. Rename an fts3 table.
+*/
+static int fts3RenameMethod(
+ sqlite3_vtab *pVtab, /* Virtual table handle */
+ const char *zName /* New name of table */
+){
+ Fts3Table *p = (Fts3Table *)pVtab;
+ sqlite3 *db = p->db; /* Database connection */
+ int rc; /* Return Code */
+
+ /* At this point it must be known if the %_stat table exists or not.
+ ** So bHasStat may not be 2. */
+ rc = fts3SetHasStat(p);
+
+ /* As it happens, the pending terms table is always empty here. This is
+ ** because an "ALTER TABLE RENAME TABLE" statement inside a transaction
+ ** always opens a savepoint transaction. And the xSavepoint() method
+ ** flushes the pending terms table. But leave the (no-op) call to
+ ** PendingTermsFlush() in in case that changes.
+ */
+ assert( p->nPendingData==0 );
+ if( rc==SQLITE_OK ){
+ rc = sqlite3Fts3PendingTermsFlush(p);
+ }
+
+ if( p->zContentTbl==0 ){
+ fts3DbExec(&rc, db,
+ "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';",
+ p->zDb, p->zName, zName
+ );
+ }
+
+ if( p->bHasDocsize ){
+ fts3DbExec(&rc, db,
+ "ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';",
+ p->zDb, p->zName, zName
+ );
+ }
+ if( p->bHasStat ){
+ fts3DbExec(&rc, db,
+ "ALTER TABLE %Q.'%q_stat' RENAME TO '%q_stat';",
+ p->zDb, p->zName, zName
+ );
+ }
+ fts3DbExec(&rc, db,
+ "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';",
+ p->zDb, p->zName, zName
+ );
+ fts3DbExec(&rc, db,
+ "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';",
+ p->zDb, p->zName, zName
+ );
+ return rc;
+}
+
+/*
+** The xSavepoint() method.
+**
+** Flush the contents of the pending-terms table to disk.
+*/
+static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
+ int rc = SQLITE_OK;
+ UNUSED_PARAMETER(iSavepoint);
+ assert( ((Fts3Table *)pVtab)->inTransaction );
+ assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint );
+ TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint );
+ if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){
+ rc = fts3SyncMethod(pVtab);
+ }
+ return rc;
+}
+
+/*
+** The xRelease() method.
+**
+** This is a no-op.
+*/
+static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
+ TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
+ UNUSED_PARAMETER(iSavepoint);
+ UNUSED_PARAMETER(pVtab);
+ assert( p->inTransaction );
+ assert( p->mxSavepoint >= iSavepoint );
+ TESTONLY( p->mxSavepoint = iSavepoint-1 );
+ return SQLITE_OK;
+}
+
+/*
+** The xRollbackTo() method.
+**
+** Discard the contents of the pending terms table.
+*/
+static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
+ Fts3Table *p = (Fts3Table*)pVtab;
+ UNUSED_PARAMETER(iSavepoint);
+ assert( p->inTransaction );
+ assert( p->mxSavepoint >= iSavepoint );
+ TESTONLY( p->mxSavepoint = iSavepoint );
+ sqlite3Fts3PendingTermsClear(p);
+ return SQLITE_OK;
+}
+
+static const sqlite3_module fts3Module = {
+ /* iVersion */ 2,
+ /* xCreate */ fts3CreateMethod,
+ /* xConnect */ fts3ConnectMethod,
+ /* xBestIndex */ fts3BestIndexMethod,
+ /* xDisconnect */ fts3DisconnectMethod,
+ /* xDestroy */ fts3DestroyMethod,
+ /* xOpen */ fts3OpenMethod,
+ /* xClose */ fts3CloseMethod,
+ /* xFilter */ fts3FilterMethod,
+ /* xNext */ fts3NextMethod,
+ /* xEof */ fts3EofMethod,
+ /* xColumn */ fts3ColumnMethod,
+ /* xRowid */ fts3RowidMethod,
+ /* xUpdate */ fts3UpdateMethod,
+ /* xBegin */ fts3BeginMethod,
+ /* xSync */ fts3SyncMethod,
+ /* xCommit */ fts3CommitMethod,
+ /* xRollback */ fts3RollbackMethod,
+ /* xFindFunction */ fts3FindFunctionMethod,
+ /* xRename */ fts3RenameMethod,
+ /* xSavepoint */ fts3SavepointMethod,
+ /* xRelease */ fts3ReleaseMethod,
+ /* xRollbackTo */ fts3RollbackToMethod,
+};
+
+/*
+** This function is registered as the module destructor (called when an
+** FTS3 enabled database connection is closed). It frees the memory
+** allocated for the tokenizer hash table.
+*/
+static void hashDestroy(void *p){
+ Fts3Hash *pHash = (Fts3Hash *)p;
+ sqlite3Fts3HashClear(pHash);
+ sqlite3_free(pHash);
+}
+
+/*
+** The fts3 built-in tokenizers - "simple", "porter" and "icu"- are
+** implemented in files fts3_tokenizer1.c, fts3_porter.c and fts3_icu.c
+** respectively. The following three forward declarations are for functions
+** declared in these files used to retrieve the respective implementations.
+**
+** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed
+** to by the argument to point to the "simple" tokenizer implementation.
+** And so on.
+*/
+SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+#ifndef SQLITE_DISABLE_FTS3_UNICODE
+SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const**ppModule);
+#endif
+#ifdef SQLITE_ENABLE_ICU
+SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+#endif
+
+/*
+** Initialize the fts3 extension. If this extension is built as part
+** of the sqlite library, then this function is called directly by
+** SQLite. If fts3 is built as a dynamically loadable extension, this
+** function is called by the sqlite3_extension_init() entry point.
+*/
+SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){
+ int rc = SQLITE_OK;
+ Fts3Hash *pHash = 0;
+ const sqlite3_tokenizer_module *pSimple = 0;
+ const sqlite3_tokenizer_module *pPorter = 0;
+#ifndef SQLITE_DISABLE_FTS3_UNICODE
+ const sqlite3_tokenizer_module *pUnicode = 0;
+#endif
+
+#ifdef SQLITE_ENABLE_ICU
+ const sqlite3_tokenizer_module *pIcu = 0;
+ sqlite3Fts3IcuTokenizerModule(&pIcu);
+#endif
+
+#ifndef SQLITE_DISABLE_FTS3_UNICODE
+ sqlite3Fts3UnicodeTokenizer(&pUnicode);
+#endif
+
+#ifdef SQLITE_TEST
+ rc = sqlite3Fts3InitTerm(db);
+ if( rc!=SQLITE_OK ) return rc;
+#endif
+
+ rc = sqlite3Fts3InitAux(db);
+ if( rc!=SQLITE_OK ) return rc;
+
+ sqlite3Fts3SimpleTokenizerModule(&pSimple);
+ sqlite3Fts3PorterTokenizerModule(&pPorter);
+
+ /* Allocate and initialize the hash-table used to store tokenizers. */
+ pHash = sqlite3_malloc(sizeof(Fts3Hash));
+ if( !pHash ){
+ rc = SQLITE_NOMEM;
+ }else{
+ sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
+ }
+
+ /* Load the built-in tokenizers into the hash table */
+ if( rc==SQLITE_OK ){
+ if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple)
+ || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter)
+
+#ifndef SQLITE_DISABLE_FTS3_UNICODE
+ || sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode)
+#endif
+#ifdef SQLITE_ENABLE_ICU
+ || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu))
+#endif
+ ){
+ rc = SQLITE_NOMEM;
+ }
+ }
+
+#ifdef SQLITE_TEST
+ if( rc==SQLITE_OK ){
+ rc = sqlite3Fts3ExprInitTestInterface(db);
+ }
+#endif
+
+ /* Create the virtual table wrapper around the hash-table and overload
+ ** the two scalar functions. If this is successful, register the
+ ** module with sqlite.
+ */
+ if( SQLITE_OK==rc
+ && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer"))
+ && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
+ && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1))
+ && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1))
+ && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 2))
+ && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1))
+ ){
+ rc = sqlite3_create_module_v2(
+ db, "fts3", &fts3Module, (void *)pHash, hashDestroy
+ );
+ if( rc==SQLITE_OK ){
+ rc = sqlite3_create_module_v2(
+ db, "fts4", &fts3Module, (void *)pHash, 0
+ );
+ }
+ if( rc==SQLITE_OK ){
+ rc = sqlite3Fts3InitTok(db, (void *)pHash);
+ }
+ return rc;
+ }
+
+
+ /* An error has occurred. Delete the hash table and return the error code. */
+ assert( rc!=SQLITE_OK );
+ if( pHash ){
+ sqlite3Fts3HashClear(pHash);
+ sqlite3_free(pHash);
+ }
+ return rc;
+}
+
+/*
+** Allocate an Fts3MultiSegReader for each token in the expression headed
+** by pExpr.
+**
+** An Fts3SegReader object is a cursor that can seek or scan a range of
+** entries within a single segment b-tree. An Fts3MultiSegReader uses multiple
+** Fts3SegReader objects internally to provide an interface to seek or scan
+** within the union of all segments of a b-tree. Hence the name.
+**
+** If the allocated Fts3MultiSegReader just seeks to a single entry in a
+** segment b-tree (if the term is not a prefix or it is a prefix for which
+** there exists prefix b-tree of the right length) then it may be traversed
+** and merged incrementally. Otherwise, it has to be merged into an in-memory
+** doclist and then traversed.
+*/
+static void fts3EvalAllocateReaders(
+ Fts3Cursor *pCsr, /* FTS cursor handle */
+ Fts3Expr *pExpr, /* Allocate readers for this expression */
+ int *pnToken, /* OUT: Total number of tokens in phrase. */
+ int *pnOr, /* OUT: Total number of OR nodes in expr. */
+ int *pRc /* IN/OUT: Error code */
+){
+ if( pExpr && SQLITE_OK==*pRc ){
+ if( pExpr->eType==FTSQUERY_PHRASE ){
+ int i;
+ int nToken = pExpr->pPhrase->nToken;
+ *pnToken += nToken;
+ for(i=0; i<nToken; i++){
+ Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i];
+ int rc = fts3TermSegReaderCursor(pCsr,
+ pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr
+ );
+ if( rc!=SQLITE_OK ){
+ *pRc = rc;
+ return;
+ }
+ }
+ assert( pExpr->pPhrase->iDoclistToken==0 );
+ pExpr->pPhrase->iDoclistToken = -1;
+ }else{
+ *pnOr += (pExpr->eType==FTSQUERY_OR);
+ fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc);
+ fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc);
+ }
+ }
+}
+
+/*
+** Arguments pList/nList contain the doclist for token iToken of phrase p.
+** It is merged into the main doclist stored in p->doclist.aAll/nAll.
+**
+** This function assumes that pList points to a buffer allocated using
+** sqlite3_malloc(). This function takes responsibility for eventually
+** freeing the buffer.
+**
+** SQLITE_OK is returned if successful, or SQLITE_NOMEM if an error occurs.
+*/
+static int fts3EvalPhraseMergeToken(
+ Fts3Table *pTab, /* FTS Table pointer */
+ Fts3Phrase *p, /* Phrase to merge pList/nList into */
+ int iToken, /* Token pList/nList corresponds to */
+ char *pList, /* Pointer to doclist */
+ int nList /* Number of bytes in pList */
+){
+ int rc = SQLITE_OK;
+ assert( iToken!=p->iDoclistToken );
+
+ if( pList==0 ){
+ sqlite3_free(p->doclist.aAll);
+ p->doclist.aAll = 0;
+ p->doclist.nAll = 0;
+ }
+
+ else if( p->iDoclistToken<0 ){
+ p->doclist.aAll = pList;
+ p->doclist.nAll = nList;
+ }
+
+ else if( p->doclist.aAll==0 ){
+ sqlite3_free(pList);
+ }
+
+ else {
+ char *pLeft;
+ char *pRight;
+ int nLeft;
+ int nRight;
+ int nDiff;
+
+ if( p->iDoclistToken<iToken ){
+ pLeft = p->doclist.aAll;
+ nLeft = p->doclist.nAll;
+ pRight = pList;
+ nRight = nList;
+ nDiff = iToken - p->iDoclistToken;
+ }else{
+ pRight = p->doclist.aAll;
+ nRight = p->doclist.nAll;
+ pLeft = pList;
+ nLeft = nList;
+ nDiff = p->iDoclistToken - iToken;
+ }
+
+ rc = fts3DoclistPhraseMerge(
+ pTab->bDescIdx, nDiff, pLeft, nLeft, &pRight, &nRight
+ );
+ sqlite3_free(pLeft);
+ p->doclist.aAll = pRight;
+ p->doclist.nAll = nRight;
+ }
+
+ if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken;
+ return rc;
+}
+
+/*
+** Load the doclist for phrase p into p->doclist.aAll/nAll. The loaded doclist
+** does not take deferred tokens into account.
+**
+** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
+*/
+static int fts3EvalPhraseLoad(
+ Fts3Cursor *pCsr, /* FTS Cursor handle */
+ Fts3Phrase *p /* Phrase object */
+){
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+ int iToken;
+ int rc = SQLITE_OK;
+
+ for(iToken=0; rc==SQLITE_OK && iToken<p->nToken; iToken++){
+ Fts3PhraseToken *pToken = &p->aToken[iToken];
+ assert( pToken->pDeferred==0 || pToken->pSegcsr==0 );
+
+ if( pToken->pSegcsr ){
+ int nThis = 0;
+ char *pThis = 0;
+ rc = fts3TermSelect(pTab, pToken, p->iColumn, &nThis, &pThis);
+ if( rc==SQLITE_OK ){
+ rc = fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis);
+ }
+ }
+ assert( pToken->pSegcsr==0 );
+ }
+
+ return rc;
+}
+
+/*
+** This function is called on each phrase after the position lists for
+** any deferred tokens have been loaded into memory. It updates the phrases
+** current position list to include only those positions that are really
+** instances of the phrase (after considering deferred tokens). If this
+** means that the phrase does not appear in the current row, doclist.pList
+** and doclist.nList are both zeroed.
+**
+** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
+*/
+static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
+ int iToken; /* Used to iterate through phrase tokens */
+ char *aPoslist = 0; /* Position list for deferred tokens */
+ int nPoslist = 0; /* Number of bytes in aPoslist */
+ int iPrev = -1; /* Token number of previous deferred token */
+
+ assert( pPhrase->doclist.bFreeList==0 );
+
+ for(iToken=0; iToken<pPhrase->nToken; iToken++){
+ Fts3PhraseToken *pToken = &pPhrase->aToken[iToken];
+ Fts3DeferredToken *pDeferred = pToken->pDeferred;
+
+ if( pDeferred ){
+ char *pList;
+ int nList;
+ int rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList);
+ if( rc!=SQLITE_OK ) return rc;
+
+ if( pList==0 ){
+ sqlite3_free(aPoslist);
+ pPhrase->doclist.pList = 0;
+ pPhrase->doclist.nList = 0;
+ return SQLITE_OK;
+
+ }else if( aPoslist==0 ){
+ aPoslist = pList;
+ nPoslist = nList;
+
+ }else{
+ char *aOut = pList;
+ char *p1 = aPoslist;
+ char *p2 = aOut;
+
+ assert( iPrev>=0 );
+ fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2);
+ sqlite3_free(aPoslist);
+ aPoslist = pList;
+ nPoslist = (int)(aOut - aPoslist);
+ if( nPoslist==0 ){
+ sqlite3_free(aPoslist);
+ pPhrase->doclist.pList = 0;
+ pPhrase->doclist.nList = 0;
+ return SQLITE_OK;
+ }
+ }
+ iPrev = iToken;
+ }
+ }
+
+ if( iPrev>=0 ){
+ int nMaxUndeferred = pPhrase->iDoclistToken;
+ if( nMaxUndeferred<0 ){
+ pPhrase->doclist.pList = aPoslist;
+ pPhrase->doclist.nList = nPoslist;
+ pPhrase->doclist.iDocid = pCsr->iPrevId;
+ pPhrase->doclist.bFreeList = 1;
+ }else{
+ int nDistance;
+ char *p1;
+ char *p2;
+ char *aOut;
+
+ if( nMaxUndeferred>iPrev ){
+ p1 = aPoslist;
+ p2 = pPhrase->doclist.pList;
+ nDistance = nMaxUndeferred - iPrev;
+ }else{
+ p1 = pPhrase->doclist.pList;
+ p2 = aPoslist;
+ nDistance = iPrev - nMaxUndeferred;
+ }
+
+ aOut = (char *)sqlite3_malloc(nPoslist+8);
+ if( !aOut ){
+ sqlite3_free(aPoslist);
+ return SQLITE_NOMEM;
+ }
+
+ pPhrase->doclist.pList = aOut;
+ if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){
+ pPhrase->doclist.bFreeList = 1;
+ pPhrase->doclist.nList = (int)(aOut - pPhrase->doclist.pList);
+ }else{
+ sqlite3_free(aOut);
+ pPhrase->doclist.pList = 0;
+ pPhrase->doclist.nList = 0;
+ }
+ sqlite3_free(aPoslist);
+ }
+ }
+
+ return SQLITE_OK;
+}
+
+/*
+** Maximum number of tokens a phrase may have to be considered for the
+** incremental doclists strategy.
+*/
+#define MAX_INCR_PHRASE_TOKENS 4
+
+/*
+** This function is called for each Fts3Phrase in a full-text query
+** expression to initialize the mechanism for returning rows. Once this
+** function has been called successfully on an Fts3Phrase, it may be
+** used with fts3EvalPhraseNext() to iterate through the matching docids.
+**
+** If parameter bOptOk is true, then the phrase may (or may not) use the
+** incremental loading strategy. Otherwise, the entire doclist is loaded into
+** memory within this call.
+**
+** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
+*/
+static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+ int rc = SQLITE_OK; /* Error code */
+ int i;
+
+ /* Determine if doclists may be loaded from disk incrementally. This is
+ ** possible if the bOptOk argument is true, the FTS doclists will be
+ ** scanned in forward order, and the phrase consists of
+ ** MAX_INCR_PHRASE_TOKENS or fewer tokens, none of which are are "^first"
+ ** tokens or prefix tokens that cannot use a prefix-index. */
+ int bHaveIncr = 0;
+ int bIncrOk = (bOptOk
+ && pCsr->bDesc==pTab->bDescIdx
+ && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0
+#ifdef SQLITE_TEST
+ && pTab->bNoIncrDoclist==0
+#endif
+ );
+ for(i=0; bIncrOk==1 && i<p->nToken; i++){
+ Fts3PhraseToken *pToken = &p->aToken[i];
+ if( pToken->bFirst || (pToken->pSegcsr!=0 && !pToken->pSegcsr->bLookup) ){
+ bIncrOk = 0;
+ }
+ if( pToken->pSegcsr ) bHaveIncr = 1;
+ }
+
+ if( bIncrOk && bHaveIncr ){
+ /* Use the incremental approach. */
+ int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn);
+ for(i=0; rc==SQLITE_OK && i<p->nToken; i++){
+ Fts3PhraseToken *pToken = &p->aToken[i];
+ Fts3MultiSegReader *pSegcsr = pToken->pSegcsr;
+ if( pSegcsr ){
+ rc = sqlite3Fts3MsrIncrStart(pTab, pSegcsr, iCol, pToken->z, pToken->n);
+ }
+ }
+ p->bIncr = 1;
+ }else{
+ /* Load the full doclist for the phrase into memory. */
+ rc = fts3EvalPhraseLoad(pCsr, p);
+ p->bIncr = 0;
+ }
+
+ assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr );
+ return rc;
+}
+
+/*
+** This function is used to iterate backwards (from the end to start)
+** through doclists. It is used by this module to iterate through phrase
+** doclists in reverse and by the fts3_write.c module to iterate through
+** pending-terms lists when writing to databases with "order=desc".
+**
+** The doclist may be sorted in ascending (parameter bDescIdx==0) or
+** descending (parameter bDescIdx==1) order of docid. Regardless, this
+** function iterates from the end of the doclist to the beginning.
+*/
+SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(
+ int bDescIdx, /* True if the doclist is desc */
+ char *aDoclist, /* Pointer to entire doclist */
+ int nDoclist, /* Length of aDoclist in bytes */
+ char **ppIter, /* IN/OUT: Iterator pointer */
+ sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */
+ int *pnList, /* OUT: List length pointer */
+ u8 *pbEof /* OUT: End-of-file flag */
+){
+ char *p = *ppIter;
+
+ assert( nDoclist>0 );
+ assert( *pbEof==0 );
+ assert( p || *piDocid==0 );
+ assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) );
+
+ if( p==0 ){
+ sqlite3_int64 iDocid = 0;
+ char *pNext = 0;
+ char *pDocid = aDoclist;
+ char *pEnd = &aDoclist[nDoclist];
+ int iMul = 1;
+
+ while( pDocid<pEnd ){
+ sqlite3_int64 iDelta;
+ pDocid += sqlite3Fts3GetVarint(pDocid, &iDelta);
+ iDocid += (iMul * iDelta);
+ pNext = pDocid;
+ fts3PoslistCopy(0, &pDocid);
+ while( pDocid<pEnd && *pDocid==0 ) pDocid++;
+ iMul = (bDescIdx ? -1 : 1);
+ }
+
+ *pnList = (int)(pEnd - pNext);
+ *ppIter = pNext;
+ *piDocid = iDocid;
+ }else{
+ int iMul = (bDescIdx ? -1 : 1);
+ sqlite3_int64 iDelta;
+ fts3GetReverseVarint(&p, aDoclist, &iDelta);
+ *piDocid -= (iMul * iDelta);
+
+ if( p==aDoclist ){
+ *pbEof = 1;
+ }else{
+ char *pSave = p;
+ fts3ReversePoslist(aDoclist, &p);
+ *pnList = (int)(pSave - p);
+ }
+ *ppIter = p;
+ }
+}
+
+/*
+** Iterate forwards through a doclist.
+*/
+SQLITE_PRIVATE void sqlite3Fts3DoclistNext(
+ int bDescIdx, /* True if the doclist is desc */
+ char *aDoclist, /* Pointer to entire doclist */
+ int nDoclist, /* Length of aDoclist in bytes */
+ char **ppIter, /* IN/OUT: Iterator pointer */
+ sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */
+ u8 *pbEof /* OUT: End-of-file flag */
+){
+ char *p = *ppIter;
+
+ assert( nDoclist>0 );
+ assert( *pbEof==0 );
+ assert( p || *piDocid==0 );
+ assert( !p || (p>=aDoclist && p<=&aDoclist[nDoclist]) );
+
+ if( p==0 ){
+ p = aDoclist;
+ p += sqlite3Fts3GetVarint(p, piDocid);
+ }else{
+ fts3PoslistCopy(0, &p);
+ while( p<&aDoclist[nDoclist] && *p==0 ) p++;
+ if( p>=&aDoclist[nDoclist] ){
+ *pbEof = 1;
+ }else{
+ sqlite3_int64 iVar;
+ p += sqlite3Fts3GetVarint(p, &iVar);
+ *piDocid += ((bDescIdx ? -1 : 1) * iVar);
+ }
+ }
+
+ *ppIter = p;
+}
+
+/*
+** Advance the iterator pDL to the next entry in pDL->aAll/nAll. Set *pbEof
+** to true if EOF is reached.
+*/
+static void fts3EvalDlPhraseNext(
+ Fts3Table *pTab,
+ Fts3Doclist *pDL,
+ u8 *pbEof
+){
+ char *pIter; /* Used to iterate through aAll */
+ char *pEnd = &pDL->aAll[pDL->nAll]; /* 1 byte past end of aAll */
+
+ if( pDL->pNextDocid ){
+ pIter = pDL->pNextDocid;
+ }else{
+ pIter = pDL->aAll;
+ }
+
+ if( pIter>=pEnd ){
+ /* We have already reached the end of this doclist. EOF. */
+ *pbEof = 1;
+ }else{
+ sqlite3_int64 iDelta;
+ pIter += sqlite3Fts3GetVarint(pIter, &iDelta);
+ if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){
+ pDL->iDocid += iDelta;
+ }else{
+ pDL->iDocid -= iDelta;
+ }
+ pDL->pList = pIter;
+ fts3PoslistCopy(0, &pIter);
+ pDL->nList = (int)(pIter - pDL->pList);
+
+ /* pIter now points just past the 0x00 that terminates the position-
+ ** list for document pDL->iDocid. However, if this position-list was
+ ** edited in place by fts3EvalNearTrim(), then pIter may not actually
+ ** point to the start of the next docid value. The following line deals
+ ** with this case by advancing pIter past the zero-padding added by
+ ** fts3EvalNearTrim(). */
+ while( pIter<pEnd && *pIter==0 ) pIter++;
+
+ pDL->pNextDocid = pIter;
+ assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter );
+ *pbEof = 0;
+ }
+}
+
+/*
+** Helper type used by fts3EvalIncrPhraseNext() and incrPhraseTokenNext().
+*/
+typedef struct TokenDoclist TokenDoclist;
+struct TokenDoclist {
+ int bIgnore;
+ sqlite3_int64 iDocid;
+ char *pList;
+ int nList;
+};
+
+/*
+** Token pToken is an incrementally loaded token that is part of a
+** multi-token phrase. Advance it to the next matching document in the
+** database and populate output variable *p with the details of the new
+** entry. Or, if the iterator has reached EOF, set *pbEof to true.
+**
+** If an error occurs, return an SQLite error code. Otherwise, return
+** SQLITE_OK.
+*/
+static int incrPhraseTokenNext(
+ Fts3Table *pTab, /* Virtual table handle */
+ Fts3Phrase *pPhrase, /* Phrase to advance token of */
+ int iToken, /* Specific token to advance */
+ TokenDoclist *p, /* OUT: Docid and doclist for new entry */
+ u8 *pbEof /* OUT: True if iterator is at EOF */
+){
+ int rc = SQLITE_OK;
+
+ if( pPhrase->iDoclistToken==iToken ){
+ assert( p->bIgnore==0 );
+ assert( pPhrase->aToken[iToken].pSegcsr==0 );
+ fts3EvalDlPhraseNext(pTab, &pPhrase->doclist, pbEof);
+ p->pList = pPhrase->doclist.pList;
+ p->nList = pPhrase->doclist.nList;
+ p->iDocid = pPhrase->doclist.iDocid;
+ }else{
+ Fts3PhraseToken *pToken = &pPhrase->aToken[iToken];
+ assert( pToken->pDeferred==0 );
+ assert( pToken->pSegcsr || pPhrase->iDoclistToken>=0 );
+ if( pToken->pSegcsr ){
+ assert( p->bIgnore==0 );
+ rc = sqlite3Fts3MsrIncrNext(
+ pTab, pToken->pSegcsr, &p->iDocid, &p->pList, &p->nList
+ );
+ if( p->pList==0 ) *pbEof = 1;
+ }else{
+ p->bIgnore = 1;
+ }
+ }
+
+ return rc;
+}
+
+
+/*
+** The phrase iterator passed as the second argument:
+**
+** * features at least one token that uses an incremental doclist, and
+**
+** * does not contain any deferred tokens.
+**
+** Advance it to the next matching documnent in the database and populate
+** the Fts3Doclist.pList and nList fields.
+**
+** If there is no "next" entry and no error occurs, then *pbEof is set to
+** 1 before returning. Otherwise, if no error occurs and the iterator is
+** successfully advanced, *pbEof is set to 0.
+**
+** If an error occurs, return an SQLite error code. Otherwise, return
+** SQLITE_OK.
+*/
+static int fts3EvalIncrPhraseNext(
+ Fts3Cursor *pCsr, /* FTS Cursor handle */
+ Fts3Phrase *p, /* Phrase object to advance to next docid */
+ u8 *pbEof /* OUT: Set to 1 if EOF */
+){
+ int rc = SQLITE_OK;
+ Fts3Doclist *pDL = &p->doclist;
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+ u8 bEof = 0;
+
+ /* This is only called if it is guaranteed that the phrase has at least
+ ** one incremental token. In which case the bIncr flag is set. */
+ assert( p->bIncr==1 );
+
+ if( p->nToken==1 && p->bIncr ){
+ rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr,
+ &pDL->iDocid, &pDL->pList, &pDL->nList
+ );
+ if( pDL->pList==0 ) bEof = 1;
+ }else{
+ int bDescDoclist = pCsr->bDesc;
+ struct TokenDoclist a[MAX_INCR_PHRASE_TOKENS];
+
+ memset(a, 0, sizeof(a));
+ assert( p->nToken<=MAX_INCR_PHRASE_TOKENS );
+ assert( p->iDoclistToken<MAX_INCR_PHRASE_TOKENS );
+
+ while( bEof==0 ){
+ int bMaxSet = 0;
+ sqlite3_int64 iMax = 0; /* Largest docid for all iterators */
+ int i; /* Used to iterate through tokens */
+
+ /* Advance the iterator for each token in the phrase once. */
+ for(i=0; rc==SQLITE_OK && i<p->nToken && bEof==0; i++){
+ rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof);
+ if( a[i].bIgnore==0 && (bMaxSet==0 || DOCID_CMP(iMax, a[i].iDocid)<0) ){
+ iMax = a[i].iDocid;
+ bMaxSet = 1;
+ }
+ }
+ assert( rc!=SQLITE_OK || (p->nToken>=1 && a[p->nToken-1].bIgnore==0) );
+ assert( rc!=SQLITE_OK || bMaxSet );
+
+ /* Keep advancing iterators until they all point to the same document */
+ for(i=0; i<p->nToken; i++){
+ while( rc==SQLITE_OK && bEof==0
+ && a[i].bIgnore==0 && DOCID_CMP(a[i].iDocid, iMax)<0
+ ){
+ rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof);
+ if( DOCID_CMP(a[i].iDocid, iMax)>0 ){
+ iMax = a[i].iDocid;
+ i = 0;
+ }
+ }
+ }
+
+ /* Check if the current entries really are a phrase match */
+ if( bEof==0 ){
+ int nList = 0;
+ int nByte = a[p->nToken-1].nList;
+ char *aDoclist = sqlite3_malloc(nByte+1);
+ if( !aDoclist ) return SQLITE_NOMEM;
+ memcpy(aDoclist, a[p->nToken-1].pList, nByte+1);
+
+ for(i=0; i<(p->nToken-1); i++){
+ if( a[i].bIgnore==0 ){
+ char *pL = a[i].pList;
+ char *pR = aDoclist;
+ char *pOut = aDoclist;
+ int nDist = p->nToken-1-i;
+ int res = fts3PoslistPhraseMerge(&pOut, nDist, 0, 1, &pL, &pR);
+ if( res==0 ) break;
+ nList = (int)(pOut - aDoclist);
+ }
+ }
+ if( i==(p->nToken-1) ){
+ pDL->iDocid = iMax;
+ pDL->pList = aDoclist;
+ pDL->nList = nList;
+ pDL->bFreeList = 1;
+ break;
+ }
+ sqlite3_free(aDoclist);
+ }
+ }
+ }
+
+ *pbEof = bEof;
+ return rc;
+}
+
+/*
+** Attempt to move the phrase iterator to point to the next matching docid.
+** If an error occurs, return an SQLite error code. Otherwise, return
+** SQLITE_OK.
+**
+** If there is no "next" entry and no error occurs, then *pbEof is set to
+** 1 before returning. Otherwise, if no error occurs and the iterator is
+** successfully advanced, *pbEof is set to 0.
+*/
+static int fts3EvalPhraseNext(
+ Fts3Cursor *pCsr, /* FTS Cursor handle */
+ Fts3Phrase *p, /* Phrase object to advance to next docid */
+ u8 *pbEof /* OUT: Set to 1 if EOF */
+){
+ int rc = SQLITE_OK;
+ Fts3Doclist *pDL = &p->doclist;
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+
+ if( p->bIncr ){
+ rc = fts3EvalIncrPhraseNext(pCsr, p, pbEof);
+ }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){
+ sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll,
+ &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof
+ );
+ pDL->pList = pDL->pNextDocid;
+ }else{
+ fts3EvalDlPhraseNext(pTab, pDL, pbEof);
+ }
+
+ return rc;
+}
+
+/*
+**
+** If *pRc is not SQLITE_OK when this function is called, it is a no-op.
+** Otherwise, fts3EvalPhraseStart() is called on all phrases within the
+** expression. Also the Fts3Expr.bDeferred variable is set to true for any
+** expressions for which all descendent tokens are deferred.
+**
+** If parameter bOptOk is zero, then it is guaranteed that the
+** Fts3Phrase.doclist.aAll/nAll variables contain the entire doclist for
+** each phrase in the expression (subject to deferred token processing).
+** Or, if bOptOk is non-zero, then one or more tokens within the expression
+** may be loaded incrementally, meaning doclist.aAll/nAll is not available.
+**
+** If an error occurs within this function, *pRc is set to an SQLite error
+** code before returning.
+*/
+static void fts3EvalStartReaders(
+ Fts3Cursor *pCsr, /* FTS Cursor handle */
+ Fts3Expr *pExpr, /* Expression to initialize phrases in */
+ int *pRc /* IN/OUT: Error code */
+){
+ if( pExpr && SQLITE_OK==*pRc ){
+ if( pExpr->eType==FTSQUERY_PHRASE ){
+ int nToken = pExpr->pPhrase->nToken;
+ if( nToken ){
+ int i;
+ for(i=0; i<nToken; i++){
+ if( pExpr->pPhrase->aToken[i].pDeferred==0 ) break;
+ }
+ pExpr->bDeferred = (i==nToken);
+ }
+ *pRc = fts3EvalPhraseStart(pCsr, 1, pExpr->pPhrase);
+ }else{
+ fts3EvalStartReaders(pCsr, pExpr->pLeft, pRc);
+ fts3EvalStartReaders(pCsr, pExpr->pRight, pRc);
+ pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred);
+ }
+ }
+}
+
+/*
+** An array of the following structures is assembled as part of the process
+** of selecting tokens to defer before the query starts executing (as part
+** of the xFilter() method). There is one element in the array for each
+** token in the FTS expression.
+**
+** Tokens are divided into AND/NEAR clusters. All tokens in a cluster belong
+** to phrases that are connected only by AND and NEAR operators (not OR or
+** NOT). When determining tokens to defer, each AND/NEAR cluster is considered
+** separately. The root of a tokens AND/NEAR cluster is stored in
+** Fts3TokenAndCost.pRoot.
+*/
+typedef struct Fts3TokenAndCost Fts3TokenAndCost;
+struct Fts3TokenAndCost {
+ Fts3Phrase *pPhrase; /* The phrase the token belongs to */
+ int iToken; /* Position of token in phrase */
+ Fts3PhraseToken *pToken; /* The token itself */
+ Fts3Expr *pRoot; /* Root of NEAR/AND cluster */
+ int nOvfl; /* Number of overflow pages to load doclist */
+ int iCol; /* The column the token must match */
+};
+
+/*
+** This function is used to populate an allocated Fts3TokenAndCost array.
+**
+** If *pRc is not SQLITE_OK when this function is called, it is a no-op.
+** Otherwise, if an error occurs during execution, *pRc is set to an
+** SQLite error code.
+*/
+static void fts3EvalTokenCosts(
+ Fts3Cursor *pCsr, /* FTS Cursor handle */
+ Fts3Expr *pRoot, /* Root of current AND/NEAR cluster */
+ Fts3Expr *pExpr, /* Expression to consider */
+ Fts3TokenAndCost **ppTC, /* Write new entries to *(*ppTC)++ */
+ Fts3Expr ***ppOr, /* Write new OR root to *(*ppOr)++ */
+ int *pRc /* IN/OUT: Error code */
+){
+ if( *pRc==SQLITE_OK ){
+ if( pExpr->eType==FTSQUERY_PHRASE ){
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
+ int i;
+ for(i=0; *pRc==SQLITE_OK && i<pPhrase->nToken; i++){
+ Fts3TokenAndCost *pTC = (*ppTC)++;
+ pTC->pPhrase = pPhrase;
+ pTC->iToken = i;
+ pTC->pRoot = pRoot;
+ pTC->pToken = &pPhrase->aToken[i];
+ pTC->iCol = pPhrase->iColumn;
+ *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl);
+ }
+ }else if( pExpr->eType!=FTSQUERY_NOT ){
+ assert( pExpr->eType==FTSQUERY_OR
+ || pExpr->eType==FTSQUERY_AND
+ || pExpr->eType==FTSQUERY_NEAR
+ );
+ assert( pExpr->pLeft && pExpr->pRight );
+ if( pExpr->eType==FTSQUERY_OR ){
+ pRoot = pExpr->pLeft;
+ **ppOr = pRoot;
+ (*ppOr)++;
+ }
+ fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc);
+ if( pExpr->eType==FTSQUERY_OR ){
+ pRoot = pExpr->pRight;
+ **ppOr = pRoot;
+ (*ppOr)++;
+ }
+ fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc);
+ }
+ }
+}
+
+/*
+** Determine the average document (row) size in pages. If successful,
+** write this value to *pnPage and return SQLITE_OK. Otherwise, return
+** an SQLite error code.
+**
+** The average document size in pages is calculated by first calculating
+** determining the average size in bytes, B. If B is less than the amount
+** of data that will fit on a single leaf page of an intkey table in
+** this database, then the average docsize is 1. Otherwise, it is 1 plus
+** the number of overflow pages consumed by a record B bytes in size.
+*/
+static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){
+ if( pCsr->nRowAvg==0 ){
+ /* The average document size, which is required to calculate the cost
+ ** of each doclist, has not yet been determined. Read the required
+ ** data from the %_stat table to calculate it.
+ **
+ ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3
+ ** varints, where nCol is the number of columns in the FTS3 table.
+ ** The first varint is the number of documents currently stored in
+ ** the table. The following nCol varints contain the total amount of
+ ** data stored in all rows of each column of the table, from left
+ ** to right.
+ */
+ int rc;
+ Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
+ sqlite3_stmt *pStmt;
+ sqlite3_int64 nDoc = 0;
+ sqlite3_int64 nByte = 0;
+ const char *pEnd;
+ const char *a;
+
+ rc = sqlite3Fts3SelectDoctotal(p, &pStmt);
+ if( rc!=SQLITE_OK ) return rc;
+ a = sqlite3_column_blob(pStmt, 0);
+ assert( a );
+
+ pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
+ a += sqlite3Fts3GetVarint(a, &nDoc);
+ while( a<pEnd ){
+ a += sqlite3Fts3GetVarint(a, &nByte);
+ }
+ if( nDoc==0 || nByte==0 ){
+ sqlite3_reset(pStmt);
+ return FTS_CORRUPT_VTAB;
+ }
+
+ pCsr->nDoc = nDoc;
+ pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz);
+ assert( pCsr->nRowAvg>0 );
+ rc = sqlite3_reset(pStmt);
+ if( rc!=SQLITE_OK ) return rc;
+ }
+
+ *pnPage = pCsr->nRowAvg;
+ return SQLITE_OK;
+}
+
+/*
+** This function is called to select the tokens (if any) that will be
+** deferred. The array aTC[] has already been populated when this is
+** called.
+**
+** This function is called once for each AND/NEAR cluster in the
+** expression. Each invocation determines which tokens to defer within
+** the cluster with root node pRoot. See comments above the definition
+** of struct Fts3TokenAndCost for more details.
+**
+** If no error occurs, SQLITE_OK is returned and sqlite3Fts3DeferToken()
+** called on each token to defer. Otherwise, an SQLite error code is
+** returned.
+*/
+static int fts3EvalSelectDeferred(
+ Fts3Cursor *pCsr, /* FTS Cursor handle */
+ Fts3Expr *pRoot, /* Consider tokens with this root node */
+ Fts3TokenAndCost *aTC, /* Array of expression tokens and costs */
+ int nTC /* Number of entries in aTC[] */
+){
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+ int nDocSize = 0; /* Number of pages per doc loaded */
+ int rc = SQLITE_OK; /* Return code */
+ int ii; /* Iterator variable for various purposes */
+ int nOvfl = 0; /* Total overflow pages used by doclists */
+ int nToken = 0; /* Total number of tokens in cluster */
+
+ int nMinEst = 0; /* The minimum count for any phrase so far. */
+ int nLoad4 = 1; /* (Phrases that will be loaded)^4. */
+
+ /* Tokens are never deferred for FTS tables created using the content=xxx
+ ** option. The reason being that it is not guaranteed that the content
+ ** table actually contains the same data as the index. To prevent this from
+ ** causing any problems, the deferred token optimization is completely
+ ** disabled for content=xxx tables. */
+ if( pTab->zContentTbl ){
+ return SQLITE_OK;
+ }
+
+ /* Count the tokens in this AND/NEAR cluster. If none of the doclists
+ ** associated with the tokens spill onto overflow pages, or if there is
+ ** only 1 token, exit early. No tokens to defer in this case. */
+ for(ii=0; ii<nTC; ii++){
+ if( aTC[ii].pRoot==pRoot ){
+ nOvfl += aTC[ii].nOvfl;
+ nToken++;
+ }
+ }
+ if( nOvfl==0 || nToken<2 ) return SQLITE_OK;
+
+ /* Obtain the average docsize (in pages). */
+ rc = fts3EvalAverageDocsize(pCsr, &nDocSize);
+ assert( rc!=SQLITE_OK || nDocSize>0 );
+
+
+ /* Iterate through all tokens in this AND/NEAR cluster, in ascending order
+ ** of the number of overflow pages that will be loaded by the pager layer
+ ** to retrieve the entire doclist for the token from the full-text index.
+ ** Load the doclists for tokens that are either:
+ **
+ ** a. The cheapest token in the entire query (i.e. the one visited by the
+ ** first iteration of this loop), or
+ **
+ ** b. Part of a multi-token phrase.
+ **
+ ** After each token doclist is loaded, merge it with the others from the
+ ** same phrase and count the number of documents that the merged doclist
+ ** contains. Set variable "nMinEst" to the smallest number of documents in
+ ** any phrase doclist for which 1 or more token doclists have been loaded.
+ ** Let nOther be the number of other phrases for which it is certain that
+ ** one or more tokens will not be deferred.
+ **
+ ** Then, for each token, defer it if loading the doclist would result in
+ ** loading N or more overflow pages into memory, where N is computed as:
+ **
+ ** (nMinEst + 4^nOther - 1) / (4^nOther)
+ */
+ for(ii=0; ii<nToken && rc==SQLITE_OK; ii++){
+ int iTC; /* Used to iterate through aTC[] array. */
+ Fts3TokenAndCost *pTC = 0; /* Set to cheapest remaining token. */
+
+ /* Set pTC to point to the cheapest remaining token. */
+ for(iTC=0; iTC<nTC; iTC++){
+ if( aTC[iTC].pToken && aTC[iTC].pRoot==pRoot
+ && (!pTC || aTC[iTC].nOvfl<pTC->nOvfl)
+ ){
+ pTC = &aTC[iTC];
+ }
+ }
+ assert( pTC );
+
+ if( ii && pTC->nOvfl>=((nMinEst+(nLoad4/4)-1)/(nLoad4/4))*nDocSize ){
+ /* The number of overflow pages to load for this (and therefore all
+ ** subsequent) tokens is greater than the estimated number of pages
+ ** that will be loaded if all subsequent tokens are deferred.
+ */
+ Fts3PhraseToken *pToken = pTC->pToken;
+ rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol);
+ fts3SegReaderCursorFree(pToken->pSegcsr);
+ pToken->pSegcsr = 0;
+ }else{
+ /* Set nLoad4 to the value of (4^nOther) for the next iteration of the
+ ** for-loop. Except, limit the value to 2^24 to prevent it from
+ ** overflowing the 32-bit integer it is stored in. */
+ if( ii<12 ) nLoad4 = nLoad4*4;
+
+ if( ii==0 || (pTC->pPhrase->nToken>1 && ii!=nToken-1) ){
+ /* Either this is the cheapest token in the entire query, or it is
+ ** part of a multi-token phrase. Either way, the entire doclist will
+ ** (eventually) be loaded into memory. It may as well be now. */
+ Fts3PhraseToken *pToken = pTC->pToken;
+ int nList = 0;
+ char *pList = 0;
+ rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList);
+ assert( rc==SQLITE_OK || pList==0 );
+ if( rc==SQLITE_OK ){
+ rc = fts3EvalPhraseMergeToken(
+ pTab, pTC->pPhrase, pTC->iToken,pList,nList
+ );
+ }
+ if( rc==SQLITE_OK ){
+ int nCount;
+ nCount = fts3DoclistCountDocids(
+ pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll
+ );
+ if( ii==0 || nCount<nMinEst ) nMinEst = nCount;
+ }
+ }
+ }
+ pTC->pToken = 0;
+ }
+
+ return rc;
+}
+
+/*
+** This function is called from within the xFilter method. It initializes
+** the full-text query currently stored in pCsr->pExpr. To iterate through
+** the results of a query, the caller does:
+**
+** fts3EvalStart(pCsr);
+** while( 1 ){
+** fts3EvalNext(pCsr);
+** if( pCsr->bEof ) break;
+** ... return row pCsr->iPrevId to the caller ...
+** }
+*/
+static int fts3EvalStart(Fts3Cursor *pCsr){
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+ int rc = SQLITE_OK;
+ int nToken = 0;
+ int nOr = 0;
+
+ /* Allocate a MultiSegReader for each token in the expression. */
+ fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc);
+
+ /* Determine which, if any, tokens in the expression should be deferred. */
+#ifndef SQLITE_DISABLE_FTS4_DEFERRED
+ if( rc==SQLITE_OK && nToken>1 && pTab->bFts4 ){
+ Fts3TokenAndCost *aTC;
+ Fts3Expr **apOr;
+ aTC = (Fts3TokenAndCost *)sqlite3_malloc(
+ sizeof(Fts3TokenAndCost) * nToken
+ + sizeof(Fts3Expr *) * nOr * 2
+ );
+ apOr = (Fts3Expr **)&aTC[nToken];
+
+ if( !aTC ){
+ rc = SQLITE_NOMEM;
+ }else{
+ int ii;
+ Fts3TokenAndCost *pTC = aTC;
+ Fts3Expr **ppOr = apOr;
+
+ fts3EvalTokenCosts(pCsr, 0, pCsr->pExpr, &pTC, &ppOr, &rc);
+ nToken = (int)(pTC-aTC);
+ nOr = (int)(ppOr-apOr);
+
+ if( rc==SQLITE_OK ){
+ rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken);
+ for(ii=0; rc==SQLITE_OK && ii<nOr; ii++){
+ rc = fts3EvalSelectDeferred(pCsr, apOr[ii], aTC, nToken);
+ }
+ }
+
+ sqlite3_free(aTC);
+ }
+ }
+#endif
+
+ fts3EvalStartReaders(pCsr, pCsr->pExpr, &rc);
+ return rc;
+}
+
+/*
+** Invalidate the current position list for phrase pPhrase.
+*/
+static void fts3EvalInvalidatePoslist(Fts3Phrase *pPhrase){
+ if( pPhrase->doclist.bFreeList ){
+ sqlite3_free(pPhrase->doclist.pList);
+ }
+ pPhrase->doclist.pList = 0;
+ pPhrase->doclist.nList = 0;
+ pPhrase->doclist.bFreeList = 0;
+}
+
+/*
+** This function is called to edit the position list associated with
+** the phrase object passed as the fifth argument according to a NEAR
+** condition. For example:
+**
+** abc NEAR/5 "def ghi"
+**
+** Parameter nNear is passed the NEAR distance of the expression (5 in
+** the example above). When this function is called, *paPoslist points to
+** the position list, and *pnToken is the number of phrase tokens in, the
+** phrase on the other side of the NEAR operator to pPhrase. For example,
+** if pPhrase refers to the "def ghi" phrase, then *paPoslist points to
+** the position list associated with phrase "abc".
+**
+** All positions in the pPhrase position list that are not sufficiently
+** close to a position in the *paPoslist position list are removed. If this
+** leaves 0 positions, zero is returned. Otherwise, non-zero.
+**
+** Before returning, *paPoslist is set to point to the position lsit
+** associated with pPhrase. And *pnToken is set to the number of tokens in
+** pPhrase.
+*/
+static int fts3EvalNearTrim(
+ int nNear, /* NEAR distance. As in "NEAR/nNear". */
+ char *aTmp, /* Temporary space to use */
+ char **paPoslist, /* IN/OUT: Position list */
+ int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */
+ Fts3Phrase *pPhrase /* The phrase object to trim the doclist of */
+){
+ int nParam1 = nNear + pPhrase->nToken;
+ int nParam2 = nNear + *pnToken;
+ int nNew;
+ char *p2;
+ char *pOut;
+ int res;
+
+ assert( pPhrase->doclist.pList );
+
+ p2 = pOut = pPhrase->doclist.pList;
+ res = fts3PoslistNearMerge(
+ &pOut, aTmp, nParam1, nParam2, paPoslist, &p2
+ );
+ if( res ){
+ nNew = (int)(pOut - pPhrase->doclist.pList) - 1;
+ assert( pPhrase->doclist.pList[nNew]=='\0' );
+ assert( nNew<=pPhrase->doclist.nList && nNew>0 );
+ memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew);
+ pPhrase->doclist.nList = nNew;
+ *paPoslist = pPhrase->doclist.pList;
+ *pnToken = pPhrase->nToken;
+ }
+
+ return res;
+}
+
+/*
+** This function is a no-op if *pRc is other than SQLITE_OK when it is called.
+** Otherwise, it advances the expression passed as the second argument to
+** point to the next matching row in the database. Expressions iterate through
+** matching rows in docid order. Ascending order if Fts3Cursor.bDesc is zero,
+** or descending if it is non-zero.
+**
+** If an error occurs, *pRc is set to an SQLite error code. Otherwise, if
+** successful, the following variables in pExpr are set:
+**
+** Fts3Expr.bEof (non-zero if EOF - there is no next row)
+** Fts3Expr.iDocid (valid if bEof==0. The docid of the next row)
+**
+** If the expression is of type FTSQUERY_PHRASE, and the expression is not
+** at EOF, then the following variables are populated with the position list
+** for the phrase for the visited row:
+**
+** FTs3Expr.pPhrase->doclist.nList (length of pList in bytes)
+** FTs3Expr.pPhrase->doclist.pList (pointer to position list)
+**
+** It says above that this function advances the expression to the next
+** matching row. This is usually true, but there are the following exceptions:
+**
+** 1. Deferred tokens are not taken into account. If a phrase consists
+** entirely of deferred tokens, it is assumed to match every row in
+** the db. In this case the position-list is not populated at all.
+**
+** Or, if a phrase contains one or more deferred tokens and one or
+** more non-deferred tokens, then the expression is advanced to the
+** next possible match, considering only non-deferred tokens. In other
+** words, if the phrase is "A B C", and "B" is deferred, the expression
+** is advanced to the next row that contains an instance of "A * C",
+** where "*" may match any single token. The position list in this case
+** is populated as for "A * C" before returning.
+**
+** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is
+** advanced to point to the next row that matches "x AND y".
+**
+** See sqlite3Fts3EvalTestDeferred() for details on testing if a row is
+** really a match, taking into account deferred tokens and NEAR operators.
+*/
+static void fts3EvalNextRow(
+ Fts3Cursor *pCsr, /* FTS Cursor handle */
+ Fts3Expr *pExpr, /* Expr. to advance to next matching row */
+ int *pRc /* IN/OUT: Error code */
+){
+ if( *pRc==SQLITE_OK ){
+ int bDescDoclist = pCsr->bDesc; /* Used by DOCID_CMP() macro */
+ assert( pExpr->bEof==0 );
+ pExpr->bStart = 1;
+
+ switch( pExpr->eType ){
+ case FTSQUERY_NEAR:
+ case FTSQUERY_AND: {
+ Fts3Expr *pLeft = pExpr->pLeft;
+ Fts3Expr *pRight = pExpr->pRight;
+ assert( !pLeft->bDeferred || !pRight->bDeferred );
+
+ if( pLeft->bDeferred ){
+ /* LHS is entirely deferred. So we assume it matches every row.
+ ** Advance the RHS iterator to find the next row visited. */
+ fts3EvalNextRow(pCsr, pRight, pRc);
+ pExpr->iDocid = pRight->iDocid;
+ pExpr->bEof = pRight->bEof;
+ }else if( pRight->bDeferred ){
+ /* RHS is entirely deferred. So we assume it matches every row.
+ ** Advance the LHS iterator to find the next row visited. */
+ fts3EvalNextRow(pCsr, pLeft, pRc);
+ pExpr->iDocid = pLeft->iDocid;
+ pExpr->bEof = pLeft->bEof;
+ }else{
+ /* Neither the RHS or LHS are deferred. */
+ fts3EvalNextRow(pCsr, pLeft, pRc);
+ fts3EvalNextRow(pCsr, pRight, pRc);
+ while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){
+ sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
+ if( iDiff==0 ) break;
+ if( iDiff<0 ){
+ fts3EvalNextRow(pCsr, pLeft, pRc);
+ }else{
+ fts3EvalNextRow(pCsr, pRight, pRc);
+ }
+ }
+ pExpr->iDocid = pLeft->iDocid;
+ pExpr->bEof = (pLeft->bEof || pRight->bEof);
+ if( pExpr->eType==FTSQUERY_NEAR && pExpr->bEof ){
+ if( pRight->pPhrase && pRight->pPhrase->doclist.aAll ){
+ Fts3Doclist *pDl = &pRight->pPhrase->doclist;
+ while( *pRc==SQLITE_OK && pRight->bEof==0 ){
+ memset(pDl->pList, 0, pDl->nList);
+ fts3EvalNextRow(pCsr, pRight, pRc);
+ }
+ }
+ if( pLeft->pPhrase && pLeft->pPhrase->doclist.aAll ){
+ Fts3Doclist *pDl = &pLeft->pPhrase->doclist;
+ while( *pRc==SQLITE_OK && pLeft->bEof==0 ){
+ memset(pDl->pList, 0, pDl->nList);
+ fts3EvalNextRow(pCsr, pLeft, pRc);
+ }
+ }
+ }
+ }
+ break;
+ }
+
+ case FTSQUERY_OR: {
+ Fts3Expr *pLeft = pExpr->pLeft;
+ Fts3Expr *pRight = pExpr->pRight;
+ sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
+
+ assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid );
+ assert( pRight->bStart || pLeft->iDocid==pRight->iDocid );
+
+ if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){
+ fts3EvalNextRow(pCsr, pLeft, pRc);
+ }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){
+ fts3EvalNextRow(pCsr, pRight, pRc);
+ }else{
+ fts3EvalNextRow(pCsr, pLeft, pRc);
+ fts3EvalNextRow(pCsr, pRight, pRc);
+ }
+
+ pExpr->bEof = (pLeft->bEof && pRight->bEof);
+ iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
+ if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){
+ pExpr->iDocid = pLeft->iDocid;
+ }else{
+ pExpr->iDocid = pRight->iDocid;
+ }
+
+ break;
+ }
+
+ case FTSQUERY_NOT: {
+ Fts3Expr *pLeft = pExpr->pLeft;
+ Fts3Expr *pRight = pExpr->pRight;
+
+ if( pRight->bStart==0 ){
+ fts3EvalNextRow(pCsr, pRight, pRc);
+ assert( *pRc!=SQLITE_OK || pRight->bStart );
+ }
+
+ fts3EvalNextRow(pCsr, pLeft, pRc);
+ if( pLeft->bEof==0 ){
+ while( !*pRc
+ && !pRight->bEof
+ && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0
+ ){
+ fts3EvalNextRow(pCsr, pRight, pRc);
+ }
+ }
+ pExpr->iDocid = pLeft->iDocid;
+ pExpr->bEof = pLeft->bEof;
+ break;
+ }
+
+ default: {
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
+ fts3EvalInvalidatePoslist(pPhrase);
+ *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof);
+ pExpr->iDocid = pPhrase->doclist.iDocid;
+ break;
+ }
+ }
+ }
+}
+
+/*
+** If *pRc is not SQLITE_OK, or if pExpr is not the root node of a NEAR
+** cluster, then this function returns 1 immediately.
+**
+** Otherwise, it checks if the current row really does match the NEAR
+** expression, using the data currently stored in the position lists
+** (Fts3Expr->pPhrase.doclist.pList/nList) for each phrase in the expression.
+**
+** If the current row is a match, the position list associated with each
+** phrase in the NEAR expression is edited in place to contain only those
+** phrase instances sufficiently close to their peers to satisfy all NEAR
+** constraints. In this case it returns 1. If the NEAR expression does not
+** match the current row, 0 is returned. The position lists may or may not
+** be edited if 0 is returned.
+*/
+static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){
+ int res = 1;
+
+ /* The following block runs if pExpr is the root of a NEAR query.
+ ** For example, the query:
+ **
+ ** "w" NEAR "x" NEAR "y" NEAR "z"
+ **
+ ** which is represented in tree form as:
+ **
+ ** |
+ ** +--NEAR--+ <-- root of NEAR query
+ ** | |
+ ** +--NEAR--+ "z"
+ ** | |
+ ** +--NEAR--+ "y"
+ ** | |
+ ** "w" "x"
+ **
+ ** The right-hand child of a NEAR node is always a phrase. The
+ ** left-hand child may be either a phrase or a NEAR node. There are
+ ** no exceptions to this - it's the way the parser in fts3_expr.c works.
+ */
+ if( *pRc==SQLITE_OK
+ && pExpr->eType==FTSQUERY_NEAR
+ && pExpr->bEof==0
+ && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR)
+ ){
+ Fts3Expr *p;
+ int nTmp = 0; /* Bytes of temp space */
+ char *aTmp; /* Temp space for PoslistNearMerge() */
+
+ /* Allocate temporary working space. */
+ for(p=pExpr; p->pLeft; p=p->pLeft){
+ nTmp += p->pRight->pPhrase->doclist.nList;
+ }
+ nTmp += p->pPhrase->doclist.nList;
+ if( nTmp==0 ){
+ res = 0;
+ }else{
+ aTmp = sqlite3_malloc(nTmp*2);
+ if( !aTmp ){
+ *pRc = SQLITE_NOMEM;
+ res = 0;
+ }else{
+ char *aPoslist = p->pPhrase->doclist.pList;
+ int nToken = p->pPhrase->nToken;
+
+ for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){
+ Fts3Phrase *pPhrase = p->pRight->pPhrase;
+ int nNear = p->nNear;
+ res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
+ }
+
+ aPoslist = pExpr->pRight->pPhrase->doclist.pList;
+ nToken = pExpr->pRight->pPhrase->nToken;
+ for(p=pExpr->pLeft; p && res; p=p->pLeft){
+ int nNear;
+ Fts3Phrase *pPhrase;
+ assert( p->pParent && p->pParent->pLeft==p );
+ nNear = p->pParent->nNear;
+ pPhrase = (
+ p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase
+ );
+ res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
+ }
+ }
+
+ sqlite3_free(aTmp);
+ }
+ }
+
+ return res;
+}
+
+/*
+** This function is a helper function for sqlite3Fts3EvalTestDeferred().
+** Assuming no error occurs or has occurred, It returns non-zero if the
+** expression passed as the second argument matches the row that pCsr
+** currently points to, or zero if it does not.
+**
+** If *pRc is not SQLITE_OK when this function is called, it is a no-op.
+** If an error occurs during execution of this function, *pRc is set to
+** the appropriate SQLite error code. In this case the returned value is
+** undefined.
+*/
+static int fts3EvalTestExpr(
+ Fts3Cursor *pCsr, /* FTS cursor handle */
+ Fts3Expr *pExpr, /* Expr to test. May or may not be root. */
+ int *pRc /* IN/OUT: Error code */
+){
+ int bHit = 1; /* Return value */
+ if( *pRc==SQLITE_OK ){
+ switch( pExpr->eType ){
+ case FTSQUERY_NEAR:
+ case FTSQUERY_AND:
+ bHit = (
+ fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc)
+ && fts3EvalTestExpr(pCsr, pExpr->pRight, pRc)
+ && fts3EvalNearTest(pExpr, pRc)
+ );
+
+ /* If the NEAR expression does not match any rows, zero the doclist for
+ ** all phrases involved in the NEAR. This is because the snippet(),
+ ** offsets() and matchinfo() functions are not supposed to recognize
+ ** any instances of phrases that are part of unmatched NEAR queries.
+ ** For example if this expression:
+ **
+ ** ... MATCH 'a OR (b NEAR c)'
+ **
+ ** is matched against a row containing:
+ **
+ ** 'a b d e'
+ **
+ ** then any snippet() should ony highlight the "a" term, not the "b"
+ ** (as "b" is part of a non-matching NEAR clause).
+ */
+ if( bHit==0
+ && pExpr->eType==FTSQUERY_NEAR
+ && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR)
+ ){
+ Fts3Expr *p;
+ for(p=pExpr; p->pPhrase==0; p=p->pLeft){
+ if( p->pRight->iDocid==pCsr->iPrevId ){
+ fts3EvalInvalidatePoslist(p->pRight->pPhrase);
+ }
+ }
+ if( p->iDocid==pCsr->iPrevId ){
+ fts3EvalInvalidatePoslist(p->pPhrase);
+ }
+ }
+
+ break;
+
+ case FTSQUERY_OR: {
+ int bHit1 = fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc);
+ int bHit2 = fts3EvalTestExpr(pCsr, pExpr->pRight, pRc);
+ bHit = bHit1 || bHit2;
+ break;
+ }
+
+ case FTSQUERY_NOT:
+ bHit = (
+ fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc)
+ && !fts3EvalTestExpr(pCsr, pExpr->pRight, pRc)
+ );
+ break;
+
+ default: {
+#ifndef SQLITE_DISABLE_FTS4_DEFERRED
+ if( pCsr->pDeferred
+ && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred)
+ ){
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
+ assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 );
+ if( pExpr->bDeferred ){
+ fts3EvalInvalidatePoslist(pPhrase);
+ }
+ *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase);
+ bHit = (pPhrase->doclist.pList!=0);
+ pExpr->iDocid = pCsr->iPrevId;
+ }else
+#endif
+ {
+ bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId);
+ }
+ break;
+ }
+ }
+ }
+ return bHit;
+}
+
+/*
+** This function is called as the second part of each xNext operation when
+** iterating through the results of a full-text query. At this point the
+** cursor points to a row that matches the query expression, with the
+** following caveats:
+**
+** * Up until this point, "NEAR" operators in the expression have been
+** treated as "AND".
+**
+** * Deferred tokens have not yet been considered.
+**
+** If *pRc is not SQLITE_OK when this function is called, it immediately
+** returns 0. Otherwise, it tests whether or not after considering NEAR
+** operators and deferred tokens the current row is still a match for the
+** expression. It returns 1 if both of the following are true:
+**
+** 1. *pRc is SQLITE_OK when this function returns, and
+**
+** 2. After scanning the current FTS table row for the deferred tokens,
+** it is determined that the row does *not* match the query.
+**
+** Or, if no error occurs and it seems the current row does match the FTS
+** query, return 0.
+*/
+SQLITE_PRIVATE int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc){
+ int rc = *pRc;
+ int bMiss = 0;
+ if( rc==SQLITE_OK ){
+
+ /* If there are one or more deferred tokens, load the current row into
+ ** memory and scan it to determine the position list for each deferred
+ ** token. Then, see if this row is really a match, considering deferred
+ ** tokens and NEAR operators (neither of which were taken into account
+ ** earlier, by fts3EvalNextRow()).
+ */
+ if( pCsr->pDeferred ){
+ rc = fts3CursorSeek(0, pCsr);
+ if( rc==SQLITE_OK ){
+ rc = sqlite3Fts3CacheDeferredDoclists(pCsr);
+ }
+ }
+ bMiss = (0==fts3EvalTestExpr(pCsr, pCsr->pExpr, &rc));
+
+ /* Free the position-lists accumulated for each deferred token above. */
+ sqlite3Fts3FreeDeferredDoclists(pCsr);
+ *pRc = rc;
+ }
+ return (rc==SQLITE_OK && bMiss);
+}
+
+/*
+** Advance to the next document that matches the FTS expression in
+** Fts3Cursor.pExpr.
+*/
+static int fts3EvalNext(Fts3Cursor *pCsr){
+ int rc = SQLITE_OK; /* Return Code */
+ Fts3Expr *pExpr = pCsr->pExpr;
+ assert( pCsr->isEof==0 );
+ if( pExpr==0 ){
+ pCsr->isEof = 1;
+ }else{
+ do {
+ if( pCsr->isRequireSeek==0 ){
+ sqlite3_reset(pCsr->pStmt);
+ }
+ assert( sqlite3_data_count(pCsr->pStmt)==0 );
+ fts3EvalNextRow(pCsr, pExpr, &rc);
+ pCsr->isEof = pExpr->bEof;
+ pCsr->isRequireSeek = 1;
+ pCsr->isMatchinfoNeeded = 1;
+ pCsr->iPrevId = pExpr->iDocid;
+ }while( pCsr->isEof==0 && sqlite3Fts3EvalTestDeferred(pCsr, &rc) );
+ }
+
+ /* Check if the cursor is past the end of the docid range specified
+ ** by Fts3Cursor.iMinDocid/iMaxDocid. If so, set the EOF flag. */
+ if( rc==SQLITE_OK && (
+ (pCsr->bDesc==0 && pCsr->iPrevId>pCsr->iMaxDocid)
+ || (pCsr->bDesc!=0 && pCsr->iPrevId<pCsr->iMinDocid)
+ )){
+ pCsr->isEof = 1;
+ }
+
+ return rc;
+}
+
+/*
+** Restart interation for expression pExpr so that the next call to
+** fts3EvalNext() visits the first row. Do not allow incremental
+** loading or merging of phrase doclists for this iteration.
+**
+** If *pRc is other than SQLITE_OK when this function is called, it is
+** a no-op. If an error occurs within this function, *pRc is set to an
+** SQLite error code before returning.
+*/
+static void fts3EvalRestart(
+ Fts3Cursor *pCsr,
+ Fts3Expr *pExpr,
+ int *pRc
+){
+ if( pExpr && *pRc==SQLITE_OK ){
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
+
+ if( pPhrase ){
+ fts3EvalInvalidatePoslist(pPhrase);
+ if( pPhrase->bIncr ){
+ int i;
+ for(i=0; i<pPhrase->nToken; i++){
+ Fts3PhraseToken *pToken = &pPhrase->aToken[i];
+ assert( pToken->pDeferred==0 );
+ if( pToken->pSegcsr ){
+ sqlite3Fts3MsrIncrRestart(pToken->pSegcsr);
+ }
+ }
+ *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase);
+ }
+ pPhrase->doclist.pNextDocid = 0;
+ pPhrase->doclist.iDocid = 0;
+ pPhrase->pOrPoslist = 0;
+ }
+
+ pExpr->iDocid = 0;
+ pExpr->bEof = 0;
+ pExpr->bStart = 0;
+
+ fts3EvalRestart(pCsr, pExpr->pLeft, pRc);
+ fts3EvalRestart(pCsr, pExpr->pRight, pRc);
+ }
+}
+
+/*
+** After allocating the Fts3Expr.aMI[] array for each phrase in the
+** expression rooted at pExpr, the cursor iterates through all rows matched
+** by pExpr, calling this function for each row. This function increments
+** the values in Fts3Expr.aMI[] according to the position-list currently
+** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase
+** expression nodes.
+*/
+static void fts3EvalUpdateCounts(Fts3Expr *pExpr){
+ if( pExpr ){
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
+ if( pPhrase && pPhrase->doclist.pList ){
+ int iCol = 0;
+ char *p = pPhrase->doclist.pList;
+
+ assert( *p );
+ while( 1 ){
+ u8 c = 0;
+ int iCnt = 0;
+ while( 0xFE & (*p | c) ){
+ if( (c&0x80)==0 ) iCnt++;
+ c = *p++ & 0x80;
+ }
+
+ /* aMI[iCol*3 + 1] = Number of occurrences
+ ** aMI[iCol*3 + 2] = Number of rows containing at least one instance
+ */
+ pExpr->aMI[iCol*3 + 1] += iCnt;
+ pExpr->aMI[iCol*3 + 2] += (iCnt>0);
+ if( *p==0x00 ) break;
+ p++;
+ p += fts3GetVarint32(p, &iCol);
+ }
+ }
+
+ fts3EvalUpdateCounts(pExpr->pLeft);
+ fts3EvalUpdateCounts(pExpr->pRight);
+ }
+}
+
+/*
+** Expression pExpr must be of type FTSQUERY_PHRASE.
+**
+** If it is not already allocated and populated, this function allocates and
+** populates the Fts3Expr.aMI[] array for expression pExpr. If pExpr is part
+** of a NEAR expression, then it also allocates and populates the same array
+** for all other phrases that are part of the NEAR expression.
+**
+** SQLITE_OK is returned if the aMI[] array is successfully allocated and
+** populated. Otherwise, if an error occurs, an SQLite error code is returned.
+*/
+static int fts3EvalGatherStats(
+ Fts3Cursor *pCsr, /* Cursor object */
+ Fts3Expr *pExpr /* FTSQUERY_PHRASE expression */
+){
+ int rc = SQLITE_OK; /* Return code */
+
+ assert( pExpr->eType==FTSQUERY_PHRASE );
+ if( pExpr->aMI==0 ){
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+ Fts3Expr *pRoot; /* Root of NEAR expression */
+ Fts3Expr *p; /* Iterator used for several purposes */
+
+ sqlite3_int64 iPrevId = pCsr->iPrevId;
+ sqlite3_int64 iDocid;
+ u8 bEof;
+
+ /* Find the root of the NEAR expression */
+ pRoot = pExpr;
+ while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){
+ pRoot = pRoot->pParent;
+ }
+ iDocid = pRoot->iDocid;
+ bEof = pRoot->bEof;
+ assert( pRoot->bStart );
+
+ /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */
+ for(p=pRoot; p; p=p->pLeft){
+ Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight);
+ assert( pE->aMI==0 );
+ pE->aMI = (u32 *)sqlite3_malloc(pTab->nColumn * 3 * sizeof(u32));
+ if( !pE->aMI ) return SQLITE_NOMEM;
+ memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32));
+ }
+
+ fts3EvalRestart(pCsr, pRoot, &rc);
+
+ while( pCsr->isEof==0 && rc==SQLITE_OK ){
+
+ do {
+ /* Ensure the %_content statement is reset. */
+ if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt);
+ assert( sqlite3_data_count(pCsr->pStmt)==0 );
+
+ /* Advance to the next document */
+ fts3EvalNextRow(pCsr, pRoot, &rc);
+ pCsr->isEof = pRoot->bEof;
+ pCsr->isRequireSeek = 1;
+ pCsr->isMatchinfoNeeded = 1;
+ pCsr->iPrevId = pRoot->iDocid;
+ }while( pCsr->isEof==0
+ && pRoot->eType==FTSQUERY_NEAR
+ && sqlite3Fts3EvalTestDeferred(pCsr, &rc)
+ );
+
+ if( rc==SQLITE_OK && pCsr->isEof==0 ){
+ fts3EvalUpdateCounts(pRoot);
+ }
+ }
+
+ pCsr->isEof = 0;
+ pCsr->iPrevId = iPrevId;
+
+ if( bEof ){
+ pRoot->bEof = bEof;
+ }else{
+ /* Caution: pRoot may iterate through docids in ascending or descending
+ ** order. For this reason, even though it seems more defensive, the
+ ** do loop can not be written:
+ **
+ ** do {...} while( pRoot->iDocid<iDocid && rc==SQLITE_OK );
+ */
+ fts3EvalRestart(pCsr, pRoot, &rc);
+ do {
+ fts3EvalNextRow(pCsr, pRoot, &rc);
+ assert( pRoot->bEof==0 );
+ }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK );
+ }
+ }
+ return rc;
+}
+
+/*
+** This function is used by the matchinfo() module to query a phrase
+** expression node for the following information:
+**
+** 1. The total number of occurrences of the phrase in each column of
+** the FTS table (considering all rows), and
+**
+** 2. For each column, the number of rows in the table for which the
+** column contains at least one instance of the phrase.
+**
+** If no error occurs, SQLITE_OK is returned and the values for each column
+** written into the array aiOut as follows:
+**
+** aiOut[iCol*3 + 1] = Number of occurrences
+** aiOut[iCol*3 + 2] = Number of rows containing at least one instance
+**
+** Caveats:
+**
+** * If a phrase consists entirely of deferred tokens, then all output
+** values are set to the number of documents in the table. In other
+** words we assume that very common tokens occur exactly once in each
+** column of each row of the table.
+**
+** * If a phrase contains some deferred tokens (and some non-deferred
+** tokens), count the potential occurrence identified by considering
+** the non-deferred tokens instead of actual phrase occurrences.
+**
+** * If the phrase is part of a NEAR expression, then only phrase instances
+** that meet the NEAR constraint are included in the counts.
+*/
+SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats(
+ Fts3Cursor *pCsr, /* FTS cursor handle */
+ Fts3Expr *pExpr, /* Phrase expression */
+ u32 *aiOut /* Array to write results into (see above) */
+){
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+ int rc = SQLITE_OK;
+ int iCol;
+
+ if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){
+ assert( pCsr->nDoc>0 );
+ for(iCol=0; iCol<pTab->nColumn; iCol++){
+ aiOut[iCol*3 + 1] = (u32)pCsr->nDoc;
+ aiOut[iCol*3 + 2] = (u32)pCsr->nDoc;
+ }
+ }else{
+ rc = fts3EvalGatherStats(pCsr, pExpr);
+ if( rc==SQLITE_OK ){
+ assert( pExpr->aMI );
+ for(iCol=0; iCol<pTab->nColumn; iCol++){
+ aiOut[iCol*3 + 1] = pExpr->aMI[iCol*3 + 1];
+ aiOut[iCol*3 + 2] = pExpr->aMI[iCol*3 + 2];
+ }
+ }
+ }
+
+ return rc;
+}
+
+/*
+** The expression pExpr passed as the second argument to this function
+** must be of type FTSQUERY_PHRASE.
+**
+** The returned value is either NULL or a pointer to a buffer containing
+** a position-list indicating the occurrences of the phrase in column iCol
+** of the current row.
+**
+** More specifically, the returned buffer contains 1 varint for each
+** occurrence of the phrase in the column, stored using the normal (delta+2)
+** compression and is terminated by either an 0x01 or 0x00 byte. For example,
+** if the requested column contains "a b X c d X X" and the position-list
+** for 'X' is requested, the buffer returned may contain:
+**
+** 0x04 0x05 0x03 0x01 or 0x04 0x05 0x03 0x00
+**
+** This function works regardless of whether or not the phrase is deferred,
+** incremental, or neither.
+*/
+SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist(
+ Fts3Cursor *pCsr, /* FTS3 cursor object */
+ Fts3Expr *pExpr, /* Phrase to return doclist for */
+ int iCol, /* Column to return position list for */
+ char **ppOut /* OUT: Pointer to position list */
+){
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+ char *pIter;
+ int iThis;
+ sqlite3_int64 iDocid;
+
+ /* If this phrase is applies specifically to some column other than
+ ** column iCol, return a NULL pointer. */
+ *ppOut = 0;
+ assert( iCol>=0 && iCol<pTab->nColumn );
+ if( (pPhrase->iColumn<pTab->nColumn && pPhrase->iColumn!=iCol) ){
+ return SQLITE_OK;
+ }
+
+ iDocid = pExpr->iDocid;
+ pIter = pPhrase->doclist.pList;
+ if( iDocid!=pCsr->iPrevId || pExpr->bEof ){
+ int rc = SQLITE_OK;
+ int bDescDoclist = pTab->bDescIdx; /* For DOCID_CMP macro */
+ int bOr = 0;
+ u8 bTreeEof = 0;
+ Fts3Expr *p; /* Used to iterate from pExpr to root */
+ Fts3Expr *pNear; /* Most senior NEAR ancestor (or pExpr) */
+ int bMatch;
+
+ /* Check if this phrase descends from an OR expression node. If not,
+ ** return NULL. Otherwise, the entry that corresponds to docid
+ ** pCsr->iPrevId may lie earlier in the doclist buffer. Or, if the
+ ** tree that the node is part of has been marked as EOF, but the node
+ ** itself is not EOF, then it may point to an earlier entry. */
+ pNear = pExpr;
+ for(p=pExpr->pParent; p; p=p->pParent){
+ if( p->eType==FTSQUERY_OR ) bOr = 1;
+ if( p->eType==FTSQUERY_NEAR ) pNear = p;
+ if( p->bEof ) bTreeEof = 1;
+ }
+ if( bOr==0 ) return SQLITE_OK;
+
+ /* This is the descendent of an OR node. In this case we cannot use
+ ** an incremental phrase. Load the entire doclist for the phrase
+ ** into memory in this case. */
+ if( pPhrase->bIncr ){
+ int bEofSave = pNear->bEof;
+ fts3EvalRestart(pCsr, pNear, &rc);
+ while( rc==SQLITE_OK && !pNear->bEof ){
+ fts3EvalNextRow(pCsr, pNear, &rc);
+ if( bEofSave==0 && pNear->iDocid==iDocid ) break;
+ }
+ assert( rc!=SQLITE_OK || pPhrase->bIncr==0 );
+ }
+ if( bTreeEof ){
+ while( rc==SQLITE_OK && !pNear->bEof ){
+ fts3EvalNextRow(pCsr, pNear, &rc);
+ }
+ }
+ if( rc!=SQLITE_OK ) return rc;
+
+ bMatch = 1;
+ for(p=pNear; p; p=p->pLeft){
+ u8 bEof = 0;
+ Fts3Expr *pTest = p;
+ Fts3Phrase *pPh;
+ assert( pTest->eType==FTSQUERY_NEAR || pTest->eType==FTSQUERY_PHRASE );
+ if( pTest->eType==FTSQUERY_NEAR ) pTest = pTest->pRight;
+ assert( pTest->eType==FTSQUERY_PHRASE );
+ pPh = pTest->pPhrase;
+
+ pIter = pPh->pOrPoslist;
+ iDocid = pPh->iOrDocid;
+ if( pCsr->bDesc==bDescDoclist ){
+ bEof = !pPh->doclist.nAll ||
+ (pIter >= (pPh->doclist.aAll + pPh->doclist.nAll));
+ while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){
+ sqlite3Fts3DoclistNext(
+ bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll,
+ &pIter, &iDocid, &bEof
+ );
+ }
+ }else{
+ bEof = !pPh->doclist.nAll || (pIter && pIter<=pPh->doclist.aAll);
+ while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){
+ int dummy;
+ sqlite3Fts3DoclistPrev(
+ bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll,
+ &pIter, &iDocid, &dummy, &bEof
+ );
+ }
+ }
+ pPh->pOrPoslist = pIter;
+ pPh->iOrDocid = iDocid;
+ if( bEof || iDocid!=pCsr->iPrevId ) bMatch = 0;
+ }
+
+ if( bMatch ){
+ pIter = pPhrase->pOrPoslist;
+ }else{
+ pIter = 0;
+ }
+ }
+ if( pIter==0 ) return SQLITE_OK;
+
+ if( *pIter==0x01 ){
+ pIter++;
+ pIter += fts3GetVarint32(pIter, &iThis);
+ }else{
+ iThis = 0;
+ }
+ while( iThis<iCol ){
+ fts3ColumnlistCopy(0, &pIter);
+ if( *pIter==0x00 ) return SQLITE_OK;
+ pIter++;
+ pIter += fts3GetVarint32(pIter, &iThis);
+ }
+ if( *pIter==0x00 ){
+ pIter = 0;
+ }
+
+ *ppOut = ((iCol==iThis)?pIter:0);
+ return SQLITE_OK;
+}
+
+/*
+** Free all components of the Fts3Phrase structure that were allocated by
+** the eval module. Specifically, this means to free:
+**
+** * the contents of pPhrase->doclist, and
+** * any Fts3MultiSegReader objects held by phrase tokens.
+*/
+SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){
+ if( pPhrase ){
+ int i;
+ sqlite3_free(pPhrase->doclist.aAll);
+ fts3EvalInvalidatePoslist(pPhrase);
+ memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist));
+ for(i=0; i<pPhrase->nToken; i++){
+ fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr);
+ pPhrase->aToken[i].pSegcsr = 0;
+ }
+ }
+}
+
+
+/*
+** Return SQLITE_CORRUPT_VTAB.
+*/
+#ifdef SQLITE_DEBUG
+SQLITE_PRIVATE int sqlite3Fts3Corrupt(){
+ return SQLITE_CORRUPT_VTAB;
+}
+#endif
+
+#if !SQLITE_CORE
+/*
+** Initialize API pointer table, if required.
+*/
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+SQLITE_API int SQLITE_STDCALL sqlite3_fts3_init(
+ sqlite3 *db,
+ char **pzErrMsg,
+ const sqlite3_api_routines *pApi
+){
+ SQLITE_EXTENSION_INIT2(pApi)
+ return sqlite3Fts3Init(db);
+}
+#endif
+
+#endif
+
+/************** End of fts3.c ************************************************/
+/************** Begin file fts3_aux.c ****************************************/
+/*
+** 2011 Jan 27
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+*/
+/* #include "fts3Int.h" */
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+/* #include <string.h> */
+/* #include <assert.h> */
+
+typedef struct Fts3auxTable Fts3auxTable;
+typedef struct Fts3auxCursor Fts3auxCursor;
+
+struct Fts3auxTable {
+ sqlite3_vtab base; /* Base class used by SQLite core */
+ Fts3Table *pFts3Tab;
+};
+
+struct Fts3auxCursor {
+ sqlite3_vtab_cursor base; /* Base class used by SQLite core */
+ Fts3MultiSegReader csr; /* Must be right after "base" */
+ Fts3SegFilter filter;
+ char *zStop;
+ int nStop; /* Byte-length of string zStop */
+ int iLangid; /* Language id to query */
+ int isEof; /* True if cursor is at EOF */
+ sqlite3_int64 iRowid; /* Current rowid */
+
+ int iCol; /* Current value of 'col' column */
+ int nStat; /* Size of aStat[] array */
+ struct Fts3auxColstats {
+ sqlite3_int64 nDoc; /* 'documents' values for current csr row */
+ sqlite3_int64 nOcc; /* 'occurrences' values for current csr row */
+ } *aStat;
+};
+
+/*
+** Schema of the terms table.
+*/
+#define FTS3_AUX_SCHEMA \
+ "CREATE TABLE x(term, col, documents, occurrences, languageid HIDDEN)"
+
+/*
+** This function does all the work for both the xConnect and xCreate methods.
+** These tables have no persistent representation of their own, so xConnect
+** and xCreate are identical operations.
+*/
+static int fts3auxConnectMethod(
+ sqlite3 *db, /* Database connection */
+ void *pUnused, /* Unused */
+ int argc, /* Number of elements in argv array */
+ const char * const *argv, /* xCreate/xConnect argument array */
+ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
+ char **pzErr /* OUT: sqlite3_malloc'd error message */
+){
+ char const *zDb; /* Name of database (e.g. "main") */
+ char const *zFts3; /* Name of fts3 table */
+ int nDb; /* Result of strlen(zDb) */
+ int nFts3; /* Result of strlen(zFts3) */
+ int nByte; /* Bytes of space to allocate here */
+ int rc; /* value returned by declare_vtab() */
+ Fts3auxTable *p; /* Virtual table object to return */
+
+ UNUSED_PARAMETER(pUnused);
+
+ /* The user should invoke this in one of two forms:
+ **
+ ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table);
+ ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table-db, fts4-table);
+ */
+ if( argc!=4 && argc!=5 ) goto bad_args;
+
+ zDb = argv[1];
+ nDb = (int)strlen(zDb);
+ if( argc==5 ){
+ if( nDb==4 && 0==sqlite3_strnicmp("temp", zDb, 4) ){
+ zDb = argv[3];
+ nDb = (int)strlen(zDb);
+ zFts3 = argv[4];
+ }else{
+ goto bad_args;
+ }
+ }else{
+ zFts3 = argv[3];
+ }
+ nFts3 = (int)strlen(zFts3);
+
+ rc = sqlite3_declare_vtab(db, FTS3_AUX_SCHEMA);
+ if( rc!=SQLITE_OK ) return rc;
+
+ nByte = sizeof(Fts3auxTable) + sizeof(Fts3Table) + nDb + nFts3 + 2;
+ p = (Fts3auxTable *)sqlite3_malloc(nByte);
+ if( !p ) return SQLITE_NOMEM;
+ memset(p, 0, nByte);
+
+ p->pFts3Tab = (Fts3Table *)&p[1];
+ p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1];
+ p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1];
+ p->pFts3Tab->db = db;
+ p->pFts3Tab->nIndex = 1;
+
+ memcpy((char *)p->pFts3Tab->zDb, zDb, nDb);
+ memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3);
+ sqlite3Fts3Dequote((char *)p->pFts3Tab->zName);
+
+ *ppVtab = (sqlite3_vtab *)p;
+ return SQLITE_OK;
+
+ bad_args:
+ sqlite3Fts3ErrMsg(pzErr, "invalid arguments to fts4aux constructor");
+ return SQLITE_ERROR;
+}
+
+/*
+** This function does the work for both the xDisconnect and xDestroy methods.
+** These tables have no persistent representation of their own, so xDisconnect
+** and xDestroy are identical operations.
+*/
+static int fts3auxDisconnectMethod(sqlite3_vtab *pVtab){
+ Fts3auxTable *p = (Fts3auxTable *)pVtab;
+ Fts3Table *pFts3 = p->pFts3Tab;
+ int i;
+
+ /* Free any prepared statements held */
+ for(i=0; i<SizeofArray(pFts3->aStmt); i++){
+ sqlite3_finalize(pFts3->aStmt[i]);
+ }
+ sqlite3_free(pFts3->zSegmentsTbl);
+ sqlite3_free(p);
+ return SQLITE_OK;
+}
+
+#define FTS4AUX_EQ_CONSTRAINT 1
+#define FTS4AUX_GE_CONSTRAINT 2
+#define FTS4AUX_LE_CONSTRAINT 4
+
+/*
+** xBestIndex - Analyze a WHERE and ORDER BY clause.
+*/
+static int fts3auxBestIndexMethod(
+ sqlite3_vtab *pVTab,
+ sqlite3_index_info *pInfo
+){
+ int i;
+ int iEq = -1;
+ int iGe = -1;
+ int iLe = -1;
+ int iLangid = -1;
+ int iNext = 1; /* Next free argvIndex value */
+
+ UNUSED_PARAMETER(pVTab);
+
+ /* This vtab delivers always results in "ORDER BY term ASC" order. */
+ if( pInfo->nOrderBy==1
+ && pInfo->aOrderBy[0].iColumn==0
+ && pInfo->aOrderBy[0].desc==0
+ ){
+ pInfo->orderByConsumed = 1;
+ }
+
+ /* Search for equality and range constraints on the "term" column.
+ ** And equality constraints on the hidden "languageid" column. */
+ for(i=0; i<pInfo->nConstraint; i++){
+ if( pInfo->aConstraint[i].usable ){
+ int op = pInfo->aConstraint[i].op;
+ int iCol = pInfo->aConstraint[i].iColumn;
+
+ if( iCol==0 ){
+ if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iEq = i;
+ if( op==SQLITE_INDEX_CONSTRAINT_LT ) iLe = i;
+ if( op==SQLITE_INDEX_CONSTRAINT_LE ) iLe = i;
+ if( op==SQLITE_INDEX_CONSTRAINT_GT ) iGe = i;
+ if( op==SQLITE_INDEX_CONSTRAINT_GE ) iGe = i;
+ }
+ if( iCol==4 ){
+ if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iLangid = i;
+ }
+ }
+ }
+
+ if( iEq>=0 ){
+ pInfo->idxNum = FTS4AUX_EQ_CONSTRAINT;
+ pInfo->aConstraintUsage[iEq].argvIndex = iNext++;
+ pInfo->estimatedCost = 5;
+ }else{
+ pInfo->idxNum = 0;
+ pInfo->estimatedCost = 20000;
+ if( iGe>=0 ){
+ pInfo->idxNum += FTS4AUX_GE_CONSTRAINT;
+ pInfo->aConstraintUsage[iGe].argvIndex = iNext++;
+ pInfo->estimatedCost /= 2;
+ }
+ if( iLe>=0 ){
+ pInfo->idxNum += FTS4AUX_LE_CONSTRAINT;
+ pInfo->aConstraintUsage[iLe].argvIndex = iNext++;
+ pInfo->estimatedCost /= 2;
+ }
+ }
+ if( iLangid>=0 ){
+ pInfo->aConstraintUsage[iLangid].argvIndex = iNext++;
+ pInfo->estimatedCost--;
+ }
+
+ return SQLITE_OK;
+}
+
+/*
+** xOpen - Open a cursor.
+*/
+static int fts3auxOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
+ Fts3auxCursor *pCsr; /* Pointer to cursor object to return */
+
+ UNUSED_PARAMETER(pVTab);
+
+ pCsr = (Fts3auxCursor *)sqlite3_malloc(sizeof(Fts3auxCursor));
+ if( !pCsr ) return SQLITE_NOMEM;
+ memset(pCsr, 0, sizeof(Fts3auxCursor));
+
+ *ppCsr = (sqlite3_vtab_cursor *)pCsr;
+ return SQLITE_OK;
+}
+
+/*
+** xClose - Close a cursor.
+*/
+static int fts3auxCloseMethod(sqlite3_vtab_cursor *pCursor){
+ Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab;
+ Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor;
+
+ sqlite3Fts3SegmentsClose(pFts3);
+ sqlite3Fts3SegReaderFinish(&pCsr->csr);
+ sqlite3_free((void *)pCsr->filter.zTerm);
+ sqlite3_free(pCsr->zStop);
+ sqlite3_free(pCsr->aStat);
+ sqlite3_free(pCsr);
+ return SQLITE_OK;
+}
+
+static int fts3auxGrowStatArray(Fts3auxCursor *pCsr, int nSize){
+ if( nSize>pCsr->nStat ){
+ struct Fts3auxColstats *aNew;
+ aNew = (struct Fts3auxColstats *)sqlite3_realloc(pCsr->aStat,
+ sizeof(struct Fts3auxColstats) * nSize
+ );
+ if( aNew==0 ) return SQLITE_NOMEM;
+ memset(&aNew[pCsr->nStat], 0,
+ sizeof(struct Fts3auxColstats) * (nSize - pCsr->nStat)
+ );
+ pCsr->aStat = aNew;
+ pCsr->nStat = nSize;
+ }
+ return SQLITE_OK;
+}
+
+/*
+** xNext - Advance the cursor to the next row, if any.
+*/
+static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){
+ Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor;
+ Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab;
+ int rc;
+
+ /* Increment our pretend rowid value. */
+ pCsr->iRowid++;
+
+ for(pCsr->iCol++; pCsr->iCol<pCsr->nStat; pCsr->iCol++){
+ if( pCsr->aStat[pCsr->iCol].nDoc>0 ) return SQLITE_OK;
+ }
+
+ rc = sqlite3Fts3SegReaderStep(pFts3, &pCsr->csr);
+ if( rc==SQLITE_ROW ){
+ int i = 0;
+ int nDoclist = pCsr->csr.nDoclist;
+ char *aDoclist = pCsr->csr.aDoclist;
+ int iCol;
+
+ int eState = 0;
+
+ if( pCsr->zStop ){
+ int n = (pCsr->nStop<pCsr->csr.nTerm) ? pCsr->nStop : pCsr->csr.nTerm;
+ int mc = memcmp(pCsr->zStop, pCsr->csr.zTerm, n);
+ if( mc<0 || (mc==0 && pCsr->csr.nTerm>pCsr->nStop) ){
+ pCsr->isEof = 1;
+ return SQLITE_OK;
+ }
+ }
+
+ if( fts3auxGrowStatArray(pCsr, 2) ) return SQLITE_NOMEM;
+ memset(pCsr->aStat, 0, sizeof(struct Fts3auxColstats) * pCsr->nStat);
+ iCol = 0;
+
+ while( i<nDoclist ){
+ sqlite3_int64 v = 0;
+
+ i += sqlite3Fts3GetVarint(&aDoclist[i], &v);
+ switch( eState ){
+ /* State 0. In this state the integer just read was a docid. */
+ case 0:
+ pCsr->aStat[0].nDoc++;
+ eState = 1;
+ iCol = 0;
+ break;
+
+ /* State 1. In this state we are expecting either a 1, indicating
+ ** that the following integer will be a column number, or the
+ ** start of a position list for column 0.
+ **
+ ** The only difference between state 1 and state 2 is that if the
+ ** integer encountered in state 1 is not 0 or 1, then we need to
+ ** increment the column 0 "nDoc" count for this term.
+ */
+ case 1:
+ assert( iCol==0 );
+ if( v>1 ){
+ pCsr->aStat[1].nDoc++;
+ }
+ eState = 2;
+ /* fall through */
+
+ case 2:
+ if( v==0 ){ /* 0x00. Next integer will be a docid. */
+ eState = 0;
+ }else if( v==1 ){ /* 0x01. Next integer will be a column number. */
+ eState = 3;
+ }else{ /* 2 or greater. A position. */
+ pCsr->aStat[iCol+1].nOcc++;
+ pCsr->aStat[0].nOcc++;
+ }
+ break;
+
+ /* State 3. The integer just read is a column number. */
+ default: assert( eState==3 );
+ iCol = (int)v;
+ if( fts3auxGrowStatArray(pCsr, iCol+2) ) return SQLITE_NOMEM;
+ pCsr->aStat[iCol+1].nDoc++;
+ eState = 2;
+ break;
+ }
+ }
+
+ pCsr->iCol = 0;
+ rc = SQLITE_OK;
+ }else{
+ pCsr->isEof = 1;
+ }
+ return rc;
+}
+
+/*
+** xFilter - Initialize a cursor to point at the start of its data.
+*/
+static int fts3auxFilterMethod(
+ sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
+ int idxNum, /* Strategy index */
+ const char *idxStr, /* Unused */
+ int nVal, /* Number of elements in apVal */
+ sqlite3_value **apVal /* Arguments for the indexing scheme */
+){
+ Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor;
+ Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab;
+ int rc;
+ int isScan = 0;
+ int iLangVal = 0; /* Language id to query */
+
+ int iEq = -1; /* Index of term=? value in apVal */
+ int iGe = -1; /* Index of term>=? value in apVal */
+ int iLe = -1; /* Index of term<=? value in apVal */
+ int iLangid = -1; /* Index of languageid=? value in apVal */
+ int iNext = 0;
+
+ UNUSED_PARAMETER(nVal);
+ UNUSED_PARAMETER(idxStr);
+
+ assert( idxStr==0 );
+ assert( idxNum==FTS4AUX_EQ_CONSTRAINT || idxNum==0
+ || idxNum==FTS4AUX_LE_CONSTRAINT || idxNum==FTS4AUX_GE_CONSTRAINT
+ || idxNum==(FTS4AUX_LE_CONSTRAINT|FTS4AUX_GE_CONSTRAINT)
+ );
+
+ if( idxNum==FTS4AUX_EQ_CONSTRAINT ){
+ iEq = iNext++;
+ }else{
+ isScan = 1;
+ if( idxNum & FTS4AUX_GE_CONSTRAINT ){
+ iGe = iNext++;
+ }
+ if( idxNum & FTS4AUX_LE_CONSTRAINT ){
+ iLe = iNext++;
+ }
+ }
+ if( iNext<nVal ){
+ iLangid = iNext++;
+ }
+
+ /* In case this cursor is being reused, close and zero it. */
+ testcase(pCsr->filter.zTerm);
+ sqlite3Fts3SegReaderFinish(&pCsr->csr);
+ sqlite3_free((void *)pCsr->filter.zTerm);
+ sqlite3_free(pCsr->aStat);
+ memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr);
+
+ pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY;
+ if( isScan ) pCsr->filter.flags |= FTS3_SEGMENT_SCAN;
+
+ if( iEq>=0 || iGe>=0 ){
+ const unsigned char *zStr = sqlite3_value_text(apVal[0]);
+ assert( (iEq==0 && iGe==-1) || (iEq==-1 && iGe==0) );
+ if( zStr ){
+ pCsr->filter.zTerm = sqlite3_mprintf("%s", zStr);
+ pCsr->filter.nTerm = sqlite3_value_bytes(apVal[0]);
+ if( pCsr->filter.zTerm==0 ) return SQLITE_NOMEM;
+ }
+ }
+
+ if( iLe>=0 ){
+ pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iLe]));
+ pCsr->nStop = sqlite3_value_bytes(apVal[iLe]);
+ if( pCsr->zStop==0 ) return SQLITE_NOMEM;
+ }
+
+ if( iLangid>=0 ){
+ iLangVal = sqlite3_value_int(apVal[iLangid]);
+
+ /* If the user specified a negative value for the languageid, use zero
+ ** instead. This works, as the "languageid=?" constraint will also
+ ** be tested by the VDBE layer. The test will always be false (since
+ ** this module will not return a row with a negative languageid), and
+ ** so the overall query will return zero rows. */
+ if( iLangVal<0 ) iLangVal = 0;
+ }
+ pCsr->iLangid = iLangVal;
+
+ rc = sqlite3Fts3SegReaderCursor(pFts3, iLangVal, 0, FTS3_SEGCURSOR_ALL,
+ pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr
+ );
+ if( rc==SQLITE_OK ){
+ rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter);
+ }
+
+ if( rc==SQLITE_OK ) rc = fts3auxNextMethod(pCursor);
+ return rc;
+}
+
+/*
+** xEof - Return true if the cursor is at EOF, or false otherwise.
+*/
+static int fts3auxEofMethod(sqlite3_vtab_cursor *pCursor){
+ Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor;
+ return pCsr->isEof;
+}
+
+/*
+** xColumn - Return a column value.
+*/
+static int fts3auxColumnMethod(
+ sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
+ sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */
+ int iCol /* Index of column to read value from */
+){
+ Fts3auxCursor *p = (Fts3auxCursor *)pCursor;
+
+ assert( p->isEof==0 );
+ switch( iCol ){
+ case 0: /* term */
+ sqlite3_result_text(pCtx, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT);
+ break;
+
+ case 1: /* col */
+ if( p->iCol ){
+ sqlite3_result_int(pCtx, p->iCol-1);
+ }else{
+ sqlite3_result_text(pCtx, "*", -1, SQLITE_STATIC);
+ }
+ break;
+
+ case 2: /* documents */
+ sqlite3_result_int64(pCtx, p->aStat[p->iCol].nDoc);
+ break;
+
+ case 3: /* occurrences */
+ sqlite3_result_int64(pCtx, p->aStat[p->iCol].nOcc);
+ break;
+
+ default: /* languageid */
+ assert( iCol==4 );
+ sqlite3_result_int(pCtx, p->iLangid);
+ break;
+ }
+
+ return SQLITE_OK;
+}
+
+/*
+** xRowid - Return the current rowid for the cursor.
+*/
+static int fts3auxRowidMethod(
+ sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
+ sqlite_int64 *pRowid /* OUT: Rowid value */
+){
+ Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor;
+ *pRowid = pCsr->iRowid;
+ return SQLITE_OK;
+}
+
+/*
+** Register the fts3aux module with database connection db. Return SQLITE_OK
+** if successful or an error code if sqlite3_create_module() fails.
+*/
+SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db){
+ static const sqlite3_module fts3aux_module = {
+ 0, /* iVersion */
+ fts3auxConnectMethod, /* xCreate */
+ fts3auxConnectMethod, /* xConnect */
+ fts3auxBestIndexMethod, /* xBestIndex */
+ fts3auxDisconnectMethod, /* xDisconnect */
+ fts3auxDisconnectMethod, /* xDestroy */
+ fts3auxOpenMethod, /* xOpen */
+ fts3auxCloseMethod, /* xClose */
+ fts3auxFilterMethod, /* xFilter */
+ fts3auxNextMethod, /* xNext */
+ fts3auxEofMethod, /* xEof */
+ fts3auxColumnMethod, /* xColumn */
+ fts3auxRowidMethod, /* xRowid */
+ 0, /* xUpdate */
+ 0, /* xBegin */
+ 0, /* xSync */
+ 0, /* xCommit */
+ 0, /* xRollback */
+ 0, /* xFindFunction */
+ 0, /* xRename */
+ 0, /* xSavepoint */
+ 0, /* xRelease */
+ 0 /* xRollbackTo */
+ };
+ int rc; /* Return code */
+
+ rc = sqlite3_create_module(db, "fts4aux", &fts3aux_module, 0);
+ return rc;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
+
+/************** End of fts3_aux.c ********************************************/
+/************** Begin file fts3_expr.c ***************************************/
+/*
+** 2008 Nov 28
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This module contains code that implements a parser for fts3 query strings
+** (the right-hand argument to the MATCH operator). Because the supported
+** syntax is relatively simple, the whole tokenizer/parser system is
+** hand-coded.
+*/
+/* #include "fts3Int.h" */
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+/*
+** By default, this module parses the legacy syntax that has been
+** traditionally used by fts3. Or, if SQLITE_ENABLE_FTS3_PARENTHESIS
+** is defined, then it uses the new syntax. The differences between
+** the new and the old syntaxes are:
+**
+** a) The new syntax supports parenthesis. The old does not.
+**
+** b) The new syntax supports the AND and NOT operators. The old does not.
+**
+** c) The old syntax supports the "-" token qualifier. This is not
+** supported by the new syntax (it is replaced by the NOT operator).
+**
+** d) When using the old syntax, the OR operator has a greater precedence
+** than an implicit AND. When using the new, both implicity and explicit
+** AND operators have a higher precedence than OR.
+**
+** If compiled with SQLITE_TEST defined, then this module exports the
+** symbol "int sqlite3_fts3_enable_parentheses". Setting this variable
+** to zero causes the module to use the old syntax. If it is set to
+** non-zero the new syntax is activated. This is so both syntaxes can
+** be tested using a single build of testfixture.
+**
+** The following describes the syntax supported by the fts3 MATCH
+** operator in a similar format to that used by the lemon parser
+** generator. This module does not use actually lemon, it uses a
+** custom parser.
+**
+** query ::= andexpr (OR andexpr)*.
+**
+** andexpr ::= notexpr (AND? notexpr)*.
+**
+** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*.
+** notexpr ::= LP query RP.
+**
+** nearexpr ::= phrase (NEAR distance_opt nearexpr)*.
+**
+** distance_opt ::= .
+** distance_opt ::= / INTEGER.
+**
+** phrase ::= TOKEN.
+** phrase ::= COLUMN:TOKEN.
+** phrase ::= "TOKEN TOKEN TOKEN...".
+*/
+
+#ifdef SQLITE_TEST
+SQLITE_API int sqlite3_fts3_enable_parentheses = 0;
+#else
+# ifdef SQLITE_ENABLE_FTS3_PARENTHESIS
+# define sqlite3_fts3_enable_parentheses 1
+# else
+# define sqlite3_fts3_enable_parentheses 0
+# endif
+#endif
+
+/*
+** Default span for NEAR operators.
+*/
+#define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10
+
+/* #include <string.h> */
+/* #include <assert.h> */
+
+/*
+** isNot:
+** This variable is used by function getNextNode(). When getNextNode() is
+** called, it sets ParseContext.isNot to true if the 'next node' is a
+** FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the
+** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to
+** zero.
+*/
+typedef struct ParseContext ParseContext;
+struct ParseContext {
+ sqlite3_tokenizer *pTokenizer; /* Tokenizer module */
+ int iLangid; /* Language id used with tokenizer */
+ const char **azCol; /* Array of column names for fts3 table */
+ int bFts4; /* True to allow FTS4-only syntax */
+ int nCol; /* Number of entries in azCol[] */
+ int iDefaultCol; /* Default column to query */
+ int isNot; /* True if getNextNode() sees a unary - */
+ sqlite3_context *pCtx; /* Write error message here */
+ int nNest; /* Number of nested brackets */
+};
+
+/*
+** This function is equivalent to the standard isspace() function.
+**
+** The standard isspace() can be awkward to use safely, because although it
+** is defined to accept an argument of type int, its behavior when passed
+** an integer that falls outside of the range of the unsigned char type
+** is undefined (and sometimes, "undefined" means segfault). This wrapper
+** is defined to accept an argument of type char, and always returns 0 for
+** any values that fall outside of the range of the unsigned char type (i.e.
+** negative values).
+*/
+static int fts3isspace(char c){
+ return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f';
+}
+
+/*
+** Allocate nByte bytes of memory using sqlite3_malloc(). If successful,
+** zero the memory before returning a pointer to it. If unsuccessful,
+** return NULL.
+*/
+static void *fts3MallocZero(int nByte){
+ void *pRet = sqlite3_malloc(nByte);
+ if( pRet ) memset(pRet, 0, nByte);
+ return pRet;
+}
+
+SQLITE_PRIVATE int sqlite3Fts3OpenTokenizer(
+ sqlite3_tokenizer *pTokenizer,
+ int iLangid,
+ const char *z,
+ int n,
+ sqlite3_tokenizer_cursor **ppCsr
+){
+ sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
+ sqlite3_tokenizer_cursor *pCsr = 0;
+ int rc;
+
+ rc = pModule->xOpen(pTokenizer, z, n, &pCsr);
+ assert( rc==SQLITE_OK || pCsr==0 );
+ if( rc==SQLITE_OK ){
+ pCsr->pTokenizer = pTokenizer;
+ if( pModule->iVersion>=1 ){
+ rc = pModule->xLanguageid(pCsr, iLangid);
+ if( rc!=SQLITE_OK ){
+ pModule->xClose(pCsr);
+ pCsr = 0;
+ }
+ }
+ }
+ *ppCsr = pCsr;
+ return rc;
+}
+
+/*
+** Function getNextNode(), which is called by fts3ExprParse(), may itself
+** call fts3ExprParse(). So this forward declaration is required.
+*/
+static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *);
+
+/*
+** Extract the next token from buffer z (length n) using the tokenizer
+** and other information (column names etc.) in pParse. Create an Fts3Expr
+** structure of type FTSQUERY_PHRASE containing a phrase consisting of this
+** single token and set *ppExpr to point to it. If the end of the buffer is
+** reached before a token is found, set *ppExpr to zero. It is the
+** responsibility of the caller to eventually deallocate the allocated
+** Fts3Expr structure (if any) by passing it to sqlite3_free().
+**
+** Return SQLITE_OK if successful, or SQLITE_NOMEM if a memory allocation
+** fails.
+*/
+static int getNextToken(
+ ParseContext *pParse, /* fts3 query parse context */
+ int iCol, /* Value for Fts3Phrase.iColumn */
+ const char *z, int n, /* Input string */
+ Fts3Expr **ppExpr, /* OUT: expression */
+ int *pnConsumed /* OUT: Number of bytes consumed */
+){
+ sqlite3_tokenizer *pTokenizer = pParse->pTokenizer;
+ sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
+ int rc;
+ sqlite3_tokenizer_cursor *pCursor;
+ Fts3Expr *pRet = 0;
+ int i = 0;
+
+ /* Set variable i to the maximum number of bytes of input to tokenize. */
+ for(i=0; i<n; i++){
+ if( sqlite3_fts3_enable_parentheses && (z[i]=='(' || z[i]==')') ) break;
+ if( z[i]=='"' ) break;
+ }
+
+ *pnConsumed = i;
+ rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, i, &pCursor);
+ if( rc==SQLITE_OK ){
+ const char *zToken;
+ int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0;
+ int nByte; /* total space to allocate */
+
+ rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition);
+ if( rc==SQLITE_OK ){
+ nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken;
+ pRet = (Fts3Expr *)fts3MallocZero(nByte);
+ if( !pRet ){
+ rc = SQLITE_NOMEM;
+ }else{
+ pRet->eType = FTSQUERY_PHRASE;
+ pRet->pPhrase = (Fts3Phrase *)&pRet[1];
+ pRet->pPhrase->nToken = 1;
+ pRet->pPhrase->iColumn = iCol;
+ pRet->pPhrase->aToken[0].n = nToken;
+ pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1];
+ memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken);
+
+ if( iEnd<n && z[iEnd]=='*' ){
+ pRet->pPhrase->aToken[0].isPrefix = 1;
+ iEnd++;
+ }
+
+ while( 1 ){
+ if( !sqlite3_fts3_enable_parentheses
+ && iStart>0 && z[iStart-1]=='-'
+ ){
+ pParse->isNot = 1;
+ iStart--;
+ }else if( pParse->bFts4 && iStart>0 && z[iStart-1]=='^' ){
+ pRet->pPhrase->aToken[0].bFirst = 1;
+ iStart--;
+ }else{
+ break;
+ }
+ }
+
+ }
+ *pnConsumed = iEnd;
+ }else if( i && rc==SQLITE_DONE ){
+ rc = SQLITE_OK;
+ }
+
+ pModule->xClose(pCursor);
+ }
+
+ *ppExpr = pRet;
+ return rc;
+}
+
+
+/*
+** Enlarge a memory allocation. If an out-of-memory allocation occurs,
+** then free the old allocation.
+*/
+static void *fts3ReallocOrFree(void *pOrig, int nNew){
+ void *pRet = sqlite3_realloc(pOrig, nNew);
+ if( !pRet ){
+ sqlite3_free(pOrig);
+ }
+ return pRet;
+}
+
+/*
+** Buffer zInput, length nInput, contains the contents of a quoted string
+** that appeared as part of an fts3 query expression. Neither quote character
+** is included in the buffer. This function attempts to tokenize the entire
+** input buffer and create an Fts3Expr structure of type FTSQUERY_PHRASE
+** containing the results.
+**
+** If successful, SQLITE_OK is returned and *ppExpr set to point at the
+** allocated Fts3Expr structure. Otherwise, either SQLITE_NOMEM (out of memory
+** error) or SQLITE_ERROR (tokenization error) is returned and *ppExpr set
+** to 0.
+*/
+static int getNextString(
+ ParseContext *pParse, /* fts3 query parse context */
+ const char *zInput, int nInput, /* Input string */
+ Fts3Expr **ppExpr /* OUT: expression */
+){
+ sqlite3_tokenizer *pTokenizer = pParse->pTokenizer;
+ sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
+ int rc;
+ Fts3Expr *p = 0;
+ sqlite3_tokenizer_cursor *pCursor = 0;
+ char *zTemp = 0;
+ int nTemp = 0;
+
+ const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
+ int nToken = 0;
+
+ /* The final Fts3Expr data structure, including the Fts3Phrase,
+ ** Fts3PhraseToken structures token buffers are all stored as a single
+ ** allocation so that the expression can be freed with a single call to
+ ** sqlite3_free(). Setting this up requires a two pass approach.
+ **
+ ** The first pass, in the block below, uses a tokenizer cursor to iterate
+ ** through the tokens in the expression. This pass uses fts3ReallocOrFree()
+ ** to assemble data in two dynamic buffers:
+ **
+ ** Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase
+ ** structure, followed by the array of Fts3PhraseToken
+ ** structures. This pass only populates the Fts3PhraseToken array.
+ **
+ ** Buffer zTemp: Contains copies of all tokens.
+ **
+ ** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below,
+ ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase
+ ** structures.
+ */
+ rc = sqlite3Fts3OpenTokenizer(
+ pTokenizer, pParse->iLangid, zInput, nInput, &pCursor);
+ if( rc==SQLITE_OK ){
+ int ii;
+ for(ii=0; rc==SQLITE_OK; ii++){
+ const char *zByte;
+ int nByte = 0, iBegin = 0, iEnd = 0, iPos = 0;
+ rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos);
+ if( rc==SQLITE_OK ){
+ Fts3PhraseToken *pToken;
+
+ p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken));
+ if( !p ) goto no_mem;
+
+ zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte);
+ if( !zTemp ) goto no_mem;
+
+ assert( nToken==ii );
+ pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii];
+ memset(pToken, 0, sizeof(Fts3PhraseToken));
+
+ memcpy(&zTemp[nTemp], zByte, nByte);
+ nTemp += nByte;
+
+ pToken->n = nByte;
+ pToken->isPrefix = (iEnd<nInput && zInput[iEnd]=='*');
+ pToken->bFirst = (iBegin>0 && zInput[iBegin-1]=='^');
+ nToken = ii+1;
+ }
+ }
+
+ pModule->xClose(pCursor);
+ pCursor = 0;
+ }
+
+ if( rc==SQLITE_DONE ){
+ int jj;
+ char *zBuf = 0;
+
+ p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp);
+ if( !p ) goto no_mem;
+ memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p);
+ p->eType = FTSQUERY_PHRASE;
+ p->pPhrase = (Fts3Phrase *)&p[1];
+ p->pPhrase->iColumn = pParse->iDefaultCol;
+ p->pPhrase->nToken = nToken;
+
+ zBuf = (char *)&p->pPhrase->aToken[nToken];
+ if( zTemp ){
+ memcpy(zBuf, zTemp, nTemp);
+ sqlite3_free(zTemp);
+ }else{
+ assert( nTemp==0 );
+ }
+
+ for(jj=0; jj<p->pPhrase->nToken; jj++){
+ p->pPhrase->aToken[jj].z = zBuf;
+ zBuf += p->pPhrase->aToken[jj].n;
+ }
+ rc = SQLITE_OK;
+ }
+
+ *ppExpr = p;
+ return rc;
+no_mem:
+
+ if( pCursor ){
+ pModule->xClose(pCursor);
+ }
+ sqlite3_free(zTemp);
+ sqlite3_free(p);
+ *ppExpr = 0;
+ return SQLITE_NOMEM;
+}
+
+/*
+** The output variable *ppExpr is populated with an allocated Fts3Expr
+** structure, or set to 0 if the end of the input buffer is reached.
+**
+** Returns an SQLite error code. SQLITE_OK if everything works, SQLITE_NOMEM
+** if a malloc failure occurs, or SQLITE_ERROR if a parse error is encountered.
+** If SQLITE_ERROR is returned, pContext is populated with an error message.
+*/
+static int getNextNode(
+ ParseContext *pParse, /* fts3 query parse context */
+ const char *z, int n, /* Input string */
+ Fts3Expr **ppExpr, /* OUT: expression */
+ int *pnConsumed /* OUT: Number of bytes consumed */
+){
+ static const struct Fts3Keyword {
+ char *z; /* Keyword text */
+ unsigned char n; /* Length of the keyword */
+ unsigned char parenOnly; /* Only valid in paren mode */
+ unsigned char eType; /* Keyword code */
+ } aKeyword[] = {
+ { "OR" , 2, 0, FTSQUERY_OR },
+ { "AND", 3, 1, FTSQUERY_AND },
+ { "NOT", 3, 1, FTSQUERY_NOT },
+ { "NEAR", 4, 0, FTSQUERY_NEAR }
+ };
+ int ii;
+ int iCol;
+ int iColLen;
+ int rc;
+ Fts3Expr *pRet = 0;
+
+ const char *zInput = z;
+ int nInput = n;
+
+ pParse->isNot = 0;
+
+ /* Skip over any whitespace before checking for a keyword, an open or
+ ** close bracket, or a quoted string.
+ */
+ while( nInput>0 && fts3isspace(*zInput) ){
+ nInput--;
+ zInput++;
+ }
+ if( nInput==0 ){
+ return SQLITE_DONE;
+ }
+
+ /* See if we are dealing with a keyword. */
+ for(ii=0; ii<(int)(sizeof(aKeyword)/sizeof(struct Fts3Keyword)); ii++){
+ const struct Fts3Keyword *pKey = &aKeyword[ii];
+
+ if( (pKey->parenOnly & ~sqlite3_fts3_enable_parentheses)!=0 ){
+ continue;
+ }
+
+ if( nInput>=pKey->n && 0==memcmp(zInput, pKey->z, pKey->n) ){
+ int nNear = SQLITE_FTS3_DEFAULT_NEAR_PARAM;
+ int nKey = pKey->n;
+ char cNext;
+
+ /* If this is a "NEAR" keyword, check for an explicit nearness. */
+ if( pKey->eType==FTSQUERY_NEAR ){
+ assert( nKey==4 );
+ if( zInput[4]=='/' && zInput[5]>='0' && zInput[5]<='9' ){
+ nNear = 0;
+ for(nKey=5; zInput[nKey]>='0' && zInput[nKey]<='9'; nKey++){
+ nNear = nNear * 10 + (zInput[nKey] - '0');
+ }
+ }
+ }
+
+ /* At this point this is probably a keyword. But for that to be true,
+ ** the next byte must contain either whitespace, an open or close
+ ** parenthesis, a quote character, or EOF.
+ */
+ cNext = zInput[nKey];
+ if( fts3isspace(cNext)
+ || cNext=='"' || cNext=='(' || cNext==')' || cNext==0
+ ){
+ pRet = (Fts3Expr *)fts3MallocZero(sizeof(Fts3Expr));
+ if( !pRet ){
+ return SQLITE_NOMEM;
+ }
+ pRet->eType = pKey->eType;
+ pRet->nNear = nNear;
+ *ppExpr = pRet;
+ *pnConsumed = (int)((zInput - z) + nKey);
+ return SQLITE_OK;
+ }
+
+ /* Turns out that wasn't a keyword after all. This happens if the
+ ** user has supplied a token such as "ORacle". Continue.
+ */
+ }
+ }
+
+ /* See if we are dealing with a quoted phrase. If this is the case, then
+ ** search for the closing quote and pass the whole string to getNextString()
+ ** for processing. This is easy to do, as fts3 has no syntax for escaping
+ ** a quote character embedded in a string.
+ */
+ if( *zInput=='"' ){
+ for(ii=1; ii<nInput && zInput[ii]!='"'; ii++);
+ *pnConsumed = (int)((zInput - z) + ii + 1);
+ if( ii==nInput ){
+ return SQLITE_ERROR;
+ }
+ return getNextString(pParse, &zInput[1], ii-1, ppExpr);
+ }
+
+ if( sqlite3_fts3_enable_parentheses ){
+ if( *zInput=='(' ){
+ int nConsumed = 0;
+ pParse->nNest++;
+ rc = fts3ExprParse(pParse, zInput+1, nInput-1, ppExpr, &nConsumed);
+ if( rc==SQLITE_OK && !*ppExpr ){ rc = SQLITE_DONE; }
+ *pnConsumed = (int)(zInput - z) + 1 + nConsumed;
+ return rc;
+ }else if( *zInput==')' ){
+ pParse->nNest--;
+ *pnConsumed = (int)((zInput - z) + 1);
+ *ppExpr = 0;
+ return SQLITE_DONE;
+ }
+ }
+
+ /* If control flows to this point, this must be a regular token, or
+ ** the end of the input. Read a regular token using the sqlite3_tokenizer
+ ** interface. Before doing so, figure out if there is an explicit
+ ** column specifier for the token.
+ **
+ ** TODO: Strangely, it is not possible to associate a column specifier
+ ** with a quoted phrase, only with a single token. Not sure if this was
+ ** an implementation artifact or an intentional decision when fts3 was
+ ** first implemented. Whichever it was, this module duplicates the
+ ** limitation.
+ */
+ iCol = pParse->iDefaultCol;
+ iColLen = 0;
+ for(ii=0; ii<pParse->nCol; ii++){
+ const char *zStr = pParse->azCol[ii];
+ int nStr = (int)strlen(zStr);
+ if( nInput>nStr && zInput[nStr]==':'
+ && sqlite3_strnicmp(zStr, zInput, nStr)==0
+ ){
+ iCol = ii;
+ iColLen = (int)((zInput - z) + nStr + 1);
+ break;
+ }
+ }
+ rc = getNextToken(pParse, iCol, &z[iColLen], n-iColLen, ppExpr, pnConsumed);
+ *pnConsumed += iColLen;
+ return rc;
+}
+
+/*
+** The argument is an Fts3Expr structure for a binary operator (any type
+** except an FTSQUERY_PHRASE). Return an integer value representing the
+** precedence of the operator. Lower values have a higher precedence (i.e.
+** group more tightly). For example, in the C language, the == operator
+** groups more tightly than ||, and would therefore have a higher precedence.
+**
+** When using the new fts3 query syntax (when SQLITE_ENABLE_FTS3_PARENTHESIS
+** is defined), the order of the operators in precedence from highest to
+** lowest is:
+**
+** NEAR
+** NOT
+** AND (including implicit ANDs)
+** OR
+**
+** Note that when using the old query syntax, the OR operator has a higher
+** precedence than the AND operator.
+*/
+static int opPrecedence(Fts3Expr *p){
+ assert( p->eType!=FTSQUERY_PHRASE );
+ if( sqlite3_fts3_enable_parentheses ){
+ return p->eType;
+ }else if( p->eType==FTSQUERY_NEAR ){
+ return 1;
+ }else if( p->eType==FTSQUERY_OR ){
+ return 2;
+ }
+ assert( p->eType==FTSQUERY_AND );
+ return 3;
+}
+
+/*
+** Argument ppHead contains a pointer to the current head of a query
+** expression tree being parsed. pPrev is the expression node most recently
+** inserted into the tree. This function adds pNew, which is always a binary
+** operator node, into the expression tree based on the relative precedence
+** of pNew and the existing nodes of the tree. This may result in the head
+** of the tree changing, in which case *ppHead is set to the new root node.
+*/
+static void insertBinaryOperator(
+ Fts3Expr **ppHead, /* Pointer to the root node of a tree */
+ Fts3Expr *pPrev, /* Node most recently inserted into the tree */
+ Fts3Expr *pNew /* New binary node to insert into expression tree */
+){
+ Fts3Expr *pSplit = pPrev;
+ while( pSplit->pParent && opPrecedence(pSplit->pParent)<=opPrecedence(pNew) ){
+ pSplit = pSplit->pParent;
+ }
+
+ if( pSplit->pParent ){
+ assert( pSplit->pParent->pRight==pSplit );
+ pSplit->pParent->pRight = pNew;
+ pNew->pParent = pSplit->pParent;
+ }else{
+ *ppHead = pNew;
+ }
+ pNew->pLeft = pSplit;
+ pSplit->pParent = pNew;
+}
+
+/*
+** Parse the fts3 query expression found in buffer z, length n. This function
+** returns either when the end of the buffer is reached or an unmatched
+** closing bracket - ')' - is encountered.
+**
+** If successful, SQLITE_OK is returned, *ppExpr is set to point to the
+** parsed form of the expression and *pnConsumed is set to the number of
+** bytes read from buffer z. Otherwise, *ppExpr is set to 0 and SQLITE_NOMEM
+** (out of memory error) or SQLITE_ERROR (parse error) is returned.
+*/
+static int fts3ExprParse(
+ ParseContext *pParse, /* fts3 query parse context */
+ const char *z, int n, /* Text of MATCH query */
+ Fts3Expr **ppExpr, /* OUT: Parsed query structure */
+ int *pnConsumed /* OUT: Number of bytes consumed */
+){
+ Fts3Expr *pRet = 0;
+ Fts3Expr *pPrev = 0;
+ Fts3Expr *pNotBranch = 0; /* Only used in legacy parse mode */
+ int nIn = n;
+ const char *zIn = z;
+ int rc = SQLITE_OK;
+ int isRequirePhrase = 1;
+
+ while( rc==SQLITE_OK ){
+ Fts3Expr *p = 0;
+ int nByte = 0;
+
+ rc = getNextNode(pParse, zIn, nIn, &p, &nByte);
+ assert( nByte>0 || (rc!=SQLITE_OK && p==0) );
+ if( rc==SQLITE_OK ){
+ if( p ){
+ int isPhrase;
+
+ if( !sqlite3_fts3_enable_parentheses
+ && p->eType==FTSQUERY_PHRASE && pParse->isNot
+ ){
+ /* Create an implicit NOT operator. */
+ Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr));
+ if( !pNot ){
+ sqlite3Fts3ExprFree(p);
+ rc = SQLITE_NOMEM;
+ goto exprparse_out;
+ }
+ pNot->eType = FTSQUERY_NOT;
+ pNot->pRight = p;
+ p->pParent = pNot;
+ if( pNotBranch ){
+ pNot->pLeft = pNotBranch;
+ pNotBranch->pParent = pNot;
+ }
+ pNotBranch = pNot;
+ p = pPrev;
+ }else{
+ int eType = p->eType;
+ isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft);
+
+ /* The isRequirePhrase variable is set to true if a phrase or
+ ** an expression contained in parenthesis is required. If a
+ ** binary operator (AND, OR, NOT or NEAR) is encounted when
+ ** isRequirePhrase is set, this is a syntax error.
+ */
+ if( !isPhrase && isRequirePhrase ){
+ sqlite3Fts3ExprFree(p);
+ rc = SQLITE_ERROR;
+ goto exprparse_out;
+ }
+
+ if( isPhrase && !isRequirePhrase ){
+ /* Insert an implicit AND operator. */
+ Fts3Expr *pAnd;
+ assert( pRet && pPrev );
+ pAnd = fts3MallocZero(sizeof(Fts3Expr));
+ if( !pAnd ){
+ sqlite3Fts3ExprFree(p);
+ rc = SQLITE_NOMEM;
+ goto exprparse_out;
+ }
+ pAnd->eType = FTSQUERY_AND;
+ insertBinaryOperator(&pRet, pPrev, pAnd);
+ pPrev = pAnd;
+ }
+
+ /* This test catches attempts to make either operand of a NEAR
+ ** operator something other than a phrase. For example, either of
+ ** the following:
+ **
+ ** (bracketed expression) NEAR phrase
+ ** phrase NEAR (bracketed expression)
+ **
+ ** Return an error in either case.
+ */
+ if( pPrev && (
+ (eType==FTSQUERY_NEAR && !isPhrase && pPrev->eType!=FTSQUERY_PHRASE)
+ || (eType!=FTSQUERY_PHRASE && isPhrase && pPrev->eType==FTSQUERY_NEAR)
+ )){
+ sqlite3Fts3ExprFree(p);
+ rc = SQLITE_ERROR;
+ goto exprparse_out;
+ }
+
+ if( isPhrase ){
+ if( pRet ){
+ assert( pPrev && pPrev->pLeft && pPrev->pRight==0 );
+ pPrev->pRight = p;
+ p->pParent = pPrev;
+ }else{
+ pRet = p;
+ }
+ }else{
+ insertBinaryOperator(&pRet, pPrev, p);
+ }
+ isRequirePhrase = !isPhrase;
+ }
+ pPrev = p;
+ }
+ assert( nByte>0 );
+ }
+ assert( rc!=SQLITE_OK || (nByte>0 && nByte<=nIn) );
+ nIn -= nByte;
+ zIn += nByte;
+ }
+
+ if( rc==SQLITE_DONE && pRet && isRequirePhrase ){
+ rc = SQLITE_ERROR;
+ }
+
+ if( rc==SQLITE_DONE ){
+ rc = SQLITE_OK;
+ if( !sqlite3_fts3_enable_parentheses && pNotBranch ){
+ if( !pRet ){
+ rc = SQLITE_ERROR;
+ }else{
+ Fts3Expr *pIter = pNotBranch;
+ while( pIter->pLeft ){
+ pIter = pIter->pLeft;
+ }
+ pIter->pLeft = pRet;
+ pRet->pParent = pIter;
+ pRet = pNotBranch;
+ }
+ }
+ }
+ *pnConsumed = n - nIn;
+
+exprparse_out:
+ if( rc!=SQLITE_OK ){
+ sqlite3Fts3ExprFree(pRet);
+ sqlite3Fts3ExprFree(pNotBranch);
+ pRet = 0;
+ }
+ *ppExpr = pRet;
+ return rc;
+}
+
+/*
+** Return SQLITE_ERROR if the maximum depth of the expression tree passed
+** as the only argument is more than nMaxDepth.
+*/
+static int fts3ExprCheckDepth(Fts3Expr *p, int nMaxDepth){
+ int rc = SQLITE_OK;
+ if( p ){
+ if( nMaxDepth<0 ){
+ rc = SQLITE_TOOBIG;
+ }else{
+ rc = fts3ExprCheckDepth(p->pLeft, nMaxDepth-1);
+ if( rc==SQLITE_OK ){
+ rc = fts3ExprCheckDepth(p->pRight, nMaxDepth-1);
+ }
+ }
+ }
+ return rc;
+}
+
+/*
+** This function attempts to transform the expression tree at (*pp) to
+** an equivalent but more balanced form. The tree is modified in place.
+** If successful, SQLITE_OK is returned and (*pp) set to point to the
+** new root expression node.
+**
+** nMaxDepth is the maximum allowable depth of the balanced sub-tree.
+**
+** Otherwise, if an error occurs, an SQLite error code is returned and
+** expression (*pp) freed.
+*/
+static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){
+ int rc = SQLITE_OK; /* Return code */
+ Fts3Expr *pRoot = *pp; /* Initial root node */
+ Fts3Expr *pFree = 0; /* List of free nodes. Linked by pParent. */
+ int eType = pRoot->eType; /* Type of node in this tree */
+
+ if( nMaxDepth==0 ){
+ rc = SQLITE_ERROR;
+ }
+
+ if( rc==SQLITE_OK ){
+ if( (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){
+ Fts3Expr **apLeaf;
+ apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth);
+ if( 0==apLeaf ){
+ rc = SQLITE_NOMEM;
+ }else{
+ memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth);
+ }
+
+ if( rc==SQLITE_OK ){
+ int i;
+ Fts3Expr *p;
+
+ /* Set $p to point to the left-most leaf in the tree of eType nodes. */
+ for(p=pRoot; p->eType==eType; p=p->pLeft){
+ assert( p->pParent==0 || p->pParent->pLeft==p );
+ assert( p->pLeft && p->pRight );
+ }
+
+ /* This loop runs once for each leaf in the tree of eType nodes. */
+ while( 1 ){
+ int iLvl;
+ Fts3Expr *pParent = p->pParent; /* Current parent of p */
+
+ assert( pParent==0 || pParent->pLeft==p );
+ p->pParent = 0;
+ if( pParent ){
+ pParent->pLeft = 0;
+ }else{
+ pRoot = 0;
+ }
+ rc = fts3ExprBalance(&p, nMaxDepth-1);
+ if( rc!=SQLITE_OK ) break;
+
+ for(iLvl=0; p && iLvl<nMaxDepth; iLvl++){
+ if( apLeaf[iLvl]==0 ){
+ apLeaf[iLvl] = p;
+ p = 0;
+ }else{
+ assert( pFree );
+ pFree->pLeft = apLeaf[iLvl];
+ pFree->pRight = p;
+ pFree->pLeft->pParent = pFree;
+ pFree->pRight->pParent = pFree;
+
+ p = pFree;
+ pFree = pFree->pParent;
+ p->pParent = 0;
+ apLeaf[iLvl] = 0;
+ }
+ }
+ if( p ){
+ sqlite3Fts3ExprFree(p);
+ rc = SQLITE_TOOBIG;
+ break;
+ }
+
+ /* If that was the last leaf node, break out of the loop */
+ if( pParent==0 ) break;
+
+ /* Set $p to point to the next leaf in the tree of eType nodes */
+ for(p=pParent->pRight; p->eType==eType; p=p->pLeft);
+
+ /* Remove pParent from the original tree. */
+ assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent );
+ pParent->pRight->pParent = pParent->pParent;
+ if( pParent->pParent ){
+ pParent->pParent->pLeft = pParent->pRight;
+ }else{
+ assert( pParent==pRoot );
+ pRoot = pParent->pRight;
+ }
+
+ /* Link pParent into the free node list. It will be used as an
+ ** internal node of the new tree. */
+ pParent->pParent = pFree;
+ pFree = pParent;
+ }
+
+ if( rc==SQLITE_OK ){
+ p = 0;
+ for(i=0; i<nMaxDepth; i++){
+ if( apLeaf[i] ){
+ if( p==0 ){
+ p = apLeaf[i];
+ p->pParent = 0;
+ }else{
+ assert( pFree!=0 );
+ pFree->pRight = p;
+ pFree->pLeft = apLeaf[i];
+ pFree->pLeft->pParent = pFree;
+ pFree->pRight->pParent = pFree;
+
+ p = pFree;
+ pFree = pFree->pParent;
+ p->pParent = 0;
+ }
+ }
+ }
+ pRoot = p;
+ }else{
+ /* An error occurred. Delete the contents of the apLeaf[] array
+ ** and pFree list. Everything else is cleaned up by the call to
+ ** sqlite3Fts3ExprFree(pRoot) below. */
+ Fts3Expr *pDel;
+ for(i=0; i<nMaxDepth; i++){
+ sqlite3Fts3ExprFree(apLeaf[i]);
+ }
+ while( (pDel=pFree)!=0 ){
+ pFree = pDel->pParent;
+ sqlite3_free(pDel);
+ }
+ }
+
+ assert( pFree==0 );
+ sqlite3_free( apLeaf );
+ }
+ }else if( eType==FTSQUERY_NOT ){
+ Fts3Expr *pLeft = pRoot->pLeft;
+ Fts3Expr *pRight = pRoot->pRight;
+
+ pRoot->pLeft = 0;
+ pRoot->pRight = 0;
+ pLeft->pParent = 0;
+ pRight->pParent = 0;
+
+ rc = fts3ExprBalance(&pLeft, nMaxDepth-1);
+ if( rc==SQLITE_OK ){
+ rc = fts3ExprBalance(&pRight, nMaxDepth-1);
+ }
+
+ if( rc!=SQLITE_OK ){
+ sqlite3Fts3ExprFree(pRight);
+ sqlite3Fts3ExprFree(pLeft);
+ }else{
+ assert( pLeft && pRight );
+ pRoot->pLeft = pLeft;
+ pLeft->pParent = pRoot;
+ pRoot->pRight = pRight;
+ pRight->pParent = pRoot;
+ }
+ }
+ }
+
+ if( rc!=SQLITE_OK ){
+ sqlite3Fts3ExprFree(pRoot);
+ pRoot = 0;
+ }
+ *pp = pRoot;
+ return rc;
+}
+
+/*
+** This function is similar to sqlite3Fts3ExprParse(), with the following
+** differences:
+**
+** 1. It does not do expression rebalancing.
+** 2. It does not check that the expression does not exceed the
+** maximum allowable depth.
+** 3. Even if it fails, *ppExpr may still be set to point to an
+** expression tree. It should be deleted using sqlite3Fts3ExprFree()
+** in this case.
+*/
+static int fts3ExprParseUnbalanced(
+ sqlite3_tokenizer *pTokenizer, /* Tokenizer module */
+ int iLangid, /* Language id for tokenizer */
+ char **azCol, /* Array of column names for fts3 table */
+ int bFts4, /* True to allow FTS4-only syntax */
+ int nCol, /* Number of entries in azCol[] */
+ int iDefaultCol, /* Default column to query */
+ const char *z, int n, /* Text of MATCH query */
+ Fts3Expr **ppExpr /* OUT: Parsed query structure */
+){
+ int nParsed;
+ int rc;
+ ParseContext sParse;
+
+ memset(&sParse, 0, sizeof(ParseContext));
+ sParse.pTokenizer = pTokenizer;
+ sParse.iLangid = iLangid;
+ sParse.azCol = (const char **)azCol;
+ sParse.nCol = nCol;
+ sParse.iDefaultCol = iDefaultCol;
+ sParse.bFts4 = bFts4;
+ if( z==0 ){
+ *ppExpr = 0;
+ return SQLITE_OK;
+ }
+ if( n<0 ){
+ n = (int)strlen(z);
+ }
+ rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed);
+ assert( rc==SQLITE_OK || *ppExpr==0 );
+
+ /* Check for mismatched parenthesis */
+ if( rc==SQLITE_OK && sParse.nNest ){
+ rc = SQLITE_ERROR;
+ }
+
+ return rc;
+}
+
+/*
+** Parameters z and n contain a pointer to and length of a buffer containing
+** an fts3 query expression, respectively. This function attempts to parse the
+** query expression and create a tree of Fts3Expr structures representing the
+** parsed expression. If successful, *ppExpr is set to point to the head
+** of the parsed expression tree and SQLITE_OK is returned. If an error
+** occurs, either SQLITE_NOMEM (out-of-memory error) or SQLITE_ERROR (parse
+** error) is returned and *ppExpr is set to 0.
+**
+** If parameter n is a negative number, then z is assumed to point to a
+** nul-terminated string and the length is determined using strlen().
+**
+** The first parameter, pTokenizer, is passed the fts3 tokenizer module to
+** use to normalize query tokens while parsing the expression. The azCol[]
+** array, which is assumed to contain nCol entries, should contain the names
+** of each column in the target fts3 table, in order from left to right.
+** Column names must be nul-terminated strings.
+**
+** The iDefaultCol parameter should be passed the index of the table column
+** that appears on the left-hand-side of the MATCH operator (the default
+** column to match against for tokens for which a column name is not explicitly
+** specified as part of the query string), or -1 if tokens may by default
+** match any table column.
+*/
+SQLITE_PRIVATE int sqlite3Fts3ExprParse(
+ sqlite3_tokenizer *pTokenizer, /* Tokenizer module */
+ int iLangid, /* Language id for tokenizer */
+ char **azCol, /* Array of column names for fts3 table */
+ int bFts4, /* True to allow FTS4-only syntax */
+ int nCol, /* Number of entries in azCol[] */
+ int iDefaultCol, /* Default column to query */
+ const char *z, int n, /* Text of MATCH query */
+ Fts3Expr **ppExpr, /* OUT: Parsed query structure */
+ char **pzErr /* OUT: Error message (sqlite3_malloc) */
+){
+ int rc = fts3ExprParseUnbalanced(
+ pTokenizer, iLangid, azCol, bFts4, nCol, iDefaultCol, z, n, ppExpr
+ );
+
+ /* Rebalance the expression. And check that its depth does not exceed
+ ** SQLITE_FTS3_MAX_EXPR_DEPTH. */
+ if( rc==SQLITE_OK && *ppExpr ){
+ rc = fts3ExprBalance(ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH);
+ if( rc==SQLITE_OK ){
+ rc = fts3ExprCheckDepth(*ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH);
+ }
+ }
+
+ if( rc!=SQLITE_OK ){
+ sqlite3Fts3ExprFree(*ppExpr);
+ *ppExpr = 0;
+ if( rc==SQLITE_TOOBIG ){
+ sqlite3Fts3ErrMsg(pzErr,
+ "FTS expression tree is too large (maximum depth %d)",
+ SQLITE_FTS3_MAX_EXPR_DEPTH
+ );
+ rc = SQLITE_ERROR;
+ }else if( rc==SQLITE_ERROR ){
+ sqlite3Fts3ErrMsg(pzErr, "malformed MATCH expression: [%s]", z);
+ }
+ }
+
+ return rc;
+}
+
+/*
+** Free a single node of an expression tree.
+*/
+static void fts3FreeExprNode(Fts3Expr *p){
+ assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 );
+ sqlite3Fts3EvalPhraseCleanup(p->pPhrase);
+ sqlite3_free(p->aMI);
+ sqlite3_free(p);
+}
+
+/*
+** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse().
+**
+** This function would be simpler if it recursively called itself. But
+** that would mean passing a sufficiently large expression to ExprParse()
+** could cause a stack overflow.
+*/
+SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *pDel){
+ Fts3Expr *p;
+ assert( pDel==0 || pDel->pParent==0 );
+ for(p=pDel; p && (p->pLeft||p->pRight); p=(p->pLeft ? p->pLeft : p->pRight)){
+ assert( p->pParent==0 || p==p->pParent->pRight || p==p->pParent->pLeft );
+ }
+ while( p ){
+ Fts3Expr *pParent = p->pParent;
+ fts3FreeExprNode(p);
+ if( pParent && p==pParent->pLeft && pParent->pRight ){
+ p = pParent->pRight;
+ while( p && (p->pLeft || p->pRight) ){
+ assert( p==p->pParent->pRight || p==p->pParent->pLeft );
+ p = (p->pLeft ? p->pLeft : p->pRight);
+ }
+ }else{
+ p = pParent;
+ }
+ }
+}
+
+/****************************************************************************
+*****************************************************************************
+** Everything after this point is just test code.
+*/
+
+#ifdef SQLITE_TEST
+
+/* #include <stdio.h> */
+
+/*
+** Function to query the hash-table of tokenizers (see README.tokenizers).
+*/
+static int queryTestTokenizer(
+ sqlite3 *db,
+ const char *zName,
+ const sqlite3_tokenizer_module **pp
+){
+ int rc;
+ sqlite3_stmt *pStmt;
+ const char zSql[] = "SELECT fts3_tokenizer(?)";
+
+ *pp = 0;
+ rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+
+ sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){
+ if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
+ memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
+ }
+ }
+
+ return sqlite3_finalize(pStmt);
+}
+
+/*
+** Return a pointer to a buffer containing a text representation of the
+** expression passed as the first argument. The buffer is obtained from
+** sqlite3_malloc(). It is the responsibility of the caller to use
+** sqlite3_free() to release the memory. If an OOM condition is encountered,
+** NULL is returned.
+**
+** If the second argument is not NULL, then its contents are prepended to
+** the returned expression text and then freed using sqlite3_free().
+*/
+static char *exprToString(Fts3Expr *pExpr, char *zBuf){
+ if( pExpr==0 ){
+ return sqlite3_mprintf("");
+ }
+ switch( pExpr->eType ){
+ case FTSQUERY_PHRASE: {
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
+ int i;
+ zBuf = sqlite3_mprintf(
+ "%zPHRASE %d 0", zBuf, pPhrase->iColumn);
+ for(i=0; zBuf && i<pPhrase->nToken; i++){
+ zBuf = sqlite3_mprintf("%z %.*s%s", zBuf,
+ pPhrase->aToken[i].n, pPhrase->aToken[i].z,
+ (pPhrase->aToken[i].isPrefix?"+":"")
+ );
+ }
+ return zBuf;
+ }
+
+ case FTSQUERY_NEAR:
+ zBuf = sqlite3_mprintf("%zNEAR/%d ", zBuf, pExpr->nNear);
+ break;
+ case FTSQUERY_NOT:
+ zBuf = sqlite3_mprintf("%zNOT ", zBuf);
+ break;
+ case FTSQUERY_AND:
+ zBuf = sqlite3_mprintf("%zAND ", zBuf);
+ break;
+ case FTSQUERY_OR:
+ zBuf = sqlite3_mprintf("%zOR ", zBuf);
+ break;
+ }
+
+ if( zBuf ) zBuf = sqlite3_mprintf("%z{", zBuf);
+ if( zBuf ) zBuf = exprToString(pExpr->pLeft, zBuf);
+ if( zBuf ) zBuf = sqlite3_mprintf("%z} {", zBuf);
+
+ if( zBuf ) zBuf = exprToString(pExpr->pRight, zBuf);
+ if( zBuf ) zBuf = sqlite3_mprintf("%z}", zBuf);
+
+ return zBuf;
+}
+
+/*
+** This is the implementation of a scalar SQL function used to test the
+** expression parser. It should be called as follows:
+**
+** fts3_exprtest(<tokenizer>, <expr>, <column 1>, ...);
+**
+** The first argument, <tokenizer>, is the name of the fts3 tokenizer used
+** to parse the query expression (see README.tokenizers). The second argument
+** is the query expression to parse. Each subsequent argument is the name
+** of a column of the fts3 table that the query expression may refer to.
+** For example:
+**
+** SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2');
+*/
+static void fts3ExprTest(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ sqlite3_tokenizer_module const *pModule = 0;
+ sqlite3_tokenizer *pTokenizer = 0;
+ int rc;
+ char **azCol = 0;
+ const char *zExpr;
+ int nExpr;
+ int nCol;
+ int ii;
+ Fts3Expr *pExpr;
+ char *zBuf = 0;
+ sqlite3 *db = sqlite3_context_db_handle(context);
+
+ if( argc<3 ){
+ sqlite3_result_error(context,
+ "Usage: fts3_exprtest(tokenizer, expr, col1, ...", -1
+ );
+ return;
+ }
+
+ rc = queryTestTokenizer(db,
+ (const char *)sqlite3_value_text(argv[0]), &pModule);
+ if( rc==SQLITE_NOMEM ){
+ sqlite3_result_error_nomem(context);
+ goto exprtest_out;
+ }else if( !pModule ){
+ sqlite3_result_error(context, "No such tokenizer module", -1);
+ goto exprtest_out;
+ }
+
+ rc = pModule->xCreate(0, 0, &pTokenizer);
+ assert( rc==SQLITE_NOMEM || rc==SQLITE_OK );
+ if( rc==SQLITE_NOMEM ){
+ sqlite3_result_error_nomem(context);
+ goto exprtest_out;
+ }
+ pTokenizer->pModule = pModule;
+
+ zExpr = (const char *)sqlite3_value_text(argv[1]);
+ nExpr = sqlite3_value_bytes(argv[1]);
+ nCol = argc-2;
+ azCol = (char **)sqlite3_malloc(nCol*sizeof(char *));
+ if( !azCol ){
+ sqlite3_result_error_nomem(context);
+ goto exprtest_out;
+ }
+ for(ii=0; ii<nCol; ii++){
+ azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]);
+ }
+
+ if( sqlite3_user_data(context) ){
+ char *zDummy = 0;
+ rc = sqlite3Fts3ExprParse(
+ pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr, &zDummy
+ );
+ assert( rc==SQLITE_OK || pExpr==0 );
+ sqlite3_free(zDummy);
+ }else{
+ rc = fts3ExprParseUnbalanced(
+ pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr
+ );
+ }
+
+ if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){
+ sqlite3Fts3ExprFree(pExpr);
+ sqlite3_result_error(context, "Error parsing expression", -1);
+ }else if( rc==SQLITE_NOMEM || !(zBuf = exprToString(pExpr, 0)) ){
+ sqlite3_result_error_nomem(context);
+ }else{
+ sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
+ sqlite3_free(zBuf);
+ }
+
+ sqlite3Fts3ExprFree(pExpr);
+
+exprtest_out:
+ if( pModule && pTokenizer ){
+ rc = pModule->xDestroy(pTokenizer);
+ }
+ sqlite3_free(azCol);
+}
+
+/*
+** Register the query expression parser test function fts3_exprtest()
+** with database connection db.
+*/
+SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3* db){
+ int rc = sqlite3_create_function(
+ db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0
+ );
+ if( rc==SQLITE_OK ){
+ rc = sqlite3_create_function(db, "fts3_exprtest_rebalance",
+ -1, SQLITE_UTF8, (void *)1, fts3ExprTest, 0, 0
+ );
+ }
+ return rc;
+}
+
+#endif
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
+
+/************** End of fts3_expr.c *******************************************/
+/************** Begin file fts3_hash.c ***************************************/
+/*
+** 2001 September 22
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This is the implementation of generic hash-tables used in SQLite.
+** We've modified it slightly to serve as a standalone hash table
+** implementation for the full-text indexing module.
+*/
+
+/*
+** The code in this file is only compiled if:
+**
+** * The FTS3 module is being built as an extension
+** (in which case SQLITE_CORE is not defined), or
+**
+** * The FTS3 module is being built into the core of
+** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
+*/
+/* #include "fts3Int.h" */
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+/* #include <assert.h> */
+/* #include <stdlib.h> */
+/* #include <string.h> */
+
+/* #include "fts3_hash.h" */
+
+/*
+** Malloc and Free functions
+*/
+static void *fts3HashMalloc(int n){
+ void *p = sqlite3_malloc(n);
+ if( p ){
+ memset(p, 0, n);
+ }
+ return p;
+}
+static void fts3HashFree(void *p){
+ sqlite3_free(p);
+}
+
+/* Turn bulk memory into a hash table object by initializing the
+** fields of the Hash structure.
+**
+** "pNew" is a pointer to the hash table that is to be initialized.
+** keyClass is one of the constants
+** FTS3_HASH_BINARY or FTS3_HASH_STRING. The value of keyClass
+** determines what kind of key the hash table will use. "copyKey" is
+** true if the hash table should make its own private copy of keys and
+** false if it should just use the supplied pointer.
+*/
+SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey){
+ assert( pNew!=0 );
+ assert( keyClass>=FTS3_HASH_STRING && keyClass<=FTS3_HASH_BINARY );
+ pNew->keyClass = keyClass;
+ pNew->copyKey = copyKey;
+ pNew->first = 0;
+ pNew->count = 0;
+ pNew->htsize = 0;
+ pNew->ht = 0;
+}
+
+/* Remove all entries from a hash table. Reclaim all memory.
+** Call this routine to delete a hash table or to reset a hash table
+** to the empty state.
+*/
+SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash *pH){
+ Fts3HashElem *elem; /* For looping over all elements of the table */
+
+ assert( pH!=0 );
+ elem = pH->first;
+ pH->first = 0;
+ fts3HashFree(pH->ht);
+ pH->ht = 0;
+ pH->htsize = 0;
+ while( elem ){
+ Fts3HashElem *next_elem = elem->next;
+ if( pH->copyKey && elem->pKey ){
+ fts3HashFree(elem->pKey);
+ }
+ fts3HashFree(elem);
+ elem = next_elem;
+ }
+ pH->count = 0;
+}
+
+/*
+** Hash and comparison functions when the mode is FTS3_HASH_STRING
+*/
+static int fts3StrHash(const void *pKey, int nKey){
+ const char *z = (const char *)pKey;
+ unsigned h = 0;
+ if( nKey<=0 ) nKey = (int) strlen(z);
+ while( nKey > 0 ){
+ h = (h<<3) ^ h ^ *z++;
+ nKey--;
+ }
+ return (int)(h & 0x7fffffff);
+}
+static int fts3StrCompare(const void *pKey1, int n1, const void *pKey2, int n2){
+ if( n1!=n2 ) return 1;
+ return strncmp((const char*)pKey1,(const char*)pKey2,n1);
+}
+
+/*
+** Hash and comparison functions when the mode is FTS3_HASH_BINARY
+*/
+static int fts3BinHash(const void *pKey, int nKey){
+ int h = 0;
+ const char *z = (const char *)pKey;
+ while( nKey-- > 0 ){
+ h = (h<<3) ^ h ^ *(z++);
+ }
+ return h & 0x7fffffff;
+}
+static int fts3BinCompare(const void *pKey1, int n1, const void *pKey2, int n2){
+ if( n1!=n2 ) return 1;
+ return memcmp(pKey1,pKey2,n1);
+}
+
+/*
+** Return a pointer to the appropriate hash function given the key class.
+**
+** The C syntax in this function definition may be unfamilar to some
+** programmers, so we provide the following additional explanation:
+**
+** The name of the function is "ftsHashFunction". The function takes a
+** single parameter "keyClass". The return value of ftsHashFunction()
+** is a pointer to another function. Specifically, the return value
+** of ftsHashFunction() is a pointer to a function that takes two parameters
+** with types "const void*" and "int" and returns an "int".
+*/
+static int (*ftsHashFunction(int keyClass))(const void*,int){
+ if( keyClass==FTS3_HASH_STRING ){
+ return &fts3StrHash;
+ }else{
+ assert( keyClass==FTS3_HASH_BINARY );
+ return &fts3BinHash;
+ }
+}
+
+/*
+** Return a pointer to the appropriate hash function given the key class.
+**
+** For help in interpreted the obscure C code in the function definition,
+** see the header comment on the previous function.
+*/
+static int (*ftsCompareFunction(int keyClass))(const void*,int,const void*,int){
+ if( keyClass==FTS3_HASH_STRING ){
+ return &fts3StrCompare;
+ }else{
+ assert( keyClass==FTS3_HASH_BINARY );
+ return &fts3BinCompare;
+ }
+}
+
+/* Link an element into the hash table
+*/
+static void fts3HashInsertElement(
+ Fts3Hash *pH, /* The complete hash table */
+ struct _fts3ht *pEntry, /* The entry into which pNew is inserted */
+ Fts3HashElem *pNew /* The element to be inserted */
+){
+ Fts3HashElem *pHead; /* First element already in pEntry */
+ pHead = pEntry->chain;
+ if( pHead ){
+ pNew->next = pHead;
+ pNew->prev = pHead->prev;
+ if( pHead->prev ){ pHead->prev->next = pNew; }
+ else { pH->first = pNew; }
+ pHead->prev = pNew;
+ }else{
+ pNew->next = pH->first;
+ if( pH->first ){ pH->first->prev = pNew; }
+ pNew->prev = 0;
+ pH->first = pNew;
+ }
+ pEntry->count++;
+ pEntry->chain = pNew;
+}
+
+
+/* Resize the hash table so that it cantains "new_size" buckets.
+** "new_size" must be a power of 2. The hash table might fail
+** to resize if sqliteMalloc() fails.
+**
+** Return non-zero if a memory allocation error occurs.
+*/
+static int fts3Rehash(Fts3Hash *pH, int new_size){
+ struct _fts3ht *new_ht; /* The new hash table */
+ Fts3HashElem *elem, *next_elem; /* For looping over existing elements */
+ int (*xHash)(const void*,int); /* The hash function */
+
+ assert( (new_size & (new_size-1))==0 );
+ new_ht = (struct _fts3ht *)fts3HashMalloc( new_size*sizeof(struct _fts3ht) );
+ if( new_ht==0 ) return 1;
+ fts3HashFree(pH->ht);
+ pH->ht = new_ht;
+ pH->htsize = new_size;
+ xHash = ftsHashFunction(pH->keyClass);
+ for(elem=pH->first, pH->first=0; elem; elem = next_elem){
+ int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1);
+ next_elem = elem->next;
+ fts3HashInsertElement(pH, &new_ht[h], elem);
+ }
+ return 0;
+}
+
+/* This function (for internal use only) locates an element in an
+** hash table that matches the given key. The hash for this key has
+** already been computed and is passed as the 4th parameter.
+*/
+static Fts3HashElem *fts3FindElementByHash(
+ const Fts3Hash *pH, /* The pH to be searched */
+ const void *pKey, /* The key we are searching for */
+ int nKey,
+ int h /* The hash for this key. */
+){
+ Fts3HashElem *elem; /* Used to loop thru the element list */
+ int count; /* Number of elements left to test */
+ int (*xCompare)(const void*,int,const void*,int); /* comparison function */
+
+ if( pH->ht ){
+ struct _fts3ht *pEntry = &pH->ht[h];
+ elem = pEntry->chain;
+ count = pEntry->count;
+ xCompare = ftsCompareFunction(pH->keyClass);
+ while( count-- && elem ){
+ if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){
+ return elem;
+ }
+ elem = elem->next;
+ }
+ }
+ return 0;
+}
+
+/* Remove a single entry from the hash table given a pointer to that
+** element and a hash on the element's key.
+*/
+static void fts3RemoveElementByHash(
+ Fts3Hash *pH, /* The pH containing "elem" */
+ Fts3HashElem* elem, /* The element to be removed from the pH */
+ int h /* Hash value for the element */
+){
+ struct _fts3ht *pEntry;
+ if( elem->prev ){
+ elem->prev->next = elem->next;
+ }else{
+ pH->first = elem->next;
+ }
+ if( elem->next ){
+ elem->next->prev = elem->prev;
+ }
+ pEntry = &pH->ht[h];
+ if( pEntry->chain==elem ){
+ pEntry->chain = elem->next;
+ }
+ pEntry->count--;
+ if( pEntry->count<=0 ){
+ pEntry->chain = 0;
+ }
+ if( pH->copyKey && elem->pKey ){
+ fts3HashFree(elem->pKey);
+ }
+ fts3HashFree( elem );
+ pH->count--;
+ if( pH->count<=0 ){
+ assert( pH->first==0 );
+ assert( pH->count==0 );
+ fts3HashClear(pH);
+ }
+}
+
+SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem(
+ const Fts3Hash *pH,
+ const void *pKey,
+ int nKey
+){
+ int h; /* A hash on key */
+ int (*xHash)(const void*,int); /* The hash function */
+
+ if( pH==0 || pH->ht==0 ) return 0;
+ xHash = ftsHashFunction(pH->keyClass);
+ assert( xHash!=0 );
+ h = (*xHash)(pKey,nKey);
+ assert( (pH->htsize & (pH->htsize-1))==0 );
+ return fts3FindElementByHash(pH,pKey,nKey, h & (pH->htsize-1));
+}
+
+/*
+** Attempt to locate an element of the hash table pH with a key
+** that matches pKey,nKey. Return the data for this element if it is
+** found, or NULL if there is no match.
+*/
+SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash *pH, const void *pKey, int nKey){
+ Fts3HashElem *pElem; /* The element that matches key (if any) */
+
+ pElem = sqlite3Fts3HashFindElem(pH, pKey, nKey);
+ return pElem ? pElem->data : 0;
+}
+
+/* Insert an element into the hash table pH. The key is pKey,nKey
+** and the data is "data".
+**
+** If no element exists with a matching key, then a new
+** element is created. A copy of the key is made if the copyKey
+** flag is set. NULL is returned.
+**
+** If another element already exists with the same key, then the
+** new data replaces the old data and the old data is returned.
+** The key is not copied in this instance. If a malloc fails, then
+** the new data is returned and the hash table is unchanged.
+**
+** If the "data" parameter to this function is NULL, then the
+** element corresponding to "key" is removed from the hash table.
+*/
+SQLITE_PRIVATE void *sqlite3Fts3HashInsert(
+ Fts3Hash *pH, /* The hash table to insert into */
+ const void *pKey, /* The key */
+ int nKey, /* Number of bytes in the key */
+ void *data /* The data */
+){
+ int hraw; /* Raw hash value of the key */
+ int h; /* the hash of the key modulo hash table size */
+ Fts3HashElem *elem; /* Used to loop thru the element list */
+ Fts3HashElem *new_elem; /* New element added to the pH */
+ int (*xHash)(const void*,int); /* The hash function */
+
+ assert( pH!=0 );
+ xHash = ftsHashFunction(pH->keyClass);
+ assert( xHash!=0 );
+ hraw = (*xHash)(pKey, nKey);
+ assert( (pH->htsize & (pH->htsize-1))==0 );
+ h = hraw & (pH->htsize-1);
+ elem = fts3FindElementByHash(pH,pKey,nKey,h);
+ if( elem ){
+ void *old_data = elem->data;
+ if( data==0 ){
+ fts3RemoveElementByHash(pH,elem,h);
+ }else{
+ elem->data = data;
+ }
+ return old_data;
+ }
+ if( data==0 ) return 0;
+ if( (pH->htsize==0 && fts3Rehash(pH,8))
+ || (pH->count>=pH->htsize && fts3Rehash(pH, pH->htsize*2))
+ ){
+ pH->count = 0;
+ return data;
+ }
+ assert( pH->htsize>0 );
+ new_elem = (Fts3HashElem*)fts3HashMalloc( sizeof(Fts3HashElem) );
+ if( new_elem==0 ) return data;
+ if( pH->copyKey && pKey!=0 ){
+ new_elem->pKey = fts3HashMalloc( nKey );
+ if( new_elem->pKey==0 ){
+ fts3HashFree(new_elem);
+ return data;
+ }
+ memcpy((void*)new_elem->pKey, pKey, nKey);
+ }else{
+ new_elem->pKey = (void*)pKey;
+ }
+ new_elem->nKey = nKey;
+ pH->count++;
+ assert( pH->htsize>0 );
+ assert( (pH->htsize & (pH->htsize-1))==0 );
+ h = hraw & (pH->htsize-1);
+ fts3HashInsertElement(pH, &pH->ht[h], new_elem);
+ new_elem->data = data;
+ return 0;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
+
+/************** End of fts3_hash.c *******************************************/
+/************** Begin file fts3_porter.c *************************************/
+/*
+** 2006 September 30
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+*************************************************************************
+** Implementation of the full-text-search tokenizer that implements
+** a Porter stemmer.
+*/
+
+/*
+** The code in this file is only compiled if:
+**
+** * The FTS3 module is being built as an extension
+** (in which case SQLITE_CORE is not defined), or
+**
+** * The FTS3 module is being built into the core of
+** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
+*/
+/* #include "fts3Int.h" */
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+/* #include <assert.h> */
+/* #include <stdlib.h> */
+/* #include <stdio.h> */
+/* #include <string.h> */
+
+/* #include "fts3_tokenizer.h" */
+
+/*
+** Class derived from sqlite3_tokenizer
+*/
+typedef struct porter_tokenizer {
+ sqlite3_tokenizer base; /* Base class */
+} porter_tokenizer;
+
+/*
+** Class derived from sqlite3_tokenizer_cursor
+*/
+typedef struct porter_tokenizer_cursor {
+ sqlite3_tokenizer_cursor base;
+ const char *zInput; /* input we are tokenizing */
+ int nInput; /* size of the input */
+ int iOffset; /* current position in zInput */
+ int iToken; /* index of next token to be returned */
+ char *zToken; /* storage for current token */
+ int nAllocated; /* space allocated to zToken buffer */
+} porter_tokenizer_cursor;
+
+
+/*
+** Create a new tokenizer instance.
+*/
+static int porterCreate(
+ int argc, const char * const *argv,
+ sqlite3_tokenizer **ppTokenizer
+){
+ porter_tokenizer *t;
+
+ UNUSED_PARAMETER(argc);
+ UNUSED_PARAMETER(argv);
+
+ t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t));
+ if( t==NULL ) return SQLITE_NOMEM;
+ memset(t, 0, sizeof(*t));
+ *ppTokenizer = &t->base;
+ return SQLITE_OK;
+}
+
+/*
+** Destroy a tokenizer
+*/
+static int porterDestroy(sqlite3_tokenizer *pTokenizer){
+ sqlite3_free(pTokenizer);
+ return SQLITE_OK;
+}
+
+/*
+** Prepare to begin tokenizing a particular string. The input
+** string to be tokenized is zInput[0..nInput-1]. A cursor
+** used to incrementally tokenize this string is returned in
+** *ppCursor.
+*/
+static int porterOpen(
+ sqlite3_tokenizer *pTokenizer, /* The tokenizer */
+ const char *zInput, int nInput, /* String to be tokenized */
+ sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
+){
+ porter_tokenizer_cursor *c;
+
+ UNUSED_PARAMETER(pTokenizer);
+
+ c = (porter_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
+ if( c==NULL ) return SQLITE_NOMEM;
+
+ c->zInput = zInput;
+ if( zInput==0 ){
+ c->nInput = 0;
+ }else if( nInput<0 ){
+ c->nInput = (int)strlen(zInput);
+ }else{
+ c->nInput = nInput;
+ }
+ c->iOffset = 0; /* start tokenizing at the beginning */
+ c->iToken = 0;
+ c->zToken = NULL; /* no space allocated, yet. */
+ c->nAllocated = 0;
+
+ *ppCursor = &c->base;
+ return SQLITE_OK;
+}
+
+/*
+** Close a tokenization cursor previously opened by a call to
+** porterOpen() above.
+*/
+static int porterClose(sqlite3_tokenizer_cursor *pCursor){
+ porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
+ sqlite3_free(c->zToken);
+ sqlite3_free(c);
+ return SQLITE_OK;
+}
+/*
+** Vowel or consonant
+*/
+static const char cType[] = {
+ 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0,
+ 1, 1, 1, 2, 1
+};
+
+/*
+** isConsonant() and isVowel() determine if their first character in
+** the string they point to is a consonant or a vowel, according
+** to Porter ruls.
+**
+** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'.
+** 'Y' is a consonant unless it follows another consonant,
+** in which case it is a vowel.
+**
+** In these routine, the letters are in reverse order. So the 'y' rule
+** is that 'y' is a consonant unless it is followed by another
+** consonent.
+*/
+static int isVowel(const char*);
+static int isConsonant(const char *z){
+ int j;
+ char x = *z;
+ if( x==0 ) return 0;
+ assert( x>='a' && x<='z' );
+ j = cType[x-'a'];
+ if( j<2 ) return j;
+ return z[1]==0 || isVowel(z + 1);
+}
+static int isVowel(const char *z){
+ int j;
+ char x = *z;
+ if( x==0 ) return 0;
+ assert( x>='a' && x<='z' );
+ j = cType[x-'a'];
+ if( j<2 ) return 1-j;
+ return isConsonant(z + 1);
+}
+
+/*
+** Let any sequence of one or more vowels be represented by V and let
+** C be sequence of one or more consonants. Then every word can be
+** represented as:
+**
+** [C] (VC){m} [V]
+**
+** In prose: A word is an optional consonant followed by zero or
+** vowel-consonant pairs followed by an optional vowel. "m" is the
+** number of vowel consonant pairs. This routine computes the value
+** of m for the first i bytes of a word.
+**
+** Return true if the m-value for z is 1 or more. In other words,
+** return true if z contains at least one vowel that is followed
+** by a consonant.
+**
+** In this routine z[] is in reverse order. So we are really looking
+** for an instance of a consonant followed by a vowel.
+*/
+static int m_gt_0(const char *z){
+ while( isVowel(z) ){ z++; }
+ if( *z==0 ) return 0;
+ while( isConsonant(z) ){ z++; }
+ return *z!=0;
+}
+
+/* Like mgt0 above except we are looking for a value of m which is
+** exactly 1
+*/
+static int m_eq_1(const char *z){
+ while( isVowel(z) ){ z++; }
+ if( *z==0 ) return 0;
+ while( isConsonant(z) ){ z++; }
+ if( *z==0 ) return 0;
+ while( isVowel(z) ){ z++; }
+ if( *z==0 ) return 1;
+ while( isConsonant(z) ){ z++; }
+ return *z==0;
+}
+
+/* Like mgt0 above except we are looking for a value of m>1 instead
+** or m>0
+*/
+static int m_gt_1(const char *z){
+ while( isVowel(z) ){ z++; }
+ if( *z==0 ) return 0;
+ while( isConsonant(z) ){ z++; }
+ if( *z==0 ) return 0;
+ while( isVowel(z) ){ z++; }
+ if( *z==0 ) return 0;
+ while( isConsonant(z) ){ z++; }
+ return *z!=0;
+}
+
+/*
+** Return TRUE if there is a vowel anywhere within z[0..n-1]
+*/
+static int hasVowel(const char *z){
+ while( isConsonant(z) ){ z++; }
+ return *z!=0;
+}
+
+/*
+** Return TRUE if the word ends in a double consonant.
+**
+** The text is reversed here. So we are really looking at
+** the first two characters of z[].
+*/
+static int doubleConsonant(const char *z){
+ return isConsonant(z) && z[0]==z[1];
+}
+
+/*
+** Return TRUE if the word ends with three letters which
+** are consonant-vowel-consonent and where the final consonant
+** is not 'w', 'x', or 'y'.
+**
+** The word is reversed here. So we are really checking the
+** first three letters and the first one cannot be in [wxy].
+*/
+static int star_oh(const char *z){
+ return
+ isConsonant(z) &&
+ z[0]!='w' && z[0]!='x' && z[0]!='y' &&
+ isVowel(z+1) &&
+ isConsonant(z+2);
+}
+
+/*
+** If the word ends with zFrom and xCond() is true for the stem
+** of the word that preceeds the zFrom ending, then change the
+** ending to zTo.
+**
+** The input word *pz and zFrom are both in reverse order. zTo
+** is in normal order.
+**
+** Return TRUE if zFrom matches. Return FALSE if zFrom does not
+** match. Not that TRUE is returned even if xCond() fails and
+** no substitution occurs.
+*/
+static int stem(
+ char **pz, /* The word being stemmed (Reversed) */
+ const char *zFrom, /* If the ending matches this... (Reversed) */
+ const char *zTo, /* ... change the ending to this (not reversed) */
+ int (*xCond)(const char*) /* Condition that must be true */
+){
+ char *z = *pz;
+ while( *zFrom && *zFrom==*z ){ z++; zFrom++; }
+ if( *zFrom!=0 ) return 0;
+ if( xCond && !xCond(z) ) return 1;
+ while( *zTo ){
+ *(--z) = *(zTo++);
+ }
+ *pz = z;
+ return 1;
+}
+
+/*
+** This is the fallback stemmer used when the porter stemmer is
+** inappropriate. The input word is copied into the output with
+** US-ASCII case folding. If the input word is too long (more
+** than 20 bytes if it contains no digits or more than 6 bytes if
+** it contains digits) then word is truncated to 20 or 6 bytes
+** by taking 10 or 3 bytes from the beginning and end.
+*/
+static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
+ int i, mx, j;
+ int hasDigit = 0;
+ for(i=0; i<nIn; i++){
+ char c = zIn[i];
+ if( c>='A' && c<='Z' ){
+ zOut[i] = c - 'A' + 'a';
+ }else{
+ if( c>='0' && c<='9' ) hasDigit = 1;
+ zOut[i] = c;
+ }
+ }
+ mx = hasDigit ? 3 : 10;
+ if( nIn>mx*2 ){
+ for(j=mx, i=nIn-mx; i<nIn; i++, j++){
+ zOut[j] = zOut[i];
+ }
+ i = j;
+ }
+ zOut[i] = 0;
+ *pnOut = i;
+}
+
+
+/*
+** Stem the input word zIn[0..nIn-1]. Store the output in zOut.
+** zOut is at least big enough to hold nIn bytes. Write the actual
+** size of the output word (exclusive of the '\0' terminator) into *pnOut.
+**
+** Any upper-case characters in the US-ASCII character set ([A-Z])
+** are converted to lower case. Upper-case UTF characters are
+** unchanged.
+**
+** Words that are longer than about 20 bytes are stemmed by retaining
+** a few bytes from the beginning and the end of the word. If the
+** word contains digits, 3 bytes are taken from the beginning and
+** 3 bytes from the end. For long words without digits, 10 bytes
+** are taken from each end. US-ASCII case folding still applies.
+**
+** If the input word contains not digits but does characters not
+** in [a-zA-Z] then no stemming is attempted and this routine just
+** copies the input into the input into the output with US-ASCII
+** case folding.
+**
+** Stemming never increases the length of the word. So there is
+** no chance of overflowing the zOut buffer.
+*/
+static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
+ int i, j;
+ char zReverse[28];
+ char *z, *z2;
+ if( nIn<3 || nIn>=(int)sizeof(zReverse)-7 ){
+ /* The word is too big or too small for the porter stemmer.
+ ** Fallback to the copy stemmer */
+ copy_stemmer(zIn, nIn, zOut, pnOut);
+ return;
+ }
+ for(i=0, j=sizeof(zReverse)-6; i<nIn; i++, j--){
+ char c = zIn[i];
+ if( c>='A' && c<='Z' ){
+ zReverse[j] = c + 'a' - 'A';
+ }else if( c>='a' && c<='z' ){
+ zReverse[j] = c;
+ }else{
+ /* The use of a character not in [a-zA-Z] means that we fallback
+ ** to the copy stemmer */
+ copy_stemmer(zIn, nIn, zOut, pnOut);
+ return;
+ }
+ }
+ memset(&zReverse[sizeof(zReverse)-5], 0, 5);
+ z = &zReverse[j+1];
+
+
+ /* Step 1a */
+ if( z[0]=='s' ){
+ if(
+ !stem(&z, "sess", "ss", 0) &&
+ !stem(&z, "sei", "i", 0) &&
+ !stem(&z, "ss", "ss", 0)
+ ){
+ z++;
+ }
+ }
+
+ /* Step 1b */
+ z2 = z;
+ if( stem(&z, "dee", "ee", m_gt_0) ){
+ /* Do nothing. The work was all in the test */
+ }else if(
+ (stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel))
+ && z!=z2
+ ){
+ if( stem(&z, "ta", "ate", 0) ||
+ stem(&z, "lb", "ble", 0) ||
+ stem(&z, "zi", "ize", 0) ){
+ /* Do nothing. The work was all in the test */
+ }else if( doubleConsonant(z) && (*z!='l' && *z!='s' && *z!='z') ){
+ z++;
+ }else if( m_eq_1(z) && star_oh(z) ){
+ *(--z) = 'e';
+ }
+ }
+
+ /* Step 1c */
+ if( z[0]=='y' && hasVowel(z+1) ){
+ z[0] = 'i';
+ }
+
+ /* Step 2 */
+ switch( z[1] ){
+ case 'a':
+ if( !stem(&z, "lanoita", "ate", m_gt_0) ){
+ stem(&z, "lanoit", "tion", m_gt_0);
+ }
+ break;
+ case 'c':
+ if( !stem(&z, "icne", "ence", m_gt_0) ){
+ stem(&z, "icna", "ance", m_gt_0);
+ }
+ break;
+ case 'e':
+ stem(&z, "rezi", "ize", m_gt_0);
+ break;
+ case 'g':
+ stem(&z, "igol", "log", m_gt_0);
+ break;
+ case 'l':
+ if( !stem(&z, "ilb", "ble", m_gt_0)
+ && !stem(&z, "illa", "al", m_gt_0)
+ && !stem(&z, "iltne", "ent", m_gt_0)
+ && !stem(&z, "ile", "e", m_gt_0)
+ ){
+ stem(&z, "ilsuo", "ous", m_gt_0);
+ }
+ break;
+ case 'o':
+ if( !stem(&z, "noitazi", "ize", m_gt_0)
+ && !stem(&z, "noita", "ate", m_gt_0)
+ ){
+ stem(&z, "rota", "ate", m_gt_0);
+ }
+ break;
+ case 's':
+ if( !stem(&z, "msila", "al", m_gt_0)
+ && !stem(&z, "ssenevi", "ive", m_gt_0)
+ && !stem(&z, "ssenluf", "ful", m_gt_0)
+ ){
+ stem(&z, "ssensuo", "ous", m_gt_0);
+ }
+ break;
+ case 't':
+ if( !stem(&z, "itila", "al", m_gt_0)
+ && !stem(&z, "itivi", "ive", m_gt_0)
+ ){
+ stem(&z, "itilib", "ble", m_gt_0);
+ }
+ break;
+ }
+
+ /* Step 3 */
+ switch( z[0] ){
+ case 'e':
+ if( !stem(&z, "etaci", "ic", m_gt_0)
+ && !stem(&z, "evita", "", m_gt_0)
+ ){
+ stem(&z, "ezila", "al", m_gt_0);
+ }
+ break;
+ case 'i':
+ stem(&z, "itici", "ic", m_gt_0);
+ break;
+ case 'l':
+ if( !stem(&z, "laci", "ic", m_gt_0) ){
+ stem(&z, "luf", "", m_gt_0);
+ }
+ break;
+ case 's':
+ stem(&z, "ssen", "", m_gt_0);
+ break;
+ }
+
+ /* Step 4 */
+ switch( z[1] ){
+ case 'a':
+ if( z[0]=='l' && m_gt_1(z+2) ){
+ z += 2;
+ }
+ break;
+ case 'c':
+ if( z[0]=='e' && z[2]=='n' && (z[3]=='a' || z[3]=='e') && m_gt_1(z+4) ){
+ z += 4;
+ }
+ break;
+ case 'e':
+ if( z[0]=='r' && m_gt_1(z+2) ){
+ z += 2;
+ }
+ break;
+ case 'i':
+ if( z[0]=='c' && m_gt_1(z+2) ){
+ z += 2;
+ }
+ break;
+ case 'l':
+ if( z[0]=='e' && z[2]=='b' && (z[3]=='a' || z[3]=='i') && m_gt_1(z+4) ){
+ z += 4;
+ }
+ break;
+ case 'n':
+ if( z[0]=='t' ){
+ if( z[2]=='a' ){
+ if( m_gt_1(z+3) ){
+ z += 3;
+ }
+ }else if( z[2]=='e' ){
+ if( !stem(&z, "tneme", "", m_gt_1)
+ && !stem(&z, "tnem", "", m_gt_1)
+ ){
+ stem(&z, "tne", "", m_gt_1);
+ }
+ }
+ }
+ break;
+ case 'o':
+ if( z[0]=='u' ){
+ if( m_gt_1(z+2) ){
+ z += 2;
+ }
+ }else if( z[3]=='s' || z[3]=='t' ){
+ stem(&z, "noi", "", m_gt_1);
+ }
+ break;
+ case 's':
+ if( z[0]=='m' && z[2]=='i' && m_gt_1(z+3) ){
+ z += 3;
+ }
+ break;
+ case 't':
+ if( !stem(&z, "eta", "", m_gt_1) ){
+ stem(&z, "iti", "", m_gt_1);
+ }
+ break;
+ case 'u':
+ if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){
+ z += 3;
+ }
+ break;
+ case 'v':
+ case 'z':
+ if( z[0]=='e' && z[2]=='i' && m_gt_1(z+3) ){
+ z += 3;
+ }
+ break;
+ }
+
+ /* Step 5a */
+ if( z[0]=='e' ){
+ if( m_gt_1(z+1) ){
+ z++;
+ }else if( m_eq_1(z+1) && !star_oh(z+1) ){
+ z++;
+ }
+ }
+
+ /* Step 5b */
+ if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){
+ z++;
+ }
+
+ /* z[] is now the stemmed word in reverse order. Flip it back
+ ** around into forward order and return.
+ */
+ *pnOut = i = (int)strlen(z);
+ zOut[i] = 0;
+ while( *z ){
+ zOut[--i] = *(z++);
+ }
+}
+
+/*
+** Characters that can be part of a token. We assume any character
+** whose value is greater than 0x80 (any UTF character) can be
+** part of a token. In other words, delimiters all must have
+** values of 0x7f or lower.
+*/
+static const char porterIdChar[] = {
+/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
+};
+#define isDelim(C) (((ch=C)&0x80)==0 && (ch<0x30 || !porterIdChar[ch-0x30]))
+
+/*
+** Extract the next token from a tokenization cursor. The cursor must
+** have been opened by a prior call to porterOpen().
+*/
+static int porterNext(
+ sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by porterOpen */
+ const char **pzToken, /* OUT: *pzToken is the token text */
+ int *pnBytes, /* OUT: Number of bytes in token */
+ int *piStartOffset, /* OUT: Starting offset of token */
+ int *piEndOffset, /* OUT: Ending offset of token */
+ int *piPosition /* OUT: Position integer of token */
+){
+ porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
+ const char *z = c->zInput;
+
+ while( c->iOffset<c->nInput ){
+ int iStartOffset, ch;
+
+ /* Scan past delimiter characters */
+ while( c->iOffset<c->nInput && isDelim(z[c->iOffset]) ){
+ c->iOffset++;
+ }
+
+ /* Count non-delimiter characters. */
+ iStartOffset = c->iOffset;
+ while( c->iOffset<c->nInput && !isDelim(z[c->iOffset]) ){
+ c->iOffset++;
+ }
+
+ if( c->iOffset>iStartOffset ){
+ int n = c->iOffset-iStartOffset;
+ if( n>c->nAllocated ){
+ char *pNew;
+ c->nAllocated = n+20;
+ pNew = sqlite3_realloc(c->zToken, c->nAllocated);
+ if( !pNew ) return SQLITE_NOMEM;
+ c->zToken = pNew;
+ }
+ porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes);
+ *pzToken = c->zToken;
+ *piStartOffset = iStartOffset;
+ *piEndOffset = c->iOffset;
+ *piPosition = c->iToken++;
+ return SQLITE_OK;
+ }
+ }
+ return SQLITE_DONE;
+}
+
+/*
+** The set of routines that implement the porter-stemmer tokenizer
+*/
+static const sqlite3_tokenizer_module porterTokenizerModule = {
+ 0,
+ porterCreate,
+ porterDestroy,
+ porterOpen,
+ porterClose,
+ porterNext,
+ 0
+};
+
+/*
+** Allocate a new porter tokenizer. Return a pointer to the new
+** tokenizer in *ppModule
+*/
+SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(
+ sqlite3_tokenizer_module const**ppModule
+){
+ *ppModule = &porterTokenizerModule;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
+
+/************** End of fts3_porter.c *****************************************/
+/************** Begin file fts3_tokenizer.c **********************************/
+/*
+** 2007 June 22
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This is part of an SQLite module implementing full-text search.
+** This particular file implements the generic tokenizer interface.
+*/
+
+/*
+** The code in this file is only compiled if:
+**
+** * The FTS3 module is being built as an extension
+** (in which case SQLITE_CORE is not defined), or
+**
+** * The FTS3 module is being built into the core of
+** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
+*/
+/* #include "fts3Int.h" */
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+/* #include <assert.h> */
+/* #include <string.h> */
+
+/*
+** Implementation of the SQL scalar function for accessing the underlying
+** hash table. This function may be called as follows:
+**
+** SELECT <function-name>(<key-name>);
+** SELECT <function-name>(<key-name>, <pointer>);
+**
+** where <function-name> is the name passed as the second argument
+** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer').
+**
+** If the <pointer> argument is specified, it must be a blob value
+** containing a pointer to be stored as the hash data corresponding
+** to the string <key-name>. If <pointer> is not specified, then
+** the string <key-name> must already exist in the has table. Otherwise,
+** an error is returned.
+**
+** Whether or not the <pointer> argument is specified, the value returned
+** is a blob containing the pointer stored as the hash data corresponding
+** to string <key-name> (after the hash-table is updated, if applicable).
+*/
+static void scalarFunc(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ Fts3Hash *pHash;
+ void *pPtr = 0;
+ const unsigned char *zName;
+ int nName;
+
+ assert( argc==1 || argc==2 );
+
+ pHash = (Fts3Hash *)sqlite3_user_data(context);
+
+ zName = sqlite3_value_text(argv[0]);
+ nName = sqlite3_value_bytes(argv[0])+1;
+
+ if( argc==2 ){
+ void *pOld;
+ int n = sqlite3_value_bytes(argv[1]);
+ if( zName==0 || n!=sizeof(pPtr) ){
+ sqlite3_result_error(context, "argument type mismatch", -1);
+ return;
+ }
+ pPtr = *(void **)sqlite3_value_blob(argv[1]);
+ pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr);
+ if( pOld==pPtr ){
+ sqlite3_result_error(context, "out of memory", -1);
+ return;
+ }
+ }else{
+ if( zName ){
+ pPtr = sqlite3Fts3HashFind(pHash, zName, nName);
+ }
+ if( !pPtr ){
+ char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
+ sqlite3_result_error(context, zErr, -1);
+ sqlite3_free(zErr);
+ return;
+ }
+ }
+
+ sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
+}
+
+SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char c){
+ static const char isFtsIdChar[] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
+ 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
+ };
+ return (c&0x80 || isFtsIdChar[(int)(c)]);
+}
+
+SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *zStr, int *pn){
+ const char *z1;
+ const char *z2 = 0;
+
+ /* Find the start of the next token. */
+ z1 = zStr;
+ while( z2==0 ){
+ char c = *z1;
+ switch( c ){
+ case '\0': return 0; /* No more tokens here */
+ case '\'':
+ case '"':
+ case '`': {
+ z2 = z1;
+ while( *++z2 && (*z2!=c || *++z2==c) );
+ break;
+ }
+ case '[':
+ z2 = &z1[1];
+ while( *z2 && z2[0]!=']' ) z2++;
+ if( *z2 ) z2++;
+ break;
+
+ default:
+ if( sqlite3Fts3IsIdChar(*z1) ){
+ z2 = &z1[1];
+ while( sqlite3Fts3IsIdChar(*z2) ) z2++;
+ }else{
+ z1++;
+ }
+ }
+ }
+
+ *pn = (int)(z2-z1);
+ return z1;
+}
+
+SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(
+ Fts3Hash *pHash, /* Tokenizer hash table */
+ const char *zArg, /* Tokenizer name */
+ sqlite3_tokenizer **ppTok, /* OUT: Tokenizer (if applicable) */
+ char **pzErr /* OUT: Set to malloced error message */
+){
+ int rc;
+ char *z = (char *)zArg;
+ int n = 0;
+ char *zCopy;
+ char *zEnd; /* Pointer to nul-term of zCopy */
+ sqlite3_tokenizer_module *m;
+
+ zCopy = sqlite3_mprintf("%s", zArg);
+ if( !zCopy ) return SQLITE_NOMEM;
+ zEnd = &zCopy[strlen(zCopy)];
+
+ z = (char *)sqlite3Fts3NextToken(zCopy, &n);
+ if( z==0 ){
+ assert( n==0 );
+ z = zCopy;
+ }
+ z[n] = '\0';
+ sqlite3Fts3Dequote(z);
+
+ m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1);
+ if( !m ){
+ sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", z);
+ rc = SQLITE_ERROR;
+ }else{
+ char const **aArg = 0;
+ int iArg = 0;
+ z = &z[n+1];
+ while( z<zEnd && (NULL!=(z = (char *)sqlite3Fts3NextToken(z, &n))) ){
+ int nNew = sizeof(char *)*(iArg+1);
+ char const **aNew = (const char **)sqlite3_realloc((void *)aArg, nNew);
+ if( !aNew ){
+ sqlite3_free(zCopy);
+ sqlite3_free((void *)aArg);
+ return SQLITE_NOMEM;
+ }
+ aArg = aNew;
+ aArg[iArg++] = z;
+ z[n] = '\0';
+ sqlite3Fts3Dequote(z);
+ z = &z[n+1];
+ }
+ rc = m->xCreate(iArg, aArg, ppTok);
+ assert( rc!=SQLITE_OK || *ppTok );
+ if( rc!=SQLITE_OK ){
+ sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer");
+ }else{
+ (*ppTok)->pModule = m;
+ }
+ sqlite3_free((void *)aArg);
+ }
+
+ sqlite3_free(zCopy);
+ return rc;
+}
+
+
+#ifdef SQLITE_TEST
+
+#include <tcl.h>
+/* #include <string.h> */
+
+/*
+** Implementation of a special SQL scalar function for testing tokenizers
+** designed to be used in concert with the Tcl testing framework. This
+** function must be called with two or more arguments:
+**
+** SELECT <function-name>(<key-name>, ..., <input-string>);
+**
+** where <function-name> is the name passed as the second argument
+** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer')
+** concatenated with the string '_test' (e.g. 'fts3_tokenizer_test').
+**
+** The return value is a string that may be interpreted as a Tcl
+** list. For each token in the <input-string>, three elements are
+** added to the returned list. The first is the token position, the
+** second is the token text (folded, stemmed, etc.) and the third is the
+** substring of <input-string> associated with the token. For example,
+** using the built-in "simple" tokenizer:
+**
+** SELECT fts_tokenizer_test('simple', 'I don't see how');
+**
+** will return the string:
+**
+** "{0 i I 1 dont don't 2 see see 3 how how}"
+**
+*/
+static void testFunc(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ Fts3Hash *pHash;
+ sqlite3_tokenizer_module *p;
+ sqlite3_tokenizer *pTokenizer = 0;
+ sqlite3_tokenizer_cursor *pCsr = 0;
+
+ const char *zErr = 0;
+
+ const char *zName;
+ int nName;
+ const char *zInput;
+ int nInput;
+
+ const char *azArg[64];
+
+ const char *zToken;
+ int nToken = 0;
+ int iStart = 0;
+ int iEnd = 0;
+ int iPos = 0;
+ int i;
+
+ Tcl_Obj *pRet;
+
+ if( argc<2 ){
+ sqlite3_result_error(context, "insufficient arguments", -1);
+ return;
+ }
+
+ nName = sqlite3_value_bytes(argv[0]);
+ zName = (const char *)sqlite3_value_text(argv[0]);
+ nInput = sqlite3_value_bytes(argv[argc-1]);
+ zInput = (const char *)sqlite3_value_text(argv[argc-1]);
+
+ pHash = (Fts3Hash *)sqlite3_user_data(context);
+ p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1);
+
+ if( !p ){
+ char *zErr2 = sqlite3_mprintf("unknown tokenizer: %s", zName);
+ sqlite3_result_error(context, zErr2, -1);
+ sqlite3_free(zErr2);
+ return;
+ }
+
+ pRet = Tcl_NewObj();
+ Tcl_IncrRefCount(pRet);
+
+ for(i=1; i<argc-1; i++){
+ azArg[i-1] = (const char *)sqlite3_value_text(argv[i]);
+ }
+
+ if( SQLITE_OK!=p->xCreate(argc-2, azArg, &pTokenizer) ){
+ zErr = "error in xCreate()";
+ goto finish;
+ }
+ pTokenizer->pModule = p;
+ if( sqlite3Fts3OpenTokenizer(pTokenizer, 0, zInput, nInput, &pCsr) ){
+ zErr = "error in xOpen()";
+ goto finish;
+ }
+
+ while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){
+ Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos));
+ Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
+ zToken = &zInput[iStart];
+ nToken = iEnd-iStart;
+ Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
+ }
+
+ if( SQLITE_OK!=p->xClose(pCsr) ){
+ zErr = "error in xClose()";
+ goto finish;
+ }
+ if( SQLITE_OK!=p->xDestroy(pTokenizer) ){
+ zErr = "error in xDestroy()";
+ goto finish;
+ }
+
+finish:
+ if( zErr ){
+ sqlite3_result_error(context, zErr, -1);
+ }else{
+ sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT);
+ }
+ Tcl_DecrRefCount(pRet);
+}
+
+static
+int registerTokenizer(
+ sqlite3 *db,
+ char *zName,
+ const sqlite3_tokenizer_module *p
+){
+ int rc;
+ sqlite3_stmt *pStmt;
+ const char zSql[] = "SELECT fts3_tokenizer(?, ?)";
+
+ rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+
+ sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
+ sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
+ sqlite3_step(pStmt);
+
+ return sqlite3_finalize(pStmt);
+}
+
+static
+int queryTokenizer(
+ sqlite3 *db,
+ char *zName,
+ const sqlite3_tokenizer_module **pp
+){
+ int rc;
+ sqlite3_stmt *pStmt;
+ const char zSql[] = "SELECT fts3_tokenizer(?)";
+
+ *pp = 0;
+ rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+
+ sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){
+ if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
+ memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
+ }
+ }
+
+ return sqlite3_finalize(pStmt);
+}
+
+SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+
+/*
+** Implementation of the scalar function fts3_tokenizer_internal_test().
+** This function is used for testing only, it is not included in the
+** build unless SQLITE_TEST is defined.
+**
+** The purpose of this is to test that the fts3_tokenizer() function
+** can be used as designed by the C-code in the queryTokenizer and
+** registerTokenizer() functions above. These two functions are repeated
+** in the README.tokenizer file as an example, so it is important to
+** test them.
+**
+** To run the tests, evaluate the fts3_tokenizer_internal_test() scalar
+** function with no arguments. An assert() will fail if a problem is
+** detected. i.e.:
+**
+** SELECT fts3_tokenizer_internal_test();
+**
+*/
+static void intTestFunc(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ int rc;
+ const sqlite3_tokenizer_module *p1;
+ const sqlite3_tokenizer_module *p2;
+ sqlite3 *db = (sqlite3 *)sqlite3_user_data(context);
+
+ UNUSED_PARAMETER(argc);
+ UNUSED_PARAMETER(argv);
+
+ /* Test the query function */
+ sqlite3Fts3SimpleTokenizerModule(&p1);
+ rc = queryTokenizer(db, "simple", &p2);
+ assert( rc==SQLITE_OK );
+ assert( p1==p2 );
+ rc = queryTokenizer(db, "nosuchtokenizer", &p2);
+ assert( rc==SQLITE_ERROR );
+ assert( p2==0 );
+ assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
+
+ /* Test the storage function */
+ rc = registerTokenizer(db, "nosuchtokenizer", p1);
+ assert( rc==SQLITE_OK );
+ rc = queryTokenizer(db, "nosuchtokenizer", &p2);
+ assert( rc==SQLITE_OK );
+ assert( p2==p1 );
+
+ sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
+}
+
+#endif
+
+/*
+** Set up SQL objects in database db used to access the contents of
+** the hash table pointed to by argument pHash. The hash table must
+** been initialized to use string keys, and to take a private copy
+** of the key when a value is inserted. i.e. by a call similar to:
+**
+** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
+**
+** This function adds a scalar function (see header comment above
+** scalarFunc() in this file for details) and, if ENABLE_TABLE is
+** defined at compilation time, a temporary virtual table (see header
+** comment above struct HashTableVtab) to the database schema. Both
+** provide read/write access to the contents of *pHash.
+**
+** The third argument to this function, zName, is used as the name
+** of both the scalar and, if created, the virtual table.
+*/
+SQLITE_PRIVATE int sqlite3Fts3InitHashTable(
+ sqlite3 *db,
+ Fts3Hash *pHash,
+ const char *zName
+){
+ int rc = SQLITE_OK;
+ void *p = (void *)pHash;
+ const int any = SQLITE_ANY;
+
+#ifdef SQLITE_TEST
+ char *zTest = 0;
+ char *zTest2 = 0;
+ void *pdb = (void *)db;
+ zTest = sqlite3_mprintf("%s_test", zName);
+ zTest2 = sqlite3_mprintf("%s_internal_test", zName);
+ if( !zTest || !zTest2 ){
+ rc = SQLITE_NOMEM;
+ }
+#endif
+
+ if( SQLITE_OK==rc ){
+ rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0);
+ }
+ if( SQLITE_OK==rc ){
+ rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0);
+ }
+#ifdef SQLITE_TEST
+ if( SQLITE_OK==rc ){
+ rc = sqlite3_create_function(db, zTest, -1, any, p, testFunc, 0, 0);
+ }
+ if( SQLITE_OK==rc ){
+ rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0);
+ }
+#endif
+
+#ifdef SQLITE_TEST
+ sqlite3_free(zTest);
+ sqlite3_free(zTest2);
+#endif
+
+ return rc;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
+
+/************** End of fts3_tokenizer.c **************************************/
+/************** Begin file fts3_tokenizer1.c *********************************/
+/*
+** 2006 Oct 10
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** Implementation of the "simple" full-text-search tokenizer.
+*/
+
+/*
+** The code in this file is only compiled if:
+**
+** * The FTS3 module is being built as an extension
+** (in which case SQLITE_CORE is not defined), or
+**
+** * The FTS3 module is being built into the core of
+** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
+*/
+/* #include "fts3Int.h" */
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+/* #include <assert.h> */
+/* #include <stdlib.h> */
+/* #include <stdio.h> */
+/* #include <string.h> */
+
+/* #include "fts3_tokenizer.h" */
+
+typedef struct simple_tokenizer {
+ sqlite3_tokenizer base;
+ char delim[128]; /* flag ASCII delimiters */
+} simple_tokenizer;
+
+typedef struct simple_tokenizer_cursor {
+ sqlite3_tokenizer_cursor base;
+ const char *pInput; /* input we are tokenizing */
+ int nBytes; /* size of the input */
+ int iOffset; /* current position in pInput */
+ int iToken; /* index of next token to be returned */
+ char *pToken; /* storage for current token */
+ int nTokenAllocated; /* space allocated to zToken buffer */
+} simple_tokenizer_cursor;
+
+
+static int simpleDelim(simple_tokenizer *t, unsigned char c){
+ return c<0x80 && t->delim[c];
+}
+static int fts3_isalnum(int x){
+ return (x>='0' && x<='9') || (x>='A' && x<='Z') || (x>='a' && x<='z');
+}
+
+/*
+** Create a new tokenizer instance.
+*/
+static int simpleCreate(
+ int argc, const char * const *argv,
+ sqlite3_tokenizer **ppTokenizer
+){
+ simple_tokenizer *t;
+
+ t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t));
+ if( t==NULL ) return SQLITE_NOMEM;
+ memset(t, 0, sizeof(*t));
+
+ /* TODO(shess) Delimiters need to remain the same from run to run,
+ ** else we need to reindex. One solution would be a meta-table to
+ ** track such information in the database, then we'd only want this
+ ** information on the initial create.
+ */
+ if( argc>1 ){
+ int i, n = (int)strlen(argv[1]);
+ for(i=0; i<n; i++){
+ unsigned char ch = argv[1][i];
+ /* We explicitly don't support UTF-8 delimiters for now. */
+ if( ch>=0x80 ){
+ sqlite3_free(t);
+ return SQLITE_ERROR;
+ }
+ t->delim[ch] = 1;
+ }
+ } else {
+ /* Mark non-alphanumeric ASCII characters as delimiters */
+ int i;
+ for(i=1; i<0x80; i++){
+ t->delim[i] = !fts3_isalnum(i) ? -1 : 0;
+ }
+ }
+
+ *ppTokenizer = &t->base;
+ return SQLITE_OK;
+}
+
+/*
+** Destroy a tokenizer
+*/
+static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
+ sqlite3_free(pTokenizer);
+ return SQLITE_OK;
+}
+
+/*
+** Prepare to begin tokenizing a particular string. The input
+** string to be tokenized is pInput[0..nBytes-1]. A cursor
+** used to incrementally tokenize this string is returned in
+** *ppCursor.
+*/
+static int simpleOpen(
+ sqlite3_tokenizer *pTokenizer, /* The tokenizer */
+ const char *pInput, int nBytes, /* String to be tokenized */
+ sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
+){
+ simple_tokenizer_cursor *c;
+
+ UNUSED_PARAMETER(pTokenizer);
+
+ c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
+ if( c==NULL ) return SQLITE_NOMEM;
+
+ c->pInput = pInput;
+ if( pInput==0 ){
+ c->nBytes = 0;
+ }else if( nBytes<0 ){
+ c->nBytes = (int)strlen(pInput);
+ }else{
+ c->nBytes = nBytes;
+ }
+ c->iOffset = 0; /* start tokenizing at the beginning */
+ c->iToken = 0;
+ c->pToken = NULL; /* no space allocated, yet. */
+ c->nTokenAllocated = 0;
+
+ *ppCursor = &c->base;
+ return SQLITE_OK;
+}
+
+/*
+** Close a tokenization cursor previously opened by a call to
+** simpleOpen() above.
+*/
+static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
+ simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
+ sqlite3_free(c->pToken);
+ sqlite3_free(c);
+ return SQLITE_OK;
+}
+
+/*
+** Extract the next token from a tokenization cursor. The cursor must
+** have been opened by a prior call to simpleOpen().
+*/
+static int simpleNext(
+ sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */
+ const char **ppToken, /* OUT: *ppToken is the token text */
+ int *pnBytes, /* OUT: Number of bytes in token */
+ int *piStartOffset, /* OUT: Starting offset of token */
+ int *piEndOffset, /* OUT: Ending offset of token */
+ int *piPosition /* OUT: Position integer of token */
+){
+ simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
+ simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
+ unsigned char *p = (unsigned char *)c->pInput;
+
+ while( c->iOffset<c->nBytes ){
+ int iStartOffset;
+
+ /* Scan past delimiter characters */
+ while( c->iOffset<c->nBytes && simpleDelim(t, p[c->iOffset]) ){
+ c->iOffset++;
+ }
+
+ /* Count non-delimiter characters. */
+ iStartOffset = c->iOffset;
+ while( c->iOffset<c->nBytes && !simpleDelim(t, p[c->iOffset]) ){
+ c->iOffset++;
+ }
+
+ if( c->iOffset>iStartOffset ){
+ int i, n = c->iOffset-iStartOffset;
+ if( n>c->nTokenAllocated ){
+ char *pNew;
+ c->nTokenAllocated = n+20;
+ pNew = sqlite3_realloc(c->pToken, c->nTokenAllocated);
+ if( !pNew ) return SQLITE_NOMEM;
+ c->pToken = pNew;
+ }
+ for(i=0; i<n; i++){
+ /* TODO(shess) This needs expansion to handle UTF-8
+ ** case-insensitivity.
+ */
+ unsigned char ch = p[iStartOffset+i];
+ c->pToken[i] = (char)((ch>='A' && ch<='Z') ? ch-'A'+'a' : ch);
+ }
+ *ppToken = c->pToken;
+ *pnBytes = n;
+ *piStartOffset = iStartOffset;
+ *piEndOffset = c->iOffset;
+ *piPosition = c->iToken++;
+
+ return SQLITE_OK;
+ }
+ }
+ return SQLITE_DONE;
+}
+
+/*
+** The set of routines that implement the simple tokenizer
+*/
+static const sqlite3_tokenizer_module simpleTokenizerModule = {
+ 0,
+ simpleCreate,
+ simpleDestroy,
+ simpleOpen,
+ simpleClose,
+ simpleNext,
+ 0,
+};
+
+/*
+** Allocate a new simple tokenizer. Return a pointer to the new
+** tokenizer in *ppModule
+*/
+SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(
+ sqlite3_tokenizer_module const**ppModule
+){
+ *ppModule = &simpleTokenizerModule;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
+
+/************** End of fts3_tokenizer1.c *************************************/
+/************** Begin file fts3_tokenize_vtab.c ******************************/
+/*
+** 2013 Apr 22
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file contains code for the "fts3tokenize" virtual table module.
+** An fts3tokenize virtual table is created as follows:
+**
+** CREATE VIRTUAL TABLE <tbl> USING fts3tokenize(
+** <tokenizer-name>, <arg-1>, ...
+** );
+**
+** The table created has the following schema:
+**
+** CREATE TABLE <tbl>(input, token, start, end, position)
+**
+** When queried, the query must include a WHERE clause of type:
+**
+** input = <string>
+**
+** The virtual table module tokenizes this <string>, using the FTS3
+** tokenizer specified by the arguments to the CREATE VIRTUAL TABLE
+** statement and returns one row for each token in the result. With
+** fields set as follows:
+**
+** input: Always set to a copy of <string>
+** token: A token from the input.
+** start: Byte offset of the token within the input <string>.
+** end: Byte offset of the byte immediately following the end of the
+** token within the input string.
+** pos: Token offset of token within input.
+**
+*/
+/* #include "fts3Int.h" */
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+/* #include <string.h> */
+/* #include <assert.h> */
+
+typedef struct Fts3tokTable Fts3tokTable;
+typedef struct Fts3tokCursor Fts3tokCursor;
+
+/*
+** Virtual table structure.
+*/
+struct Fts3tokTable {
+ sqlite3_vtab base; /* Base class used by SQLite core */
+ const sqlite3_tokenizer_module *pMod;
+ sqlite3_tokenizer *pTok;
+};
+
+/*
+** Virtual table cursor structure.
+*/
+struct Fts3tokCursor {
+ sqlite3_vtab_cursor base; /* Base class used by SQLite core */
+ char *zInput; /* Input string */
+ sqlite3_tokenizer_cursor *pCsr; /* Cursor to iterate through zInput */
+ int iRowid; /* Current 'rowid' value */
+ const char *zToken; /* Current 'token' value */
+ int nToken; /* Size of zToken in bytes */
+ int iStart; /* Current 'start' value */
+ int iEnd; /* Current 'end' value */
+ int iPos; /* Current 'pos' value */
+};
+
+/*
+** Query FTS for the tokenizer implementation named zName.
+*/
+static int fts3tokQueryTokenizer(
+ Fts3Hash *pHash,
+ const char *zName,
+ const sqlite3_tokenizer_module **pp,
+ char **pzErr
+){
+ sqlite3_tokenizer_module *p;
+ int nName = (int)strlen(zName);
+
+ p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1);
+ if( !p ){
+ sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", zName);
+ return SQLITE_ERROR;
+ }
+
+ *pp = p;
+ return SQLITE_OK;
+}
+
+/*
+** The second argument, argv[], is an array of pointers to nul-terminated
+** strings. This function makes a copy of the array and strings into a
+** single block of memory. It then dequotes any of the strings that appear
+** to be quoted.
+**
+** If successful, output parameter *pazDequote is set to point at the
+** array of dequoted strings and SQLITE_OK is returned. The caller is
+** responsible for eventually calling sqlite3_free() to free the array
+** in this case. Or, if an error occurs, an SQLite error code is returned.
+** The final value of *pazDequote is undefined in this case.
+*/
+static int fts3tokDequoteArray(
+ int argc, /* Number of elements in argv[] */
+ const char * const *argv, /* Input array */
+ char ***pazDequote /* Output array */
+){
+ int rc = SQLITE_OK; /* Return code */
+ if( argc==0 ){
+ *pazDequote = 0;
+ }else{
+ int i;
+ int nByte = 0;
+ char **azDequote;
+
+ for(i=0; i<argc; i++){
+ nByte += (int)(strlen(argv[i]) + 1);
+ }
+
+ *pazDequote = azDequote = sqlite3_malloc(sizeof(char *)*argc + nByte);
+ if( azDequote==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ char *pSpace = (char *)&azDequote[argc];
+ for(i=0; i<argc; i++){
+ int n = (int)strlen(argv[i]);
+ azDequote[i] = pSpace;
+ memcpy(pSpace, argv[i], n+1);
+ sqlite3Fts3Dequote(pSpace);
+ pSpace += (n+1);
+ }
+ }
+ }
+
+ return rc;
+}
+
+/*
+** Schema of the tokenizer table.
+*/
+#define FTS3_TOK_SCHEMA "CREATE TABLE x(input, token, start, end, position)"
+
+/*
+** This function does all the work for both the xConnect and xCreate methods.
+** These tables have no persistent representation of their own, so xConnect
+** and xCreate are identical operations.
+**
+** argv[0]: module name
+** argv[1]: database name
+** argv[2]: table name
+** argv[3]: first argument (tokenizer name)
+*/
+static int fts3tokConnectMethod(
+ sqlite3 *db, /* Database connection */
+ void *pHash, /* Hash table of tokenizers */
+ int argc, /* Number of elements in argv array */
+ const char * const *argv, /* xCreate/xConnect argument array */
+ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
+ char **pzErr /* OUT: sqlite3_malloc'd error message */
+){
+ Fts3tokTable *pTab = 0;
+ const sqlite3_tokenizer_module *pMod = 0;
+ sqlite3_tokenizer *pTok = 0;
+ int rc;
+ char **azDequote = 0;
+ int nDequote;
+
+ rc = sqlite3_declare_vtab(db, FTS3_TOK_SCHEMA);
+ if( rc!=SQLITE_OK ) return rc;
+
+ nDequote = argc-3;
+ rc = fts3tokDequoteArray(nDequote, &argv[3], &azDequote);
+
+ if( rc==SQLITE_OK ){
+ const char *zModule;
+ if( nDequote<1 ){
+ zModule = "simple";
+ }else{
+ zModule = azDequote[0];
+ }
+ rc = fts3tokQueryTokenizer((Fts3Hash*)pHash, zModule, &pMod, pzErr);
+ }
+
+ assert( (rc==SQLITE_OK)==(pMod!=0) );
+ if( rc==SQLITE_OK ){
+ const char * const *azArg = (const char * const *)&azDequote[1];
+ rc = pMod->xCreate((nDequote>1 ? nDequote-1 : 0), azArg, &pTok);
+ }
+
+ if( rc==SQLITE_OK ){
+ pTab = (Fts3tokTable *)sqlite3_malloc(sizeof(Fts3tokTable));
+ if( pTab==0 ){
+ rc = SQLITE_NOMEM;
+ }
+ }
+
+ if( rc==SQLITE_OK ){
+ memset(pTab, 0, sizeof(Fts3tokTable));
+ pTab->pMod = pMod;
+ pTab->pTok = pTok;
+ *ppVtab = &pTab->base;
+ }else{
+ if( pTok ){
+ pMod->xDestroy(pTok);
+ }
+ }
+
+ sqlite3_free(azDequote);
+ return rc;
+}
+
+/*
+** This function does the work for both the xDisconnect and xDestroy methods.
+** These tables have no persistent representation of their own, so xDisconnect
+** and xDestroy are identical operations.
+*/
+static int fts3tokDisconnectMethod(sqlite3_vtab *pVtab){
+ Fts3tokTable *pTab = (Fts3tokTable *)pVtab;
+
+ pTab->pMod->xDestroy(pTab->pTok);
+ sqlite3_free(pTab);
+ return SQLITE_OK;
+}
+
+/*
+** xBestIndex - Analyze a WHERE and ORDER BY clause.
+*/
+static int fts3tokBestIndexMethod(
+ sqlite3_vtab *pVTab,
+ sqlite3_index_info *pInfo
+){
+ int i;
+ UNUSED_PARAMETER(pVTab);
+
+ for(i=0; i<pInfo->nConstraint; i++){
+ if( pInfo->aConstraint[i].usable
+ && pInfo->aConstraint[i].iColumn==0
+ && pInfo->aConstraint[i].op==SQLITE_INDEX_CONSTRAINT_EQ
+ ){
+ pInfo->idxNum = 1;
+ pInfo->aConstraintUsage[i].argvIndex = 1;
+ pInfo->aConstraintUsage[i].omit = 1;
+ pInfo->estimatedCost = 1;
+ return SQLITE_OK;
+ }
+ }
+
+ pInfo->idxNum = 0;
+ assert( pInfo->estimatedCost>1000000.0 );
+
+ return SQLITE_OK;
+}
+
+/*
+** xOpen - Open a cursor.
+*/
+static int fts3tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
+ Fts3tokCursor *pCsr;
+ UNUSED_PARAMETER(pVTab);
+
+ pCsr = (Fts3tokCursor *)sqlite3_malloc(sizeof(Fts3tokCursor));
+ if( pCsr==0 ){
+ return SQLITE_NOMEM;
+ }
+ memset(pCsr, 0, sizeof(Fts3tokCursor));
+
+ *ppCsr = (sqlite3_vtab_cursor *)pCsr;
+ return SQLITE_OK;
+}
+
+/*
+** Reset the tokenizer cursor passed as the only argument. As if it had
+** just been returned by fts3tokOpenMethod().
+*/
+static void fts3tokResetCursor(Fts3tokCursor *pCsr){
+ if( pCsr->pCsr ){
+ Fts3tokTable *pTab = (Fts3tokTable *)(pCsr->base.pVtab);
+ pTab->pMod->xClose(pCsr->pCsr);
+ pCsr->pCsr = 0;
+ }
+ sqlite3_free(pCsr->zInput);
+ pCsr->zInput = 0;
+ pCsr->zToken = 0;
+ pCsr->nToken = 0;
+ pCsr->iStart = 0;
+ pCsr->iEnd = 0;
+ pCsr->iPos = 0;
+ pCsr->iRowid = 0;
+}
+
+/*
+** xClose - Close a cursor.
+*/
+static int fts3tokCloseMethod(sqlite3_vtab_cursor *pCursor){
+ Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor;
+
+ fts3tokResetCursor(pCsr);
+ sqlite3_free(pCsr);
+ return SQLITE_OK;
+}
+
+/*
+** xNext - Advance the cursor to the next row, if any.
+*/
+static int fts3tokNextMethod(sqlite3_vtab_cursor *pCursor){
+ Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor;
+ Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab);
+ int rc; /* Return code */
+
+ pCsr->iRowid++;
+ rc = pTab->pMod->xNext(pCsr->pCsr,
+ &pCsr->zToken, &pCsr->nToken,
+ &pCsr->iStart, &pCsr->iEnd, &pCsr->iPos
+ );
+
+ if( rc!=SQLITE_OK ){
+ fts3tokResetCursor(pCsr);
+ if( rc==SQLITE_DONE ) rc = SQLITE_OK;
+ }
+
+ return rc;
+}
+
+/*
+** xFilter - Initialize a cursor to point at the start of its data.
+*/
+static int fts3tokFilterMethod(
+ sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
+ int idxNum, /* Strategy index */
+ const char *idxStr, /* Unused */
+ int nVal, /* Number of elements in apVal */
+ sqlite3_value **apVal /* Arguments for the indexing scheme */
+){
+ int rc = SQLITE_ERROR;
+ Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor;
+ Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab);
+ UNUSED_PARAMETER(idxStr);
+ UNUSED_PARAMETER(nVal);
+
+ fts3tokResetCursor(pCsr);
+ if( idxNum==1 ){
+ const char *zByte = (const char *)sqlite3_value_text(apVal[0]);
+ int nByte = sqlite3_value_bytes(apVal[0]);
+ pCsr->zInput = sqlite3_malloc(nByte+1);
+ if( pCsr->zInput==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ memcpy(pCsr->zInput, zByte, nByte);
+ pCsr->zInput[nByte] = 0;
+ rc = pTab->pMod->xOpen(pTab->pTok, pCsr->zInput, nByte, &pCsr->pCsr);
+ if( rc==SQLITE_OK ){
+ pCsr->pCsr->pTokenizer = pTab->pTok;
+ }
+ }
+ }
+
+ if( rc!=SQLITE_OK ) return rc;
+ return fts3tokNextMethod(pCursor);
+}
+
+/*
+** xEof - Return true if the cursor is at EOF, or false otherwise.
+*/
+static int fts3tokEofMethod(sqlite3_vtab_cursor *pCursor){
+ Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor;
+ return (pCsr->zToken==0);
+}
+
+/*
+** xColumn - Return a column value.
+*/
+static int fts3tokColumnMethod(
+ sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
+ sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */
+ int iCol /* Index of column to read value from */
+){
+ Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor;
+
+ /* CREATE TABLE x(input, token, start, end, position) */
+ switch( iCol ){
+ case 0:
+ sqlite3_result_text(pCtx, pCsr->zInput, -1, SQLITE_TRANSIENT);
+ break;
+ case 1:
+ sqlite3_result_text(pCtx, pCsr->zToken, pCsr->nToken, SQLITE_TRANSIENT);
+ break;
+ case 2:
+ sqlite3_result_int(pCtx, pCsr->iStart);
+ break;
+ case 3:
+ sqlite3_result_int(pCtx, pCsr->iEnd);
+ break;
+ default:
+ assert( iCol==4 );
+ sqlite3_result_int(pCtx, pCsr->iPos);
+ break;
+ }
+ return SQLITE_OK;
+}
+
+/*
+** xRowid - Return the current rowid for the cursor.
+*/
+static int fts3tokRowidMethod(
+ sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
+ sqlite_int64 *pRowid /* OUT: Rowid value */
+){
+ Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor;
+ *pRowid = (sqlite3_int64)pCsr->iRowid;
+ return SQLITE_OK;
+}
+
+/*
+** Register the fts3tok module with database connection db. Return SQLITE_OK
+** if successful or an error code if sqlite3_create_module() fails.
+*/
+SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){
+ static const sqlite3_module fts3tok_module = {
+ 0, /* iVersion */
+ fts3tokConnectMethod, /* xCreate */
+ fts3tokConnectMethod, /* xConnect */
+ fts3tokBestIndexMethod, /* xBestIndex */
+ fts3tokDisconnectMethod, /* xDisconnect */
+ fts3tokDisconnectMethod, /* xDestroy */
+ fts3tokOpenMethod, /* xOpen */
+ fts3tokCloseMethod, /* xClose */
+ fts3tokFilterMethod, /* xFilter */
+ fts3tokNextMethod, /* xNext */
+ fts3tokEofMethod, /* xEof */
+ fts3tokColumnMethod, /* xColumn */
+ fts3tokRowidMethod, /* xRowid */
+ 0, /* xUpdate */
+ 0, /* xBegin */
+ 0, /* xSync */
+ 0, /* xCommit */
+ 0, /* xRollback */
+ 0, /* xFindFunction */
+ 0, /* xRename */
+ 0, /* xSavepoint */
+ 0, /* xRelease */
+ 0 /* xRollbackTo */
+ };
+ int rc; /* Return code */
+
+ rc = sqlite3_create_module(db, "fts3tokenize", &fts3tok_module, (void*)pHash);
+ return rc;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
+
+/************** End of fts3_tokenize_vtab.c **********************************/
+/************** Begin file fts3_write.c **************************************/
+/*
+** 2009 Oct 23
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file is part of the SQLite FTS3 extension module. Specifically,
+** this file contains code to insert, update and delete rows from FTS3
+** tables. It also contains code to merge FTS3 b-tree segments. Some
+** of the sub-routines used to merge segments are also used by the query
+** code in fts3.c.
+*/
+
+/* #include "fts3Int.h" */
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+/* #include <string.h> */
+/* #include <assert.h> */
+/* #include <stdlib.h> */
+
+
+#define FTS_MAX_APPENDABLE_HEIGHT 16
+
+/*
+** When full-text index nodes are loaded from disk, the buffer that they
+** are loaded into has the following number of bytes of padding at the end
+** of it. i.e. if a full-text index node is 900 bytes in size, then a buffer
+** of 920 bytes is allocated for it.
+**
+** This means that if we have a pointer into a buffer containing node data,
+** it is always safe to read up to two varints from it without risking an
+** overread, even if the node data is corrupted.
+*/
+#define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2)
+
+/*
+** Under certain circumstances, b-tree nodes (doclists) can be loaded into
+** memory incrementally instead of all at once. This can be a big performance
+** win (reduced IO and CPU) if SQLite stops calling the virtual table xNext()
+** method before retrieving all query results (as may happen, for example,
+** if a query has a LIMIT clause).
+**
+** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD
+** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes.
+** The code is written so that the hard lower-limit for each of these values
+** is 1. Clearly such small values would be inefficient, but can be useful
+** for testing purposes.
+**
+** If this module is built with SQLITE_TEST defined, these constants may
+** be overridden at runtime for testing purposes. File fts3_test.c contains
+** a Tcl interface to read and write the values.
+*/
+#ifdef SQLITE_TEST
+int test_fts3_node_chunksize = (4*1024);
+int test_fts3_node_chunk_threshold = (4*1024)*4;
+# define FTS3_NODE_CHUNKSIZE test_fts3_node_chunksize
+# define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold
+#else
+# define FTS3_NODE_CHUNKSIZE (4*1024)
+# define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4)
+#endif
+
+/*
+** The two values that may be meaningfully bound to the :1 parameter in
+** statements SQL_REPLACE_STAT and SQL_SELECT_STAT.
+*/
+#define FTS_STAT_DOCTOTAL 0
+#define FTS_STAT_INCRMERGEHINT 1
+#define FTS_STAT_AUTOINCRMERGE 2
+
+/*
+** If FTS_LOG_MERGES is defined, call sqlite3_log() to report each automatic
+** and incremental merge operation that takes place. This is used for
+** debugging FTS only, it should not usually be turned on in production
+** systems.
+*/
+#ifdef FTS3_LOG_MERGES
+static void fts3LogMerge(int nMerge, sqlite3_int64 iAbsLevel){
+ sqlite3_log(SQLITE_OK, "%d-way merge from level %d", nMerge, (int)iAbsLevel);
+}
+#else
+#define fts3LogMerge(x, y)
+#endif
+
+
+typedef struct PendingList PendingList;
+typedef struct SegmentNode SegmentNode;
+typedef struct SegmentWriter SegmentWriter;
+
+/*
+** An instance of the following data structure is used to build doclists
+** incrementally. See function fts3PendingListAppend() for details.
+*/
+struct PendingList {
+ int nData;
+ char *aData;
+ int nSpace;
+ sqlite3_int64 iLastDocid;
+ sqlite3_int64 iLastCol;
+ sqlite3_int64 iLastPos;
+};
+
+
+/*
+** Each cursor has a (possibly empty) linked list of the following objects.
+*/
+struct Fts3DeferredToken {
+ Fts3PhraseToken *pToken; /* Pointer to corresponding expr token */
+ int iCol; /* Column token must occur in */
+ Fts3DeferredToken *pNext; /* Next in list of deferred tokens */
+ PendingList *pList; /* Doclist is assembled here */
+};
+
+/*
+** An instance of this structure is used to iterate through the terms on
+** a contiguous set of segment b-tree leaf nodes. Although the details of
+** this structure are only manipulated by code in this file, opaque handles
+** of type Fts3SegReader* are also used by code in fts3.c to iterate through
+** terms when querying the full-text index. See functions:
+**
+** sqlite3Fts3SegReaderNew()
+** sqlite3Fts3SegReaderFree()
+** sqlite3Fts3SegReaderIterate()
+**
+** Methods used to manipulate Fts3SegReader structures:
+**
+** fts3SegReaderNext()
+** fts3SegReaderFirstDocid()
+** fts3SegReaderNextDocid()
+*/
+struct Fts3SegReader {
+ int iIdx; /* Index within level, or 0x7FFFFFFF for PT */
+ u8 bLookup; /* True for a lookup only */
+ u8 rootOnly; /* True for a root-only reader */
+
+ sqlite3_int64 iStartBlock; /* Rowid of first leaf block to traverse */
+ sqlite3_int64 iLeafEndBlock; /* Rowid of final leaf block to traverse */
+ sqlite3_int64 iEndBlock; /* Rowid of final block in segment (or 0) */
+ sqlite3_int64 iCurrentBlock; /* Current leaf block (or 0) */
+
+ char *aNode; /* Pointer to node data (or NULL) */
+ int nNode; /* Size of buffer at aNode (or 0) */
+ int nPopulate; /* If >0, bytes of buffer aNode[] loaded */
+ sqlite3_blob *pBlob; /* If not NULL, blob handle to read node */
+
+ Fts3HashElem **ppNextElem;
+
+ /* Variables set by fts3SegReaderNext(). These may be read directly
+ ** by the caller. They are valid from the time SegmentReaderNew() returns
+ ** until SegmentReaderNext() returns something other than SQLITE_OK
+ ** (i.e. SQLITE_DONE).
+ */
+ int nTerm; /* Number of bytes in current term */
+ char *zTerm; /* Pointer to current term */
+ int nTermAlloc; /* Allocated size of zTerm buffer */
+ char *aDoclist; /* Pointer to doclist of current entry */
+ int nDoclist; /* Size of doclist in current entry */
+
+ /* The following variables are used by fts3SegReaderNextDocid() to iterate
+ ** through the current doclist (aDoclist/nDoclist).
+ */
+ char *pOffsetList;
+ int nOffsetList; /* For descending pending seg-readers only */
+ sqlite3_int64 iDocid;
+};
+
+#define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0)
+#define fts3SegReaderIsRootOnly(p) ((p)->rootOnly!=0)
+
+/*
+** An instance of this structure is used to create a segment b-tree in the
+** database. The internal details of this type are only accessed by the
+** following functions:
+**
+** fts3SegWriterAdd()
+** fts3SegWriterFlush()
+** fts3SegWriterFree()
+*/
+struct SegmentWriter {
+ SegmentNode *pTree; /* Pointer to interior tree structure */
+ sqlite3_int64 iFirst; /* First slot in %_segments written */
+ sqlite3_int64 iFree; /* Next free slot in %_segments */
+ char *zTerm; /* Pointer to previous term buffer */
+ int nTerm; /* Number of bytes in zTerm */
+ int nMalloc; /* Size of malloc'd buffer at zMalloc */
+ char *zMalloc; /* Malloc'd space (possibly) used for zTerm */
+ int nSize; /* Size of allocation at aData */
+ int nData; /* Bytes of data in aData */
+ char *aData; /* Pointer to block from malloc() */
+ i64 nLeafData; /* Number of bytes of leaf data written */
+};
+
+/*
+** Type SegmentNode is used by the following three functions to create
+** the interior part of the segment b+-tree structures (everything except
+** the leaf nodes). These functions and type are only ever used by code
+** within the fts3SegWriterXXX() family of functions described above.
+**
+** fts3NodeAddTerm()
+** fts3NodeWrite()
+** fts3NodeFree()
+**
+** When a b+tree is written to the database (either as a result of a merge
+** or the pending-terms table being flushed), leaves are written into the
+** database file as soon as they are completely populated. The interior of
+** the tree is assembled in memory and written out only once all leaves have
+** been populated and stored. This is Ok, as the b+-tree fanout is usually
+** very large, meaning that the interior of the tree consumes relatively
+** little memory.
+*/
+struct SegmentNode {
+ SegmentNode *pParent; /* Parent node (or NULL for root node) */
+ SegmentNode *pRight; /* Pointer to right-sibling */
+ SegmentNode *pLeftmost; /* Pointer to left-most node of this depth */
+ int nEntry; /* Number of terms written to node so far */
+ char *zTerm; /* Pointer to previous term buffer */
+ int nTerm; /* Number of bytes in zTerm */
+ int nMalloc; /* Size of malloc'd buffer at zMalloc */
+ char *zMalloc; /* Malloc'd space (possibly) used for zTerm */
+ int nData; /* Bytes of valid data so far */
+ char *aData; /* Node data */
+};
+
+/*
+** Valid values for the second argument to fts3SqlStmt().
+*/
+#define SQL_DELETE_CONTENT 0
+#define SQL_IS_EMPTY 1
+#define SQL_DELETE_ALL_CONTENT 2
+#define SQL_DELETE_ALL_SEGMENTS 3
+#define SQL_DELETE_ALL_SEGDIR 4
+#define SQL_DELETE_ALL_DOCSIZE 5
+#define SQL_DELETE_ALL_STAT 6
+#define SQL_SELECT_CONTENT_BY_ROWID 7
+#define SQL_NEXT_SEGMENT_INDEX 8
+#define SQL_INSERT_SEGMENTS 9
+#define SQL_NEXT_SEGMENTS_ID 10
+#define SQL_INSERT_SEGDIR 11
+#define SQL_SELECT_LEVEL 12
+#define SQL_SELECT_LEVEL_RANGE 13
+#define SQL_SELECT_LEVEL_COUNT 14
+#define SQL_SELECT_SEGDIR_MAX_LEVEL 15
+#define SQL_DELETE_SEGDIR_LEVEL 16
+#define SQL_DELETE_SEGMENTS_RANGE 17
+#define SQL_CONTENT_INSERT 18
+#define SQL_DELETE_DOCSIZE 19
+#define SQL_REPLACE_DOCSIZE 20
+#define SQL_SELECT_DOCSIZE 21
+#define SQL_SELECT_STAT 22
+#define SQL_REPLACE_STAT 23
+
+#define SQL_SELECT_ALL_PREFIX_LEVEL 24
+#define SQL_DELETE_ALL_TERMS_SEGDIR 25
+#define SQL_DELETE_SEGDIR_RANGE 26
+#define SQL_SELECT_ALL_LANGID 27
+#define SQL_FIND_MERGE_LEVEL 28
+#define SQL_MAX_LEAF_NODE_ESTIMATE 29
+#define SQL_DELETE_SEGDIR_ENTRY 30
+#define SQL_SHIFT_SEGDIR_ENTRY 31
+#define SQL_SELECT_SEGDIR 32
+#define SQL_CHOMP_SEGDIR 33
+#define SQL_SEGMENT_IS_APPENDABLE 34
+#define SQL_SELECT_INDEXES 35
+#define SQL_SELECT_MXLEVEL 36
+
+#define SQL_SELECT_LEVEL_RANGE2 37
+#define SQL_UPDATE_LEVEL_IDX 38
+#define SQL_UPDATE_LEVEL 39
+
+/*
+** This function is used to obtain an SQLite prepared statement handle
+** for the statement identified by the second argument. If successful,
+** *pp is set to the requested statement handle and SQLITE_OK returned.
+** Otherwise, an SQLite error code is returned and *pp is set to 0.
+**
+** If argument apVal is not NULL, then it must point to an array with
+** at least as many entries as the requested statement has bound
+** parameters. The values are bound to the statements parameters before
+** returning.
+*/
+static int fts3SqlStmt(
+ Fts3Table *p, /* Virtual table handle */
+ int eStmt, /* One of the SQL_XXX constants above */
+ sqlite3_stmt **pp, /* OUT: Statement handle */
+ sqlite3_value **apVal /* Values to bind to statement */
+){
+ const char *azSql[] = {
+/* 0 */ "DELETE FROM %Q.'%q_content' WHERE rowid = ?",
+/* 1 */ "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)",
+/* 2 */ "DELETE FROM %Q.'%q_content'",
+/* 3 */ "DELETE FROM %Q.'%q_segments'",
+/* 4 */ "DELETE FROM %Q.'%q_segdir'",
+/* 5 */ "DELETE FROM %Q.'%q_docsize'",
+/* 6 */ "DELETE FROM %Q.'%q_stat'",
+/* 7 */ "SELECT %s WHERE rowid=?",
+/* 8 */ "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1",
+/* 9 */ "REPLACE INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)",
+/* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)",
+/* 11 */ "REPLACE INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)",
+
+ /* Return segments in order from oldest to newest.*/
+/* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root "
+ "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC",
+/* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root "
+ "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?"
+ "ORDER BY level DESC, idx ASC",
+
+/* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?",
+/* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?",
+
+/* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?",
+/* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?",
+/* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%s)",
+/* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?",
+/* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",
+/* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
+/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=?",
+/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(?,?)",
+/* 24 */ "",
+/* 25 */ "",
+
+/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?",
+/* 27 */ "SELECT ? UNION SELECT level / (1024 * ?) FROM %Q.'%q_segdir'",
+
+/* This statement is used to determine which level to read the input from
+** when performing an incremental merge. It returns the absolute level number
+** of the oldest level in the db that contains at least ? segments. Or,
+** if no level in the FTS index contains more than ? segments, the statement
+** returns zero rows. */
+/* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?"
+ " ORDER BY (level %% 1024) ASC LIMIT 1",
+
+/* Estimate the upper limit on the number of leaf nodes in a new segment
+** created by merging the oldest :2 segments from absolute level :1. See
+** function sqlite3Fts3Incrmerge() for details. */
+/* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) "
+ " FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?",
+
+/* SQL_DELETE_SEGDIR_ENTRY
+** Delete the %_segdir entry on absolute level :1 with index :2. */
+/* 30 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?",
+
+/* SQL_SHIFT_SEGDIR_ENTRY
+** Modify the idx value for the segment with idx=:3 on absolute level :2
+** to :1. */
+/* 31 */ "UPDATE %Q.'%q_segdir' SET idx = ? WHERE level=? AND idx=?",
+
+/* SQL_SELECT_SEGDIR
+** Read a single entry from the %_segdir table. The entry from absolute
+** level :1 with index value :2. */
+/* 32 */ "SELECT idx, start_block, leaves_end_block, end_block, root "
+ "FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?",
+
+/* SQL_CHOMP_SEGDIR
+** Update the start_block (:1) and root (:2) fields of the %_segdir
+** entry located on absolute level :3 with index :4. */
+/* 33 */ "UPDATE %Q.'%q_segdir' SET start_block = ?, root = ?"
+ "WHERE level = ? AND idx = ?",
+
+/* SQL_SEGMENT_IS_APPENDABLE
+** Return a single row if the segment with end_block=? is appendable. Or
+** no rows otherwise. */
+/* 34 */ "SELECT 1 FROM %Q.'%q_segments' WHERE blockid=? AND block IS NULL",
+
+/* SQL_SELECT_INDEXES
+** Return the list of valid segment indexes for absolute level ? */
+/* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC",
+
+/* SQL_SELECT_MXLEVEL
+** Return the largest relative level in the FTS index or indexes. */
+/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'",
+
+ /* Return segments in order from oldest to newest.*/
+/* 37 */ "SELECT level, idx, end_block "
+ "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? "
+ "ORDER BY level DESC, idx ASC",
+
+ /* Update statements used while promoting segments */
+/* 38 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=-1,idx=? "
+ "WHERE level=? AND idx=?",
+/* 39 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=? WHERE level=-1"
+
+ };
+ int rc = SQLITE_OK;
+ sqlite3_stmt *pStmt;
+
+ assert( SizeofArray(azSql)==SizeofArray(p->aStmt) );
+ assert( eStmt<SizeofArray(azSql) && eStmt>=0 );
+
+ pStmt = p->aStmt[eStmt];
+ if( !pStmt ){
+ char *zSql;
+ if( eStmt==SQL_CONTENT_INSERT ){
+ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist);
+ }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){
+ zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist);
+ }else{
+ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName);
+ }
+ if( !zSql ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL);
+ sqlite3_free(zSql);
+ assert( rc==SQLITE_OK || pStmt==0 );
+ p->aStmt[eStmt] = pStmt;
+ }
+ }
+ if( apVal ){
+ int i;
+ int nParam = sqlite3_bind_parameter_count(pStmt);
+ for(i=0; rc==SQLITE_OK && i<nParam; i++){
+ rc = sqlite3_bind_value(pStmt, i+1, apVal[i]);
+ }
+ }
+ *pp = pStmt;
+ return rc;
+}
+
+
+static int fts3SelectDocsize(
+ Fts3Table *pTab, /* FTS3 table handle */
+ sqlite3_int64 iDocid, /* Docid to bind for SQL_SELECT_DOCSIZE */
+ sqlite3_stmt **ppStmt /* OUT: Statement handle */
+){
+ sqlite3_stmt *pStmt = 0; /* Statement requested from fts3SqlStmt() */
+ int rc; /* Return code */
+
+ rc = fts3SqlStmt(pTab, SQL_SELECT_DOCSIZE, &pStmt, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int64(pStmt, 1, iDocid);
+ rc = sqlite3_step(pStmt);
+ if( rc!=SQLITE_ROW || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){
+ rc = sqlite3_reset(pStmt);
+ if( rc==SQLITE_OK ) rc = FTS_CORRUPT_VTAB;
+ pStmt = 0;
+ }else{
+ rc = SQLITE_OK;
+ }
+ }
+
+ *ppStmt = pStmt;
+ return rc;
+}
+
+SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(
+ Fts3Table *pTab, /* Fts3 table handle */
+ sqlite3_stmt **ppStmt /* OUT: Statement handle */
+){
+ sqlite3_stmt *pStmt = 0;
+ int rc;
+ rc = fts3SqlStmt(pTab, SQL_SELECT_STAT, &pStmt, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL);
+ if( sqlite3_step(pStmt)!=SQLITE_ROW
+ || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB
+ ){
+ rc = sqlite3_reset(pStmt);
+ if( rc==SQLITE_OK ) rc = FTS_CORRUPT_VTAB;
+ pStmt = 0;
+ }
+ }
+ *ppStmt = pStmt;
+ return rc;
+}
+
+SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(
+ Fts3Table *pTab, /* Fts3 table handle */
+ sqlite3_int64 iDocid, /* Docid to read size data for */
+ sqlite3_stmt **ppStmt /* OUT: Statement handle */
+){
+ return fts3SelectDocsize(pTab, iDocid, ppStmt);
+}
+
+/*
+** Similar to fts3SqlStmt(). Except, after binding the parameters in
+** array apVal[] to the SQL statement identified by eStmt, the statement
+** is executed.
+**
+** Returns SQLITE_OK if the statement is successfully executed, or an
+** SQLite error code otherwise.
+*/
+static void fts3SqlExec(
+ int *pRC, /* Result code */
+ Fts3Table *p, /* The FTS3 table */
+ int eStmt, /* Index of statement to evaluate */
+ sqlite3_value **apVal /* Parameters to bind */
+){
+ sqlite3_stmt *pStmt;
+ int rc;
+ if( *pRC ) return;
+ rc = fts3SqlStmt(p, eStmt, &pStmt, apVal);
+ if( rc==SQLITE_OK ){
+ sqlite3_step(pStmt);
+ rc = sqlite3_reset(pStmt);
+ }
+ *pRC = rc;
+}
+
+
+/*
+** This function ensures that the caller has obtained an exclusive
+** shared-cache table-lock on the %_segdir table. This is required before
+** writing data to the fts3 table. If this lock is not acquired first, then
+** the caller may end up attempting to take this lock as part of committing
+** a transaction, causing SQLite to return SQLITE_LOCKED or
+** LOCKED_SHAREDCACHEto a COMMIT command.
+**
+** It is best to avoid this because if FTS3 returns any error when
+** committing a transaction, the whole transaction will be rolled back.
+** And this is not what users expect when they get SQLITE_LOCKED_SHAREDCACHE.
+** It can still happen if the user locks the underlying tables directly
+** instead of accessing them via FTS.
+*/
+static int fts3Writelock(Fts3Table *p){
+ int rc = SQLITE_OK;
+
+ if( p->nPendingData==0 ){
+ sqlite3_stmt *pStmt;
+ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pStmt, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_null(pStmt, 1);
+ sqlite3_step(pStmt);
+ rc = sqlite3_reset(pStmt);
+ }
+ }
+
+ return rc;
+}
+
+/*
+** FTS maintains a separate indexes for each language-id (a 32-bit integer).
+** Within each language id, a separate index is maintained to store the
+** document terms, and each configured prefix size (configured the FTS
+** "prefix=" option). And each index consists of multiple levels ("relative
+** levels").
+**
+** All three of these values (the language id, the specific index and the
+** level within the index) are encoded in 64-bit integer values stored
+** in the %_segdir table on disk. This function is used to convert three
+** separate component values into the single 64-bit integer value that
+** can be used to query the %_segdir table.
+**
+** Specifically, each language-id/index combination is allocated 1024
+** 64-bit integer level values ("absolute levels"). The main terms index
+** for language-id 0 is allocate values 0-1023. The first prefix index
+** (if any) for language-id 0 is allocated values 1024-2047. And so on.
+** Language 1 indexes are allocated immediately following language 0.
+**
+** So, for a system with nPrefix prefix indexes configured, the block of
+** absolute levels that corresponds to language-id iLangid and index
+** iIndex starts at absolute level ((iLangid * (nPrefix+1) + iIndex) * 1024).
+*/
+static sqlite3_int64 getAbsoluteLevel(
+ Fts3Table *p, /* FTS3 table handle */
+ int iLangid, /* Language id */
+ int iIndex, /* Index in p->aIndex[] */
+ int iLevel /* Level of segments */
+){
+ sqlite3_int64 iBase; /* First absolute level for iLangid/iIndex */
+ assert( iLangid>=0 );
+ assert( p->nIndex>0 );
+ assert( iIndex>=0 && iIndex<p->nIndex );
+
+ iBase = ((sqlite3_int64)iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL;
+ return iBase + iLevel;
+}
+
+/*
+** Set *ppStmt to a statement handle that may be used to iterate through
+** all rows in the %_segdir table, from oldest to newest. If successful,
+** return SQLITE_OK. If an error occurs while preparing the statement,
+** return an SQLite error code.
+**
+** There is only ever one instance of this SQL statement compiled for
+** each FTS3 table.
+**
+** The statement returns the following columns from the %_segdir table:
+**
+** 0: idx
+** 1: start_block
+** 2: leaves_end_block
+** 3: end_block
+** 4: root
+*/
+SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(
+ Fts3Table *p, /* FTS3 table */
+ int iLangid, /* Language being queried */
+ int iIndex, /* Index for p->aIndex[] */
+ int iLevel, /* Level to select (relative level) */
+ sqlite3_stmt **ppStmt /* OUT: Compiled statement */
+){
+ int rc;
+ sqlite3_stmt *pStmt = 0;
+
+ assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 );
+ assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
+ assert( iIndex>=0 && iIndex<p->nIndex );
+
+ if( iLevel<0 ){
+ /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */
+ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0));
+ sqlite3_bind_int64(pStmt, 2,
+ getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1)
+ );
+ }
+ }else{
+ /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */
+ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel));
+ }
+ }
+ *ppStmt = pStmt;
+ return rc;
+}
+
+
+/*
+** Append a single varint to a PendingList buffer. SQLITE_OK is returned
+** if successful, or an SQLite error code otherwise.
+**
+** This function also serves to allocate the PendingList structure itself.
+** For example, to create a new PendingList structure containing two
+** varints:
+**
+** PendingList *p = 0;
+** fts3PendingListAppendVarint(&p, 1);
+** fts3PendingListAppendVarint(&p, 2);
+*/
+static int fts3PendingListAppendVarint(
+ PendingList **pp, /* IN/OUT: Pointer to PendingList struct */
+ sqlite3_int64 i /* Value to append to data */
+){
+ PendingList *p = *pp;
+
+ /* Allocate or grow the PendingList as required. */
+ if( !p ){
+ p = sqlite3_malloc(sizeof(*p) + 100);
+ if( !p ){
+ return SQLITE_NOMEM;
+ }
+ p->nSpace = 100;
+ p->aData = (char *)&p[1];
+ p->nData = 0;
+ }
+ else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){
+ int nNew = p->nSpace * 2;
+ p = sqlite3_realloc(p, sizeof(*p) + nNew);
+ if( !p ){
+ sqlite3_free(*pp);
+ *pp = 0;
+ return SQLITE_NOMEM;
+ }
+ p->nSpace = nNew;
+ p->aData = (char *)&p[1];
+ }
+
+ /* Append the new serialized varint to the end of the list. */
+ p->nData += sqlite3Fts3PutVarint(&p->aData[p->nData], i);
+ p->aData[p->nData] = '\0';
+ *pp = p;
+ return SQLITE_OK;
+}
+
+/*
+** Add a docid/column/position entry to a PendingList structure. Non-zero
+** is returned if the structure is sqlite3_realloced as part of adding
+** the entry. Otherwise, zero.
+**
+** If an OOM error occurs, *pRc is set to SQLITE_NOMEM before returning.
+** Zero is always returned in this case. Otherwise, if no OOM error occurs,
+** it is set to SQLITE_OK.
+*/
+static int fts3PendingListAppend(
+ PendingList **pp, /* IN/OUT: PendingList structure */
+ sqlite3_int64 iDocid, /* Docid for entry to add */
+ sqlite3_int64 iCol, /* Column for entry to add */
+ sqlite3_int64 iPos, /* Position of term for entry to add */
+ int *pRc /* OUT: Return code */
+){
+ PendingList *p = *pp;
+ int rc = SQLITE_OK;
+
+ assert( !p || p->iLastDocid<=iDocid );
+
+ if( !p || p->iLastDocid!=iDocid ){
+ sqlite3_int64 iDelta = iDocid - (p ? p->iLastDocid : 0);
+ if( p ){
+ assert( p->nData<p->nSpace );
+ assert( p->aData[p->nData]==0 );
+ p->nData++;
+ }
+ if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iDelta)) ){
+ goto pendinglistappend_out;
+ }
+ p->iLastCol = -1;
+ p->iLastPos = 0;
+ p->iLastDocid = iDocid;
+ }
+ if( iCol>0 && p->iLastCol!=iCol ){
+ if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, 1))
+ || SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iCol))
+ ){
+ goto pendinglistappend_out;
+ }
+ p->iLastCol = iCol;
+ p->iLastPos = 0;
+ }
+ if( iCol>=0 ){
+ assert( iPos>p->iLastPos || (iPos==0 && p->iLastPos==0) );
+ rc = fts3PendingListAppendVarint(&p, 2+iPos-p->iLastPos);
+ if( rc==SQLITE_OK ){
+ p->iLastPos = iPos;
+ }
+ }
+
+ pendinglistappend_out:
+ *pRc = rc;
+ if( p!=*pp ){
+ *pp = p;
+ return 1;
+ }
+ return 0;
+}
+
+/*
+** Free a PendingList object allocated by fts3PendingListAppend().
+*/
+static void fts3PendingListDelete(PendingList *pList){
+ sqlite3_free(pList);
+}
+
+/*
+** Add an entry to one of the pending-terms hash tables.
+*/
+static int fts3PendingTermsAddOne(
+ Fts3Table *p,
+ int iCol,
+ int iPos,
+ Fts3Hash *pHash, /* Pending terms hash table to add entry to */
+ const char *zToken,
+ int nToken
+){
+ PendingList *pList;
+ int rc = SQLITE_OK;
+
+ pList = (PendingList *)fts3HashFind(pHash, zToken, nToken);
+ if( pList ){
+ p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem));
+ }
+ if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){
+ if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){
+ /* Malloc failed while inserting the new entry. This can only
+ ** happen if there was no previous entry for this token.
+ */
+ assert( 0==fts3HashFind(pHash, zToken, nToken) );
+ sqlite3_free(pList);
+ rc = SQLITE_NOMEM;
+ }
+ }
+ if( rc==SQLITE_OK ){
+ p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem));
+ }
+ return rc;
+}
+
+/*
+** Tokenize the nul-terminated string zText and add all tokens to the
+** pending-terms hash-table. The docid used is that currently stored in
+** p->iPrevDocid, and the column is specified by argument iCol.
+**
+** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code.
+*/
+static int fts3PendingTermsAdd(
+ Fts3Table *p, /* Table into which text will be inserted */
+ int iLangid, /* Language id to use */
+ const char *zText, /* Text of document to be inserted */
+ int iCol, /* Column into which text is being inserted */
+ u32 *pnWord /* IN/OUT: Incr. by number tokens inserted */
+){
+ int rc;
+ int iStart = 0;
+ int iEnd = 0;
+ int iPos = 0;
+ int nWord = 0;
+
+ char const *zToken;
+ int nToken = 0;
+
+ sqlite3_tokenizer *pTokenizer = p->pTokenizer;
+ sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
+ sqlite3_tokenizer_cursor *pCsr;
+ int (*xNext)(sqlite3_tokenizer_cursor *pCursor,
+ const char**,int*,int*,int*,int*);
+
+ assert( pTokenizer && pModule );
+
+ /* If the user has inserted a NULL value, this function may be called with
+ ** zText==0. In this case, add zero token entries to the hash table and
+ ** return early. */
+ if( zText==0 ){
+ *pnWord = 0;
+ return SQLITE_OK;
+ }
+
+ rc = sqlite3Fts3OpenTokenizer(pTokenizer, iLangid, zText, -1, &pCsr);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+
+ xNext = pModule->xNext;
+ while( SQLITE_OK==rc
+ && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos))
+ ){
+ int i;
+ if( iPos>=nWord ) nWord = iPos+1;
+
+ /* Positions cannot be negative; we use -1 as a terminator internally.
+ ** Tokens must have a non-zero length.
+ */
+ if( iPos<0 || !zToken || nToken<=0 ){
+ rc = SQLITE_ERROR;
+ break;
+ }
+
+ /* Add the term to the terms index */
+ rc = fts3PendingTermsAddOne(
+ p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken
+ );
+
+ /* Add the term to each of the prefix indexes that it is not too
+ ** short for. */
+ for(i=1; rc==SQLITE_OK && i<p->nIndex; i++){
+ struct Fts3Index *pIndex = &p->aIndex[i];
+ if( nToken<pIndex->nPrefix ) continue;
+ rc = fts3PendingTermsAddOne(
+ p, iCol, iPos, &pIndex->hPending, zToken, pIndex->nPrefix
+ );
+ }
+ }
+
+ pModule->xClose(pCsr);
+ *pnWord += nWord;
+ return (rc==SQLITE_DONE ? SQLITE_OK : rc);
+}
+
+/*
+** Calling this function indicates that subsequent calls to
+** fts3PendingTermsAdd() are to add term/position-list pairs for the
+** contents of the document with docid iDocid.
+*/
+static int fts3PendingTermsDocid(
+ Fts3Table *p, /* Full-text table handle */
+ int bDelete, /* True if this op is a delete */
+ int iLangid, /* Language id of row being written */
+ sqlite_int64 iDocid /* Docid of row being written */
+){
+ assert( iLangid>=0 );
+ assert( bDelete==1 || bDelete==0 );
+
+ /* TODO(shess) Explore whether partially flushing the buffer on
+ ** forced-flush would provide better performance. I suspect that if
+ ** we ordered the doclists by size and flushed the largest until the
+ ** buffer was half empty, that would let the less frequent terms
+ ** generate longer doclists.
+ */
+ if( iDocid<p->iPrevDocid
+ || (iDocid==p->iPrevDocid && p->bPrevDelete==0)
+ || p->iPrevLangid!=iLangid
+ || p->nPendingData>p->nMaxPendingData
+ ){
+ int rc = sqlite3Fts3PendingTermsFlush(p);
+ if( rc!=SQLITE_OK ) return rc;
+ }
+ p->iPrevDocid = iDocid;
+ p->iPrevLangid = iLangid;
+ p->bPrevDelete = bDelete;
+ return SQLITE_OK;
+}
+
+/*
+** Discard the contents of the pending-terms hash tables.
+*/
+SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *p){
+ int i;
+ for(i=0; i<p->nIndex; i++){
+ Fts3HashElem *pElem;
+ Fts3Hash *pHash = &p->aIndex[i].hPending;
+ for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){
+ PendingList *pList = (PendingList *)fts3HashData(pElem);
+ fts3PendingListDelete(pList);
+ }
+ fts3HashClear(pHash);
+ }
+ p->nPendingData = 0;
+}
+
+/*
+** This function is called by the xUpdate() method as part of an INSERT
+** operation. It adds entries for each term in the new record to the
+** pendingTerms hash table.
+**
+** Argument apVal is the same as the similarly named argument passed to
+** fts3InsertData(). Parameter iDocid is the docid of the new row.
+*/
+static int fts3InsertTerms(
+ Fts3Table *p,
+ int iLangid,
+ sqlite3_value **apVal,
+ u32 *aSz
+){
+ int i; /* Iterator variable */
+ for(i=2; i<p->nColumn+2; i++){
+ int iCol = i-2;
+ if( p->abNotindexed[iCol]==0 ){
+ const char *zText = (const char *)sqlite3_value_text(apVal[i]);
+ int rc = fts3PendingTermsAdd(p, iLangid, zText, iCol, &aSz[iCol]);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+ aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]);
+ }
+ }
+ return SQLITE_OK;
+}
+
+/*
+** This function is called by the xUpdate() method for an INSERT operation.
+** The apVal parameter is passed a copy of the apVal argument passed by
+** SQLite to the xUpdate() method. i.e:
+**
+** apVal[0] Not used for INSERT.
+** apVal[1] rowid
+** apVal[2] Left-most user-defined column
+** ...
+** apVal[p->nColumn+1] Right-most user-defined column
+** apVal[p->nColumn+2] Hidden column with same name as table
+** apVal[p->nColumn+3] Hidden "docid" column (alias for rowid)
+** apVal[p->nColumn+4] Hidden languageid column
+*/
+static int fts3InsertData(
+ Fts3Table *p, /* Full-text table */
+ sqlite3_value **apVal, /* Array of values to insert */
+ sqlite3_int64 *piDocid /* OUT: Docid for row just inserted */
+){
+ int rc; /* Return code */
+ sqlite3_stmt *pContentInsert; /* INSERT INTO %_content VALUES(...) */
+
+ if( p->zContentTbl ){
+ sqlite3_value *pRowid = apVal[p->nColumn+3];
+ if( sqlite3_value_type(pRowid)==SQLITE_NULL ){
+ pRowid = apVal[1];
+ }
+ if( sqlite3_value_type(pRowid)!=SQLITE_INTEGER ){
+ return SQLITE_CONSTRAINT;
+ }
+ *piDocid = sqlite3_value_int64(pRowid);
+ return SQLITE_OK;
+ }
+
+ /* Locate the statement handle used to insert data into the %_content
+ ** table. The SQL for this statement is:
+ **
+ ** INSERT INTO %_content VALUES(?, ?, ?, ...)
+ **
+ ** The statement features N '?' variables, where N is the number of user
+ ** defined columns in the FTS3 table, plus one for the docid field.
+ */
+ rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]);
+ if( rc==SQLITE_OK && p->zLanguageid ){
+ rc = sqlite3_bind_int(
+ pContentInsert, p->nColumn+2,
+ sqlite3_value_int(apVal[p->nColumn+4])
+ );
+ }
+ if( rc!=SQLITE_OK ) return rc;
+
+ /* There is a quirk here. The users INSERT statement may have specified
+ ** a value for the "rowid" field, for the "docid" field, or for both.
+ ** Which is a problem, since "rowid" and "docid" are aliases for the
+ ** same value. For example:
+ **
+ ** INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2);
+ **
+ ** In FTS3, this is an error. It is an error to specify non-NULL values
+ ** for both docid and some other rowid alias.
+ */
+ if( SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){
+ if( SQLITE_NULL==sqlite3_value_type(apVal[0])
+ && SQLITE_NULL!=sqlite3_value_type(apVal[1])
+ ){
+ /* A rowid/docid conflict. */
+ return SQLITE_ERROR;
+ }
+ rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]);
+ if( rc!=SQLITE_OK ) return rc;
+ }
+
+ /* Execute the statement to insert the record. Set *piDocid to the
+ ** new docid value.
+ */
+ sqlite3_step(pContentInsert);
+ rc = sqlite3_reset(pContentInsert);
+
+ *piDocid = sqlite3_last_insert_rowid(p->db);
+ return rc;
+}
+
+
+
+/*
+** Remove all data from the FTS3 table. Clear the hash table containing
+** pending terms.
+*/
+static int fts3DeleteAll(Fts3Table *p, int bContent){
+ int rc = SQLITE_OK; /* Return code */
+
+ /* Discard the contents of the pending-terms hash table. */
+ sqlite3Fts3PendingTermsClear(p);
+
+ /* Delete everything from the shadow tables. Except, leave %_content as
+ ** is if bContent is false. */
+ assert( p->zContentTbl==0 || bContent==0 );
+ if( bContent ) fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0);
+ fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGMENTS, 0);
+ fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0);
+ if( p->bHasDocsize ){
+ fts3SqlExec(&rc, p, SQL_DELETE_ALL_DOCSIZE, 0);
+ }
+ if( p->bHasStat ){
+ fts3SqlExec(&rc, p, SQL_DELETE_ALL_STAT, 0);
+ }
+ return rc;
+}
+
+/*
+**
+*/
+static int langidFromSelect(Fts3Table *p, sqlite3_stmt *pSelect){
+ int iLangid = 0;
+ if( p->zLanguageid ) iLangid = sqlite3_column_int(pSelect, p->nColumn+1);
+ return iLangid;
+}
+
+/*
+** The first element in the apVal[] array is assumed to contain the docid
+** (an integer) of a row about to be deleted. Remove all terms from the
+** full-text index.
+*/
+static void fts3DeleteTerms(
+ int *pRC, /* Result code */
+ Fts3Table *p, /* The FTS table to delete from */
+ sqlite3_value *pRowid, /* The docid to be deleted */
+ u32 *aSz, /* Sizes of deleted document written here */
+ int *pbFound /* OUT: Set to true if row really does exist */
+){
+ int rc;
+ sqlite3_stmt *pSelect;
+
+ assert( *pbFound==0 );
+ if( *pRC ) return;
+ rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, &pRowid);
+ if( rc==SQLITE_OK ){
+ if( SQLITE_ROW==sqlite3_step(pSelect) ){
+ int i;
+ int iLangid = langidFromSelect(p, pSelect);
+ i64 iDocid = sqlite3_column_int64(pSelect, 0);
+ rc = fts3PendingTermsDocid(p, 1, iLangid, iDocid);
+ for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){
+ int iCol = i-1;
+ if( p->abNotindexed[iCol]==0 ){
+ const char *zText = (const char *)sqlite3_column_text(pSelect, i);
+ rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[iCol]);
+ aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i);
+ }
+ }
+ if( rc!=SQLITE_OK ){
+ sqlite3_reset(pSelect);
+ *pRC = rc;
+ return;
+ }
+ *pbFound = 1;
+ }
+ rc = sqlite3_reset(pSelect);
+ }else{
+ sqlite3_reset(pSelect);
+ }
+ *pRC = rc;
+}
+
+/*
+** Forward declaration to account for the circular dependency between
+** functions fts3SegmentMerge() and fts3AllocateSegdirIdx().
+*/
+static int fts3SegmentMerge(Fts3Table *, int, int, int);
+
+/*
+** This function allocates a new level iLevel index in the segdir table.
+** Usually, indexes are allocated within a level sequentially starting
+** with 0, so the allocated index is one greater than the value returned
+** by:
+**
+** SELECT max(idx) FROM %_segdir WHERE level = :iLevel
+**
+** However, if there are already FTS3_MERGE_COUNT indexes at the requested
+** level, they are merged into a single level (iLevel+1) segment and the
+** allocated index is 0.
+**
+** If successful, *piIdx is set to the allocated index slot and SQLITE_OK
+** returned. Otherwise, an SQLite error code is returned.
+*/
+static int fts3AllocateSegdirIdx(
+ Fts3Table *p,
+ int iLangid, /* Language id */
+ int iIndex, /* Index for p->aIndex */
+ int iLevel,
+ int *piIdx
+){
+ int rc; /* Return Code */
+ sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */
+ int iNext = 0; /* Result of query pNextIdx */
+
+ assert( iLangid>=0 );
+ assert( p->nIndex>=1 );
+
+ /* Set variable iNext to the next available segdir index at level iLevel. */
+ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int64(
+ pNextIdx, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel)
+ );
+ if( SQLITE_ROW==sqlite3_step(pNextIdx) ){
+ iNext = sqlite3_column_int(pNextIdx, 0);
+ }
+ rc = sqlite3_reset(pNextIdx);
+ }
+
+ if( rc==SQLITE_OK ){
+ /* If iNext is FTS3_MERGE_COUNT, indicating that level iLevel is already
+ ** full, merge all segments in level iLevel into a single iLevel+1
+ ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise,
+ ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext.
+ */
+ if( iNext>=FTS3_MERGE_COUNT ){
+ fts3LogMerge(16, getAbsoluteLevel(p, iLangid, iIndex, iLevel));
+ rc = fts3SegmentMerge(p, iLangid, iIndex, iLevel);
+ *piIdx = 0;
+ }else{
+ *piIdx = iNext;
+ }
+ }
+
+ return rc;
+}
+
+/*
+** The %_segments table is declared as follows:
+**
+** CREATE TABLE %_segments(blockid INTEGER PRIMARY KEY, block BLOB)
+**
+** This function reads data from a single row of the %_segments table. The
+** specific row is identified by the iBlockid parameter. If paBlob is not
+** NULL, then a buffer is allocated using sqlite3_malloc() and populated
+** with the contents of the blob stored in the "block" column of the
+** identified table row is. Whether or not paBlob is NULL, *pnBlob is set
+** to the size of the blob in bytes before returning.
+**
+** If an error occurs, or the table does not contain the specified row,
+** an SQLite error code is returned. Otherwise, SQLITE_OK is returned. If
+** paBlob is non-NULL, then it is the responsibility of the caller to
+** eventually free the returned buffer.
+**
+** This function may leave an open sqlite3_blob* handle in the
+** Fts3Table.pSegments variable. This handle is reused by subsequent calls
+** to this function. The handle may be closed by calling the
+** sqlite3Fts3SegmentsClose() function. Reusing a blob handle is a handy
+** performance improvement, but the blob handle should always be closed
+** before control is returned to the user (to prevent a lock being held
+** on the database file for longer than necessary). Thus, any virtual table
+** method (xFilter etc.) that may directly or indirectly call this function
+** must call sqlite3Fts3SegmentsClose() before returning.
+*/
+SQLITE_PRIVATE int sqlite3Fts3ReadBlock(
+ Fts3Table *p, /* FTS3 table handle */
+ sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */
+ char **paBlob, /* OUT: Blob data in malloc'd buffer */
+ int *pnBlob, /* OUT: Size of blob data */
+ int *pnLoad /* OUT: Bytes actually loaded */
+){
+ int rc; /* Return code */
+
+ /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */
+ assert( pnBlob );
+
+ if( p->pSegments ){
+ rc = sqlite3_blob_reopen(p->pSegments, iBlockid);
+ }else{
+ if( 0==p->zSegmentsTbl ){
+ p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName);
+ if( 0==p->zSegmentsTbl ) return SQLITE_NOMEM;
+ }
+ rc = sqlite3_blob_open(
+ p->db, p->zDb, p->zSegmentsTbl, "block", iBlockid, 0, &p->pSegments
+ );
+ }
+
+ if( rc==SQLITE_OK ){
+ int nByte = sqlite3_blob_bytes(p->pSegments);
+ *pnBlob = nByte;
+ if( paBlob ){
+ char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING);
+ if( !aByte ){
+ rc = SQLITE_NOMEM;
+ }else{
+ if( pnLoad && nByte>(FTS3_NODE_CHUNK_THRESHOLD) ){
+ nByte = FTS3_NODE_CHUNKSIZE;
+ *pnLoad = nByte;
+ }
+ rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0);
+ memset(&aByte[nByte], 0, FTS3_NODE_PADDING);
+ if( rc!=SQLITE_OK ){
+ sqlite3_free(aByte);
+ aByte = 0;
+ }
+ }
+ *paBlob = aByte;
+ }
+ }
+
+ return rc;
+}
+
+/*
+** Close the blob handle at p->pSegments, if it is open. See comments above
+** the sqlite3Fts3ReadBlock() function for details.
+*/
+SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *p){
+ sqlite3_blob_close(p->pSegments);
+ p->pSegments = 0;
+}
+
+static int fts3SegReaderIncrRead(Fts3SegReader *pReader){
+ int nRead; /* Number of bytes to read */
+ int rc; /* Return code */
+
+ nRead = MIN(pReader->nNode - pReader->nPopulate, FTS3_NODE_CHUNKSIZE);
+ rc = sqlite3_blob_read(
+ pReader->pBlob,
+ &pReader->aNode[pReader->nPopulate],
+ nRead,
+ pReader->nPopulate
+ );
+
+ if( rc==SQLITE_OK ){
+ pReader->nPopulate += nRead;
+ memset(&pReader->aNode[pReader->nPopulate], 0, FTS3_NODE_PADDING);
+ if( pReader->nPopulate==pReader->nNode ){
+ sqlite3_blob_close(pReader->pBlob);
+ pReader->pBlob = 0;
+ pReader->nPopulate = 0;
+ }
+ }
+ return rc;
+}
+
+static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){
+ int rc = SQLITE_OK;
+ assert( !pReader->pBlob
+ || (pFrom>=pReader->aNode && pFrom<&pReader->aNode[pReader->nNode])
+ );
+ while( pReader->pBlob && rc==SQLITE_OK
+ && (pFrom - pReader->aNode + nByte)>pReader->nPopulate
+ ){
+ rc = fts3SegReaderIncrRead(pReader);
+ }
+ return rc;
+}
+
+/*
+** Set an Fts3SegReader cursor to point at EOF.
+*/
+static void fts3SegReaderSetEof(Fts3SegReader *pSeg){
+ if( !fts3SegReaderIsRootOnly(pSeg) ){
+ sqlite3_free(pSeg->aNode);
+ sqlite3_blob_close(pSeg->pBlob);
+ pSeg->pBlob = 0;
+ }
+ pSeg->aNode = 0;
+}
+
+/*
+** Move the iterator passed as the first argument to the next term in the
+** segment. If successful, SQLITE_OK is returned. If there is no next term,
+** SQLITE_DONE. Otherwise, an SQLite error code.
+*/
+static int fts3SegReaderNext(
+ Fts3Table *p,
+ Fts3SegReader *pReader,
+ int bIncr
+){
+ int rc; /* Return code of various sub-routines */
+ char *pNext; /* Cursor variable */
+ int nPrefix; /* Number of bytes in term prefix */
+ int nSuffix; /* Number of bytes in term suffix */
+
+ if( !pReader->aDoclist ){
+ pNext = pReader->aNode;
+ }else{
+ pNext = &pReader->aDoclist[pReader->nDoclist];
+ }
+
+ if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){
+
+ if( fts3SegReaderIsPending(pReader) ){
+ Fts3HashElem *pElem = *(pReader->ppNextElem);
+ sqlite3_free(pReader->aNode);
+ pReader->aNode = 0;
+ if( pElem ){
+ char *aCopy;
+ PendingList *pList = (PendingList *)fts3HashData(pElem);
+ int nCopy = pList->nData+1;
+ pReader->zTerm = (char *)fts3HashKey(pElem);
+ pReader->nTerm = fts3HashKeysize(pElem);
+ aCopy = (char*)sqlite3_malloc(nCopy);
+ if( !aCopy ) return SQLITE_NOMEM;
+ memcpy(aCopy, pList->aData, nCopy);
+ pReader->nNode = pReader->nDoclist = nCopy;
+ pReader->aNode = pReader->aDoclist = aCopy;
+ pReader->ppNextElem++;
+ assert( pReader->aNode );
+ }
+ return SQLITE_OK;
+ }
+
+ fts3SegReaderSetEof(pReader);
+
+ /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf
+ ** blocks have already been traversed. */
+ assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock );
+ if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){
+ return SQLITE_OK;
+ }
+
+ rc = sqlite3Fts3ReadBlock(
+ p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode,
+ (bIncr ? &pReader->nPopulate : 0)
+ );
+ if( rc!=SQLITE_OK ) return rc;
+ assert( pReader->pBlob==0 );
+ if( bIncr && pReader->nPopulate<pReader->nNode ){
+ pReader->pBlob = p->pSegments;
+ p->pSegments = 0;
+ }
+ pNext = pReader->aNode;
+ }
+
+ assert( !fts3SegReaderIsPending(pReader) );
+
+ rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2);
+ if( rc!=SQLITE_OK ) return rc;
+
+ /* Because of the FTS3_NODE_PADDING bytes of padding, the following is
+ ** safe (no risk of overread) even if the node data is corrupted. */
+ pNext += fts3GetVarint32(pNext, &nPrefix);
+ pNext += fts3GetVarint32(pNext, &nSuffix);
+ if( nPrefix<0 || nSuffix<=0
+ || &pNext[nSuffix]>&pReader->aNode[pReader->nNode]
+ ){
+ return FTS_CORRUPT_VTAB;
+ }
+
+ if( nPrefix+nSuffix>pReader->nTermAlloc ){
+ int nNew = (nPrefix+nSuffix)*2;
+ char *zNew = sqlite3_realloc(pReader->zTerm, nNew);
+ if( !zNew ){
+ return SQLITE_NOMEM;
+ }
+ pReader->zTerm = zNew;
+ pReader->nTermAlloc = nNew;
+ }
+
+ rc = fts3SegReaderRequire(pReader, pNext, nSuffix+FTS3_VARINT_MAX);
+ if( rc!=SQLITE_OK ) return rc;
+
+ memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix);
+ pReader->nTerm = nPrefix+nSuffix;
+ pNext += nSuffix;
+ pNext += fts3GetVarint32(pNext, &pReader->nDoclist);
+ pReader->aDoclist = pNext;
+ pReader->pOffsetList = 0;
+
+ /* Check that the doclist does not appear to extend past the end of the
+ ** b-tree node. And that the final byte of the doclist is 0x00. If either
+ ** of these statements is untrue, then the data structure is corrupt.
+ */
+ if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode]
+ || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1])
+ ){
+ return FTS_CORRUPT_VTAB;
+ }
+ return SQLITE_OK;
+}
+
+/*
+** Set the SegReader to point to the first docid in the doclist associated
+** with the current term.
+*/
+static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){
+ int rc = SQLITE_OK;
+ assert( pReader->aDoclist );
+ assert( !pReader->pOffsetList );
+ if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){
+ u8 bEof = 0;
+ pReader->iDocid = 0;
+ pReader->nOffsetList = 0;
+ sqlite3Fts3DoclistPrev(0,
+ pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList,
+ &pReader->iDocid, &pReader->nOffsetList, &bEof
+ );
+ }else{
+ rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX);
+ if( rc==SQLITE_OK ){
+ int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid);
+ pReader->pOffsetList = &pReader->aDoclist[n];
+ }
+ }
+ return rc;
+}
+
+/*
+** Advance the SegReader to point to the next docid in the doclist
+** associated with the current term.
+**
+** If arguments ppOffsetList and pnOffsetList are not NULL, then
+** *ppOffsetList is set to point to the first column-offset list
+** in the doclist entry (i.e. immediately past the docid varint).
+** *pnOffsetList is set to the length of the set of column-offset
+** lists, not including the nul-terminator byte. For example:
+*/
+static int fts3SegReaderNextDocid(
+ Fts3Table *pTab,
+ Fts3SegReader *pReader, /* Reader to advance to next docid */
+ char **ppOffsetList, /* OUT: Pointer to current position-list */
+ int *pnOffsetList /* OUT: Length of *ppOffsetList in bytes */
+){
+ int rc = SQLITE_OK;
+ char *p = pReader->pOffsetList;
+ char c = 0;
+
+ assert( p );
+
+ if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){
+ /* A pending-terms seg-reader for an FTS4 table that uses order=desc.
+ ** Pending-terms doclists are always built up in ascending order, so
+ ** we have to iterate through them backwards here. */
+ u8 bEof = 0;
+ if( ppOffsetList ){
+ *ppOffsetList = pReader->pOffsetList;
+ *pnOffsetList = pReader->nOffsetList - 1;
+ }
+ sqlite3Fts3DoclistPrev(0,
+ pReader->aDoclist, pReader->nDoclist, &p, &pReader->iDocid,
+ &pReader->nOffsetList, &bEof
+ );
+ if( bEof ){
+ pReader->pOffsetList = 0;
+ }else{
+ pReader->pOffsetList = p;
+ }
+ }else{
+ char *pEnd = &pReader->aDoclist[pReader->nDoclist];
+
+ /* Pointer p currently points at the first byte of an offset list. The
+ ** following block advances it to point one byte past the end of
+ ** the same offset list. */
+ while( 1 ){
+
+ /* The following line of code (and the "p++" below the while() loop) is
+ ** normally all that is required to move pointer p to the desired
+ ** position. The exception is if this node is being loaded from disk
+ ** incrementally and pointer "p" now points to the first byte past
+ ** the populated part of pReader->aNode[].
+ */
+ while( *p | c ) c = *p++ & 0x80;
+ assert( *p==0 );
+
+ if( pReader->pBlob==0 || p<&pReader->aNode[pReader->nPopulate] ) break;
+ rc = fts3SegReaderIncrRead(pReader);
+ if( rc!=SQLITE_OK ) return rc;
+ }
+ p++;
+
+ /* If required, populate the output variables with a pointer to and the
+ ** size of the previous offset-list.
+ */
+ if( ppOffsetList ){
+ *ppOffsetList = pReader->pOffsetList;
+ *pnOffsetList = (int)(p - pReader->pOffsetList - 1);
+ }
+
+ /* List may have been edited in place by fts3EvalNearTrim() */
+ while( p<pEnd && *p==0 ) p++;
+
+ /* If there are no more entries in the doclist, set pOffsetList to
+ ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and
+ ** Fts3SegReader.pOffsetList to point to the next offset list before
+ ** returning.
+ */
+ if( p>=pEnd ){
+ pReader->pOffsetList = 0;
+ }else{
+ rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX);
+ if( rc==SQLITE_OK ){
+ sqlite3_int64 iDelta;
+ pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
+ if( pTab->bDescIdx ){
+ pReader->iDocid -= iDelta;
+ }else{
+ pReader->iDocid += iDelta;
+ }
+ }
+ }
+ }
+
+ return SQLITE_OK;
+}
+
+
+SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(
+ Fts3Cursor *pCsr,
+ Fts3MultiSegReader *pMsr,
+ int *pnOvfl
+){
+ Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
+ int nOvfl = 0;
+ int ii;
+ int rc = SQLITE_OK;
+ int pgsz = p->nPgsz;
+
+ assert( p->bFts4 );
+ assert( pgsz>0 );
+
+ for(ii=0; rc==SQLITE_OK && ii<pMsr->nSegment; ii++){
+ Fts3SegReader *pReader = pMsr->apSegment[ii];
+ if( !fts3SegReaderIsPending(pReader)
+ && !fts3SegReaderIsRootOnly(pReader)
+ ){
+ sqlite3_int64 jj;
+ for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){
+ int nBlob;
+ rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob, 0);
+ if( rc!=SQLITE_OK ) break;
+ if( (nBlob+35)>pgsz ){
+ nOvfl += (nBlob + 34)/pgsz;
+ }
+ }
+ }
+ }
+ *pnOvfl = nOvfl;
+ return rc;
+}
+
+/*
+** Free all allocations associated with the iterator passed as the
+** second argument.
+*/
+SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){
+ if( pReader ){
+ if( !fts3SegReaderIsPending(pReader) ){
+ sqlite3_free(pReader->zTerm);
+ }
+ if( !fts3SegReaderIsRootOnly(pReader) ){
+ sqlite3_free(pReader->aNode);
+ }
+ sqlite3_blob_close(pReader->pBlob);
+ }
+ sqlite3_free(pReader);
+}
+
+/*
+** Allocate a new SegReader object.
+*/
+SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(
+ int iAge, /* Segment "age". */
+ int bLookup, /* True for a lookup only */
+ sqlite3_int64 iStartLeaf, /* First leaf to traverse */
+ sqlite3_int64 iEndLeaf, /* Final leaf to traverse */
+ sqlite3_int64 iEndBlock, /* Final block of segment */
+ const char *zRoot, /* Buffer containing root node */
+ int nRoot, /* Size of buffer containing root node */
+ Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */
+){
+ Fts3SegReader *pReader; /* Newly allocated SegReader object */
+ int nExtra = 0; /* Bytes to allocate segment root node */
+
+ assert( iStartLeaf<=iEndLeaf );
+ if( iStartLeaf==0 ){
+ nExtra = nRoot + FTS3_NODE_PADDING;
+ }
+
+ pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra);
+ if( !pReader ){
+ return SQLITE_NOMEM;
+ }
+ memset(pReader, 0, sizeof(Fts3SegReader));
+ pReader->iIdx = iAge;
+ pReader->bLookup = bLookup!=0;
+ pReader->iStartBlock = iStartLeaf;
+ pReader->iLeafEndBlock = iEndLeaf;
+ pReader->iEndBlock = iEndBlock;
+
+ if( nExtra ){
+ /* The entire segment is stored in the root node. */
+ pReader->aNode = (char *)&pReader[1];
+ pReader->rootOnly = 1;
+ pReader->nNode = nRoot;
+ memcpy(pReader->aNode, zRoot, nRoot);
+ memset(&pReader->aNode[nRoot], 0, FTS3_NODE_PADDING);
+ }else{
+ pReader->iCurrentBlock = iStartLeaf-1;
+ }
+ *ppReader = pReader;
+ return SQLITE_OK;
+}
+
+/*
+** This is a comparison function used as a qsort() callback when sorting
+** an array of pending terms by term. This occurs as part of flushing
+** the contents of the pending-terms hash table to the database.
+*/
+static int SQLITE_CDECL fts3CompareElemByTerm(
+ const void *lhs,
+ const void *rhs
+){
+ char *z1 = fts3HashKey(*(Fts3HashElem **)lhs);
+ char *z2 = fts3HashKey(*(Fts3HashElem **)rhs);
+ int n1 = fts3HashKeysize(*(Fts3HashElem **)lhs);
+ int n2 = fts3HashKeysize(*(Fts3HashElem **)rhs);
+
+ int n = (n1<n2 ? n1 : n2);
+ int c = memcmp(z1, z2, n);
+ if( c==0 ){
+ c = n1 - n2;
+ }
+ return c;
+}
+
+/*
+** This function is used to allocate an Fts3SegReader that iterates through
+** a subset of the terms stored in the Fts3Table.pendingTerms array.
+**
+** If the isPrefixIter parameter is zero, then the returned SegReader iterates
+** through each term in the pending-terms table. Or, if isPrefixIter is
+** non-zero, it iterates through each term and its prefixes. For example, if
+** the pending terms hash table contains the terms "sqlite", "mysql" and
+** "firebird", then the iterator visits the following 'terms' (in the order
+** shown):
+**
+** f fi fir fire fireb firebi firebir firebird
+** m my mys mysq mysql
+** s sq sql sqli sqlit sqlite
+**
+** Whereas if isPrefixIter is zero, the terms visited are:
+**
+** firebird mysql sqlite
+*/
+SQLITE_PRIVATE int sqlite3Fts3SegReaderPending(
+ Fts3Table *p, /* Virtual table handle */
+ int iIndex, /* Index for p->aIndex */
+ const char *zTerm, /* Term to search for */
+ int nTerm, /* Size of buffer zTerm */
+ int bPrefix, /* True for a prefix iterator */
+ Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */
+){
+ Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */
+ Fts3HashElem *pE; /* Iterator variable */
+ Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */
+ int nElem = 0; /* Size of array at aElem */
+ int rc = SQLITE_OK; /* Return Code */
+ Fts3Hash *pHash;
+
+ pHash = &p->aIndex[iIndex].hPending;
+ if( bPrefix ){
+ int nAlloc = 0; /* Size of allocated array at aElem */
+
+ for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){
+ char *zKey = (char *)fts3HashKey(pE);
+ int nKey = fts3HashKeysize(pE);
+ if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){
+ if( nElem==nAlloc ){
+ Fts3HashElem **aElem2;
+ nAlloc += 16;
+ aElem2 = (Fts3HashElem **)sqlite3_realloc(
+ aElem, nAlloc*sizeof(Fts3HashElem *)
+ );
+ if( !aElem2 ){
+ rc = SQLITE_NOMEM;
+ nElem = 0;
+ break;
+ }
+ aElem = aElem2;
+ }
+
+ aElem[nElem++] = pE;
+ }
+ }
+
+ /* If more than one term matches the prefix, sort the Fts3HashElem
+ ** objects in term order using qsort(). This uses the same comparison
+ ** callback as is used when flushing terms to disk.
+ */
+ if( nElem>1 ){
+ qsort(aElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm);
+ }
+
+ }else{
+ /* The query is a simple term lookup that matches at most one term in
+ ** the index. All that is required is a straight hash-lookup.
+ **
+ ** Because the stack address of pE may be accessed via the aElem pointer
+ ** below, the "Fts3HashElem *pE" must be declared so that it is valid
+ ** within this entire function, not just this "else{...}" block.
+ */
+ pE = fts3HashFindElem(pHash, zTerm, nTerm);
+ if( pE ){
+ aElem = &pE;
+ nElem = 1;
+ }
+ }
+
+ if( nElem>0 ){
+ int nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *);
+ pReader = (Fts3SegReader *)sqlite3_malloc(nByte);
+ if( !pReader ){
+ rc = SQLITE_NOMEM;
+ }else{
+ memset(pReader, 0, nByte);
+ pReader->iIdx = 0x7FFFFFFF;
+ pReader->ppNextElem = (Fts3HashElem **)&pReader[1];
+ memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *));
+ }
+ }
+
+ if( bPrefix ){
+ sqlite3_free(aElem);
+ }
+ *ppReader = pReader;
+ return rc;
Ecere Software / sdk / commitdiff