[tracker/fts4: 1/21] libtracker-fts: Update to FTS4
- From: Carlos Garnacho <carlosg src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [tracker/fts4: 1/21] libtracker-fts: Update to FTS4
- Date: Mon, 4 Feb 2013 10:02:11 +0000 (UTC)
commit 049387a3d315760a1cf57287997c0258c43b3fa7
Author: Carlos Garnacho <carlos lanedo com>
Date: Thu Oct 27 17:06:49 2011 +0200
libtracker-fts: Update to FTS4
The code isn't yet feature complete, but cleans up the tracker
integration with the FTS module, which means that fts3* files
are taken verbatim from sqlite, and are licenced as such.
src/libtracker-data/tracker-data-manager.c | 6 +
src/libtracker-data/tracker-db-interface-sqlite.c | 85 +-
src/libtracker-fts/Makefile.am | 20 +-
src/libtracker-fts/fts3.c | 4843 ++++++++++++
src/libtracker-fts/fts3.h | 29 +
src/libtracker-fts/fts3Int.h | 513 ++
src/libtracker-fts/fts3_aux.c | 474 ++
src/libtracker-fts/fts3_expr.c | 964 +++
.../{tracker-fts-hash.c => fts3_hash.c} | 106 +-
.../{tracker-fts-hash.h => fts3_hash.h} | 68 +-
src/libtracker-fts/fts3_icu.c | 258 +
src/libtracker-fts/fts3_porter.c | 645 ++
src/libtracker-fts/fts3_snippet.c | 1499 ++++
src/libtracker-fts/fts3_term.c | 369 +
src/libtracker-fts/fts3_test.c | 324 +
src/libtracker-fts/fts3_tokenizer.c | 489 ++
src/libtracker-fts/fts3_tokenizer.h | 152 +
src/libtracker-fts/fts3_tokenizer1.c | 233 +
src/libtracker-fts/fts3_write.c | 3268 ++++++++
src/libtracker-fts/tracker-fts.c | 7997 +-------------------
src/libtracker-fts/tracker-fts.h | 53 +-
21 files changed, 14304 insertions(+), 8091 deletions(-)
---
diff --git a/src/libtracker-data/tracker-data-manager.c b/src/libtracker-data/tracker-data-manager.c
index 88b1763..e700da6 100644
--- a/src/libtracker-data/tracker-data-manager.c
+++ b/src/libtracker-data/tracker-data-manager.c
@@ -3616,6 +3616,12 @@ tracker_data_manager_init (TrackerDBManagerFlags flags,
tracker_data_update_init ();
+#ifdef TRACKER_FTS
+ if (tracker_fts_init ()) {
+ g_warning ("FTS module loading failed");
+ }
+#endif
+
/* First set defaults for return values */
if (first_time) {
*first_time = FALSE;
diff --git a/src/libtracker-data/tracker-db-interface-sqlite.c b/src/libtracker-data/tracker-db-interface-sqlite.c
index c16edc1..fbd8d66 100644
--- a/src/libtracker-data/tracker-db-interface-sqlite.c
+++ b/src/libtracker-data/tracker-db-interface-sqlite.c
@@ -86,9 +86,6 @@ struct TrackerDBInterface {
gint n_active_cursors;
guint ro : 1;
-#if HAVE_TRACKER_FTS
- TrackerFts *fts;
-#endif
GCancellable *cancellable;
TrackerDBStatementLru select_stmt_lru;
@@ -988,12 +985,6 @@ close_database (TrackerDBInterface *db_interface)
db_interface->function_data = NULL;
}
-#if HAVE_TRACKER_FTS
- if (db_interface->fts) {
- tracker_fts_free (db_interface->fts);
- }
-#endif
-
if (db_interface->db) {
rc = sqlite3_close (db_interface->db);
g_warn_if_fail (rc == SQLITE_OK);
@@ -1005,9 +996,9 @@ tracker_db_interface_sqlite_fts_init (TrackerDBInterface *db_interface,
gboolean create)
{
#if HAVE_TRACKER_FTS
- db_interface->fts = tracker_fts_new (db_interface->db, create);
-#else
- g_message ("FTS support is disabled");
+ if (!tracker_fts_init_db (db_interface->db, create)) {
+ g_warning ("FTS tables creation failed");
+ }
#endif
}
@@ -1016,29 +1007,89 @@ int
tracker_db_interface_sqlite_fts_update_init (TrackerDBInterface *db_interface,
int id)
{
- return tracker_fts_update_init (db_interface->fts, id);
+ return 0;
}
-int
+gboolean
tracker_db_interface_sqlite_fts_update_text (TrackerDBInterface *db_interface,
int id,
int column_id,
const char *text,
gboolean limit_word_length)
{
- return tracker_fts_update_text (db_interface->fts, id, column_id, text, limit_word_length);
+ TrackerDBStatement *stmt;
+ GError *error = NULL;
+
+ if (!text || !*text || column_id < 0) {
+ return FALSE;
+ }
+
+ /* Insert docid/property into property map */
+ stmt = tracker_db_interface_create_statement (db_interface,
+ TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE,
+ &error,
+ "INSERT OR REPLACE "
+ "INTO fts_prop_map "
+ " (resourceid, propid) "
+ "VALUES (?, ?)");
+
+ if (!stmt || error) {
+ if (error) {
+ g_warning ("Could not create FTS property map insert statement: %s\n",
+ error->message);
+ g_error_free (error);
+ }
+ return FALSE;
+ }
+
+ tracker_db_statement_bind_int (stmt, 0, id);
+ tracker_db_statement_bind_int (stmt, 1, column_id);
+
+ tracker_db_statement_execute (stmt, &error);
+ g_object_unref (stmt);
+
+ if (error) {
+ g_warning ("Could not insert FTS property map: %s", error->message);
+ g_error_free (error);
+ return FALSE;
+ }
+
+ stmt = tracker_db_interface_create_statement (db_interface,
+ TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE,
+ &error,
+ "INSERT OR REPLACE "
+ "INTO fts (docid,content) "
+ "VALUES (last_insert_rowid(), ?)");
+ if (!stmt || error) {
+ if (error) {
+ g_warning ("Could not create FTS insert statement: %s\n",
+ error->message);
+ g_error_free (error);
+ }
+ return FALSE;
+ }
+
+ tracker_db_statement_bind_text (stmt, 0, text);
+ tracker_db_statement_execute (stmt, &error);
+ g_object_unref (stmt);
+
+ if (error) {
+ g_warning ("Could not insert FTS text: %s", error->message);
+ g_error_free (error);
+ return FALSE;
+ }
+
+ return TRUE;
}
void
tracker_db_interface_sqlite_fts_update_commit (TrackerDBInterface *db_interface)
{
- return tracker_fts_update_commit (db_interface->fts);
}
void
tracker_db_interface_sqlite_fts_update_rollback (TrackerDBInterface *db_interface)
{
- return tracker_fts_update_rollback (db_interface->fts);
}
#endif
diff --git a/src/libtracker-fts/Makefile.am b/src/libtracker-fts/Makefile.am
index 0dacc44..a0306e0 100644
--- a/src/libtracker-fts/Makefile.am
+++ b/src/libtracker-fts/Makefile.am
@@ -8,13 +8,29 @@ AM_CPPFLAGS = \
noinst_LTLIBRARIES = libtracker-fts.la
+fts4_sources = \
+ fts3_aux.c \
+ fts3.c \
+ fts3_expr.c \
+ fts3.h \
+ fts3_hash.c \
+ fts3_hash.h \
+ fts3_icu.c \
+ fts3Int.h \
+ fts3_porter.c \
+ fts3_snippet.c \
+ fts3_term.c \
+ fts3_tokenizer1.c \
+ fts3_tokenizer.c \
+ fts3_tokenizer.h \
+ fts3_write.c
+
libtracker_fts_la_SOURCES = \
+ $(fts4_sources) \
tracker-fts.c \
tracker-fts.h \
tracker-fts-config.c \
tracker-fts-config.h \
- tracker-fts-hash.c \
- tracker-fts-hash.h \
tracker-parser-utils.c \
tracker-parser-utils.h \
tracker-parser.h
diff --git a/src/libtracker-fts/fts3.c b/src/libtracker-fts/fts3.c
new file mode 100644
index 0000000..f29970d
--- /dev/null
+++ b/src/libtracker-fts/fts3.c
@@ -0,0 +1,4843 @@
+/*
+** 2006 Oct 10
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This is an SQLite module implementing full-text search.
+*/
+
+/*
+** The code in this file is only compiled if:
+**
+** * The FTS3 module is being built as an extension
+** (in which case SQLITE_CORE is not defined), or
+**
+** * The FTS3 module is being built into the core of
+** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
+*/
+
+/* The full-text index is stored in a series of b+tree (-like)
+** structures called segments which map terms to doclists. The
+** structures are like b+trees in layout, but are constructed from the
+** bottom up in optimal fashion and are not updatable. Since trees
+** are built from the bottom up, things will be described from the
+** bottom up.
+**
+**
+**** Varints ****
+** The basic unit of encoding is a variable-length integer called a
+** varint. We encode variable-length integers in little-endian order
+** using seven bits * per byte as follows:
+**
+** KEY:
+** A = 0xxxxxxx 7 bits of data and one flag bit
+** B = 1xxxxxxx 7 bits of data and one flag bit
+**
+** 7 bits - A
+** 14 bits - BA
+** 21 bits - BBA
+** and so on.
+**
+** This is similar in concept to how sqlite encodes "varints" but
+** the encoding is not the same. SQLite varints are big-endian
+** are are limited to 9 bytes in length whereas FTS3 varints are
+** little-endian and can be up to 10 bytes in length (in theory).
+**
+** Example encodings:
+**
+** 1: 0x01
+** 127: 0x7f
+** 128: 0x81 0x00
+**
+**
+**** Document lists ****
+** A doclist (document list) holds a docid-sorted list of hits for a
+** given term. Doclists hold docids and associated token positions.
+** A docid is the unique integer identifier for a single document.
+** A position is the index of a word within the document. The first
+** word of the document has a position of 0.
+**
+** FTS3 used to optionally store character offsets using a compile-time
+** option. But that functionality is no longer supported.
+**
+** A doclist is stored like this:
+**
+** array {
+** varint docid;
+** array { (position list for column 0)
+** varint position; (2 more than the delta from previous position)
+** }
+** array {
+** varint POS_COLUMN; (marks start of position list for new column)
+** varint column; (index of new column)
+** array {
+** varint position; (2 more than the delta from previous position)
+** }
+** }
+** varint POS_END; (marks end of positions for this document.
+** }
+**
+** Here, array { X } means zero or more occurrences of X, adjacent in
+** memory. A "position" is an index of a token in the token stream
+** generated by the tokenizer. Note that POS_END and POS_COLUMN occur
+** in the same logical place as the position element, and act as sentinals
+** ending a position list array. POS_END is 0. POS_COLUMN is 1.
+** The positions numbers are not stored literally but rather as two more
+** than the difference from the prior position, or the just the position plus
+** 2 for the first position. Example:
+**
+** label: A B C D E F G H I J K
+** value: 123 5 9 1 1 14 35 0 234 72 0
+**
+** The 123 value is the first docid. For column zero in this document
+** there are two matches at positions 3 and 10 (5-2 and 9-2+3). The 1
+** at D signals the start of a new column; the 1 at E indicates that the
+** new column is column number 1. There are two positions at 12 and 45
+** (14-2 and 35-2+12). The 0 at H indicate the end-of-document. The
+** 234 at I is the next docid. It has one position 72 (72-2) and then
+** terminates with the 0 at K.
+**
+** A "position-list" is the list of positions for multiple columns for
+** a single docid. A "column-list" is the set of positions for a single
+** column. Hence, a position-list consists of one or more column-lists,
+** a document record consists of a docid followed by a position-list and
+** a doclist consists of one or more document records.
+**
+** A bare doclist omits the position information, becoming an
+** array of varint-encoded docids.
+**
+**** Segment leaf nodes ****
+** Segment leaf nodes store terms and doclists, ordered by term. Leaf
+** nodes are written using LeafWriter, and read using LeafReader (to
+** iterate through a single leaf node's data) and LeavesReader (to
+** iterate through a segment's entire leaf layer). Leaf nodes have
+** the format:
+**
+** varint iHeight; (height from leaf level, always 0)
+** varint nTerm; (length of first term)
+** char pTerm[nTerm]; (content of first term)
+** varint nDoclist; (length of term's associated doclist)
+** char pDoclist[nDoclist]; (content of doclist)
+** array {
+** (further terms are delta-encoded)
+** varint nPrefix; (length of prefix shared with previous term)
+** varint nSuffix; (length of unshared suffix)
+** char pTermSuffix[nSuffix];(unshared suffix of next term)
+** varint nDoclist; (length of term's associated doclist)
+** char pDoclist[nDoclist]; (content of doclist)
+** }
+**
+** Here, array { X } means zero or more occurrences of X, adjacent in
+** memory.
+**
+** Leaf nodes are broken into blocks which are stored contiguously in
+** the %_segments table in sorted order. This means that when the end
+** of a node is reached, the next term is in the node with the next
+** greater node id.
+**
+** New data is spilled to a new leaf node when the current node
+** exceeds LEAF_MAX bytes (default 2048). New data which itself is
+** larger than STANDALONE_MIN (default 1024) is placed in a standalone
+** node (a leaf node with a single term and doclist). The goal of
+** these settings is to pack together groups of small doclists while
+** making it efficient to directly access large doclists. The
+** assumption is that large doclists represent terms which are more
+** likely to be query targets.
+**
+** TODO(shess) It may be useful for blocking decisions to be more
+** dynamic. For instance, it may make more sense to have a 2.5k leaf
+** node rather than splitting into 2k and .5k nodes. My intuition is
+** that this might extend through 2x or 4x the pagesize.
+**
+**
+**** Segment interior nodes ****
+** Segment interior nodes store blockids for subtree nodes and terms
+** to describe what data is stored by the each subtree. Interior
+** nodes are written using InteriorWriter, and read using
+** InteriorReader. InteriorWriters are created as needed when
+** SegmentWriter creates new leaf nodes, or when an interior node
+** itself grows too big and must be split. The format of interior
+** nodes:
+**
+** varint iHeight; (height from leaf level, always >0)
+** varint iBlockid; (block id of node's leftmost subtree)
+** optional {
+** varint nTerm; (length of first term)
+** char pTerm[nTerm]; (content of first term)
+** array {
+** (further terms are delta-encoded)
+** varint nPrefix; (length of shared prefix with previous term)
+** varint nSuffix; (length of unshared suffix)
+** char pTermSuffix[nSuffix]; (unshared suffix of next term)
+** }
+** }
+**
+** Here, optional { X } means an optional element, while array { X }
+** means zero or more occurrences of X, adjacent in memory.
+**
+** An interior node encodes n terms separating n+1 subtrees. The
+** subtree blocks are contiguous, so only the first subtree's blockid
+** is encoded. The subtree at iBlockid will contain all terms less
+** than the first term encoded (or all terms if no term is encoded).
+** Otherwise, for terms greater than or equal to pTerm[i] but less
+** than pTerm[i+1], the subtree for that term will be rooted at
+** iBlockid+i. Interior nodes only store enough term data to
+** distinguish adjacent children (if the rightmost term of the left
+** child is "something", and the leftmost term of the right child is
+** "wicked", only "w" is stored).
+**
+** New data is spilled to a new interior node at the same height when
+** the current node exceeds INTERIOR_MAX bytes (default 2048).
+** INTERIOR_MIN_TERMS (default 7) keeps large terms from monopolizing
+** interior nodes and making the tree too skinny. The interior nodes
+** at a given height are naturally tracked by interior nodes at
+** height+1, and so on.
+**
+**
+**** Segment directory ****
+** The segment directory in table %_segdir stores meta-information for
+** merging and deleting segments, and also the root node of the
+** segment's tree.
+**
+** The root node is the top node of the segment's tree after encoding
+** the entire segment, restricted to ROOT_MAX bytes (default 1024).
+** This could be either a leaf node or an interior node. If the top
+** node requires more than ROOT_MAX bytes, it is flushed to %_segments
+** and a new root interior node is generated (which should always fit
+** within ROOT_MAX because it only needs space for 2 varints, the
+** height and the blockid of the previous root).
+**
+** The meta-information in the segment directory is:
+** level - segment level (see below)
+** idx - index within level
+** - (level,idx uniquely identify a segment)
+** start_block - first leaf node
+** leaves_end_block - last leaf node
+** end_block - last block (including interior nodes)
+** root - contents of root node
+**
+** If the root node is a leaf node, then start_block,
+** leaves_end_block, and end_block are all 0.
+**
+**
+**** Segment merging ****
+** To amortize update costs, segments are grouped into levels and
+** merged in batches. Each increase in level represents exponentially
+** more documents.
+**
+** New documents (actually, document updates) are tokenized and
+** written individually (using LeafWriter) to a level 0 segment, with
+** incrementing idx. When idx reaches MERGE_COUNT (default 16), all
+** level 0 segments are merged into a single level 1 segment. Level 1
+** is populated like level 0, and eventually MERGE_COUNT level 1
+** segments are merged to a single level 2 segment (representing
+** MERGE_COUNT^2 updates), and so on.
+**
+** A segment merge traverses all segments at a given level in
+** parallel, performing a straightforward sorted merge. Since segment
+** leaf nodes are written in to the %_segments table in order, this
+** merge traverses the underlying sqlite disk structures efficiently.
+** After the merge, all segment blocks from the merged level are
+** deleted.
+**
+** MERGE_COUNT controls how often we merge segments. 16 seems to be
+** somewhat of a sweet spot for insertion performance. 32 and 64 show
+** very similar performance numbers to 16 on insertion, though they're
+** a tiny bit slower (perhaps due to more overhead in merge-time
+** sorting). 8 is about 20% slower than 16, 4 about 50% slower than
+** 16, 2 about 66% slower than 16.
+**
+** At query time, high MERGE_COUNT increases the number of segments
+** which need to be scanned and merged. For instance, with 100k docs
+** inserted:
+**
+** MERGE_COUNT segments
+** 16 25
+** 8 12
+** 4 10
+** 2 6
+**
+** This appears to have only a moderate impact on queries for very
+** frequent terms (which are somewhat dominated by segment merge
+** costs), and infrequent and non-existent terms still seem to be fast
+** even with many segments.
+**
+** TODO(shess) That said, it would be nice to have a better query-side
+** argument for MERGE_COUNT of 16. Also, it is possible/likely that
+** optimizations to things like doclist merging will swing the sweet
+** spot around.
+**
+**
+**
+**** Handling of deletions and updates ****
+** Since we're using a segmented structure, with no docid-oriented
+** index into the term index, we clearly cannot simply update the term
+** index when a document is deleted or updated. For deletions, we
+** write an empty doclist (varint(docid) varint(POS_END)), for updates
+** we simply write the new doclist. Segment merges overwrite older
+** data for a particular docid with newer data, so deletes or updates
+** will eventually overtake the earlier data and knock it out. The
+** query logic likewise merges doclists so that newer data knocks out
+** older data.
+**
+** TODO(shess) Provide a VACUUM type operation to clear out all
+** deletions and duplications. This would basically be a forced merge
+** into a single segment.
+*/
+
+#include "fts3Int.h"
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE)
+# define SQLITE_CORE 1
+#endif
+
+#include <assert.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdarg.h>
+
+#include "fts3.h"
+#ifndef SQLITE_CORE
+ # include "sqlite3ext.h"
+ SQLITE_EXTENSION_INIT1
+#endif
+
+static int fts3EvalNext(Fts3Cursor *pCsr);
+static int fts3EvalStart(Fts3Cursor *pCsr);
+static int fts3TermSegReaderCursor(
+ Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **);
+
+/*
+** Write a 64-bit variable-length integer to memory starting at p[0].
+** The length of data written will be between 1 and FTS3_VARINT_MAX bytes.
+** The number of bytes written is returned.
+*/
+int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){
+ unsigned char *q = (unsigned char *) p;
+ sqlite_uint64 vu = v;
+ do{
+ *q++ = (unsigned char) ((vu & 0x7f) | 0x80);
+ vu >>= 7;
+ }while( vu!=0 );
+ q[-1] &= 0x7f; /* turn off high bit in final byte */
+ assert( q - (unsigned char *)p <= FTS3_VARINT_MAX );
+ return (int) (q - (unsigned char *)p);
+}
+
+/*
+** Read a 64-bit variable-length integer from memory starting at p[0].
+** Return the number of bytes read, or 0 on error.
+** The value is stored in *v.
+*/
+int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){
+ const unsigned char *q = (const unsigned char *) p;
+ sqlite_uint64 x = 0, y = 1;
+ while( (*q&0x80)==0x80 && q-(unsigned char *)p<FTS3_VARINT_MAX ){
+ x += y * (*q++ & 0x7f);
+ y <<= 7;
+ }
+ x += y * (*q++);
+ *v = (sqlite_int64) x;
+ return (int) (q - (unsigned char *)p);
+}
+
+/*
+** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to a
+** 32-bit integer before it is returned.
+*/
+int sqlite3Fts3GetVarint32(const char *p, int *pi){
+ sqlite_int64 i;
+ int ret = sqlite3Fts3GetVarint(p, &i);
+ *pi = (int) i;
+ return ret;
+}
+
+/*
+** Return the number of bytes required to encode v as a varint
+*/
+int sqlite3Fts3VarintLen(sqlite3_uint64 v){
+ int i = 0;
+ do{
+ i++;
+ v >>= 7;
+ }while( v!=0 );
+ return i;
+}
+
+/*
+** Convert an SQL-style quoted string into a normal string by removing
+** the quote characters. The conversion is done in-place. If the
+** input does not begin with a quote character, then this routine
+** is a no-op.
+**
+** Examples:
+**
+** "abc" becomes abc
+** 'xyz' becomes xyz
+** [pqr] becomes pqr
+** `mno` becomes mno
+**
+*/
+void sqlite3Fts3Dequote(char *z){
+ char quote; /* Quote character (if any ) */
+
+ quote = z[0];
+ if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){
+ int iIn = 1; /* Index of next byte to read from input */
+ int iOut = 0; /* Index of next byte to write to output */
+
+ /* If the first byte was a '[', then the close-quote character is a ']' */
+ if( quote=='[' ) quote = ']';
+
+ while( ALWAYS(z[iIn]) ){
+ if( z[iIn]==quote ){
+ if( z[iIn+1]!=quote ) break;
+ z[iOut++] = quote;
+ iIn += 2;
+ }else{
+ z[iOut++] = z[iIn++];
+ }
+ }
+ z[iOut] = '\0';
+ }
+}
+
+/*
+** Read a single varint from the doclist at *pp and advance *pp to point
+** to the first byte past the end of the varint. Add the value of the varint
+** to *pVal.
+*/
+static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){
+ sqlite3_int64 iVal;
+ *pp += sqlite3Fts3GetVarint(*pp, &iVal);
+ *pVal += iVal;
+}
+
+/*
+** When this function is called, *pp points to the first byte following a
+** varint that is part of a doclist (or position-list, or any other list
+** of varints). This function moves *pp to point to the start of that varint,
+** and sets *pVal by the varint value.
+**
+** Argument pStart points to the first byte of the doclist that the
+** varint is part of.
+*/
+static void fts3GetReverseVarint(
+ char **pp,
+ char *pStart,
+ sqlite3_int64 *pVal
+){
+ sqlite3_int64 iVal;
+ char *p = *pp;
+
+ /* Pointer p now points at the first byte past the varint we are
+ ** interested in. So, unless the doclist is corrupt, the 0x80 bit is
+ ** clear on character p[-1]. */
+ for(p = (*pp)-2; p>=pStart && *p&0x80; p--);
+ p++;
+ *pp = p;
+
+ sqlite3Fts3GetVarint(p, &iVal);
+ *pVal = iVal;
+}
+
+/*
+** The xDisconnect() virtual table method.
+*/
+static int fts3DisconnectMethod(sqlite3_vtab *pVtab){
+ Fts3Table *p = (Fts3Table *)pVtab;
+ int i;
+
+ assert( p->nPendingData==0 );
+ assert( p->pSegments==0 );
+
+ /* Free any prepared statements held */
+ for(i=0; i<SizeofArray(p->aStmt); i++){
+ sqlite3_finalize(p->aStmt[i]);
+ }
+ sqlite3_free(p->zSegmentsTbl);
+ sqlite3_free(p->zReadExprlist);
+ sqlite3_free(p->zWriteExprlist);
+
+ /* Invoke the tokenizer destructor to free the tokenizer. */
+ p->pTokenizer->pModule->xDestroy(p->pTokenizer);
+
+ sqlite3_free(p);
+ return SQLITE_OK;
+}
+
+/*
+** Construct one or more SQL statements from the format string given
+** and then evaluate those statements. The success code is written
+** into *pRc.
+**
+** If *pRc is initially non-zero then this routine is a no-op.
+*/
+static void fts3DbExec(
+ int *pRc, /* Success code */
+ sqlite3 *db, /* Database in which to run SQL */
+ const char *zFormat, /* Format string for SQL */
+ ... /* Arguments to the format string */
+){
+ va_list ap;
+ char *zSql;
+ if( *pRc ) return;
+ va_start(ap, zFormat);
+ zSql = sqlite3_vmprintf(zFormat, ap);
+ va_end(ap);
+ if( zSql==0 ){
+ *pRc = SQLITE_NOMEM;
+ }else{
+ *pRc = sqlite3_exec(db, zSql, 0, 0, 0);
+ sqlite3_free(zSql);
+ }
+}
+
+/*
+** The xDestroy() virtual table method.
+*/
+static int fts3DestroyMethod(sqlite3_vtab *pVtab){
+ int rc = SQLITE_OK; /* Return code */
+ Fts3Table *p = (Fts3Table *)pVtab;
+ sqlite3 *db = p->db;
+
+ /* Drop the shadow tables */
+ fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", p->zDb, p->zName);
+ fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", p->zDb,p->zName);
+ fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", p->zDb, p->zName);
+ fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", p->zDb, p->zName);
+ fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", p->zDb, p->zName);
+
+ /* If everything has worked, invoke fts3DisconnectMethod() to free the
+ ** memory associated with the Fts3Table structure and return SQLITE_OK.
+ ** Otherwise, return an SQLite error code.
+ */
+ return (rc==SQLITE_OK ? fts3DisconnectMethod(pVtab) : rc);
+}
+
+
+/*
+** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table
+** passed as the first argument. This is done as part of the xConnect()
+** and xCreate() methods.
+**
+** If *pRc is non-zero when this function is called, it is a no-op.
+** Otherwise, if an error occurs, an SQLite error code is stored in *pRc
+** before returning.
+*/
+static void fts3DeclareVtab(int *pRc, Fts3Table *p){
+ if( *pRc==SQLITE_OK ){
+ int i; /* Iterator variable */
+ int rc; /* Return code */
+ char *zSql; /* SQL statement passed to declare_vtab() */
+ char *zCols; /* List of user defined columns */
+
+ sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1);
+
+ /* Create a list of user columns for the virtual table */
+ zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]);
+ for(i=1; zCols && i<p->nColumn; i++){
+ zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]);
+ }
+
+ /* Create the whole "CREATE TABLE" statement to pass to SQLite */
+ zSql = sqlite3_mprintf(
+ "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName
+ );
+ if( !zCols || !zSql ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_declare_vtab(p->db, zSql);
+ }
+
+ sqlite3_free(zSql);
+ sqlite3_free(zCols);
+ *pRc = rc;
+ }
+}
+
+/*
+** Create the backing store tables (%_content, %_segments and %_segdir)
+** required by the FTS3 table passed as the only argument. This is done
+** as part of the vtab xCreate() method.
+**
+** If the p->bHasDocsize boolean is true (indicating that this is an
+** FTS4 table, not an FTS3 table) then also create the %_docsize and
+** %_stat tables required by FTS4.
+*/
+static int fts3CreateTables(Fts3Table *p){
+ int rc = SQLITE_OK; /* Return code */
+ int i; /* Iterator variable */
+ char *zContentCols; /* Columns of %_content table */
+ sqlite3 *db = p->db; /* The database connection */
+
+ /* Create a list of user columns for the content table */
+ zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY");
+ for(i=0; zContentCols && i<p->nColumn; i++){
+ char *z = p->azColumn[i];
+ zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z);
+ }
+ if( zContentCols==0 ) rc = SQLITE_NOMEM;
+
+ /* Create the content table */
+ fts3DbExec(&rc, db,
+ "CREATE TABLE %Q.'%q_content'(%s)",
+ p->zDb, p->zName, zContentCols
+ );
+ sqlite3_free(zContentCols);
+ /* Create other tables */
+ fts3DbExec(&rc, db,
+ "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);",
+ p->zDb, p->zName
+ );
+ fts3DbExec(&rc, db,
+ "CREATE TABLE %Q.'%q_segdir'("
+ "level INTEGER,"
+ "idx INTEGER,"
+ "start_block INTEGER,"
+ "leaves_end_block INTEGER,"
+ "end_block INTEGER,"
+ "root BLOB,"
+ "PRIMARY KEY(level, idx)"
+ ");",
+ p->zDb, p->zName
+ );
+ if( p->bHasDocsize ){
+ fts3DbExec(&rc, db,
+ "CREATE TABLE %Q.'%q_docsize'(docid INTEGER PRIMARY KEY, size BLOB);",
+ p->zDb, p->zName
+ );
+ }
+ if( p->bHasStat ){
+ fts3DbExec(&rc, db,
+ "CREATE TABLE %Q.'%q_stat'(id INTEGER PRIMARY KEY, value BLOB);",
+ p->zDb, p->zName
+ );
+ }
+ return rc;
+}
+
+/*
+** Store the current database page-size in bytes in p->nPgsz.
+**
+** If *pRc is non-zero when this function is called, it is a no-op.
+** Otherwise, if an error occurs, an SQLite error code is stored in *pRc
+** before returning.
+*/
+static void fts3DatabasePageSize(int *pRc, Fts3Table *p){
+ if( *pRc==SQLITE_OK ){
+ int rc; /* Return code */
+ char *zSql; /* SQL text "PRAGMA %Q.page_size" */
+ sqlite3_stmt *pStmt; /* Compiled "PRAGMA %Q.page_size" statement */
+
+ zSql = sqlite3_mprintf("PRAGMA %Q.page_size", p->zDb);
+ if( !zSql ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_step(pStmt);
+ p->nPgsz = sqlite3_column_int(pStmt, 0);
+ rc = sqlite3_finalize(pStmt);
+ }else if( rc==SQLITE_AUTH ){
+ p->nPgsz = 1024;
+ rc = SQLITE_OK;
+ }
+ }
+ assert( p->nPgsz>0 || rc!=SQLITE_OK );
+ sqlite3_free(zSql);
+ *pRc = rc;
+ }
+}
+
+/*
+** "Special" FTS4 arguments are column specifications of the following form:
+**
+** <key> = <value>
+**
+** There may not be whitespace surrounding the "=" character. The <value>
+** term may be quoted, but the <key> may not.
+*/
+static int fts3IsSpecialColumn(
+ const char *z,
+ int *pnKey,
+ char **pzValue
+){
+ char *zValue;
+ const char *zCsr = z;
+
+ while( *zCsr!='=' ){
+ if( *zCsr=='\0' ) return 0;
+ zCsr++;
+ }
+
+ *pnKey = (int)(zCsr-z);
+ zValue = sqlite3_mprintf("%s", &zCsr[1]);
+ if( zValue ){
+ sqlite3Fts3Dequote(zValue);
+ }
+ *pzValue = zValue;
+ return 1;
+}
+
+/*
+** Append the output of a printf() style formatting to an existing string.
+*/
+static void fts3Appendf(
+ int *pRc, /* IN/OUT: Error code */
+ char **pz, /* IN/OUT: Pointer to string buffer */
+ const char *zFormat, /* Printf format string to append */
+ ... /* Arguments for printf format string */
+){
+ if( *pRc==SQLITE_OK ){
+ va_list ap;
+ char *z;
+ va_start(ap, zFormat);
+ z = sqlite3_vmprintf(zFormat, ap);
+ if( z && *pz ){
+ char *z2 = sqlite3_mprintf("%s%s", *pz, z);
+ sqlite3_free(z);
+ z = z2;
+ }
+ if( z==0 ) *pRc = SQLITE_NOMEM;
+ sqlite3_free(*pz);
+ *pz = z;
+ }
+}
+
+/*
+** Return a copy of input string zInput enclosed in double-quotes (") and
+** with all double quote characters escaped. For example:
+**
+** fts3QuoteId("un \"zip\"") -> "un \"\"zip\"\""
+**
+** The pointer returned points to memory obtained from sqlite3_malloc(). It
+** is the callers responsibility to call sqlite3_free() to release this
+** memory.
+*/
+static char *fts3QuoteId(char const *zInput){
+ int nRet;
+ char *zRet;
+ nRet = 2 + strlen(zInput)*2 + 1;
+ zRet = sqlite3_malloc(nRet);
+ if( zRet ){
+ int i;
+ char *z = zRet;
+ *(z++) = '"';
+ for(i=0; zInput[i]; i++){
+ if( zInput[i]=='"' ) *(z++) = '"';
+ *(z++) = zInput[i];
+ }
+ *(z++) = '"';
+ *(z++) = '\0';
+ }
+ return zRet;
+}
+
+/*
+** Return a list of comma separated SQL expressions that could be used
+** in a SELECT statement such as the following:
+**
+** SELECT <list of expressions> FROM %_content AS x ...
+**
+** to return the docid, followed by each column of text data in order
+** from left to write. If parameter zFunc is not NULL, then instead of
+** being returned directly each column of text data is passed to an SQL
+** function named zFunc first. For example, if zFunc is "unzip" and the
+** table has the three user-defined columns "a", "b", and "c", the following
+** string is returned:
+**
+** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c')"
+**
+** The pointer returned points to a buffer allocated by sqlite3_malloc(). It
+** is the responsibility of the caller to eventually free it.
+**
+** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and
+** a NULL pointer is returned). Otherwise, if an OOM error is encountered
+** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If
+** no error occurs, *pRc is left unmodified.
+*/
+static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){
+ char *zRet = 0;
+ char *zFree = 0;
+ char *zFunction;
+ int i;
+
+ if( !zFunc ){
+ zFunction = "";
+ }else{
+ zFree = zFunction = fts3QuoteId(zFunc);
+ }
+ fts3Appendf(pRc, &zRet, "docid");
+ for(i=0; i<p->nColumn; i++){
+ fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]);
+ }
+ sqlite3_free(zFree);
+ return zRet;
+}
+
+/*
+** Return a list of N comma separated question marks, where N is the number
+** of columns in the %_content table (one for the docid plus one for each
+** user-defined text column).
+**
+** If argument zFunc is not NULL, then all but the first question mark
+** is preceded by zFunc and an open bracket, and followed by a closed
+** bracket. For example, if zFunc is "zip" and the FTS3 table has three
+** user-defined text columns, the following string is returned:
+**
+** "?, zip(?), zip(?), zip(?)"
+**
+** The pointer returned points to a buffer allocated by sqlite3_malloc(). It
+** is the responsibility of the caller to eventually free it.
+**
+** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and
+** a NULL pointer is returned). Otherwise, if an OOM error is encountered
+** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If
+** no error occurs, *pRc is left unmodified.
+*/
+static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){
+ char *zRet = 0;
+ char *zFree = 0;
+ char *zFunction;
+ int i;
+
+ if( !zFunc ){
+ zFunction = "";
+ }else{
+ zFree = zFunction = fts3QuoteId(zFunc);
+ }
+ fts3Appendf(pRc, &zRet, "?");
+ for(i=0; i<p->nColumn; i++){
+ fts3Appendf(pRc, &zRet, ",%s(?)", zFunction);
+ }
+ sqlite3_free(zFree);
+ return zRet;
+}
+
+/*
+** This function interprets the string at (*pp) as a non-negative integer
+** value. It reads the integer and sets *pnOut to the value read, then
+** sets *pp to point to the byte immediately following the last byte of
+** the integer value.
+**
+** Only decimal digits ('0'..'9') may be part of an integer value.
+**
+** If *pp does not being with a decimal digit SQLITE_ERROR is returned and
+** the output value undefined. Otherwise SQLITE_OK is returned.
+**
+** This function is used when parsing the "prefix=" FTS4 parameter.
+*/
+static int fts3GobbleInt(const char **pp, int *pnOut){
+ const char *p = *pp; /* Iterator pointer */
+ int nInt = 0; /* Output value */
+
+ for(p=*pp; p[0]>='0' && p[0]<='9'; p++){
+ nInt = nInt * 10 + (p[0] - '0');
+ }
+ if( p==*pp ) return SQLITE_ERROR;
+ *pnOut = nInt;
+ *pp = p;
+ return SQLITE_OK;
+}
+
+/*
+** This function is called to allocate an array of Fts3Index structures
+** representing the indexes maintained by the current FTS table. FTS tables
+** always maintain the main "terms" index, but may also maintain one or
+** more "prefix" indexes, depending on the value of the "prefix=" parameter
+** (if any) specified as part of the CREATE VIRTUAL TABLE statement.
+**
+** Argument zParam is passed the value of the "prefix=" option if one was
+** specified, or NULL otherwise.
+**
+** If no error occurs, SQLITE_OK is returned and *apIndex set to point to
+** the allocated array. *pnIndex is set to the number of elements in the
+** array. If an error does occur, an SQLite error code is returned.
+**
+** Regardless of whether or not an error is returned, it is the responsibility
+** of the caller to call sqlite3_free() on the output array to free it.
+*/
+static int fts3PrefixParameter(
+ const char *zParam, /* ABC in prefix=ABC parameter to parse */
+ int *pnIndex, /* OUT: size of *apIndex[] array */
+ struct Fts3Index **apIndex /* OUT: Array of indexes for this table */
+){
+ struct Fts3Index *aIndex; /* Allocated array */
+ int nIndex = 1; /* Number of entries in array */
+
+ if( zParam && zParam[0] ){
+ const char *p;
+ nIndex++;
+ for(p=zParam; *p; p++){
+ if( *p==',' ) nIndex++;
+ }
+ }
+
+ aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex);
+ *apIndex = aIndex;
+ *pnIndex = nIndex;
+ if( !aIndex ){
+ return SQLITE_NOMEM;
+ }
+
+ memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex);
+ if( zParam ){
+ const char *p = zParam;
+ int i;
+ for(i=1; i<nIndex; i++){
+ int nPrefix;
+ if( fts3GobbleInt(&p, &nPrefix) ) return SQLITE_ERROR;
+ aIndex[i].nPrefix = nPrefix;
+ p++;
+ }
+ }
+
+ return SQLITE_OK;
+}
+
+/*
+** This function is the implementation of both the xConnect and xCreate
+** methods of the FTS3 virtual table.
+**
+** The argv[] array contains the following:
+**
+** argv[0] -> module name ("fts3" or "fts4")
+** argv[1] -> database name
+** argv[2] -> table name
+** argv[...] -> "column name" and other module argument fields.
+*/
+static int fts3InitVtab(
+ int isCreate, /* True for xCreate, false for xConnect */
+ sqlite3 *db, /* The SQLite database connection */
+ void *pAux, /* Hash table containing tokenizers */
+ int argc, /* Number of elements in argv array */
+ const char * const *argv, /* xCreate/xConnect argument array */
+ sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */
+ char **pzErr /* Write any error message here */
+){
+ Fts3Hash *pHash = (Fts3Hash *)pAux;
+ Fts3Table *p = 0; /* Pointer to allocated vtab */
+ int rc = SQLITE_OK; /* Return code */
+ int i; /* Iterator variable */
+ int nByte; /* Size of allocation used for *p */
+ int iCol; /* Column index */
+ int nString = 0; /* Bytes required to hold all column names */
+ int nCol = 0; /* Number of columns in the FTS table */
+ char *zCsr; /* Space for holding column names */
+ int nDb; /* Bytes required to hold database name */
+ int nName; /* Bytes required to hold table name */
+ int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */
+ const char **aCol; /* Array of column names */
+ sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */
+
+ int nIndex; /* Size of aIndex[] array */
+ struct Fts3Index *aIndex = 0; /* Array of indexes for this table */
+
+ /* The results of parsing supported FTS4 key=value options: */
+ int bNoDocsize = 0; /* True to omit %_docsize table */
+ int bDescIdx = 0; /* True to store descending indexes */
+ char *zPrefix = 0; /* Prefix parameter value (or NULL) */
+ char *zCompress = 0; /* compress=? parameter (or NULL) */
+ char *zUncompress = 0; /* uncompress=? parameter (or NULL) */
+
+ assert( strlen(argv[0])==4 );
+ assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4)
+ || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4)
+ );
+
+ nDb = (int)strlen(argv[1]) + 1;
+ nName = (int)strlen(argv[2]) + 1;
+
+ aCol = (const char **)sqlite3_malloc(sizeof(const char *) * (argc-2) );
+ if( !aCol ) return SQLITE_NOMEM;
+ memset((void *)aCol, 0, sizeof(const char *) * (argc-2));
+
+ /* Loop through all of the arguments passed by the user to the FTS3/4
+ ** module (i.e. all the column names and special arguments). This loop
+ ** does the following:
+ **
+ ** + Figures out the number of columns the FTSX table will have, and
+ ** the number of bytes of space that must be allocated to store copies
+ ** of the column names.
+ **
+ ** + If there is a tokenizer specification included in the arguments,
+ ** initializes the tokenizer pTokenizer.
+ */
+ for(i=3; rc==SQLITE_OK && i<argc; i++){
+ char const *z = argv[i];
+ int nKey;
+ char *zVal;
+
+ /* Check if this is a tokenizer specification */
+ if( !pTokenizer
+ && strlen(z)>8
+ && 0==sqlite3_strnicmp(z, "tokenize", 8)
+ && 0==sqlite3Fts3IsIdChar(z[8])
+ ){
+ rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr);
+ }
+
+ /* Check if it is an FTS4 special argument. */
+ else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){
+ struct Fts4Option {
+ const char *zOpt;
+ int nOpt;
+ char **pzVar;
+ } aFts4Opt[] = {
+ { "matchinfo", 9, 0 }, /* 0 -> MATCHINFO */
+ { "prefix", 6, 0 }, /* 1 -> PREFIX */
+ { "compress", 8, 0 }, /* 2 -> COMPRESS */
+ { "uncompress", 10, 0 }, /* 3 -> UNCOMPRESS */
+ { "order", 5, 0 } /* 4 -> ORDER */
+ };
+
+ int iOpt;
+ if( !zVal ){
+ rc = SQLITE_NOMEM;
+ }else{
+ for(iOpt=0; iOpt<SizeofArray(aFts4Opt); iOpt++){
+ struct Fts4Option *pOp = &aFts4Opt[iOpt];
+ if( nKey==pOp->nOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){
+ break;
+ }
+ }
+ if( iOpt==SizeofArray(aFts4Opt) ){
+ *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z);
+ rc = SQLITE_ERROR;
+ }else{
+ switch( iOpt ){
+ case 0: /* MATCHINFO */
+ if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){
+ *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal);
+ rc = SQLITE_ERROR;
+ }
+ bNoDocsize = 1;
+ break;
+
+ case 1: /* PREFIX */
+ sqlite3_free(zPrefix);
+ zPrefix = zVal;
+ zVal = 0;
+ break;
+
+ case 2: /* COMPRESS */
+ sqlite3_free(zCompress);
+ zCompress = zVal;
+ zVal = 0;
+ break;
+
+ case 3: /* UNCOMPRESS */
+ sqlite3_free(zUncompress);
+ zUncompress = zVal;
+ zVal = 0;
+ break;
+
+ case 4: /* ORDER */
+ if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3))
+ && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 3))
+ ){
+ *pzErr = sqlite3_mprintf("unrecognized order: %s", zVal);
+ rc = SQLITE_ERROR;
+ }
+ bDescIdx = (zVal[0]=='d' || zVal[0]=='D');
+ break;
+ }
+ }
+ sqlite3_free(zVal);
+ }
+ }
+
+ /* Otherwise, the argument is a column name. */
+ else {
+ nString += (int)(strlen(z) + 1);
+ aCol[nCol++] = z;
+ }
+ }
+ if( rc!=SQLITE_OK ) goto fts3_init_out;
+
+ if( nCol==0 ){
+ assert( nString==0 );
+ aCol[0] = "content";
+ nString = 8;
+ nCol = 1;
+ }
+
+ if( pTokenizer==0 ){
+ rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr);
+ if( rc!=SQLITE_OK ) goto fts3_init_out;
+ }
+ assert( pTokenizer );
+
+ rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex);
+ if( rc==SQLITE_ERROR ){
+ assert( zPrefix );
+ *pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix);
+ }
+ if( rc!=SQLITE_OK ) goto fts3_init_out;
+
+ /* Allocate and populate the Fts3Table structure. */
+ nByte = sizeof(Fts3Table) + /* Fts3Table */
+ nCol * sizeof(char *) + /* azColumn */
+ nIndex * sizeof(struct Fts3Index) + /* aIndex */
+ nName + /* zName */
+ nDb + /* zDb */
+ nString; /* Space for azColumn strings */
+ p = (Fts3Table*)sqlite3_malloc(nByte);
+ if( p==0 ){
+ rc = SQLITE_NOMEM;
+ goto fts3_init_out;
+ }
+ memset(p, 0, nByte);
+ p->db = db;
+ p->nColumn = nCol;
+ p->nPendingData = 0;
+ p->azColumn = (char **)&p[1];
+ p->pTokenizer = pTokenizer;
+ p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
+ p->bHasDocsize = (isFts4 && bNoDocsize==0);
+ p->bHasStat = isFts4;
+ p->bDescIdx = bDescIdx;
+ TESTONLY( p->inTransaction = -1 );
+ TESTONLY( p->mxSavepoint = -1 );
+
+ p->aIndex = (struct Fts3Index *)&p->azColumn[nCol];
+ memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex);
+ p->nIndex = nIndex;
+ for(i=0; i<nIndex; i++){
+ fts3HashInit(&p->aIndex[i].hPending, FTS3_HASH_STRING, 1);
+ }
+
+ /* Fill in the zName and zDb fields of the vtab structure. */
+ zCsr = (char *)&p->aIndex[nIndex];
+ p->zName = zCsr;
+ memcpy(zCsr, argv[2], nName);
+ zCsr += nName;
+ p->zDb = zCsr;
+ memcpy(zCsr, argv[1], nDb);
+ zCsr += nDb;
+
+ /* Fill in the azColumn array */
+ for(iCol=0; iCol<nCol; iCol++){
+ char *z;
+ int n = 0;
+ z = (char *)sqlite3Fts3NextToken(aCol[iCol], &n);
+ memcpy(zCsr, z, n);
+ zCsr[n] = '\0';
+ sqlite3Fts3Dequote(zCsr);
+ p->azColumn[iCol] = zCsr;
+ zCsr += n+1;
+ assert( zCsr <= &((char *)p)[nByte] );
+ }
+
+ if( (zCompress==0)!=(zUncompress==0) ){
+ char const *zMiss = (zCompress==0 ? "compress" : "uncompress");
+ rc = SQLITE_ERROR;
+ *pzErr = sqlite3_mprintf("missing %s parameter in fts4 constructor", zMiss);
+ }
+ p->zReadExprlist = fts3ReadExprList(p, zUncompress, &rc);
+ p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc);
+ if( rc!=SQLITE_OK ) goto fts3_init_out;
+
+ /* If this is an xCreate call, create the underlying tables in the
+ ** database. TODO: For xConnect(), it could verify that said tables exist.
+ */
+ if( isCreate ){
+ rc = fts3CreateTables(p);
+ }
+
+ /* Figure out the page-size for the database. This is required in order to
+ ** estimate the cost of loading large doclists from the database. */
+ fts3DatabasePageSize(&rc, p);
+ p->nNodeSize = p->nPgsz-35;
+
+ /* Declare the table schema to SQLite. */
+ fts3DeclareVtab(&rc, p);
+
+fts3_init_out:
+ sqlite3_free(zPrefix);
+ sqlite3_free(aIndex);
+ sqlite3_free(zCompress);
+ sqlite3_free(zUncompress);
+ sqlite3_free((void *)aCol);
+ if( rc!=SQLITE_OK ){
+ if( p ){
+ fts3DisconnectMethod((sqlite3_vtab *)p);
+ }else if( pTokenizer ){
+ pTokenizer->pModule->xDestroy(pTokenizer);
+ }
+ }else{
+ assert( p->pSegments==0 );
+ *ppVTab = &p->base;
+ }
+ return rc;
+}
+
+/*
+** The xConnect() and xCreate() methods for the virtual table. All the
+** work is done in function fts3InitVtab().
+*/
+static int fts3ConnectMethod(
+ sqlite3 *db, /* Database connection */
+ void *pAux, /* Pointer to tokenizer hash table */
+ int argc, /* Number of elements in argv array */
+ const char * const *argv, /* xCreate/xConnect argument array */
+ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
+ char **pzErr /* OUT: sqlite3_malloc'd error message */
+){
+ return fts3InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr);
+}
+static int fts3CreateMethod(
+ sqlite3 *db, /* Database connection */
+ void *pAux, /* Pointer to tokenizer hash table */
+ int argc, /* Number of elements in argv array */
+ const char * const *argv, /* xCreate/xConnect argument array */
+ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
+ char **pzErr /* OUT: sqlite3_malloc'd error message */
+){
+ return fts3InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr);
+}
+
+/*
+** Implementation of the xBestIndex method for FTS3 tables. There
+** are three possible strategies, in order of preference:
+**
+** 1. Direct lookup by rowid or docid.
+** 2. Full-text search using a MATCH operator on a non-docid column.
+** 3. Linear scan of %_content table.
+*/
+static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
+ Fts3Table *p = (Fts3Table *)pVTab;
+ int i; /* Iterator variable */
+ int iCons = -1; /* Index of constraint to use */
+
+ /* By default use a full table scan. This is an expensive option,
+ ** so search through the constraints to see if a more efficient
+ ** strategy is possible.
+ */
+ pInfo->idxNum = FTS3_FULLSCAN_SEARCH;
+ pInfo->estimatedCost = 500000;
+ for(i=0; i<pInfo->nConstraint; i++){
+ struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i];
+ if( pCons->usable==0 ) continue;
+
+ /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */
+ if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ
+ && (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1 )
+ ){
+ pInfo->idxNum = FTS3_DOCID_SEARCH;
+ pInfo->estimatedCost = 1.0;
+ iCons = i;
+ }
+
+ /* A MATCH constraint. Use a full-text search.
+ **
+ ** If there is more than one MATCH constraint available, use the first
+ ** one encountered. If there is both a MATCH constraint and a direct
+ ** rowid/docid lookup, prefer the MATCH strategy. This is done even
+ ** though the rowid/docid lookup is faster than a MATCH query, selecting
+ ** it would lead to an "unable to use function MATCH in the requested
+ ** context" error.
+ */
+ if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH
+ && pCons->iColumn>=0 && pCons->iColumn<=p->nColumn
+ ){
+ pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn;
+ pInfo->estimatedCost = 2.0;
+ iCons = i;
+ break;
+ }
+ }
+
+ if( iCons>=0 ){
+ pInfo->aConstraintUsage[iCons].argvIndex = 1;
+ pInfo->aConstraintUsage[iCons].omit = 1;
+ }
+
+ /* Regardless of the strategy selected, FTS can deliver rows in rowid (or
+ ** docid) order. Both ascending and descending are possible.
+ */
+ if( pInfo->nOrderBy==1 ){
+ struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0];
+ if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){
+ if( pOrder->desc ){
+ pInfo->idxStr = "DESC";
+ }else{
+ pInfo->idxStr = "ASC";
+ }
+ pInfo->orderByConsumed = 1;
+ }
+ }
+
+ assert( p->pSegments==0 );
+ return SQLITE_OK;
+}
+
+/*
+** Implementation of xOpen method.
+*/
+static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
+ sqlite3_vtab_cursor *pCsr; /* Allocated cursor */
+
+ UNUSED_PARAMETER(pVTab);
+
+ /* Allocate a buffer large enough for an Fts3Cursor structure. If the
+ ** allocation succeeds, zero it and return SQLITE_OK. Otherwise,
+ ** if the allocation fails, return SQLITE_NOMEM.
+ */
+ *ppCsr = pCsr = (sqlite3_vtab_cursor *)sqlite3_malloc(sizeof(Fts3Cursor));
+ if( !pCsr ){
+ return SQLITE_NOMEM;
+ }
+ memset(pCsr, 0, sizeof(Fts3Cursor));
+ return SQLITE_OK;
+}
+
+/*
+** Close the cursor. For additional information see the documentation
+** on the xClose method of the virtual table interface.
+*/
+static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){
+ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
+ assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
+ sqlite3_finalize(pCsr->pStmt);
+ sqlite3Fts3ExprFree(pCsr->pExpr);
+ sqlite3Fts3FreeDeferredTokens(pCsr);
+ sqlite3_free(pCsr->aDoclist);
+ sqlite3_free(pCsr->aMatchinfo);
+ assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
+ sqlite3_free(pCsr);
+ return SQLITE_OK;
+}
+
+/*
+** Position the pCsr->pStmt statement so that it is on the row
+** of the %_content table that contains the last match. Return
+** SQLITE_OK on success.
+*/
+static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){
+ if( pCsr->isRequireSeek ){
+ sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId);
+ pCsr->isRequireSeek = 0;
+ if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){
+ return SQLITE_OK;
+ }else{
+ int rc = sqlite3_reset(pCsr->pStmt);
+ if( rc==SQLITE_OK ){
+ /* If no row was found and no error has occured, then the %_content
+ ** table is missing a row that is present in the full-text index.
+ ** The data structures are corrupt.
+ */
+ rc = SQLITE_CORRUPT_VTAB;
+ }
+ pCsr->isEof = 1;
+ if( pContext ){
+ sqlite3_result_error_code(pContext, rc);
+ }
+ return rc;
+ }
+ }else{
+ return SQLITE_OK;
+ }
+}
+
+/*
+** This function is used to process a single interior node when searching
+** a b-tree for a term or term prefix. The node data is passed to this
+** function via the zNode/nNode parameters. The term to search for is
+** passed in zTerm/nTerm.
+**
+** If piFirst is not NULL, then this function sets *piFirst to the blockid
+** of the child node that heads the sub-tree that may contain the term.
+**
+** If piLast is not NULL, then *piLast is set to the right-most child node
+** that heads a sub-tree that may contain a term for which zTerm/nTerm is
+** a prefix.
+**
+** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK.
+*/
+static int fts3ScanInteriorNode(
+ const char *zTerm, /* Term to select leaves for */
+ int nTerm, /* Size of term zTerm in bytes */
+ const char *zNode, /* Buffer containing segment interior node */
+ int nNode, /* Size of buffer at zNode */
+ sqlite3_int64 *piFirst, /* OUT: Selected child node */
+ sqlite3_int64 *piLast /* OUT: Selected child node */
+){
+ int rc = SQLITE_OK; /* Return code */
+ const char *zCsr = zNode; /* Cursor to iterate through node */
+ const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
+ char *zBuffer = 0; /* Buffer to load terms into */
+ int nAlloc = 0; /* Size of allocated buffer */
+ int isFirstTerm = 1; /* True when processing first term on page */
+ sqlite3_int64 iChild; /* Block id of child node to descend to */
+
+ /* Skip over the 'height' varint that occurs at the start of every
+ ** interior node. Then load the blockid of the left-child of the b-tree
+ ** node into variable iChild.
+ **
+ ** Even if the data structure on disk is corrupted, this (reading two
+ ** varints from the buffer) does not risk an overread. If zNode is a
+ ** root node, then the buffer comes from a SELECT statement. SQLite does
+ ** not make this guarantee explicitly, but in practice there are always
+ ** either more than 20 bytes of allocated space following the nNode bytes of
+ ** contents, or two zero bytes. Or, if the node is read from the %_segments
+ ** table, then there are always 20 bytes of zeroed padding following the
+ ** nNode bytes of content (see sqlite3Fts3ReadBlock() for details).
+ */
+ zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
+ zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
+ if( zCsr>zEnd ){
+ return SQLITE_CORRUPT_VTAB;
+ }
+
+ while( zCsr<zEnd && (piFirst || piLast) ){
+ int cmp; /* memcmp() result */
+ int nSuffix; /* Size of term suffix */
+ int nPrefix = 0; /* Size of term prefix */
+ int nBuffer; /* Total term size */
+
+ /* Load the next term on the node into zBuffer. Use realloc() to expand
+ ** the size of zBuffer if required. */
+ if( !isFirstTerm ){
+ zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix);
+ }
+ isFirstTerm = 0;
+ zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix);
+
+ if( nPrefix<0 || nSuffix<0 || &zCsr[nSuffix]>zEnd ){
+ rc = SQLITE_CORRUPT_VTAB;
+ goto finish_scan;
+ }
+ if( nPrefix+nSuffix>nAlloc ){
+ char *zNew;
+ nAlloc = (nPrefix+nSuffix) * 2;
+ zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
+ if( !zNew ){
+ rc = SQLITE_NOMEM;
+ goto finish_scan;
+ }
+ zBuffer = zNew;
+ }
+ memcpy(&zBuffer[nPrefix], zCsr, nSuffix);
+ nBuffer = nPrefix + nSuffix;
+ zCsr += nSuffix;
+
+ /* Compare the term we are searching for with the term just loaded from
+ ** the interior node. If the specified term is greater than or equal
+ ** to the term from the interior node, then all terms on the sub-tree
+ ** headed by node iChild are smaller than zTerm. No need to search
+ ** iChild.
+ **
+ ** If the interior node term is larger than the specified term, then
+ ** the tree headed by iChild may contain the specified term.
+ */
+ cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer));
+ if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){
+ *piFirst = iChild;
+ piFirst = 0;
+ }
+
+ if( piLast && cmp<0 ){
+ *piLast = iChild;
+ piLast = 0;
+ }
+
+ iChild++;
+ };
+
+ if( piFirst ) *piFirst = iChild;
+ if( piLast ) *piLast = iChild;
+
+ finish_scan:
+ sqlite3_free(zBuffer);
+ return rc;
+}
+
+
+/*
+** The buffer pointed to by argument zNode (size nNode bytes) contains an
+** interior node of a b-tree segment. The zTerm buffer (size nTerm bytes)
+** contains a term. This function searches the sub-tree headed by the zNode
+** node for the range of leaf nodes that may contain the specified term
+** or terms for which the specified term is a prefix.
+**
+** If piLeaf is not NULL, then *piLeaf is set to the blockid of the
+** left-most leaf node in the tree that may contain the specified term.
+** If piLeaf2 is not NULL, then *piLeaf2 is set to the blockid of the
+** right-most leaf node that may contain a term for which the specified
+** term is a prefix.
+**
+** It is possible that the range of returned leaf nodes does not contain
+** the specified term or any terms for which it is a prefix. However, if the
+** segment does contain any such terms, they are stored within the identified
+** range. Because this function only inspects interior segment nodes (and
+** never loads leaf nodes into memory), it is not possible to be sure.
+**
+** If an error occurs, an error code other than SQLITE_OK is returned.
+*/
+static int fts3SelectLeaf(
+ Fts3Table *p, /* Virtual table handle */
+ const char *zTerm, /* Term to select leaves for */
+ int nTerm, /* Size of term zTerm in bytes */
+ const char *zNode, /* Buffer containing segment interior node */
+ int nNode, /* Size of buffer at zNode */
+ sqlite3_int64 *piLeaf, /* Selected leaf node */
+ sqlite3_int64 *piLeaf2 /* Selected leaf node */
+){
+ int rc; /* Return code */
+ int iHeight; /* Height of this node in tree */
+
+ assert( piLeaf || piLeaf2 );
+
+ sqlite3Fts3GetVarint32(zNode, &iHeight);
+ rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2);
+ assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) );
+
+ if( rc==SQLITE_OK && iHeight>1 ){
+ char *zBlob = 0; /* Blob read from %_segments table */
+ int nBlob; /* Size of zBlob in bytes */
+
+ if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){
+ rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0);
+ if( rc==SQLITE_OK ){
+ rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0);
+ }
+ sqlite3_free(zBlob);
+ piLeaf = 0;
+ zBlob = 0;
+ }
+
+ if( rc==SQLITE_OK ){
+ rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0);
+ }
+ if( rc==SQLITE_OK ){
+ rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2);
+ }
+ sqlite3_free(zBlob);
+ }
+
+ return rc;
+}
+
+/*
+** This function is used to create delta-encoded serialized lists of FTS3
+** varints. Each call to this function appends a single varint to a list.
+*/
+static void fts3PutDeltaVarint(
+ char **pp, /* IN/OUT: Output pointer */
+ sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */
+ sqlite3_int64 iVal /* Write this value to the list */
+){
+ assert( iVal-*piPrev > 0 || (*piPrev==0 && iVal==0) );
+ *pp += sqlite3Fts3PutVarint(*pp, iVal-*piPrev);
+ *piPrev = iVal;
+}
+
+/*
+** When this function is called, *ppPoslist is assumed to point to the
+** start of a position-list. After it returns, *ppPoslist points to the
+** first byte after the position-list.
+**
+** A position list is list of positions (delta encoded) and columns for
+** a single document record of a doclist. So, in other words, this
+** routine advances *ppPoslist so that it points to the next docid in
+** the doclist, or to the first byte past the end of the doclist.
+**
+** If pp is not NULL, then the contents of the position list are copied
+** to *pp. *pp is set to point to the first byte past the last byte copied
+** before this function returns.
+*/
+static void fts3PoslistCopy(char **pp, char **ppPoslist){
+ char *pEnd = *ppPoslist;
+ char c = 0;
+
+ /* The end of a position list is marked by a zero encoded as an FTS3
+ ** varint. A single POS_END (0) byte. Except, if the 0 byte is preceded by
+ ** a byte with the 0x80 bit set, then it is not a varint 0, but the tail
+ ** of some other, multi-byte, value.
+ **
+ ** The following while-loop moves pEnd to point to the first byte that is not
+ ** immediately preceded by a byte with the 0x80 bit set. Then increments
+ ** pEnd once more so that it points to the byte immediately following the
+ ** last byte in the position-list.
+ */
+ while( *pEnd | c ){
+ c = *pEnd++ & 0x80;
+ testcase( c!=0 && (*pEnd)==0 );
+ }
+ pEnd++; /* Advance past the POS_END terminator byte */
+
+ if( pp ){
+ int n = (int)(pEnd - *ppPoslist);
+ char *p = *pp;
+ memcpy(p, *ppPoslist, n);
+ p += n;
+ *pp = p;
+ }
+ *ppPoslist = pEnd;
+}
+
+/*
+** When this function is called, *ppPoslist is assumed to point to the
+** start of a column-list. After it returns, *ppPoslist points to the
+** to the terminator (POS_COLUMN or POS_END) byte of the column-list.
+**
+** A column-list is list of delta-encoded positions for a single column
+** within a single document within a doclist.
+**
+** The column-list is terminated either by a POS_COLUMN varint (1) or
+** a POS_END varint (0). This routine leaves *ppPoslist pointing to
+** the POS_COLUMN or POS_END that terminates the column-list.
+**
+** If pp is not NULL, then the contents of the column-list are copied
+** to *pp. *pp is set to point to the first byte past the last byte copied
+** before this function returns. The POS_COLUMN or POS_END terminator
+** is not copied into *pp.
+*/
+static void fts3ColumnlistCopy(char **pp, char **ppPoslist){
+ char *pEnd = *ppPoslist;
+ char c = 0;
+
+ /* A column-list is terminated by either a 0x01 or 0x00 byte that is
+ ** not part of a multi-byte varint.
+ */
+ while( 0xFE & (*pEnd | c) ){
+ c = *pEnd++ & 0x80;
+ testcase( c!=0 && ((*pEnd)&0xfe)==0 );
+ }
+ if( pp ){
+ int n = (int)(pEnd - *ppPoslist);
+ char *p = *pp;
+ memcpy(p, *ppPoslist, n);
+ p += n;
+ *pp = p;
+ }
+ *ppPoslist = pEnd;
+}
+
+/*
+** Value used to signify the end of an position-list. This is safe because
+** it is not possible to have a document with 2^31 terms.
+*/
+#define POSITION_LIST_END 0x7fffffff
+
+/*
+** This function is used to help parse position-lists. When this function is
+** called, *pp may point to the start of the next varint in the position-list
+** being parsed, or it may point to 1 byte past the end of the position-list
+** (in which case **pp will be a terminator bytes POS_END (0) or
+** (1)).
+**
+** If *pp points past the end of the current position-list, set *pi to
+** POSITION_LIST_END and return. Otherwise, read the next varint from *pp,
+** increment the current value of *pi by the value read, and set *pp to
+** point to the next value before returning.
+**
+** Before calling this routine *pi must be initialized to the value of
+** the previous position, or zero if we are reading the first position
+** in the position-list. Because positions are delta-encoded, the value
+** of the previous position is needed in order to compute the value of
+** the next position.
+*/
+static void fts3ReadNextPos(
+ char **pp, /* IN/OUT: Pointer into position-list buffer */
+ sqlite3_int64 *pi /* IN/OUT: Value read from position-list */
+){
+ if( (**pp)&0xFE ){
+ fts3GetDeltaVarint(pp, pi);
+ *pi -= 2;
+ }else{
+ *pi = POSITION_LIST_END;
+ }
+}
+
+/*
+** If parameter iCol is not 0, write an POS_COLUMN (1) byte followed by
+** the value of iCol encoded as a varint to *pp. This will start a new
+** column list.
+**
+** Set *pp to point to the byte just after the last byte written before
+** returning (do not modify it if iCol==0). Return the total number of bytes
+** written (0 if iCol==0).
+*/
+static int fts3PutColNumber(char **pp, int iCol){
+ int n = 0; /* Number of bytes written */
+ if( iCol ){
+ char *p = *pp; /* Output pointer */
+ n = 1 + sqlite3Fts3PutVarint(&p[1], iCol);
+ *p = 0x01;
+ *pp = &p[n];
+ }
+ return n;
+}
+
+/*
+** Compute the union of two position lists. The output written
+** into *pp contains all positions of both *pp1 and *pp2 in sorted
+** order and with any duplicates removed. All pointers are
+** updated appropriately. The caller is responsible for insuring
+** that there is enough space in *pp to hold the complete output.
+*/
+static void fts3PoslistMerge(
+ char **pp, /* Output buffer */
+ char **pp1, /* Left input list */
+ char **pp2 /* Right input list */
+){
+ char *p = *pp;
+ char *p1 = *pp1;
+ char *p2 = *pp2;
+
+ while( *p1 || *p2 ){
+ int iCol1; /* The current column index in pp1 */
+ int iCol2; /* The current column index in pp2 */
+
+ if( *p1==POS_COLUMN ) sqlite3Fts3GetVarint32(&p1[1], &iCol1);
+ else if( *p1==POS_END ) iCol1 = POSITION_LIST_END;
+ else iCol1 = 0;
+
+ if( *p2==POS_COLUMN ) sqlite3Fts3GetVarint32(&p2[1], &iCol2);
+ else if( *p2==POS_END ) iCol2 = POSITION_LIST_END;
+ else iCol2 = 0;
+
+ if( iCol1==iCol2 ){
+ sqlite3_int64 i1 = 0; /* Last position from pp1 */
+ sqlite3_int64 i2 = 0; /* Last position from pp2 */
+ sqlite3_int64 iPrev = 0;
+ int n = fts3PutColNumber(&p, iCol1);
+ p1 += n;
+ p2 += n;
+
+ /* At this point, both p1 and p2 point to the start of column-lists
+ ** for the same column (the column with index iCol1 and iCol2).
+ ** A column-list is a list of non-negative delta-encoded varints, each
+ ** incremented by 2 before being stored. Each list is terminated by a
+ ** POS_END (0) or POS_COLUMN (1). The following block merges the two lists
+ ** and writes the results to buffer p. p is left pointing to the byte
+ ** after the list written. No terminator (POS_END or POS_COLUMN) is
+ ** written to the output.
+ */
+ fts3GetDeltaVarint(&p1, &i1);
+ fts3GetDeltaVarint(&p2, &i2);
+ do {
+ fts3PutDeltaVarint(&p, &iPrev, (i1<i2) ? i1 : i2);
+ iPrev -= 2;
+ if( i1==i2 ){
+ fts3ReadNextPos(&p1, &i1);
+ fts3ReadNextPos(&p2, &i2);
+ }else if( i1<i2 ){
+ fts3ReadNextPos(&p1, &i1);
+ }else{
+ fts3ReadNextPos(&p2, &i2);
+ }
+ }while( i1!=POSITION_LIST_END || i2!=POSITION_LIST_END );
+ }else if( iCol1<iCol2 ){
+ p1 += fts3PutColNumber(&p, iCol1);
+ fts3ColumnlistCopy(&p, &p1);
+ }else{
+ p2 += fts3PutColNumber(&p, iCol2);
+ fts3ColumnlistCopy(&p, &p2);
+ }
+ }
+
+ *p++ = POS_END;
+ *pp = p;
+ *pp1 = p1 + 1;
+ *pp2 = p2 + 1;
+}
+
+/*
+** This function is used to merge two position lists into one. When it is
+** called, *pp1 and *pp2 must both point to position lists. A position-list is
+** the part of a doclist that follows each document id. For example, if a row
+** contains:
+**
+** 'a b c'|'x y z'|'a b b a'
+**
+** Then the position list for this row for token 'b' would consist of:
+**
+** 0x02 0x01 0x02 0x03 0x03 0x00
+**
+** When this function returns, both *pp1 and *pp2 are left pointing to the
+** byte following the 0x00 terminator of their respective position lists.
+**
+** If isSaveLeft is 0, an entry is added to the output position list for
+** each position in *pp2 for which there exists one or more positions in
+** *pp1 so that (pos(*pp2)>pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e.
+** when the *pp1 token appears before the *pp2 token, but not more than nToken
+** slots before it.
+**
+** e.g. nToken==1 searches for adjacent positions.
+*/
+static int fts3PoslistPhraseMerge(
+ char **pp, /* IN/OUT: Preallocated output buffer */
+ int nToken, /* Maximum difference in token positions */
+ int isSaveLeft, /* Save the left position */
+ int isExact, /* If *pp1 is exactly nTokens before *pp2 */
+ char **pp1, /* IN/OUT: Left input list */
+ char **pp2 /* IN/OUT: Right input list */
+){
+ char *p = (pp ? *pp : 0);
+ char *p1 = *pp1;
+ char *p2 = *pp2;
+ int iCol1 = 0;
+ int iCol2 = 0;
+
+ /* Never set both isSaveLeft and isExact for the same invocation. */
+ assert( isSaveLeft==0 || isExact==0 );
+
+ assert( *p1!=0 && *p2!=0 );
+ if( *p1==POS_COLUMN ){
+ p1++;
+ p1 += sqlite3Fts3GetVarint32(p1, &iCol1);
+ }
+ if( *p2==POS_COLUMN ){
+ p2++;
+ p2 += sqlite3Fts3GetVarint32(p2, &iCol2);
+ }
+
+ while( 1 ){
+ if( iCol1==iCol2 ){
+ char *pSave = p;
+ sqlite3_int64 iPrev = 0;
+ sqlite3_int64 iPos1 = 0;
+ sqlite3_int64 iPos2 = 0;
+
+ if( pp && iCol1 ){
+ *p++ = POS_COLUMN;
+ p += sqlite3Fts3PutVarint(p, iCol1);
+ }
+
+ assert( *p1!=POS_END && *p1!=POS_COLUMN );
+ assert( *p2!=POS_END && *p2!=POS_COLUMN );
+ fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2;
+ fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2;
+
+ while( 1 ){
+ if( iPos2==iPos1+nToken
+ || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken)
+ ){
+ sqlite3_int64 iSave;
+ if( !pp ){
+ fts3PoslistCopy(0, &p2);
+ fts3PoslistCopy(0, &p1);
+ *pp1 = p1;
+ *pp2 = p2;
+ return 1;
+ }
+ iSave = isSaveLeft ? iPos1 : iPos2;
+ fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2;
+ pSave = 0;
+ }
+ if( (!isSaveLeft && iPos2<=(iPos1+nToken)) || iPos2<=iPos1 ){
+ if( (*p2&0xFE)==0 ) break;
+ fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2;
+ }else{
+ if( (*p1&0xFE)==0 ) break;
+ fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2;
+ }
+ }
+
+ if( pSave ){
+ assert( pp && p );
+ p = pSave;
+ }
+
+ fts3ColumnlistCopy(0, &p1);
+ fts3ColumnlistCopy(0, &p2);
+ assert( (*p1&0xFE)==0 && (*p2&0xFE)==0 );
+ if( 0==*p1 || 0==*p2 ) break;
+
+ p1++;
+ p1 += sqlite3Fts3GetVarint32(p1, &iCol1);
+ p2++;
+ p2 += sqlite3Fts3GetVarint32(p2, &iCol2);
+ }
+
+ /* Advance pointer p1 or p2 (whichever corresponds to the smaller of
+ ** iCol1 and iCol2) so that it points to either the 0x00 that marks the
+ ** end of the position list, or the 0x01 that precedes the next
+ ** column-number in the position list.
+ */
+ else if( iCol1<iCol2 ){
+ fts3ColumnlistCopy(0, &p1);
+ if( 0==*p1 ) break;
+ p1++;
+ p1 += sqlite3Fts3GetVarint32(p1, &iCol1);
+ }else{
+ fts3ColumnlistCopy(0, &p2);
+ if( 0==*p2 ) break;
+ p2++;
+ p2 += sqlite3Fts3GetVarint32(p2, &iCol2);
+ }
+ }
+
+ fts3PoslistCopy(0, &p2);
+ fts3PoslistCopy(0, &p1);
+ *pp1 = p1;
+ *pp2 = p2;
+ if( !pp || *pp==p ){
+ return 0;
+ }
+ *p++ = 0x00;
+ *pp = p;
+ return 1;
+}
+
+/*
+** Merge two position-lists as required by the NEAR operator. The argument
+** position lists correspond to the left and right phrases of an expression
+** like:
+**
+** "phrase 1" NEAR "phrase number 2"
+**
+** Position list *pp1 corresponds to the left-hand side of the NEAR
+** expression and *pp2 to the right. As usual, the indexes in the position
+** lists are the offsets of the last token in each phrase (tokens "1" and "2"
+** in the example above).
+**
+** The output position list - written to *pp - is a copy of *pp2 with those
+** entries that are not sufficiently NEAR entries in *pp1 removed.
+*/
+static int fts3PoslistNearMerge(
+ char **pp, /* Output buffer */
+ char *aTmp, /* Temporary buffer space */
+ int nRight, /* Maximum difference in token positions */
+ int nLeft, /* Maximum difference in token positions */
+ char **pp1, /* IN/OUT: Left input list */
+ char **pp2 /* IN/OUT: Right input list */
+){
+ char *p1 = *pp1;
+ char *p2 = *pp2;
+
+ char *pTmp1 = aTmp;
+ char *pTmp2;
+ char *aTmp2;
+ int res = 1;
+
+ fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2);
+ aTmp2 = pTmp2 = pTmp1;
+ *pp1 = p1;
+ *pp2 = p2;
+ fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1);
+ if( pTmp1!=aTmp && pTmp2!=aTmp2 ){
+ fts3PoslistMerge(pp, &aTmp, &aTmp2);
+ }else if( pTmp1!=aTmp ){
+ fts3PoslistCopy(pp, &aTmp);
+ }else if( pTmp2!=aTmp2 ){
+ fts3PoslistCopy(pp, &aTmp2);
+ }else{
+ res = 0;
+ }
+
+ return res;
+}
+
+/*
+** An instance of this function is used to merge together the (potentially
+** large number of) doclists for each term that matches a prefix query.
+** See function fts3TermSelectMerge() for details.
+*/
+typedef struct TermSelect TermSelect;
+struct TermSelect {
+ char *aaOutput[16]; /* Malloc'd output buffers */
+ int anOutput[16]; /* Size each output buffer in bytes */
+};
+
+/*
+** This function is used to read a single varint from a buffer. Parameter
+** pEnd points 1 byte past the end of the buffer. When this function is
+** called, if *pp points to pEnd or greater, then the end of the buffer
+** has been reached. In this case *pp is set to 0 and the function returns.
+**
+** If *pp does not point to or past pEnd, then a single varint is read
+** from *pp. *pp is then set to point 1 byte past the end of the read varint.
+**
+** If bDescIdx is false, the value read is added to *pVal before returning.
+** If it is true, the value read is subtracted from *pVal before this
+** function returns.
+*/
+static void fts3GetDeltaVarint3(
+ char **pp, /* IN/OUT: Point to read varint from */
+ char *pEnd, /* End of buffer */
+ int bDescIdx, /* True if docids are descending */
+ sqlite3_int64 *pVal /* IN/OUT: Integer value */
+){
+ if( *pp>=pEnd ){
+ *pp = 0;
+ }else{
+ sqlite3_int64 iVal;
+ *pp += sqlite3Fts3GetVarint(*pp, &iVal);
+ if( bDescIdx ){
+ *pVal -= iVal;
+ }else{
+ *pVal += iVal;
+ }
+ }
+}
+
+/*
+** This function is used to write a single varint to a buffer. The varint
+** is written to *pp. Before returning, *pp is set to point 1 byte past the
+** end of the value written.
+**
+** If *pbFirst is zero when this function is called, the value written to
+** the buffer is that of parameter iVal.
+**
+** If *pbFirst is non-zero when this function is called, then the value
+** written is either (iVal-*piPrev) (if bDescIdx is zero) or (*piPrev-iVal)
+** (if bDescIdx is non-zero).
+**
+** Before returning, this function always sets *pbFirst to 1 and *piPrev
+** to the value of parameter iVal.
+*/
+static void fts3PutDeltaVarint3(
+ char **pp, /* IN/OUT: Output pointer */
+ int bDescIdx, /* True for descending docids */
+ sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */
+ int *pbFirst, /* IN/OUT: True after first int written */
+ sqlite3_int64 iVal /* Write this value to the list */
+){
+ sqlite3_int64 iWrite;
+ if( bDescIdx==0 || *pbFirst==0 ){
+ iWrite = iVal - *piPrev;
+ }else{
+ iWrite = *piPrev - iVal;
+ }
+ assert( *pbFirst || *piPrev==0 );
+ assert( *pbFirst==0 || iWrite>0 );
+ *pp += sqlite3Fts3PutVarint(*pp, iWrite);
+ *piPrev = iVal;
+ *pbFirst = 1;
+}
+
+
+/*
+** This macro is used by various functions that merge doclists. The two
+** arguments are 64-bit docid values. If the value of the stack variable
+** bDescDoclist is 0 when this macro is invoked, then it returns (i1-i2).
+** Otherwise, (i2-i1).
+**
+** Using this makes it easier to write code that can merge doclists that are
+** sorted in either ascending or descending order.
+*/
+#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1-i2))
+
+/*
+** This function does an "OR" merge of two doclists (output contains all
+** positions contained in either argument doclist). If the docids in the
+** input doclists are sorted in ascending order, parameter bDescDoclist
+** should be false. If they are sorted in ascending order, it should be
+** passed a non-zero value.
+**
+** If no error occurs, *paOut is set to point at an sqlite3_malloc'd buffer
+** containing the output doclist and SQLITE_OK is returned. In this case
+** *pnOut is set to the number of bytes in the output doclist.
+**
+** If an error occurs, an SQLite error code is returned. The output values
+** are undefined in this case.
+*/
+static int fts3DoclistOrMerge(
+ int bDescDoclist, /* True if arguments are desc */
+ char *a1, int n1, /* First doclist */
+ char *a2, int n2, /* Second doclist */
+ char **paOut, int *pnOut /* OUT: Malloc'd doclist */
+){
+ sqlite3_int64 i1 = 0;
+ sqlite3_int64 i2 = 0;
+ sqlite3_int64 iPrev = 0;
+ char *pEnd1 = &a1[n1];
+ char *pEnd2 = &a2[n2];
+ char *p1 = a1;
+ char *p2 = a2;
+ char *p;
+ char *aOut;
+ int bFirstOut = 0;
+
+ *paOut = 0;
+ *pnOut = 0;
+
+ /* Allocate space for the output. Both the input and output doclists
+ ** are delta encoded. If they are in ascending order (bDescDoclist==0),
+ ** then the first docid in each list is simply encoded as a varint. For
+ ** each subsequent docid, the varint stored is the difference between the
+ ** current and previous docid (a positive number - since the list is in
+ ** ascending order).
+ **
+ ** The first docid written to the output is therefore encoded using the
+ ** same number of bytes as it is in whichever of the input lists it is
+ ** read from. And each subsequent docid read from the same input list
+ ** consumes either the same or less bytes as it did in the input (since
+ ** the difference between it and the previous value in the output must
+ ** be a positive value less than or equal to the delta value read from
+ ** the input list). The same argument applies to all but the first docid
+ ** read from the 'other' list. And to the contents of all position lists
+ ** that will be copied and merged from the input to the output.
+ **
+ ** However, if the first docid copied to the output is a negative number,
+ ** then the encoding of the first docid from the 'other' input list may
+ ** be larger in the output than it was in the input (since the delta value
+ ** may be a larger positive integer than the actual docid).
+ **
+ ** The space required to store the output is therefore the sum of the
+ ** sizes of the two inputs, plus enough space for exactly one of the input
+ ** docids to grow.
+ **
+ ** A symetric argument may be made if the doclists are in descending
+ ** order.
+ */
+ aOut = sqlite3_malloc(n1+n2+FTS3_VARINT_MAX-1);
+ if( !aOut ) return SQLITE_NOMEM;
+
+ p = aOut;
+ fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
+ fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);
+ while( p1 || p2 ){
+ sqlite3_int64 iDiff = DOCID_CMP(i1, i2);
+
+ if( p2 && p1 && iDiff==0 ){
+ fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1);
+ fts3PoslistMerge(&p, &p1, &p2);
+ fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1);
+ fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
+ }else if( !p2 || (p1 && iDiff<0) ){
+ fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1);
+ fts3PoslistCopy(&p, &p1);
+ fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1);
+ }else{
+ fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i2);
+ fts3PoslistCopy(&p, &p2);
+ fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
+ }
+ }
+
+ *paOut = aOut;
+ *pnOut = (p-aOut);
+ assert( *pnOut<=n1+n2+FTS3_VARINT_MAX-1 );
+ return SQLITE_OK;
+}
+
+/*
+** This function does a "phrase" merge of two doclists. In a phrase merge,
+** the output contains a copy of each position from the right-hand input
+** doclist for which there is a position in the left-hand input doclist
+** exactly nDist tokens before it.
+**
+** If the docids in the input doclists are sorted in ascending order,
+** parameter bDescDoclist should be false. If they are sorted in ascending
+** order, it should be passed a non-zero value.
+**
+** The right-hand input doclist is overwritten by this function.
+*/
+static void fts3DoclistPhraseMerge(
+ int bDescDoclist, /* True if arguments are desc */
+ int nDist, /* Distance from left to right (1=adjacent) */
+ char *aLeft, int nLeft, /* Left doclist */
+ char *aRight, int *pnRight /* IN/OUT: Right/output doclist */
+){
+ sqlite3_int64 i1 = 0;
+ sqlite3_int64 i2 = 0;
+ sqlite3_int64 iPrev = 0;
+ char *pEnd1 = &aLeft[nLeft];
+ char *pEnd2 = &aRight[*pnRight];
+ char *p1 = aLeft;
+ char *p2 = aRight;
+ char *p;
+ int bFirstOut = 0;
+ char *aOut = aRight;
+
+ assert( nDist>0 );
+
+ p = aOut;
+ fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
+ fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);
+
+ while( p1 && p2 ){
+ sqlite3_int64 iDiff = DOCID_CMP(i1, i2);
+ if( iDiff==0 ){
+ char *pSave = p;
+ sqlite3_int64 iPrevSave = iPrev;
+ int bFirstOutSave = bFirstOut;
+
+ fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1);
+ if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){
+ p = pSave;
+ iPrev = iPrevSave;
+ bFirstOut = bFirstOutSave;
+ }
+ fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1);
+ fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
+ }else if( iDiff<0 ){
+ fts3PoslistCopy(0, &p1);
+ fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1);
+ }else{
+ fts3PoslistCopy(0, &p2);
+ fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
+ }
+ }
+
+ *pnRight = p - aOut;
+}
+
+
+/*
+** Merge all doclists in the TermSelect.aaOutput[] array into a single
+** doclist stored in TermSelect.aaOutput[0]. If successful, delete all
+** other doclists (except the aaOutput[0] one) and return SQLITE_OK.
+**
+** If an OOM error occurs, return SQLITE_NOMEM. In this case it is
+** the responsibility of the caller to free any doclists left in the
+** TermSelect.aaOutput[] array.
+*/
+static int fts3TermSelectFinishMerge(Fts3Table *p, TermSelect *pTS){
+ char *aOut = 0;
+ int nOut = 0;
+ int i;
+
+ /* Loop through the doclists in the aaOutput[] array. Merge them all
+ ** into a single doclist.
+ */
+ for(i=0; i<SizeofArray(pTS->aaOutput); i++){
+ if( pTS->aaOutput[i] ){
+ if( !aOut ){
+ aOut = pTS->aaOutput[i];
+ nOut = pTS->anOutput[i];
+ pTS->aaOutput[i] = 0;
+ }else{
+ int nNew;
+ char *aNew;
+
+ int rc = fts3DoclistOrMerge(p->bDescIdx,
+ pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew
+ );
+ if( rc!=SQLITE_OK ){
+ sqlite3_free(aOut);
+ return rc;
+ }
+
+ sqlite3_free(pTS->aaOutput[i]);
+ sqlite3_free(aOut);
+ pTS->aaOutput[i] = 0;
+ aOut = aNew;
+ nOut = nNew;
+ }
+ }
+ }
+
+ pTS->aaOutput[0] = aOut;
+ pTS->anOutput[0] = nOut;
+ return SQLITE_OK;
+}
+
+/*
+** Merge the doclist aDoclist/nDoclist into the TermSelect object passed
+** as the first argument. The merge is an "OR" merge (see function
+** fts3DoclistOrMerge() for details).
+**
+** This function is called with the doclist for each term that matches
+** a queried prefix. It merges all these doclists into one, the doclist
+** for the specified prefix. Since there can be a very large number of
+** doclists to merge, the merging is done pair-wise using the TermSelect
+** object.
+**
+** This function returns SQLITE_OK if the merge is successful, or an
+** SQLite error code (SQLITE_NOMEM) if an error occurs.
+*/
+static int fts3TermSelectMerge(
+ Fts3Table *p, /* FTS table handle */
+ TermSelect *pTS, /* TermSelect object to merge into */
+ char *aDoclist, /* Pointer to doclist */
+ int nDoclist /* Size of aDoclist in bytes */
+){
+ if( pTS->aaOutput[0]==0 ){
+ /* If this is the first term selected, copy the doclist to the output
+ ** buffer using memcpy(). */
+ pTS->aaOutput[0] = sqlite3_malloc(nDoclist);
+ pTS->anOutput[0] = nDoclist;
+ if( pTS->aaOutput[0] ){
+ memcpy(pTS->aaOutput[0], aDoclist, nDoclist);
+ }else{
+ return SQLITE_NOMEM;
+ }
+ }else{
+ char *aMerge = aDoclist;
+ int nMerge = nDoclist;
+ int iOut;
+
+ for(iOut=0; iOut<SizeofArray(pTS->aaOutput); iOut++){
+ if( pTS->aaOutput[iOut]==0 ){
+ assert( iOut>0 );
+ pTS->aaOutput[iOut] = aMerge;
+ pTS->anOutput[iOut] = nMerge;
+ break;
+ }else{
+ char *aNew;
+ int nNew;
+
+ int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge,
+ pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew
+ );
+ if( rc!=SQLITE_OK ){
+ if( aMerge!=aDoclist ) sqlite3_free(aMerge);
+ return rc;
+ }
+
+ if( aMerge!=aDoclist ) sqlite3_free(aMerge);
+ sqlite3_free(pTS->aaOutput[iOut]);
+ pTS->aaOutput[iOut] = 0;
+
+ aMerge = aNew;
+ nMerge = nNew;
+ if( (iOut+1)==SizeofArray(pTS->aaOutput) ){
+ pTS->aaOutput[iOut] = aMerge;
+ pTS->anOutput[iOut] = nMerge;
+ }
+ }
+ }
+ }
+ return SQLITE_OK;
+}
+
+/*
+** Append SegReader object pNew to the end of the pCsr->apSegment[] array.
+*/
+static int fts3SegReaderCursorAppend(
+ Fts3MultiSegReader *pCsr,
+ Fts3SegReader *pNew
+){
+ if( (pCsr->nSegment%16)==0 ){
+ Fts3SegReader **apNew;
+ int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*);
+ apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte);
+ if( !apNew ){
+ sqlite3Fts3SegReaderFree(pNew);
+ return SQLITE_NOMEM;
+ }
+ pCsr->apSegment = apNew;
+ }
+ pCsr->apSegment[pCsr->nSegment++] = pNew;
+ return SQLITE_OK;
+}
+
+/*
+** Add seg-reader objects to the Fts3MultiSegReader object passed as the
+** 8th argument.
+**
+** This function returns SQLITE_OK if successful, or an SQLite error code
+** otherwise.
+*/
+static int fts3SegReaderCursor(
+ Fts3Table *p, /* FTS3 table handle */
+ int iIndex, /* Index to search (from 0 to p->nIndex-1) */
+ int iLevel, /* Level of segments to scan */
+ const char *zTerm, /* Term to query for */
+ int nTerm, /* Size of zTerm in bytes */
+ int isPrefix, /* True for a prefix search */
+ int isScan, /* True to scan from zTerm to EOF */
+ Fts3MultiSegReader *pCsr /* Cursor object to populate */
+){
+ int rc = SQLITE_OK; /* Error code */
+ sqlite3_stmt *pStmt = 0; /* Statement to iterate through segments */
+ int rc2; /* Result of sqlite3_reset() */
+
+ /* If iLevel is less than 0 and this is not a scan, include a seg-reader
+ ** for the pending-terms. If this is a scan, then this call must be being
+ ** made by an fts4aux module, not an FTS table. In this case calling
+ ** Fts3SegReaderPending might segfault, as the data structures used by
+ ** fts4aux are not completely populated. So it's easiest to filter these
+ ** calls out here. */
+ if( iLevel<0 && p->aIndex ){
+ Fts3SegReader *pSeg = 0;
+ rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix, &pSeg);
+ if( rc==SQLITE_OK && pSeg ){
+ rc = fts3SegReaderCursorAppend(pCsr, pSeg);
+ }
+ }
+
+ if( iLevel!=FTS3_SEGCURSOR_PENDING ){
+ if( rc==SQLITE_OK ){
+ rc = sqlite3Fts3AllSegdirs(p, iIndex, iLevel, &pStmt);
+ }
+
+ while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
+ Fts3SegReader *pSeg = 0;
+
+ /* Read the values returned by the SELECT into local variables. */
+ sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1);
+ sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2);
+ sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3);
+ int nRoot = sqlite3_column_bytes(pStmt, 4);
+ char const *zRoot = sqlite3_column_blob(pStmt, 4);
+
+ /* If zTerm is not NULL, and this segment is not stored entirely on its
+ ** root node, the range of leaves scanned can be reduced. Do this. */
+ if( iStartBlock && zTerm ){
+ sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0);
+ rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi);
+ if( rc!=SQLITE_OK ) goto finished;
+ if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock;
+ }
+
+ rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1,
+ iStartBlock, iLeavesEndBlock, iEndBlock, zRoot, nRoot, &pSeg
+ );
+ if( rc!=SQLITE_OK ) goto finished;
+ rc = fts3SegReaderCursorAppend(pCsr, pSeg);
+ }
+ }
+
+ finished:
+ rc2 = sqlite3_reset(pStmt);
+ if( rc==SQLITE_DONE ) rc = rc2;
+
+ return rc;
+}
+
+/*
+** Set up a cursor object for iterating through a full-text index or a
+** single level therein.
+*/
+int sqlite3Fts3SegReaderCursor(
+ Fts3Table *p, /* FTS3 table handle */
+ int iIndex, /* Index to search (from 0 to p->nIndex-1) */
+ int iLevel, /* Level of segments to scan */
+ const char *zTerm, /* Term to query for */
+ int nTerm, /* Size of zTerm in bytes */
+ int isPrefix, /* True for a prefix search */
+ int isScan, /* True to scan from zTerm to EOF */
+ Fts3MultiSegReader *pCsr /* Cursor object to populate */
+){
+ assert( iIndex>=0 && iIndex<p->nIndex );
+ assert( iLevel==FTS3_SEGCURSOR_ALL
+ || iLevel==FTS3_SEGCURSOR_PENDING
+ || iLevel>=0
+ );
+ assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
+ assert( FTS3_SEGCURSOR_ALL<0 && FTS3_SEGCURSOR_PENDING<0 );
+ assert( isPrefix==0 || isScan==0 );
+
+ /* "isScan" is only set to true by the ft4aux module, an ordinary
+ ** full-text tables. */
+ assert( isScan==0 || p->aIndex==0 );
+
+ memset(pCsr, 0, sizeof(Fts3MultiSegReader));
+
+ return fts3SegReaderCursor(
+ p, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr
+ );
+}
+
+/*
+** In addition to its current configuration, have the Fts3MultiSegReader
+** passed as the 4th argument also scan the doclist for term zTerm/nTerm.
+**
+** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
+*/
+static int fts3SegReaderCursorAddZero(
+ Fts3Table *p, /* FTS virtual table handle */
+ const char *zTerm, /* Term to scan doclist of */
+ int nTerm, /* Number of bytes in zTerm */
+ Fts3MultiSegReader *pCsr /* Fts3MultiSegReader to modify */
+){
+ return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr);
+}
+
+/*
+** Open an Fts3MultiSegReader to scan the doclist for term zTerm/nTerm. Or,
+** if isPrefix is true, to scan the doclist for all terms for which
+** zTerm/nTerm is a prefix. If successful, return SQLITE_OK and write
+** a pointer to the new Fts3MultiSegReader to *ppSegcsr. Otherwise, return
+** an SQLite error code.
+**
+** It is the responsibility of the caller to free this object by eventually
+** passing it to fts3SegReaderCursorFree()
+**
+** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
+** Output parameter *ppSegcsr is set to 0 if an error occurs.
+*/
+static int fts3TermSegReaderCursor(
+ Fts3Cursor *pCsr, /* Virtual table cursor handle */
+ const char *zTerm, /* Term to query for */
+ int nTerm, /* Size of zTerm in bytes */
+ int isPrefix, /* True for a prefix search */
+ Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */
+){
+ Fts3MultiSegReader *pSegcsr; /* Object to allocate and return */
+ int rc = SQLITE_NOMEM; /* Return code */
+
+ pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader));
+ if( pSegcsr ){
+ int i;
+ int bFound = 0; /* True once an index has been found */
+ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
+
+ if( isPrefix ){
+ for(i=1; bFound==0 && i<p->nIndex; i++){
+ if( p->aIndex[i].nPrefix==nTerm ){
+ bFound = 1;
+ rc = sqlite3Fts3SegReaderCursor(
+ p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr);
+ pSegcsr->bLookup = 1;
+ }
+ }
+
+ for(i=1; bFound==0 && i<p->nIndex; i++){
+ if( p->aIndex[i].nPrefix==nTerm+1 ){
+ bFound = 1;
+ rc = sqlite3Fts3SegReaderCursor(
+ p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr
+ );
+ if( rc==SQLITE_OK ){
+ rc = fts3SegReaderCursorAddZero(p, zTerm, nTerm, pSegcsr);
+ }
+ }
+ }
+ }
+
+ if( bFound==0 ){
+ rc = sqlite3Fts3SegReaderCursor(
+ p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr
+ );
+ pSegcsr->bLookup = !isPrefix;
+ }
+ }
+
+ *ppSegcsr = pSegcsr;
+ return rc;
+}
+
+/*
+** Free an Fts3MultiSegReader allocated by fts3TermSegReaderCursor().
+*/
+static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){
+ sqlite3Fts3SegReaderFinish(pSegcsr);
+ sqlite3_free(pSegcsr);
+}
+
+/*
+** This function retreives the doclist for the specified term (or term
+** prefix) from the database.
+*/
+static int fts3TermSelect(
+ Fts3Table *p, /* Virtual table handle */
+ Fts3PhraseToken *pTok, /* Token to query for */
+ int iColumn, /* Column to query (or -ve for all columns) */
+ int *pnOut, /* OUT: Size of buffer at *ppOut */
+ char **ppOut /* OUT: Malloced result buffer */
+){
+ int rc; /* Return code */
+ Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */
+ TermSelect tsc; /* Object for pair-wise doclist merging */
+ Fts3SegFilter filter; /* Segment term filter configuration */
+
+ pSegcsr = pTok->pSegcsr;
+ memset(&tsc, 0, sizeof(TermSelect));
+
+ filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | FTS3_SEGMENT_REQUIRE_POS
+ | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0)
+ | (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0);
+ filter.iCol = iColumn;
+ filter.zTerm = pTok->z;
+ filter.nTerm = pTok->n;
+
+ rc = sqlite3Fts3SegReaderStart(p, pSegcsr, &filter);
+ while( SQLITE_OK==rc
+ && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr))
+ ){
+ rc = fts3TermSelectMerge(p, &tsc, pSegcsr->aDoclist, pSegcsr->nDoclist);
+ }
+
+ if( rc==SQLITE_OK ){
+ rc = fts3TermSelectFinishMerge(p, &tsc);
+ }
+ if( rc==SQLITE_OK ){
+ *ppOut = tsc.aaOutput[0];
+ *pnOut = tsc.anOutput[0];
+ }else{
+ int i;
+ for(i=0; i<SizeofArray(tsc.aaOutput); i++){
+ sqlite3_free(tsc.aaOutput[i]);
+ }
+ }
+
+ fts3SegReaderCursorFree(pSegcsr);
+ pTok->pSegcsr = 0;
+ return rc;
+}
+
+/*
+** This function counts the total number of docids in the doclist stored
+** in buffer aList[], size nList bytes.
+**
+** If the isPoslist argument is true, then it is assumed that the doclist
+** contains a position-list following each docid. Otherwise, it is assumed
+** that the doclist is simply a list of docids stored as delta encoded
+** varints.
+*/
+static int fts3DoclistCountDocids(char *aList, int nList){
+ int nDoc = 0; /* Return value */
+ if( aList ){
+ char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */
+ char *p = aList; /* Cursor */
+ while( p<aEnd ){
+ nDoc++;
+ while( (*p++)&0x80 ); /* Skip docid varint */
+ fts3PoslistCopy(0, &p); /* Skip over position list */
+ }
+ }
+
+ return nDoc;
+}
+
+/*
+** Advance the cursor to the next row in the %_content table that
+** matches the search criteria. For a MATCH search, this will be
+** the next row that matches. For a full-table scan, this will be
+** simply the next row in the %_content table. For a docid lookup,
+** this routine simply sets the EOF flag.
+**
+** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned
+** even if we reach end-of-file. The fts3EofMethod() will be called
+** subsequently to determine whether or not an EOF was hit.
+*/
+static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
+ int rc;
+ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
+ if( pCsr->eSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){
+ if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
+ pCsr->isEof = 1;
+ rc = sqlite3_reset(pCsr->pStmt);
+ }else{
+ pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0);
+ rc = SQLITE_OK;
+ }
+ }else{
+ rc = fts3EvalNext((Fts3Cursor *)pCursor);
+ }
+ assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
+ return rc;
+}
+
+/*
+** This is the xFilter interface for the virtual table. See
+** the virtual table xFilter method documentation for additional
+** information.
+**
+** If idxNum==FTS3_FULLSCAN_SEARCH then do a full table scan against
+** the %_content table.
+**
+** If idxNum==FTS3_DOCID_SEARCH then do a docid lookup for a single entry
+** in the %_content table.
+**
+** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index. The
+** column on the left-hand side of the MATCH operator is column
+** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand
+** side of the MATCH operator.
+*/
+static int fts3FilterMethod(
+ sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
+ int idxNum, /* Strategy index */
+ const char *idxStr, /* Unused */
+ int nVal, /* Number of elements in apVal */
+ sqlite3_value **apVal /* Arguments for the indexing scheme */
+){
+ int rc;
+ char *zSql; /* SQL statement used to access %_content */
+ Fts3Table *p = (Fts3Table *)pCursor->pVtab;
+ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
+
+ UNUSED_PARAMETER(idxStr);
+ UNUSED_PARAMETER(nVal);
+
+ assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) );
+ assert( nVal==0 || nVal==1 );
+ assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) );
+ assert( p->pSegments==0 );
+
+ /* In case the cursor has been used before, clear it now. */
+ sqlite3_finalize(pCsr->pStmt);
+ sqlite3_free(pCsr->aDoclist);
+ sqlite3Fts3ExprFree(pCsr->pExpr);
+ memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
+
+ if( idxStr ){
+ pCsr->bDesc = (idxStr[0]=='D');
+ }else{
+ pCsr->bDesc = p->bDescIdx;
+ }
+ pCsr->eSearch = (i16)idxNum;
+
+ if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){
+ int iCol = idxNum-FTS3_FULLTEXT_SEARCH;
+ const char *zQuery = (const char *)sqlite3_value_text(apVal[0]);
+
+ if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
+ return SQLITE_NOMEM;
+ }
+
+ rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn,
+ iCol, zQuery, -1, &pCsr->pExpr
+ );
+ if( rc!=SQLITE_OK ){
+ if( rc==SQLITE_ERROR ){
+ static const char *zErr = "malformed MATCH expression: [%s]";
+ p->base.zErrMsg = sqlite3_mprintf(zErr, zQuery);
+ }
+ return rc;
+ }
+
+ rc = sqlite3Fts3ReadLock(p);
+ if( rc!=SQLITE_OK ) return rc;
+
+ rc = fts3EvalStart(pCsr);
+
+ sqlite3Fts3SegmentsClose(p);
+ if( rc!=SQLITE_OK ) return rc;
+ pCsr->pNextId = pCsr->aDoclist;
+ pCsr->iPrevId = 0;
+ }
+
+ /* Compile a SELECT statement for this cursor. For a full-table-scan, the
+ ** statement loops through all rows of the %_content table. For a
+ ** full-text query or docid lookup, the statement retrieves a single
+ ** row by docid.
+ */
+ if( idxNum==FTS3_FULLSCAN_SEARCH ){
+ const char *zSort = (pCsr->bDesc ? "DESC" : "ASC");
+ const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s";
+ zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName, zSort);
+ }else{
+ const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?";
+ zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName);
+ }
+ if( !zSql ) return SQLITE_NOMEM;
+ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
+ sqlite3_free(zSql);
+ if( rc!=SQLITE_OK ) return rc;
+
+ if( idxNum==FTS3_DOCID_SEARCH ){
+ rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
+ if( rc!=SQLITE_OK ) return rc;
+ }
+
+ return fts3NextMethod(pCursor);
+}
+
+/*
+** This is the xEof method of the virtual table. SQLite calls this
+** routine to find out if it has reached the end of a result set.
+*/
+static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){
+ return ((Fts3Cursor *)pCursor)->isEof;
+}
+
+/*
+** This is the xRowid method. The SQLite core calls this routine to
+** retrieve the rowid for the current row of the result set. fts3
+** exposes %_content.docid as the rowid for the virtual table. The
+** rowid should be written to *pRowid.
+*/
+static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
+ Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
+ *pRowid = pCsr->iPrevId;
+ return SQLITE_OK;
+}
+
+/*
+** This is the xColumn method, called by SQLite to request a value from
+** the row that the supplied cursor currently points to.
+*/
+static int fts3ColumnMethod(
+ sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
+ sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */
+ int iCol /* Index of column to read value from */
+){
+ int rc = SQLITE_OK; /* Return Code */
+ Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
+ Fts3Table *p = (Fts3Table *)pCursor->pVtab;
+
+ /* The column value supplied by SQLite must be in range. */
+ assert( iCol>=0 && iCol<=p->nColumn+1 );
+
+ if( iCol==p->nColumn+1 ){
+ /* This call is a request for the "docid" column. Since "docid" is an
+ ** alias for "rowid", use the xRowid() method to obtain the value.
+ */
+ sqlite3_result_int64(pContext, pCsr->iPrevId);
+ }else if( iCol==p->nColumn ){
+ /* The extra column whose name is the same as the table.
+ ** Return a blob which is a pointer to the cursor.
+ */
+ sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
+ }else{
+ rc = fts3CursorSeek(0, pCsr);
+ if( rc==SQLITE_OK ){
+ sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1));
+ }
+ }
+
+ assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
+ return rc;
+}
+
+/*
+** This function is the implementation of the xUpdate callback used by
+** FTS3 virtual tables. It is invoked by SQLite each time a row is to be
+** inserted, updated or deleted.
+*/
+static int fts3UpdateMethod(
+ sqlite3_vtab *pVtab, /* Virtual table handle */
+ int nArg, /* Size of argument array */
+ sqlite3_value **apVal, /* Array of arguments */
+ sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */
+){
+ return sqlite3Fts3UpdateMethod(pVtab, nArg, apVal, pRowid);
+}
+
+/*
+** Implementation of xSync() method. Flush the contents of the pending-terms
+** hash-table to the database.
+*/
+static int fts3SyncMethod(sqlite3_vtab *pVtab){
+ int rc = sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab);
+ sqlite3Fts3SegmentsClose((Fts3Table *)pVtab);
+ return rc;
+}
+
+/*
+** Implementation of xBegin() method. This is a no-op.
+*/
+static int fts3BeginMethod(sqlite3_vtab *pVtab){
+ TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
+ UNUSED_PARAMETER(pVtab);
+ assert( p->pSegments==0 );
+ assert( p->nPendingData==0 );
+ assert( p->inTransaction!=1 );
+ TESTONLY( p->inTransaction = 1 );
+ TESTONLY( p->mxSavepoint = -1; );
+ return SQLITE_OK;
+}
+
+/*
+** Implementation of xCommit() method. This is a no-op. The contents of
+** the pending-terms hash-table have already been flushed into the database
+** by fts3SyncMethod().
+*/
+static int fts3CommitMethod(sqlite3_vtab *pVtab){
+ TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
+ UNUSED_PARAMETER(pVtab);
+ assert( p->nPendingData==0 );
+ assert( p->inTransaction!=0 );
+ assert( p->pSegments==0 );
+ TESTONLY( p->inTransaction = 0 );
+ TESTONLY( p->mxSavepoint = -1; );
+ return SQLITE_OK;
+}
+
+/*
+** Implementation of xRollback(). Discard the contents of the pending-terms
+** hash-table. Any changes made to the database are reverted by SQLite.
+*/
+static int fts3RollbackMethod(sqlite3_vtab *pVtab){
+ Fts3Table *p = (Fts3Table*)pVtab;
+ sqlite3Fts3PendingTermsClear(p);
+ assert( p->inTransaction!=0 );
+ TESTONLY( p->inTransaction = 0 );
+ TESTONLY( p->mxSavepoint = -1; );
+ return SQLITE_OK;
+}
+
+/*
+** When called, *ppPoslist must point to the byte immediately following the
+** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function
+** moves *ppPoslist so that it instead points to the first byte of the
+** same position list.
+*/
+static void fts3ReversePoslist(char *pStart, char **ppPoslist){
+ char *p = &(*ppPoslist)[-2];
+ char c;
+
+ while( p>pStart && (c=*p--)==0 );
+ while( p>pStart && (*p & 0x80) | c ){
+ c = *p--;
+ }
+ if( p>pStart ){ p = &p[2]; }
+ while( *p++&0x80 );
+ *ppPoslist = p;
+}
+
+/*
+** Helper function used by the implementation of the overloaded snippet(),
+** offsets() and optimize() SQL functions.
+**
+** If the value passed as the third argument is a blob of size
+** sizeof(Fts3Cursor*), then the blob contents are copied to the
+** output variable *ppCsr and SQLITE_OK is returned. Otherwise, an error
+** message is written to context pContext and SQLITE_ERROR returned. The
+** string passed via zFunc is used as part of the error message.
+*/
+static int fts3FunctionArg(
+ sqlite3_context *pContext, /* SQL function call context */
+ const char *zFunc, /* Function name */
+ sqlite3_value *pVal, /* argv[0] passed to function */
+ Fts3Cursor **ppCsr /* OUT: Store cursor handle here */
+){
+ Fts3Cursor *pRet;
+ if( sqlite3_value_type(pVal)!=SQLITE_BLOB
+ || sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *)
+ ){
+ char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc);
+ sqlite3_result_error(pContext, zErr, -1);
+ sqlite3_free(zErr);
+ return SQLITE_ERROR;
+ }
+ memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *));
+ *ppCsr = pRet;
+ return SQLITE_OK;
+}
+
+/*
+** Implementation of the snippet() function for FTS3
+*/
+static void fts3SnippetFunc(
+ sqlite3_context *pContext, /* SQLite function call context */
+ int nVal, /* Size of apVal[] array */
+ sqlite3_value **apVal /* Array of arguments */
+){
+ Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */
+ const char *zStart = "<b>";
+ const char *zEnd = "</b>";
+ const char *zEllipsis = "<b>...</b>";
+ int iCol = -1;
+ int nToken = 15; /* Default number of tokens in snippet */
+
+ /* There must be at least one argument passed to this function (otherwise
+ ** the non-overloaded version would have been called instead of this one).
+ */
+ assert( nVal>=1 );
+
+ if( nVal>6 ){
+ sqlite3_result_error(pContext,
+ "wrong number of arguments to function snippet()", -1);
+ return;
+ }
+ if( fts3FunctionArg(pContext, "snippet", apVal[0], &pCsr) ) return;
+
+ switch( nVal ){
+ case 6: nToken = sqlite3_value_int(apVal[5]);
+ case 5: iCol = sqlite3_value_int(apVal[4]);
+ case 4: zEllipsis = (const char*)sqlite3_value_text(apVal[3]);
+ case 3: zEnd = (const char*)sqlite3_value_text(apVal[2]);
+ case 2: zStart = (const char*)sqlite3_value_text(apVal[1]);
+ }
+ if( !zEllipsis || !zEnd || !zStart ){
+ sqlite3_result_error_nomem(pContext);
+ }else if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){
+ sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis, iCol, nToken);
+ }
+}
+
+/*
+** Implementation of the offsets() function for FTS3
+*/
+static void fts3OffsetsFunc(
+ sqlite3_context *pContext, /* SQLite function call context */
+ int nVal, /* Size of argument array */
+ sqlite3_value **apVal /* Array of arguments */
+){
+ Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */
+
+ UNUSED_PARAMETER(nVal);
+
+ assert( nVal==1 );
+ if( fts3FunctionArg(pContext, "offsets", apVal[0], &pCsr) ) return;
+ assert( pCsr );
+ if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){
+ sqlite3Fts3Offsets(pContext, pCsr);
+ }
+}
+
+/*
+** Implementation of the special optimize() function for FTS3. This
+** function merges all segments in the database to a single segment.
+** Example usage is:
+**
+** SELECT optimize(t) FROM t LIMIT 1;
+**
+** where 't' is the name of an FTS3 table.
+*/
+static void fts3OptimizeFunc(
+ sqlite3_context *pContext, /* SQLite function call context */
+ int nVal, /* Size of argument array */
+ sqlite3_value **apVal /* Array of arguments */
+){
+ int rc; /* Return code */
+ Fts3Table *p; /* Virtual table handle */
+ Fts3Cursor *pCursor; /* Cursor handle passed through apVal[0] */
+
+ UNUSED_PARAMETER(nVal);
+
+ assert( nVal==1 );
+ if( fts3FunctionArg(pContext, "optimize", apVal[0], &pCursor) ) return;
+ p = (Fts3Table *)pCursor->base.pVtab;
+ assert( p );
+
+ rc = sqlite3Fts3Optimize(p);
+
+ switch( rc ){
+ case SQLITE_OK:
+ sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC);
+ break;
+ case SQLITE_DONE:
+ sqlite3_result_text(pContext, "Index already optimal", -1, SQLITE_STATIC);
+ break;
+ default:
+ sqlite3_result_error_code(pContext, rc);
+ break;
+ }
+}
+
+/*
+** Implementation of the matchinfo() function for FTS3
+*/
+static void fts3MatchinfoFunc(
+ sqlite3_context *pContext, /* SQLite function call context */
+ int nVal, /* Size of argument array */
+ sqlite3_value **apVal /* Array of arguments */
+){
+ Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */
+ assert( nVal==1 || nVal==2 );
+ if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){
+ const char *zArg = 0;
+ if( nVal>1 ){
+ zArg = (const char *)sqlite3_value_text(apVal[1]);
+ }
+ sqlite3Fts3Matchinfo(pContext, pCsr, zArg);
+ }
+}
+
+/*
+** This routine implements the xFindFunction method for the FTS3
+** virtual table.
+*/
+static int fts3FindFunctionMethod(
+ sqlite3_vtab *pVtab, /* Virtual table handle */
+ int nArg, /* Number of SQL function arguments */
+ const char *zName, /* Name of SQL function */
+ void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */
+ void **ppArg /* Unused */
+){
+ struct Overloaded {
+ const char *zName;
+ void (*xFunc)(sqlite3_context*,int,sqlite3_value**);
+ } aOverload[] = {
+ { "snippet", fts3SnippetFunc },
+ { "offsets", fts3OffsetsFunc },
+ { "optimize", fts3OptimizeFunc },
+ { "matchinfo", fts3MatchinfoFunc },
+ };
+ int i; /* Iterator variable */
+
+ UNUSED_PARAMETER(pVtab);
+ UNUSED_PARAMETER(nArg);
+ UNUSED_PARAMETER(ppArg);
+
+ for(i=0; i<SizeofArray(aOverload); i++){
+ if( strcmp(zName, aOverload[i].zName)==0 ){
+ *pxFunc = aOverload[i].xFunc;
+ return 1;
+ }
+ }
+
+ /* No function of the specified name was found. Return 0. */
+ return 0;
+}
+
+/*
+** Implementation of FTS3 xRename method. Rename an fts3 table.
+*/
+static int fts3RenameMethod(
+ sqlite3_vtab *pVtab, /* Virtual table handle */
+ const char *zName /* New name of table */
+){
+ Fts3Table *p = (Fts3Table *)pVtab;
+ sqlite3 *db = p->db; /* Database connection */
+ int rc; /* Return Code */
+
+ rc = sqlite3Fts3PendingTermsFlush(p);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+
+ fts3DbExec(&rc, db,
+ "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';",
+ p->zDb, p->zName, zName
+ );
+ if( p->bHasDocsize ){
+ fts3DbExec(&rc, db,
+ "ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';",
+ p->zDb, p->zName, zName
+ );
+ }
+ if( p->bHasStat ){
+ fts3DbExec(&rc, db,
+ "ALTER TABLE %Q.'%q_stat' RENAME TO '%q_stat';",
+ p->zDb, p->zName, zName
+ );
+ }
+ fts3DbExec(&rc, db,
+ "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';",
+ p->zDb, p->zName, zName
+ );
+ fts3DbExec(&rc, db,
+ "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';",
+ p->zDb, p->zName, zName
+ );
+ return rc;
+}
+
+/*
+** The xSavepoint() method.
+**
+** Flush the contents of the pending-terms table to disk.
+*/
+static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
+ UNUSED_PARAMETER(iSavepoint);
+ assert( ((Fts3Table *)pVtab)->inTransaction );
+ assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint );
+ TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint );
+ return fts3SyncMethod(pVtab);
+}
+
+/*
+** The xRelease() method.
+**
+** This is a no-op.
+*/
+static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
+ TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
+ UNUSED_PARAMETER(iSavepoint);
+ UNUSED_PARAMETER(pVtab);
+ assert( p->inTransaction );
+ assert( p->mxSavepoint >= iSavepoint );
+ TESTONLY( p->mxSavepoint = iSavepoint-1 );
+ return SQLITE_OK;
+}
+
+/*
+** The xRollbackTo() method.
+**
+** Discard the contents of the pending terms table.
+*/
+static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
+ Fts3Table *p = (Fts3Table*)pVtab;
+ UNUSED_PARAMETER(iSavepoint);
+ assert( p->inTransaction );
+ assert( p->mxSavepoint >= iSavepoint );
+ TESTONLY( p->mxSavepoint = iSavepoint );
+ sqlite3Fts3PendingTermsClear(p);
+ return SQLITE_OK;
+}
+
+static const sqlite3_module fts3Module = {
+ /* iVersion */ 2,
+ /* xCreate */ fts3CreateMethod,
+ /* xConnect */ fts3ConnectMethod,
+ /* xBestIndex */ fts3BestIndexMethod,
+ /* xDisconnect */ fts3DisconnectMethod,
+ /* xDestroy */ fts3DestroyMethod,
+ /* xOpen */ fts3OpenMethod,
+ /* xClose */ fts3CloseMethod,
+ /* xFilter */ fts3FilterMethod,
+ /* xNext */ fts3NextMethod,
+ /* xEof */ fts3EofMethod,
+ /* xColumn */ fts3ColumnMethod,
+ /* xRowid */ fts3RowidMethod,
+ /* xUpdate */ fts3UpdateMethod,
+ /* xBegin */ fts3BeginMethod,
+ /* xSync */ fts3SyncMethod,
+ /* xCommit */ fts3CommitMethod,
+ /* xRollback */ fts3RollbackMethod,
+ /* xFindFunction */ fts3FindFunctionMethod,
+ /* xRename */ fts3RenameMethod,
+ /* xSavepoint */ fts3SavepointMethod,
+ /* xRelease */ fts3ReleaseMethod,
+ /* xRollbackTo */ fts3RollbackToMethod,
+};
+
+/*
+** This function is registered as the module destructor (called when an
+** FTS3 enabled database connection is closed). It frees the memory
+** allocated for the tokenizer hash table.
+*/
+static void hashDestroy(void *p){
+ Fts3Hash *pHash = (Fts3Hash *)p;
+ sqlite3Fts3HashClear(pHash);
+ sqlite3_free(pHash);
+}
+
+/*
+** The fts3 built-in tokenizers - "simple", "porter" and "icu"- are
+** implemented in files fts3_tokenizer1.c, fts3_porter.c and fts3_icu.c
+** respectively. The following three forward declarations are for functions
+** declared in these files used to retrieve the respective implementations.
+**
+** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed
+** to by the argument to point to the "simple" tokenizer implementation.
+** And so on.
+*/
+void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+#ifdef SQLITE_ENABLE_ICU
+void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+#endif
+
+/*
+** Initialise the fts3 extension. If this extension is built as part
+** of the sqlite library, then this function is called directly by
+** SQLite. If fts3 is built as a dynamically loadable extension, this
+** function is called by the sqlite3_extension_init() entry point.
+*/
+int sqlite3Fts3Init(sqlite3 *db){
+ int rc = SQLITE_OK;
+ Fts3Hash *pHash = 0;
+ const sqlite3_tokenizer_module *pSimple = 0;
+ const sqlite3_tokenizer_module *pPorter = 0;
+
+#ifdef SQLITE_ENABLE_ICU
+ const sqlite3_tokenizer_module *pIcu = 0;
+ sqlite3Fts3IcuTokenizerModule(&pIcu);
+#endif
+
+#ifdef SQLITE_TEST
+ rc = sqlite3Fts3InitTerm(db);
+ if( rc!=SQLITE_OK ) return rc;
+#endif
+
+ rc = sqlite3Fts3InitAux(db);
+ if( rc!=SQLITE_OK ) return rc;
+
+ sqlite3Fts3SimpleTokenizerModule(&pSimple);
+ sqlite3Fts3PorterTokenizerModule(&pPorter);
+
+ /* Allocate and initialise the hash-table used to store tokenizers. */
+ pHash = sqlite3_malloc(sizeof(Fts3Hash));
+ if( !pHash ){
+ rc = SQLITE_NOMEM;
+ }else{
+ sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
+ }
+
+ /* Load the built-in tokenizers into the hash table */
+ if( rc==SQLITE_OK ){
+ if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple)
+ || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter)
+#ifdef SQLITE_ENABLE_ICU
+ || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu))
+#endif
+ ){
+ rc = SQLITE_NOMEM;
+ }
+ }
+
+#ifdef SQLITE_TEST
+ if( rc==SQLITE_OK ){
+ rc = sqlite3Fts3ExprInitTestInterface(db);
+ }
+#endif
+
+ /* Create the virtual table wrapper around the hash-table and overload
+ ** the two scalar functions. If this is successful, register the
+ ** module with sqlite.
+ */
+ if( SQLITE_OK==rc
+ && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer"))
+ && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
+ && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1))
+ && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1))
+ && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 2))
+ && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1))
+ ){
+ rc = sqlite3_create_module_v2(
+ db, "fts3", &fts3Module, (void *)pHash, hashDestroy
+ );
+ if( rc==SQLITE_OK ){
+ rc = sqlite3_create_module_v2(
+ db, "fts4", &fts3Module, (void *)pHash, 0
+ );
+ }
+ return rc;
+ }
+
+ /* An error has occurred. Delete the hash table and return the error code. */
+ assert( rc!=SQLITE_OK );
+ if( pHash ){
+ sqlite3Fts3HashClear(pHash);
+ sqlite3_free(pHash);
+ }
+ return rc;
+}
+
+/*
+** Allocate an Fts3MultiSegReader for each token in the expression headed
+** by pExpr.
+**
+** An Fts3SegReader object is a cursor that can seek or scan a range of
+** entries within a single segment b-tree. An Fts3MultiSegReader uses multiple
+** Fts3SegReader objects internally to provide an interface to seek or scan
+** within the union of all segments of a b-tree. Hence the name.
+**
+** If the allocated Fts3MultiSegReader just seeks to a single entry in a
+** segment b-tree (if the term is not a prefix or it is a prefix for which
+** there exists prefix b-tree of the right length) then it may be traversed
+** and merged incrementally. Otherwise, it has to be merged into an in-memory
+** doclist and then traversed.
+*/
+static void fts3EvalAllocateReaders(
+ Fts3Cursor *pCsr, /* FTS cursor handle */
+ Fts3Expr *pExpr, /* Allocate readers for this expression */
+ int *pnToken, /* OUT: Total number of tokens in phrase. */
+ int *pnOr, /* OUT: Total number of OR nodes in expr. */
+ int *pRc /* IN/OUT: Error code */
+){
+ if( pExpr && SQLITE_OK==*pRc ){
+ if( pExpr->eType==FTSQUERY_PHRASE ){
+ int i;
+ int nToken = pExpr->pPhrase->nToken;
+ *pnToken += nToken;
+ for(i=0; i<nToken; i++){
+ Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i];
+ int rc = fts3TermSegReaderCursor(pCsr,
+ pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr
+ );
+ if( rc!=SQLITE_OK ){
+ *pRc = rc;
+ return;
+ }
+ }
+ assert( pExpr->pPhrase->iDoclistToken==0 );
+ pExpr->pPhrase->iDoclistToken = -1;
+ }else{
+ *pnOr += (pExpr->eType==FTSQUERY_OR);
+ fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc);
+ fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc);
+ }
+ }
+}
+
+/*
+** Arguments pList/nList contain the doclist for token iToken of phrase p.
+** It is merged into the main doclist stored in p->doclist.aAll/nAll.
+**
+** This function assumes that pList points to a buffer allocated using
+** sqlite3_malloc(). This function takes responsibility for eventually
+** freeing the buffer.
+*/
+static void fts3EvalPhraseMergeToken(
+ Fts3Table *pTab, /* FTS Table pointer */
+ Fts3Phrase *p, /* Phrase to merge pList/nList into */
+ int iToken, /* Token pList/nList corresponds to */
+ char *pList, /* Pointer to doclist */
+ int nList /* Number of bytes in pList */
+){
+ assert( iToken!=p->iDoclistToken );
+
+ if( pList==0 ){
+ sqlite3_free(p->doclist.aAll);
+ p->doclist.aAll = 0;
+ p->doclist.nAll = 0;
+ }
+
+ else if( p->iDoclistToken<0 ){
+ p->doclist.aAll = pList;
+ p->doclist.nAll = nList;
+ }
+
+ else if( p->doclist.aAll==0 ){
+ sqlite3_free(pList);
+ }
+
+ else {
+ char *pLeft;
+ char *pRight;
+ int nLeft;
+ int nRight;
+ int nDiff;
+
+ if( p->iDoclistToken<iToken ){
+ pLeft = p->doclist.aAll;
+ nLeft = p->doclist.nAll;
+ pRight = pList;
+ nRight = nList;
+ nDiff = iToken - p->iDoclistToken;
+ }else{
+ pRight = p->doclist.aAll;
+ nRight = p->doclist.nAll;
+ pLeft = pList;
+ nLeft = nList;
+ nDiff = p->iDoclistToken - iToken;
+ }
+
+ fts3DoclistPhraseMerge(pTab->bDescIdx, nDiff, pLeft, nLeft, pRight,&nRight);
+ sqlite3_free(pLeft);
+ p->doclist.aAll = pRight;
+ p->doclist.nAll = nRight;
+ }
+
+ if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken;
+}
+
+/*
+** Load the doclist for phrase p into p->doclist.aAll/nAll. The loaded doclist
+** does not take deferred tokens into account.
+**
+** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
+*/
+static int fts3EvalPhraseLoad(
+ Fts3Cursor *pCsr, /* FTS Cursor handle */
+ Fts3Phrase *p /* Phrase object */
+){
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+ int iToken;
+ int rc = SQLITE_OK;
+
+ for(iToken=0; rc==SQLITE_OK && iToken<p->nToken; iToken++){
+ Fts3PhraseToken *pToken = &p->aToken[iToken];
+ assert( pToken->pDeferred==0 || pToken->pSegcsr==0 );
+
+ if( pToken->pSegcsr ){
+ int nThis = 0;
+ char *pThis = 0;
+ rc = fts3TermSelect(pTab, pToken, p->iColumn, &nThis, &pThis);
+ if( rc==SQLITE_OK ){
+ fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis);
+ }
+ }
+ assert( pToken->pSegcsr==0 );
+ }
+
+ return rc;
+}
+
+/*
+** This function is called on each phrase after the position lists for
+** any deferred tokens have been loaded into memory. It updates the phrases
+** current position list to include only those positions that are really
+** instances of the phrase (after considering deferred tokens). If this
+** means that the phrase does not appear in the current row, doclist.pList
+** and doclist.nList are both zeroed.
+**
+** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
+*/
+static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
+ int iToken; /* Used to iterate through phrase tokens */
+ int rc = SQLITE_OK; /* Return code */
+ char *aPoslist = 0; /* Position list for deferred tokens */
+ int nPoslist = 0; /* Number of bytes in aPoslist */
+ int iPrev = -1; /* Token number of previous deferred token */
+
+ assert( pPhrase->doclist.bFreeList==0 );
+
+ for(iToken=0; rc==SQLITE_OK && iToken<pPhrase->nToken; iToken++){
+ Fts3PhraseToken *pToken = &pPhrase->aToken[iToken];
+ Fts3DeferredToken *pDeferred = pToken->pDeferred;
+
+ if( pDeferred ){
+ char *pList;
+ int nList;
+ rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList);
+ if( rc!=SQLITE_OK ) return rc;
+
+ if( pList==0 ){
+ sqlite3_free(aPoslist);
+ pPhrase->doclist.pList = 0;
+ pPhrase->doclist.nList = 0;
+ return SQLITE_OK;
+
+ }else if( aPoslist==0 ){
+ aPoslist = pList;
+ nPoslist = nList;
+
+ }else{
+ char *aOut = pList;
+ char *p1 = aPoslist;
+ char *p2 = aOut;
+
+ assert( iPrev>=0 );
+ fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2);
+ sqlite3_free(aPoslist);
+ aPoslist = pList;
+ nPoslist = aOut - aPoslist;
+ if( nPoslist==0 ){
+ sqlite3_free(aPoslist);
+ pPhrase->doclist.pList = 0;
+ pPhrase->doclist.nList = 0;
+ return SQLITE_OK;
+ }
+ }
+ iPrev = iToken;
+ }
+ }
+
+ if( iPrev>=0 ){
+ int nMaxUndeferred = pPhrase->iDoclistToken;
+ if( nMaxUndeferred<0 ){
+ pPhrase->doclist.pList = aPoslist;
+ pPhrase->doclist.nList = nPoslist;
+ pPhrase->doclist.iDocid = pCsr->iPrevId;
+ pPhrase->doclist.bFreeList = 1;
+ }else{
+ int nDistance;
+ char *p1;
+ char *p2;
+ char *aOut;
+
+ if( nMaxUndeferred>iPrev ){
+ p1 = aPoslist;
+ p2 = pPhrase->doclist.pList;
+ nDistance = nMaxUndeferred - iPrev;
+ }else{
+ p1 = pPhrase->doclist.pList;
+ p2 = aPoslist;
+ nDistance = iPrev - nMaxUndeferred;
+ }
+
+ aOut = (char *)sqlite3_malloc(nPoslist+8);
+ if( !aOut ){
+ sqlite3_free(aPoslist);
+ return SQLITE_NOMEM;
+ }
+
+ pPhrase->doclist.pList = aOut;
+ if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){
+ pPhrase->doclist.bFreeList = 1;
+ pPhrase->doclist.nList = (aOut - pPhrase->doclist.pList);
+ }else{
+ sqlite3_free(aOut);
+ pPhrase->doclist.pList = 0;
+ pPhrase->doclist.nList = 0;
+ }
+ sqlite3_free(aPoslist);
+ }
+ }
+
+ return SQLITE_OK;
+}
+
+/*
+** This function is called for each Fts3Phrase in a full-text query
+** expression to initialize the mechanism for returning rows. Once this
+** function has been called successfully on an Fts3Phrase, it may be
+** used with fts3EvalPhraseNext() to iterate through the matching docids.
+**
+** If parameter bOptOk is true, then the phrase may (or may not) use the
+** incremental loading strategy. Otherwise, the entire doclist is loaded into
+** memory within this call.
+**
+** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
+*/
+static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){
+ int rc; /* Error code */
+ Fts3PhraseToken *pFirst = &p->aToken[0];
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+
+ if( pCsr->bDesc==pTab->bDescIdx
+ && bOptOk==1
+ && p->nToken==1
+ && pFirst->pSegcsr
+ && pFirst->pSegcsr->bLookup
+ ){
+ /* Use the incremental approach. */
+ int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn);
+ rc = sqlite3Fts3MsrIncrStart(
+ pTab, pFirst->pSegcsr, iCol, pFirst->z, pFirst->n);
+ p->bIncr = 1;
+
+ }else{
+ /* Load the full doclist for the phrase into memory. */
+ rc = fts3EvalPhraseLoad(pCsr, p);
+ p->bIncr = 0;
+ }
+
+ assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr );
+ return rc;
+}
+
+/*
+** This function is used to iterate backwards (from the end to start)
+** through doclists. It is used by this module to iterate through phrase
+** doclists in reverse and by the fts3_write.c module to iterate through
+** pending-terms lists when writing to databases with "order=desc".
+**
+** The doclist may be sorted in ascending (parameter bDescIdx==0) or
+** descending (parameter bDescIdx==1) order of docid. Regardless, this
+** function iterates from the end of the doclist to the beginning.
+*/
+void sqlite3Fts3DoclistPrev(
+ int bDescIdx, /* True if the doclist is desc */
+ char *aDoclist, /* Pointer to entire doclist */
+ int nDoclist, /* Length of aDoclist in bytes */
+ char **ppIter, /* IN/OUT: Iterator pointer */
+ sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */
+ int *pnList, /* IN/OUT: List length pointer */
+ u8 *pbEof /* OUT: End-of-file flag */
+){
+ char *p = *ppIter;
+
+ assert( nDoclist>0 );
+ assert( *pbEof==0 );
+ assert( p || *piDocid==0 );
+ assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) );
+
+ if( p==0 ){
+ sqlite3_int64 iDocid = 0;
+ char *pNext = 0;
+ char *pDocid = aDoclist;
+ char *pEnd = &aDoclist[nDoclist];
+ int iMul = 1;
+
+ while( pDocid<pEnd ){
+ sqlite3_int64 iDelta;
+ pDocid += sqlite3Fts3GetVarint(pDocid, &iDelta);
+ iDocid += (iMul * iDelta);
+ pNext = pDocid;
+ fts3PoslistCopy(0, &pDocid);
+ while( pDocid<pEnd && *pDocid==0 ) pDocid++;
+ iMul = (bDescIdx ? -1 : 1);
+ }
+
+ *pnList = pEnd - pNext;
+ *ppIter = pNext;
+ *piDocid = iDocid;
+ }else{
+ int iMul = (bDescIdx ? -1 : 1);
+ sqlite3_int64 iDelta;
+ fts3GetReverseVarint(&p, aDoclist, &iDelta);
+ *piDocid -= (iMul * iDelta);
+
+ if( p==aDoclist ){
+ *pbEof = 1;
+ }else{
+ char *pSave = p;
+ fts3ReversePoslist(aDoclist, &p);
+ *pnList = (pSave - p);
+ }
+ *ppIter = p;
+ }
+}
+
+/*
+** Attempt to move the phrase iterator to point to the next matching docid.
+** If an error occurs, return an SQLite error code. Otherwise, return
+** SQLITE_OK.
+**
+** If there is no "next" entry and no error occurs, then *pbEof is set to
+** 1 before returning. Otherwise, if no error occurs and the iterator is
+** successfully advanced, *pbEof is set to 0.
+*/
+static int fts3EvalPhraseNext(
+ Fts3Cursor *pCsr, /* FTS Cursor handle */
+ Fts3Phrase *p, /* Phrase object to advance to next docid */
+ u8 *pbEof /* OUT: Set to 1 if EOF */
+){
+ int rc = SQLITE_OK;
+ Fts3Doclist *pDL = &p->doclist;
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+
+ if( p->bIncr ){
+ assert( p->nToken==1 );
+ assert( pDL->pNextDocid==0 );
+ rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr,
+ &pDL->iDocid, &pDL->pList, &pDL->nList
+ );
+ if( rc==SQLITE_OK && !pDL->pList ){
+ *pbEof = 1;
+ }
+ }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){
+ sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll,
+ &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof
+ );
+ pDL->pList = pDL->pNextDocid;
+ }else{
+ char *pIter; /* Used to iterate through aAll */
+ char *pEnd = &pDL->aAll[pDL->nAll]; /* 1 byte past end of aAll */
+ if( pDL->pNextDocid ){
+ pIter = pDL->pNextDocid;
+ }else{
+ pIter = pDL->aAll;
+ }
+
+ if( pIter>=pEnd ){
+ /* We have already reached the end of this doclist. EOF. */
+ *pbEof = 1;
+ }else{
+ sqlite3_int64 iDelta;
+ pIter += sqlite3Fts3GetVarint(pIter, &iDelta);
+ if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){
+ pDL->iDocid += iDelta;
+ }else{
+ pDL->iDocid -= iDelta;
+ }
+ pDL->pList = pIter;
+ fts3PoslistCopy(0, &pIter);
+ pDL->nList = (pIter - pDL->pList);
+
+ /* pIter now points just past the 0x00 that terminates the position-
+ ** list for document pDL->iDocid. However, if this position-list was
+ ** edited in place by fts3EvalNearTrim(), then pIter may not actually
+ ** point to the start of the next docid value. The following line deals
+ ** with this case by advancing pIter past the zero-padding added by
+ ** fts3EvalNearTrim(). */
+ while( pIter<pEnd && *pIter==0 ) pIter++;
+
+ pDL->pNextDocid = pIter;
+ assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter );
+ *pbEof = 0;
+ }
+ }
+
+ return rc;
+}
+
+/*
+**
+** If *pRc is not SQLITE_OK when this function is called, it is a no-op.
+** Otherwise, fts3EvalPhraseStart() is called on all phrases within the
+** expression. Also the Fts3Expr.bDeferred variable is set to true for any
+** expressions for which all descendent tokens are deferred.
+**
+** If parameter bOptOk is zero, then it is guaranteed that the
+** Fts3Phrase.doclist.aAll/nAll variables contain the entire doclist for
+** each phrase in the expression (subject to deferred token processing).
+** Or, if bOptOk is non-zero, then one or more tokens within the expression
+** may be loaded incrementally, meaning doclist.aAll/nAll is not available.
+**
+** If an error occurs within this function, *pRc is set to an SQLite error
+** code before returning.
+*/
+static void fts3EvalStartReaders(
+ Fts3Cursor *pCsr, /* FTS Cursor handle */
+ Fts3Expr *pExpr, /* Expression to initialize phrases in */
+ int bOptOk, /* True to enable incremental loading */
+ int *pRc /* IN/OUT: Error code */
+){
+ if( pExpr && SQLITE_OK==*pRc ){
+ if( pExpr->eType==FTSQUERY_PHRASE ){
+ int i;
+ int nToken = pExpr->pPhrase->nToken;
+ for(i=0; i<nToken; i++){
+ if( pExpr->pPhrase->aToken[i].pDeferred==0 ) break;
+ }
+ pExpr->bDeferred = (i==nToken);
+ *pRc = fts3EvalPhraseStart(pCsr, bOptOk, pExpr->pPhrase);
+ }else{
+ fts3EvalStartReaders(pCsr, pExpr->pLeft, bOptOk, pRc);
+ fts3EvalStartReaders(pCsr, pExpr->pRight, bOptOk, pRc);
+ pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred);
+ }
+ }
+}
+
+/*
+** An array of the following structures is assembled as part of the process
+** of selecting tokens to defer before the query starts executing (as part
+** of the xFilter() method). There is one element in the array for each
+** token in the FTS expression.
+**
+** Tokens are divided into AND/NEAR clusters. All tokens in a cluster belong
+** to phrases that are connected only by AND and NEAR operators (not OR or
+** NOT). When determining tokens to defer, each AND/NEAR cluster is considered
+** separately. The root of a tokens AND/NEAR cluster is stored in
+** Fts3TokenAndCost.pRoot.
+*/
+typedef struct Fts3TokenAndCost Fts3TokenAndCost;
+struct Fts3TokenAndCost {
+ Fts3Phrase *pPhrase; /* The phrase the token belongs to */
+ int iToken; /* Position of token in phrase */
+ Fts3PhraseToken *pToken; /* The token itself */
+ Fts3Expr *pRoot; /* Root of NEAR/AND cluster */
+ int nOvfl; /* Number of overflow pages to load doclist */
+ int iCol; /* The column the token must match */
+};
+
+/*
+** This function is used to populate an allocated Fts3TokenAndCost array.
+**
+** If *pRc is not SQLITE_OK when this function is called, it is a no-op.
+** Otherwise, if an error occurs during execution, *pRc is set to an
+** SQLite error code.
+*/
+static void fts3EvalTokenCosts(
+ Fts3Cursor *pCsr, /* FTS Cursor handle */
+ Fts3Expr *pRoot, /* Root of current AND/NEAR cluster */
+ Fts3Expr *pExpr, /* Expression to consider */
+ Fts3TokenAndCost **ppTC, /* Write new entries to *(*ppTC)++ */
+ Fts3Expr ***ppOr, /* Write new OR root to *(*ppOr)++ */
+ int *pRc /* IN/OUT: Error code */
+){
+ if( *pRc==SQLITE_OK && pExpr ){
+ if( pExpr->eType==FTSQUERY_PHRASE ){
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
+ int i;
+ for(i=0; *pRc==SQLITE_OK && i<pPhrase->nToken; i++){
+ Fts3TokenAndCost *pTC = (*ppTC)++;
+ pTC->pPhrase = pPhrase;
+ pTC->iToken = i;
+ pTC->pRoot = pRoot;
+ pTC->pToken = &pPhrase->aToken[i];
+ pTC->iCol = pPhrase->iColumn;
+ *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl);
+ }
+ }else if( pExpr->eType!=FTSQUERY_NOT ){
+ if( pExpr->eType==FTSQUERY_OR ){
+ pRoot = pExpr->pLeft;
+ **ppOr = pRoot;
+ (*ppOr)++;
+ }
+ fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc);
+ if( pExpr->eType==FTSQUERY_OR ){
+ pRoot = pExpr->pRight;
+ **ppOr = pRoot;
+ (*ppOr)++;
+ }
+ fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc);
+ }
+ }
+}
+
+/*
+** Determine the average document (row) size in pages. If successful,
+** write this value to *pnPage and return SQLITE_OK. Otherwise, return
+** an SQLite error code.
+**
+** The average document size in pages is calculated by first calculating
+** determining the average size in bytes, B. If B is less than the amount
+** of data that will fit on a single leaf page of an intkey table in
+** this database, then the average docsize is 1. Otherwise, it is 1 plus
+** the number of overflow pages consumed by a record B bytes in size.
+*/
+static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){
+ if( pCsr->nRowAvg==0 ){
+ /* The average document size, which is required to calculate the cost
+ ** of each doclist, has not yet been determined. Read the required
+ ** data from the %_stat table to calculate it.
+ **
+ ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3
+ ** varints, where nCol is the number of columns in the FTS3 table.
+ ** The first varint is the number of documents currently stored in
+ ** the table. The following nCol varints contain the total amount of
+ ** data stored in all rows of each column of the table, from left
+ ** to right.
+ */
+ int rc;
+ Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
+ sqlite3_stmt *pStmt;
+ sqlite3_int64 nDoc = 0;
+ sqlite3_int64 nByte = 0;
+ const char *pEnd;
+ const char *a;
+
+ rc = sqlite3Fts3SelectDoctotal(p, &pStmt);
+ if( rc!=SQLITE_OK ) return rc;
+ a = sqlite3_column_blob(pStmt, 0);
+ assert( a );
+
+ pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
+ a += sqlite3Fts3GetVarint(a, &nDoc);
+ while( a<pEnd ){
+ a += sqlite3Fts3GetVarint(a, &nByte);
+ }
+ if( nDoc==0 || nByte==0 ){
+ sqlite3_reset(pStmt);
+ return SQLITE_CORRUPT_VTAB;
+ }
+
+ pCsr->nDoc = nDoc;
+ pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz);
+ assert( pCsr->nRowAvg>0 );
+ rc = sqlite3_reset(pStmt);
+ if( rc!=SQLITE_OK ) return rc;
+ }
+
+ *pnPage = pCsr->nRowAvg;
+ return SQLITE_OK;
+}
+
+/*
+** This function is called to select the tokens (if any) that will be
+** deferred. The array aTC[] has already been populated when this is
+** called.
+**
+** This function is called once for each AND/NEAR cluster in the
+** expression. Each invocation determines which tokens to defer within
+** the cluster with root node pRoot. See comments above the definition
+** of struct Fts3TokenAndCost for more details.
+**
+** If no error occurs, SQLITE_OK is returned and sqlite3Fts3DeferToken()
+** called on each token to defer. Otherwise, an SQLite error code is
+** returned.
+*/
+static int fts3EvalSelectDeferred(
+ Fts3Cursor *pCsr, /* FTS Cursor handle */
+ Fts3Expr *pRoot, /* Consider tokens with this root node */
+ Fts3TokenAndCost *aTC, /* Array of expression tokens and costs */
+ int nTC /* Number of entries in aTC[] */
+){
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+ int nDocSize = 0; /* Number of pages per doc loaded */
+ int rc = SQLITE_OK; /* Return code */
+ int ii; /* Iterator variable for various purposes */
+ int nOvfl = 0; /* Total overflow pages used by doclists */
+ int nToken = 0; /* Total number of tokens in cluster */
+
+ int nMinEst = 0; /* The minimum count for any phrase so far. */
+ int nLoad4 = 1; /* (Phrases that will be loaded)^4. */
+
+ /* Count the tokens in this AND/NEAR cluster. If none of the doclists
+ ** associated with the tokens spill onto overflow pages, or if there is
+ ** only 1 token, exit early. No tokens to defer in this case. */
+ for(ii=0; ii<nTC; ii++){
+ if( aTC[ii].pRoot==pRoot ){
+ nOvfl += aTC[ii].nOvfl;
+ nToken++;
+ }
+ }
+ if( nOvfl==0 || nToken<2 ) return SQLITE_OK;
+
+ /* Obtain the average docsize (in pages). */
+ rc = fts3EvalAverageDocsize(pCsr, &nDocSize);
+ assert( rc!=SQLITE_OK || nDocSize>0 );
+
+
+ /* Iterate through all tokens in this AND/NEAR cluster, in ascending order
+ ** of the number of overflow pages that will be loaded by the pager layer
+ ** to retrieve the entire doclist for the token from the full-text index.
+ ** Load the doclists for tokens that are either:
+ **
+ ** a. The cheapest token in the entire query (i.e. the one visited by the
+ ** first iteration of this loop), or
+ **
+ ** b. Part of a multi-token phrase.
+ **
+ ** After each token doclist is loaded, merge it with the others from the
+ ** same phrase and count the number of documents that the merged doclist
+ ** contains. Set variable "nMinEst" to the smallest number of documents in
+ ** any phrase doclist for which 1 or more token doclists have been loaded.
+ ** Let nOther be the number of other phrases for which it is certain that
+ ** one or more tokens will not be deferred.
+ **
+ ** Then, for each token, defer it if loading the doclist would result in
+ ** loading N or more overflow pages into memory, where N is computed as:
+ **
+ ** (nMinEst + 4^nOther - 1) / (4^nOther)
+ */
+ for(ii=0; ii<nToken && rc==SQLITE_OK; ii++){
+ int iTC; /* Used to iterate through aTC[] array. */
+ Fts3TokenAndCost *pTC = 0; /* Set to cheapest remaining token. */
+
+ /* Set pTC to point to the cheapest remaining token. */
+ for(iTC=0; iTC<nTC; iTC++){
+ if( aTC[iTC].pToken && aTC[iTC].pRoot==pRoot
+ && (!pTC || aTC[iTC].nOvfl<pTC->nOvfl)
+ ){
+ pTC = &aTC[iTC];
+ }
+ }
+ assert( pTC );
+
+ if( ii && pTC->nOvfl>=((nMinEst+(nLoad4/4)-1)/(nLoad4/4))*nDocSize ){
+ /* The number of overflow pages to load for this (and therefore all
+ ** subsequent) tokens is greater than the estimated number of pages
+ ** that will be loaded if all subsequent tokens are deferred.
+ */
+ Fts3PhraseToken *pToken = pTC->pToken;
+ rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol);
+ fts3SegReaderCursorFree(pToken->pSegcsr);
+ pToken->pSegcsr = 0;
+ }else{
+ nLoad4 = nLoad4*4;
+ if( ii==0 || pTC->pPhrase->nToken>1 ){
+ /* Either this is the cheapest token in the entire query, or it is
+ ** part of a multi-token phrase. Either way, the entire doclist will
+ ** (eventually) be loaded into memory. It may as well be now. */
+ Fts3PhraseToken *pToken = pTC->pToken;
+ int nList = 0;
+ char *pList = 0;
+ rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList);
+ assert( rc==SQLITE_OK || pList==0 );
+ if( rc==SQLITE_OK ){
+ int nCount;
+ fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList);
+ nCount = fts3DoclistCountDocids(
+ pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll
+ );
+ if( ii==0 || nCount<nMinEst ) nMinEst = nCount;
+ }
+ }
+ }
+ pTC->pToken = 0;
+ }
+
+ return rc;
+}
+
+/*
+** This function is called from within the xFilter method. It initializes
+** the full-text query currently stored in pCsr->pExpr. To iterate through
+** the results of a query, the caller does:
+**
+** fts3EvalStart(pCsr);
+** while( 1 ){
+** fts3EvalNext(pCsr);
+** if( pCsr->bEof ) break;
+** ... return row pCsr->iPrevId to the caller ...
+** }
+*/
+static int fts3EvalStart(Fts3Cursor *pCsr){
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+ int rc = SQLITE_OK;
+ int nToken = 0;
+ int nOr = 0;
+
+ /* Allocate a MultiSegReader for each token in the expression. */
+ fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc);
+
+ /* Determine which, if any, tokens in the expression should be deferred. */
+ if( rc==SQLITE_OK && nToken>1 && pTab->bHasStat ){
+ Fts3TokenAndCost *aTC;
+ Fts3Expr **apOr;
+ aTC = (Fts3TokenAndCost *)sqlite3_malloc(
+ sizeof(Fts3TokenAndCost) * nToken
+ + sizeof(Fts3Expr *) * nOr * 2
+ );
+ apOr = (Fts3Expr **)&aTC[nToken];
+
+ if( !aTC ){
+ rc = SQLITE_NOMEM;
+ }else{
+ int ii;
+ Fts3TokenAndCost *pTC = aTC;
+ Fts3Expr **ppOr = apOr;
+
+ fts3EvalTokenCosts(pCsr, 0, pCsr->pExpr, &pTC, &ppOr, &rc);
+ nToken = pTC-aTC;
+ nOr = ppOr-apOr;
+
+ if( rc==SQLITE_OK ){
+ rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken);
+ for(ii=0; rc==SQLITE_OK && ii<nOr; ii++){
+ rc = fts3EvalSelectDeferred(pCsr, apOr[ii], aTC, nToken);
+ }
+ }
+
+ sqlite3_free(aTC);
+ }
+ }
+
+ fts3EvalStartReaders(pCsr, pCsr->pExpr, 1, &rc);
+ return rc;
+}
+
+/*
+** Invalidate the current position list for phrase pPhrase.
+*/
+static void fts3EvalInvalidatePoslist(Fts3Phrase *pPhrase){
+ if( pPhrase->doclist.bFreeList ){
+ sqlite3_free(pPhrase->doclist.pList);
+ }
+ pPhrase->doclist.pList = 0;
+ pPhrase->doclist.nList = 0;
+ pPhrase->doclist.bFreeList = 0;
+}
+
+/*
+** This function is called to edit the position list associated with
+** the phrase object passed as the fifth argument according to a NEAR
+** condition. For example:
+**
+** abc NEAR/5 "def ghi"
+**
+** Parameter nNear is passed the NEAR distance of the expression (5 in
+** the example above). When this function is called, *paPoslist points to
+** the position list, and *pnToken is the number of phrase tokens in, the
+** phrase on the other side of the NEAR operator to pPhrase. For example,
+** if pPhrase refers to the "def ghi" phrase, then *paPoslist points to
+** the position list associated with phrase "abc".
+**
+** All positions in the pPhrase position list that are not sufficiently
+** close to a position in the *paPoslist position list are removed. If this
+** leaves 0 positions, zero is returned. Otherwise, non-zero.
+**
+** Before returning, *paPoslist is set to point to the position lsit
+** associated with pPhrase. And *pnToken is set to the number of tokens in
+** pPhrase.
+*/
+static int fts3EvalNearTrim(
+ int nNear, /* NEAR distance. As in "NEAR/nNear". */
+ char *aTmp, /* Temporary space to use */
+ char **paPoslist, /* IN/OUT: Position list */
+ int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */
+ Fts3Phrase *pPhrase /* The phrase object to trim the doclist of */
+){
+ int nParam1 = nNear + pPhrase->nToken;
+ int nParam2 = nNear + *pnToken;
+ int nNew;
+ char *p2;
+ char *pOut;
+ int res;
+
+ assert( pPhrase->doclist.pList );
+
+ p2 = pOut = pPhrase->doclist.pList;
+ res = fts3PoslistNearMerge(
+ &pOut, aTmp, nParam1, nParam2, paPoslist, &p2
+ );
+ if( res ){
+ nNew = (pOut - pPhrase->doclist.pList) - 1;
+ assert( pPhrase->doclist.pList[nNew]=='\0' );
+ assert( nNew<=pPhrase->doclist.nList && nNew>0 );
+ memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew);
+ pPhrase->doclist.nList = nNew;
+ *paPoslist = pPhrase->doclist.pList;
+ *pnToken = pPhrase->nToken;
+ }
+
+ return res;
+}
+
+/*
+** This function is a no-op if *pRc is other than SQLITE_OK when it is called.
+** Otherwise, it advances the expression passed as the second argument to
+** point to the next matching row in the database. Expressions iterate through
+** matching rows in docid order. Ascending order if Fts3Cursor.bDesc is zero,
+** or descending if it is non-zero.
+**
+** If an error occurs, *pRc is set to an SQLite error code. Otherwise, if
+** successful, the following variables in pExpr are set:
+**
+** Fts3Expr.bEof (non-zero if EOF - there is no next row)
+** Fts3Expr.iDocid (valid if bEof==0. The docid of the next row)
+**
+** If the expression is of type FTSQUERY_PHRASE, and the expression is not
+** at EOF, then the following variables are populated with the position list
+** for the phrase for the visited row:
+**
+** FTs3Expr.pPhrase->doclist.nList (length of pList in bytes)
+** FTs3Expr.pPhrase->doclist.pList (pointer to position list)
+**
+** It says above that this function advances the expression to the next
+** matching row. This is usually true, but there are the following exceptions:
+**
+** 1. Deferred tokens are not taken into account. If a phrase consists
+** entirely of deferred tokens, it is assumed to match every row in
+** the db. In this case the position-list is not populated at all.
+**
+** Or, if a phrase contains one or more deferred tokens and one or
+** more non-deferred tokens, then the expression is advanced to the
+** next possible match, considering only non-deferred tokens. In other
+** words, if the phrase is "A B C", and "B" is deferred, the expression
+** is advanced to the next row that contains an instance of "A * C",
+** where "*" may match any single token. The position list in this case
+** is populated as for "A * C" before returning.
+**
+** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is
+** advanced to point to the next row that matches "x AND y".
+**
+** See fts3EvalTestDeferredAndNear() for details on testing if a row is
+** really a match, taking into account deferred tokens and NEAR operators.
+*/
+static void fts3EvalNextRow(
+ Fts3Cursor *pCsr, /* FTS Cursor handle */
+ Fts3Expr *pExpr, /* Expr. to advance to next matching row */
+ int *pRc /* IN/OUT: Error code */
+){
+ if( *pRc==SQLITE_OK ){
+ int bDescDoclist = pCsr->bDesc; /* Used by DOCID_CMP() macro */
+ assert( pExpr->bEof==0 );
+ pExpr->bStart = 1;
+
+ switch( pExpr->eType ){
+ case FTSQUERY_NEAR:
+ case FTSQUERY_AND: {
+ Fts3Expr *pLeft = pExpr->pLeft;
+ Fts3Expr *pRight = pExpr->pRight;
+ assert( !pLeft->bDeferred || !pRight->bDeferred );
+
+ if( pLeft->bDeferred ){
+ /* LHS is entirely deferred. So we assume it matches every row.
+ ** Advance the RHS iterator to find the next row visited. */
+ fts3EvalNextRow(pCsr, pRight, pRc);
+ pExpr->iDocid = pRight->iDocid;
+ pExpr->bEof = pRight->bEof;
+ }else if( pRight->bDeferred ){
+ /* RHS is entirely deferred. So we assume it matches every row.
+ ** Advance the LHS iterator to find the next row visited. */
+ fts3EvalNextRow(pCsr, pLeft, pRc);
+ pExpr->iDocid = pLeft->iDocid;
+ pExpr->bEof = pLeft->bEof;
+ }else{
+ /* Neither the RHS or LHS are deferred. */
+ fts3EvalNextRow(pCsr, pLeft, pRc);
+ fts3EvalNextRow(pCsr, pRight, pRc);
+ while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){
+ sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
+ if( iDiff==0 ) break;
+ if( iDiff<0 ){
+ fts3EvalNextRow(pCsr, pLeft, pRc);
+ }else{
+ fts3EvalNextRow(pCsr, pRight, pRc);
+ }
+ }
+ pExpr->iDocid = pLeft->iDocid;
+ pExpr->bEof = (pLeft->bEof || pRight->bEof);
+ }
+ break;
+ }
+
+ case FTSQUERY_OR: {
+ Fts3Expr *pLeft = pExpr->pLeft;
+ Fts3Expr *pRight = pExpr->pRight;
+ sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
+
+ assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid );
+ assert( pRight->bStart || pLeft->iDocid==pRight->iDocid );
+
+ if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){
+ fts3EvalNextRow(pCsr, pLeft, pRc);
+ }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){
+ fts3EvalNextRow(pCsr, pRight, pRc);
+ }else{
+ fts3EvalNextRow(pCsr, pLeft, pRc);
+ fts3EvalNextRow(pCsr, pRight, pRc);
+ }
+
+ pExpr->bEof = (pLeft->bEof && pRight->bEof);
+ iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
+ if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){
+ pExpr->iDocid = pLeft->iDocid;
+ }else{
+ pExpr->iDocid = pRight->iDocid;
+ }
+
+ break;
+ }
+
+ case FTSQUERY_NOT: {
+ Fts3Expr *pLeft = pExpr->pLeft;
+ Fts3Expr *pRight = pExpr->pRight;
+
+ if( pRight->bStart==0 ){
+ fts3EvalNextRow(pCsr, pRight, pRc);
+ assert( *pRc!=SQLITE_OK || pRight->bStart );
+ }
+
+ fts3EvalNextRow(pCsr, pLeft, pRc);
+ if( pLeft->bEof==0 ){
+ while( !*pRc
+ && !pRight->bEof
+ && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0
+ ){
+ fts3EvalNextRow(pCsr, pRight, pRc);
+ }
+ }
+ pExpr->iDocid = pLeft->iDocid;
+ pExpr->bEof = pLeft->bEof;
+ break;
+ }
+
+ default: {
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
+ fts3EvalInvalidatePoslist(pPhrase);
+ *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof);
+ pExpr->iDocid = pPhrase->doclist.iDocid;
+ break;
+ }
+ }
+ }
+}
+
+/*
+** If *pRc is not SQLITE_OK, or if pExpr is not the root node of a NEAR
+** cluster, then this function returns 1 immediately.
+**
+** Otherwise, it checks if the current row really does match the NEAR
+** expression, using the data currently stored in the position lists
+** (Fts3Expr->pPhrase.doclist.pList/nList) for each phrase in the expression.
+**
+** If the current row is a match, the position list associated with each
+** phrase in the NEAR expression is edited in place to contain only those
+** phrase instances sufficiently close to their peers to satisfy all NEAR
+** constraints. In this case it returns 1. If the NEAR expression does not
+** match the current row, 0 is returned. The position lists may or may not
+** be edited if 0 is returned.
+*/
+static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){
+ int res = 1;
+
+ /* The following block runs if pExpr is the root of a NEAR query.
+ ** For example, the query:
+ **
+ ** "w" NEAR "x" NEAR "y" NEAR "z"
+ **
+ ** which is represented in tree form as:
+ **
+ ** |
+ ** +--NEAR--+ <-- root of NEAR query
+ ** | |
+ ** +--NEAR--+ "z"
+ ** | |
+ ** +--NEAR--+ "y"
+ ** | |
+ ** "w" "x"
+ **
+ ** The right-hand child of a NEAR node is always a phrase. The
+ ** left-hand child may be either a phrase or a NEAR node. There are
+ ** no exceptions to this - it's the way the parser in fts3_expr.c works.
+ */
+ if( *pRc==SQLITE_OK
+ && pExpr->eType==FTSQUERY_NEAR
+ && pExpr->bEof==0
+ && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR)
+ ){
+ Fts3Expr *p;
+ int nTmp = 0; /* Bytes of temp space */
+ char *aTmp; /* Temp space for PoslistNearMerge() */
+
+ /* Allocate temporary working space. */
+ for(p=pExpr; p->pLeft; p=p->pLeft){
+ nTmp += p->pRight->pPhrase->doclist.nList;
+ }
+ nTmp += p->pPhrase->doclist.nList;
+ aTmp = sqlite3_malloc(nTmp*2);
+ if( !aTmp ){
+ *pRc = SQLITE_NOMEM;
+ res = 0;
+ }else{
+ char *aPoslist = p->pPhrase->doclist.pList;
+ int nToken = p->pPhrase->nToken;
+
+ for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){
+ Fts3Phrase *pPhrase = p->pRight->pPhrase;
+ int nNear = p->nNear;
+ res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
+ }
+
+ aPoslist = pExpr->pRight->pPhrase->doclist.pList;
+ nToken = pExpr->pRight->pPhrase->nToken;
+ for(p=pExpr->pLeft; p && res; p=p->pLeft){
+ int nNear = p->pParent->nNear;
+ Fts3Phrase *pPhrase = (
+ p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase
+ );
+ res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
+ }
+ }
+
+ sqlite3_free(aTmp);
+ }
+
+ return res;
+}
+
+/*
+** This function is a helper function for fts3EvalTestDeferredAndNear().
+** Assuming no error occurs or has occurred, It returns non-zero if the
+** expression passed as the second argument matches the row that pCsr
+** currently points to, or zero if it does not.
+**
+** If *pRc is not SQLITE_OK when this function is called, it is a no-op.
+** If an error occurs during execution of this function, *pRc is set to
+** the appropriate SQLite error code. In this case the returned value is
+** undefined.
+*/
+static int fts3EvalTestExpr(
+ Fts3Cursor *pCsr, /* FTS cursor handle */
+ Fts3Expr *pExpr, /* Expr to test. May or may not be root. */
+ int *pRc /* IN/OUT: Error code */
+){
+ int bHit = 1; /* Return value */
+ if( *pRc==SQLITE_OK ){
+ switch( pExpr->eType ){
+ case FTSQUERY_NEAR:
+ case FTSQUERY_AND:
+ bHit = (
+ fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc)
+ && fts3EvalTestExpr(pCsr, pExpr->pRight, pRc)
+ && fts3EvalNearTest(pExpr, pRc)
+ );
+
+ /* If the NEAR expression does not match any rows, zero the doclist for
+ ** all phrases involved in the NEAR. This is because the snippet(),
+ ** offsets() and matchinfo() functions are not supposed to recognize
+ ** any instances of phrases that are part of unmatched NEAR queries.
+ ** For example if this expression:
+ **
+ ** ... MATCH 'a OR (b NEAR c)'
+ **
+ ** is matched against a row containing:
+ **
+ ** 'a b d e'
+ **
+ ** then any snippet() should ony highlight the "a" term, not the "b"
+ ** (as "b" is part of a non-matching NEAR clause).
+ */
+ if( bHit==0
+ && pExpr->eType==FTSQUERY_NEAR
+ && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR)
+ ){
+ Fts3Expr *p;
+ for(p=pExpr; p->pPhrase==0; p=p->pLeft){
+ if( p->pRight->iDocid==pCsr->iPrevId ){
+ fts3EvalInvalidatePoslist(p->pRight->pPhrase);
+ }
+ }
+ if( p->iDocid==pCsr->iPrevId ){
+ fts3EvalInvalidatePoslist(p->pPhrase);
+ }
+ }
+
+ break;
+
+ case FTSQUERY_OR: {
+ int bHit1 = fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc);
+ int bHit2 = fts3EvalTestExpr(pCsr, pExpr->pRight, pRc);
+ bHit = bHit1 || bHit2;
+ break;
+ }
+
+ case FTSQUERY_NOT:
+ bHit = (
+ fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc)
+ && !fts3EvalTestExpr(pCsr, pExpr->pRight, pRc)
+ );
+ break;
+
+ default: {
+ if( pCsr->pDeferred
+ && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred)
+ ){
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
+ assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 );
+ if( pExpr->bDeferred ){
+ fts3EvalInvalidatePoslist(pPhrase);
+ }
+ *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase);
+ bHit = (pPhrase->doclist.pList!=0);
+ pExpr->iDocid = pCsr->iPrevId;
+ }else{
+ bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId);
+ }
+ break;
+ }
+ }
+ }
+ return bHit;
+}
+
+/*
+** This function is called as the second part of each xNext operation when
+** iterating through the results of a full-text query. At this point the
+** cursor points to a row that matches the query expression, with the
+** following caveats:
+**
+** * Up until this point, "NEAR" operators in the expression have been
+** treated as "AND".
+**
+** * Deferred tokens have not yet been considered.
+**
+** If *pRc is not SQLITE_OK when this function is called, it immediately
+** returns 0. Otherwise, it tests whether or not after considering NEAR
+** operators and deferred tokens the current row is still a match for the
+** expression. It returns 1 if both of the following are true:
+**
+** 1. *pRc is SQLITE_OK when this function returns, and
+**
+** 2. After scanning the current FTS table row for the deferred tokens,
+** it is determined that the row does *not* match the query.
+**
+** Or, if no error occurs and it seems the current row does match the FTS
+** query, return 0.
+*/
+static int fts3EvalTestDeferredAndNear(Fts3Cursor *pCsr, int *pRc){
+ int rc = *pRc;
+ int bMiss = 0;
+ if( rc==SQLITE_OK ){
+
+ /* If there are one or more deferred tokens, load the current row into
+ ** memory and scan it to determine the position list for each deferred
+ ** token. Then, see if this row is really a match, considering deferred
+ ** tokens and NEAR operators (neither of which were taken into account
+ ** earlier, by fts3EvalNextRow()).
+ */
+ if( pCsr->pDeferred ){
+ rc = fts3CursorSeek(0, pCsr);
+ if( rc==SQLITE_OK ){
+ rc = sqlite3Fts3CacheDeferredDoclists(pCsr);
+ }
+ }
+ bMiss = (0==fts3EvalTestExpr(pCsr, pCsr->pExpr, &rc));
+
+ /* Free the position-lists accumulated for each deferred token above. */
+ sqlite3Fts3FreeDeferredDoclists(pCsr);
+ *pRc = rc;
+ }
+ return (rc==SQLITE_OK && bMiss);
+}
+
+/*
+** Advance to the next document that matches the FTS expression in
+** Fts3Cursor.pExpr.
+*/
+static int fts3EvalNext(Fts3Cursor *pCsr){
+ int rc = SQLITE_OK; /* Return Code */
+ Fts3Expr *pExpr = pCsr->pExpr;
+ assert( pCsr->isEof==0 );
+ if( pExpr==0 ){
+ pCsr->isEof = 1;
+ }else{
+ do {
+ if( pCsr->isRequireSeek==0 ){
+ sqlite3_reset(pCsr->pStmt);
+ }
+ assert( sqlite3_data_count(pCsr->pStmt)==0 );
+ fts3EvalNextRow(pCsr, pExpr, &rc);
+ pCsr->isEof = pExpr->bEof;
+ pCsr->isRequireSeek = 1;
+ pCsr->isMatchinfoNeeded = 1;
+ pCsr->iPrevId = pExpr->iDocid;
+ }while( pCsr->isEof==0 && fts3EvalTestDeferredAndNear(pCsr, &rc) );
+ }
+ return rc;
+}
+
+/*
+** Restart interation for expression pExpr so that the next call to
+** fts3EvalNext() visits the first row. Do not allow incremental
+** loading or merging of phrase doclists for this iteration.
+**
+** If *pRc is other than SQLITE_OK when this function is called, it is
+** a no-op. If an error occurs within this function, *pRc is set to an
+** SQLite error code before returning.
+*/
+static void fts3EvalRestart(
+ Fts3Cursor *pCsr,
+ Fts3Expr *pExpr,
+ int *pRc
+){
+ if( pExpr && *pRc==SQLITE_OK ){
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
+
+ if( pPhrase ){
+ fts3EvalInvalidatePoslist(pPhrase);
+ if( pPhrase->bIncr ){
+ assert( pPhrase->nToken==1 );
+ assert( pPhrase->aToken[0].pSegcsr );
+ sqlite3Fts3MsrIncrRestart(pPhrase->aToken[0].pSegcsr);
+ *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase);
+ }
+
+ pPhrase->doclist.pNextDocid = 0;
+ pPhrase->doclist.iDocid = 0;
+ }
+
+ pExpr->iDocid = 0;
+ pExpr->bEof = 0;
+ pExpr->bStart = 0;
+
+ fts3EvalRestart(pCsr, pExpr->pLeft, pRc);
+ fts3EvalRestart(pCsr, pExpr->pRight, pRc);
+ }
+}
+
+/*
+** After allocating the Fts3Expr.aMI[] array for each phrase in the
+** expression rooted at pExpr, the cursor iterates through all rows matched
+** by pExpr, calling this function for each row. This function increments
+** the values in Fts3Expr.aMI[] according to the position-list currently
+** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase
+** expression nodes.
+*/
+static void fts3EvalUpdateCounts(Fts3Expr *pExpr){
+ if( pExpr ){
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
+ if( pPhrase && pPhrase->doclist.pList ){
+ int iCol = 0;
+ char *p = pPhrase->doclist.pList;
+
+ assert( *p );
+ while( 1 ){
+ u8 c = 0;
+ int iCnt = 0;
+ while( 0xFE & (*p | c) ){
+ if( (c&0x80)==0 ) iCnt++;
+ c = *p++ & 0x80;
+ }
+
+ /* aMI[iCol*3 + 1] = Number of occurrences
+ ** aMI[iCol*3 + 2] = Number of rows containing at least one instance
+ */
+ pExpr->aMI[iCol*3 + 1] += iCnt;
+ pExpr->aMI[iCol*3 + 2] += (iCnt>0);
+ if( *p==0x00 ) break;
+ p++;
+ p += sqlite3Fts3GetVarint32(p, &iCol);
+ }
+ }
+
+ fts3EvalUpdateCounts(pExpr->pLeft);
+ fts3EvalUpdateCounts(pExpr->pRight);
+ }
+}
+
+/*
+** Expression pExpr must be of type FTSQUERY_PHRASE.
+**
+** If it is not already allocated and populated, this function allocates and
+** populates the Fts3Expr.aMI[] array for expression pExpr. If pExpr is part
+** of a NEAR expression, then it also allocates and populates the same array
+** for all other phrases that are part of the NEAR expression.
+**
+** SQLITE_OK is returned if the aMI[] array is successfully allocated and
+** populated. Otherwise, if an error occurs, an SQLite error code is returned.
+*/
+static int fts3EvalGatherStats(
+ Fts3Cursor *pCsr, /* Cursor object */
+ Fts3Expr *pExpr /* FTSQUERY_PHRASE expression */
+){
+ int rc = SQLITE_OK; /* Return code */
+
+ assert( pExpr->eType==FTSQUERY_PHRASE );
+ if( pExpr->aMI==0 ){
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+ Fts3Expr *pRoot; /* Root of NEAR expression */
+ Fts3Expr *p; /* Iterator used for several purposes */
+
+ sqlite3_int64 iPrevId = pCsr->iPrevId;
+ sqlite3_int64 iDocid;
+ u8 bEof;
+
+ /* Find the root of the NEAR expression */
+ pRoot = pExpr;
+ while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){
+ pRoot = pRoot->pParent;
+ }
+ iDocid = pRoot->iDocid;
+ bEof = pRoot->bEof;
+ assert( pRoot->bStart );
+
+ /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */
+ for(p=pRoot; p; p=p->pLeft){
+ Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight);
+ assert( pE->aMI==0 );
+ pE->aMI = (u32 *)sqlite3_malloc(pTab->nColumn * 3 * sizeof(u32));
+ if( !pE->aMI ) return SQLITE_NOMEM;
+ memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32));
+ }
+
+ fts3EvalRestart(pCsr, pRoot, &rc);
+
+ while( pCsr->isEof==0 && rc==SQLITE_OK ){
+
+ do {
+ /* Ensure the %_content statement is reset. */
+ if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt);
+ assert( sqlite3_data_count(pCsr->pStmt)==0 );
+
+ /* Advance to the next document */
+ fts3EvalNextRow(pCsr, pRoot, &rc);
+ pCsr->isEof = pRoot->bEof;
+ pCsr->isRequireSeek = 1;
+ pCsr->isMatchinfoNeeded = 1;
+ pCsr->iPrevId = pRoot->iDocid;
+ }while( pCsr->isEof==0
+ && pRoot->eType==FTSQUERY_NEAR
+ && fts3EvalTestDeferredAndNear(pCsr, &rc)
+ );
+
+ if( rc==SQLITE_OK && pCsr->isEof==0 ){
+ fts3EvalUpdateCounts(pRoot);
+ }
+ }
+
+ pCsr->isEof = 0;
+ pCsr->iPrevId = iPrevId;
+
+ if( bEof ){
+ pRoot->bEof = bEof;
+ }else{
+ /* Caution: pRoot may iterate through docids in ascending or descending
+ ** order. For this reason, even though it seems more defensive, the
+ ** do loop can not be written:
+ **
+ ** do {...} while( pRoot->iDocid<iDocid && rc==SQLITE_OK );
+ */
+ fts3EvalRestart(pCsr, pRoot, &rc);
+ do {
+ fts3EvalNextRow(pCsr, pRoot, &rc);
+ assert( pRoot->bEof==0 );
+ }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK );
+ fts3EvalTestDeferredAndNear(pCsr, &rc);
+ }
+ }
+ return rc;
+}
+
+/*
+** This function is used by the matchinfo() module to query a phrase
+** expression node for the following information:
+**
+** 1. The total number of occurrences of the phrase in each column of
+** the FTS table (considering all rows), and
+**
+** 2. For each column, the number of rows in the table for which the
+** column contains at least one instance of the phrase.
+**
+** If no error occurs, SQLITE_OK is returned and the values for each column
+** written into the array aiOut as follows:
+**
+** aiOut[iCol*3 + 1] = Number of occurrences
+** aiOut[iCol*3 + 2] = Number of rows containing at least one instance
+**
+** Caveats:
+**
+** * If a phrase consists entirely of deferred tokens, then all output
+** values are set to the number of documents in the table. In other
+** words we assume that very common tokens occur exactly once in each
+** column of each row of the table.
+**
+** * If a phrase contains some deferred tokens (and some non-deferred
+** tokens), count the potential occurrence identified by considering
+** the non-deferred tokens instead of actual phrase occurrences.
+**
+** * If the phrase is part of a NEAR expression, then only phrase instances
+** that meet the NEAR constraint are included in the counts.
+*/
+int sqlite3Fts3EvalPhraseStats(
+ Fts3Cursor *pCsr, /* FTS cursor handle */
+ Fts3Expr *pExpr, /* Phrase expression */
+ u32 *aiOut /* Array to write results into (see above) */
+){
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+ int rc = SQLITE_OK;
+ int iCol;
+
+ if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){
+ assert( pCsr->nDoc>0 );
+ for(iCol=0; iCol<pTab->nColumn; iCol++){
+ aiOut[iCol*3 + 1] = (u32)pCsr->nDoc;
+ aiOut[iCol*3 + 2] = (u32)pCsr->nDoc;
+ }
+ }else{
+ rc = fts3EvalGatherStats(pCsr, pExpr);
+ if( rc==SQLITE_OK ){
+ assert( pExpr->aMI );
+ for(iCol=0; iCol<pTab->nColumn; iCol++){
+ aiOut[iCol*3 + 1] = pExpr->aMI[iCol*3 + 1];
+ aiOut[iCol*3 + 2] = pExpr->aMI[iCol*3 + 2];
+ }
+ }
+ }
+
+ return rc;
+}
+
+/*
+** The expression pExpr passed as the second argument to this function
+** must be of type FTSQUERY_PHRASE.
+**
+** The returned value is either NULL or a pointer to a buffer containing
+** a position-list indicating the occurrences of the phrase in column iCol
+** of the current row.
+**
+** More specifically, the returned buffer contains 1 varint for each
+** occurence of the phrase in the column, stored using the normal (delta+2)
+** compression and is terminated by either an 0x01 or 0x00 byte. For example,
+** if the requested column contains "a b X c d X X" and the position-list
+** for 'X' is requested, the buffer returned may contain:
+**
+** 0x04 0x05 0x03 0x01 or 0x04 0x05 0x03 0x00
+**
+** This function works regardless of whether or not the phrase is deferred,
+** incremental, or neither.
+*/
+char *sqlite3Fts3EvalPhrasePoslist(
+ Fts3Cursor *pCsr, /* FTS3 cursor object */
+ Fts3Expr *pExpr, /* Phrase to return doclist for */
+ int iCol /* Column to return position list for */
+){
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+ char *pIter = pPhrase->doclist.pList;
+ int iThis;
+
+ assert( iCol>=0 && iCol<pTab->nColumn );
+ if( !pIter
+ || pExpr->bEof
+ || pExpr->iDocid!=pCsr->iPrevId
+ || (pPhrase->iColumn<pTab->nColumn && pPhrase->iColumn!=iCol)
+ ){
+ return 0;
+ }
+
+ assert( pPhrase->doclist.nList>0 );
+ if( *pIter==0x01 ){
+ pIter++;
+ pIter += sqlite3Fts3GetVarint32(pIter, &iThis);
+ }else{
+ iThis = 0;
+ }
+ while( iThis<iCol ){
+ fts3ColumnlistCopy(0, &pIter);
+ if( *pIter==0x00 ) return 0;
+ pIter++;
+ pIter += sqlite3Fts3GetVarint32(pIter, &iThis);
+ }
+
+ return ((iCol==iThis)?pIter:0);
+}
+
+/*
+** Free all components of the Fts3Phrase structure that were allocated by
+** the eval module. Specifically, this means to free:
+**
+** * the contents of pPhrase->doclist, and
+** * any Fts3MultiSegReader objects held by phrase tokens.
+*/
+void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){
+ if( pPhrase ){
+ int i;
+ sqlite3_free(pPhrase->doclist.aAll);
+ fts3EvalInvalidatePoslist(pPhrase);
+ memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist));
+ for(i=0; i<pPhrase->nToken; i++){
+ fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr);
+ pPhrase->aToken[i].pSegcsr = 0;
+ }
+ }
+}
+
+#if !SQLITE_CORE
+/*
+** Initialize API pointer table, if required.
+*/
+int fts4_extension_init(
+ sqlite3 *db,
+ char **pzErrMsg,
+ void *pApi
+){
+ SQLITE_EXTENSION_INIT2(pApi)
+ return sqlite3Fts3Init(db);
+}
+#endif
+
+#endif
diff --git a/src/libtracker-fts/fts3.h b/src/libtracker-fts/fts3.h
new file mode 100644
index 0000000..4a06f63
--- /dev/null
+++ b/src/libtracker-fts/fts3.h
@@ -0,0 +1,29 @@
+/*
+** 2006 Oct 10
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This header file is used by programs that want to link against the
+** FTS3 library. All it does is declare the sqlite3Fts3Init() interface.
+*/
+#include "sqlite3.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif /* __cplusplus */
+
+int sqlite3Fts3Init(sqlite3 *db);
+int fts4_extension_init(sqlite3 *db,
+ char **pzErrMsg,
+ void *pApi);
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif /* __cplusplus */
diff --git a/src/libtracker-fts/fts3Int.h b/src/libtracker-fts/fts3Int.h
new file mode 100644
index 0000000..ed8043a
--- /dev/null
+++ b/src/libtracker-fts/fts3Int.h
@@ -0,0 +1,513 @@
+/*
+** 2009 Nov 12
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+*/
+#ifndef _FTSINT_H
+#define _FTSINT_H
+
+#if !defined(NDEBUG) && !defined(SQLITE_DEBUG)
+# define NDEBUG 1
+#endif
+
+/*
+** FTS4 is really an extension for FTS3. It is enabled using the
+** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all
+** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3.
+*/
+#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3)
+# define SQLITE_ENABLE_FTS3
+#endif
+
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+/* If not building as part of the core, include sqlite3ext.h. */
+#ifndef SQLITE_CORE
+# include "sqlite3ext.h"
+extern const sqlite3_api_routines *sqlite3_api;
+#endif
+
+#include "sqlite3.h"
+#include "fts3_tokenizer.h"
+#include "fts3_hash.h"
+
+/*
+** This constant controls how often segments are merged. Once there are
+** FTS3_MERGE_COUNT segments of level N, they are merged into a single
+** segment of level N+1.
+*/
+#define FTS3_MERGE_COUNT 16
+
+/*
+** This is the maximum amount of data (in bytes) to store in the
+** Fts3Table.pendingTerms hash table. Normally, the hash table is
+** populated as documents are inserted/updated/deleted in a transaction
+** and used to create a new segment when the transaction is committed.
+** However if this limit is reached midway through a transaction, a new
+** segment is created and the hash table cleared immediately.
+*/
+#define FTS3_MAX_PENDING_DATA (1*1024*1024)
+
+/*
+** Macro to return the number of elements in an array. SQLite has a
+** similar macro called ArraySize(). Use a different name to avoid
+** a collision when building an amalgamation with built-in FTS3.
+*/
+#define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0])))
+
+
+#ifndef MIN
+# define MIN(x,y) ((x)<(y)?(x):(y))
+#endif
+
+/*
+** Maximum length of a varint encoded integer. The varint format is different
+** from that used by SQLite, so the maximum length is 10, not 9.
+*/
+#define FTS3_VARINT_MAX 10
+
+/*
+** FTS4 virtual tables may maintain multiple indexes - one index of all terms
+** in the document set and zero or more prefix indexes. All indexes are stored
+** as one or more b+-trees in the %_segments and %_segdir tables.
+**
+** It is possible to determine which index a b+-tree belongs to based on the
+** value stored in the "%_segdir.level" column. Given this value L, the index
+** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with
+** level values between 0 and 1023 (inclusive) belong to index 0, all levels
+** between 1024 and 2047 to index 1, and so on.
+**
+** It is considered impossible for an index to use more than 1024 levels. In
+** theory though this may happen, but only after at least
+** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables.
+*/
+#define FTS3_SEGDIR_MAXLEVEL 1024
+#define FTS3_SEGDIR_MAXLEVEL_STR "1024"
+
+/*
+** The testcase() macro is only used by the amalgamation. If undefined,
+** make it a no-op.
+*/
+#ifndef testcase
+# define testcase(X)
+#endif
+
+/*
+** Terminator values for position-lists and column-lists.
+*/
+#define POS_COLUMN (1) /* Column-list terminator */
+#define POS_END (0) /* Position-list terminator */
+
+/*
+** This section provides definitions to allow the
+** FTS3 extension to be compiled outside of the
+** amalgamation.
+*/
+#ifndef SQLITE_AMALGAMATION
+/*
+** Macros indicating that conditional expressions are always true or
+** false.
+*/
+#ifdef SQLITE_COVERAGE_TEST
+# define ALWAYS(x) (1)
+# define NEVER(X) (0)
+#else
+# define ALWAYS(x) (x)
+# define NEVER(X) (x)
+#endif
+
+/*
+** Internal types used by SQLite.
+*/
+typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */
+typedef short int i16; /* 2-byte (or larger) signed integer */
+typedef unsigned int u32; /* 4-byte unsigned integer */
+typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */
+
+/*
+** Macro used to suppress compiler warnings for unused parameters.
+*/
+#define UNUSED_PARAMETER(x) (void)(x)
+
+/*
+** Activate assert() only if SQLITE_TEST is enabled.
+*/
+#if !defined(NDEBUG) && !defined(SQLITE_DEBUG)
+# define NDEBUG 1
+#endif
+
+/*
+** The TESTONLY macro is used to enclose variable declarations or
+** other bits of code that are needed to support the arguments
+** within testcase() and assert() macros.
+*/
+#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST)
+# define TESTONLY(X) X
+#else
+# define TESTONLY(X)
+#endif
+
+#endif /* SQLITE_AMALGAMATION */
+
+typedef struct Fts3Table Fts3Table;
+typedef struct Fts3Cursor Fts3Cursor;
+typedef struct Fts3Expr Fts3Expr;
+typedef struct Fts3Phrase Fts3Phrase;
+typedef struct Fts3PhraseToken Fts3PhraseToken;
+
+typedef struct Fts3Doclist Fts3Doclist;
+typedef struct Fts3SegFilter Fts3SegFilter;
+typedef struct Fts3DeferredToken Fts3DeferredToken;
+typedef struct Fts3SegReader Fts3SegReader;
+typedef struct Fts3MultiSegReader Fts3MultiSegReader;
+
+/*
+** A connection to a fulltext index is an instance of the following
+** structure. The xCreate and xConnect methods create an instance
+** of this structure and xDestroy and xDisconnect free that instance.
+** All other methods receive a pointer to the structure as one of their
+** arguments.
+*/
+struct Fts3Table {
+ sqlite3_vtab base; /* Base class used by SQLite core */
+ sqlite3 *db; /* The database connection */
+ const char *zDb; /* logical database name */
+ const char *zName; /* virtual table name */
+ int nColumn; /* number of named columns in virtual table */
+ char **azColumn; /* column names. malloced */
+ sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */
+
+ /* Precompiled statements used by the implementation. Each of these
+ ** statements is run and reset within a single virtual table API call.
+ */
+ sqlite3_stmt *aStmt[27];
+
+ char *zReadExprlist;
+ char *zWriteExprlist;
+
+ int nNodeSize; /* Soft limit for node size */
+ u8 bHasStat; /* True if %_stat table exists */
+ u8 bHasDocsize; /* True if %_docsize table exists */
+ u8 bDescIdx; /* True if doclists are in reverse order */
+ int nPgsz; /* Page size for host database */
+ char *zSegmentsTbl; /* Name of %_segments table */
+ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */
+
+ /* TODO: Fix the first paragraph of this comment.
+ **
+ ** The following hash table is used to buffer pending index updates during
+ ** transactions. Variable nPendingData estimates the memory size of the
+ ** pending data, including hash table overhead, but not malloc overhead.
+ ** When nPendingData exceeds nMaxPendingData, the buffer is flushed
+ ** automatically. Variable iPrevDocid is the docid of the most recently
+ ** inserted record.
+ **
+ ** A single FTS4 table may have multiple full-text indexes. For each index
+ ** there is an entry in the aIndex[] array. Index 0 is an index of all the
+ ** terms that appear in the document set. Each subsequent index in aIndex[]
+ ** is an index of prefixes of a specific length.
+ */
+ int nIndex; /* Size of aIndex[] */
+ struct Fts3Index {
+ int nPrefix; /* Prefix length (0 for main terms index) */
+ Fts3Hash hPending; /* Pending terms table for this index */
+ } *aIndex;
+ int nMaxPendingData; /* Max pending data before flush to disk */
+ int nPendingData; /* Current bytes of pending data */
+ sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */
+
+#if defined(SQLITE_DEBUG)
+ /* State variables used for validating that the transaction control
+ ** methods of the virtual table are called at appropriate times. These
+ ** values do not contribution to the FTS computation; they are used for
+ ** verifying the SQLite core.
+ */
+ int inTransaction; /* True after xBegin but before xCommit/xRollback */
+ int mxSavepoint; /* Largest valid xSavepoint integer */
+#endif
+};
+
+/*
+** When the core wants to read from the virtual table, it creates a
+** virtual table cursor (an instance of the following structure) using
+** the xOpen method. Cursors are destroyed using the xClose method.
+*/
+struct Fts3Cursor {
+ sqlite3_vtab_cursor base; /* Base class used by SQLite core */
+ i16 eSearch; /* Search strategy (see below) */
+ u8 isEof; /* True if at End Of Results */
+ u8 isRequireSeek; /* True if must seek pStmt to %_content row */
+ sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */
+ Fts3Expr *pExpr; /* Parsed MATCH query string */
+ int nPhrase; /* Number of matchable phrases in query */
+ Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */
+ sqlite3_int64 iPrevId; /* Previous id read from aDoclist */
+ char *pNextId; /* Pointer into the body of aDoclist */
+ char *aDoclist; /* List of docids for full-text queries */
+ int nDoclist; /* Size of buffer at aDoclist */
+ u8 bDesc; /* True to sort in descending order */
+ int eEvalmode; /* An FTS3_EVAL_XX constant */
+ int nRowAvg; /* Average size of database rows, in pages */
+ sqlite3_int64 nDoc; /* Documents in table */
+
+ int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */
+ u32 *aMatchinfo; /* Information about most recent match */
+ int nMatchinfo; /* Number of elements in aMatchinfo[] */
+ char *zMatchinfo; /* Matchinfo specification */
+};
+
+#define FTS3_EVAL_FILTER 0
+#define FTS3_EVAL_NEXT 1
+#define FTS3_EVAL_MATCHINFO 2
+
+/*
+** The Fts3Cursor.eSearch member is always set to one of the following.
+** Actualy, Fts3Cursor.eSearch can be greater than or equal to
+** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index
+** of the column to be searched. For example, in
+**
+** CREATE VIRTUAL TABLE ex1 USING fts3(a,b,c,d);
+** SELECT docid FROM ex1 WHERE b MATCH 'one two three';
+**
+** Because the LHS of the MATCH operator is 2nd column "b",
+** Fts3Cursor.eSearch will be set to FTS3_FULLTEXT_SEARCH+1. (+0 for a,
+** +1 for b, +2 for c, +3 for d.) If the LHS of MATCH were "ex1"
+** indicating that all columns should be searched,
+** then eSearch would be set to FTS3_FULLTEXT_SEARCH+4.
+*/
+#define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */
+#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */
+#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */
+
+
+struct Fts3Doclist {
+ char *aAll; /* Array containing doclist (or NULL) */
+ int nAll; /* Size of a[] in bytes */
+ char *pNextDocid; /* Pointer to next docid */
+
+ sqlite3_int64 iDocid; /* Current docid (if pList!=0) */
+ int bFreeList; /* True if pList should be sqlite3_free()d */
+ char *pList; /* Pointer to position list following iDocid */
+ int nList; /* Length of position list */
+};
+
+/*
+** A "phrase" is a sequence of one or more tokens that must match in
+** sequence. A single token is the base case and the most common case.
+** For a sequence of tokens contained in double-quotes (i.e. "one two three")
+** nToken will be the number of tokens in the string.
+*/
+struct Fts3PhraseToken {
+ char *z; /* Text of the token */
+ int n; /* Number of bytes in buffer z */
+ int isPrefix; /* True if token ends with a "*" character */
+
+ /* Variables above this point are populated when the expression is
+ ** parsed (by code in fts3_expr.c). Below this point the variables are
+ ** used when evaluating the expression. */
+ Fts3DeferredToken *pDeferred; /* Deferred token object for this token */
+ Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */
+};
+
+struct Fts3Phrase {
+ /* Cache of doclist for this phrase. */
+ Fts3Doclist doclist;
+ int bIncr; /* True if doclist is loaded incrementally */
+ int iDoclistToken;
+
+ /* Variables below this point are populated by fts3_expr.c when parsing
+ ** a MATCH expression. Everything above is part of the evaluation phase.
+ */
+ int nToken; /* Number of tokens in the phrase */
+ int iColumn; /* Index of column this phrase must match */
+ Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */
+};
+
+/*
+** A tree of these objects forms the RHS of a MATCH operator.
+**
+** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist
+** points to a malloced buffer, size nDoclist bytes, containing the results
+** of this phrase query in FTS3 doclist format. As usual, the initial
+** "Length" field found in doclists stored on disk is omitted from this
+** buffer.
+**
+** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global
+** matchinfo data. If it is not NULL, it points to an array of size nCol*3,
+** where nCol is the number of columns in the queried FTS table. The array
+** is populated as follows:
+**
+** aMI[iCol*3 + 0] = Undefined
+** aMI[iCol*3 + 1] = Number of occurrences
+** aMI[iCol*3 + 2] = Number of rows containing at least one instance
+**
+** The aMI array is allocated using sqlite3_malloc(). It should be freed
+** when the expression node is.
+*/
+struct Fts3Expr {
+ int eType; /* One of the FTSQUERY_XXX values defined below */
+ int nNear; /* Valid if eType==FTSQUERY_NEAR */
+ Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */
+ Fts3Expr *pLeft; /* Left operand */
+ Fts3Expr *pRight; /* Right operand */
+ Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */
+
+ /* The following are used by the fts3_eval.c module. */
+ sqlite3_int64 iDocid; /* Current docid */
+ u8 bEof; /* True this expression is at EOF already */
+ u8 bStart; /* True if iDocid is valid */
+ u8 bDeferred; /* True if this expression is entirely deferred */
+
+ u32 *aMI;
+};
+
+/*
+** Candidate values for Fts3Query.eType. Note that the order of the first
+** four values is in order of precedence when parsing expressions. For
+** example, the following:
+**
+** "a OR b AND c NOT d NEAR e"
+**
+** is equivalent to:
+**
+** "a OR (b AND (c NOT (d NEAR e)))"
+*/
+#define FTSQUERY_NEAR 1
+#define FTSQUERY_NOT 2
+#define FTSQUERY_AND 3
+#define FTSQUERY_OR 4
+#define FTSQUERY_PHRASE 5
+
+
+/* fts3_write.c */
+int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*);
+int sqlite3Fts3PendingTermsFlush(Fts3Table *);
+void sqlite3Fts3PendingTermsClear(Fts3Table *);
+int sqlite3Fts3Optimize(Fts3Table *);
+int sqlite3Fts3SegReaderNew(int, sqlite3_int64,
+ sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
+int sqlite3Fts3SegReaderPending(
+ Fts3Table*,int,const char*,int,int,Fts3SegReader**);
+void sqlite3Fts3SegReaderFree(Fts3SegReader *);
+int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, sqlite3_stmt **);
+int sqlite3Fts3ReadLock(Fts3Table *);
+int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*);
+
+int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **);
+int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **);
+
+void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *);
+int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int);
+int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *);
+void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *);
+void sqlite3Fts3SegmentsClose(Fts3Table *);
+
+/* Special values interpreted by sqlite3SegReaderCursor() */
+#define FTS3_SEGCURSOR_PENDING -1
+#define FTS3_SEGCURSOR_ALL -2
+
+int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*);
+int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *);
+void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *);
+
+int sqlite3Fts3SegReaderCursor(
+ Fts3Table *, int, int, const char *, int, int, int, Fts3MultiSegReader *);
+
+/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */
+#define FTS3_SEGMENT_REQUIRE_POS 0x00000001
+#define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002
+#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004
+#define FTS3_SEGMENT_PREFIX 0x00000008
+#define FTS3_SEGMENT_SCAN 0x00000010
+
+/* Type passed as 4th argument to SegmentReaderIterate() */
+struct Fts3SegFilter {
+ const char *zTerm;
+ int nTerm;
+ int iCol;
+ int flags;
+};
+
+struct Fts3MultiSegReader {
+ /* Used internally by sqlite3Fts3SegReaderXXX() calls */
+ Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */
+ int nSegment; /* Size of apSegment array */
+ int nAdvance; /* How many seg-readers to advance */
+ Fts3SegFilter *pFilter; /* Pointer to filter object */
+ char *aBuffer; /* Buffer to merge doclists in */
+ int nBuffer; /* Allocated size of aBuffer[] in bytes */
+
+ int iColFilter; /* If >=0, filter for this column */
+ int bRestart;
+
+ /* Used by fts3.c only. */
+ int nCost; /* Cost of running iterator */
+ int bLookup; /* True if a lookup of a single entry. */
+
+ /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */
+ char *zTerm; /* Pointer to term buffer */
+ int nTerm; /* Size of zTerm in bytes */
+ char *aDoclist; /* Pointer to doclist buffer */
+ int nDoclist; /* Size of aDoclist[] in bytes */
+};
+
+/* fts3.c */
+int sqlite3Fts3PutVarint(char *, sqlite3_int64);
+int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
+int sqlite3Fts3GetVarint32(const char *, int *);
+int sqlite3Fts3VarintLen(sqlite3_uint64);
+void sqlite3Fts3Dequote(char *);
+void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*);
+
+int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *);
+
+/* fts3_tokenizer.c */
+const char *sqlite3Fts3NextToken(const char *, int *);
+int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *);
+int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *,
+ sqlite3_tokenizer **, char **
+);
+int sqlite3Fts3IsIdChar(char);
+
+/* fts3_snippet.c */
+void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*);
+void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *,
+ const char *, const char *, int, int
+);
+void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *);
+
+/* fts3_expr.c */
+int sqlite3Fts3ExprParse(sqlite3_tokenizer *,
+ char **, int, int, const char *, int, Fts3Expr **
+);
+void sqlite3Fts3ExprFree(Fts3Expr *);
+#ifdef SQLITE_TEST
+int sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
+int sqlite3Fts3InitTerm(sqlite3 *db);
+#endif
+
+/* fts3_aux.c */
+int sqlite3Fts3InitAux(sqlite3 *db);
+
+void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *);
+
+int sqlite3Fts3MsrIncrStart(
+ Fts3Table*, Fts3MultiSegReader*, int, const char*, int);
+int sqlite3Fts3MsrIncrNext(
+ Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *);
+char *sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol);
+int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *);
+int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr);
+
+int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *);
+
+#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */
+#endif /* _FTSINT_H */
diff --git a/src/libtracker-fts/fts3_aux.c b/src/libtracker-fts/fts3_aux.c
new file mode 100644
index 0000000..ada85d7
--- /dev/null
+++ b/src/libtracker-fts/fts3_aux.c
@@ -0,0 +1,474 @@
+/*
+** 2011 Jan 27
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+*/
+#include "fts3Int.h"
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+#include <string.h>
+#include <assert.h>
+
+typedef struct Fts3auxTable Fts3auxTable;
+typedef struct Fts3auxCursor Fts3auxCursor;
+
+struct Fts3auxTable {
+ sqlite3_vtab base; /* Base class used by SQLite core */
+ Fts3Table *pFts3Tab;
+};
+
+struct Fts3auxCursor {
+ sqlite3_vtab_cursor base; /* Base class used by SQLite core */
+ Fts3MultiSegReader csr; /* Must be right after "base" */
+ Fts3SegFilter filter;
+ char *zStop;
+ int nStop; /* Byte-length of string zStop */
+ int isEof; /* True if cursor is at EOF */
+ sqlite3_int64 iRowid; /* Current rowid */
+
+ int iCol; /* Current value of 'col' column */
+ int nStat; /* Size of aStat[] array */
+ struct Fts3auxColstats {
+ sqlite3_int64 nDoc; /* 'documents' values for current csr row */
+ sqlite3_int64 nOcc; /* 'occurrences' values for current csr row */
+ } *aStat;
+};
+
+/*
+** Schema of the terms table.
+*/
+#define FTS3_TERMS_SCHEMA "CREATE TABLE x(term, col, documents, occurrences)"
+
+/*
+** This function does all the work for both the xConnect and xCreate methods.
+** These tables have no persistent representation of their own, so xConnect
+** and xCreate are identical operations.
+*/
+static int fts3auxConnectMethod(
+ sqlite3 *db, /* Database connection */
+ void *pUnused, /* Unused */
+ int argc, /* Number of elements in argv array */
+ const char * const *argv, /* xCreate/xConnect argument array */
+ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
+ char **pzErr /* OUT: sqlite3_malloc'd error message */
+){
+ char const *zDb; /* Name of database (e.g. "main") */
+ char const *zFts3; /* Name of fts3 table */
+ int nDb; /* Result of strlen(zDb) */
+ int nFts3; /* Result of strlen(zFts3) */
+ int nByte; /* Bytes of space to allocate here */
+ int rc; /* value returned by declare_vtab() */
+ Fts3auxTable *p; /* Virtual table object to return */
+
+ UNUSED_PARAMETER(pUnused);
+
+ /* The user should specify a single argument - the name of an fts3 table. */
+ if( argc!=4 ){
+ *pzErr = sqlite3_mprintf(
+ "wrong number of arguments to fts4aux constructor"
+ );
+ return SQLITE_ERROR;
+ }
+
+ zDb = argv[1];
+ nDb = strlen(zDb);
+ zFts3 = argv[3];
+ nFts3 = strlen(zFts3);
+
+ rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA);
+ if( rc!=SQLITE_OK ) return rc;
+
+ nByte = sizeof(Fts3auxTable) + sizeof(Fts3Table) + nDb + nFts3 + 2;
+ p = (Fts3auxTable *)sqlite3_malloc(nByte);
+ if( !p ) return SQLITE_NOMEM;
+ memset(p, 0, nByte);
+
+ p->pFts3Tab = (Fts3Table *)&p[1];
+ p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1];
+ p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1];
+ p->pFts3Tab->db = db;
+ p->pFts3Tab->nIndex = 1;
+
+ memcpy((char *)p->pFts3Tab->zDb, zDb, nDb);
+ memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3);
+ sqlite3Fts3Dequote((char *)p->pFts3Tab->zName);
+
+ *ppVtab = (sqlite3_vtab *)p;
+ return SQLITE_OK;
+}
+
+/*
+** This function does the work for both the xDisconnect and xDestroy methods.
+** These tables have no persistent representation of their own, so xDisconnect
+** and xDestroy are identical operations.
+*/
+static int fts3auxDisconnectMethod(sqlite3_vtab *pVtab){
+ Fts3auxTable *p = (Fts3auxTable *)pVtab;
+ Fts3Table *pFts3 = p->pFts3Tab;
+ int i;
+
+ /* Free any prepared statements held */
+ for(i=0; i<SizeofArray(pFts3->aStmt); i++){
+ sqlite3_finalize(pFts3->aStmt[i]);
+ }
+ sqlite3_free(pFts3->zSegmentsTbl);
+ sqlite3_free(p);
+ return SQLITE_OK;
+}
+
+#define FTS4AUX_EQ_CONSTRAINT 1
+#define FTS4AUX_GE_CONSTRAINT 2
+#define FTS4AUX_LE_CONSTRAINT 4
+
+/*
+** xBestIndex - Analyze a WHERE and ORDER BY clause.
+*/
+static int fts3auxBestIndexMethod(
+ sqlite3_vtab *pVTab,
+ sqlite3_index_info *pInfo
+){
+ int i;
+ int iEq = -1;
+ int iGe = -1;
+ int iLe = -1;
+
+ UNUSED_PARAMETER(pVTab);
+
+ /* This vtab delivers always results in "ORDER BY term ASC" order. */
+ if( pInfo->nOrderBy==1
+ && pInfo->aOrderBy[0].iColumn==0
+ && pInfo->aOrderBy[0].desc==0
+ ){
+ pInfo->orderByConsumed = 1;
+ }
+
+ /* Search for equality and range constraints on the "term" column. */
+ for(i=0; i<pInfo->nConstraint; i++){
+ if( pInfo->aConstraint[i].usable && pInfo->aConstraint[i].iColumn==0 ){
+ int op = pInfo->aConstraint[i].op;
+ if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iEq = i;
+ if( op==SQLITE_INDEX_CONSTRAINT_LT ) iLe = i;
+ if( op==SQLITE_INDEX_CONSTRAINT_LE ) iLe = i;
+ if( op==SQLITE_INDEX_CONSTRAINT_GT ) iGe = i;
+ if( op==SQLITE_INDEX_CONSTRAINT_GE ) iGe = i;
+ }
+ }
+
+ if( iEq>=0 ){
+ pInfo->idxNum = FTS4AUX_EQ_CONSTRAINT;
+ pInfo->aConstraintUsage[iEq].argvIndex = 1;
+ pInfo->estimatedCost = 5;
+ }else{
+ pInfo->idxNum = 0;
+ pInfo->estimatedCost = 20000;
+ if( iGe>=0 ){
+ pInfo->idxNum += FTS4AUX_GE_CONSTRAINT;
+ pInfo->aConstraintUsage[iGe].argvIndex = 1;
+ pInfo->estimatedCost /= 2;
+ }
+ if( iLe>=0 ){
+ pInfo->idxNum += FTS4AUX_LE_CONSTRAINT;
+ pInfo->aConstraintUsage[iLe].argvIndex = 1 + (iGe>=0);
+ pInfo->estimatedCost /= 2;
+ }
+ }
+
+ return SQLITE_OK;
+}
+
+/*
+** xOpen - Open a cursor.
+*/
+static int fts3auxOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
+ Fts3auxCursor *pCsr; /* Pointer to cursor object to return */
+
+ UNUSED_PARAMETER(pVTab);
+
+ pCsr = (Fts3auxCursor *)sqlite3_malloc(sizeof(Fts3auxCursor));
+ if( !pCsr ) return SQLITE_NOMEM;
+ memset(pCsr, 0, sizeof(Fts3auxCursor));
+
+ *ppCsr = (sqlite3_vtab_cursor *)pCsr;
+ return SQLITE_OK;
+}
+
+/*
+** xClose - Close a cursor.
+*/
+static int fts3auxCloseMethod(sqlite3_vtab_cursor *pCursor){
+ Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab;
+ Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor;
+
+ sqlite3Fts3SegmentsClose(pFts3);
+ sqlite3Fts3SegReaderFinish(&pCsr->csr);
+ sqlite3_free((void *)pCsr->filter.zTerm);
+ sqlite3_free(pCsr->zStop);
+ sqlite3_free(pCsr->aStat);
+ sqlite3_free(pCsr);
+ return SQLITE_OK;
+}
+
+static int fts3auxGrowStatArray(Fts3auxCursor *pCsr, int nSize){
+ if( nSize>pCsr->nStat ){
+ struct Fts3auxColstats *aNew;
+ aNew = (struct Fts3auxColstats *)sqlite3_realloc(pCsr->aStat,
+ sizeof(struct Fts3auxColstats) * nSize
+ );
+ if( aNew==0 ) return SQLITE_NOMEM;
+ memset(&aNew[pCsr->nStat], 0,
+ sizeof(struct Fts3auxColstats) * (nSize - pCsr->nStat)
+ );
+ pCsr->aStat = aNew;
+ pCsr->nStat = nSize;
+ }
+ return SQLITE_OK;
+}
+
+/*
+** xNext - Advance the cursor to the next row, if any.
+*/
+static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){
+ Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor;
+ Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab;
+ int rc;
+
+ /* Increment our pretend rowid value. */
+ pCsr->iRowid++;
+
+ for(pCsr->iCol++; pCsr->iCol<pCsr->nStat; pCsr->iCol++){
+ if( pCsr->aStat[pCsr->iCol].nDoc>0 ) return SQLITE_OK;
+ }
+
+ rc = sqlite3Fts3SegReaderStep(pFts3, &pCsr->csr);
+ if( rc==SQLITE_ROW ){
+ int i = 0;
+ int nDoclist = pCsr->csr.nDoclist;
+ char *aDoclist = pCsr->csr.aDoclist;
+ int iCol;
+
+ int eState = 0;
+
+ if( pCsr->zStop ){
+ int n = (pCsr->nStop<pCsr->csr.nTerm) ? pCsr->nStop : pCsr->csr.nTerm;
+ int mc = memcmp(pCsr->zStop, pCsr->csr.zTerm, n);
+ if( mc<0 || (mc==0 && pCsr->csr.nTerm>pCsr->nStop) ){
+ pCsr->isEof = 1;
+ return SQLITE_OK;
+ }
+ }
+
+ if( fts3auxGrowStatArray(pCsr, 2) ) return SQLITE_NOMEM;
+ memset(pCsr->aStat, 0, sizeof(struct Fts3auxColstats) * pCsr->nStat);
+ iCol = 0;
+
+ while( i<nDoclist ){
+ sqlite3_int64 v = 0;
+
+ i += sqlite3Fts3GetVarint(&aDoclist[i], &v);
+ switch( eState ){
+ /* State 0. In this state the integer just read was a docid. */
+ case 0:
+ pCsr->aStat[0].nDoc++;
+ eState = 1;
+ iCol = 0;
+ break;
+
+ /* State 1. In this state we are expecting either a 1, indicating
+ ** that the following integer will be a column number, or the
+ ** start of a position list for column 0.
+ **
+ ** The only difference between state 1 and state 2 is that if the
+ ** integer encountered in state 1 is not 0 or 1, then we need to
+ ** increment the column 0 "nDoc" count for this term.
+ */
+ case 1:
+ assert( iCol==0 );
+ if( v>1 ){
+ pCsr->aStat[1].nDoc++;
+ }
+ eState = 2;
+ /* fall through */
+
+ case 2:
+ if( v==0 ){ /* 0x00. Next integer will be a docid. */
+ eState = 0;
+ }else if( v==1 ){ /* 0x01. Next integer will be a column number. */
+ eState = 3;
+ }else{ /* 2 or greater. A position. */
+ pCsr->aStat[iCol+1].nOcc++;
+ pCsr->aStat[0].nOcc++;
+ }
+ break;
+
+ /* State 3. The integer just read is a column number. */
+ default: assert( eState==3 );
+ iCol = (int)v;
+ if( fts3auxGrowStatArray(pCsr, iCol+2) ) return SQLITE_NOMEM;
+ pCsr->aStat[iCol+1].nDoc++;
+ eState = 2;
+ break;
+ }
+ }
+
+ pCsr->iCol = 0;
+ rc = SQLITE_OK;
+ }else{
+ pCsr->isEof = 1;
+ }
+ return rc;
+}
+
+/*
+** xFilter - Initialize a cursor to point at the start of its data.
+*/
+static int fts3auxFilterMethod(
+ sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
+ int idxNum, /* Strategy index */
+ const char *idxStr, /* Unused */
+ int nVal, /* Number of elements in apVal */
+ sqlite3_value **apVal /* Arguments for the indexing scheme */
+){
+ Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor;
+ Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab;
+ int rc;
+ int isScan;
+
+ UNUSED_PARAMETER(nVal);
+ UNUSED_PARAMETER(idxStr);
+
+ assert( idxStr==0 );
+ assert( idxNum==FTS4AUX_EQ_CONSTRAINT || idxNum==0
+ || idxNum==FTS4AUX_LE_CONSTRAINT || idxNum==FTS4AUX_GE_CONSTRAINT
+ || idxNum==(FTS4AUX_LE_CONSTRAINT|FTS4AUX_GE_CONSTRAINT)
+ );
+ isScan = (idxNum!=FTS4AUX_EQ_CONSTRAINT);
+
+ /* In case this cursor is being reused, close and zero it. */
+ testcase(pCsr->filter.zTerm);
+ sqlite3Fts3SegReaderFinish(&pCsr->csr);
+ sqlite3_free((void *)pCsr->filter.zTerm);
+ sqlite3_free(pCsr->aStat);
+ memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr);
+
+ pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY;
+ if( isScan ) pCsr->filter.flags |= FTS3_SEGMENT_SCAN;
+
+ if( idxNum&(FTS4AUX_EQ_CONSTRAINT|FTS4AUX_GE_CONSTRAINT) ){
+ const unsigned char *zStr = sqlite3_value_text(apVal[0]);
+ if( zStr ){
+ pCsr->filter.zTerm = sqlite3_mprintf("%s", zStr);
+ pCsr->filter.nTerm = sqlite3_value_bytes(apVal[0]);
+ if( pCsr->filter.zTerm==0 ) return SQLITE_NOMEM;
+ }
+ }
+ if( idxNum&FTS4AUX_LE_CONSTRAINT ){
+ int iIdx = (idxNum&FTS4AUX_GE_CONSTRAINT) ? 1 : 0;
+ pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iIdx]));
+ pCsr->nStop = sqlite3_value_bytes(apVal[iIdx]);
+ if( pCsr->zStop==0 ) return SQLITE_NOMEM;
+ }
+
+ rc = sqlite3Fts3SegReaderCursor(pFts3, 0, FTS3_SEGCURSOR_ALL,
+ pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr
+ );
+ if( rc==SQLITE_OK ){
+ rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter);
+ }
+
+ if( rc==SQLITE_OK ) rc = fts3auxNextMethod(pCursor);
+ return rc;
+}
+
+/*
+** xEof - Return true if the cursor is at EOF, or false otherwise.
+*/
+static int fts3auxEofMethod(sqlite3_vtab_cursor *pCursor){
+ Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor;
+ return pCsr->isEof;
+}
+
+/*
+** xColumn - Return a column value.
+*/
+static int fts3auxColumnMethod(
+ sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
+ sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */
+ int iCol /* Index of column to read value from */
+){
+ Fts3auxCursor *p = (Fts3auxCursor *)pCursor;
+
+ assert( p->isEof==0 );
+ if( iCol==0 ){ /* Column "term" */
+ sqlite3_result_text(pContext, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT);
+ }else if( iCol==1 ){ /* Column "col" */
+ if( p->iCol ){
+ sqlite3_result_int(pContext, p->iCol-1);
+ }else{
+ sqlite3_result_text(pContext, "*", -1, SQLITE_STATIC);
+ }
+ }else if( iCol==2 ){ /* Column "documents" */
+ sqlite3_result_int64(pContext, p->aStat[p->iCol].nDoc);
+ }else{ /* Column "occurrences" */
+ sqlite3_result_int64(pContext, p->aStat[p->iCol].nOcc);
+ }
+
+ return SQLITE_OK;
+}
+
+/*
+** xRowid - Return the current rowid for the cursor.
+*/
+static int fts3auxRowidMethod(
+ sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
+ sqlite_int64 *pRowid /* OUT: Rowid value */
+){
+ Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor;
+ *pRowid = pCsr->iRowid;
+ return SQLITE_OK;
+}
+
+/*
+** Register the fts3aux module with database connection db. Return SQLITE_OK
+** if successful or an error code if sqlite3_create_module() fails.
+*/
+int sqlite3Fts3InitAux(sqlite3 *db){
+ static const sqlite3_module fts3aux_module = {
+ 0, /* iVersion */
+ fts3auxConnectMethod, /* xCreate */
+ fts3auxConnectMethod, /* xConnect */
+ fts3auxBestIndexMethod, /* xBestIndex */
+ fts3auxDisconnectMethod, /* xDisconnect */
+ fts3auxDisconnectMethod, /* xDestroy */
+ fts3auxOpenMethod, /* xOpen */
+ fts3auxCloseMethod, /* xClose */
+ fts3auxFilterMethod, /* xFilter */
+ fts3auxNextMethod, /* xNext */
+ fts3auxEofMethod, /* xEof */
+ fts3auxColumnMethod, /* xColumn */
+ fts3auxRowidMethod, /* xRowid */
+ 0, /* xUpdate */
+ 0, /* xBegin */
+ 0, /* xSync */
+ 0, /* xCommit */
+ 0, /* xRollback */
+ 0, /* xFindFunction */
+ 0, /* xRename */
+ 0, /* xSavepoint */
+ 0, /* xRelease */
+ 0 /* xRollbackTo */
+ };
+ int rc; /* Return code */
+
+ rc = sqlite3_create_module(db, "fts4aux", &fts3aux_module, 0);
+ return rc;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
diff --git a/src/libtracker-fts/fts3_expr.c b/src/libtracker-fts/fts3_expr.c
new file mode 100644
index 0000000..7eb2962
--- /dev/null
+++ b/src/libtracker-fts/fts3_expr.c
@@ -0,0 +1,964 @@
+/*
+** 2008 Nov 28
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This module contains code that implements a parser for fts3 query strings
+** (the right-hand argument to the MATCH operator). Because the supported
+** syntax is relatively simple, the whole tokenizer/parser system is
+** hand-coded.
+*/
+#include "fts3Int.h"
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+/*
+** By default, this module parses the legacy syntax that has been
+** traditionally used by fts3. Or, if SQLITE_ENABLE_FTS3_PARENTHESIS
+** is defined, then it uses the new syntax. The differences between
+** the new and the old syntaxes are:
+**
+** a) The new syntax supports parenthesis. The old does not.
+**
+** b) The new syntax supports the AND and NOT operators. The old does not.
+**
+** c) The old syntax supports the "-" token qualifier. This is not
+** supported by the new syntax (it is replaced by the NOT operator).
+**
+** d) When using the old syntax, the OR operator has a greater precedence
+** than an implicit AND. When using the new, both implicity and explicit
+** AND operators have a higher precedence than OR.
+**
+** If compiled with SQLITE_TEST defined, then this module exports the
+** symbol "int sqlite3_fts3_enable_parentheses". Setting this variable
+** to zero causes the module to use the old syntax. If it is set to
+** non-zero the new syntax is activated. This is so both syntaxes can
+** be tested using a single build of testfixture.
+**
+** The following describes the syntax supported by the fts3 MATCH
+** operator in a similar format to that used by the lemon parser
+** generator. This module does not use actually lemon, it uses a
+** custom parser.
+**
+** query ::= andexpr (OR andexpr)*.
+**
+** andexpr ::= notexpr (AND? notexpr)*.
+**
+** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*.
+** notexpr ::= LP query RP.
+**
+** nearexpr ::= phrase (NEAR distance_opt nearexpr)*.
+**
+** distance_opt ::= .
+** distance_opt ::= / INTEGER.
+**
+** phrase ::= TOKEN.
+** phrase ::= COLUMN:TOKEN.
+** phrase ::= "TOKEN TOKEN TOKEN...".
+*/
+
+#ifdef SQLITE_TEST
+int sqlite3_fts3_enable_parentheses = 0;
+#else
+# ifdef SQLITE_ENABLE_FTS3_PARENTHESIS
+# define sqlite3_fts3_enable_parentheses 1
+# else
+# define sqlite3_fts3_enable_parentheses 0
+# endif
+#endif
+
+/*
+** Default span for NEAR operators.
+*/
+#define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10
+
+#include <string.h>
+#include <assert.h>
+
+/*
+** isNot:
+** This variable is used by function getNextNode(). When getNextNode() is
+** called, it sets ParseContext.isNot to true if the 'next node' is a
+** FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the
+** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to
+** zero.
+*/
+typedef struct ParseContext ParseContext;
+struct ParseContext {
+ sqlite3_tokenizer *pTokenizer; /* Tokenizer module */
+ const char **azCol; /* Array of column names for fts3 table */
+ int nCol; /* Number of entries in azCol[] */
+ int iDefaultCol; /* Default column to query */
+ int isNot; /* True if getNextNode() sees a unary - */
+ sqlite3_context *pCtx; /* Write error message here */
+ int nNest; /* Number of nested brackets */
+};
+
+/*
+** This function is equivalent to the standard isspace() function.
+**
+** The standard isspace() can be awkward to use safely, because although it
+** is defined to accept an argument of type int, its behaviour when passed
+** an integer that falls outside of the range of the unsigned char type
+** is undefined (and sometimes, "undefined" means segfault). This wrapper
+** is defined to accept an argument of type char, and always returns 0 for
+** any values that fall outside of the range of the unsigned char type (i.e.
+** negative values).
+*/
+static int fts3isspace(char c){
+ return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f';
+}
+
+/*
+** Allocate nByte bytes of memory using sqlite3_malloc(). If successful,
+** zero the memory before returning a pointer to it. If unsuccessful,
+** return NULL.
+*/
+static void *fts3MallocZero(int nByte){
+ void *pRet = sqlite3_malloc(nByte);
+ if( pRet ) memset(pRet, 0, nByte);
+ return pRet;
+}
+
+
+/*
+** Extract the next token from buffer z (length n) using the tokenizer
+** and other information (column names etc.) in pParse. Create an Fts3Expr
+** structure of type FTSQUERY_PHRASE containing a phrase consisting of this
+** single token and set *ppExpr to point to it. If the end of the buffer is
+** reached before a token is found, set *ppExpr to zero. It is the
+** responsibility of the caller to eventually deallocate the allocated
+** Fts3Expr structure (if any) by passing it to sqlite3_free().
+**
+** Return SQLITE_OK if successful, or SQLITE_NOMEM if a memory allocation
+** fails.
+*/
+static int getNextToken(
+ ParseContext *pParse, /* fts3 query parse context */
+ int iCol, /* Value for Fts3Phrase.iColumn */
+ const char *z, int n, /* Input string */
+ Fts3Expr **ppExpr, /* OUT: expression */
+ int *pnConsumed /* OUT: Number of bytes consumed */
+){
+ sqlite3_tokenizer *pTokenizer = pParse->pTokenizer;
+ sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
+ int rc;
+ sqlite3_tokenizer_cursor *pCursor;
+ Fts3Expr *pRet = 0;
+ int nConsumed = 0;
+
+ rc = pModule->xOpen(pTokenizer, z, n, &pCursor);
+ if( rc==SQLITE_OK ){
+ const char *zToken;
+ int nToken, iStart, iEnd, iPosition;
+ int nByte; /* total space to allocate */
+
+ pCursor->pTokenizer = pTokenizer;
+ rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition);
+
+ if( rc==SQLITE_OK ){
+ nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken;
+ pRet = (Fts3Expr *)fts3MallocZero(nByte);
+ if( !pRet ){
+ rc = SQLITE_NOMEM;
+ }else{
+ pRet->eType = FTSQUERY_PHRASE;
+ pRet->pPhrase = (Fts3Phrase *)&pRet[1];
+ pRet->pPhrase->nToken = 1;
+ pRet->pPhrase->iColumn = iCol;
+ pRet->pPhrase->aToken[0].n = nToken;
+ pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1];
+ memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken);
+
+ if( iEnd<n && z[iEnd]=='*' ){
+ pRet->pPhrase->aToken[0].isPrefix = 1;
+ iEnd++;
+ }
+ if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){
+ pParse->isNot = 1;
+ }
+ }
+ nConsumed = iEnd;
+ }
+
+ pModule->xClose(pCursor);
+ }
+
+ *pnConsumed = nConsumed;
+ *ppExpr = pRet;
+ return rc;
+}
+
+
+/*
+** Enlarge a memory allocation. If an out-of-memory allocation occurs,
+** then free the old allocation.
+*/
+static void *fts3ReallocOrFree(void *pOrig, int nNew){
+ void *pRet = sqlite3_realloc(pOrig, nNew);
+ if( !pRet ){
+ sqlite3_free(pOrig);
+ }
+ return pRet;
+}
+
+/*
+** Buffer zInput, length nInput, contains the contents of a quoted string
+** that appeared as part of an fts3 query expression. Neither quote character
+** is included in the buffer. This function attempts to tokenize the entire
+** input buffer and create an Fts3Expr structure of type FTSQUERY_PHRASE
+** containing the results.
+**
+** If successful, SQLITE_OK is returned and *ppExpr set to point at the
+** allocated Fts3Expr structure. Otherwise, either SQLITE_NOMEM (out of memory
+** error) or SQLITE_ERROR (tokenization error) is returned and *ppExpr set
+** to 0.
+*/
+static int getNextString(
+ ParseContext *pParse, /* fts3 query parse context */
+ const char *zInput, int nInput, /* Input string */
+ Fts3Expr **ppExpr /* OUT: expression */
+){
+ sqlite3_tokenizer *pTokenizer = pParse->pTokenizer;
+ sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
+ int rc;
+ Fts3Expr *p = 0;
+ sqlite3_tokenizer_cursor *pCursor = 0;
+ char *zTemp = 0;
+ int nTemp = 0;
+
+ const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
+ int nToken = 0;
+
+ /* The final Fts3Expr data structure, including the Fts3Phrase,
+ ** Fts3PhraseToken structures token buffers are all stored as a single
+ ** allocation so that the expression can be freed with a single call to
+ ** sqlite3_free(). Setting this up requires a two pass approach.
+ **
+ ** The first pass, in the block below, uses a tokenizer cursor to iterate
+ ** through the tokens in the expression. This pass uses fts3ReallocOrFree()
+ ** to assemble data in two dynamic buffers:
+ **
+ ** Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase
+ ** structure, followed by the array of Fts3PhraseToken
+ ** structures. This pass only populates the Fts3PhraseToken array.
+ **
+ ** Buffer zTemp: Contains copies of all tokens.
+ **
+ ** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below,
+ ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase
+ ** structures.
+ */
+ rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor);
+ if( rc==SQLITE_OK ){
+ int ii;
+ pCursor->pTokenizer = pTokenizer;
+ for(ii=0; rc==SQLITE_OK; ii++){
+ const char *zByte;
+ int nByte, iBegin, iEnd, iPos;
+ rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos);
+ if( rc==SQLITE_OK ){
+ Fts3PhraseToken *pToken;
+
+ p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken));
+ if( !p ) goto no_mem;
+
+ zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte);
+ if( !zTemp ) goto no_mem;
+
+ assert( nToken==ii );
+ pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii];
+ memset(pToken, 0, sizeof(Fts3PhraseToken));
+
+ memcpy(&zTemp[nTemp], zByte, nByte);
+ nTemp += nByte;
+
+ pToken->n = nByte;
+ pToken->isPrefix = (iEnd<nInput && zInput[iEnd]=='*');
+ nToken = ii+1;
+ }
+ }
+
+ pModule->xClose(pCursor);
+ pCursor = 0;
+ }
+
+ if( rc==SQLITE_DONE ){
+ int jj;
+ char *zBuf = 0;
+
+ p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp);
+ if( !p ) goto no_mem;
+ memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p);
+ p->eType = FTSQUERY_PHRASE;
+ p->pPhrase = (Fts3Phrase *)&p[1];
+ p->pPhrase->iColumn = pParse->iDefaultCol;
+ p->pPhrase->nToken = nToken;
+
+ zBuf = (char *)&p->pPhrase->aToken[nToken];
+ memcpy(zBuf, zTemp, nTemp);
+ sqlite3_free(zTemp);
+
+ for(jj=0; jj<p->pPhrase->nToken; jj++){
+ p->pPhrase->aToken[jj].z = zBuf;
+ zBuf += p->pPhrase->aToken[jj].n;
+ }
+ rc = SQLITE_OK;
+ }
+
+ *ppExpr = p;
+ return rc;
+no_mem:
+
+ if( pCursor ){
+ pModule->xClose(pCursor);
+ }
+ sqlite3_free(zTemp);
+ sqlite3_free(p);
+ *ppExpr = 0;
+ return SQLITE_NOMEM;
+}
+
+/*
+** Function getNextNode(), which is called by fts3ExprParse(), may itself
+** call fts3ExprParse(). So this forward declaration is required.
+*/
+static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *);
+
+/*
+** The output variable *ppExpr is populated with an allocated Fts3Expr
+** structure, or set to 0 if the end of the input buffer is reached.
+**
+** Returns an SQLite error code. SQLITE_OK if everything works, SQLITE_NOMEM
+** if a malloc failure occurs, or SQLITE_ERROR if a parse error is encountered.
+** If SQLITE_ERROR is returned, pContext is populated with an error message.
+*/
+static int getNextNode(
+ ParseContext *pParse, /* fts3 query parse context */
+ const char *z, int n, /* Input string */
+ Fts3Expr **ppExpr, /* OUT: expression */
+ int *pnConsumed /* OUT: Number of bytes consumed */
+){
+ static const struct Fts3Keyword {
+ char *z; /* Keyword text */
+ unsigned char n; /* Length of the keyword */
+ unsigned char parenOnly; /* Only valid in paren mode */
+ unsigned char eType; /* Keyword code */
+ } aKeyword[] = {
+ { "OR" , 2, 0, FTSQUERY_OR },
+ { "AND", 3, 1, FTSQUERY_AND },
+ { "NOT", 3, 1, FTSQUERY_NOT },
+ { "NEAR", 4, 0, FTSQUERY_NEAR }
+ };
+ int ii;
+ int iCol;
+ int iColLen;
+ int rc;
+ Fts3Expr *pRet = 0;
+
+ const char *zInput = z;
+ int nInput = n;
+
+ pParse->isNot = 0;
+
+ /* Skip over any whitespace before checking for a keyword, an open or
+ ** close bracket, or a quoted string.
+ */
+ while( nInput>0 && fts3isspace(*zInput) ){
+ nInput--;
+ zInput++;
+ }
+ if( nInput==0 ){
+ return SQLITE_DONE;
+ }
+
+ /* See if we are dealing with a keyword. */
+ for(ii=0; ii<(int)(sizeof(aKeyword)/sizeof(struct Fts3Keyword)); ii++){
+ const struct Fts3Keyword *pKey = &aKeyword[ii];
+
+ if( (pKey->parenOnly & ~sqlite3_fts3_enable_parentheses)!=0 ){
+ continue;
+ }
+
+ if( nInput>=pKey->n && 0==memcmp(zInput, pKey->z, pKey->n) ){
+ int nNear = SQLITE_FTS3_DEFAULT_NEAR_PARAM;
+ int nKey = pKey->n;
+ char cNext;
+
+ /* If this is a "NEAR" keyword, check for an explicit nearness. */
+ if( pKey->eType==FTSQUERY_NEAR ){
+ assert( nKey==4 );
+ if( zInput[4]=='/' && zInput[5]>='0' && zInput[5]<='9' ){
+ nNear = 0;
+ for(nKey=5; zInput[nKey]>='0' && zInput[nKey]<='9'; nKey++){
+ nNear = nNear * 10 + (zInput[nKey] - '0');
+ }
+ }
+ }
+
+ /* At this point this is probably a keyword. But for that to be true,
+ ** the next byte must contain either whitespace, an open or close
+ ** parenthesis, a quote character, or EOF.
+ */
+ cNext = zInput[nKey];
+ if( fts3isspace(cNext)
+ || cNext=='"' || cNext=='(' || cNext==')' || cNext==0
+ ){
+ pRet = (Fts3Expr *)fts3MallocZero(sizeof(Fts3Expr));
+ if( !pRet ){
+ return SQLITE_NOMEM;
+ }
+ pRet->eType = pKey->eType;
+ pRet->nNear = nNear;
+ *ppExpr = pRet;
+ *pnConsumed = (int)((zInput - z) + nKey);
+ return SQLITE_OK;
+ }
+
+ /* Turns out that wasn't a keyword after all. This happens if the
+ ** user has supplied a token such as "ORacle". Continue.
+ */
+ }
+ }
+
+ /* Check for an open bracket. */
+ if( sqlite3_fts3_enable_parentheses ){
+ if( *zInput=='(' ){
+ int nConsumed;
+ pParse->nNest++;
+ rc = fts3ExprParse(pParse, &zInput[1], nInput-1, ppExpr, &nConsumed);
+ if( rc==SQLITE_OK && !*ppExpr ){
+ rc = SQLITE_DONE;
+ }
+ *pnConsumed = (int)((zInput - z) + 1 + nConsumed);
+ return rc;
+ }
+
+ /* Check for a close bracket. */
+ if( *zInput==')' ){
+ pParse->nNest--;
+ *pnConsumed = (int)((zInput - z) + 1);
+ return SQLITE_DONE;
+ }
+ }
+
+ /* See if we are dealing with a quoted phrase. If this is the case, then
+ ** search for the closing quote and pass the whole string to getNextString()
+ ** for processing. This is easy to do, as fts3 has no syntax for escaping
+ ** a quote character embedded in a string.
+ */
+ if( *zInput=='"' ){
+ for(ii=1; ii<nInput && zInput[ii]!='"'; ii++);
+ *pnConsumed = (int)((zInput - z) + ii + 1);
+ if( ii==nInput ){
+ return SQLITE_ERROR;
+ }
+ return getNextString(pParse, &zInput[1], ii-1, ppExpr);
+ }
+
+
+ /* If control flows to this point, this must be a regular token, or
+ ** the end of the input. Read a regular token using the sqlite3_tokenizer
+ ** interface. Before doing so, figure out if there is an explicit
+ ** column specifier for the token.
+ **
+ ** TODO: Strangely, it is not possible to associate a column specifier
+ ** with a quoted phrase, only with a single token. Not sure if this was
+ ** an implementation artifact or an intentional decision when fts3 was
+ ** first implemented. Whichever it was, this module duplicates the
+ ** limitation.
+ */
+ iCol = pParse->iDefaultCol;
+ iColLen = 0;
+ for(ii=0; ii<pParse->nCol; ii++){
+ const char *zStr = pParse->azCol[ii];
+ int nStr = (int)strlen(zStr);
+ if( nInput>nStr && zInput[nStr]==':'
+ && sqlite3_strnicmp(zStr, zInput, nStr)==0
+ ){
+ iCol = ii;
+ iColLen = (int)((zInput - z) + nStr + 1);
+ break;
+ }
+ }
+ rc = getNextToken(pParse, iCol, &z[iColLen], n-iColLen, ppExpr, pnConsumed);
+ *pnConsumed += iColLen;
+ return rc;
+}
+
+/*
+** The argument is an Fts3Expr structure for a binary operator (any type
+** except an FTSQUERY_PHRASE). Return an integer value representing the
+** precedence of the operator. Lower values have a higher precedence (i.e.
+** group more tightly). For example, in the C language, the == operator
+** groups more tightly than ||, and would therefore have a higher precedence.
+**
+** When using the new fts3 query syntax (when SQLITE_ENABLE_FTS3_PARENTHESIS
+** is defined), the order of the operators in precedence from highest to
+** lowest is:
+**
+** NEAR
+** NOT
+** AND (including implicit ANDs)
+** OR
+**
+** Note that when using the old query syntax, the OR operator has a higher
+** precedence than the AND operator.
+*/
+static int opPrecedence(Fts3Expr *p){
+ assert( p->eType!=FTSQUERY_PHRASE );
+ if( sqlite3_fts3_enable_parentheses ){
+ return p->eType;
+ }else if( p->eType==FTSQUERY_NEAR ){
+ return 1;
+ }else if( p->eType==FTSQUERY_OR ){
+ return 2;
+ }
+ assert( p->eType==FTSQUERY_AND );
+ return 3;
+}
+
+/*
+** Argument ppHead contains a pointer to the current head of a query
+** expression tree being parsed. pPrev is the expression node most recently
+** inserted into the tree. This function adds pNew, which is always a binary
+** operator node, into the expression tree based on the relative precedence
+** of pNew and the existing nodes of the tree. This may result in the head
+** of the tree changing, in which case *ppHead is set to the new root node.
+*/
+static void insertBinaryOperator(
+ Fts3Expr **ppHead, /* Pointer to the root node of a tree */
+ Fts3Expr *pPrev, /* Node most recently inserted into the tree */
+ Fts3Expr *pNew /* New binary node to insert into expression tree */
+){
+ Fts3Expr *pSplit = pPrev;
+ while( pSplit->pParent && opPrecedence(pSplit->pParent)<=opPrecedence(pNew) ){
+ pSplit = pSplit->pParent;
+ }
+
+ if( pSplit->pParent ){
+ assert( pSplit->pParent->pRight==pSplit );
+ pSplit->pParent->pRight = pNew;
+ pNew->pParent = pSplit->pParent;
+ }else{
+ *ppHead = pNew;
+ }
+ pNew->pLeft = pSplit;
+ pSplit->pParent = pNew;
+}
+
+/*
+** Parse the fts3 query expression found in buffer z, length n. This function
+** returns either when the end of the buffer is reached or an unmatched
+** closing bracket - ')' - is encountered.
+**
+** If successful, SQLITE_OK is returned, *ppExpr is set to point to the
+** parsed form of the expression and *pnConsumed is set to the number of
+** bytes read from buffer z. Otherwise, *ppExpr is set to 0 and SQLITE_NOMEM
+** (out of memory error) or SQLITE_ERROR (parse error) is returned.
+*/
+static int fts3ExprParse(
+ ParseContext *pParse, /* fts3 query parse context */
+ const char *z, int n, /* Text of MATCH query */
+ Fts3Expr **ppExpr, /* OUT: Parsed query structure */
+ int *pnConsumed /* OUT: Number of bytes consumed */
+){
+ Fts3Expr *pRet = 0;
+ Fts3Expr *pPrev = 0;
+ Fts3Expr *pNotBranch = 0; /* Only used in legacy parse mode */
+ int nIn = n;
+ const char *zIn = z;
+ int rc = SQLITE_OK;
+ int isRequirePhrase = 1;
+
+ while( rc==SQLITE_OK ){
+ Fts3Expr *p = 0;
+ int nByte = 0;
+ rc = getNextNode(pParse, zIn, nIn, &p, &nByte);
+ if( rc==SQLITE_OK ){
+ int isPhrase;
+
+ if( !sqlite3_fts3_enable_parentheses
+ && p->eType==FTSQUERY_PHRASE && pParse->isNot
+ ){
+ /* Create an implicit NOT operator. */
+ Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr));
+ if( !pNot ){
+ sqlite3Fts3ExprFree(p);
+ rc = SQLITE_NOMEM;
+ goto exprparse_out;
+ }
+ pNot->eType = FTSQUERY_NOT;
+ pNot->pRight = p;
+ if( pNotBranch ){
+ pNot->pLeft = pNotBranch;
+ }
+ pNotBranch = pNot;
+ p = pPrev;
+ }else{
+ int eType = p->eType;
+ isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft);
+
+ /* The isRequirePhrase variable is set to true if a phrase or
+ ** an expression contained in parenthesis is required. If a
+ ** binary operator (AND, OR, NOT or NEAR) is encounted when
+ ** isRequirePhrase is set, this is a syntax error.
+ */
+ if( !isPhrase && isRequirePhrase ){
+ sqlite3Fts3ExprFree(p);
+ rc = SQLITE_ERROR;
+ goto exprparse_out;
+ }
+
+ if( isPhrase && !isRequirePhrase ){
+ /* Insert an implicit AND operator. */
+ Fts3Expr *pAnd;
+ assert( pRet && pPrev );
+ pAnd = fts3MallocZero(sizeof(Fts3Expr));
+ if( !pAnd ){
+ sqlite3Fts3ExprFree(p);
+ rc = SQLITE_NOMEM;
+ goto exprparse_out;
+ }
+ pAnd->eType = FTSQUERY_AND;
+ insertBinaryOperator(&pRet, pPrev, pAnd);
+ pPrev = pAnd;
+ }
+
+ /* This test catches attempts to make either operand of a NEAR
+ ** operator something other than a phrase. For example, either of
+ ** the following:
+ **
+ ** (bracketed expression) NEAR phrase
+ ** phrase NEAR (bracketed expression)
+ **
+ ** Return an error in either case.
+ */
+ if( pPrev && (
+ (eType==FTSQUERY_NEAR && !isPhrase && pPrev->eType!=FTSQUERY_PHRASE)
+ || (eType!=FTSQUERY_PHRASE && isPhrase && pPrev->eType==FTSQUERY_NEAR)
+ )){
+ sqlite3Fts3ExprFree(p);
+ rc = SQLITE_ERROR;
+ goto exprparse_out;
+ }
+
+ if( isPhrase ){
+ if( pRet ){
+ assert( pPrev && pPrev->pLeft && pPrev->pRight==0 );
+ pPrev->pRight = p;
+ p->pParent = pPrev;
+ }else{
+ pRet = p;
+ }
+ }else{
+ insertBinaryOperator(&pRet, pPrev, p);
+ }
+ isRequirePhrase = !isPhrase;
+ }
+ assert( nByte>0 );
+ }
+ assert( rc!=SQLITE_OK || (nByte>0 && nByte<=nIn) );
+ nIn -= nByte;
+ zIn += nByte;
+ pPrev = p;
+ }
+
+ if( rc==SQLITE_DONE && pRet && isRequirePhrase ){
+ rc = SQLITE_ERROR;
+ }
+
+ if( rc==SQLITE_DONE ){
+ rc = SQLITE_OK;
+ if( !sqlite3_fts3_enable_parentheses && pNotBranch ){
+ if( !pRet ){
+ rc = SQLITE_ERROR;
+ }else{
+ Fts3Expr *pIter = pNotBranch;
+ while( pIter->pLeft ){
+ pIter = pIter->pLeft;
+ }
+ pIter->pLeft = pRet;
+ pRet = pNotBranch;
+ }
+ }
+ }
+ *pnConsumed = n - nIn;
+
+exprparse_out:
+ if( rc!=SQLITE_OK ){
+ sqlite3Fts3ExprFree(pRet);
+ sqlite3Fts3ExprFree(pNotBranch);
+ pRet = 0;
+ }
+ *ppExpr = pRet;
+ return rc;
+}
+
+/*
+** Parameters z and n contain a pointer to and length of a buffer containing
+** an fts3 query expression, respectively. This function attempts to parse the
+** query expression and create a tree of Fts3Expr structures representing the
+** parsed expression. If successful, *ppExpr is set to point to the head
+** of the parsed expression tree and SQLITE_OK is returned. If an error
+** occurs, either SQLITE_NOMEM (out-of-memory error) or SQLITE_ERROR (parse
+** error) is returned and *ppExpr is set to 0.
+**
+** If parameter n is a negative number, then z is assumed to point to a
+** nul-terminated string and the length is determined using strlen().
+**
+** The first parameter, pTokenizer, is passed the fts3 tokenizer module to
+** use to normalize query tokens while parsing the expression. The azCol[]
+** array, which is assumed to contain nCol entries, should contain the names
+** of each column in the target fts3 table, in order from left to right.
+** Column names must be nul-terminated strings.
+**
+** The iDefaultCol parameter should be passed the index of the table column
+** that appears on the left-hand-side of the MATCH operator (the default
+** column to match against for tokens for which a column name is not explicitly
+** specified as part of the query string), or -1 if tokens may by default
+** match any table column.
+*/
+int sqlite3Fts3ExprParse(
+ sqlite3_tokenizer *pTokenizer, /* Tokenizer module */
+ char **azCol, /* Array of column names for fts3 table */
+ int nCol, /* Number of entries in azCol[] */
+ int iDefaultCol, /* Default column to query */
+ const char *z, int n, /* Text of MATCH query */
+ Fts3Expr **ppExpr /* OUT: Parsed query structure */
+){
+ int nParsed;
+ int rc;
+ ParseContext sParse;
+ sParse.pTokenizer = pTokenizer;
+ sParse.azCol = (const char **)azCol;
+ sParse.nCol = nCol;
+ sParse.iDefaultCol = iDefaultCol;
+ sParse.nNest = 0;
+ if( z==0 ){
+ *ppExpr = 0;
+ return SQLITE_OK;
+ }
+ if( n<0 ){
+ n = (int)strlen(z);
+ }
+ rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed);
+
+ /* Check for mismatched parenthesis */
+ if( rc==SQLITE_OK && sParse.nNest ){
+ rc = SQLITE_ERROR;
+ sqlite3Fts3ExprFree(*ppExpr);
+ *ppExpr = 0;
+ }
+
+ return rc;
+}
+
+/*
+** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse().
+*/
+void sqlite3Fts3ExprFree(Fts3Expr *p){
+ if( p ){
+ assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 );
+ sqlite3Fts3ExprFree(p->pLeft);
+ sqlite3Fts3ExprFree(p->pRight);
+ sqlite3Fts3EvalPhraseCleanup(p->pPhrase);
+ sqlite3_free(p->aMI);
+ sqlite3_free(p);
+ }
+}
+
+/****************************************************************************
+*****************************************************************************
+** Everything after this point is just test code.
+*/
+
+#ifdef SQLITE_TEST
+
+#include <stdio.h>
+
+/*
+** Function to query the hash-table of tokenizers (see README.tokenizers).
+*/
+static int queryTestTokenizer(
+ sqlite3 *db,
+ const char *zName,
+ const sqlite3_tokenizer_module **pp
+){
+ int rc;
+ sqlite3_stmt *pStmt;
+ const char zSql[] = "SELECT fts3_tokenizer(?)";
+
+ *pp = 0;
+ rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+
+ sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){
+ if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
+ memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
+ }
+ }
+
+ return sqlite3_finalize(pStmt);
+}
+
+/*
+** Return a pointer to a buffer containing a text representation of the
+** expression passed as the first argument. The buffer is obtained from
+** sqlite3_malloc(). It is the responsibility of the caller to use
+** sqlite3_free() to release the memory. If an OOM condition is encountered,
+** NULL is returned.
+**
+** If the second argument is not NULL, then its contents are prepended to
+** the returned expression text and then freed using sqlite3_free().
+*/
+static char *exprToString(Fts3Expr *pExpr, char *zBuf){
+ switch( pExpr->eType ){
+ case FTSQUERY_PHRASE: {
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
+ int i;
+ zBuf = sqlite3_mprintf(
+ "%zPHRASE %d 0", zBuf, pPhrase->iColumn);
+ for(i=0; zBuf && i<pPhrase->nToken; i++){
+ zBuf = sqlite3_mprintf("%z %.*s%s", zBuf,
+ pPhrase->aToken[i].n, pPhrase->aToken[i].z,
+ (pPhrase->aToken[i].isPrefix?"+":"")
+ );
+ }
+ return zBuf;
+ }
+
+ case FTSQUERY_NEAR:
+ zBuf = sqlite3_mprintf("%zNEAR/%d ", zBuf, pExpr->nNear);
+ break;
+ case FTSQUERY_NOT:
+ zBuf = sqlite3_mprintf("%zNOT ", zBuf);
+ break;
+ case FTSQUERY_AND:
+ zBuf = sqlite3_mprintf("%zAND ", zBuf);
+ break;
+ case FTSQUERY_OR:
+ zBuf = sqlite3_mprintf("%zOR ", zBuf);
+ break;
+ }
+
+ if( zBuf ) zBuf = sqlite3_mprintf("%z{", zBuf);
+ if( zBuf ) zBuf = exprToString(pExpr->pLeft, zBuf);
+ if( zBuf ) zBuf = sqlite3_mprintf("%z} {", zBuf);
+
+ if( zBuf ) zBuf = exprToString(pExpr->pRight, zBuf);
+ if( zBuf ) zBuf = sqlite3_mprintf("%z}", zBuf);
+
+ return zBuf;
+}
+
+/*
+** This is the implementation of a scalar SQL function used to test the
+** expression parser. It should be called as follows:
+**
+** fts3_exprtest(<tokenizer>, <expr>, <column 1>, ...);
+**
+** The first argument, <tokenizer>, is the name of the fts3 tokenizer used
+** to parse the query expression (see README.tokenizers). The second argument
+** is the query expression to parse. Each subsequent argument is the name
+** of a column of the fts3 table that the query expression may refer to.
+** For example:
+**
+** SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2');
+*/
+static void fts3ExprTest(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ sqlite3_tokenizer_module const *pModule = 0;
+ sqlite3_tokenizer *pTokenizer = 0;
+ int rc;
+ char **azCol = 0;
+ const char *zExpr;
+ int nExpr;
+ int nCol;
+ int ii;
+ Fts3Expr *pExpr;
+ char *zBuf = 0;
+ sqlite3 *db = sqlite3_context_db_handle(context);
+
+ if( argc<3 ){
+ sqlite3_result_error(context,
+ "Usage: fts3_exprtest(tokenizer, expr, col1, ...", -1
+ );
+ return;
+ }
+
+ rc = queryTestTokenizer(db,
+ (const char *)sqlite3_value_text(argv[0]), &pModule);
+ if( rc==SQLITE_NOMEM ){
+ sqlite3_result_error_nomem(context);
+ goto exprtest_out;
+ }else if( !pModule ){
+ sqlite3_result_error(context, "No such tokenizer module", -1);
+ goto exprtest_out;
+ }
+
+ rc = pModule->xCreate(0, 0, &pTokenizer);
+ assert( rc==SQLITE_NOMEM || rc==SQLITE_OK );
+ if( rc==SQLITE_NOMEM ){
+ sqlite3_result_error_nomem(context);
+ goto exprtest_out;
+ }
+ pTokenizer->pModule = pModule;
+
+ zExpr = (const char *)sqlite3_value_text(argv[1]);
+ nExpr = sqlite3_value_bytes(argv[1]);
+ nCol = argc-2;
+ azCol = (char **)sqlite3_malloc(nCol*sizeof(char *));
+ if( !azCol ){
+ sqlite3_result_error_nomem(context);
+ goto exprtest_out;
+ }
+ for(ii=0; ii<nCol; ii++){
+ azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]);
+ }
+
+ rc = sqlite3Fts3ExprParse(
+ pTokenizer, azCol, nCol, nCol, zExpr, nExpr, &pExpr
+ );
+ if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){
+ sqlite3_result_error(context, "Error parsing expression", -1);
+ }else if( rc==SQLITE_NOMEM || !(zBuf = exprToString(pExpr, 0)) ){
+ sqlite3_result_error_nomem(context);
+ }else{
+ sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
+ sqlite3_free(zBuf);
+ }
+
+ sqlite3Fts3ExprFree(pExpr);
+
+exprtest_out:
+ if( pModule && pTokenizer ){
+ rc = pModule->xDestroy(pTokenizer);
+ }
+ sqlite3_free(azCol);
+}
+
+/*
+** Register the query expression parser test function fts3_exprtest()
+** with database connection db.
+*/
+int sqlite3Fts3ExprInitTestInterface(sqlite3* db){
+ return sqlite3_create_function(
+ db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0
+ );
+}
+
+#endif
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
diff --git a/src/libtracker-fts/tracker-fts-hash.c b/src/libtracker-fts/fts3_hash.c
similarity index 80%
rename from src/libtracker-fts/tracker-fts-hash.c
rename to src/libtracker-fts/fts3_hash.c
index e924afd..57c59b5 100644
--- a/src/libtracker-fts/tracker-fts-hash.c
+++ b/src/libtracker-fts/fts3_hash.c
@@ -23,13 +23,14 @@
** * The FTS3 module is being built into the core of
** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
*/
+#include "fts3Int.h"
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
#include <assert.h>
#include <stdlib.h>
#include <string.h>
-#include <sqlite3.h>
-#include "tracker-fts-hash.h"
+#include "fts3_hash.h"
/*
** Malloc and Free functions
@@ -49,13 +50,13 @@ static void fts3HashFree(void *p){
** fields of the Hash structure.
**
** "pNew" is a pointer to the hash table that is to be initialized.
-** keyClass is one of the constants
-** FTS3_HASH_BINARY or FTS3_HASH_STRING. The value of keyClass
+** keyClass is one of the constants
+** FTS3_HASH_BINARY or FTS3_HASH_STRING. The value of keyClass
** determines what kind of key the hash table will use. "copyKey" is
** true if the hash table should make its own private copy of keys and
** false if it should just use the supplied pointer.
*/
-void sqlite3Fts3HashInit(fts3Hash *pNew, int keyClass, int copyKey){
+void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey){
assert( pNew!=0 );
assert( keyClass>=FTS3_HASH_STRING && keyClass<=FTS3_HASH_BINARY );
pNew->keyClass = keyClass;
@@ -70,8 +71,8 @@ void sqlite3Fts3HashInit(fts3Hash *pNew, int keyClass, int copyKey){
** Call this routine to delete a hash table or to reset a hash table
** to the empty state.
*/
-void sqlite3Fts3HashClear(fts3Hash *pH){
- fts3HashElem *elem; /* For looping over all elements of the table */
+void sqlite3Fts3HashClear(Fts3Hash *pH){
+ Fts3HashElem *elem; /* For looping over all elements of the table */
assert( pH!=0 );
elem = pH->first;
@@ -80,7 +81,7 @@ void sqlite3Fts3HashClear(fts3Hash *pH){
pH->ht = 0;
pH->htsize = 0;
while( elem ){
- fts3HashElem *next_elem = elem->next;
+ Fts3HashElem *next_elem = elem->next;
if( pH->copyKey && elem->pKey ){
fts3HashFree(elem->pKey);
}
@@ -127,7 +128,7 @@ static int fts3BinCompare(const void *pKey1, int n1, const void *pKey2, int n2){
/*
** Return a pointer to the appropriate hash function given the key class.
**
-** The C syntax in this function definition may be unfamilar to some
+** The C syntax in this function definition may be unfamilar to some
** programmers, so we provide the following additional explanation:
**
** The name of the function is "ftsHashFunction". The function takes a
@@ -163,11 +164,11 @@ static int (*ftsCompareFunction(int keyClass))(const void*,int,const void*,int){
/* Link an element into the hash table
*/
static void fts3HashInsertElement(
- fts3Hash *pH, /* The complete hash table */
+ Fts3Hash *pH, /* The complete hash table */
struct _fts3ht *pEntry, /* The entry into which pNew is inserted */
- fts3HashElem *pNew /* The element to be inserted */
+ Fts3HashElem *pNew /* The element to be inserted */
){
- fts3HashElem *pHead; /* First element already in pEntry */
+ Fts3HashElem *pHead; /* First element already in pEntry */
pHead = pEntry->chain;
if( pHead ){
pNew->next = pHead;
@@ -187,17 +188,19 @@ static void fts3HashInsertElement(
/* Resize the hash table so that it cantains "new_size" buckets.
-** "new_size" must be a power of 2. The hash table might fail
+** "new_size" must be a power of 2. The hash table might fail
** to resize if sqliteMalloc() fails.
+**
+** Return non-zero if a memory allocation error occurs.
*/
-static void fts3Rehash(fts3Hash *pH, int new_size){
+static int fts3Rehash(Fts3Hash *pH, int new_size){
struct _fts3ht *new_ht; /* The new hash table */
- fts3HashElem *elem, *next_elem; /* For looping over existing elements */
+ Fts3HashElem *elem, *next_elem; /* For looping over existing elements */
int (*xHash)(const void*,int); /* The hash function */
assert( (new_size & (new_size-1))==0 );
new_ht = (struct _fts3ht *)fts3HashMalloc( new_size*sizeof(struct _fts3ht) );
- if( new_ht==0 ) return;
+ if( new_ht==0 ) return 1;
fts3HashFree(pH->ht);
pH->ht = new_ht;
pH->htsize = new_size;
@@ -207,19 +210,20 @@ static void fts3Rehash(fts3Hash *pH, int new_size){
next_elem = elem->next;
fts3HashInsertElement(pH, &new_ht[h], elem);
}
+ return 0;
}
/* This function (for internal use only) locates an element in an
** hash table that matches the given key. The hash for this key has
** already been computed and is passed as the 4th parameter.
*/
-static fts3HashElem *fts3FindElementByHash(
- const fts3Hash *pH, /* The pH to be searched */
+static Fts3HashElem *fts3FindElementByHash(
+ const Fts3Hash *pH, /* The pH to be searched */
const void *pKey, /* The key we are searching for */
int nKey,
int h /* The hash for this key. */
){
- fts3HashElem *elem; /* Used to loop thru the element list */
+ Fts3HashElem *elem; /* Used to loop thru the element list */
int count; /* Number of elements left to test */
int (*xCompare)(const void*,int,const void*,int); /* comparison function */
@@ -229,7 +233,7 @@ static fts3HashElem *fts3FindElementByHash(
count = pEntry->count;
xCompare = ftsCompareFunction(pH->keyClass);
while( count-- && elem ){
- if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){
+ if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){
return elem;
}
elem = elem->next;
@@ -242,13 +246,13 @@ static fts3HashElem *fts3FindElementByHash(
** element and a hash on the element's key.
*/
static void fts3RemoveElementByHash(
- fts3Hash *pH, /* The pH containing "elem" */
- fts3HashElem* elem, /* The element to be removed from the pH */
+ Fts3Hash *pH, /* The pH containing "elem" */
+ Fts3HashElem* elem, /* The element to be removed from the pH */
int h /* Hash value for the element */
){
struct _fts3ht *pEntry;
if( elem->prev ){
- elem->prev->next = elem->next;
+ elem->prev->next = elem->next;
}else{
pH->first = elem->next;
}
@@ -275,13 +279,12 @@ static void fts3RemoveElementByHash(
}
}
-/* Attempt to locate an element of the hash table pH with a key
-** that matches pKey,nKey. Return the data for this element if it is
-** found, or NULL if there is no match.
-*/
-void *sqlite3Fts3HashFind(const fts3Hash *pH, const void *pKey, int nKey){
- int h; /* A hash on key */
- fts3HashElem *elem; /* The element that matches key */
+Fts3HashElem *sqlite3Fts3HashFindElem(
+ const Fts3Hash *pH,
+ const void *pKey,
+ int nKey
+){
+ int h; /* A hash on key */
int (*xHash)(const void*,int); /* The hash function */
if( pH==0 || pH->ht==0 ) return 0;
@@ -289,8 +292,19 @@ void *sqlite3Fts3HashFind(const fts3Hash *pH, const void *pKey, int nKey){
assert( xHash!=0 );
h = (*xHash)(pKey,nKey);
assert( (pH->htsize & (pH->htsize-1))==0 );
- elem = fts3FindElementByHash(pH,pKey,nKey, h & (pH->htsize-1));
- return elem ? elem->data : 0;
+ return fts3FindElementByHash(pH,pKey,nKey, h & (pH->htsize-1));
+}
+
+/*
+** Attempt to locate an element of the hash table pH with a key
+** that matches pKey,nKey. Return the data for this element if it is
+** found, or NULL if there is no match.
+*/
+void *sqlite3Fts3HashFind(const Fts3Hash *pH, const void *pKey, int nKey){
+ Fts3HashElem *pElem; /* The element that matches key (if any) */
+
+ pElem = sqlite3Fts3HashFindElem(pH, pKey, nKey);
+ return pElem ? pElem->data : 0;
}
/* Insert an element into the hash table pH. The key is pKey,nKey
@@ -309,15 +323,15 @@ void *sqlite3Fts3HashFind(const fts3Hash *pH, const void *pKey, int nKey){
** element corresponding to "key" is removed from the hash table.
*/
void *sqlite3Fts3HashInsert(
- fts3Hash *pH, /* The hash table to insert into */
+ Fts3Hash *pH, /* The hash table to insert into */
const void *pKey, /* The key */
int nKey, /* Number of bytes in the key */
void *data /* The data */
){
int hraw; /* Raw hash value of the key */
int h; /* the hash of the key modulo hash table size */
- fts3HashElem *elem; /* Used to loop thru the element list */
- fts3HashElem *new_elem; /* New element added to the pH */
+ Fts3HashElem *elem; /* Used to loop thru the element list */
+ Fts3HashElem *new_elem; /* New element added to the pH */
int (*xHash)(const void*,int); /* The hash function */
assert( pH!=0 );
@@ -337,7 +351,14 @@ void *sqlite3Fts3HashInsert(
return old_data;
}
if( data==0 ) return 0;
- new_elem = (fts3HashElem*)fts3HashMalloc( sizeof(fts3HashElem) );
+ if( (pH->htsize==0 && fts3Rehash(pH,8))
+ || (pH->count>=pH->htsize && fts3Rehash(pH, pH->htsize*2))
+ ){
+ pH->count = 0;
+ return data;
+ }
+ assert( pH->htsize>0 );
+ new_elem = (Fts3HashElem*)fts3HashMalloc( sizeof(Fts3HashElem) );
if( new_elem==0 ) return data;
if( pH->copyKey && pKey!=0 ){
new_elem->pKey = fts3HashMalloc( nKey );
@@ -351,17 +372,6 @@ void *sqlite3Fts3HashInsert(
}
new_elem->nKey = nKey;
pH->count++;
- if( pH->htsize==0 ){
- fts3Rehash(pH,8);
- if( pH->htsize==0 ){
- pH->count = 0;
- fts3HashFree(new_elem);
- return data;
- }
- }
- if( pH->count > pH->htsize ){
- fts3Rehash(pH,pH->htsize*2);
- }
assert( pH->htsize>0 );
assert( (pH->htsize & (pH->htsize-1))==0 );
h = hraw & (pH->htsize-1);
@@ -370,4 +380,4 @@ void *sqlite3Fts3HashInsert(
return 0;
}
-
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
diff --git a/src/libtracker-fts/tracker-fts-hash.h b/src/libtracker-fts/fts3_hash.h
similarity index 55%
rename from src/libtracker-fts/tracker-fts-hash.h
rename to src/libtracker-fts/fts3_hash.h
index 15b77dc..399f515 100644
--- a/src/libtracker-fts/tracker-fts-hash.h
+++ b/src/libtracker-fts/fts3_hash.h
@@ -1,7 +1,7 @@
/*
** 2001 September 22
**
-** The author disclaims copyright to this source code. In place of
+** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
@@ -18,8 +18,8 @@
#define _FTS3_HASH_H_
/* Forward declarations of structures. */
-typedef struct fts3Hash fts3Hash;
-typedef struct fts3HashElem fts3HashElem;
+typedef struct Fts3Hash Fts3Hash;
+typedef struct Fts3HashElem Fts3HashElem;
/* A complete hash table is an instance of the following structure.
** The internals of this structure are intended to be opaque -- client
@@ -29,41 +29,41 @@ typedef struct fts3HashElem fts3HashElem;
** accessing this structure are really macros, so we can't really make
** this structure opaque.
*/
-struct fts3Hash {
- char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */
- char copyKey; /* True if copy of key made on insert */
- int count; /* Number of entries in this table */
- fts3HashElem *first; /* The first element of the array */
- int htsize; /* Number of buckets in the hash table */
- struct _fts3ht { /* the hash table */
- int count; /* Number of entries with this hash */
- fts3HashElem *chain; /* Pointer to first entry with this hash */
+struct Fts3Hash {
+ char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */
+ char copyKey; /* True if copy of key made on insert */
+ int count; /* Number of entries in this table */
+ Fts3HashElem *first; /* The first element of the array */
+ int htsize; /* Number of buckets in the hash table */
+ struct _fts3ht { /* the hash table */
+ int count; /* Number of entries with this hash */
+ Fts3HashElem *chain; /* Pointer to first entry with this hash */
} *ht;
};
-/* Each element in the hash table is an instance of the following
+/* Each element in the hash table is an instance of the following
** structure. All elements are stored on a single doubly-linked list.
**
** Again, this structure is intended to be opaque, but it can't really
** be opaque because it is used by macros.
*/
-struct fts3HashElem {
- fts3HashElem *next, *prev; /* Next and previous elements in the table */
- void *data; /* Data associated with this element */
+struct Fts3HashElem {
+ Fts3HashElem *next, *prev; /* Next and previous elements in the table */
+ void *data; /* Data associated with this element */
void *pKey; int nKey; /* Key associated with this element */
};
/*
** There are 2 different modes of operation for a hash table:
**
-** FTS3_HASH_STRING pKey points to a string that is nKey bytes long
-** (including the null-terminator, if any). Case
-** is respected in comparisons.
+** FTS3_HASH_STRING pKey points to a string that is nKey bytes long
+** (including the null-terminator, if any). Case
+** is respected in comparisons.
**
-** FTS3_HASH_BINARY pKey points to binary data nKey bytes long.
-** memcmp() is used to compare keys.
+** FTS3_HASH_BINARY pKey points to binary data nKey bytes long.
+** memcmp() is used to compare keys.
**
-** A copy of the key is made if the copyKey parameter to fts3HashInit is 1.
+** A copy of the key is made if the copyKey parameter to fts3HashInit is 1.
*/
#define FTS3_HASH_STRING 1
#define FTS3_HASH_BINARY 2
@@ -71,25 +71,27 @@ struct fts3HashElem {
/*
** Access routines. To delete, insert a NULL pointer.
*/
-void sqlite3Fts3HashInit(fts3Hash*, int keytype, int copyKey);
-void *sqlite3Fts3HashInsert(fts3Hash*, const void *pKey, int nKey, void *pData);
-void *sqlite3Fts3HashFind(const fts3Hash*, const void *pKey, int nKey);
-void sqlite3Fts3HashClear(fts3Hash*);
+void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey);
+void *sqlite3Fts3HashInsert(Fts3Hash*, const void *pKey, int nKey, void *pData);
+void *sqlite3Fts3HashFind(const Fts3Hash*, const void *pKey, int nKey);
+void sqlite3Fts3HashClear(Fts3Hash*);
+Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const void *, int);
/*
** Shorthand for the functions above
*/
-#define fts3HashInit sqlite3Fts3HashInit
-#define fts3HashInsert sqlite3Fts3HashInsert
-#define fts3HashFind sqlite3Fts3HashFind
-#define fts3HashClear sqlite3Fts3HashClear
+#define fts3HashInit sqlite3Fts3HashInit
+#define fts3HashInsert sqlite3Fts3HashInsert
+#define fts3HashFind sqlite3Fts3HashFind
+#define fts3HashClear sqlite3Fts3HashClear
+#define fts3HashFindElem sqlite3Fts3HashFindElem
/*
** Macros for looping over all elements of a hash table. The idiom is
** like this:
**
-** fts3Hash h;
-** fts3HashElem *p;
+** Fts3Hash h;
+** Fts3HashElem *p;
** ...
** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){
** SomeStructure *pData = fts3HashData(p);
@@ -99,7 +101,7 @@ void sqlite3Fts3HashClear(fts3Hash*);
#define fts3HashFirst(H) ((H)->first)
#define fts3HashNext(E) ((E)->next)
#define fts3HashData(E) ((E)->data)
-#define fts3HashKey(E) ((E)->pKey)
+#define fts3HashKey(E) ((E)->pKey)
#define fts3HashKeysize(E) ((E)->nKey)
/*
diff --git a/src/libtracker-fts/fts3_icu.c b/src/libtracker-fts/fts3_icu.c
new file mode 100644
index 0000000..a10a55d
--- /dev/null
+++ b/src/libtracker-fts/fts3_icu.c
@@ -0,0 +1,258 @@
+/*
+** 2007 June 22
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file implements a tokenizer for fts3 based on the ICU library.
+*/
+#include "fts3Int.h"
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+#ifdef SQLITE_ENABLE_ICU
+
+#include <assert.h>
+#include <string.h>
+#include "fts3_tokenizer.h"
+
+#include <unicode/ubrk.h>
+#include <unicode/ucol.h>
+#include <unicode/ustring.h>
+#include <unicode/utf16.h>
+
+typedef struct IcuTokenizer IcuTokenizer;
+typedef struct IcuCursor IcuCursor;
+
+struct IcuTokenizer {
+ sqlite3_tokenizer base;
+ char *zLocale;
+};
+
+struct IcuCursor {
+ sqlite3_tokenizer_cursor base;
+
+ UBreakIterator *pIter; /* ICU break-iterator object */
+ int nChar; /* Number of UChar elements in pInput */
+ UChar *aChar; /* Copy of input using utf-16 encoding */
+ int *aOffset; /* Offsets of each character in utf-8 input */
+
+ int nBuffer;
+ char *zBuffer;
+
+ int iToken;
+};
+
+/*
+** Create a new tokenizer instance.
+*/
+static int icuCreate(
+ int argc, /* Number of entries in argv[] */
+ const char * const *argv, /* Tokenizer creation arguments */
+ sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */
+){
+ IcuTokenizer *p;
+ int n = 0;
+
+ if( argc>0 ){
+ n = strlen(argv[0])+1;
+ }
+ p = (IcuTokenizer *)sqlite3_malloc(sizeof(IcuTokenizer)+n);
+ if( !p ){
+ return SQLITE_NOMEM;
+ }
+ memset(p, 0, sizeof(IcuTokenizer));
+
+ if( n ){
+ p->zLocale = (char *)&p[1];
+ memcpy(p->zLocale, argv[0], n);
+ }
+
+ *ppTokenizer = (sqlite3_tokenizer *)p;
+
+ return SQLITE_OK;
+}
+
+/*
+** Destroy a tokenizer
+*/
+static int icuDestroy(sqlite3_tokenizer *pTokenizer){
+ IcuTokenizer *p = (IcuTokenizer *)pTokenizer;
+ sqlite3_free(p);
+ return SQLITE_OK;
+}
+
+/*
+** Prepare to begin tokenizing a particular string. The input
+** string to be tokenized is pInput[0..nBytes-1]. A cursor
+** used to incrementally tokenize this string is returned in
+** *ppCursor.
+*/
+static int icuOpen(
+ sqlite3_tokenizer *pTokenizer, /* The tokenizer */
+ const char *zInput, /* Input string */
+ int nInput, /* Length of zInput in bytes */
+ sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
+){
+ IcuTokenizer *p = (IcuTokenizer *)pTokenizer;
+ IcuCursor *pCsr;
+
+ const int32_t opt = U_FOLD_CASE_DEFAULT;
+ UErrorCode status = U_ZERO_ERROR;
+ int nChar;
+
+ UChar32 c;
+ int iInput = 0;
+ int iOut = 0;
+
+ *ppCursor = 0;
+
+ if( nInput<0 ){
+ nInput = strlen(zInput);
+ }
+ nChar = nInput+1;
+ pCsr = (IcuCursor *)sqlite3_malloc(
+ sizeof(IcuCursor) + /* IcuCursor */
+ nChar * sizeof(UChar) + /* IcuCursor.aChar[] */
+ (nChar+1) * sizeof(int) /* IcuCursor.aOffset[] */
+ );
+ if( !pCsr ){
+ return SQLITE_NOMEM;
+ }
+ memset(pCsr, 0, sizeof(IcuCursor));
+ pCsr->aChar = (UChar *)&pCsr[1];
+ pCsr->aOffset = (int *)&pCsr->aChar[nChar];
+
+ pCsr->aOffset[iOut] = iInput;
+ U8_NEXT(zInput, iInput, nInput, c);
+ while( c>0 ){
+ int isError = 0;
+ c = u_foldCase(c, opt);
+ U16_APPEND(pCsr->aChar, iOut, nChar, c, isError);
+ if( isError ){
+ sqlite3_free(pCsr);
+ return SQLITE_ERROR;
+ }
+ pCsr->aOffset[iOut] = iInput;
+
+ if( iInput<nInput ){
+ U8_NEXT(zInput, iInput, nInput, c);
+ }else{
+ c = 0;
+ }
+ }
+
+ pCsr->pIter = ubrk_open(UBRK_WORD, p->zLocale, pCsr->aChar, iOut, &status);
+ if( !U_SUCCESS(status) ){
+ sqlite3_free(pCsr);
+ return SQLITE_ERROR;
+ }
+ pCsr->nChar = iOut;
+
+ ubrk_first(pCsr->pIter);
+ *ppCursor = (sqlite3_tokenizer_cursor *)pCsr;
+ return SQLITE_OK;
+}
+
+/*
+** Close a tokenization cursor previously opened by a call to icuOpen().
+*/
+static int icuClose(sqlite3_tokenizer_cursor *pCursor){
+ IcuCursor *pCsr = (IcuCursor *)pCursor;
+ ubrk_close(pCsr->pIter);
+ sqlite3_free(pCsr->zBuffer);
+ sqlite3_free(pCsr);
+ return SQLITE_OK;
+}
+
+/*
+** Extract the next token from a tokenization cursor.
+*/
+static int icuNext(
+ sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */
+ const char **ppToken, /* OUT: *ppToken is the token text */
+ int *pnBytes, /* OUT: Number of bytes in token */
+ int *piStartOffset, /* OUT: Starting offset of token */
+ int *piEndOffset, /* OUT: Ending offset of token */
+ int *piPosition /* OUT: Position integer of token */
+){
+ IcuCursor *pCsr = (IcuCursor *)pCursor;
+
+ int iStart = 0;
+ int iEnd = 0;
+ int nByte = 0;
+
+ while( iStart==iEnd ){
+ UChar32 c;
+
+ iStart = ubrk_current(pCsr->pIter);
+ iEnd = ubrk_next(pCsr->pIter);
+ if( iEnd==UBRK_DONE ){
+ return SQLITE_DONE;
+ }
+
+ while( iStart<iEnd ){
+ int iWhite = iStart;
+ U8_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c);
+ if( u_isspace(c) ){
+ iStart = iWhite;
+ }else{
+ break;
+ }
+ }
+ assert(iStart<=iEnd);
+ }
+
+ do {
+ UErrorCode status = U_ZERO_ERROR;
+ if( nByte ){
+ char *zNew = sqlite3_realloc(pCsr->zBuffer, nByte);
+ if( !zNew ){
+ return SQLITE_NOMEM;
+ }
+ pCsr->zBuffer = zNew;
+ pCsr->nBuffer = nByte;
+ }
+
+ u_strToUTF8(
+ pCsr->zBuffer, pCsr->nBuffer, &nByte, /* Output vars */
+ &pCsr->aChar[iStart], iEnd-iStart, /* Input vars */
+ &status /* Output success/failure */
+ );
+ } while( nByte>pCsr->nBuffer );
+
+ *ppToken = pCsr->zBuffer;
+ *pnBytes = nByte;
+ *piStartOffset = pCsr->aOffset[iStart];
+ *piEndOffset = pCsr->aOffset[iEnd];
+ *piPosition = pCsr->iToken++;
+
+ return SQLITE_OK;
+}
+
+/*
+** The set of routines that implement the simple tokenizer
+*/
+static const sqlite3_tokenizer_module icuTokenizerModule = {
+ 0, /* iVersion */
+ icuCreate, /* xCreate */
+ icuDestroy, /* xCreate */
+ icuOpen, /* xOpen */
+ icuClose, /* xClose */
+ icuNext, /* xNext */
+};
+
+/*
+** Set *ppModule to point at the implementation of the ICU tokenizer.
+*/
+void sqlite3Fts3IcuTokenizerModule(
+ sqlite3_tokenizer_module const**ppModule
+){
+ *ppModule = &icuTokenizerModule;
+}
+
+#endif /* defined(SQLITE_ENABLE_ICU) */
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
diff --git a/src/libtracker-fts/fts3_porter.c b/src/libtracker-fts/fts3_porter.c
new file mode 100644
index 0000000..148c570
--- /dev/null
+++ b/src/libtracker-fts/fts3_porter.c
@@ -0,0 +1,645 @@
+/*
+** 2006 September 30
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+*************************************************************************
+** Implementation of the full-text-search tokenizer that implements
+** a Porter stemmer.
+*/
+
+/*
+** The code in this file is only compiled if:
+**
+** * The FTS3 module is being built as an extension
+** (in which case SQLITE_CORE is not defined), or
+**
+** * The FTS3 module is being built into the core of
+** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
+*/
+#include "fts3Int.h"
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "fts3_tokenizer.h"
+
+/*
+** Class derived from sqlite3_tokenizer
+*/
+typedef struct porter_tokenizer {
+ sqlite3_tokenizer base; /* Base class */
+} porter_tokenizer;
+
+/*
+** Class derived from sqlit3_tokenizer_cursor
+*/
+typedef struct porter_tokenizer_cursor {
+ sqlite3_tokenizer_cursor base;
+ const char *zInput; /* input we are tokenizing */
+ int nInput; /* size of the input */
+ int iOffset; /* current position in zInput */
+ int iToken; /* index of next token to be returned */
+ char *zToken; /* storage for current token */
+ int nAllocated; /* space allocated to zToken buffer */
+} porter_tokenizer_cursor;
+
+
+/*
+** Create a new tokenizer instance.
+*/
+static int porterCreate(
+ int argc, const char * const *argv,
+ sqlite3_tokenizer **ppTokenizer
+){
+ porter_tokenizer *t;
+
+ UNUSED_PARAMETER(argc);
+ UNUSED_PARAMETER(argv);
+
+ t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t));
+ if( t==NULL ) return SQLITE_NOMEM;
+ memset(t, 0, sizeof(*t));
+ *ppTokenizer = &t->base;
+ return SQLITE_OK;
+}
+
+/*
+** Destroy a tokenizer
+*/
+static int porterDestroy(sqlite3_tokenizer *pTokenizer){
+ sqlite3_free(pTokenizer);
+ return SQLITE_OK;
+}
+
+/*
+** Prepare to begin tokenizing a particular string. The input
+** string to be tokenized is zInput[0..nInput-1]. A cursor
+** used to incrementally tokenize this string is returned in
+** *ppCursor.
+*/
+static int porterOpen(
+ sqlite3_tokenizer *pTokenizer, /* The tokenizer */
+ const char *zInput, int nInput, /* String to be tokenized */
+ sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
+){
+ porter_tokenizer_cursor *c;
+
+ UNUSED_PARAMETER(pTokenizer);
+
+ c = (porter_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
+ if( c==NULL ) return SQLITE_NOMEM;
+
+ c->zInput = zInput;
+ if( zInput==0 ){
+ c->nInput = 0;
+ }else if( nInput<0 ){
+ c->nInput = (int)strlen(zInput);
+ }else{
+ c->nInput = nInput;
+ }
+ c->iOffset = 0; /* start tokenizing at the beginning */
+ c->iToken = 0;
+ c->zToken = NULL; /* no space allocated, yet. */
+ c->nAllocated = 0;
+
+ *ppCursor = &c->base;
+ return SQLITE_OK;
+}
+
+/*
+** Close a tokenization cursor previously opened by a call to
+** porterOpen() above.
+*/
+static int porterClose(sqlite3_tokenizer_cursor *pCursor){
+ porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
+ sqlite3_free(c->zToken);
+ sqlite3_free(c);
+ return SQLITE_OK;
+}
+/*
+** Vowel or consonant
+*/
+static const char cType[] = {
+ 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0,
+ 1, 1, 1, 2, 1
+};
+
+/*
+** isConsonant() and isVowel() determine if their first character in
+** the string they point to is a consonant or a vowel, according
+** to Porter ruls.
+**
+** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'.
+** 'Y' is a consonant unless it follows another consonant,
+** in which case it is a vowel.
+**
+** In these routine, the letters are in reverse order. So the 'y' rule
+** is that 'y' is a consonant unless it is followed by another
+** consonent.
+*/
+static int isVowel(const char*);
+static int isConsonant(const char *z){
+ int j;
+ char x = *z;
+ if( x==0 ) return 0;
+ assert( x>='a' && x<='z' );
+ j = cType[x-'a'];
+ if( j<2 ) return j;
+ return z[1]==0 || isVowel(z + 1);
+}
+static int isVowel(const char *z){
+ int j;
+ char x = *z;
+ if( x==0 ) return 0;
+ assert( x>='a' && x<='z' );
+ j = cType[x-'a'];
+ if( j<2 ) return 1-j;
+ return isConsonant(z + 1);
+}
+
+/*
+** Let any sequence of one or more vowels be represented by V and let
+** C be sequence of one or more consonants. Then every word can be
+** represented as:
+**
+** [C] (VC){m} [V]
+**
+** In prose: A word is an optional consonant followed by zero or
+** vowel-consonant pairs followed by an optional vowel. "m" is the
+** number of vowel consonant pairs. This routine computes the value
+** of m for the first i bytes of a word.
+**
+** Return true if the m-value for z is 1 or more. In other words,
+** return true if z contains at least one vowel that is followed
+** by a consonant.
+**
+** In this routine z[] is in reverse order. So we are really looking
+** for an instance of of a consonant followed by a vowel.
+*/
+static int m_gt_0(const char *z){
+ while( isVowel(z) ){ z++; }
+ if( *z==0 ) return 0;
+ while( isConsonant(z) ){ z++; }
+ return *z!=0;
+}
+
+/* Like mgt0 above except we are looking for a value of m which is
+** exactly 1
+*/
+static int m_eq_1(const char *z){
+ while( isVowel(z) ){ z++; }
+ if( *z==0 ) return 0;
+ while( isConsonant(z) ){ z++; }
+ if( *z==0 ) return 0;
+ while( isVowel(z) ){ z++; }
+ if( *z==0 ) return 1;
+ while( isConsonant(z) ){ z++; }
+ return *z==0;
+}
+
+/* Like mgt0 above except we are looking for a value of m>1 instead
+** or m>0
+*/
+static int m_gt_1(const char *z){
+ while( isVowel(z) ){ z++; }
+ if( *z==0 ) return 0;
+ while( isConsonant(z) ){ z++; }
+ if( *z==0 ) return 0;
+ while( isVowel(z) ){ z++; }
+ if( *z==0 ) return 0;
+ while( isConsonant(z) ){ z++; }
+ return *z!=0;
+}
+
+/*
+** Return TRUE if there is a vowel anywhere within z[0..n-1]
+*/
+static int hasVowel(const char *z){
+ while( isConsonant(z) ){ z++; }
+ return *z!=0;
+}
+
+/*
+** Return TRUE if the word ends in a double consonant.
+**
+** The text is reversed here. So we are really looking at
+** the first two characters of z[].
+*/
+static int doubleConsonant(const char *z){
+ return isConsonant(z) && z[0]==z[1];
+}
+
+/*
+** Return TRUE if the word ends with three letters which
+** are consonant-vowel-consonent and where the final consonant
+** is not 'w', 'x', or 'y'.
+**
+** The word is reversed here. So we are really checking the
+** first three letters and the first one cannot be in [wxy].
+*/
+static int star_oh(const char *z){
+ return
+ isConsonant(z) &&
+ z[0]!='w' && z[0]!='x' && z[0]!='y' &&
+ isVowel(z+1) &&
+ isConsonant(z+2);
+}
+
+/*
+** If the word ends with zFrom and xCond() is true for the stem
+** of the word that preceeds the zFrom ending, then change the
+** ending to zTo.
+**
+** The input word *pz and zFrom are both in reverse order. zTo
+** is in normal order.
+**
+** Return TRUE if zFrom matches. Return FALSE if zFrom does not
+** match. Not that TRUE is returned even if xCond() fails and
+** no substitution occurs.
+*/
+static int stem(
+ char **pz, /* The word being stemmed (Reversed) */
+ const char *zFrom, /* If the ending matches this... (Reversed) */
+ const char *zTo, /* ... change the ending to this (not reversed) */
+ int (*xCond)(const char*) /* Condition that must be true */
+){
+ char *z = *pz;
+ while( *zFrom && *zFrom==*z ){ z++; zFrom++; }
+ if( *zFrom!=0 ) return 0;
+ if( xCond && !xCond(z) ) return 1;
+ while( *zTo ){
+ *(--z) = *(zTo++);
+ }
+ *pz = z;
+ return 1;
+}
+
+/*
+** This is the fallback stemmer used when the porter stemmer is
+** inappropriate. The input word is copied into the output with
+** US-ASCII case folding. If the input word is too long (more
+** than 20 bytes if it contains no digits or more than 6 bytes if
+** it contains digits) then word is truncated to 20 or 6 bytes
+** by taking 10 or 3 bytes from the beginning and end.
+*/
+static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
+ int i, mx, j;
+ int hasDigit = 0;
+ for(i=0; i<nIn; i++){
+ char c = zIn[i];
+ if( c>='A' && c<='Z' ){
+ zOut[i] = c - 'A' + 'a';
+ }else{
+ if( c>='0' && c<='9' ) hasDigit = 1;
+ zOut[i] = c;
+ }
+ }
+ mx = hasDigit ? 3 : 10;
+ if( nIn>mx*2 ){
+ for(j=mx, i=nIn-mx; i<nIn; i++, j++){
+ zOut[j] = zOut[i];
+ }
+ i = j;
+ }
+ zOut[i] = 0;
+ *pnOut = i;
+}
+
+
+/*
+** Stem the input word zIn[0..nIn-1]. Store the output in zOut.
+** zOut is at least big enough to hold nIn bytes. Write the actual
+** size of the output word (exclusive of the '\0' terminator) into *pnOut.
+**
+** Any upper-case characters in the US-ASCII character set ([A-Z])
+** are converted to lower case. Upper-case UTF characters are
+** unchanged.
+**
+** Words that are longer than about 20 bytes are stemmed by retaining
+** a few bytes from the beginning and the end of the word. If the
+** word contains digits, 3 bytes are taken from the beginning and
+** 3 bytes from the end. For long words without digits, 10 bytes
+** are taken from each end. US-ASCII case folding still applies.
+**
+** If the input word contains not digits but does characters not
+** in [a-zA-Z] then no stemming is attempted and this routine just
+** copies the input into the input into the output with US-ASCII
+** case folding.
+**
+** Stemming never increases the length of the word. So there is
+** no chance of overflowing the zOut buffer.
+*/
+static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
+ int i, j;
+ char zReverse[28];
+ char *z, *z2;
+ if( nIn<3 || nIn>=(int)sizeof(zReverse)-7 ){
+ /* The word is too big or too small for the porter stemmer.
+ ** Fallback to the copy stemmer */
+ copy_stemmer(zIn, nIn, zOut, pnOut);
+ return;
+ }
+ for(i=0, j=sizeof(zReverse)-6; i<nIn; i++, j--){
+ char c = zIn[i];
+ if( c>='A' && c<='Z' ){
+ zReverse[j] = c + 'a' - 'A';
+ }else if( c>='a' && c<='z' ){
+ zReverse[j] = c;
+ }else{
+ /* The use of a character not in [a-zA-Z] means that we fallback
+ ** to the copy stemmer */
+ copy_stemmer(zIn, nIn, zOut, pnOut);
+ return;
+ }
+ }
+ memset(&zReverse[sizeof(zReverse)-5], 0, 5);
+ z = &zReverse[j+1];
+
+
+ /* Step 1a */
+ if( z[0]=='s' ){
+ if(
+ !stem(&z, "sess", "ss", 0) &&
+ !stem(&z, "sei", "i", 0) &&
+ !stem(&z, "ss", "ss", 0)
+ ){
+ z++;
+ }
+ }
+
+ /* Step 1b */
+ z2 = z;
+ if( stem(&z, "dee", "ee", m_gt_0) ){
+ /* Do nothing. The work was all in the test */
+ }else if(
+ (stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel))
+ && z!=z2
+ ){
+ if( stem(&z, "ta", "ate", 0) ||
+ stem(&z, "lb", "ble", 0) ||
+ stem(&z, "zi", "ize", 0) ){
+ /* Do nothing. The work was all in the test */
+ }else if( doubleConsonant(z) && (*z!='l' && *z!='s' && *z!='z') ){
+ z++;
+ }else if( m_eq_1(z) && star_oh(z) ){
+ *(--z) = 'e';
+ }
+ }
+
+ /* Step 1c */
+ if( z[0]=='y' && hasVowel(z+1) ){
+ z[0] = 'i';
+ }
+
+ /* Step 2 */
+ switch( z[1] ){
+ case 'a':
+ stem(&z, "lanoita", "ate", m_gt_0) ||
+ stem(&z, "lanoit", "tion", m_gt_0);
+ break;
+ case 'c':
+ stem(&z, "icne", "ence", m_gt_0) ||
+ stem(&z, "icna", "ance", m_gt_0);
+ break;
+ case 'e':
+ stem(&z, "rezi", "ize", m_gt_0);
+ break;
+ case 'g':
+ stem(&z, "igol", "log", m_gt_0);
+ break;
+ case 'l':
+ stem(&z, "ilb", "ble", m_gt_0) ||
+ stem(&z, "illa", "al", m_gt_0) ||
+ stem(&z, "iltne", "ent", m_gt_0) ||
+ stem(&z, "ile", "e", m_gt_0) ||
+ stem(&z, "ilsuo", "ous", m_gt_0);
+ break;
+ case 'o':
+ stem(&z, "noitazi", "ize", m_gt_0) ||
+ stem(&z, "noita", "ate", m_gt_0) ||
+ stem(&z, "rota", "ate", m_gt_0);
+ break;
+ case 's':
+ stem(&z, "msila", "al", m_gt_0) ||
+ stem(&z, "ssenevi", "ive", m_gt_0) ||
+ stem(&z, "ssenluf", "ful", m_gt_0) ||
+ stem(&z, "ssensuo", "ous", m_gt_0);
+ break;
+ case 't':
+ stem(&z, "itila", "al", m_gt_0) ||
+ stem(&z, "itivi", "ive", m_gt_0) ||
+ stem(&z, "itilib", "ble", m_gt_0);
+ break;
+ }
+
+ /* Step 3 */
+ switch( z[0] ){
+ case 'e':
+ stem(&z, "etaci", "ic", m_gt_0) ||
+ stem(&z, "evita", "", m_gt_0) ||
+ stem(&z, "ezila", "al", m_gt_0);
+ break;
+ case 'i':
+ stem(&z, "itici", "ic", m_gt_0);
+ break;
+ case 'l':
+ stem(&z, "laci", "ic", m_gt_0) ||
+ stem(&z, "luf", "", m_gt_0);
+ break;
+ case 's':
+ stem(&z, "ssen", "", m_gt_0);
+ break;
+ }
+
+ /* Step 4 */
+ switch( z[1] ){
+ case 'a':
+ if( z[0]=='l' && m_gt_1(z+2) ){
+ z += 2;
+ }
+ break;
+ case 'c':
+ if( z[0]=='e' && z[2]=='n' && (z[3]=='a' || z[3]=='e') && m_gt_1(z+4) ){
+ z += 4;
+ }
+ break;
+ case 'e':
+ if( z[0]=='r' && m_gt_1(z+2) ){
+ z += 2;
+ }
+ break;
+ case 'i':
+ if( z[0]=='c' && m_gt_1(z+2) ){
+ z += 2;
+ }
+ break;
+ case 'l':
+ if( z[0]=='e' && z[2]=='b' && (z[3]=='a' || z[3]=='i') && m_gt_1(z+4) ){
+ z += 4;
+ }
+ break;
+ case 'n':
+ if( z[0]=='t' ){
+ if( z[2]=='a' ){
+ if( m_gt_1(z+3) ){
+ z += 3;
+ }
+ }else if( z[2]=='e' ){
+ stem(&z, "tneme", "", m_gt_1) ||
+ stem(&z, "tnem", "", m_gt_1) ||
+ stem(&z, "tne", "", m_gt_1);
+ }
+ }
+ break;
+ case 'o':
+ if( z[0]=='u' ){
+ if( m_gt_1(z+2) ){
+ z += 2;
+ }
+ }else if( z[3]=='s' || z[3]=='t' ){
+ stem(&z, "noi", "", m_gt_1);
+ }
+ break;
+ case 's':
+ if( z[0]=='m' && z[2]=='i' && m_gt_1(z+3) ){
+ z += 3;
+ }
+ break;
+ case 't':
+ stem(&z, "eta", "", m_gt_1) ||
+ stem(&z, "iti", "", m_gt_1);
+ break;
+ case 'u':
+ if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){
+ z += 3;
+ }
+ break;
+ case 'v':
+ case 'z':
+ if( z[0]=='e' && z[2]=='i' && m_gt_1(z+3) ){
+ z += 3;
+ }
+ break;
+ }
+
+ /* Step 5a */
+ if( z[0]=='e' ){
+ if( m_gt_1(z+1) ){
+ z++;
+ }else if( m_eq_1(z+1) && !star_oh(z+1) ){
+ z++;
+ }
+ }
+
+ /* Step 5b */
+ if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){
+ z++;
+ }
+
+ /* z[] is now the stemmed word in reverse order. Flip it back
+ ** around into forward order and return.
+ */
+ *pnOut = i = (int)strlen(z);
+ zOut[i] = 0;
+ while( *z ){
+ zOut[--i] = *(z++);
+ }
+}
+
+/*
+** Characters that can be part of a token. We assume any character
+** whose value is greater than 0x80 (any UTF character) can be
+** part of a token. In other words, delimiters all must have
+** values of 0x7f or lower.
+*/
+static const char porterIdChar[] = {
+/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
+};
+#define isDelim(C) (((ch=C)&0x80)==0 && (ch<0x30 || !porterIdChar[ch-0x30]))
+
+/*
+** Extract the next token from a tokenization cursor. The cursor must
+** have been opened by a prior call to porterOpen().
+*/
+static int porterNext(
+ sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by porterOpen */
+ const char **pzToken, /* OUT: *pzToken is the token text */
+ int *pnBytes, /* OUT: Number of bytes in token */
+ int *piStartOffset, /* OUT: Starting offset of token */
+ int *piEndOffset, /* OUT: Ending offset of token */
+ int *piPosition /* OUT: Position integer of token */
+){
+ porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
+ const char *z = c->zInput;
+
+ while( c->iOffset<c->nInput ){
+ int iStartOffset, ch;
+
+ /* Scan past delimiter characters */
+ while( c->iOffset<c->nInput && isDelim(z[c->iOffset]) ){
+ c->iOffset++;
+ }
+
+ /* Count non-delimiter characters. */
+ iStartOffset = c->iOffset;
+ while( c->iOffset<c->nInput && !isDelim(z[c->iOffset]) ){
+ c->iOffset++;
+ }
+
+ if( c->iOffset>iStartOffset ){
+ int n = c->iOffset-iStartOffset;
+ if( n>c->nAllocated ){
+ char *pNew;
+ c->nAllocated = n+20;
+ pNew = sqlite3_realloc(c->zToken, c->nAllocated);
+ if( !pNew ) return SQLITE_NOMEM;
+ c->zToken = pNew;
+ }
+ porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes);
+ *pzToken = c->zToken;
+ *piStartOffset = iStartOffset;
+ *piEndOffset = c->iOffset;
+ *piPosition = c->iToken++;
+ return SQLITE_OK;
+ }
+ }
+ return SQLITE_DONE;
+}
+
+/*
+** The set of routines that implement the porter-stemmer tokenizer
+*/
+static const sqlite3_tokenizer_module porterTokenizerModule = {
+ 0,
+ porterCreate,
+ porterDestroy,
+ porterOpen,
+ porterClose,
+ porterNext,
+};
+
+/*
+** Allocate a new porter tokenizer. Return a pointer to the new
+** tokenizer in *ppModule
+*/
+void sqlite3Fts3PorterTokenizerModule(
+ sqlite3_tokenizer_module const**ppModule
+){
+ *ppModule = &porterTokenizerModule;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
diff --git a/src/libtracker-fts/fts3_snippet.c b/src/libtracker-fts/fts3_snippet.c
new file mode 100644
index 0000000..b569eb1
--- /dev/null
+++ b/src/libtracker-fts/fts3_snippet.c
@@ -0,0 +1,1499 @@
+/*
+** 2009 Oct 23
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+*/
+
+#include "fts3Int.h"
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+#include <string.h>
+#include <assert.h>
+
+/*
+** Characters that may appear in the second argument to matchinfo().
+*/
+#define FTS3_MATCHINFO_NPHRASE 'p' /* 1 value */
+#define FTS3_MATCHINFO_NCOL 'c' /* 1 value */
+#define FTS3_MATCHINFO_NDOC 'n' /* 1 value */
+#define FTS3_MATCHINFO_AVGLENGTH 'a' /* nCol values */
+#define FTS3_MATCHINFO_LENGTH 'l' /* nCol values */
+#define FTS3_MATCHINFO_LCS 's' /* nCol values */
+#define FTS3_MATCHINFO_HITS 'x' /* 3*nCol*nPhrase values */
+
+/*
+** The default value for the second argument to matchinfo().
+*/
+#define FTS3_MATCHINFO_DEFAULT "pcx"
+
+
+/*
+** Used as an fts3ExprIterate() context when loading phrase doclists to
+** Fts3Expr.aDoclist[]/nDoclist.
+*/
+typedef struct LoadDoclistCtx LoadDoclistCtx;
+struct LoadDoclistCtx {
+ Fts3Cursor *pCsr; /* FTS3 Cursor */
+ int nPhrase; /* Number of phrases seen so far */
+ int nToken; /* Number of tokens seen so far */
+};
+
+/*
+** The following types are used as part of the implementation of the
+** fts3BestSnippet() routine.
+*/
+typedef struct SnippetIter SnippetIter;
+typedef struct SnippetPhrase SnippetPhrase;
+typedef struct SnippetFragment SnippetFragment;
+
+struct SnippetIter {
+ Fts3Cursor *pCsr; /* Cursor snippet is being generated from */
+ int iCol; /* Extract snippet from this column */
+ int nSnippet; /* Requested snippet length (in tokens) */
+ int nPhrase; /* Number of phrases in query */
+ SnippetPhrase *aPhrase; /* Array of size nPhrase */
+ int iCurrent; /* First token of current snippet */
+};
+
+struct SnippetPhrase {
+ int nToken; /* Number of tokens in phrase */
+ char *pList; /* Pointer to start of phrase position list */
+ int iHead; /* Next value in position list */
+ char *pHead; /* Position list data following iHead */
+ int iTail; /* Next value in trailing position list */
+ char *pTail; /* Position list data following iTail */
+};
+
+struct SnippetFragment {
+ int iCol; /* Column snippet is extracted from */
+ int iPos; /* Index of first token in snippet */
+ u64 covered; /* Mask of query phrases covered */
+ u64 hlmask; /* Mask of snippet terms to highlight */
+};
+
+/*
+** This type is used as an fts3ExprIterate() context object while
+** accumulating the data returned by the matchinfo() function.
+*/
+typedef struct MatchInfo MatchInfo;
+struct MatchInfo {
+ Fts3Cursor *pCursor; /* FTS3 Cursor */
+ int nCol; /* Number of columns in table */
+ int nPhrase; /* Number of matchable phrases in query */
+ sqlite3_int64 nDoc; /* Number of docs in database */
+ u32 *aMatchinfo; /* Pre-allocated buffer */
+};
+
+
+
+/*
+** The snippet() and offsets() functions both return text values. An instance
+** of the following structure is used to accumulate those values while the
+** functions are running. See fts3StringAppend() for details.
+*/
+typedef struct StrBuffer StrBuffer;
+struct StrBuffer {
+ char *z; /* Pointer to buffer containing string */
+ int n; /* Length of z in bytes (excl. nul-term) */
+ int nAlloc; /* Allocated size of buffer z in bytes */
+};
+
+
+/*
+** This function is used to help iterate through a position-list. A position
+** list is a list of unique integers, sorted from smallest to largest. Each
+** element of the list is represented by an FTS3 varint that takes the value
+** of the difference between the current element and the previous one plus
+** two. For example, to store the position-list:
+**
+** 4 9 113
+**
+** the three varints:
+**
+** 6 7 106
+**
+** are encoded.
+**
+** When this function is called, *pp points to the start of an element of
+** the list. *piPos contains the value of the previous entry in the list.
+** After it returns, *piPos contains the value of the next element of the
+** list and *pp is advanced to the following varint.
+*/
+static void fts3GetDeltaPosition(char **pp, int *piPos){
+ int iVal;
+ *pp += sqlite3Fts3GetVarint32(*pp, &iVal);
+ *piPos += (iVal-2);
+}
+
+/*
+** Helper function for fts3ExprIterate() (see below).
+*/
+static int fts3ExprIterate2(
+ Fts3Expr *pExpr, /* Expression to iterate phrases of */
+ int *piPhrase, /* Pointer to phrase counter */
+ int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */
+ void *pCtx /* Second argument to pass to callback */
+){
+ int rc; /* Return code */
+ int eType = pExpr->eType; /* Type of expression node pExpr */
+
+ if( eType!=FTSQUERY_PHRASE ){
+ assert( pExpr->pLeft && pExpr->pRight );
+ rc = fts3ExprIterate2(pExpr->pLeft, piPhrase, x, pCtx);
+ if( rc==SQLITE_OK && eType!=FTSQUERY_NOT ){
+ rc = fts3ExprIterate2(pExpr->pRight, piPhrase, x, pCtx);
+ }
+ }else{
+ rc = x(pExpr, *piPhrase, pCtx);
+ (*piPhrase)++;
+ }
+ return rc;
+}
+
+/*
+** Iterate through all phrase nodes in an FTS3 query, except those that
+** are part of a sub-tree that is the right-hand-side of a NOT operator.
+** For each phrase node found, the supplied callback function is invoked.
+**
+** If the callback function returns anything other than SQLITE_OK,
+** the iteration is abandoned and the error code returned immediately.
+** Otherwise, SQLITE_OK is returned after a callback has been made for
+** all eligible phrase nodes.
+*/
+static int fts3ExprIterate(
+ Fts3Expr *pExpr, /* Expression to iterate phrases of */
+ int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */
+ void *pCtx /* Second argument to pass to callback */
+){
+ int iPhrase = 0; /* Variable used as the phrase counter */
+ return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx);
+}
+
+/*
+** This is an fts3ExprIterate() callback used while loading the doclists
+** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
+** fts3ExprLoadDoclists().
+*/
+static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
+ int rc = SQLITE_OK;
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
+ LoadDoclistCtx *p = (LoadDoclistCtx *)ctx;
+
+ UNUSED_PARAMETER(iPhrase);
+
+ p->nPhrase++;
+ p->nToken += pPhrase->nToken;
+
+ return rc;
+}
+
+/*
+** Load the doclists for each phrase in the query associated with FTS3 cursor
+** pCsr.
+**
+** If pnPhrase is not NULL, then *pnPhrase is set to the number of matchable
+** phrases in the expression (all phrases except those directly or
+** indirectly descended from the right-hand-side of a NOT operator). If
+** pnToken is not NULL, then it is set to the number of tokens in all
+** matchable phrases of the expression.
+*/
+static int fts3ExprLoadDoclists(
+ Fts3Cursor *pCsr, /* Fts3 cursor for current query */
+ int *pnPhrase, /* OUT: Number of phrases in query */
+ int *pnToken /* OUT: Number of tokens in query */
+){
+ int rc; /* Return Code */
+ LoadDoclistCtx sCtx = {0,0,0}; /* Context for fts3ExprIterate() */
+ sCtx.pCsr = pCsr;
+ rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb, (void *)&sCtx);
+ if( pnPhrase ) *pnPhrase = sCtx.nPhrase;
+ if( pnToken ) *pnToken = sCtx.nToken;
+ return rc;
+}
+
+static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
+ (*(int *)ctx)++;
+ UNUSED_PARAMETER(pExpr);
+ UNUSED_PARAMETER(iPhrase);
+ return SQLITE_OK;
+}
+static int fts3ExprPhraseCount(Fts3Expr *pExpr){
+ int nPhrase = 0;
+ (void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase);
+ return nPhrase;
+}
+
+/*
+** Advance the position list iterator specified by the first two
+** arguments so that it points to the first element with a value greater
+** than or equal to parameter iNext.
+*/
+static void fts3SnippetAdvance(char **ppIter, int *piIter, int iNext){
+ char *pIter = *ppIter;
+ if( pIter ){
+ int iIter = *piIter;
+
+ while( iIter<iNext ){
+ if( 0==(*pIter & 0xFE) ){
+ iIter = -1;
+ pIter = 0;
+ break;
+ }
+ fts3GetDeltaPosition(&pIter, &iIter);
+ }
+
+ *piIter = iIter;
+ *ppIter = pIter;
+ }
+}
+
+/*
+** Advance the snippet iterator to the next candidate snippet.
+*/
+static int fts3SnippetNextCandidate(SnippetIter *pIter){
+ int i; /* Loop counter */
+
+ if( pIter->iCurrent<0 ){
+ /* The SnippetIter object has just been initialized. The first snippet
+ ** candidate always starts at offset 0 (even if this candidate has a
+ ** score of 0.0).
+ */
+ pIter->iCurrent = 0;
+
+ /* Advance the 'head' iterator of each phrase to the first offset that
+ ** is greater than or equal to (iNext+nSnippet).
+ */
+ for(i=0; i<pIter->nPhrase; i++){
+ SnippetPhrase *pPhrase = &pIter->aPhrase[i];
+ fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, pIter->nSnippet);
+ }
+ }else{
+ int iStart;
+ int iEnd = 0x7FFFFFFF;
+
+ for(i=0; i<pIter->nPhrase; i++){
+ SnippetPhrase *pPhrase = &pIter->aPhrase[i];
+ if( pPhrase->pHead && pPhrase->iHead<iEnd ){
+ iEnd = pPhrase->iHead;
+ }
+ }
+ if( iEnd==0x7FFFFFFF ){
+ return 1;
+ }
+
+ pIter->iCurrent = iStart = iEnd - pIter->nSnippet + 1;
+ for(i=0; i<pIter->nPhrase; i++){
+ SnippetPhrase *pPhrase = &pIter->aPhrase[i];
+ fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, iEnd+1);
+ fts3SnippetAdvance(&pPhrase->pTail, &pPhrase->iTail, iStart);
+ }
+ }
+
+ return 0;
+}
+
+/*
+** Retrieve information about the current candidate snippet of snippet
+** iterator pIter.
+*/
+static void fts3SnippetDetails(
+ SnippetIter *pIter, /* Snippet iterator */
+ u64 mCovered, /* Bitmask of phrases already covered */
+ int *piToken, /* OUT: First token of proposed snippet */
+ int *piScore, /* OUT: "Score" for this snippet */
+ u64 *pmCover, /* OUT: Bitmask of phrases covered */
+ u64 *pmHighlight /* OUT: Bitmask of terms to highlight */
+){
+ int iStart = pIter->iCurrent; /* First token of snippet */
+ int iScore = 0; /* Score of this snippet */
+ int i; /* Loop counter */
+ u64 mCover = 0; /* Mask of phrases covered by this snippet */
+ u64 mHighlight = 0; /* Mask of tokens to highlight in snippet */
+
+ for(i=0; i<pIter->nPhrase; i++){
+ SnippetPhrase *pPhrase = &pIter->aPhrase[i];
+ if( pPhrase->pTail ){
+ char *pCsr = pPhrase->pTail;
+ int iCsr = pPhrase->iTail;
+
+ while( iCsr<(iStart+pIter->nSnippet) ){
+ int j;
+ u64 mPhrase = (u64)1 << i;
+ u64 mPos = (u64)1 << (iCsr - iStart);
+ assert( iCsr>=iStart );
+ if( (mCover|mCovered)&mPhrase ){
+ iScore++;
+ }else{
+ iScore += 1000;
+ }
+ mCover |= mPhrase;
+
+ for(j=0; j<pPhrase->nToken; j++){
+ mHighlight |= (mPos>>j);
+ }
+
+ if( 0==(*pCsr & 0x0FE) ) break;
+ fts3GetDeltaPosition(&pCsr, &iCsr);
+ }
+ }
+ }
+
+ /* Set the output variables before returning. */
+ *piToken = iStart;
+ *piScore = iScore;
+ *pmCover = mCover;
+ *pmHighlight = mHighlight;
+}
+
+/*
+** This function is an fts3ExprIterate() callback used by fts3BestSnippet().
+** Each invocation populates an element of the SnippetIter.aPhrase[] array.
+*/
+static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){
+ SnippetIter *p = (SnippetIter *)ctx;
+ SnippetPhrase *pPhrase = &p->aPhrase[iPhrase];
+ char *pCsr;
+
+ pPhrase->nToken = pExpr->pPhrase->nToken;
+
+ pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol);
+ if( pCsr ){
+ int iFirst = 0;
+ pPhrase->pList = pCsr;
+ fts3GetDeltaPosition(&pCsr, &iFirst);
+ pPhrase->pHead = pCsr;
+ pPhrase->pTail = pCsr;
+ pPhrase->iHead = iFirst;
+ pPhrase->iTail = iFirst;
+ }else{
+ assert( pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 );
+ }
+
+ return SQLITE_OK;
+}
+
+/*
+** Select the fragment of text consisting of nFragment contiguous tokens
+** from column iCol that represent the "best" snippet. The best snippet
+** is the snippet with the highest score, where scores are calculated
+** by adding:
+**
+** (a) +1 point for each occurence of a matchable phrase in the snippet.
+**
+** (b) +1000 points for the first occurence of each matchable phrase in
+** the snippet for which the corresponding mCovered bit is not set.
+**
+** The selected snippet parameters are stored in structure *pFragment before
+** returning. The score of the selected snippet is stored in *piScore
+** before returning.
+*/
+static int fts3BestSnippet(
+ int nSnippet, /* Desired snippet length */
+ Fts3Cursor *pCsr, /* Cursor to create snippet for */
+ int iCol, /* Index of column to create snippet from */
+ u64 mCovered, /* Mask of phrases already covered */
+ u64 *pmSeen, /* IN/OUT: Mask of phrases seen */
+ SnippetFragment *pFragment, /* OUT: Best snippet found */
+ int *piScore /* OUT: Score of snippet pFragment */
+){
+ int rc; /* Return Code */
+ int nList; /* Number of phrases in expression */
+ SnippetIter sIter; /* Iterates through snippet candidates */
+ int nByte; /* Number of bytes of space to allocate */
+ int iBestScore = -1; /* Best snippet score found so far */
+ int i; /* Loop counter */
+
+ memset(&sIter, 0, sizeof(sIter));
+
+ /* Iterate through the phrases in the expression to count them. The same
+ ** callback makes sure the doclists are loaded for each phrase.
+ */
+ rc = fts3ExprLoadDoclists(pCsr, &nList, 0);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+
+ /* Now that it is known how many phrases there are, allocate and zero
+ ** the required space using malloc().
+ */
+ nByte = sizeof(SnippetPhrase) * nList;
+ sIter.aPhrase = (SnippetPhrase *)sqlite3_malloc(nByte);
+ if( !sIter.aPhrase ){
+ return SQLITE_NOMEM;
+ }
+ memset(sIter.aPhrase, 0, nByte);
+
+ /* Initialize the contents of the SnippetIter object. Then iterate through
+ ** the set of phrases in the expression to populate the aPhrase[] array.
+ */
+ sIter.pCsr = pCsr;
+ sIter.iCol = iCol;
+ sIter.nSnippet = nSnippet;
+ sIter.nPhrase = nList;
+ sIter.iCurrent = -1;
+ (void)fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void *)&sIter);
+
+ /* Set the *pmSeen output variable. */
+ for(i=0; i<nList; i++){
+ if( sIter.aPhrase[i].pHead ){
+ *pmSeen |= (u64)1 << i;
+ }
+ }
+
+ /* Loop through all candidate snippets. Store the best snippet in
+ ** *pFragment. Store its associated 'score' in iBestScore.
+ */
+ pFragment->iCol = iCol;
+ while( !fts3SnippetNextCandidate(&sIter) ){
+ int iPos;
+ int iScore;
+ u64 mCover;
+ u64 mHighlight;
+ fts3SnippetDetails(&sIter, mCovered, &iPos, &iScore, &mCover, &mHighlight);
+ assert( iScore>=0 );
+ if( iScore>iBestScore ){
+ pFragment->iPos = iPos;
+ pFragment->hlmask = mHighlight;
+ pFragment->covered = mCover;
+ iBestScore = iScore;
+ }
+ }
+
+ sqlite3_free(sIter.aPhrase);
+ *piScore = iBestScore;
+ return SQLITE_OK;
+}
+
+
+/*
+** Append a string to the string-buffer passed as the first argument.
+**
+** If nAppend is negative, then the length of the string zAppend is
+** determined using strlen().
+*/
+static int fts3StringAppend(
+ StrBuffer *pStr, /* Buffer to append to */
+ const char *zAppend, /* Pointer to data to append to buffer */
+ int nAppend /* Size of zAppend in bytes (or -1) */
+){
+ if( nAppend<0 ){
+ nAppend = (int)strlen(zAppend);
+ }
+
+ /* If there is insufficient space allocated at StrBuffer.z, use realloc()
+ ** to grow the buffer until so that it is big enough to accomadate the
+ ** appended data.
+ */
+ if( pStr->n+nAppend+1>=pStr->nAlloc ){
+ int nAlloc = pStr->nAlloc+nAppend+100;
+ char *zNew = sqlite3_realloc(pStr->z, nAlloc);
+ if( !zNew ){
+ return SQLITE_NOMEM;
+ }
+ pStr->z = zNew;
+ pStr->nAlloc = nAlloc;
+ }
+
+ /* Append the data to the string buffer. */
+ memcpy(&pStr->z[pStr->n], zAppend, nAppend);
+ pStr->n += nAppend;
+ pStr->z[pStr->n] = '\0';
+
+ return SQLITE_OK;
+}
+
+/*
+** The fts3BestSnippet() function often selects snippets that end with a
+** query term. That is, the final term of the snippet is always a term
+** that requires highlighting. For example, if 'X' is a highlighted term
+** and '.' is a non-highlighted term, BestSnippet() may select:
+**
+** ........X.....X
+**
+** This function "shifts" the beginning of the snippet forward in the
+** document so that there are approximately the same number of
+** non-highlighted terms to the right of the final highlighted term as there
+** are to the left of the first highlighted term. For example, to this:
+**
+** ....X.....X....
+**
+** This is done as part of extracting the snippet text, not when selecting
+** the snippet. Snippet selection is done based on doclists only, so there
+** is no way for fts3BestSnippet() to know whether or not the document
+** actually contains terms that follow the final highlighted term.
+*/
+static int fts3SnippetShift(
+ Fts3Table *pTab, /* FTS3 table snippet comes from */
+ int nSnippet, /* Number of tokens desired for snippet */
+ const char *zDoc, /* Document text to extract snippet from */
+ int nDoc, /* Size of buffer zDoc in bytes */
+ int *piPos, /* IN/OUT: First token of snippet */
+ u64 *pHlmask /* IN/OUT: Mask of tokens to highlight */
+){
+ u64 hlmask = *pHlmask; /* Local copy of initial highlight-mask */
+
+ if( hlmask ){
+ int nLeft; /* Tokens to the left of first highlight */
+ int nRight; /* Tokens to the right of last highlight */
+ int nDesired; /* Ideal number of tokens to shift forward */
+
+ for(nLeft=0; !(hlmask & ((u64)1 << nLeft)); nLeft++);
+ for(nRight=0; !(hlmask & ((u64)1 << (nSnippet-1-nRight))); nRight++);
+ nDesired = (nLeft-nRight)/2;
+
+ /* Ideally, the start of the snippet should be pushed forward in the
+ ** document nDesired tokens. This block checks if there are actually
+ ** nDesired tokens to the right of the snippet. If so, *piPos and
+ ** *pHlMask are updated to shift the snippet nDesired tokens to the
+ ** right. Otherwise, the snippet is shifted by the number of tokens
+ ** available.
+ */
+ if( nDesired>0 ){
+ int nShift; /* Number of tokens to shift snippet by */
+ int iCurrent = 0; /* Token counter */
+ int rc; /* Return Code */
+ sqlite3_tokenizer_module *pMod;
+ sqlite3_tokenizer_cursor *pC;
+ pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule;
+
+ /* Open a cursor on zDoc/nDoc. Check if there are (nSnippet+nDesired)
+ ** or more tokens in zDoc/nDoc.
+ */
+ rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+ pC->pTokenizer = pTab->pTokenizer;
+ while( rc==SQLITE_OK && iCurrent<(nSnippet+nDesired) ){
+ const char *ZDUMMY; int DUMMY1, DUMMY2, DUMMY3;
+ rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &DUMMY2, &DUMMY3, &iCurrent);
+ }
+ pMod->xClose(pC);
+ if( rc!=SQLITE_OK && rc!=SQLITE_DONE ){ return rc; }
+
+ nShift = (rc==SQLITE_DONE)+iCurrent-nSnippet;
+ assert( nShift<=nDesired );
+ if( nShift>0 ){
+ *piPos += nShift;
+ *pHlmask = hlmask >> nShift;
+ }
+ }
+ }
+ return SQLITE_OK;
+}
+
+/*
+** Extract the snippet text for fragment pFragment from cursor pCsr and
+** append it to string buffer pOut.
+*/
+static int fts3SnippetText(
+ Fts3Cursor *pCsr, /* FTS3 Cursor */
+ SnippetFragment *pFragment, /* Snippet to extract */
+ int iFragment, /* Fragment number */
+ int isLast, /* True for final fragment in snippet */
+ int nSnippet, /* Number of tokens in extracted snippet */
+ const char *zOpen, /* String inserted before highlighted term */
+ const char *zClose, /* String inserted after highlighted term */
+ const char *zEllipsis, /* String inserted between snippets */
+ StrBuffer *pOut /* Write output here */
+){
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+ int rc; /* Return code */
+ const char *zDoc; /* Document text to extract snippet from */
+ int nDoc; /* Size of zDoc in bytes */
+ int iCurrent = 0; /* Current token number of document */
+ int iEnd = 0; /* Byte offset of end of current token */
+ int isShiftDone = 0; /* True after snippet is shifted */
+ int iPos = pFragment->iPos; /* First token of snippet */
+ u64 hlmask = pFragment->hlmask; /* Highlight-mask for snippet */
+ int iCol = pFragment->iCol+1; /* Query column to extract text from */
+ sqlite3_tokenizer_module *pMod; /* Tokenizer module methods object */
+ sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor open on zDoc/nDoc */
+ const char *ZDUMMY; /* Dummy argument used with tokenizer */
+ int DUMMY1; /* Dummy argument used with tokenizer */
+
+ zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol);
+ if( zDoc==0 ){
+ if( sqlite3_column_type(pCsr->pStmt, iCol)!=SQLITE_NULL ){
+ return SQLITE_NOMEM;
+ }
+ return SQLITE_OK;
+ }
+ nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol);
+
+ /* Open a token cursor on the document. */
+ pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule;
+ rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+ pC->pTokenizer = pTab->pTokenizer;
+
+ while( rc==SQLITE_OK ){
+ int iBegin; /* Offset in zDoc of start of token */
+ int iFin; /* Offset in zDoc of end of token */
+ int isHighlight; /* True for highlighted terms */
+
+ rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &iBegin, &iFin, &iCurrent);
+ if( rc!=SQLITE_OK ){
+ if( rc==SQLITE_DONE ){
+ /* Special case - the last token of the snippet is also the last token
+ ** of the column. Append any punctuation that occurred between the end
+ ** of the previous token and the end of the document to the output.
+ ** Then break out of the loop. */
+ rc = fts3StringAppend(pOut, &zDoc[iEnd], -1);
+ }
+ break;
+ }
+ if( iCurrent<iPos ){ continue; }
+
+ if( !isShiftDone ){
+ int n = nDoc - iBegin;
+ rc = fts3SnippetShift(pTab, nSnippet, &zDoc[iBegin], n, &iPos, &hlmask);
+ isShiftDone = 1;
+
+ /* Now that the shift has been done, check if the initial "..." are
+ ** required. They are required if (a) this is not the first fragment,
+ ** or (b) this fragment does not begin at position 0 of its column.
+ */
+ if( rc==SQLITE_OK && (iPos>0 || iFragment>0) ){
+ rc = fts3StringAppend(pOut, zEllipsis, -1);
+ }
+ if( rc!=SQLITE_OK || iCurrent<iPos ) continue;
+ }
+
+ if( iCurrent>=(iPos+nSnippet) ){
+ if( isLast ){
+ rc = fts3StringAppend(pOut, zEllipsis, -1);
+ }
+ break;
+ }
+
+ /* Set isHighlight to true if this term should be highlighted. */
+ isHighlight = (hlmask & ((u64)1 << (iCurrent-iPos)))!=0;
+
+ if( iCurrent>iPos ) rc = fts3StringAppend(pOut, &zDoc[iEnd], iBegin-iEnd);
+ if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zOpen, -1);
+ if( rc==SQLITE_OK ) rc = fts3StringAppend(pOut, &zDoc[iBegin], iFin-iBegin);
+ if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zClose, -1);
+
+ iEnd = iFin;
+ }
+
+ pMod->xClose(pC);
+ return rc;
+}
+
+
+/*
+** This function is used to count the entries in a column-list (a
+** delta-encoded list of term offsets within a single column of a single
+** row). When this function is called, *ppCollist should point to the
+** beginning of the first varint in the column-list (the varint that
+** contains the position of the first matching term in the column data).
+** Before returning, *ppCollist is set to point to the first byte after
+** the last varint in the column-list (either the 0x00 signifying the end
+** of the position-list, or the 0x01 that precedes the column number of
+** the next column in the position-list).
+**
+** The number of elements in the column-list is returned.
+*/
+static int fts3ColumnlistCount(char **ppCollist){
+ char *pEnd = *ppCollist;
+ char c = 0;
+ int nEntry = 0;
+
+ /* A column-list is terminated by either a 0x01 or 0x00. */
+ while( 0xFE & (*pEnd | c) ){
+ c = *pEnd++ & 0x80;
+ if( !c ) nEntry++;
+ }
+
+ *ppCollist = pEnd;
+ return nEntry;
+}
+
+/*
+** fts3ExprIterate() callback used to collect the "global" matchinfo stats
+** for a single query.
+**
+** fts3ExprIterate() callback to load the 'global' elements of a
+** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements
+** of the matchinfo array that are constant for all rows returned by the
+** current query.
+**
+** Argument pCtx is actually a pointer to a struct of type MatchInfo. This
+** function populates Matchinfo.aMatchinfo[] as follows:
+**
+** for(iCol=0; iCol<nCol; iCol++){
+** aMatchinfo[3*iPhrase*nCol + 3*iCol + 1] = X;
+** aMatchinfo[3*iPhrase*nCol + 3*iCol + 2] = Y;
+** }
+**
+** where X is the number of matches for phrase iPhrase is column iCol of all
+** rows of the table. Y is the number of rows for which column iCol contains
+** at least one instance of phrase iPhrase.
+**
+** If the phrase pExpr consists entirely of deferred tokens, then all X and
+** Y values are set to nDoc, where nDoc is the number of documents in the
+** file system. This is done because the full-text index doclist is required
+** to calculate these values properly, and the full-text index doclist is
+** not available for deferred tokens.
+*/
+static int fts3ExprGlobalHitsCb(
+ Fts3Expr *pExpr, /* Phrase expression node */
+ int iPhrase, /* Phrase number (numbered from zero) */
+ void *pCtx /* Pointer to MatchInfo structure */
+){
+ MatchInfo *p = (MatchInfo *)pCtx;
+ return sqlite3Fts3EvalPhraseStats(
+ p->pCursor, pExpr, &p->aMatchinfo[3*iPhrase*p->nCol]
+ );
+}
+
+/*
+** fts3ExprIterate() callback used to collect the "local" part of the
+** FTS3_MATCHINFO_HITS array. The local stats are those elements of the
+** array that are different for each row returned by the query.
+*/
+static int fts3ExprLocalHitsCb(
+ Fts3Expr *pExpr, /* Phrase expression node */
+ int iPhrase, /* Phrase number */
+ void *pCtx /* Pointer to MatchInfo structure */
+){
+ MatchInfo *p = (MatchInfo *)pCtx;
+ int iStart = iPhrase * p->nCol * 3;
+ int i;
+
+ for(i=0; i<p->nCol; i++){
+ char *pCsr;
+ pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i);
+ if( pCsr ){
+ p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr);
+ }else{
+ p->aMatchinfo[iStart+i*3] = 0;
+ }
+ }
+
+ return SQLITE_OK;
+}
+
+static int fts3MatchinfoCheck(
+ Fts3Table *pTab,
+ char cArg,
+ char **pzErr
+){
+ if( (cArg==FTS3_MATCHINFO_NPHRASE)
+ || (cArg==FTS3_MATCHINFO_NCOL)
+ || (cArg==FTS3_MATCHINFO_NDOC && pTab->bHasStat)
+ || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bHasStat)
+ || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize)
+ || (cArg==FTS3_MATCHINFO_LCS)
+ || (cArg==FTS3_MATCHINFO_HITS)
+ ){
+ return SQLITE_OK;
+ }
+ *pzErr = sqlite3_mprintf("unrecognized matchinfo request: %c", cArg);
+ return SQLITE_ERROR;
+}
+
+static int fts3MatchinfoSize(MatchInfo *pInfo, char cArg){
+ int nVal; /* Number of integers output by cArg */
+
+ switch( cArg ){
+ case FTS3_MATCHINFO_NDOC:
+ case FTS3_MATCHINFO_NPHRASE:
+ case FTS3_MATCHINFO_NCOL:
+ nVal = 1;
+ break;
+
+ case FTS3_MATCHINFO_AVGLENGTH:
+ case FTS3_MATCHINFO_LENGTH:
+ case FTS3_MATCHINFO_LCS:
+ nVal = pInfo->nCol;
+ break;
+
+ default:
+ assert( cArg==FTS3_MATCHINFO_HITS );
+ nVal = pInfo->nCol * pInfo->nPhrase * 3;
+ break;
+ }
+
+ return nVal;
+}
+
+static int fts3MatchinfoSelectDoctotal(
+ Fts3Table *pTab,
+ sqlite3_stmt **ppStmt,
+ sqlite3_int64 *pnDoc,
+ const char **paLen
+){
+ sqlite3_stmt *pStmt;
+ const char *a;
+ sqlite3_int64 nDoc;
+
+ if( !*ppStmt ){
+ int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt);
+ if( rc!=SQLITE_OK ) return rc;
+ }
+ pStmt = *ppStmt;
+ assert( sqlite3_data_count(pStmt)==1 );
+
+ a = sqlite3_column_blob(pStmt, 0);
+ a += sqlite3Fts3GetVarint(a, &nDoc);
+ if( nDoc==0 ) return SQLITE_CORRUPT_VTAB;
+ *pnDoc = (u32)nDoc;
+
+ if( paLen ) *paLen = a;
+ return SQLITE_OK;
+}
+
+/*
+** An instance of the following structure is used to store state while
+** iterating through a multi-column position-list corresponding to the
+** hits for a single phrase on a single row in order to calculate the
+** values for a matchinfo() FTS3_MATCHINFO_LCS request.
+*/
+typedef struct LcsIterator LcsIterator;
+struct LcsIterator {
+ Fts3Expr *pExpr; /* Pointer to phrase expression */
+ int iPosOffset; /* Tokens count up to end of this phrase */
+ char *pRead; /* Cursor used to iterate through aDoclist */
+ int iPos; /* Current position */
+};
+
+/*
+** If LcsIterator.iCol is set to the following value, the iterator has
+** finished iterating through all offsets for all columns.
+*/
+#define LCS_ITERATOR_FINISHED 0x7FFFFFFF;
+
+static int fts3MatchinfoLcsCb(
+ Fts3Expr *pExpr, /* Phrase expression node */
+ int iPhrase, /* Phrase number (numbered from zero) */
+ void *pCtx /* Pointer to MatchInfo structure */
+){
+ LcsIterator *aIter = (LcsIterator *)pCtx;
+ aIter[iPhrase].pExpr = pExpr;
+ return SQLITE_OK;
+}
+
+/*
+** Advance the iterator passed as an argument to the next position. Return
+** 1 if the iterator is at EOF or if it now points to the start of the
+** position list for the next column.
+*/
+static int fts3LcsIteratorAdvance(LcsIterator *pIter){
+ char *pRead = pIter->pRead;
+ sqlite3_int64 iRead;
+ int rc = 0;
+
+ pRead += sqlite3Fts3GetVarint(pRead, &iRead);
+ if( iRead==0 || iRead==1 ){
+ pRead = 0;
+ rc = 1;
+ }else{
+ pIter->iPos += (int)(iRead-2);
+ }
+
+ pIter->pRead = pRead;
+ return rc;
+}
+
+/*
+** This function implements the FTS3_MATCHINFO_LCS matchinfo() flag.
+**
+** If the call is successful, the longest-common-substring lengths for each
+** column are written into the first nCol elements of the pInfo->aMatchinfo[]
+** array before returning. SQLITE_OK is returned in this case.
+**
+** Otherwise, if an error occurs, an SQLite error code is returned and the
+** data written to the first nCol elements of pInfo->aMatchinfo[] is
+** undefined.
+*/
+static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){
+ LcsIterator *aIter;
+ int i;
+ int iCol;
+ int nToken = 0;
+
+ /* Allocate and populate the array of LcsIterator objects. The array
+ ** contains one element for each matchable phrase in the query.
+ **/
+ aIter = sqlite3_malloc(sizeof(LcsIterator) * pCsr->nPhrase);
+ if( !aIter ) return SQLITE_NOMEM;
+ memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase);
+ (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter);
+
+ for(i=0; i<pInfo->nPhrase; i++){
+ LcsIterator *pIter = &aIter[i];
+ nToken -= pIter->pExpr->pPhrase->nToken;
+ pIter->iPosOffset = nToken;
+ }
+
+ for(iCol=0; iCol<pInfo->nCol; iCol++){
+ int nLcs = 0; /* LCS value for this column */
+ int nLive = 0; /* Number of iterators in aIter not at EOF */
+
+ for(i=0; i<pInfo->nPhrase; i++){
+ LcsIterator *pIt = &aIter[i];
+ pIt->pRead = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol);
+ if( pIt->pRead ){
+ pIt->iPos = pIt->iPosOffset;
+ fts3LcsIteratorAdvance(&aIter[i]);
+ nLive++;
+ }
+ }
+
+ while( nLive>0 ){
+ LcsIterator *pAdv = 0; /* The iterator to advance by one position */
+ int nThisLcs = 0; /* LCS for the current iterator positions */
+
+ for(i=0; i<pInfo->nPhrase; i++){
+ LcsIterator *pIter = &aIter[i];
+ if( pIter->pRead==0 ){
+ /* This iterator is already at EOF for this column. */
+ nThisLcs = 0;
+ }else{
+ if( pAdv==0 || pIter->iPos<pAdv->iPos ){
+ pAdv = pIter;
+ }
+ if( nThisLcs==0 || pIter->iPos==pIter[-1].iPos ){
+ nThisLcs++;
+ }else{
+ nThisLcs = 1;
+ }
+ if( nThisLcs>nLcs ) nLcs = nThisLcs;
+ }
+ }
+ if( fts3LcsIteratorAdvance(pAdv) ) nLive--;
+ }
+
+ pInfo->aMatchinfo[iCol] = nLcs;
+ }
+
+ sqlite3_free(aIter);
+ return SQLITE_OK;
+}
+
+/*
+** Populate the buffer pInfo->aMatchinfo[] with an array of integers to
+** be returned by the matchinfo() function. Argument zArg contains the
+** format string passed as the second argument to matchinfo (or the
+** default value "pcx" if no second argument was specified). The format
+** string has already been validated and the pInfo->aMatchinfo[] array
+** is guaranteed to be large enough for the output.
+**
+** If bGlobal is true, then populate all fields of the matchinfo() output.
+** If it is false, then assume that those fields that do not change between
+** rows (i.e. FTS3_MATCHINFO_NPHRASE, NCOL, NDOC, AVGLENGTH and part of HITS)
+** have already been populated.
+**
+** Return SQLITE_OK if successful, or an SQLite error code if an error
+** occurs. If a value other than SQLITE_OK is returned, the state the
+** pInfo->aMatchinfo[] buffer is left in is undefined.
+*/
+static int fts3MatchinfoValues(
+ Fts3Cursor *pCsr, /* FTS3 cursor object */
+ int bGlobal, /* True to grab the global stats */
+ MatchInfo *pInfo, /* Matchinfo context object */
+ const char *zArg /* Matchinfo format string */
+){
+ int rc = SQLITE_OK;
+ int i;
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+ sqlite3_stmt *pSelect = 0;
+
+ for(i=0; rc==SQLITE_OK && zArg[i]; i++){
+
+ switch( zArg[i] ){
+ case FTS3_MATCHINFO_NPHRASE:
+ if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase;
+ break;
+
+ case FTS3_MATCHINFO_NCOL:
+ if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nCol;
+ break;
+
+ case FTS3_MATCHINFO_NDOC:
+ if( bGlobal ){
+ sqlite3_int64 nDoc = 0;
+ rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0);
+ pInfo->aMatchinfo[0] = (u32)nDoc;
+ }
+ break;
+
+ case FTS3_MATCHINFO_AVGLENGTH:
+ if( bGlobal ){
+ sqlite3_int64 nDoc; /* Number of rows in table */
+ const char *a; /* Aggregate column length array */
+
+ rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a);
+ if( rc==SQLITE_OK ){
+ int iCol;
+ for(iCol=0; iCol<pInfo->nCol; iCol++){
+ u32 iVal;
+ sqlite3_int64 nToken;
+ a += sqlite3Fts3GetVarint(a, &nToken);
+ iVal = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc);
+ pInfo->aMatchinfo[iCol] = iVal;
+ }
+ }
+ }
+ break;
+
+ case FTS3_MATCHINFO_LENGTH: {
+ sqlite3_stmt *pSelectDocsize = 0;
+ rc = sqlite3Fts3SelectDocsize(pTab, pCsr->iPrevId, &pSelectDocsize);
+ if( rc==SQLITE_OK ){
+ int iCol;
+ const char *a = sqlite3_column_blob(pSelectDocsize, 0);
+ for(iCol=0; iCol<pInfo->nCol; iCol++){
+ sqlite3_int64 nToken;
+ a += sqlite3Fts3GetVarint(a, &nToken);
+ pInfo->aMatchinfo[iCol] = (u32)nToken;
+ }
+ }
+ sqlite3_reset(pSelectDocsize);
+ break;
+ }
+
+ case FTS3_MATCHINFO_LCS:
+ rc = fts3ExprLoadDoclists(pCsr, 0, 0);
+ if( rc==SQLITE_OK ){
+ rc = fts3MatchinfoLcs(pCsr, pInfo);
+ }
+ break;
+
+ default: {
+ Fts3Expr *pExpr;
+ assert( zArg[i]==FTS3_MATCHINFO_HITS );
+ pExpr = pCsr->pExpr;
+ rc = fts3ExprLoadDoclists(pCsr, 0, 0);
+ if( rc!=SQLITE_OK ) break;
+ if( bGlobal ){
+ if( pCsr->pDeferred ){
+ rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc, 0);
+ if( rc!=SQLITE_OK ) break;
+ }
+ rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo);
+ if( rc!=SQLITE_OK ) break;
+ }
+ (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo);
+ break;
+ }
+ }
+
+ pInfo->aMatchinfo += fts3MatchinfoSize(pInfo, zArg[i]);
+ }
+
+ sqlite3_reset(pSelect);
+ return rc;
+}
+
+
+/*
+** Populate pCsr->aMatchinfo[] with data for the current row. The
+** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32).
+*/
+static int fts3GetMatchinfo(
+ Fts3Cursor *pCsr, /* FTS3 Cursor object */
+ const char *zArg /* Second argument to matchinfo() function */
+){
+ MatchInfo sInfo;
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+ int rc = SQLITE_OK;
+ int bGlobal = 0; /* Collect 'global' stats as well as local */
+
+ memset(&sInfo, 0, sizeof(MatchInfo));
+ sInfo.pCursor = pCsr;
+ sInfo.nCol = pTab->nColumn;
+
+ /* If there is cached matchinfo() data, but the format string for the
+ ** cache does not match the format string for this request, discard
+ ** the cached data. */
+ if( pCsr->zMatchinfo && strcmp(pCsr->zMatchinfo, zArg) ){
+ assert( pCsr->aMatchinfo );
+ sqlite3_free(pCsr->aMatchinfo);
+ pCsr->zMatchinfo = 0;
+ pCsr->aMatchinfo = 0;
+ }
+
+ /* If Fts3Cursor.aMatchinfo[] is NULL, then this is the first time the
+ ** matchinfo function has been called for this query. In this case
+ ** allocate the array used to accumulate the matchinfo data and
+ ** initialize those elements that are constant for every row.
+ */
+ if( pCsr->aMatchinfo==0 ){
+ int nMatchinfo = 0; /* Number of u32 elements in match-info */
+ int nArg; /* Bytes in zArg */
+ int i; /* Used to iterate through zArg */
+
+ /* Determine the number of phrases in the query */
+ pCsr->nPhrase = fts3ExprPhraseCount(pCsr->pExpr);
+ sInfo.nPhrase = pCsr->nPhrase;
+
+ /* Determine the number of integers in the buffer returned by this call. */
+ for(i=0; zArg[i]; i++){
+ nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]);
+ }
+
+ /* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */
+ nArg = (int)strlen(zArg);
+ pCsr->aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo + nArg + 1);
+ if( !pCsr->aMatchinfo ) return SQLITE_NOMEM;
+
+ pCsr->zMatchinfo = (char *)&pCsr->aMatchinfo[nMatchinfo];
+ pCsr->nMatchinfo = nMatchinfo;
+ memcpy(pCsr->zMatchinfo, zArg, nArg+1);
+ memset(pCsr->aMatchinfo, 0, sizeof(u32)*nMatchinfo);
+ pCsr->isMatchinfoNeeded = 1;
+ bGlobal = 1;
+ }
+
+ sInfo.aMatchinfo = pCsr->aMatchinfo;
+ sInfo.nPhrase = pCsr->nPhrase;
+ if( pCsr->isMatchinfoNeeded ){
+ rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg);
+ pCsr->isMatchinfoNeeded = 0;
+ }
+
+ return rc;
+}
+
+/*
+** Implementation of snippet() function.
+*/
+void sqlite3Fts3Snippet(
+ sqlite3_context *pCtx, /* SQLite function call context */
+ Fts3Cursor *pCsr, /* Cursor object */
+ const char *zStart, /* Snippet start text - "<b>" */
+ const char *zEnd, /* Snippet end text - "</b>" */
+ const char *zEllipsis, /* Snippet ellipsis text - "<b>...</b>" */
+ int iCol, /* Extract snippet from this column */
+ int nToken /* Approximate number of tokens in snippet */
+){
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+ int rc = SQLITE_OK;
+ int i;
+ StrBuffer res = {0, 0, 0};
+
+ /* The returned text includes up to four fragments of text extracted from
+ ** the data in the current row. The first iteration of the for(...) loop
+ ** below attempts to locate a single fragment of text nToken tokens in
+ ** size that contains at least one instance of all phrases in the query
+ ** expression that appear in the current row. If such a fragment of text
+ ** cannot be found, the second iteration of the loop attempts to locate
+ ** a pair of fragments, and so on.
+ */
+ int nSnippet = 0; /* Number of fragments in this snippet */
+ SnippetFragment aSnippet[4]; /* Maximum of 4 fragments per snippet */
+ int nFToken = -1; /* Number of tokens in each fragment */
+
+ if( !pCsr->pExpr ){
+ sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC);
+ return;
+ }
+
+ for(nSnippet=1; 1; nSnippet++){
+
+ int iSnip; /* Loop counter 0..nSnippet-1 */
+ u64 mCovered = 0; /* Bitmask of phrases covered by snippet */
+ u64 mSeen = 0; /* Bitmask of phrases seen by BestSnippet() */
+
+ if( nToken>=0 ){
+ nFToken = (nToken+nSnippet-1) / nSnippet;
+ }else{
+ nFToken = -1 * nToken;
+ }
+
+ for(iSnip=0; iSnip<nSnippet; iSnip++){
+ int iBestScore = -1; /* Best score of columns checked so far */
+ int iRead; /* Used to iterate through columns */
+ SnippetFragment *pFragment = &aSnippet[iSnip];
+
+ memset(pFragment, 0, sizeof(*pFragment));
+
+ /* Loop through all columns of the table being considered for snippets.
+ ** If the iCol argument to this function was negative, this means all
+ ** columns of the FTS3 table. Otherwise, only column iCol is considered.
+ */
+ for(iRead=0; iRead<pTab->nColumn; iRead++){
+ SnippetFragment sF = {0, 0, 0, 0};
+ int iS;
+ if( iCol>=0 && iRead!=iCol ) continue;
+
+ /* Find the best snippet of nFToken tokens in column iRead. */
+ rc = fts3BestSnippet(nFToken, pCsr, iRead, mCovered, &mSeen, &sF, &iS);
+ if( rc!=SQLITE_OK ){
+ goto snippet_out;
+ }
+ if( iS>iBestScore ){
+ *pFragment = sF;
+ iBestScore = iS;
+ }
+ }
+
+ mCovered |= pFragment->covered;
+ }
+
+ /* If all query phrases seen by fts3BestSnippet() are present in at least
+ ** one of the nSnippet snippet fragments, break out of the loop.
+ */
+ assert( (mCovered&mSeen)==mCovered );
+ if( mSeen==mCovered || nSnippet==SizeofArray(aSnippet) ) break;
+ }
+
+ assert( nFToken>0 );
+
+ for(i=0; i<nSnippet && rc==SQLITE_OK; i++){
+ rc = fts3SnippetText(pCsr, &aSnippet[i],
+ i, (i==nSnippet-1), nFToken, zStart, zEnd, zEllipsis, &res
+ );
+ }
+
+ snippet_out:
+ sqlite3Fts3SegmentsClose(pTab);
+ if( rc!=SQLITE_OK ){
+ sqlite3_result_error_code(pCtx, rc);
+ sqlite3_free(res.z);
+ }else{
+ sqlite3_result_text(pCtx, res.z, -1, sqlite3_free);
+ }
+}
+
+
+typedef struct TermOffset TermOffset;
+typedef struct TermOffsetCtx TermOffsetCtx;
+
+struct TermOffset {
+ char *pList; /* Position-list */
+ int iPos; /* Position just read from pList */
+ int iOff; /* Offset of this term from read positions */
+};
+
+struct TermOffsetCtx {
+ Fts3Cursor *pCsr;
+ int iCol; /* Column of table to populate aTerm for */
+ int iTerm;
+ sqlite3_int64 iDocid;
+ TermOffset *aTerm;
+};
+
+/*
+** This function is an fts3ExprIterate() callback used by sqlite3Fts3Offsets().
+*/
+static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){
+ TermOffsetCtx *p = (TermOffsetCtx *)ctx;
+ int nTerm; /* Number of tokens in phrase */
+ int iTerm; /* For looping through nTerm phrase terms */
+ char *pList; /* Pointer to position list for phrase */
+ int iPos = 0; /* First position in position-list */
+
+ UNUSED_PARAMETER(iPhrase);
+ pList = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol);
+ nTerm = pExpr->pPhrase->nToken;
+ if( pList ){
+ fts3GetDeltaPosition(&pList, &iPos);
+ assert( iPos>=0 );
+ }
+
+ for(iTerm=0; iTerm<nTerm; iTerm++){
+ TermOffset *pT = &p->aTerm[p->iTerm++];
+ pT->iOff = nTerm-iTerm-1;
+ pT->pList = pList;
+ pT->iPos = iPos;
+ }
+
+ return SQLITE_OK;
+}
+
+/*
+** Implementation of offsets() function.
+*/
+void sqlite3Fts3Offsets(
+ sqlite3_context *pCtx, /* SQLite function call context */
+ Fts3Cursor *pCsr /* Cursor object */
+){
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+ sqlite3_tokenizer_module const *pMod = pTab->pTokenizer->pModule;
+ const char *ZDUMMY; /* Dummy argument used with xNext() */
+ int NDUMMY; /* Dummy argument used with xNext() */
+ int rc; /* Return Code */
+ int nToken; /* Number of tokens in query */
+ int iCol; /* Column currently being processed */
+ StrBuffer res = {0, 0, 0}; /* Result string */
+ TermOffsetCtx sCtx; /* Context for fts3ExprTermOffsetInit() */
+
+ if( !pCsr->pExpr ){
+ sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC);
+ return;
+ }
+
+ memset(&sCtx, 0, sizeof(sCtx));
+ assert( pCsr->isRequireSeek==0 );
+
+ /* Count the number of terms in the query */
+ rc = fts3ExprLoadDoclists(pCsr, 0, &nToken);
+ if( rc!=SQLITE_OK ) goto offsets_out;
+
+ /* Allocate the array of TermOffset iterators. */
+ sCtx.aTerm = (TermOffset *)sqlite3_malloc(sizeof(TermOffset)*nToken);
+ if( 0==sCtx.aTerm ){
+ rc = SQLITE_NOMEM;
+ goto offsets_out;
+ }
+ sCtx.iDocid = pCsr->iPrevId;
+ sCtx.pCsr = pCsr;
+
+ /* Loop through the table columns, appending offset information to
+ ** string-buffer res for each column.
+ */
+ for(iCol=0; iCol<pTab->nColumn; iCol++){
+ sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor */
+ int iStart;
+ int iEnd;
+ int iCurrent;
+ const char *zDoc;
+ int nDoc;
+
+ /* Initialize the contents of sCtx.aTerm[] for column iCol. There is
+ ** no way that this operation can fail, so the return code from
+ ** fts3ExprIterate() can be discarded.
+ */
+ sCtx.iCol = iCol;
+ sCtx.iTerm = 0;
+ (void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void *)&sCtx);
+
+ /* Retreive the text stored in column iCol. If an SQL NULL is stored
+ ** in column iCol, jump immediately to the next iteration of the loop.
+ ** If an OOM occurs while retrieving the data (this can happen if SQLite
+ ** needs to transform the data from utf-16 to utf-8), return SQLITE_NOMEM
+ ** to the caller.
+ */
+ zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol+1);
+ nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol+1);
+ if( zDoc==0 ){
+ if( sqlite3_column_type(pCsr->pStmt, iCol+1)==SQLITE_NULL ){
+ continue;
+ }
+ rc = SQLITE_NOMEM;
+ goto offsets_out;
+ }
+
+ /* Initialize a tokenizer iterator to iterate through column iCol. */
+ rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC);
+ if( rc!=SQLITE_OK ) goto offsets_out;
+ pC->pTokenizer = pTab->pTokenizer;
+
+ rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent);
+ while( rc==SQLITE_OK ){
+ int i; /* Used to loop through terms */
+ int iMinPos = 0x7FFFFFFF; /* Position of next token */
+ TermOffset *pTerm = 0; /* TermOffset associated with next token */
+
+ for(i=0; i<nToken; i++){
+ TermOffset *pT = &sCtx.aTerm[i];
+ if( pT->pList && (pT->iPos-pT->iOff)<iMinPos ){
+ iMinPos = pT->iPos-pT->iOff;
+ pTerm = pT;
+ }
+ }
+
+ if( !pTerm ){
+ /* All offsets for this column have been gathered. */
+ break;
+ }else{
+ assert( iCurrent<=iMinPos );
+ if( 0==(0xFE&*pTerm->pList) ){
+ pTerm->pList = 0;
+ }else{
+ fts3GetDeltaPosition(&pTerm->pList, &pTerm->iPos);
+ }
+ while( rc==SQLITE_OK && iCurrent<iMinPos ){
+ rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent);
+ }
+ if( rc==SQLITE_OK ){
+ char aBuffer[64];
+ sqlite3_snprintf(sizeof(aBuffer), aBuffer,
+ "%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart
+ );
+ rc = fts3StringAppend(&res, aBuffer, -1);
+ }else if( rc==SQLITE_DONE ){
+ rc = SQLITE_CORRUPT_VTAB;
+ }
+ }
+ }
+ if( rc==SQLITE_DONE ){
+ rc = SQLITE_OK;
+ }
+
+ pMod->xClose(pC);
+ if( rc!=SQLITE_OK ) goto offsets_out;
+ }
+
+ offsets_out:
+ sqlite3_free(sCtx.aTerm);
+ assert( rc!=SQLITE_DONE );
+ sqlite3Fts3SegmentsClose(pTab);
+ if( rc!=SQLITE_OK ){
+ sqlite3_result_error_code(pCtx, rc);
+ sqlite3_free(res.z);
+ }else{
+ sqlite3_result_text(pCtx, res.z, res.n-1, sqlite3_free);
+ }
+ return;
+}
+
+/*
+** Implementation of matchinfo() function.
+*/
+void sqlite3Fts3Matchinfo(
+ sqlite3_context *pContext, /* Function call context */
+ Fts3Cursor *pCsr, /* FTS3 table cursor */
+ const char *zArg /* Second arg to matchinfo() function */
+){
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+ int rc;
+ int i;
+ const char *zFormat;
+
+ if( zArg ){
+ for(i=0; zArg[i]; i++){
+ char *zErr = 0;
+ if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){
+ sqlite3_result_error(pContext, zErr, -1);
+ sqlite3_free(zErr);
+ return;
+ }
+ }
+ zFormat = zArg;
+ }else{
+ zFormat = FTS3_MATCHINFO_DEFAULT;
+ }
+
+ if( !pCsr->pExpr ){
+ sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC);
+ return;
+ }
+
+ /* Retrieve matchinfo() data. */
+ rc = fts3GetMatchinfo(pCsr, zFormat);
+ sqlite3Fts3SegmentsClose(pTab);
+
+ if( rc!=SQLITE_OK ){
+ sqlite3_result_error_code(pContext, rc);
+ }else{
+ int n = pCsr->nMatchinfo * sizeof(u32);
+ sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT);
+ }
+}
+
+#endif
diff --git a/src/libtracker-fts/fts3_term.c b/src/libtracker-fts/fts3_term.c
new file mode 100644
index 0000000..d3eb690
--- /dev/null
+++ b/src/libtracker-fts/fts3_term.c
@@ -0,0 +1,369 @@
+/*
+** 2011 Jan 27
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file is not part of the production FTS code. It is only used for
+** testing. It contains a virtual table implementation that provides direct
+** access to the full-text index of an FTS table.
+*/
+
+#include "fts3Int.h"
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+#ifdef SQLITE_TEST
+
+#include <string.h>
+#include <assert.h>
+#include <stdlib.h>
+
+typedef struct Fts3termTable Fts3termTable;
+typedef struct Fts3termCursor Fts3termCursor;
+
+struct Fts3termTable {
+ sqlite3_vtab base; /* Base class used by SQLite core */
+ int iIndex; /* Index for Fts3Table.aIndex[] */
+ Fts3Table *pFts3Tab;
+};
+
+struct Fts3termCursor {
+ sqlite3_vtab_cursor base; /* Base class used by SQLite core */
+ Fts3MultiSegReader csr; /* Must be right after "base" */
+ Fts3SegFilter filter;
+
+ int isEof; /* True if cursor is at EOF */
+ char *pNext;
+
+ sqlite3_int64 iRowid; /* Current 'rowid' value */
+ sqlite3_int64 iDocid; /* Current 'docid' value */
+ int iCol; /* Current 'col' value */
+ int iPos; /* Current 'pos' value */
+};
+
+/*
+** Schema of the terms table.
+*/
+#define FTS3_TERMS_SCHEMA "CREATE TABLE x(term, docid, col, pos)"
+
+/*
+** This function does all the work for both the xConnect and xCreate methods.
+** These tables have no persistent representation of their own, so xConnect
+** and xCreate are identical operations.
+*/
+static int fts3termConnectMethod(
+ sqlite3 *db, /* Database connection */
+ void *pCtx, /* Non-zero for an fts4prefix table */
+ int argc, /* Number of elements in argv array */
+ const char * const *argv, /* xCreate/xConnect argument array */
+ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
+ char **pzErr /* OUT: sqlite3_malloc'd error message */
+){
+ char const *zDb; /* Name of database (e.g. "main") */
+ char const *zFts3; /* Name of fts3 table */
+ int nDb; /* Result of strlen(zDb) */
+ int nFts3; /* Result of strlen(zFts3) */
+ int nByte; /* Bytes of space to allocate here */
+ int rc; /* value returned by declare_vtab() */
+ Fts3termTable *p; /* Virtual table object to return */
+ int iIndex = 0;
+
+ if( argc==5 ){
+ iIndex = atoi(argv[4]);
+ argc--;
+ }
+
+ /* The user should specify a single argument - the name of an fts3 table. */
+ if( argc!=4 ){
+ *pzErr = sqlite3_mprintf(
+ "wrong number of arguments to fts4term constructor"
+ );
+ return SQLITE_ERROR;
+ }
+
+ zDb = argv[1];
+ nDb = strlen(zDb);
+ zFts3 = argv[3];
+ nFts3 = strlen(zFts3);
+
+ rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA);
+ if( rc!=SQLITE_OK ) return rc;
+
+ nByte = sizeof(Fts3termTable) + sizeof(Fts3Table) + nDb + nFts3 + 2;
+ p = (Fts3termTable *)sqlite3_malloc(nByte);
+ if( !p ) return SQLITE_NOMEM;
+ memset(p, 0, nByte);
+
+ p->pFts3Tab = (Fts3Table *)&p[1];
+ p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1];
+ p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1];
+ p->pFts3Tab->db = db;
+ p->pFts3Tab->nIndex = iIndex+1;
+ p->iIndex = iIndex;
+
+ memcpy((char *)p->pFts3Tab->zDb, zDb, nDb);
+ memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3);
+ sqlite3Fts3Dequote((char *)p->pFts3Tab->zName);
+
+ *ppVtab = (sqlite3_vtab *)p;
+ return SQLITE_OK;
+}
+
+/*
+** This function does the work for both the xDisconnect and xDestroy methods.
+** These tables have no persistent representation of their own, so xDisconnect
+** and xDestroy are identical operations.
+*/
+static int fts3termDisconnectMethod(sqlite3_vtab *pVtab){
+ Fts3termTable *p = (Fts3termTable *)pVtab;
+ Fts3Table *pFts3 = p->pFts3Tab;
+ int i;
+
+ /* Free any prepared statements held */
+ for(i=0; i<SizeofArray(pFts3->aStmt); i++){
+ sqlite3_finalize(pFts3->aStmt[i]);
+ }
+ sqlite3_free(pFts3->zSegmentsTbl);
+ sqlite3_free(p);
+ return SQLITE_OK;
+}
+
+#define FTS4AUX_EQ_CONSTRAINT 1
+#define FTS4AUX_GE_CONSTRAINT 2
+#define FTS4AUX_LE_CONSTRAINT 4
+
+/*
+** xBestIndex - Analyze a WHERE and ORDER BY clause.
+*/
+static int fts3termBestIndexMethod(
+ sqlite3_vtab *pVTab,
+ sqlite3_index_info *pInfo
+){
+ UNUSED_PARAMETER(pVTab);
+
+ /* This vtab naturally does "ORDER BY term, docid, col, pos". */
+ if( pInfo->nOrderBy ){
+ int i;
+ for(i=0; i<pInfo->nOrderBy; i++){
+ if( pInfo->aOrderBy[i].iColumn!=i || pInfo->aOrderBy[i].desc ) break;
+ }
+ if( i==pInfo->nOrderBy ){
+ pInfo->orderByConsumed = 1;
+ }
+ }
+
+ return SQLITE_OK;
+}
+
+/*
+** xOpen - Open a cursor.
+*/
+static int fts3termOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
+ Fts3termCursor *pCsr; /* Pointer to cursor object to return */
+
+ UNUSED_PARAMETER(pVTab);
+
+ pCsr = (Fts3termCursor *)sqlite3_malloc(sizeof(Fts3termCursor));
+ if( !pCsr ) return SQLITE_NOMEM;
+ memset(pCsr, 0, sizeof(Fts3termCursor));
+
+ *ppCsr = (sqlite3_vtab_cursor *)pCsr;
+ return SQLITE_OK;
+}
+
+/*
+** xClose - Close a cursor.
+*/
+static int fts3termCloseMethod(sqlite3_vtab_cursor *pCursor){
+ Fts3Table *pFts3 = ((Fts3termTable *)pCursor->pVtab)->pFts3Tab;
+ Fts3termCursor *pCsr = (Fts3termCursor *)pCursor;
+
+ sqlite3Fts3SegmentsClose(pFts3);
+ sqlite3Fts3SegReaderFinish(&pCsr->csr);
+ sqlite3_free(pCsr);
+ return SQLITE_OK;
+}
+
+/*
+** xNext - Advance the cursor to the next row, if any.
+*/
+static int fts3termNextMethod(sqlite3_vtab_cursor *pCursor){
+ Fts3termCursor *pCsr = (Fts3termCursor *)pCursor;
+ Fts3Table *pFts3 = ((Fts3termTable *)pCursor->pVtab)->pFts3Tab;
+ int rc;
+ sqlite3_int64 v;
+
+ /* Increment our pretend rowid value. */
+ pCsr->iRowid++;
+
+ /* Advance to the next term in the full-text index. */
+ if( pCsr->csr.aDoclist==0
+ || pCsr->pNext>=&pCsr->csr.aDoclist[pCsr->csr.nDoclist-1]
+ ){
+ rc = sqlite3Fts3SegReaderStep(pFts3, &pCsr->csr);
+ if( rc!=SQLITE_ROW ){
+ pCsr->isEof = 1;
+ return rc;
+ }
+
+ pCsr->iCol = 0;
+ pCsr->iPos = 0;
+ pCsr->iDocid = 0;
+ pCsr->pNext = pCsr->csr.aDoclist;
+
+ /* Read docid */
+ pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &pCsr->iDocid);
+ }
+
+ pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v);
+ if( v==0 ){
+ pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v);
+ pCsr->iDocid += v;
+ pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v);
+ pCsr->iCol = 0;
+ pCsr->iPos = 0;
+ }
+
+ if( v==1 ){
+ pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v);
+ pCsr->iCol += v;
+ pCsr->iPos = 0;
+ pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v);
+ }
+
+ pCsr->iPos += (v - 2);
+
+ return SQLITE_OK;
+}
+
+/*
+** xFilter - Initialize a cursor to point at the start of its data.
+*/
+static int fts3termFilterMethod(
+ sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
+ int idxNum, /* Strategy index */
+ const char *idxStr, /* Unused */
+ int nVal, /* Number of elements in apVal */
+ sqlite3_value **apVal /* Arguments for the indexing scheme */
+){
+ Fts3termCursor *pCsr = (Fts3termCursor *)pCursor;
+ Fts3termTable *p = (Fts3termTable *)pCursor->pVtab;
+ Fts3Table *pFts3 = p->pFts3Tab;
+ int rc;
+
+ UNUSED_PARAMETER(nVal);
+ UNUSED_PARAMETER(idxNum);
+ UNUSED_PARAMETER(idxStr);
+ UNUSED_PARAMETER(apVal);
+
+ assert( idxStr==0 && idxNum==0 );
+
+ /* In case this cursor is being reused, close and zero it. */
+ testcase(pCsr->filter.zTerm);
+ sqlite3Fts3SegReaderFinish(&pCsr->csr);
+ memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr);
+
+ pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY;
+ pCsr->filter.flags |= FTS3_SEGMENT_SCAN;
+
+ rc = sqlite3Fts3SegReaderCursor(pFts3, p->iIndex, FTS3_SEGCURSOR_ALL,
+ pCsr->filter.zTerm, pCsr->filter.nTerm, 0, 1, &pCsr->csr
+ );
+ if( rc==SQLITE_OK ){
+ rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter);
+ }
+ if( rc==SQLITE_OK ){
+ rc = fts3termNextMethod(pCursor);
+ }
+ return rc;
+}
+
+/*
+** xEof - Return true if the cursor is at EOF, or false otherwise.
+*/
+static int fts3termEofMethod(sqlite3_vtab_cursor *pCursor){
+ Fts3termCursor *pCsr = (Fts3termCursor *)pCursor;
+ return pCsr->isEof;
+}
+
+/*
+** xColumn - Return a column value.
+*/
+static int fts3termColumnMethod(
+ sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
+ sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */
+ int iCol /* Index of column to read value from */
+){
+ Fts3termCursor *p = (Fts3termCursor *)pCursor;
+
+ assert( iCol>=0 && iCol<=3 );
+ switch( iCol ){
+ case 0:
+ sqlite3_result_text(pCtx, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT);
+ break;
+ case 1:
+ sqlite3_result_int64(pCtx, p->iDocid);
+ break;
+ case 2:
+ sqlite3_result_int64(pCtx, p->iCol);
+ break;
+ default:
+ sqlite3_result_int64(pCtx, p->iPos);
+ break;
+ }
+
+ return SQLITE_OK;
+}
+
+/*
+** xRowid - Return the current rowid for the cursor.
+*/
+static int fts3termRowidMethod(
+ sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
+ sqlite_int64 *pRowid /* OUT: Rowid value */
+){
+ Fts3termCursor *pCsr = (Fts3termCursor *)pCursor;
+ *pRowid = pCsr->iRowid;
+ return SQLITE_OK;
+}
+
+/*
+** Register the fts3term module with database connection db. Return SQLITE_OK
+** if successful or an error code if sqlite3_create_module() fails.
+*/
+int sqlite3Fts3InitTerm(sqlite3 *db){
+ static const sqlite3_module fts3term_module = {
+ 0, /* iVersion */
+ fts3termConnectMethod, /* xCreate */
+ fts3termConnectMethod, /* xConnect */
+ fts3termBestIndexMethod, /* xBestIndex */
+ fts3termDisconnectMethod, /* xDisconnect */
+ fts3termDisconnectMethod, /* xDestroy */
+ fts3termOpenMethod, /* xOpen */
+ fts3termCloseMethod, /* xClose */
+ fts3termFilterMethod, /* xFilter */
+ fts3termNextMethod, /* xNext */
+ fts3termEofMethod, /* xEof */
+ fts3termColumnMethod, /* xColumn */
+ fts3termRowidMethod, /* xRowid */
+ 0, /* xUpdate */
+ 0, /* xBegin */
+ 0, /* xSync */
+ 0, /* xCommit */
+ 0, /* xRollback */
+ 0, /* xFindFunction */
+ 0 /* xRename */
+ };
+ int rc; /* Return code */
+
+ rc = sqlite3_create_module(db, "fts4term", &fts3term_module, 0);
+ return rc;
+}
+
+#endif
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
diff --git a/src/libtracker-fts/fts3_test.c b/src/libtracker-fts/fts3_test.c
new file mode 100644
index 0000000..72735f3
--- /dev/null
+++ b/src/libtracker-fts/fts3_test.c
@@ -0,0 +1,324 @@
+/*
+** 2011 Jun 13
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file is not part of the production FTS code. It is only used for
+** testing. It contains a Tcl command that can be used to test if a document
+** matches an FTS NEAR expression.
+*/
+
+#include <tcl.h>
+#include <string.h>
+#include <assert.h>
+
+#ifdef SQLITE_TEST
+
+/* Required so that the "ifdef SQLITE_ENABLE_FTS3" below works */
+#include "fts3Int.h"
+
+#define NM_MAX_TOKEN 12
+
+typedef struct NearPhrase NearPhrase;
+typedef struct NearDocument NearDocument;
+typedef struct NearToken NearToken;
+
+struct NearDocument {
+ int nToken; /* Length of token in bytes */
+ NearToken *aToken; /* Token array */
+};
+
+struct NearToken {
+ int n; /* Length of token in bytes */
+ const char *z; /* Pointer to token string */
+};
+
+struct NearPhrase {
+ int nNear; /* Preceding NEAR value */
+ int nToken; /* Number of tokens in this phrase */
+ NearToken aToken[NM_MAX_TOKEN]; /* Array of tokens in this phrase */
+};
+
+static int nm_phrase_match(
+ NearPhrase *p,
+ NearToken *aToken
+){
+ int ii;
+
+ for(ii=0; ii<p->nToken; ii++){
+ NearToken *pToken = &p->aToken[ii];
+ if( pToken->n>0 && pToken->z[pToken->n-1]=='*' ){
+ if( aToken[ii].n<(pToken->n-1) ) return 0;
+ if( memcmp(aToken[ii].z, pToken->z, pToken->n-1) ) return 0;
+ }else{
+ if( aToken[ii].n!=pToken->n ) return 0;
+ if( memcmp(aToken[ii].z, pToken->z, pToken->n) ) return 0;
+ }
+ }
+
+ return 1;
+}
+
+static int nm_near_chain(
+ int iDir, /* Direction to iterate through aPhrase[] */
+ NearDocument *pDoc, /* Document to match against */
+ int iPos, /* Position at which iPhrase was found */
+ int nPhrase, /* Size of phrase array */
+ NearPhrase *aPhrase, /* Phrase array */
+ int iPhrase /* Index of phrase found */
+){
+ int iStart;
+ int iStop;
+ int ii;
+ int nNear;
+ int iPhrase2;
+ NearPhrase *p;
+ NearPhrase *pPrev;
+
+ assert( iDir==1 || iDir==-1 );
+
+ if( iDir==1 ){
+ if( (iPhrase+1)==nPhrase ) return 1;
+ nNear = aPhrase[iPhrase+1].nNear;
+ }else{
+ if( iPhrase==0 ) return 1;
+ nNear = aPhrase[iPhrase].nNear;
+ }
+ pPrev = &aPhrase[iPhrase];
+ iPhrase2 = iPhrase+iDir;
+ p = &aPhrase[iPhrase2];
+
+ iStart = iPos - nNear - p->nToken;
+ iStop = iPos + nNear + pPrev->nToken;
+
+ if( iStart<0 ) iStart = 0;
+ if( iStop > pDoc->nToken - p->nToken ) iStop = pDoc->nToken - p->nToken;
+
+ for(ii=iStart; ii<=iStop; ii++){
+ if( nm_phrase_match(p, &pDoc->aToken[ii]) ){
+ if( nm_near_chain(iDir, pDoc, ii, nPhrase, aPhrase, iPhrase2) ) return 1;
+ }
+ }
+
+ return 0;
+}
+
+static int nm_match_count(
+ NearDocument *pDoc, /* Document to match against */
+ int nPhrase, /* Size of phrase array */
+ NearPhrase *aPhrase, /* Phrase array */
+ int iPhrase /* Index of phrase to count matches for */
+){
+ int nOcc = 0;
+ int ii;
+ NearPhrase *p = &aPhrase[iPhrase];
+
+ for(ii=0; ii<(pDoc->nToken + 1 - p->nToken); ii++){
+ if( nm_phrase_match(p, &pDoc->aToken[ii]) ){
+ /* Test forward NEAR chain (i>iPhrase) */
+ if( 0==nm_near_chain(1, pDoc, ii, nPhrase, aPhrase, iPhrase) ) continue;
+
+ /* Test reverse NEAR chain (i<iPhrase) */
+ if( 0==nm_near_chain(-1, pDoc, ii, nPhrase, aPhrase, iPhrase) ) continue;
+
+ /* This is a real match. Increment the counter. */
+ nOcc++;
+ }
+ }
+
+ return nOcc;
+}
+
+/*
+** Tclcmd: fts3_near_match DOCUMENT EXPR ?OPTIONS?
+*/
+static int fts3_near_match_cmd(
+ ClientData clientData,
+ Tcl_Interp *interp,
+ int objc,
+ Tcl_Obj *CONST objv[]
+){
+ int nTotal = 0;
+ int rc;
+ int ii;
+ int nPhrase;
+ NearPhrase *aPhrase = 0;
+ NearDocument doc = {0, 0};
+ Tcl_Obj **apDocToken;
+ Tcl_Obj *pRet;
+ Tcl_Obj *pPhrasecount = 0;
+
+ Tcl_Obj **apExprToken;
+ int nExprToken;
+
+ /* Must have 3 or more arguments. */
+ if( objc<3 || (objc%2)==0 ){
+ Tcl_WrongNumArgs(interp, 1, objv, "DOCUMENT EXPR ?OPTION VALUE?...");
+ rc = TCL_ERROR;
+ goto near_match_out;
+ }
+
+ for(ii=3; ii<objc; ii+=2){
+ enum NM_enum { NM_PHRASECOUNTS };
+ struct TestnmSubcmd {
+ char *zName;
+ enum NM_enum eOpt;
+ } aOpt[] = {
+ { "-phrasecountvar", NM_PHRASECOUNTS },
+ { 0, 0 }
+ };
+ int iOpt;
+ if( Tcl_GetIndexFromObjStruct(
+ interp, objv[ii], aOpt, sizeof(aOpt[0]), "option", 0, &iOpt)
+ ){
+ return TCL_ERROR;
+ }
+
+ switch( aOpt[iOpt].eOpt ){
+ case NM_PHRASECOUNTS:
+ pPhrasecount = objv[ii+1];
+ break;
+ }
+ }
+
+ rc = Tcl_ListObjGetElements(interp, objv[1], &doc.nToken, &apDocToken);
+ if( rc!=TCL_OK ) goto near_match_out;
+ doc.aToken = (NearToken *)ckalloc(doc.nToken*sizeof(NearToken));
+ for(ii=0; ii<doc.nToken; ii++){
+ doc.aToken[ii].z = Tcl_GetStringFromObj(apDocToken[ii], &doc.aToken[ii].n);
+ }
+
+ rc = Tcl_ListObjGetElements(interp, objv[2], &nExprToken, &apExprToken);
+ if( rc!=TCL_OK ) goto near_match_out;
+
+ nPhrase = (nExprToken + 1) / 2;
+ aPhrase = (NearPhrase *)ckalloc(nPhrase * sizeof(NearPhrase));
+ memset(aPhrase, 0, nPhrase * sizeof(NearPhrase));
+ for(ii=0; ii<nPhrase; ii++){
+ Tcl_Obj *pPhrase = apExprToken[ii*2];
+ Tcl_Obj **apToken;
+ int nToken;
+ int jj;
+
+ rc = Tcl_ListObjGetElements(interp, pPhrase, &nToken, &apToken);
+ if( rc!=TCL_OK ) goto near_match_out;
+ if( nToken>NM_MAX_TOKEN ){
+ Tcl_AppendResult(interp, "Too many tokens in phrase", 0);
+ rc = TCL_ERROR;
+ goto near_match_out;
+ }
+ for(jj=0; jj<nToken; jj++){
+ NearToken *pT = &aPhrase[ii].aToken[jj];
+ pT->z = Tcl_GetStringFromObj(apToken[jj], &pT->n);
+ }
+ aPhrase[ii].nToken = nToken;
+ }
+ for(ii=1; ii<nPhrase; ii++){
+ Tcl_Obj *pNear = apExprToken[2*ii-1];
+ int nNear;
+ rc = Tcl_GetIntFromObj(interp, pNear, &nNear);
+ if( rc!=TCL_OK ) goto near_match_out;
+ aPhrase[ii].nNear = nNear;
+ }
+
+ pRet = Tcl_NewObj();
+ Tcl_IncrRefCount(pRet);
+ for(ii=0; ii<nPhrase; ii++){
+ int nOcc = nm_match_count(&doc, nPhrase, aPhrase, ii);
+ Tcl_ListObjAppendElement(interp, pRet, Tcl_NewIntObj(nOcc));
+ nTotal += nOcc;
+ }
+ if( pPhrasecount ){
+ Tcl_ObjSetVar2(interp, pPhrasecount, 0, pRet, 0);
+ }
+ Tcl_DecrRefCount(pRet);
+ Tcl_SetObjResult(interp, Tcl_NewBooleanObj(nTotal>0));
+
+ near_match_out:
+ ckfree((char *)aPhrase);
+ ckfree((char *)doc.aToken);
+ return rc;
+}
+
+/*
+** Tclcmd: fts3_configure_incr_load ?CHUNKSIZE THRESHOLD?
+**
+** Normally, FTS uses hard-coded values to determine the minimum doclist
+** size eligible for incremental loading, and the size of the chunks loaded
+** when a doclist is incrementally loaded. This command allows the built-in
+** values to be overridden for testing purposes.
+**
+** If present, the first argument is the chunksize in bytes to load doclists
+** in. The second argument is the minimum doclist size in bytes to use
+** incremental loading with.
+**
+** Whether or not the arguments are present, this command returns a list of
+** two integers - the initial chunksize and threshold when the command is
+** invoked. This can be used to restore the default behaviour after running
+** tests. For example:
+**
+** # Override incr-load settings for testing:
+** set cfg [fts3_configure_incr_load $new_chunksize $new_threshold]
+**
+** .... run tests ....
+**
+** # Restore initial incr-load settings:
+** eval fts3_configure_incr_load $cfg
+*/
+static int fts3_configure_incr_load_cmd(
+ ClientData clientData,
+ Tcl_Interp *interp,
+ int objc,
+ Tcl_Obj *CONST objv[]
+){
+#ifdef SQLITE_ENABLE_FTS3
+ extern int test_fts3_node_chunksize;
+ extern int test_fts3_node_chunk_threshold;
+ Tcl_Obj *pRet;
+
+ if( objc!=1 && objc!=3 ){
+ Tcl_WrongNumArgs(interp, 1, objv, "?CHUNKSIZE THRESHOLD?");
+ return TCL_ERROR;
+ }
+
+ pRet = Tcl_NewObj();
+ Tcl_IncrRefCount(pRet);
+ Tcl_ListObjAppendElement(
+ interp, pRet, Tcl_NewIntObj(test_fts3_node_chunksize));
+ Tcl_ListObjAppendElement(
+ interp, pRet, Tcl_NewIntObj(test_fts3_node_chunk_threshold));
+
+ if( objc==3 ){
+ int iArg1;
+ int iArg2;
+ if( Tcl_GetIntFromObj(interp, objv[1], &iArg1)
+ || Tcl_GetIntFromObj(interp, objv[2], &iArg2)
+ ){
+ Tcl_DecrRefCount(pRet);
+ return TCL_ERROR;
+ }
+ test_fts3_node_chunksize = iArg1;
+ test_fts3_node_chunk_threshold = iArg2;
+ }
+
+ Tcl_SetObjResult(interp, pRet);
+ Tcl_DecrRefCount(pRet);
+#endif
+ return TCL_OK;
+}
+
+int Sqlitetestfts3_Init(Tcl_Interp *interp){
+ Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0);
+ Tcl_CreateObjCommand(interp,
+ "fts3_configure_incr_load", fts3_configure_incr_load_cmd, 0, 0
+ );
+ return TCL_OK;
+}
+#endif /* ifdef SQLITE_TEST */
diff --git a/src/libtracker-fts/fts3_tokenizer.c b/src/libtracker-fts/fts3_tokenizer.c
new file mode 100644
index 0000000..6494bb9
--- /dev/null
+++ b/src/libtracker-fts/fts3_tokenizer.c
@@ -0,0 +1,489 @@
+/*
+** 2007 June 22
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This is part of an SQLite module implementing full-text search.
+** This particular file implements the generic tokenizer interface.
+*/
+
+/*
+** The code in this file is only compiled if:
+**
+** * The FTS3 module is being built as an extension
+** (in which case SQLITE_CORE is not defined), or
+**
+** * The FTS3 module is being built into the core of
+** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
+*/
+#include "fts3Int.h"
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+#include <assert.h>
+#include <string.h>
+
+/*
+** Implementation of the SQL scalar function for accessing the underlying
+** hash table. This function may be called as follows:
+**
+** SELECT <function-name>(<key-name>);
+** SELECT <function-name>(<key-name>, <pointer>);
+**
+** where <function-name> is the name passed as the second argument
+** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer').
+**
+** If the <pointer> argument is specified, it must be a blob value
+** containing a pointer to be stored as the hash data corresponding
+** to the string <key-name>. If <pointer> is not specified, then
+** the string <key-name> must already exist in the has table. Otherwise,
+** an error is returned.
+**
+** Whether or not the <pointer> argument is specified, the value returned
+** is a blob containing the pointer stored as the hash data corresponding
+** to string <key-name> (after the hash-table is updated, if applicable).
+*/
+static void scalarFunc(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ Fts3Hash *pHash;
+ void *pPtr = 0;
+ const unsigned char *zName;
+ int nName;
+
+ assert( argc==1 || argc==2 );
+
+ pHash = (Fts3Hash *)sqlite3_user_data(context);
+
+ zName = sqlite3_value_text(argv[0]);
+ nName = sqlite3_value_bytes(argv[0])+1;
+
+ if( argc==2 ){
+ void *pOld;
+ int n = sqlite3_value_bytes(argv[1]);
+ if( n!=sizeof(pPtr) ){
+ sqlite3_result_error(context, "argument type mismatch", -1);
+ return;
+ }
+ pPtr = *(void **)sqlite3_value_blob(argv[1]);
+ pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr);
+ if( pOld==pPtr ){
+ sqlite3_result_error(context, "out of memory", -1);
+ return;
+ }
+ }else{
+ pPtr = sqlite3Fts3HashFind(pHash, zName, nName);
+ if( !pPtr ){
+ char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
+ sqlite3_result_error(context, zErr, -1);
+ sqlite3_free(zErr);
+ return;
+ }
+ }
+
+ sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
+}
+
+int sqlite3Fts3IsIdChar(char c){
+ static const char isFtsIdChar[] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
+ 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
+ };
+ return (c&0x80 || isFtsIdChar[(int)(c)]);
+}
+
+const char *sqlite3Fts3NextToken(const char *zStr, int *pn){
+ const char *z1;
+ const char *z2 = 0;
+
+ /* Find the start of the next token. */
+ z1 = zStr;
+ while( z2==0 ){
+ char c = *z1;
+ switch( c ){
+ case '\0': return 0; /* No more tokens here */
+ case '\'':
+ case '"':
+ case '`': {
+ z2 = z1;
+ while( *++z2 && (*z2!=c || *++z2==c) );
+ break;
+ }
+ case '[':
+ z2 = &z1[1];
+ while( *z2 && z2[0]!=']' ) z2++;
+ if( *z2 ) z2++;
+ break;
+
+ default:
+ if( sqlite3Fts3IsIdChar(*z1) ){
+ z2 = &z1[1];
+ while( sqlite3Fts3IsIdChar(*z2) ) z2++;
+ }else{
+ z1++;
+ }
+ }
+ }
+
+ *pn = (int)(z2-z1);
+ return z1;
+}
+
+int sqlite3Fts3InitTokenizer(
+ Fts3Hash *pHash, /* Tokenizer hash table */
+ const char *zArg, /* Tokenizer name */
+ sqlite3_tokenizer **ppTok, /* OUT: Tokenizer (if applicable) */
+ char **pzErr /* OUT: Set to malloced error message */
+){
+ int rc;
+ char *z = (char *)zArg;
+ int n = 0;
+ char *zCopy;
+ char *zEnd; /* Pointer to nul-term of zCopy */
+ sqlite3_tokenizer_module *m;
+
+ zCopy = sqlite3_mprintf("%s", zArg);
+ if( !zCopy ) return SQLITE_NOMEM;
+ zEnd = &zCopy[strlen(zCopy)];
+
+ z = (char *)sqlite3Fts3NextToken(zCopy, &n);
+ z[n] = '\0';
+ sqlite3Fts3Dequote(z);
+
+ m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1);
+ if( !m ){
+ *pzErr = sqlite3_mprintf("unknown tokenizer: %s", z);
+ rc = SQLITE_ERROR;
+ }else{
+ char const **aArg = 0;
+ int iArg = 0;
+ z = &z[n+1];
+ while( z<zEnd && (NULL!=(z = (char *)sqlite3Fts3NextToken(z, &n))) ){
+ int nNew = sizeof(char *)*(iArg+1);
+ char const **aNew = (const char **)sqlite3_realloc((void *)aArg, nNew);
+ if( !aNew ){
+ sqlite3_free(zCopy);
+ sqlite3_free((void *)aArg);
+ return SQLITE_NOMEM;
+ }
+ aArg = aNew;
+ aArg[iArg++] = z;
+ z[n] = '\0';
+ sqlite3Fts3Dequote(z);
+ z = &z[n+1];
+ }
+ rc = m->xCreate(iArg, aArg, ppTok);
+ assert( rc!=SQLITE_OK || *ppTok );
+ if( rc!=SQLITE_OK ){
+ *pzErr = sqlite3_mprintf("unknown tokenizer");
+ }else{
+ (*ppTok)->pModule = m;
+ }
+ sqlite3_free((void *)aArg);
+ }
+
+ sqlite3_free(zCopy);
+ return rc;
+}
+
+
+#ifdef SQLITE_TEST
+
+#include <tcl.h>
+#include <string.h>
+
+/*
+** Implementation of a special SQL scalar function for testing tokenizers
+** designed to be used in concert with the Tcl testing framework. This
+** function must be called with two arguments:
+**
+** SELECT <function-name>(<key-name>, <input-string>);
+** SELECT <function-name>(<key-name>, <pointer>);
+**
+** where <function-name> is the name passed as the second argument
+** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer')
+** concatenated with the string '_test' (e.g. 'fts3_tokenizer_test').
+**
+** The return value is a string that may be interpreted as a Tcl
+** list. For each token in the <input-string>, three elements are
+** added to the returned list. The first is the token position, the
+** second is the token text (folded, stemmed, etc.) and the third is the
+** substring of <input-string> associated with the token. For example,
+** using the built-in "simple" tokenizer:
+**
+** SELECT fts_tokenizer_test('simple', 'I don't see how');
+**
+** will return the string:
+**
+** "{0 i I 1 dont don't 2 see see 3 how how}"
+**
+*/
+static void testFunc(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ Fts3Hash *pHash;
+ sqlite3_tokenizer_module *p;
+ sqlite3_tokenizer *pTokenizer = 0;
+ sqlite3_tokenizer_cursor *pCsr = 0;
+
+ const char *zErr = 0;
+
+ const char *zName;
+ int nName;
+ const char *zInput;
+ int nInput;
+
+ const char *zArg = 0;
+
+ const char *zToken;
+ int nToken;
+ int iStart;
+ int iEnd;
+ int iPos;
+
+ Tcl_Obj *pRet;
+
+ assert( argc==2 || argc==3 );
+
+ nName = sqlite3_value_bytes(argv[0]);
+ zName = (const char *)sqlite3_value_text(argv[0]);
+ nInput = sqlite3_value_bytes(argv[argc-1]);
+ zInput = (const char *)sqlite3_value_text(argv[argc-1]);
+
+ if( argc==3 ){
+ zArg = (const char *)sqlite3_value_text(argv[1]);
+ }
+
+ pHash = (Fts3Hash *)sqlite3_user_data(context);
+ p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1);
+
+ if( !p ){
+ char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
+ sqlite3_result_error(context, zErr, -1);
+ sqlite3_free(zErr);
+ return;
+ }
+
+ pRet = Tcl_NewObj();
+ Tcl_IncrRefCount(pRet);
+
+ if( SQLITE_OK!=p->xCreate(zArg ? 1 : 0, &zArg, &pTokenizer) ){
+ zErr = "error in xCreate()";
+ goto finish;
+ }
+ pTokenizer->pModule = p;
+ if( SQLITE_OK!=p->xOpen(pTokenizer, zInput, nInput, &pCsr) ){
+ zErr = "error in xOpen()";
+ goto finish;
+ }
+ pCsr->pTokenizer = pTokenizer;
+
+ while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){
+ Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos));
+ Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
+ zToken = &zInput[iStart];
+ nToken = iEnd-iStart;
+ Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
+ }
+
+ if( SQLITE_OK!=p->xClose(pCsr) ){
+ zErr = "error in xClose()";
+ goto finish;
+ }
+ if( SQLITE_OK!=p->xDestroy(pTokenizer) ){
+ zErr = "error in xDestroy()";
+ goto finish;
+ }
+
+finish:
+ if( zErr ){
+ sqlite3_result_error(context, zErr, -1);
+ }else{
+ sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT);
+ }
+ Tcl_DecrRefCount(pRet);
+}
+
+static
+int registerTokenizer(
+ sqlite3 *db,
+ char *zName,
+ const sqlite3_tokenizer_module *p
+){
+ int rc;
+ sqlite3_stmt *pStmt;
+ const char zSql[] = "SELECT fts3_tokenizer(?, ?)";
+
+ rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+
+ sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
+ sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
+ sqlite3_step(pStmt);
+
+ return sqlite3_finalize(pStmt);
+}
+
+static
+int queryTokenizer(
+ sqlite3 *db,
+ char *zName,
+ const sqlite3_tokenizer_module **pp
+){
+ int rc;
+ sqlite3_stmt *pStmt;
+ const char zSql[] = "SELECT fts3_tokenizer(?)";
+
+ *pp = 0;
+ rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+
+ sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){
+ if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
+ memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
+ }
+ }
+
+ return sqlite3_finalize(pStmt);
+}
+
+void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+
+/*
+** Implementation of the scalar function fts3_tokenizer_internal_test().
+** This function is used for testing only, it is not included in the
+** build unless SQLITE_TEST is defined.
+**
+** The purpose of this is to test that the fts3_tokenizer() function
+** can be used as designed by the C-code in the queryTokenizer and
+** registerTokenizer() functions above. These two functions are repeated
+** in the README.tokenizer file as an example, so it is important to
+** test them.
+**
+** To run the tests, evaluate the fts3_tokenizer_internal_test() scalar
+** function with no arguments. An assert() will fail if a problem is
+** detected. i.e.:
+**
+** SELECT fts3_tokenizer_internal_test();
+**
+*/
+static void intTestFunc(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ int rc;
+ const sqlite3_tokenizer_module *p1;
+ const sqlite3_tokenizer_module *p2;
+ sqlite3 *db = (sqlite3 *)sqlite3_user_data(context);
+
+ UNUSED_PARAMETER(argc);
+ UNUSED_PARAMETER(argv);
+
+ /* Test the query function */
+ sqlite3Fts3SimpleTokenizerModule(&p1);
+ rc = queryTokenizer(db, "simple", &p2);
+ assert( rc==SQLITE_OK );
+ assert( p1==p2 );
+ rc = queryTokenizer(db, "nosuchtokenizer", &p2);
+ assert( rc==SQLITE_ERROR );
+ assert( p2==0 );
+ assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
+
+ /* Test the storage function */
+ rc = registerTokenizer(db, "nosuchtokenizer", p1);
+ assert( rc==SQLITE_OK );
+ rc = queryTokenizer(db, "nosuchtokenizer", &p2);
+ assert( rc==SQLITE_OK );
+ assert( p2==p1 );
+
+ sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
+}
+
+#endif
+
+/*
+** Set up SQL objects in database db used to access the contents of
+** the hash table pointed to by argument pHash. The hash table must
+** been initialised to use string keys, and to take a private copy
+** of the key when a value is inserted. i.e. by a call similar to:
+**
+** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
+**
+** This function adds a scalar function (see header comment above
+** scalarFunc() in this file for details) and, if ENABLE_TABLE is
+** defined at compilation time, a temporary virtual table (see header
+** comment above struct HashTableVtab) to the database schema. Both
+** provide read/write access to the contents of *pHash.
+**
+** The third argument to this function, zName, is used as the name
+** of both the scalar and, if created, the virtual table.
+*/
+int sqlite3Fts3InitHashTable(
+ sqlite3 *db,
+ Fts3Hash *pHash,
+ const char *zName
+){
+ int rc = SQLITE_OK;
+ void *p = (void *)pHash;
+ const int any = SQLITE_ANY;
+
+#ifdef SQLITE_TEST
+ char *zTest = 0;
+ char *zTest2 = 0;
+ void *pdb = (void *)db;
+ zTest = sqlite3_mprintf("%s_test", zName);
+ zTest2 = sqlite3_mprintf("%s_internal_test", zName);
+ if( !zTest || !zTest2 ){
+ rc = SQLITE_NOMEM;
+ }
+#endif
+
+ if( SQLITE_OK==rc ){
+ rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0);
+ }
+ if( SQLITE_OK==rc ){
+ rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0);
+ }
+#ifdef SQLITE_TEST
+ if( SQLITE_OK==rc ){
+ rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0);
+ }
+ if( SQLITE_OK==rc ){
+ rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0);
+ }
+ if( SQLITE_OK==rc ){
+ rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0);
+ }
+#endif
+
+#ifdef SQLITE_TEST
+ sqlite3_free(zTest);
+ sqlite3_free(zTest2);
+#endif
+
+ return rc;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
diff --git a/src/libtracker-fts/fts3_tokenizer.h b/src/libtracker-fts/fts3_tokenizer.h
new file mode 100644
index 0000000..6156445
--- /dev/null
+++ b/src/libtracker-fts/fts3_tokenizer.h
@@ -0,0 +1,152 @@
+/*
+** 2006 July 10
+**
+** The author disclaims copyright to this source code.
+**
+*************************************************************************
+** Defines the interface to tokenizers used by fulltext-search. There
+** are three basic components:
+**
+** sqlite3_tokenizer_module is a singleton defining the tokenizer
+** interface functions. This is essentially the class structure for
+** tokenizers.
+**
+** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
+** including customization information defined at creation time.
+**
+** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
+** tokens from a particular input.
+*/
+#ifndef _FTS3_TOKENIZER_H_
+#define _FTS3_TOKENIZER_H_
+
+/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
+** If tokenizers are to be allowed to call sqlite3_*() functions, then
+** we will need a way to register the API consistently.
+*/
+#include "sqlite3.h"
+
+/*
+** Structures used by the tokenizer interface. When a new tokenizer
+** implementation is registered, the caller provides a pointer to
+** an sqlite3_tokenizer_module containing pointers to the callback
+** functions that make up an implementation.
+**
+** When an fts3 table is created, it passes any arguments passed to
+** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the
+** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer
+** implementation. The xCreate() function in turn returns an
+** sqlite3_tokenizer structure representing the specific tokenizer to
+** be used for the fts3 table (customized by the tokenizer clause arguments).
+**
+** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen()
+** method is called. It returns an sqlite3_tokenizer_cursor object
+** that may be used to tokenize a specific input buffer based on
+** the tokenization rules supplied by a specific sqlite3_tokenizer
+** object.
+*/
+typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
+typedef struct sqlite3_tokenizer sqlite3_tokenizer;
+typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
+
+struct sqlite3_tokenizer_module {
+
+ /*
+ ** Structure version. Should always be set to 0.
+ */
+ int iVersion;
+
+ /*
+ ** Create a new tokenizer. The values in the argv[] array are the
+ ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL
+ ** TABLE statement that created the fts3 table. For example, if
+ ** the following SQL is executed:
+ **
+ ** CREATE .. USING fts3( ... , tokenizer <tokenizer-name> arg1 arg2)
+ **
+ ** then argc is set to 2, and the argv[] array contains pointers
+ ** to the strings "arg1" and "arg2".
+ **
+ ** This method should return either SQLITE_OK (0), or an SQLite error
+ ** code. If SQLITE_OK is returned, then *ppTokenizer should be set
+ ** to point at the newly created tokenizer structure. The generic
+ ** sqlite3_tokenizer.pModule variable should not be initialised by
+ ** this callback. The caller will do so.
+ */
+ int (*xCreate)(
+ int argc, /* Size of argv array */
+ const char *const*argv, /* Tokenizer argument strings */
+ sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */
+ );
+
+ /*
+ ** Destroy an existing tokenizer. The fts3 module calls this method
+ ** exactly once for each successful call to xCreate().
+ */
+ int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
+
+ /*
+ ** Create a tokenizer cursor to tokenize an input buffer. The caller
+ ** is responsible for ensuring that the input buffer remains valid
+ ** until the cursor is closed (using the xClose() method).
+ */
+ int (*xOpen)(
+ sqlite3_tokenizer *pTokenizer, /* Tokenizer object */
+ const char *pInput, int nBytes, /* Input buffer */
+ sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */
+ );
+
+ /*
+ ** Destroy an existing tokenizer cursor. The fts3 module calls this
+ ** method exactly once for each successful call to xOpen().
+ */
+ int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
+
+ /*
+ ** Retrieve the next token from the tokenizer cursor pCursor. This
+ ** method should either return SQLITE_OK and set the values of the
+ ** "OUT" variables identified below, or SQLITE_DONE to indicate that
+ ** the end of the buffer has been reached, or an SQLite error code.
+ **
+ ** *ppToken should be set to point at a buffer containing the
+ ** normalized version of the token (i.e. after any case-folding and/or
+ ** stemming has been performed). *pnBytes should be set to the length
+ ** of this buffer in bytes. The input text that generated the token is
+ ** identified by the byte offsets returned in *piStartOffset and
+ ** *piEndOffset. *piStartOffset should be set to the index of the first
+ ** byte of the token in the input buffer. *piEndOffset should be set
+ ** to the index of the first byte just past the end of the token in
+ ** the input buffer.
+ **
+ ** The buffer *ppToken is set to point at is managed by the tokenizer
+ ** implementation. It is only required to be valid until the next call
+ ** to xNext() or xClose().
+ */
+ /* TODO(shess) current implementation requires pInput to be
+ ** nul-terminated. This should either be fixed, or pInput/nBytes
+ ** should be converted to zInput.
+ */
+ int (*xNext)(
+ sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */
+ const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */
+ int *piStartOffset, /* OUT: Byte offset of token in input buffer */
+ int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */
+ int *piPosition /* OUT: Number of tokens returned before this one */
+ );
+};
+
+struct sqlite3_tokenizer {
+ const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */
+ /* Tokenizer implementations will typically add additional fields */
+};
+
+struct sqlite3_tokenizer_cursor {
+ sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */
+ /* Tokenizer implementations will typically add additional fields */
+};
+
+int fts3_global_term_cnt(int iTerm, int iCol);
+int fts3_term_cnt(int iTerm, int iCol);
+
+
+#endif /* _FTS3_TOKENIZER_H_ */
diff --git a/src/libtracker-fts/fts3_tokenizer1.c b/src/libtracker-fts/fts3_tokenizer1.c
new file mode 100644
index 0000000..d11a499
--- /dev/null
+++ b/src/libtracker-fts/fts3_tokenizer1.c
@@ -0,0 +1,233 @@
+/*
+** 2006 Oct 10
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** Implementation of the "simple" full-text-search tokenizer.
+*/
+
+/*
+** The code in this file is only compiled if:
+**
+** * The FTS3 module is being built as an extension
+** (in which case SQLITE_CORE is not defined), or
+**
+** * The FTS3 module is being built into the core of
+** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
+*/
+#include "fts3Int.h"
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "fts3_tokenizer.h"
+
+typedef struct simple_tokenizer {
+ sqlite3_tokenizer base;
+ char delim[128]; /* flag ASCII delimiters */
+} simple_tokenizer;
+
+typedef struct simple_tokenizer_cursor {
+ sqlite3_tokenizer_cursor base;
+ const char *pInput; /* input we are tokenizing */
+ int nBytes; /* size of the input */
+ int iOffset; /* current position in pInput */
+ int iToken; /* index of next token to be returned */
+ char *pToken; /* storage for current token */
+ int nTokenAllocated; /* space allocated to zToken buffer */
+} simple_tokenizer_cursor;
+
+
+static int simpleDelim(simple_tokenizer *t, unsigned char c){
+ return c<0x80 && t->delim[c];
+}
+static int fts3_isalnum(int x){
+ return (x>='0' && x<='9') || (x>='A' && x<='Z') || (x>='a' && x<='z');
+}
+
+/*
+** Create a new tokenizer instance.
+*/
+static int simpleCreate(
+ int argc, const char * const *argv,
+ sqlite3_tokenizer **ppTokenizer
+){
+ simple_tokenizer *t;
+
+ t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t));
+ if( t==NULL ) return SQLITE_NOMEM;
+ memset(t, 0, sizeof(*t));
+
+ /* TODO(shess) Delimiters need to remain the same from run to run,
+ ** else we need to reindex. One solution would be a meta-table to
+ ** track such information in the database, then we'd only want this
+ ** information on the initial create.
+ */
+ if( argc>1 ){
+ int i, n = (int)strlen(argv[1]);
+ for(i=0; i<n; i++){
+ unsigned char ch = argv[1][i];
+ /* We explicitly don't support UTF-8 delimiters for now. */
+ if( ch>=0x80 ){
+ sqlite3_free(t);
+ return SQLITE_ERROR;
+ }
+ t->delim[ch] = 1;
+ }
+ } else {
+ /* Mark non-alphanumeric ASCII characters as delimiters */
+ int i;
+ for(i=1; i<0x80; i++){
+ t->delim[i] = !fts3_isalnum(i) ? -1 : 0;
+ }
+ }
+
+ *ppTokenizer = &t->base;
+ return SQLITE_OK;
+}
+
+/*
+** Destroy a tokenizer
+*/
+static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
+ sqlite3_free(pTokenizer);
+ return SQLITE_OK;
+}
+
+/*
+** Prepare to begin tokenizing a particular string. The input
+** string to be tokenized is pInput[0..nBytes-1]. A cursor
+** used to incrementally tokenize this string is returned in
+** *ppCursor.
+*/
+static int simpleOpen(
+ sqlite3_tokenizer *pTokenizer, /* The tokenizer */
+ const char *pInput, int nBytes, /* String to be tokenized */
+ sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
+){
+ simple_tokenizer_cursor *c;
+
+ UNUSED_PARAMETER(pTokenizer);
+
+ c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
+ if( c==NULL ) return SQLITE_NOMEM;
+
+ c->pInput = pInput;
+ if( pInput==0 ){
+ c->nBytes = 0;
+ }else if( nBytes<0 ){
+ c->nBytes = (int)strlen(pInput);
+ }else{
+ c->nBytes = nBytes;
+ }
+ c->iOffset = 0; /* start tokenizing at the beginning */
+ c->iToken = 0;
+ c->pToken = NULL; /* no space allocated, yet. */
+ c->nTokenAllocated = 0;
+
+ *ppCursor = &c->base;
+ return SQLITE_OK;
+}
+
+/*
+** Close a tokenization cursor previously opened by a call to
+** simpleOpen() above.
+*/
+static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
+ simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
+ sqlite3_free(c->pToken);
+ sqlite3_free(c);
+ return SQLITE_OK;
+}
+
+/*
+** Extract the next token from a tokenization cursor. The cursor must
+** have been opened by a prior call to simpleOpen().
+*/
+static int simpleNext(
+ sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */
+ const char **ppToken, /* OUT: *ppToken is the token text */
+ int *pnBytes, /* OUT: Number of bytes in token */
+ int *piStartOffset, /* OUT: Starting offset of token */
+ int *piEndOffset, /* OUT: Ending offset of token */
+ int *piPosition /* OUT: Position integer of token */
+){
+ simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
+ simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
+ unsigned char *p = (unsigned char *)c->pInput;
+
+ while( c->iOffset<c->nBytes ){
+ int iStartOffset;
+
+ /* Scan past delimiter characters */
+ while( c->iOffset<c->nBytes && simpleDelim(t, p[c->iOffset]) ){
+ c->iOffset++;
+ }
+
+ /* Count non-delimiter characters. */
+ iStartOffset = c->iOffset;
+ while( c->iOffset<c->nBytes && !simpleDelim(t, p[c->iOffset]) ){
+ c->iOffset++;
+ }
+
+ if( c->iOffset>iStartOffset ){
+ int i, n = c->iOffset-iStartOffset;
+ if( n>c->nTokenAllocated ){
+ char *pNew;
+ c->nTokenAllocated = n+20;
+ pNew = sqlite3_realloc(c->pToken, c->nTokenAllocated);
+ if( !pNew ) return SQLITE_NOMEM;
+ c->pToken = pNew;
+ }
+ for(i=0; i<n; i++){
+ /* TODO(shess) This needs expansion to handle UTF-8
+ ** case-insensitivity.
+ */
+ unsigned char ch = p[iStartOffset+i];
+ c->pToken[i] = (char)((ch>='A' && ch<='Z') ? ch-'A'+'a' : ch);
+ }
+ *ppToken = c->pToken;
+ *pnBytes = n;
+ *piStartOffset = iStartOffset;
+ *piEndOffset = c->iOffset;
+ *piPosition = c->iToken++;
+
+ return SQLITE_OK;
+ }
+ }
+ return SQLITE_DONE;
+}
+
+/*
+** The set of routines that implement the simple tokenizer
+*/
+static const sqlite3_tokenizer_module simpleTokenizerModule = {
+ 0,
+ simpleCreate,
+ simpleDestroy,
+ simpleOpen,
+ simpleClose,
+ simpleNext,
+};
+
+/*
+** Allocate a new simple tokenizer. Return a pointer to the new
+** tokenizer in *ppModule
+*/
+void sqlite3Fts3SimpleTokenizerModule(
+ sqlite3_tokenizer_module const**ppModule
+){
+ *ppModule = &simpleTokenizerModule;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
diff --git a/src/libtracker-fts/fts3_write.c b/src/libtracker-fts/fts3_write.c
new file mode 100644
index 0000000..36f2249
--- /dev/null
+++ b/src/libtracker-fts/fts3_write.c
@@ -0,0 +1,3268 @@
+/*
+** 2009 Oct 23
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file is part of the SQLite FTS3 extension module. Specifically,
+** this file contains code to insert, update and delete rows from FTS3
+** tables. It also contains code to merge FTS3 b-tree segments. Some
+** of the sub-routines used to merge segments are also used by the query
+** code in fts3.c.
+*/
+
+#include "fts3Int.h"
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+#include <string.h>
+#include <assert.h>
+#include <stdlib.h>
+
+/*
+** When full-text index nodes are loaded from disk, the buffer that they
+** are loaded into has the following number of bytes of padding at the end
+** of it. i.e. if a full-text index node is 900 bytes in size, then a buffer
+** of 920 bytes is allocated for it.
+**
+** This means that if we have a pointer into a buffer containing node data,
+** it is always safe to read up to two varints from it without risking an
+** overread, even if the node data is corrupted.
+*/
+#define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2)
+
+/*
+** Under certain circumstances, b-tree nodes (doclists) can be loaded into
+** memory incrementally instead of all at once. This can be a big performance
+** win (reduced IO and CPU) if SQLite stops calling the virtual table xNext()
+** method before retrieving all query results (as may happen, for example,
+** if a query has a LIMIT clause).
+**
+** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD
+** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes.
+** The code is written so that the hard lower-limit for each of these values
+** is 1. Clearly such small values would be inefficient, but can be useful
+** for testing purposes.
+**
+** If this module is built with SQLITE_TEST defined, these constants may
+** be overridden at runtime for testing purposes. File fts3_test.c contains
+** a Tcl interface to read and write the values.
+*/
+#ifdef SQLITE_TEST
+int test_fts3_node_chunksize = (4*1024);
+int test_fts3_node_chunk_threshold = (4*1024)*4;
+# define FTS3_NODE_CHUNKSIZE test_fts3_node_chunksize
+# define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold
+#else
+# define FTS3_NODE_CHUNKSIZE (4*1024)
+# define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4)
+#endif
+
+typedef struct PendingList PendingList;
+typedef struct SegmentNode SegmentNode;
+typedef struct SegmentWriter SegmentWriter;
+
+/*
+** An instance of the following data structure is used to build doclists
+** incrementally. See function fts3PendingListAppend() for details.
+*/
+struct PendingList {
+ int nData;
+ char *aData;
+ int nSpace;
+ sqlite3_int64 iLastDocid;
+ sqlite3_int64 iLastCol;
+ sqlite3_int64 iLastPos;
+};
+
+
+/*
+** Each cursor has a (possibly empty) linked list of the following objects.
+*/
+struct Fts3DeferredToken {
+ Fts3PhraseToken *pToken; /* Pointer to corresponding expr token */
+ int iCol; /* Column token must occur in */
+ Fts3DeferredToken *pNext; /* Next in list of deferred tokens */
+ PendingList *pList; /* Doclist is assembled here */
+};
+
+/*
+** An instance of this structure is used to iterate through the terms on
+** a contiguous set of segment b-tree leaf nodes. Although the details of
+** this structure are only manipulated by code in this file, opaque handles
+** of type Fts3SegReader* are also used by code in fts3.c to iterate through
+** terms when querying the full-text index. See functions:
+**
+** sqlite3Fts3SegReaderNew()
+** sqlite3Fts3SegReaderFree()
+** sqlite3Fts3SegReaderIterate()
+**
+** Methods used to manipulate Fts3SegReader structures:
+**
+** fts3SegReaderNext()
+** fts3SegReaderFirstDocid()
+** fts3SegReaderNextDocid()
+*/
+struct Fts3SegReader {
+ int iIdx; /* Index within level, or 0x7FFFFFFF for PT */
+
+ sqlite3_int64 iStartBlock; /* Rowid of first leaf block to traverse */
+ sqlite3_int64 iLeafEndBlock; /* Rowid of final leaf block to traverse */
+ sqlite3_int64 iEndBlock; /* Rowid of final block in segment (or 0) */
+ sqlite3_int64 iCurrentBlock; /* Current leaf block (or 0) */
+
+ char *aNode; /* Pointer to node data (or NULL) */
+ int nNode; /* Size of buffer at aNode (or 0) */
+ int nPopulate; /* If >0, bytes of buffer aNode[] loaded */
+ sqlite3_blob *pBlob; /* If not NULL, blob handle to read node */
+
+ Fts3HashElem **ppNextElem;
+
+ /* Variables set by fts3SegReaderNext(). These may be read directly
+ ** by the caller. They are valid from the time SegmentReaderNew() returns
+ ** until SegmentReaderNext() returns something other than SQLITE_OK
+ ** (i.e. SQLITE_DONE).
+ */
+ int nTerm; /* Number of bytes in current term */
+ char *zTerm; /* Pointer to current term */
+ int nTermAlloc; /* Allocated size of zTerm buffer */
+ char *aDoclist; /* Pointer to doclist of current entry */
+ int nDoclist; /* Size of doclist in current entry */
+
+ /* The following variables are used by fts3SegReaderNextDocid() to iterate
+ ** through the current doclist (aDoclist/nDoclist).
+ */
+ char *pOffsetList;
+ int nOffsetList; /* For descending pending seg-readers only */
+ sqlite3_int64 iDocid;
+};
+
+#define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0)
+#define fts3SegReaderIsRootOnly(p) ((p)->aNode==(char *)&(p)[1])
+
+/*
+** An instance of this structure is used to create a segment b-tree in the
+** database. The internal details of this type are only accessed by the
+** following functions:
+**
+** fts3SegWriterAdd()
+** fts3SegWriterFlush()
+** fts3SegWriterFree()
+*/
+struct SegmentWriter {
+ SegmentNode *pTree; /* Pointer to interior tree structure */
+ sqlite3_int64 iFirst; /* First slot in %_segments written */
+ sqlite3_int64 iFree; /* Next free slot in %_segments */
+ char *zTerm; /* Pointer to previous term buffer */
+ int nTerm; /* Number of bytes in zTerm */
+ int nMalloc; /* Size of malloc'd buffer at zMalloc */
+ char *zMalloc; /* Malloc'd space (possibly) used for zTerm */
+ int nSize; /* Size of allocation at aData */
+ int nData; /* Bytes of data in aData */
+ char *aData; /* Pointer to block from malloc() */
+};
+
+/*
+** Type SegmentNode is used by the following three functions to create
+** the interior part of the segment b+-tree structures (everything except
+** the leaf nodes). These functions and type are only ever used by code
+** within the fts3SegWriterXXX() family of functions described above.
+**
+** fts3NodeAddTerm()
+** fts3NodeWrite()
+** fts3NodeFree()
+**
+** When a b+tree is written to the database (either as a result of a merge
+** or the pending-terms table being flushed), leaves are written into the
+** database file as soon as they are completely populated. The interior of
+** the tree is assembled in memory and written out only once all leaves have
+** been populated and stored. This is Ok, as the b+-tree fanout is usually
+** very large, meaning that the interior of the tree consumes relatively
+** little memory.
+*/
+struct SegmentNode {
+ SegmentNode *pParent; /* Parent node (or NULL for root node) */
+ SegmentNode *pRight; /* Pointer to right-sibling */
+ SegmentNode *pLeftmost; /* Pointer to left-most node of this depth */
+ int nEntry; /* Number of terms written to node so far */
+ char *zTerm; /* Pointer to previous term buffer */
+ int nTerm; /* Number of bytes in zTerm */
+ int nMalloc; /* Size of malloc'd buffer at zMalloc */
+ char *zMalloc; /* Malloc'd space (possibly) used for zTerm */
+ int nData; /* Bytes of valid data so far */
+ char *aData; /* Node data */
+};
+
+/*
+** Valid values for the second argument to fts3SqlStmt().
+*/
+#define SQL_DELETE_CONTENT 0
+#define SQL_IS_EMPTY 1
+#define SQL_DELETE_ALL_CONTENT 2
+#define SQL_DELETE_ALL_SEGMENTS 3
+#define SQL_DELETE_ALL_SEGDIR 4
+#define SQL_DELETE_ALL_DOCSIZE 5
+#define SQL_DELETE_ALL_STAT 6
+#define SQL_SELECT_CONTENT_BY_ROWID 7
+#define SQL_NEXT_SEGMENT_INDEX 8
+#define SQL_INSERT_SEGMENTS 9
+#define SQL_NEXT_SEGMENTS_ID 10
+#define SQL_INSERT_SEGDIR 11
+#define SQL_SELECT_LEVEL 12
+#define SQL_SELECT_LEVEL_RANGE 13
+#define SQL_SELECT_LEVEL_COUNT 14
+#define SQL_SELECT_SEGDIR_MAX_LEVEL 15
+#define SQL_DELETE_SEGDIR_LEVEL 16
+#define SQL_DELETE_SEGMENTS_RANGE 17
+#define SQL_CONTENT_INSERT 18
+#define SQL_DELETE_DOCSIZE 19
+#define SQL_REPLACE_DOCSIZE 20
+#define SQL_SELECT_DOCSIZE 21
+#define SQL_SELECT_DOCTOTAL 22
+#define SQL_REPLACE_DOCTOTAL 23
+
+#define SQL_SELECT_ALL_PREFIX_LEVEL 24
+#define SQL_DELETE_ALL_TERMS_SEGDIR 25
+
+#define SQL_DELETE_SEGDIR_RANGE 26
+
+/*
+** This function is used to obtain an SQLite prepared statement handle
+** for the statement identified by the second argument. If successful,
+** *pp is set to the requested statement handle and SQLITE_OK returned.
+** Otherwise, an SQLite error code is returned and *pp is set to 0.
+**
+** If argument apVal is not NULL, then it must point to an array with
+** at least as many entries as the requested statement has bound
+** parameters. The values are bound to the statements parameters before
+** returning.
+*/
+static int fts3SqlStmt(
+ Fts3Table *p, /* Virtual table handle */
+ int eStmt, /* One of the SQL_XXX constants above */
+ sqlite3_stmt **pp, /* OUT: Statement handle */
+ sqlite3_value **apVal /* Values to bind to statement */
+){
+ const char *azSql[] = {
+/* 0 */ "DELETE FROM %Q.'%q_content' WHERE rowid = ?",
+/* 1 */ "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)",
+/* 2 */ "DELETE FROM %Q.'%q_content'",
+/* 3 */ "DELETE FROM %Q.'%q_segments'",
+/* 4 */ "DELETE FROM %Q.'%q_segdir'",
+/* 5 */ "DELETE FROM %Q.'%q_docsize'",
+/* 6 */ "DELETE FROM %Q.'%q_stat'",
+/* 7 */ "SELECT %s FROM %Q.'%q_content' AS x WHERE rowid=?",
+/* 8 */ "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1",
+/* 9 */ "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)",
+/* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)",
+/* 11 */ "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)",
+
+ /* Return segments in order from oldest to newest.*/
+/* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root "
+ "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC",
+/* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root "
+ "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?"
+ "ORDER BY level DESC, idx ASC",
+
+/* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?",
+/* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?",
+
+/* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?",
+/* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?",
+/* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%s)",
+/* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?",
+/* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",
+/* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
+/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0",
+/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)",
+/* 24 */ "",
+/* 25 */ "",
+
+/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?",
+
+ };
+ int rc = SQLITE_OK;
+ sqlite3_stmt *pStmt;
+
+ assert( SizeofArray(azSql)==SizeofArray(p->aStmt) );
+ assert( eStmt<SizeofArray(azSql) && eStmt>=0 );
+
+ pStmt = p->aStmt[eStmt];
+ if( !pStmt ){
+ char *zSql;
+ if( eStmt==SQL_CONTENT_INSERT ){
+ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist);
+ }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){
+ zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist, p->zDb, p->zName);
+ }else{
+ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName);
+ }
+ if( !zSql ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL);
+ sqlite3_free(zSql);
+ assert( rc==SQLITE_OK || pStmt==0 );
+ p->aStmt[eStmt] = pStmt;
+ }
+ }
+ if( apVal ){
+ int i;
+ int nParam = sqlite3_bind_parameter_count(pStmt);
+ for(i=0; rc==SQLITE_OK && i<nParam; i++){
+ rc = sqlite3_bind_value(pStmt, i+1, apVal[i]);
+ }
+ }
+ *pp = pStmt;
+ return rc;
+}
+
+static int fts3SelectDocsize(
+ Fts3Table *pTab, /* FTS3 table handle */
+ int eStmt, /* Either SQL_SELECT_DOCSIZE or DOCTOTAL */
+ sqlite3_int64 iDocid, /* Docid to bind for SQL_SELECT_DOCSIZE */
+ sqlite3_stmt **ppStmt /* OUT: Statement handle */
+){
+ sqlite3_stmt *pStmt = 0; /* Statement requested from fts3SqlStmt() */
+ int rc; /* Return code */
+
+ assert( eStmt==SQL_SELECT_DOCSIZE || eStmt==SQL_SELECT_DOCTOTAL );
+
+ rc = fts3SqlStmt(pTab, eStmt, &pStmt, 0);
+ if( rc==SQLITE_OK ){
+ if( eStmt==SQL_SELECT_DOCSIZE ){
+ sqlite3_bind_int64(pStmt, 1, iDocid);
+ }
+ rc = sqlite3_step(pStmt);
+ if( rc!=SQLITE_ROW || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){
+ rc = sqlite3_reset(pStmt);
+ if( rc==SQLITE_OK ) rc = SQLITE_CORRUPT_VTAB;
+ pStmt = 0;
+ }else{
+ rc = SQLITE_OK;
+ }
+ }
+
+ *ppStmt = pStmt;
+ return rc;
+}
+
+int sqlite3Fts3SelectDoctotal(
+ Fts3Table *pTab, /* Fts3 table handle */
+ sqlite3_stmt **ppStmt /* OUT: Statement handle */
+){
+ return fts3SelectDocsize(pTab, SQL_SELECT_DOCTOTAL, 0, ppStmt);
+}
+
+int sqlite3Fts3SelectDocsize(
+ Fts3Table *pTab, /* Fts3 table handle */
+ sqlite3_int64 iDocid, /* Docid to read size data for */
+ sqlite3_stmt **ppStmt /* OUT: Statement handle */
+){
+ return fts3SelectDocsize(pTab, SQL_SELECT_DOCSIZE, iDocid, ppStmt);
+}
+
+/*
+** Similar to fts3SqlStmt(). Except, after binding the parameters in
+** array apVal[] to the SQL statement identified by eStmt, the statement
+** is executed.
+**
+** Returns SQLITE_OK if the statement is successfully executed, or an
+** SQLite error code otherwise.
+*/
+static void fts3SqlExec(
+ int *pRC, /* Result code */
+ Fts3Table *p, /* The FTS3 table */
+ int eStmt, /* Index of statement to evaluate */
+ sqlite3_value **apVal /* Parameters to bind */
+){
+ sqlite3_stmt *pStmt;
+ int rc;
+ if( *pRC ) return;
+ rc = fts3SqlStmt(p, eStmt, &pStmt, apVal);
+ if( rc==SQLITE_OK ){
+ sqlite3_step(pStmt);
+ rc = sqlite3_reset(pStmt);
+ }
+ *pRC = rc;
+}
+
+
+/*
+** This function ensures that the caller has obtained a shared-cache
+** table-lock on the %_content table. This is required before reading
+** data from the fts3 table. If this lock is not acquired first, then
+** the caller may end up holding read-locks on the %_segments and %_segdir
+** tables, but no read-lock on the %_content table. If this happens
+** a second connection will be able to write to the fts3 table, but
+** attempting to commit those writes might return SQLITE_LOCKED or
+** SQLITE_LOCKED_SHAREDCACHE (because the commit attempts to obtain
+** write-locks on the %_segments and %_segdir ** tables).
+**
+** We try to avoid this because if FTS3 returns any error when committing
+** a transaction, the whole transaction will be rolled back. And this is
+** not what users expect when they get SQLITE_LOCKED_SHAREDCACHE. It can
+** still happen if the user reads data directly from the %_segments or
+** %_segdir tables instead of going through FTS3 though.
+*/
+int sqlite3Fts3ReadLock(Fts3Table *p){
+ int rc; /* Return code */
+ sqlite3_stmt *pStmt; /* Statement used to obtain lock */
+
+ rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pStmt, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_null(pStmt, 1);
+ sqlite3_step(pStmt);
+ rc = sqlite3_reset(pStmt);
+ }
+ return rc;
+}
+
+/*
+** Set *ppStmt to a statement handle that may be used to iterate through
+** all rows in the %_segdir table, from oldest to newest. If successful,
+** return SQLITE_OK. If an error occurs while preparing the statement,
+** return an SQLite error code.
+**
+** There is only ever one instance of this SQL statement compiled for
+** each FTS3 table.
+**
+** The statement returns the following columns from the %_segdir table:
+**
+** 0: idx
+** 1: start_block
+** 2: leaves_end_block
+** 3: end_block
+** 4: root
+*/
+int sqlite3Fts3AllSegdirs(
+ Fts3Table *p, /* FTS3 table */
+ int iIndex, /* Index for p->aIndex[] */
+ int iLevel, /* Level to select */
+ sqlite3_stmt **ppStmt /* OUT: Compiled statement */
+){
+ int rc;
+ sqlite3_stmt *pStmt = 0;
+
+ assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 );
+ assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
+ assert( iIndex>=0 && iIndex<p->nIndex );
+
+ if( iLevel<0 ){
+ /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */
+ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL);
+ sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL-1);
+ }
+ }else{
+ /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */
+ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int(pStmt, 1, iLevel+iIndex*FTS3_SEGDIR_MAXLEVEL);
+ }
+ }
+ *ppStmt = pStmt;
+ return rc;
+}
+
+
+/*
+** Append a single varint to a PendingList buffer. SQLITE_OK is returned
+** if successful, or an SQLite error code otherwise.
+**
+** This function also serves to allocate the PendingList structure itself.
+** For example, to create a new PendingList structure containing two
+** varints:
+**
+** PendingList *p = 0;
+** fts3PendingListAppendVarint(&p, 1);
+** fts3PendingListAppendVarint(&p, 2);
+*/
+static int fts3PendingListAppendVarint(
+ PendingList **pp, /* IN/OUT: Pointer to PendingList struct */
+ sqlite3_int64 i /* Value to append to data */
+){
+ PendingList *p = *pp;
+
+ /* Allocate or grow the PendingList as required. */
+ if( !p ){
+ p = sqlite3_malloc(sizeof(*p) + 100);
+ if( !p ){
+ return SQLITE_NOMEM;
+ }
+ p->nSpace = 100;
+ p->aData = (char *)&p[1];
+ p->nData = 0;
+ }
+ else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){
+ int nNew = p->nSpace * 2;
+ p = sqlite3_realloc(p, sizeof(*p) + nNew);
+ if( !p ){
+ sqlite3_free(*pp);
+ *pp = 0;
+ return SQLITE_NOMEM;
+ }
+ p->nSpace = nNew;
+ p->aData = (char *)&p[1];
+ }
+
+ /* Append the new serialized varint to the end of the list. */
+ p->nData += sqlite3Fts3PutVarint(&p->aData[p->nData], i);
+ p->aData[p->nData] = '\0';
+ *pp = p;
+ return SQLITE_OK;
+}
+
+/*
+** Add a docid/column/position entry to a PendingList structure. Non-zero
+** is returned if the structure is sqlite3_realloced as part of adding
+** the entry. Otherwise, zero.
+**
+** If an OOM error occurs, *pRc is set to SQLITE_NOMEM before returning.
+** Zero is always returned in this case. Otherwise, if no OOM error occurs,
+** it is set to SQLITE_OK.
+*/
+static int fts3PendingListAppend(
+ PendingList **pp, /* IN/OUT: PendingList structure */
+ sqlite3_int64 iDocid, /* Docid for entry to add */
+ sqlite3_int64 iCol, /* Column for entry to add */
+ sqlite3_int64 iPos, /* Position of term for entry to add */
+ int *pRc /* OUT: Return code */
+){
+ PendingList *p = *pp;
+ int rc = SQLITE_OK;
+
+ assert( !p || p->iLastDocid<=iDocid );
+
+ if( !p || p->iLastDocid!=iDocid ){
+ sqlite3_int64 iDelta = iDocid - (p ? p->iLastDocid : 0);
+ if( p ){
+ assert( p->nData<p->nSpace );
+ assert( p->aData[p->nData]==0 );
+ p->nData++;
+ }
+ if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iDelta)) ){
+ goto pendinglistappend_out;
+ }
+ p->iLastCol = -1;
+ p->iLastPos = 0;
+ p->iLastDocid = iDocid;
+ }
+ if( iCol>0 && p->iLastCol!=iCol ){
+ if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, 1))
+ || SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iCol))
+ ){
+ goto pendinglistappend_out;
+ }
+ p->iLastCol = iCol;
+ p->iLastPos = 0;
+ }
+ if( iCol>=0 ){
+ assert( iPos>p->iLastPos || (iPos==0 && p->iLastPos==0) );
+ rc = fts3PendingListAppendVarint(&p, 2+iPos-p->iLastPos);
+ if( rc==SQLITE_OK ){
+ p->iLastPos = iPos;
+ }
+ }
+
+ pendinglistappend_out:
+ *pRc = rc;
+ if( p!=*pp ){
+ *pp = p;
+ return 1;
+ }
+ return 0;
+}
+
+/*
+** Free a PendingList object allocated by fts3PendingListAppend().
+*/
+static void fts3PendingListDelete(PendingList *pList){
+ sqlite3_free(pList);
+}
+
+/*
+** Add an entry to one of the pending-terms hash tables.
+*/
+static int fts3PendingTermsAddOne(
+ Fts3Table *p,
+ int iCol,
+ int iPos,
+ Fts3Hash *pHash, /* Pending terms hash table to add entry to */
+ const char *zToken,
+ int nToken
+){
+ PendingList *pList;
+ int rc = SQLITE_OK;
+
+ pList = (PendingList *)fts3HashFind(pHash, zToken, nToken);
+ if( pList ){
+ p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem));
+ }
+ if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){
+ if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){
+ /* Malloc failed while inserting the new entry. This can only
+ ** happen if there was no previous entry for this token.
+ */
+ assert( 0==fts3HashFind(pHash, zToken, nToken) );
+ sqlite3_free(pList);
+ rc = SQLITE_NOMEM;
+ }
+ }
+ if( rc==SQLITE_OK ){
+ p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem));
+ }
+ return rc;
+}
+
+/*
+** Tokenize the nul-terminated string zText and add all tokens to the
+** pending-terms hash-table. The docid used is that currently stored in
+** p->iPrevDocid, and the column is specified by argument iCol.
+**
+** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code.
+*/
+static int fts3PendingTermsAdd(
+ Fts3Table *p, /* Table into which text will be inserted */
+ const char *zText, /* Text of document to be inserted */
+ int iCol, /* Column into which text is being inserted */
+ u32 *pnWord /* OUT: Number of tokens inserted */
+){
+ int rc;
+ int iStart;
+ int iEnd;
+ int iPos;
+ int nWord = 0;
+
+ char const *zToken;
+ int nToken;
+
+ sqlite3_tokenizer *pTokenizer = p->pTokenizer;
+ sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
+ sqlite3_tokenizer_cursor *pCsr;
+ int (*xNext)(sqlite3_tokenizer_cursor *pCursor,
+ const char**,int*,int*,int*,int*);
+
+ assert( pTokenizer && pModule );
+
+ /* If the user has inserted a NULL value, this function may be called with
+ ** zText==0. In this case, add zero token entries to the hash table and
+ ** return early. */
+ if( zText==0 ){
+ *pnWord = 0;
+ return SQLITE_OK;
+ }
+
+ rc = pModule->xOpen(pTokenizer, zText, -1, &pCsr);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+ pCsr->pTokenizer = pTokenizer;
+
+ xNext = pModule->xNext;
+ while( SQLITE_OK==rc
+ && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos))
+ ){
+ int i;
+ if( iPos>=nWord ) nWord = iPos+1;
+
+ /* Positions cannot be negative; we use -1 as a terminator internally.
+ ** Tokens must have a non-zero length.
+ */
+ if( iPos<0 || !zToken || nToken<=0 ){
+ rc = SQLITE_ERROR;
+ break;
+ }
+
+ /* Add the term to the terms index */
+ rc = fts3PendingTermsAddOne(
+ p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken
+ );
+
+ /* Add the term to each of the prefix indexes that it is not too
+ ** short for. */
+ for(i=1; rc==SQLITE_OK && i<p->nIndex; i++){
+ struct Fts3Index *pIndex = &p->aIndex[i];
+ if( nToken<pIndex->nPrefix ) continue;
+ rc = fts3PendingTermsAddOne(
+ p, iCol, iPos, &pIndex->hPending, zToken, pIndex->nPrefix
+ );
+ }
+ }
+
+ pModule->xClose(pCsr);
+ *pnWord = nWord;
+ return (rc==SQLITE_DONE ? SQLITE_OK : rc);
+}
+
+/*
+** Calling this function indicates that subsequent calls to
+** fts3PendingTermsAdd() are to add term/position-list pairs for the
+** contents of the document with docid iDocid.
+*/
+static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){
+ /* TODO(shess) Explore whether partially flushing the buffer on
+ ** forced-flush would provide better performance. I suspect that if
+ ** we ordered the doclists by size and flushed the largest until the
+ ** buffer was half empty, that would let the less frequent terms
+ ** generate longer doclists.
+ */
+ if( iDocid<=p->iPrevDocid || p->nPendingData>p->nMaxPendingData ){
+ int rc = sqlite3Fts3PendingTermsFlush(p);
+ if( rc!=SQLITE_OK ) return rc;
+ }
+ p->iPrevDocid = iDocid;
+ return SQLITE_OK;
+}
+
+/*
+** Discard the contents of the pending-terms hash tables.
+*/
+void sqlite3Fts3PendingTermsClear(Fts3Table *p){
+ int i;
+ for(i=0; i<p->nIndex; i++){
+ Fts3HashElem *pElem;
+ Fts3Hash *pHash = &p->aIndex[i].hPending;
+ for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){
+ PendingList *pList = (PendingList *)fts3HashData(pElem);
+ fts3PendingListDelete(pList);
+ }
+ fts3HashClear(pHash);
+ }
+ p->nPendingData = 0;
+}
+
+/*
+** This function is called by the xUpdate() method as part of an INSERT
+** operation. It adds entries for each term in the new record to the
+** pendingTerms hash table.
+**
+** Argument apVal is the same as the similarly named argument passed to
+** fts3InsertData(). Parameter iDocid is the docid of the new row.
+*/
+static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal, u32 *aSz){
+ int i; /* Iterator variable */
+ for(i=2; i<p->nColumn+2; i++){
+ const char *zText = (const char *)sqlite3_value_text(apVal[i]);
+ int rc = fts3PendingTermsAdd(p, zText, i-2, &aSz[i-2]);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+ aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]);
+ }
+ return SQLITE_OK;
+}
+
+/*
+** This function is called by the xUpdate() method for an INSERT operation.
+** The apVal parameter is passed a copy of the apVal argument passed by
+** SQLite to the xUpdate() method. i.e:
+**
+** apVal[0] Not used for INSERT.
+** apVal[1] rowid
+** apVal[2] Left-most user-defined column
+** ...
+** apVal[p->nColumn+1] Right-most user-defined column
+** apVal[p->nColumn+2] Hidden column with same name as table
+** apVal[p->nColumn+3] Hidden "docid" column (alias for rowid)
+*/
+static int fts3InsertData(
+ Fts3Table *p, /* Full-text table */
+ sqlite3_value **apVal, /* Array of values to insert */
+ sqlite3_int64 *piDocid /* OUT: Docid for row just inserted */
+){
+ int rc; /* Return code */
+ sqlite3_stmt *pContentInsert; /* INSERT INTO %_content VALUES(...) */
+
+ /* Locate the statement handle used to insert data into the %_content
+ ** table. The SQL for this statement is:
+ **
+ ** INSERT INTO %_content VALUES(?, ?, ?, ...)
+ **
+ ** The statement features N '?' variables, where N is the number of user
+ ** defined columns in the FTS3 table, plus one for the docid field.
+ */
+ rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+
+ /* There is a quirk here. The users INSERT statement may have specified
+ ** a value for the "rowid" field, for the "docid" field, or for both.
+ ** Which is a problem, since "rowid" and "docid" are aliases for the
+ ** same value. For example:
+ **
+ ** INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2);
+ **
+ ** In FTS3, this is an error. It is an error to specify non-NULL values
+ ** for both docid and some other rowid alias.
+ */
+ if( SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){
+ if( SQLITE_NULL==sqlite3_value_type(apVal[0])
+ && SQLITE_NULL!=sqlite3_value_type(apVal[1])
+ ){
+ /* A rowid/docid conflict. */
+ return SQLITE_ERROR;
+ }
+ rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]);
+ if( rc!=SQLITE_OK ) return rc;
+ }
+
+ /* Execute the statement to insert the record. Set *piDocid to the
+ ** new docid value.
+ */
+ sqlite3_step(pContentInsert);
+ rc = sqlite3_reset(pContentInsert);
+
+ *piDocid = sqlite3_last_insert_rowid(p->db);
+ return rc;
+}
+
+
+
+/*
+** Remove all data from the FTS3 table. Clear the hash table containing
+** pending terms.
+*/
+static int fts3DeleteAll(Fts3Table *p){
+ int rc = SQLITE_OK; /* Return code */
+
+ /* Discard the contents of the pending-terms hash table. */
+ sqlite3Fts3PendingTermsClear(p);
+
+ /* Delete everything from the %_content, %_segments and %_segdir tables. */
+ fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0);
+ fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGMENTS, 0);
+ fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0);
+ if( p->bHasDocsize ){
+ fts3SqlExec(&rc, p, SQL_DELETE_ALL_DOCSIZE, 0);
+ }
+ if( p->bHasStat ){
+ fts3SqlExec(&rc, p, SQL_DELETE_ALL_STAT, 0);
+ }
+ return rc;
+}
+
+/*
+** The first element in the apVal[] array is assumed to contain the docid
+** (an integer) of a row about to be deleted. Remove all terms from the
+** full-text index.
+*/
+static void fts3DeleteTerms(
+ int *pRC, /* Result code */
+ Fts3Table *p, /* The FTS table to delete from */
+ sqlite3_value *pRowid, /* The docid to be deleted */
+ u32 *aSz /* Sizes of deleted document written here */
+){
+ int rc;
+ sqlite3_stmt *pSelect;
+
+ if( *pRC ) return;
+ rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, &pRowid);
+ if( rc==SQLITE_OK ){
+ if( SQLITE_ROW==sqlite3_step(pSelect) ){
+ int i;
+ for(i=1; i<=p->nColumn; i++){
+ const char *zText = (const char *)sqlite3_column_text(pSelect, i);
+ rc = fts3PendingTermsAdd(p, zText, -1, &aSz[i-1]);
+ if( rc!=SQLITE_OK ){
+ sqlite3_reset(pSelect);
+ *pRC = rc;
+ return;
+ }
+ aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i);
+ }
+ }
+ rc = sqlite3_reset(pSelect);
+ }else{
+ sqlite3_reset(pSelect);
+ }
+ *pRC = rc;
+}
+
+/*
+** Forward declaration to account for the circular dependency between
+** functions fts3SegmentMerge() and fts3AllocateSegdirIdx().
+*/
+static int fts3SegmentMerge(Fts3Table *, int, int);
+
+/*
+** This function allocates a new level iLevel index in the segdir table.
+** Usually, indexes are allocated within a level sequentially starting
+** with 0, so the allocated index is one greater than the value returned
+** by:
+**
+** SELECT max(idx) FROM %_segdir WHERE level = :iLevel
+**
+** However, if there are already FTS3_MERGE_COUNT indexes at the requested
+** level, they are merged into a single level (iLevel+1) segment and the
+** allocated index is 0.
+**
+** If successful, *piIdx is set to the allocated index slot and SQLITE_OK
+** returned. Otherwise, an SQLite error code is returned.
+*/
+static int fts3AllocateSegdirIdx(
+ Fts3Table *p,
+ int iIndex, /* Index for p->aIndex */
+ int iLevel,
+ int *piIdx
+){
+ int rc; /* Return Code */
+ sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */
+ int iNext = 0; /* Result of query pNextIdx */
+
+ /* Set variable iNext to the next available segdir index at level iLevel. */
+ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int(pNextIdx, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel);
+ if( SQLITE_ROW==sqlite3_step(pNextIdx) ){
+ iNext = sqlite3_column_int(pNextIdx, 0);
+ }
+ rc = sqlite3_reset(pNextIdx);
+ }
+
+ if( rc==SQLITE_OK ){
+ /* If iNext is FTS3_MERGE_COUNT, indicating that level iLevel is already
+ ** full, merge all segments in level iLevel into a single iLevel+1
+ ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise,
+ ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext.
+ */
+ if( iNext>=FTS3_MERGE_COUNT ){
+ rc = fts3SegmentMerge(p, iIndex, iLevel);
+ *piIdx = 0;
+ }else{
+ *piIdx = iNext;
+ }
+ }
+
+ return rc;
+}
+
+/*
+** The %_segments table is declared as follows:
+**
+** CREATE TABLE %_segments(blockid INTEGER PRIMARY KEY, block BLOB)
+**
+** This function reads data from a single row of the %_segments table. The
+** specific row is identified by the iBlockid parameter. If paBlob is not
+** NULL, then a buffer is allocated using sqlite3_malloc() and populated
+** with the contents of the blob stored in the "block" column of the
+** identified table row is. Whether or not paBlob is NULL, *pnBlob is set
+** to the size of the blob in bytes before returning.
+**
+** If an error occurs, or the table does not contain the specified row,
+** an SQLite error code is returned. Otherwise, SQLITE_OK is returned. If
+** paBlob is non-NULL, then it is the responsibility of the caller to
+** eventually free the returned buffer.
+**
+** This function may leave an open sqlite3_blob* handle in the
+** Fts3Table.pSegments variable. This handle is reused by subsequent calls
+** to this function. The handle may be closed by calling the
+** sqlite3Fts3SegmentsClose() function. Reusing a blob handle is a handy
+** performance improvement, but the blob handle should always be closed
+** before control is returned to the user (to prevent a lock being held
+** on the database file for longer than necessary). Thus, any virtual table
+** method (xFilter etc.) that may directly or indirectly call this function
+** must call sqlite3Fts3SegmentsClose() before returning.
+*/
+int sqlite3Fts3ReadBlock(
+ Fts3Table *p, /* FTS3 table handle */
+ sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */
+ char **paBlob, /* OUT: Blob data in malloc'd buffer */
+ int *pnBlob, /* OUT: Size of blob data */
+ int *pnLoad /* OUT: Bytes actually loaded */
+){
+ int rc; /* Return code */
+
+ /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */
+ assert( pnBlob);
+
+ if( p->pSegments ){
+ rc = sqlite3_blob_reopen(p->pSegments, iBlockid);
+ }else{
+ if( 0==p->zSegmentsTbl ){
+ p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName);
+ if( 0==p->zSegmentsTbl ) return SQLITE_NOMEM;
+ }
+ rc = sqlite3_blob_open(
+ p->db, p->zDb, p->zSegmentsTbl, "block", iBlockid, 0, &p->pSegments
+ );
+ }
+
+ if( rc==SQLITE_OK ){
+ int nByte = sqlite3_blob_bytes(p->pSegments);
+ *pnBlob = nByte;
+ if( paBlob ){
+ char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING);
+ if( !aByte ){
+ rc = SQLITE_NOMEM;
+ }else{
+ if( pnLoad && nByte>(FTS3_NODE_CHUNK_THRESHOLD) ){
+ nByte = FTS3_NODE_CHUNKSIZE;
+ *pnLoad = nByte;
+ }
+ rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0);
+ memset(&aByte[nByte], 0, FTS3_NODE_PADDING);
+ if( rc!=SQLITE_OK ){
+ sqlite3_free(aByte);
+ aByte = 0;
+ }
+ }
+ *paBlob = aByte;
+ }
+ }
+
+ return rc;
+}
+
+/*
+** Close the blob handle at p->pSegments, if it is open. See comments above
+** the sqlite3Fts3ReadBlock() function for details.
+*/
+void sqlite3Fts3SegmentsClose(Fts3Table *p){
+ sqlite3_blob_close(p->pSegments);
+ p->pSegments = 0;
+}
+
+static int fts3SegReaderIncrRead(Fts3SegReader *pReader){
+ int nRead; /* Number of bytes to read */
+ int rc; /* Return code */
+
+ nRead = MIN(pReader->nNode - pReader->nPopulate, FTS3_NODE_CHUNKSIZE);
+ rc = sqlite3_blob_read(
+ pReader->pBlob,
+ &pReader->aNode[pReader->nPopulate],
+ nRead,
+ pReader->nPopulate
+ );
+
+ if( rc==SQLITE_OK ){
+ pReader->nPopulate += nRead;
+ memset(&pReader->aNode[pReader->nPopulate], 0, FTS3_NODE_PADDING);
+ if( pReader->nPopulate==pReader->nNode ){
+ sqlite3_blob_close(pReader->pBlob);
+ pReader->pBlob = 0;
+ pReader->nPopulate = 0;
+ }
+ }
+ return rc;
+}
+
+static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){
+ int rc = SQLITE_OK;
+ assert( !pReader->pBlob
+ || (pFrom>=pReader->aNode && pFrom<&pReader->aNode[pReader->nNode])
+ );
+ while( pReader->pBlob && rc==SQLITE_OK
+ && (pFrom - pReader->aNode + nByte)>pReader->nPopulate
+ ){
+ rc = fts3SegReaderIncrRead(pReader);
+ }
+ return rc;
+}
+
+/*
+** Move the iterator passed as the first argument to the next term in the
+** segment. If successful, SQLITE_OK is returned. If there is no next term,
+** SQLITE_DONE. Otherwise, an SQLite error code.
+*/
+static int fts3SegReaderNext(
+ Fts3Table *p,
+ Fts3SegReader *pReader,
+ int bIncr
+){
+ int rc; /* Return code of various sub-routines */
+ char *pNext; /* Cursor variable */
+ int nPrefix; /* Number of bytes in term prefix */
+ int nSuffix; /* Number of bytes in term suffix */
+
+ if( !pReader->aDoclist ){
+ pNext = pReader->aNode;
+ }else{
+ pNext = &pReader->aDoclist[pReader->nDoclist];
+ }
+
+ if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){
+
+ if( fts3SegReaderIsPending(pReader) ){
+ Fts3HashElem *pElem = *(pReader->ppNextElem);
+ if( pElem==0 ){
+ pReader->aNode = 0;
+ }else{
+ PendingList *pList = (PendingList *)fts3HashData(pElem);
+ pReader->zTerm = (char *)fts3HashKey(pElem);
+ pReader->nTerm = fts3HashKeysize(pElem);
+ pReader->nNode = pReader->nDoclist = pList->nData + 1;
+ pReader->aNode = pReader->aDoclist = pList->aData;
+ pReader->ppNextElem++;
+ assert( pReader->aNode );
+ }
+ return SQLITE_OK;
+ }
+
+ if( !fts3SegReaderIsRootOnly(pReader) ){
+ sqlite3_free(pReader->aNode);
+ sqlite3_blob_close(pReader->pBlob);
+ pReader->pBlob = 0;
+ }
+ pReader->aNode = 0;
+
+ /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf
+ ** blocks have already been traversed. */
+ assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock );
+ if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){
+ return SQLITE_OK;
+ }
+
+ rc = sqlite3Fts3ReadBlock(
+ p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode,
+ (bIncr ? &pReader->nPopulate : 0)
+ );
+ if( rc!=SQLITE_OK ) return rc;
+ assert( pReader->pBlob==0 );
+ if( bIncr && pReader->nPopulate<pReader->nNode ){
+ pReader->pBlob = p->pSegments;
+ p->pSegments = 0;
+ }
+ pNext = pReader->aNode;
+ }
+
+ assert( !fts3SegReaderIsPending(pReader) );
+
+ rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2);
+ if( rc!=SQLITE_OK ) return rc;
+
+ /* Because of the FTS3_NODE_PADDING bytes of padding, the following is
+ ** safe (no risk of overread) even if the node data is corrupted. */
+ pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix);
+ pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix);
+ if( nPrefix<0 || nSuffix<=0
+ || &pNext[nSuffix]>&pReader->aNode[pReader->nNode]
+ ){
+ return SQLITE_CORRUPT_VTAB;
+ }
+
+ if( nPrefix+nSuffix>pReader->nTermAlloc ){
+ int nNew = (nPrefix+nSuffix)*2;
+ char *zNew = sqlite3_realloc(pReader->zTerm, nNew);
+ if( !zNew ){
+ return SQLITE_NOMEM;
+ }
+ pReader->zTerm = zNew;
+ pReader->nTermAlloc = nNew;
+ }
+
+ rc = fts3SegReaderRequire(pReader, pNext, nSuffix+FTS3_VARINT_MAX);
+ if( rc!=SQLITE_OK ) return rc;
+
+ memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix);
+ pReader->nTerm = nPrefix+nSuffix;
+ pNext += nSuffix;
+ pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist);
+ pReader->aDoclist = pNext;
+ pReader->pOffsetList = 0;
+
+ /* Check that the doclist does not appear to extend past the end of the
+ ** b-tree node. And that the final byte of the doclist is 0x00. If either
+ ** of these statements is untrue, then the data structure is corrupt.
+ */
+ if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode]
+ || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1])
+ ){
+ return SQLITE_CORRUPT_VTAB;
+ }
+ return SQLITE_OK;
+}
+
+/*
+** Set the SegReader to point to the first docid in the doclist associated
+** with the current term.
+*/
+static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){
+ int rc = SQLITE_OK;
+ assert( pReader->aDoclist );
+ assert( !pReader->pOffsetList );
+ if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){
+ u8 bEof = 0;
+ pReader->iDocid = 0;
+ pReader->nOffsetList = 0;
+ sqlite3Fts3DoclistPrev(0,
+ pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList,
+ &pReader->iDocid, &pReader->nOffsetList, &bEof
+ );
+ }else{
+ rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX);
+ if( rc==SQLITE_OK ){
+ int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid);
+ pReader->pOffsetList = &pReader->aDoclist[n];
+ }
+ }
+ return rc;
+}
+
+/*
+** Advance the SegReader to point to the next docid in the doclist
+** associated with the current term.
+**
+** If arguments ppOffsetList and pnOffsetList are not NULL, then
+** *ppOffsetList is set to point to the first column-offset list
+** in the doclist entry (i.e. immediately past the docid varint).
+** *pnOffsetList is set to the length of the set of column-offset
+** lists, not including the nul-terminator byte. For example:
+*/
+static int fts3SegReaderNextDocid(
+ Fts3Table *pTab,
+ Fts3SegReader *pReader, /* Reader to advance to next docid */
+ char **ppOffsetList, /* OUT: Pointer to current position-list */
+ int *pnOffsetList /* OUT: Length of *ppOffsetList in bytes */
+){
+ int rc = SQLITE_OK;
+ char *p = pReader->pOffsetList;
+ char c = 0;
+
+ assert( p );
+
+ if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){
+ /* A pending-terms seg-reader for an FTS4 table that uses order=desc.
+ ** Pending-terms doclists are always built up in ascending order, so
+ ** we have to iterate through them backwards here. */
+ u8 bEof = 0;
+ if( ppOffsetList ){
+ *ppOffsetList = pReader->pOffsetList;
+ *pnOffsetList = pReader->nOffsetList - 1;
+ }
+ sqlite3Fts3DoclistPrev(0,
+ pReader->aDoclist, pReader->nDoclist, &p, &pReader->iDocid,
+ &pReader->nOffsetList, &bEof
+ );
+ if( bEof ){
+ pReader->pOffsetList = 0;
+ }else{
+ pReader->pOffsetList = p;
+ }
+ }else{
+ char *pEnd = &pReader->aDoclist[pReader->nDoclist];
+
+ /* Pointer p currently points at the first byte of an offset list. The
+ ** following block advances it to point one byte past the end of
+ ** the same offset list. */
+ while( 1 ){
+
+ /* The following line of code (and the "p++" below the while() loop) is
+ ** normally all that is required to move pointer p to the desired
+ ** position. The exception is if this node is being loaded from disk
+ ** incrementally and pointer "p" now points to the first byte passed
+ ** the populated part of pReader->aNode[].
+ */
+ while( *p | c ) c = *p++ & 0x80;
+ assert( *p==0 );
+
+ if( pReader->pBlob==0 || p<&pReader->aNode[pReader->nPopulate] ) break;
+ rc = fts3SegReaderIncrRead(pReader);
+ if( rc!=SQLITE_OK ) return rc;
+ }
+ p++;
+
+ /* If required, populate the output variables with a pointer to and the
+ ** size of the previous offset-list.
+ */
+ if( ppOffsetList ){
+ *ppOffsetList = pReader->pOffsetList;
+ *pnOffsetList = (int)(p - pReader->pOffsetList - 1);
+ }
+
+ while( p<pEnd && *p==0 ) p++;
+
+ /* If there are no more entries in the doclist, set pOffsetList to
+ ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and
+ ** Fts3SegReader.pOffsetList to point to the next offset list before
+ ** returning.
+ */
+ if( p>=pEnd ){
+ pReader->pOffsetList = 0;
+ }else{
+ rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX);
+ if( rc==SQLITE_OK ){
+ sqlite3_int64 iDelta;
+ pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
+ if( pTab->bDescIdx ){
+ pReader->iDocid -= iDelta;
+ }else{
+ pReader->iDocid += iDelta;
+ }
+ }
+ }
+ }
+
+ return SQLITE_OK;
+}
+
+
+int sqlite3Fts3MsrOvfl(
+ Fts3Cursor *pCsr,
+ Fts3MultiSegReader *pMsr,
+ int *pnOvfl
+){
+ Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
+ int nOvfl = 0;
+ int ii;
+ int rc = SQLITE_OK;
+ int pgsz = p->nPgsz;
+
+ assert( p->bHasStat );
+ assert( pgsz>0 );
+
+ for(ii=0; rc==SQLITE_OK && ii<pMsr->nSegment; ii++){
+ Fts3SegReader *pReader = pMsr->apSegment[ii];
+ if( !fts3SegReaderIsPending(pReader)
+ && !fts3SegReaderIsRootOnly(pReader)
+ ){
+ sqlite3_int64 jj;
+ for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){
+ int nBlob;
+ rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob, 0);
+ if( rc!=SQLITE_OK ) break;
+ if( (nBlob+35)>pgsz ){
+ nOvfl += (nBlob + 34)/pgsz;
+ }
+ }
+ }
+ }
+ *pnOvfl = nOvfl;
+ return rc;
+}
+
+/*
+** Free all allocations associated with the iterator passed as the
+** second argument.
+*/
+void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){
+ if( pReader && !fts3SegReaderIsPending(pReader) ){
+ sqlite3_free(pReader->zTerm);
+ if( !fts3SegReaderIsRootOnly(pReader) ){
+ sqlite3_free(pReader->aNode);
+ sqlite3_blob_close(pReader->pBlob);
+ }
+ }
+ sqlite3_free(pReader);
+}
+
+/*
+** Allocate a new SegReader object.
+*/
+int sqlite3Fts3SegReaderNew(
+ int iAge, /* Segment "age". */
+ sqlite3_int64 iStartLeaf, /* First leaf to traverse */
+ sqlite3_int64 iEndLeaf, /* Final leaf to traverse */
+ sqlite3_int64 iEndBlock, /* Final block of segment */
+ const char *zRoot, /* Buffer containing root node */
+ int nRoot, /* Size of buffer containing root node */
+ Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */
+){
+ int rc = SQLITE_OK; /* Return code */
+ Fts3SegReader *pReader; /* Newly allocated SegReader object */
+ int nExtra = 0; /* Bytes to allocate segment root node */
+
+ assert( iStartLeaf<=iEndLeaf );
+ if( iStartLeaf==0 ){
+ nExtra = nRoot + FTS3_NODE_PADDING;
+ }
+
+ pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra);
+ if( !pReader ){
+ return SQLITE_NOMEM;
+ }
+ memset(pReader, 0, sizeof(Fts3SegReader));
+ pReader->iIdx = iAge;
+ pReader->iStartBlock = iStartLeaf;
+ pReader->iLeafEndBlock = iEndLeaf;
+ pReader->iEndBlock = iEndBlock;
+
+ if( nExtra ){
+ /* The entire segment is stored in the root node. */
+ pReader->aNode = (char *)&pReader[1];
+ pReader->nNode = nRoot;
+ memcpy(pReader->aNode, zRoot, nRoot);
+ memset(&pReader->aNode[nRoot], 0, FTS3_NODE_PADDING);
+ }else{
+ pReader->iCurrentBlock = iStartLeaf-1;
+ }
+
+ if( rc==SQLITE_OK ){
+ *ppReader = pReader;
+ }else{
+ sqlite3Fts3SegReaderFree(pReader);
+ }
+ return rc;
+}
+
+/*
+** This is a comparison function used as a qsort() callback when sorting
+** an array of pending terms by term. This occurs as part of flushing
+** the contents of the pending-terms hash table to the database.
+*/
+static int fts3CompareElemByTerm(const void *lhs, const void *rhs){
+ char *z1 = fts3HashKey(*(Fts3HashElem **)lhs);
+ char *z2 = fts3HashKey(*(Fts3HashElem **)rhs);
+ int n1 = fts3HashKeysize(*(Fts3HashElem **)lhs);
+ int n2 = fts3HashKeysize(*(Fts3HashElem **)rhs);
+
+ int n = (n1<n2 ? n1 : n2);
+ int c = memcmp(z1, z2, n);
+ if( c==0 ){
+ c = n1 - n2;
+ }
+ return c;
+}
+
+/*
+** This function is used to allocate an Fts3SegReader that iterates through
+** a subset of the terms stored in the Fts3Table.pendingTerms array.
+**
+** If the isPrefixIter parameter is zero, then the returned SegReader iterates
+** through each term in the pending-terms table. Or, if isPrefixIter is
+** non-zero, it iterates through each term and its prefixes. For example, if
+** the pending terms hash table contains the terms "sqlite", "mysql" and
+** "firebird", then the iterator visits the following 'terms' (in the order
+** shown):
+**
+** f fi fir fire fireb firebi firebir firebird
+** m my mys mysq mysql
+** s sq sql sqli sqlit sqlite
+**
+** Whereas if isPrefixIter is zero, the terms visited are:
+**
+** firebird mysql sqlite
+*/
+int sqlite3Fts3SegReaderPending(
+ Fts3Table *p, /* Virtual table handle */
+ int iIndex, /* Index for p->aIndex */
+ const char *zTerm, /* Term to search for */
+ int nTerm, /* Size of buffer zTerm */
+ int bPrefix, /* True for a prefix iterator */
+ Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */
+){
+ Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */
+ Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */
+ int nElem = 0; /* Size of array at aElem */
+ int rc = SQLITE_OK; /* Return Code */
+ Fts3Hash *pHash;
+
+ pHash = &p->aIndex[iIndex].hPending;
+ if( bPrefix ){
+ int nAlloc = 0; /* Size of allocated array at aElem */
+ Fts3HashElem *pE = 0; /* Iterator variable */
+
+ for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){
+ char *zKey = (char *)fts3HashKey(pE);
+ int nKey = fts3HashKeysize(pE);
+ if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){
+ if( nElem==nAlloc ){
+ Fts3HashElem **aElem2;
+ nAlloc += 16;
+ aElem2 = (Fts3HashElem **)sqlite3_realloc(
+ aElem, nAlloc*sizeof(Fts3HashElem *)
+ );
+ if( !aElem2 ){
+ rc = SQLITE_NOMEM;
+ nElem = 0;
+ break;
+ }
+ aElem = aElem2;
+ }
+
+ aElem[nElem++] = pE;
+ }
+ }
+
+ /* If more than one term matches the prefix, sort the Fts3HashElem
+ ** objects in term order using qsort(). This uses the same comparison
+ ** callback as is used when flushing terms to disk.
+ */
+ if( nElem>1 ){
+ qsort(aElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm);
+ }
+
+ }else{
+ /* The query is a simple term lookup that matches at most one term in
+ ** the index. All that is required is a straight hash-lookup. */
+ Fts3HashElem *pE = fts3HashFindElem(pHash, zTerm, nTerm);
+ if( pE ){
+ aElem = &pE;
+ nElem = 1;
+ }
+ }
+
+ if( nElem>0 ){
+ int nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *);
+ pReader = (Fts3SegReader *)sqlite3_malloc(nByte);
+ if( !pReader ){
+ rc = SQLITE_NOMEM;
+ }else{
+ memset(pReader, 0, nByte);
+ pReader->iIdx = 0x7FFFFFFF;
+ pReader->ppNextElem = (Fts3HashElem **)&pReader[1];
+ memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *));
+ }
+ }
+
+ if( bPrefix ){
+ sqlite3_free(aElem);
+ }
+ *ppReader = pReader;
+ return rc;
+}
+
+/*
+** Compare the entries pointed to by two Fts3SegReader structures.
+** Comparison is as follows:
+**
+** 1) EOF is greater than not EOF.
+**
+** 2) The current terms (if any) are compared using memcmp(). If one
+** term is a prefix of another, the longer term is considered the
+** larger.
+**
+** 3) By segment age. An older segment is considered larger.
+*/
+static int fts3SegReaderCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){
+ int rc;
+ if( pLhs->aNode && pRhs->aNode ){
+ int rc2 = pLhs->nTerm - pRhs->nTerm;
+ if( rc2<0 ){
+ rc = memcmp(pLhs->zTerm, pRhs->zTerm, pLhs->nTerm);
+ }else{
+ rc = memcmp(pLhs->zTerm, pRhs->zTerm, pRhs->nTerm);
+ }
+ if( rc==0 ){
+ rc = rc2;
+ }
+ }else{
+ rc = (pLhs->aNode==0) - (pRhs->aNode==0);
+ }
+ if( rc==0 ){
+ rc = pRhs->iIdx - pLhs->iIdx;
+ }
+ assert( rc!=0 );
+ return rc;
+}
+
+/*
+** A different comparison function for SegReader structures. In this
+** version, it is assumed that each SegReader points to an entry in
+** a doclist for identical terms. Comparison is made as follows:
+**
+** 1) EOF (end of doclist in this case) is greater than not EOF.
+**
+** 2) By current docid.
+**
+** 3) By segment age. An older segment is considered larger.
+*/
+static int fts3SegReaderDoclistCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){
+ int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0);
+ if( rc==0 ){
+ if( pLhs->iDocid==pRhs->iDocid ){
+ rc = pRhs->iIdx - pLhs->iIdx;
+ }else{
+ rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1;
+ }
+ }
+ assert( pLhs->aNode && pRhs->aNode );
+ return rc;
+}
+static int fts3SegReaderDoclistCmpRev(Fts3SegReader *pLhs, Fts3SegReader *pRhs){
+ int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0);
+ if( rc==0 ){
+ if( pLhs->iDocid==pRhs->iDocid ){
+ rc = pRhs->iIdx - pLhs->iIdx;
+ }else{
+ rc = (pLhs->iDocid < pRhs->iDocid) ? 1 : -1;
+ }
+ }
+ assert( pLhs->aNode && pRhs->aNode );
+ return rc;
+}
+
+/*
+** Compare the term that the Fts3SegReader object passed as the first argument
+** points to with the term specified by arguments zTerm and nTerm.
+**
+** If the pSeg iterator is already at EOF, return 0. Otherwise, return
+** -ve if the pSeg term is less than zTerm/nTerm, 0 if the two terms are
+** equal, or +ve if the pSeg term is greater than zTerm/nTerm.
+*/
+static int fts3SegReaderTermCmp(
+ Fts3SegReader *pSeg, /* Segment reader object */
+ const char *zTerm, /* Term to compare to */
+ int nTerm /* Size of term zTerm in bytes */
+){
+ int res = 0;
+ if( pSeg->aNode ){
+ if( pSeg->nTerm>nTerm ){
+ res = memcmp(pSeg->zTerm, zTerm, nTerm);
+ }else{
+ res = memcmp(pSeg->zTerm, zTerm, pSeg->nTerm);
+ }
+ if( res==0 ){
+ res = pSeg->nTerm-nTerm;
+ }
+ }
+ return res;
+}
+
+/*
+** Argument apSegment is an array of nSegment elements. It is known that
+** the final (nSegment-nSuspect) members are already in sorted order
+** (according to the comparison function provided). This function shuffles
+** the array around until all entries are in sorted order.
+*/
+static void fts3SegReaderSort(
+ Fts3SegReader **apSegment, /* Array to sort entries of */
+ int nSegment, /* Size of apSegment array */
+ int nSuspect, /* Unsorted entry count */
+ int (*xCmp)(Fts3SegReader *, Fts3SegReader *) /* Comparison function */
+){
+ int i; /* Iterator variable */
+
+ assert( nSuspect<=nSegment );
+
+ if( nSuspect==nSegment ) nSuspect--;
+ for(i=nSuspect-1; i>=0; i--){
+ int j;
+ for(j=i; j<(nSegment-1); j++){
+ Fts3SegReader *pTmp;
+ if( xCmp(apSegment[j], apSegment[j+1])<0 ) break;
+ pTmp = apSegment[j+1];
+ apSegment[j+1] = apSegment[j];
+ apSegment[j] = pTmp;
+ }
+ }
+
+#ifndef NDEBUG
+ /* Check that the list really is sorted now. */
+ for(i=0; i<(nSuspect-1); i++){
+ assert( xCmp(apSegment[i], apSegment[i+1])<0 );
+ }
+#endif
+}
+
+/*
+** Insert a record into the %_segments table.
+*/
+static int fts3WriteSegment(
+ Fts3Table *p, /* Virtual table handle */
+ sqlite3_int64 iBlock, /* Block id for new block */
+ char *z, /* Pointer to buffer containing block data */
+ int n /* Size of buffer z in bytes */
+){
+ sqlite3_stmt *pStmt;
+ int rc = fts3SqlStmt(p, SQL_INSERT_SEGMENTS, &pStmt, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int64(pStmt, 1, iBlock);
+ sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC);
+ sqlite3_step(pStmt);
+ rc = sqlite3_reset(pStmt);
+ }
+ return rc;
+}
+
+/*
+** Insert a record into the %_segdir table.
+*/
+static int fts3WriteSegdir(
+ Fts3Table *p, /* Virtual table handle */
+ int iLevel, /* Value for "level" field */
+ int iIdx, /* Value for "idx" field */
+ sqlite3_int64 iStartBlock, /* Value for "start_block" field */
+ sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */
+ sqlite3_int64 iEndBlock, /* Value for "end_block" field */
+ char *zRoot, /* Blob value for "root" field */
+ int nRoot /* Number of bytes in buffer zRoot */
+){
+ sqlite3_stmt *pStmt;
+ int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int(pStmt, 1, iLevel);
+ sqlite3_bind_int(pStmt, 2, iIdx);
+ sqlite3_bind_int64(pStmt, 3, iStartBlock);
+ sqlite3_bind_int64(pStmt, 4, iLeafEndBlock);
+ sqlite3_bind_int64(pStmt, 5, iEndBlock);
+ sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC);
+ sqlite3_step(pStmt);
+ rc = sqlite3_reset(pStmt);
+ }
+ return rc;
+}
+
+/*
+** Return the size of the common prefix (if any) shared by zPrev and
+** zNext, in bytes. For example,
+**
+** fts3PrefixCompress("abc", 3, "abcdef", 6) // returns 3
+** fts3PrefixCompress("abX", 3, "abcdef", 6) // returns 2
+** fts3PrefixCompress("abX", 3, "Xbcdef", 6) // returns 0
+*/
+static int fts3PrefixCompress(
+ const char *zPrev, /* Buffer containing previous term */
+ int nPrev, /* Size of buffer zPrev in bytes */
+ const char *zNext, /* Buffer containing next term */
+ int nNext /* Size of buffer zNext in bytes */
+){
+ int n;
+ UNUSED_PARAMETER(nNext);
+ for(n=0; n<nPrev && zPrev[n]==zNext[n]; n++);
+ return n;
+}
+
+/*
+** Add term zTerm to the SegmentNode. It is guaranteed that zTerm is larger
+** (according to memcmp) than the previous term.
+*/
+static int fts3NodeAddTerm(
+ Fts3Table *p, /* Virtual table handle */
+ SegmentNode **ppTree, /* IN/OUT: SegmentNode handle */
+ int isCopyTerm, /* True if zTerm/nTerm is transient */
+ const char *zTerm, /* Pointer to buffer containing term */
+ int nTerm /* Size of term in bytes */
+){
+ SegmentNode *pTree = *ppTree;
+ int rc;
+ SegmentNode *pNew;
+
+ /* First try to append the term to the current node. Return early if
+ ** this is possible.
+ */
+ if( pTree ){
+ int nData = pTree->nData; /* Current size of node in bytes */
+ int nReq = nData; /* Required space after adding zTerm */
+ int nPrefix; /* Number of bytes of prefix compression */
+ int nSuffix; /* Suffix length */
+
+ nPrefix = fts3PrefixCompress(pTree->zTerm, pTree->nTerm, zTerm, nTerm);
+ nSuffix = nTerm-nPrefix;
+
+ nReq += sqlite3Fts3VarintLen(nPrefix)+sqlite3Fts3VarintLen(nSuffix)+nSuffix;
+ if( nReq<=p->nNodeSize || !pTree->zTerm ){
+
+ if( nReq>p->nNodeSize ){
+ /* An unusual case: this is the first term to be added to the node
+ ** and the static node buffer (p->nNodeSize bytes) is not large
+ ** enough. Use a separately malloced buffer instead This wastes
+ ** p->nNodeSize bytes, but since this scenario only comes about when
+ ** the database contain two terms that share a prefix of almost 2KB,
+ ** this is not expected to be a serious problem.
+ */
+ assert( pTree->aData==(char *)&pTree[1] );
+ pTree->aData = (char *)sqlite3_malloc(nReq);
+ if( !pTree->aData ){
+ return SQLITE_NOMEM;
+ }
+ }
+
+ if( pTree->zTerm ){
+ /* There is no prefix-length field for first term in a node */
+ nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nPrefix);
+ }
+
+ nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nSuffix);
+ memcpy(&pTree->aData[nData], &zTerm[nPrefix], nSuffix);
+ pTree->nData = nData + nSuffix;
+ pTree->nEntry++;
+
+ if( isCopyTerm ){
+ if( pTree->nMalloc<nTerm ){
+ char *zNew = sqlite3_realloc(pTree->zMalloc, nTerm*2);
+ if( !zNew ){
+ return SQLITE_NOMEM;
+ }
+ pTree->nMalloc = nTerm*2;
+ pTree->zMalloc = zNew;
+ }
+ pTree->zTerm = pTree->zMalloc;
+ memcpy(pTree->zTerm, zTerm, nTerm);
+ pTree->nTerm = nTerm;
+ }else{
+ pTree->zTerm = (char *)zTerm;
+ pTree->nTerm = nTerm;
+ }
+ return SQLITE_OK;
+ }
+ }
+
+ /* If control flows to here, it was not possible to append zTerm to the
+ ** current node. Create a new node (a right-sibling of the current node).
+ ** If this is the first node in the tree, the term is added to it.
+ **
+ ** Otherwise, the term is not added to the new node, it is left empty for
+ ** now. Instead, the term is inserted into the parent of pTree. If pTree
+ ** has no parent, one is created here.
+ */
+ pNew = (SegmentNode *)sqlite3_malloc(sizeof(SegmentNode) + p->nNodeSize);
+ if( !pNew ){
+ return SQLITE_NOMEM;
+ }
+ memset(pNew, 0, sizeof(SegmentNode));
+ pNew->nData = 1 + FTS3_VARINT_MAX;
+ pNew->aData = (char *)&pNew[1];
+
+ if( pTree ){
+ SegmentNode *pParent = pTree->pParent;
+ rc = fts3NodeAddTerm(p, &pParent, isCopyTerm, zTerm, nTerm);
+ if( pTree->pParent==0 ){
+ pTree->pParent = pParent;
+ }
+ pTree->pRight = pNew;
+ pNew->pLeftmost = pTree->pLeftmost;
+ pNew->pParent = pParent;
+ pNew->zMalloc = pTree->zMalloc;
+ pNew->nMalloc = pTree->nMalloc;
+ pTree->zMalloc = 0;
+ }else{
+ pNew->pLeftmost = pNew;
+ rc = fts3NodeAddTerm(p, &pNew, isCopyTerm, zTerm, nTerm);
+ }
+
+ *ppTree = pNew;
+ return rc;
+}
+
+/*
+** Helper function for fts3NodeWrite().
+*/
+static int fts3TreeFinishNode(
+ SegmentNode *pTree,
+ int iHeight,
+ sqlite3_int64 iLeftChild
+){
+ int nStart;
+ assert( iHeight>=1 && iHeight<128 );
+ nStart = FTS3_VARINT_MAX - sqlite3Fts3VarintLen(iLeftChild);
+ pTree->aData[nStart] = (char)iHeight;
+ sqlite3Fts3PutVarint(&pTree->aData[nStart+1], iLeftChild);
+ return nStart;
+}
+
+/*
+** Write the buffer for the segment node pTree and all of its peers to the
+** database. Then call this function recursively to write the parent of
+** pTree and its peers to the database.
+**
+** Except, if pTree is a root node, do not write it to the database. Instead,
+** set output variables *paRoot and *pnRoot to contain the root node.
+**
+** If successful, SQLITE_OK is returned and output variable *piLast is
+** set to the largest blockid written to the database (or zero if no
+** blocks were written to the db). Otherwise, an SQLite error code is
+** returned.
+*/
+static int fts3NodeWrite(
+ Fts3Table *p, /* Virtual table handle */
+ SegmentNode *pTree, /* SegmentNode handle */
+ int iHeight, /* Height of this node in tree */
+ sqlite3_int64 iLeaf, /* Block id of first leaf node */
+ sqlite3_int64 iFree, /* Block id of next free slot in %_segments */
+ sqlite3_int64 *piLast, /* OUT: Block id of last entry written */
+ char **paRoot, /* OUT: Data for root node */
+ int *pnRoot /* OUT: Size of root node in bytes */
+){
+ int rc = SQLITE_OK;
+
+ if( !pTree->pParent ){
+ /* Root node of the tree. */
+ int nStart = fts3TreeFinishNode(pTree, iHeight, iLeaf);
+ *piLast = iFree-1;
+ *pnRoot = pTree->nData - nStart;
+ *paRoot = &pTree->aData[nStart];
+ }else{
+ SegmentNode *pIter;
+ sqlite3_int64 iNextFree = iFree;
+ sqlite3_int64 iNextLeaf = iLeaf;
+ for(pIter=pTree->pLeftmost; pIter && rc==SQLITE_OK; pIter=pIter->pRight){
+ int nStart = fts3TreeFinishNode(pIter, iHeight, iNextLeaf);
+ int nWrite = pIter->nData - nStart;
+
+ rc = fts3WriteSegment(p, iNextFree, &pIter->aData[nStart], nWrite);
+ iNextFree++;
+ iNextLeaf += (pIter->nEntry+1);
+ }
+ if( rc==SQLITE_OK ){
+ assert( iNextLeaf==iFree );
+ rc = fts3NodeWrite(
+ p, pTree->pParent, iHeight+1, iFree, iNextFree, piLast, paRoot, pnRoot
+ );
+ }
+ }
+
+ return rc;
+}
+
+/*
+** Free all memory allocations associated with the tree pTree.
+*/
+static void fts3NodeFree(SegmentNode *pTree){
+ if( pTree ){
+ SegmentNode *p = pTree->pLeftmost;
+ fts3NodeFree(p->pParent);
+ while( p ){
+ SegmentNode *pRight = p->pRight;
+ if( p->aData!=(char *)&p[1] ){
+ sqlite3_free(p->aData);
+ }
+ assert( pRight==0 || p->zMalloc==0 );
+ sqlite3_free(p->zMalloc);
+ sqlite3_free(p);
+ p = pRight;
+ }
+ }
+}
+
+/*
+** Add a term to the segment being constructed by the SegmentWriter object
+** *ppWriter. When adding the first term to a segment, *ppWriter should
+** be passed NULL. This function will allocate a new SegmentWriter object
+** and return it via the input/output variable *ppWriter in this case.
+**
+** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code.
+*/
+static int fts3SegWriterAdd(
+ Fts3Table *p, /* Virtual table handle */
+ SegmentWriter **ppWriter, /* IN/OUT: SegmentWriter handle */
+ int isCopyTerm, /* True if buffer zTerm must be copied */
+ const char *zTerm, /* Pointer to buffer containing term */
+ int nTerm, /* Size of term in bytes */
+ const char *aDoclist, /* Pointer to buffer containing doclist */
+ int nDoclist /* Size of doclist in bytes */
+){
+ int nPrefix; /* Size of term prefix in bytes */
+ int nSuffix; /* Size of term suffix in bytes */
+ int nReq; /* Number of bytes required on leaf page */
+ int nData;
+ SegmentWriter *pWriter = *ppWriter;
+
+ if( !pWriter ){
+ int rc;
+ sqlite3_stmt *pStmt;
+
+ /* Allocate the SegmentWriter structure */
+ pWriter = (SegmentWriter *)sqlite3_malloc(sizeof(SegmentWriter));
+ if( !pWriter ) return SQLITE_NOMEM;
+ memset(pWriter, 0, sizeof(SegmentWriter));
+ *ppWriter = pWriter;
+
+ /* Allocate a buffer in which to accumulate data */
+ pWriter->aData = (char *)sqlite3_malloc(p->nNodeSize);
+ if( !pWriter->aData ) return SQLITE_NOMEM;
+ pWriter->nSize = p->nNodeSize;
+
+ /* Find the next free blockid in the %_segments table */
+ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pStmt, 0);
+ if( rc!=SQLITE_OK ) return rc;
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){
+ pWriter->iFree = sqlite3_column_int64(pStmt, 0);
+ pWriter->iFirst = pWriter->iFree;
+ }
+ rc = sqlite3_reset(pStmt);
+ if( rc!=SQLITE_OK ) return rc;
+ }
+ nData = pWriter->nData;
+
+ nPrefix = fts3PrefixCompress(pWriter->zTerm, pWriter->nTerm, zTerm, nTerm);
+ nSuffix = nTerm-nPrefix;
+
+ /* Figure out how many bytes are required by this new entry */
+ nReq = sqlite3Fts3VarintLen(nPrefix) + /* varint containing prefix size */
+ sqlite3Fts3VarintLen(nSuffix) + /* varint containing suffix size */
+ nSuffix + /* Term suffix */
+ sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */
+ nDoclist; /* Doclist data */
+
+ if( nData>0 && nData+nReq>p->nNodeSize ){
+ int rc;
+
+ /* The current leaf node is full. Write it out to the database. */
+ rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData);
+ if( rc!=SQLITE_OK ) return rc;
+
+ /* Add the current term to the interior node tree. The term added to
+ ** the interior tree must:
+ **
+ ** a) be greater than the largest term on the leaf node just written
+ ** to the database (still available in pWriter->zTerm), and
+ **
+ ** b) be less than or equal to the term about to be added to the new
+ ** leaf node (zTerm/nTerm).
+ **
+ ** In other words, it must be the prefix of zTerm 1 byte longer than
+ ** the common prefix (if any) of zTerm and pWriter->zTerm.
+ */
+ assert( nPrefix<nTerm );
+ rc = fts3NodeAddTerm(p, &pWriter->pTree, isCopyTerm, zTerm, nPrefix+1);
+ if( rc!=SQLITE_OK ) return rc;
+
+ nData = 0;
+ pWriter->nTerm = 0;
+
+ nPrefix = 0;
+ nSuffix = nTerm;
+ nReq = 1 + /* varint containing prefix size */
+ sqlite3Fts3VarintLen(nTerm) + /* varint containing suffix size */
+ nTerm + /* Term suffix */
+ sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */
+ nDoclist; /* Doclist data */
+ }
+
+ /* If the buffer currently allocated is too small for this entry, realloc
+ ** the buffer to make it large enough.
+ */
+ if( nReq>pWriter->nSize ){
+ char *aNew = sqlite3_realloc(pWriter->aData, nReq);
+ if( !aNew ) return SQLITE_NOMEM;
+ pWriter->aData = aNew;
+ pWriter->nSize = nReq;
+ }
+ assert( nData+nReq<=pWriter->nSize );
+
+ /* Append the prefix-compressed term and doclist to the buffer. */
+ nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nPrefix);
+ nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nSuffix);
+ memcpy(&pWriter->aData[nData], &zTerm[nPrefix], nSuffix);
+ nData += nSuffix;
+ nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nDoclist);
+ memcpy(&pWriter->aData[nData], aDoclist, nDoclist);
+ pWriter->nData = nData + nDoclist;
+
+ /* Save the current term so that it can be used to prefix-compress the next.
+ ** If the isCopyTerm parameter is true, then the buffer pointed to by
+ ** zTerm is transient, so take a copy of the term data. Otherwise, just
+ ** store a copy of the pointer.
+ */
+ if( isCopyTerm ){
+ if( nTerm>pWriter->nMalloc ){
+ char *zNew = sqlite3_realloc(pWriter->zMalloc, nTerm*2);
+ if( !zNew ){
+ return SQLITE_NOMEM;
+ }
+ pWriter->nMalloc = nTerm*2;
+ pWriter->zMalloc = zNew;
+ pWriter->zTerm = zNew;
+ }
+ assert( pWriter->zTerm==pWriter->zMalloc );
+ memcpy(pWriter->zTerm, zTerm, nTerm);
+ }else{
+ pWriter->zTerm = (char *)zTerm;
+ }
+ pWriter->nTerm = nTerm;
+
+ return SQLITE_OK;
+}
+
+/*
+** Flush all data associated with the SegmentWriter object pWriter to the
+** database. This function must be called after all terms have been added
+** to the segment using fts3SegWriterAdd(). If successful, SQLITE_OK is
+** returned. Otherwise, an SQLite error code.
+*/
+static int fts3SegWriterFlush(
+ Fts3Table *p, /* Virtual table handle */
+ SegmentWriter *pWriter, /* SegmentWriter to flush to the db */
+ int iLevel, /* Value for 'level' column of %_segdir */
+ int iIdx /* Value for 'idx' column of %_segdir */
+){
+ int rc; /* Return code */
+ if( pWriter->pTree ){
+ sqlite3_int64 iLast = 0; /* Largest block id written to database */
+ sqlite3_int64 iLastLeaf; /* Largest leaf block id written to db */
+ char *zRoot = NULL; /* Pointer to buffer containing root node */
+ int nRoot = 0; /* Size of buffer zRoot */
+
+ iLastLeaf = pWriter->iFree;
+ rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, pWriter->nData);
+ if( rc==SQLITE_OK ){
+ rc = fts3NodeWrite(p, pWriter->pTree, 1,
+ pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot);
+ }
+ if( rc==SQLITE_OK ){
+ rc = fts3WriteSegdir(
+ p, iLevel, iIdx, pWriter->iFirst, iLastLeaf, iLast, zRoot, nRoot);
+ }
+ }else{
+ /* The entire tree fits on the root node. Write it to the segdir table. */
+ rc = fts3WriteSegdir(
+ p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData);
+ }
+ return rc;
+}
+
+/*
+** Release all memory held by the SegmentWriter object passed as the
+** first argument.
+*/
+static void fts3SegWriterFree(SegmentWriter *pWriter){
+ if( pWriter ){
+ sqlite3_free(pWriter->aData);
+ sqlite3_free(pWriter->zMalloc);
+ fts3NodeFree(pWriter->pTree);
+ sqlite3_free(pWriter);
+ }
+}
+
+/*
+** The first value in the apVal[] array is assumed to contain an integer.
+** This function tests if there exist any documents with docid values that
+** are different from that integer. i.e. if deleting the document with docid
+** pRowid would mean the FTS3 table were empty.
+**
+** If successful, *pisEmpty is set to true if the table is empty except for
+** document pRowid, or false otherwise, and SQLITE_OK is returned. If an
+** error occurs, an SQLite error code is returned.
+*/
+static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){
+ sqlite3_stmt *pStmt;
+ int rc;
+ rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid);
+ if( rc==SQLITE_OK ){
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){
+ *pisEmpty = sqlite3_column_int(pStmt, 0);
+ }
+ rc = sqlite3_reset(pStmt);
+ }
+ return rc;
+}
+
+/*
+** Set *pnMax to the largest segment level in the database for the index
+** iIndex.
+**
+** Segment levels are stored in the 'level' column of the %_segdir table.
+**
+** Return SQLITE_OK if successful, or an SQLite error code if not.
+*/
+static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){
+ sqlite3_stmt *pStmt;
+ int rc;
+ assert( iIndex>=0 && iIndex<p->nIndex );
+
+ /* Set pStmt to the compiled version of:
+ **
+ ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?
+ **
+ ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR).
+ */
+ rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0);
+ if( rc!=SQLITE_OK ) return rc;
+ sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL);
+ sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL - 1);
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){
+ *pnMax = sqlite3_column_int(pStmt, 0);
+ }
+ return sqlite3_reset(pStmt);
+}
+
+/*
+** This function is used after merging multiple segments into a single large
+** segment to delete the old, now redundant, segment b-trees. Specifically,
+** it:
+**
+** 1) Deletes all %_segments entries for the segments associated with
+** each of the SegReader objects in the array passed as the third
+** argument, and
+**
+** 2) deletes all %_segdir entries with level iLevel, or all %_segdir
+** entries regardless of level if (iLevel<0).
+**
+** SQLITE_OK is returned if successful, otherwise an SQLite error code.
+*/
+static int fts3DeleteSegdir(
+ Fts3Table *p, /* Virtual table handle */
+ int iIndex, /* Index for p->aIndex */
+ int iLevel, /* Level of %_segdir entries to delete */
+ Fts3SegReader **apSegment, /* Array of SegReader objects */
+ int nReader /* Size of array apSegment */
+){
+ int rc; /* Return Code */
+ int i; /* Iterator variable */
+ sqlite3_stmt *pDelete; /* SQL statement to delete rows */
+
+ rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0);
+ for(i=0; rc==SQLITE_OK && i<nReader; i++){
+ Fts3SegReader *pSegment = apSegment[i];
+ if( pSegment->iStartBlock ){
+ sqlite3_bind_int64(pDelete, 1, pSegment->iStartBlock);
+ sqlite3_bind_int64(pDelete, 2, pSegment->iEndBlock);
+ sqlite3_step(pDelete);
+ rc = sqlite3_reset(pDelete);
+ }
+ }
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+
+ assert( iLevel>=0 || iLevel==FTS3_SEGCURSOR_ALL );
+ if( iLevel==FTS3_SEGCURSOR_ALL ){
+ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL);
+ sqlite3_bind_int(pDelete, 2, (iIndex+1) * FTS3_SEGDIR_MAXLEVEL - 1);
+ }
+ }else{
+ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel);
+ }
+ }
+
+ if( rc==SQLITE_OK ){
+ sqlite3_step(pDelete);
+ rc = sqlite3_reset(pDelete);
+ }
+
+ return rc;
+}
+
+/*
+** When this function is called, buffer *ppList (size *pnList bytes) contains
+** a position list that may (or may not) feature multiple columns. This
+** function adjusts the pointer *ppList and the length *pnList so that they
+** identify the subset of the position list that corresponds to column iCol.
+**
+** If there are no entries in the input position list for column iCol, then
+** *pnList is set to zero before returning.
+*/
+static void fts3ColumnFilter(
+ int iCol, /* Column to filter on */
+ char **ppList, /* IN/OUT: Pointer to position list */
+ int *pnList /* IN/OUT: Size of buffer *ppList in bytes */
+){
+ char *pList = *ppList;
+ int nList = *pnList;
+ char *pEnd = &pList[nList];
+ int iCurrent = 0;
+ char *p = pList;
+
+ assert( iCol>=0 );
+ while( 1 ){
+ char c = 0;
+ while( p<pEnd && (c | *p)&0xFE ) c = *p++ & 0x80;
+
+ if( iCol==iCurrent ){
+ nList = (int)(p - pList);
+ break;
+ }
+
+ nList -= (int)(p - pList);
+ pList = p;
+ if( nList==0 ){
+ break;
+ }
+ p = &pList[1];
+ p += sqlite3Fts3GetVarint32(p, &iCurrent);
+ }
+
+ *ppList = pList;
+ *pnList = nList;
+}
+
+/*
+** Cache data in the Fts3MultiSegReader.aBuffer[] buffer (overwriting any
+** existing data). Grow the buffer if required.
+**
+** If successful, return SQLITE_OK. Otherwise, if an OOM error is encountered
+** trying to resize the buffer, return SQLITE_NOMEM.
+*/
+static int fts3MsrBufferData(
+ Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */
+ char *pList,
+ int nList
+){
+ if( nList>pMsr->nBuffer ){
+ char *pNew;
+ pMsr->nBuffer = nList*2;
+ pNew = (char *)sqlite3_realloc(pMsr->aBuffer, pMsr->nBuffer);
+ if( !pNew ) return SQLITE_NOMEM;
+ pMsr->aBuffer = pNew;
+ }
+
+ memcpy(pMsr->aBuffer, pList, nList);
+ return SQLITE_OK;
+}
+
+int sqlite3Fts3MsrIncrNext(
+ Fts3Table *p, /* Virtual table handle */
+ Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */
+ sqlite3_int64 *piDocid, /* OUT: Docid value */
+ char **paPoslist, /* OUT: Pointer to position list */
+ int *pnPoslist /* OUT: Size of position list in bytes */
+){
+ int nMerge = pMsr->nAdvance;
+ Fts3SegReader **apSegment = pMsr->apSegment;
+ int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = (
+ p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp
+ );
+
+ if( nMerge==0 ){
+ *paPoslist = 0;
+ return SQLITE_OK;
+ }
+
+ while( 1 ){
+ Fts3SegReader *pSeg;
+ pSeg = pMsr->apSegment[0];
+
+ if( pSeg->pOffsetList==0 ){
+ *paPoslist = 0;
+ break;
+ }else{
+ int rc;
+ char *pList;
+ int nList;
+ int j;
+ sqlite3_int64 iDocid = apSegment[0]->iDocid;
+
+ rc = fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList);
+ j = 1;
+ while( rc==SQLITE_OK
+ && j<nMerge
+ && apSegment[j]->pOffsetList
+ && apSegment[j]->iDocid==iDocid
+ ){
+ rc = fts3SegReaderNextDocid(p, apSegment[j], 0, 0);
+ j++;
+ }
+ if( rc!=SQLITE_OK ) return rc;
+ fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp);
+
+ if( pMsr->iColFilter>=0 ){
+ fts3ColumnFilter(pMsr->iColFilter, &pList, &nList);
+ }
+
+ if( nList>0 ){
+ if( fts3SegReaderIsPending(apSegment[0]) ){
+ rc = fts3MsrBufferData(pMsr, pList, nList+1);
+ if( rc!=SQLITE_OK ) return rc;
+ *paPoslist = pMsr->aBuffer;
+ assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 );
+ }else{
+ *paPoslist = pList;
+ }
+ *piDocid = iDocid;
+ *pnPoslist = nList;
+ break;
+ }
+ }
+ }
+
+ return SQLITE_OK;
+}
+
+static int fts3SegReaderStart(
+ Fts3Table *p, /* Virtual table handle */
+ Fts3MultiSegReader *pCsr, /* Cursor object */
+ const char *zTerm, /* Term searched for (or NULL) */
+ int nTerm /* Length of zTerm in bytes */
+){
+ int i;
+ int nSeg = pCsr->nSegment;
+
+ /* If the Fts3SegFilter defines a specific term (or term prefix) to search
+ ** for, then advance each segment iterator until it points to a term of
+ ** equal or greater value than the specified term. This prevents many
+ ** unnecessary merge/sort operations for the case where single segment
+ ** b-tree leaf nodes contain more than one term.
+ */
+ for(i=0; pCsr->bRestart==0 && i<pCsr->nSegment; i++){
+ Fts3SegReader *pSeg = pCsr->apSegment[i];
+ do {
+ int rc = fts3SegReaderNext(p, pSeg, 0);
+ if( rc!=SQLITE_OK ) return rc;
+ }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 );
+ }
+ fts3SegReaderSort(pCsr->apSegment, nSeg, nSeg, fts3SegReaderCmp);
+
+ return SQLITE_OK;
+}
+
+int sqlite3Fts3SegReaderStart(
+ Fts3Table *p, /* Virtual table handle */
+ Fts3MultiSegReader *pCsr, /* Cursor object */
+ Fts3SegFilter *pFilter /* Restrictions on range of iteration */
+){
+ pCsr->pFilter = pFilter;
+ return fts3SegReaderStart(p, pCsr, pFilter->zTerm, pFilter->nTerm);
+}
+
+int sqlite3Fts3MsrIncrStart(
+ Fts3Table *p, /* Virtual table handle */
+ Fts3MultiSegReader *pCsr, /* Cursor object */
+ int iCol, /* Column to match on. */
+ const char *zTerm, /* Term to iterate through a doclist for */
+ int nTerm /* Number of bytes in zTerm */
+){
+ int i;
+ int rc;
+ int nSegment = pCsr->nSegment;
+ int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = (
+ p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp
+ );
+
+ assert( pCsr->pFilter==0 );
+ assert( zTerm && nTerm>0 );
+
+ /* Advance each segment iterator until it points to the term zTerm/nTerm. */
+ rc = fts3SegReaderStart(p, pCsr, zTerm, nTerm);
+ if( rc!=SQLITE_OK ) return rc;
+
+ /* Determine how many of the segments actually point to zTerm/nTerm. */
+ for(i=0; i<nSegment; i++){
+ Fts3SegReader *pSeg = pCsr->apSegment[i];
+ if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){
+ break;
+ }
+ }
+ pCsr->nAdvance = i;
+
+ /* Advance each of the segments to point to the first docid. */
+ for(i=0; i<pCsr->nAdvance; i++){
+ rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]);
+ if( rc!=SQLITE_OK ) return rc;
+ }
+ fts3SegReaderSort(pCsr->apSegment, i, i, xCmp);
+
+ assert( iCol<0 || iCol<p->nColumn );
+ pCsr->iColFilter = iCol;
+
+ return SQLITE_OK;
+}
+
+/*
+** This function is called on a MultiSegReader that has been started using
+** sqlite3Fts3MsrIncrStart(). One or more calls to MsrIncrNext() may also
+** have been made. Calling this function puts the MultiSegReader in such
+** a state that if the next two calls are:
+**
+** sqlite3Fts3SegReaderStart()
+** sqlite3Fts3SegReaderStep()
+**
+** then the entire doclist for the term is available in
+** MultiSegReader.aDoclist/nDoclist.
+*/
+int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){
+ int i; /* Used to iterate through segment-readers */
+
+ assert( pCsr->zTerm==0 );
+ assert( pCsr->nTerm==0 );
+ assert( pCsr->aDoclist==0 );
+ assert( pCsr->nDoclist==0 );
+
+ pCsr->nAdvance = 0;
+ pCsr->bRestart = 1;
+ for(i=0; i<pCsr->nSegment; i++){
+ pCsr->apSegment[i]->pOffsetList = 0;
+ pCsr->apSegment[i]->nOffsetList = 0;
+ pCsr->apSegment[i]->iDocid = 0;
+ }
+
+ return SQLITE_OK;
+}
+
+
+int sqlite3Fts3SegReaderStep(
+ Fts3Table *p, /* Virtual table handle */
+ Fts3MultiSegReader *pCsr /* Cursor object */
+){
+ int rc = SQLITE_OK;
+
+ int isIgnoreEmpty = (pCsr->pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY);
+ int isRequirePos = (pCsr->pFilter->flags & FTS3_SEGMENT_REQUIRE_POS);
+ int isColFilter = (pCsr->pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER);
+ int isPrefix = (pCsr->pFilter->flags & FTS3_SEGMENT_PREFIX);
+ int isScan = (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN);
+
+ Fts3SegReader **apSegment = pCsr->apSegment;
+ int nSegment = pCsr->nSegment;
+ Fts3SegFilter *pFilter = pCsr->pFilter;
+ int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = (
+ p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp
+ );
+
+ if( pCsr->nSegment==0 ) return SQLITE_OK;
+
+ do {
+ int nMerge;
+ int i;
+
+ /* Advance the first pCsr->nAdvance entries in the apSegment[] array
+ ** forward. Then sort the list in order of current term again.
+ */
+ for(i=0; i<pCsr->nAdvance; i++){
+ rc = fts3SegReaderNext(p, apSegment[i], 0);
+ if( rc!=SQLITE_OK ) return rc;
+ }
+ fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp);
+ pCsr->nAdvance = 0;
+
+ /* If all the seg-readers are at EOF, we're finished. return SQLITE_OK. */
+ assert( rc==SQLITE_OK );
+ if( apSegment[0]->aNode==0 ) break;
+
+ pCsr->nTerm = apSegment[0]->nTerm;
+ pCsr->zTerm = apSegment[0]->zTerm;
+
+ /* If this is a prefix-search, and if the term that apSegment[0] points
+ ** to does not share a suffix with pFilter->zTerm/nTerm, then all
+ ** required callbacks have been made. In this case exit early.
+ **
+ ** Similarly, if this is a search for an exact match, and the first term
+ ** of segment apSegment[0] is not a match, exit early.
+ */
+ if( pFilter->zTerm && !isScan ){
+ if( pCsr->nTerm<pFilter->nTerm
+ || (!isPrefix && pCsr->nTerm>pFilter->nTerm)
+ || memcmp(pCsr->zTerm, pFilter->zTerm, pFilter->nTerm)
+ ){
+ break;
+ }
+ }
+
+ nMerge = 1;
+ while( nMerge<nSegment
+ && apSegment[nMerge]->aNode
+ && apSegment[nMerge]->nTerm==pCsr->nTerm
+ && 0==memcmp(pCsr->zTerm, apSegment[nMerge]->zTerm, pCsr->nTerm)
+ ){
+ nMerge++;
+ }
+
+ assert( isIgnoreEmpty || (isRequirePos && !isColFilter) );
+ if( nMerge==1
+ && !isIgnoreEmpty
+ && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0)
+ ){
+ pCsr->nDoclist = apSegment[0]->nDoclist;
+ if( fts3SegReaderIsPending(apSegment[0]) ){
+ rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, pCsr->nDoclist);
+ pCsr->aDoclist = pCsr->aBuffer;
+ }else{
+ pCsr->aDoclist = apSegment[0]->aDoclist;
+ }
+ if( rc==SQLITE_OK ) rc = SQLITE_ROW;
+ }else{
+ int nDoclist = 0; /* Size of doclist */
+ sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */
+
+ /* The current term of the first nMerge entries in the array
+ ** of Fts3SegReader objects is the same. The doclists must be merged
+ ** and a single term returned with the merged doclist.
+ */
+ for(i=0; i<nMerge; i++){
+ fts3SegReaderFirstDocid(p, apSegment[i]);
+ }
+ fts3SegReaderSort(apSegment, nMerge, nMerge, xCmp);
+ while( apSegment[0]->pOffsetList ){
+ int j; /* Number of segments that share a docid */
+ char *pList;
+ int nList;
+ int nByte;
+ sqlite3_int64 iDocid = apSegment[0]->iDocid;
+ fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList);
+ j = 1;
+ while( j<nMerge
+ && apSegment[j]->pOffsetList
+ && apSegment[j]->iDocid==iDocid
+ ){
+ fts3SegReaderNextDocid(p, apSegment[j], 0, 0);
+ j++;
+ }
+
+ if( isColFilter ){
+ fts3ColumnFilter(pFilter->iCol, &pList, &nList);
+ }
+
+ if( !isIgnoreEmpty || nList>0 ){
+
+ /* Calculate the 'docid' delta value to write into the merged
+ ** doclist. */
+ sqlite3_int64 iDelta;
+ if( p->bDescIdx && nDoclist>0 ){
+ iDelta = iPrev - iDocid;
+ }else{
+ iDelta = iDocid - iPrev;
+ }
+ assert( iDelta>0 || (nDoclist==0 && iDelta==iDocid) );
+ assert( nDoclist>0 || iDelta==iDocid );
+
+ nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0);
+ if( nDoclist+nByte>pCsr->nBuffer ){
+ char *aNew;
+ pCsr->nBuffer = (nDoclist+nByte)*2;
+ aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer);
+ if( !aNew ){
+ return SQLITE_NOMEM;
+ }
+ pCsr->aBuffer = aNew;
+ }
+ nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta);
+ iPrev = iDocid;
+ if( isRequirePos ){
+ memcpy(&pCsr->aBuffer[nDoclist], pList, nList);
+ nDoclist += nList;
+ pCsr->aBuffer[nDoclist++] = '\0';
+ }
+ }
+
+ fts3SegReaderSort(apSegment, nMerge, j, xCmp);
+ }
+ if( nDoclist>0 ){
+ pCsr->aDoclist = pCsr->aBuffer;
+ pCsr->nDoclist = nDoclist;
+ rc = SQLITE_ROW;
+ }
+ }
+ pCsr->nAdvance = nMerge;
+ }while( rc==SQLITE_OK );
+
+ return rc;
+}
+
+
+void sqlite3Fts3SegReaderFinish(
+ Fts3MultiSegReader *pCsr /* Cursor object */
+){
+ if( pCsr ){
+ int i;
+ for(i=0; i<pCsr->nSegment; i++){
+ sqlite3Fts3SegReaderFree(pCsr->apSegment[i]);
+ }
+ sqlite3_free(pCsr->apSegment);
+ sqlite3_free(pCsr->aBuffer);
+
+ pCsr->nSegment = 0;
+ pCsr->apSegment = 0;
+ pCsr->aBuffer = 0;
+ }
+}
+
+/*
+** Merge all level iLevel segments in the database into a single
+** iLevel+1 segment. Or, if iLevel<0, merge all segments into a
+** single segment with a level equal to the numerically largest level
+** currently present in the database.
+**
+** If this function is called with iLevel<0, but there is only one
+** segment in the database, SQLITE_DONE is returned immediately.
+** Otherwise, if successful, SQLITE_OK is returned. If an error occurs,
+** an SQLite error code is returned.
+*/
+static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){
+ int rc; /* Return code */
+ int iIdx = 0; /* Index of new segment */
+ int iNewLevel = 0; /* Level/index to create new segment at */
+ SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */
+ Fts3SegFilter filter; /* Segment term filter condition */
+ Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */
+ int bIgnoreEmpty = 0; /* True to ignore empty segments */
+
+ assert( iLevel==FTS3_SEGCURSOR_ALL
+ || iLevel==FTS3_SEGCURSOR_PENDING
+ || iLevel>=0
+ );
+ assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
+ assert( iIndex>=0 && iIndex<p->nIndex );
+
+ rc = sqlite3Fts3SegReaderCursor(p, iIndex, iLevel, 0, 0, 1, 0, &csr);
+ if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished;
+
+ if( iLevel==FTS3_SEGCURSOR_ALL ){
+ /* This call is to merge all segments in the database to a single
+ ** segment. The level of the new segment is equal to the the numerically
+ ** greatest segment level currently present in the database for this
+ ** index. The idx of the new segment is always 0. */
+ if( csr.nSegment==1 ){
+ rc = SQLITE_DONE;
+ goto finished;
+ }
+ rc = fts3SegmentMaxLevel(p, iIndex, &iNewLevel);
+ bIgnoreEmpty = 1;
+
+ }else if( iLevel==FTS3_SEGCURSOR_PENDING ){
+ iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL;
+ rc = fts3AllocateSegdirIdx(p, iIndex, 0, &iIdx);
+ }else{
+ /* This call is to merge all segments at level iLevel. find the next
+ ** available segment index at level iLevel+1. The call to
+ ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to
+ ** a single iLevel+2 segment if necessary. */
+ rc = fts3AllocateSegdirIdx(p, iIndex, iLevel+1, &iIdx);
+ iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL + iLevel+1;
+ }
+ if( rc!=SQLITE_OK ) goto finished;
+ assert( csr.nSegment>0 );
+ assert( iNewLevel>=(iIndex*FTS3_SEGDIR_MAXLEVEL) );
+ assert( iNewLevel<((iIndex+1)*FTS3_SEGDIR_MAXLEVEL) );
+
+ memset(&filter, 0, sizeof(Fts3SegFilter));
+ filter.flags = FTS3_SEGMENT_REQUIRE_POS;
+ filter.flags |= (bIgnoreEmpty ? FTS3_SEGMENT_IGNORE_EMPTY : 0);
+
+ rc = sqlite3Fts3SegReaderStart(p, &csr, &filter);
+ while( SQLITE_OK==rc ){
+ rc = sqlite3Fts3SegReaderStep(p, &csr);
+ if( rc!=SQLITE_ROW ) break;
+ rc = fts3SegWriterAdd(p, &pWriter, 1,
+ csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist);
+ }
+ if( rc!=SQLITE_OK ) goto finished;
+ assert( pWriter );
+
+ if( iLevel!=FTS3_SEGCURSOR_PENDING ){
+ rc = fts3DeleteSegdir(p, iIndex, iLevel, csr.apSegment, csr.nSegment);
+ if( rc!=SQLITE_OK ) goto finished;
+ }
+ rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx);
+
+ finished:
+ fts3SegWriterFree(pWriter);
+ sqlite3Fts3SegReaderFinish(&csr);
+ return rc;
+}
+
+
+/*
+** Flush the contents of pendingTerms to level 0 segments.
+*/
+int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
+ int rc = SQLITE_OK;
+ int i;
+ for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){
+ rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_PENDING);
+ if( rc==SQLITE_DONE ) rc = SQLITE_OK;
+ }
+ sqlite3Fts3PendingTermsClear(p);
+ return rc;
+}
+
+/*
+** Encode N integers as varints into a blob.
+*/
+static void fts3EncodeIntArray(
+ int N, /* The number of integers to encode */
+ u32 *a, /* The integer values */
+ char *zBuf, /* Write the BLOB here */
+ int *pNBuf /* Write number of bytes if zBuf[] used here */
+){
+ int i, j;
+ for(i=j=0; i<N; i++){
+ j += sqlite3Fts3PutVarint(&zBuf[j], (sqlite3_int64)a[i]);
+ }
+ *pNBuf = j;
+}
+
+/*
+** Decode a blob of varints into N integers
+*/
+static void fts3DecodeIntArray(
+ int N, /* The number of integers to decode */
+ u32 *a, /* Write the integer values */
+ const char *zBuf, /* The BLOB containing the varints */
+ int nBuf /* size of the BLOB */
+){
+ int i, j;
+ UNUSED_PARAMETER(nBuf);
+ for(i=j=0; i<N; i++){
+ sqlite3_int64 x;
+ j += sqlite3Fts3GetVarint(&zBuf[j], &x);
+ assert(j<=nBuf);
+ a[i] = (u32)(x & 0xffffffff);
+ }
+}
+
+/*
+** Insert the sizes (in tokens) for each column of the document
+** with docid equal to p->iPrevDocid. The sizes are encoded as
+** a blob of varints.
+*/
+static void fts3InsertDocsize(
+ int *pRC, /* Result code */
+ Fts3Table *p, /* Table into which to insert */
+ u32 *aSz /* Sizes of each column */
+){
+ char *pBlob; /* The BLOB encoding of the document size */
+ int nBlob; /* Number of bytes in the BLOB */
+ sqlite3_stmt *pStmt; /* Statement used to insert the encoding */
+ int rc; /* Result code from subfunctions */
+
+ if( *pRC ) return;
+ pBlob = sqlite3_malloc( 10*p->nColumn );
+ if( pBlob==0 ){
+ *pRC = SQLITE_NOMEM;
+ return;
+ }
+ fts3EncodeIntArray(p->nColumn, aSz, pBlob, &nBlob);
+ rc = fts3SqlStmt(p, SQL_REPLACE_DOCSIZE, &pStmt, 0);
+ if( rc ){
+ sqlite3_free(pBlob);
+ *pRC = rc;
+ return;
+ }
+ sqlite3_bind_int64(pStmt, 1, p->iPrevDocid);
+ sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, sqlite3_free);
+ sqlite3_step(pStmt);
+ *pRC = sqlite3_reset(pStmt);
+}
+
+/*
+** Record 0 of the %_stat table contains a blob consisting of N varints,
+** where N is the number of user defined columns in the fts3 table plus
+** two. If nCol is the number of user defined columns, then values of the
+** varints are set as follows:
+**
+** Varint 0: Total number of rows in the table.
+**
+** Varint 1..nCol: For each column, the total number of tokens stored in
+** the column for all rows of the table.
+**
+** Varint 1+nCol: The total size, in bytes, of all text values in all
+** columns of all rows of the table.
+**
+*/
+static void fts3UpdateDocTotals(
+ int *pRC, /* The result code */
+ Fts3Table *p, /* Table being updated */
+ u32 *aSzIns, /* Size increases */
+ u32 *aSzDel, /* Size decreases */
+ int nChng /* Change in the number of documents */
+){
+ char *pBlob; /* Storage for BLOB written into %_stat */
+ int nBlob; /* Size of BLOB written into %_stat */
+ u32 *a; /* Array of integers that becomes the BLOB */
+ sqlite3_stmt *pStmt; /* Statement for reading and writing */
+ int i; /* Loop counter */
+ int rc; /* Result code from subfunctions */
+
+ const int nStat = p->nColumn+2;
+
+ if( *pRC ) return;
+ a = sqlite3_malloc( (sizeof(u32)+10)*nStat );
+ if( a==0 ){
+ *pRC = SQLITE_NOMEM;
+ return;
+ }
+ pBlob = (char*)&a[nStat];
+ rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0);
+ if( rc ){
+ sqlite3_free(a);
+ *pRC = rc;
+ return;
+ }
+ if( sqlite3_step(pStmt)==SQLITE_ROW ){
+ fts3DecodeIntArray(nStat, a,
+ sqlite3_column_blob(pStmt, 0),
+ sqlite3_column_bytes(pStmt, 0));
+ }else{
+ memset(a, 0, sizeof(u32)*(nStat) );
+ }
+ sqlite3_reset(pStmt);
+ if( nChng<0 && a[0]<(u32)(-nChng) ){
+ a[0] = 0;
+ }else{
+ a[0] += nChng;
+ }
+ for(i=0; i<p->nColumn+1; i++){
+ u32 x = a[i+1];
+ if( x+aSzIns[i] < aSzDel[i] ){
+ x = 0;
+ }else{
+ x = x + aSzIns[i] - aSzDel[i];
+ }
+ a[i+1] = x;
+ }
+ fts3EncodeIntArray(nStat, a, pBlob, &nBlob);
+ rc = fts3SqlStmt(p, SQL_REPLACE_DOCTOTAL, &pStmt, 0);
+ if( rc ){
+ sqlite3_free(a);
+ *pRC = rc;
+ return;
+ }
+ sqlite3_bind_blob(pStmt, 1, pBlob, nBlob, SQLITE_STATIC);
+ sqlite3_step(pStmt);
+ *pRC = sqlite3_reset(pStmt);
+ sqlite3_free(a);
+}
+
+static int fts3DoOptimize(Fts3Table *p, int bReturnDone){
+ int i;
+ int bSeenDone = 0;
+ int rc = SQLITE_OK;
+ for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){
+ rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_ALL);
+ if( rc==SQLITE_DONE ){
+ bSeenDone = 1;
+ rc = SQLITE_OK;
+ }
+ }
+ sqlite3Fts3SegmentsClose(p);
+ sqlite3Fts3PendingTermsClear(p);
+
+ return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc;
+}
+
+/*
+** Handle a 'special' INSERT of the form:
+**
+** "INSERT INTO tbl(tbl) VALUES(<expr>)"
+**
+** Argument pVal contains the result of <expr>. Currently the only
+** meaningful value to insert is the text 'optimize'.
+*/
+static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
+ int rc; /* Return Code */
+ const char *zVal = (const char *)sqlite3_value_text(pVal);
+ int nVal = sqlite3_value_bytes(pVal);
+
+ if( !zVal ){
+ return SQLITE_NOMEM;
+ }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){
+ rc = fts3DoOptimize(p, 0);
+#ifdef SQLITE_TEST
+ }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
+ p->nNodeSize = atoi(&zVal[9]);
+ rc = SQLITE_OK;
+ }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){
+ p->nMaxPendingData = atoi(&zVal[11]);
+ rc = SQLITE_OK;
+#endif
+ }else{
+ rc = SQLITE_ERROR;
+ }
+
+ return rc;
+}
+
+/*
+** Delete all cached deferred doclists. Deferred doclists are cached
+** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function.
+*/
+void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){
+ Fts3DeferredToken *pDef;
+ for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){
+ fts3PendingListDelete(pDef->pList);
+ pDef->pList = 0;
+ }
+}
+
+/*
+** Free all entries in the pCsr->pDeffered list. Entries are added to
+** this list using sqlite3Fts3DeferToken().
+*/
+void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){
+ Fts3DeferredToken *pDef;
+ Fts3DeferredToken *pNext;
+ for(pDef=pCsr->pDeferred; pDef; pDef=pNext){
+ pNext = pDef->pNext;
+ fts3PendingListDelete(pDef->pList);
+ sqlite3_free(pDef);
+ }
+ pCsr->pDeferred = 0;
+}
+
+/*
+** Generate deferred-doclists for all tokens in the pCsr->pDeferred list
+** based on the row that pCsr currently points to.
+**
+** A deferred-doclist is like any other doclist with position information
+** included, except that it only contains entries for a single row of the
+** table, not for all rows.
+*/
+int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){
+ int rc = SQLITE_OK; /* Return code */
+ if( pCsr->pDeferred ){
+ int i; /* Used to iterate through table columns */
+ sqlite3_int64 iDocid; /* Docid of the row pCsr points to */
+ Fts3DeferredToken *pDef; /* Used to iterate through deferred tokens */
+
+ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
+ sqlite3_tokenizer *pT = p->pTokenizer;
+ sqlite3_tokenizer_module const *pModule = pT->pModule;
+
+ assert( pCsr->isRequireSeek==0 );
+ iDocid = sqlite3_column_int64(pCsr->pStmt, 0);
+
+ for(i=0; i<p->nColumn && rc==SQLITE_OK; i++){
+ const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1);
+ sqlite3_tokenizer_cursor *pTC = 0;
+
+ rc = pModule->xOpen(pT, zText, -1, &pTC);
+ while( rc==SQLITE_OK ){
+ char const *zToken; /* Buffer containing token */
+ int nToken; /* Number of bytes in token */
+ int iDum1, iDum2; /* Dummy variables */
+ int iPos; /* Position of token in zText */
+
+ pTC->pTokenizer = pT;
+ rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos);
+ for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){
+ Fts3PhraseToken *pPT = pDef->pToken;
+ if( (pDef->iCol>=p->nColumn || pDef->iCol==i)
+ && (pPT->n==nToken || (pPT->isPrefix && pPT->n<nToken))
+ && (0==memcmp(zToken, pPT->z, pPT->n))
+ ){
+ fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc);
+ }
+ }
+ }
+ if( pTC ) pModule->xClose(pTC);
+ if( rc==SQLITE_DONE ) rc = SQLITE_OK;
+ }
+
+ for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){
+ if( pDef->pList ){
+ rc = fts3PendingListAppendVarint(&pDef->pList, 0);
+ }
+ }
+ }
+
+ return rc;
+}
+
+int sqlite3Fts3DeferredTokenList(
+ Fts3DeferredToken *p,
+ char **ppData,
+ int *pnData
+){
+ char *pRet;
+ int nSkip;
+ sqlite3_int64 dummy;
+
+ *ppData = 0;
+ *pnData = 0;
+
+ if( p->pList==0 ){
+ return SQLITE_OK;
+ }
+
+ pRet = (char *)sqlite3_malloc(p->pList->nData);
+ if( !pRet ) return SQLITE_NOMEM;
+
+ nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy);
+ *pnData = p->pList->nData - nSkip;
+ *ppData = pRet;
+
+ memcpy(pRet, &p->pList->aData[nSkip], *pnData);
+ return SQLITE_OK;
+}
+
+/*
+** Add an entry for token pToken to the pCsr->pDeferred list.
+*/
+int sqlite3Fts3DeferToken(
+ Fts3Cursor *pCsr, /* Fts3 table cursor */
+ Fts3PhraseToken *pToken, /* Token to defer */
+ int iCol /* Column that token must appear in (or -1) */
+){
+ Fts3DeferredToken *pDeferred;
+ pDeferred = sqlite3_malloc(sizeof(*pDeferred));
+ if( !pDeferred ){
+ return SQLITE_NOMEM;
+ }
+ memset(pDeferred, 0, sizeof(*pDeferred));
+ pDeferred->pToken = pToken;
+ pDeferred->pNext = pCsr->pDeferred;
+ pDeferred->iCol = iCol;
+ pCsr->pDeferred = pDeferred;
+
+ assert( pToken->pDeferred==0 );
+ pToken->pDeferred = pDeferred;
+
+ return SQLITE_OK;
+}
+
+/*
+** SQLite value pRowid contains the rowid of a row that may or may not be
+** present in the FTS3 table. If it is, delete it and adjust the contents
+** of subsiduary data structures accordingly.
+*/
+static int fts3DeleteByRowid(
+ Fts3Table *p,
+ sqlite3_value *pRowid,
+ int *pnDoc,
+ u32 *aSzDel
+){
+ int isEmpty = 0;
+ int rc = fts3IsEmpty(p, pRowid, &isEmpty);
+ if( rc==SQLITE_OK ){
+ if( isEmpty ){
+ /* Deleting this row means the whole table is empty. In this case
+ ** delete the contents of all three tables and throw away any
+ ** data in the pendingTerms hash table. */
+ rc = fts3DeleteAll(p);
+ *pnDoc = *pnDoc - 1;
+ }else{
+ sqlite3_int64 iRemove = sqlite3_value_int64(pRowid);
+ rc = fts3PendingTermsDocid(p, iRemove);
+ fts3DeleteTerms(&rc, p, pRowid, aSzDel);
+ fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid);
+ if( sqlite3_changes(p->db) ) *pnDoc = *pnDoc - 1;
+ if( p->bHasDocsize ){
+ fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid);
+ }
+ }
+ }
+
+ return rc;
+}
+
+/*
+** This function does the work for the xUpdate method of FTS3 virtual
+** tables.
+*/
+int sqlite3Fts3UpdateMethod(
+ sqlite3_vtab *pVtab, /* FTS3 vtab object */
+ int nArg, /* Size of argument array */
+ sqlite3_value **apVal, /* Array of arguments */
+ sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */
+){
+ Fts3Table *p = (Fts3Table *)pVtab;
+ int rc = SQLITE_OK; /* Return Code */
+ int isRemove = 0; /* True for an UPDATE or DELETE */
+ sqlite3_int64 iRemove = 0; /* Rowid removed by UPDATE or DELETE */
+ u32 *aSzIns = 0; /* Sizes of inserted documents */
+ u32 *aSzDel; /* Sizes of deleted documents */
+ int nChng = 0; /* Net change in number of documents */
+ int bInsertDone = 0;
+
+ assert( p->pSegments==0 );
+
+ /* Check for a "special" INSERT operation. One of the form:
+ **
+ ** INSERT INTO xyz(xyz) VALUES('command');
+ */
+ if( nArg>1
+ && sqlite3_value_type(apVal[0])==SQLITE_NULL
+ && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL
+ ){
+ rc = fts3SpecialInsert(p, apVal[p->nColumn+2]);
+ goto update_out;
+ }
+
+ /* Allocate space to hold the change in document sizes */
+ aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 );
+ if( aSzIns==0 ){
+ rc = SQLITE_NOMEM;
+ goto update_out;
+ }
+ aSzDel = &aSzIns[p->nColumn+1];
+ memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2);
+
+ /* If this is an INSERT operation, or an UPDATE that modifies the rowid
+ ** value, then this operation requires constraint handling.
+ **
+ ** If the on-conflict mode is REPLACE, this means that the existing row
+ ** should be deleted from the database before inserting the new row. Or,
+ ** if the on-conflict mode is other than REPLACE, then this method must
+ ** detect the conflict and return SQLITE_CONSTRAINT before beginning to
+ ** modify the database file.
+ */
+ if( nArg>1 ){
+ /* Find the value object that holds the new rowid value. */
+ sqlite3_value *pNewRowid = apVal[3+p->nColumn];
+ if( sqlite3_value_type(pNewRowid)==SQLITE_NULL ){
+ pNewRowid = apVal[1];
+ }
+
+ if( sqlite3_value_type(pNewRowid)!=SQLITE_NULL && (
+ sqlite3_value_type(apVal[0])==SQLITE_NULL
+ || sqlite3_value_int64(apVal[0])!=sqlite3_value_int64(pNewRowid)
+ )){
+ /* The new rowid is not NULL (in this case the rowid will be
+ ** automatically assigned and there is no chance of a conflict), and
+ ** the statement is either an INSERT or an UPDATE that modifies the
+ ** rowid column. So if the conflict mode is REPLACE, then delete any
+ ** existing row with rowid=pNewRowid.
+ **
+ ** Or, if the conflict mode is not REPLACE, insert the new record into
+ ** the %_content table. If we hit the duplicate rowid constraint (or any
+ ** other error) while doing so, return immediately.
+ **
+ ** This branch may also run if pNewRowid contains a value that cannot
+ ** be losslessly converted to an integer. In this case, the eventual
+ ** call to fts3InsertData() (either just below or further on in this
+ ** function) will return SQLITE_MISMATCH. If fts3DeleteByRowid is
+ ** invoked, it will delete zero rows (since no row will have
+ ** docid=$pNewRowid if $pNewRowid is not an integer value).
+ */
+ if( sqlite3_vtab_on_conflict(p->db)==SQLITE_REPLACE ){
+ rc = fts3DeleteByRowid(p, pNewRowid, &nChng, aSzDel);
+ }else{
+ rc = fts3InsertData(p, apVal, pRowid);
+ bInsertDone = 1;
+ }
+ }
+ }
+ if( rc!=SQLITE_OK ){
+ goto update_out;
+ }
+
+ /* If this is a DELETE or UPDATE operation, remove the old record. */
+ if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
+ assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER );
+ rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel);
+ isRemove = 1;
+ iRemove = sqlite3_value_int64(apVal[0]);
+ }
+
+ /* If this is an INSERT or UPDATE operation, insert the new record. */
+ if( nArg>1 && rc==SQLITE_OK ){
+ if( bInsertDone==0 ){
+ rc = fts3InsertData(p, apVal, pRowid);
+ if( rc==SQLITE_CONSTRAINT ) rc = SQLITE_CORRUPT_VTAB;
+ }
+ if( rc==SQLITE_OK && (!isRemove || *pRowid!=iRemove) ){
+ rc = fts3PendingTermsDocid(p, *pRowid);
+ }
+ if( rc==SQLITE_OK ){
+ rc = fts3InsertTerms(p, apVal, aSzIns);
+ }
+ if( p->bHasDocsize ){
+ fts3InsertDocsize(&rc, p, aSzIns);
+ }
+ nChng++;
+ }
+
+ if( p->bHasStat ){
+ fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng);
+ }
+
+ update_out:
+ sqlite3_free(aSzIns);
+ sqlite3Fts3SegmentsClose(p);
+ return rc;
+}
+
+/*
+** Flush any data in the pending-terms hash table to disk. If successful,
+** merge all segments in the database (including the new segment, if
+** there was any data to flush) into a single segment.
+*/
+int sqlite3Fts3Optimize(Fts3Table *p){
+ int rc;
+ rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0);
+ if( rc==SQLITE_OK ){
+ rc = fts3DoOptimize(p, 1);
+ if( rc==SQLITE_OK || rc==SQLITE_DONE ){
+ int rc2 = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
+ if( rc2!=SQLITE_OK ) rc = rc2;
+ }else{
+ sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0);
+ sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
+ }
+ }
+ sqlite3Fts3SegmentsClose(p);
+ return rc;
+}
+
+#endif
diff --git a/src/libtracker-fts/tracker-fts.c b/src/libtracker-fts/tracker-fts.c
index 02ed3ec..9b23b65 100644
--- a/src/libtracker-fts/tracker-fts.c
+++ b/src/libtracker-fts/tracker-fts.c
@@ -1,7974 +1,51 @@
/*
-** 2006 Oct 10
-**
-** The author disclaims copyright to this source code. In place of
-** a legal notice, here is a blessing:
-**
-** May you do good and not evil.
-** May you find forgiveness for yourself and forgive others.
-** May you share freely, never taking more than you give.
-**
-******************************************************************************
-**
-** This is an SQLite module implementing full-text search.
-*/
-
-/*
-** The code in this file is only compiled if:
-**
-** * The FTS3 module is being built as an extension
-** (in which case SQLITE_CORE is not defined), or
-**
-** * The FTS3 module is being built into the core of
-** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
-*/
-
-/* TODO(shess) Consider exporting this comment to an HTML file or the
-** wiki.
-*/
-/* The full-text index is stored in a series of b+tree (-like)
-** structures called segments which map terms to doclists. The
-** structures are like b+trees in layout, but are constructed from the
-** bottom up in optimal fashion and are not updatable. Since trees
-** are built from the bottom up, things will be described from the
-** bottom up.
-**
-**
-**** Varints ****
-** The basic unit of encoding is a variable-length integer called a
-** varint. We encode variable-length integers in little-endian order
-** using seven bits * per byte as follows:
-**
-** KEY:
-** A = 0xxxxxxx 7 bits of data and one flag bit
-** B = 1xxxxxxx 7 bits of data and one flag bit
-**
-** 7 bits - A
-** 14 bits - BA
-** 21 bits - BBA
-** and so on.
-**
-** This is identical to how sqlite encodes varints (see util.c).
-**
-**
-**** Document lists ****
-** A doclist (document list) holds a docid-sorted list of hits for a
-** given term. Doclists hold docids, and can optionally associate
-** token positions and offsets with docids.
-**
-** A DL_POSITIONS_OFFSETS doclist is stored like this:
-**
-** array {
-** varint docid;
-** array { (position list for column 0)
-** varint position; (delta from previous position plus POS_BASE)
-** varint startOffset; (delta from previous startOffset)
-** varint endOffset; (delta from startOffset)
-** }
-** array {
-** varint POS_COLUMN; (marks start of position list for new column)
-** varint column; (index of new column)
-** array {
-** varint position; (delta from previous position plus POS_BASE)
-** varint startOffset;(delta from previous startOffset)
-** varint endOffset; (delta from startOffset)
-** }
-** }
-** varint POS_END; (marks end of positions for this document.
-** }
-**
-** Here, array { X } means zero or more occurrences of X, adjacent in
-** memory. A "position" is an index of a token in the token stream
-** generated by the tokenizer, while an "offset" is a byte offset,
-** both based at 0. Note that POS_END and POS_COLUMN occur in the
-** same logical place as the position element, and act as sentinals
-** ending a position list array.
-**
-** A DL_POSITIONS doclist omits the startOffset and endOffset
-** information. A DL_DOCIDS doclist omits both the position and
-** offset information, becoming an array of varint-encoded docids.
-**
-** On-disk data is stored as type DL_DEFAULT, so we don't serialize
-** the type. Due to how deletion is implemented in the segmentation
-** system, on-disk doclists MUST store at least positions.
-**
-**
-**** Segment leaf nodes ****
-** Segment leaf nodes store terms and doclists, ordered by term. Leaf
-** nodes are written using LeafWriter, and read using LeafReader (to
-** iterate through a single leaf node's data) and LeavesReader (to
-** iterate through a segment's entire leaf layer). Leaf nodes have
-** the format:
-**
-** varint iHeight; (height from leaf level, always 0)
-** varint nTerm; (length of first term)
-** char pTerm[nTerm]; (content of first term)
-** varint nDoclist; (length of term's associated doclist)
-** char pDoclist[nDoclist]; (content of doclist)
-** array {
-** (further terms are delta-encoded)
-** varint nPrefix; (length of prefix shared with previous term)
-** varint nSuffix; (length of unshared suffix)
-** char pTermSuffix[nSuffix];(unshared suffix of next term)
-** varint nDoclist; (length of term's associated doclist)
-** char pDoclist[nDoclist]; (content of doclist)
-** }
-**
-** Here, array { X } means zero or more occurrences of X, adjacent in
-** memory.
-**
-** Leaf nodes are broken into blocks which are stored contiguously in
-** the %_segments table in sorted order. This means that when the end
-** of a node is reached, the next term is in the node with the next
-** greater node id.
-**
-** New data is spilled to a new leaf node when the current node
-** exceeds LEAF_MAX bytes (default 2048). New data which itself is
-** larger than STANDALONE_MIN (default 1024) is placed in a standalone
-** node (a leaf node with a single term and doclist). The goal of
-** these settings is to pack together groups of small doclists while
-** making it efficient to directly access large doclists. The
-** assumption is that large doclists represent terms which are more
-** likely to be query targets.
-**
-** TODO(shess) It may be useful for blocking decisions to be more
-** dynamic. For instance, it may make more sense to have a 2.5k leaf
-** node rather than splitting into 2k and .5k nodes. My intuition is
-** that this might extend through 2x or 4x the pagesize.
-**
-**
-**** Segment interior nodes ****
-** Segment interior nodes store blockids for subtree nodes and terms
-** to describe what data is stored by the each subtree. Interior
-** nodes are written using InteriorWriter, and read using
-** InteriorReader. InteriorWriters are created as needed when
-** SegmentWriter creates new leaf nodes, or when an interior node
-** itself grows too big and must be split. The format of interior
-** nodes:
-**
-** varint iHeight; (height from leaf level, always >0)
-** varint iBlockid; (block id of node's leftmost subtree)
-** optional {
-** varint nTerm; (length of first term)
-** char pTerm[nTerm]; (content of first term)
-** array {
-** (further terms are delta-encoded)
-** varint nPrefix; (length of shared prefix with previous term)
-** varint nSuffix; (length of unshared suffix)
-** char pTermSuffix[nSuffix]; (unshared suffix of next term)
-** }
-** }
-**
-** Here, optional { X } means an optional element, while array { X }
-** means zero or more occurrences of X, adjacent in memory.
-**
-** An interior node encodes n terms separating n+1 subtrees. The
-** subtree blocks are contiguous, so only the first subtree's blockid
-** is encoded. The subtree at iBlockid will contain all terms less
-** than the first term encoded (or all terms if no term is encoded).
-** Otherwise, for terms greater than or equal to pTerm[i] but less
-** than pTerm[i+1], the subtree for that term will be rooted at
-** iBlockid+i. Interior nodes only store enough term data to
-** distinguish adjacent children (if the rightmost term of the left
-** child is "something", and the leftmost term of the right child is
-** "wicked", only "w" is stored).
-**
-** New data is spilled to a new interior node at the same height when
-** the current node exceeds INTERIOR_MAX bytes (default 2048).
-** INTERIOR_MIN_TERMS (default 7) keeps large terms from monopolizing
-** interior nodes and making the tree too skinny. The interior nodes
-** at a given height are naturally tracked by interior nodes at
-** height+1, and so on.
-**
-**
-**** Segment directory ****
-** The segment directory in table %_segdir stores meta-information for
-** merging and deleting segments, and also the root node of the
-** segment's tree.
-**
-** The root node is the top node of the segment's tree after encoding
-** the entire segment, restricted to ROOT_MAX bytes (default 1024).
-** This could be either a leaf node or an interior node. If the top
-** node requires more than ROOT_MAX bytes, it is flushed to %_segments
-** and a new root interior node is generated (which should always fit
-** within ROOT_MAX because it only needs space for 2 varints, the
-** height and the blockid of the previous root).
-**
-** The meta-information in the segment directory is:
-** level - segment level (see below)
-** idx - index within level
-** - (level,idx uniquely identify a segment)
-** start_block - first leaf node
-** leaves_end_block - last leaf node
-** end_block - last block (including interior nodes)
-** root - contents of root node
-**
-** If the root node is a leaf node, then start_block,
-** leaves_end_block, and end_block are all 0.
-**
-**
-**** Segment merging ****
-** To amortize update costs, segments are grouped into levels and
-** merged in batches. Each increase in level represents exponentially
-** more documents.
-**
-** New documents (actually, document updates) are tokenized and
-** written individually (using LeafWriter) to a level 0 segment, with
-** incrementing idx. When idx reaches MERGE_COUNT (default 16), all
-** level 0 segments are merged into a single level 1 segment. Level 1
-** is populated like level 0, and eventually MERGE_COUNT level 1
-** segments are merged to a single level 2 segment (representing
-** MERGE_COUNT^2 updates), and so on.
-**
-** A segment merge traverses all segments at a given level in
-** parallel, performing a straightforward sorted merge. Since segment
-** leaf nodes are written in to the %_segments table in order, this
-** merge traverses the underlying sqlite disk structures efficiently.
-** After the merge, all segment blocks from the merged level are
-** deleted.
-**
-** MERGE_COUNT controls how often we merge segments. 16 seems to be
-** somewhat of a sweet spot for insertion performance. 32 and 64 show
-** very similar performance numbers to 16 on insertion, though they're
-** a tiny bit slower (perhaps due to more overhead in merge-time
-** sorting). 8 is about 20% slower than 16, 4 about 50% slower than
-** 16, 2 about 66% slower than 16.
-**
-** At query time, high MERGE_COUNT increases the number of segments
-** which need to be scanned and merged. For instance, with 100k docs
-** inserted:
-**
-** MERGE_COUNT segments
-** 16 25
-** 8 12
-** 4 10
-** 2 6
-**
-** This appears to have only a moderate impact on queries for very
-** frequent terms (which are somewhat dominated by segment merge
-** costs), and infrequent and non-existent terms still seem to be fast
-** even with many segments.
-**
-** TODO(shess) That said, it would be nice to have a better query-side
-** argument for MERGE_COUNT of 16. Also, it is possible/likely that
-** optimizations to things like doclist merging will swing the sweet
-** spot around.
-**
-**
-**
-**** Handling of deletions and updates ****
-** Since we're using a segmented structure, with no docid-oriented
-** index into the term index, we clearly cannot simply update the term
-** index when a document is deleted or updated. For deletions, we
-** write an empty doclist (varint(docid) varint(POS_END)), for updates
-** we simply write the new doclist. Segment merges overwrite older
-** data for a particular docid with newer data, so deletes or updates
-** will eventually overtake the earlier data and knock it out. The
-** query logic likewise merges doclists so that newer data knocks out
-** older data.
-**
-** TODO(shess) Provide a VACUUM type operation to clear out all
-** deletions and duplications. This would basically be a forced merge
-** into a single segment.
-*/
+ * Copyright (C) 2011 Nokia <ivan frade nokia com>
+ *
+ * Author: Carlos Garnacho <carlos lanedo com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA
+ */
-#include <assert.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <ctype.h>
#include <sqlite3.h>
-
-#include <libtracker-common/tracker-language.h>
-#include <libtracker-data/tracker-db-manager.h>
-
#include "tracker-fts.h"
-#include "tracker-fts-config.h"
-#include "tracker-fts-hash.h"
-#include "tracker-parser.h"
-
-/* TODO(shess) MAN, this thing needs some refactoring. At minimum, it
-** would be nice to order the file better, perhaps something along the
-** lines of:
-**
-** - utility functions
-** - table setup functions
-** - table update functions
-** - table query functions
-**
-** Put the query functions last because they're likely to reference
-** typedefs or functions from the table update section.
-*/
-
-#if 0
-# define FTSTRACE(A) printf A; fflush(stdout)
-#else
-# define FTSTRACE(A)
-#endif
-
-#if 0
-static int default_column = 0;
-#endif
-
-
-/* Define to 1 if you want to get debug logs with the parsed query,
- * quite useful to understand the FTS syntax */
-#define PRINT_PARSED_QUERY 0
-
-/*
- * ** Default span for NEAR operators.
- * */
-#define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10
-
-/* It is not safe to call isspace(), tolower(), or isalnum() on
-** hi-bit-set characters. This is the same solution used in the
-** tokenizer.
-*/
-/* TODO(shess) The snippet-generation code should be using the
-** tokenizer-generated tokens rather than doing its own local
-** tokenization.
-*/
-/* TODO(shess) Is __isascii() a portable version of (c&0x80)==0? */
-static int safe_isspace(char c){
- return (c&0x80)==0 ? isspace(c) : 0;
-}
-#if 0
-static int safe_tolower(char c){
- return (c&0x80)==0 ? tolower(c) : c;
-}
-static int safe_isalnum(char c){
- return (c&0x80)==0 ? isalnum(c) : 0;
-}
-#endif
-
-
-typedef enum DocListType {
-
- DL_DOCIDS, /* docids only */
- DL_POSITIONS, /* docids + positions */
- DL_POSITIONS_RANK, /* docids + catid + rank + positions */
- DL_POSITIONS_OFFSETS /* docids + positions + offsets */
-} DocListType;
-
-/*
-** By default, only positions and not offsets are stored in the doclists.
-** To change this so that offsets are stored too, compile with
-**
-** -DDL_DEFAULT=DL_POSITIONS_OFFSETS
-**
-** If DL_DEFAULT is set to DL_DOCIDS, your table can only be inserted
-** into (no deletes or updates).
-*/
-#ifndef DL_DEFAULT
-# define DL_DEFAULT DL_POSITIONS
-#endif
-
-#define RANK
-
-enum {
- POS_END = 0, /* end of this position list */
- POS_COLUMN, /* followed by new column number */
- POS_BASE
-};
-
-/* MERGE_COUNT controls how often we merge segments (see comment at
-** top of file).
-*/
-#define MERGE_COUNT 16
-
-/* utility functions */
-
-/* CLEAR() and SCRAMBLE() abstract memset() on a pointer to a single
-** record to prevent errors of the form:
-**
-** my_function(SomeType *b){
-** memset(b, '\0', sizeof(b)); // sizeof(b)!=sizeof(*b)
-** }
-*/
-/* TODO(shess) Obvious candidates for a header file. */
-#define CLEAR(b) memset(b, '\0', sizeof(*(b)))
-
-#ifndef NDEBUG
-# define SCRAMBLE(b) memset(b, 0x55, sizeof(*(b)))
-#else
-# define SCRAMBLE(b)
-#endif
-
-/* We may need up to VARINT_MAX bytes to store an encoded 64-bit integer. */
-#define VARINT_MAX 10
-
-/* Write a 64-bit variable-length integer to memory starting at p[0].
- * The length of data written will be between 1 and VARINT_MAX bytes.
- * The number of bytes written is returned. */
-static int fts3PutVarint(char *p, sqlite_int64 v){
- unsigned char *q = (unsigned char *) p;
- sqlite_uint64 vu = v;
- do{
- *q++ = (unsigned char) ((vu & 0x7f) | 0x80);
- vu >>= 7;
- }while( vu!=0 );
- q[-1] &= 0x7f; /* turn off high bit in final byte */
- assert( q - (unsigned char *)p <= VARINT_MAX );
- return (int) (q - (unsigned char *)p);
-}
-
-/* Read a 64-bit variable-length integer from memory starting at p[0].
- * Return the number of bytes read, or 0 on error.
- * The value is stored in *v. */
-static int fts3GetVarint(const char *p, sqlite_int64 *v){
- const unsigned char *q = (const unsigned char *) p;
- sqlite_uint64 x = 0, y = 1;
- while( (*q & 0x80) == 0x80 ){
- x += y * (*q++ & 0x7f);
- y <<= 7;
- if( q - (unsigned char *)p >= VARINT_MAX ){ /* bad data */
- assert( 0 );
- return 0;
- }
- }
- x += y * (*q++);
- *v = (sqlite_int64) x;
- return (int) (q - (unsigned char *)p);
-}
-
-static int fts3GetVarint32(const char *p, int *pi){
- sqlite_int64 i = 0;
- int ret = fts3GetVarint(p, &i);
- *pi = (int) i;
- assert( *pi==i );
- return ret;
-}
-
-/*******************************************************************/
-/* DataBuffer is used to collect data into a buffer in piecemeal
-** fashion. It implements the usual distinction between amount of
-** data currently stored (nData) and buffer capacity (nCapacity).
-**
-** dataBufferInit - create a buffer with given initial capacity.
-** dataBufferReset - forget buffer's data, retaining capacity.
-** dataBufferDestroy - free buffer's data.
-** dataBufferSwap - swap contents of two buffers.
-** dataBufferExpand - expand capacity without adding data.
-** dataBufferAppend - append data.
-** dataBufferAppend2 - append two pieces of data at once.
-** dataBufferReplace - replace buffer's data.
-*/
-typedef struct DataBuffer {
- char *pData; /* Pointer to malloc'ed buffer. */
- int nCapacity; /* Size of pData buffer. */
- int nData; /* End of data loaded into pData. */
-} DataBuffer;
-
-static void dataBufferInit(DataBuffer *pBuffer, int nCapacity){
- assert( nCapacity>=0 );
- pBuffer->nData = 0;
- pBuffer->nCapacity = nCapacity;
- pBuffer->pData = nCapacity==0 ? NULL : sqlite3_malloc(nCapacity);
-}
-static void dataBufferReset(DataBuffer *pBuffer){
- pBuffer->nData = 0;
-}
-static void dataBufferDestroy(DataBuffer *pBuffer){
- if( pBuffer->pData!=NULL ) sqlite3_free(pBuffer->pData);
- SCRAMBLE(pBuffer);
-}
-static void dataBufferSwap(DataBuffer *pBuffer1, DataBuffer *pBuffer2){
- DataBuffer tmp = *pBuffer1;
- *pBuffer1 = *pBuffer2;
- *pBuffer2 = tmp;
-}
-static void dataBufferExpand(DataBuffer *pBuffer, int nAddCapacity){
- assert( nAddCapacity>0 );
- /* TODO(shess) Consider expanding more aggressively. Note that the
- ** underlying malloc implementation may take care of such things for
- ** us already.
- */
- if( pBuffer->nData+nAddCapacity>pBuffer->nCapacity ){
- pBuffer->nCapacity = pBuffer->nData+nAddCapacity;
- pBuffer->pData = sqlite3_realloc(pBuffer->pData, pBuffer->nCapacity);
- }
-}
-static void dataBufferAppend(DataBuffer *pBuffer,
- const char *pSource, int nSource){
- assert( nSource>0 && pSource!=NULL );
- dataBufferExpand(pBuffer, nSource);
- memcpy(pBuffer->pData+pBuffer->nData, pSource, nSource);
- pBuffer->nData += nSource;
-}
-static void dataBufferAppend2(DataBuffer *pBuffer,
- const char *pSource1, int nSource1,
- const char *pSource2, int nSource2){
- assert( nSource1>0 && pSource1!=NULL );
- assert( nSource2>0 && pSource2!=NULL );
- dataBufferExpand(pBuffer, nSource1+nSource2);
- memcpy(pBuffer->pData+pBuffer->nData, pSource1, nSource1);
- memcpy(pBuffer->pData+pBuffer->nData+nSource1, pSource2, nSource2);
- pBuffer->nData += nSource1+nSource2;
-}
-static void dataBufferReplace(DataBuffer *pBuffer,
- const char *pSource, int nSource){
- dataBufferReset(pBuffer);
- dataBufferAppend(pBuffer, pSource, nSource);
-}
-
-/* StringBuffer is a null-terminated version of DataBuffer. */
-typedef struct StringBuffer {
- DataBuffer b; /* Includes null terminator. */
-} StringBuffer;
-
-static void initStringBuffer(StringBuffer *sb){
- dataBufferInit(&sb->b, 100);
- dataBufferReplace(&sb->b, "", 1);
-}
-static int stringBufferLength(StringBuffer *sb){
- return sb->b.nData-1;
-}
-static char *stringBufferData(StringBuffer *sb){
- return sb->b.pData;
-}
-static void stringBufferDestroy(StringBuffer *sb){
- dataBufferDestroy(&sb->b);
-}
-
-static void nappend(StringBuffer *sb, const char *zFrom, int nFrom){
- assert( sb->b.nData>0 );
- if( nFrom>0 ){
- sb->b.nData--;
- dataBufferAppend2(&sb->b, zFrom, nFrom, "", 1);
- }
-}
-static void append(StringBuffer *sb, const char *zFrom){
- nappend(sb, zFrom, strlen(zFrom));
-}
-
-#if 0
-/* Append a list of strings separated by commas. */
-static void appendList(StringBuffer *sb, int nString, char **azString){
- int i;
- for(i=0; i<nString; ++i){
- if( i>0 ) append(sb, ", ");
- append(sb, azString[i]);
- }
-}
-#endif
-
-static int endsInWhiteSpace(StringBuffer *p){
- return stringBufferLength(p)>0 &&
- safe_isspace(stringBufferData(p)[stringBufferLength(p)-1]);
-}
-
-/* If the StringBuffer ends in something other than white space, add a
-** single space character to the end.
-*/
-static void appendWhiteSpace(StringBuffer *p){
- if( stringBufferLength(p)==0 ) return;
- if( !endsInWhiteSpace(p) ) append(p, " ");
-}
-
-/* Remove white space from the end of the StringBuffer */
-static void trimWhiteSpace(StringBuffer *p){
- while( endsInWhiteSpace(p) ){
- p->b.pData[--p->b.nData-1] = '\0';
- }
-}
-
-/*******************************************************************/
-/* DLReader is used to read document elements from a doclist. The
-** current docid is cached, so dlrDocid() is fast. DLReader does not
-** own the doclist buffer.
-**
-** dlrAtEnd - true if there's no more data to read.
-** dlrDocid - docid of current document.
-** dlrDocData - doclist data for current document (including docid).
-** dlrDocDataBytes - length of same.
-** dlrAllDataBytes - length of all remaining data.
-** dlrPosData - position data for current document.
-** dlrPosDataLen - length of pos data for current document (incl POS_END).
-** dlrStep - step to current document.
-** dlrInit - initial for doclist of given type against given data.
-** dlrDestroy - clean up.
-**
-** Expected usage is something like:
-**
-** DLReader reader;
-** dlrInit(&reader, pData, nData);
-** while( !dlrAtEnd(&reader) ){
-** // calls to dlrDocid() and kin.
-** dlrStep(&reader);
-** }
-** dlrDestroy(&reader);
-*/
-typedef struct DLReader {
- DocListType iType;
- const char *pData;
- int nData;
-
- sqlite_int64 iDocid;
-
-#ifdef STORE_CATEGORY
- int Catid;
-#endif
-
-
- int nElement;
-} DLReader;
-
-static int dlrAtEnd(DLReader *pReader){
- assert( pReader->nData>=0 );
- return pReader->nData==0;
-}
-static sqlite_int64 dlrDocid(DLReader *pReader){
- assert( !dlrAtEnd(pReader) );
- return pReader->iDocid;
-}
-
-#ifdef STORE_CATEGORY
-static int dlrCatid(DLReader *pReader){
- //assert( !dlrAtEnd(pReader) );
- return pReader->Catid;
-}
-#endif
-
-static const char *dlrDocData(DLReader *pReader){
- assert( !dlrAtEnd(pReader) );
- return pReader->pData;
-}
-static int dlrDocDataBytes(DLReader *pReader){
- assert( !dlrAtEnd(pReader) );
- return pReader->nElement;
-}
-static int dlrAllDataBytes(DLReader *pReader){
- assert( !dlrAtEnd(pReader) );
- return pReader->nData;
-}
-/* TODO(shess) Consider adding a field to track iDocid varint length
-** to make these two functions faster. This might matter (a tiny bit)
-** for queries.
-*/
-static const char *dlrPosData(DLReader *pReader){
- sqlite_int64 iDummy;
- int n = fts3GetVarint(pReader->pData, &iDummy);
-
-#ifdef STORE_CATEGORY
- int Catid;
- n += fts3GetVarint32(pReader->pData+n, &Catid);
-#endif
-
- assert( !dlrAtEnd(pReader) );
- return pReader->pData+n;
-}
-static int dlrPosDataLen(DLReader *pReader){
- sqlite_int64 iDummy;
- int n = fts3GetVarint(pReader->pData, &iDummy);
-
-#ifdef STORE_CATEGORY
- int Catid;
- n += fts3GetVarint32(pReader->pData+n, &Catid);
-#endif
-
- assert( !dlrAtEnd(pReader) );
- return pReader->nElement-n;
-}
-static void dlrStep(DLReader *pReader){
- assert( !dlrAtEnd(pReader) );
-
- /* Skip past current doclist element. */
- assert( pReader->nElement<=pReader->nData );
- pReader->pData += pReader->nElement;
- pReader->nData -= pReader->nElement;
-
- /* If there is more data, read the next doclist element. */
- if( pReader->nData!=0 ){
- sqlite_int64 iDocidDelta = 0;
-
- int iDummy, n = fts3GetVarint(pReader->pData, &iDocidDelta);
- pReader->iDocid += iDocidDelta;
-
-#ifdef STORE_CATEGORY
- int Catid;
- n += fts3GetVarint32(pReader->pData+n, &Catid);
- pReader->Catid = Catid;
-#endif
-
- if( pReader->iType>=DL_POSITIONS ){
- assert( n<pReader->nData );
- while( 1 ){
- n += fts3GetVarint32(pReader->pData+n, &iDummy);
- assert( n<=pReader->nData );
- if( iDummy==POS_END ) break;
- if( iDummy==POS_COLUMN ){
- n += fts3GetVarint32(pReader->pData+n, &iDummy);
- assert( n<pReader->nData );
- }else if( pReader->iType==DL_POSITIONS_OFFSETS ){
- n += fts3GetVarint32(pReader->pData+n, &iDummy);
- n += fts3GetVarint32(pReader->pData+n, &iDummy);
- assert( n<pReader->nData );
- }
- }
- }
- pReader->nElement = n;
- assert( pReader->nElement<=pReader->nData );
- }
-}
-static void dlrInit(DLReader *pReader, DocListType iType,
- const char *pData, int nData){
- assert( pData!=NULL && nData!=0 );
- pReader->iType = iType;
- pReader->pData = pData;
- pReader->nData = nData;
- pReader->nElement = 0;
- pReader->iDocid = 0;
-
-#ifdef STORE_CATEGORY
- pReader->Catid = 0;
-#endif
-
- /* Load the first element's data. There must be a first element. */
- dlrStep(pReader);
-}
-static void dlrDestroy(DLReader *pReader){
- SCRAMBLE(pReader);
-}
-
-#ifndef NDEBUG
-/* Verify that the doclist can be validly decoded. Also returns the
-** last docid found because it is convenient in other assertions for
-** DLWriter.
-*/
-static void docListValidate(DocListType iType, const char *pData, int nData,
- sqlite_int64 *pLastDocid){
- sqlite_int64 iPrevDocid = 0;
- assert( nData>0 );
- assert( pData!=0 );
- assert( pData+nData>pData );
- while( nData!=0 ){
- sqlite_int64 iDocidDelta;
- int n = fts3GetVarint(pData, &iDocidDelta);
- iPrevDocid += iDocidDelta;
-
-#ifdef STORE_CATEGORY
- int Catid;
- n += fts3GetVarint32(pData+n, &Catid);
-#endif
-
- if( iType>DL_DOCIDS ){
- int iDummy;
- while( 1 ){
- n += fts3GetVarint32(pData+n, &iDummy);
- if( iDummy==POS_END ) break;
- if( iDummy==POS_COLUMN ){
- n += fts3GetVarint32(pData+n, &iDummy);
- }else if( iType>DL_POSITIONS ){
- n += fts3GetVarint32(pData+n, &iDummy);
- n += fts3GetVarint32(pData+n, &iDummy);
- }
- assert( n<=nData );
- }
- }
- assert( n<=nData );
- pData += n;
- nData -= n;
- }
- if( pLastDocid ) *pLastDocid = iPrevDocid;
-}
-#define ASSERT_VALID_DOCLIST(i, p, n, o) docListValidate(i, p, n, o)
-#else
-#define ASSERT_VALID_DOCLIST(i, p, n, o) assert( 1 )
-#endif
-
-/*******************************************************************/
-/* DLWriter is used to write doclist data to a DataBuffer. DLWriter
-** always appends to the buffer and does not own it.
-**
-** dlwInit - initialize to write a given type doclistto a buffer.
-** dlwDestroy - clear the writer's memory. Does not free buffer.
-** dlwAppend - append raw doclist data to buffer.
-** dlwCopy - copy next doclist from reader to writer.
-** dlwAdd - construct doclist element and append to buffer.
-** Only apply dlwAdd() to DL_DOCIDS doclists (else use PLWriter).
-*/
-typedef struct DLWriter {
- DocListType iType;
- DataBuffer *b;
- sqlite_int64 iPrevDocid;
-#ifndef NDEBUG
- int has_iPrevDocid;
-#endif
-} DLWriter;
-
-static void dlwInit(DLWriter *pWriter, DocListType iType, DataBuffer *b){
- pWriter->b = b;
- pWriter->iType = iType;
- pWriter->iPrevDocid = 0;
-#ifndef NDEBUG
- pWriter->has_iPrevDocid = 0;
-#endif
-}
-static void dlwDestroy(DLWriter *pWriter){
- SCRAMBLE(pWriter);
-}
-/* iFirstDocid is the first docid in the doclist in pData. It is
-** needed because pData may point within a larger doclist, in which
-** case the first item would be delta-encoded.
-**
-** iLastDocid is the final docid in the doclist in pData. It is
-** needed to create the new iPrevDocid for future delta-encoding. The
-** code could decode the passed doclist to recreate iLastDocid, but
-** the only current user (docListMerge) already has decoded this
-** information.
-*/
-/* TODO(shess) This has become just a helper for docListMerge.
-** Consider a refactor to make this cleaner.
-*/
-static void dlwAppend(DLWriter *pWriter,
- const char *pData, int nData,
- sqlite_int64 iFirstDocid, sqlite_int64 iLastDocid){
- sqlite_int64 iDocid = 0;
- char c[VARINT_MAX];
- int nFirstOld, nFirstNew; /* Old and new varint len of first docid. */
-#ifndef NDEBUG
- sqlite_int64 iLastDocidDelta;
-#endif
-
- /* Recode the initial docid as delta from iPrevDocid. */
- nFirstOld = fts3GetVarint(pData, &iDocid);
- assert( nFirstOld<nData || (nFirstOld==nData && pWriter->iType==DL_DOCIDS) );
- nFirstNew = fts3PutVarint(c, iFirstDocid-pWriter->iPrevDocid);
-
- /* Verify that the incoming doclist is valid AND that it ends with
- ** the expected docid. This is essential because we'll trust this
- ** docid in future delta-encoding.
- */
- ASSERT_VALID_DOCLIST(pWriter->iType, pData, nData, &iLastDocidDelta);
- assert( iLastDocid==iFirstDocid-iDocid+iLastDocidDelta );
-
- /* Append recoded initial docid and everything else. Rest of docids
- ** should have been delta-encoded from previous initial docid.
- */
- if( nFirstOld<nData ){
- dataBufferAppend2(pWriter->b, c, nFirstNew,
- pData+nFirstOld, nData-nFirstOld);
- }else{
- dataBufferAppend(pWriter->b, c, nFirstNew);
- }
- pWriter->iPrevDocid = iLastDocid;
-}
-static void dlwCopy(DLWriter *pWriter, DLReader *pReader){
- dlwAppend(pWriter, dlrDocData(pReader), dlrDocDataBytes(pReader),
- dlrDocid(pReader), dlrDocid(pReader));
-}
-
-
-#ifndef RANK
-#ifdef STORE_CATEGORY
-static void dlwAdd(DLWriter *pWriter, sqlite_int64 iDocid, int Catid){
- char c[VARINT_MAX];
- int n = fts3PutVarint(c, iDocid-pWriter->iPrevDocid);
-
- /* Docids must ascend. */
- assert( !pWriter->has_iPrevDocid || iDocid>pWriter->iPrevDocid );
- assert( pWriter->iType==DL_DOCIDS );
-
- dataBufferAppend(pWriter->b, c, n);
- pWriter->iPrevDocid = iDocid;
-
- n = fts3PutVarint(c, Catid);
- dataBufferAppend(pWriter->b, c, n);
-
-#ifndef NDEBUG
- pWriter->has_iPrevDocid = 1;
-#endif
-}
-#else
-
-static void dlwAdd(DLWriter *pWriter, sqlite_int64 iDocid){
- char c[VARINT_MAX];
- int n = fts3PutVarint(c, iDocid-pWriter->iPrevDocid);
-
- /* Docids must ascend. */
- assert( !pWriter->has_iPrevDocid || iDocid>pWriter->iPrevDocid );
- assert( pWriter->iType==DL_DOCIDS );
-
- dataBufferAppend(pWriter->b, c, n);
- pWriter->iPrevDocid = iDocid;
-#ifndef NDEBUG
- pWriter->has_iPrevDocid = 1;
-#endif
-}
-
-#endif
-#endif
-
-/*******************************************************************/
-/* PLReader is used to read data from a document's position list. As
-** the caller steps through the list, data is cached so that varints
-** only need to be decoded once.
-**
-** plrInit, plrDestroy - create/destroy a reader.
-** plrColumn, plrPosition, plrStartOffset, plrEndOffset - accessors
-** plrAtEnd - at end of stream, only call plrDestroy once true.
-** plrStep - step to the next element.
-*/
-typedef struct PLReader {
- /* These refer to the next position's data. nData will reach 0 when
- ** reading the last position, so plrStep() signals EOF by setting
- ** pData to NULL.
- */
- const char *pData;
- int nData;
-
- DocListType iType;
- int iColumn; /* the last column read */
- int iPosition; /* the last position read */
- int iStartOffset; /* the last start offset read */
- int iEndOffset; /* the last end offset read */
-} PLReader;
-
-static int plrAtEnd(PLReader *pReader){
- return pReader->pData==NULL;
-}
-static int plrColumn(PLReader *pReader){
- assert( !plrAtEnd(pReader) );
- return pReader->iColumn;
-}
-static int plrPosition(PLReader *pReader){
- assert( !plrAtEnd(pReader) );
- return pReader->iPosition;
-}
-static int plrStartOffset(PLReader *pReader){
- assert( !plrAtEnd(pReader) );
- return pReader->iStartOffset;
-}
-static int plrEndOffset(PLReader *pReader){
- assert( !plrAtEnd(pReader) );
- return pReader->iEndOffset;
-}
-static void plrStep(PLReader *pReader){
- int i, n;
-
- assert( !plrAtEnd(pReader) );
-
- if( pReader->nData==0 ){
- pReader->pData = NULL;
- return;
- }
-
- n = fts3GetVarint32(pReader->pData, &i);
- if( i==POS_COLUMN ){
- n += fts3GetVarint32(pReader->pData+n, &pReader->iColumn);
- pReader->iPosition = 0;
- pReader->iStartOffset = 0;
- n += fts3GetVarint32(pReader->pData+n, &i);
- }
- /* Should never see adjacent column changes. */
- assert( i!=POS_COLUMN );
-
- if( i==POS_END ){
- pReader->nData = 0;
- pReader->pData = NULL;
- return;
- }
-
- pReader->iPosition += i-POS_BASE;
- if( pReader->iType==DL_POSITIONS_OFFSETS ){
- n += fts3GetVarint32(pReader->pData+n, &i);
- pReader->iStartOffset += i;
- n += fts3GetVarint32(pReader->pData+n, &i);
- pReader->iEndOffset = pReader->iStartOffset+i;
- }
- assert( n<=pReader->nData );
- pReader->pData += n;
- pReader->nData -= n;
-}
-
-static void plrInit(PLReader *pReader, DLReader *pDLReader){
- pReader->pData = dlrPosData(pDLReader);
- pReader->nData = dlrPosDataLen(pDLReader);
- pReader->iType = pDLReader->iType;
- pReader->iColumn = 0;
- pReader->iPosition = 0;
- pReader->iStartOffset = 0;
- pReader->iEndOffset = 0;
- plrStep(pReader);
-}
-static void plrDestroy(PLReader *pReader){
- SCRAMBLE(pReader);
-}
+#include "fts3.h"
-#if 0
-/* because plrDestroy should only be called when plrAtEnd is true,
-we create a new convenience function to do this in one call */
-static void plrEndAndDestroy (PLReader *pReader){
+gboolean tracker_fts_init (void) {
+ static gsize module_initialized = 0;
+ int rc = SQLITE_OK;
- while (!plrAtEnd(pReader)) {
- plrStep (pReader);
+ if (g_once_init_enter (&module_initialized)) {
+ rc = sqlite3_auto_extension ((void (*) (void)) fts4_extension_init);
+ g_once_init_leave (&module_initialized, (rc == SQLITE_OK));
}
- plrDestroy (pReader);
-}
-#endif
-
-/*******************************************************************/
-/* PLWriter is used in constructing a document's position list. As a
-** convenience, if iType is DL_DOCIDS, PLWriter becomes a no-op.
-** PLWriter writes to the associated DLWriter's buffer.
-**
-** plwInit - init for writing a document's poslist.
-** plwDestroy - clear a writer.
-** plwAdd - append position and offset information.
-** plwCopy - copy next position's data from reader to writer.
-** plwTerminate - add any necessary doclist terminator.
-**
-** Calling plwAdd() after plwTerminate() may result in a corrupt
-** doclist.
-*/
-/* TODO(shess) Until we've written the second item, we can cache the
-** first item's information. Then we'd have three states:
-**
-** - initialized with docid, no positions.
-** - docid and one position.
-** - docid and multiple positions.
-**
-** Only the last state needs to actually write to dlw->b, which would
-** be an improvement in the DLCollector case.
-*/
-typedef struct PLWriter {
- DLWriter *dlw;
-
- int iColumn; /* the last column written */
- int iPos; /* the last position written */
- int iOffset; /* the last start offset written */
-} PLWriter;
-
-/* TODO(shess) In the case where the parent is reading these values
-** from a PLReader, we could optimize to a copy if that PLReader has
-** the same type as pWriter.
-*/
-static void plwAdd(PLWriter *pWriter, int iColumn, int iPos,
- int iStartOffset, int iEndOffset){
- /* Worst-case space for POS_COLUMN, iColumn, iPosDelta,
- ** iStartOffsetDelta, and iEndOffsetDelta.
- */
- char c[5*VARINT_MAX];
- int n = 0;
-
- /* Ban plwAdd() after plwTerminate(). */
- assert( pWriter->iPos!=-1 );
-
- if( pWriter->dlw->iType==DL_DOCIDS ) return;
-
- if( iColumn!=pWriter->iColumn ){
- n += fts3PutVarint(c+n, POS_COLUMN);
- n += fts3PutVarint(c+n, iColumn);
- pWriter->iColumn = iColumn;
- pWriter->iPos = 0;
- pWriter->iOffset = 0;
- }
- assert( iPos>=pWriter->iPos );
- n += fts3PutVarint(c+n, POS_BASE+(iPos-pWriter->iPos));
- pWriter->iPos = iPos;
- if( pWriter->dlw->iType==DL_POSITIONS_OFFSETS ){
- assert( iStartOffset>=pWriter->iOffset );
- n += fts3PutVarint(c+n, iStartOffset-pWriter->iOffset);
- pWriter->iOffset = iStartOffset;
- assert( iEndOffset>=iStartOffset );
- n += fts3PutVarint(c+n, iEndOffset-iStartOffset);
- }
- dataBufferAppend(pWriter->dlw->b, c, n);
-}
-static void plwCopy(PLWriter *pWriter, PLReader *pReader){
- plwAdd(pWriter, plrColumn(pReader), plrPosition(pReader),
- plrStartOffset(pReader), plrEndOffset(pReader));
-}
-
-
-#ifdef STORE_CATEGORY
-static void plwInit(PLWriter *pWriter, DLWriter *dlw, sqlite_int64 iDocid, int Catid){
- char c[VARINT_MAX];
- int n;
-
- pWriter->dlw = dlw;
-
- /* Docids must ascend. */
- assert( !pWriter->dlw->has_iPrevDocid || iDocid>pWriter->dlw->iPrevDocid );
- n = fts3PutVarint(c, iDocid-pWriter->dlw->iPrevDocid);
- dataBufferAppend(pWriter->dlw->b, c, n);
- pWriter->dlw->iPrevDocid = iDocid;
-
- n = fts3PutVarint(c, Catid);
- dataBufferAppend(pWriter->dlw->b, c, n);
-
-#ifndef NDEBUG
- pWriter->dlw->has_iPrevDocid = 1;
-#endif
-
- /* [tracker] - the default column (ID = 0) should be that of File:Contents for Files db and Email:Body for email db
- that way we avoid wrting a column ID and cosuming more bytes for the most voluminous cases */
- pWriter->iColumn = default_column;
- pWriter->iPos = 0;
- pWriter->iOffset = 0;
-}
-
-#else
-
-static void plwInit(PLWriter *pWriter, DLWriter *dlw, sqlite_int64 iDocid){
- char c[VARINT_MAX];
- int n;
-
- pWriter->dlw = dlw;
-
- /* Docids must ascend. */
- assert( !pWriter->dlw->has_iPrevDocid || iDocid>pWriter->dlw->iPrevDocid );
- n = fts3PutVarint(c, iDocid-pWriter->dlw->iPrevDocid);
- dataBufferAppend(pWriter->dlw->b, c, n);
- pWriter->dlw->iPrevDocid = iDocid;
-
-#ifndef NDEBUG
- pWriter->dlw->has_iPrevDocid = 1;
-#endif
-
- pWriter->iColumn = 0;
- pWriter->iPos = 0;
- pWriter->iOffset = 0;
-}
-
-#endif
-
-
-
-/* TODO(shess) Should plwDestroy() also terminate the doclist? But
-
-** then plwDestroy() would no longer be just a destructor, it would
-** also be doing work, which isn't consistent with the overall idiom.
-** Another option would be for plwAdd() to always append any necessary
-** terminator, so that the output is always correct. But that would
-** add incremental work to the common case with the only benefit being
-** API elegance. Punt for now.
-*/
-static void plwTerminate(PLWriter *pWriter){
- if( pWriter->dlw->iType>DL_DOCIDS ){
- char c[VARINT_MAX];
- int n = fts3PutVarint(c, POS_END);
- dataBufferAppend(pWriter->dlw->b, c, n);
- }
-#ifndef NDEBUG
- /* Mark as terminated for assert in plwAdd(). */
- pWriter->iPos = -1;
-#endif
-}
-static void plwDestroy(PLWriter *pWriter){
- SCRAMBLE(pWriter);
-}
-
-/*******************************************************************/
-/* DLCollector wraps PLWriter and DLWriter to provide a
-** dynamically-allocated doclist area to use during tokenization.
-**
-** dlcNew - malloc up and initialize a collector.
-** dlcDelete - destroy a collector and all contained items.
-** dlcAddPos - append position and offset information.
-** dlcAddDoclist - add the collected doclist to the given buffer.
-** dlcNext - terminate the current document and open another.
-*/
-typedef struct DLCollector {
- DataBuffer b;
- DLWriter dlw;
- PLWriter plw;
-} DLCollector;
-
-/* TODO(shess) This could also be done by calling plwTerminate() and
-** dataBufferAppend(). I tried that, expecting nominal performance
-** differences, but it seemed to pretty reliably be worth 1% to code
-** it this way. I suspect it is the incremental malloc overhead (some
-** percentage of the plwTerminate() calls will cause a realloc), so
-** this might be worth revisiting if the DataBuffer implementation
-** changes.
-*/
-static void dlcAddDoclist(DLCollector *pCollector, DataBuffer *b){
- if( pCollector->dlw.iType>DL_DOCIDS ){
- char c[VARINT_MAX];
- int n = fts3PutVarint(c, POS_END);
- dataBufferAppend2(b, pCollector->b.pData, pCollector->b.nData, c, n);
- }else{
- dataBufferAppend(b, pCollector->b.pData, pCollector->b.nData);
- }
-}
-
-static void dlcAddPos(DLCollector *pCollector, int iColumn, int iPos,
- int iStartOffset, int iEndOffset){
- plwAdd(&pCollector->plw, iColumn, iPos, iStartOffset, iEndOffset);
-}
-
-
-#ifdef STORE_CATEGORY
-static void dlcNext(DLCollector *pCollector, sqlite_int64 iDocid, int Catid){
- plwTerminate(&pCollector->plw);
- plwDestroy(&pCollector->plw);
- plwInit(&pCollector->plw, &pCollector->dlw, iDocid, Catid);
-}
-
-
-static DLCollector *dlcNew(sqlite_int64 iDocid, DocListType iType, int Catid){
- DLCollector *pCollector = sqlite3_malloc(sizeof(DLCollector));
- dataBufferInit(&pCollector->b, 0);
- dlwInit(&pCollector->dlw, iType, &pCollector->b);
- plwInit(&pCollector->plw, &pCollector->dlw, iDocid, Catid);
- return pCollector;
-}
-
-#else
-
-static void dlcNext(DLCollector *pCollector, sqlite_int64 iDocid){
- plwTerminate(&pCollector->plw);
- plwDestroy(&pCollector->plw);
- plwInit(&pCollector->plw, &pCollector->dlw, iDocid);
-}
-
-static DLCollector *dlcNew(sqlite_int64 iDocid, DocListType iType){
- DLCollector *pCollector = sqlite3_malloc(sizeof(DLCollector));
- dataBufferInit(&pCollector->b, 0);
- dlwInit(&pCollector->dlw, iType, &pCollector->b);
- plwInit(&pCollector->plw, &pCollector->dlw, iDocid);
- return pCollector;
-}
-
-#endif
-
-static void dlcDelete(DLCollector *pCollector){
- plwDestroy(&pCollector->plw);
- dlwDestroy(&pCollector->dlw);
- dataBufferDestroy(&pCollector->b);
- SCRAMBLE(pCollector);
- sqlite3_free(pCollector);
-}
-
-
-/* Copy the doclist data of iType in pData/nData into *out, trimming
-** unnecessary data as we go. Only columns matching iColumn are
-** copied, all columns copied if iColumn is -1. Elements with no
-** matching columns are dropped. The output is an iOutType doclist.
-*/
-/* NOTE(shess) This code is only valid after all doclists are merged.
-** If this is run before merges, then doclist items which represent
-** deletion will be trimmed, and will thus not effect a deletion
-** during the merge.
-*/
-static void docListTrim(DocListType iType, const char *pData, int nData,
- int iColumn, DocListType iOutType, DataBuffer *out){
- DLReader dlReader;
- DLWriter dlWriter;
-
- assert( iOutType<=iType );
-
- dlrInit(&dlReader, iType, pData, nData);
- dlwInit(&dlWriter, iOutType, out);
-
- while( !dlrAtEnd(&dlReader) ){
- PLReader plReader;
- PLWriter plWriter;
- int match = 0;
-
- plrInit(&plReader, &dlReader);
-
- while( !plrAtEnd(&plReader) ){
- if( iColumn==-1 || plrColumn(&plReader)==iColumn ){
-
- if( !match ){
-
-#ifdef STORE_CATEGORY
- plwInit(&plWriter, &dlWriter, dlrDocid(&dlReader), dlrCatid(&dlReader));
-#else
- plwInit(&plWriter, &dlWriter, dlrDocid(&dlReader));
-#endif
- match = 1;
- }
- plwAdd(&plWriter, plrColumn(&plReader), plrPosition(&plReader),
- plrStartOffset(&plReader), plrEndOffset(&plReader));
- }
- plrStep(&plReader);
- }
- if( match ){
- plwTerminate(&plWriter);
- plwDestroy(&plWriter);
- }
-
- plrDestroy(&plReader);
- dlrStep(&dlReader);
- }
- dlwDestroy(&dlWriter);
- dlrDestroy(&dlReader);
-}
-
-/* Used by docListMerge() to keep doclists in the ascending order by
-** docid, then ascending order by age (so the newest comes first).
-*/
-typedef struct OrderedDLReader {
- DLReader *pReader;
-
- /* TODO(shess) If we assume that docListMerge pReaders is ordered by
- ** age (which we do), then we could use pReader comparisons to break
- ** ties.
- */
- int idx;
-} OrderedDLReader;
-
-/* Order eof to end, then by docid asc, idx desc. */
-static int orderedDLReaderCmp(OrderedDLReader *r1, OrderedDLReader *r2){
- if( dlrAtEnd(r1->pReader) ){
- if( dlrAtEnd(r2->pReader) ) return 0; /* Both atEnd(). */
- return 1; /* Only r1 atEnd(). */
- }
- if( dlrAtEnd(r2->pReader) ) return -1; /* Only r2 atEnd(). */
-
- if( dlrDocid(r1->pReader)<dlrDocid(r2->pReader) ) return -1;
- if( dlrDocid(r1->pReader)>dlrDocid(r2->pReader) ) return 1;
-
- /* Descending on idx. */
- return r2->idx-r1->idx;
-}
-
-/* Bubble p[0] to appropriate place in p[1..n-1]. Assumes that
-** p[1..n-1] is already sorted.
-*/
-/* TODO(shess) Is this frequent enough to warrant a binary search?
-** Before implementing that, instrument the code to check. In most
-** current usage, I expect that p[0] will be less than p[1] a very
-** high proportion of the time.
-*/
-static void orderedDLReaderReorder(OrderedDLReader *p, int n){
- while( n>1 && orderedDLReaderCmp(p, p+1)>0 ){
- OrderedDLReader tmp = p[0];
- p[0] = p[1];
- p[1] = tmp;
- n--;
- p++;
- }
-}
-
-/* Given an array of doclist readers, merge their doclist elements
-** into out in sorted order (by docid), dropping elements from older
-** readers when there is a duplicate docid. pReaders is assumed to be
-** ordered by age, oldest first.
-*/
-/* TODO(shess) nReaders must be <= MERGE_COUNT. This should probably
-** be fixed.
-*/
-static void docListMerge(DataBuffer *out,
- DLReader *pReaders, int nReaders){
- OrderedDLReader readers[MERGE_COUNT];
- DLWriter writer;
- int i, n;
- const char *pStart = 0;
- int nStart = 0;
- sqlite_int64 iFirstDocid = 0, iLastDocid = 0;
-
- assert( nReaders>0 );
- if( nReaders==1 ){
- dataBufferAppend(out, dlrDocData(pReaders), dlrAllDataBytes(pReaders));
- return;
- }
-
- assert( nReaders<=MERGE_COUNT );
- n = 0;
- for(i=0; i<nReaders; i++){
- assert( pReaders[i].iType==pReaders[0].iType );
- readers[i].pReader = pReaders+i;
- readers[i].idx = i;
- n += dlrAllDataBytes(&pReaders[i]);
- }
- /* Conservatively size output to sum of inputs. Output should end
- ** up strictly smaller than input.
- */
- dataBufferExpand(out, n);
-
- /* Get the readers into sorted order. */
- while( i-->0 ){
- orderedDLReaderReorder(readers+i, nReaders-i);
- }
-
- dlwInit(&writer, pReaders[0].iType, out);
- while( !dlrAtEnd(readers[0].pReader) ){
- sqlite_int64 iDocid = dlrDocid(readers[0].pReader);
-
- /* If this is a continuation of the current buffer to copy, extend
- ** that buffer. memcpy() seems to be more efficient if it has a
- ** lots of data to copy.
- */
- if( dlrDocData(readers[0].pReader)==pStart+nStart ){
- nStart += dlrDocDataBytes(readers[0].pReader);
- }else{
- if( pStart!=0 ){
- dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid);
- }
- pStart = dlrDocData(readers[0].pReader);
- nStart = dlrDocDataBytes(readers[0].pReader);
- iFirstDocid = iDocid;
- }
- iLastDocid = iDocid;
- dlrStep(readers[0].pReader);
-
- /* Drop all of the older elements with the same docid. */
- for(i=1; i<nReaders &&
- !dlrAtEnd(readers[i].pReader) &&
- dlrDocid(readers[i].pReader)==iDocid; i++){
- dlrStep(readers[i].pReader);
- }
-
- /* Get the readers back into order. */
- while( i-->0 ){
- orderedDLReaderReorder(readers+i, nReaders-i);
- }
- }
-
- /* Copy over any remaining elements. */
- if( nStart>0 ) dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid);
- dlwDestroy(&writer);
-}
-
-/* Helper function for posListUnion(). Compares the current position
-** between left and right, returning as standard C idiom of <0 if
-** left<right, >0 if left>right, and 0 if left==right. "End" always
-** compares greater.
-*/
-static int posListCmp(PLReader *pLeft, PLReader *pRight){
- assert( pLeft->iType==pRight->iType );
- if( pLeft->iType==DL_DOCIDS ) return 0;
-
- if( plrAtEnd(pLeft) ) return plrAtEnd(pRight) ? 0 : 1;
- if( plrAtEnd(pRight) ) return -1;
-
- if( plrColumn(pLeft)<plrColumn(pRight) ) return -1;
- if( plrColumn(pLeft)>plrColumn(pRight) ) return 1;
-
- if( plrPosition(pLeft)<plrPosition(pRight) ) return -1;
- if( plrPosition(pLeft)>plrPosition(pRight) ) return 1;
- if( pLeft->iType==DL_POSITIONS ) return 0;
-
- if( plrStartOffset(pLeft)<plrStartOffset(pRight) ) return -1;
- if( plrStartOffset(pLeft)>plrStartOffset(pRight) ) return 1;
-
- if( plrEndOffset(pLeft)<plrEndOffset(pRight) ) return -1;
- if( plrEndOffset(pLeft)>plrEndOffset(pRight) ) return 1;
-
- return 0;
+ return (module_initialized != 0);
}
-/* Write the union of position lists in pLeft and pRight to pOut.
-** "Union" in this case meaning "All unique position tuples". Should
-** work with any doclist type, though both inputs and the output
-** should be the same type.
-*/
-static void posListUnion(DLReader *pLeft, DLReader *pRight, DLWriter *pOut){
- PLReader left, right;
- PLWriter writer;
+gboolean tracker_fts_init_db (sqlite3 *db, int create){
+ int rc = SQLITE_OK;
- assert( dlrDocid(pLeft)==dlrDocid(pRight) );
- assert( pLeft->iType==pRight->iType );
- assert( pLeft->iType==pOut->iType );
-
- plrInit(&left, pLeft);
- plrInit(&right, pRight);
-
-#ifdef STORE_CATEGORY
- plwInit(&writer, pOut, dlrDocid(pLeft), dlrCatid(pLeft));
-#else
- plwInit(&writer, pOut, dlrDocid(pLeft));
-#endif
-
-
- while( !plrAtEnd(&left) || !plrAtEnd(&right) ){
- int c = posListCmp(&left, &right);
- if( c<0 ){
- plwCopy(&writer, &left);
- plrStep(&left);
- }else if( c>0 ){
- plwCopy(&writer, &right);
- plrStep(&right);
- }else{
- plwCopy(&writer, &left);
- plrStep(&left);
- plrStep(&right);
- }
- }
-
- plwTerminate(&writer);
- plwDestroy(&writer);
- plrDestroy(&left);
- plrDestroy(&right);
-}
-
-/* Write the union of doclists in pLeft and pRight to pOut. For
-** docids in common between the inputs, the union of the position
-** lists is written. Inputs and outputs are always type DL_DEFAULT.
-*/
-static void docListUnion(
- const char *pLeft, int nLeft,
- const char *pRight, int nRight,
- DataBuffer *pOut /* Write the combined doclist here */
-){
- DLReader left, right;
- DLWriter writer;
-
- if( nLeft==0 ){
- if( nRight!=0) dataBufferAppend(pOut, pRight, nRight);
- return;
- }
- if( nRight==0 ){
- dataBufferAppend(pOut, pLeft, nLeft);
- return;
- }
-
- dlrInit(&left, DL_DEFAULT, pLeft, nLeft);
- dlrInit(&right, DL_DEFAULT, pRight, nRight);
- dlwInit(&writer, DL_DEFAULT, pOut);
-
- while( !dlrAtEnd(&left) || !dlrAtEnd(&right) ){
- if( dlrAtEnd(&right) ){
- dlwCopy(&writer, &left);
- dlrStep(&left);
- }else if( dlrAtEnd(&left) ){
- dlwCopy(&writer, &right);
- dlrStep(&right);
- }else if( dlrDocid(&left)<dlrDocid(&right) ){
- dlwCopy(&writer, &left);
- dlrStep(&left);
- }else if( dlrDocid(&left)>dlrDocid(&right) ){
- dlwCopy(&writer, &right);
- dlrStep(&right);
- }else{
- posListUnion(&left, &right, &writer);
- dlrStep(&left);
- dlrStep(&right);
- }
- }
-
- dlrDestroy(&left);
- dlrDestroy(&right);
- dlwDestroy(&writer);
-}
-
-/*
-** This function is used as part of the implementation of phrase and
-** NEAR matching.
-**
-** pLeft and pRight are DLReaders positioned to the same docid in
-** lists of type DL_POSITION. This function writes an entry to the
-** DLWriter pOut for each position in pRight that is less than
-** (nNear+1) greater (but not equal to or smaller) than a position
-** in pLeft. For example, if nNear is 0, and the positions contained
-** by pLeft and pRight are:
-**
-** pLeft: 5 10 15 20
-** pRight: 6 9 17 21
-**
-** then the docid is added to pOut. If pOut is of type DL_POSITIONS,
-** then a positionids "6" and "21" are also added to pOut.
-**
-** If boolean argument isSaveLeft is true, then positionids are copied
-** from pLeft instead of pRight. In the example above, the positions "5"
-** and "20" would be added instead of "6" and "21".
-** If isSaveLeft = 2 then both positions are added, 3 or above and postions are appended to left
-*/
-static void posListPhraseMerge(
- DLReader *pLeft,
- DLReader *pRight,
- int nNear,
- int isSaveLeft,
- int term_num,
- DLWriter *pOut
-){
- PLReader left, right;
- PLWriter writer;
- int match = 0;
-
- assert( dlrDocid(pLeft)==dlrDocid(pRight) );
- assert( pOut->iType!=DL_POSITIONS_OFFSETS );
-
- plrInit(&left, pLeft);
- plrInit(&right, pRight);
-
-
-
- while( !plrAtEnd(&left) && !plrAtEnd(&right) ){
- if( plrColumn(&left)<plrColumn(&right) ){
- plrStep(&left);
- }else if( plrColumn(&left)>plrColumn(&right) ){
- plrStep(&right);
- }else if( plrPosition(&left)>=plrPosition(&right) ){
- plrStep(&right);
- }else{
- if( (plrPosition(&right)-plrPosition(&left))<=(nNear+1) ){
-
- if( !match ){
-
-#ifdef STORE_CATEGORY
- plwInit(&writer, pOut, dlrDocid(pLeft), dlrCatid(pLeft));
-#else
- plwInit(&writer, pOut, dlrDocid(pLeft));
-#endif
- match = 1;
- }
- if( isSaveLeft == 0 ){
- plwAdd(&writer, plrColumn(&right), plrPosition(&right), 0, 0);
- }else if( isSaveLeft == 1 ){
- plwAdd(&writer, plrColumn(&left), plrPosition(&left), 0, 0);
- } else {
-
- int iPosRight = plrPosition(&right);
- int iColumnRight = plrColumn(&right);
- int i;
-
- for (i=term_num; i>=0; i--) {
- plwAdd(&writer, iColumnRight, iPosRight - i, 0, 0);
- }
-
- }
-
- plrStep(&right);
- }else{
- plrStep(&left);
- }
- }
- }
-
- if( match ){
- plwTerminate(&writer);
- plwDestroy(&writer);
- }
-
- plrDestroy(&left);
- plrDestroy(&right);
-}
+ if (create){
+ rc = sqlite3_exec(db, "CREATE VIRTUAL TABLE fts USING fts4",
+ NULL, 0, NULL);
+ }
-/*
-** Compare the values pointed to by the PLReaders passed as arguments.
-** Return -1 if the value pointed to by pLeft is considered less than
-** the value pointed to by pRight, +1 if it is considered greater
-** than it, or 0 if it is equal. i.e.
-**
-** (*pLeft - *pRight)
-**
-** A PLReader that is in the EOF condition is considered greater than
-** any other. If neither argument is in EOF state, the return value of
-** plrColumn() is used. If the plrColumn() values are equal, the
-** comparison is on the basis of plrPosition().
-*/
-static int plrCompare(PLReader *pLeft, PLReader *pRight){
- assert(!plrAtEnd(pLeft) || !plrAtEnd(pRight));
+ if (SQLITE_OK != rc){
+ return FALSE;
+ }
- if( plrAtEnd(pRight) || plrAtEnd(pLeft) ){
- return (plrAtEnd(pRight) ? -1 : 1);
- }
- if( plrColumn(pLeft)!=plrColumn(pRight) ){
- return ((plrColumn(pLeft)<plrColumn(pRight)) ? -1 : 1);
- }
- if( plrPosition(pLeft)!=plrPosition(pRight) ){
- return ((plrPosition(pLeft)<plrPosition(pRight)) ? -1 : 1);
- }
- return 0;
+ return TRUE;
}
-
-/* We have two doclists with positions: pLeft and pRight. Depending
-** on the value of the nNear parameter, perform either a phrase
-** intersection (if nNear==0) or a NEAR intersection (if nNear>0)
-** and write the results into pOut.
-**
-** A phrase intersection means that two documents only match
-** if pLeft.iPos+1==pRight.iPos.
-**
-** A NEAR intersection means that two documents only match if
-** (abs(pLeft.iPos-pRight.iPos)<nNear).
-**
-** If a NEAR intersection is requested, then the nPhrase argument should
-** be passed the number of tokens in the two operands to the NEAR operator
-** combined. For example:
-**
-** Query syntax nPhrase
-** ------------------------------------
-** "A B C" NEAR "D E" 5
-** A NEAR B 2
-**
-** iType controls the type of data written to pOut. If iType is
-** DL_POSITIONS, the positions are those from pRight.
-*/
-static void docListPhraseMerge(
- const char *pLeft, int nLeft,
- const char *pRight, int nRight,
- int nNear, /* 0 for a phrase merge, non-zero for a NEAR merge */
- int nPhrase, /* Number of tokens in left+right operands to NEAR */
- DocListType iType, /* Type of doclist to write to pOut */
- DataBuffer *pOut, /* Write the combined doclist here */
- int term /* (tracker) term number */
-){
- DLReader left, right;
- DLWriter writer;
-
- if( nLeft==0 || nRight==0 ) return;
-
- assert( iType!=DL_POSITIONS_OFFSETS );
-
- dlrInit(&left, DL_POSITIONS, pLeft, nLeft);
- dlrInit(&right, DL_POSITIONS, pRight, nRight);
- dlwInit(&writer, iType, pOut);
- while( !dlrAtEnd(&left) && !dlrAtEnd(&right) ){
- if( dlrDocid(&left)<dlrDocid(&right) ){
- dlrStep(&left);
- }else if( dlrDocid(&right)<dlrDocid(&left) ){
- dlrStep(&right);
- }else{
- if( nNear==0 ){
- posListPhraseMerge(&left, &right, 0, 2, term, &writer);
- }else{
- /* This case occurs when two terms (simple terms or phrases) are
- * connected by a NEAR operator, span (nNear+1). i.e.
- *
- * '"terrible company" NEAR widget'
- */
- DataBuffer one = {0, 0, 0};
- DataBuffer two = {0, 0, 0};
-
- DLWriter dlwriter2;
- DLReader dr1 = {0, 0, 0, 0, 0};
- DLReader dr2 = {0, 0, 0, 0, 0};
-
- dlwInit(&dlwriter2, iType, &one);
- posListPhraseMerge(&right, &left, nNear-3+nPhrase, 1, term, &dlwriter2);
- dlwInit(&dlwriter2, iType, &two);
- posListPhraseMerge(&left, &right, nNear-1, 0, term, &dlwriter2);
-
- if( one.nData) dlrInit(&dr1, iType, one.pData, one.nData);
- if( two.nData) dlrInit(&dr2, iType, two.pData, two.nData);
-
- if( !dlrAtEnd(&dr1) || !dlrAtEnd(&dr2) ){
- PLReader pr1 = {0};
- PLReader pr2 = {0};
-
- PLWriter plwriter;
-
-
-#ifdef STORE_CATEGORY
- plwInit(&plwriter, &writer, dlrDocid(dlrAtEnd(&dr1)?&dr2:&dr1), dlrCatid(dlrAtEnd(&dr1)?&dr2:&dr1));
-#else
- plwInit(&plwriter, &writer, dlrDocid(dlrAtEnd(&dr1)?&dr2:&dr1));
-#endif
-
- if( one.nData ) plrInit(&pr1, &dr1);
- if( two.nData ) plrInit(&pr2, &dr2);
- while( !plrAtEnd(&pr1) || !plrAtEnd(&pr2) ){
- int iCompare = plrCompare(&pr1, &pr2);
- switch( iCompare ){
- case -1:
- plwCopy(&plwriter, &pr1);
- plrStep(&pr1);
- break;
- case 1:
- plwCopy(&plwriter, &pr2);
- plrStep(&pr2);
- break;
- case 0:
- plwCopy(&plwriter, &pr1);
- plrStep(&pr1);
- plrStep(&pr2);
- break;
- }
- }
- plwTerminate(&plwriter);
- }
- dataBufferDestroy(&one);
- dataBufferDestroy(&two);
- }
- dlrStep(&left);
- dlrStep(&right);
- }
- }
-
- dlrDestroy(&left);
- dlrDestroy(&right);
- dlwDestroy(&writer);
-}
-
-/* We have two DL_DOCIDS doclists: pLeft and pRight.
-** Write the intersection of these two doclists into pOut as a
-** DL_DOCIDS doclist.
-*/
-#ifdef RANK
-static void docListAndMerge(
- const char *pLeft, int nLeft,
- const char *pRight, int nRight,
- DataBuffer *pOut /* Write the combined doclist here */
-){
- DLReader left, right;
- DLWriter writer;
-
- if( nLeft==0 || nRight==0 ) return;
-
-
- dlrInit(&left, DL_POSITIONS, pLeft, nLeft);
- dlrInit(&right, DL_POSITIONS, pRight, nRight);
- dlwInit(&writer, DL_POSITIONS, pOut);
-
-
-
- while( !dlrAtEnd(&left) && !dlrAtEnd(&right) ){
- if(dlrDocid(&left)<dlrDocid(&right) ){
- dlrStep(&left);
- }else if( dlrDocid(&right)<dlrDocid(&left) ){
- dlrStep(&right);
- }else{
- posListUnion(&left, &right, &writer);
- dlrStep(&left);
- dlrStep(&right);
- }
- }
-
- dlrDestroy(&left);
- dlrDestroy(&right);
- dlwDestroy(&writer);
-}
-
-#else
-static void docListAndMerge(
- const char *pLeft, int nLeft,
- const char *pRight, int nRight,
- DataBuffer *pOut /* Write the combined doclist here */
-){
- DLReader left, right;
- DLWriter writer;
-
- if( nLeft==0 || nRight==0 ) return;
-
- dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
- dlrInit(&right, DL_DOCIDS, pRight, nRight);
- dlwInit(&writer, DL_DOCIDS, pOut);
-
- while( !dlrAtEnd(&left) && !dlrAtEnd(&right) ){
- if( dlrDocid(&left)<dlrDocid(&right) ){
- dlrStep(&left);
- }else if( dlrDocid(&right)<dlrDocid(&left) ){
- dlrStep(&right);
- }else{
- dlwAdd(&writer, dlrDocid(&left));
- dlrStep(&left);
- dlrStep(&right);
- }
- }
-
- dlrDestroy(&left);
- dlrDestroy(&right);
- dlwDestroy(&writer);
-}
-#endif
-
-/* We have two DL_DOCIDS doclists: pLeft and pRight.
-** Write the union of these two doclists into pOut as a
-** DL_DOCIDS doclist.
-*/
-
-#ifdef RANK
-static void docListOrMerge(
- const char *pLeft, int nLeft,
- const char *pRight, int nRight,
- DataBuffer *pOut /* Write the combined doclist here */
-){
- DLReader left, right;
- DLWriter writer;
-
- if( nLeft==0 ){
- if( nRight!=0 ) dataBufferAppend(pOut, pRight, nRight);
- return;
- }
- if( nRight==0 ){
- dataBufferAppend(pOut, pLeft, nLeft);
- return;
- }
-
- dlrInit(&left, DL_POSITIONS, pLeft, nLeft);
- dlrInit(&right, DL_POSITIONS, pRight, nRight);
- dlwInit(&writer, DL_POSITIONS, pOut);
-
- while( !dlrAtEnd(&left) || !dlrAtEnd(&right) ){
- if( dlrAtEnd(&right) ){
- dlwCopy (&writer, &left);
- dlrStep(&left);
- }else if( dlrAtEnd(&left) ){
-
- dlwCopy (&writer, &right);
- dlrStep(&right);
- }else if( dlrDocid(&left)<dlrDocid(&right) ){
-
- dlwCopy (&writer, &left);
-
- dlrStep(&left);
- }else if( dlrDocid(&right)<dlrDocid(&left) ){
-
- dlwCopy (&writer, &right);
-
- dlrStep(&right);
- }else{
-
- posListUnion(&left, &right, &writer);
-
- dlrStep(&left);
- dlrStep(&right);
- }
- }
-
- dlrDestroy(&left);
- dlrDestroy(&right);
- dlwDestroy(&writer);
-}
-
-#else
-
-static void docListOrMerge(
- const char *pLeft, int nLeft,
- const char *pRight, int nRight,
- DataBuffer *pOut /* Write the combined doclist here */
-){
- DLReader left, right;
- DLWriter writer;
-
- if( nLeft==0 ){
- if( nRight!=0 ) dataBufferAppend(pOut, pRight, nRight);
- return;
- }
- if( nRight==0 ){
- dataBufferAppend(pOut, pLeft, nLeft);
- return;
- }
-
- dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
- dlrInit(&right, DL_DOCIDS, pRight, nRight);
- dlwInit(&writer, DL_DOCIDS, pOut);
-
- while( !dlrAtEnd(&left) || !dlrAtEnd(&right) ){
- if( dlrAtEnd(&right) ){
- dlwAdd(&writer, dlrDocid(&left));
-
-
- dlrStep(&left);
- }else if( dlrAtEnd(&left) ){
-
- dlwAdd(&writer, dlrDocid(&right));
-
- dlrStep(&right);
- }else if( dlrDocid(&left)<dlrDocid(&right) ){
-
- dlwAdd(&writer, dlrDocid(&left));
-
-
- dlrStep(&left);
- }else if( dlrDocid(&right)<dlrDocid(&left) ){
-
- dlwAdd(&writer, dlrDocid(&right));
-
-
-
- dlrStep(&right);
- }else{
-
- dlwAdd(&writer, dlrDocid(&left));
-
-
- dlrStep(&left);
- dlrStep(&right);
- }
- }
-
- dlrDestroy(&left);
- dlrDestroy(&right);
- dlwDestroy(&writer);
-}
-#endif
-
-/* We have two DL_DOCIDS doclists: pLeft and pRight.
-** Write into pOut as DL_DOCIDS doclist containing all documents that
-** occur in pLeft but not in pRight.
-*/
-#ifdef RANK
-static void docListExceptMerge(
- const char *pLeft, int nLeft,
- const char *pRight, int nRight,
- DataBuffer *pOut /* Write the combined doclist here */
-){
- DLReader left, right;
- DLWriter writer;
-
- if( nLeft==0 ) return;
- if( nRight==0 ){
- dataBufferAppend(pOut, pLeft, nLeft);
- return;
- }
-
- dlrInit(&left, DL_POSITIONS, pLeft, nLeft);
- dlrInit(&right, DL_POSITIONS, pRight, nRight);
- dlwInit(&writer, DL_POSITIONS, pOut);
-
- while( !dlrAtEnd(&left) ){
- while( !dlrAtEnd(&right) && dlrDocid(&right)<dlrDocid(&left) ){
- dlrStep(&right);
- }
- if( dlrAtEnd(&right) || dlrDocid(&left)<dlrDocid(&right) ){
- dlwCopy (&writer, &left);
- }
- dlrStep(&left);
- }
-
- dlrDestroy(&left);
- dlrDestroy(&right);
- dlwDestroy(&writer);
-}
-#else
-static void docListExceptMerge(
- const char *pLeft, int nLeft,
- const char *pRight, int nRight,
- DataBuffer *pOut /* Write the combined doclist here */
-){
- DLReader left, right;
- DLWriter writer;
-
- if( nLeft==0 ) return;
- if( nRight==0 ){
- dataBufferAppend(pOut, pLeft, nLeft);
- return;
- }
-
- dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
- dlrInit(&right, DL_DOCIDS, pRight, nRight);
- dlwInit(&writer, DL_DOCIDS, pOut);
-
- while( !dlrAtEnd(&left) ){
- while( !dlrAtEnd(&right) && dlrDocid(&right)<dlrDocid(&left) ){
- dlrStep(&right);
- }
- if( dlrAtEnd(&right) || dlrDocid(&left)<dlrDocid(&right) ){
-
- dlwAdd(&writer, dlrDocid(&left));
- }
- dlrStep(&left);
- }
-
- dlrDestroy(&left);
- dlrDestroy(&right);
- dlwDestroy(&writer);
-}
-#endif
-
-#if 0
-static char *string_dup_n(const char *s, int n){
- char *str = sqlite3_malloc(n + 1);
- memcpy(str, s, n);
- str[n] = '\0';
- return str;
-}
-
-/* Duplicate a string; the caller must free() the returned string.
- * (We don't use strdup() since it is not part of the standard C library and
- * may not be available everywhere.) */
-static char *string_dup(const char *s){
- return string_dup_n(s, strlen(s));
-}
-#endif
-
-/* Format a string, replacing each occurrence of the % character with
- * zDb.zName. This may be more convenient than sqlite_mprintf()
- * when one string is used repeatedly in a format string.
- * The caller must free() the returned string. */
-static char *string_format(const char *zFormat,
- const char *zDb, const char *zName){
- const char *p;
- size_t len = 0;
- size_t nDb = strlen(zDb);
- size_t nName = strlen(zName);
- size_t nFullTableName = nDb+1+nName;
- char *result;
- char *r;
-
- /* first compute length needed */
- for(p = zFormat ; *p ; ++p){
- len += (*p=='%' ? nFullTableName : 1);
- }
- len += 1; /* for null terminator */
-
- r = result = sqlite3_malloc(len);
- for(p = zFormat; *p; ++p){
- if( *p=='%' ){
- memcpy(r, zDb, nDb);
- r += nDb;
- *r++ = '.';
- memcpy(r, zName, nName);
- r += nName;
- } else {
- *r++ = *p;
- }
- }
- *r++ = '\0';
- assert( r == result + len );
- return result;
-}
-
-static int sql_exec(sqlite3 *db, const char *zDb, const char *zName,
- const char *zFormat){
- char *zCommand = string_format(zFormat, zDb, zName);
- int rc;
- FTSTRACE(("FTS3 sql: %s\n", zCommand));
- rc = sqlite3_exec(db, zCommand, NULL, 0, NULL);
- sqlite3_free(zCommand);
- return rc;
-}
-
-static int sql_prepare(sqlite3 *db, const char *zDb, const char *zName,
- sqlite3_stmt **ppStmt, const char *zFormat){
- char *zCommand = string_format(zFormat, zDb, zName);
- int rc;
- FTSTRACE(("FTS3 prepare: %s\n", zCommand));
- rc = sqlite3_prepare_v2(db, zCommand, -1, ppStmt, NULL);
- sqlite3_free(zCommand);
- return rc;
-}
-
-/* end utility functions */
-
-/* Forward reference */
-typedef struct fulltext_vtab fulltext_vtab;
-typedef struct fulltext_sqlite_vtab fulltext_sqlite_vtab;
-
-/* A single term in a query is represented by an instances of
-** the following structure. Each word which may match against
-** document content is a term. Operators, like NEAR or OR, are
-** not terms. Query terms are organized as a flat list stored
-** in the Query.pTerms array.
-**
-** If the QueryTerm.nPhrase variable is non-zero, then the QueryTerm
-** is the first in a contiguous string of terms that are either part
-** of the same phrase, or connected by the NEAR operator.
-**
-** If the QueryTerm.nNear variable is non-zero, then the token is followed
-** by a NEAR operator with span set to (nNear-1). For example, the
-** following query:
-**
-** The QueryTerm.iPhrase variable stores the index of the token within
-** its phrase, indexed starting at 1, or 1 if the token is not part
-** of any phrase.
-**
-** For example, the data structure used to represent the following query:
-**
-** ... MATCH 'sqlite NEAR/5 google NEAR/2 "search engine"'
-**
-** is:
-**
-** {nPhrase=4, iPhrase=1, nNear=6, pTerm="sqlite"},
-** {nPhrase=0, iPhrase=1, nNear=3, pTerm="google"},
-** {nPhrase=0, iPhrase=1, nNear=0, pTerm="search"},
-** {nPhrase=0, iPhrase=2, nNear=0, pTerm="engine"},
-**
-** compiling the FTS3 syntax to Query structures is done by the parseQuery()
-** function.
-*/
-typedef struct QueryTerm {
- short int nPhrase; /* How many following terms are part of the same phrase */
- short int iPhrase; /* This is the i-th term of a phrase. */
- short int iColumn; /* Column of the index that must match this term */
- signed char nNear; /* term followed by a NEAR operator with span=(nNear-1) */
- signed char isOr; /* this term is preceded by "OR" */
- signed char isNot; /* this term is preceded by "-" */
- signed char isPrefix; /* this term is followed by "*" */
- char *pTerm; /* text of the term. '\000' terminated. malloced */
- int nTerm; /* Number of bytes in pTerm[] */
-} QueryTerm;
-
-
-/* A query string is parsed into a Query structure.
- *
- * We could, in theory, allow query strings to be complicated
- * nested expressions with precedence determined by parentheses.
- * But none of the major search engines do this. (Perhaps the
- * feeling is that an parenthesized expression is two complex of
- * an idea for the average user to grasp.) Taking our lead from
- * the major search engines, we will allow queries to be a list
- * of terms (with an implied AND operator) or phrases in double-quotes,
- * with a single optional "-" before each non-phrase term to designate
- * negation and an optional OR connector.
- *
- * OR binds more tightly than the implied AND, which is what the
- * major search engines seem to do. So, for example:
- *
- * [one two OR three] ==> one AND (two OR three)
- * [one OR two three] ==> (one OR two) AND three
- *
- * A "-" before a term matches all entries that lack that term.
- * The "-" must occur immediately before the term with in intervening
- * space. This is how the search engines do it.
- *
- * A NOT term cannot be the right-hand operand of an OR. If this
- * occurs in the query string, the NOT is ignored:
- *
- * [one OR -two] ==> one OR two
- *
- */
-typedef struct Query {
- fulltext_vtab *pFts; /* The full text index */
- int nTerms; /* Number of terms in the query */
- QueryTerm *pTerms; /* Array of terms. Space obtained from malloc() */
- int nextIsOr; /* Set the isOr flag on the next inserted term */
- int nextIsNear; /* Set the isOr flag on the next inserted term */
- int nextColumn; /* Next word parsed must be in this column */
- int dfltColumn; /* The default column */
-} Query;
-
-
-/*
-** An instance of the following structure keeps track of generated
-** matching-word offset information and snippets.
-*/
-typedef struct Snippet {
- int nMatch; /* Total number of matches */
- int nAlloc; /* Space allocated for aMatch[] */
- struct snippetMatch { /* One entry for each matching term */
- char snStatus; /* Status flag for use while constructing snippets */
- short int iCol; /* The column that contains the match */
- short int iTerm; /* The index in Query.pTerms[] of the matching term */
- int iToken; /* The index of the matching document token */
- short int nByte; /* Number of bytes in the term */
- int iStart; /* The offset to the first character of the term */
- int rank; /* the rank of the snippet */
- } *aMatch; /* Points to space obtained from malloc */
- char *zOffset; /* Text rendering of aMatch[] */
- int nOffset; /* strlen(zOffset) */
- char *zSnippet; /* Snippet text */
- int nSnippet; /* strlen(zSnippet) */
-} Snippet;
-
-
-typedef enum QueryType {
- QUERY_GENERIC, /* table scan */
- QUERY_DOCID, /* lookup by docid */
- QUERY_FULLTEXT /* QUERY_FULLTEXT + [i] is a full-text search for column i*/
-} QueryType;
-
-typedef enum fulltext_statement {
-#if 0
- CONTENT_INSERT_STMT,
- CONTENT_SELECT_STMT,
- CONTENT_DELETE_STMT,
- CONTENT_EXISTS_STMT,
-#endif
-
- BLOCK_INSERT_STMT,
- BLOCK_SELECT_STMT,
- BLOCK_DELETE_STMT,
- BLOCK_DELETE_ALL_STMT,
-
- SEGDIR_MAX_INDEX_STMT,
- SEGDIR_SET_STMT,
- SEGDIR_SELECT_LEVEL_STMT,
- SEGDIR_SPAN_STMT,
- SEGDIR_DELETE_STMT,
- SEGDIR_SELECT_SEGMENT_STMT,
- SEGDIR_SELECT_ALL_STMT,
- SEGDIR_DELETE_ALL_STMT,
- SEGDIR_COUNT_STMT,
-
- PROPERTY_WEIGHT_STMT,
-
- MAX_STMT /* Always at end! */
-} fulltext_statement;
-
-/* These must exactly match the enum above. */
-/* TODO(shess): Is there some risk that a statement will be used in two
-** cursors at once, e.g. if a query joins a virtual table to itself?
-** If so perhaps we should move some of these to the cursor object.
-*/
-static const char *const fulltext_zStatement[MAX_STMT] = {
-#if 0
- /* CONTENT_INSERT */ NULL, /* generated in contentInsertStatement() */
- /* CONTENT_SELECT */ NULL, /* generated in contentSelectStatement() */
- /* CONTENT_DELETE */ "delete from %_content where docid = ?",
- /* CONTENT_EXISTS */ "select docid from %_content limit 1",
-#endif
-
- /* BLOCK_INSERT */
- "insert into %_segments (blockid, block) values (null, ?)",
- /* BLOCK_SELECT */ "select block from %_segments where blockid = ?",
- /* BLOCK_DELETE */ "delete from %_segments where blockid between ? and ?",
- /* BLOCK_DELETE_ALL */ "delete from %_segments",
-
- /* SEGDIR_MAX_INDEX */ "select max(idx) from %_segdir where level = ?",
- /* SEGDIR_SET */ "insert into %_segdir values (?, ?, ?, ?, ?, ?)",
- /* SEGDIR_SELECT_LEVEL */
- "select start_block, leaves_end_block, root from %_segdir "
- " where level = ? order by idx",
- /* SEGDIR_SPAN */
- "select min(start_block), max(end_block) from %_segdir "
- " where level = ? and start_block <> 0",
- /* SEGDIR_DELETE */ "delete from %_segdir where level = ?",
-
- /* NOTE(shess): The first three results of the following two
- ** statements must match.
- */
- /* SEGDIR_SELECT_SEGMENT */
- "select start_block, leaves_end_block, root from %_segdir "
- " where level = ? and idx = ?",
- /* SEGDIR_SELECT_ALL */
- "select start_block, leaves_end_block, root from %_segdir "
- " order by level desc, idx asc",
- /* SEGDIR_DELETE_ALL */ "delete from %_segdir",
- /* SEGDIR_COUNT */ "select count(*), ifnull(max(level),0) from %_segdir",
-
- /* PROPERTY_WEIGHT */ "SELECT \"tracker:weight\", (SELECT Uri FROM Resource WHERE ID = \"rdf:Property\".ID) FROM \"rdf:Property\" WHERE ID = ?",
-};
-
-/*
-** A connection to a fulltext index is an instance of the following
-** structure. The xCreate and xConnect methods create an instance
-** of this structure and xDestroy and xDisconnect free that instance.
-** All other methods receive a pointer to the structure as one of their
-** arguments.
-*/
-struct fulltext_sqlite_vtab {
- sqlite3_vtab base; /* Base class used by SQLite core */
- fulltext_vtab *fulltext;
-};
-
-struct fulltext_vtab {
- sqlite3 *db; /* The database connection */
- const char *zDb; /* logical database name */
- const char *zName; /* virtual table name */
- int nColumn; /* number of columns in virtual table */
- TrackerParser *parser; /* tokenizer for inserts and queries */
- gboolean enable_stemmer;
- gboolean enable_unaccent;
- gboolean ignore_numbers;
- gboolean ignore_stop_words;
- int max_words;
- int min_word_length;
- int max_word_length;
-
- /* Precompiled statements which we keep as long as the table is
- ** open.
- */
- sqlite3_stmt *pFulltextStatements[MAX_STMT];
-
- /* Precompiled statements used for segment merges. We run a
- ** separate select across the leaf level of each tree being merged.
- */
- sqlite3_stmt *pLeafSelectStmts[MERGE_COUNT];
- /* The statement used to prepare pLeafSelectStmts. */
-#define LEAF_SELECT \
- "select block from %_segments where blockid between ? and ? order by blockid"
-
- /* These buffer pending index updates during transactions.
- ** nPendingData estimates the memory size of the pending data. It
- ** doesn't include the hash-bucket overhead, nor any malloc
- ** overhead. When nPendingData exceeds kPendingThreshold, the
- ** buffer is flushed even before the transaction closes.
- ** pendingTerms stores the data, and is only valid when nPendingData
- ** is >=0 (nPendingData<0 means pendingTerms has not been
- ** initialized). iPrevDocid is the last docid written, used to make
- ** certain we're inserting in sorted order.
- */
- int nPendingData;
-#define kPendingThreshold (1*1024*1024)
- sqlite_int64 iPrevDocid;
- fts3Hash pendingTerms;
-};
-
-/*
-** When the core wants to do a query, it create a cursor using a
-** call to xOpen. This structure is an instance of a cursor. It
-** is destroyed by xClose.
-*/
-typedef struct fulltext_cursor {
- sqlite3_vtab_cursor base; /* Base class used by SQLite core */
- QueryType iCursorType; /* Copy of sqlite3_index_info.idxNum */
- sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */
- int eof; /* True if at End Of Results */
- Query q; /* Parsed query string */
- Snippet snippet; /* Cached snippet for the current row */
- int iColumn; /* Column being searched */
- DataBuffer result; /* Doclist results from fulltextQuery */
- DLReader reader; /* Result reader if result not empty */
- sqlite_int64 currentDocid;
- int currentCatid; /* (tracker) Category (service type ID) of the document */
- GString *offsets; /* (tracker) pre computed offsets from position data in index */
- double rank; /* (tracker) pre computed rank from position data in index */
-} fulltext_cursor;
-
-static fulltext_vtab *get_fulltext_vtab (sqlite3_vtab *sqlite_vtab){
- return ((fulltext_sqlite_vtab *) sqlite_vtab)->fulltext;
-}
-
-static struct fulltext_vtab *cursor_vtab(fulltext_cursor *c){
- return get_fulltext_vtab (c->base.pVtab);
-}
-
-static const sqlite3_module fts3Module; /* forward declaration */
-
-#if 0
-/* Return a dynamically generated statement of the form
- * insert into %_content (docid, ...) values (?, ...)
- */
-static const char *contentInsertStatement(fulltext_vtab *v){
- StringBuffer sb;
-
- initStringBuffer(&sb);
- append(&sb, "insert into %_content (docid) values (?)");
- return stringBufferData(&sb);
-}
-#endif
-
-
-#if 0
-/* Return a dynamically generated statement of the form
- * select <content columns> from %_content where docid = ?
- */
-static const char *contentSelectStatement(fulltext_vtab *v){
- StringBuffer sb;
- int i;
-
- initStringBuffer(&sb);
- append(&sb, "SELECT ");
-
- for (i = 0; i < v->nColumn; i++) {
- if (i > 0) {
- append(&sb, ", ");
- }
- append(&sb, "NULL");
- }
-
- return stringBufferData(&sb);
-}
-#endif
-
-/* Puts a freshly-prepared statement determined by iStmt in *ppStmt.
-** If the indicated statement has never been prepared, it is prepared
-** and cached, otherwise the cached version is reset.
-*/
-static int sql_get_statement(fulltext_vtab *v, fulltext_statement iStmt,
- sqlite3_stmt **ppStmt){
- assert( iStmt<MAX_STMT );
- if( v->pFulltextStatements[iStmt]==NULL ){
- const char *zStmt;
- int rc;
- switch( iStmt ){
-#if 0
- case CONTENT_INSERT_STMT:
- zStmt = contentInsertStatement(v); break;
- case CONTENT_SELECT_STMT:
- zStmt = contentSelectStatement(v); break;
-#endif
- default:
- zStmt = fulltext_zStatement[iStmt];
- }
- rc = sql_prepare(v->db, v->zDb, v->zName, &v->pFulltextStatements[iStmt],
- zStmt);
- if( zStmt != fulltext_zStatement[iStmt]) sqlite3_free((void *) zStmt);
- if( rc!=SQLITE_OK ) return rc;
- } else {
- int rc = sqlite3_reset(v->pFulltextStatements[iStmt]);
- if( rc!=SQLITE_OK ) return rc;
- }
-
- *ppStmt = v->pFulltextStatements[iStmt];
- return SQLITE_OK;
-}
-
-
-/* Function from Tracker */
-static inline int
-get_metadata_weight (fulltext_vtab *v, int id, gchar **uri)
-{
- sqlite3_stmt *stmt;
- int rc;
- int weight;
-
- weight = 1;
-
- rc = sql_get_statement(v, PROPERTY_WEIGHT_STMT, &stmt);
- if (rc != SQLITE_OK) return weight;
-
- rc = sqlite3_bind_int (stmt, 1, id);
- if (rc != SQLITE_OK) return weight;
-
- rc = sqlite3_step (stmt);
- if (rc != SQLITE_ROW) return weight;
-
- weight = sqlite3_column_int (stmt, 0);
- if (weight == 0) {
- weight = 1;
- }
-
- *uri = g_strdup (sqlite3_column_text (stmt, 1));
-
- /* We expect only one row. We must execute another sqlite3_step()
- * to complete the iteration; otherwise the table will remain locked. */
- sqlite3_step (stmt);
-
- return weight;
-}
-
-/* Like sqlite3_step(), but convert SQLITE_DONE to SQLITE_OK and
-** SQLITE_ROW to SQLITE_ERROR. Useful for statements like UPDATE,
-** where we expect no results.
-*/
-static int sql_single_step(sqlite3_stmt *s){
- int rc = sqlite3_step(s);
- return (rc==SQLITE_DONE) ? SQLITE_OK : rc;
-}
-
-/* Like sql_get_statement(), but for special replicated LEAF_SELECT
-** statements. idx -1 is a special case for an uncached version of
-** the statement (used in the optimize implementation).
-*/
-/* TODO(shess) Write version for generic statements and then share
-** that between the cached-statement functions.
-*/
-static int sql_get_leaf_statement(fulltext_vtab *v, int idx,
- sqlite3_stmt **ppStmt){
- assert( idx>=-1 && idx<MERGE_COUNT );
- if( idx==-1 ){
- return sql_prepare(v->db, v->zDb, v->zName, ppStmt, LEAF_SELECT);
- }else if( v->pLeafSelectStmts[idx]==NULL ){
- int rc = sql_prepare(v->db, v->zDb, v->zName, &v->pLeafSelectStmts[idx],
- LEAF_SELECT);
- if( rc!=SQLITE_OK ) return rc;
- }else{
- int rc = sqlite3_reset(v->pLeafSelectStmts[idx]);
- if( rc!=SQLITE_OK ) return rc;
- }
-
- *ppStmt = v->pLeafSelectStmts[idx];
- return SQLITE_OK;
-}
-
-#if 0
-/* insert into %_content (docid, ...) values ([docid], [pValues])
-** If the docid contains SQL NULL, then a unique docid will be
-** generated.
-*/
-static int content_insert(fulltext_vtab *v, sqlite3_value *docid,
- sqlite3_value **pValues){
- sqlite3_stmt *s;
-
- int rc = sql_get_statement(v, CONTENT_INSERT_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_value(s, 1, docid);
- if( rc!=SQLITE_OK ) return rc;
-
- return sql_single_step(s);
-}
-
-static void freeStringArray(int nString, const char **pString){
- int i;
-
- for (i=0 ; i < nString ; ++i) {
- if( pString[i]!=NULL ) sqlite3_free((void *) pString[i]);
- }
- sqlite3_free((void *) pString);
-}
-
-/* select * from %_content where docid = [iDocid]
- * The caller must delete the returned array and all strings in it.
- * null fields will be NULL in the returned array.
- *
- * TODO: Perhaps we should return pointer/length strings here for consistency
- * with other code which uses pointer/length. */
-static int content_select(fulltext_vtab *v, sqlite_int64 iDocid,
- const char ***pValues){
- sqlite3_stmt *s;
- const char **values;
- int i;
- int rc;
-
- *pValues = NULL;
-
- rc = sql_get_statement(v, CONTENT_SELECT_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 1, iDocid);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_step(s);
- if( rc!=SQLITE_ROW ) return rc;
-
- values = (const char **) sqlite3_malloc(v->nColumn * sizeof(const char *));
- for(i=0; i<v->nColumn; ++i){
- if( sqlite3_column_type(s, i)==SQLITE_NULL ){
- values[i] = NULL;
- }else{
- values[i] = string_dup((char*)sqlite3_column_text(s, i));
- }
- }
-
- /* We expect only one row. We must execute another sqlite3_step()
- * to complete the iteration; otherwise the table will remain locked. */
- rc = sqlite3_step(s);
- if( rc==SQLITE_DONE ){
- *pValues = values;
- return SQLITE_OK;
- }
-
- freeStringArray(v->nColumn, values);
- return rc;
-}
-
-/* delete from %_content where docid = [iDocid ] */
-static int content_delete(fulltext_vtab *v, sqlite_int64 iDocid){
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, CONTENT_DELETE_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 1, iDocid);
- if( rc!=SQLITE_OK ) return rc;
-
- return sql_single_step(s);
-}
-
-/* Returns SQLITE_ROW if any rows exist in %_content, SQLITE_DONE if
-** no rows exist, and any error in case of failure.
-*/
-static int content_exists(fulltext_vtab *v){
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, CONTENT_EXISTS_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_step(s);
- if( rc!=SQLITE_ROW ) return rc;
-
- /* We expect only one row. We must execute another sqlite3_step()
- * to complete the iteration; otherwise the table will remain locked. */
- rc = sqlite3_step(s);
- if( rc==SQLITE_DONE ) return SQLITE_ROW;
- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
- return rc;
-}
-#endif
-
-/* insert into %_segments values ([pData])
-** returns assigned blockid in *piBlockid
-*/
-static int block_insert(fulltext_vtab *v, const char *pData, int nData,
- sqlite_int64 *piBlockid){
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, BLOCK_INSERT_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_blob(s, 1, pData, nData, SQLITE_STATIC);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_step(s);
- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
- if( rc!=SQLITE_DONE ) return rc;
-
- /* blockid column is an alias for rowid. */
- *piBlockid = sqlite3_last_insert_rowid(v->db);
- return SQLITE_OK;
-}
-
-/* delete from %_segments
-** where blockid between [iStartBlockid] and [iEndBlockid]
-**
-** Deletes the range of blocks, inclusive, used to delete the blocks
-** which form a segment.
-*/
-static int block_delete(fulltext_vtab *v,
- sqlite_int64 iStartBlockid, sqlite_int64 iEndBlockid){
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, BLOCK_DELETE_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 1, iStartBlockid);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 2, iEndBlockid);
- if( rc!=SQLITE_OK ) return rc;
-
- return sql_single_step(s);
-}
-
-/* Returns SQLITE_ROW with *pidx set to the maximum segment idx found
-** at iLevel. Returns SQLITE_DONE if there are no segments at
-** iLevel. Otherwise returns an error.
-*/
-static int segdir_max_index(fulltext_vtab *v, int iLevel, int *pidx){
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, SEGDIR_MAX_INDEX_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int(s, 1, iLevel);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_step(s);
- /* Should always get at least one row due to how max() works. */
- if( rc==SQLITE_DONE ) return SQLITE_DONE;
- if( rc!=SQLITE_ROW ) return rc;
-
- /* NULL means that there were no inputs to max(). */
- if( SQLITE_NULL==sqlite3_column_type(s, 0) ){
- rc = sqlite3_step(s);
- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
- return rc;
- }
-
- *pidx = sqlite3_column_int(s, 0);
-
- /* We expect only one row. We must execute another sqlite3_step()
- * to complete the iteration; otherwise the table will remain locked. */
- rc = sqlite3_step(s);
- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
- if( rc!=SQLITE_DONE ) return rc;
- return SQLITE_ROW;
-}
-
-/* insert into %_segdir values (
-** [iLevel], [idx],
-** [iStartBlockid], [iLeavesEndBlockid], [iEndBlockid],
-** [pRootData]
-** )
-*/
-static int segdir_set(fulltext_vtab *v, int iLevel, int idx,
- sqlite_int64 iStartBlockid,
- sqlite_int64 iLeavesEndBlockid,
- sqlite_int64 iEndBlockid,
- const char *pRootData, int nRootData){
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, SEGDIR_SET_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int(s, 1, iLevel);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int(s, 2, idx);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 3, iStartBlockid);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 4, iLeavesEndBlockid);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 5, iEndBlockid);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_blob(s, 6, pRootData, nRootData, SQLITE_STATIC);
- if( rc!=SQLITE_OK ) return rc;
-
- return sql_single_step(s);
-}
-
-/* Queries %_segdir for the block span of the segments in level
-** iLevel. Returns SQLITE_DONE if there are no blocks for iLevel,
-** SQLITE_ROW if there are blocks, else an error.
-*/
-static int segdir_span(fulltext_vtab *v, int iLevel,
- sqlite_int64 *piStartBlockid,
- sqlite_int64 *piEndBlockid){
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, SEGDIR_SPAN_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int(s, 1, iLevel);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_step(s);
- if( rc==SQLITE_DONE ) return SQLITE_DONE; /* Should never happen */
- if( rc!=SQLITE_ROW ) return rc;
-
- /* This happens if all segments at this level are entirely inline. */
- if( SQLITE_NULL==sqlite3_column_type(s, 0) ){
- /* We expect only one row. We must execute another sqlite3_step()
- * to complete the iteration; otherwise the table will remain locked. */
- int rc2 = sqlite3_step(s);
- if( rc2==SQLITE_ROW ) return SQLITE_ERROR;
- return rc2;
- }
-
- *piStartBlockid = sqlite3_column_int64(s, 0);
- *piEndBlockid = sqlite3_column_int64(s, 1);
-
- /* We expect only one row. We must execute another sqlite3_step()
- * to complete the iteration; otherwise the table will remain locked. */
- rc = sqlite3_step(s);
- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
- if( rc!=SQLITE_DONE ) return rc;
- return SQLITE_ROW;
-}
-
-/* Delete the segment blocks and segment directory records for all
-** segments at iLevel.
-*/
-static int segdir_delete(fulltext_vtab *v, int iLevel){
- sqlite3_stmt *s;
- sqlite_int64 iStartBlockid = 0, iEndBlockid = 0;
- int rc = segdir_span(v, iLevel, &iStartBlockid, &iEndBlockid);
- if( rc!=SQLITE_ROW && rc!=SQLITE_DONE ) return rc;
-
- if( rc==SQLITE_ROW ){
- rc = block_delete(v, iStartBlockid, iEndBlockid);
- if( rc!=SQLITE_OK ) return rc;
- }
-
- /* Delete the segment directory itself. */
- rc = sql_get_statement(v, SEGDIR_DELETE_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 1, iLevel);
- if( rc!=SQLITE_OK ) return rc;
-
- return sql_single_step(s);
-}
-
-#if 0
-/* Delete entire fts index, SQLITE_OK on success, relevant error on
-** failure.
-*/
-static int segdir_delete_all(fulltext_vtab *v){
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, SEGDIR_DELETE_ALL_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sql_single_step(s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sql_get_statement(v, BLOCK_DELETE_ALL_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- return sql_single_step(s);
-}
-#endif
-
-/* Returns SQLITE_OK with *pnSegments set to the number of entries in
-** %_segdir and *piMaxLevel set to the highest level which has a
-** segment. Otherwise returns the SQLite error which caused failure.
-*/
-static int segdir_count(fulltext_vtab *v, int *pnSegments, int *piMaxLevel){
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, SEGDIR_COUNT_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_step(s);
- /* TODO(shess): This case should not be possible? Should stronger
- ** measures be taken if it happens?
- */
- if( rc==SQLITE_DONE ){
- *pnSegments = 0;
- *piMaxLevel = 0;
- return SQLITE_OK;
- }
- if( rc!=SQLITE_ROW ) return rc;
-
- *pnSegments = sqlite3_column_int(s, 0);
- *piMaxLevel = sqlite3_column_int(s, 1);
-
- /* We expect only one row. We must execute another sqlite3_step()
- * to complete the iteration; otherwise the table will remain locked. */
- rc = sqlite3_step(s);
- if( rc==SQLITE_DONE ) return SQLITE_OK;
- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
- return rc;
-}
-
-/* TODO(shess) clearPendingTerms() is far down the file because
-** writeZeroSegment() is far down the file because LeafWriter is far
-** down the file. Consider refactoring the code to move the non-vtab
-** code above the vtab code so that we don't need this forward
-** reference.
-*/
-static int clearPendingTerms(fulltext_vtab *v);
-
-/*
-** Free the memory used to contain a fulltext_vtab structure.
-*/
-static void fulltext_vtab_destroy(fulltext_vtab *v){
- int iStmt, i;
-
- FTSTRACE(("FTS3 Destroy %p\n", v));
- for( iStmt=0; iStmt<MAX_STMT; iStmt++ ){
- if( v->pFulltextStatements[iStmt]!=NULL ){
- sqlite3_finalize(v->pFulltextStatements[iStmt]);
- v->pFulltextStatements[iStmt] = NULL;
- }
- }
-
- for( i=0; i<MERGE_COUNT; i++ ){
- if( v->pLeafSelectStmts[i]!=NULL ){
- sqlite3_finalize(v->pLeafSelectStmts[i]);
- v->pLeafSelectStmts[i] = NULL;
- }
- }
-
- if( v->parser!=NULL ){
- tracker_parser_free (v->parser);
- v->parser = NULL;
- }
-
- clearPendingTerms(v);
-
- sqlite3_free(v);
-}
-
-#if 0
-/*
-** Token types for parsing the arguments to xConnect or xCreate.
-*/
-#define TOKEN_EOF 0 /* End of file */
-#define TOKEN_SPACE 1 /* Any kind of whitespace */
-#define TOKEN_ID 2 /* An identifier */
-#define TOKEN_STRING 3 /* A string literal */
-#define TOKEN_PUNCT 4 /* A single punctuation character */
-
-/*
-** If X is a character that can be used in an identifier then
-** ftsIdChar(X) will be true. Otherwise it is false.
-**
-** For ASCII, any character with the high-order bit set is
-** allowed in an identifier. For 7-bit characters,
-** isFtsIdChar[X] must be 1.
-**
-** Ticket #1066. the SQL standard does not allow '$' in the
-** middle of identfiers. But many SQL implementations do.
-** SQLite will allow '$' in identifiers for compatibility.
-** But the feature is undocumented.
-*/
-static const char isFtsIdChar[] = {
-/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
- 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
- 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
- 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
-};
-#define ftsIdChar(C) (((c=C)&0x80)!=0 || (c>0x1f && isFtsIdChar[c-0x20]))
-
-
-/*
-** Return the length of the token that begins at z[0].
-** Store the token type in *tokenType before returning.
-*/
-static int ftsGetToken(const char *z, int *tokenType){
- int i, c;
- switch( *z ){
- case 0: {
- *tokenType = TOKEN_EOF;
- return 0;
- }
- case ' ': case '\t': case '\n': case '\f': case '\r': {
- for(i=1; safe_isspace(z[i]); i++){}
- *tokenType = TOKEN_SPACE;
- return i;
- }
- case '`':
- case '\'':
- case '"': {
- int delim = z[0];
- for(i=1; (c=z[i])!=0; i++){
- if( c==delim ){
- if( z[i+1]==delim ){
- i++;
- }else{
- break;
- }
- }
- }
- *tokenType = TOKEN_STRING;
- return i + (c!=0);
- }
- case '[': {
- for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){}
- *tokenType = TOKEN_ID;
- return i;
- }
- default: {
- if( !ftsIdChar(*z) ){
- break;
- }
- for(i=1; ftsIdChar(z[i]); i++){}
- *tokenType = TOKEN_ID;
- return i;
- }
- }
- *tokenType = TOKEN_PUNCT;
- return 1;
-}
-
-/*
-** A token extracted from a string is an instance of the following
-** structure.
-*/
-typedef struct FtsToken {
- const char *z; /* Pointer to token text. Not '\000' terminated */
- short int n; /* Length of the token text in bytes. */
-} FtsToken;
-
-/*
-** Given a input string (which is really one of the argv[] parameters
-** passed into xConnect or xCreate) split the string up into tokens.
-** Return an array of pointers to '\000' terminated strings, one string
-** for each non-whitespace token.
-**
-** The returned array is terminated by a single NULL pointer.
-**
-** Space to hold the returned array is obtained from a single
-** malloc and should be freed by passing the return value to free().
-** The individual strings within the token list are all a part of
-** the single memory allocation and will all be freed at once.
-*/
-static char **tokenizeString(const char *z, int *pnToken){
- int nToken = 0;
- FtsToken *aToken = sqlite3_malloc( strlen(z) * sizeof(aToken[0]) );
- int n = 1;
- int e, i;
- int totalSize = 0;
- char **azToken;
- char *zCopy;
- while( n>0 ){
- n = ftsGetToken(z, &e);
- if( e!=TOKEN_SPACE ){
- aToken[nToken].z = z;
- aToken[nToken].n = n;
- nToken++;
- totalSize += n+1;
- }
- z += n;
- }
- azToken = (char**)sqlite3_malloc( nToken*sizeof(char*) + totalSize );
- zCopy = (char*)&azToken[nToken];
- nToken--;
- for(i=0; i<nToken; i++){
- azToken[i] = zCopy;
- n = aToken[i].n;
- memcpy(zCopy, aToken[i].z, n);
- zCopy[n] = 0;
- zCopy += n+1;
- }
- azToken[nToken] = 0;
- sqlite3_free(aToken);
- *pnToken = nToken;
- return azToken;
-}
-
-/*
-** Convert an SQL-style quoted string into a normal string by removing
-** the quote characters. The conversion is done in-place. If the
-** input does not begin with a quote character, then this routine
-** is a no-op.
-**
-** Examples:
-**
-** "abc" becomes abc
-** 'xyz' becomes xyz
-** [pqr] becomes pqr
-** `mno` becomes mno
-*/
-static void dequoteString(char *z){
- int quote;
- int i, j;
- if( z==0 ) return;
- quote = z[0];
- switch( quote ){
- case '\'': break;
- case '"': break;
- case '`': break; /* For MySQL compatibility */
- case '[': quote = ']'; break; /* For MS SqlServer compatibility */
- default: return;
- }
- for(i=1, j=0; z[i]; i++){
- if( z[i]==quote ){
- if( z[i+1]==quote ){
- z[j++] = quote;
- i++;
- }else{
- z[j++] = 0;
- break;
- }
- }else{
- z[j++] = z[i];
- }
- }
-}
-
-/*
-** The input azIn is a NULL-terminated list of tokens. Remove the first
-** token and all punctuation tokens. Remove the quotes from
-** around string literal tokens.
-**
-** Example:
-**
-** input: tokenize chinese ( 'simplifed' , 'mixed' )
-** output: chinese simplifed mixed
-**
-** Another example:
-**
-** input: delimiters ( '[' , ']' , '...' )
-** output: [ ] ...
-*/
-static void tokenListToIdList(char **azIn){
- int i, j;
- if( azIn ){
- for(i=0, j=-1; azIn[i]; i++){
- if( safe_isalnum(azIn[i][0]) || azIn[i][1] ){
- dequoteString(azIn[i]);
- if( j>=0 ){
- azIn[j] = azIn[i];
- }
- j++;
- }
- }
- azIn[j] = 0;
- }
-}
-
-
-/*
-** Find the first alphanumeric token in the string zIn. Null-terminate
-** this token. Remove any quotation marks. And return a pointer to
-** the result.
-*/
-static char *firstToken(char *zIn, char **pzTail){
- int n, ttype;
- while(1){
- n = ftsGetToken(zIn, &ttype);
- if( ttype==TOKEN_SPACE ){
- zIn += n;
- }else if( ttype==TOKEN_EOF ){
- *pzTail = zIn;
- return 0;
- }else{
- zIn[n] = 0;
- *pzTail = &zIn[1];
- dequoteString(zIn);
- return zIn;
- }
- }
- /*NOTREACHED*/
-}
-
-/* Return true if...
-**
-** * s begins with the string t, ignoring case
-** * s is longer than t
-** * The first character of s beyond t is not a alphanumeric
-**
-** Ignore leading space in *s.
-**
-** To put it another way, return true if the first token of
-** s[] is t[].
-*/
-static int startsWith(const char *s, const char *t){
- while( safe_isspace(*s) ){ s++; }
- while( *t ){
- if( safe_tolower(*s++)!=safe_tolower(*t++) ) return 0;
- }
- return *s!='_' && !safe_isalnum(*s);
-}
-
-/*
-** An instance of this structure defines the "spec" of a
-** full text index. This structure is populated by parseSpec
-** and use by fulltextConnect and fulltextCreate.
-*/
-typedef struct TableSpec {
- const char *zDb; /* Logical database name */
- const char *zName; /* Name of the full-text index */
- int nColumn; /* Number of columns to be indexed */
- char **azColumn; /* Original names of columns to be indexed */
- char **azContentColumn; /* Column names for %_content */
- char **azTokenizer; /* Name of tokenizer and its arguments */
-} TableSpec;
-
-/*
-** Reclaim all of the memory used by a TableSpec
-*/
-static void clearTableSpec(TableSpec *p) {
- sqlite3_free(p->azColumn);
- sqlite3_free(p->azContentColumn);
- sqlite3_free(p->azTokenizer);
-}
-
-/* Parse a CREATE VIRTUAL TABLE statement, which looks like this:
- *
- * CREATE VIRTUAL TABLE email
- * USING fts3(subject, body, tokenize mytokenizer(myarg))
- *
- * We return parsed information in a TableSpec structure.
- *
- */
-static int parseSpec(TableSpec *pSpec, int argc, const char *const*argv,
- char**pzErr){
- int i, n;
- char *z, *zDummy;
- char **azArg;
- const char *zTokenizer = 0; /* argv[] entry describing the tokenizer */
-
- assert( argc>=3 );
- /* Current interface:
- ** argv[0] - module name
- ** argv[1] - database name
- ** argv[2] - table name
- ** argv[3..] - columns, optionally followed by tokenizer specification
- ** and snippet delimiters specification.
- */
-
- /* Make a copy of the complete argv[][] array in a single allocation.
- ** The argv[][] array is read-only and transient. We can write to the
- ** copy in order to modify things and the copy is persistent.
- */
- CLEAR(pSpec);
- for(i=n=0; i<argc; i++){
- n += strlen(argv[i]) + 1;
- }
- azArg = sqlite3_malloc( sizeof(char*)*argc + n );
- if( azArg==0 ){
- return SQLITE_NOMEM;
- }
- z = (char*)&azArg[argc];
- for(i=0; i<argc; i++){
- azArg[i] = z;
- strcpy(z, argv[i]);
- z += strlen(z)+1;
- }
-
- /* Identify the column names and the tokenizer and delimiter arguments
- ** in the argv[][] array.
- */
- pSpec->zDb = azArg[1];
- pSpec->zName = azArg[2];
- pSpec->nColumn = 0;
- pSpec->azColumn = azArg;
- zTokenizer = "tokenize simple";
- for(i=3; i<argc; ++i){
- if( startsWith(azArg[i],"tokenize") ){
- zTokenizer = azArg[i];
- }else{
- z = azArg[pSpec->nColumn] = firstToken(azArg[i], &zDummy);
- pSpec->nColumn++;
- }
- }
- if( pSpec->nColumn==0 ){
- azArg[0] = (char *) "content";
- pSpec->nColumn = 1;
- }
-
- /*
- ** Construct the list of content column names.
- **
- ** Each content column name will be of the form cNNAAAA
- ** where NN is the column number and AAAA is the sanitized
- ** column name. "sanitized" means that special characters are
- ** converted to "_". The cNN prefix guarantees that all column
- ** names are unique.
- **
- ** The AAAA suffix is not strictly necessary. It is included
- ** for the convenience of people who might examine the generated
- ** %_content table and wonder what the columns are used for.
- */
- pSpec->azContentColumn = sqlite3_malloc( pSpec->nColumn * sizeof(char *) );
- if( pSpec->azContentColumn==0 ){
- clearTableSpec(pSpec);
- return SQLITE_NOMEM;
- }
- for(i=0; i<pSpec->nColumn; i++){
- char *p;
- pSpec->azContentColumn[i] = sqlite3_mprintf("c%d%s", i, azArg[i]);
- for (p = pSpec->azContentColumn[i]; *p ; ++p) {
- if( !safe_isalnum(*p) ) *p = '_';
- }
- }
-
- /*
- ** Parse the tokenizer specification string.
- */
- pSpec->azTokenizer = tokenizeString(zTokenizer, &n);
- tokenListToIdList(pSpec->azTokenizer);
-
- return SQLITE_OK;
-}
-#endif
-
-/*
-** Generate a CREATE TABLE statement that describes the schema of
-** the virtual table. Return a pointer to this schema string.
-**
-** Space is obtained from sqlite3_mprintf() and should be freed
-** using sqlite3_free().
-*/
-static char *fulltextSchema(
- int nColumn, /* Number of columns */
- const char *const* azColumn, /* List of columns */
- const char *zTableName /* Name of the table */
-){
- int i;
- char *zSchema, *zNext;
- const char *zSep = "(";
- zSchema = sqlite3_mprintf("CREATE TABLE x");
- for(i=0; i<nColumn; i++){
- zNext = sqlite3_mprintf("%s%s%Q", zSchema, zSep, azColumn[i]);
- sqlite3_free(zSchema);
- zSchema = zNext;
- zSep = ",";
- }
- zNext = sqlite3_mprintf("%s%s%Q HIDDEN", zSchema, zSep, zTableName);
- sqlite3_free(zSchema);
- zSchema = zNext;
- zNext = sqlite3_mprintf("%s,docid HIDDEN)", zSchema);
- sqlite3_free(zSchema);
- return zNext;
-}
-
-/*
-** Build a new sqlite3_vtab structure that will describe the
-** fulltext index defined by spec.
-*/
-static fulltext_vtab *constructVtab(
- sqlite3 *db, /* The SQLite database connection */
- const char *zDb,
- const char *zName,
- char **pzErr /* Write any error message here */
-){
- fulltext_vtab *v = 0;
- TrackerFTSConfig *config;
- TrackerLanguage *language;
-
- v = (fulltext_vtab *) sqlite3_malloc(sizeof(fulltext_vtab));
- if( v==0 ) return NULL;
- CLEAR(v);
- /* sqlite will initialize v->base */
- v->db = db;
- v->zDb = zDb;
- v->zName = zName;
- v->nColumn = 0;
-
-/* comment out tokenizer stuff
- if( spec->azTokenizer==0 ){
- return SQLITE_NOMEM;
- }
-
- zTok = spec->azTokenizer[0];
- if( !zTok ){
- zTok = "simple";
- }
- nTok = strlen(zTok)+1;
-
- m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zTok, nTok);
- if( !m ){
- *pzErr = sqlite3_mprintf("unknown tokenizer: %s", spec->azTokenizer[0]);
- rc = SQLITE_ERROR;
- goto err;
- }
-
- for(n=0; spec->azTokenizer[n]; n++){}
- if( n ){
- rc = m->xCreate(n-1, (const char*const*)&spec->azTokenizer[1],
- &v->pTokenizer);
- }else{
- rc = m->xCreate(0, 0, &v->pTokenizer);
- }
- if( rc!=SQLITE_OK ) goto err;
- */
-
-
- /* set up our parser */
-
- config = tracker_fts_config_new ();
-
- language = tracker_language_new (NULL);
-
- v->min_word_length = tracker_fts_config_get_min_word_length (config);
- v->max_word_length = tracker_fts_config_get_max_word_length (config);
- v->enable_stemmer = tracker_fts_config_get_enable_stemmer (config);
- v->enable_unaccent = tracker_fts_config_get_enable_unaccent (config);
- v->ignore_numbers = tracker_fts_config_get_ignore_numbers (config);
-
- /* disable stop words if TRACKER_FTS_STOP_WORDS is set to 0 - used by tests
- * otherwise, get value from the conf file */
- v->ignore_stop_words = (g_strcmp0 (g_getenv ("TRACKER_FTS_STOP_WORDS"), "0") == 0 ?
- FALSE : tracker_fts_config_get_ignore_stop_words (config));
-
- v->max_words = tracker_fts_config_get_max_words_to_index (config);
-
- v->parser = tracker_parser_new (language);
-
- g_object_unref (language);
-
-
- memset(v->pFulltextStatements, 0, sizeof(v->pFulltextStatements));
-
- /* Indicate that the buffer is not live. */
- v->nPendingData = -1;
-
- FTSTRACE(("FTS3 Connect %p\n", v));
-
- /* Config no longer needed */
- g_object_unref (config);
-
- return v;
-}
-
-static int constructSqliteVtab(
- sqlite3 *db, /* The SQLite database connection */
- void *pAux,
- const char *zDb,
- const char *zName,
- sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */
- char **pzErr /* Write any error message here */
-){
- int rc;
- fulltext_sqlite_vtab *v = 0;
- char *schema;
-
- v = sqlite3_malloc(sizeof(fulltext_sqlite_vtab));
- if( v==0 ) return SQLITE_NOMEM;
- CLEAR(v);
-
- v->fulltext = pAux;
-
- schema = fulltextSchema(0, NULL,
- zName);
- rc = sqlite3_declare_vtab(db, schema);
- sqlite3_free(schema);
- if( rc!=SQLITE_OK ) goto err;
-
- *ppVTab = (sqlite3_vtab *) v;
- FTSTRACE(("FTS3 Connect %p\n", v));
-
- return rc;
-
-err:
- sqlite3_free(v);
- return rc;
-}
-
-static int fulltextConnect(
- sqlite3 *db,
- void *pAux,
- int argc, const char *const*argv,
- sqlite3_vtab **ppVTab,
- char **pzErr
-){
- const char *zDb, *zName;
-
- zDb = argv[1];
- zName = argv[2];
-
- return constructSqliteVtab(db, pAux, zDb, zName, ppVTab, pzErr);
-}
-
-/* The %_content table holds the text of each document, with
-** the docid column exposed as the SQLite rowid for the table.
-*/
-/* TODO(shess) This comment needs elaboration to match the updated
-** code. Work it into the top-of-file comment at that time.
-*/
-static int createTables(sqlite3 *db, const char *zDb, const char *zName) {
- int rc;
- StringBuffer schema;
- FTSTRACE(("FTS3 Create\n"));
-
- initStringBuffer(&schema);
- append(&schema, "CREATE TABLE %_content(");
- append(&schema, " docid INTEGER PRIMARY KEY");
- append(&schema, ")");
- rc = sql_exec(db, zDb, zName, stringBufferData(&schema));
- stringBufferDestroy(&schema);
- if( rc!=SQLITE_OK ) goto out;
-
- rc = sql_exec(db, zDb, zName,
- "create table %_segments("
- " blockid INTEGER PRIMARY KEY,"
- " block blob"
- ");"
- );
- if( rc!=SQLITE_OK ) goto out;
-
- rc = sql_exec(db, zDb, zName,
- "create table %_segdir("
- " level integer,"
- " idx integer,"
- " start_block integer,"
- " leaves_end_block integer,"
- " end_block integer,"
- " root blob,"
- " primary key(level, idx)"
- ");");
- if( rc!=SQLITE_OK ) goto out;
-
-out:
- return rc;
-}
-
-static int fulltextCreate(sqlite3 *db, void *pAux,
- int argc, const char * const *argv,
- sqlite3_vtab **ppVTab, char **pzErr){
- const char *zDb, *zName;
- FTSTRACE(("FTS3 Create\n"));
-
- zDb = argv[1];
- zName = argv[2];
-
- return constructSqliteVtab(db, pAux, zDb, zName, ppVTab, pzErr);
-}
-
-/* Decide how to handle an SQL query. */
-static int fulltextBestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
- fulltext_vtab *v = get_fulltext_vtab (pVTab);
- int i;
- int iCons = -1; /* Index of constraint to use */
- FTSTRACE(("FTS3 BestIndex\n"));
-
- pInfo->idxNum = QUERY_GENERIC;
- pInfo->estimatedCost = 500000;
- for(i=0; i<pInfo->nConstraint; ++i){
- const struct sqlite3_index_constraint *pConstraint;
- pConstraint = &pInfo->aConstraint[i];
- if( pConstraint->usable ) {
- if( (pConstraint->iColumn==-1 || pConstraint->iColumn==v->nColumn+1) &&
- pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){
- pInfo->idxNum = QUERY_DOCID; /* lookup by docid */
- pInfo->estimatedCost = 1.0;
- iCons = i;
- FTSTRACE(("FTS3 QUERY_DOCID\n"));
- } else if( pConstraint->iColumn>=0 && pConstraint->iColumn<=v->nColumn &&
- pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH ){
- /* full-text search */
- pInfo->idxNum = QUERY_FULLTEXT + pConstraint->iColumn;
- pInfo->estimatedCost = 2.0;
- iCons = i;
- FTSTRACE(("FTS3 QUERY_FULLTEXT %d\n", pConstraint->iColumn));
- break;
- }
- }
- }
-
- if( iCons>=0 ){
- pInfo->aConstraintUsage[iCons].argvIndex = 1;
- pInfo->aConstraintUsage[iCons].omit = 1;
- }
- return SQLITE_OK;
-}
-
-static int fulltextDisconnect(sqlite3_vtab *pVTab){
- FTSTRACE(("FTS3 Disconnect %p\n", pVTab));
- sqlite3_free (pVTab);
- return SQLITE_OK;
-}
-
-static int fulltextDestroy(sqlite3_vtab *pVTab){
- fulltext_vtab *v = get_fulltext_vtab (pVTab);
- int rc;
-
- FTSTRACE(("FTS3 Destroy %p\n", pVTab));
- rc = sql_exec(v->db, v->zDb, v->zName,
- "drop table if exists %_content;"
- "drop table if exists %_segments;"
- "drop table if exists %_segdir;"
- );
- if( rc!=SQLITE_OK ) return rc;
-
- sqlite3_free (pVTab);
- return SQLITE_OK;
-}
-
-static int fulltextOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
- fulltext_cursor *c;
-
- /* fulltext_vtab *v = (fulltext_vtab *)pVTab; */
- c = (fulltext_cursor *) sqlite3_malloc(sizeof(fulltext_cursor));
- if( c ){
- memset(c, 0, sizeof(fulltext_cursor));
- /* sqlite will initialize c->base */
- *ppCursor = &c->base;
-
- c->offsets = g_string_new ("");
-
- FTSTRACE(("FTS3 Open %p: %p\n", pVTab, c));
- return SQLITE_OK;
- }else{
- return SQLITE_NOMEM;
- }
-}
-
-
-/* Free all of the dynamically allocated memory held by *q
-*/
-static void queryClear(Query *q){
- int i;
- for(i = 0; i < q->nTerms; ++i){
- sqlite3_free(q->pTerms[i].pTerm);
- }
- sqlite3_free(q->pTerms);
- CLEAR(q);
-}
-
-/* Free all of the dynamically allocated memory held by the
-** Snippet
-*/
-static void snippetClear(Snippet *p){
- sqlite3_free(p->aMatch);
- sqlite3_free(p->zOffset);
- sqlite3_free(p->zSnippet);
- CLEAR(p);
-}
-/*
-** Append a single entry to the p->aMatch[] log.
-*/
-static void snippetAppendMatch(
- Snippet *p, /* Append the entry to this snippet */
- int iCol, int iTerm, /* The column and query term */
- int iToken, /* Matching token in document */
- int iStart, int nByte /* Offset and size of the match */
-){
- int i;
- struct snippetMatch *pMatch;
- if( p->nMatch+1>=p->nAlloc ){
- p->nAlloc = p->nAlloc*2 + 10;
- p->aMatch = sqlite3_realloc(p->aMatch, p->nAlloc*sizeof(p->aMatch[0]) );
- if( p->aMatch==0 ){
- p->nMatch = 0;
- p->nAlloc = 0;
- return;
- }
- }
- i = p->nMatch++;
- pMatch = &p->aMatch[i];
- pMatch->iCol = iCol;
- pMatch->iTerm = iTerm;
- pMatch->iToken = iToken;
- pMatch->iStart = iStart;
- pMatch->nByte = nByte;
-}
-
-
-/*
-** Sizing information for the circular buffer used in snippetOffsetsOfColumn()
-*/
-#define FTS3_ROTOR_SZ (32)
-#define FTS3_ROTOR_MASK (FTS3_ROTOR_SZ-1)
-
-/*
-** Add entries to pSnippet->aMatch[] for every match that occurs against
-** document zDoc[0..nDoc-1] which is stored in column iColumn.
-*/
-static void snippetOffsetsOfColumn(
- Query *pQuery,
- Snippet *pSnippet,
- int iColumn,
- const char *zDoc,
- int nDoc
-#ifdef STORE_CATEGORY
- , int position
-#endif
-){
-
- fulltext_vtab *pVtab; /* The full text index */
- int nColumn; /* Number of columns in the index */
- const QueryTerm *aTerm; /* Query string terms */
- int nTerm; /* Number of query string terms */
- int i, j; /* Loop counters */
- unsigned int match, prevMatch; /* Phrase search bitmasks */
- const char *zToken; /* Next token from the tokenizer */
- int nToken; /* Size of zToken */
- int iBegin, iEnd, iPos; /* Offsets of beginning and end */
- gboolean stop_word;
-
- /* The following variables keep a circular buffer of the last
- ** few tokens */
- unsigned int iRotor = 0; /* Index of current token */
- int iRotorBegin[FTS3_ROTOR_SZ]; /* Beginning offset of token */
- int iRotorLen[FTS3_ROTOR_SZ]; /* Length of token */
- int nWords;
-
- pVtab = pQuery->pFts;
- nColumn = pVtab->nColumn;
-
- FTSTRACE (("FTS parsing started for Snippets, limiting '%d' bytes to '%d' words",
- nDoc, pVtab->max_words));
-
- tracker_parser_reset (pVtab->parser,
- zDoc,
- nDoc,
- pVtab->max_word_length,
- pVtab->enable_stemmer,
- pVtab->enable_unaccent,
- pVtab->ignore_stop_words,
- TRUE,
- pVtab->ignore_numbers);
-
- aTerm = pQuery->pTerms;
- nTerm = pQuery->nTerms;
-
- if( nTerm>=FTS3_ROTOR_SZ ){
- nTerm = FTS3_ROTOR_SZ - 1;
- }
-
- prevMatch = 0;
- nWords = 0;
- while(nWords < pVtab->max_words){
-// rc = pTModule->xNext(pTCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos);
-
-
- zToken = tracker_parser_next (pVtab->parser,
- &iPos,
- &iBegin,
- &iEnd,
- &stop_word,
- &nToken);
-
- if (!zToken) break;
-
- if (pVtab->ignore_stop_words && stop_word) {
- continue;
- }
-
- nWords++;
-
- iRotorBegin[iRotor&FTS3_ROTOR_MASK] = iBegin;
- iRotorLen[iRotor&FTS3_ROTOR_MASK] = iEnd-iBegin;
- match = 0;
- for(i=0; i<nTerm; i++){
- int iCol;
- iCol = aTerm[i].iColumn;
- if( iCol>=0 && iCol<nColumn && iCol!=iColumn ) continue;
- if( aTerm[i].nTerm>nToken ) continue;
- if( !aTerm[i].isPrefix && aTerm[i].nTerm<nToken ) continue;
- assert( aTerm[i].nTerm<=nToken );
- if( memcmp(aTerm[i].pTerm, zToken, aTerm[i].nTerm) ) continue;
- if( aTerm[i].iPhrase>1 && (prevMatch & (1<<i))==0 ) continue;
- match |= 1<<i;
- if( i==nTerm-1 || aTerm[i+1].iPhrase==1 ){
- for(j=aTerm[i].iPhrase-1; j>=0; j--){
- int k = (iRotor-j) & FTS3_ROTOR_MASK;
- snippetAppendMatch(pSnippet, iColumn, i-j, iPos-j,
- iRotorBegin[k], iRotorLen[k]);
- }
- }
- }
- prevMatch = match<<1;
- iRotor++;
- }
-// pTModule->xClose(pTCursor);
-}
-
-/*
-** Remove entries from the pSnippet structure to account for the NEAR
-** operator. When this is called, pSnippet contains the list of token
-** offsets produced by treating all NEAR operators as AND operators.
-** This function removes any entries that should not be present after
-** accounting for the NEAR restriction. For example, if the queried
-** document is:
-**
-** "A B C D E A"
-**
-** and the query is:
-**
-** A NEAR/0 E
-**
-** then when this function is called the Snippet contains token offsets
-** 0, 4 and 5. This function removes the "0" entry (because the first A
-** is not near enough to an E).
-*/
-static void trimSnippetOffsetsForNear(Query *pQuery, Snippet *pSnippet){
- int ii;
- int iDir = 1;
-
- while(iDir>-2) {
- assert( iDir==1 || iDir==-1 );
- for(ii=0; ii<pSnippet->nMatch; ii++){
- int jj;
- int nNear;
- struct snippetMatch *pMatch = &pSnippet->aMatch[ii];
- QueryTerm *pQueryTerm = &pQuery->pTerms[pMatch->iTerm];
-
- if( (pMatch->iTerm+iDir)<0
- || (pMatch->iTerm+iDir)>=pQuery->nTerms
- ){
- continue;
- }
-
- nNear = pQueryTerm->nNear;
- if( iDir<0 ){
- nNear = pQueryTerm[-1].nNear;
- }
-
- if( pMatch->iTerm>=0 && nNear ){
- int isOk = 0;
- int iNextTerm = pMatch->iTerm+iDir;
- int iPrevTerm = iNextTerm;
-
- int iEndToken;
- int iStartToken;
-
- if( iDir<0 ){
- int nPhrase = 1;
- iStartToken = pMatch->iToken;
- while( (pMatch->iTerm+nPhrase)<pQuery->nTerms
- && pQuery->pTerms[pMatch->iTerm+nPhrase].iPhrase>1
- ){
- nPhrase++;
- }
- iEndToken = iStartToken + nPhrase - 1;
- }else{
- iEndToken = pMatch->iToken;
- iStartToken = pMatch->iToken+1-pQueryTerm->iPhrase;
- }
-
- while( pQuery->pTerms[iNextTerm].iPhrase>1 ){
- iNextTerm--;
- }
- while( (iPrevTerm+1)<pQuery->nTerms &&
- pQuery->pTerms[iPrevTerm+1].iPhrase>1
- ){
- iPrevTerm++;
- }
-
- for(jj=0; isOk==0 && jj<pSnippet->nMatch; jj++){
- struct snippetMatch *p = &pSnippet->aMatch[jj];
- if( p->iCol==pMatch->iCol && ((
- p->iTerm==iNextTerm &&
- p->iToken>iEndToken &&
- p->iToken<=iEndToken+nNear
- ) || (
- p->iTerm==iPrevTerm &&
- p->iToken<iStartToken &&
- p->iToken>=iStartToken-nNear
- ))){
- isOk = 1;
- }
- }
- if( !isOk ){
- for(jj=1-pQueryTerm->iPhrase; jj<=0; jj++){
- pMatch[jj].iTerm = -1;
- }
- ii = -1;
- iDir = 1;
- }
- }
- }
- iDir -= 2;
- }
-}
-
-
-
-/*
-** Compute all offsets for the current row of the query.
-** If the offsets have already been computed, this routine is a no-op.
-*/
-static void snippetAllOffsets(fulltext_cursor *p){
- int iColumn, i;
- fulltext_vtab *pFts;
-
-#ifndef STORE_CATEGORY
- int iFirst, iLast;
- int nColumn;
-#endif
-
- if( p->snippet.nMatch ) return;
- if( p->q.nTerms==0 ) return;
- pFts = p->q.pFts;
-
-#ifdef STORE_CATEGORY
- PLReader plReader;
- int col_array[255];
- gpointer pos_array[255];
-
- for (i=0; i<255; i++) {
- col_array[i] = 0;
- pos_array[i] = NULL;
- }
-
-
- int iPos = 0;
-
- if (dlrAtEnd (&p->reader)) return;
-
- plrInit(&plReader, &p->reader);
-
- if (plrAtEnd(&plReader)) return;
-
- iColumn = -1;
-
- for ( ; !plrAtEnd(&plReader); plrStep(&plReader) ){
-
- if (plrColumn (&plReader) != iColumn) {
-
- iColumn = plrColumn(&plReader);
- col_array[iColumn] += 1;
- }
-
- iPos = plrPosition(&plReader);
- GSList *l = pos_array[iColumn];
- l = g_slist_prepend (l, GINT_TO_POINTER (iPos));
- }
-
- plrEndAndDestroy(&plReader);
-
-
- /* get the column with most hits */
- int hit_column = 0;
- int hit_column_count = col_array[0];
-
- /*bias field id 0 more as its the main content field */
- // if (hit_column_count > 0) hit_column_count++;
-
- for (i=1; i<255; i++) {
- if (col_array [i] > hit_column_count) {
- hit_column = i;
- hit_column_count =col_array[i];
- }
-
- g_slist_free (pos_array[i]);
- }
-
-
- const char *zDoc;
- int nDoc;
- zDoc = (const char*)sqlite3_column_text(p->pStmt, hit_column+1);
- nDoc = sqlite3_column_bytes(p->pStmt, hit_column+1);
- snippetOffsetsOfColumn(&p->q, &p->snippet, hit_column, zDoc, nDoc, iPos);
-
-
-#else
-
- nColumn = pFts->nColumn;
- iColumn = (p->iCursorType - QUERY_FULLTEXT);
- if( iColumn<0 || iColumn>=nColumn ){
- iFirst = 0;
- iLast = nColumn-1;
- }else{
- iFirst = iColumn;
- iLast = iColumn;
- }
-
-
- for(i=iFirst; i<=iLast; i++){
- const char *zDoc;
- int nDoc;
- zDoc = (const char*)sqlite3_column_text(p->pStmt, i+1);
- nDoc = sqlite3_column_bytes(p->pStmt, i+1);
- snippetOffsetsOfColumn(&p->q, &p->snippet, i, zDoc, nDoc);
- }
-#endif
-
- trimSnippetOffsetsForNear(&p->q, &p->snippet);
-}
-
-#if 0
-/*
-** Convert the information in the aMatch[] array of the snippet
-** into the string zOffset[0..nOffset-1]. This string is used as
-** the return of the SQL offsets() function.
-*/
-static void snippetOffsetText(Snippet *p){
- int i;
- int cnt = 0;
- StringBuffer sb;
- char zBuf[200];
- if( p->zOffset ) return;
- initStringBuffer(&sb);
- for(i=0; i<p->nMatch; i++){
- struct snippetMatch *pMatch = &p->aMatch[i];
- if( pMatch->iTerm>=0 ){
- /* If snippetMatch.iTerm is less than 0, then the match was
- ** discarded as part of processing the NEAR operator (see the
- ** trimSnippetOffsetsForNear() function for details). Ignore
- ** it in this case
- */
- zBuf[0] = ' ';
- sqlite3_snprintf(sizeof(zBuf)-1, &zBuf[cnt>0], "%d %d %d %d",
- pMatch->iCol, pMatch->iTerm, pMatch->iStart, pMatch->nByte);
- append(&sb, zBuf);
- cnt++;
- }
- }
- p->zOffset = stringBufferData(&sb);
- p->nOffset = stringBufferLength(&sb);
-}
-#endif
-
-/*
-** zDoc[0..nDoc-1] is phrase of text. aMatch[0..nMatch-1] are a set
-** of matching words some of which might be in zDoc. zDoc is column
-** number iCol.
-**
-** iBreak is suggested spot in zDoc where we could begin or end an
-** excerpt. Return a value similar to iBreak but possibly adjusted
-** to be a little left or right so that the break point is better.
-*/
-static int wordBoundary(
- int iBreak, /* The suggested break point */
- const char *zDoc, /* Document text */
- int nDoc, /* Number of bytes in zDoc[] */
- struct snippetMatch *aMatch, /* Matching words */
- int nMatch, /* Number of entries in aMatch[] */
- int iCol /* The column number for zDoc[] */
-){
- int i;
- if( iBreak<=10 ){
- return 0;
- }
- if( iBreak>=nDoc-10 ){
- return nDoc;
- }
- for(i=0; i<nMatch && aMatch[i].iCol<iCol; i++){}
- while( i<nMatch && aMatch[i].iStart+aMatch[i].nByte<iBreak ){ i++; }
- if( i<nMatch ){
- if( aMatch[i].iStart<iBreak+10 ){
- return aMatch[i].iStart;
- }
- if( i>0 && aMatch[i-1].iStart+aMatch[i-1].nByte>=iBreak ){
- return aMatch[i-1].iStart;
- }
- }
- for(i=1; i<=10; i++){
- if( safe_isspace(zDoc[iBreak-i]) ){
- return iBreak - i + 1;
- }
- if( safe_isspace(zDoc[iBreak+i]) ){
- return iBreak + i + 1;
- }
- }
- return iBreak;
-}
-
-
-
-/*
-** Allowed values for Snippet.aMatch[].snStatus
-*/
-#define SNIPPET_IGNORE 0 /* It is ok to omit this match from the snippet */
-#define SNIPPET_DESIRED 1 /* We want to include this match in the snippet */
-
-/*
-** Generate the text of a snippet.
-*/
-static void snippetText(
- fulltext_cursor *pCursor, /* The cursor we need the snippet for */
- const char *zStartMark, /* Markup to appear before each match */
- const char *zEndMark, /* Markup to appear after each match */
- const char *zEllipsis /* Ellipsis mark */
-){
- int i, j;
- struct snippetMatch *aMatch;
- int nMatch;
- int nDesired;
- StringBuffer sb;
- int tailCol;
- int tailOffset;
- int iCol;
- int nDoc;
- const char *zDoc;
- int iStart, iEnd;
- int tailEllipsis = 0;
- int iMatch;
-
-
- sqlite3_free(pCursor->snippet.zSnippet);
- pCursor->snippet.zSnippet = 0;
- aMatch = pCursor->snippet.aMatch;
- nMatch = pCursor->snippet.nMatch;
- initStringBuffer(&sb);
-
- for(i=0; i<nMatch; i++){
- aMatch[i].snStatus = SNIPPET_IGNORE;
- }
- nDesired = 0;
- for(i=0; i<pCursor->q.nTerms; i++){
- for(j=0; j<nMatch; j++){
- if( aMatch[j].iTerm==i ){
- aMatch[j].snStatus = SNIPPET_DESIRED;
- nDesired++;
- break;
- }
- }
- }
-
- iMatch = 0;
- tailCol = -1;
- tailOffset = 0;
- for(i=0; i<nMatch && nDesired>0; i++){
- if( aMatch[i].snStatus!=SNIPPET_DESIRED ) continue;
- nDesired--;
- iCol = aMatch[i].iCol;
- zDoc = (const char*)sqlite3_column_text(pCursor->pStmt, iCol+1);
- nDoc = sqlite3_column_bytes(pCursor->pStmt, iCol+1);
- iStart = aMatch[i].iStart - 40;
- iStart = wordBoundary(iStart, zDoc, nDoc, aMatch, nMatch, iCol);
- if( iStart<=10 ){
- iStart = 0;
- }
- if( iCol==tailCol && iStart<=tailOffset+20 ){
- iStart = tailOffset;
- }
- if( (iCol!=tailCol && tailCol>=0) || iStart!=tailOffset ){
- trimWhiteSpace(&sb);
- appendWhiteSpace(&sb);
- append(&sb, zEllipsis);
- appendWhiteSpace(&sb);
- }
- iEnd = aMatch[i].iStart + aMatch[i].nByte + 40;
- iEnd = wordBoundary(iEnd, zDoc, nDoc, aMatch, nMatch, iCol);
- if( iEnd>=nDoc-10 ){
- iEnd = nDoc;
- tailEllipsis = 0;
- }else{
- tailEllipsis = 1;
- }
- while( iMatch<nMatch && aMatch[iMatch].iCol<iCol ){ iMatch++; }
- while( iStart<iEnd ){
- while( iMatch<nMatch && aMatch[iMatch].iStart<iStart
- && aMatch[iMatch].iCol<=iCol ){
- iMatch++;
- }
- if( iMatch<nMatch && aMatch[iMatch].iStart<iEnd
- && aMatch[iMatch].iCol==iCol ){
- nappend(&sb, &zDoc[iStart], aMatch[iMatch].iStart - iStart);
- iStart = aMatch[iMatch].iStart;
- append(&sb, zStartMark);
- nappend(&sb, &zDoc[iStart], aMatch[iMatch].nByte);
- append(&sb, zEndMark);
- iStart += aMatch[iMatch].nByte;
- for(j=iMatch+1; j<nMatch; j++){
- if( aMatch[j].iTerm==aMatch[iMatch].iTerm
- && aMatch[j].snStatus==SNIPPET_DESIRED ){
- nDesired--;
- aMatch[j].snStatus = SNIPPET_IGNORE;
- }
- }
- }else{
- nappend(&sb, &zDoc[iStart], iEnd - iStart);
- iStart = iEnd;
- }
- }
- tailCol = iCol;
- tailOffset = iEnd;
- }
- trimWhiteSpace(&sb);
- if( tailEllipsis ){
- appendWhiteSpace(&sb);
- append(&sb, zEllipsis);
- }
- pCursor->snippet.zSnippet = stringBufferData(&sb);
- pCursor->snippet.nSnippet = stringBufferLength(&sb);
-}
-
-
-/*
-** Close the cursor. For additional information see the documentation
-** on the xClose method of the virtual table interface.
-*/
-static int fulltextClose(sqlite3_vtab_cursor *pCursor){
- fulltext_cursor *c = (fulltext_cursor *) pCursor;
- FTSTRACE(("FTS3 Close %p\n", c));
- sqlite3_finalize(c->pStmt);
- queryClear(&c->q);
- snippetClear(&c->snippet);
- g_string_free (c->offsets, TRUE);
- if( c->result.nData!=0 ) dlrDestroy(&c->reader);
- dataBufferDestroy(&c->result);
- sqlite3_free(c);
- return SQLITE_OK;
-}
-
-static int fulltextNext(sqlite3_vtab_cursor *pCursor){
- fulltext_cursor *c = (fulltext_cursor *) pCursor;
- fulltext_vtab *v = cursor_vtab (c);
- int rc;
- PLReader plReader;
- gboolean first_pos = TRUE;
-
- FTSTRACE(("FTS3 Next %p\n", pCursor));
- snippetClear(&c->snippet);
- if( c->iCursorType < QUERY_FULLTEXT ){
- /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
- rc = sqlite3_step(c->pStmt);
- switch( rc ){
- case SQLITE_ROW:
- c->eof = 0;
- return SQLITE_OK;
- case SQLITE_DONE:
- c->eof = 1;
- return SQLITE_OK;
- default:
- c->eof = 1;
- return rc;
- }
- } else { /* full-text query */
- rc = sqlite3_reset(c->pStmt);
- if( rc!=SQLITE_OK ) return rc;
-
- if( c->result.nData==0 || dlrAtEnd(&c->reader) ){
- c->eof = 1;
- return SQLITE_OK;
- }
- c->currentDocid = dlrDocid(&c->reader);
-#ifdef STORE_CATEGORY
- c->currentCatid = dlrCatid(&c->reader);
-#endif
-
- /* (tracker) read position offsets here */
-
-
- c->offsets = g_string_assign (c->offsets, "");
- c->rank = 0;
-
- plrInit(&plReader, &c->reader);
-
-
- for ( ; !plrAtEnd(&plReader); plrStep(&plReader) ){
- gchar *uri = NULL;
- int col = plrColumn (&plReader);
-
- c->rank += get_metadata_weight (v, col, &uri);
-
- if (uri && first_pos) {
- g_string_append_printf (c->offsets, "%s,%d", uri, plrPosition (&plReader));
- first_pos = FALSE;
- } else if (uri) {
- g_string_append_printf (c->offsets, ",%s,%d", uri, plrPosition (&plReader));
- } else {
- g_warning ("Type '%d' for FTS offset doesn't exist in ontology", col);
- }
-
- g_free (uri);
- }
-
- plrDestroy(&plReader);
-
- dlrStep(&c->reader);
-
- c->eof = 0;
- return SQLITE_OK;
- }
-}
-
-
-/* TODO(shess) If we pushed LeafReader to the top of the file, or to
-** another file, term_select() could be pushed above
-** docListOfTerm().
-*/
-static int termSelect(fulltext_vtab *v, int iColumn,
- const char *pTerm, int nTerm, int isPrefix,
- DocListType iType, DataBuffer *out);
-
-/*
-** Return a DocList corresponding to the phrase *pPhrase.
-**
-** The resulting DL_DOCIDS doclist is stored in pResult, which is
-** overwritten.
-*/
-static int docListOfTerm(
- fulltext_vtab *v, /* The full text index */
- int iColumn, /* column to restrict to. No restriction if >=nColumn */
- QueryTerm *pQTerm, /* Term we are looking for, or 1st term of a phrase */
- DataBuffer *pResult /* Write the result here */
-){
- DataBuffer left, right, new;
- int i, rc;
-
- /* No phrase search if no position info. */
- assert( pQTerm->nPhrase==0 || DL_DEFAULT!=DL_DOCIDS );
-
- /* This code should never be called with buffered updates. */
- assert( v->nPendingData<0 );
-
- dataBufferInit(&left, 0);
-
- #ifdef RANK
- rc = termSelect(v, iColumn, pQTerm->pTerm, pQTerm->nTerm, pQTerm->isPrefix,
- DL_POSITIONS, &left);
- #else
- rc = termSelect(v, iColumn, pQTerm->pTerm, pQTerm->nTerm, pQTerm->isPrefix,
- (0<pQTerm->nPhrase ? DL_POSITIONS : DL_DOCIDS), &left);
- #endif
-
- if( rc ) return rc;
- for(i=1; i<=pQTerm->nPhrase && left.nData>0; i++){
- /* If this token is connected to the next by a NEAR operator, and
- ** the next token is the start of a phrase, then set nPhraseRight
- ** to the number of tokens in the phrase. Otherwise leave it at 1.
- */
- int nPhraseRight = 1;
- while( (i+nPhraseRight)<=pQTerm->nPhrase
- && pQTerm[i+nPhraseRight].nNear==0
- ){
- nPhraseRight++;
- }
-
- dataBufferInit(&right, 0);
- rc = termSelect(v, iColumn, pQTerm[i].pTerm, pQTerm[i].nTerm,
- pQTerm[i].isPrefix, DL_POSITIONS, &right);
- if( rc ){
- dataBufferDestroy(&left);
- return rc;
- }
- dataBufferInit(&new, 0);
-
- #ifdef RANK
- docListPhraseMerge(left.pData, left.nData, right.pData, right.nData,
- pQTerm[i-1].nNear, pQTerm[i-1].iPhrase + nPhraseRight,
- DL_POSITIONS,
- &new, i);
-
- #else
- docListPhraseMerge(left.pData, left.nData, right.pData, right.nData,
- pQTerm[i-1].nNear, pQTerm[i-1].iPhrase + nPhraseRight,
- ((i<pQTerm->nPhrase) ? DL_POSITIONS : DL_DOCIDS),
- &new, i);
-
- #endif
- dataBufferDestroy(&left);
- dataBufferDestroy(&right);
- left = new;
- }
- *pResult = left;
- return SQLITE_OK;
-}
-
-/* Add a new term pTerm[0..nTerm-1] to the query *q.
-*/
-static void queryAdd(Query *q, const char *pTerm, int nTerm){
- QueryTerm *t;
- ++q->nTerms;
- q->pTerms = sqlite3_realloc(q->pTerms, q->nTerms * sizeof(q->pTerms[0]));
- if( q->pTerms==0 ){
- q->nTerms = 0;
- return;
- }
- t = &q->pTerms[q->nTerms - 1];
- CLEAR(t);
- t->pTerm = sqlite3_malloc(nTerm+1);
- memcpy(t->pTerm, pTerm, nTerm);
- t->pTerm[nTerm] = 0;
- t->nTerm = nTerm;
- t->isOr = q->nextIsOr;
- t->isPrefix = 0;
- q->nextIsOr = 0;
- t->iColumn = q->nextColumn;
- q->nextColumn = q->dfltColumn;
-}
-
-#if 0
-/*
-** Check to see if the string zToken[0...nToken-1] matches any
-** column name in the virtual table. If it does,
-** return the zero-indexed column number. If not, return -1.
-*/
-static int checkColumnSpecifier(
- fulltext_vtab *pVtab, /* The virtual table */
- const char *zToken, /* Text of the token */
- int nToken /* Number of characters in the token */
-){
- int i;
- for(i=0; i<pVtab->nColumn; i++){
- if( memcmp(pVtab->azColumn[i], zToken, nToken)==0
- && pVtab->azColumn[i][nToken]==0 ){
- return i;
- }
- }
- return -1;
-}
-#endif
-
-/*
-** Parse the text at pSegment[0..nSegment-1]. Add additional terms
-** to the query being assemblied in pQuery.
-**
-** inPhrase is true if pSegment[0..nSegement-1] is contained within
-** double-quotes. If inPhrase is true, then the first term
-** is marked with the number of terms in the phrase less one and
-** OR and "-" syntax is ignored. If inPhrase is false, then every
-** term found is marked with nPhrase=0 and OR and "-" syntax is significant.
-*/
-static int tokenizeSegment(
- fulltext_vtab *v, /* The tokenizer to use */
- const char *pSegment, int nSegment, /* Query expression being parsed */
- int inPhrase, /* True if within "..." */
- Query *pQuery /* Append results here */
-){
- TrackerParser *parser = v->parser;
- int firstIndex = pQuery->nTerms;
- int nTerm = 1;
- int nWords;
- int last_near_offset = -1;
-
- FTSTRACE (("FTS parsing started for Segments, limiting '%d' bytes to '%d' words",
- nSegment, v->max_words));
-
- tracker_parser_reset (parser,
- pSegment,
- nSegment,
- v->max_word_length,
- v->enable_stemmer,
- v->enable_unaccent,
- v->ignore_stop_words,
- FALSE,
- v->ignore_numbers);
-
- nWords = 0;
- while(nWords < v->max_words){
- const char *pToken;
- int nToken, iBegin, iEnd, iPos, stop_word;
-
-
- pToken = tracker_parser_next (parser, &iPos,
- &iBegin,
- &iEnd,
- &stop_word,
- &nToken);
- if (!pToken) {
- break;
- }
-
- if (last_near_offset > 0 && iEnd <= last_near_offset) {
- /* Skip this word */
- continue;
- }
-
- nWords ++;
-
-// printf("token being indexed is %s, pos is %d, begin is %d, end is %d and length is %d\n", pToken, iPos, iBegin, iEnd, nToken);
-
-#if 0
- if( !inPhrase &&
- pSegment[iEnd]==':') {
-
- int len = iEnd - iBegin;
- char *field = g_strndup (pSegment + iBegin, len);
-
- // printf ("field is %s\n", field);
-
- if ((iCol = checkColumnSpecifier(pQuery->pFts, field, len))>=0 ){
- pQuery->nextColumn = iCol;
- g_free (field);
- continue;
- }
- }
-#endif
-
- if( !inPhrase && pQuery->nTerms>0 && nToken==2
- && pToken[0] == 'o' && pToken[1] == 'r'
- ){
- pQuery->nextIsOr = 1;
- continue;
- }
- if( !inPhrase && pQuery->nTerms>0 && !pQuery->nextIsOr && nToken==4
- && pToken[0]=='n'
- && pToken[1]=='e'
- && pToken[2]=='a'
- && pToken[3]=='r'
- ){
- QueryTerm *pTerm;
-
- /* Make sure pQuery->pTerms is non-NULL */
- g_return_val_if_fail (pQuery->pTerms, SQLITE_ERROR);
-
- pTerm = &pQuery->pTerms[pQuery->nTerms-1];
- if( (iBegin+6)<nSegment
- && pSegment[iBegin+4] == '/'
- && pSegment[iBegin+5]>='0' && pSegment[iBegin+5]<='9'
- ){
- pTerm->nNear = (pSegment[iBegin+5] - '0');
- nToken += 2;
- if( pSegment[iBegin+6]>='0' && pSegment[iBegin+6]<='9' ){
- pTerm->nNear = pTerm->nNear * 10 + (pSegment[iBegin+6] - '0');
- nToken++;
- }
-
- /* Set last near offset, so that any new word that comes
- * after the near which ends before this last near offset is
- * discarded. This is done because NEAR/10 will actually get
- * split into 3 words, "NEAR", "/" and "10" */
- last_near_offset = iBegin + nToken;
- } else {
- pTerm->nNear = SQLITE_FTS3_DEFAULT_NEAR_PARAM;
- }
- pTerm->nNear++;
- continue;
- }
-
- /* If prefix search ignore the word length limit and stop words */
- if (!(iEnd<nSegment && pSegment[iEnd] == '*')) {
- if (nToken < v->min_word_length) {
- continue;
- }
- if (v->ignore_stop_words && stop_word) {
- continue;
- }
- }
-
- queryAdd(pQuery, pToken, nToken);
-
- /* After queryAdd, make sure pQuery->pTerms is non-NULL */
- g_return_val_if_fail (pQuery->pTerms, SQLITE_ERROR);
-
- if( !inPhrase && iBegin>0) {
-
- // printf("first char is %c, prev char is %c\n", pSegment[iBegin], pSegment[iBegin-1]);
-
- if (pSegment[iBegin-1]=='-' ){
- pQuery->pTerms[pQuery->nTerms-1].isNot = 1;
- }
- }
- if( iEnd<nSegment && pSegment[iEnd]=='*' ){
- pQuery->pTerms[pQuery->nTerms-1].isPrefix = 1;
- }
- pQuery->pTerms[pQuery->nTerms-1].iPhrase = nTerm;
- if( inPhrase ){
- nTerm++;
- }
- }
-
- if( inPhrase && pQuery->nTerms>firstIndex ){
- g_return_val_if_fail (pQuery->pTerms, SQLITE_ERROR);
- pQuery->pTerms[firstIndex].nPhrase = pQuery->nTerms - firstIndex - 1;
- }
-
- return SQLITE_OK;
-}
-
-/* Parse a query string, yielding a Query object pQuery.
-**
-** The calling function will need to queryClear() to clean up
-** the dynamically allocated memory held by pQuery.
-*/
-static int parseQuery(
- fulltext_vtab *v, /* The fulltext index */
- const char *zInput, /* Input text of the query string */
- int nInput, /* Size of the input text */
- int dfltColumn, /* Default column of the index to match against */
- Query *pQuery /* Write the parse results here. */
-){
- int iInput, inPhrase = 0;
-
- if( zInput==0 ) nInput = 0;
- if( nInput<0 ) nInput = strlen(zInput);
- pQuery->nTerms = 0;
- pQuery->pTerms = NULL;
- pQuery->nextIsOr = 0;
- pQuery->nextColumn = dfltColumn;
- pQuery->dfltColumn = dfltColumn;
- pQuery->pFts = v;
-
- for(iInput=0; iInput<nInput; ++iInput){
- int i;
- for(i=iInput; i<nInput && zInput[i]!='"'; ++i){}
- if( i>iInput ){
- tokenizeSegment(v, zInput+iInput, i-iInput, inPhrase,
- pQuery);
- }
- iInput = i;
- if( i<nInput ){
- assert( zInput[i]=='"' );
- inPhrase = !inPhrase;
- }
- }
-
- if( inPhrase ){
- /* unmatched quote
- proceed as if there is an extra quote at the end
- do not report error as this may be user input */
- }
-
- if (pQuery->pTerms) {
- QueryTerm *aTerm;
- int ii;
-
- /* Modify the values of the QueryTerm.nPhrase variables to account for
- ** the NEAR operator. For the purposes of QueryTerm.nPhrase, phrases
- ** and tokens connected by the NEAR operator are handled as a single
- ** phrase. See comments above the QueryTerm structure for details.
- */
- aTerm = pQuery->pTerms;
- for(ii=0; ii<pQuery->nTerms; ii++){
- if( aTerm[ii].nNear || aTerm[ii].nPhrase ){
- while (aTerm[ii+aTerm[ii].nPhrase].nNear) {
- aTerm[ii].nPhrase += (1 + aTerm[ii+aTerm[ii].nPhrase+1].nPhrase);
- }
- }
-#if PRINT_PARSED_QUERY
- g_debug ("\n"
- "[Term %d] '%s' (%d)\n"
- " nPhrase: %d\n"
- " iPhrase: %d\n"
- " iColumn: %d\n"
- " nNear: %d\n"
- " isOr: %s\n"
- " isNot: %s\n"
- " isPrefix: %s\n",
- ii, aTerm[ii].pTerm, aTerm[ii].nTerm,
- aTerm[ii].nPhrase,
- aTerm[ii].iPhrase,
- aTerm[ii].iColumn,
- aTerm[ii].nNear,
- aTerm[ii].isOr ? "yes" : "no",
- aTerm[ii].isNot ? "yes" : "no",
- aTerm[ii].isPrefix ? "yes" : "no");
-#endif /* PRINT_PARSED_QUERY */
- }
- }
-
- return SQLITE_OK;
-}
-
-/* TODO(shess) Refactor the code to remove this forward decl. */
-static int flushPendingTerms(fulltext_vtab *v);
-
-/* Perform a full-text query using the search expression in
-** zInput[0..nInput-1]. Return a list of matching documents
-** in pResult.
-**
-** Queries must match column iColumn. Or if iColumn>=nColumn
-** they are allowed to match against any column.
-*/
-static int fulltextQuery(
- fulltext_vtab *v, /* The full text index */
- int iColumn, /* Match against this column by default */
- const char *zInput, /* The query string */
- int nInput, /* Number of bytes in zInput[] */
- DataBuffer *pResult, /* Write the result doclist here */
- Query *pQuery /* Put parsed query string here */
-){
- int i, iNext, rc;
- DataBuffer left, right, or, new;
- int nNot = 0;
- QueryTerm *aTerm;
-
- /* TODO(shess) Instead of flushing pendingTerms, we could query for
- ** the relevant term and merge the doclist into what we receive from
- ** the database. Wait and see if this is a common issue, first.
- **
- ** A good reason not to flush is to not generate update-related
- ** error codes from here.
- */
-
- /* Flush any buffered updates before executing the query. */
- rc = flushPendingTerms(v);
- if( rc!=SQLITE_OK ) return rc;
-
- /* TODO(shess) I think that the queryClear() calls below are not
- ** necessary, because fulltextClose() already clears the query.
- */
- rc = parseQuery(v, zInput, nInput, iColumn, pQuery);
- if( rc!=SQLITE_OK ) return rc;
-
- /* Empty or NULL queries return no results. */
- if( pQuery->nTerms==0 ){
- dataBufferInit(pResult, 0);
- return SQLITE_OK;
- }
-
- /* Initialize empty buffers */
- dataBufferInit (&left, 0);
- dataBufferInit (&right, 0);
- dataBufferInit (&or, 0);
- dataBufferInit (&new, 0);
-
- /* Merge AND terms. */
- /* TODO(shess) I think we can early-exit if( i>nNot && left.nData==0 ). */
- aTerm = pQuery->pTerms;
- for(i = 0; i<pQuery->nTerms; i=iNext){
- if( aTerm[i].isNot ){
- /* Handle all NOT terms in a separate pass */
- nNot++;
- iNext = i + aTerm[i].nPhrase+1;
- continue;
- }
- iNext = i + aTerm[i].nPhrase + 1;
- rc = docListOfTerm(v, aTerm[i].iColumn, &aTerm[i], &right);
- if( rc ){
- if( i!=nNot ) dataBufferDestroy(&left);
- queryClear(pQuery);
- return rc;
- }
- while( iNext<pQuery->nTerms && aTerm[iNext].isOr ){
- rc = docListOfTerm(v, aTerm[iNext].iColumn, &aTerm[iNext], &or);
- iNext += aTerm[iNext].nPhrase + 1;
- if( rc ){
- if( i!=nNot ) dataBufferDestroy(&left);
- dataBufferDestroy(&right);
- queryClear(pQuery);
- return rc;
- }
- dataBufferInit(&new, 0);
- docListOrMerge(right.pData, right.nData, or.pData, or.nData, &new);
- dataBufferDestroy(&right);
- dataBufferDestroy(&or);
- right = new;
- }
- if( i==nNot ){ /* first term processed. */
- left = right;
- }else{
- dataBufferInit(&new, 0);
- docListAndMerge(left.pData, left.nData, right.pData, right.nData, &new);
- dataBufferDestroy(&right);
- dataBufferDestroy(&left);
- left = new;
- }
- }
-
- if( nNot==pQuery->nTerms ){
- /* We do not yet know how to handle a query of only NOT terms */
- return SQLITE_ERROR;
- }
-
- /* Do the EXCEPT terms */
- for(i=0; i<pQuery->nTerms; i += aTerm[i].nPhrase + 1){
- if( !aTerm[i].isNot ) continue;
- rc = docListOfTerm(v, aTerm[i].iColumn, &aTerm[i], &right);
- if( rc ){
- queryClear(pQuery);
- dataBufferDestroy(&left);
- return rc;
- }
- dataBufferInit(&new, 0);
- docListExceptMerge(left.pData, left.nData, right.pData, right.nData, &new);
- dataBufferDestroy(&right);
- dataBufferDestroy(&left);
- left = new;
- }
-
- *pResult = left;
- return rc;
-}
-
-/*
-** This is the xFilter interface for the virtual table. See
-** the virtual table xFilter method documentation for additional
-** information.
-**
-** If idxNum==QUERY_GENERIC then do a full table scan against
-** the %_content table.
-**
-** If idxNum==QUERY_DOCID then do a docid lookup for a single entry
-** in the %_content table.
-**
-** If idxNum>=QUERY_FULLTEXT then use the full text index. The
-** column on the left-hand side of the MATCH operator is column
-** number idxNum-QUERY_FULLTEXT, 0 indexed. argv[0] is the right-hand
-** side of the MATCH operator.
-*/
-/* TODO(shess) Upgrade the cursor initialization and destruction to
-** account for fulltextFilter() being called multiple times on the
-** same cursor. The current solution is very fragile. Apply fix to
-** fts3 as appropriate.
-*/
-static int fulltextFilter(
- sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
- int idxNum, const char *idxStr, /* Which indexing scheme to use */
- int argc, sqlite3_value **argv /* Arguments for the indexing scheme */
-){
- fulltext_cursor *c = (fulltext_cursor *) pCursor;
- fulltext_vtab *v = cursor_vtab (c);
- int rc;
- int i;
- StringBuffer sb;
-
- FTSTRACE(("FTS3 Filter %p\n",pCursor));
-
- initStringBuffer(&sb);
- append(&sb, "SELECT docid");
-
- for (i = 0; i < v->nColumn; i++) {
- append(&sb, ", NULL");
- }
-
- append(&sb, " FROM %_content");
- if( idxNum!=QUERY_GENERIC ) append(&sb, " WHERE docid = ?");
- sqlite3_finalize(c->pStmt);
- rc = sql_prepare(v->db, v->zDb, v->zName, &c->pStmt, stringBufferData(&sb));
- stringBufferDestroy(&sb);
- if( rc!=SQLITE_OK ) return rc;
-
- c->iCursorType = idxNum;
- switch( idxNum ){
- case QUERY_GENERIC:
- break;
-
- case QUERY_DOCID:
- rc = sqlite3_bind_int64(c->pStmt, 1, sqlite3_value_int64(argv[0]));
- if( rc!=SQLITE_OK ) return rc;
- break;
-
- default: /* full-text search */
- {
- const char *zQuery = (const char *)sqlite3_value_text(argv[0]);
- assert( idxNum<=QUERY_FULLTEXT+v->nColumn);
- assert( argc==1 );
- queryClear(&c->q);
- if( c->result.nData!=0 ){
- /* This case happens if the same cursor is used repeatedly. */
- dlrDestroy(&c->reader);
- dataBufferReset(&c->result);
- }else{
- dataBufferInit(&c->result, 0);
- }
- rc = fulltextQuery(v, idxNum-QUERY_FULLTEXT, zQuery, -1, &c->result, &c->q);
- if( rc!=SQLITE_OK ) return rc;
- if( c->result.nData!=0 ){
-
-#ifdef RANK
- dlrInit(&c->reader, DL_POSITIONS, c->result.pData, c->result.nData);
-#else
- dlrInit(&c->reader, DL_DOCIDS, c->result.pData, c->result.nData);
-
-#endif
-
- }
- break;
- }
- }
-
- return fulltextNext(pCursor);
-}
-
-/* This is the xEof method of the virtual table. The SQLite core
-** calls this routine to find out if it has reached the end of
-** a query's results set.
-*/
-static int fulltextEof(sqlite3_vtab_cursor *pCursor){
- fulltext_cursor *c = (fulltext_cursor *) pCursor;
- return c->eof;
-}
-
-/* This is the xColumn method of the virtual table. The SQLite
-** core calls this method during a query when it needs the value
-** of a column from the virtual table. This method needs to use
-** one of the sqlite3_result_*() routines to store the requested
-** value back in the pContext.
-*/
-static int fulltextColumn(sqlite3_vtab_cursor *pCursor,
- sqlite3_context *pContext, int idxCol){
- fulltext_cursor *c = (fulltext_cursor *) pCursor;
- fulltext_vtab *v = cursor_vtab (c);
-
-#ifdef STORE_CATEGORY
- if (idxCol == 0) {
- sqlite3_result_int (pContext, c->currentCatid);
- return SQLITE_OK;
- }
-#endif
-
- if( idxCol<v->nColumn ){
- sqlite3_value *pVal = sqlite3_column_value(c->pStmt, idxCol+1);
- sqlite3_result_value(pContext, pVal);
- }else if( idxCol==v->nColumn ){
- /* The extra column whose name is the same as the table.
- ** Return a blob which is a pointer to the cursor
- */
- sqlite3_result_blob(pContext, &c, sizeof(c), SQLITE_TRANSIENT);
- }else if( idxCol==v->nColumn+1 ){
- /* The docid column, which is an alias for rowid. */
- sqlite3_value *pVal = sqlite3_column_value(c->pStmt, 0);
- sqlite3_result_value(pContext, pVal);
- }
- return SQLITE_OK;
-}
-
-/* This is the xRowid method. The SQLite core calls this routine to
-** retrieve the rowid for the current row of the result set. fts3
-** exposes %_content.docid as the rowid for the virtual table. The
-** rowid should be written to *pRowid.
-*/
-static int fulltextRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
- fulltext_cursor *c = (fulltext_cursor *) pCursor;
-
- *pRowid = c->currentDocid;
- return SQLITE_OK;
-}
-
-/* Add all terms in [zText] to pendingTerms table. If [iColumn] > 0,
-** we also store positions and offsets in the hash table using that
-** column number.
-*/
-static int buildTerms(fulltext_vtab *v, sqlite_int64 iDocid,
-
-#ifdef STORE_CATEGORY
-int Catid,
-#endif
- const char *zText, int iColumn,
- gboolean limit_word_length){
-
- const char *pToken;
- int nTokenBytes;
- int iStartOffset, iEndOffset, iPosition, stop_word;
- int rc;
- TrackerParser *parser = v->parser;
- DLCollector *p;
- int nData; /* Size of doclist before our update. */
- gint nText;
- gint nWords;
-
- if (!zText) return SQLITE_OK;
-
- nText = strlen (zText);
-
- if (!nText) return SQLITE_OK;
-
- FTSTRACE (("FTS parsing started for Terms, limiting '%d' bytes to '%d' words",
- nText, v->max_words));
-
- tracker_parser_reset (parser,
- zText,
- nText,
- v->max_word_length,
- v->enable_stemmer,
- v->enable_unaccent,
- v->ignore_stop_words,
- TRUE,
- v->ignore_numbers);
- nWords = 0;
- while(nWords < v->max_words){
-
- pToken = tracker_parser_next (parser, &iPosition,
- &iStartOffset,
- &iEndOffset,
- &stop_word,
- &nTokenBytes);
- if (!pToken) {
- break;
- }
-
- if (limit_word_length && nTokenBytes < v->min_word_length) {
- continue;
- }
-
- nWords++;
-
- // printf("token being indexed is %s, begin is %d, end is %d and length is %d\n", pToken, iStartOffset, iEndOffset, nTokenBytes);
-
- if (v->ignore_stop_words && stop_word) {
- continue;
- }
-
- /* Positions can't be negative; we use -1 as a terminator
- * internally. Token can't be NULL or empty. */
- if( iPosition<0 || pToken == NULL || nTokenBytes == 0 ){
- rc = SQLITE_ERROR;
- break;
- }
-
- p = fts3HashFind(&v->pendingTerms, pToken, nTokenBytes);
- if( p==NULL ){
- nData = 0;
-
-#ifdef STORE_CATEGORY
- p = dlcNew(iDocid, DL_DEFAULT, Catid);
-#else
- p = dlcNew(iDocid, DL_DEFAULT);
-#endif
-
- fts3HashInsert(&v->pendingTerms, pToken, nTokenBytes, p);
-
- /* Overhead for our hash table entry, the key, and the value. */
- v->nPendingData += sizeof(struct fts3HashElem)+sizeof(*p)+nTokenBytes;
- }else{
- nData = p->b.nData;
- if( p->dlw.iPrevDocid!=iDocid ) {
-#ifdef STORE_CATEGORY
- dlcNext(p, iDocid, Catid);
-#else
- dlcNext(p, iDocid);
-#endif
-
-
- }
- }
- if( iColumn>=0 ){
- dlcAddPos(p, iColumn, iPosition, iStartOffset, iEndOffset);
- }
-
- /* Accumulate data added by dlcNew or dlcNext, and dlcAddPos. */
- v->nPendingData += p->b.nData-nData;
- }
-
- /* TODO(shess) Check return? Should this be able to cause errors at
- ** this point? Actually, same question about sqlite3_finalize(),
- ** though one could argue that failure there means that the data is
- ** not durable. *ponder*
- */
-
- return SQLITE_OK;
-
-}
-
-#if 0
-/* Add doclists for all terms in [pValues] to pendingTerms table. */
-static int insertTerms(fulltext_vtab *v, sqlite_int64 iDocid,
- sqlite3_value **pValues){
- int i;
-
-#ifdef STORE_CATEGORY
-
- /* tracker- category is at column 0 so we dont want to add that value to index */
- for(i = 1; i < v->nColumn ; ++i){
- char *zText = (char*)sqlite3_value_text(pValues[i]);
-
- /* tracker - as for col id we want col 0 to be the default metadata field (file:contents or email:body) ,
- col 1 to be meatdata id 1, col 2 to be metadat id 2 etc so need to decrement i here */
- int rc = buildTerms(v, iDocid, sqlite3_value_int (pValues[0]), zText, i-1);
- if( rc!=SQLITE_OK ) return rc;
- }
-
-#else
-
- for(i = 0; i < v->nColumn ; ++i){
- char *zText = (char*)sqlite3_value_text(pValues[i]);
- int rc = buildTerms(v, iDocid, zText, i);
- if( rc!=SQLITE_OK ) return rc;
- }
-
-#endif
-
- return SQLITE_OK;
-}
-
-/* Add empty doclists for all terms in the given row's content to
-** pendingTerms.
-*/
-static int deleteTerms(fulltext_vtab *v, sqlite_int64 iDocid){
- const char **pValues;
- int i, rc;
-
- /* TODO(shess) Should we allow such tables at all? */
- if( DL_DEFAULT==DL_DOCIDS ) return SQLITE_ERROR;
-
- rc = content_select(v, iDocid, &pValues);
- if( rc!=SQLITE_OK ) return rc;
-
-#ifdef STORE_CATEGORY
-
- for(i = 1 ; i < v->nColumn; ++i) {
- rc = buildTerms(v, iDocid, atoi(pValues[0]), pValues[i], -1);
- if( rc!=SQLITE_OK ) break;
- }
-
-#else
- for(i = 0 ; i < v->nColumn; ++i) {
- rc = buildTerms(v, iDocid, pValues[i], -1);
- if( rc!=SQLITE_OK ) break;
- }
-#endif
-
- freeStringArray(v->nColumn, pValues);
- return SQLITE_OK;
-}
-#endif
-
-/* TODO(shess) Refactor the code to remove this forward decl. */
-static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid);
-
-#if 0
-/* Insert a row into the %_content table; set *piDocid to be the ID of the
-** new row. Add doclists for terms to pendingTerms.
-*/
-static int index_insert(fulltext_vtab *v, sqlite3_value *pRequestDocid,
- sqlite3_value **pValues, sqlite_int64 *piDocid){
- return content_insert(v, pRequestDocid, pValues); /* execute an SQL INSERT */
-}
-
-/* Delete a row from the %_content table; add empty doclists for terms
-** to pendingTerms.
-*/
-static int index_delete(fulltext_vtab *v, sqlite_int64 iRow){
- return content_delete(v, iRow); /* execute an SQL DELETE */
-}
-
-/* Update a row in the %_content table; add delete doclists to
-** pendingTerms for old terms not in the new data, add insert doclists
-** to pendingTerms for terms in the new data.
-*/
-static int index_update(fulltext_vtab *v, sqlite_int64 iRow,
- sqlite3_value **pValues){
- int i;
- int delete;
-
- int rc = initPendingTerms(v, iRow);
- if( rc!=SQLITE_OK ) return rc;
-
- /* delete if magic column is set to -1,
- otherwise insert */
- delete = (sqlite3_value_int (pValues[v->nColumn]) == -1);
-
-#ifdef STORE_CATEGORY
-
- /* tracker- category is at column 0 so we dont want to add that value to index */
- for(i = 1; i < v->nColumn ; ++i){
- char *zText = (char*)sqlite3_value_text(pValues[i]);
-
- /* tracker - as for col id we want col 0 to be the default metadata field (file:contents or email:body) ,
- col 1 to be meatdata id 1, col 2 to be metadat id 2 etc so need to decrement i here */
- int rc = buildTerms(v, iRow, sqlite3_value_int (pValues[0]), zText, delete ? -1 : (i-1), TRUE);
- if( rc!=SQLITE_OK ) return rc;
- }
-
-#else
-
- for(i = 0; i < v->nColumn ; ++i){
- char *zText = (char*)sqlite3_value_text(pValues[i]);
- rc = buildTerms(v, iRow, zText, delete ? -1 : i, TRUE);
- if( rc!=SQLITE_OK ) return rc;
- }
-
-#endif
-
- return SQLITE_OK;
-}
-#endif
-
-/*******************************************************************/
-/* InteriorWriter is used to collect terms and block references into
-** interior nodes in %_segments. See commentary at top of file for
-** format.
-*/
-
-/* How large interior nodes can grow. */
-#define INTERIOR_MAX 2048
-
-/* Minimum number of terms per interior node (except the root). This
-** prevents large terms from making the tree too skinny - must be >0
-** so that the tree always makes progress. Note that the min tree
-** fanout will be INTERIOR_MIN_TERMS+1.
-*/
-#define INTERIOR_MIN_TERMS 7
-#if INTERIOR_MIN_TERMS<1
-# error INTERIOR_MIN_TERMS must be greater than 0.
-#endif
-
-/* ROOT_MAX controls how much data is stored inline in the segment
-** directory.
-*/
-/* TODO(shess) Push ROOT_MAX down to whoever is writing things. It's
-** only here so that interiorWriterRootInfo() and leafWriterRootInfo()
-** can both see it, but if the caller passed it in, we wouldn't even
-** need a define.
-*/
-#define ROOT_MAX 1024
-#if ROOT_MAX<VARINT_MAX*2
-# error ROOT_MAX must have enough space for a header.
-#endif
-
-/* InteriorBlock stores a linked-list of interior blocks while a lower
-** layer is being constructed.
-*/
-typedef struct InteriorBlock {
- DataBuffer term; /* Leftmost term in block's subtree. */
- DataBuffer data; /* Accumulated data for the block. */
- struct InteriorBlock *next;
-} InteriorBlock;
-
-static InteriorBlock *interiorBlockNew(int iHeight, sqlite_int64 iChildBlock,
- const char *pTerm, int nTerm){
- InteriorBlock *block = sqlite3_malloc(sizeof(InteriorBlock));
- char c[VARINT_MAX+VARINT_MAX];
- int n;
-
- if( block ){
- memset(block, 0, sizeof(*block));
- dataBufferInit(&block->term, 0);
- dataBufferReplace(&block->term, pTerm, nTerm);
-
- n = fts3PutVarint(c, iHeight);
- n += fts3PutVarint(c+n, iChildBlock);
- dataBufferInit(&block->data, INTERIOR_MAX);
- dataBufferReplace(&block->data, c, n);
- }
- return block;
-}
-
-#ifndef NDEBUG
-/* Verify that the data is readable as an interior node. */
-static void interiorBlockValidate(InteriorBlock *pBlock){
- const char *pData = pBlock->data.pData;
- int nData = pBlock->data.nData;
- int n, iDummy;
- sqlite_int64 iBlockid;
-
- assert( nData>0 );
- assert( pData!=0 );
- assert( pData+nData>pData );
-
- /* Must lead with height of node as a varint(n), n>0 */
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n<nData );
- pData += n;
- nData -= n;
-
- /* Must contain iBlockid. */
- n = fts3GetVarint(pData, &iBlockid);
- assert( n>0 );
- assert( n<=nData );
- pData += n;
- nData -= n;
-
- /* Zero or more terms of positive length */
- if( nData!=0 ){
- /* First term is not delta-encoded. */
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n+iDummy>0);
- assert( n+iDummy<=nData );
- pData += n+iDummy;
- nData -= n+iDummy;
-
- /* Following terms delta-encoded. */
- while( nData!=0 ){
- /* Length of shared prefix. */
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>=0 );
- assert( n<nData );
- pData += n;
- nData -= n;
-
- /* Length and data of distinct suffix. */
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n+iDummy>0);
- assert( n+iDummy<=nData );
- pData += n+iDummy;
- nData -= n+iDummy;
- }
- }
-}
-#define ASSERT_VALID_INTERIOR_BLOCK(x) interiorBlockValidate(x)
-#else
-#define ASSERT_VALID_INTERIOR_BLOCK(x) assert( 1 )
-#endif
-
-typedef struct InteriorWriter {
- int iHeight; /* from 0 at leaves. */
- InteriorBlock *first, *last;
- struct InteriorWriter *parentWriter;
-
- DataBuffer term; /* Last term written to block "last". */
- sqlite_int64 iOpeningChildBlock; /* First child block in block "last". */
-#ifndef NDEBUG
- sqlite_int64 iLastChildBlock; /* for consistency checks. */
-#endif
-} InteriorWriter;
-
-/* Initialize an interior node where pTerm[nTerm] marks the leftmost
-** term in the tree. iChildBlock is the leftmost child block at the
-** next level down the tree.
-*/
-static void interiorWriterInit(int iHeight, const char *pTerm, int nTerm,
- sqlite_int64 iChildBlock,
- InteriorWriter *pWriter){
- InteriorBlock *block;
- assert( iHeight>0 );
- CLEAR(pWriter);
-
- pWriter->iHeight = iHeight;
- pWriter->iOpeningChildBlock = iChildBlock;
-#ifndef NDEBUG
- pWriter->iLastChildBlock = iChildBlock;
-#endif
- block = interiorBlockNew(iHeight, iChildBlock, pTerm, nTerm);
- pWriter->last = pWriter->first = block;
- ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
- dataBufferInit(&pWriter->term, 0);
-}
-
-/* Append the child node rooted at iChildBlock to the interior node,
-** with pTerm[nTerm] as the leftmost term in iChildBlock's subtree.
-*/
-static void interiorWriterAppend(InteriorWriter *pWriter,
- const char *pTerm, int nTerm,
- sqlite_int64 iChildBlock){
- char c[VARINT_MAX+VARINT_MAX];
- int n, nPrefix = 0;
-
- ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
-
- /* The first term written into an interior node is actually
- ** associated with the second child added (the first child was added
- ** in interiorWriterInit, or in the if clause at the bottom of this
- ** function). That term gets encoded straight up, with nPrefix left
- ** at 0.
- */
- if( pWriter->term.nData==0 ){
- n = fts3PutVarint(c, nTerm);
- }else{
- while( nPrefix<pWriter->term.nData &&
- pTerm[nPrefix]==pWriter->term.pData[nPrefix] ){
- nPrefix++;
- }
-
- n = fts3PutVarint(c, nPrefix);
- n += fts3PutVarint(c+n, nTerm-nPrefix);
- }
-
-#ifndef NDEBUG
- pWriter->iLastChildBlock++;
-#endif
- assert( pWriter->iLastChildBlock==iChildBlock );
-
- /* Overflow to a new block if the new term makes the current block
- ** too big, and the current block already has enough terms.
- */
- if( pWriter->last->data.nData+n+nTerm-nPrefix>INTERIOR_MAX &&
- iChildBlock-pWriter->iOpeningChildBlock>INTERIOR_MIN_TERMS ){
- pWriter->last->next = interiorBlockNew(pWriter->iHeight, iChildBlock,
- pTerm, nTerm);
- pWriter->last = pWriter->last->next;
- pWriter->iOpeningChildBlock = iChildBlock;
- dataBufferReset(&pWriter->term);
- }else{
- dataBufferAppend2(&pWriter->last->data, c, n,
- pTerm+nPrefix, nTerm-nPrefix);
- dataBufferReplace(&pWriter->term, pTerm, nTerm);
- }
- ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
-}
-
-/* Free the space used by pWriter, including the linked-list of
-** InteriorBlocks, and parentWriter, if present.
-*/
-static int interiorWriterDestroy(InteriorWriter *pWriter){
- InteriorBlock *block = pWriter->first;
-
- while( block!=NULL ){
- InteriorBlock *b = block;
- block = block->next;
- dataBufferDestroy(&b->term);
- dataBufferDestroy(&b->data);
- sqlite3_free(b);
- }
- if( pWriter->parentWriter!=NULL ){
- interiorWriterDestroy(pWriter->parentWriter);
- sqlite3_free(pWriter->parentWriter);
- }
- dataBufferDestroy(&pWriter->term);
- SCRAMBLE(pWriter);
- return SQLITE_OK;
-}
-
-/* If pWriter can fit entirely in ROOT_MAX, return it as the root info
-** directly, leaving *piEndBlockid unchanged. Otherwise, flush
-** pWriter to %_segments, building a new layer of interior nodes, and
-** recursively ask for their root into.
-*/
-static int interiorWriterRootInfo(fulltext_vtab *v, InteriorWriter *pWriter,
- char **ppRootInfo, int *pnRootInfo,
- sqlite_int64 *piEndBlockid){
- InteriorBlock *block = pWriter->first;
- sqlite_int64 iBlockid = 0;
- int rc;
-
- /* If we can fit the segment inline */
- if( block==pWriter->last && block->data.nData<ROOT_MAX ){
- *ppRootInfo = block->data.pData;
- *pnRootInfo = block->data.nData;
- return SQLITE_OK;
- }
-
- /* Flush the first block to %_segments, and create a new level of
- ** interior node.
- */
- ASSERT_VALID_INTERIOR_BLOCK(block);
- rc = block_insert(v, block->data.pData, block->data.nData, &iBlockid);
- if( rc!=SQLITE_OK ) return rc;
- *piEndBlockid = iBlockid;
-
- pWriter->parentWriter = sqlite3_malloc(sizeof(*pWriter->parentWriter));
- interiorWriterInit(pWriter->iHeight+1,
- block->term.pData, block->term.nData,
- iBlockid, pWriter->parentWriter);
-
- /* Flush additional blocks and append to the higher interior
- ** node.
- */
- for(block=block->next; block!=NULL; block=block->next){
- ASSERT_VALID_INTERIOR_BLOCK(block);
- rc = block_insert(v, block->data.pData, block->data.nData, &iBlockid);
- if( rc!=SQLITE_OK ) return rc;
- *piEndBlockid = iBlockid;
-
- interiorWriterAppend(pWriter->parentWriter,
- block->term.pData, block->term.nData, iBlockid);
- }
-
- /* Parent node gets the chance to be the root. */
- return interiorWriterRootInfo(v, pWriter->parentWriter,
- ppRootInfo, pnRootInfo, piEndBlockid);
-}
-
-/****************************************************************/
-/* InteriorReader is used to read off the data from an interior node
-** (see comment at top of file for the format).
-*/
-typedef struct InteriorReader {
- const char *pData;
- int nData;
-
- DataBuffer term; /* previous term, for decoding term delta. */
-
- sqlite_int64 iBlockid;
-} InteriorReader;
-
-static void interiorReaderDestroy(InteriorReader *pReader){
- dataBufferDestroy(&pReader->term);
- SCRAMBLE(pReader);
-}
-
-/* TODO(shess) The assertions are great, but what if we're in NDEBUG
-** and the blob is empty or otherwise contains suspect data?
-*/
-static void interiorReaderInit(const char *pData, int nData,
- InteriorReader *pReader){
- int n, nTerm;
-
- /* Require at least the leading flag byte */
- assert( nData>0 );
- assert( pData[0]!='\0' );
-
- CLEAR(pReader);
-
- /* Decode the base blockid, and set the cursor to the first term. */
- n = fts3GetVarint(pData+1, &pReader->iBlockid);
- assert( 1+n<=nData );
- pReader->pData = pData+1+n;
- pReader->nData = nData-(1+n);
-
- /* A single-child interior node (such as when a leaf node was too
- ** large for the segment directory) won't have any terms.
- ** Otherwise, decode the first term.
- */
- if( pReader->nData==0 ){
- dataBufferInit(&pReader->term, 0);
- }else{
- n = fts3GetVarint32(pReader->pData, &nTerm);
- dataBufferInit(&pReader->term, nTerm);
- dataBufferReplace(&pReader->term, pReader->pData+n, nTerm);
- assert( n+nTerm<=pReader->nData );
- pReader->pData += n+nTerm;
- pReader->nData -= n+nTerm;
- }
-}
-
-static int interiorReaderAtEnd(InteriorReader *pReader){
- return pReader->term.nData==0;
-}
-
-static sqlite_int64 interiorReaderCurrentBlockid(InteriorReader *pReader){
- return pReader->iBlockid;
-}
-
-static int interiorReaderTermBytes(InteriorReader *pReader){
- assert( !interiorReaderAtEnd(pReader) );
- return pReader->term.nData;
-}
-static const char *interiorReaderTerm(InteriorReader *pReader){
- assert( !interiorReaderAtEnd(pReader) );
- return pReader->term.pData;
-}
-
-/* Step forward to the next term in the node. */
-static void interiorReaderStep(InteriorReader *pReader){
- assert( !interiorReaderAtEnd(pReader) );
-
- /* If the last term has been read, signal eof, else construct the
- ** next term.
- */
- if( pReader->nData==0 ){
- dataBufferReset(&pReader->term);
- }else{
- int n, nPrefix, nSuffix;
-
- n = fts3GetVarint32(pReader->pData, &nPrefix);
- n += fts3GetVarint32(pReader->pData+n, &nSuffix);
-
- /* Truncate the current term and append suffix data. */
- pReader->term.nData = nPrefix;
- dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix);
-
- assert( n+nSuffix<=pReader->nData );
- pReader->pData += n+nSuffix;
- pReader->nData -= n+nSuffix;
- }
- pReader->iBlockid++;
-}
-
-/* Compare the current term to pTerm[nTerm], returning strcmp-style
-** results. If isPrefix, equality means equal through nTerm bytes.
-*/
-static int interiorReaderTermCmp(InteriorReader *pReader,
- const char *pTerm, int nTerm, int isPrefix){
- const char *pReaderTerm = interiorReaderTerm(pReader);
- int nReaderTerm = interiorReaderTermBytes(pReader);
- int c, n = nReaderTerm<nTerm ? nReaderTerm : nTerm;
-
- if( n==0 ){
- if( nReaderTerm>0 ) return -1;
- if( nTerm>0 ) return 1;
- return 0;
- }
-
- c = memcmp(pReaderTerm, pTerm, n);
- if( c!=0 ) return c;
- if( isPrefix && n==nTerm ) return 0;
- return nReaderTerm - nTerm;
-}
-
-/****************************************************************/
-/* LeafWriter is used to collect terms and associated doclist data
-** into leaf blocks in %_segments (see top of file for format info).
-** Expected usage is:
-**
-** LeafWriter writer;
-** leafWriterInit(0, 0, &writer);
-** while( sorted_terms_left_to_process ){
-** // data is doclist data for that term.
-** rc = leafWriterStep(v, &writer, pTerm, nTerm, pData, nData);
-** if( rc!=SQLITE_OK ) goto err;
-** }
-** rc = leafWriterFinalize(v, &writer);
-**err:
-** leafWriterDestroy(&writer);
-** return rc;
-**
-** leafWriterStep() may write a collected leaf out to %_segments.
-** leafWriterFinalize() finishes writing any buffered data and stores
-** a root node in %_segdir. leafWriterDestroy() frees all buffers and
-** InteriorWriters allocated as part of writing this segment.
-**
-** TODO(shess) Document leafWriterStepMerge().
-*/
-
-/* Put terms with data this big in their own block. */
-#define STANDALONE_MIN 1024
-
-/* Keep leaf blocks below this size. */
-#define LEAF_MAX 2048
-
-typedef struct LeafWriter {
- int iLevel;
- int idx;
- sqlite_int64 iStartBlockid; /* needed to create the root info */
- sqlite_int64 iEndBlockid; /* when we're done writing. */
-
- DataBuffer term; /* previous encoded term */
- DataBuffer data; /* encoding buffer */
-
- /* bytes of first term in the current node which distinguishes that
- ** term from the last term of the previous node.
- */
- int nTermDistinct;
-
- InteriorWriter parentWriter; /* if we overflow */
- int has_parent;
-} LeafWriter;
-
-static void leafWriterInit(int iLevel, int idx, LeafWriter *pWriter){
- CLEAR(pWriter);
- pWriter->iLevel = iLevel;
- pWriter->idx = idx;
-
- dataBufferInit(&pWriter->term, 32);
-
- /* Start out with a reasonably sized block, though it can grow. */
- dataBufferInit(&pWriter->data, LEAF_MAX);
-}
-
-#ifndef NDEBUG
-/* Verify that the data is readable as a leaf node. */
-static void leafNodeValidate(const char *pData, int nData){
- int n, iDummy;
-
- if( nData==0 ) return;
- assert( nData>0 );
- assert( pData!=0 );
- assert( pData+nData>pData );
-
- /* Must lead with a varint(0) */
- n = fts3GetVarint32(pData, &iDummy);
- assert( iDummy==0 );
- assert( n>0 );
- assert( n<nData );
- pData += n;
- nData -= n;
-
- /* Leading term length and data must fit in buffer. */
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n+iDummy>0 );
- assert( n+iDummy<nData );
- pData += n+iDummy;
- nData -= n+iDummy;
-
- /* Leading term's doclist length and data must fit. */
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n+iDummy>0 );
- assert( n+iDummy<=nData );
- ASSERT_VALID_DOCLIST(DL_DEFAULT, pData+n, iDummy, NULL);
- pData += n+iDummy;
- nData -= n+iDummy;
-
- /* Verify that trailing terms and doclists also are readable. */
- while( nData!=0 ){
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>=0 );
- assert( n<nData );
- pData += n;
- nData -= n;
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n+iDummy>0 );
- assert( n+iDummy<nData );
- pData += n+iDummy;
- nData -= n+iDummy;
-
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n+iDummy>0 );
- assert( n+iDummy<=nData );
- ASSERT_VALID_DOCLIST(DL_DEFAULT, pData+n, iDummy, NULL);
- pData += n+iDummy;
- nData -= n+iDummy;
- }
-}
-#define ASSERT_VALID_LEAF_NODE(p, n) leafNodeValidate(p, n)
-#else
-#define ASSERT_VALID_LEAF_NODE(p, n) assert( 1 )
-#endif
-
-/* Flush the current leaf node to %_segments, and adding the resulting
-** blockid and the starting term to the interior node which will
-** contain it.
-*/
-static int leafWriterInternalFlush(fulltext_vtab *v, LeafWriter *pWriter,
- int iData, int nData){
- sqlite_int64 iBlockid = 0;
- const char *pStartingTerm;
- int nStartingTerm, rc, n;
-
- /* Must have the leading varint(0) flag, plus at least some
- ** valid-looking data.
- */
- assert( nData>2 );
- assert( iData>=0 );
- assert( iData+nData<=pWriter->data.nData );
- ASSERT_VALID_LEAF_NODE(pWriter->data.pData+iData, nData);
-
- rc = block_insert(v, pWriter->data.pData+iData, nData, &iBlockid);
- if( rc!=SQLITE_OK ) return rc;
- assert( iBlockid!=0 );
-
- /* Reconstruct the first term in the leaf for purposes of building
- ** the interior node.
- */
- n = fts3GetVarint32(pWriter->data.pData+iData+1, &nStartingTerm);
- pStartingTerm = pWriter->data.pData+iData+1+n;
- assert( pWriter->data.nData>iData+1+n+nStartingTerm );
- assert( pWriter->nTermDistinct>0 );
- assert( pWriter->nTermDistinct<=nStartingTerm );
- nStartingTerm = pWriter->nTermDistinct;
-
- if( pWriter->has_parent ){
- interiorWriterAppend(&pWriter->parentWriter,
- pStartingTerm, nStartingTerm, iBlockid);
- }else{
- interiorWriterInit(1, pStartingTerm, nStartingTerm, iBlockid,
- &pWriter->parentWriter);
- pWriter->has_parent = 1;
- }
-
- /* Track the span of this segment's leaf nodes. */
- if( pWriter->iEndBlockid==0 ){
- pWriter->iEndBlockid = pWriter->iStartBlockid = iBlockid;
- }else{
- pWriter->iEndBlockid++;
- assert( iBlockid==pWriter->iEndBlockid );
- }
-
- return SQLITE_OK;
-}
-static int leafWriterFlush(fulltext_vtab *v, LeafWriter *pWriter){
- int rc = leafWriterInternalFlush(v, pWriter, 0, pWriter->data.nData);
- if( rc!=SQLITE_OK ) return rc;
-
- /* Re-initialize the output buffer. */
- dataBufferReset(&pWriter->data);
-
- return SQLITE_OK;
-}
-
-/* Fetch the root info for the segment. If the entire leaf fits
-** within ROOT_MAX, then it will be returned directly, otherwise it
-** will be flushed and the root info will be returned from the
-** interior node. *piEndBlockid is set to the blockid of the last
-** interior or leaf node written to disk (0 if none are written at
-** all).
-*/
-static int leafWriterRootInfo(fulltext_vtab *v, LeafWriter *pWriter,
- char **ppRootInfo, int *pnRootInfo,
- sqlite_int64 *piEndBlockid){
- /* we can fit the segment entirely inline */
- if( !pWriter->has_parent && pWriter->data.nData<ROOT_MAX ){
- *ppRootInfo = pWriter->data.pData;
- *pnRootInfo = pWriter->data.nData;
- *piEndBlockid = 0;
- return SQLITE_OK;
- }
-
- /* Flush remaining leaf data. */
- if( pWriter->data.nData>0 ){
- int rc = leafWriterFlush(v, pWriter);
- if( rc!=SQLITE_OK ) return rc;
- }
-
- /* We must have flushed a leaf at some point. */
- assert( pWriter->has_parent );
-
- /* Tenatively set the end leaf blockid as the end blockid. If the
- ** interior node can be returned inline, this will be the final
- ** blockid, otherwise it will be overwritten by
- ** interiorWriterRootInfo().
- */
- *piEndBlockid = pWriter->iEndBlockid;
-
- return interiorWriterRootInfo(v, &pWriter->parentWriter,
- ppRootInfo, pnRootInfo, piEndBlockid);
-}
-
-/* Collect the rootInfo data and store it into the segment directory.
-** This has the effect of flushing the segment's leaf data to
-** %_segments, and also flushing any interior nodes to %_segments.
-*/
-static int leafWriterFinalize(fulltext_vtab *v, LeafWriter *pWriter){
- sqlite_int64 iEndBlockid;
- char *pRootInfo;
- int rc, nRootInfo;
-
- rc = leafWriterRootInfo(v, pWriter, &pRootInfo, &nRootInfo, &iEndBlockid);
- if( rc!=SQLITE_OK ) return rc;
-
- /* Don't bother storing an entirely empty segment. */
- if( iEndBlockid==0 && nRootInfo==0 ) return SQLITE_OK;
-
- return segdir_set(v, pWriter->iLevel, pWriter->idx,
- pWriter->iStartBlockid, pWriter->iEndBlockid,
- iEndBlockid, pRootInfo, nRootInfo);
-}
-
-static void leafWriterDestroy(LeafWriter *pWriter){
- if( pWriter->has_parent ) interiorWriterDestroy(&pWriter->parentWriter);
- dataBufferDestroy(&pWriter->term);
- dataBufferDestroy(&pWriter->data);
-}
-
-/* Encode a term into the leafWriter, delta-encoding as appropriate.
-** Returns the length of the new term which distinguishes it from the
-** previous term, which can be used to set nTermDistinct when a node
-** boundary is crossed.
-*/
-static int leafWriterEncodeTerm(LeafWriter *pWriter,
- const char *pTerm, int nTerm){
- char c[VARINT_MAX+VARINT_MAX];
- int n, nPrefix = 0;
-
- assert( nTerm>0 );
- while( nPrefix<pWriter->term.nData &&
- pTerm[nPrefix]==pWriter->term.pData[nPrefix] ){
- nPrefix++;
- /* Failing this implies that the terms weren't in order. */
- assert( nPrefix<nTerm );
- }
-
- if( pWriter->data.nData==0 ){
- /* Encode the node header and leading term as:
- ** varint(0)
- ** varint(nTerm)
- ** char pTerm[nTerm]
- */
- n = fts3PutVarint(c, '\0');
- n += fts3PutVarint(c+n, nTerm);
- dataBufferAppend2(&pWriter->data, c, n, pTerm, nTerm);
- }else{
- /* Delta-encode the term as:
- ** varint(nPrefix)
- ** varint(nSuffix)
- ** char pTermSuffix[nSuffix]
- */
- n = fts3PutVarint(c, nPrefix);
- n += fts3PutVarint(c+n, nTerm-nPrefix);
- dataBufferAppend2(&pWriter->data, c, n, pTerm+nPrefix, nTerm-nPrefix);
- }
- dataBufferReplace(&pWriter->term, pTerm, nTerm);
-
- return nPrefix+1;
-}
-
-/* Used to avoid a memmove when a large amount of doclist data is in
-** the buffer. This constructs a node and term header before
-** iDoclistData and flushes the resulting complete node using
-** leafWriterInternalFlush().
-*/
-static int leafWriterInlineFlush(fulltext_vtab *v, LeafWriter *pWriter,
- const char *pTerm, int nTerm,
- int iDoclistData){
- char c[VARINT_MAX+VARINT_MAX];
- int iData, n = fts3PutVarint(c, 0);
- n += fts3PutVarint(c+n, nTerm);
-
- /* There should always be room for the header. Even if pTerm shared
- ** a substantial prefix with the previous term, the entire prefix
- ** could be constructed from earlier data in the doclist, so there
- ** should be room.
- */
- assert( iDoclistData>=n+nTerm );
-
- iData = iDoclistData-(n+nTerm);
- memcpy(pWriter->data.pData+iData, c, n);
- memcpy(pWriter->data.pData+iData+n, pTerm, nTerm);
-
- return leafWriterInternalFlush(v, pWriter, iData, pWriter->data.nData-iData);
-}
-
-/* Push pTerm[nTerm] along with the doclist data to the leaf layer of
-** %_segments.
-*/
-static int leafWriterStepMerge(fulltext_vtab *v, LeafWriter *pWriter,
- const char *pTerm, int nTerm,
- DLReader *pReaders, int nReaders){
- char c[VARINT_MAX+VARINT_MAX];
- int iTermData = pWriter->data.nData, iDoclistData;
- int i, nData, n, nActualData, nActual, rc, nTermDistinct;
-
- ASSERT_VALID_LEAF_NODE(pWriter->data.pData, pWriter->data.nData);
- nTermDistinct = leafWriterEncodeTerm(pWriter, pTerm, nTerm);
-
- /* Remember nTermDistinct if opening a new node. */
- if( iTermData==0 ) pWriter->nTermDistinct = nTermDistinct;
-
- iDoclistData = pWriter->data.nData;
-
- /* Estimate the length of the merged doclist so we can leave space
- ** to encode it.
- */
- for(i=0, nData=0; i<nReaders; i++){
- nData += dlrAllDataBytes(&pReaders[i]);
- }
- n = fts3PutVarint(c, nData);
- dataBufferAppend(&pWriter->data, c, n);
-
- docListMerge(&pWriter->data, pReaders, nReaders);
- ASSERT_VALID_DOCLIST(DL_DEFAULT,
- pWriter->data.pData+iDoclistData+n,
- pWriter->data.nData-iDoclistData-n, NULL);
-
- /* The actual amount of doclist data at this point could be smaller
- ** than the length we encoded. Additionally, the space required to
- ** encode this length could be smaller. For small doclists, this is
- ** not a big deal, we can just use memmove() to adjust things.
- */
- nActualData = pWriter->data.nData-(iDoclistData+n);
- nActual = fts3PutVarint(c, nActualData);
- assert( nActualData<=nData );
- assert( nActual<=n );
-
- /* If the new doclist is big enough for force a standalone leaf
- ** node, we can immediately flush it inline without doing the
- ** memmove().
- */
- /* TODO(shess) This test matches leafWriterStep(), which does this
- ** test before it knows the cost to varint-encode the term and
- ** doclist lengths. At some point, change to
- ** pWriter->data.nData-iTermData>STANDALONE_MIN.
- */
- if( nTerm+nActualData>STANDALONE_MIN ){
- /* Push leaf node from before this term. */
- if( iTermData>0 ){
- rc = leafWriterInternalFlush(v, pWriter, 0, iTermData);
- if( rc!=SQLITE_OK ) return rc;
-
- pWriter->nTermDistinct = nTermDistinct;
- }
-
- /* Fix the encoded doclist length. */
- iDoclistData += n - nActual;
- memcpy(pWriter->data.pData+iDoclistData, c, nActual);
-
- /* Push the standalone leaf node. */
- rc = leafWriterInlineFlush(v, pWriter, pTerm, nTerm, iDoclistData);
- if( rc!=SQLITE_OK ) return rc;
-
- /* Leave the node empty. */
- dataBufferReset(&pWriter->data);
-
- return rc;
- }
-
- /* At this point, we know that the doclist was small, so do the
- ** memmove if indicated.
- */
- if( nActual<n ){
- memmove(pWriter->data.pData+iDoclistData+nActual,
- pWriter->data.pData+iDoclistData+n,
- pWriter->data.nData-(iDoclistData+n));
- pWriter->data.nData -= n-nActual;
- }
-
- /* Replace written length with actual length. */
- memcpy(pWriter->data.pData+iDoclistData, c, nActual);
-
- /* If the node is too large, break things up. */
- /* TODO(shess) This test matches leafWriterStep(), which does this
- ** test before it knows the cost to varint-encode the term and
- ** doclist lengths. At some point, change to
- ** pWriter->data.nData>LEAF_MAX.
- */
- if( iTermData+nTerm+nActualData>LEAF_MAX ){
- /* Flush out the leading data as a node */
- rc = leafWriterInternalFlush(v, pWriter, 0, iTermData);
- if( rc!=SQLITE_OK ) return rc;
-
- pWriter->nTermDistinct = nTermDistinct;
-
- /* Rebuild header using the current term */
- n = fts3PutVarint(pWriter->data.pData, 0);
- n += fts3PutVarint(pWriter->data.pData+n, nTerm);
- memcpy(pWriter->data.pData+n, pTerm, nTerm);
- n += nTerm;
-
- /* There should always be room, because the previous encoding
- ** included all data necessary to construct the term.
- */
- assert( n<iDoclistData );
- /* So long as STANDALONE_MIN is half or less of LEAF_MAX, the
- ** following memcpy() is safe (as opposed to needing a memmove).
- */
- assert( 2*STANDALONE_MIN<=LEAF_MAX );
- assert( n+pWriter->data.nData-iDoclistData<iDoclistData );
- memcpy(pWriter->data.pData+n,
- pWriter->data.pData+iDoclistData,
- pWriter->data.nData-iDoclistData);
- pWriter->data.nData -= iDoclistData-n;
- }
- ASSERT_VALID_LEAF_NODE(pWriter->data.pData, pWriter->data.nData);
-
- return SQLITE_OK;
-}
-
-/* Push pTerm[nTerm] along with the doclist data to the leaf layer of
-** %_segments.
-*/
-/* TODO(shess) Revise writeZeroSegment() so that doclists are
-** constructed directly in pWriter->data.
-*/
-static int leafWriterStep(fulltext_vtab *v, LeafWriter *pWriter,
- const char *pTerm, int nTerm,
- const char *pData, int nData){
- int rc;
- DLReader reader;
-
- dlrInit(&reader, DL_DEFAULT, pData, nData);
- rc = leafWriterStepMerge(v, pWriter, pTerm, nTerm, &reader, 1);
- dlrDestroy(&reader);
-
- return rc;
-}
-
-
-/****************************************************************/
-/* LeafReader is used to iterate over an individual leaf node. */
-typedef struct LeafReader {
- DataBuffer term; /* copy of current term. */
-
- const char *pData; /* data for current term. */
- int nData;
-} LeafReader;
-
-static void leafReaderDestroy(LeafReader *pReader){
- dataBufferDestroy(&pReader->term);
- SCRAMBLE(pReader);
-}
-
-static int leafReaderAtEnd(LeafReader *pReader){
- return pReader->nData<=0;
-}
-
-/* Access the current term. */
-static int leafReaderTermBytes(LeafReader *pReader){
- return pReader->term.nData;
-}
-static const char *leafReaderTerm(LeafReader *pReader){
- assert( pReader->term.nData>0 );
- return pReader->term.pData;
-}
-
-/* Access the doclist data for the current term. */
-static int leafReaderDataBytes(LeafReader *pReader){
- int nData;
- assert( pReader->term.nData>0 );
- fts3GetVarint32(pReader->pData, &nData);
- return nData;
-}
-static const char *leafReaderData(LeafReader *pReader){
- int n, nData;
- assert( pReader->term.nData>0 );
- n = fts3GetVarint32(pReader->pData, &nData);
- return pReader->pData+n;
-}
-
-static void leafReaderInit(const char *pData, int nData,
- LeafReader *pReader){
- int nTerm, n;
-
- assert( nData>0 );
- assert( pData[0]=='\0' );
-
- CLEAR(pReader);
-
- /* Read the first term, skipping the header byte. */
- n = fts3GetVarint32(pData+1, &nTerm);
- dataBufferInit(&pReader->term, nTerm);
- dataBufferReplace(&pReader->term, pData+1+n, nTerm);
-
- /* Position after the first term. */
- assert( 1+n+nTerm<nData );
- pReader->pData = pData+1+n+nTerm;
- pReader->nData = nData-1-n-nTerm;
-}
-
-/* Step the reader forward to the next term. */
-static void leafReaderStep(LeafReader *pReader){
- int n, nData, nPrefix, nSuffix;
- assert( !leafReaderAtEnd(pReader) );
-
- /* Skip previous entry's data block. */
- n = fts3GetVarint32(pReader->pData, &nData);
- assert( n+nData<=pReader->nData );
- pReader->pData += n+nData;
- pReader->nData -= n+nData;
-
- if( !leafReaderAtEnd(pReader) ){
- /* Construct the new term using a prefix from the old term plus a
- ** suffix from the leaf data.
- */
- n = fts3GetVarint32(pReader->pData, &nPrefix);
- n += fts3GetVarint32(pReader->pData+n, &nSuffix);
- assert( n+nSuffix<pReader->nData );
- pReader->term.nData = nPrefix;
- dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix);
-
- pReader->pData += n+nSuffix;
- pReader->nData -= n+nSuffix;
- }
-}
-
-/* strcmp-style comparison of pReader's current term against pTerm.
-** If isPrefix, equality means equal through nTerm bytes.
-*/
-static int leafReaderTermCmp(LeafReader *pReader,
- const char *pTerm, int nTerm, int isPrefix){
- int c, n = pReader->term.nData<nTerm ? pReader->term.nData : nTerm;
- if( n==0 ){
- if( pReader->term.nData>0 ) return -1;
- if(nTerm>0 ) return 1;
- return 0;
- }
-
- c = memcmp(pReader->term.pData, pTerm, n);
- if( c!=0 ) return c;
- if( isPrefix && n==nTerm ) return 0;
- return pReader->term.nData - nTerm;
-}
-
-
-/****************************************************************/
-/* LeavesReader wraps LeafReader to allow iterating over the entire
-** leaf layer of the tree.
-*/
-typedef struct LeavesReader {
- int idx; /* Index within the segment. */
-
- sqlite3_stmt *pStmt; /* Statement we're streaming leaves from. */
- int eof; /* we've seen SQLITE_DONE from pStmt. */
-
- LeafReader leafReader; /* reader for the current leaf. */
- DataBuffer rootData; /* root data for inline. */
-} LeavesReader;
-
-/* Access the current term. */
-static int leavesReaderTermBytes(LeavesReader *pReader){
- assert( !pReader->eof );
- return leafReaderTermBytes(&pReader->leafReader);
-}
-static const char *leavesReaderTerm(LeavesReader *pReader){
- assert( !pReader->eof );
- return leafReaderTerm(&pReader->leafReader);
-}
-
-/* Access the doclist data for the current term. */
-static int leavesReaderDataBytes(LeavesReader *pReader){
- assert( !pReader->eof );
- return leafReaderDataBytes(&pReader->leafReader);
-}
-static const char *leavesReaderData(LeavesReader *pReader){
- assert( !pReader->eof );
- return leafReaderData(&pReader->leafReader);
-}
-
-static int leavesReaderAtEnd(LeavesReader *pReader){
- return pReader->eof;
-}
-
-/* loadSegmentLeaves() may not read all the way to SQLITE_DONE, thus
-** leaving the statement handle open, which locks the table.
-*/
-/* TODO(shess) This "solution" is not satisfactory. Really, there
-** should be check-in function for all statement handles which
-** arranges to call sqlite3_reset(). This most likely will require
-** modification to control flow all over the place, though, so for now
-** just punt.
-**
-** Note the the current system assumes that segment merges will run to
-** completion, which is why this particular probably hasn't arisen in
-** this case. Probably a brittle assumption.
-*/
-static int leavesReaderReset(LeavesReader *pReader){
- return sqlite3_reset(pReader->pStmt);
-}
-
-static void leavesReaderDestroy(LeavesReader *pReader){
- /* If idx is -1, that means we're using a non-cached statement
- ** handle in the optimize() case, so we need to release it.
- */
- if( pReader->pStmt!=NULL && pReader->idx==-1 ){
- sqlite3_finalize(pReader->pStmt);
- }
- leafReaderDestroy(&pReader->leafReader);
- dataBufferDestroy(&pReader->rootData);
- SCRAMBLE(pReader);
-}
-
-/* Initialize pReader with the given root data (if iStartBlockid==0
-** the leaf data was entirely contained in the root), or from the
-** stream of blocks between iStartBlockid and iEndBlockid, inclusive.
-*/
-static int leavesReaderInit(fulltext_vtab *v,
- int idx,
- sqlite_int64 iStartBlockid,
- sqlite_int64 iEndBlockid,
- const char *pRootData, int nRootData,
- LeavesReader *pReader){
- CLEAR(pReader);
- pReader->idx = idx;
-
- dataBufferInit(&pReader->rootData, 0);
- if( iStartBlockid==0 ){
- /* Entire leaf level fit in root data. */
- dataBufferReplace(&pReader->rootData, pRootData, nRootData);
- leafReaderInit(pReader->rootData.pData, pReader->rootData.nData,
- &pReader->leafReader);
- }else{
- sqlite3_stmt *s;
- int rc = sql_get_leaf_statement(v, idx, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 1, iStartBlockid);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 2, iEndBlockid);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_step(s);
- if( rc==SQLITE_DONE ){
- pReader->eof = 1;
- return SQLITE_OK;
- }
- if( rc!=SQLITE_ROW ) return rc;
-
- pReader->pStmt = s;
- leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
- sqlite3_column_bytes(pReader->pStmt, 0),
- &pReader->leafReader);
- }
- return SQLITE_OK;
-}
-
-/* Step the current leaf forward to the next term. If we reach the
-** end of the current leaf, step forward to the next leaf block.
-*/
-static int leavesReaderStep(fulltext_vtab *v, LeavesReader *pReader){
- assert( !leavesReaderAtEnd(pReader) );
- leafReaderStep(&pReader->leafReader);
-
- if( leafReaderAtEnd(&pReader->leafReader) ){
- int rc;
- if( pReader->rootData.pData ){
- pReader->eof = 1;
- return SQLITE_OK;
- }
- rc = sqlite3_step(pReader->pStmt);
- if( rc!=SQLITE_ROW ){
- pReader->eof = 1;
- return rc==SQLITE_DONE ? SQLITE_OK : rc;
- }
- leafReaderDestroy(&pReader->leafReader);
- leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
- sqlite3_column_bytes(pReader->pStmt, 0),
- &pReader->leafReader);
- }
- return SQLITE_OK;
-}
-
-/* Order LeavesReaders by their term, ignoring idx. Readers at eof
-** always sort to the end.
-*/
-static int leavesReaderTermCmp(LeavesReader *lr1, LeavesReader *lr2){
- if( leavesReaderAtEnd(lr1) ){
- if( leavesReaderAtEnd(lr2) ) return 0;
- return 1;
- }
- if( leavesReaderAtEnd(lr2) ) return -1;
-
- return leafReaderTermCmp(&lr1->leafReader,
- leavesReaderTerm(lr2), leavesReaderTermBytes(lr2),
- 0);
-}
-
-/* Similar to leavesReaderTermCmp(), with additional ordering by idx
-** so that older segments sort before newer segments.
-*/
-static int leavesReaderCmp(LeavesReader *lr1, LeavesReader *lr2){
- int c = leavesReaderTermCmp(lr1, lr2);
- if( c!=0 ) return c;
- return lr1->idx-lr2->idx;
-}
-
-/* Assume that pLr[1]..pLr[nLr] are sorted. Bubble pLr[0] into its
-** sorted position.
-*/
-static void leavesReaderReorder(LeavesReader *pLr, int nLr){
- while( nLr>1 && leavesReaderCmp(pLr, pLr+1)>0 ){
- LeavesReader tmp = pLr[0];
- pLr[0] = pLr[1];
- pLr[1] = tmp;
- nLr--;
- pLr++;
- }
-}
-
-/* Initializes pReaders with the segments from level iLevel, returning
-** the number of segments in *piReaders. Leaves pReaders in sorted
-** order.
-*/
-static int leavesReadersInit(fulltext_vtab *v, int iLevel,
- LeavesReader *pReaders, int *piReaders){
- sqlite3_stmt *s;
- int i, rc = sql_get_statement(v, SEGDIR_SELECT_LEVEL_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int(s, 1, iLevel);
- if( rc!=SQLITE_OK ) return rc;
-
- i = 0;
- while( (rc = sqlite3_step(s))==SQLITE_ROW ){
- sqlite_int64 iStart = sqlite3_column_int64(s, 0);
- sqlite_int64 iEnd = sqlite3_column_int64(s, 1);
- const char *pRootData = sqlite3_column_blob(s, 2);
- int nRootData = sqlite3_column_bytes(s, 2);
-
- assert( i<MERGE_COUNT );
- rc = leavesReaderInit(v, i, iStart, iEnd, pRootData, nRootData,
- &pReaders[i]);
- if( rc!=SQLITE_OK ) break;
-
- i++;
- }
- if( rc!=SQLITE_DONE ){
- while( i-->0 ){
- leavesReaderDestroy(&pReaders[i]);
- }
- return rc;
- }
-
- *piReaders = i;
-
- /* Leave our results sorted by term, then age. */
- while( i-- ){
- leavesReaderReorder(pReaders+i, *piReaders-i);
- }
- return SQLITE_OK;
-}
-
-/* Merge doclists from pReaders[nReaders] into a single doclist, which
-** is written to pWriter. Assumes pReaders is ordered oldest to
-** newest.
-*/
-/* TODO(shess) Consider putting this inline in segmentMerge(). */
-static int leavesReadersMerge(fulltext_vtab *v,
- LeavesReader *pReaders, int nReaders,
- LeafWriter *pWriter){
- DLReader dlReaders[MERGE_COUNT];
- const char *pTerm = leavesReaderTerm(pReaders);
- int i, nTerm = leavesReaderTermBytes(pReaders);
-
- assert( nReaders<=MERGE_COUNT );
-
- for(i=0; i<nReaders; i++){
- dlrInit(&dlReaders[i], DL_DEFAULT,
- leavesReaderData(pReaders+i),
- leavesReaderDataBytes(pReaders+i));
- }
-
- return leafWriterStepMerge(v, pWriter, pTerm, nTerm, dlReaders, nReaders);
-}
-
-/* Forward ref due to mutual recursion with segdirNextIndex(). */
-static int segmentMerge(fulltext_vtab *v, int iLevel);
-
-/* Put the next available index at iLevel into *pidx. If iLevel
-** already has MERGE_COUNT segments, they are merged to a higher
-** level to make room.
-*/
-static int segdirNextIndex(fulltext_vtab *v, int iLevel, int *pidx){
- int rc = segdir_max_index(v, iLevel, pidx);
- if( rc==SQLITE_DONE ){ /* No segments at iLevel. */
- *pidx = 0;
- }else if( rc==SQLITE_ROW ){
- if( *pidx==(MERGE_COUNT-1) ){
- rc = segmentMerge(v, iLevel);
- if( rc!=SQLITE_OK ) return rc;
- *pidx = 0;
- }else{
- (*pidx)++;
- }
- }else{
- return rc;
- }
- return SQLITE_OK;
-}
-
-/* Merge MERGE_COUNT segments at iLevel into a new segment at
-** iLevel+1. If iLevel+1 is already full of segments, those will be
-** merged to make room.
-*/
-static int segmentMerge(fulltext_vtab *v, int iLevel){
- LeafWriter writer;
- LeavesReader lrs[MERGE_COUNT];
- int i, rc, idx = 0;
-
- /* Determine the next available segment index at the next level,
- ** merging as necessary.
- */
- rc = segdirNextIndex(v, iLevel+1, &idx);
- if( rc!=SQLITE_OK ) return rc;
-
- /* TODO(shess) This assumes that we'll always see exactly
- ** MERGE_COUNT segments to merge at a given level. That will be
- ** broken if we allow the developer to request preemptive or
- ** deferred merging.
- */
- memset(&lrs, '\0', sizeof(lrs));
- rc = leavesReadersInit(v, iLevel, lrs, &i);
- if( rc!=SQLITE_OK ) return rc;
- assert( i==MERGE_COUNT );
-
- leafWriterInit(iLevel+1, idx, &writer);
-
- /* Since leavesReaderReorder() pushes readers at eof to the end,
- ** when the first reader is empty, all will be empty.
- */
- while( !leavesReaderAtEnd(lrs) ){
- /* Figure out how many readers share their next term. */
- for(i=1; i<MERGE_COUNT && !leavesReaderAtEnd(lrs+i); i++){
- if( 0!=leavesReaderTermCmp(lrs, lrs+i) ) break;
- }
-
- rc = leavesReadersMerge(v, lrs, i, &writer);
- if( rc!=SQLITE_OK ) goto err;
-
- /* Step forward those that were merged. */
- while( i-->0 ){
- rc = leavesReaderStep(v, lrs+i);
- if( rc!=SQLITE_OK ) goto err;
-
- /* Reorder by term, then by age. */
- leavesReaderReorder(lrs+i, MERGE_COUNT-i);
- }
- }
-
- for(i=0; i<MERGE_COUNT; i++){
- leavesReaderDestroy(&lrs[i]);
- }
-
- rc = leafWriterFinalize(v, &writer);
- leafWriterDestroy(&writer);
- if( rc!=SQLITE_OK ) return rc;
-
- /* Delete the merged segment data. */
- return segdir_delete(v, iLevel);
-
- err:
- for(i=0; i<MERGE_COUNT; i++){
- leavesReaderDestroy(&lrs[i]);
- }
- leafWriterDestroy(&writer);
- return rc;
-}
-
-/* Accumulate the union of *acc and *pData into *acc. */
-static void docListAccumulateUnion(DataBuffer *acc,
- const char *pData, int nData) {
- DataBuffer tmp = *acc;
- dataBufferInit(acc, tmp.nData+nData);
- docListUnion(tmp.pData, tmp.nData, pData, nData, acc);
- dataBufferDestroy(&tmp);
-}
-
-/* TODO(shess) It might be interesting to explore different merge
-** strategies, here. For instance, since this is a sorted merge, we
-** could easily merge many doclists in parallel. With some
-** comprehension of the storage format, we could merge all of the
-** doclists within a leaf node directly from the leaf node's storage.
-** It may be worthwhile to merge smaller doclists before larger
-** doclists, since they can be traversed more quickly - but the
-** results may have less overlap, making them more expensive in a
-** different way.
-*/
-
-/* Scan pReader for pTerm/nTerm, and merge the term's doclist over
-** *out (any doclists with duplicate docids overwrite those in *out).
-** Internal function for loadSegmentLeaf().
-*/
-static int loadSegmentLeavesInt(fulltext_vtab *v, LeavesReader *pReader,
- const char *pTerm, int nTerm, int isPrefix,
- DataBuffer *out){
- /* doclist data is accumulated into pBuffers similar to how one does
- ** increment in binary arithmetic. If index 0 is empty, the data is
- ** stored there. If there is data there, it is merged and the
- ** results carried into position 1, with further merge-and-carry
- ** until an empty position is found.
- */
- DataBuffer *pBuffers = NULL;
- int nBuffers = 0, nMaxBuffers = 0, rc;
-
- assert( nTerm>0 );
-
- for(rc=SQLITE_OK; rc==SQLITE_OK && !leavesReaderAtEnd(pReader);
- rc=leavesReaderStep(v, pReader)){
- /* TODO(shess) Really want leavesReaderTermCmp(), but that name is
- ** already taken to compare the terms of two LeavesReaders. Think
- ** on a better name. [Meanwhile, break encapsulation rather than
- ** use a confusing name.]
- */
- int c = leafReaderTermCmp(&pReader->leafReader, pTerm, nTerm, isPrefix);
- if( c>0 ) break; /* Past any possible matches. */
- if( c==0 ){
- const char *pData = leavesReaderData(pReader);
- int iBuffer, nData = leavesReaderDataBytes(pReader);
-
- /* Find the first empty buffer. */
- for(iBuffer=0; iBuffer<nBuffers; ++iBuffer){
- if( 0==pBuffers[iBuffer].nData ) break;
- }
-
- /* Out of buffers, add an empty one. */
- if( iBuffer==nBuffers ){
- if( nBuffers==nMaxBuffers ){
- DataBuffer *p;
- nMaxBuffers += 20;
-
- /* Manual realloc so we can handle NULL appropriately. */
- p = sqlite3_malloc(nMaxBuffers*sizeof(*pBuffers));
- if( p==NULL ){
- rc = SQLITE_NOMEM;
- break;
- }
-
- if( nBuffers>0 ){
- assert(pBuffers!=NULL);
- memcpy(p, pBuffers, nBuffers*sizeof(*pBuffers));
- sqlite3_free(pBuffers);
- }
- pBuffers = p;
- }
- dataBufferInit(&(pBuffers[nBuffers]), 0);
- nBuffers++;
- }
-
- /* At this point, must have an empty at iBuffer. */
- assert(iBuffer<nBuffers && pBuffers[iBuffer].nData==0);
-
- /* If empty was first buffer, no need for merge logic. */
- if( iBuffer==0 ){
- dataBufferReplace(&(pBuffers[0]), pData, nData);
- }else{
- /* pAcc is the empty buffer the merged data will end up in. */
- DataBuffer *pAcc = &(pBuffers[iBuffer]);
- DataBuffer *p = &(pBuffers[0]);
-
- /* Handle position 0 specially to avoid need to prime pAcc
- ** with pData/nData.
- */
- dataBufferSwap(p, pAcc);
- docListAccumulateUnion(pAcc, pData, nData);
-
- /* Accumulate remaining doclists into pAcc. */
- for(++p; p<pAcc; ++p){
- docListAccumulateUnion(pAcc, p->pData, p->nData);
-
- /* dataBufferReset() could allow a large doclist to blow up
- ** our memory requirements.
- */
- if( p->nCapacity<1024 ){
- dataBufferReset(p);
- }else{
- dataBufferDestroy(p);
- dataBufferInit(p, 0);
- }
- }
- }
- }
- }
-
- /* Union all the doclists together into *out. */
- /* TODO(shess) What if *out is big? Sigh. */
- if( rc==SQLITE_OK && nBuffers>0 ){
- int iBuffer;
- for(iBuffer=0; iBuffer<nBuffers; ++iBuffer){
- if( pBuffers[iBuffer].nData>0 ){
- if( out->nData==0 ){
- dataBufferSwap(out, &(pBuffers[iBuffer]));
- }else{
- docListAccumulateUnion(out, pBuffers[iBuffer].pData,
- pBuffers[iBuffer].nData);
- }
- }
- }
- }
-
- while( nBuffers-- ){
- dataBufferDestroy(&(pBuffers[nBuffers]));
- }
- if( pBuffers!=NULL ) sqlite3_free(pBuffers);
-
- return rc;
-}
-
-/* Call loadSegmentLeavesInt() with pData/nData as input. */
-static int loadSegmentLeaf(fulltext_vtab *v, const char *pData, int nData,
- const char *pTerm, int nTerm, int isPrefix,
- DataBuffer *out){
- LeavesReader reader;
- int rc;
-
- assert( nData>1 );
- assert( *pData=='\0' );
- rc = leavesReaderInit(v, 0, 0, 0, pData, nData, &reader);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = loadSegmentLeavesInt(v, &reader, pTerm, nTerm, isPrefix, out);
- leavesReaderReset(&reader);
- leavesReaderDestroy(&reader);
- return rc;
-}
-
-/* Call loadSegmentLeavesInt() with the leaf nodes from iStartLeaf to
-** iEndLeaf (inclusive) as input, and merge the resulting doclist into
-** out.
-*/
-static int loadSegmentLeaves(fulltext_vtab *v,
- sqlite_int64 iStartLeaf, sqlite_int64 iEndLeaf,
- const char *pTerm, int nTerm, int isPrefix,
- DataBuffer *out){
- int rc;
- LeavesReader reader;
-
- assert( iStartLeaf<=iEndLeaf );
- rc = leavesReaderInit(v, 0, iStartLeaf, iEndLeaf, NULL, 0, &reader);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = loadSegmentLeavesInt(v, &reader, pTerm, nTerm, isPrefix, out);
- leavesReaderReset(&reader);
- leavesReaderDestroy(&reader);
- return rc;
-}
-
-/* Taking pData/nData as an interior node, find the sequence of child
-** nodes which could include pTerm/nTerm/isPrefix. Note that the
-** interior node terms logically come between the blocks, so there is
-** one more blockid than there are terms (that block contains terms >=
-** the last interior-node term).
-*/
-/* TODO(shess) The calling code may already know that the end child is
-** not worth calculating, because the end may be in a later sibling
-** node. Consider whether breaking symmetry is worthwhile. I suspect
-** it is not worthwhile.
-*/
-static void getChildrenContaining(const char *pData, int nData,
- const char *pTerm, int nTerm, int isPrefix,
- sqlite_int64 *piStartChild,
- sqlite_int64 *piEndChild){
- InteriorReader reader;
-
- assert( nData>1 );
- assert( *pData!='\0' );
- interiorReaderInit(pData, nData, &reader);
-
- /* Scan for the first child which could contain pTerm/nTerm. */
- while( !interiorReaderAtEnd(&reader) ){
- if( interiorReaderTermCmp(&reader, pTerm, nTerm, 0)>0 ) break;
- interiorReaderStep(&reader);
- }
- *piStartChild = interiorReaderCurrentBlockid(&reader);
-
- /* Keep scanning to find a term greater than our term, using prefix
- ** comparison if indicated. If isPrefix is false, this will be the
- ** same blockid as the starting block.
- */
- while( !interiorReaderAtEnd(&reader) ){
- if( interiorReaderTermCmp(&reader, pTerm, nTerm, isPrefix)>0 ) break;
- interiorReaderStep(&reader);
- }
- *piEndChild = interiorReaderCurrentBlockid(&reader);
-
- interiorReaderDestroy(&reader);
-
- /* Children must ascend, and if !prefix, both must be the same. */
- assert( *piEndChild>=*piStartChild );
- assert( isPrefix || *piStartChild==*piEndChild );
-}
-
-/* Read block at iBlockid and pass it with other params to
-** getChildrenContaining().
-*/
-static int loadAndGetChildrenContaining(
- fulltext_vtab *v,
- sqlite_int64 iBlockid,
- const char *pTerm, int nTerm, int isPrefix,
- sqlite_int64 *piStartChild, sqlite_int64 *piEndChild
-){
- sqlite3_stmt *s = NULL;
- int rc;
-
- assert( iBlockid!=0 );
- assert( pTerm!=NULL );
- assert( nTerm!=0 ); /* TODO(shess) Why not allow this? */
- assert( piStartChild!=NULL );
- assert( piEndChild!=NULL );
-
- rc = sql_get_statement(v, BLOCK_SELECT_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 1, iBlockid);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_step(s);
- if( rc==SQLITE_DONE ) return SQLITE_ERROR;
- if( rc!=SQLITE_ROW ) return rc;
-
- getChildrenContaining(sqlite3_column_blob(s, 0), sqlite3_column_bytes(s, 0),
- pTerm, nTerm, isPrefix, piStartChild, piEndChild);
-
- /* We expect only one row. We must execute another sqlite3_step()
- * to complete the iteration; otherwise the table will remain
- * locked. */
- rc = sqlite3_step(s);
- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
- if( rc!=SQLITE_DONE ) return rc;
-
- return SQLITE_OK;
-}
-
-/* Traverse the tree represented by pData[nData] looking for
-** pTerm[nTerm], placing its doclist into *out. This is internal to
-** loadSegment() to make error-handling cleaner.
-*/
-static int loadSegmentInt(fulltext_vtab *v, const char *pData, int nData,
- sqlite_int64 iLeavesEnd,
- const char *pTerm, int nTerm, int isPrefix,
- DataBuffer *out){
- /* Special case where root is a leaf. */
- if( *pData=='\0' ){
- return loadSegmentLeaf(v, pData, nData, pTerm, nTerm, isPrefix, out);
- }else{
- int rc;
- sqlite_int64 iStartChild, iEndChild;
-
- /* Process pData as an interior node, then loop down the tree
- ** until we find the set of leaf nodes to scan for the term.
- */
- getChildrenContaining(pData, nData, pTerm, nTerm, isPrefix,
- &iStartChild, &iEndChild);
- while( iStartChild>iLeavesEnd ){
- sqlite_int64 iNextStart, iNextEnd;
- rc = loadAndGetChildrenContaining(v, iStartChild, pTerm, nTerm, isPrefix,
- &iNextStart, &iNextEnd);
- if( rc!=SQLITE_OK ) return rc;
-
- /* If we've branched, follow the end branch, too. */
- if( iStartChild!=iEndChild ){
- sqlite_int64 iDummy;
- rc = loadAndGetChildrenContaining(v, iEndChild, pTerm, nTerm, isPrefix,
- &iDummy, &iNextEnd);
- if( rc!=SQLITE_OK ) return rc;
- }
-
- assert( iNextStart<=iNextEnd );
- iStartChild = iNextStart;
- iEndChild = iNextEnd;
- }
- assert( iStartChild<=iLeavesEnd );
- assert( iEndChild<=iLeavesEnd );
-
- /* Scan through the leaf segments for doclists. */
- return loadSegmentLeaves(v, iStartChild, iEndChild,
- pTerm, nTerm, isPrefix, out);
- }
-}
-
-/* Call loadSegmentInt() to collect the doclist for pTerm/nTerm, then
-** merge its doclist over *out (any duplicate doclists read from the
-** segment rooted at pData will overwrite those in *out).
-*/
-/* TODO(shess) Consider changing this to determine the depth of the
-** leaves using either the first characters of interior nodes (when
-** ==1, we're one level above the leaves), or the first character of
-** the root (which will describe the height of the tree directly).
-** Either feels somewhat tricky to me.
-*/
-/* TODO(shess) The current merge is likely to be slow for large
-** doclists (though it should process from newest/smallest to
-** oldest/largest, so it may not be that bad). It might be useful to
-** modify things to allow for N-way merging. This could either be
-** within a segment, with pairwise merges across segments, or across
-** all segments at once.
-*/
-static int loadSegment(fulltext_vtab *v, const char *pData, int nData,
- sqlite_int64 iLeavesEnd,
- const char *pTerm, int nTerm, int isPrefix,
- DataBuffer *out){
- DataBuffer result;
- int rc;
-
- assert( nData>1 );
-
- /* This code should never be called with buffered updates. */
- assert( v->nPendingData<0 );
-
- dataBufferInit(&result, 0);
- rc = loadSegmentInt(v, pData, nData, iLeavesEnd,
- pTerm, nTerm, isPrefix, &result);
- if( rc==SQLITE_OK && result.nData>0 ){
- if( out->nData==0 ){
- DataBuffer tmp = *out;
- *out = result;
- result = tmp;
- }else{
- DataBuffer merged;
- DLReader readers[2];
-
- dlrInit(&readers[0], DL_DEFAULT, out->pData, out->nData);
- dlrInit(&readers[1], DL_DEFAULT, result.pData, result.nData);
- dataBufferInit(&merged, out->nData+result.nData);
- docListMerge(&merged, readers, 2);
- dataBufferDestroy(out);
- *out = merged;
- dlrDestroy(&readers[0]);
- dlrDestroy(&readers[1]);
- }
- }
- dataBufferDestroy(&result);
- return rc;
-}
-
-/* Scan the database and merge together the posting lists for the term
-** into *out.
-*/
-static int termSelect(fulltext_vtab *v, int iColumn,
- const char *pTerm, int nTerm, int isPrefix,
- DocListType iType, DataBuffer *out){
- DataBuffer doclist;
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- /* This code should never be called with buffered updates. */
- assert( v->nPendingData<0 );
-
- dataBufferInit(&doclist, 0);
-
- /* Traverse the segments from oldest to newest so that newer doclist
- ** elements for given docids overwrite older elements.
- */
- while( (rc = sqlite3_step(s))==SQLITE_ROW ){
- const char *pData = sqlite3_column_blob(s, 2);
- const int nData = sqlite3_column_bytes(s, 2);
- const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1);
- rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, isPrefix,
- &doclist);
- if( rc!=SQLITE_OK ) goto err;
- }
- if( rc==SQLITE_DONE ){
- if( doclist.nData!=0 ){
- /* TODO(shess) The old term_select_all() code applied the column
- ** restrict as we merged segments, leading to smaller buffers.
- ** This is probably worthwhile to bring back, once the new storage
- ** system is checked in.
- */
- if( iColumn==v->nColumn) iColumn = -1;
- docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
- iColumn, iType, out);
- }
- rc = SQLITE_OK;
- }
-
- err:
- dataBufferDestroy(&doclist);
- return rc;
-}
-
-/****************************************************************/
-/* Used to hold hashtable data for sorting. */
-typedef struct TermData {
- const char *pTerm;
- int nTerm;
- DLCollector *pCollector;
-} TermData;
-
-/* Orders TermData elements in strcmp fashion ( <0 for less-than, 0
-** for equal, >0 for greater-than).
-*/
-static int termDataCmp(const void *av, const void *bv){
- const TermData *a = (const TermData *)av;
- const TermData *b = (const TermData *)bv;
- int n = a->nTerm<b->nTerm ? a->nTerm : b->nTerm;
- int c = memcmp(a->pTerm, b->pTerm, n);
- if( c!=0 ) return c;
- return a->nTerm-b->nTerm;
-}
-
-/* Order pTerms data by term, then write a new level 0 segment using
-** LeafWriter.
-*/
-static int writeZeroSegment(fulltext_vtab *v, fts3Hash *pTerms){
- fts3HashElem *e;
- int idx, rc, i, n;
- TermData *pData;
- LeafWriter writer;
- DataBuffer dl;
-
- /* Determine the next index at level 0, merging as necessary. */
- rc = segdirNextIndex(v, 0, &idx);
- if( rc!=SQLITE_OK ) return rc;
-
- n = fts3HashCount(pTerms);
- pData = sqlite3_malloc(n*sizeof(TermData));
-
- for(i = 0, e = fts3HashFirst(pTerms); e; i++, e = fts3HashNext(e)){
- assert( i<n );
- pData[i].pTerm = fts3HashKey(e);
- pData[i].nTerm = fts3HashKeysize(e);
- pData[i].pCollector = fts3HashData(e);
- }
- assert( i==n );
-
- /* TODO(shess) Should we allow user-defined collation sequences,
- ** here? I think we only need that once we support prefix searches.
- */
- if( n>1 ) qsort(pData, n, sizeof(*pData), termDataCmp);
-
- /* TODO(shess) Refactor so that we can write directly to the segment
- ** DataBuffer, as happens for segment merges.
- */
- leafWriterInit(0, idx, &writer);
- dataBufferInit(&dl, 0);
- for(i=0; i<n; i++){
- dataBufferReset(&dl);
- dlcAddDoclist(pData[i].pCollector, &dl);
- rc = leafWriterStep(v, &writer,
- pData[i].pTerm, pData[i].nTerm, dl.pData, dl.nData);
- if( rc!=SQLITE_OK ) goto err;
- }
- rc = leafWriterFinalize(v, &writer);
-
- err:
- dataBufferDestroy(&dl);
- sqlite3_free(pData);
- leafWriterDestroy(&writer);
- return rc;
-}
-
-/* If pendingTerms has data, free it. */
-static int clearPendingTerms(fulltext_vtab *v){
- if( v->nPendingData>=0 ){
- fts3HashElem *e;
- for(e=fts3HashFirst(&v->pendingTerms); e; e=fts3HashNext(e)){
- dlcDelete(fts3HashData(e));
- }
- fts3HashClear(&v->pendingTerms);
- v->nPendingData = -1;
- }
- return SQLITE_OK;
-}
-
-/* If pendingTerms has data, flush it to a level-zero segment, and
-** free it.
-*/
-static int flushPendingTerms(fulltext_vtab *v){
- if( v->nPendingData>=0 ){
- int rc = writeZeroSegment(v, &v->pendingTerms);
- if( rc==SQLITE_OK ) clearPendingTerms(v);
- return rc;
- }
- return SQLITE_OK;
-}
-
-/* If pendingTerms is "too big", or docid is out of order, flush it.
-** Regardless, be certain that pendingTerms is initialized for use.
-*/
-static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid){
- /* TODO(shess) Explore whether partially flushing the buffer on
- ** forced-flush would provide better performance. I suspect that if
- ** we ordered the doclists by size and flushed the largest until the
- ** buffer was half empty, that would let the less frequent terms
- ** generate longer doclists.
- */
- if( iDocid<=v->iPrevDocid || v->nPendingData>kPendingThreshold ){
- int rc = flushPendingTerms(v);
- if( rc!=SQLITE_OK ) return rc;
- }
- if( v->nPendingData<0 ){
- fts3HashInit(&v->pendingTerms, FTS3_HASH_STRING, 1);
- v->nPendingData = 0;
- }
- v->iPrevDocid = iDocid;
- return SQLITE_OK;
-}
-
-#if 0
-/* This function implements the xUpdate callback; it is the top-level entry
- * point for inserting, deleting or updating a row in a full-text table. */
-static int fulltextUpdate(sqlite3_vtab *pVtab, int nArg, sqlite3_value **ppArg,
- sqlite_int64 *pRowid){
- fulltext_vtab *v = (fulltext_vtab *) pVtab;
- int rc;
-
- FTSTRACE(("FTS3 Update %p\n", pVtab));
-
- if( nArg<2 ){
- rc = index_delete(v, sqlite3_value_int64(ppArg[0]));
- if( rc==SQLITE_OK ){
- /* If we just deleted the last row in the table, clear out the
- ** index data.
- */
- rc = content_exists(v);
- if( rc==SQLITE_ROW ){
- rc = SQLITE_OK;
- }else if( rc==SQLITE_DONE ){
- /* Clear the pending terms so we don't flush a useless level-0
- ** segment when the transaction closes.
- */
- rc = clearPendingTerms(v);
- if( rc==SQLITE_OK ){
- rc = segdir_delete_all(v);
- }
- }
- }
- } else if( sqlite3_value_type(ppArg[0]) != SQLITE_NULL ){
- /* An update:
- * ppArg[0] = old rowid
- * ppArg[1] = new rowid
- * ppArg[2..2+v->nColumn-1] = values
- * ppArg[2+v->nColumn] = value for magic column (we ignore this)
- * ppArg[2+v->nColumn+1] = value for docid
- */
- sqlite_int64 rowid = sqlite3_value_int64(ppArg[0]);
- if( sqlite3_value_type(ppArg[1]) != SQLITE_INTEGER ||
- sqlite3_value_int64(ppArg[1]) != rowid ){
- rc = SQLITE_ERROR; /* we don't allow changing the rowid */
- }else if( sqlite3_value_type(ppArg[2+v->nColumn+1]) != SQLITE_INTEGER ||
- sqlite3_value_int64(ppArg[2+v->nColumn+1]) != rowid ){
- rc = SQLITE_ERROR; /* we don't allow changing the docid */
- }else{
- assert( nArg==2+v->nColumn+2);
- rc = index_update(v, rowid, &ppArg[2]);
- }
- } else {
- /* An insert:
- * ppArg[1] = requested rowid
- * ppArg[2..2+v->nColumn-1] = values
- * ppArg[2+v->nColumn] = value for magic column (we ignore this)
- * ppArg[2+v->nColumn+1] = value for docid
- */
- sqlite3_value *pRequestDocid = ppArg[2+v->nColumn+1];
- assert( nArg==2+v->nColumn+2);
- if( SQLITE_NULL != sqlite3_value_type(pRequestDocid) &&
- SQLITE_NULL != sqlite3_value_type(ppArg[1]) ){
- /* TODO(shess) Consider allowing this to work if the values are
- ** identical. I'm inclined to discourage that usage, though,
- ** given that both rowid and docid are special columns. Better
- ** would be to define one or the other as the default winner,
- ** but should it be fts3-centric (docid) or SQLite-centric
- ** (rowid)?
- */
- rc = SQLITE_ERROR;
- }else{
- if( SQLITE_NULL == sqlite3_value_type(pRequestDocid) ){
- pRequestDocid = ppArg[1];
- }
- rc = index_insert(v, pRequestDocid, &ppArg[2], pRowid);
- }
- }
-
- return rc;
-}
-#endif
-
-static int fulltextSync(sqlite3_vtab *pVtab){
- fulltext_vtab *v = get_fulltext_vtab (pVtab);
-
- FTSTRACE(("FTS3 xSync()\n"));
- return flushPendingTerms(v);
-}
-
-static int fulltextBegin(sqlite3_vtab *pVtab){
- fulltext_vtab *v = get_fulltext_vtab (pVtab);
- FTSTRACE(("FTS3 xBegin()\n"));
-
- /* Any buffered updates should have been cleared by the previous
- ** transaction.
- */
- assert( v->nPendingData<0 );
- return clearPendingTerms(v);
-}
-
-static int fulltextCommit(sqlite3_vtab *pVtab){
- fulltext_vtab *v = get_fulltext_vtab (pVtab);
- FTSTRACE(("FTS3 xCommit()\n"));
-
- /* Buffered updates should have been cleared by fulltextSync(). */
- assert( v->nPendingData<0 );
- return clearPendingTerms(v);
-}
-
-static int fulltextRollback(sqlite3_vtab *pVtab){
- fulltext_vtab *v = get_fulltext_vtab (pVtab);
- FTSTRACE(("FTS3 xRollback()\n"));
- return clearPendingTerms(v);
-}
-
-/*
-** Implementation of the snippet() function for FTS3
-*/
-static void snippetFunc(
- sqlite3_context *pContext,
- int argc,
- sqlite3_value **argv
-){
- fulltext_cursor *pCursor;
- if( argc<1 ) return;
- if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
- sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
- sqlite3_result_error(pContext, "illegal first argument to html_snippet",-1);
- }else{
- const char *zStart = "<b>";
- const char *zEnd = "</b>";
- const char *zEllipsis = "<b>...</b>";
- memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
- if( argc>=2 ){
- zStart = (const char*)sqlite3_value_text(argv[1]);
- if( argc>=3 ){
- zEnd = (const char*)sqlite3_value_text(argv[2]);
- if( argc>=4 ){
- zEllipsis = (const char*)sqlite3_value_text(argv[3]);
- }
- }
- }
- snippetAllOffsets(pCursor);
- snippetText(pCursor, zStart, zEnd, zEllipsis);
- sqlite3_result_text(pContext, pCursor->snippet.zSnippet,
- pCursor->snippet.nSnippet, SQLITE_STATIC);
- }
-}
-
-
-/*
-** Implementation of the rank() function for FTS3
-*/
-static void rankFunc(
- sqlite3_context *pContext,
- int argc,
- sqlite3_value **argv
-){
-
- // TODO
-
- fulltext_cursor *pCursor;
- if( argc<1 ) return;
- if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
- sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
- sqlite3_result_error(pContext, "illegal first argument to html_snippet",-1);
- }else{
- memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
- sqlite3_result_double(pContext, pCursor->rank);
- }
-}
-
-/*
-** Implementation of the offsets() function for FTS3
-** altered by tracker to omit query term position as that
-** info is not stored in the poisiton data in the index
-*/
-static void snippetOffsetsFunc(
- sqlite3_context *pContext,
- int argc,
- sqlite3_value **argv
-){
- fulltext_cursor *pCursor;
- if( argc<1 ) return;
- if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
- sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
- sqlite3_result_error(pContext, "illegal first argument to offsets",-1);
- }else{
- memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
-
- /* (tracker) output caches position data in column, position string format */
- sqlite3_result_text(pContext,
- pCursor->offsets->str, pCursor->offsets->len,
- SQLITE_STATIC);
- }
-}
-
-/* OptLeavesReader is nearly identical to LeavesReader, except that
-** where LeavesReader is geared towards the merging of complete
-** segment levels (with exactly MERGE_COUNT segments), OptLeavesReader
-** is geared towards implementation of the optimize() function, and
-** can merge all segments simultaneously. This version may be
-** somewhat less efficient than LeavesReader because it merges into an
-** accumulator rather than doing an N-way merge, but since segment
-** size grows exponentially (so segment count logrithmically) this is
-** probably not an immediate problem.
-*/
-/* TODO(shess): Prove that assertion, or extend the merge code to
-** merge tree fashion (like the prefix-searching code does).
-*/
-/* TODO(shess): OptLeavesReader and LeavesReader could probably be
-** merged with little or no loss of performance for LeavesReader. The
-** merged code would need to handle >MERGE_COUNT segments, and would
-** also need to be able to optionally optimize away deletes.
-*/
-typedef struct OptLeavesReader {
- /* Segment number, to order readers by age. */
- int segment;
- LeavesReader reader;
-} OptLeavesReader;
-
-static int optLeavesReaderAtEnd(OptLeavesReader *pReader){
- return leavesReaderAtEnd(&pReader->reader);
-}
-static int optLeavesReaderTermBytes(OptLeavesReader *pReader){
- return leavesReaderTermBytes(&pReader->reader);
-}
-static const char *optLeavesReaderData(OptLeavesReader *pReader){
- return leavesReaderData(&pReader->reader);
-}
-static int optLeavesReaderDataBytes(OptLeavesReader *pReader){
- return leavesReaderDataBytes(&pReader->reader);
-}
-static const char *optLeavesReaderTerm(OptLeavesReader *pReader){
- return leavesReaderTerm(&pReader->reader);
-}
-static int optLeavesReaderStep(fulltext_vtab *v, OptLeavesReader *pReader){
- return leavesReaderStep(v, &pReader->reader);
-}
-static int optLeavesReaderTermCmp(OptLeavesReader *lr1, OptLeavesReader *lr2){
- return leavesReaderTermCmp(&lr1->reader, &lr2->reader);
-}
-/* Order by term ascending, segment ascending (oldest to newest), with
-** exhausted readers to the end.
-*/
-static int optLeavesReaderCmp(OptLeavesReader *lr1, OptLeavesReader *lr2){
- int c = optLeavesReaderTermCmp(lr1, lr2);
- if( c!=0 ) return c;
- return lr1->segment-lr2->segment;
-}
-/* Bubble pLr[0] to appropriate place in pLr[1..nLr-1]. Assumes that
-** pLr[1..nLr-1] is already sorted.
-*/
-static void optLeavesReaderReorder(OptLeavesReader *pLr, int nLr){
- while( nLr>1 && optLeavesReaderCmp(pLr, pLr+1)>0 ){
- OptLeavesReader tmp = pLr[0];
- pLr[0] = pLr[1];
- pLr[1] = tmp;
- nLr--;
- pLr++;
- }
-}
-
-/* optimize() helper function. Put the readers in order and iterate
-** through them, merging doclists for matching terms into pWriter.
-** Returns SQLITE_OK on success, or the SQLite error code which
-** prevented success.
-*/
-static int optimizeInternal(fulltext_vtab *v,
- OptLeavesReader *readers, int nReaders,
- LeafWriter *pWriter){
- int i, rc = SQLITE_OK;
- DataBuffer doclist, merged, tmp;
-
- /* Order the readers. */
- i = nReaders;
- while( i-- > 0 ){
- optLeavesReaderReorder(&readers[i], nReaders-i);
- }
-
- dataBufferInit(&doclist, LEAF_MAX);
- dataBufferInit(&merged, LEAF_MAX);
-
- /* Exhausted readers bubble to the end, so when the first reader is
- ** at eof, all are at eof.
- */
- while( !optLeavesReaderAtEnd(&readers[0]) ){
-
- /* Figure out how many readers share the next term. */
- for(i=1; i<nReaders && !optLeavesReaderAtEnd(&readers[i]); i++){
- if( 0!=optLeavesReaderTermCmp(&readers[0], &readers[i]) ) break;
- }
-
- /* Special-case for no merge. */
- if( i==1 ){
- /* Trim deletions from the doclist. */
- dataBufferReset(&merged);
- docListTrim(DL_DEFAULT,
- optLeavesReaderData(&readers[0]),
- optLeavesReaderDataBytes(&readers[0]),
- -1, DL_DEFAULT, &merged);
- }else{
- DLReader dlReaders[MERGE_COUNT];
- int iReader, pnReaders;
-
- /* Prime the pipeline with the first reader's doclist. After
- ** one pass index 0 will reference the accumulated doclist.
- */
- dlrInit(&dlReaders[0], DL_DEFAULT,
- optLeavesReaderData(&readers[0]),
- optLeavesReaderDataBytes(&readers[0]));
- iReader = 1;
-
- assert( iReader<i ); /* Must execute the loop at least once. */
- while( iReader<i ){
- /* Merge 16 inputs per pass. */
- for( pnReaders=1; iReader<i && pnReaders<MERGE_COUNT;
- iReader++, pnReaders++ ){
- dlrInit(&dlReaders[pnReaders], DL_DEFAULT,
- optLeavesReaderData(&readers[iReader]),
- optLeavesReaderDataBytes(&readers[iReader]));
- }
-
- /* Merge doclists and swap result into accumulator. */
- dataBufferReset(&merged);
- docListMerge(&merged, dlReaders, pnReaders);
- tmp = merged;
- merged = doclist;
- doclist = tmp;
-
- while( pnReaders-- > 0 ){
- dlrDestroy(&dlReaders[pnReaders]);
- }
-
- /* Accumulated doclist to reader 0 for next pass. */
- dlrInit(&dlReaders[0], DL_DEFAULT, doclist.pData, doclist.nData);
- }
-
- /* Destroy reader that was left in the pipeline. */
- dlrDestroy(&dlReaders[0]);
-
- /* Trim deletions from the doclist. */
- dataBufferReset(&merged);
- docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
- -1, DL_DEFAULT, &merged);
- }
-
- /* Only pass doclists with hits (skip if all hits deleted). */
- if( merged.nData>0 ){
- rc = leafWriterStep(v, pWriter,
- optLeavesReaderTerm(&readers[0]),
- optLeavesReaderTermBytes(&readers[0]),
- merged.pData, merged.nData);
- if( rc!=SQLITE_OK ) goto err;
- }
-
- /* Step merged readers to next term and reorder. */
- while( i-- > 0 ){
- rc = optLeavesReaderStep(v, &readers[i]);
- if( rc!=SQLITE_OK ) goto err;
-
- optLeavesReaderReorder(&readers[i], nReaders-i);
- }
- }
-
- err:
- dataBufferDestroy(&doclist);
- dataBufferDestroy(&merged);
- return rc;
-}
-
-/* Implement optimize() function for FTS3. optimize(t) merges all
-** segments in the fts index into a single segment. 't' is the magic
-** table-named column.
-*/
-static void optimizeFunc(sqlite3_context *pContext,
- int argc, sqlite3_value **argv){
- fulltext_cursor *pCursor;
- if( argc>1 ){
- sqlite3_result_error(pContext, "excess arguments to optimize()",-1);
- }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
- sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
- sqlite3_result_error(pContext, "illegal first argument to optimize",-1);
- }else{
- fulltext_vtab *v;
- int i, rc, iMaxLevel = 0;
- OptLeavesReader *readers;
- int nReaders = 0;
- LeafWriter writer;
- sqlite3_stmt *s;
-
- memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
- v = cursor_vtab(pCursor);
-
- /* Flush any buffered updates before optimizing. */
- rc = flushPendingTerms(v);
- if( rc!=SQLITE_OK ) goto err;
-
- rc = segdir_count(v, &nReaders, &iMaxLevel);
- if( rc!=SQLITE_OK ) goto err;
- if( nReaders==0 || nReaders==1 ){
- sqlite3_result_text(pContext, "Index already optimal", -1,
- SQLITE_STATIC);
- return;
- }
-
- rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
- if( rc!=SQLITE_OK ) goto err;
-
- readers = sqlite3_malloc(nReaders*sizeof(readers[0]));
- if( readers==NULL ) goto err;
-
- /* Note that there will already be a segment at this position
- ** until we call segdir_delete() on iMaxLevel.
- */
- leafWriterInit(iMaxLevel, 0, &writer);
-
- i = 0;
- while( (rc = sqlite3_step(s))==SQLITE_ROW ){
- sqlite_int64 iStart = sqlite3_column_int64(s, 0);
- sqlite_int64 iEnd = sqlite3_column_int64(s, 1);
- const char *pRootData = sqlite3_column_blob(s, 2);
- int nRootData = sqlite3_column_bytes(s, 2);
-
- assert( i<nReaders );
- rc = leavesReaderInit(v, -1, iStart, iEnd, pRootData, nRootData,
- &readers[i].reader);
- if( rc!=SQLITE_OK ) break;
-
- readers[i].segment = i;
- i++;
- }
-
- /* If we managed to successfully read them all, optimize them. */
- if( rc==SQLITE_DONE ){
- assert( i==nReaders );
- rc = optimizeInternal(v, readers, nReaders, &writer);
- }
-
- while( i-- > 0 ){
- leavesReaderDestroy(&readers[i].reader);
- }
- sqlite3_free(readers);
-
- /* If we've successfully gotten to here, delete the old segments
- ** and flush the interior structure of the new segment.
- */
- if( rc==SQLITE_OK ){
- for( i=0; i<=iMaxLevel; i++ ){
- rc = segdir_delete(v, i);
- if( rc!=SQLITE_OK ) break;
- }
-
- if( rc==SQLITE_OK ) rc = leafWriterFinalize(v, &writer);
- }
-
- leafWriterDestroy(&writer);
-
- if( rc!=SQLITE_OK ) goto err;
-
- sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC);
- return;
-
- /* TODO(shess): Error-handling needs to be improved along the
- ** lines of the dump_ functions.
- */
- err:
- {
- char buf[512];
- sqlite3_snprintf(sizeof(buf), buf, "Error in optimize: %s",
- sqlite3_errmsg(sqlite3_context_db_handle(pContext)));
- sqlite3_result_error(pContext, buf, -1);
- }
- }
-}
-
-#ifdef SQLITE_TEST
-/* Generate an error of the form "<prefix>: <msg>". If msg is NULL,
-** pull the error from the context's db handle.
-*/
-static void generateError(sqlite3_context *pContext,
- const char *prefix, const char *msg){
- char buf[512];
- if( msg==NULL ) msg = sqlite3_errmsg(sqlite3_context_db_handle(pContext));
- sqlite3_snprintf(sizeof(buf), buf, "%s: %s", prefix, msg);
- sqlite3_result_error(pContext, buf, -1);
-}
-
-/* Helper function to collect the set of terms in the segment into
-** pTerms. The segment is defined by the leaf nodes between
-** iStartBlockid and iEndBlockid, inclusive, or by the contents of
-** pRootData if iStartBlockid is 0 (in which case the entire segment
-** fit in a leaf).
-*/
-static int collectSegmentTerms(fulltext_vtab *v, sqlite3_stmt *s,
- fts3Hash *pTerms){
- const sqlite_int64 iStartBlockid = sqlite3_column_int64(s, 0);
- const sqlite_int64 iEndBlockid = sqlite3_column_int64(s, 1);
- const char *pRootData = sqlite3_column_blob(s, 2);
- const int nRootData = sqlite3_column_bytes(s, 2);
- LeavesReader reader;
- int rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid,
- pRootData, nRootData, &reader);
- if( rc!=SQLITE_OK ) return rc;
-
- while( rc==SQLITE_OK && !leavesReaderAtEnd(&reader) ){
- const char *pTerm = leavesReaderTerm(&reader);
- const int nTerm = leavesReaderTermBytes(&reader);
- void *oldValue = sqlite3Fts3HashFind(pTerms, pTerm, nTerm);
- void *newValue = (void *)((char *)oldValue+1);
-
- /* From the comment before sqlite3Fts3HashInsert in fts3_hash.c,
- ** the data value passed is returned in case of malloc failure.
- */
- if( newValue==sqlite3Fts3HashInsert(pTerms, pTerm, nTerm, newValue) ){
- rc = SQLITE_NOMEM;
- }else{
- rc = leavesReaderStep(v, &reader);
- }
- }
-
- leavesReaderDestroy(&reader);
- return rc;
-}
-
-/* Helper function to build the result string for dump_terms(). */
-static int generateTermsResult(sqlite3_context *pContext, fts3Hash *pTerms){
- int iTerm, nTerms, nResultBytes, iByte;
- char *result;
- TermData *pData;
- fts3HashElem *e;
-
- /* Iterate pTerms to generate an array of terms in pData for
- ** sorting.
- */
- nTerms = fts3HashCount(pTerms);
- assert( nTerms>0 );
- pData = sqlite3_malloc(nTerms*sizeof(TermData));
- if( pData==NULL ) return SQLITE_NOMEM;
-
- nResultBytes = 0;
- for(iTerm = 0, e = fts3HashFirst(pTerms); e; iTerm++, e = fts3HashNext(e)){
- nResultBytes += fts3HashKeysize(e)+1; /* Term plus trailing space */
- assert( iTerm<nTerms );
- pData[iTerm].pTerm = fts3HashKey(e);
- pData[iTerm].nTerm = fts3HashKeysize(e);
- pData[iTerm].pCollector = fts3HashData(e); /* unused */
- }
- assert( iTerm==nTerms );
-
- assert( nResultBytes>0 ); /* nTerms>0, nResultsBytes must be, too. */
- result = sqlite3_malloc(nResultBytes);
- if( result==NULL ){
- sqlite3_free(pData);
- return SQLITE_NOMEM;
- }
-
- if( nTerms>1 ) qsort(pData, nTerms, sizeof(*pData), termDataCmp);
-
- /* Read the terms in order to build the result. */
- iByte = 0;
- for(iTerm=0; iTerm<nTerms; ++iTerm){
- memcpy(result+iByte, pData[iTerm].pTerm, pData[iTerm].nTerm);
- iByte += pData[iTerm].nTerm;
- result[iByte++] = ' ';
- }
- assert( iByte==nResultBytes );
- assert( result[nResultBytes-1]==' ' );
- result[nResultBytes-1] = '\0';
-
- /* Passes away ownership of result. */
- sqlite3_result_text(pContext, result, nResultBytes-1, sqlite3_free);
- sqlite3_free(pData);
- return SQLITE_OK;
-}
-
-/* Implements dump_terms() for use in inspecting the fts3 index from
-** tests. TEXT result containing the ordered list of terms joined by
-** spaces. dump_terms(t, level, idx) dumps the terms for the segment
-** specified by level, idx (in %_segdir), while dump_terms(t) dumps
-** all terms in the index. In both cases t is the fts table's magic
-** table-named column.
-*/
-static void dumpTermsFunc(
- sqlite3_context *pContext,
- int argc, sqlite3_value **argv
-){
- fulltext_cursor *pCursor;
- if( argc!=3 && argc!=1 ){
- generateError(pContext, "dump_terms", "incorrect arguments");
- }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
- sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
- generateError(pContext, "dump_terms", "illegal first argument");
- }else{
- fulltext_vtab *v;
- fts3Hash terms;
- sqlite3_stmt *s = NULL;
- int rc;
-
- memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
- v = cursor_vtab(pCursor);
-
- /* If passed only the cursor column, get all segments. Otherwise
- ** get the segment described by the following two arguments.
- */
- if( argc==1 ){
- rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
- }else{
- rc = sql_get_statement(v, SEGDIR_SELECT_SEGMENT_STMT, &s);
- if( rc==SQLITE_OK ){
- rc = sqlite3_bind_int(s, 1, sqlite3_value_int(argv[1]));
- if( rc==SQLITE_OK ){
- rc = sqlite3_bind_int(s, 2, sqlite3_value_int(argv[2]));
- }
- }
- }
-
- if( rc!=SQLITE_OK ){
- generateError(pContext, "dump_terms", NULL);
- return;
- }
-
- /* Collect the terms for each segment. */
- sqlite3Fts3HashInit(&terms, FTS3_HASH_STRING, 1);
- while( (rc = sqlite3_step(s))==SQLITE_ROW ){
- rc = collectSegmentTerms(v, s, &terms);
- if( rc!=SQLITE_OK ) break;
- }
-
- if( rc!=SQLITE_DONE ){
- sqlite3_reset(s);
- generateError(pContext, "dump_terms", NULL);
- }else{
- const int nTerms = fts3HashCount(&terms);
- if( nTerms>0 ){
- rc = generateTermsResult(pContext, &terms);
- if( rc==SQLITE_NOMEM ){
- generateError(pContext, "dump_terms", "out of memory");
- }else{
- assert( rc==SQLITE_OK );
- }
- }else if( argc==3 ){
- /* The specific segment asked for could not be found. */
- generateError(pContext, "dump_terms", "segment not found");
- }else{
- /* No segments found. */
- /* TODO(shess): It should be impossible to reach this. This
- ** case can only happen for an empty table, in which case
- ** SQLite has no rows to call this function on.
- */
- sqlite3_result_null(pContext);
- }
- }
- sqlite3Fts3HashClear(&terms);
- }
-}
-
-/* Expand the DL_DEFAULT doclist in pData into a text result in
-** pContext.
-*/
-static void createDoclistResult(sqlite3_context *pContext,
- const char *pData, int nData){
- DataBuffer dump;
- DLReader dlReader;
-
- assert( pData!=NULL && nData>0 );
-
- dataBufferInit(&dump, 0);
- dlrInit(&dlReader, DL_DEFAULT, pData, nData);
- for( ; !dlrAtEnd(&dlReader); dlrStep(&dlReader) ){
- char buf[256];
- PLReader plReader;
-
- plrInit(&plReader, &dlReader);
- if( DL_DEFAULT==DL_DOCIDS || plrAtEnd(&plReader) ){
- sqlite3_snprintf(sizeof(buf), buf, "[%lld] ", dlrDocid(&dlReader));
- dataBufferAppend(&dump, buf, strlen(buf));
- }else{
- int iColumn = plrColumn(&plReader);
-
- sqlite3_snprintf(sizeof(buf), buf, "[%lld %d[",
- dlrDocid(&dlReader), iColumn);
- dataBufferAppend(&dump, buf, strlen(buf));
-
- for( ; !plrAtEnd(&plReader); plrStep(&plReader) ){
- if( plrColumn(&plReader)!=iColumn ){
- iColumn = plrColumn(&plReader);
- sqlite3_snprintf(sizeof(buf), buf, "] %d[", iColumn);
- assert( dump.nData>0 );
- dump.nData--; /* Overwrite trailing space. */
- assert( dump.pData[dump.nData]==' ');
- dataBufferAppend(&dump, buf, strlen(buf));
- }
- if( DL_DEFAULT==DL_POSITIONS_OFFSETS ){
- sqlite3_snprintf(sizeof(buf), buf, "%d,%d,%d ",
- plrPosition(&plReader),
- plrStartOffset(&plReader), plrEndOffset(&plReader));
- }else if( DL_DEFAULT==DL_POSITIONS ){
- sqlite3_snprintf(sizeof(buf), buf, "%d ", plrPosition(&plReader));
- }else{
- assert( NULL=="Unhandled DL_DEFAULT value");
- }
- dataBufferAppend(&dump, buf, strlen(buf));
- }
- plrDestroy(&plReader);
-
- assert( dump.nData>0 );
- dump.nData--; /* Overwrite trailing space. */
- assert( dump.pData[dump.nData]==' ');
- dataBufferAppend(&dump, "]] ", 3);
- }
- }
- dlrDestroy(&dlReader);
-
- assert( dump.nData>0 );
- dump.nData--; /* Overwrite trailing space. */
- assert( dump.pData[dump.nData]==' ');
- dump.pData[dump.nData] = '\0';
- assert( dump.nData>0 );
-
- /* Passes ownership of dump's buffer to pContext. */
- sqlite3_result_text(pContext, dump.pData, dump.nData, sqlite3_free);
- dump.pData = NULL;
- dump.nData = dump.nCapacity = 0;
-}
-
-/* Implements dump_doclist() for use in inspecting the fts3 index from
-** tests. TEXT result containing a string representation of the
-** doclist for the indicated term. dump_doclist(t, term, level, idx)
-** dumps the doclist for term from the segment specified by level, idx
-** (in %_segdir), while dump_doclist(t, term) dumps the logical
-** doclist for the term across all segments. The per-segment doclist
-** can contain deletions, while the full-index doclist will not
-** (deletions are omitted).
-**
-** Result formats differ with the setting of DL_DEFAULTS. Examples:
-**
-** DL_DOCIDS: [1] [3] [7]
-** DL_POSITIONS: [1 0[0 4] 1[17]] [3 1[5]]
-** DL_POSITIONS_OFFSETS: [1 0[0,0,3 4,23,26] 1[17,102,105]] [3 1[5,20,23]]
-**
-** In each case the number after the outer '[' is the docid. In the
-** latter two cases, the number before the inner '[' is the column
-** associated with the values within. For DL_POSITIONS the numbers
-** within are the positions, for DL_POSITIONS_OFFSETS they are the
-** position, the start offset, and the end offset.
-*/
-static void dumpDoclistFunc(
- sqlite3_context *pContext,
- int argc, sqlite3_value **argv
-){
- fulltext_cursor *pCursor;
- if( argc!=2 && argc!=4 ){
- generateError(pContext, "dump_doclist", "incorrect arguments");
- }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
- sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
- generateError(pContext, "dump_doclist", "illegal first argument");
- }else if( sqlite3_value_text(argv[1])==NULL ||
- sqlite3_value_text(argv[1])[0]=='\0' ){
- generateError(pContext, "dump_doclist", "empty second argument");
- }else{
- const char *pTerm = (const char *)sqlite3_value_text(argv[1]);
- const int nTerm = strlen(pTerm);
- fulltext_vtab *v;
- int rc;
- DataBuffer doclist;
-
- memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
- v = cursor_vtab(pCursor);
-
- dataBufferInit(&doclist, 0);
-
- /* termSelect() yields the same logical doclist that queries are
- ** run against.
- */
- if( argc==2 ){
- rc = termSelect(v, v->nColumn, pTerm, nTerm, 0, DL_DEFAULT, &doclist);
- }else{
- sqlite3_stmt *s = NULL;
-
- /* Get our specific segment's information. */
- rc = sql_get_statement(v, SEGDIR_SELECT_SEGMENT_STMT, &s);
- if( rc==SQLITE_OK ){
- rc = sqlite3_bind_int(s, 1, sqlite3_value_int(argv[2]));
- if( rc==SQLITE_OK ){
- rc = sqlite3_bind_int(s, 2, sqlite3_value_int(argv[3]));
- }
- }
-
- if( rc==SQLITE_OK ){
- rc = sqlite3_step(s);
-
- if( rc==SQLITE_DONE ){
- dataBufferDestroy(&doclist);
- generateError(pContext, "dump_doclist", "segment not found");
- return;
- }
-
- /* Found a segment, load it into doclist. */
- if( rc==SQLITE_ROW ){
- const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1);
- const char *pData = sqlite3_column_blob(s, 2);
- const int nData = sqlite3_column_bytes(s, 2);
-
- /* loadSegment() is used by termSelect() to load each
- ** segment's data.
- */
- rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, 0,
- &doclist);
- if( rc==SQLITE_OK ){
- rc = sqlite3_step(s);
-
- /* Should not have more than one matching segment. */
- if( rc!=SQLITE_DONE ){
- sqlite3_reset(s);
- dataBufferDestroy(&doclist);
- generateError(pContext, "dump_doclist", "invalid segdir");
- return;
- }
- rc = SQLITE_OK;
- }
- }
- }
-
- sqlite3_reset(s);
- }
-
- if( rc==SQLITE_OK ){
- if( doclist.nData>0 ){
- createDoclistResult(pContext, doclist.pData, doclist.nData);
- }else{
- /* TODO(shess): This can happen if the term is not present, or
- ** if all instances of the term have been deleted and this is
- ** an all-index dump. It may be interesting to distinguish
- ** these cases.
- */
- sqlite3_result_text(pContext, "", 0, SQLITE_STATIC);
- }
- }else if( rc==SQLITE_NOMEM ){
- /* Handle out-of-memory cases specially because if they are
- ** generated in fts3 code they may not be reflected in the db
- ** handle.
- */
- /* TODO(shess): Handle this more comprehensively.
- ** sqlite3ErrStr() has what I need, but is internal.
- */
- generateError(pContext, "dump_doclist", "out of memory");
- }else{
- generateError(pContext, "dump_doclist", NULL);
- }
-
- dataBufferDestroy(&doclist);
- }
-}
-#endif
-
-/*
-** This routine implements the xFindFunction method for the FTS3
-** virtual table.
-*/
-static int fulltextFindFunction(
- sqlite3_vtab *pVtab,
- int nArg,
- const char *zName,
- void (**pxFunc)(sqlite3_context*,int,sqlite3_value**),
- void **ppArg
-){
- if( strcmp(zName,"snippet")==0 ){
- *pxFunc = snippetFunc;
- return 1;
- }else if( strcmp(zName,"offsets")==0 ){
- *pxFunc = snippetOffsetsFunc;
- return 1;
- }else if( strcmp(zName,"rank")==0 ){
- *pxFunc = rankFunc;
- return 1;
- }else if( strcmp(zName,"optimize")==0 ){
- *pxFunc = optimizeFunc;
- return 1;
-#ifdef SQLITE_TEST
- /* NOTE(shess): These functions are present only for testing
- ** purposes. No particular effort is made to optimize their
- ** execution or how they build their results.
- */
- }else if( strcmp(zName,"dump_terms")==0 ){
- /* fprintf(stderr, "Found dump_terms\n"); */
- *pxFunc = dumpTermsFunc;
- return 1;
- }else if( strcmp(zName,"dump_doclist")==0 ){
- /* fprintf(stderr, "Found dump_doclist\n"); */
- *pxFunc = dumpDoclistFunc;
- return 1;
-#endif
- }
- return 0;
-}
-
-/*
-** Rename an fts3 table.
-*/
-static int fulltextRename(
- sqlite3_vtab *pVtab,
- const char *zName
-){
- fulltext_vtab *p = get_fulltext_vtab (pVtab);
- int rc = SQLITE_NOMEM;
- char *zSql = sqlite3_mprintf(
- "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';"
- "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';"
- "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';"
- , p->zDb, p->zName, zName
- , p->zDb, p->zName, zName
- , p->zDb, p->zName, zName
- );
- if( zSql ){
- rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
- sqlite3_free(zSql);
- }
- return rc;
-}
-
-static const sqlite3_module fts3Module = {
- /* iVersion */ 0,
- /* xCreate */ fulltextCreate,
- /* xConnect */ fulltextConnect,
- /* xBestIndex */ fulltextBestIndex,
- /* xDisconnect */ fulltextDisconnect,
- /* xDestroy */ fulltextDestroy,
- /* xOpen */ fulltextOpen,
- /* xClose */ fulltextClose,
- /* xFilter */ fulltextFilter,
- /* xNext */ fulltextNext,
- /* xEof */ fulltextEof,
- /* xColumn */ fulltextColumn,
- /* xRowid */ fulltextRowid,
- /* xUpdate */ NULL,
- /* xBegin */ fulltextBegin,
- /* xSync */ fulltextSync,
- /* xCommit */ fulltextCommit,
- /* xRollback */ fulltextRollback,
- /* xFindFunction */ fulltextFindFunction,
- /* xRename */ fulltextRename,
-};
-
-int sqlite3Fts3InitHashTable(sqlite3 *, fts3Hash *, const char *);
-
-/*
-** Initialise the fts3 extension. If this extension is built as part
-** of the sqlite library, then this function is called directly by
-** SQLite. If fts3 is built as a dynamically loadable extension, this
-** function is called by the sqlite3_extension_init() entry point.
-*/
-TrackerFts *tracker_fts_new(sqlite3 *db, int create){
- int rc = SQLITE_OK;
- fulltext_vtab *v;
-
- if (create){
- createTables (db, "main", "fts");
- }
-
- v = constructVtab(db, "main", "fts", NULL);
-
- if (v == NULL) {
- return NULL;
- }
-
- /* Create the virtual table wrapper around the hash-table and overload
- ** the two scalar functions. If this is successful, register the
- ** module with sqlite.
- */
- if( SQLITE_OK==rc
- && SQLITE_OK==(rc = sqlite3_overload_function(db, "rank", -1))
- && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
- && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", -1))
- && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", -1))
-#ifdef SQLITE_TEST
- && SQLITE_OK==(rc = sqlite3_overload_function(db, "dump_terms", -1))
- && SQLITE_OK==(rc = sqlite3_overload_function(db, "dump_doclist", -1))
-#endif
- ){
- rc = sqlite3_create_module_v2(
- db, "trackerfts", &fts3Module, v, NULL
- );
- if (SQLITE_OK != rc){
- fulltext_vtab_destroy (v);
- return NULL;
- }
-
- if (create){
- rc = sqlite3_exec(db, "CREATE VIRTUAL TABLE fts USING trackerfts", NULL, 0, NULL);
- }
- }
-
- if (SQLITE_OK != rc){
- fulltext_vtab_destroy (v);
- return NULL;
- }
-
- return v;
-}
-
-void tracker_fts_free (TrackerFts *fts){
- fulltext_vtab_destroy (fts);
-}
-
-int tracker_fts_update_init(TrackerFts *fts, int id){
- return initPendingTerms(fts, id);
-}
-
-int tracker_fts_update_text(TrackerFts *fts, int id, int column_id,
- const char *text, gboolean limit_word_length){
- return buildTerms(fts, id, text, column_id, limit_word_length);
-}
-
-void tracker_fts_update_commit(TrackerFts *fts){
- flushPendingTerms(fts);
- clearPendingTerms(fts);
-}
-
-void tracker_fts_update_rollback(TrackerFts *fts){
- clearPendingTerms(fts);
-}
-
diff --git a/src/libtracker-fts/tracker-fts.h b/src/libtracker-fts/tracker-fts.h
index ebc0266..a1beb66 100644
--- a/src/libtracker-fts/tracker-fts.h
+++ b/src/libtracker-fts/tracker-fts.h
@@ -1,44 +1,35 @@
/*
-** 2006 Oct 10
-**
-** The author disclaims copyright to this source code. In place of
-** a legal notice, here is a blessing:
-**
-** May you do good and not evil.
-** May you find forgiveness for yourself and forgive others.
-** May you share freely, never taking more than you give.
-**
-******************************************************************************
-**
-** This header file is used by programs that want to link against the
-** FTS3 library.
-*/
+ * Copyright (C) 2011 Nokia <ivan frade nokia com>
+ *
+ * Author: Carlos Garnacho <carlos lanedo com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA
+ */
#ifndef __TRACKER_FTS_H__
#define __TRACKER_FTS_H__
#include <sqlite3.h>
#include <glib.h>
-#include <glib-object.h>
G_BEGIN_DECLS
-typedef const gchar *(*TrackerFtsMapFunc) (gint id);
-
-typedef struct fulltext_vtab TrackerFts;
-
-TrackerFts *tracker_fts_new (sqlite3 *db,
- int create);
-void tracker_fts_free (TrackerFts *fts);
-int tracker_fts_update_init (TrackerFts *fts,
- int id);
-int tracker_fts_update_text (TrackerFts *fts,
- int id,
- int column_id,
- const char *text,
- gboolean limit_word_length);
-void tracker_fts_update_commit (TrackerFts *fts);
-void tracker_fts_update_rollback (TrackerFts *fts);
+gboolean tracker_fts_init (void);
+gboolean tracker_fts_init_db (sqlite3 *db,
+ int create);
G_END_DECLS
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
