Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Reorganize comments in fts2_tokenizer.h. No code changes. (CVS 4132) |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
b331e30395e9fc90abe40ab802972a67 |
User & Date: | danielk1977 2007-06-26 12:54:07.000 |
Context
2007-06-26
| ||
22:10 | Remove vestigal code from Makefile.linux-gcc. Ticket #2466. (CVS 4133) (check-in: 338c187910 user: drh tags: trunk) | |
12:54 | Reorganize comments in fts2_tokenizer.h. No code changes. (CVS 4132) (check-in: b331e30395 user: danielk1977 tags: trunk) | |
12:52 | Use (((i64)1)<<63) instead of just (1<<63) to compute the most negative 64-bit integer. (CVS 4131) (check-in: be2570c061 user: danielk1977 tags: trunk) | |
Changes
Changes to ext/fts2/fts2_tokenizer.h.
︙ | ︙ | |||
23 24 25 26 27 28 29 | /* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. ** If tokenizers are to be allowed to call sqlite3_*() functions, then ** we will need a way to register the API consistently. */ #include "sqlite3.h" /* | | > > > > > > > > > > > > > > > > | | | > > > > | | > > | > > > > > > > > > > > | > > | > > > > > > > > > > > | > > > | > > > > | | > > > > > > > > > > > > > > | | | < | < < | | > > > | 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 | /* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. ** If tokenizers are to be allowed to call sqlite3_*() functions, then ** we will need a way to register the API consistently. */ #include "sqlite3.h" /* ** Structures used by the tokenizer interface. When a new tokenizer ** implementation is registered, the caller provides a pointer to ** an sqlite3_tokenizer_module containing pointers to the callback ** functions that make up an implementation. ** ** When an fts2 table is created, it passes any arguments passed to ** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the ** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer ** implementation. The xCreate() function in turn returns an ** sqlite3_tokenizer structure representing the specific tokenizer to ** be used for the fts2 table (customized by the tokenizer clause arguments). ** ** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen() ** method is called. It returns an sqlite3_tokenizer_cursor object ** that may be used to tokenize a specific input buffer based on ** the tokenization rules supplied by a specific sqlite3_tokenizer ** object. */ typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module; typedef struct sqlite3_tokenizer sqlite3_tokenizer; typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; struct sqlite3_tokenizer_module { /* ** Structure version. Should always be set to 0. */ int iVersion; /* ** Create a new tokenizer. The values in the argv[] array are the ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL ** TABLE statement that created the fts2 table. For example, if ** the following SQL is executed: ** ** CREATE .. USING fts2( ... , tokenizer <tokenizer-name> arg1 arg2) ** ** then argc is set to 2, and the argv[] array contains pointers ** to the strings "arg1" and "arg2". ** ** This method should return either SQLITE_OK (0), or an SQLite error ** code. If SQLITE_OK is returned, then *ppTokenizer should be set ** to point at the newly created tokenizer structure. The generic ** sqlite3_tokenizer.pModule variable should not be initialised by ** this callback. The caller will do so. */ int (*xCreate)( int argc, /* Size of argv array */ const char *const*argv, /* Tokenizer argument strings */ sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ ); /* ** Destroy an existing tokenizer. The fts2 module calls this method ** exactly once for each successful call to xCreate(). */ int (*xDestroy)(sqlite3_tokenizer *pTokenizer); /* ** Create a tokenizer cursor to tokenize an input buffer. The caller ** is responsible for ensuring that the input buffer remains valid ** until the cursor is closed (using the xClose() method). */ int (*xOpen)( sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ const char *pInput, int nBytes, /* Input buffer */ sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */ ); /* ** Destroy an existing tokenizer cursor. The fts2 module calls this ** method exactly once for each successful call to xOpen(). */ int (*xClose)(sqlite3_tokenizer_cursor *pCursor); /* ** Retrieve the next token from the tokenizer cursor pCursor. This ** method should either return SQLITE_OK and set the values of the ** "OUT" variables identified below, or SQLITE_DONE to indicate that ** the end of the buffer has been reached, or an SQLite error code. ** ** *ppToken should be set to point at a buffer containing the ** normalized version of the token (i.e. after any case-folding and/or ** stemming has been performed). *pnBytes should be set to the length ** of this buffer in bytes. The input text that generated the token is ** identified by the byte offsets returned in *piStartOffset and ** *piEndOffset. ** ** The buffer *ppToken is set to point at is managed by the tokenizer ** implementation. It is only required to be valid until the next call ** to xNext() or xClose(). */ /* TODO(shess) current implementation requires pInput to be ** nul-terminated. This should either be fixed, or pInput/nBytes ** should be converted to zInput. */ int (*xNext)( sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */ const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */ int *piStartOffset, /* OUT: Byte offset of token in input buffer */ int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ int *piPosition /* OUT: Number of tokens returned before this one */ ); }; struct sqlite3_tokenizer { const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ /* Tokenizer implementations will typically add additional fields */ }; |
︙ | ︙ |