Difference between revisions of "Full-Text Module"
m (Text replacement - "<br />" to "<br/>") |
m (Text replacement - "<syntaxhighlight lang="xquery">" to "<pre lang='xquery'>") Tags: Mobile web edit Mobile edit |
||
(5 intermediate revisions by the same user not shown) | |||
Line 11: | Line 11: | ||
{| width='100%' | {| width='100%' | ||
|- valign="top" | |- valign="top" | ||
− | | width='120' | ''' | + | | width='120' | '''Signature''' |
− | | | + | |<pre>ft:search( |
+ | $db as xs:string, | ||
+ | $terms as item()*, | ||
+ | $options as map(*)? := map { } | ||
+ | ) as text()*</pre> | ||
|- valign="top" | |- valign="top" | ||
| '''Summary''' | | '''Summary''' | ||
Line 42: | Line 46: | ||
* Return all text nodes of the database {{Code|DB}} that contain the numbers {{Code|2010}} and {{Code|2020}}:<br/><code>ft:search("DB", ("2010", "2020"), map { 'mode': 'all' })</code> | * Return all text nodes of the database {{Code|DB}} that contain the numbers {{Code|2010}} and {{Code|2020}}:<br/><code>ft:search("DB", ("2010", "2020"), map { 'mode': 'all' })</code> | ||
* Return text nodes that contain the terms {{Code|A}} and {{Code|B|}} in a distance of at most 5 words: | * Return text nodes that contain the terms {{Code|A}} and {{Code|B|}} in a distance of at most 5 words: | ||
− | < | + | <pre lang='xquery'> |
ft:search("db", ("A", "B"), map { | ft:search("db", ("A", "B"), map { | ||
"mode": "all words", | "mode": "all words", | ||
Line 50: | Line 54: | ||
} | } | ||
}) | }) | ||
− | </ | + | </pre> |
* Iterate over three databases and return all elements containing terms similar to {{Code|Hello World}} in the text nodes: | * Iterate over three databases and return all elements containing terms similar to {{Code|Hello World}} in the text nodes: | ||
− | < | + | <pre lang='xquery'> |
let $terms := "Hello Worlds" | let $terms := "Hello Worlds" | ||
let $fuzzy := true() | let $fuzzy := true() | ||
Line 58: | Line 62: | ||
let $dbname := 'DB' || $db | let $dbname := 'DB' || $db | ||
return ft:search($dbname, $terms, map { 'fuzzy': $fuzzy })/.. | return ft:search($dbname, $terms, map { 'fuzzy': $fuzzy })/.. | ||
− | </ | + | </pre> |
|} | |} | ||
Line 65: | Line 69: | ||
{| width='100%' | {| width='100%' | ||
|- valign="top" | |- valign="top" | ||
− | | width='120' | ''' | + | | width='120' | '''Signature''' |
− | | | + | |<pre>ft:tokens( |
+ | $db as xs:string, | ||
+ | $prefix as xs:string := () | ||
+ | ) as element(value)*</pre> | ||
|- valign="top" | |- valign="top" | ||
| '''Summary''' | | '''Summary''' | ||
Line 76: | Line 83: | ||
| '''Examples''' | | '''Examples''' | ||
|Returns the number of occurrences for a single, specific index entry: | |Returns the number of occurrences for a single, specific index entry: | ||
− | < | + | <pre lang='xquery'> |
let $term := ft:tokenize($term) | let $term := ft:tokenize($term) | ||
return number(ft:tokens('db', $term)[. = $term]/@count) | return number(ft:tokens('db', $term)[. = $term]/@count) | ||
− | </ | + | </pre> |
|} | |} | ||
Line 88: | Line 95: | ||
{| width='100%' | {| width='100%' | ||
|- valign="top" | |- valign="top" | ||
− | | width='120' | ''' | + | | width='120' | '''Signature''' |
− | | | + | |<pre>ft:contains( |
+ | $input as item()*, | ||
+ | $terms as item()*, | ||
+ | $options as map(*)? := map { } | ||
+ | ) as xs:boolean</pre> | ||
|- valign="top" | |- valign="top" | ||
| '''Summary''' | | '''Summary''' | ||
Line 104: | Line 115: | ||
| | | | ||
* Checks if {{Code|jack}} or {{Code|john}} occurs in the input string {{Code|John Doe}}: | * Checks if {{Code|jack}} or {{Code|john}} occurs in the input string {{Code|John Doe}}: | ||
− | < | + | <pre lang='xquery'> |
ft:contains("John Doe", ("jack", "john"), map { "mode": "any" }) | ft:contains("John Doe", ("jack", "john"), map { "mode": "any" }) | ||
− | </ | + | </pre> |
* Calls the function with stemming turned on and off: | * Calls the function with stemming turned on and off: | ||
− | < | + | <pre lang='xquery'> |
(true(), false()) ! ft:contains("Häuser", "Haus", map { 'stemming': ., 'language':'de' }) | (true(), false()) ! ft:contains("Häuser", "Haus", map { 'stemming': ., 'language':'de' }) | ||
− | </ | + | </pre> |
|} | |} | ||
Line 117: | Line 128: | ||
{| width='100%' | {| width='100%' | ||
|- valign="top" | |- valign="top" | ||
− | | width='120' | ''' | + | | width='120' | '''Signature''' |
− | | | + | |<pre>ft:count( |
+ | $nodes as node()* | ||
+ | ) as xs:integer</pre> | ||
|- valign="top" | |- valign="top" | ||
| '''Summary''' | | '''Summary''' | ||
Line 132: | Line 145: | ||
{| width='100%' | {| width='100%' | ||
|- valign="top" | |- valign="top" | ||
− | | width='120' | ''' | + | | width='120' | '''Signature''' |
− | | | + | |<pre>ft:score( |
+ | $item as item()* | ||
+ | ) as xs:double*</pre> | ||
|- valign="top" | |- valign="top" | ||
| '''Summary''' | | '''Summary''' | ||
Line 147: | Line 162: | ||
{| width='100%' | {| width='100%' | ||
|- valign="top" | |- valign="top" | ||
− | | width='120' | ''' | + | | width='120' | '''Signature''' |
− | | | + | |<pre>ft:tokenize( |
+ | $string as xs:string?, | ||
+ | $options as map(*)? := map { } | ||
+ | ) as xs:string*</pre> | ||
|- valign="top" | |- valign="top" | ||
| '''Summary''' | | '''Summary''' | ||
Line 169: | Line 187: | ||
{| width='100%' | {| width='100%' | ||
|- valign="top" | |- valign="top" | ||
− | | width='120' | ''' | + | | width='120' | '''Signature''' |
− | | | + | |<pre>ft:normalize( |
+ | $string as xs:string?, | ||
+ | $options as map(*)? := map { } | ||
+ | ) as xs:string</pre> | ||
|- valign="top" | |- valign="top" | ||
| '''Summary''' | | '''Summary''' | ||
Line 184: | Line 205: | ||
{| width='100%' | {| width='100%' | ||
|- valign="top" | |- valign="top" | ||
− | | width='120' | ''' | + | | width='120' | '''Signature''' |
− | | | + | |<pre>ft:thesaurus( |
+ | $node as node(), | ||
+ | $term as xs:string, | ||
+ | $options as map(*)? := map { } | ||
+ | ) as xs:string*</pre> | ||
|- valign="top" | |- valign="top" | ||
| '''Summary''' | | '''Summary''' | ||
Line 194: | Line 219: | ||
| '''Examples''' | | '''Examples''' | ||
| Returns {{Code|happy}} and {{Code|lucky}}: | | Returns {{Code|happy}} and {{Code|lucky}}: | ||
− | < | + | <pre lang='xquery'> |
ft:thesaurus( | ft:thesaurus( | ||
<thesaurus> | <thesaurus> | ||
Line 207: | Line 232: | ||
'happy' | 'happy' | ||
) | ) | ||
− | </ | + | </pre> |
|} | |} | ||
Line 216: | Line 241: | ||
{| width='100%' | {| width='100%' | ||
|- valign="top" | |- valign="top" | ||
− | | width='120' | ''' | + | | width='120' | '''Signature''' |
− | | | + | |<pre>ft:mark( |
+ | $nodes as node()*, | ||
+ | $name as xs:string := () | ||
+ | ) as node()*</pre> | ||
|- valign="top" | |- valign="top" | ||
| '''Summary''' | | '''Summary''' | ||
Line 226: | Line 254: | ||
| '''Examples''' | | '''Examples''' | ||
|'''Example 1''': The following query returns {{Code|<XML><mark>hello</mark> world</XML>}}, if one text node of the database {{Code|DB}} has the value "hello world": | |'''Example 1''': The following query returns {{Code|<XML><mark>hello</mark> world</XML>}}, if one text node of the database {{Code|DB}} has the value "hello world": | ||
− | < | + | <pre lang='xquery'> |
ft:mark(db:get('DB')//*[text() contains text 'hello']) | ft:mark(db:get('DB')//*[text() contains text 'hello']) | ||
− | </ | + | </pre> |
'''Example 2''': The following expression loops through the first ten full-text results and marks the results in a second expression: | '''Example 2''': The following expression loops through the first ten full-text results and marks the results in a second expression: | ||
− | < | + | <pre lang='xquery'> |
let $start := 1 | let $start := 1 | ||
let $end := 10 | let $end := 10 | ||
Line 237: | Line 265: | ||
for $ft in (db:get('DB')//*[$test(.)])[position() = $start to $end] | for $ft in (db:get('DB')//*[$test(.)])[position() = $start to $end] | ||
return ft:mark($ft[$test(.)]) | return ft:mark($ft[$test(.)]) | ||
− | </ | + | </pre> |
'''Example 3''': The following expression returns <code><xml>hello <b>word</b></xml></code>: | '''Example 3''': The following expression returns <code><xml>hello <b>word</b></xml></code>: | ||
− | < | + | <pre lang='xquery'> |
copy $p := <xml>hello world</xml> | copy $p := <xml>hello world</xml> | ||
modify () | modify () | ||
return ft:mark($p[text() contains text 'word'], 'b') | return ft:mark($p[text() contains text 'word'], 'b') | ||
− | </ | + | </pre> |
|} | |} | ||
Line 250: | Line 278: | ||
{| width='100%' | {| width='100%' | ||
|- valign="top" | |- valign="top" | ||
− | | width='120' | ''' | + | | width='120' | '''Signature''' |
− | | | + | |<pre>ft:extract( |
+ | $nodes as node()*, | ||
+ | $name as xs:string := (), | ||
+ | $length as xs:integer := () | ||
+ | ) as node()*</pre> | ||
|- valign="top" | |- valign="top" | ||
| '''Summary''' | | '''Summary''' | ||
Line 259: | Line 291: | ||
| | | | ||
* The following query may return {{Code|<XML>...<b>hello</b>...<XML>}} if a text node of the database {{Code|DB}} contains the string "hello world": | * The following query may return {{Code|<XML>...<b>hello</b>...<XML>}} if a text node of the database {{Code|DB}} contains the string "hello world": | ||
− | < | + | <pre lang='xquery'> |
ft:extract(db:get('DB')//*[text() contains text 'hello'], 'b', 1) | ft:extract(db:get('DB')//*[text() contains text 'hello'], 'b', 1) | ||
− | </ | + | </pre> |
|} | |} | ||
Latest revision as of 18:34, 1 December 2023
This XQuery Module extends the Full-Text features of BaseX: The index can be directly accessed, full-text results can be marked with additional elements, or the relevant parts can be extracted. Moreover, the score value, which is generated by the contains text
expression, can be explicitly requested from items.
Contents
Conventions[edit]
All functions and errors in this module are assigned to the http://basex.org/modules/ft
namespace, which is statically bound to the ft
prefix.
Database Functions[edit]
ft:search[edit]
Signature | ft:search( $db as xs:string, $terms as item()*, $options as map(*)? := map { } ) as text()* |
Summary | Returns all text nodes from the full-text index of the database $db that contain the specified $terms .The options used for tokenizing the input and building the full-text will also be applied to the search terms. As an example, if the index terms have been stemmed, the search string will be stemmed as well. The
|
Errors | db:get : The addressed database does not exist or could not be opened.db:no-index : the index is not available.options : the fuzzy and wildcard option cannot be both specified.
|
Examples |
ft:search("db", ("A", "B"), map {
"mode": "all words",
"distance": map {
"max": "5",
"unit": "words"
}
})
let $terms := "Hello Worlds"
let $fuzzy := true()
for $db in 1 to 3
let $dbname := 'DB' || $db
return ft:search($dbname, $terms, map { 'fuzzy': $fuzzy })/..
|
ft:tokens[edit]
Signature | ft:tokens( $db as xs:string, $prefix as xs:string := () ) as element(value)* |
Summary | Returns all full-text tokens stored in the index of the database $db , along with their numbers of occurrences.If $prefix is specified, the returned nodes will be refined to the strings starting with that prefix. The prefix will be tokenized according to the full-text used for creating the index.
|
Errors | db:get : The addressed database does not exist or could not be opened.db:no-index : the full-text index is not available.
|
Examples | Returns the number of occurrences for a single, specific index entry:
let $term := ft:tokenize($term)
return number(ft:tokens('db', $term)[. = $term]/@count)
|
General Functions[edit]
ft:contains[edit]
Signature | ft:contains( $input as item()*, $terms as item()*, $options as map(*)? := map { } ) as xs:boolean |
Summary | Checks if the specified $input items contain the specified $terms .The function does the same as the Full-Text expression contains text , but options can be specified more dynamically. The $options are the same as for ft:search , and the following ones exist:
|
Errors | options : specified options are conflicting.
|
Examples |
ft:contains("John Doe", ("jack", "john"), map { "mode": "any" })
(true(), false()) ! ft:contains("Häuser", "Haus", map { 'stemming': ., 'language':'de' })
|
ft:count[edit]
Signature | ft:count( $nodes as node()* ) as xs:integer |
Summary | Returns the number of occurrences of the search terms specified in a full-text expression. |
Examples |
|
ft:score[edit]
Signature | ft:score( $item as item()* ) as xs:double* |
Summary | Returns the score values (0.0 - 1.0) that have been attached to the specified items. 0 is returned a value if no score was attached.
|
Examples |
|
ft:tokenize[edit]
Signature | ft:tokenize( $string as xs:string?, $options as map(*)? := map { } ) as xs:string* |
Summary | Tokenizes the given $string , using the current default full-text options or the $options specified as second argument, and returns a sequence with the tokenized string. The following options are available:
The |
Examples |
|
ft:normalize[edit]
Signature | ft:normalize( $string as xs:string?, $options as map(*)? := map { } ) as xs:string |
Summary | Normalizes the given $string , using the current default full-text options or the $options specified as second argument. The function accepts the same arguments as ft:tokenize ; special characters and separators will be preserved.
|
Examples |
|
ft:thesaurus[edit]
Signature | ft:thesaurus( $node as node(), $term as xs:string, $options as map(*)? := map { } ) as xs:string* |
Summary | Looks up a $term in a Thesaurus Structure supplied by $node . The following $options exist:
|
Examples | Returns happy and lucky :
ft:thesaurus(
<thesaurus>
<entry>
<term>happy</term>
<synonym>
<term>lucky</term>
<relationship>RT</relationship>
</synonym>
</entry>
</thesaurus>,
'happy'
)
|
Highlighting Functions[edit]
ft:mark[edit]
Signature | ft:mark( $nodes as node()*, $name as xs:string := () ) as node()* |
Summary | Puts a marker element around the resulting $nodes of a full-text request.The default name of the marker element is mark . An alternative name can be chosen via the optional $name argument.Please note that:
|
Examples | Example 1: The following query returns <XML><mark>hello</mark> world</XML> , if one text node of the database DB has the value "hello world":
ft:mark(db:get('DB')//*[text() contains text 'hello'])
Example 2: The following expression loops through the first ten full-text results and marks the results in a second expression: let $start := 1
let $end := 10
let $term := 'welcome'
let $test := function($node) { $node/text() contains text { $term } }
for $ft in (db:get('DB')//*[$test(.)])[position() = $start to $end]
return ft:mark($ft[$test(.)])
Example 3: The following expression returns copy $p := <xml>hello world</xml>
modify ()
return ft:mark($p[text() contains text 'word'], 'b')
|
ft:extract[edit]
Signature | ft:extract( $nodes as node()*, $name as xs:string := (), $length as xs:integer := () ) as node()* |
Summary | Extracts and returns relevant parts of full-text results. It puts a marker element around the resulting $nodes of a full-text index request and chops irrelevant sections of the result.The default element name of the marker element is mark . An alternative element name can be chosen via the optional $name argument.The default length of the returned text is 150 characters. An alternative length can be specified via the optional $length argument. Note that the effective text length may differ from the specified text due to formatting and readibility issues.For more details on this function, please have a look at ft:mark .
|
Examples |
ft:extract(db:get('DB')//*[text() contains text 'hello'], 'b', 1)
|
Errors[edit]
Code | Description |
---|---|
options
|
Both wildcards and fuzzy search have been specified as search options. |
Changelog[edit]
- Version 9.6
- Added:
ft:thesaurus
- Updated:
ft:search
,ft:contains
: newerrors
option.
- Version 9.1
- Updated:
ft:tokenize
andft:normalize
can be called with empty sequence.
- Version 9.0
- Updated: error codes updated; errors now use the module namespace
- Version 8.0
- Added:
ft:contains
,ft:normalize
- Updated: Options added to
ft:tokenize
- Version 7.8
- Added:
ft:contains
- Updated: Options added to
ft:search
- Version 7.7
- Updated: the functions no longer accept Database Nodes as reference. Instead, the name of a database must now be specified.
- Version 7.2
- Updated:
ft:search
(second argument generalized, third parameter added)
- Version 7.1
- Added:
ft:tokens
,ft:tokenize