Edge-ngram filter index, when querying _all field not working for certain words

Full Text Search
1

We use a FTS index for searching patients. The index use a custom filter edge-ngram for auto-complete .
field address.city is include in _all field.
Example
When we search word ‘Brusel’ it only returns results as expected if we specify the field in query:

Query returning 27 document works ok:
{
“explain”: false,
“fields”: [
“*”
],
“highlight”: {},
“query”: {“conjuncts”:[{“field”:“address.city”,“match”:“Brusel”},{“field”:“managingOrganization.reference”, “match”: “Organization/302”}]}
}

Query not returning documents with same word but searching without specifying field:

{
“explain”: false,
“fields”: [
“*”
],
“highlight”: {},
“query”: {“conjuncts”:[{“match”:“Brusel”},{“field”:“managingOrganization.reference”, “match”: “Organization/302”}]}
}

Index definition:
field addres.city is included in _all field:

{
“name”: “patient-search-index-v2”,
“type”: “fulltext-index”,
“params”: {
“doc_config”: {
“docid_prefix_delim”: “”,
“docid_regexp”: “”,
“mode”: “type_field”,
“type_field”: “resourceType”
},
“mapping”: {
“analysis”: {
“analyzers”: {
“autocomplete”: {
“token_filters”: [
“to_lower”,
“autocomplete-filter”,
“stop_fr”
],
“tokenizer”: “unicode”,
“type”: “custom”
},
“organization”: {
“tokenizer”: “whitespace”,
“type”: “custom”
}
},
“token_filters”: {
“autocomplete-filter”: {
“back”: “false”,
“max”: 20,
“min”: 2,
“type”: “edge_ngram”
}
}
},
“default_analyzer”: “fr”,
“default_datetime_parser”: “dateTimeOptional”,
“default_field”: “_all”,
“default_mapping”: {
“dynamic”: true,
“enabled”: false
},
“default_type”: “_default”,
“docvalues_dynamic”: true,
“index_dynamic”: true,
“store_dynamic”: false,
“type_field”: “_type”,
“types”: {
“Patient”: {
“default_analyzer”: “autocomplete”,
“dynamic”: false,
“enabled”: true,
“properties”: {
“address”: {
“default_analyzer”: “autocomplete”,
“dynamic”: false,
“enabled”: true,
“properties”: {
“city”: {
“enabled”: true,
“dynamic”: false,
“fields”: [
{
“include_in_all”: true,
“include_term_vectors”: true,
“index”: true,
“name”: “city”,
“store”: true,
“type”: “text”
}
]
},
“line”: {
“enabled”: true,
“dynamic”: false,
“fields”: [
{
“include_in_all”: true,
“include_term_vectors”: true,
“index”: true,
“name”: “line”,
“store”: true,
“type”: “text”
}
]
},
“postalCode”: {
“enabled”: true,
“dynamic”: false,
“fields”: [
{
“include_in_all”: true,
“include_term_vectors”: true,
“index”: true,
“name”: “postalCode”,
“store”: true,
“type”: “text”
}
]
},
“use”: {
“enabled”: true,
“dynamic”: false,
“fields”: [
{
“include_in_all”: true,
“include_term_vectors”: true,
“index”: true,
“name”: “use”,
“store”: true,
“type”: “text”
}
]
}
}
},
“identifier”: {
“default_analyzer”: “autocomplete”,
“dynamic”: false,
“enabled”: true,
“properties”: {
“system”: {
“enabled”: true,
“dynamic”: false,
“fields”: [
{
“name”: “system”,
“store”: true,
“type”: “text”
}
]
},
“use”: {
“enabled”: true,
“dynamic”: false,
“fields”: [
{
“name”: “use”,
“store”: true,
“type”: “text”
}
]
},
“value”: {
“enabled”: true,
“dynamic”: false,
“fields”: [
{
“analyzer”: “autocomplete”,
“include_in_all”: true,
“index”: true,
“name”: “value”,
“store”: true,
“type”: “text”
}
]
}
}
},
“managingOrganization”: {
“default_analyzer”: “organization”,
“dynamic”: false,
“enabled”: true,
“properties”: {
“reference”: {
“enabled”: true,
“dynamic”: false,
“fields”: [
{
“analyzer”: “organization”,
“include_in_all”: true,
“index”: true,
“name”: “reference”,
“store”: true,
“type”: “text”
}
]
}
}
},
“name”: {
“default_analyzer”: “autocomplete”,
“dynamic”: false,
“enabled”: true,
“properties”: {
“family”: {
“enabled”: true,
“dynamic”: false,
“fields”: [
{
“analyzer”: “autocomplete”,
“include_in_all”: true,
“include_term_vectors”: true,
“index”: true,
“name”: “family”,
“store”: true,
“type”: “text”
}
]
},
“given”: {
“enabled”: true,
“dynamic”: false,
“fields”: [
{
“analyzer”: “autocomplete”,
“include_in_all”: true,
“include_term_vectors”: true,
“index”: true,
“name”: “given”,
“store”: true,
“type”: “text”
}
]
}
}
},
“telecom”: {
“default_analyzer”: “autocomplete”,
“dynamic”: false,
“enabled”: true,
“properties”: {
“system”: {
“enabled”: true,
“dynamic”: false,
“fields”: [
{
“name”: “system”,
“store”: true,
“type”: “text”
}
]
},
“use”: {
“enabled”: true,
“dynamic”: false,
“fields”: [
{
“name”: “use”,
“store”: true,
“type”: “text”
}
]
},
“value”: {
“enabled”: true,
“dynamic”: false,
“fields”: [
{
“analyzer”: “autocomplete”,
“include_in_all”: true,
“include_term_vectors”: true,
“index”: true,
“name”: “value”,
“store”: true,
“type”: “text”
}
]
}
}
},
“birthDate”: {
“enabled”: true,
“dynamic”: false,
“fields”: [
{
“analyzer”: “organization”,
“include_in_all”: true,
“include_term_vectors”: true,
“index”: true,
“name”: “birthDate”,
“store”: true,
“type”: “datetime”
}
]
},
“gender”: {
“enabled”: true,
“dynamic”: false,
“fields”: [
{
“include_in_all”: true,
“include_term_vectors”: true,
“index”: true,
“name”: “gender”,
“store”: true,
“type”: “text”
}
]
}
}
}
}
},
“store”: {
“indexType”: “scorch”,
“kvStoreName”: “”
}
},
“sourceType”: “couchbase”,
“sourceName”: “fhirDatabaseClevehrLatest”,
“sourceUUID”: “92eff48e18a84e2045e1569b7c4cc365”,
“sourceParams”: {},
“planParams”: {
“maxPartitionsPerPIndex”: 171,
“numReplicas”: 0
},
“uuid”: “11d545b00e281bfc”
}

2

Hi @gunter,

The given index definition works for my sample document for the below query.

curl -XPOST -H "Content-Type: application/json" -u Administrator:asdasd http://localhost:9200/api/index/FTS/query -d '{"query": {"conjuncts": [{"match":"Brusel"},{"field":"managingOrganization.reference", "match": "Organization/302"}]}}'

{"status":{"total":6,"failed":0,"successful":6},"request":{"query":{"conjuncts":[{"match":"Brusel","prefix_length":0,"fuzziness":0,"operator":"or"},{"match":"Organization/302","field":"managingOrganization.reference","prefix_length":0,"fuzziness":0,"operator":"or"}]},"size":10,"from":0,"highlight":null,"fields":null,"facets":null,"explain":false,"sort":["-_score"],"includeLocations":false,"search_after":null,"search_before":null},"hits":[{"index":"FTS_152a1c3387e06629_54820232","id":"doc1","score":0.30555849162854065,"sort":["_score"]}],"total_hits":1,"max_score":0.30555849162854065,"took":139432,"facets":null}

document

{
  "resourceType": "Patient",
  "address": {
    "city": "Brusel"
  },
  "managingOrganization": {
    "reference": "Organization/302"
  }
}

A sample document from your side would be preferred.

Cheers!

3

thanks Sheekanh, please modify your document with this one and relaunch the query:
{
“resourceType”: “Patient”,
“address”: [
{
“city”: “Brussel”,
“line”: [
“impasse Lemaire”,
“72”
],
“postalCode”: “4794”,
“use”: “home”
}
],
“managingOrganization”: {
“reference”: “Organization/302”
}
}

4

The default analyser getting applied for _all field is different, and hence the query text is getting analysed to
“brussel” => “brusel”. So it is not working.

_all field is generic field and we can’t use different analysers for the different fields getting mapped there.
Here we use the default analyser “fr” in this case.

In such custom analyser use cases, its always recommended to use field scoped queries.

5

Thank a lot !
I have changed the default analyzer for _all to the custom “autocomplete” and now It works !!