Improve issue search (#2387)
* Improve issue indexer * Fix new issue sqlite bug * Different test indexer paths for each db * Add integration indexer paths to make cleanrelease/v1.3
parent
52e11b24bf
commit
b0f7457d9e
@ -0,0 +1,143 @@
|
||||
// Copyright 2017 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package indexer
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
|
||||
"github.com/blevesearch/bleve"
|
||||
"github.com/blevesearch/bleve/analysis/analyzer/custom"
|
||||
"github.com/blevesearch/bleve/analysis/token/lowercase"
|
||||
"github.com/blevesearch/bleve/analysis/token/unicodenorm"
|
||||
"github.com/blevesearch/bleve/analysis/tokenizer/unicode"
|
||||
)
|
||||
|
||||
// issueIndexer (thread-safe) index for searching issues
|
||||
var issueIndexer bleve.Index
|
||||
|
||||
// IssueIndexerData data stored in the issue indexer
|
||||
type IssueIndexerData struct {
|
||||
RepoID int64
|
||||
Title string
|
||||
Content string
|
||||
Comments []string
|
||||
}
|
||||
|
||||
// IssueIndexerUpdate an update to the issue indexer
|
||||
type IssueIndexerUpdate struct {
|
||||
IssueID int64
|
||||
Data *IssueIndexerData
|
||||
}
|
||||
|
||||
const issueIndexerAnalyzer = "issueIndexer"
|
||||
|
||||
// InitIssueIndexer initialize issue indexer
|
||||
func InitIssueIndexer(populateIndexer func() error) {
|
||||
_, err := os.Stat(setting.Indexer.IssuePath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
if err = createIssueIndexer(); err != nil {
|
||||
log.Fatal(4, "CreateIssuesIndexer: %v", err)
|
||||
}
|
||||
if err = populateIndexer(); err != nil {
|
||||
log.Fatal(4, "PopulateIssuesIndex: %v", err)
|
||||
}
|
||||
} else {
|
||||
log.Fatal(4, "InitIssuesIndexer: %v", err)
|
||||
}
|
||||
} else {
|
||||
issueIndexer, err = bleve.Open(setting.Indexer.IssuePath)
|
||||
if err != nil {
|
||||
log.Error(4, "Unable to open issues indexer (%s)."+
|
||||
" If the error is due to incompatible versions, try deleting the indexer files;"+
|
||||
" gitea will recreate them with the appropriate version the next time it runs."+
|
||||
" Deleting the indexer files will not result in loss of data.",
|
||||
setting.Indexer.IssuePath)
|
||||
log.Fatal(4, "InitIssuesIndexer, open index: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// createIssueIndexer create an issue indexer if one does not already exist
|
||||
func createIssueIndexer() error {
|
||||
mapping := bleve.NewIndexMapping()
|
||||
docMapping := bleve.NewDocumentMapping()
|
||||
|
||||
docMapping.AddFieldMappingsAt("RepoID", bleve.NewNumericFieldMapping())
|
||||
|
||||
textFieldMapping := bleve.NewTextFieldMapping()
|
||||
docMapping.AddFieldMappingsAt("Title", textFieldMapping)
|
||||
docMapping.AddFieldMappingsAt("Content", textFieldMapping)
|
||||
docMapping.AddFieldMappingsAt("Comments", textFieldMapping)
|
||||
|
||||
const unicodeNormNFC = "unicodeNormNFC"
|
||||
if err := mapping.AddCustomTokenFilter(unicodeNormNFC, map[string]interface{}{
|
||||
"type": unicodenorm.Name,
|
||||
"form": unicodenorm.NFC,
|
||||
}); err != nil {
|
||||
return err
|
||||
} else if err = mapping.AddCustomAnalyzer(issueIndexerAnalyzer, map[string]interface{}{
|
||||
"type": custom.Name,
|
||||
"char_filters": []string{},
|
||||
"tokenizer": unicode.Name,
|
||||
"token_filters": []string{unicodeNormNFC, lowercase.Name},
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
mapping.DefaultAnalyzer = issueIndexerAnalyzer
|
||||
mapping.AddDocumentMapping("issues", docMapping)
|
||||
|
||||
var err error
|
||||
issueIndexer, err = bleve.New(setting.Indexer.IssuePath, mapping)
|
||||
return err
|
||||
}
|
||||
|
||||
// UpdateIssue update the issue indexer
|
||||
func UpdateIssue(update IssueIndexerUpdate) error {
|
||||
return issueIndexer.Index(indexerID(update.IssueID), update.Data)
|
||||
}
|
||||
|
||||
// BatchUpdateIssues perform a batch update of the issue indexer
|
||||
func BatchUpdateIssues(updates ...IssueIndexerUpdate) error {
|
||||
batch := issueIndexer.NewBatch()
|
||||
for _, update := range updates {
|
||||
err := batch.Index(indexerID(update.IssueID), update.Data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return issueIndexer.Batch(batch)
|
||||
}
|
||||
|
||||
// SearchIssuesByKeyword searches for issues by given conditions.
|
||||
// Returns the matching issue IDs
|
||||
func SearchIssuesByKeyword(repoID int64, keyword string) ([]int64, error) {
|
||||
indexerQuery := bleve.NewConjunctionQuery(
|
||||
numericEqualityQuery(repoID, "RepoID"),
|
||||
bleve.NewDisjunctionQuery(
|
||||
newMatchPhraseQuery(keyword, "Title", issueIndexerAnalyzer),
|
||||
newMatchPhraseQuery(keyword, "Content", issueIndexerAnalyzer),
|
||||
newMatchPhraseQuery(keyword, "Comments", issueIndexerAnalyzer),
|
||||
))
|
||||
search := bleve.NewSearchRequestOptions(indexerQuery, 2147483647, 0, false)
|
||||
|
||||
result, err := issueIndexer.Search(search)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
issueIDs := make([]int64, len(result.Hits))
|
||||
for i, hit := range result.Hits {
|
||||
issueIDs[i], err = idOfIndexerID(hit.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return issueIDs, nil
|
||||
}
|
@ -0,0 +1,145 @@
|
||||
// Copyright (c) 2014 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package custom
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/blevesearch/bleve/analysis"
|
||||
"github.com/blevesearch/bleve/registry"
|
||||
)
|
||||
|
||||
const Name = "custom"
|
||||
|
||||
func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) {
|
||||
|
||||
var err error
|
||||
var charFilters []analysis.CharFilter
|
||||
charFiltersValue, ok := config["char_filters"]
|
||||
if ok {
|
||||
switch charFiltersValue := charFiltersValue.(type) {
|
||||
case []string:
|
||||
charFilters, err = getCharFilters(charFiltersValue, cache)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
case []interface{}:
|
||||
charFiltersNames, err := convertInterfaceSliceToStringSlice(charFiltersValue, "char filter")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
charFilters, err = getCharFilters(charFiltersNames, cache)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported type for char_filters, must be slice")
|
||||
}
|
||||
}
|
||||
|
||||
var tokenizerName string
|
||||
tokenizerValue, ok := config["tokenizer"]
|
||||
if ok {
|
||||
tokenizerName, ok = tokenizerValue.(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("must specify tokenizer as string")
|
||||
}
|
||||
} else {
|
||||
return nil, fmt.Errorf("must specify tokenizer")
|
||||
}
|
||||
|
||||
tokenizer, err := cache.TokenizerNamed(tokenizerName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var tokenFilters []analysis.TokenFilter
|
||||
tokenFiltersValue, ok := config["token_filters"]
|
||||
if ok {
|
||||
switch tokenFiltersValue := tokenFiltersValue.(type) {
|
||||
case []string:
|
||||
tokenFilters, err = getTokenFilters(tokenFiltersValue, cache)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
case []interface{}:
|
||||
tokenFiltersNames, err := convertInterfaceSliceToStringSlice(tokenFiltersValue, "token filter")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tokenFilters, err = getTokenFilters(tokenFiltersNames, cache)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported type for token_filters, must be slice")
|
||||
}
|
||||
}
|
||||
|
||||
rv := analysis.Analyzer{
|
||||
Tokenizer: tokenizer,
|
||||
}
|
||||
if charFilters != nil {
|
||||
rv.CharFilters = charFilters
|
||||
}
|
||||
if tokenFilters != nil {
|
||||
rv.TokenFilters = tokenFilters
|
||||
}
|
||||
return &rv, nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
registry.RegisterAnalyzer(Name, AnalyzerConstructor)
|
||||
}
|
||||
|
||||
func getCharFilters(charFilterNames []string, cache *registry.Cache) ([]analysis.CharFilter, error) {
|
||||
charFilters := make([]analysis.CharFilter, len(charFilterNames))
|
||||
for i, charFilterName := range charFilterNames {
|
||||
charFilter, err := cache.CharFilterNamed(charFilterName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
charFilters[i] = charFilter
|
||||
}
|
||||
|
||||
return charFilters, nil
|
||||
}
|
||||
|
||||
func getTokenFilters(tokenFilterNames []string, cache *registry.Cache) ([]analysis.TokenFilter, error) {
|
||||
tokenFilters := make([]analysis.TokenFilter, len(tokenFilterNames))
|
||||
for i, tokenFilterName := range tokenFilterNames {
|
||||
tokenFilter, err := cache.TokenFilterNamed(tokenFilterName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tokenFilters[i] = tokenFilter
|
||||
}
|
||||
|
||||
return tokenFilters, nil
|
||||
}
|
||||
|
||||
func convertInterfaceSliceToStringSlice(interfaceSlice []interface{}, objType string) ([]string, error) {
|
||||
stringSlice := make([]string, len(interfaceSlice))
|
||||
for i, interfaceObj := range interfaceSlice {
|
||||
stringObj, ok := interfaceObj.(string)
|
||||
if ok {
|
||||
stringSlice[i] = stringObj
|
||||
} else {
|
||||
return nil, fmt.Errorf(objType + " name must be a string")
|
||||
}
|
||||
}
|
||||
|
||||
return stringSlice, nil
|
||||
}
|
@ -1,46 +0,0 @@
|
||||
// Copyright (c) 2014 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package simple
|
||||
|
||||
import (
|
||||
"github.com/blevesearch/bleve/analysis"
|
||||
"github.com/blevesearch/bleve/analysis/token/lowercase"
|
||||
"github.com/blevesearch/bleve/analysis/tokenizer/letter"
|
||||
"github.com/blevesearch/bleve/registry"
|
||||
)
|
||||
|
||||
const Name = "simple"
|
||||
|
||||
func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) {
|
||||
tokenizer, err := cache.TokenizerNamed(letter.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rv := analysis.Analyzer{
|
||||
Tokenizer: tokenizer,
|
||||
TokenFilters: []analysis.TokenFilter{
|
||||
toLowerFilter,
|
||||
},
|
||||
}
|
||||
return &rv, nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
registry.RegisterAnalyzer(Name, AnalyzerConstructor)
|
||||
}
|
@ -0,0 +1,79 @@
|
||||
// Copyright (c) 2014 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package unicodenorm
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/blevesearch/bleve/analysis"
|
||||
"github.com/blevesearch/bleve/registry"
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
const Name = "normalize_unicode"
|
||||
|
||||
const NFC = "nfc"
|
||||
const NFD = "nfd"
|
||||
const NFKC = "nfkc"
|
||||
const NFKD = "nfkd"
|
||||
|
||||
var forms = map[string]norm.Form{
|
||||
NFC: norm.NFC,
|
||||
NFD: norm.NFD,
|
||||
NFKC: norm.NFKC,
|
||||
NFKD: norm.NFKD,
|
||||
}
|
||||
|
||||
type UnicodeNormalizeFilter struct {
|
||||
form norm.Form
|
||||
}
|
||||
|
||||
func NewUnicodeNormalizeFilter(formName string) (*UnicodeNormalizeFilter, error) {
|
||||
form, ok := forms[formName]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no form named %s", formName)
|
||||
}
|
||||
return &UnicodeNormalizeFilter{
|
||||
form: form,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func MustNewUnicodeNormalizeFilter(formName string) *UnicodeNormalizeFilter {
|
||||
filter, err := NewUnicodeNormalizeFilter(formName)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return filter
|
||||
}
|
||||
|
||||
func (s *UnicodeNormalizeFilter) Filter(input analysis.TokenStream) analysis.TokenStream {
|
||||
for _, token := range input {
|
||||
token.Term = s.form.Bytes(token.Term)
|
||||
}
|
||||
return input
|
||||
}
|
||||
|
||||
func UnicodeNormalizeFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {
|
||||
formVal, ok := config["form"].(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("must specify form")
|
||||
}
|
||||
form := formVal
|
||||
return NewUnicodeNormalizeFilter(form)
|
||||
}
|
||||
|
||||
func init() {
|
||||
registry.RegisterTokenFilter(Name, UnicodeNormalizeFilterConstructor)
|
||||
}
|
@ -1,76 +0,0 @@
|
||||
// Copyright (c) 2016 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package character
|
||||
|
||||
import (
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/blevesearch/bleve/analysis"
|
||||
)
|
||||
|
||||
type IsTokenRune func(r rune) bool
|
||||
|
||||
type CharacterTokenizer struct {
|
||||
isTokenRun IsTokenRune
|
||||
}
|
||||
|
||||
func NewCharacterTokenizer(f IsTokenRune) *CharacterTokenizer {
|
||||
return &CharacterTokenizer{
|
||||
isTokenRun: f,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *CharacterTokenizer) Tokenize(input []byte) analysis.TokenStream {
|
||||
|
||||
rv := make(analysis.TokenStream, 0, 1024)
|
||||
|
||||
offset := 0
|
||||
start := 0
|
||||
end := 0
|
||||
count := 0
|
||||
for currRune, size := utf8.DecodeRune(input[offset:]); currRune != utf8.RuneError; currRune, size = utf8.DecodeRune(input[offset:]) {
|
||||
isToken := c.isTokenRun(currRune)
|
||||
if isToken {
|
||||
end = offset + size
|
||||
} else {
|
||||
if end-start > 0 {
|
||||
// build token
|
||||
rv = append(rv, &analysis.Token{
|
||||
Term: input[start:end],
|
||||
Start: start,
|
||||
End: end,
|
||||
Position: count + 1,
|
||||
Type: analysis.AlphaNumeric,
|
||||
})
|
||||
count++
|
||||
}
|
||||
start = offset + size
|
||||
end = start
|
||||
}
|
||||
offset += size
|
||||
}
|
||||
// if we ended in the middle of a token, finish it
|
||||
if end-start > 0 {
|
||||
// build token
|
||||
rv = append(rv, &analysis.Token{
|
||||
Term: input[start:end],
|
||||
Start: start,
|
||||
End: end,
|
||||
Position: count + 1,
|
||||
Type: analysis.AlphaNumeric,
|
||||
})
|
||||
}
|
||||
return rv
|
||||
}
|
@ -1,33 +0,0 @@
|
||||
// Copyright (c) 2016 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package letter
|
||||
|
||||
import (
|
||||
"unicode"
|
||||
|
||||
"github.com/blevesearch/bleve/analysis"
|
||||
"github.com/blevesearch/bleve/analysis/tokenizer/character"
|
||||
"github.com/blevesearch/bleve/registry"
|
||||
)
|
||||
|
||||
const Name = "letter"
|
||||
|
||||
func TokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) {
|
||||
return character.NewCharacterTokenizer(unicode.IsLetter), nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
registry.RegisterTokenizer(Name, TokenizerConstructor)
|
||||
}
|
@ -1,23 +0,0 @@
|
||||
// Copyright (c) 2014 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build appengine appenginevm
|
||||
|
||||
package bleve
|
||||
|
||||
// in the appengine environment we cannot support disk based indexes
|
||||
// so we do no extra configuration in this method
|
||||
func initDisk() {
|
||||
|
||||
}
|
@ -0,0 +1,137 @@
|
||||
// Copyright (c) 2017 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package document
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/blevesearch/bleve/analysis"
|
||||
"github.com/blevesearch/bleve/geo"
|
||||
"github.com/blevesearch/bleve/numeric"
|
||||
)
|
||||
|
||||
var GeoPrecisionStep uint = 9
|
||||
|
||||
type GeoPointField struct {
|
||||
name string
|
||||
arrayPositions []uint64
|
||||
options IndexingOptions
|
||||
value numeric.PrefixCoded
|
||||
numPlainTextBytes uint64
|
||||
}
|
||||
|
||||
func (n *GeoPointField) Name() string {
|
||||
return n.name
|
||||
}
|
||||
|
||||
func (n *GeoPointField) ArrayPositions() []uint64 {
|
||||
return n.arrayPositions
|
||||
}
|
||||
|
||||
func (n *GeoPointField) Options() IndexingOptions {
|
||||
return n.options
|
||||
}
|
||||
|
||||
func (n *GeoPointField) Analyze() (int, analysis.TokenFrequencies) {
|
||||
tokens := make(analysis.TokenStream, 0)
|
||||
tokens = append(tokens, &analysis.Token{
|
||||
Start: 0,
|
||||
End: len(n.value),
|
||||
Term: n.value,
|
||||
Position: 1,
|
||||
Type: analysis.Numeric,
|
||||
})
|
||||
|
||||
original, err := n.value.Int64()
|
||||
if err == nil {
|
||||
|
||||
shift := GeoPrecisionStep
|
||||
for shift < 64 {
|
||||
shiftEncoded, err := numeric.NewPrefixCodedInt64(original, shift)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
token := analysis.Token{
|
||||
Start: 0,
|
||||
End: len(shiftEncoded),
|
||||
Term: shiftEncoded,
|
||||
Position: 1,
|
||||
Type: analysis.Numeric,
|
||||
}
|
||||
tokens = append(tokens, &token)
|
||||
shift += GeoPrecisionStep
|
||||
}
|
||||
}
|
||||
|
||||
fieldLength := len(tokens)
|
||||
tokenFreqs := analysis.TokenFrequency(tokens, n.arrayPositions, n.options.IncludeTermVectors())
|
||||
return fieldLength, tokenFreqs
|
||||
}
|
||||
|
||||
func (n *GeoPointField) Value() []byte {
|
||||
return n.value
|
||||
}
|
||||
|
||||
func (n *GeoPointField) Lon() (float64, error) {
|
||||
i64, err := n.value.Int64()
|
||||
if err != nil {
|
||||
return 0.0, err
|
||||
}
|
||||
return geo.MortonUnhashLon(uint64(i64)), nil
|
||||
}
|
||||
|
||||
func (n *GeoPointField) Lat() (float64, error) {
|
||||
i64, err := n.value.Int64()
|
||||
if err != nil {
|
||||
return 0.0, err
|
||||
}
|
||||
return geo.MortonUnhashLat(uint64(i64)), nil
|
||||
}
|
||||
|
||||
func (n *GeoPointField) GoString() string {
|
||||
return fmt.Sprintf("&document.GeoPointField{Name:%s, Options: %s, Value: %s}", n.name, n.options, n.value)
|
||||
}
|
||||
|
||||
func (n *GeoPointField) NumPlainTextBytes() uint64 {
|
||||
return n.numPlainTextBytes
|
||||
}
|
||||
|
||||
func NewGeoPointFieldFromBytes(name string, arrayPositions []uint64, value []byte) *GeoPointField {
|
||||
return &GeoPointField{
|
||||
name: name,
|
||||
arrayPositions: arrayPositions,
|
||||
value: value,
|
||||
options: DefaultNumericIndexingOptions,
|
||||
numPlainTextBytes: uint64(len(value)),
|
||||
}
|
||||
}
|
||||
|
||||
func NewGeoPointField(name string, arrayPositions []uint64, lon, lat float64) *GeoPointField {
|
||||
return NewGeoPointFieldWithIndexingOptions(name, arrayPositions, lon, lat, DefaultNumericIndexingOptions)
|
||||
}
|
||||
|
||||
func NewGeoPointFieldWithIndexingOptions(name string, arrayPositions []uint64, lon, lat float64, options IndexingOptions) *GeoPointField {
|
||||
mhash := geo.MortonHash(lon, lat)
|
||||
prefixCoded := numeric.MustNewPrefixCodedInt64(int64(mhash), 0)
|
||||
return &GeoPointField{
|
||||
name: name,
|
||||
arrayPositions: arrayPositions,
|
||||
value: prefixCoded,
|
||||
options: options,
|
||||
// not correct, just a place holder until we revisit how fields are
|
||||
// represented and can fix this better
|
||||
numPlainTextBytes: uint64(8),
|
||||
}
|
||||
}
|
@ -0,0 +1,9 @@
|
||||
# geo support in bleve
|
||||
|
||||
First, all of this geo code is a Go adaptation of the [Lucene 5.3.2 sandbox geo support](https://lucene.apache.org/core/5_3_2/sandbox/org/apache/lucene/util/package-summary.html).
|
||||
|
||||
## Notes
|
||||
|
||||
- All of the APIs will use float64 for lon/lat values.
|
||||
- When describing a point in function arguments or return values, we always use the order lon, lat.
|
||||
- High level APIs will use TopLeft and BottomRight to describe bounding boxes. This may not map cleanly to min/max lon/lat when crossing the dateline. The lower level APIs will use min/max lon/lat and require the higher-level code to split boxes accordingly.
|
@ -0,0 +1,170 @@
|
||||
// Copyright (c) 2017 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package geo
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
|
||||
"github.com/blevesearch/bleve/numeric"
|
||||
)
|
||||
|
||||
// GeoBits is the number of bits used for a single geo point
|
||||
// Currently this is 32bits for lon and 32bits for lat
|
||||
var GeoBits uint = 32
|
||||
|
||||
var minLon = -180.0
|
||||
var minLat = -90.0
|
||||
var maxLon = 180.0
|
||||
var maxLat = 90.0
|
||||
var minLonRad = minLon * degreesToRadian
|
||||
var minLatRad = minLat * degreesToRadian
|
||||
var maxLonRad = maxLon * degreesToRadian
|
||||
var maxLatRad = maxLat * degreesToRadian
|
||||
var geoTolerance = 1E-6
|
||||
var lonScale = float64((uint64(0x1)<<GeoBits)-1) / 360.0
|
||||
var latScale = float64((uint64(0x1)<<GeoBits)-1) / 180.0
|
||||
|
||||
// MortonHash computes the morton hash value for the provided geo point
|
||||
// This point is ordered as lon, lat.
|
||||
func MortonHash(lon, lat float64) uint64 {
|
||||
return numeric.Interleave(scaleLon(lon), scaleLat(lat))
|
||||
}
|
||||
|
||||
func scaleLon(lon float64) uint64 {
|
||||
rv := uint64((lon - minLon) * lonScale)
|
||||
return rv
|
||||
}
|
||||
|
||||
func scaleLat(lat float64) uint64 {
|
||||
rv := uint64((lat - minLat) * latScale)
|
||||
return rv
|
||||
}
|
||||
|
||||
// MortonUnhashLon extracts the longitude value from the provided morton hash.
|
||||
func MortonUnhashLon(hash uint64) float64 {
|
||||
return unscaleLon(numeric.Deinterleave(hash))
|
||||
}
|
||||
|
||||
// MortonUnhashLat extracts the latitude value from the provided morton hash.
|
||||
func MortonUnhashLat(hash uint64) float64 {
|
||||
return unscaleLat(numeric.Deinterleave(hash >> 1))
|
||||
}
|
||||
|
||||
func unscaleLon(lon uint64) float64 {
|
||||
return (float64(lon) / lonScale) + minLon
|
||||
}
|
||||
|
||||
func unscaleLat(lat uint64) float64 {
|
||||
return (float64(lat) / latScale) + minLat
|
||||
}
|
||||
|
||||
// compareGeo will compare two float values and see if they are the same
|
||||
// taking into consideration a known geo tolerance.
|
||||
func compareGeo(a, b float64) float64 {
|
||||
compare := a - b
|
||||
if math.Abs(compare) <= geoTolerance {
|
||||
return 0
|
||||
}
|
||||
return compare
|
||||
}
|
||||
|
||||
// RectIntersects checks whether rectangles a and b intersect
|
||||
func RectIntersects(aMinX, aMinY, aMaxX, aMaxY, bMinX, bMinY, bMaxX, bMaxY float64) bool {
|
||||
return !(aMaxX < bMinX || aMinX > bMaxX || aMaxY < bMinY || aMinY > bMaxY)
|
||||
}
|
||||
|
||||
// RectWithin checks whether box a is within box b
|
||||
func RectWithin(aMinX, aMinY, aMaxX, aMaxY, bMinX, bMinY, bMaxX, bMaxY float64) bool {
|
||||
rv := !(aMinX < bMinX || aMinY < bMinY || aMaxX > bMaxX || aMaxY > bMaxY)
|
||||
return rv
|
||||
}
|
||||
|
||||
// BoundingBoxContains checks whether the lon/lat point is within the box
|
||||
func BoundingBoxContains(lon, lat, minLon, minLat, maxLon, maxLat float64) bool {
|
||||
return compareGeo(lon, minLon) >= 0 && compareGeo(lon, maxLon) <= 0 &&
|
||||
compareGeo(lat, minLat) >= 0 && compareGeo(lat, maxLat) <= 0
|
||||
}
|
||||
|
||||
const degreesToRadian = math.Pi / 180
|
||||
const radiansToDegrees = 180 / math.Pi
|
||||
|
||||
// DegreesToRadians converts an angle in degrees to radians
|
||||
func DegreesToRadians(d float64) float64 {
|
||||
return d * degreesToRadian
|
||||
}
|
||||
|
||||
// RadiansToDegrees converts an angle in radians to degress
|
||||
func RadiansToDegrees(r float64) float64 {
|
||||
return r * radiansToDegrees
|
||||
}
|
||||
|
||||
var earthMeanRadiusMeters = 6371008.7714
|
||||
|
||||
func RectFromPointDistance(lon, lat, dist float64) (float64, float64, float64, float64, error) {
|
||||
err := checkLongitude(lon)
|
||||
if err != nil {
|
||||
return 0, 0, 0, 0, err
|
||||
}
|
||||
err = checkLatitude(lat)
|
||||
if err != nil {
|
||||
return 0, 0, 0, 0, err
|
||||
}
|
||||
radLon := DegreesToRadians(lon)
|
||||
radLat := DegreesToRadians(lat)
|
||||
radDistance := (dist + 7e-2) / earthMeanRadiusMeters
|
||||
|
||||
minLatL := radLat - radDistance
|
||||
maxLatL := radLat + radDistance
|
||||
|
||||
var minLonL, maxLonL float64
|
||||
if minLatL > minLatRad && maxLatL < maxLatRad {
|
||||
deltaLon := asin(sin(radDistance) / cos(radLat))
|
||||
minLonL = radLon - deltaLon
|
||||
if minLonL < minLonRad {
|
||||
minLonL += 2 * math.Pi
|
||||
}
|
||||
maxLonL = radLon + deltaLon
|
||||
if maxLonL > maxLonRad {
|
||||
maxLonL -= 2 * math.Pi
|
||||
}
|
||||
} else {
|
||||
// pole is inside distance
|
||||
minLatL = math.Max(minLatL, minLatRad)
|
||||
maxLatL = math.Min(maxLatL, maxLatRad)
|
||||
minLonL = minLonRad
|
||||
maxLonL = maxLonRad
|
||||
}
|
||||
|
||||
return RadiansToDegrees(minLonL),
|
||||
RadiansToDegrees(maxLatL),
|
||||
RadiansToDegrees(maxLonL),
|
||||
RadiansToDegrees(minLatL),
|
||||
nil
|
||||
}
|
||||
|
||||
func checkLatitude(latitude float64) error {
|
||||
if math.IsNaN(latitude) || latitude < minLat || latitude > maxLat {
|
||||
return fmt.Errorf("invalid latitude %f; must be between %f and %f", latitude, minLat, maxLat)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkLongitude(longitude float64) error {
|
||||
if math.IsNaN(longitude) || longitude < minLon || longitude > maxLon {
|
||||
return fmt.Errorf("invalid longitude %f; must be between %f and %f", longitude, minLon, maxLon)
|
||||
}
|
||||
return nil
|
||||
}
|
@ -0,0 +1,98 @@
|
||||
// Copyright (c) 2017 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package geo
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type distanceUnit struct {
|
||||
conv float64
|
||||
suffixes []string
|
||||
}
|
||||
|
||||
var inch = distanceUnit{0.0254, []string{"in", "inch"}}
|
||||
var yard = distanceUnit{0.9144, []string{"yd", "yards"}}
|
||||
var feet = distanceUnit{0.3048, []string{"ft", "feet"}}
|
||||
var kilom = distanceUnit{1000, []string{"km", "kilometers"}}
|
||||
var nauticalm = distanceUnit{1852.0, []string{"nm", "nauticalmiles"}}
|
||||
var millim = distanceUnit{0.001, []string{"mm", "millimeters"}}
|
||||
var centim = distanceUnit{0.01, []string{"cm", "centimeters"}}
|
||||
var miles = distanceUnit{1609.344, []string{"mi", "miles"}}
|
||||
var meters = distanceUnit{1, []string{"m", "meters"}}
|
||||
|
||||
var distanceUnits = []*distanceUnit{
|
||||
&inch, &yard, &feet, &kilom, &nauticalm, &millim, ¢im, &miles, &meters,
|
||||
}
|
||||
|
||||
// ParseDistance attempts to parse a distance string and return distance in
|
||||
// meters. Example formats supported:
|
||||
// "5in" "5inch" "7yd" "7yards" "9ft" "9feet" "11km" "11kilometers"
|
||||
// "3nm" "3nauticalmiles" "13mm" "13millimeters" "15cm" "15centimeters"
|
||||
// "17mi" "17miles" "19m" "19meters"
|
||||
// If the unit cannot be determined, the entire string is parsed and the
|
||||
// unit of meters is assumed.
|
||||
// If the number portion cannot be parsed, 0 and the parse error are returned.
|
||||
func ParseDistance(d string) (float64, error) {
|
||||
for _, unit := range distanceUnits {
|
||||
for _, unitSuffix := range unit.suffixes {
|
||||
if strings.HasSuffix(d, unitSuffix) {
|
||||
parsedNum, err := strconv.ParseFloat(d[0:len(d)-len(unitSuffix)], 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return parsedNum * unit.conv, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
// no unit matched, try assuming meters?
|
||||
parsedNum, err := strconv.ParseFloat(d, 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return parsedNum, nil
|
||||
}
|
||||
|
||||
// ParseDistanceUnit attempts to parse a distance unit and return the
|
||||
// multiplier for converting this to meters. If the unit cannot be parsed
|
||||
// then 0 and the error message is returned.
|
||||
func ParseDistanceUnit(u string) (float64, error) {
|
||||
for _, unit := range distanceUnits {
|
||||
for _, unitSuffix := range unit.suffixes {
|
||||
if u == unitSuffix {
|
||||
return unit.conv, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0, fmt.Errorf("unknown distance unit: %s", u)
|
||||
}
|
||||
|
||||
// Haversin computes the distance between two points.
|
||||
// This implemenation uses the sloppy math implemenations which trade off
|
||||
// accuracy for performance. The distance returned is in kilometers.
|
||||
func Haversin(lon1, lat1, lon2, lat2 float64) float64 {
|
||||
x1 := lat1 * degreesToRadian
|
||||
x2 := lat2 * degreesToRadian
|
||||
h1 := 1 - cos(x1-x2)
|
||||
h2 := 1 - cos((lon1-lon2)*degreesToRadian)
|
||||
h := (h1 + cos(x1)*cos(x2)*h2) / 2
|
||||
avgLat := (x1 + x2) / 2
|
||||
diameter := earthDiameter(avgLat)
|
||||
|
||||
return diameter * asin(math.Min(1, math.Sqrt(h)))
|
||||
}
|
@ -0,0 +1,140 @@
|
||||
// Copyright (c) 2017 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package geo
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ExtractGeoPoint takes an arbitrary interface{} and tries it's best to
|
||||
// interpret it is as geo point. Supported formats:
|
||||
// Container:
|
||||
// slice length 2 (GeoJSON)
|
||||
// first element lon, second element lat
|
||||
// map[string]interface{}
|
||||
// exact keys lat and lon or lng
|
||||
// struct
|
||||
// w/exported fields case-insensitive match on lat and lon or lng
|
||||
// struct
|
||||
// satisfying Later and Loner or Lnger interfaces
|
||||
//
|
||||
// in all cases values must be some sort of numeric-like thing: int/uint/float
|
||||
func ExtractGeoPoint(thing interface{}) (lon, lat float64, success bool) {
|
||||
var foundLon, foundLat bool
|
||||
|
||||
thingVal := reflect.ValueOf(thing)
|
||||
thingTyp := thingVal.Type()
|
||||
|
||||
// is it a slice
|
||||
if thingVal.IsValid() && thingVal.Kind() == reflect.Slice {
|
||||
// must be length 2
|
||||
if thingVal.Len() == 2 {
|
||||
first := thingVal.Index(0)
|
||||
if first.CanInterface() {
|
||||
firstVal := first.Interface()
|
||||
lon, foundLon = extractNumericVal(firstVal)
|
||||
}
|
||||
second := thingVal.Index(1)
|
||||
if second.CanInterface() {
|
||||
secondVal := second.Interface()
|
||||
lat, foundLat = extractNumericVal(secondVal)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// is it a map
|
||||
if l, ok := thing.(map[string]interface{}); ok {
|
||||
if lval, ok := l["lon"]; ok {
|
||||
lon, foundLon = extractNumericVal(lval)
|
||||
} else if lval, ok := l["lng"]; ok {
|
||||
lon, foundLon = extractNumericVal(lval)
|
||||
}
|
||||
if lval, ok := l["lat"]; ok {
|
||||
lat, foundLat = extractNumericVal(lval)
|
||||
}
|
||||
}
|
||||
|
||||
// now try reflection on struct fields
|
||||
if thingVal.IsValid() && thingVal.Kind() == reflect.Struct {
|
||||
for i := 0; i < thingVal.NumField(); i++ {
|
||||
fieldName := thingTyp.Field(i).Name
|
||||
if strings.HasPrefix(strings.ToLower(fieldName), "lon") {
|
||||
if thingVal.Field(i).CanInterface() {
|
||||
fieldVal := thingVal.Field(i).Interface()
|
||||
lon, foundLon = extractNumericVal(fieldVal)
|
||||
}
|
||||
}
|
||||
if strings.HasPrefix(strings.ToLower(fieldName), "lng") {
|
||||
if thingVal.Field(i).CanInterface() {
|
||||
fieldVal := thingVal.Field(i).Interface()
|
||||
lon, foundLon = extractNumericVal(fieldVal)
|
||||
}
|
||||
}
|
||||
if strings.HasPrefix(strings.ToLower(fieldName), "lat") {
|
||||
if thingVal.Field(i).CanInterface() {
|
||||
fieldVal := thingVal.Field(i).Interface()
|
||||
lat, foundLat = extractNumericVal(fieldVal)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// last hope, some interfaces
|
||||
// lon
|
||||
if l, ok := thing.(loner); ok {
|
||||
lon = l.Lon()
|
||||
foundLon = true
|
||||
} else if l, ok := thing.(lnger); ok {
|
||||
lon = l.Lng()
|
||||
foundLon = true
|
||||
}
|
||||
// lat
|
||||
if l, ok := thing.(later); ok {
|
||||
lat = l.Lat()
|
||||
foundLat = true
|
||||
}
|
||||
|
||||
return lon, lat, foundLon && foundLat
|
||||
}
|
||||
|
||||
// extract numeric value (if possible) and returns a float64
|
||||
func extractNumericVal(v interface{}) (float64, bool) {
|
||||
val := reflect.ValueOf(v)
|
||||
typ := val.Type()
|
||||
switch typ.Kind() {
|
||||
case reflect.Float32, reflect.Float64:
|
||||
return val.Float(), true
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||
return float64(val.Int()), true
|
||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
|
||||
return float64(val.Uint()), true
|
||||
}
|
||||
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// various support interfaces which can be used to find lat/lon
|
||||
type loner interface {
|
||||
Lon() float64
|
||||
}
|
||||
|
||||
type later interface {
|
||||
Lat() float64
|
||||
}
|
||||
|
||||
type lnger interface {
|
||||
Lng() float64
|
||||
}
|
@ -0,0 +1,212 @@
|
||||
// Copyright (c) 2017 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package geo
|
||||
|
||||
import (
|
||||
"math"
|
||||
)
|
||||
|
||||
var earthDiameterPerLatitude []float64
|
||||
var sinTab []float64
|
||||
var cosTab []float64
|
||||
var asinTab []float64
|
||||
var asinDer1DivF1Tab []float64
|
||||
var asinDer2DivF2Tab []float64
|
||||
var asinDer3DivF3Tab []float64
|
||||
var asinDer4DivF4Tab []float64
|
||||
|
||||
const radiusTabsSize = (1 << 10) + 1
|
||||
const radiusDelta = (math.Pi / 2) / (radiusTabsSize - 1)
|
||||
const radiusIndexer = 1 / radiusDelta
|
||||
const sinCosTabsSize = (1 << 11) + 1
|
||||
const asinTabsSize = (1 << 13) + 1
|
||||
const oneDivF2 = 1 / 2.0
|
||||
const oneDivF3 = 1 / 6.0
|
||||
const oneDivF4 = 1 / 24.0
|
||||
|
||||
// 1.57079632673412561417e+00 first 33 bits of pi/2
|
||||
var pio2Hi = math.Float64frombits(0x3FF921FB54400000)
|
||||
|
||||
// 6.07710050650619224932e-11 pi/2 - PIO2_HI
|
||||
var pio2Lo = math.Float64frombits(0x3DD0B4611A626331)
|
||||
|
||||
var asinPio2Hi = math.Float64frombits(0x3FF921FB54442D18) // 1.57079632679489655800e+00
|
||||
var asinPio2Lo = math.Float64frombits(0x3C91A62633145C07) // 6.12323399573676603587e-17
|
||||
var asinPs0 = math.Float64frombits(0x3fc5555555555555) // 1.66666666666666657415e-01
|
||||
var asinPs1 = math.Float64frombits(0xbfd4d61203eb6f7d) // -3.25565818622400915405e-01
|
||||
var asinPs2 = math.Float64frombits(0x3fc9c1550e884455) // 2.01212532134862925881e-01
|
||||
var asinPs3 = math.Float64frombits(0xbfa48228b5688f3b) // -4.00555345006794114027e-02
|
||||
var asinPs4 = math.Float64frombits(0x3f49efe07501b288) // 7.91534994289814532176e-04
|
||||
var asinPs5 = math.Float64frombits(0x3f023de10dfdf709) // 3.47933107596021167570e-05
|
||||
var asinQs1 = math.Float64frombits(0xc0033a271c8a2d4b) // -2.40339491173441421878e+00
|
||||
var asinQs2 = math.Float64frombits(0x40002ae59c598ac8) // 2.02094576023350569471e+00
|
||||
var asinQs3 = math.Float64frombits(0xbfe6066c1b8d0159) // -6.88283971605453293030e-01
|
||||
var asinQs4 = math.Float64frombits(0x3fb3b8c5b12e9282) // 7.70381505559019352791e-02
|
||||
|
||||
var twoPiHi = 4 * pio2Hi
|
||||
var twoPiLo = 4 * pio2Lo
|
||||
var sinCosDeltaHi = twoPiHi/sinCosTabsSize - 1
|
||||
var sinCosDeltaLo = twoPiLo/sinCosTabsSize - 1
|
||||
var sinCosIndexer = 1 / (sinCosDeltaHi + sinCosDeltaLo)
|
||||
var sinCosMaxValueForIntModulo = ((math.MaxInt64 >> 9) / sinCosIndexer) * 0.99
|
||||
var asinMaxValueForTabs = math.Sin(73.0 * degreesToRadian)
|
||||
|
||||
var asinDelta = asinMaxValueForTabs / (asinTabsSize - 1)
|
||||
var asinIndexer = 1 / asinDelta
|
||||
|
||||
func init() {
|
||||
// initializes the tables used for the sloppy math functions
|
||||
|
||||
// sin and cos
|
||||
sinTab = make([]float64, sinCosTabsSize)
|
||||
cosTab = make([]float64, sinCosTabsSize)
|
||||
sinCosPiIndex := (sinCosTabsSize - 1) / 2
|
||||
sinCosPiMul2Index := 2 * sinCosPiIndex
|
||||
sinCosPiMul05Index := sinCosPiIndex / 2
|
||||
sinCosPiMul15Index := 3 * sinCosPiIndex / 2
|
||||
for i := 0; i < sinCosTabsSize; i++ {
|
||||
// angle: in [0,2*PI].
|
||||
angle := float64(i)*sinCosDeltaHi + float64(i)*sinCosDeltaLo
|
||||
sinAngle := math.Sin(angle)
|
||||
cosAngle := math.Cos(angle)
|
||||
// For indexes corresponding to null cosine or sine, we make sure the value is zero
|
||||
// and not an epsilon. This allows for a much better accuracy for results close to zero.
|
||||
if i == sinCosPiIndex {
|
||||
sinAngle = 0.0
|
||||
} else if i == sinCosPiMul2Index {
|
||||
sinAngle = 0.0
|
||||
} else if i == sinCosPiMul05Index {
|
||||
sinAngle = 0.0
|
||||
} else if i == sinCosPiMul15Index {
|
||||
sinAngle = 0.0
|
||||
}
|
||||
sinTab[i] = sinAngle
|
||||
cosTab[i] = cosAngle
|
||||
}
|
||||
|
||||
// asin
|
||||
asinTab = make([]float64, asinTabsSize)
|
||||
asinDer1DivF1Tab = make([]float64, asinTabsSize)
|
||||
asinDer2DivF2Tab = make([]float64, asinTabsSize)
|
||||
asinDer3DivF3Tab = make([]float64, asinTabsSize)
|
||||
asinDer4DivF4Tab = make([]float64, asinTabsSize)
|
||||
for i := 0; i < asinTabsSize; i++ {
|
||||
// x: in [0,ASIN_MAX_VALUE_FOR_TABS].
|
||||
x := float64(i) * asinDelta
|
||||
asinTab[i] = math.Asin(x)
|
||||
oneMinusXSqInv := 1.0 / (1 - x*x)
|
||||
oneMinusXSqInv05 := math.Sqrt(oneMinusXSqInv)
|
||||
oneMinusXSqInv15 := oneMinusXSqInv05 * oneMinusXSqInv
|
||||
oneMinusXSqInv25 := oneMinusXSqInv15 * oneMinusXSqInv
|
||||
oneMinusXSqInv35 := oneMinusXSqInv25 * oneMinusXSqInv
|
||||
asinDer1DivF1Tab[i] = oneMinusXSqInv05
|
||||
asinDer2DivF2Tab[i] = (x * oneMinusXSqInv15) * oneDivF2
|
||||
asinDer3DivF3Tab[i] = ((1 + 2*x*x) * oneMinusXSqInv25) * oneDivF3
|
||||
asinDer4DivF4Tab[i] = ((5 + 2*x*(2+x*(5-2*x))) * oneMinusXSqInv35) * oneDivF4
|
||||
}
|
||||
|
||||
// earth radius
|
||||
a := 6378137.0
|
||||
b := 6356752.31420
|
||||
a2 := a * a
|
||||
b2 := b * b
|
||||
earthDiameterPerLatitude = make([]float64, radiusTabsSize)
|
||||
earthDiameterPerLatitude[0] = 2.0 * a / 1000
|
||||
earthDiameterPerLatitude[radiusTabsSize-1] = 2.0 * b / 1000
|
||||
for i := 1; i < radiusTabsSize-1; i++ {
|
||||
lat := math.Pi * float64(i) / (2*radiusTabsSize - 1)
|
||||
one := math.Pow(a2*math.Cos(lat), 2)
|
||||
two := math.Pow(b2*math.Sin(lat), 2)
|
||||
three := math.Pow(float64(a)*math.Cos(lat), 2)
|
||||
four := math.Pow(b*math.Sin(lat), 2)
|
||||
radius := math.Sqrt((one + two) / (three + four))
|
||||
earthDiameterPerLatitude[i] = 2 * radius / 1000
|
||||
}
|
||||
}
|
||||
|
||||
// earthDiameter returns an estimation of the earth's diameter at the specified
|
||||
// latitude in kilometers
|
||||
func earthDiameter(lat float64) float64 {
|
||||
index := math.Mod(math.Abs(lat)*radiusIndexer+0.5, float64(len(earthDiameterPerLatitude)))
|
||||
if math.IsNaN(index) {
|
||||
return 0
|
||||
}
|
||||
return earthDiameterPerLatitude[int(index)]
|
||||
}
|
||||
|
||||
var pio2 = math.Pi / 2
|
||||
|
||||
func sin(a float64) float64 {
|
||||
return cos(a - pio2)
|
||||
}
|
||||
|
||||
// cos is a sloppy math (faster) implementation of math.Cos
|
||||
func cos(a float64) float64 {
|
||||
if a < 0.0 {
|
||||
a = -a
|
||||
}
|
||||
if a > sinCosMaxValueForIntModulo {
|
||||
return math.Cos(a)
|
||||
}
|
||||
// index: possibly outside tables range.
|
||||
index := int(a*sinCosIndexer + 0.5)
|
||||
delta := (a - float64(index)*sinCosDeltaHi) - float64(index)*sinCosDeltaLo
|
||||
// Making sure index is within tables range.
|
||||
// Last value of each table is the same than first, so we ignore it (tabs size minus one) for modulo.
|
||||
index &= (sinCosTabsSize - 2) // index % (SIN_COS_TABS_SIZE-1)
|
||||
indexCos := cosTab[index]
|
||||
indexSin := sinTab[index]
|
||||
return indexCos + delta*(-indexSin+delta*(-indexCos*oneDivF2+delta*(indexSin*oneDivF3+delta*indexCos*oneDivF4)))
|
||||
}
|
||||
|
||||
// asin is a sloppy math (faster) implementation of math.Asin
|
||||
func asin(a float64) float64 {
|
||||
var negateResult bool
|
||||
if a < 0 {
|
||||
a = -a
|
||||
negateResult = true
|
||||
}
|
||||
if a <= asinMaxValueForTabs {
|
||||
index := int(a*asinIndexer + 0.5)
|
||||
delta := a - float64(index)*asinDelta
|
||||
result := asinTab[index] + delta*(asinDer1DivF1Tab[index]+delta*(asinDer2DivF2Tab[index]+delta*(asinDer3DivF3Tab[index]+delta*asinDer4DivF4Tab[index])))
|
||||
if negateResult {
|
||||
return -result
|
||||
}
|
||||
return result
|
||||
}
|
||||
// value > ASIN_MAX_VALUE_FOR_TABS, or value is NaN
|
||||
// This part is derived from fdlibm.
|
||||
if a < 1 {
|
||||
t := (1.0 - a) * 0.5
|
||||
p := t * (asinPs0 + t*(asinPs1+t*(asinPs2+t*(asinPs3+t*(asinPs4+t+asinPs5)))))
|
||||
q := 1.0 + t*(asinQs1+t*(asinQs2+t*(asinQs3+t*asinQs4)))
|
||||
s := math.Sqrt(t)
|
||||
z := s + s*(p/q)
|
||||
result := asinPio2Hi - ((z + z) - asinPio2Lo)
|
||||
if negateResult {
|
||||
return -result
|
||||
}
|
||||
return result
|
||||
}
|
||||
// value >= 1.0, or value is NaN
|
||||
if a == 1.0 {
|
||||
if negateResult {
|
||||
return -math.Pi / 2
|
||||
}
|
||||
return math.Pi / 2
|
||||
}
|
||||
return math.NaN()
|
||||
}
|
@ -0,0 +1,43 @@
|
||||
package numeric
|
||||
|
||||
var interleaveMagic = []uint64{
|
||||
0x5555555555555555,
|
||||
0x3333333333333333,
|
||||
0x0F0F0F0F0F0F0F0F,
|
||||
0x00FF00FF00FF00FF,
|
||||
0x0000FFFF0000FFFF,
|
||||
0x00000000FFFFFFFF,
|
||||
0xAAAAAAAAAAAAAAAA,
|
||||
}
|
||||
|
||||
var interleaveShift = []uint{1, 2, 4, 8, 16}
|
||||
|
||||
// Interleave the first 32 bits of each uint64
|
||||
// apdated from org.apache.lucene.util.BitUtil
|
||||
// whcih was adapted from:
|
||||
// http://graphics.stanford.edu/~seander/bithacks.html#InterleaveBMN
|
||||
func Interleave(v1, v2 uint64) uint64 {
|
||||
v1 = (v1 | (v1 << interleaveShift[4])) & interleaveMagic[4]
|
||||
v1 = (v1 | (v1 << interleaveShift[3])) & interleaveMagic[3]
|
||||
v1 = (v1 | (v1 << interleaveShift[2])) & interleaveMagic[2]
|
||||
v1 = (v1 | (v1 << interleaveShift[1])) & interleaveMagic[1]
|
||||
v1 = (v1 | (v1 << interleaveShift[0])) & interleaveMagic[0]
|
||||
v2 = (v2 | (v2 << interleaveShift[4])) & interleaveMagic[4]
|
||||
v2 = (v2 | (v2 << interleaveShift[3])) & interleaveMagic[3]
|
||||
v2 = (v2 | (v2 << interleaveShift[2])) & interleaveMagic[2]
|
||||
v2 = (v2 | (v2 << interleaveShift[1])) & interleaveMagic[1]
|
||||
v2 = (v2 | (v2 << interleaveShift[0])) & interleaveMagic[0]
|
||||
return (v2 << 1) | v1
|
||||
}
|
||||
|
||||
// Deinterleave the 32-bit value starting at position 0
|
||||
// to get the other 32-bit value, shift it by 1 first
|
||||
func Deinterleave(b uint64) uint64 {
|
||||
b &= interleaveMagic[0]
|
||||
b = (b ^ (b >> interleaveShift[0])) & interleaveMagic[1]
|
||||
b = (b ^ (b >> interleaveShift[1])) & interleaveMagic[2]
|
||||
b = (b ^ (b >> interleaveShift[2])) & interleaveMagic[3]
|
||||
b = (b ^ (b >> interleaveShift[3])) & interleaveMagic[4]
|
||||
b = (b ^ (b >> interleaveShift[4])) & interleaveMagic[5]
|
||||
return b
|
||||
}
|
@ -0,0 +1,113 @@
|
||||
// Copyright (c) 2017 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package query
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/blevesearch/bleve/geo"
|
||||
"github.com/blevesearch/bleve/index"
|
||||
"github.com/blevesearch/bleve/mapping"
|
||||
"github.com/blevesearch/bleve/search"
|
||||
"github.com/blevesearch/bleve/search/searcher"
|
||||
)
|
||||
|
||||
type GeoBoundingBoxQuery struct {
|
||||
TopLeft []float64 `json:"top_left,omitempty"`
|
||||
BottomRight []float64 `json:"bottom_right,omitempty"`
|
||||
FieldVal string `json:"field,omitempty"`
|
||||
BoostVal *Boost `json:"boost,omitempty"`
|
||||
}
|
||||
|
||||
func NewGeoBoundingBoxQuery(topLeftLon, topLeftLat, bottomRightLon, bottomRightLat float64) *GeoBoundingBoxQuery {
|
||||
return &GeoBoundingBoxQuery{
|
||||
TopLeft: []float64{topLeftLon, topLeftLat},
|
||||
BottomRight: []float64{bottomRightLon, bottomRightLat},
|
||||
}
|
||||
}
|
||||
|
||||
func (q *GeoBoundingBoxQuery) SetBoost(b float64) {
|
||||
boost := Boost(b)
|
||||
q.BoostVal = &boost
|
||||
}
|
||||
|
||||
func (q *GeoBoundingBoxQuery) Boost() float64 {
|
||||
return q.BoostVal.Value()
|
||||
}
|
||||
|
||||
func (q *GeoBoundingBoxQuery) SetField(f string) {
|
||||
q.FieldVal = f
|
||||
}
|
||||
|
||||
func (q *GeoBoundingBoxQuery) Field() string {
|
||||
return q.FieldVal
|
||||
}
|
||||
|
||||
func (q *GeoBoundingBoxQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {
|
||||
field := q.FieldVal
|
||||
if q.FieldVal == "" {
|
||||
field = m.DefaultSearchField()
|
||||
}
|
||||
|
||||
if q.BottomRight[0] < q.TopLeft[0] {
|
||||
// cross date line, rewrite as two parts
|
||||
|
||||
leftSearcher, err := searcher.NewGeoBoundingBoxSearcher(i, -180, q.BottomRight[1], q.BottomRight[0], q.TopLeft[1], field, q.BoostVal.Value(), options, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rightSearcher, err := searcher.NewGeoBoundingBoxSearcher(i, q.TopLeft[0], q.BottomRight[1], 180, q.TopLeft[1], field, q.BoostVal.Value(), options, true)
|
||||
if err != nil {
|
||||
_ = leftSearcher.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return searcher.NewDisjunctionSearcher(i, []search.Searcher{leftSearcher, rightSearcher}, 0, options)
|
||||
}
|
||||
|
||||
return searcher.NewGeoBoundingBoxSearcher(i, q.TopLeft[0], q.BottomRight[1], q.BottomRight[0], q.TopLeft[1], field, q.BoostVal.Value(), options, true)
|
||||
}
|
||||
|
||||
func (q *GeoBoundingBoxQuery) Validate() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *GeoBoundingBoxQuery) UnmarshalJSON(data []byte) error {
|
||||
tmp := struct {
|
||||
TopLeft interface{} `json:"top_left,omitempty"`
|
||||
BottomRight interface{} `json:"bottom_right,omitempty"`
|
||||
FieldVal string `json:"field,omitempty"`
|
||||
BoostVal *Boost `json:"boost,omitempty"`
|
||||
}{}
|
||||
err := json.Unmarshal(data, &tmp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// now use our generic point parsing code from the geo package
|
||||
lon, lat, found := geo.ExtractGeoPoint(tmp.TopLeft)
|
||||
if !found {
|
||||
return fmt.Errorf("geo location top_left not in a valid format")
|
||||
}
|
||||
q.TopLeft = []float64{lon, lat}
|
||||
lon, lat, found = geo.ExtractGeoPoint(tmp.BottomRight)
|
||||
if !found {
|
||||
return fmt.Errorf("geo location bottom_right not in a valid format")
|
||||
}
|
||||
q.BottomRight = []float64{lon, lat}
|
||||
q.FieldVal = tmp.FieldVal
|
||||
q.BoostVal = tmp.BoostVal
|
||||
return nil
|
||||
}
|
@ -0,0 +1,100 @@
|
||||
// Copyright (c) 2017 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package query
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/blevesearch/bleve/geo"
|
||||
"github.com/blevesearch/bleve/index"
|
||||
"github.com/blevesearch/bleve/mapping"
|
||||
"github.com/blevesearch/bleve/search"
|
||||
"github.com/blevesearch/bleve/search/searcher"
|
||||
)
|
||||
|
||||
type GeoDistanceQuery struct {
|
||||
Location []float64 `json:"location,omitempty"`
|
||||
Distance string `json:"distance,omitempty"`
|
||||
FieldVal string `json:"field,omitempty"`
|
||||
BoostVal *Boost `json:"boost,omitempty"`
|
||||
}
|
||||
|
||||
func NewGeoDistanceQuery(lon, lat float64, distance string) *GeoDistanceQuery {
|
||||
return &GeoDistanceQuery{
|
||||
Location: []float64{lon, lat},
|
||||
Distance: distance,
|
||||
}
|
||||
}
|
||||
|
||||
func (q *GeoDistanceQuery) SetBoost(b float64) {
|
||||
boost := Boost(b)
|
||||
q.BoostVal = &boost
|
||||
}
|
||||
|
||||
func (q *GeoDistanceQuery) Boost() float64 {
|
||||
return q.BoostVal.Value()
|
||||
}
|
||||
|
||||
func (q *GeoDistanceQuery) SetField(f string) {
|
||||
q.FieldVal = f
|
||||
}
|
||||
|
||||
func (q *GeoDistanceQuery) Field() string {
|
||||
return q.FieldVal
|
||||
}
|
||||
|
||||
func (q *GeoDistanceQuery) Searcher(i index.IndexReader, m mapping.IndexMapping,
|
||||
options search.SearcherOptions) (search.Searcher, error) {
|
||||
field := q.FieldVal
|
||||
if q.FieldVal == "" {
|
||||
field = m.DefaultSearchField()
|
||||
}
|
||||
|
||||
dist, err := geo.ParseDistance(q.Distance)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return searcher.NewGeoPointDistanceSearcher(i, q.Location[0], q.Location[1],
|
||||
dist, field, q.BoostVal.Value(), options)
|
||||
}
|
||||
|
||||
func (q *GeoDistanceQuery) Validate() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *GeoDistanceQuery) UnmarshalJSON(data []byte) error {
|
||||
tmp := struct {
|
||||
Location interface{} `json:"location,omitempty"`
|
||||
Distance string `json:"distance,omitempty"`
|
||||
FieldVal string `json:"field,omitempty"`
|
||||
BoostVal *Boost `json:"boost,omitempty"`
|
||||
}{}
|
||||
err := json.Unmarshal(data, &tmp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// now use our generic point parsing code from the geo package
|
||||
lon, lat, found := geo.ExtractGeoPoint(tmp.Location)
|
||||
if !found {
|
||||
return fmt.Errorf("geo location not in a valid format")
|
||||
}
|
||||
q.Location = []float64{lon, lat}
|
||||
q.Distance = tmp.Distance
|
||||
q.FieldVal = tmp.FieldVal
|
||||
q.BoostVal = tmp.BoostVal
|
||||
return nil
|
||||
}
|
@ -0,0 +1,80 @@
|
||||
// Copyright (c) 2014 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package query
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/blevesearch/bleve/index"
|
||||
"github.com/blevesearch/bleve/mapping"
|
||||
"github.com/blevesearch/bleve/search"
|
||||
"github.com/blevesearch/bleve/search/searcher"
|
||||
)
|
||||
|
||||
type MultiPhraseQuery struct {
|
||||
Terms [][]string `json:"terms"`
|
||||
Field string `json:"field,omitempty"`
|
||||
BoostVal *Boost `json:"boost,omitempty"`
|
||||
}
|
||||
|
||||
// NewMultiPhraseQuery creates a new Query for finding
|
||||
// term phrases in the index.
|
||||
// It is like PhraseQuery, but each position in the
|
||||
// phrase may be satisfied by a list of terms
|
||||
// as opposed to just one.
|
||||
// At least one of the terms must exist in the correct
|
||||
// order, at the correct index offsets, in the
|
||||
// specified field. Queried field must have been indexed with
|
||||
// IncludeTermVectors set to true.
|
||||
func NewMultiPhraseQuery(terms [][]string, field string) *MultiPhraseQuery {
|
||||
return &MultiPhraseQuery{
|
||||
Terms: terms,
|
||||
Field: field,
|
||||
}
|
||||
}
|
||||
|
||||
func (q *MultiPhraseQuery) SetBoost(b float64) {
|
||||
boost := Boost(b)
|
||||
q.BoostVal = &boost
|
||||
}
|
||||
|
||||
func (q *MultiPhraseQuery) Boost() float64 {
|
||||
return q.BoostVal.Value()
|
||||
}
|
||||
|
||||
func (q *MultiPhraseQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {
|
||||
return searcher.NewMultiPhraseSearcher(i, q.Terms, q.Field, options)
|
||||
}
|
||||
|
||||
func (q *MultiPhraseQuery) Validate() error {
|
||||
if len(q.Terms) < 1 {
|
||||
return fmt.Errorf("phrase query must contain at least one term")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *MultiPhraseQuery) UnmarshalJSON(data []byte) error {
|
||||
type _mphraseQuery MultiPhraseQuery
|
||||
tmp := _mphraseQuery{}
|
||||
err := json.Unmarshal(data, &tmp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
q.Terms = tmp.Terms
|
||||
q.Field = tmp.Field
|
||||
q.BoostVal = tmp.BoostVal
|
||||
return nil
|
||||
}
|
@ -0,0 +1,95 @@
|
||||
// Copyright (c) 2017 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package query
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/blevesearch/bleve/index"
|
||||
"github.com/blevesearch/bleve/mapping"
|
||||
"github.com/blevesearch/bleve/search"
|
||||
"github.com/blevesearch/bleve/search/searcher"
|
||||
)
|
||||
|
||||
type TermRangeQuery struct {
|
||||
Min string `json:"min,omitempty"`
|
||||
Max string `json:"max,omitempty"`
|
||||
InclusiveMin *bool `json:"inclusive_min,omitempty"`
|
||||
InclusiveMax *bool `json:"inclusive_max,omitempty"`
|
||||
FieldVal string `json:"field,omitempty"`
|
||||
BoostVal *Boost `json:"boost,omitempty"`
|
||||
}
|
||||
|
||||
// NewTermRangeQuery creates a new Query for ranges
|
||||
// of text term values.
|
||||
// Either, but not both endpoints can be nil.
|
||||
// The minimum value is inclusive.
|
||||
// The maximum value is exclusive.
|
||||
func NewTermRangeQuery(min, max string) *TermRangeQuery {
|
||||
return NewTermRangeInclusiveQuery(min, max, nil, nil)
|
||||
}
|
||||
|
||||
// NewTermRangeInclusiveQuery creates a new Query for ranges
|
||||
// of numeric values.
|
||||
// Either, but not both endpoints can be nil.
|
||||
// Control endpoint inclusion with inclusiveMin, inclusiveMax.
|
||||
func NewTermRangeInclusiveQuery(min, max string, minInclusive, maxInclusive *bool) *TermRangeQuery {
|
||||
return &TermRangeQuery{
|
||||
Min: min,
|
||||
Max: max,
|
||||
InclusiveMin: minInclusive,
|
||||
InclusiveMax: maxInclusive,
|
||||
}
|
||||
}
|
||||
|
||||
func (q *TermRangeQuery) SetBoost(b float64) {
|
||||
boost := Boost(b)
|
||||
q.BoostVal = &boost
|
||||
}
|
||||
|
||||
func (q *TermRangeQuery) Boost() float64 {
|
||||
return q.BoostVal.Value()
|
||||
}
|
||||
|
||||
func (q *TermRangeQuery) SetField(f string) {
|
||||
q.FieldVal = f
|
||||
}
|
||||
|
||||
func (q *TermRangeQuery) Field() string {
|
||||
return q.FieldVal
|
||||
}
|
||||
|
||||
func (q *TermRangeQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {
|
||||
field := q.FieldVal
|
||||
if q.FieldVal == "" {
|
||||
field = m.DefaultSearchField()
|
||||
}
|
||||
var minTerm []byte
|
||||
if q.Min != "" {
|
||||
minTerm = []byte(q.Min)
|
||||
}
|
||||
var maxTerm []byte
|
||||
if q.Max != "" {
|
||||
maxTerm = []byte(q.Max)
|
||||
}
|
||||
return searcher.NewTermRangeSearcher(i, minTerm, maxTerm, q.InclusiveMin, q.InclusiveMax, field, q.BoostVal.Value(), options)
|
||||
}
|
||||
|
||||
func (q *TermRangeQuery) Validate() error {
|
||||
if q.Min == "" && q.Min == q.Max {
|
||||
return fmt.Errorf("term range query must specify min or max")
|
||||
}
|
||||
return nil
|
||||
}
|
@ -0,0 +1,88 @@
|
||||
// Copyright (c) 2017 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"github.com/blevesearch/bleve/index"
|
||||
"github.com/blevesearch/bleve/search"
|
||||
)
|
||||
|
||||
// FilterFunc defines a function which can filter documents
|
||||
// returning true means keep the document
|
||||
// returning false means do not keep the document
|
||||
type FilterFunc func(d *search.DocumentMatch) bool
|
||||
|
||||
// FilteringSearcher wraps any other searcher, but checks any Next/Advance
|
||||
// call against the supplied FilterFunc
|
||||
type FilteringSearcher struct {
|
||||
child search.Searcher
|
||||
accept FilterFunc
|
||||
}
|
||||
|
||||
func NewFilteringSearcher(s search.Searcher, filter FilterFunc) *FilteringSearcher {
|
||||
return &FilteringSearcher{
|
||||
child: s,
|
||||
accept: filter,
|
||||
}
|
||||
}
|
||||
|
||||
func (f *FilteringSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
|
||||
next, err := f.child.Next(ctx)
|
||||
for next != nil && err == nil {
|
||||
if f.accept(next) {
|
||||
return next, nil
|
||||
}
|
||||
next, err = f.child.Next(ctx)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func (f *FilteringSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
|
||||
adv, err := f.child.Advance(ctx, ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if adv == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if f.accept(adv) {
|
||||
return adv, nil
|
||||
}
|
||||
return f.Next(ctx)
|
||||
}
|
||||
|
||||
func (f *FilteringSearcher) Close() error {
|
||||
return f.child.Close()
|
||||
}
|
||||
|
||||
func (f *FilteringSearcher) Weight() float64 {
|
||||
return f.child.Weight()
|
||||
}
|
||||
|
||||
func (f *FilteringSearcher) SetQueryNorm(n float64) {
|
||||
f.child.SetQueryNorm(n)
|
||||
}
|
||||
|
||||
func (f *FilteringSearcher) Count() uint64 {
|
||||
return f.child.Count()
|
||||
}
|
||||
|
||||
func (f *FilteringSearcher) Min() int {
|
||||
return f.child.Min()
|
||||
}
|
||||
|
||||
func (f *FilteringSearcher) DocumentMatchPoolSize() int {
|
||||
return f.child.DocumentMatchPoolSize()
|
||||
}
|
@ -0,0 +1,173 @@
|
||||
// Copyright (c) 2017 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"github.com/blevesearch/bleve/document"
|
||||
"github.com/blevesearch/bleve/geo"
|
||||
"github.com/blevesearch/bleve/index"
|
||||
"github.com/blevesearch/bleve/numeric"
|
||||
"github.com/blevesearch/bleve/search"
|
||||
)
|
||||
|
||||
func NewGeoBoundingBoxSearcher(indexReader index.IndexReader, minLon, minLat,
|
||||
maxLon, maxLat float64, field string, boost float64,
|
||||
options search.SearcherOptions, checkBoundaries bool) (
|
||||
search.Searcher, error) {
|
||||
|
||||
// track list of opened searchers, for cleanup on early exit
|
||||
var openedSearchers []search.Searcher
|
||||
cleanupOpenedSearchers := func() {
|
||||
for _, s := range openedSearchers {
|
||||
_ = s.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// do math to produce list of terms needed for this search
|
||||
onBoundaryTerms, notOnBoundaryTerms := ComputeGeoRange(0, (geo.GeoBits<<1)-1,
|
||||
minLon, minLat, maxLon, maxLat, checkBoundaries)
|
||||
|
||||
var onBoundarySearcher search.Searcher
|
||||
if len(onBoundaryTerms) > 0 {
|
||||
rawOnBoundarySearcher, err := NewMultiTermSearcherBytes(indexReader,
|
||||
onBoundaryTerms, field, boost, options, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// add filter to check points near the boundary
|
||||
onBoundarySearcher = NewFilteringSearcher(rawOnBoundarySearcher,
|
||||
buildRectFilter(indexReader, field, minLon, minLat, maxLon, maxLat))
|
||||
openedSearchers = append(openedSearchers, onBoundarySearcher)
|
||||
}
|
||||
|
||||
var notOnBoundarySearcher search.Searcher
|
||||
if len(notOnBoundaryTerms) > 0 {
|
||||
var err error
|
||||
notOnBoundarySearcher, err = NewMultiTermSearcherBytes(indexReader,
|
||||
notOnBoundaryTerms, field, boost, options, false)
|
||||
if err != nil {
|
||||
cleanupOpenedSearchers()
|
||||
return nil, err
|
||||
}
|
||||
openedSearchers = append(openedSearchers, notOnBoundarySearcher)
|
||||
}
|
||||
|
||||
if onBoundarySearcher != nil && notOnBoundarySearcher != nil {
|
||||
rv, err := NewDisjunctionSearcher(indexReader,
|
||||
[]search.Searcher{
|
||||
onBoundarySearcher,
|
||||
notOnBoundarySearcher,
|
||||
},
|
||||
0, options)
|
||||
if err != nil {
|
||||
cleanupOpenedSearchers()
|
||||
return nil, err
|
||||
}
|
||||
return rv, nil
|
||||
} else if onBoundarySearcher != nil {
|
||||
return onBoundarySearcher, nil
|
||||
} else if notOnBoundarySearcher != nil {
|
||||
return notOnBoundarySearcher, nil
|
||||
}
|
||||
|
||||
return NewMatchNoneSearcher(indexReader)
|
||||
}
|
||||
|
||||
var geoMaxShift = document.GeoPrecisionStep * 4
|
||||
var geoDetailLevel = ((geo.GeoBits << 1) - geoMaxShift) / 2
|
||||
|
||||
func ComputeGeoRange(term uint64, shift uint,
|
||||
sminLon, sminLat, smaxLon, smaxLat float64,
|
||||
checkBoundaries bool) (
|
||||
onBoundary [][]byte, notOnBoundary [][]byte) {
|
||||
split := term | uint64(0x1)<<shift
|
||||
var upperMax uint64
|
||||
if shift < 63 {
|
||||
upperMax = term | ((uint64(1) << (shift + 1)) - 1)
|
||||
} else {
|
||||
upperMax = 0xffffffffffffffff
|
||||
}
|
||||
lowerMax := split - 1
|
||||
onBoundary, notOnBoundary = relateAndRecurse(term, lowerMax, shift,
|
||||
sminLon, sminLat, smaxLon, smaxLat, checkBoundaries)
|
||||
plusOnBoundary, plusNotOnBoundary := relateAndRecurse(split, upperMax, shift,
|
||||
sminLon, sminLat, smaxLon, smaxLat, checkBoundaries)
|
||||
onBoundary = append(onBoundary, plusOnBoundary...)
|
||||
notOnBoundary = append(notOnBoundary, plusNotOnBoundary...)
|
||||
return
|
||||
}
|
||||
|
||||
func relateAndRecurse(start, end uint64, res uint,
|
||||
sminLon, sminLat, smaxLon, smaxLat float64,
|
||||
checkBoundaries bool) (
|
||||
onBoundary [][]byte, notOnBoundary [][]byte) {
|
||||
minLon := geo.MortonUnhashLon(start)
|
||||
minLat := geo.MortonUnhashLat(start)
|
||||
maxLon := geo.MortonUnhashLon(end)
|
||||
maxLat := geo.MortonUnhashLat(end)
|
||||
|
||||
level := ((geo.GeoBits << 1) - res) >> 1
|
||||
|
||||
within := res%document.GeoPrecisionStep == 0 &&
|
||||
geo.RectWithin(minLon, minLat, maxLon, maxLat,
|
||||
sminLon, sminLat, smaxLon, smaxLat)
|
||||
if within || (level == geoDetailLevel &&
|
||||
geo.RectIntersects(minLon, minLat, maxLon, maxLat,
|
||||
sminLon, sminLat, smaxLon, smaxLat)) {
|
||||
if !within && checkBoundaries {
|
||||
return [][]byte{
|
||||
numeric.MustNewPrefixCodedInt64(int64(start), res),
|
||||
}, nil
|
||||
}
|
||||
return nil,
|
||||
[][]byte{
|
||||
numeric.MustNewPrefixCodedInt64(int64(start), res),
|
||||
}
|
||||
} else if level < geoDetailLevel &&
|
||||
geo.RectIntersects(minLon, minLat, maxLon, maxLat,
|
||||
sminLon, sminLat, smaxLon, smaxLat) {
|
||||
return ComputeGeoRange(start, res-1, sminLon, sminLat, smaxLon, smaxLat,
|
||||
checkBoundaries)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func buildRectFilter(indexReader index.IndexReader, field string,
|
||||
minLon, minLat, maxLon, maxLat float64) FilterFunc {
|
||||
return func(d *search.DocumentMatch) bool {
|
||||
var lon, lat float64
|
||||
var found bool
|
||||
err := indexReader.DocumentVisitFieldTerms(d.IndexInternalID,
|
||||
[]string{field}, func(field string, term []byte) {
|
||||
// only consider the values which are shifted 0
|
||||
prefixCoded := numeric.PrefixCoded(term)
|
||||
shift, err := prefixCoded.Shift()
|
||||
if err == nil && shift == 0 {
|
||||
var i64 int64
|
||||
i64, err = prefixCoded.Int64()
|
||||
if err == nil {
|
||||
lon = geo.MortonUnhashLon(uint64(i64))
|
||||
lat = geo.MortonUnhashLat(uint64(i64))
|
||||
found = true
|
||||
}
|
||||
}
|
||||
})
|
||||
if err == nil && found {
|
||||
return geo.BoundingBoxContains(lon, lat,
|
||||
minLon, minLat, maxLon, maxLat)
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue