mirrors
/
forgejo
mirror of
1
Fork 0
Code Issues Releases Activity

enh(search): improve issue search 

- new sort by relevency option for issue search
- rework bleve fuzzy search to consider each term rather than matching the entire phrase
This commit is contained in:
Shiny Nematoda 2024-11-01 13:06:02 +00:00
parent c8ba330833
commit a265574821
10 changed files with 56 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
modules/indexer/internal/bleve/query.go
View File

@ -19,6 +19,15 @@ func NumericEqualityQuery(value int64, field string) *query.NumericRangeQuery {
return q
}
// MatchQuery generates a match query for the given phrase, field and analyzer
func MatchQuery(matchTerm, field, analyzer string, fuzziness int) *query.MatchQuery {
q := bleve.NewMatchQuery(matchTerm)
q.FieldVal = field
q.Analyzer = analyzer
q.Fuzziness = fuzziness
return q
}
// MatchPhraseQuery generates a match phrase query for the given phrase, field and analyzer
func MatchPhraseQuery(matchPhrase, field, analyzer string, fuzziness int) *query.MatchPhraseQuery {
q := bleve.NewMatchPhraseQuery(matchPhrase)

29
modules/indexer/issues/bleve/bleve.go
View File

@ -35,13 +35,7 @@ func addUnicodeNormalizeTokenFilter(m *mapping.IndexMappingImpl) error {
})
}
const (
maxBatchSize = 16
// fuzzyDenominator determines the levenshtein distance per each character of a keyword
fuzzyDenominator = 4
// see https://github.com/blevesearch/bleve/issues/1563#issuecomment-786822311
maxFuzziness = 2
)
const maxBatchSize = 16
// IndexerData an update to the issue indexer
type IndexerData internal.IndexerData
@ -162,16 +156,25 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
var queries []query.Query
if options.Keyword != "" {
fuzziness := 0
if options.IsFuzzyKeyword {
fuzziness = min(maxFuzziness, len(options.Keyword)/fuzzyDenominator)
fuzziness := 1
if kl := len(options.Keyword); kl > 3 {
fuzziness = 2
} else if kl < 2 {
fuzziness = 0
}
queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{
inner_bleve.MatchPhraseQuery(options.Keyword, "title", issueIndexerAnalyzer, fuzziness),
inner_bleve.MatchPhraseQuery(options.Keyword, "content", issueIndexerAnalyzer, fuzziness),
inner_bleve.MatchPhraseQuery(options.Keyword, "comments", issueIndexerAnalyzer, fuzziness),
inner_bleve.MatchQuery(options.Keyword, "title", issueIndexerAnalyzer, fuzziness),
inner_bleve.MatchQuery(options.Keyword, "content", issueIndexerAnalyzer, fuzziness),
inner_bleve.MatchQuery(options.Keyword, "comments", issueIndexerAnalyzer, fuzziness),
}...))
} else {
queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{
inner_bleve.MatchPhraseQuery(options.Keyword, "title", issueIndexerAnalyzer, 0),
inner_bleve.MatchPhraseQuery(options.Keyword, "content", issueIndexerAnalyzer, 0),
inner_bleve.MatchPhraseQuery(options.Keyword, "comments", issueIndexerAnalyzer, 0),
}...))
}
}
if len(options.RepoIDs) > 0 || options.AllPublic {

4
modules/indexer/issues/dboptions.go
View File

@ -78,7 +78,9 @@ func ToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *SearchOp
searchOpt.Paginator = opts.Paginator
switch opts.SortType {
case "", "latest":
case "", "relevance":
searchOpt.SortBy = SortByScore
case "latest":
searchOpt.SortBy = SortByCreatedDesc
case "oldest":
searchOpt.SortBy = SortByCreatedAsc

2
modules/indexer/issues/elasticsearch/elasticsearch.go
View File

@ -236,7 +236,7 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
}
if options.SortBy == "" {
options.SortBy = internal.SortByCreatedAsc
options.SortBy = internal.SortByScore
}
sortBy := []elastic.Sorter{
parseSortBy(options.SortBy),

1
modules/indexer/issues/indexer.go
View File

@ -269,6 +269,7 @@ func IsAvailable(ctx context.Context) bool {
type SearchOptions = internal.SearchOptions
const (
SortByScore = internal.SortByScore
SortByCreatedDesc = internal.SortByCreatedDesc
SortByUpdatedDesc = internal.SortByUpdatedDesc
SortByCommentsDesc = internal.SortByCommentsDesc

1
modules/indexer/issues/internal/model.go
View File

@ -127,6 +127,7 @@ func (o *SearchOptions) Copy(edit ...func(options *SearchOptions)) *SearchOption
type SortBy string
const (
SortByScore SortBy = "-_score"
SortByCreatedDesc SortBy = "-created_unix"
SortByUpdatedDesc SortBy = "-updated_unix"
SortByCommentsDesc SortBy = "-comment_count"

14
modules/indexer/issues/meilisearch/meilisearch.go
View File

@ -208,13 +208,19 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
query.And(inner_meilisearch.NewFilterLte("updated_unix", options.UpdatedBeforeUnix.Value()))
}
if options.SortBy == "" {
options.SortBy = internal.SortByCreatedAsc
}
sortBy := []string{
var sortBy []string
switch options.SortBy {
// sort by relevancy (no explicit sorting)
case internal.SortByScore:
fallthrough
case "":
sortBy = []string{}
default:
sortBy = []string{
parseSortBy(options.SortBy),
"id:desc",
}
}
skip, limit := indexer_internal.ParsePaginator(options.Paginator, maxTotalHits)

4
modules/templates/util_string.go
View File

@ -19,6 +19,10 @@ func NewStringUtils() *StringUtils {
return &stringUtils
}
func (su *StringUtils) Make(arr ...string) []string {
return arr
}
func (su *StringUtils) HasPrefix(s any, prefix string) bool {
switch v := s.(type) {
case string:

1
options/locale/locale_en-US.ini
View File

@ -1586,6 +1586,7 @@ issues.filter_type.mentioning_you = Mentioning you
issues.filter_type.review_requested = Review requested
issues.filter_type.reviewed_by_you = Reviewed by you
issues.filter_sort = Sort
issues.filter_sort.relevance = Relevance
issues.filter_sort.latest = Newest
issues.filter_sort.oldest = Oldest
issues.filter_sort.recentupdate = Recently updated

14
templates/repo/issue/filter_list.tmpl
View File

@ -146,13 +146,11 @@
</span>
{{svg "octicon-triangle-down" 14 "dropdown icon"}}
<div class="menu">
<a rel="nofollow" class="{{if or (eq .SortType "latest") (not .SortType)}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=latest&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.latest"}}</a>
<a rel="nofollow" class="{{if eq .SortType "oldest"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=oldest&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.oldest"}}</a>
<a rel="nofollow" class="{{if eq .SortType "recentupdate"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=recentupdate&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.recentupdate"}}</a>
<a rel="nofollow" class="{{if eq .SortType "leastupdate"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=leastupdate&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.leastupdate"}}</a>
<a rel="nofollow" class="{{if eq .SortType "mostcomment"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=mostcomment&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.mostcomment"}}</a>
<a rel="nofollow" class="{{if eq .SortType "leastcomment"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=leastcomment&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.leastcomment"}}</a>
<a rel="nofollow" class="{{if eq .SortType "nearduedate"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=nearduedate&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.nearduedate"}}</a>
<a rel="nofollow" class="{{if eq .SortType "farduedate"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=farduedate&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.farduedate"}}</a>
<a rel="nofollow" class="{{if or (eq .SortType "relevance") (not .SortType)}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=relevency&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.relevance"}}</a>
{{$o := .}}
{{range $opt := StringUtils.Make "latest" "oldest" "recentupdate" "mostcomment" "leastcomment" "nearduedate" "farduedate"}}
{{$text := ctx.Locale.Tr (printf "repo.issues.filter_sort.%s" $opt)}}
<a rel="nofollow" class="{{if eq $o.SortType $opt}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort={{$opt}}&state={{$.State}}&labels={{$o.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{$text}}</a>
{{end}}
</div>
</div>