Skip to content

Commit 7eba450

Browse files
silvanocerzasilvanocerza
authored
Change core and lib search commands to use fuzzy search (arduino#1193)
* Change lib search command to use fuzzy search * Change core search command to use fuzzy search * Avoid splitting search arguments when doing fuzzy search * Check ranking when running fuzzy search * Some other enhancements to fuzzy search * Fix duplicated results in lib search command
1 parent b8c9e89 commit 7eba450

File tree

8 files changed

+235
-121
lines changed

8 files changed

+235
-121
lines changed

‎commands/core/search.go

Lines changed: 40 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -23,19 +23,16 @@ import (
2323
"github.com/arduino/arduino-cli/arduino/cores"
2424
"github.com/arduino/arduino-cli/commands"
2525
rpc "github.com/arduino/arduino-cli/rpc/commands"
26+
"github.com/lithammer/fuzzysearch/fuzzy"
2627
)
2728

28-
func match(line, searchArgs string) bool {
29-
return strings.Contains(strings.ToLower(line), strings.ToLower(searchArgs))
30-
}
31-
32-
func exactMatch(line, searchArgs string) bool {
33-
return strings.Compare(strings.ToLower(line), strings.ToLower(searchArgs)) == 0
34-
}
29+
// maximumSearchDistance is the maximum Levenshtein distance accepted when using fuzzy search.
30+
// This value is completely arbitrary and picked randomly.
31+
const maximumSearchDistance = 20
3532

3633
// PlatformSearch FIXMEDOC
3734
func PlatformSearch(req *rpc.PlatformSearchReq) (*rpc.PlatformSearchResp, error) {
38-
searchArgs := req.SearchArgs
35+
searchArgs := strings.Trim(req.SearchArgs, " ")
3936
allVersions := req.AllVersions
4037
pm := commands.GetPackageManager(req.Instance.Id)
4138
if pm == nil {
@@ -63,29 +60,54 @@ func PlatformSearch(req *rpc.PlatformSearchReq) (*rpc.PlatformSearchResp, error)
6360
continue
6461
}
6562

66-
// platform has a valid release, check if it matches the search arguments
67-
if match(platform.Name, searchArgs) || match(platform.Architecture, searchArgs) ||
68-
exactMatch(platform.String(), searchArgs) || match(targetPackage.Name, searchArgs) ||
69-
match(targetPackage.Maintainer, searchArgs) || match(targetPackage.WebsiteURL, searchArgs) {
63+
if searchArgs == "" {
7064
if allVersions {
7165
res = append(res, platform.GetAllReleases()...)
7266
} else {
7367
res = append(res, platformRelease)
7468
}
75-
} else {
76-
// if we didn't find a match in the platform data, search for
77-
// a match in the boards manifest
78-
for _, board := range platformRelease.BoardsManifest {
79-
if match(board.Name, searchArgs) {
69+
continue
70+
}
71+
72+
// Gather all strings that can be used for searching
73+
toTest := []string{
74+
platform.String(),
75+
platform.Name,
76+
platform.Architecture,
77+
targetPackage.Name,
78+
targetPackage.Maintainer,
79+
targetPackage.WebsiteURL,
80+
}
81+
for _, board := range platformRelease.BoardsManifest {
82+
toTest = append(toTest, board.Name)
83+
}
84+
85+
// Removes some chars from query strings to enhance results
86+
cleanSearchArgs := strings.Map(func(r rune) rune {
87+
switch r {
88+
case '_':
89+
case '-':
90+
case ' ':
91+
return -1
92+
}
93+
return r
94+
}, searchArgs)
95+
96+
// Fuzzy search
97+
for _, arg := range []string{searchArgs, cleanSearchArgs} {
98+
for _, rank := range fuzzy.RankFindNormalizedFold(arg, toTest) {
99+
// Accepts only results that close to the searched terms
100+
if rank.Distance < maximumSearchDistance {
80101
if allVersions {
81102
res = append(res, platform.GetAllReleases()...)
82103
} else {
83104
res = append(res, platformRelease)
84105
}
85-
break
106+
goto nextPlatform
86107
}
87108
}
88109
}
110+
nextPlatform:
89111
}
90112
}
91113
}

‎commands/core/search_test.go

Lines changed: 46 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,17 +24,9 @@ import (
2424
"github.com/arduino/arduino-cli/rpc/commands"
2525
rpc "github.com/arduino/arduino-cli/rpc/commands"
2626
"github.com/arduino/go-paths-helper"
27-
"github.com/stretchr/testify/assert"
2827
"github.com/stretchr/testify/require"
2928
)
3029

31-
func TestMatch(t *testing.T) {
32-
assert.True(t, match("this is platform Foo", "foo"))
33-
assert.True(t, match("this is platform Foo", "FOO"))
34-
assert.True(t, match("this is platform Foo", ""))
35-
assert.False(t, match("this is platform Foo", "Bar"))
36-
}
37-
3830
func TestPlatformSearch(t *testing.T) {
3931

4032
dataDir := paths.TempDir().Join("test", "data_dir")
@@ -238,4 +230,50 @@ func TestPlatformSearch(t *testing.T) {
238230
{Name: "Linino One"},
239231
},
240232
})
233+
234+
res, err = PlatformSearch(&rpc.PlatformSearchReq{
235+
Instance: inst,
236+
SearchArgs: "yun",
237+
AllVersions: true,
238+
})
239+
require.Nil(t, err)
240+
require.NotNil(t, res)
241+
require.Len(t, res.SearchOutput, 1)
242+
require.Contains(t, res.SearchOutput, &commands.Platform{
243+
ID: "arduino:avr",
244+
Installed: "",
245+
Latest: "1.8.3",
246+
Name: "Arduino AVR Boards",
247+
Maintainer: "Arduino",
248+
Website: "https://www.arduino.cc/",
249+
Email: "packages@arduino.cc",
250+
Boards: []*commands.Board{
251+
{Name: "Arduino Yún"},
252+
{Name: "Arduino Uno"},
253+
{Name: "Arduino Uno WiFi"},
254+
{Name: "Arduino Diecimila"},
255+
{Name: "Arduino Nano"},
256+
{Name: "Arduino Mega"},
257+
{Name: "Arduino MegaADK"},
258+
{Name: "Arduino Leonardo"},
259+
{Name: "Arduino Leonardo Ethernet"},
260+
{Name: "Arduino Micro"},
261+
{Name: "Arduino Esplora"},
262+
{Name: "Arduino Mini"},
263+
{Name: "Arduino Ethernet"},
264+
{Name: "Arduino Fio"},
265+
{Name: "Arduino BT"},
266+
{Name: "Arduino LilyPadUSB"},
267+
{Name: "Arduino Lilypad"},
268+
{Name: "Arduino Pro"},
269+
{Name: "Arduino ATMegaNG"},
270+
{Name: "Arduino Robot Control"},
271+
{Name: "Arduino Robot Motor"},
272+
{Name: "Arduino Gemma"},
273+
{Name: "Adafruit Circuit Playground"},
274+
{Name: "Arduino Yún Mini"},
275+
{Name: "Arduino Industrial 101"},
276+
{Name: "Linino One"},
277+
},
278+
})
241279
}

‎commands/lib/search.go

Lines changed: 52 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,10 @@ import (
2424
"github.com/arduino/arduino-cli/arduino/libraries/librariesmanager"
2525
"github.com/arduino/arduino-cli/commands"
2626
rpc "github.com/arduino/arduino-cli/rpc/commands"
27-
"github.com/imjasonmiller/godice"
27+
"github.com/lithammer/fuzzysearch/fuzzy"
2828
semver "go.bug.st/relaxed-semver"
2929
)
3030

31-
var similarityThreshold = 0.7
32-
3331
// LibrarySearch FIXMEDOC
3432
func LibrarySearch(ctx context.Context, req *rpc.LibrarySearchReq) (*rpc.LibrarySearchResp, error) {
3533
lm := commands.GetLibraryManager(req.GetInstance().GetId())
@@ -41,45 +39,70 @@ func LibrarySearch(ctx context.Context, req *rpc.LibrarySearchReq) (*rpc.Library
4139
}
4240

4341
func searchLibrary(req *rpc.LibrarySearchReq, lm *librariesmanager.LibrariesManager) (*rpc.LibrarySearchResp, error) {
42+
query := req.GetQuery()
4443
res := []*rpc.SearchedLibrary{}
4544
status := rpc.LibrarySearchStatus_success
4645

47-
for _, lib := range lm.Index.Libraries {
48-
qry := strings.ToLower(req.GetQuery())
49-
if strings.Contains(strings.ToLower(lib.Name), qry) ||
50-
strings.Contains(strings.ToLower(lib.Latest.Paragraph), qry) ||
51-
strings.Contains(strings.ToLower(lib.Latest.Sentence), qry) {
52-
releases := map[string]*rpc.LibraryRelease{}
53-
for str, rel := range lib.Releases {
54-
releases[str] = GetLibraryParameters(rel)
55-
}
56-
latest := GetLibraryParameters(lib.Latest)
57-
58-
searchedLib := &rpc.SearchedLibrary{
59-
Name: lib.Name,
60-
Releases: releases,
61-
Latest: latest,
62-
}
63-
res = append(res, searchedLib)
46+
// If the query is empty all libraries are returned
47+
if strings.Trim(query, " ") == "" {
48+
for _, lib := range lm.Index.Libraries {
49+
res = append(res, indexLibraryToRPCSearchLibrary(lib))
6450
}
51+
return &rpc.LibrarySearchResp{Libraries: res, Status: status}, nil
6552
}
6653

67-
if len(res) == 0 {
68-
status = rpc.LibrarySearchStatus_failed
69-
for _, lib := range lm.Index.Libraries {
70-
if godice.CompareString(req.GetQuery(), lib.Name) > similarityThreshold {
71-
res = append(res, &rpc.SearchedLibrary{
72-
Name: lib.Name,
73-
})
54+
// maximumSearchDistance is the maximum Levenshtein distance accepted when using fuzzy search.
55+
// This value is completely arbitrary and picked randomly.
56+
maximumSearchDistance := 150
57+
// Use a lower distance for shorter query or the user might be flooded with unrelated results
58+
if len(query) <= 4 {
59+
maximumSearchDistance = 40
60+
}
61+
62+
// Removes some chars from query strings to enhance results
63+
cleanQuery := strings.Map(func(r rune) rune {
64+
switch r {
65+
case '_':
66+
case '-':
67+
case ' ':
68+
return -1
69+
}
70+
return r
71+
}, query)
72+
for _, lib := range lm.Index.Libraries {
73+
// Use both uncleaned and cleaned query
74+
for _, q := range []string{query, cleanQuery} {
75+
toTest := []string{lib.Name, lib.Latest.Paragraph, lib.Latest.Sentence}
76+
for _, rank := range fuzzy.RankFindNormalizedFold(q, toTest) {
77+
if rank.Distance < maximumSearchDistance {
78+
res = append(res, indexLibraryToRPCSearchLibrary(lib))
79+
goto nextLib
80+
}
7481
}
7582
}
83+
nextLib:
7684
}
7785

7886
return &rpc.LibrarySearchResp{Libraries: res, Status: status}, nil
7987
}
8088

81-
// GetLibraryParameters FIXMEDOC
82-
func GetLibraryParameters(rel *librariesindex.Release) *rpc.LibraryRelease {
89+
// indexLibraryToRPCSearchLibrary converts a librariindex.Library to rpc.SearchLibrary
90+
func indexLibraryToRPCSearchLibrary(lib *librariesindex.Library) *rpc.SearchedLibrary {
91+
releases := map[string]*rpc.LibraryRelease{}
92+
for str, rel := range lib.Releases {
93+
releases[str] = getLibraryParameters(rel)
94+
}
95+
latest := getLibraryParameters(lib.Latest)
96+
97+
return &rpc.SearchedLibrary{
98+
Name: lib.Name,
99+
Releases: releases,
100+
Latest: latest,
101+
}
102+
}
103+
104+
// getLibraryParameters FIXMEDOC
105+
func getLibraryParameters(rel *librariesindex.Release) *rpc.LibraryRelease {
83106
return &rpc.LibraryRelease{
84107
Author: rel.Author,
85108
Version: rel.Version.String(),

‎commands/lib/search_test.go

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,12 @@ func TestSearchLibrarySimilar(t *testing.T) {
4848
}
4949

5050
assert := assert.New(t)
51-
assert.Equal(resp.GetStatus(), rpc.LibrarySearchStatus_failed)
52-
assert.Equal(len(resp.GetLibraries()), 1)
53-
assert.Equal(resp.GetLibraries()[0].Name, "Arduino")
51+
assert.Equal(resp.GetStatus(), rpc.LibrarySearchStatus_success)
52+
assert.Equal(len(resp.GetLibraries()), 2)
53+
libs := map[string]*rpc.SearchedLibrary{}
54+
for _, l := range resp.GetLibraries() {
55+
libs[l.Name] = l
56+
}
57+
assert.Contains(libs, "ArduinoTestPackage")
58+
assert.Contains(libs, "Arduino")
5459
}

‎go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,10 @@ require (
1919
github.com/gofrs/uuid v3.2.0+incompatible
2020
github.com/golang/protobuf v1.4.2
2121
github.com/h2non/filetype v1.0.8 // indirect
22-
github.com/imjasonmiller/godice v0.1.2
2322
github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 // indirect
2423
github.com/kr/text v0.2.0 // indirect
2524
github.com/leonelquinteros/gotext v1.4.0
25+
github.com/lithammer/fuzzysearch v1.1.1
2626
github.com/mattn/go-colorable v0.1.2
2727
github.com/mattn/go-isatty v0.0.8
2828
github.com/mattn/go-runewidth v0.0.9 // indirect

‎go.sum

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
22
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
33
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
4-
github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/jt0CW30vsg=
54
github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0=
65
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
76
github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c=
@@ -117,8 +116,6 @@ github.com/h2non/filetype v1.0.8 h1:le8gpf+FQA0/DlDABbtisA1KiTS0Xi+YSC/E8yY3Y14=
117116
github.com/h2non/filetype v1.0.8/go.mod h1:isekKqOuhMj+s/7r3rIeTErIRy4Rub5uBWHfvMusLMU=
118117
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
119118
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
120-
github.com/imjasonmiller/godice v0.1.2 h1:T1/sW/HoDzFeuwzOOuQjmeMELz9CzZ53I2CnD+08zD4=
121-
github.com/imjasonmiller/godice v0.1.2/go.mod h1:8cTkdnVI+NglU2d6sv+ilYcNaJ5VSTBwvMbFULJd/QQ=
122119
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
123120
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
124121
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
@@ -155,6 +152,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
155152
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
156153
github.com/leonelquinteros/gotext v1.4.0 h1:2NHPCto5IoMXbrT0bldPrxj0qM5asOCwtb1aUQZ1tys=
157154
github.com/leonelquinteros/gotext v1.4.0/go.mod h1:yZGXREmoGTtBvZHNcc+Yfug49G/2spuF/i/Qlsvz1Us=
155+
github.com/lithammer/fuzzysearch v1.1.1 h1:8F9OAV2xPuYblToVohjanztdnPjbtA0MLgMvDKQ0Z08=
156+
github.com/lithammer/fuzzysearch v1.1.1/go.mod h1:H2bng+w5gsR7NlfIJM8ElGZI0sX6C/9uzGqicVXGU6c=
158157
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
159158
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
160159
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -215,7 +214,6 @@ github.com/segmentio/objconv v1.0.1 h1:QjfLzwriJj40JibCV3MGSEiAoXixbp4ybhwfTB8RX
215214
github.com/segmentio/objconv v1.0.1/go.mod h1:auayaH5k3137Cl4SoXTgrzQcuQDmvuVtZgS0fb1Ahys=
216215
github.com/segmentio/stats/v4 v4.5.3 h1:Y/DSUWZ4c8ICgqJ9rQohzKvGqGWbLPWad5zmxVoKN+Y=
217216
github.com/segmentio/stats/v4 v4.5.3/go.mod h1:LsaahUJR7iiSs8mnkvQvdQ/RLHAS5adGLxuntg0ydGo=
218-
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
219217
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
220218
github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
221219
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
@@ -281,7 +279,6 @@ golang.org/x/crypto v0.0.0-20180214000028-650f4a345ab4/go.mod h1:6SG95UA2DQfeDnf
281279
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
282280
golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
283281
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
284-
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
285282
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
286283
golang.org/x/crypto v0.0.0-20200406173513-056763e48d71 h1:DOmugCavvUtnUD114C1Wh+UgTgQZ4pMLzXxi1pSt+/Y=
287284
golang.org/x/crypto v0.0.0-20200406173513-056763e48d71/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -334,7 +331,6 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
334331
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
335332
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
336333
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
337-
golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a h1:mEQZbbaBjWyLNy0tmZmgEuQAR8XOQ3hL8GYi3J/NG64=
338334
golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
339335
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
340336
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy