diff --git a/.travis.yml b/.travis.yml
index 958072620..b945ab73b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -23,8 +23,9 @@ before_install:
- docker ps -a
install:
- - go get -u github.com/pilosa/go-pilosa
- - cd "$GOPATH/src/github.com/pilosa/go-pilosa" && git checkout v0.9.0 && cd "$TRAVIS_BUILD_DIR"
+ - go get -u github.com/golang/dep/cmd/dep
+ - touch Gopkg.toml
+ - dep ensure -v -add "github.com/pilosa/go-pilosa@0.9.0" "github.com/pilosa/pilosa@1.0.2"
- make dependencies
script:
diff --git a/server/context.go b/server/context.go
index 2f982d7aa..bb472b210 100644
--- a/server/context.go
+++ b/server/context.go
@@ -60,11 +60,7 @@ func (s *SessionManager) NewContext(conn *mysql.Conn) (*sql.Context, DoneFunc, e
sess := s.sessions[conn.ConnectionID]
s.mu.Unlock()
context := sql.NewContext(ctx, sql.WithSession(sess), sql.WithTracer(s.tracer))
- id, err := uuid.NewV4()
- if err != nil {
- cancel()
- return nil, nil, err
- }
+ id := uuid.NewV4()
s.mu.Lock()
s.sessionContexts[conn.ConnectionID] = append(s.sessionContexts[conn.ConnectionID], id)
diff --git a/sql/index/config.go b/sql/index/config.go
index d3eb88576..fefa5f791 100644
--- a/sql/index/config.go
+++ b/sql/index/config.go
@@ -4,18 +4,10 @@ import (
"io"
"io/ioutil"
"os"
- "path/filepath"
yaml "gopkg.in/yaml.v2"
)
-const (
- // ConfigFileName is the name of an index config file.
- ConfigFileName = "config.yml"
- // ProcessingFileName is the name of the processing index file.
- ProcessingFileName = ".processing"
-)
-
// Config represents index configuration
type Config struct {
DB string
@@ -60,9 +52,8 @@ func WriteConfig(w io.Writer, cfg *Config) error {
return err
}
-// WriteConfigFile writes the configuration to dir/config.yml file.
-func WriteConfigFile(dir string, cfg *Config) error {
- path := filepath.Join(dir, ConfigFileName)
+// WriteConfigFile writes the configuration to file.
+func WriteConfigFile(path string, cfg *Config) error {
f, err := os.Create(path)
if err != nil {
return err
@@ -84,9 +75,8 @@ func ReadConfig(r io.Reader) (*Config, error) {
return &cfg, err
}
-// ReadConfigFile reads an configuration from dir/config.yml file.
-func ReadConfigFile(dir string) (*Config, error) {
- path := filepath.Join(dir, ConfigFileName)
+// ReadConfigFile reads an configuration from file.
+func ReadConfigFile(path string) (*Config, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
@@ -96,10 +86,9 @@ func ReadConfigFile(dir string) (*Config, error) {
return ReadConfig(f)
}
-// CreateProcessingFile creates a file inside the directory saying whether
-// the index is being created.
-func CreateProcessingFile(dir string) error {
- f, err := os.Create(filepath.Join(dir, ProcessingFileName))
+// CreateProcessingFile creates a file saying whether the index is being created.
+func CreateProcessingFile(path string) error {
+ f, err := os.Create(path)
if err != nil {
return err
}
@@ -109,16 +98,14 @@ func CreateProcessingFile(dir string) error {
return nil
}
-// RemoveProcessingFile removes the file that says whether the index is still
-// being created.
-func RemoveProcessingFile(dir string) error {
- return os.Remove(filepath.Join(dir, ProcessingFileName))
+// RemoveProcessingFile removes the file that says whether the index is still being created.
+func RemoveProcessingFile(path string) error {
+ return os.Remove(path)
}
-// ExistsProcessingFile returns whether the processing file exists inside an
-// index directory.
-func ExistsProcessingFile(dir string) (bool, error) {
- _, err := os.Stat(filepath.Join(dir, ProcessingFileName))
+// ExistsProcessingFile returns whether the processing file exists.
+func ExistsProcessingFile(path string) (bool, error) {
+ _, err := os.Stat(path)
if err != nil {
if os.IsNotExist(err) {
return false, nil
diff --git a/sql/index/config_test.go b/sql/index/config_test.go
index e3fa49d54..851520719 100644
--- a/sql/index/config_test.go
+++ b/sql/index/config_test.go
@@ -11,13 +11,17 @@ import (
func TestConfig(t *testing.T) {
require := require.New(t)
+ tmpDir, err := ioutil.TempDir("", "index")
+ require.NoError(err)
+ defer func() { require.NoError(os.RemoveAll(tmpDir)) }()
+ driver := "driver"
db, table, id := "db_name", "table_name", "index_id"
- path := filepath.Join(os.TempDir(), db, table, id)
- err := os.MkdirAll(path, 0750)
-
+ dir := filepath.Join(tmpDir, driver)
+ subdir := filepath.Join(dir, db, table)
+ err = os.MkdirAll(subdir, 0750)
require.NoError(err)
- defer os.RemoveAll(path)
+ file := filepath.Join(subdir, id+".cfg")
cfg1 := NewConfig(
db,
@@ -31,36 +35,35 @@ func TestConfig(t *testing.T) {
},
)
- err = WriteConfigFile(path, cfg1)
+ err = WriteConfigFile(file, cfg1)
require.NoError(err)
- cfg2, err := ReadConfigFile(path)
+ cfg2, err := ReadConfigFile(file)
require.NoError(err)
require.Equal(cfg1, cfg2)
}
func TestProcessingFile(t *testing.T) {
require := require.New(t)
-
- dir, err := ioutil.TempDir(os.TempDir(), "processing-file")
+ tmpDir, err := ioutil.TempDir("", "index")
require.NoError(err)
- defer func() {
- require.NoError(os.RemoveAll(dir))
- }()
+ defer func() { require.NoError(os.RemoveAll(tmpDir)) }()
+
+ file := filepath.Join(tmpDir, ".processing")
- ok, err := ExistsProcessingFile(dir)
+ ok, err := ExistsProcessingFile(file)
require.NoError(err)
require.False(ok)
- require.NoError(CreateProcessingFile(dir))
+ require.NoError(CreateProcessingFile(file))
- ok, err = ExistsProcessingFile(dir)
+ ok, err = ExistsProcessingFile(file)
require.NoError(err)
require.True(ok)
- require.NoError(RemoveProcessingFile(dir))
+ require.NoError(RemoveProcessingFile(file))
- ok, err = ExistsProcessingFile(dir)
+ ok, err = ExistsProcessingFile(file)
require.NoError(err)
require.False(ok)
}
diff --git a/sql/index/pilosa/driver.go b/sql/index/pilosa/driver.go
index 198b50d1f..559cfcaf8 100644
--- a/sql/index/pilosa/driver.go
+++ b/sql/index/pilosa/driver.go
@@ -4,6 +4,7 @@ import (
"crypto/sha1"
"fmt"
"io"
+ "io/ioutil"
"os"
"path/filepath"
"strings"
@@ -24,6 +25,15 @@ const (
IndexNamePrefix = "idx"
// FrameNamePrefix the pilosa's frames prefix
FrameNamePrefix = "frm"
+
+ // ConfigFileName is the name of an index config file.
+ ConfigFileName = "config.yml"
+
+ // ProcessingFileName is the name of the lock/processing index file.
+ ProcessingFileName = ".processing"
+
+ // MappingFileName is the name of the mapping file.
+ MappingFileName = "mapping.db"
)
var (
@@ -65,7 +75,7 @@ func (*Driver) ID() string {
// Create a new index.
func (d *Driver) Create(db, table, id string, expressions []sql.Expression, config map[string]string) (sql.Index, error) {
- path, err := mkdir(d.root, db, table, id)
+ _, err := mkdir(d.root, db, table, id)
if err != nil {
return nil, err
}
@@ -76,79 +86,88 @@ func (d *Driver) Create(db, table, id string, expressions []sql.Expression, conf
}
cfg := index.NewConfig(db, table, id, exprs, d.ID(), config)
- err = index.WriteConfigFile(path, cfg)
+ err = index.WriteConfigFile(d.configFilePath(db, table, id), cfg)
if err != nil {
return nil, err
}
- return newPilosaIndex(path, d.client, cfg), nil
+ return newPilosaIndex(d.mappingFilePath(db, table, id), d.client, cfg), nil
}
// LoadAll loads all indexes for given db and table
func (d *Driver) LoadAll(db, table string) ([]sql.Index, error) {
- root := filepath.Join(d.root, db, table)
-
var (
indexes []sql.Index
errors []string
- err error
+ root = filepath.Join(d.root, db, table)
)
- filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
- if err != nil {
- if path != root || !os.IsNotExist(err) {
- errors = append(errors, err.Error())
- }
- return filepath.SkipDir
- }
- if info.IsDir() && path != root && info.Name() != "." && info.Name() != ".." {
- idx, err := d.loadIndex(path)
+ dirs, err := ioutil.ReadDir(root)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return indexes, nil
+ }
+ return nil, err
+ }
+ for _, info := range dirs {
+ if info.IsDir() && !strings.HasPrefix(info.Name(), ".") {
+ idx, err := d.loadIndex(db, table, info.Name())
if err != nil {
if !errCorruptedIndex.Is(err) {
errors = append(errors, err.Error())
}
-
- return filepath.SkipDir
+ continue
}
indexes = append(indexes, idx)
}
-
- return nil
- })
+ }
if len(errors) > 0 {
- err = fmt.Errorf(strings.Join(errors, "\n"))
+ return nil, fmt.Errorf(strings.Join(errors, "\n"))
}
- return indexes, err
+
+ return indexes, nil
}
-func (d *Driver) loadIndex(path string) (sql.Index, error) {
- ok, err := index.ExistsProcessingFile(path)
+func (d *Driver) loadIndex(db, table, id string) (sql.Index, error) {
+ dir := filepath.Join(d.root, db, table, id)
+ config := d.configFilePath(db, table, id)
+ if _, err := os.Stat(config); err != nil {
+ return nil, errCorruptedIndex.New(dir)
+ }
+
+ mapping := d.mappingFilePath(db, table, id)
+ processing := d.processingFilePath(db, table, id)
+ ok, err := index.ExistsProcessingFile(processing)
if err != nil {
return nil, err
}
-
if ok {
log := logrus.WithFields(logrus.Fields{
- "err": err,
- "path": path,
+ "err": err,
+ "db": db,
+ "table": table,
+ "id": id,
+ "dir": dir,
})
log.Warn("could not read index file, index is corrupt and will be deleted")
-
- if err := os.RemoveAll(path); err != nil {
- log.Warn("unable to remove folder of corrupted index")
+ if err := os.RemoveAll(dir); err != nil {
+ log.Warn("unable to remove corrupted index: " + dir)
}
- return nil, errCorruptedIndex.New(path)
+ return nil, errCorruptedIndex.New(dir)
}
- cfg, err := index.ReadConfigFile(path)
+ cfg, err := index.ReadConfigFile(config)
if err != nil {
return nil, err
}
+ if cfg.Driver(DriverID) == nil {
+ return nil, errCorruptedIndex.New(dir)
+ }
- idx := newPilosaIndex(path, d.client, cfg)
+ idx := newPilosaIndex(mapping, d.client, cfg)
return idx, nil
}
@@ -166,12 +185,8 @@ func (d *Driver) Save(
return errInvalidIndexType.New(i)
}
- path, err := mkdir(d.root, i.Database(), i.Table(), i.ID())
- if err != nil {
- return err
- }
-
- if err = index.CreateProcessingFile(path); err != nil {
+ processingFile := d.processingFilePath(idx.Database(), idx.Table(), idx.ID())
+ if err = index.CreateProcessingFile(processingFile); err != nil {
return err
}
@@ -285,13 +300,12 @@ func (d *Driver) Save(
"id": i.ID(),
}).Debugf("finished pilosa indexing")
- return index.RemoveProcessingFile(path)
+ return index.RemoveProcessingFile(processingFile)
}
// Delete the index with the given path.
func (d *Driver) Delete(idx sql.Index) error {
- path := filepath.Join(d.root, idx.Database(), idx.Table(), idx.ID())
- if err := os.RemoveAll(path); err != nil {
+ if err := os.RemoveAll(filepath.Join(d.root, idx.Database(), idx.Table(), idx.ID())); err != nil {
return err
}
@@ -426,3 +440,15 @@ func mkdir(elem ...string) (string, error) {
path := filepath.Join(elem...)
return path, os.MkdirAll(path, 0750)
}
+
+func (d *Driver) configFilePath(db, table, id string) string {
+ return filepath.Join(d.root, db, table, id, ConfigFileName)
+}
+
+func (d *Driver) processingFilePath(db, table, id string) string {
+ return filepath.Join(d.root, db, table, id, ProcessingFileName)
+}
+
+func (d *Driver) mappingFilePath(db, table, id string) string {
+ return filepath.Join(d.root, db, table, id, MappingFileName)
+}
diff --git a/sql/index/pilosa/driver_test.go b/sql/index/pilosa/driver_test.go
index 63ec77373..dc66ff6c1 100644
--- a/sql/index/pilosa/driver_test.go
+++ b/sql/index/pilosa/driver_test.go
@@ -25,6 +25,8 @@ import (
var (
dockerIsRunning bool
dockerCmdOutput string
+
+ tmpDir string
)
func init() {
@@ -34,6 +36,22 @@ func init() {
dockerCmdOutput, dockerIsRunning = string(b), (err == nil)
}
+func setup(t *testing.T) {
+ var err error
+
+ tmpDir, err = ioutil.TempDir("", "pilosa")
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func cleanup(t *testing.T) {
+ err := os.RemoveAll(tmpDir)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
func TestID(t *testing.T) {
d := &Driver{}
@@ -44,11 +62,10 @@ func TestID(t *testing.T) {
func TestLoadAll(t *testing.T) {
require := require.New(t)
- path, err := ioutil.TempDir(os.TempDir(), "indexes")
- require.NoError(err)
- defer os.RemoveAll(path)
+ setup(t)
+ defer cleanup(t)
- d := NewIndexDriver(path)
+ d := NewIndexDriver(tmpDir)
idx1, err := d.Create("db", "table", "id1", makeExpressions("table", "field1"), nil)
require.NoError(err)
@@ -91,14 +108,13 @@ func TestSaveAndLoad(t *testing.T) {
t.Skipf("Skip TestSaveAndLoad: %s", dockerCmdOutput)
}
require := require.New(t)
+ setup(t)
+ defer cleanup(t)
db, table, id := "db_name", "table_name", "index_id"
expressions := makeExpressions(table, "lang", "hash")
- path, err := ioutil.TempDir(os.TempDir(), "indexes")
- require.NoError(err)
- defer os.RemoveAll(path)
- d := NewDriver(path, newClientWithTimeout(200*time.Millisecond))
+ d := NewDriver(tmpDir, newClientWithTimeout(200*time.Millisecond))
sqlIdx, err := d.Create(db, table, id, expressions, nil)
require.NoError(err)
@@ -184,15 +200,13 @@ func TestSaveAndGetAll(t *testing.T) {
if !dockerIsRunning {
t.Skipf("Skip TestSaveAndGetAll: %s", dockerCmdOutput)
}
+ setup(t)
+ defer cleanup(t)
require := require.New(t)
db, table, id := "db_name", "table_name", "index_id"
expressions := makeExpressions(table, "lang", "hash")
- path, err := ioutil.TempDir(os.TempDir(), "indexes")
- require.NoError(err)
- defer os.RemoveAll(path)
-
- d := NewDriver(path, newClientWithTimeout(200*time.Millisecond))
+ d := NewDriver(tmpDir, newClientWithTimeout(200*time.Millisecond))
sqlIdx, err := d.Create(db, table, id, expressions, nil)
require.NoError(err)
@@ -218,17 +232,20 @@ func TestSaveAndGetAll(t *testing.T) {
func TestLoadCorruptedIndex(t *testing.T) {
require := require.New(t)
- path, err := ioutil.TempDir(os.TempDir(), "indexes")
+ setup(t)
+ defer cleanup(t)
+
+ d := NewIndexDriver(tmpDir).(*Driver)
+ _, err := d.Create("db", "table", "id", nil, nil)
require.NoError(err)
- defer os.RemoveAll(path)
- require.NoError(index.CreateProcessingFile(path))
+ require.NoError(index.CreateProcessingFile(d.processingFilePath("db", "table", "id")))
- _, err = new(Driver).loadIndex(path)
+ _, err = d.loadIndex("db", "table", "id")
require.Error(err)
require.True(errCorruptedIndex.Is(err))
- _, err = os.Stat(path)
+ _, err = os.Stat(d.processingFilePath("db", "table", "id"))
require.Error(err)
require.True(os.IsNotExist(err))
}
@@ -238,18 +255,17 @@ func TestDelete(t *testing.T) {
t.Skipf("Skip TestDelete: %s", dockerCmdOutput)
}
require := require.New(t)
+ setup(t)
+ defer cleanup(t)
db, table, id := "db_name", "table_name", "index_id"
- path, err := ioutil.TempDir(os.TempDir(), "indexes")
- require.NoError(err)
- defer os.RemoveAll(path)
expressions := []sql.Expression{
expression.NewGetFieldWithTable(0, sql.Int64, table, "lang", true),
expression.NewGetFieldWithTable(1, sql.Int64, table, "field", true),
}
- d := NewIndexDriver(path)
+ d := NewIndexDriver(tmpDir)
sqlIdx, err := d.Create(db, table, id, expressions, nil)
require.NoError(err)
@@ -259,12 +275,8 @@ func TestDelete(t *testing.T) {
func TestLoadAllDirectoryDoesNotExist(t *testing.T) {
require := require.New(t)
- tmpDir, err := ioutil.TempDir(os.TempDir(), "pilosa-")
- require.NoError(err)
-
- defer func() {
- require.NoError(os.RemoveAll(tmpDir))
- }()
+ setup(t)
+ defer cleanup(t)
driver := &Driver{root: tmpDir}
drivers, err := driver.LoadAll("foo", "bar")
@@ -339,15 +351,11 @@ func TestNegateIndex(t *testing.T) {
t.Skipf("Skip test: %s", dockerCmdOutput)
}
require := require.New(t)
+ setup(t)
+ defer cleanup(t)
db, table := "db_name", "table_name"
- path, err := mkdir(os.TempDir(), "indexes")
- require.NoError(err)
- defer func() {
- require.NoError(os.RemoveAll(path))
- }()
-
- d := NewDriver(path, newClientWithTimeout(200*time.Millisecond))
+ d := NewDriver(tmpDir, newClientWithTimeout(200*time.Millisecond))
idx, err := d.Create(db, table, "index_id", makeExpressions(table, "a"), nil)
require.NoError(err)
@@ -413,16 +421,14 @@ func TestIntersection(t *testing.T) {
t.Skipf("Skip TestUnion: %s", dockerCmdOutput)
}
require := require.New(t)
+ setup(t)
+ defer cleanup(t)
db, table := "db_name", "table_name"
idxLang, expLang := "idx_lang", makeExpressions(table, "lang")
idxPath, expPath := "idx_path", makeExpressions(table, "path")
- path, err := ioutil.TempDir(os.TempDir(), "indexes")
- require.NoError(err)
- defer os.RemoveAll(path)
-
- d := NewDriver(path, newClientWithTimeout(200*time.Millisecond))
+ d := NewDriver(tmpDir, newClientWithTimeout(200*time.Millisecond))
sqlIdxLang, err := d.Create(db, table, idxLang, expLang, nil)
require.NoError(err)
@@ -491,16 +497,14 @@ func TestUnion(t *testing.T) {
t.Skipf("Skip TestUnion: %s", dockerCmdOutput)
}
require := require.New(t)
+ setup(t)
+ defer cleanup(t)
db, table := "db_name", "table_name"
idxLang, expLang := "idx_lang", makeExpressions(table, "lang")
idxPath, expPath := "idx_path", makeExpressions(table, "path")
- path, err := ioutil.TempDir(os.TempDir(), "indexes")
- require.NoError(err)
- defer os.RemoveAll(path)
-
- d := NewDriver(path, newClientWithTimeout(200*time.Millisecond))
+ d := NewDriver(tmpDir, newClientWithTimeout(200*time.Millisecond))
sqlIdxLang, err := d.Create(db, table, idxLang, expLang, nil)
require.NoError(err)
@@ -581,16 +585,14 @@ func TestDifference(t *testing.T) {
t.Skipf("Skip TestUnion: %s", dockerCmdOutput)
}
require := require.New(t)
+ setup(t)
+ defer cleanup(t)
db, table := "db_name", "table_name"
idxLang, expLang := "idx_lang", makeExpressions(table, "lang")
idxPath, expPath := "idx_path", makeExpressions(table, "path")
- path, err := ioutil.TempDir(os.TempDir(), "indexes")
- require.NoError(err)
- defer os.RemoveAll(path)
-
- d := NewDriver(path, newClientWithTimeout(200*time.Millisecond))
+ d := NewDriver(tmpDir, newClientWithTimeout(200*time.Millisecond))
sqlIdxLang, err := d.Create(db, table, idxLang, expLang, nil)
require.NoError(err)
@@ -654,15 +656,13 @@ func TestUnionDiffAsc(t *testing.T) {
t.Skipf("Skip TestUnion: %s", dockerCmdOutput)
}
require := require.New(t)
+ setup(t)
+ defer cleanup(t)
db, table := "db_name", "table_name"
idx, exp := "idx_lang", makeExpressions(table, "lang")
- path, err := ioutil.TempDir(os.TempDir(), "indexes")
- require.NoError(err)
- defer os.RemoveAll(path)
-
- d := NewDriver(path, newClientWithTimeout(200*time.Millisecond))
+ d := NewDriver(tmpDir, newClientWithTimeout(200*time.Millisecond))
sqlIdx, err := d.Create(db, table, idx, exp, nil)
require.NoError(err)
pilosaIdx, ok := sqlIdx.(*pilosaIndex)
@@ -714,15 +714,13 @@ func TestInterRanges(t *testing.T) {
t.Skipf("Skip TestUnion: %s", dockerCmdOutput)
}
require := require.New(t)
+ setup(t)
+ defer cleanup(t)
db, table := "db_name", "table_name"
idx, exp := "idx_lang", makeExpressions(table, "lang")
- path, err := ioutil.TempDir(os.TempDir(), "indexes")
- require.NoError(err)
- defer os.RemoveAll(path)
-
- d := NewDriver(path, newClientWithTimeout(200*time.Millisecond))
+ d := NewDriver(tmpDir, newClientWithTimeout(200*time.Millisecond))
sqlIdx, err := d.Create(db, table, idx, exp, nil)
require.NoError(err)
pilosaIdx, ok := sqlIdx.(*pilosaIndex)
@@ -775,10 +773,9 @@ func setupAscendDescend(t *testing.T) (*pilosaIndex, func()) {
db, table, id := "db_name", "table_name", "index_id"
expressions := makeExpressions(table, "a", "b")
- path, err := mkdir(os.TempDir(), "indexes")
- require.NoError(err)
+ setup(t)
- d := NewDriver(path, newClientWithTimeout(200*time.Millisecond))
+ d := NewDriver(tmpDir, newClientWithTimeout(200*time.Millisecond))
sqlIdx, err := d.Create(db, table, id, expressions, nil)
require.NoError(err)
@@ -801,7 +798,7 @@ func setupAscendDescend(t *testing.T) (*pilosaIndex, func()) {
require.NoError(err)
return sqlIdx.(*pilosaIndex), func() {
- require.NoError(os.RemoveAll(path))
+ cleanup(t)
}
}
diff --git a/sql/index/pilosa/index.go b/sql/index/pilosa/index.go
index f6ca61933..2048ffbb7 100644
--- a/sql/index/pilosa/index.go
+++ b/sql/index/pilosa/index.go
@@ -10,7 +10,6 @@ import (
// pilosaIndex is an pilosa implementation of sql.Index interface
type pilosaIndex struct {
- path string
client *pilosa.Client
mapping *mapping
@@ -20,15 +19,14 @@ type pilosaIndex struct {
expressions []string
}
-func newPilosaIndex(path string, client *pilosa.Client, cfg *index.Config) *pilosaIndex {
+func newPilosaIndex(mappingfile string, client *pilosa.Client, cfg *index.Config) *pilosaIndex {
return &pilosaIndex{
- path: path,
client: client,
db: cfg.DB,
table: cfg.Table,
id: cfg.ID,
expressions: cfg.Expressions,
- mapping: newMapping(path),
+ mapping: newMapping(mappingfile),
}
}
diff --git a/sql/index/pilosa/mapping.go b/sql/index/pilosa/mapping.go
index 42e9cff13..a62b6a916 100644
--- a/sql/index/pilosa/mapping.go
+++ b/sql/index/pilosa/mapping.go
@@ -6,23 +6,18 @@ import (
"encoding/gob"
"fmt"
"io"
- "path/filepath"
"sort"
"sync"
"github.com/boltdb/bolt"
)
-const (
- mappingFileName = DriverID + "-mapping.db"
-)
-
// mapping
// buckets:
// - index name: columndID uint64 -> location []byte
// - frame name: value []byte (gob encoding) -> rowID uint64
type mapping struct {
- dir string
+ path string
mut sync.RWMutex
db *bolt.DB
@@ -36,8 +31,8 @@ type mapping struct {
clients int
}
-func newMapping(dir string) *mapping {
- return &mapping{dir: dir}
+func newMapping(path string) *mapping {
+ return &mapping{path: path}
}
func (m *mapping) open() {
@@ -80,7 +75,7 @@ func (m *mapping) query(fn func() error) error {
m.mut.Lock()
if m.db == nil {
var err error
- m.db, err = bolt.Open(filepath.Join(m.dir, mappingFileName), 0640, nil)
+ m.db, err = bolt.Open(m.path, 0640, nil)
if err != nil {
m.mut.Unlock()
return err
diff --git a/sql/index/pilosa/mapping_test.go b/sql/index/pilosa/mapping_test.go
index 31f205623..2eba39ae6 100644
--- a/sql/index/pilosa/mapping_test.go
+++ b/sql/index/pilosa/mapping_test.go
@@ -2,7 +2,7 @@ package pilosa
import (
"encoding/binary"
- "os"
+ "path/filepath"
"testing"
"github.com/stretchr/testify/require"
@@ -10,12 +10,10 @@ import (
func TestRowID(t *testing.T) {
require := require.New(t)
+ setup(t)
+ defer cleanup(t)
- path, err := mkdir(os.TempDir(), "mapping_test")
- require.NoError(err)
- defer os.RemoveAll(path)
-
- m := newMapping(path)
+ m := newMapping(filepath.Join(tmpDir, "id.map"))
m.open()
defer m.close()
@@ -35,12 +33,10 @@ func TestRowID(t *testing.T) {
func TestLocation(t *testing.T) {
require := require.New(t)
+ setup(t)
+ defer cleanup(t)
- path, err := mkdir(os.TempDir(), "mapping_test")
- require.NoError(err)
- defer os.RemoveAll(path)
-
- m := newMapping(path)
+ m := newMapping(filepath.Join(tmpDir, "id.map"))
m.open()
defer m.close()
@@ -53,7 +49,7 @@ func TestLocation(t *testing.T) {
}
for colID, loc := range cases {
- err = m.putLocation("index name", colID, []byte(loc))
+ err := m.putLocation("index name", colID, []byte(loc))
require.NoError(err)
}
@@ -66,12 +62,10 @@ func TestLocation(t *testing.T) {
func TestGet(t *testing.T) {
require := require.New(t)
+ setup(t)
+ defer cleanup(t)
- path, err := mkdir(os.TempDir(), "mapping_test")
- require.NoError(err)
- defer os.RemoveAll(path)
-
- m := newMapping(path)
+ m := newMapping(filepath.Join(tmpDir, "id.map"))
m.open()
defer m.close()
diff --git a/sql/index/pilosalib/driver.go b/sql/index/pilosalib/driver.go
new file mode 100644
index 000000000..09cc2bee0
--- /dev/null
+++ b/sql/index/pilosalib/driver.go
@@ -0,0 +1,467 @@
+package pilosalib
+
+import (
+ "crypto/sha1"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "strings"
+ "time"
+
+ opentracing "github.com/opentracing/opentracing-go"
+ pilosa "github.com/pilosa/pilosa"
+ "github.com/sirupsen/logrus"
+ errors "gopkg.in/src-d/go-errors.v1"
+ "gopkg.in/src-d/go-mysql-server.v0/sql"
+ "gopkg.in/src-d/go-mysql-server.v0/sql/index"
+)
+
+const (
+ // DriverID the unique name of the pilosa driver.
+ DriverID = "pilosalib"
+
+ // IndexNamePrefix the pilosa's indexes prefix
+ IndexNamePrefix = "idx"
+
+ // FieldNamePrefix the pilosa's field prefix
+ FieldNamePrefix = "fld"
+
+ // ConfigFileName is the extension of an index config file.
+ ConfigFileName = "config.yml"
+
+ // ProcessingFileName is the extension of the lock/processing index file.
+ ProcessingFileName = ".processing"
+
+ // MappingFileName is the extension of the mapping file.
+ MappingFileName = "mapping.db"
+)
+
+var (
+ errCorruptedIndex = errors.NewKind("the index db: %s, table: %s, id: %s is corrupted")
+ errLoadingIndex = errors.NewKind("cannot load pilosa index: %s")
+ errInvalidIndexType = errors.NewKind("expecting a pilosa index, instead got %T")
+ errDeletePilosaField = errors.NewKind("error deleting pilosa field %s: %s")
+)
+
+type (
+ bitBatch struct {
+ size uint64
+ rows []uint64
+ cols []uint64
+ pos uint64
+ }
+
+ // Driver implements sql.IndexDriver interface.
+ Driver struct {
+ root string
+ holder *pilosa.Holder
+
+ // used for saving
+ bitBatches []*bitBatch
+ fields []*pilosa.Field
+ timePilosa time.Duration
+ timeMapping time.Duration
+ }
+)
+
+// NewDriver returns a new instance of pilosalib.Driver
+// which satisfies sql.IndexDriver interface
+func NewDriver(root string) *Driver {
+ return &Driver{
+ root: root,
+ holder: pilosa.NewHolder(),
+ }
+}
+
+// ID returns the unique name of the driver.
+func (*Driver) ID() string {
+ return DriverID
+}
+
+// Create a new index.
+func (d *Driver) Create(db, table, id string, expressions []sql.Expression, config map[string]string) (sql.Index, error) {
+ _, err := mkdir(d.root, db, table, id)
+ if err != nil {
+ return nil, err
+ }
+ name := indexName(db, table)
+ if config == nil {
+ config = make(map[string]string)
+ }
+ exprs := make([]string, len(expressions))
+ for i, e := range expressions {
+ name := e.String()
+
+ exprs[i] = name
+ config[fieldName(id, name)] = name
+ }
+
+ cfg := index.NewConfig(db, table, id, exprs, d.ID(), config)
+ err = index.WriteConfigFile(d.configFilePath(db, table, id), cfg)
+ if err != nil {
+ return nil, err
+ }
+
+ d.holder.Path = d.pilosaDirPath(db, table)
+ idx, err := d.holder.CreateIndexIfNotExists(name, pilosa.IndexOptions{})
+ if err != nil {
+ return nil, err
+ }
+ mapping := newMapping(d.mappingFilePath(db, table, id))
+
+ return newPilosaIndex(idx, mapping, cfg), nil
+}
+
+// LoadAll loads all indexes for given db and table
+func (d *Driver) LoadAll(db, table string) ([]sql.Index, error) {
+ var (
+ indexes []sql.Index
+ errors []string
+ root = filepath.Join(d.root, db, table)
+ )
+
+ d.holder.Path = d.pilosaDirPath(db, table)
+ err := d.holder.Open()
+ if err != nil {
+ return nil, err
+ }
+ defer d.holder.Close()
+
+ dirs, err := ioutil.ReadDir(root)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return indexes, nil
+ }
+ return nil, err
+ }
+ for _, info := range dirs {
+ if info.IsDir() && !strings.HasPrefix(info.Name(), ".") {
+ idx, err := d.loadIndex(db, table, info.Name())
+ if err != nil {
+ if !errCorruptedIndex.Is(err) {
+ errors = append(errors, err.Error())
+ }
+ continue
+ }
+
+ indexes = append(indexes, idx)
+ }
+ }
+ if len(errors) > 0 {
+ return nil, fmt.Errorf(strings.Join(errors, "\n"))
+ }
+
+ return indexes, nil
+}
+
+func (d *Driver) loadIndex(db, table, id string) (*pilosaIndex, error) {
+ name := indexName(db, table)
+ idx := d.holder.Index(name)
+ if idx == nil {
+ return nil, errLoadingIndex.New(name)
+ }
+
+ dir := filepath.Join(d.root, db, table, id)
+ config := d.configFilePath(db, table, id)
+ if _, err := os.Stat(config); err != nil {
+ return nil, errCorruptedIndex.New(dir)
+ }
+
+ mapping := d.mappingFilePath(db, table, id)
+ processing := d.processingFilePath(db, table, id)
+ ok, err := index.ExistsProcessingFile(processing)
+ if err != nil {
+ return nil, err
+ }
+ if ok {
+ log := logrus.WithFields(logrus.Fields{
+ "err": err,
+ "db": db,
+ "table": table,
+ "id": id,
+ "dir": dir,
+ })
+ log.Warn("could not read index file, index is corrupt and will be deleted")
+ if err := os.RemoveAll(dir); err != nil {
+ log.Warn("unable to remove corrupted index: " + dir)
+ }
+
+ return nil, errCorruptedIndex.New(dir)
+ }
+
+ cfg, err := index.ReadConfigFile(config)
+ if err != nil {
+ return nil, err
+ }
+ if cfg.Driver(DriverID) == nil {
+ return nil, errCorruptedIndex.New(dir)
+ }
+
+ return newPilosaIndex(idx, newMapping(mapping), cfg), nil
+}
+
+// Save the given index (mapping and bitmap)
+func (d *Driver) Save(ctx *sql.Context, i sql.Index, iter sql.IndexKeyValueIter) (err error) {
+ var colID uint64
+ start := time.Now()
+
+ idx, ok := i.(*pilosaIndex)
+ if !ok {
+ return errInvalidIndexType.New(i)
+ }
+
+ processingFile := d.processingFilePath(idx.Database(), idx.Table(), idx.ID())
+ if err = index.CreateProcessingFile(processingFile); err != nil {
+ return err
+ }
+
+ pilosaIndex := idx.index
+ if err = pilosaIndex.Open(); err != nil {
+ return err
+ }
+ defer pilosaIndex.Close()
+
+ d.fields = make([]*pilosa.Field, len(idx.Expressions()))
+ for i, e := range idx.Expressions() {
+ name := fieldName(idx.ID(), e)
+ pilosaIndex.DeleteField(name)
+ field, err := pilosaIndex.CreateField(name, pilosa.OptFieldTypeDefault())
+ if err != nil {
+ return err
+ }
+ d.fields[i] = field
+ }
+
+ rollback := true
+ idx.mapping.openCreate(true)
+ defer func() {
+ if rollback {
+ idx.mapping.rollback()
+ } else {
+ e := d.saveMapping(ctx, idx.mapping, colID, false)
+ if e != nil && err == nil {
+ err = e
+ }
+ }
+
+ idx.mapping.close()
+ }()
+
+ d.bitBatches = make([]*bitBatch, len(d.fields))
+ for i := range d.bitBatches {
+ d.bitBatches[i] = newBitBatch(sql.IndexBatchSize)
+ }
+
+ for colID = uint64(0); err == nil; colID++ {
+ // commit each batch of objects (pilosa and boltdb)
+ if colID%sql.IndexBatchSize == 0 && colID != 0 {
+ d.saveBatch(ctx, idx.mapping, colID)
+ }
+
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+
+ default:
+ var (
+ values []interface{}
+ location []byte
+ )
+ values, location, err = iter.Next()
+ if err != nil {
+ break
+ }
+
+ for i, field := range d.fields {
+ if values[i] == nil {
+ continue
+ }
+
+ rowID, err := idx.mapping.getRowID(field.Name(), values[i])
+ if err != nil {
+ return err
+ }
+
+ d.bitBatches[i].Add(rowID, colID)
+ }
+ err = idx.mapping.putLocation(pilosaIndex.Name(), colID, location)
+ }
+ }
+
+ if err != nil && err != io.EOF {
+ return err
+ }
+
+ rollback = false
+
+ err = d.savePilosa(ctx, colID)
+ if err != nil {
+ return err
+ }
+
+ logrus.WithFields(logrus.Fields{
+ "duration": time.Since(start),
+ "pilosa": d.timePilosa,
+ "mapping": d.timeMapping,
+ "rows": colID,
+ "id": i.ID(),
+ }).Debugf("finished pilosa indexing")
+
+ return index.RemoveProcessingFile(processingFile)
+}
+
+// Delete the index with the given path.
+func (d *Driver) Delete(i sql.Index) error {
+ if err := os.RemoveAll(filepath.Join(d.root, i.Database(), i.Table(), i.ID())); err != nil {
+ return err
+ }
+
+ idx, ok := i.(*pilosaIndex)
+ if !ok {
+ return errInvalidIndexType.New(i)
+ }
+
+ err := idx.index.Open()
+ if err != nil {
+ return err
+ }
+ defer idx.index.Close()
+
+ for _, ex := range idx.Expressions() {
+ name := fieldName(idx.ID(), ex)
+ field := idx.index.Field(name)
+ if field == nil {
+ continue
+ }
+
+ if err = idx.index.DeleteField(name); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (d *Driver) saveBatch(ctx *sql.Context, m *mapping, colID uint64) error {
+ err := d.savePilosa(ctx, colID)
+ if err != nil {
+ return err
+ }
+
+ return d.saveMapping(ctx, m, colID, true)
+}
+
+func (d *Driver) savePilosa(ctx *sql.Context, colID uint64) error {
+ span, _ := ctx.Span("pilosa.Save.bitBatch",
+ opentracing.Tag{Key: "cols", Value: colID},
+ opentracing.Tag{Key: "frames", Value: len(d.fields)},
+ )
+ defer span.Finish()
+
+ start := time.Now()
+
+ for i, frm := range d.fields {
+ err := frm.Import(d.bitBatches[i].rows, d.bitBatches[i].cols, nil)
+ if err != nil {
+ span.LogKV("error", err)
+ return err
+ }
+
+ d.bitBatches[i].Clean()
+ }
+
+ d.timePilosa += time.Since(start)
+
+ return nil
+}
+
+func (d *Driver) saveMapping(
+ ctx *sql.Context,
+ m *mapping,
+ colID uint64,
+ cont bool,
+) error {
+ span, _ := ctx.Span("pilosa.Save.mapping",
+ opentracing.Tag{Key: "cols", Value: colID},
+ opentracing.Tag{Key: "continues", Value: cont},
+ )
+ defer span.Finish()
+
+ start := time.Now()
+
+ err := m.commit(cont)
+ if err != nil {
+ span.LogKV("error", err)
+ return err
+ }
+
+ d.timeMapping += time.Since(start)
+
+ return nil
+}
+
+func newBitBatch(size uint64) *bitBatch {
+ b := &bitBatch{size: size}
+ b.Clean()
+
+ return b
+}
+
+func (b *bitBatch) Clean() {
+ b.rows = make([]uint64, 0, b.size)
+ b.rows = make([]uint64, 0, b.size)
+ b.pos = 0
+}
+
+func (b *bitBatch) Add(row, col uint64) {
+ b.rows = append(b.rows, row)
+ b.cols = append(b.cols, col)
+}
+
+func (b *bitBatch) NextRecord() (uint64, uint64, error) {
+ if b.pos >= uint64(len(b.rows)) {
+ return 0, 0, io.EOF
+ }
+
+ b.pos++
+ return b.rows[b.pos-1], b.cols[b.pos-1], nil
+}
+
+func indexName(db, table string) string {
+ h := sha1.New()
+ io.WriteString(h, db)
+ io.WriteString(h, table)
+
+ return fmt.Sprintf("%s-%x", IndexNamePrefix, h.Sum(nil))
+}
+
+func fieldName(id string, ex string) string {
+ h := sha1.New()
+ io.WriteString(h, id)
+ io.WriteString(h, ex)
+ return fmt.Sprintf("%s-%x", FieldNamePrefix, h.Sum(nil))
+}
+
+// mkdir makes an empty index directory (if doesn't exist) and returns a path.
+func mkdir(elem ...string) (string, error) {
+ path := filepath.Join(elem...)
+ return path, os.MkdirAll(path, 0750)
+}
+
+func (d *Driver) pilosaDirPath(db, table string) string {
+ return filepath.Join(d.root, db, table, "."+DriverID)
+}
+
+func (d *Driver) configFilePath(db, table, id string) string {
+ return filepath.Join(d.root, db, table, id, ConfigFileName)
+}
+
+func (d *Driver) processingFilePath(db, table, id string) string {
+ return filepath.Join(d.root, db, table, id, ProcessingFileName)
+}
+
+func (d *Driver) mappingFilePath(db, table, id string) string {
+ return filepath.Join(d.root, db, table, id, MappingFileName)
+}
diff --git a/sql/index/pilosalib/driver_test.go b/sql/index/pilosalib/driver_test.go
new file mode 100644
index 000000000..d7075ae60
--- /dev/null
+++ b/sql/index/pilosalib/driver_test.go
@@ -0,0 +1,982 @@
+package pilosalib
+
+import (
+ "context"
+ "crypto/rand"
+ "encoding/hex"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "reflect"
+ "testing"
+
+ "github.com/pilosa/pilosa"
+ "github.com/stretchr/testify/require"
+ "gopkg.in/src-d/go-mysql-server.v0/sql"
+ "gopkg.in/src-d/go-mysql-server.v0/sql/expression"
+ "gopkg.in/src-d/go-mysql-server.v0/sql/index"
+ "gopkg.in/src-d/go-mysql-server.v0/test"
+)
+
+var tmpDir string
+
+func setup(t *testing.T) {
+ var err error
+
+ tmpDir, err = ioutil.TempDir("", "pilosalib")
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func cleanup(t *testing.T) {
+ err := os.RemoveAll(tmpDir)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestID(t *testing.T) {
+ d := &Driver{}
+
+ require := require.New(t)
+ require.Equal(DriverID, d.ID())
+}
+
+func TestLoadAll(t *testing.T) {
+ require := require.New(t)
+ setup(t)
+ defer cleanup(t)
+
+ d := NewDriver(tmpDir)
+ idx1, err := d.Create("db", "table", "id1", makeExpressions("table", "hash1"), nil)
+ require.NoError(err)
+
+ idx2, err := d.Create("db", "table", "id2", makeExpressions("table", "hash1"), nil)
+ require.NoError(err)
+
+ indexes, err := d.LoadAll("db", "table")
+ require.NoError(err)
+
+ require.Equal(2, len(indexes))
+ i1, ok := idx1.(*pilosaIndex)
+ require.True(ok)
+ i2, ok := idx2.(*pilosaIndex)
+ require.True(ok)
+
+ require.Equal(i1.index.Name(), i2.index.Name())
+}
+
+type logLoc struct {
+ loc []byte
+ err error
+}
+
+func TestSaveAndLoad(t *testing.T) {
+ require := require.New(t)
+ setup(t)
+ defer cleanup(t)
+
+ db, table, id := "db_name", "table_name", "index_id"
+ expressions := makeExpressions(table, "lang", "hash")
+
+ d := NewDriver(tmpDir)
+ sqlIdx, err := d.Create(db, table, id, expressions, nil)
+ require.NoError(err)
+
+ it := &testIndexKeyValueIter{
+ offset: 0,
+ total: 64,
+ expressions: sqlIdx.Expressions(),
+ location: randLocation,
+ }
+
+ tracer := new(test.MemTracer)
+ ctx := sql.NewContext(context.Background(), sql.WithTracer(tracer))
+ err = d.Save(ctx, sqlIdx, it)
+ require.NoError(err)
+
+ indexes, err := d.LoadAll(db, table)
+ require.NoError(err)
+ require.Equal(1, len(indexes))
+
+ for _, r := range it.records {
+ lookup, err := sqlIdx.Get(r.values...)
+ require.NoError(err)
+
+ found, foundLoc := false, []string{}
+ lit, err := lookup.Values()
+ require.NoError(err)
+
+ var logs []logLoc
+ for i := 0; ; i++ {
+ loc, err := lit.Next()
+
+ // make a copy of location to save in the log
+ loc2 := make([]byte, len(loc))
+ copy(loc2, loc)
+ logs = append(logs, logLoc{loc2, err})
+
+ if err == io.EOF {
+ if i == 0 {
+ for j, l := range logs {
+ t.Logf("[%d] values: %v location: %x loc: %x err: %v\n",
+ j, r.values, r.location, l.loc, l.err)
+ }
+
+ t.Errorf("No data for r.values: %v\tr.location: %x",
+ r.values, r.location)
+ t.FailNow()
+ }
+
+ break
+ }
+
+ require.NoError(err)
+ found = found || reflect.DeepEqual(r.location, loc)
+ foundLoc = append(foundLoc, hex.EncodeToString(loc))
+ }
+ require.Truef(found, "Expected: %s\nGot: %v\n", hex.EncodeToString(r.location), foundLoc)
+
+ err = lit.Close()
+ require.NoError(err)
+ }
+
+ // test that not found values do not cause error
+ lookup, err := sqlIdx.Get("do not exist", "none")
+ require.NoError(err)
+ lit, err := lookup.Values()
+ require.NoError(err)
+ _, err = lit.Next()
+ require.Equal(io.EOF, err)
+
+ found := false
+ for _, span := range tracer.Spans {
+ if span == "pilosa.Save.bitBatch" {
+ found = true
+ break
+ }
+ }
+
+ require.True(found)
+}
+
+func TestSaveAndGetAll(t *testing.T) {
+ require := require.New(t)
+ setup(t)
+ defer cleanup(t)
+
+ db, table, id := "db_name", "table_name", "index_id"
+ expressions := makeExpressions(table, "lang", "hash")
+
+ d := NewDriver(tmpDir)
+ sqlIdx, err := d.Create(db, table, id, expressions, nil)
+ require.NoError(err)
+
+ it := &testIndexKeyValueIter{
+ offset: 0,
+ total: 64,
+ expressions: sqlIdx.Expressions(),
+ location: randLocation,
+ }
+
+ err = d.Save(sql.NewEmptyContext(), sqlIdx, it)
+ require.NoError(err)
+
+ indexes, err := d.LoadAll(db, table)
+ require.NoError(err)
+ require.Equal(1, len(indexes))
+
+ _, err = sqlIdx.Get()
+ require.Error(err)
+ require.True(errInvalidKeys.Is(err))
+}
+
+func TestLoadCorruptedIndex(t *testing.T) {
+ require := require.New(t)
+ setup(t)
+ defer cleanup(t)
+
+ d := NewDriver(tmpDir)
+ _, err := d.Create("db", "table", "id", nil, nil)
+ require.NoError(err)
+
+ require.NoError(index.CreateProcessingFile(d.processingFilePath("db", "table", "id")))
+
+ _, err = d.loadIndex("db", "table", "id")
+ require.Error(err)
+ require.True(errCorruptedIndex.Is(err))
+
+ _, err = os.Stat(d.processingFilePath("db", "table", "id"))
+ require.Error(err)
+ require.True(os.IsNotExist(err))
+}
+
+func TestDelete(t *testing.T) {
+ require := require.New(t)
+ setup(t)
+ defer cleanup(t)
+
+ db, table, id := "db_name", "table_name", "index_id"
+
+ expressions := []sql.Expression{
+ expression.NewGetFieldWithTable(0, sql.Int64, table, "lang", true),
+ expression.NewGetFieldWithTable(1, sql.Int64, table, "field", true),
+ }
+
+ d := NewDriver(tmpDir)
+ sqlIdx, err := d.Create(db, table, id, expressions, nil)
+ require.NoError(err)
+
+ err = d.Delete(sqlIdx)
+ require.NoError(err)
+}
+
+func TestLoadAllDirectoryDoesNotExist(t *testing.T) {
+ require := require.New(t)
+ setup(t)
+ defer cleanup(t)
+
+ driver := NewDriver(tmpDir)
+ indexes, err := driver.LoadAll("foo", "bar")
+ require.NoError(err)
+ require.Len(indexes, 0)
+}
+
+func TestAscendDescendIndex(t *testing.T) {
+ idx, cleanup := setupAscendDescend(t)
+ defer cleanup()
+
+ must := func(lookup sql.IndexLookup, err error) sql.IndexLookup {
+ require.NoError(t, err)
+ return lookup
+ }
+
+ testCases := []struct {
+ name string
+ lookup sql.IndexLookup
+ expected []string
+ }{
+ {
+ "ascend range",
+ must(idx.AscendRange(
+ []interface{}{int64(1), int64(1)},
+ []interface{}{int64(7), int64(10)},
+ )),
+ []string{"1", "5", "6", "7", "8", "9"},
+ },
+ {
+ "ascend greater or equal",
+ must(idx.AscendGreaterOrEqual(int64(7), int64(6))),
+ []string{"2", "4"},
+ },
+ {
+ "ascend less than",
+ must(idx.AscendLessThan(int64(5), int64(3))),
+ []string{"1", "10"},
+ },
+ {
+ "descend range",
+ must(idx.DescendRange(
+ []interface{}{int64(6), int64(9)},
+ []interface{}{int64(0), int64(0)},
+ )),
+ []string{"9", "8", "7", "6", "5", "1"},
+ },
+ {
+ "descend less or equal",
+ must(idx.DescendLessOrEqual(int64(4), int64(2))),
+ []string{"10", "1"},
+ },
+ {
+ "descend greater",
+ must(idx.DescendGreater(int64(6), int64(5))),
+ []string{"4", "2"},
+ },
+ }
+
+ for _, tt := range testCases {
+ t.Run(tt.name, func(t *testing.T) {
+ require := require.New(t)
+ iter, err := tt.lookup.Values()
+ require.NoError(err)
+
+ var result []string
+ for {
+ k, err := iter.Next()
+ if err == io.EOF {
+ break
+ }
+ require.NoError(err)
+
+ result = append(result, string(k))
+ }
+
+ require.Equal(tt.expected, result)
+ })
+ }
+}
+
+func TestIntersection(t *testing.T) {
+ ctx := sql.NewContext(context.Background())
+ require := require.New(t)
+ setup(t)
+ defer cleanup(t)
+
+ db, table := "db_name", "table_name"
+ idxLang, expLang := "idx_lang", makeExpressions(table, "lang")
+ idxPath, expPath := "idx_path", makeExpressions(table, "path")
+
+ d := NewDriver(tmpDir)
+ sqlIdxLang, err := d.Create(db, table, idxLang, expLang, nil)
+ require.NoError(err)
+
+ sqlIdxPath, err := d.Create(db, table, idxPath, expPath, nil)
+ require.NoError(err)
+
+ itLang := &testIndexKeyValueIter{
+ offset: 0,
+ total: 10,
+ expressions: sqlIdxLang.Expressions(),
+ location: offsetLocation,
+ }
+
+ itPath := &testIndexKeyValueIter{
+ offset: 0,
+ total: 10,
+ expressions: sqlIdxPath.Expressions(),
+ location: offsetLocation,
+ }
+
+ err = d.Save(ctx, sqlIdxLang, itLang)
+ require.NoError(err)
+
+ err = d.Save(ctx, sqlIdxPath, itPath)
+ require.NoError(err)
+
+ lookupLang, err := sqlIdxLang.Get(itLang.records[0].values...)
+ require.NoError(err)
+ lookupPath, err := sqlIdxPath.Get(itPath.records[itPath.total-1].values...)
+ require.NoError(err)
+
+ m, ok := lookupLang.(sql.Mergeable)
+ require.True(ok)
+ require.True(m.IsMergeable(lookupPath))
+
+ interLookup, ok := lookupLang.(sql.SetOperations)
+ require.True(ok)
+ interIt, err := interLookup.Intersection(lookupPath).Values()
+ require.NoError(err)
+ loc, err := interIt.Next()
+ fmt.Println(loc, err, err == io.EOF)
+
+ require.True(err == io.EOF)
+ require.NoError(interIt.Close())
+
+ lookupLang, err = sqlIdxLang.Get(itLang.records[0].values...)
+ require.NoError(err)
+ lookupPath, err = sqlIdxPath.Get(itPath.records[0].values...)
+ require.NoError(err)
+
+ interLookup, ok = lookupPath.(sql.SetOperations)
+ require.True(ok)
+ interIt, err = interLookup.Intersection(lookupLang).Values()
+ require.NoError(err)
+ loc, err = interIt.Next()
+ require.NoError(err)
+ require.Equal(loc, itPath.records[0].location)
+ _, err = interIt.Next()
+ require.True(err == io.EOF)
+
+ require.NoError(interIt.Close())
+}
+
+func TestUnion(t *testing.T) {
+ require := require.New(t)
+ setup(t)
+ defer cleanup(t)
+
+ db, table := "db_name", "table_name"
+ idxLang, expLang := "idx_lang", makeExpressions(table, "lang")
+ idxPath, expPath := "idx_path", makeExpressions(table, "path")
+
+ d := NewDriver(tmpDir)
+ sqlIdxLang, err := d.Create(db, table, idxLang, expLang, nil)
+ require.NoError(err)
+
+ sqlIdxPath, err := d.Create(db, table, idxPath, expPath, nil)
+ require.NoError(err)
+
+ itLang := &testIndexKeyValueIter{
+ offset: 0,
+ total: 10,
+ expressions: sqlIdxLang.Expressions(),
+ location: offsetLocation,
+ }
+
+ itPath := &testIndexKeyValueIter{
+ offset: 0,
+ total: 10,
+ expressions: sqlIdxPath.Expressions(),
+ location: offsetLocation,
+ }
+
+ ctx := sql.NewContext(context.Background())
+
+ err = d.Save(ctx, sqlIdxLang, itLang)
+ require.NoError(err)
+
+ err = d.Save(ctx, sqlIdxPath, itPath)
+ require.NoError(err)
+
+ lookupLang, err := sqlIdxLang.Get(itLang.records[0].values...)
+ require.NoError(err)
+ litLang, err := lookupLang.Values()
+ require.NoError(err)
+
+ loc, err := litLang.Next()
+ require.NoError(err)
+ require.Equal(itLang.records[0].location, loc)
+ _, err = litLang.Next()
+ require.True(err == io.EOF)
+ err = litLang.Close()
+ require.NoError(err)
+
+ lookupPath, err := sqlIdxPath.Get(itPath.records[itPath.total-1].values...)
+ require.NoError(err)
+ litPath, err := lookupPath.Values()
+ require.NoError(err)
+
+ loc, err = litPath.Next()
+ require.NoError(err)
+ require.Equal(itPath.records[itPath.total-1].location, loc)
+ _, err = litPath.Next()
+ require.True(err == io.EOF)
+ err = litLang.Close()
+ require.NoError(err)
+
+ m, ok := lookupLang.(sql.Mergeable)
+ require.True(ok)
+ require.True(m.IsMergeable(lookupPath))
+
+ unionLookup, ok := lookupLang.(sql.SetOperations)
+ unionIt, err := unionLookup.Union(lookupPath).Values()
+ require.NoError(err)
+ // 0
+ loc, err = unionIt.Next()
+ require.Equal(itLang.records[0].location, loc)
+
+ // total-1
+ loc, err = unionIt.Next()
+ require.Equal(itPath.records[itPath.total-1].location, loc)
+
+ _, err = unionIt.Next()
+ require.True(err == io.EOF)
+
+ require.NoError(unionIt.Close())
+}
+
+func TestDifference(t *testing.T) {
+ require := require.New(t)
+ setup(t)
+ defer cleanup(t)
+
+ db, table := "db_name", "table_name"
+ idxLang, expLang := "idx_lang", makeExpressions(table, "lang")
+ idxPath, expPath := "idx_path", makeExpressions(table, "path")
+
+ d := NewDriver(tmpDir)
+ sqlIdxLang, err := d.Create(db, table, idxLang, expLang, nil)
+ require.NoError(err)
+
+ sqlIdxPath, err := d.Create(db, table, idxPath, expPath, nil)
+ require.NoError(err)
+
+ itLang := &testIndexKeyValueIter{
+ offset: 0,
+ total: 10,
+ expressions: sqlIdxLang.Expressions(),
+ location: offsetLocation,
+ }
+
+ itPath := &testIndexKeyValueIter{
+ offset: 0,
+ total: 10,
+ expressions: sqlIdxPath.Expressions(),
+ location: offsetLocation,
+ }
+
+ ctx := sql.NewContext(context.Background())
+
+ err = d.Save(ctx, sqlIdxLang, itLang)
+ require.NoError(err)
+
+ err = d.Save(ctx, sqlIdxPath, itPath)
+ require.NoError(err)
+
+ lookupLang, err := sqlIdxLang.Get(itLang.records[0].values...)
+ require.NoError(err)
+
+ lookupPath, err := sqlIdxPath.Get(itPath.records[itPath.total-1].values...)
+ require.NoError(err)
+
+ m, ok := lookupLang.(sql.Mergeable)
+ require.True(ok)
+ require.True(m.IsMergeable(lookupPath))
+
+ unionOp, ok := lookupLang.(sql.SetOperations)
+ require.True(ok)
+ unionLookup, ok := unionOp.Union(lookupPath).(sql.SetOperations)
+ require.True(ok)
+
+ diffLookup := unionLookup.Difference(lookupLang)
+ diffIt, err := diffLookup.Values()
+ require.NoError(err)
+
+ // total-1
+ loc, err := diffIt.Next()
+ require.NoError(err)
+ require.Equal(itPath.records[itPath.total-1].location, loc)
+
+ _, err = diffIt.Next()
+ require.True(err == io.EOF)
+
+ require.NoError(diffIt.Close())
+}
+
+func TestUnionDiffAsc(t *testing.T) {
+ require := require.New(t)
+ setup(t)
+ defer cleanup(t)
+
+ db, table := "db_name", "table_name"
+ idx, exp := "idx_lang", makeExpressions(table, "lang")
+
+ d := NewDriver(tmpDir)
+ sqlIdx, err := d.Create(db, table, idx, exp, nil)
+ require.NoError(err)
+ pilosaIdx, ok := sqlIdx.(*pilosaIndex)
+ require.True(ok)
+ it := &testIndexKeyValueIter{
+ offset: 0,
+ total: 10,
+ expressions: sqlIdx.Expressions(),
+ location: offsetLocation,
+ }
+
+ ctx := sql.NewContext(context.Background())
+
+ err = d.Save(ctx, pilosaIdx, it)
+ require.NoError(err)
+
+ sqlLookup, err := pilosaIdx.AscendLessThan(it.records[it.total-1].values...)
+ require.NoError(err)
+ ascLookup, ok := sqlLookup.(*ascendLookup)
+ require.True(ok)
+
+ ls := make([]*indexLookup, it.total)
+ for i, r := range it.records {
+ l, err := pilosaIdx.Get(r.values...)
+ require.NoError(err)
+ ls[i], _ = l.(*indexLookup)
+ }
+
+ unionLookup := ls[0].Union(ls[2], ls[4], ls[6], ls[8])
+
+ diffLookup := ascLookup.Difference(unionLookup)
+ diffIt, err := diffLookup.Values()
+ require.NoError(err)
+
+ for i := 1; i < it.total-1; i += 2 {
+ loc, err := diffIt.Next()
+ require.NoError(err)
+
+ require.Equal(it.records[i].location, loc)
+ }
+
+ _, err = diffIt.Next()
+ require.True(err == io.EOF)
+ require.NoError(diffIt.Close())
+}
+
+func TestInterRanges(t *testing.T) {
+ require := require.New(t)
+ setup(t)
+ defer cleanup(t)
+
+ db, table := "db_name", "table_name"
+ idx, exp := "idx_lang", makeExpressions(table, "lang")
+
+ d := NewDriver(tmpDir)
+ sqlIdx, err := d.Create(db, table, idx, exp, nil)
+ require.NoError(err)
+ pilosaIdx, ok := sqlIdx.(*pilosaIndex)
+ require.True(ok)
+ it := &testIndexKeyValueIter{
+ offset: 0,
+ total: 10,
+ expressions: sqlIdx.Expressions(),
+ location: offsetLocation,
+ }
+
+ ctx := sql.NewContext(context.Background())
+
+ err = d.Save(ctx, pilosaIdx, it)
+ require.NoError(err)
+
+ ranges := [2]int{3, 9}
+ sqlLookup, err := pilosaIdx.AscendLessThan(it.records[ranges[1]].values...)
+ require.NoError(err)
+ lessLookup, ok := sqlLookup.(*ascendLookup)
+ require.True(ok)
+
+ sqlLookup, err = pilosaIdx.AscendGreaterOrEqual(it.records[ranges[0]].values...)
+ require.NoError(err)
+ greaterLookup, ok := sqlLookup.(*ascendLookup)
+ require.True(ok)
+
+ interLookup := lessLookup.Intersection(greaterLookup)
+ require.NotNil(interLookup)
+ interIt, err := interLookup.Values()
+ require.NoError(err)
+
+ for i := ranges[0]; i < ranges[1]; i++ {
+ loc, err := interIt.Next()
+ require.NoError(err)
+ require.Equal(it.records[i].location, loc)
+ }
+
+ _, err = interIt.Next()
+ require.True(err == io.EOF)
+ require.NoError(interIt.Close())
+}
+
+func TestNegateIndex(t *testing.T) {
+ require := require.New(t)
+ setup(t)
+ defer cleanup(t)
+
+ db, table := "db_name", "table_name"
+
+ d := NewDriver(tmpDir)
+ idx, err := d.Create(db, table, "index_id", makeExpressions(table, "a"), nil)
+ require.NoError(err)
+
+ multiIdx, err := d.Create(
+ db, table, "multi_index_id",
+ makeExpressions(table, "a", "b"),
+ nil,
+ )
+ require.NoError(err)
+
+ it := &fixtureKeyValueIter{
+ fixtures: []kvfixture{
+ {"1", []interface{}{int64(2)}},
+ {"2", []interface{}{int64(7)}},
+ {"3", []interface{}{int64(1)}},
+ {"4", []interface{}{int64(1)}},
+ {"5", []interface{}{int64(7)}},
+ },
+ }
+
+ err = d.Save(sql.NewEmptyContext(), idx, it)
+ require.NoError(err)
+
+ multiIt := &fixtureKeyValueIter{
+ fixtures: []kvfixture{
+ {"1", []interface{}{int64(2), int64(6)}},
+ {"2", []interface{}{int64(7), int64(5)}},
+ {"3", []interface{}{int64(1), int64(2)}},
+ {"4", []interface{}{int64(1), int64(3)}},
+ {"5", []interface{}{int64(7), int64(6)}},
+ {"6", []interface{}{int64(10), int64(6)}},
+ {"7", []interface{}{int64(5), int64(1)}},
+ {"8", []interface{}{int64(6), int64(2)}},
+ {"9", []interface{}{int64(4), int64(0)}},
+ {"10", []interface{}{int64(3), int64(5)}},
+ },
+ }
+
+ err = d.Save(sql.NewEmptyContext(), multiIdx, multiIt)
+ require.NoError(err)
+
+ lookup, err := idx.(sql.NegateIndex).Not(int64(1))
+ require.NoError(err)
+
+ values, err := lookupValues(lookup)
+ require.NoError(err)
+
+ expected := []string{"1", "2", "5"}
+ require.Equal(expected, values)
+
+ lookup, err = multiIdx.(sql.NegateIndex).Not(int64(1), int64(6))
+ require.NoError(err)
+
+ values, err = lookupValues(lookup)
+ require.NoError(err)
+
+ expected = []string{"2", "7", "8", "9", "10"}
+ require.Equal(expected, values)
+}
+
+func TestPilosaHolder(t *testing.T) {
+ require := require.New(t)
+ setup(t)
+ defer cleanup(t)
+
+ h := pilosa.NewHolder()
+ h.Path = tmpDir
+ err := h.Open()
+ require.NoError(err)
+
+ idx1, err := h.CreateIndexIfNotExists("idx", pilosa.IndexOptions{})
+ require.NoError(err)
+ err = idx1.Open()
+ require.NoError(err)
+
+ f1, err := idx1.CreateFieldIfNotExists("f1", pilosa.OptFieldTypeDefault())
+ require.NoError(err)
+
+ _, err = f1.SetBit(0, 0, nil)
+ require.NoError(err)
+ _, err = f1.SetBit(0, 2, nil)
+ require.NoError(err)
+ r0, err := f1.Row(0)
+ require.NoError(err)
+
+ _, err = f1.SetBit(1, 0, nil)
+ require.NoError(err)
+ _, err = f1.SetBit(1, 1, nil)
+ require.NoError(err)
+ r1, err := f1.Row(1)
+ require.NoError(err)
+
+ _, err = f1.SetBit(2, 2, nil)
+ require.NoError(err)
+ _, err = f1.SetBit(2, 3, nil)
+ require.NoError(err)
+ r2, err := f1.Row(2)
+ require.NoError(err)
+
+ row := r0.Intersect(r1).Union(r2)
+ cols := row.Columns()
+ require.Equal(3, len(cols))
+ require.Equal(uint64(0), cols[0])
+ require.Equal(uint64(2), cols[1])
+ require.Equal(uint64(3), cols[2])
+
+ f2, err := idx1.CreateFieldIfNotExists("f2", pilosa.OptFieldTypeDefault())
+ require.NoError(err)
+
+ rowIDs := []uint64{0, 0, 1, 1}
+ colIDs := []uint64{1, 2, 0, 3}
+ err = f2.Import(rowIDs, colIDs, nil)
+ require.NoError(err)
+
+ r0, err = f2.Row(0)
+ require.NoError(err)
+
+ r1, err = f2.Row(1)
+ require.NoError(err)
+
+ row = r0.Union(r1)
+ cols = row.Columns()
+ require.Equal(4, len(cols))
+ require.Equal(uint64(0), cols[0])
+ require.Equal(uint64(1), cols[1])
+ require.Equal(uint64(2), cols[2])
+ require.Equal(uint64(3), cols[3])
+
+ r1, err = f1.Row(1)
+ require.NoError(err)
+ r0, err = f2.Row(0)
+ require.NoError(err)
+
+ row = r1.Intersect(r0)
+ cols = row.Columns()
+ require.Equal(1, len(cols))
+ require.Equal(uint64(1), cols[0])
+
+ err = idx1.Close()
+ require.NoError(err)
+ // -------------------------------------------------------------------------
+
+ idx2, err := h.CreateIndexIfNotExists("idx", pilosa.IndexOptions{})
+ require.NoError(err)
+ err = idx2.Open()
+ require.NoError(err)
+
+ f1 = idx2.Field("f1")
+
+ r2, err = f1.Row(2)
+ require.NoError(err)
+
+ f2 = idx2.Field("f2")
+
+ r0, err = f2.Row(0)
+ require.NoError(err)
+
+ r1, err = f2.Row(1)
+ require.NoError(err)
+
+ row = r0.Union(r1)
+ cols = row.Columns()
+ require.Equal(4, len(cols))
+ require.Equal(uint64(0), cols[0])
+ require.Equal(uint64(1), cols[1])
+ require.Equal(uint64(2), cols[2])
+ require.Equal(uint64(3), cols[3])
+
+ err = idx2.Close()
+ require.NoError(err)
+
+ err = h.Close()
+ require.NoError(err)
+}
+
+func makeExpressions(table string, names ...string) []sql.Expression {
+ var expressions []sql.Expression
+
+ for i, n := range names {
+ expressions = append(expressions,
+ expression.NewGetFieldWithTable(i, sql.Int64, table, n, true))
+ }
+
+ return expressions
+}
+
+func randLocation(offset int) []byte {
+ b := make([]byte, 1)
+ rand.Read(b)
+ return b
+}
+
+func offsetLocation(offset int) []byte {
+ b := make([]byte, 1)
+ b[0] = byte(offset % 10)
+ return b
+}
+
+// test implementation of sql.IndexKeyValueIter interface
+type testIndexKeyValueIter struct {
+ offset int
+ total int
+ expressions []string
+ location func(int) []byte
+
+ records []struct {
+ values []interface{}
+ location []byte
+ }
+}
+
+func (it *testIndexKeyValueIter) Next() ([]interface{}, []byte, error) {
+ if it.offset >= it.total {
+ return nil, nil, io.EOF
+ }
+
+ b := it.location(it.offset)
+
+ values := make([]interface{}, len(it.expressions))
+ for i, e := range it.expressions {
+ values[i] = e + "-" + hex.EncodeToString(b)
+ }
+
+ it.records = append(it.records, struct {
+ values []interface{}
+ location []byte
+ }{
+ values,
+ b,
+ })
+ it.offset++
+
+ return values, b, nil
+}
+
+func (it *testIndexKeyValueIter) Close() error {
+ it.offset = 0
+ it.records = nil
+ return nil
+}
+
+func setupAscendDescend(t *testing.T) (*pilosaIndex, func()) {
+ t.Helper()
+ require := require.New(t)
+ setup(t)
+
+ db, table, id := "db_name", "table_name", "index_id"
+ expressions := makeExpressions(table, "a", "b")
+
+ d := NewDriver(tmpDir)
+ sqlIdx, err := d.Create(db, table, id, expressions, nil)
+ require.NoError(err)
+
+ it := &fixtureKeyValueIter{
+ fixtures: []kvfixture{
+ {"9", []interface{}{int64(2), int64(6)}},
+ {"3", []interface{}{int64(7), int64(5)}},
+ {"1", []interface{}{int64(1), int64(2)}},
+ {"7", []interface{}{int64(1), int64(3)}},
+ {"4", []interface{}{int64(7), int64(6)}},
+ {"2", []interface{}{int64(10), int64(6)}},
+ {"5", []interface{}{int64(5), int64(1)}},
+ {"6", []interface{}{int64(6), int64(2)}},
+ {"10", []interface{}{int64(4), int64(0)}},
+ {"8", []interface{}{int64(3), int64(5)}},
+ },
+ }
+
+ err = d.Save(sql.NewEmptyContext(), sqlIdx, it)
+ require.NoError(err)
+
+ return sqlIdx.(*pilosaIndex), func() {
+ cleanup(t)
+ }
+}
+
+func lookupValues(lookup sql.IndexLookup) ([]string, error) {
+ iter, err := lookup.Values()
+ if err != nil {
+ return nil, err
+ }
+
+ var result []string
+ for {
+ k, err := iter.Next()
+ if err == io.EOF {
+ break
+ }
+
+ if err != nil {
+ return nil, err
+ }
+
+ result = append(result, string(k))
+ }
+
+ return result, nil
+}
+
+type kvfixture struct {
+ key string
+ values []interface{}
+}
+
+type fixtureKeyValueIter struct {
+ fixtures []kvfixture
+ pos int
+}
+
+func (i *fixtureKeyValueIter) Next() ([]interface{}, []byte, error) {
+ if i.pos >= len(i.fixtures) {
+ return nil, nil, io.EOF
+ }
+
+ f := i.fixtures[i.pos]
+ i.pos++
+ return f.values, []byte(f.key), nil
+}
+
+func (i *fixtureKeyValueIter) Close() error { return nil }
diff --git a/sql/index/pilosalib/index.go b/sql/index/pilosalib/index.go
new file mode 100644
index 000000000..8b95276f1
--- /dev/null
+++ b/sql/index/pilosalib/index.go
@@ -0,0 +1,296 @@
+package pilosalib
+
+import (
+ "github.com/pilosa/pilosa"
+ errors "gopkg.in/src-d/go-errors.v1"
+ "gopkg.in/src-d/go-mysql-server.v0/sql"
+ "gopkg.in/src-d/go-mysql-server.v0/sql/index"
+)
+
+var (
+ errInvalidKeys = errors.NewKind("expecting %d keys for index %q, got %d")
+)
+
+// pilosaIndex is an pilosa implementation of sql.Index interface
+type pilosaIndex struct {
+ index *pilosa.Index
+ mapping *mapping
+
+ db string
+ table string
+ id string
+ expressions []string
+}
+
+func newPilosaIndex(idx *pilosa.Index, mapping *mapping, cfg *index.Config) *pilosaIndex {
+ return &pilosaIndex{
+ index: idx,
+ db: cfg.DB,
+ table: cfg.Table,
+ id: cfg.ID,
+ expressions: cfg.Expressions,
+ mapping: mapping,
+ }
+}
+
+// Get returns an IndexLookup for the given key in the index.
+// If key parameter is not present then the returned iterator
+// will go through all the locations on the index.
+func (idx *pilosaIndex) Get(keys ...interface{}) (sql.IndexLookup, error) {
+ if len(keys) != len(idx.expressions) {
+ return nil, errInvalidKeys.New(len(idx.expressions), idx.ID(), len(keys))
+ }
+
+ return &indexLookup{
+ id: idx.ID(),
+ index: idx.index,
+ mapping: idx.mapping,
+ keys: keys,
+ expressions: idx.expressions,
+ }, nil
+}
+
+// Has checks if the given key is present in the index mapping
+func (idx *pilosaIndex) Has(key ...interface{}) (bool, error) {
+ idx.mapping.open()
+ defer idx.mapping.close()
+
+ n := len(key)
+ if n > len(idx.expressions) {
+ n = len(idx.expressions)
+ }
+
+ for i, expr := range idx.expressions {
+ name := fieldName(idx.ID(), expr)
+
+ val, err := idx.mapping.get(name, key[i])
+ if err != nil || val == nil {
+ return false, err
+ }
+ }
+
+ return true, nil
+}
+
+// Database returns the database name this index belongs to.
+func (idx *pilosaIndex) Database() string {
+ return idx.db
+}
+
+// Table returns the table name this index belongs to.
+func (idx *pilosaIndex) Table() string {
+ return idx.table
+}
+
+// ID returns the identifier of the index.
+func (idx *pilosaIndex) ID() string {
+ return idx.id
+}
+
+// Expressions returns the indexed expressions. If the result is more than
+// one expression, it means the index has multiple columns indexed. If it's
+// just one, it means it may be an expression or a column.
+func (idx *pilosaIndex) Expressions() []string {
+ return idx.expressions
+}
+
+func (pilosaIndex) Driver() string { return DriverID }
+
+func (idx *pilosaIndex) AscendGreaterOrEqual(keys ...interface{}) (sql.IndexLookup, error) {
+ if len(keys) != len(idx.expressions) {
+ return nil, errInvalidKeys.New(len(idx.expressions), idx.ID(), len(keys))
+ }
+
+ return newAscendLookup(&filteredLookup{
+ id: idx.ID(),
+ index: idx.index,
+ mapping: idx.mapping,
+ keys: keys,
+ expressions: idx.expressions,
+ }, keys, nil), nil
+}
+
+func (idx *pilosaIndex) AscendLessThan(keys ...interface{}) (sql.IndexLookup, error) {
+ if len(keys) != len(idx.expressions) {
+ return nil, errInvalidKeys.New(len(idx.expressions), idx.ID(), len(keys))
+ }
+
+ return newAscendLookup(&filteredLookup{
+ id: idx.ID(),
+ index: idx.index,
+ mapping: idx.mapping,
+ keys: keys,
+ expressions: idx.expressions,
+ }, nil, keys), nil
+}
+
+func (idx *pilosaIndex) AscendRange(greaterOrEqual, lessThan []interface{}) (sql.IndexLookup, error) {
+ if len(greaterOrEqual) != len(idx.expressions) {
+ return nil, errInvalidKeys.New(len(idx.expressions), idx.ID(), len(greaterOrEqual))
+ }
+
+ if len(lessThan) != len(idx.expressions) {
+ return nil, errInvalidKeys.New(len(idx.expressions), idx.ID(), len(lessThan))
+ }
+
+ return newAscendLookup(&filteredLookup{
+ id: idx.ID(),
+ index: idx.index,
+ mapping: idx.mapping,
+ expressions: idx.expressions,
+ }, greaterOrEqual, lessThan), nil
+}
+
+func (idx *pilosaIndex) DescendGreater(keys ...interface{}) (sql.IndexLookup, error) {
+ if len(keys) != len(idx.expressions) {
+ return nil, errInvalidKeys.New(len(idx.expressions), idx.ID(), len(keys))
+ }
+
+ return newDescendLookup(&filteredLookup{
+ id: idx.ID(),
+ index: idx.index,
+ mapping: idx.mapping,
+ keys: keys,
+ expressions: idx.expressions,
+ reverse: true,
+ }, keys, nil), nil
+}
+
+func (idx *pilosaIndex) DescendLessOrEqual(keys ...interface{}) (sql.IndexLookup, error) {
+ if len(keys) != len(idx.expressions) {
+ return nil, errInvalidKeys.New(len(idx.expressions), idx.ID(), len(keys))
+ }
+
+ return newDescendLookup(&filteredLookup{
+ id: idx.ID(),
+ index: idx.index,
+ mapping: idx.mapping,
+ keys: keys,
+ expressions: idx.expressions,
+ reverse: true,
+ }, nil, keys), nil
+}
+
+func (idx *pilosaIndex) DescendRange(lessOrEqual, greaterThan []interface{}) (sql.IndexLookup, error) {
+ if len(lessOrEqual) != len(idx.expressions) {
+ return nil, errInvalidKeys.New(len(idx.expressions), idx.ID(), len(lessOrEqual))
+ }
+
+ if len(greaterThan) != len(idx.expressions) {
+ return nil, errInvalidKeys.New(len(idx.expressions), idx.ID(), len(greaterThan))
+ }
+
+ return newDescendLookup(&filteredLookup{
+ id: idx.ID(),
+ index: idx.index,
+ mapping: idx.mapping,
+ expressions: idx.expressions,
+ reverse: true,
+ }, greaterThan, lessOrEqual), nil
+}
+
+func (idx *pilosaIndex) Not(keys ...interface{}) (sql.IndexLookup, error) {
+ if len(keys) != len(idx.expressions) {
+ return nil, errInvalidKeys.New(len(idx.expressions), idx.ID(), len(keys))
+ }
+
+ return &negateLookup{
+ id: idx.ID(),
+ index: idx.index,
+ mapping: idx.mapping,
+ keys: keys,
+ expressions: idx.expressions,
+ }, nil
+}
+
+func newAscendLookup(f *filteredLookup, gte []interface{}, lt []interface{}) *ascendLookup {
+ l := &ascendLookup{filteredLookup: f, gte: gte, lt: lt}
+ if l.filter == nil {
+ l.filter = func(i int, value []byte) (bool, error) {
+ var v interface{}
+ var err error
+ if len(l.gte) > 0 {
+ v, err = decodeGob(value, l.gte[i])
+ if err != nil {
+ return false, err
+ }
+
+ cmp, err := compare(v, l.gte[i])
+ if err != nil {
+ return false, err
+ }
+
+ if cmp < 0 {
+ return false, nil
+ }
+ }
+
+ if len(l.lt) > 0 {
+ if v == nil {
+ v, err = decodeGob(value, l.lt[i])
+ if err != nil {
+ return false, err
+ }
+ }
+
+ cmp, err := compare(v, l.lt[i])
+ if err != nil {
+ return false, err
+ }
+
+ if cmp >= 0 {
+ return false, nil
+ }
+ }
+
+ return true, nil
+ }
+ }
+ return l
+}
+
+func newDescendLookup(f *filteredLookup, gt []interface{}, lte []interface{}) *descendLookup {
+ l := &descendLookup{filteredLookup: f, gt: gt, lte: lte}
+ if l.filter == nil {
+ l.filter = func(i int, value []byte) (bool, error) {
+ var v interface{}
+ var err error
+ if len(l.gt) > 0 {
+ v, err = decodeGob(value, l.gt[i])
+ if err != nil {
+ return false, err
+ }
+
+ cmp, err := compare(v, l.gt[i])
+ if err != nil {
+ return false, err
+ }
+
+ if cmp <= 0 {
+ return false, nil
+ }
+ }
+
+ if len(l.lte) > 0 {
+ if v == nil {
+ v, err = decodeGob(value, l.lte[i])
+ if err != nil {
+ return false, err
+ }
+ }
+
+ cmp, err := compare(v, l.lte[i])
+ if err != nil {
+ return false, err
+ }
+
+ if cmp > 0 {
+ return false, nil
+ }
+ }
+
+ return true, nil
+ }
+ }
+ return l
+}
diff --git a/sql/index/pilosalib/iterator.go b/sql/index/pilosalib/iterator.go
new file mode 100644
index 000000000..b7ae49c73
--- /dev/null
+++ b/sql/index/pilosalib/iterator.go
@@ -0,0 +1,83 @@
+package pilosalib
+
+import (
+ "io"
+
+ "github.com/boltdb/bolt"
+ "github.com/sirupsen/logrus"
+)
+
+type locationValueIter struct {
+ locations [][]byte
+ pos int
+}
+
+func (i *locationValueIter) Next() ([]byte, error) {
+ if i.pos >= len(i.locations) {
+ return nil, io.EOF
+ }
+
+ i.pos++
+ return i.locations[i.pos-1], nil
+}
+
+func (i *locationValueIter) Close() error {
+ i.locations = nil
+ return nil
+}
+
+type indexValueIter struct {
+ offset uint64
+ total uint64
+ bits []uint64
+ mapping *mapping
+ indexName string
+
+ // share transaction and bucket on all getLocation calls
+ bucket *bolt.Bucket
+ tx *bolt.Tx
+}
+
+func (it *indexValueIter) Next() ([]byte, error) {
+ if it.bucket == nil {
+ bucket, err := it.mapping.getBucket(it.indexName, false)
+ if err != nil {
+ return nil, err
+ }
+
+ it.bucket = bucket
+ it.tx = bucket.Tx()
+ }
+
+ if it.offset >= it.total {
+ if err := it.Close(); err != nil {
+ logrus.WithField("err", err.Error()).
+ Error("unable to close the pilosa index value iterator")
+ }
+
+ if it.tx != nil {
+ it.tx.Rollback()
+ }
+
+ return nil, io.EOF
+ }
+
+ var colID uint64
+ if it.bits == nil {
+ colID = it.offset
+ } else {
+ colID = it.bits[it.offset]
+ }
+
+ it.offset++
+
+ return it.mapping.getLocationFromBucket(it.bucket, colID)
+}
+
+func (it *indexValueIter) Close() error {
+ if it.tx != nil {
+ it.tx.Rollback()
+ }
+
+ return it.mapping.close()
+}
diff --git a/sql/index/pilosalib/lookup.go b/sql/index/pilosalib/lookup.go
new file mode 100644
index 000000000..7b8e21bcc
--- /dev/null
+++ b/sql/index/pilosalib/lookup.go
@@ -0,0 +1,674 @@
+package pilosalib
+
+import (
+ "bytes"
+ "encoding/gob"
+ "io"
+ "strings"
+ "time"
+
+ "github.com/pilosa/pilosa"
+ errors "gopkg.in/src-d/go-errors.v1"
+ "gopkg.in/src-d/go-mysql-server.v0/sql"
+)
+
+var (
+ errUnknownType = errors.NewKind("unknown type %T received as value")
+ errTypeMismatch = errors.NewKind("cannot compare type %T with type %T")
+ errUnmergeableType = errors.NewKind("unmergeable type %T")
+
+ // operation functors
+ intersect = func(r1, r2 *pilosa.Row) *pilosa.Row {
+ if r1 == nil {
+ return r2
+ }
+ return r1.Intersect(r2)
+ }
+ union = func(r1, r2 *pilosa.Row) *pilosa.Row {
+ if r1 == nil {
+ return r2
+ }
+ return r1.Union(r2)
+ }
+ difference = func(r1, r2 *pilosa.Row) *pilosa.Row {
+ if r1 == nil {
+ return r2
+ }
+ return r1.Difference(r2)
+ }
+)
+
+type (
+
+ // indexLookup implement following interfaces:
+ // sql.IndexLookup, sql.Mergeable, sql.SetOperations
+ indexLookup struct {
+ id string
+ index *pilosa.Index
+ mapping *mapping
+ keys []interface{}
+ expressions []string
+ operations []*lookupOperation
+ }
+
+ lookupOperation struct {
+ lookup sql.IndexLookup
+ operation func(*pilosa.Row, *pilosa.Row) *pilosa.Row
+ }
+
+ pilosaLookup interface {
+ indexName() string
+ values() (*pilosa.Row, error)
+ }
+)
+
+func (l *indexLookup) indexName() string {
+ return l.index.Name()
+}
+
+func (l *indexLookup) values() (*pilosa.Row, error) {
+ l.mapping.open()
+ defer l.mapping.close()
+
+ var row *pilosa.Row
+ for i, expr := range l.expressions {
+ field := l.index.Field(fieldName(l.id, expr))
+ rowID, err := l.mapping.rowID(field.Name(), l.keys[i])
+ if err == io.EOF {
+ continue
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ r, err := field.Row(rowID)
+ if err != nil {
+ return nil, err
+ }
+
+ row = intersect(row, r)
+ }
+ if row == nil {
+ return nil, nil
+ }
+
+ // evaluate composition of operations
+ for _, op := range l.operations {
+ var (
+ r *pilosa.Row
+ e error
+ )
+
+ il, ok := op.lookup.(pilosaLookup)
+ if !ok {
+ return nil, errUnmergeableType.New(op.lookup)
+ }
+
+ r, e = il.values()
+ if e != nil {
+ return nil, e
+ }
+
+ row = op.operation(row, r)
+ }
+
+ return row, nil
+}
+
+// Values implements sql.IndexLookup.Values
+func (l *indexLookup) Values() (sql.IndexValueIter, error) {
+ l.index.Open()
+ defer l.index.Close()
+
+ row, err := l.values()
+ if err != nil {
+ return nil, err
+ }
+
+ l.mapping.open()
+ if row == nil {
+ return &indexValueIter{mapping: l.mapping, indexName: l.index.Name()}, nil
+ }
+
+ bits := row.Columns()
+ return &indexValueIter{
+ total: uint64(len(bits)),
+ bits: bits,
+ mapping: l.mapping,
+ indexName: l.index.Name(),
+ }, nil
+}
+
+// IsMergeable implements sql.Mergeable interface.
+func (l *indexLookup) IsMergeable(lookup sql.IndexLookup) bool {
+ if il, ok := lookup.(pilosaLookup); ok {
+ return il.indexName() == l.indexName()
+ }
+
+ return false
+}
+
+// Intersection implements sql.SetOperations interface
+func (l *indexLookup) Intersection(lookups ...sql.IndexLookup) sql.IndexLookup {
+ lookup := *l
+ for _, li := range lookups {
+ lookup.operations = append(lookup.operations, &lookupOperation{li, intersect})
+ }
+
+ return &lookup
+}
+
+// Union implements sql.SetOperations interface
+func (l *indexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup {
+ lookup := *l
+ for _, li := range lookups {
+ lookup.operations = append(lookup.operations, &lookupOperation{li, union})
+ }
+
+ return &lookup
+}
+
+// Difference implements sql.SetOperations interface
+func (l *indexLookup) Difference(lookups ...sql.IndexLookup) sql.IndexLookup {
+ lookup := *l
+ for _, li := range lookups {
+ lookup.operations = append(lookup.operations, &lookupOperation{li, difference})
+ }
+
+ return &lookup
+}
+
+type filteredLookup struct {
+ id string
+ index *pilosa.Index
+ mapping *mapping
+ keys []interface{}
+ expressions []string
+ operations []*lookupOperation
+
+ reverse bool
+ filter func(int, []byte) (bool, error)
+}
+
+func (l *filteredLookup) indexName() string {
+ return l.index.Name()
+}
+
+func (l *filteredLookup) values() (*pilosa.Row, error) {
+ l.mapping.open()
+ defer l.mapping.close()
+
+ // evaluate Intersection of bitmaps
+ var row *pilosa.Row
+ for i, expr := range l.expressions {
+ field := l.index.Field(fieldName(l.id, expr))
+ rows, err := l.mapping.filter(field.Name(), func(b []byte) (bool, error) {
+ return l.filter(i, b)
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ var r *pilosa.Row
+ for _, ri := range rows {
+ rr, err := field.Row(ri)
+ if err != nil {
+ return nil, err
+ }
+ r = union(r, rr)
+ }
+
+ row = intersect(row, r)
+ }
+
+ if row == nil {
+ return nil, nil
+ }
+
+ // evaluate composition of operations
+ for _, op := range l.operations {
+ var (
+ r *pilosa.Row
+ e error
+ )
+
+ il, ok := op.lookup.(pilosaLookup)
+ if !ok {
+ return nil, errUnmergeableType.New(op.lookup)
+ }
+
+ r, e = il.values()
+ if e != nil {
+ return nil, e
+ }
+ if r == nil {
+ continue
+ }
+
+ row = op.operation(row, r)
+ }
+
+ return row, nil
+}
+
+func (l *filteredLookup) Values() (sql.IndexValueIter, error) {
+ l.index.Open()
+ defer l.index.Close()
+
+ row, err := l.values()
+ if err != nil {
+ return nil, err
+ }
+
+ l.mapping.open()
+ if row == nil {
+ return &indexValueIter{mapping: l.mapping, indexName: l.index.Name()}, nil
+ }
+
+ bits := row.Columns()
+ locations, err := l.mapping.sortedLocations(l.index.Name(), bits, l.reverse)
+ if err != nil {
+ return nil, err
+ }
+
+ return &locationValueIter{locations: locations}, nil
+}
+
+// IsMergeable implements sql.Mergeable interface.
+func (l *filteredLookup) IsMergeable(lookup sql.IndexLookup) bool {
+ if il, ok := lookup.(pilosaLookup); ok {
+ return il.indexName() == l.indexName()
+ }
+ return false
+}
+
+// Intersection implements sql.SetOperations interface
+func (l *filteredLookup) Intersection(lookups ...sql.IndexLookup) sql.IndexLookup {
+ lookup := *l
+ for _, li := range lookups {
+ lookup.operations = append(lookup.operations, &lookupOperation{li, intersect})
+ }
+
+ return &lookup
+}
+
+// Union implements sql.SetOperations interface
+func (l *filteredLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup {
+ lookup := *l
+ for _, li := range lookups {
+ lookup.operations = append(lookup.operations, &lookupOperation{li, union})
+ }
+
+ return &lookup
+}
+
+// Difference implements sql.SetOperations interface
+func (l *filteredLookup) Difference(lookups ...sql.IndexLookup) sql.IndexLookup {
+ lookup := *l
+ for _, li := range lookups {
+ lookup.operations = append(lookup.operations, &lookupOperation{li, difference})
+ }
+
+ return &lookup
+}
+
+type ascendLookup struct {
+ *filteredLookup
+ gte []interface{}
+ lt []interface{}
+}
+
+type descendLookup struct {
+ *filteredLookup
+ gt []interface{}
+ lte []interface{}
+}
+
+type negateLookup struct {
+ id string
+ index *pilosa.Index
+ mapping *mapping
+ keys []interface{}
+ expressions []string
+ operations []*lookupOperation
+}
+
+func (l *negateLookup) indexName() string { return l.index.Name() }
+
+func (l *negateLookup) values() (*pilosa.Row, error) {
+ l.mapping.open()
+ defer l.mapping.close()
+
+ var row *pilosa.Row
+ for i, expr := range l.expressions {
+ field := l.index.Field(fieldName(l.id, expr))
+ maxRowID, err := l.mapping.getMaxRowID(field.Name())
+ if err != nil {
+ return nil, err
+ }
+
+ // Since Pilosa does not have a negation in PQL (see:
+ // https://github.com/pilosa/pilosa/issues/807), we have to get all the
+ // ones in all the rows and join them, and then make difference between
+ // them and the ones in the row of the given value.
+ var r *pilosa.Row
+ // rowIDs start with 1
+ for ri := uint64(1); ri <= maxRowID; ri++ {
+ rr, err := field.Row(ri)
+ if err != nil {
+ return nil, err
+ }
+ r = union(r, rr)
+ }
+
+ rowID, err := l.mapping.rowID(field.Name(), l.keys[i])
+ if err == io.EOF {
+ continue
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ rr, err := field.Row(rowID)
+ if err != nil {
+ return nil, err
+ }
+ r = difference(r, rr)
+
+ row = intersect(row, r)
+ }
+ if row == nil {
+ return nil, nil
+ }
+
+ // evaluate composition of operations
+ for _, op := range l.operations {
+ var (
+ r *pilosa.Row
+ e error
+ )
+
+ il, ok := op.lookup.(pilosaLookup)
+ if !ok {
+ return nil, errUnmergeableType.New(op.lookup)
+ }
+
+ r, e = il.values()
+ if e != nil {
+ return nil, e
+ }
+
+ if r == nil {
+ continue
+ }
+
+ row = op.operation(row, r)
+ }
+
+ return row, nil
+}
+
+// Values implements sql.IndexLookup.Values
+func (l *negateLookup) Values() (sql.IndexValueIter, error) {
+ l.index.Open()
+ defer l.index.Close()
+
+ row, err := l.values()
+ if err != nil {
+ return nil, err
+ }
+
+ l.mapping.open()
+ if row == nil {
+ return &indexValueIter{mapping: l.mapping, indexName: l.index.Name()}, nil
+ }
+
+ bits := row.Columns()
+ return &indexValueIter{
+ total: uint64(len(bits)),
+ bits: bits,
+ mapping: l.mapping,
+ indexName: l.index.Name(),
+ }, nil
+}
+
+// IsMergeable implements sql.Mergeable interface.
+func (l *negateLookup) IsMergeable(lookup sql.IndexLookup) bool {
+ if il, ok := lookup.(pilosaLookup); ok {
+ return il.indexName() == l.indexName()
+ }
+
+ return false
+}
+
+// Intersection implements sql.SetOperations interface
+func (l *negateLookup) Intersection(lookups ...sql.IndexLookup) sql.IndexLookup {
+ lookup := *l
+ for _, li := range lookups {
+ lookup.operations = append(lookup.operations, &lookupOperation{li, intersect})
+ }
+
+ return &lookup
+}
+
+// Union implements sql.SetOperations interface
+func (l *negateLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup {
+ lookup := *l
+ for _, li := range lookups {
+ lookup.operations = append(lookup.operations, &lookupOperation{li, union})
+ }
+
+ return &lookup
+}
+
+// Difference implements sql.SetOperations interface
+func (l *negateLookup) Difference(lookups ...sql.IndexLookup) sql.IndexLookup {
+ lookup := *l
+ for _, li := range lookups {
+ lookup.operations = append(lookup.operations, &lookupOperation{li, difference})
+ }
+
+ return &lookup
+}
+
+func decodeGob(k []byte, value interface{}) (interface{}, error) {
+ decoder := gob.NewDecoder(bytes.NewBuffer(k))
+
+ switch value.(type) {
+ case string:
+ var v string
+ err := decoder.Decode(&v)
+ return v, err
+ case int32:
+ var v int32
+ err := decoder.Decode(&v)
+ return v, err
+ case int64:
+ var v int64
+ err := decoder.Decode(&v)
+ return v, err
+ case uint32:
+ var v uint32
+ err := decoder.Decode(&v)
+ return v, err
+ case uint64:
+ var v uint64
+ err := decoder.Decode(&v)
+ return v, err
+ case float64:
+ var v float64
+ err := decoder.Decode(&v)
+ return v, err
+ case time.Time:
+ var v time.Time
+ err := decoder.Decode(&v)
+ return v, err
+ case []byte:
+ var v []byte
+ err := decoder.Decode(&v)
+ return v, err
+ case bool:
+ var v bool
+ err := decoder.Decode(&v)
+ return v, err
+ case []interface{}:
+ var v []interface{}
+ err := decoder.Decode(&v)
+ return v, err
+ default:
+ return nil, errUnknownType.New(value)
+ }
+}
+
+// compare two values of the same underlying type. The values MUST be of the
+// same type.
+func compare(a, b interface{}) (int, error) {
+ switch a := a.(type) {
+ case bool:
+ v, ok := b.(bool)
+ if !ok {
+ return 0, errTypeMismatch.New(a, b)
+ }
+
+ if a == v {
+ return 0, nil
+ }
+
+ if a == false {
+ return -1, nil
+ }
+
+ return 1, nil
+ case string:
+ v, ok := b.(string)
+ if !ok {
+ return 0, errTypeMismatch.New(a, b)
+ }
+
+ return strings.Compare(a, v), nil
+ case int32:
+ v, ok := b.(int32)
+ if !ok {
+ return 0, errTypeMismatch.New(a, b)
+ }
+
+ if a == v {
+ return 0, nil
+ }
+
+ if a < v {
+ return -1, nil
+ }
+
+ return 1, nil
+ case int64:
+ v, ok := b.(int64)
+ if !ok {
+ return 0, errTypeMismatch.New(a, b)
+ }
+
+ if a == v {
+ return 0, nil
+ }
+
+ if a < v {
+ return -1, nil
+ }
+
+ return 1, nil
+ case uint32:
+ v, ok := b.(uint32)
+ if !ok {
+ return 0, errTypeMismatch.New(a, b)
+ }
+
+ if a == v {
+ return 0, nil
+ }
+
+ if a < v {
+ return -1, nil
+ }
+
+ return 1, nil
+ case uint64:
+ v, ok := b.(uint64)
+ if !ok {
+ return 0, errTypeMismatch.New(a, b)
+ }
+
+ if a == v {
+ return 0, nil
+ }
+
+ if a < v {
+ return -1, nil
+ }
+
+ return 1, nil
+ case float64:
+ v, ok := b.(float64)
+ if !ok {
+ return 0, errTypeMismatch.New(a, b)
+ }
+
+ if a == v {
+ return 0, nil
+ }
+
+ if a < v {
+ return -1, nil
+ }
+
+ return 1, nil
+ case []byte:
+ v, ok := b.([]byte)
+ if !ok {
+ return 0, errTypeMismatch.New(a, b)
+ }
+ return bytes.Compare(a, v), nil
+ case []interface{}:
+ v, ok := b.([]interface{})
+ if !ok {
+ return 0, errTypeMismatch.New(a, b)
+ }
+
+ if len(a) < len(v) {
+ return -1, nil
+ }
+
+ if len(a) > len(v) {
+ return 1, nil
+ }
+
+ for i := range a {
+ cmp, err := compare(a[i], v[i])
+ if err != nil {
+ return 0, err
+ }
+
+ if cmp != 0 {
+ return cmp, nil
+ }
+ }
+
+ return 0, nil
+ case time.Time:
+ v, ok := b.(time.Time)
+ if !ok {
+ return 0, errTypeMismatch.New(a, b)
+ }
+
+ if a.Equal(v) {
+ return 0, nil
+ }
+
+ if a.Before(v) {
+ return -1, nil
+ }
+
+ return 1, nil
+ default:
+ return 0, errUnknownType.New(a)
+ }
+}
diff --git a/sql/index/pilosalib/lookup_test.go b/sql/index/pilosalib/lookup_test.go
new file mode 100644
index 000000000..1514f738d
--- /dev/null
+++ b/sql/index/pilosalib/lookup_test.go
@@ -0,0 +1,203 @@
+package pilosalib
+
+import (
+ "bytes"
+ "encoding/gob"
+ "fmt"
+ "os"
+ "testing"
+ "time"
+
+ "github.com/pilosa/pilosa"
+ "github.com/stretchr/testify/require"
+ errors "gopkg.in/src-d/go-errors.v1"
+ "gopkg.in/src-d/go-mysql-server.v0/sql"
+)
+
+func TestCompare(t *testing.T) {
+ now := time.Now()
+ testCases := []struct {
+ a, b interface{}
+ err *errors.Kind
+ expected int
+ }{
+ {true, true, nil, 0},
+ {false, true, nil, -1},
+ {true, false, nil, 1},
+ {false, false, nil, 0},
+ {true, 0, errTypeMismatch, 0},
+
+ {"a", "b", nil, -1},
+ {"b", "a", nil, 1},
+ {"a", "a", nil, 0},
+ {"a", 1, errTypeMismatch, 0},
+
+ {int32(1), int32(2), nil, -1},
+ {int32(2), int32(1), nil, 1},
+ {int32(2), int32(2), nil, 0},
+ {int32(1), "", errTypeMismatch, 0},
+
+ {int64(1), int64(2), nil, -1},
+ {int64(2), int64(1), nil, 1},
+ {int64(2), int64(2), nil, 0},
+ {int64(1), "", errTypeMismatch, 0},
+
+ {uint32(1), uint32(2), nil, -1},
+ {uint32(2), uint32(1), nil, 1},
+ {uint32(2), uint32(2), nil, 0},
+ {uint32(1), "", errTypeMismatch, 0},
+
+ {uint64(1), uint64(2), nil, -1},
+ {uint64(2), uint64(1), nil, 1},
+ {uint64(2), uint64(2), nil, 0},
+ {uint64(1), "", errTypeMismatch, 0},
+
+ {float64(1), float64(2), nil, -1},
+ {float64(2), float64(1), nil, 1},
+ {float64(2), float64(2), nil, 0},
+ {float64(1), "", errTypeMismatch, 0},
+
+ {now.Add(-1 * time.Hour), now, nil, -1},
+ {now, now.Add(-1 * time.Hour), nil, 1},
+ {now, now, nil, 0},
+ {now, 1, errTypeMismatch, -1},
+
+ {[]interface{}{"a", "a"}, []interface{}{"a", "b"}, nil, -1},
+ {[]interface{}{"a", "b"}, []interface{}{"a", "a"}, nil, 1},
+ {[]interface{}{"a", "a"}, []interface{}{"a", "a"}, nil, 0},
+ {[]interface{}{"b"}, []interface{}{"a", "b"}, nil, -1},
+ {[]interface{}{"b"}, 1, errTypeMismatch, -1},
+
+ {[]byte{0, 1}, []byte{1, 1}, nil, -1},
+ {[]byte{1, 1}, []byte{0, 1}, nil, 1},
+ {[]byte{1, 1}, []byte{1, 1}, nil, 0},
+ {[]byte{1}, []byte{0, 1}, nil, 1},
+ {[]byte{0, 1}, 1, errTypeMismatch, -1},
+
+ {time.Duration(0), nil, errUnknownType, -1},
+ }
+
+ for _, tt := range testCases {
+ name := fmt.Sprintf("(%T)(%v) and (%T)(%v)", tt.a, tt.a, tt.b, tt.b)
+ t.Run(name, func(t *testing.T) {
+ require := require.New(t)
+ cmp, err := compare(tt.a, tt.b)
+ if tt.err != nil {
+ require.Error(err)
+ require.True(tt.err.Is(err))
+ } else {
+ require.NoError(err)
+ require.Equal(tt.expected, cmp)
+ }
+ })
+ }
+}
+
+func TestDecodeGob(t *testing.T) {
+ testCases := []interface{}{
+ "foo",
+ int32(1),
+ int64(1),
+ uint32(1),
+ uint64(1),
+ float64(1),
+ true,
+ time.Date(2018, time.August, 1, 1, 1, 1, 1, time.Local),
+ []byte("foo"),
+ []interface{}{1, 3, 3, 7},
+ }
+
+ for _, tt := range testCases {
+ name := fmt.Sprintf("(%T)(%v)", tt, tt)
+ t.Run(name, func(t *testing.T) {
+ require := require.New(t)
+
+ var buf bytes.Buffer
+ require.NoError(gob.NewEncoder(&buf).Encode(tt))
+
+ result, err := decodeGob(buf.Bytes(), tt)
+ require.NoError(err)
+ require.Equal(tt, result)
+ })
+ }
+}
+
+func TestMergeable(t *testing.T) {
+ require := require.New(t)
+ h := pilosa.NewHolder()
+ h.Path = os.TempDir()
+
+ i1, err := h.CreateIndexIfNotExists("i1", pilosa.IndexOptions{})
+ require.NoError(err)
+ i2, err := h.CreateIndexIfNotExists("i2", pilosa.IndexOptions{})
+ require.NoError(err)
+
+ testCases := []struct {
+ i1 sql.IndexLookup
+ i2 sql.IndexLookup
+ expected bool
+ }{
+ {
+ i1: &indexLookup{index: i1},
+ i2: &indexLookup{index: i1},
+ expected: true,
+ },
+ {
+ i1: &indexLookup{index: i1},
+ i2: &indexLookup{index: i2},
+ expected: false,
+ },
+ {
+ i1: &indexLookup{index: i1},
+ i2: &ascendLookup{filteredLookup: &filteredLookup{index: i1}},
+ expected: true,
+ },
+ {
+ i1: &descendLookup{filteredLookup: &filteredLookup{index: i1}},
+ i2: &ascendLookup{filteredLookup: &filteredLookup{index: i1}},
+ expected: true,
+ },
+ {
+ i1: &descendLookup{filteredLookup: &filteredLookup{index: i1}},
+ i2: &indexLookup{index: i2},
+ expected: false,
+ },
+ {
+ i1: &descendLookup{filteredLookup: &filteredLookup{index: i1}},
+ i2: &descendLookup{filteredLookup: &filteredLookup{index: i2}},
+ expected: false,
+ },
+ {
+ i1: &negateLookup{index: i1},
+ i2: &negateLookup{index: i1},
+ expected: true,
+ },
+ {
+ i1: &negateLookup{index: i1},
+ i2: &negateLookup{index: i2},
+ expected: false,
+ },
+ {
+ i1: &negateLookup{index: i1},
+ i2: &indexLookup{index: i1},
+ expected: true,
+ },
+ {
+ i1: &negateLookup{index: i1},
+ i2: &descendLookup{filteredLookup: &filteredLookup{index: i1}},
+ expected: true,
+ },
+ {
+ i1: &negateLookup{index: i1},
+ i2: &ascendLookup{filteredLookup: &filteredLookup{index: i1}},
+ expected: true,
+ },
+ }
+
+ for _, tc := range testCases {
+ m1, ok := tc.i1.(sql.Mergeable)
+ require.True(ok)
+
+ require.Equal(tc.expected, m1.IsMergeable(tc.i2))
+ }
+}
diff --git a/sql/index/pilosalib/mapping.go b/sql/index/pilosalib/mapping.go
new file mode 100644
index 000000000..1a7d1ff05
--- /dev/null
+++ b/sql/index/pilosalib/mapping.go
@@ -0,0 +1,403 @@
+package pilosalib
+
+import (
+ "bytes"
+ "encoding/binary"
+ "encoding/gob"
+ "fmt"
+ "io"
+ "sort"
+ "sync"
+
+ "github.com/boltdb/bolt"
+)
+
+// mapping
+// buckets:
+// - index name: columndID uint64 -> location []byte
+// - frame name: value []byte (gob encoding) -> rowID uint64
+type mapping struct {
+ path string
+
+ mut sync.RWMutex
+ db *bolt.DB
+
+ // in create mode there's only one transaction closed explicitly by
+ // commit function
+ create bool
+ tx *bolt.Tx
+
+ clientMut sync.Mutex
+ clients int
+}
+
+func newMapping(path string) *mapping {
+ return &mapping{path: path}
+}
+
+func (m *mapping) open() {
+ m.openCreate(false)
+}
+
+// openCreate opens and sets creation mode in the database.
+func (m *mapping) openCreate(create bool) {
+ m.clientMut.Lock()
+ defer m.clientMut.Unlock()
+ m.clients++
+ m.create = create
+}
+
+func (m *mapping) close() error {
+ m.clientMut.Lock()
+ defer m.clientMut.Unlock()
+
+ if m.clients > 1 {
+ m.clients--
+ return nil
+ }
+
+ m.clients = 0
+
+ m.mut.Lock()
+ defer m.mut.Unlock()
+
+ if m.db != nil {
+ if err := m.db.Close(); err != nil {
+ return err
+ }
+ m.db = nil
+ }
+
+ return nil
+}
+
+func (m *mapping) query(fn func() error) error {
+ m.mut.Lock()
+ if m.db == nil {
+ var err error
+ m.db, err = bolt.Open(m.path, 0640, nil)
+ if err != nil {
+ m.mut.Unlock()
+ return err
+ }
+ }
+ m.mut.Unlock()
+
+ m.mut.RLock()
+ defer m.mut.RUnlock()
+ return fn()
+}
+
+func (m *mapping) rowID(frameName string, value interface{}) (uint64, error) {
+ val, err := m.get(frameName, value)
+ if err != nil {
+ return 0, err
+ }
+ if val == nil {
+ return 0, io.EOF
+ }
+
+ return binary.LittleEndian.Uint64(val), err
+}
+
+// commit saves current transaction, if cont is true a new transaction will be
+// created again in the next query. Only for create mode.
+func (m *mapping) commit(cont bool) error {
+ m.clientMut.Lock()
+ defer m.clientMut.Unlock()
+
+ var err error
+ if m.create && m.tx != nil {
+ err = m.tx.Commit()
+ }
+
+ m.create = cont
+ m.tx = nil
+
+ return err
+}
+
+func (m *mapping) rollback() error {
+ m.clientMut.Lock()
+ defer m.clientMut.Unlock()
+
+ var err error
+ if m.create && m.tx != nil {
+ err = m.tx.Rollback()
+ }
+
+ m.create = false
+ m.tx = nil
+
+ return err
+}
+
+func (m *mapping) transaction(writable bool, f func(*bolt.Tx) error) error {
+ var tx *bolt.Tx
+ var err error
+ if m.create {
+ m.clientMut.Lock()
+ if m.tx == nil {
+ m.tx, err = m.db.Begin(true)
+ if err != nil {
+ m.clientMut.Unlock()
+ return err
+ }
+ }
+
+ m.clientMut.Unlock()
+
+ tx = m.tx
+ } else {
+ tx, err = m.db.Begin(writable)
+ if err != nil {
+ return err
+ }
+ }
+
+ err = f(tx)
+
+ if m.create {
+ return err
+ }
+
+ if err != nil {
+ tx.Rollback()
+ return err
+ }
+
+ return tx.Commit()
+}
+
+func (m *mapping) getRowID(frameName string, value interface{}) (uint64, error) {
+ var id uint64
+ err := m.query(func() error {
+ var buf bytes.Buffer
+ enc := gob.NewEncoder(&buf)
+ err := enc.Encode(value)
+ if err != nil {
+ return err
+ }
+
+ err = m.transaction(true, func(tx *bolt.Tx) error {
+ b, err := tx.CreateBucketIfNotExists([]byte(frameName))
+ if err != nil {
+ return err
+ }
+
+ key := buf.Bytes()
+ val := b.Get(key)
+ if val != nil {
+ id = binary.LittleEndian.Uint64(val)
+ return nil
+ }
+
+ // the first NextSequence is 1 so the first id will be 1
+ // this can only fail if the transaction is closed
+ id, _ = b.NextSequence()
+
+ val = make([]byte, 8)
+ binary.LittleEndian.PutUint64(val, id)
+ err = b.Put(key, val)
+ return err
+ })
+
+ return err
+ })
+
+ return id, err
+}
+
+func (m *mapping) getMaxRowID(frameName string) (uint64, error) {
+ var id uint64
+ err := m.query(func() error {
+ return m.transaction(true, func(tx *bolt.Tx) error {
+ b := tx.Bucket([]byte(frameName))
+ if b == nil {
+ return nil
+ }
+
+ id = b.Sequence()
+ return nil
+ })
+ })
+
+ return id, err
+}
+
+func (m *mapping) putLocation(indexName string, colID uint64, location []byte) error {
+ return m.query(func() error {
+ return m.transaction(true, func(tx *bolt.Tx) error {
+ b, err := tx.CreateBucketIfNotExists([]byte(indexName))
+ if err != nil {
+ return err
+ }
+
+ key := make([]byte, 8)
+ binary.LittleEndian.PutUint64(key, colID)
+
+ return b.Put(key, location)
+ })
+ })
+}
+
+func (m *mapping) sortedLocations(indexName string, cols []uint64, reverse bool) ([][]byte, error) {
+ var result [][]byte
+ err := m.query(func() error {
+ return m.db.View(func(tx *bolt.Tx) error {
+ b := tx.Bucket([]byte(indexName))
+ if b == nil {
+ return fmt.Errorf("bucket %s not found", indexName)
+ }
+
+ for _, col := range cols {
+ key := make([]byte, 8)
+ binary.LittleEndian.PutUint64(key, col)
+ val := b.Get(key)
+
+ // val will point to mmap addresses, so we need to copy the slice
+ dst := make([]byte, len(val))
+ copy(dst, val)
+ result = append(result, dst)
+ }
+
+ return nil
+ })
+ })
+
+ if err != nil {
+ return nil, err
+ }
+
+ if reverse {
+ sort.Stable(sort.Reverse(byBytes(result)))
+ } else {
+ sort.Stable(byBytes(result))
+ }
+
+ return result, nil
+}
+
+type byBytes [][]byte
+
+func (b byBytes) Len() int { return len(b) }
+func (b byBytes) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
+func (b byBytes) Less(i, j int) bool { return bytes.Compare(b[i], b[j]) < 0 }
+
+func (m *mapping) getLocation(indexName string, colID uint64) ([]byte, error) {
+ var location []byte
+
+ err := m.query(func() error {
+ err := m.transaction(true, func(tx *bolt.Tx) error {
+ b := tx.Bucket([]byte(indexName))
+ if b == nil {
+ return fmt.Errorf("bucket %s not found", indexName)
+ }
+
+ key := make([]byte, 8)
+ binary.LittleEndian.PutUint64(key, colID)
+
+ location = b.Get(key)
+ return nil
+ })
+
+ return err
+ })
+
+ return location, err
+}
+
+func (m *mapping) getLocationFromBucket(
+ bucket *bolt.Bucket,
+ colID uint64,
+) ([]byte, error) {
+ var location []byte
+
+ err := m.query(func() error {
+ key := make([]byte, 8)
+ binary.LittleEndian.PutUint64(key, colID)
+
+ location = bucket.Get(key)
+ return nil
+ })
+
+ return location, err
+}
+
+func (m *mapping) getBucket(
+ indexName string,
+ writable bool,
+) (*bolt.Bucket, error) {
+ var bucket *bolt.Bucket
+
+ err := m.query(func() error {
+ tx, err := m.db.Begin(writable)
+ if err != nil {
+ return err
+ }
+
+ bucket = tx.Bucket([]byte(indexName))
+ if bucket == nil {
+ tx.Rollback()
+ return fmt.Errorf("bucket %s not found", indexName)
+ }
+
+ return nil
+ })
+
+ return bucket, err
+}
+
+func (m *mapping) get(name string, key interface{}) ([]byte, error) {
+ var value []byte
+
+ err := m.query(func() error {
+ var buf bytes.Buffer
+ enc := gob.NewEncoder(&buf)
+ err := enc.Encode(key)
+ if err != nil {
+ return err
+ }
+
+ err = m.transaction(true, func(tx *bolt.Tx) error {
+ b := tx.Bucket([]byte(name))
+ if b != nil {
+ value = b.Get(buf.Bytes())
+ return nil
+ }
+
+ return fmt.Errorf("%s not found", name)
+ })
+
+ return err
+ })
+ return value, err
+}
+
+func (m *mapping) filter(name string, fn func([]byte) (bool, error)) ([]uint64, error) {
+ var result []uint64
+
+ err := m.query(func() error {
+ return m.db.View(func(tx *bolt.Tx) error {
+ b := tx.Bucket([]byte(name))
+ if b == nil {
+ return nil
+ }
+
+ return b.ForEach(func(k, v []byte) error {
+ ok, err := fn(k)
+ if err != nil {
+ return err
+ }
+
+ if ok {
+ result = append(result, binary.LittleEndian.Uint64(v))
+ }
+
+ return nil
+ })
+ })
+ })
+
+ return result, err
+}
diff --git a/sql/index/pilosalib/mapping_test.go b/sql/index/pilosalib/mapping_test.go
new file mode 100644
index 000000000..98bdb7d6f
--- /dev/null
+++ b/sql/index/pilosalib/mapping_test.go
@@ -0,0 +1,83 @@
+package pilosalib
+
+import (
+ "encoding/binary"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestRowID(t *testing.T) {
+ require := require.New(t)
+ setup(t)
+ defer cleanup(t)
+ m := newMapping(filepath.Join(tmpDir, "id.map"))
+ m.open()
+ defer m.close()
+
+ cases := []int{0, 1, 2, 3, 4, 5, 5, 0, 3, 2, 1, 5}
+ expected := []uint64{1, 2, 3, 4, 5, 6, 6, 1, 4, 3, 2, 6}
+
+ for i, c := range cases {
+ rowID, err := m.getRowID("frame name", c)
+ require.NoError(err)
+ require.Equal(expected[i], rowID)
+ }
+
+ maxRowID, err := m.getMaxRowID("frame name")
+ require.NoError(err)
+ require.Equal(uint64(6), maxRowID)
+}
+
+func TestLocation(t *testing.T) {
+ require := require.New(t)
+ setup(t)
+ defer cleanup(t)
+
+ m := newMapping(filepath.Join(tmpDir, "id.map"))
+ m.open()
+ defer m.close()
+
+ cases := map[uint64]string{
+ 0: "zero",
+ 1: "one",
+ 2: "two",
+ 3: "three",
+ 4: "four",
+ }
+
+ for colID, loc := range cases {
+ err := m.putLocation("index name", colID, []byte(loc))
+ require.NoError(err)
+ }
+
+ for colID, loc := range cases {
+ b, err := m.getLocation("index name", colID)
+ require.NoError(err)
+ require.Equal(loc, string(b))
+ }
+}
+
+func TestGet(t *testing.T) {
+ require := require.New(t)
+ setup(t)
+ defer cleanup(t)
+
+ m := newMapping(filepath.Join(tmpDir, "id.map"))
+ m.open()
+ defer m.close()
+
+ cases := []int{0, 1, 2, 3, 4, 5, 5, 0, 3, 2, 1, 5}
+ expected := []uint64{1, 2, 3, 4, 5, 6, 6, 1, 4, 3, 2, 6}
+
+ for i, c := range cases {
+ m.getRowID("frame name", c)
+
+ id, err := m.get("frame name", c)
+ val := binary.LittleEndian.Uint64(id)
+
+ require.NoError(err)
+ require.Equal(expected[i], val)
+ }
+}
pFad - Phonifier reborn
Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.
Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.
Alternative Proxies:
Alternative Proxy
pFad Proxy
pFad v3 Proxy
pFad v4 Proxy