From a94937c708edae1f1857f1debb0ba33a445aa539 Mon Sep 17 00:00:00 2001 From: kuba-- Date: Tue, 31 Jul 2018 16:08:30 +0200 Subject: [PATCH 1/5] Pilosa index driver as library Signed-off-by: kuba-- --- .travis.yml | 6 +- server/context.go | 6 +- sql/index/config.go | 43 +- sql/index/config_test.go | 38 +- sql/index/pilosa/driver.go | 119 ++-- sql/index/pilosa/driver_test.go | 26 +- sql/index/pilosa/index.go | 6 +- sql/index/pilosa/mapping.go | 13 +- sql/index/pilosa/mapping_test.go | 19 +- sql/index/pilosalib/driver.go | 458 +++++++++++++ sql/index/pilosalib/driver_test.go | 995 ++++++++++++++++++++++++++++ sql/index/pilosalib/index.go | 296 +++++++++ sql/index/pilosalib/iterator.go | 83 +++ sql/index/pilosalib/lookup.go | 674 +++++++++++++++++++ sql/index/pilosalib/lookup_test.go | 203 ++++++ sql/index/pilosalib/mapping.go | 403 +++++++++++ sql/index/pilosalib/mapping_test.go | 91 +++ 17 files changed, 3349 insertions(+), 130 deletions(-) create mode 100644 sql/index/pilosalib/driver.go create mode 100644 sql/index/pilosalib/driver_test.go create mode 100644 sql/index/pilosalib/index.go create mode 100644 sql/index/pilosalib/iterator.go create mode 100644 sql/index/pilosalib/lookup.go create mode 100644 sql/index/pilosalib/lookup_test.go create mode 100644 sql/index/pilosalib/mapping.go create mode 100644 sql/index/pilosalib/mapping_test.go diff --git a/.travis.yml b/.travis.yml index 958072620..a7bb79099 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,8 +23,10 @@ before_install: - docker ps -a install: - - go get -u github.com/pilosa/go-pilosa - - cd "$GOPATH/src/github.com/pilosa/go-pilosa" && git checkout v0.9.0 && cd "$TRAVIS_BUILD_DIR" + - go get -u github.com/golang/dep/cmd/dep + - cd "$TRAVIS_BUILD_DIR" + - touch Gopkg.toml + - dep ensure -v -add "github.com/pilosa/go-pilosa@0.9.0" "github.com/pilosa/pilosa@master" - make dependencies script: diff --git a/server/context.go b/server/context.go index 2f982d7aa..bb472b210 100644 --- a/server/context.go +++ b/server/context.go @@ -60,11 +60,7 @@ func (s *SessionManager) NewContext(conn *mysql.Conn) (*sql.Context, DoneFunc, e sess := s.sessions[conn.ConnectionID] s.mu.Unlock() context := sql.NewContext(ctx, sql.WithSession(sess), sql.WithTracer(s.tracer)) - id, err := uuid.NewV4() - if err != nil { - cancel() - return nil, nil, err - } + id := uuid.NewV4() s.mu.Lock() s.sessionContexts[conn.ConnectionID] = append(s.sessionContexts[conn.ConnectionID], id) diff --git a/sql/index/config.go b/sql/index/config.go index 5b0bb3972..fb794e4e7 100644 --- a/sql/index/config.go +++ b/sql/index/config.go @@ -4,19 +4,11 @@ import ( "io" "io/ioutil" "os" - "path/filepath" "gopkg.in/src-d/go-mysql-server.v0/sql" yaml "gopkg.in/yaml.v2" ) -const ( - // ConfigFileName is the name of an index config file. - ConfigFileName = "config.yml" - // ProcessingFileName is the name of the processing index file. - ProcessingFileName = ".processing" -) - // Config represents index configuration type Config struct { DB string @@ -77,10 +69,9 @@ func WriteConfig(w io.Writer, cfg *Config) error { return err } -// WriteConfigFile writes the configuration to dir/config.yml file. -func WriteConfigFile(dir string, cfg *Config) error { - path := filepath.Join(dir, ConfigFileName) - f, err := os.Create(path) +// WriteConfigFile writes the configuration to file. +func WriteConfigFile(file string, cfg *Config) error { + f, err := os.Create(file) if err != nil { return err } @@ -101,10 +92,9 @@ func ReadConfig(r io.Reader) (*Config, error) { return &cfg, err } -// ReadConfigFile reads an configuration from dir/config.yml file. -func ReadConfigFile(dir string) (*Config, error) { - path := filepath.Join(dir, ConfigFileName) - f, err := os.Open(path) +// ReadConfigFile reads an configuration from file. +func ReadConfigFile(file string) (*Config, error) { + f, err := os.Open(file) if err != nil { return nil, err } @@ -113,10 +103,9 @@ func ReadConfigFile(dir string) (*Config, error) { return ReadConfig(f) } -// CreateProcessingFile creates a file inside the directory saying whether -// the index is being created. -func CreateProcessingFile(dir string) error { - f, err := os.Create(filepath.Join(dir, ProcessingFileName)) +// CreateProcessingFile creates a file saying whether the index is being created. +func CreateProcessingFile(file string) error { + f, err := os.Create(file) if err != nil { return err } @@ -126,16 +115,14 @@ func CreateProcessingFile(dir string) error { return nil } -// RemoveProcessingFile removes the file that says whether the index is still -// being created. -func RemoveProcessingFile(dir string) error { - return os.Remove(filepath.Join(dir, ProcessingFileName)) +// RemoveProcessingFile removes the file that says whether the index is still being created. +func RemoveProcessingFile(file string) error { + return os.Remove(file) } -// ExistsProcessingFile returns whether the processing file exists inside an -// index directory. -func ExistsProcessingFile(dir string) (bool, error) { - _, err := os.Stat(filepath.Join(dir, ProcessingFileName)) +// ExistsProcessingFile returns whether the processing file exists. +func ExistsProcessingFile(file string) (bool, error) { + _, err := os.Stat(file) if err != nil { if os.IsNotExist(err) { return false, nil diff --git a/sql/index/config_test.go b/sql/index/config_test.go index 309382427..6a97cf150 100644 --- a/sql/index/config_test.go +++ b/sql/index/config_test.go @@ -2,7 +2,6 @@ package index import ( "crypto/sha1" - "io/ioutil" "os" "path/filepath" "testing" @@ -14,12 +13,15 @@ import ( func TestConfig(t *testing.T) { require := require.New(t) + driver := "driver" db, table, id := "db_name", "table_name", "index_id" - path := filepath.Join(os.TempDir(), db, table, id) - err := os.MkdirAll(path, 0750) - + dir := filepath.Join(os.TempDir(), driver) + subdir := filepath.Join(dir, db, table) + err := os.MkdirAll(subdir, 0750) require.NoError(err) - defer os.RemoveAll(path) + file := filepath.Join(subdir, id+".cfg") + + defer os.RemoveAll(dir) h1 := sha1.Sum([]byte("h1")) h2 := sha1.Sum([]byte("h2")) @@ -31,43 +33,41 @@ func TestConfig(t *testing.T) { table, id, []sql.ExpressionHash{exh1, exh2}, - "DriverID", + "pilosa", map[string]string{ "port": "10101", "host": "localhost", }, ) - err = WriteConfigFile(path, cfg1) + err = WriteConfigFile(file, cfg1) require.NoError(err) - cfg2, err := ReadConfigFile(path) + cfg2, err := ReadConfigFile(file) require.NoError(err) require.Equal(cfg1, cfg2) } -func TestProcessingFile(t *testing.T) { +func TestLockFile(t *testing.T) { require := require.New(t) - dir, err := ioutil.TempDir(os.TempDir(), "processing-file") - require.NoError(err) - defer func() { - require.NoError(os.RemoveAll(dir)) - }() + dir := os.TempDir() + file := filepath.Join(dir, ".processing") + defer require.NoError(os.RemoveAll(file)) - ok, err := ExistsProcessingFile(dir) + ok, err := ExistsProcessingFile(file) require.NoError(err) require.False(ok) - require.NoError(CreateProcessingFile(dir)) + require.NoError(CreateProcessingFile(file)) - ok, err = ExistsProcessingFile(dir) + ok, err = ExistsProcessingFile(file) require.NoError(err) require.True(ok) - require.NoError(RemoveProcessingFile(dir)) + require.NoError(RemoveProcessingFile(file)) - ok, err = ExistsProcessingFile(dir) + ok, err = ExistsProcessingFile(file) require.NoError(err) require.False(ok) } diff --git a/sql/index/pilosa/driver.go b/sql/index/pilosa/driver.go index edaaec991..99eb659b2 100644 --- a/sql/index/pilosa/driver.go +++ b/sql/index/pilosa/driver.go @@ -2,6 +2,7 @@ package pilosa import ( "crypto/sha1" + "encoding/hex" "fmt" "io" "os" @@ -24,6 +25,15 @@ const ( IndexNamePrefix = "idx" // FrameNamePrefix the pilosa's frames prefix FrameNamePrefix = "frm" + + // ConfigFileExt is the extension of an index config file. + ConfigFileExt = ".cfg" + + // ProcessingFileExt is the extension of the lock/processing index file. + ProcessingFileExt = ".processing" + + // MappingFileExt is the extension of the mapping file. + MappingFileExt = ".map" ) var ( @@ -48,7 +58,7 @@ type Driver struct { // which satisfies sql.IndexDriver interface func NewDriver(root string, client *pilosa.Client) *Driver { return &Driver{ - root: root, + root: filepath.Join(root, DriverID), client: client, } } @@ -65,61 +75,64 @@ func (*Driver) ID() string { // Create a new index. func (d *Driver) Create(db, table, id string, expr []sql.ExpressionHash, config map[string]string) (sql.Index, error) { - path, err := mkdir(d.root, db, table, id) + _, err := mkdir(d.root, db, table) if err != nil { return nil, err } + if config == nil { + config = make(map[string]string) + } + + config["index"] = indexName(db, table) + for _, e := range expr { + config[hex.EncodeToString(e)] = frameName(id, e) + } cfg := index.NewConfig(db, table, id, expr, d.ID(), config) - err = index.WriteConfigFile(path, cfg) + err = index.WriteConfigFile(d.configFileName(db, table, id), cfg) if err != nil { return nil, err } - return newPilosaIndex(path, d.client, cfg), nil + return newPilosaIndex(d.mappingFileName(db, table, id), d.client, cfg), nil } // LoadAll loads all indexes for given db and table func (d *Driver) LoadAll(db, table string) ([]sql.Index, error) { - root := filepath.Join(d.root, db, table) + path := filepath.Join(d.root, db, table) var ( indexes []sql.Index errors []string - err error ) - filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if err != nil { - if path != root || !os.IsNotExist(err) { - errors = append(errors, err.Error()) - } - return filepath.SkipDir - } - if info.IsDir() && path != root && info.Name() != "." && info.Name() != ".." { - idx, err := d.loadIndex(path) - if err != nil { - if !errCorruptedIndex.Is(err) { - errors = append(errors, err.Error()) - } + configfiles, err := filepath.Glob(filepath.Join(path, "*") + ConfigFileExt) + if err != nil { + return nil, err + } - return filepath.SkipDir + for _, config := range configfiles { + idx, err := d.loadIndex(config) + if err != nil { + if !errCorruptedIndex.Is(err) { + errors = append(errors, err.Error()) } - indexes = append(indexes, idx) + continue } - return nil - }) - + indexes = append(indexes, idx) + } if len(errors) > 0 { - err = fmt.Errorf(strings.Join(errors, "\n")) + return nil, fmt.Errorf(strings.Join(errors, "\n")) } - return indexes, err + return indexes, nil } -func (d *Driver) loadIndex(path string) (sql.Index, error) { - ok, err := index.ExistsProcessingFile(path) +func (d *Driver) loadIndex(configfile string) (sql.Index, error) { + mapping := strings.Replace(configfile, ConfigFileExt, MappingFileExt, 1) + processing := strings.Replace(configfile, ConfigFileExt, ProcessingFileExt, 1) + ok, err := index.ExistsProcessingFile(processing) if err != nil { return nil, err } @@ -127,23 +140,30 @@ func (d *Driver) loadIndex(path string) (sql.Index, error) { if ok { log := logrus.WithFields(logrus.Fields{ "err": err, - "path": path, + "path": configfile, }) log.Warn("could not read index file, index is corrupt and will be deleted") - if err := os.RemoveAll(path); err != nil { - log.Warn("unable to remove folder of corrupted index") + if err := os.RemoveAll(configfile); err != nil { + log.Warn("unable to remove corrupted index: " + configfile) } - return nil, errCorruptedIndex.New(path) + if err := os.RemoveAll(processing); err != nil { + log.Warn("unable to remove corrupted index: " + processing) + } + + if err := os.RemoveAll(mapping); err != nil { + log.Warn("unable to remove corrupted index: " + mapping) + } + return nil, errCorruptedIndex.New(configfile) } - cfg, err := index.ReadConfigFile(path) + cfg, err := index.ReadConfigFile(configfile) if err != nil { return nil, err } - idx := newPilosaIndex(path, d.client, cfg) + idx := newPilosaIndex(mapping, d.client, cfg) return idx, nil } @@ -161,12 +181,8 @@ func (d *Driver) Save( return errInvalidIndexType.New(i) } - path, err := mkdir(d.root, i.Database(), i.Table(), i.ID()) - if err != nil { - return err - } - - if err = index.CreateProcessingFile(path); err != nil { + processingFile := d.processingFileName(idx.Database(), idx.Table(), idx.ID()) + if err = index.CreateProcessingFile(processingFile); err != nil { return err } @@ -280,13 +296,18 @@ func (d *Driver) Save( "id": i.ID(), }).Debugf("finished pilosa indexing") - return index.RemoveProcessingFile(path) + return index.RemoveProcessingFile(processingFile) } // Delete the index with the given path. func (d *Driver) Delete(idx sql.Index) error { - path := filepath.Join(d.root, idx.Database(), idx.Table(), idx.ID()) - if err := os.RemoveAll(path); err != nil { + if err := os.RemoveAll(d.configFileName(idx.Database(), idx.Table(), idx.ID())); err != nil { + return err + } + if err := os.RemoveAll(d.mappingFileName(idx.Database(), idx.Table(), idx.ID())); err != nil { + return err + } + if err := os.RemoveAll(d.processingFileName(idx.Database(), idx.Table(), idx.ID())); err != nil { return err } @@ -421,3 +442,15 @@ func mkdir(elem ...string) (string, error) { path := filepath.Join(elem...) return path, os.MkdirAll(path, 0750) } + +func (d *Driver) configFileName(db, table, id string) string { + return filepath.Join(d.root, db, table, id) + ConfigFileExt +} + +func (d *Driver) processingFileName(db, table, id string) string { + return filepath.Join(d.root, db, table, id) + ProcessingFileExt +} + +func (d *Driver) mappingFileName(db, table, id string) string { + return filepath.Join(d.root, db, table, id) + MappingFileExt +} diff --git a/sql/index/pilosa/driver_test.go b/sql/index/pilosa/driver_test.go index c39e8c10d..cd6255854 100644 --- a/sql/index/pilosa/driver_test.go +++ b/sql/index/pilosa/driver_test.go @@ -44,7 +44,7 @@ func TestID(t *testing.T) { func TestLoadAll(t *testing.T) { require := require.New(t) - path, err := ioutil.TempDir(os.TempDir(), "indexes") + path, err := ioutil.TempDir(os.TempDir(), "indexes-") require.NoError(err) defer os.RemoveAll(path) @@ -94,7 +94,7 @@ func TestSaveAndLoad(t *testing.T) { db, table, id := "db_name", "table_name", "index_id" expressions := makeExpressions("lang", "hash") - path, err := ioutil.TempDir(os.TempDir(), "indexes") + path, err := ioutil.TempDir("", "indexes") require.NoError(err) defer os.RemoveAll(path) @@ -188,7 +188,7 @@ func TestSaveAndGetAll(t *testing.T) { db, table, id := "db_name", "table_name", "index_id" expressions := makeExpressions("lang", "hash") - path, err := ioutil.TempDir(os.TempDir(), "indexes") + path, err := ioutil.TempDir("", "indexes") require.NoError(err) defer os.RemoveAll(path) @@ -218,17 +218,21 @@ func TestSaveAndGetAll(t *testing.T) { func TestLoadCorruptedIndex(t *testing.T) { require := require.New(t) - path, err := ioutil.TempDir(os.TempDir(), "indexes") + path, err := ioutil.TempDir("", "indexes") require.NoError(err) defer os.RemoveAll(path) - require.NoError(index.CreateProcessingFile(path)) + d := NewIndexDriver(path).(*Driver) + _, err = d.Create("db", "table", "id", nil, nil) + require.NoError(err) - _, err = new(Driver).loadIndex(path) + require.NoError(index.CreateProcessingFile(d.processingFileName("db", "table", "id"))) + + _, err = d.loadIndex(d.configFileName("db", "table", "id")) require.Error(err) require.True(errCorruptedIndex.Is(err)) - _, err = os.Stat(path) + _, err = os.Stat(d.processingFileName("db", "table", "id")) require.Error(err) require.True(os.IsNotExist(err)) } @@ -240,9 +244,9 @@ func TestDelete(t *testing.T) { require := require.New(t) db, table, id := "db_name", "table_name", "index_id" - path, err := ioutil.TempDir(os.TempDir(), "indexes") + root, err := ioutil.TempDir("", "indexes") require.NoError(err) - defer os.RemoveAll(path) + defer os.RemoveAll(root) h1 := sha1.Sum([]byte("lang")) exh1 := sql.ExpressionHash(h1[:]) @@ -252,7 +256,7 @@ func TestDelete(t *testing.T) { expressions := []sql.ExpressionHash{exh1, exh2} - d := NewIndexDriver(path) + d := NewIndexDriver(root) sqlIdx, err := d.Create(db, table, id, expressions, nil) require.NoError(err) @@ -262,7 +266,7 @@ func TestDelete(t *testing.T) { func TestLoadAllDirectoryDoesNotExist(t *testing.T) { require := require.New(t) - tmpDir, err := ioutil.TempDir(os.TempDir(), "pilosa-") + tmpDir, err := ioutil.TempDir("", "pilosa-") require.NoError(err) defer func() { diff --git a/sql/index/pilosa/index.go b/sql/index/pilosa/index.go index 1b41b9e7b..9890f9d80 100644 --- a/sql/index/pilosa/index.go +++ b/sql/index/pilosa/index.go @@ -10,7 +10,6 @@ import ( // pilosaIndex is an pilosa implementation of sql.Index interface type pilosaIndex struct { - path string client *pilosa.Client mapping *mapping @@ -20,15 +19,14 @@ type pilosaIndex struct { expressions []sql.ExpressionHash } -func newPilosaIndex(path string, client *pilosa.Client, cfg *index.Config) *pilosaIndex { +func newPilosaIndex(mappingfile string, client *pilosa.Client, cfg *index.Config) *pilosaIndex { return &pilosaIndex{ - path: path, client: client, db: cfg.DB, table: cfg.Table, id: cfg.ID, expressions: cfg.ExpressionHashes(), - mapping: newMapping(path), + mapping: newMapping(mappingfile), } } diff --git a/sql/index/pilosa/mapping.go b/sql/index/pilosa/mapping.go index 42e9cff13..a62b6a916 100644 --- a/sql/index/pilosa/mapping.go +++ b/sql/index/pilosa/mapping.go @@ -6,23 +6,18 @@ import ( "encoding/gob" "fmt" "io" - "path/filepath" "sort" "sync" "github.com/boltdb/bolt" ) -const ( - mappingFileName = DriverID + "-mapping.db" -) - // mapping // buckets: // - index name: columndID uint64 -> location []byte // - frame name: value []byte (gob encoding) -> rowID uint64 type mapping struct { - dir string + path string mut sync.RWMutex db *bolt.DB @@ -36,8 +31,8 @@ type mapping struct { clients int } -func newMapping(dir string) *mapping { - return &mapping{dir: dir} +func newMapping(path string) *mapping { + return &mapping{path: path} } func (m *mapping) open() { @@ -80,7 +75,7 @@ func (m *mapping) query(fn func() error) error { m.mut.Lock() if m.db == nil { var err error - m.db, err = bolt.Open(filepath.Join(m.dir, mappingFileName), 0640, nil) + m.db, err = bolt.Open(m.path, 0640, nil) if err != nil { m.mut.Unlock() return err diff --git a/sql/index/pilosa/mapping_test.go b/sql/index/pilosa/mapping_test.go index 31f205623..f2c297052 100644 --- a/sql/index/pilosa/mapping_test.go +++ b/sql/index/pilosa/mapping_test.go @@ -3,6 +3,7 @@ package pilosa import ( "encoding/binary" "os" + "path/filepath" "testing" "github.com/stretchr/testify/require" @@ -11,11 +12,11 @@ import ( func TestRowID(t *testing.T) { require := require.New(t) - path, err := mkdir(os.TempDir(), "mapping_test") + root, err := mkdir(os.TempDir(), "mapping_test") require.NoError(err) - defer os.RemoveAll(path) + defer os.RemoveAll(root) - m := newMapping(path) + m := newMapping(filepath.Join(root, "id.map")) m.open() defer m.close() @@ -36,11 +37,11 @@ func TestRowID(t *testing.T) { func TestLocation(t *testing.T) { require := require.New(t) - path, err := mkdir(os.TempDir(), "mapping_test") + root, err := mkdir(os.TempDir(), "mapping_test") require.NoError(err) - defer os.RemoveAll(path) + defer os.RemoveAll(root) - m := newMapping(path) + m := newMapping(filepath.Join(root, "id.map")) m.open() defer m.close() @@ -67,11 +68,11 @@ func TestLocation(t *testing.T) { func TestGet(t *testing.T) { require := require.New(t) - path, err := mkdir(os.TempDir(), "mapping_test") + root, err := mkdir(os.TempDir(), "mapping_test") require.NoError(err) - defer os.RemoveAll(path) + defer os.RemoveAll(root) - m := newMapping(path) + m := newMapping(filepath.Join(root, "id.map")) m.open() defer m.close() diff --git a/sql/index/pilosalib/driver.go b/sql/index/pilosalib/driver.go new file mode 100644 index 000000000..1d7fc8c25 --- /dev/null +++ b/sql/index/pilosalib/driver.go @@ -0,0 +1,458 @@ +package pilosalib + +import ( + "crypto/sha1" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + opentracing "github.com/opentracing/opentracing-go" + "github.com/opentracing/opentracing-go/log" + pilosa "github.com/pilosa/pilosa" + "github.com/sirupsen/logrus" + errors "gopkg.in/src-d/go-errors.v1" + "gopkg.in/src-d/go-mysql-server.v0/sql" + "gopkg.in/src-d/go-mysql-server.v0/sql/index" +) + +const ( + // DriverID the unique name of the pilosa driver. + DriverID = "pilosalib" + + // IndexNamePrefix the pilosa's indexes prefix + IndexNamePrefix = "idx" + + // FieldNamePrefix the pilosa's field prefix + FieldNamePrefix = "fld" + + // ConfigFileExt is the extension of an index config file. + ConfigFileExt = ".cfg" + + // ProcessingFileExt is the extension of the lock/processing index file. + ProcessingFileExt = ".processing" + + // MappingFileExt is the extension of the mapping file. + MappingFileExt = ".map" +) + +var ( + errCorruptedIndex = errors.NewKind("the index db: %s, table: %s, id: %s is corrupted") + errLoadingIndex = errors.NewKind("cannot load pilosa index: %s") + errInvalidIndexType = errors.NewKind("expecting a pilosa index, instead got %T") + errDeletePilosaField = errors.NewKind("error deleting pilosa field %s: %s") +) + +type ( + bitBatch struct { + size uint64 + rows []uint64 + cols []uint64 + pos uint64 + } + + // Driver implements sql.IndexDriver interface. + Driver struct { + root string + holder *pilosa.Holder + + // used for saving + bitBatches []*bitBatch + fields []*pilosa.Field + timePilosa time.Duration + timeMapping time.Duration + } +) + +// NewDriver returns a new instance of pilosalib.Driver +// which satisfies sql.IndexDriver interface +func NewDriver(root string) *Driver { + return &Driver{ + root: filepath.Join(root, DriverID), + holder: pilosa.NewHolder(), + } +} + +// ID returns the unique name of the driver. +func (*Driver) ID() string { + return DriverID +} + +// Create a new index. +func (d *Driver) Create(db, table, id string, expr []sql.ExpressionHash, config map[string]string) (sql.Index, error) { + root, err := mkdir(d.root, db, table) + if err != nil { + return nil, err + } + name := indexName(db, table) + + if config == nil { + config = make(map[string]string) + } + + config["index"] = name + for _, e := range expr { + config[hex.EncodeToString(e)] = fieldName(id, e) + } + cfg := index.NewConfig(db, table, id, expr, d.ID(), config) + err = index.WriteConfigFile(d.configFileName(db, table, id), cfg) + if err != nil { + return nil, err + } + + d.holder.Path = root + idx, err := d.holder.CreateIndexIfNotExists(name, pilosa.IndexOptions{}) + if err != nil { + return nil, err + } + mapping := newMapping(d.mappingFileName(db, table, id)) + + return newPilosaIndex(idx, mapping, cfg), nil +} + +// LoadAll loads all indexes for given db and table +func (d *Driver) LoadAll(db, table string) ([]sql.Index, error) { + var ( + indexes []sql.Index + errors []string + ) + + d.holder.Path = filepath.Join(d.root, db, table) + err := d.holder.Open() + if err != nil { + return nil, err + } + defer d.holder.Close() + + configfiles, err := filepath.Glob(filepath.Join(d.holder.Path, "*") + ConfigFileExt) + if err != nil { + return nil, err + } + for _, path := range configfiles { + filename := filepath.Base(path) + id := filename[:len(filename)-len(ConfigFileExt)] + idx, err := d.loadIndex(db, table, id) + if err != nil { + if errLoadingIndex.Is(err) { + return nil, err + } + if !errCorruptedIndex.Is(err) { + errors = append(errors, err.Error()) + } + continue + } + indexes = append(indexes, idx) + } + + if len(errors) > 0 { + return nil, fmt.Errorf(strings.Join(errors, "\n")) + } + return indexes, nil +} + +func (d *Driver) loadIndex(db, table, id string) (*pilosaIndex, error) { + name := indexName(db, table) + idx := d.holder.Index(name) + if idx == nil { + return nil, errLoadingIndex.New(name) + } + + processing := d.processingFileName(db, table, id) + ok, err := index.ExistsProcessingFile(processing) + if err != nil { + return nil, err + } + if ok { + log := logrus.WithFields(logrus.Fields{ + "db": db, + "table": table, + "id": id, + }) + log.Warn("could not read index file, index is corrupt and will be deleted") + d.removeResources(db, table, id) + return nil, errCorruptedIndex.New(id) + } + + cfg, err := index.ReadConfigFile(d.configFileName(db, table, id)) + if err != nil { + return nil, err + } + + mapping := newMapping(d.mappingFileName(db, table, id)) + return newPilosaIndex(idx, mapping, cfg), nil +} + +// Save the given index (mapping and bitmap) +func (d *Driver) Save(ctx *sql.Context, i sql.Index, iter sql.IndexKeyValueIter) (err error) { + var colID uint64 + start := time.Now() + + idx, ok := i.(*pilosaIndex) + if !ok { + return errInvalidIndexType.New(i) + } + + processingFile := d.processingFileName(idx.Database(), idx.Table(), idx.ID()) + if err = index.CreateProcessingFile(processingFile); err != nil { + return err + } + + pilosaIndex := idx.index + if err = pilosaIndex.Open(); err != nil { + return err + } + defer pilosaIndex.Close() + + d.fields = make([]*pilosa.Field, len(idx.ExpressionHashes())) + for i, e := range idx.ExpressionHashes() { + name := fieldName(idx.ID(), e) + pilosaIndex.DeleteField(name) + field, err := pilosaIndex.CreateField(name, pilosa.OptFieldTypeDefault()) + if err != nil { + return err + } + d.fields[i] = field + } + + rollback := true + idx.mapping.openCreate(true) + defer func() { + if rollback { + idx.mapping.rollback() + } else { + e := d.saveMapping(ctx, idx.mapping, colID, false) + if e != nil && err == nil { + err = e + } + } + + idx.mapping.close() + }() + + d.bitBatches = make([]*bitBatch, len(d.fields)) + for i := range d.bitBatches { + d.bitBatches[i] = newBitBatch(sql.IndexBatchSize) + } + + for colID = uint64(0); err == nil; colID++ { + // commit each batch of objects (pilosa and boltdb) + if colID%sql.IndexBatchSize == 0 && colID != 0 { + d.saveBatch(ctx, idx.mapping, colID) + } + + select { + case <-ctx.Done(): + return ctx.Err() + + default: + var ( + values []interface{} + location []byte + ) + values, location, err = iter.Next() + if err != nil { + break + } + + for i, field := range d.fields { + if values[i] == nil { + continue + } + + rowID, err := idx.mapping.getRowID(field.Name(), values[i]) + if err != nil { + return err + } + + d.bitBatches[i].Add(rowID, colID) + } + err = idx.mapping.putLocation(pilosaIndex.Name(), colID, location) + } + } + + if err != nil && err != io.EOF { + return err + } + + rollback = false + + err = d.savePilosa(ctx, colID) + if err != nil { + return err + } + + logrus.WithFields(logrus.Fields{ + "duration": time.Since(start), + "pilosa": d.timePilosa, + "mapping": d.timeMapping, + "rows": colID, + "id": i.ID(), + }).Debugf("finished pilosa indexing") + + return index.RemoveProcessingFile(processingFile) +} + +// Delete the index with the given path. +func (d *Driver) Delete(i sql.Index) error { + idx, ok := i.(*pilosaIndex) + if !ok { + return errInvalidIndexType.New(i) + } + + d.removeResources(idx.Database(), idx.Table(), idx.ID()) + + err := idx.index.Open() + if err != nil { + return err + } + defer idx.index.Close() + + for _, ex := range idx.ExpressionHashes() { + name := fieldName(idx.ID(), ex) + field := idx.index.Field(name) + if field == nil { + continue + } + + if err = idx.index.DeleteField(name); err != nil { + return err + } + } + + return nil +} + +func (d *Driver) saveBatch(ctx *sql.Context, m *mapping, colID uint64) error { + err := d.savePilosa(ctx, colID) + if err != nil { + return err + } + + return d.saveMapping(ctx, m, colID, true) +} + +func (d *Driver) savePilosa(ctx *sql.Context, colID uint64) error { + span, _ := ctx.Span("pilosa.Save.bitBatch", + opentracing.Tag{Key: "cols", Value: colID}, + opentracing.Tag{Key: "frames", Value: len(d.fields)}, + ) + defer span.Finish() + + start := time.Now() + + for i, frm := range d.fields { + err := frm.Import(d.bitBatches[i].rows, d.bitBatches[i].cols, nil) + if err != nil { + span.LogKV("error", err) + return err + } + + d.bitBatches[i].Clean() + } + + d.timePilosa += time.Since(start) + + return nil +} + +func (d *Driver) saveMapping( + ctx *sql.Context, + m *mapping, + colID uint64, + cont bool, +) error { + span, _ := ctx.Span("pilosa.Save.mapping", + opentracing.Tag{Key: "cols", Value: colID}, + opentracing.Tag{Key: "continues", Value: cont}, + ) + defer span.Finish() + + start := time.Now() + + err := m.commit(cont) + if err != nil { + span.LogKV("error", err) + return err + } + + d.timeMapping += time.Since(start) + + return nil +} + +func newBitBatch(size uint64) *bitBatch { + b := &bitBatch{size: size} + b.Clean() + + return b +} + +func (b *bitBatch) Clean() { + b.rows = make([]uint64, 0, b.size) + b.rows = make([]uint64, 0, b.size) + b.pos = 0 +} + +func (b *bitBatch) Add(row, col uint64) { + b.rows = append(b.rows, row) + b.cols = append(b.cols, col) +} + +func (b *bitBatch) NextRecord() (uint64, uint64, error) { + if b.pos >= uint64(len(b.rows)) { + return 0, 0, io.EOF + } + + b.pos++ + return b.rows[b.pos-1], b.cols[b.pos-1], nil +} + +func indexName(db, table string) string { + h := sha1.New() + io.WriteString(h, db) + io.WriteString(h, table) + + return fmt.Sprintf("%s-%x", IndexNamePrefix, h.Sum(nil)) +} + +func fieldName(id string, ex sql.ExpressionHash) string { + h := sha1.New() + io.WriteString(h, id) + h.Write(ex) + return fmt.Sprintf("%s-%x", FieldNamePrefix, h.Sum(nil)) +} + +// mkdir makes an empty index directory (if doesn't exist) and returns a path. +func mkdir(elem ...string) (string, error) { + path := filepath.Join(elem...) + return path, os.MkdirAll(path, 0750) +} + +func (d *Driver) configFileName(db, table, id string) string { + return filepath.Join(d.root, db, table, id) + ConfigFileExt +} + +func (d *Driver) processingFileName(db, table, id string) string { + return filepath.Join(d.root, db, table, id) + ProcessingFileExt +} + +func (d *Driver) mappingFileName(db, table, id string) string { + return filepath.Join(d.root, db, table, id) + MappingFileExt +} + +func (d *Driver) removeResources(db, table, id string) { + if err := os.RemoveAll(d.configFileName(db, table, id)); err != nil { + log.Error(err) + } + + if err := os.RemoveAll(d.processingFileName(db, table, id)); err != nil { + log.Error(err) + } + + if err := os.RemoveAll(d.mappingFileName(db, table, id)); err != nil { + log.Error(err) + } +} diff --git a/sql/index/pilosalib/driver_test.go b/sql/index/pilosalib/driver_test.go new file mode 100644 index 000000000..dd64b6fc5 --- /dev/null +++ b/sql/index/pilosalib/driver_test.go @@ -0,0 +1,995 @@ +package pilosalib + +import ( + "context" + "crypto/rand" + "crypto/sha1" + "encoding/hex" + "fmt" + "io" + "io/ioutil" + "os" + "reflect" + "testing" + + "github.com/pilosa/pilosa" + "github.com/stretchr/testify/require" + "gopkg.in/src-d/go-mysql-server.v0/sql" + "gopkg.in/src-d/go-mysql-server.v0/sql/index" + "gopkg.in/src-d/go-mysql-server.v0/test" +) + +func TestID(t *testing.T) { + d := &Driver{} + + require := require.New(t) + require.Equal(DriverID, d.ID()) +} + +func TestLoadAll(t *testing.T) { + require := require.New(t) + + root, err := ioutil.TempDir("", "indexes") + require.NoError(err) + defer os.RemoveAll(root) + + d := NewDriver(root) + idx1, err := d.Create("db", "table", "id1", makeExpressions("hash1"), nil) + require.NoError(err) + + idx2, err := d.Create("db", "table", "id2", makeExpressions("hash1"), nil) + require.NoError(err) + + indexes, err := d.LoadAll("db", "table") + require.NoError(err) + + require.Equal(2, len(indexes)) + i1, ok := idx1.(*pilosaIndex) + require.True(ok) + i2, ok := idx2.(*pilosaIndex) + require.True(ok) + + require.Equal(i1.index.Name(), i2.index.Name()) +} + +type logLoc struct { + loc []byte + err error +} + +func TestSaveAndLoad(t *testing.T) { + require := require.New(t) + + db, table, id := "db_name", "table_name", "index_id" + expressions := makeExpressions("lang", "hash") + root, err := ioutil.TempDir("", "indexes") + require.NoError(err) + defer os.RemoveAll(root) + + d := NewDriver(root) + sqlIdx, err := d.Create(db, table, id, expressions, nil) + require.NoError(err) + + it := &testIndexKeyValueIter{ + offset: 0, + total: 64, + expressions: expressions, + location: randLocation, + } + + tracer := new(test.MemTracer) + ctx := sql.NewContext(context.Background(), sql.WithTracer(tracer)) + err = d.Save(ctx, sqlIdx, it) + require.NoError(err) + + indexes, err := d.LoadAll(db, table) + require.NoError(err) + require.Equal(1, len(indexes)) + + for _, r := range it.records { + lookup, err := sqlIdx.Get(r.values...) + require.NoError(err) + + found, foundLoc := false, []string{} + lit, err := lookup.Values() + require.NoError(err) + + var logs []logLoc + for i := 0; ; i++ { + loc, err := lit.Next() + + // make a copy of location to save in the log + loc2 := make([]byte, len(loc)) + copy(loc2, loc) + logs = append(logs, logLoc{loc2, err}) + + if err == io.EOF { + if i == 0 { + for j, l := range logs { + t.Logf("[%d] values: %v location: %x loc: %x err: %v\n", + j, r.values, r.location, l.loc, l.err) + } + + t.Errorf("No data for r.values: %v\tr.location: %x", + r.values, r.location) + t.FailNow() + } + + break + } + + require.NoError(err) + found = found || reflect.DeepEqual(r.location, loc) + foundLoc = append(foundLoc, hex.EncodeToString(loc)) + } + require.Truef(found, "Expected: %s\nGot: %v\n", hex.EncodeToString(r.location), foundLoc) + + err = lit.Close() + require.NoError(err) + } + + // test that not found values do not cause error + lookup, err := sqlIdx.Get("do not exist", "none") + require.NoError(err) + lit, err := lookup.Values() + require.NoError(err) + _, err = lit.Next() + require.Equal(io.EOF, err) + + found := false + for _, span := range tracer.Spans { + if span == "pilosa.Save.bitBatch" { + found = true + break + } + } + + require.True(found) +} + +func TestSaveAndGetAll(t *testing.T) { + require := require.New(t) + + db, table, id := "db_name", "table_name", "index_id" + expressions := makeExpressions("lang", "hash") + root, err := ioutil.TempDir("", "indexes") + require.NoError(err) + defer os.RemoveAll(root) + + d := NewDriver(root) + sqlIdx, err := d.Create(db, table, id, expressions, nil) + require.NoError(err) + + it := &testIndexKeyValueIter{ + offset: 0, + total: 64, + expressions: expressions, + location: randLocation, + } + + err = d.Save(sql.NewEmptyContext(), sqlIdx, it) + require.NoError(err) + + indexes, err := d.LoadAll(db, table) + require.NoError(err) + require.Equal(1, len(indexes)) + + _, err = sqlIdx.Get() + require.Error(err) + require.True(errInvalidKeys.Is(err)) +} + +func TestLoadCorruptedIndex(t *testing.T) { + require := require.New(t) + root, err := ioutil.TempDir("", "indexes") + require.NoError(err) + defer os.RemoveAll(root) + + d := NewDriver(root) + _, err = d.Create("db", "table", "id", nil, nil) + require.NoError(err) + + require.NoError(index.CreateProcessingFile(d.processingFileName("db", "table", "id"))) + + _, err = d.loadIndex("db", "table", "id") + require.Error(err) + require.True(errCorruptedIndex.Is(err)) + + _, err = os.Stat(d.processingFileName("db", "table", "id")) + require.Error(err) + require.True(os.IsNotExist(err)) +} + +func TestDelete(t *testing.T) { + require := require.New(t) + + db, table, id := "db_name", "table_name", "index_id" + root, err := ioutil.TempDir("", "indexes") + require.NoError(err) + defer os.RemoveAll(root) + + h1 := sha1.Sum([]byte("lang")) + exh1 := sql.ExpressionHash(h1[:]) + + h2 := sha1.Sum([]byte("hash")) + exh2 := sql.ExpressionHash(h2[:]) + + expressions := []sql.ExpressionHash{exh1, exh2} + + d := NewDriver(root) + sqlIdx, err := d.Create(db, table, id, expressions, nil) + require.NoError(err) + + err = d.Delete(sqlIdx) + require.NoError(err) +} + +func TestLoadAllDirectoryDoesNotExist(t *testing.T) { + require := require.New(t) + root, err := ioutil.TempDir("", "pilosa-") + require.NoError(err) + + defer func() { + require.NoError(os.RemoveAll(root)) + }() + + driver := NewDriver(root) + indexes, err := driver.LoadAll("foo", "bar") + require.NoError(err) + require.Len(indexes, 0) +} + +func TestAscendDescendIndex(t *testing.T) { + idx, cleanup := setupAscendDescend(t) + defer cleanup() + + must := func(lookup sql.IndexLookup, err error) sql.IndexLookup { + require.NoError(t, err) + return lookup + } + + testCases := []struct { + name string + lookup sql.IndexLookup + expected []string + }{ + { + "ascend range", + must(idx.AscendRange( + []interface{}{int64(1), int64(1)}, + []interface{}{int64(7), int64(10)}, + )), + []string{"1", "5", "6", "7", "8", "9"}, + }, + { + "ascend greater or equal", + must(idx.AscendGreaterOrEqual(int64(7), int64(6))), + []string{"2", "4"}, + }, + { + "ascend less than", + must(idx.AscendLessThan(int64(5), int64(3))), + []string{"1", "10"}, + }, + { + "descend range", + must(idx.DescendRange( + []interface{}{int64(6), int64(9)}, + []interface{}{int64(0), int64(0)}, + )), + []string{"9", "8", "7", "6", "5", "1"}, + }, + { + "descend less or equal", + must(idx.DescendLessOrEqual(int64(4), int64(2))), + []string{"10", "1"}, + }, + { + "descend greater", + must(idx.DescendGreater(int64(6), int64(5))), + []string{"4", "2"}, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + require := require.New(t) + iter, err := tt.lookup.Values() + require.NoError(err) + + var result []string + for { + k, err := iter.Next() + if err == io.EOF { + break + } + require.NoError(err) + + result = append(result, string(k)) + } + + require.Equal(tt.expected, result) + }) + } +} + +func TestIntersection(t *testing.T) { + ctx := sql.NewContext(context.Background()) + require := require.New(t) + + db, table := "db_name", "table_name" + idxLang, expLang := "idx_lang", makeExpressions("lang") + idxPath, expPath := "idx_path", makeExpressions("path") + + root, err := ioutil.TempDir("", "indexes-") + require.NoError(err) + defer os.RemoveAll(root) + + d := NewDriver(root) + sqlIdxLang, err := d.Create(db, table, idxLang, expLang, nil) + require.NoError(err) + + sqlIdxPath, err := d.Create(db, table, idxPath, expPath, nil) + require.NoError(err) + + itLang := &testIndexKeyValueIter{ + offset: 0, + total: 10, + expressions: expLang, + location: offsetLocation, + } + + itPath := &testIndexKeyValueIter{ + offset: 0, + total: 10, + expressions: expPath, + location: offsetLocation, + } + + err = d.Save(ctx, sqlIdxLang, itLang) + require.NoError(err) + + err = d.Save(ctx, sqlIdxPath, itPath) + require.NoError(err) + + lookupLang, err := sqlIdxLang.Get(itLang.records[0].values...) + require.NoError(err) + lookupPath, err := sqlIdxPath.Get(itPath.records[itPath.total-1].values...) + require.NoError(err) + + m, ok := lookupLang.(sql.Mergeable) + require.True(ok) + require.True(m.IsMergeable(lookupPath)) + + interLookup, ok := lookupLang.(sql.SetOperations) + require.True(ok) + interIt, err := interLookup.Intersection(lookupPath).Values() + require.NoError(err) + loc, err := interIt.Next() + fmt.Println(loc, err, err == io.EOF) + + require.True(err == io.EOF) + require.NoError(interIt.Close()) + + lookupLang, err = sqlIdxLang.Get(itLang.records[0].values...) + require.NoError(err) + lookupPath, err = sqlIdxPath.Get(itPath.records[0].values...) + require.NoError(err) + + interLookup, ok = lookupPath.(sql.SetOperations) + require.True(ok) + interIt, err = interLookup.Intersection(lookupLang).Values() + require.NoError(err) + loc, err = interIt.Next() + require.NoError(err) + require.Equal(loc, itPath.records[0].location) + _, err = interIt.Next() + require.True(err == io.EOF) + + require.NoError(interIt.Close()) +} + +func TestUnion(t *testing.T) { + require := require.New(t) + + db, table := "db_name", "table_name" + idxLang, expLang := "idx_lang", makeExpressions("lang") + idxPath, expPath := "idx_path", makeExpressions("path") + + root, err := ioutil.TempDir("", "indexes") + require.NoError(err) + defer os.RemoveAll(root) + + d := NewDriver(root) + sqlIdxLang, err := d.Create(db, table, idxLang, expLang, nil) + require.NoError(err) + + sqlIdxPath, err := d.Create(db, table, idxPath, expPath, nil) + require.NoError(err) + + itLang := &testIndexKeyValueIter{ + offset: 0, + total: 10, + expressions: expLang, + location: offsetLocation, + } + + itPath := &testIndexKeyValueIter{ + offset: 0, + total: 10, + expressions: expPath, + location: offsetLocation, + } + + ctx := sql.NewContext(context.Background()) + + err = d.Save(ctx, sqlIdxLang, itLang) + require.NoError(err) + + err = d.Save(ctx, sqlIdxPath, itPath) + require.NoError(err) + + lookupLang, err := sqlIdxLang.Get(itLang.records[0].values...) + require.NoError(err) + litLang, err := lookupLang.Values() + require.NoError(err) + + loc, err := litLang.Next() + require.NoError(err) + require.Equal(itLang.records[0].location, loc) + _, err = litLang.Next() + require.True(err == io.EOF) + err = litLang.Close() + require.NoError(err) + + lookupPath, err := sqlIdxPath.Get(itPath.records[itPath.total-1].values...) + require.NoError(err) + litPath, err := lookupPath.Values() + require.NoError(err) + + loc, err = litPath.Next() + require.NoError(err) + require.Equal(itPath.records[itPath.total-1].location, loc) + _, err = litPath.Next() + require.True(err == io.EOF) + err = litLang.Close() + require.NoError(err) + + m, ok := lookupLang.(sql.Mergeable) + require.True(ok) + require.True(m.IsMergeable(lookupPath)) + + unionLookup, ok := lookupLang.(sql.SetOperations) + unionIt, err := unionLookup.Union(lookupPath).Values() + require.NoError(err) + // 0 + loc, err = unionIt.Next() + require.Equal(itLang.records[0].location, loc) + + // total-1 + loc, err = unionIt.Next() + require.Equal(itPath.records[itPath.total-1].location, loc) + + _, err = unionIt.Next() + require.True(err == io.EOF) + + require.NoError(unionIt.Close()) +} + +func TestDifference(t *testing.T) { + require := require.New(t) + + db, table := "db_name", "table_name" + idxLang, expLang := "idx_lang", makeExpressions("lang") + idxPath, expPath := "idx_path", makeExpressions("path") + + root, err := ioutil.TempDir("", "indexes") + require.NoError(err) + defer os.RemoveAll(root) + + d := NewDriver(root) + sqlIdxLang, err := d.Create(db, table, idxLang, expLang, nil) + require.NoError(err) + + sqlIdxPath, err := d.Create(db, table, idxPath, expPath, nil) + require.NoError(err) + + itLang := &testIndexKeyValueIter{ + offset: 0, + total: 10, + expressions: expLang, + location: offsetLocation, + } + + itPath := &testIndexKeyValueIter{ + offset: 0, + total: 10, + expressions: expPath, + location: offsetLocation, + } + + ctx := sql.NewContext(context.Background()) + + err = d.Save(ctx, sqlIdxLang, itLang) + require.NoError(err) + + err = d.Save(ctx, sqlIdxPath, itPath) + require.NoError(err) + + lookupLang, err := sqlIdxLang.Get(itLang.records[0].values...) + require.NoError(err) + + lookupPath, err := sqlIdxPath.Get(itPath.records[itPath.total-1].values...) + require.NoError(err) + + m, ok := lookupLang.(sql.Mergeable) + require.True(ok) + require.True(m.IsMergeable(lookupPath)) + + unionOp, ok := lookupLang.(sql.SetOperations) + require.True(ok) + unionLookup, ok := unionOp.Union(lookupPath).(sql.SetOperations) + require.True(ok) + + diffLookup := unionLookup.Difference(lookupLang) + diffIt, err := diffLookup.Values() + require.NoError(err) + + // total-1 + loc, err := diffIt.Next() + require.NoError(err) + require.Equal(itPath.records[itPath.total-1].location, loc) + + _, err = diffIt.Next() + require.True(err == io.EOF) + + require.NoError(diffIt.Close()) +} + +func TestUnionDiffAsc(t *testing.T) { + require := require.New(t) + + db, table := "db_name", "table_name" + idx, exp := "idx_lang", makeExpressions("lang") + + root, err := ioutil.TempDir("", "indexes") + require.NoError(err) + defer os.RemoveAll(root) + + d := NewDriver(root) + sqlIdx, err := d.Create(db, table, idx, exp, nil) + require.NoError(err) + pilosaIdx, ok := sqlIdx.(*pilosaIndex) + require.True(ok) + it := &testIndexKeyValueIter{ + offset: 0, + total: 10, + expressions: exp, + location: offsetLocation, + } + + ctx := sql.NewContext(context.Background()) + + err = d.Save(ctx, pilosaIdx, it) + require.NoError(err) + + sqlLookup, err := pilosaIdx.AscendLessThan(it.records[it.total-1].values...) + require.NoError(err) + ascLookup, ok := sqlLookup.(*ascendLookup) + require.True(ok) + + ls := make([]*indexLookup, it.total) + for i, r := range it.records { + l, err := pilosaIdx.Get(r.values...) + require.NoError(err) + ls[i], _ = l.(*indexLookup) + } + + unionLookup := ls[0].Union(ls[2], ls[4], ls[6], ls[8]) + + diffLookup := ascLookup.Difference(unionLookup) + diffIt, err := diffLookup.Values() + require.NoError(err) + + for i := 1; i < it.total-1; i += 2 { + loc, err := diffIt.Next() + require.NoError(err) + + require.Equal(it.records[i].location, loc) + } + + _, err = diffIt.Next() + require.True(err == io.EOF) + require.NoError(diffIt.Close()) +} + +func TestInterRanges(t *testing.T) { + require := require.New(t) + + db, table := "db_name", "table_name" + idx, exp := "idx_lang", makeExpressions("lang") + + root, err := ioutil.TempDir("", "indexes") + require.NoError(err) + defer os.RemoveAll(root) + + d := NewDriver(root) + sqlIdx, err := d.Create(db, table, idx, exp, nil) + require.NoError(err) + pilosaIdx, ok := sqlIdx.(*pilosaIndex) + require.True(ok) + it := &testIndexKeyValueIter{ + offset: 0, + total: 10, + expressions: exp, + location: offsetLocation, + } + + ctx := sql.NewContext(context.Background()) + + err = d.Save(ctx, pilosaIdx, it) + require.NoError(err) + + ranges := [2]int{3, 9} + sqlLookup, err := pilosaIdx.AscendLessThan(it.records[ranges[1]].values...) + require.NoError(err) + lessLookup, ok := sqlLookup.(*ascendLookup) + require.True(ok) + + sqlLookup, err = pilosaIdx.AscendGreaterOrEqual(it.records[ranges[0]].values...) + require.NoError(err) + greaterLookup, ok := sqlLookup.(*ascendLookup) + require.True(ok) + + interLookup := lessLookup.Intersection(greaterLookup) + require.NotNil(interLookup) + interIt, err := interLookup.Values() + require.NoError(err) + + for i := ranges[0]; i < ranges[1]; i++ { + loc, err := interIt.Next() + require.NoError(err) + require.Equal(it.records[i].location, loc) + } + + _, err = interIt.Next() + require.True(err == io.EOF) + require.NoError(interIt.Close()) +} + +func TestNegateIndex(t *testing.T) { + require := require.New(t) + + db, table := "db_name", "table_name" + root, err := mkdir(os.TempDir(), "indexes") + require.NoError(err) + defer func() { + require.NoError(os.RemoveAll(root)) + }() + + d := NewDriver(root) + idx, err := d.Create(db, table, "index_id", makeExpressions("a"), nil) + require.NoError(err) + + multiIdx, err := d.Create( + db, table, "multi_index_id", + makeExpressions("a", "b"), + nil, + ) + require.NoError(err) + + it := &fixtureKeyValueIter{ + fixtures: []kvfixture{ + {"1", []interface{}{int64(2)}}, + {"2", []interface{}{int64(7)}}, + {"3", []interface{}{int64(1)}}, + {"4", []interface{}{int64(1)}}, + {"5", []interface{}{int64(7)}}, + }, + } + + err = d.Save(sql.NewEmptyContext(), idx, it) + require.NoError(err) + + multiIt := &fixtureKeyValueIter{ + fixtures: []kvfixture{ + {"1", []interface{}{int64(2), int64(6)}}, + {"2", []interface{}{int64(7), int64(5)}}, + {"3", []interface{}{int64(1), int64(2)}}, + {"4", []interface{}{int64(1), int64(3)}}, + {"5", []interface{}{int64(7), int64(6)}}, + {"6", []interface{}{int64(10), int64(6)}}, + {"7", []interface{}{int64(5), int64(1)}}, + {"8", []interface{}{int64(6), int64(2)}}, + {"9", []interface{}{int64(4), int64(0)}}, + {"10", []interface{}{int64(3), int64(5)}}, + }, + } + + err = d.Save(sql.NewEmptyContext(), multiIdx, multiIt) + require.NoError(err) + + lookup, err := idx.(sql.NegateIndex).Not(int64(1)) + require.NoError(err) + + values, err := lookupValues(lookup) + require.NoError(err) + + expected := []string{"1", "2", "5"} + require.Equal(expected, values) + + lookup, err = multiIdx.(sql.NegateIndex).Not(int64(1), int64(6)) + require.NoError(err) + + values, err = lookupValues(lookup) + require.NoError(err) + + expected = []string{"2", "7", "8", "9", "10"} + require.Equal(expected, values) +} + +func TestPilosaHolder(t *testing.T) { + require := require.New(t) + + path, err := ioutil.TempDir("", "indexes-") + require.NoError(err) + defer os.RemoveAll(path) + + h := pilosa.NewHolder() + + h.Path = path + err = h.Open() + require.NoError(err) + + idx1, err := h.CreateIndexIfNotExists("idx", pilosa.IndexOptions{}) + require.NoError(err) + err = idx1.Open() + require.NoError(err) + + f1, err := idx1.CreateFieldIfNotExists("f1", pilosa.OptFieldTypeDefault()) + require.NoError(err) + + _, err = f1.SetBit(0, 0, nil) + require.NoError(err) + _, err = f1.SetBit(0, 2, nil) + require.NoError(err) + r0, err := f1.Row(0) + require.NoError(err) + + _, err = f1.SetBit(1, 0, nil) + require.NoError(err) + _, err = f1.SetBit(1, 1, nil) + require.NoError(err) + r1, err := f1.Row(1) + require.NoError(err) + + _, err = f1.SetBit(2, 2, nil) + require.NoError(err) + _, err = f1.SetBit(2, 3, nil) + require.NoError(err) + r2, err := f1.Row(2) + require.NoError(err) + + row := r0.Intersect(r1).Union(r2) + cols := row.Columns() + require.Equal(3, len(cols)) + require.Equal(uint64(0), cols[0]) + require.Equal(uint64(2), cols[1]) + require.Equal(uint64(3), cols[2]) + + f2, err := idx1.CreateFieldIfNotExists("f2", pilosa.OptFieldTypeDefault()) + require.NoError(err) + + rowIDs := []uint64{0, 0, 1, 1} + colIDs := []uint64{1, 2, 0, 3} + err = f2.Import(rowIDs, colIDs, nil) + require.NoError(err) + + r0, err = f2.Row(0) + require.NoError(err) + + r1, err = f2.Row(1) + require.NoError(err) + + row = r0.Union(r1) + cols = row.Columns() + require.Equal(4, len(cols)) + require.Equal(uint64(0), cols[0]) + require.Equal(uint64(1), cols[1]) + require.Equal(uint64(2), cols[2]) + require.Equal(uint64(3), cols[3]) + + r1, err = f1.Row(1) + require.NoError(err) + r0, err = f2.Row(0) + require.NoError(err) + + row = r1.Intersect(r0) + cols = row.Columns() + require.Equal(1, len(cols)) + require.Equal(uint64(1), cols[0]) + + err = idx1.Close() + require.NoError(err) + // ------------------------------------------------------------------------- + + idx2, err := h.CreateIndexIfNotExists("idx", pilosa.IndexOptions{}) + require.NoError(err) + err = idx2.Open() + require.NoError(err) + + f1 = idx2.Field("f1") + + r2, err = f1.Row(2) + require.NoError(err) + + f2 = idx2.Field("f2") + + r0, err = f2.Row(0) + require.NoError(err) + + r1, err = f2.Row(1) + require.NoError(err) + + row = r0.Union(r1) + cols = row.Columns() + require.Equal(4, len(cols)) + require.Equal(uint64(0), cols[0]) + require.Equal(uint64(1), cols[1]) + require.Equal(uint64(2), cols[2]) + require.Equal(uint64(3), cols[3]) + + err = idx2.Close() + require.NoError(err) + + err = h.Close() + require.NoError(err) +} + +func makeExpressions(names ...string) []sql.ExpressionHash { + var expressions []sql.ExpressionHash + + for _, n := range names { + h := sha1.Sum([]byte(n)) + exh := sql.ExpressionHash(h[:]) + expressions = append(expressions, sql.ExpressionHash(exh)) + } + + return expressions +} + +func randLocation(offset int) []byte { + b := make([]byte, 1) + rand.Read(b) + return b +} + +func offsetLocation(offset int) []byte { + b := make([]byte, 1) + b[0] = byte(offset % 10) + return b +} + +// test implementation of sql.IndexKeyValueIter interface +type testIndexKeyValueIter struct { + offset int + total int + expressions []sql.ExpressionHash + location func(int) []byte + + records []struct { + values []interface{} + location []byte + } +} + +func (it *testIndexKeyValueIter) Next() ([]interface{}, []byte, error) { + if it.offset >= it.total { + return nil, nil, io.EOF + } + + b := it.location(it.offset) + + values := make([]interface{}, len(it.expressions)) + for i, e := range it.expressions { + values[i] = hex.EncodeToString(e) + "-" + hex.EncodeToString(b) + } + + it.records = append(it.records, struct { + values []interface{} + location []byte + }{ + values, + b, + }) + it.offset++ + + return values, b, nil +} + +func (it *testIndexKeyValueIter) Close() error { + it.offset = 0 + it.records = nil + return nil +} + +func setupAscendDescend(t *testing.T) (*pilosaIndex, func()) { + t.Helper() + require := require.New(t) + + db, table, id := "db_name", "table_name", "index_id" + expressions := makeExpressions("a", "b") + root, err := mkdir(os.TempDir(), "indexes") + require.NoError(err) + + d := NewDriver(root) + sqlIdx, err := d.Create(db, table, id, expressions, nil) + require.NoError(err) + + it := &fixtureKeyValueIter{ + fixtures: []kvfixture{ + {"9", []interface{}{int64(2), int64(6)}}, + {"3", []interface{}{int64(7), int64(5)}}, + {"1", []interface{}{int64(1), int64(2)}}, + {"7", []interface{}{int64(1), int64(3)}}, + {"4", []interface{}{int64(7), int64(6)}}, + {"2", []interface{}{int64(10), int64(6)}}, + {"5", []interface{}{int64(5), int64(1)}}, + {"6", []interface{}{int64(6), int64(2)}}, + {"10", []interface{}{int64(4), int64(0)}}, + {"8", []interface{}{int64(3), int64(5)}}, + }, + } + + err = d.Save(sql.NewEmptyContext(), sqlIdx, it) + require.NoError(err) + + return sqlIdx.(*pilosaIndex), func() { + require.NoError(os.RemoveAll(root)) + } +} + +func lookupValues(lookup sql.IndexLookup) ([]string, error) { + iter, err := lookup.Values() + if err != nil { + return nil, err + } + + var result []string + for { + k, err := iter.Next() + if err == io.EOF { + break + } + + if err != nil { + return nil, err + } + + result = append(result, string(k)) + } + + return result, nil +} + +type kvfixture struct { + key string + values []interface{} +} + +type fixtureKeyValueIter struct { + fixtures []kvfixture + pos int +} + +func (i *fixtureKeyValueIter) Next() ([]interface{}, []byte, error) { + if i.pos >= len(i.fixtures) { + return nil, nil, io.EOF + } + + f := i.fixtures[i.pos] + i.pos++ + return f.values, []byte(f.key), nil +} + +func (i *fixtureKeyValueIter) Close() error { return nil } diff --git a/sql/index/pilosalib/index.go b/sql/index/pilosalib/index.go new file mode 100644 index 000000000..ef315d3a1 --- /dev/null +++ b/sql/index/pilosalib/index.go @@ -0,0 +1,296 @@ +package pilosalib + +import ( + "github.com/pilosa/pilosa" + errors "gopkg.in/src-d/go-errors.v1" + "gopkg.in/src-d/go-mysql-server.v0/sql" + "gopkg.in/src-d/go-mysql-server.v0/sql/index" +) + +var ( + errInvalidKeys = errors.NewKind("expecting %d keys for index %q, got %d") +) + +// pilosaIndex is an pilosa implementation of sql.Index interface +type pilosaIndex struct { + index *pilosa.Index + mapping *mapping + + db string + table string + id string + expressions []sql.ExpressionHash +} + +func newPilosaIndex(idx *pilosa.Index, mapping *mapping, cfg *index.Config) *pilosaIndex { + return &pilosaIndex{ + index: idx, + db: cfg.DB, + table: cfg.Table, + id: cfg.ID, + expressions: cfg.ExpressionHashes(), + mapping: mapping, + } +} + +// Get returns an IndexLookup for the given key in the index. +// If key parameter is not present then the returned iterator +// will go through all the locations on the index. +func (idx *pilosaIndex) Get(keys ...interface{}) (sql.IndexLookup, error) { + if len(keys) != len(idx.expressions) { + return nil, errInvalidKeys.New(len(idx.expressions), idx.ID(), len(keys)) + } + + return &indexLookup{ + id: idx.ID(), + index: idx.index, + mapping: idx.mapping, + keys: keys, + expressions: idx.expressions, + }, nil +} + +// Has checks if the given key is present in the index mapping +func (idx *pilosaIndex) Has(key ...interface{}) (bool, error) { + idx.mapping.open() + defer idx.mapping.close() + + n := len(key) + if n > len(idx.expressions) { + n = len(idx.expressions) + } + + for i, expr := range idx.expressions { + name := fieldName(idx.ID(), expr) + + val, err := idx.mapping.get(name, key[i]) + if err != nil || val == nil { + return false, err + } + } + + return true, nil +} + +// Database returns the database name this index belongs to. +func (idx *pilosaIndex) Database() string { + return idx.db +} + +// Table returns the table name this index belongs to. +func (idx *pilosaIndex) Table() string { + return idx.table +} + +// ID returns the identifier of the index. +func (idx *pilosaIndex) ID() string { + return idx.id +} + +// Expressions returns the indexed expressions. If the result is more than +// one expression, it means the index has multiple columns indexed. If it's +// just one, it means it may be an expression or a column. +func (idx *pilosaIndex) ExpressionHashes() []sql.ExpressionHash { + return idx.expressions +} + +func (pilosaIndex) Driver() string { return DriverID } + +func (idx *pilosaIndex) AscendGreaterOrEqual(keys ...interface{}) (sql.IndexLookup, error) { + if len(keys) != len(idx.expressions) { + return nil, errInvalidKeys.New(len(idx.expressions), idx.ID(), len(keys)) + } + + return newAscendLookup(&filteredLookup{ + id: idx.ID(), + index: idx.index, + mapping: idx.mapping, + keys: keys, + expressions: idx.expressions, + }, keys, nil), nil +} + +func (idx *pilosaIndex) AscendLessThan(keys ...interface{}) (sql.IndexLookup, error) { + if len(keys) != len(idx.expressions) { + return nil, errInvalidKeys.New(len(idx.expressions), idx.ID(), len(keys)) + } + + return newAscendLookup(&filteredLookup{ + id: idx.ID(), + index: idx.index, + mapping: idx.mapping, + keys: keys, + expressions: idx.expressions, + }, nil, keys), nil +} + +func (idx *pilosaIndex) AscendRange(greaterOrEqual, lessThan []interface{}) (sql.IndexLookup, error) { + if len(greaterOrEqual) != len(idx.expressions) { + return nil, errInvalidKeys.New(len(idx.expressions), idx.ID(), len(greaterOrEqual)) + } + + if len(lessThan) != len(idx.expressions) { + return nil, errInvalidKeys.New(len(idx.expressions), idx.ID(), len(lessThan)) + } + + return newAscendLookup(&filteredLookup{ + id: idx.ID(), + index: idx.index, + mapping: idx.mapping, + expressions: idx.expressions, + }, greaterOrEqual, lessThan), nil +} + +func (idx *pilosaIndex) DescendGreater(keys ...interface{}) (sql.IndexLookup, error) { + if len(keys) != len(idx.expressions) { + return nil, errInvalidKeys.New(len(idx.expressions), idx.ID(), len(keys)) + } + + return newDescendLookup(&filteredLookup{ + id: idx.ID(), + index: idx.index, + mapping: idx.mapping, + keys: keys, + expressions: idx.expressions, + reverse: true, + }, keys, nil), nil +} + +func (idx *pilosaIndex) DescendLessOrEqual(keys ...interface{}) (sql.IndexLookup, error) { + if len(keys) != len(idx.expressions) { + return nil, errInvalidKeys.New(len(idx.expressions), idx.ID(), len(keys)) + } + + return newDescendLookup(&filteredLookup{ + id: idx.ID(), + index: idx.index, + mapping: idx.mapping, + keys: keys, + expressions: idx.expressions, + reverse: true, + }, nil, keys), nil +} + +func (idx *pilosaIndex) DescendRange(lessOrEqual, greaterThan []interface{}) (sql.IndexLookup, error) { + if len(lessOrEqual) != len(idx.expressions) { + return nil, errInvalidKeys.New(len(idx.expressions), idx.ID(), len(lessOrEqual)) + } + + if len(greaterThan) != len(idx.expressions) { + return nil, errInvalidKeys.New(len(idx.expressions), idx.ID(), len(greaterThan)) + } + + return newDescendLookup(&filteredLookup{ + id: idx.ID(), + index: idx.index, + mapping: idx.mapping, + expressions: idx.expressions, + reverse: true, + }, greaterThan, lessOrEqual), nil +} + +func (idx *pilosaIndex) Not(keys ...interface{}) (sql.IndexLookup, error) { + if len(keys) != len(idx.expressions) { + return nil, errInvalidKeys.New(len(idx.expressions), idx.ID(), len(keys)) + } + + return &negateLookup{ + id: idx.ID(), + index: idx.index, + mapping: idx.mapping, + keys: keys, + expressions: idx.expressions, + }, nil +} + +func newAscendLookup(f *filteredLookup, gte []interface{}, lt []interface{}) *ascendLookup { + l := &ascendLookup{filteredLookup: f, gte: gte, lt: lt} + if l.filter == nil { + l.filter = func(i int, value []byte) (bool, error) { + var v interface{} + var err error + if len(l.gte) > 0 { + v, err = decodeGob(value, l.gte[i]) + if err != nil { + return false, err + } + + cmp, err := compare(v, l.gte[i]) + if err != nil { + return false, err + } + + if cmp < 0 { + return false, nil + } + } + + if len(l.lt) > 0 { + if v == nil { + v, err = decodeGob(value, l.lt[i]) + if err != nil { + return false, err + } + } + + cmp, err := compare(v, l.lt[i]) + if err != nil { + return false, err + } + + if cmp >= 0 { + return false, nil + } + } + + return true, nil + } + } + return l +} + +func newDescendLookup(f *filteredLookup, gt []interface{}, lte []interface{}) *descendLookup { + l := &descendLookup{filteredLookup: f, gt: gt, lte: lte} + if l.filter == nil { + l.filter = func(i int, value []byte) (bool, error) { + var v interface{} + var err error + if len(l.gt) > 0 { + v, err = decodeGob(value, l.gt[i]) + if err != nil { + return false, err + } + + cmp, err := compare(v, l.gt[i]) + if err != nil { + return false, err + } + + if cmp <= 0 { + return false, nil + } + } + + if len(l.lte) > 0 { + if v == nil { + v, err = decodeGob(value, l.lte[i]) + if err != nil { + return false, err + } + } + + cmp, err := compare(v, l.lte[i]) + if err != nil { + return false, err + } + + if cmp > 0 { + return false, nil + } + } + + return true, nil + } + } + return l +} diff --git a/sql/index/pilosalib/iterator.go b/sql/index/pilosalib/iterator.go new file mode 100644 index 000000000..b7ae49c73 --- /dev/null +++ b/sql/index/pilosalib/iterator.go @@ -0,0 +1,83 @@ +package pilosalib + +import ( + "io" + + "github.com/boltdb/bolt" + "github.com/sirupsen/logrus" +) + +type locationValueIter struct { + locations [][]byte + pos int +} + +func (i *locationValueIter) Next() ([]byte, error) { + if i.pos >= len(i.locations) { + return nil, io.EOF + } + + i.pos++ + return i.locations[i.pos-1], nil +} + +func (i *locationValueIter) Close() error { + i.locations = nil + return nil +} + +type indexValueIter struct { + offset uint64 + total uint64 + bits []uint64 + mapping *mapping + indexName string + + // share transaction and bucket on all getLocation calls + bucket *bolt.Bucket + tx *bolt.Tx +} + +func (it *indexValueIter) Next() ([]byte, error) { + if it.bucket == nil { + bucket, err := it.mapping.getBucket(it.indexName, false) + if err != nil { + return nil, err + } + + it.bucket = bucket + it.tx = bucket.Tx() + } + + if it.offset >= it.total { + if err := it.Close(); err != nil { + logrus.WithField("err", err.Error()). + Error("unable to close the pilosa index value iterator") + } + + if it.tx != nil { + it.tx.Rollback() + } + + return nil, io.EOF + } + + var colID uint64 + if it.bits == nil { + colID = it.offset + } else { + colID = it.bits[it.offset] + } + + it.offset++ + + return it.mapping.getLocationFromBucket(it.bucket, colID) +} + +func (it *indexValueIter) Close() error { + if it.tx != nil { + it.tx.Rollback() + } + + return it.mapping.close() +} diff --git a/sql/index/pilosalib/lookup.go b/sql/index/pilosalib/lookup.go new file mode 100644 index 000000000..57aa0d9f8 --- /dev/null +++ b/sql/index/pilosalib/lookup.go @@ -0,0 +1,674 @@ +package pilosalib + +import ( + "bytes" + "encoding/gob" + "io" + "strings" + "time" + + "github.com/pilosa/pilosa" + errors "gopkg.in/src-d/go-errors.v1" + "gopkg.in/src-d/go-mysql-server.v0/sql" +) + +var ( + errUnknownType = errors.NewKind("unknown type %T received as value") + errTypeMismatch = errors.NewKind("cannot compare type %T with type %T") + errUnmergeableType = errors.NewKind("unmergeable type %T") + + // operation functors + intersect = func(r1, r2 *pilosa.Row) *pilosa.Row { + if r1 == nil { + return r2 + } + return r1.Intersect(r2) + } + union = func(r1, r2 *pilosa.Row) *pilosa.Row { + if r1 == nil { + return r2 + } + return r1.Union(r2) + } + difference = func(r1, r2 *pilosa.Row) *pilosa.Row { + if r1 == nil { + return r2 + } + return r1.Difference(r2) + } +) + +type ( + + // indexLookup implement following interfaces: + // sql.IndexLookup, sql.Mergeable, sql.SetOperations + indexLookup struct { + id string + index *pilosa.Index + mapping *mapping + keys []interface{} + expressions []sql.ExpressionHash + operations []*lookupOperation + } + + lookupOperation struct { + lookup sql.IndexLookup + operation func(*pilosa.Row, *pilosa.Row) *pilosa.Row + } + + pilosaLookup interface { + indexName() string + values() (*pilosa.Row, error) + } +) + +func (l *indexLookup) indexName() string { + return l.index.Name() +} + +func (l *indexLookup) values() (*pilosa.Row, error) { + l.mapping.open() + defer l.mapping.close() + + var row *pilosa.Row + for i, expr := range l.expressions { + field := l.index.Field(fieldName(l.id, expr)) + rowID, err := l.mapping.rowID(field.Name(), l.keys[i]) + if err == io.EOF { + continue + } + if err != nil { + return nil, err + } + + r, err := field.Row(rowID) + if err != nil { + return nil, err + } + + row = intersect(row, r) + } + if row == nil { + return nil, nil + } + + // evaluate composition of operations + for _, op := range l.operations { + var ( + r *pilosa.Row + e error + ) + + il, ok := op.lookup.(pilosaLookup) + if !ok { + return nil, errUnmergeableType.New(op.lookup) + } + + r, e = il.values() + if e != nil { + return nil, e + } + + row = op.operation(row, r) + } + + return row, nil +} + +// Values implements sql.IndexLookup.Values +func (l *indexLookup) Values() (sql.IndexValueIter, error) { + l.index.Open() + defer l.index.Close() + + row, err := l.values() + if err != nil { + return nil, err + } + + l.mapping.open() + if row == nil { + return &indexValueIter{mapping: l.mapping, indexName: l.index.Name()}, nil + } + + bits := row.Columns() + return &indexValueIter{ + total: uint64(len(bits)), + bits: bits, + mapping: l.mapping, + indexName: l.index.Name(), + }, nil +} + +// IsMergeable implements sql.Mergeable interface. +func (l *indexLookup) IsMergeable(lookup sql.IndexLookup) bool { + if il, ok := lookup.(pilosaLookup); ok { + return il.indexName() == l.indexName() + } + + return false +} + +// Intersection implements sql.SetOperations interface +func (l *indexLookup) Intersection(lookups ...sql.IndexLookup) sql.IndexLookup { + lookup := *l + for _, li := range lookups { + lookup.operations = append(lookup.operations, &lookupOperation{li, intersect}) + } + + return &lookup +} + +// Union implements sql.SetOperations interface +func (l *indexLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { + lookup := *l + for _, li := range lookups { + lookup.operations = append(lookup.operations, &lookupOperation{li, union}) + } + + return &lookup +} + +// Difference implements sql.SetOperations interface +func (l *indexLookup) Difference(lookups ...sql.IndexLookup) sql.IndexLookup { + lookup := *l + for _, li := range lookups { + lookup.operations = append(lookup.operations, &lookupOperation{li, difference}) + } + + return &lookup +} + +type filteredLookup struct { + id string + index *pilosa.Index + mapping *mapping + keys []interface{} + expressions []sql.ExpressionHash + operations []*lookupOperation + + reverse bool + filter func(int, []byte) (bool, error) +} + +func (l *filteredLookup) indexName() string { + return l.index.Name() +} + +func (l *filteredLookup) values() (*pilosa.Row, error) { + l.mapping.open() + defer l.mapping.close() + + // evaluate Intersection of bitmaps + var row *pilosa.Row + for i, expr := range l.expressions { + field := l.index.Field(fieldName(l.id, expr)) + rows, err := l.mapping.filter(field.Name(), func(b []byte) (bool, error) { + return l.filter(i, b) + }) + if err != nil { + return nil, err + } + + var r *pilosa.Row + for _, ri := range rows { + rr, err := field.Row(ri) + if err != nil { + return nil, err + } + r = union(r, rr) + } + + row = intersect(row, r) + } + + if row == nil { + return nil, nil + } + + // evaluate composition of operations + for _, op := range l.operations { + var ( + r *pilosa.Row + e error + ) + + il, ok := op.lookup.(pilosaLookup) + if !ok { + return nil, errUnmergeableType.New(op.lookup) + } + + r, e = il.values() + if e != nil { + return nil, e + } + if r == nil { + continue + } + + row = op.operation(row, r) + } + + return row, nil +} + +func (l *filteredLookup) Values() (sql.IndexValueIter, error) { + l.index.Open() + defer l.index.Close() + + row, err := l.values() + if err != nil { + return nil, err + } + + l.mapping.open() + if row == nil { + return &indexValueIter{mapping: l.mapping, indexName: l.index.Name()}, nil + } + + bits := row.Columns() + locations, err := l.mapping.sortedLocations(l.index.Name(), bits, l.reverse) + if err != nil { + return nil, err + } + + return &locationValueIter{locations: locations}, nil +} + +// IsMergeable implements sql.Mergeable interface. +func (l *filteredLookup) IsMergeable(lookup sql.IndexLookup) bool { + if il, ok := lookup.(pilosaLookup); ok { + return il.indexName() == l.indexName() + } + return false +} + +// Intersection implements sql.SetOperations interface +func (l *filteredLookup) Intersection(lookups ...sql.IndexLookup) sql.IndexLookup { + lookup := *l + for _, li := range lookups { + lookup.operations = append(lookup.operations, &lookupOperation{li, intersect}) + } + + return &lookup +} + +// Union implements sql.SetOperations interface +func (l *filteredLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { + lookup := *l + for _, li := range lookups { + lookup.operations = append(lookup.operations, &lookupOperation{li, union}) + } + + return &lookup +} + +// Difference implements sql.SetOperations interface +func (l *filteredLookup) Difference(lookups ...sql.IndexLookup) sql.IndexLookup { + lookup := *l + for _, li := range lookups { + lookup.operations = append(lookup.operations, &lookupOperation{li, difference}) + } + + return &lookup +} + +type ascendLookup struct { + *filteredLookup + gte []interface{} + lt []interface{} +} + +type descendLookup struct { + *filteredLookup + gt []interface{} + lte []interface{} +} + +type negateLookup struct { + id string + index *pilosa.Index + mapping *mapping + keys []interface{} + expressions []sql.ExpressionHash + operations []*lookupOperation +} + +func (l *negateLookup) indexName() string { return l.index.Name() } + +func (l *negateLookup) values() (*pilosa.Row, error) { + l.mapping.open() + defer l.mapping.close() + + var row *pilosa.Row + for i, expr := range l.expressions { + field := l.index.Field(fieldName(l.id, expr)) + maxRowID, err := l.mapping.getMaxRowID(field.Name()) + if err != nil { + return nil, err + } + + // Since Pilosa does not have a negation in PQL (see: + // https://github.com/pilosa/pilosa/issues/807), we have to get all the + // ones in all the rows and join them, and then make difference between + // them and the ones in the row of the given value. + var r *pilosa.Row + // rowIDs start with 1 + for ri := uint64(1); ri <= maxRowID; ri++ { + rr, err := field.Row(ri) + if err != nil { + return nil, err + } + r = union(r, rr) + } + + rowID, err := l.mapping.rowID(field.Name(), l.keys[i]) + if err == io.EOF { + continue + } + if err != nil { + return nil, err + } + + rr, err := field.Row(rowID) + if err != nil { + return nil, err + } + r = difference(r, rr) + + row = intersect(row, r) + } + if row == nil { + return nil, nil + } + + // evaluate composition of operations + for _, op := range l.operations { + var ( + r *pilosa.Row + e error + ) + + il, ok := op.lookup.(pilosaLookup) + if !ok { + return nil, errUnmergeableType.New(op.lookup) + } + + r, e = il.values() + if e != nil { + return nil, e + } + + if r == nil { + continue + } + + row = op.operation(row, r) + } + + return row, nil +} + +// Values implements sql.IndexLookup.Values +func (l *negateLookup) Values() (sql.IndexValueIter, error) { + l.index.Open() + defer l.index.Close() + + row, err := l.values() + if err != nil { + return nil, err + } + + l.mapping.open() + if row == nil { + return &indexValueIter{mapping: l.mapping, indexName: l.index.Name()}, nil + } + + bits := row.Columns() + return &indexValueIter{ + total: uint64(len(bits)), + bits: bits, + mapping: l.mapping, + indexName: l.index.Name(), + }, nil +} + +// IsMergeable implements sql.Mergeable interface. +func (l *negateLookup) IsMergeable(lookup sql.IndexLookup) bool { + if il, ok := lookup.(pilosaLookup); ok { + return il.indexName() == l.indexName() + } + + return false +} + +// Intersection implements sql.SetOperations interface +func (l *negateLookup) Intersection(lookups ...sql.IndexLookup) sql.IndexLookup { + lookup := *l + for _, li := range lookups { + lookup.operations = append(lookup.operations, &lookupOperation{li, intersect}) + } + + return &lookup +} + +// Union implements sql.SetOperations interface +func (l *negateLookup) Union(lookups ...sql.IndexLookup) sql.IndexLookup { + lookup := *l + for _, li := range lookups { + lookup.operations = append(lookup.operations, &lookupOperation{li, union}) + } + + return &lookup +} + +// Difference implements sql.SetOperations interface +func (l *negateLookup) Difference(lookups ...sql.IndexLookup) sql.IndexLookup { + lookup := *l + for _, li := range lookups { + lookup.operations = append(lookup.operations, &lookupOperation{li, difference}) + } + + return &lookup +} + +func decodeGob(k []byte, value interface{}) (interface{}, error) { + decoder := gob.NewDecoder(bytes.NewBuffer(k)) + + switch value.(type) { + case string: + var v string + err := decoder.Decode(&v) + return v, err + case int32: + var v int32 + err := decoder.Decode(&v) + return v, err + case int64: + var v int64 + err := decoder.Decode(&v) + return v, err + case uint32: + var v uint32 + err := decoder.Decode(&v) + return v, err + case uint64: + var v uint64 + err := decoder.Decode(&v) + return v, err + case float64: + var v float64 + err := decoder.Decode(&v) + return v, err + case time.Time: + var v time.Time + err := decoder.Decode(&v) + return v, err + case []byte: + var v []byte + err := decoder.Decode(&v) + return v, err + case bool: + var v bool + err := decoder.Decode(&v) + return v, err + case []interface{}: + var v []interface{} + err := decoder.Decode(&v) + return v, err + default: + return nil, errUnknownType.New(value) + } +} + +// compare two values of the same underlying type. The values MUST be of the +// same type. +func compare(a, b interface{}) (int, error) { + switch a := a.(type) { + case bool: + v, ok := b.(bool) + if !ok { + return 0, errTypeMismatch.New(a, b) + } + + if a == v { + return 0, nil + } + + if a == false { + return -1, nil + } + + return 1, nil + case string: + v, ok := b.(string) + if !ok { + return 0, errTypeMismatch.New(a, b) + } + + return strings.Compare(a, v), nil + case int32: + v, ok := b.(int32) + if !ok { + return 0, errTypeMismatch.New(a, b) + } + + if a == v { + return 0, nil + } + + if a < v { + return -1, nil + } + + return 1, nil + case int64: + v, ok := b.(int64) + if !ok { + return 0, errTypeMismatch.New(a, b) + } + + if a == v { + return 0, nil + } + + if a < v { + return -1, nil + } + + return 1, nil + case uint32: + v, ok := b.(uint32) + if !ok { + return 0, errTypeMismatch.New(a, b) + } + + if a == v { + return 0, nil + } + + if a < v { + return -1, nil + } + + return 1, nil + case uint64: + v, ok := b.(uint64) + if !ok { + return 0, errTypeMismatch.New(a, b) + } + + if a == v { + return 0, nil + } + + if a < v { + return -1, nil + } + + return 1, nil + case float64: + v, ok := b.(float64) + if !ok { + return 0, errTypeMismatch.New(a, b) + } + + if a == v { + return 0, nil + } + + if a < v { + return -1, nil + } + + return 1, nil + case []byte: + v, ok := b.([]byte) + if !ok { + return 0, errTypeMismatch.New(a, b) + } + return bytes.Compare(a, v), nil + case []interface{}: + v, ok := b.([]interface{}) + if !ok { + return 0, errTypeMismatch.New(a, b) + } + + if len(a) < len(v) { + return -1, nil + } + + if len(a) > len(v) { + return 1, nil + } + + for i := range a { + cmp, err := compare(a[i], v[i]) + if err != nil { + return 0, err + } + + if cmp != 0 { + return cmp, nil + } + } + + return 0, nil + case time.Time: + v, ok := b.(time.Time) + if !ok { + return 0, errTypeMismatch.New(a, b) + } + + if a.Equal(v) { + return 0, nil + } + + if a.Before(v) { + return -1, nil + } + + return 1, nil + default: + return 0, errUnknownType.New(a) + } +} diff --git a/sql/index/pilosalib/lookup_test.go b/sql/index/pilosalib/lookup_test.go new file mode 100644 index 000000000..1514f738d --- /dev/null +++ b/sql/index/pilosalib/lookup_test.go @@ -0,0 +1,203 @@ +package pilosalib + +import ( + "bytes" + "encoding/gob" + "fmt" + "os" + "testing" + "time" + + "github.com/pilosa/pilosa" + "github.com/stretchr/testify/require" + errors "gopkg.in/src-d/go-errors.v1" + "gopkg.in/src-d/go-mysql-server.v0/sql" +) + +func TestCompare(t *testing.T) { + now := time.Now() + testCases := []struct { + a, b interface{} + err *errors.Kind + expected int + }{ + {true, true, nil, 0}, + {false, true, nil, -1}, + {true, false, nil, 1}, + {false, false, nil, 0}, + {true, 0, errTypeMismatch, 0}, + + {"a", "b", nil, -1}, + {"b", "a", nil, 1}, + {"a", "a", nil, 0}, + {"a", 1, errTypeMismatch, 0}, + + {int32(1), int32(2), nil, -1}, + {int32(2), int32(1), nil, 1}, + {int32(2), int32(2), nil, 0}, + {int32(1), "", errTypeMismatch, 0}, + + {int64(1), int64(2), nil, -1}, + {int64(2), int64(1), nil, 1}, + {int64(2), int64(2), nil, 0}, + {int64(1), "", errTypeMismatch, 0}, + + {uint32(1), uint32(2), nil, -1}, + {uint32(2), uint32(1), nil, 1}, + {uint32(2), uint32(2), nil, 0}, + {uint32(1), "", errTypeMismatch, 0}, + + {uint64(1), uint64(2), nil, -1}, + {uint64(2), uint64(1), nil, 1}, + {uint64(2), uint64(2), nil, 0}, + {uint64(1), "", errTypeMismatch, 0}, + + {float64(1), float64(2), nil, -1}, + {float64(2), float64(1), nil, 1}, + {float64(2), float64(2), nil, 0}, + {float64(1), "", errTypeMismatch, 0}, + + {now.Add(-1 * time.Hour), now, nil, -1}, + {now, now.Add(-1 * time.Hour), nil, 1}, + {now, now, nil, 0}, + {now, 1, errTypeMismatch, -1}, + + {[]interface{}{"a", "a"}, []interface{}{"a", "b"}, nil, -1}, + {[]interface{}{"a", "b"}, []interface{}{"a", "a"}, nil, 1}, + {[]interface{}{"a", "a"}, []interface{}{"a", "a"}, nil, 0}, + {[]interface{}{"b"}, []interface{}{"a", "b"}, nil, -1}, + {[]interface{}{"b"}, 1, errTypeMismatch, -1}, + + {[]byte{0, 1}, []byte{1, 1}, nil, -1}, + {[]byte{1, 1}, []byte{0, 1}, nil, 1}, + {[]byte{1, 1}, []byte{1, 1}, nil, 0}, + {[]byte{1}, []byte{0, 1}, nil, 1}, + {[]byte{0, 1}, 1, errTypeMismatch, -1}, + + {time.Duration(0), nil, errUnknownType, -1}, + } + + for _, tt := range testCases { + name := fmt.Sprintf("(%T)(%v) and (%T)(%v)", tt.a, tt.a, tt.b, tt.b) + t.Run(name, func(t *testing.T) { + require := require.New(t) + cmp, err := compare(tt.a, tt.b) + if tt.err != nil { + require.Error(err) + require.True(tt.err.Is(err)) + } else { + require.NoError(err) + require.Equal(tt.expected, cmp) + } + }) + } +} + +func TestDecodeGob(t *testing.T) { + testCases := []interface{}{ + "foo", + int32(1), + int64(1), + uint32(1), + uint64(1), + float64(1), + true, + time.Date(2018, time.August, 1, 1, 1, 1, 1, time.Local), + []byte("foo"), + []interface{}{1, 3, 3, 7}, + } + + for _, tt := range testCases { + name := fmt.Sprintf("(%T)(%v)", tt, tt) + t.Run(name, func(t *testing.T) { + require := require.New(t) + + var buf bytes.Buffer + require.NoError(gob.NewEncoder(&buf).Encode(tt)) + + result, err := decodeGob(buf.Bytes(), tt) + require.NoError(err) + require.Equal(tt, result) + }) + } +} + +func TestMergeable(t *testing.T) { + require := require.New(t) + h := pilosa.NewHolder() + h.Path = os.TempDir() + + i1, err := h.CreateIndexIfNotExists("i1", pilosa.IndexOptions{}) + require.NoError(err) + i2, err := h.CreateIndexIfNotExists("i2", pilosa.IndexOptions{}) + require.NoError(err) + + testCases := []struct { + i1 sql.IndexLookup + i2 sql.IndexLookup + expected bool + }{ + { + i1: &indexLookup{index: i1}, + i2: &indexLookup{index: i1}, + expected: true, + }, + { + i1: &indexLookup{index: i1}, + i2: &indexLookup{index: i2}, + expected: false, + }, + { + i1: &indexLookup{index: i1}, + i2: &ascendLookup{filteredLookup: &filteredLookup{index: i1}}, + expected: true, + }, + { + i1: &descendLookup{filteredLookup: &filteredLookup{index: i1}}, + i2: &ascendLookup{filteredLookup: &filteredLookup{index: i1}}, + expected: true, + }, + { + i1: &descendLookup{filteredLookup: &filteredLookup{index: i1}}, + i2: &indexLookup{index: i2}, + expected: false, + }, + { + i1: &descendLookup{filteredLookup: &filteredLookup{index: i1}}, + i2: &descendLookup{filteredLookup: &filteredLookup{index: i2}}, + expected: false, + }, + { + i1: &negateLookup{index: i1}, + i2: &negateLookup{index: i1}, + expected: true, + }, + { + i1: &negateLookup{index: i1}, + i2: &negateLookup{index: i2}, + expected: false, + }, + { + i1: &negateLookup{index: i1}, + i2: &indexLookup{index: i1}, + expected: true, + }, + { + i1: &negateLookup{index: i1}, + i2: &descendLookup{filteredLookup: &filteredLookup{index: i1}}, + expected: true, + }, + { + i1: &negateLookup{index: i1}, + i2: &ascendLookup{filteredLookup: &filteredLookup{index: i1}}, + expected: true, + }, + } + + for _, tc := range testCases { + m1, ok := tc.i1.(sql.Mergeable) + require.True(ok) + + require.Equal(tc.expected, m1.IsMergeable(tc.i2)) + } +} diff --git a/sql/index/pilosalib/mapping.go b/sql/index/pilosalib/mapping.go new file mode 100644 index 000000000..1a7d1ff05 --- /dev/null +++ b/sql/index/pilosalib/mapping.go @@ -0,0 +1,403 @@ +package pilosalib + +import ( + "bytes" + "encoding/binary" + "encoding/gob" + "fmt" + "io" + "sort" + "sync" + + "github.com/boltdb/bolt" +) + +// mapping +// buckets: +// - index name: columndID uint64 -> location []byte +// - frame name: value []byte (gob encoding) -> rowID uint64 +type mapping struct { + path string + + mut sync.RWMutex + db *bolt.DB + + // in create mode there's only one transaction closed explicitly by + // commit function + create bool + tx *bolt.Tx + + clientMut sync.Mutex + clients int +} + +func newMapping(path string) *mapping { + return &mapping{path: path} +} + +func (m *mapping) open() { + m.openCreate(false) +} + +// openCreate opens and sets creation mode in the database. +func (m *mapping) openCreate(create bool) { + m.clientMut.Lock() + defer m.clientMut.Unlock() + m.clients++ + m.create = create +} + +func (m *mapping) close() error { + m.clientMut.Lock() + defer m.clientMut.Unlock() + + if m.clients > 1 { + m.clients-- + return nil + } + + m.clients = 0 + + m.mut.Lock() + defer m.mut.Unlock() + + if m.db != nil { + if err := m.db.Close(); err != nil { + return err + } + m.db = nil + } + + return nil +} + +func (m *mapping) query(fn func() error) error { + m.mut.Lock() + if m.db == nil { + var err error + m.db, err = bolt.Open(m.path, 0640, nil) + if err != nil { + m.mut.Unlock() + return err + } + } + m.mut.Unlock() + + m.mut.RLock() + defer m.mut.RUnlock() + return fn() +} + +func (m *mapping) rowID(frameName string, value interface{}) (uint64, error) { + val, err := m.get(frameName, value) + if err != nil { + return 0, err + } + if val == nil { + return 0, io.EOF + } + + return binary.LittleEndian.Uint64(val), err +} + +// commit saves current transaction, if cont is true a new transaction will be +// created again in the next query. Only for create mode. +func (m *mapping) commit(cont bool) error { + m.clientMut.Lock() + defer m.clientMut.Unlock() + + var err error + if m.create && m.tx != nil { + err = m.tx.Commit() + } + + m.create = cont + m.tx = nil + + return err +} + +func (m *mapping) rollback() error { + m.clientMut.Lock() + defer m.clientMut.Unlock() + + var err error + if m.create && m.tx != nil { + err = m.tx.Rollback() + } + + m.create = false + m.tx = nil + + return err +} + +func (m *mapping) transaction(writable bool, f func(*bolt.Tx) error) error { + var tx *bolt.Tx + var err error + if m.create { + m.clientMut.Lock() + if m.tx == nil { + m.tx, err = m.db.Begin(true) + if err != nil { + m.clientMut.Unlock() + return err + } + } + + m.clientMut.Unlock() + + tx = m.tx + } else { + tx, err = m.db.Begin(writable) + if err != nil { + return err + } + } + + err = f(tx) + + if m.create { + return err + } + + if err != nil { + tx.Rollback() + return err + } + + return tx.Commit() +} + +func (m *mapping) getRowID(frameName string, value interface{}) (uint64, error) { + var id uint64 + err := m.query(func() error { + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + err := enc.Encode(value) + if err != nil { + return err + } + + err = m.transaction(true, func(tx *bolt.Tx) error { + b, err := tx.CreateBucketIfNotExists([]byte(frameName)) + if err != nil { + return err + } + + key := buf.Bytes() + val := b.Get(key) + if val != nil { + id = binary.LittleEndian.Uint64(val) + return nil + } + + // the first NextSequence is 1 so the first id will be 1 + // this can only fail if the transaction is closed + id, _ = b.NextSequence() + + val = make([]byte, 8) + binary.LittleEndian.PutUint64(val, id) + err = b.Put(key, val) + return err + }) + + return err + }) + + return id, err +} + +func (m *mapping) getMaxRowID(frameName string) (uint64, error) { + var id uint64 + err := m.query(func() error { + return m.transaction(true, func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(frameName)) + if b == nil { + return nil + } + + id = b.Sequence() + return nil + }) + }) + + return id, err +} + +func (m *mapping) putLocation(indexName string, colID uint64, location []byte) error { + return m.query(func() error { + return m.transaction(true, func(tx *bolt.Tx) error { + b, err := tx.CreateBucketIfNotExists([]byte(indexName)) + if err != nil { + return err + } + + key := make([]byte, 8) + binary.LittleEndian.PutUint64(key, colID) + + return b.Put(key, location) + }) + }) +} + +func (m *mapping) sortedLocations(indexName string, cols []uint64, reverse bool) ([][]byte, error) { + var result [][]byte + err := m.query(func() error { + return m.db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(indexName)) + if b == nil { + return fmt.Errorf("bucket %s not found", indexName) + } + + for _, col := range cols { + key := make([]byte, 8) + binary.LittleEndian.PutUint64(key, col) + val := b.Get(key) + + // val will point to mmap addresses, so we need to copy the slice + dst := make([]byte, len(val)) + copy(dst, val) + result = append(result, dst) + } + + return nil + }) + }) + + if err != nil { + return nil, err + } + + if reverse { + sort.Stable(sort.Reverse(byBytes(result))) + } else { + sort.Stable(byBytes(result)) + } + + return result, nil +} + +type byBytes [][]byte + +func (b byBytes) Len() int { return len(b) } +func (b byBytes) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +func (b byBytes) Less(i, j int) bool { return bytes.Compare(b[i], b[j]) < 0 } + +func (m *mapping) getLocation(indexName string, colID uint64) ([]byte, error) { + var location []byte + + err := m.query(func() error { + err := m.transaction(true, func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(indexName)) + if b == nil { + return fmt.Errorf("bucket %s not found", indexName) + } + + key := make([]byte, 8) + binary.LittleEndian.PutUint64(key, colID) + + location = b.Get(key) + return nil + }) + + return err + }) + + return location, err +} + +func (m *mapping) getLocationFromBucket( + bucket *bolt.Bucket, + colID uint64, +) ([]byte, error) { + var location []byte + + err := m.query(func() error { + key := make([]byte, 8) + binary.LittleEndian.PutUint64(key, colID) + + location = bucket.Get(key) + return nil + }) + + return location, err +} + +func (m *mapping) getBucket( + indexName string, + writable bool, +) (*bolt.Bucket, error) { + var bucket *bolt.Bucket + + err := m.query(func() error { + tx, err := m.db.Begin(writable) + if err != nil { + return err + } + + bucket = tx.Bucket([]byte(indexName)) + if bucket == nil { + tx.Rollback() + return fmt.Errorf("bucket %s not found", indexName) + } + + return nil + }) + + return bucket, err +} + +func (m *mapping) get(name string, key interface{}) ([]byte, error) { + var value []byte + + err := m.query(func() error { + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + err := enc.Encode(key) + if err != nil { + return err + } + + err = m.transaction(true, func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(name)) + if b != nil { + value = b.Get(buf.Bytes()) + return nil + } + + return fmt.Errorf("%s not found", name) + }) + + return err + }) + return value, err +} + +func (m *mapping) filter(name string, fn func([]byte) (bool, error)) ([]uint64, error) { + var result []uint64 + + err := m.query(func() error { + return m.db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(name)) + if b == nil { + return nil + } + + return b.ForEach(func(k, v []byte) error { + ok, err := fn(k) + if err != nil { + return err + } + + if ok { + result = append(result, binary.LittleEndian.Uint64(v)) + } + + return nil + }) + }) + }) + + return result, err +} diff --git a/sql/index/pilosalib/mapping_test.go b/sql/index/pilosalib/mapping_test.go new file mode 100644 index 000000000..68bf157b2 --- /dev/null +++ b/sql/index/pilosalib/mapping_test.go @@ -0,0 +1,91 @@ +package pilosalib + +import ( + "encoding/binary" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRowID(t *testing.T) { + require := require.New(t) + + root, err := mkdir(os.TempDir(), "mapping_test") + require.NoError(err) + defer os.RemoveAll(root) + + m := newMapping(filepath.Join(root, "id.map")) + m.open() + defer m.close() + + cases := []int{0, 1, 2, 3, 4, 5, 5, 0, 3, 2, 1, 5} + expected := []uint64{1, 2, 3, 4, 5, 6, 6, 1, 4, 3, 2, 6} + + for i, c := range cases { + rowID, err := m.getRowID("frame name", c) + require.NoError(err) + require.Equal(expected[i], rowID) + } + + maxRowID, err := m.getMaxRowID("frame name") + require.NoError(err) + require.Equal(uint64(6), maxRowID) +} + +func TestLocation(t *testing.T) { + require := require.New(t) + + root, err := mkdir(os.TempDir(), "mapping_test") + require.NoError(err) + defer os.RemoveAll(root) + + m := newMapping(filepath.Join(root, "id.map")) + m.open() + defer m.close() + + cases := map[uint64]string{ + 0: "zero", + 1: "one", + 2: "two", + 3: "three", + 4: "four", + } + + for colID, loc := range cases { + err = m.putLocation("index name", colID, []byte(loc)) + require.NoError(err) + } + + for colID, loc := range cases { + b, err := m.getLocation("index name", colID) + require.NoError(err) + require.Equal(loc, string(b)) + } +} + +func TestGet(t *testing.T) { + require := require.New(t) + + root, err := mkdir(os.TempDir(), "mapping_test") + require.NoError(err) + defer os.RemoveAll(root) + + m := newMapping(filepath.Join(root, "id.map")) + m.open() + defer m.close() + + cases := []int{0, 1, 2, 3, 4, 5, 5, 0, 3, 2, 1, 5} + expected := []uint64{1, 2, 3, 4, 5, 6, 6, 1, 4, 3, 2, 6} + + for i, c := range cases { + m.getRowID("frame name", c) + + id, err := m.get("frame name", c) + val := binary.LittleEndian.Uint64(id) + + require.NoError(err) + require.Equal(expected[i], val) + } +} From 6f0593308ad356ce6a66ad543d08b298a7429601 Mon Sep 17 00:00:00 2001 From: kuba-- Date: Wed, 1 Aug 2018 09:59:33 +0200 Subject: [PATCH 2/5] remove TRAVIS_BUILD_DIR Signed-off-by: kuba-- --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a7bb79099..9ffc90211 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,6 @@ before_install: install: - go get -u github.com/golang/dep/cmd/dep - - cd "$TRAVIS_BUILD_DIR" - touch Gopkg.toml - dep ensure -v -add "github.com/pilosa/go-pilosa@0.9.0" "github.com/pilosa/pilosa@master" - make dependencies From 53a1de200784ef228581448ea2f41fd23bdb995e Mon Sep 17 00:00:00 2001 From: kuba-- Date: Wed, 1 Aug 2018 10:21:03 +0200 Subject: [PATCH 3/5] Lock pilosa to edf6fa9c67 Signed-off-by: kuba-- --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9ffc90211..962847b88 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ before_install: install: - go get -u github.com/golang/dep/cmd/dep - touch Gopkg.toml - - dep ensure -v -add "github.com/pilosa/go-pilosa@0.9.0" "github.com/pilosa/pilosa@master" + - dep ensure -v -add "github.com/pilosa/go-pilosa@0.9.0" "github.com/pilosa/pilosa@edf6fa9c67caeab6019fb17ab82425a74fdfaf87" - make dependencies script: From 4c245e5de49d53229c48cf29a66ed62e213b35dd Mon Sep 17 00:00:00 2001 From: kuba-- Date: Wed, 1 Aug 2018 14:44:51 +0200 Subject: [PATCH 4/5] Cleanup test tmp dir. Signed-off-by: kuba-- --- sql/index/config_test.go | 19 ++-- sql/index/pilosa/driver_test.go | 121 ++++++++++++------------- sql/index/pilosa/mapping_test.go | 27 +++--- sql/index/pilosalib/driver_test.go | 133 +++++++++++++--------------- sql/index/pilosalib/mapping_test.go | 28 +++--- 5 files changed, 150 insertions(+), 178 deletions(-) diff --git a/sql/index/config_test.go b/sql/index/config_test.go index 6a97cf150..03b6b505e 100644 --- a/sql/index/config_test.go +++ b/sql/index/config_test.go @@ -2,6 +2,7 @@ package index import ( "crypto/sha1" + "io/ioutil" "os" "path/filepath" "testing" @@ -12,17 +13,18 @@ import ( func TestConfig(t *testing.T) { require := require.New(t) + tmpDir, err := ioutil.TempDir("", "index") + require.NoError(err) + defer func() { require.NoError(os.RemoveAll(tmpDir)) }() driver := "driver" db, table, id := "db_name", "table_name", "index_id" - dir := filepath.Join(os.TempDir(), driver) + dir := filepath.Join(tmpDir, driver) subdir := filepath.Join(dir, db, table) - err := os.MkdirAll(subdir, 0750) + err = os.MkdirAll(subdir, 0750) require.NoError(err) file := filepath.Join(subdir, id+".cfg") - defer os.RemoveAll(dir) - h1 := sha1.Sum([]byte("h1")) h2 := sha1.Sum([]byte("h2")) exh1 := sql.ExpressionHash(h1[:]) @@ -48,12 +50,13 @@ func TestConfig(t *testing.T) { require.Equal(cfg1, cfg2) } -func TestLockFile(t *testing.T) { +func TestProcessingFile(t *testing.T) { require := require.New(t) + tmpDir, err := ioutil.TempDir("", "index") + require.NoError(err) + defer func() { require.NoError(os.RemoveAll(tmpDir)) }() - dir := os.TempDir() - file := filepath.Join(dir, ".processing") - defer require.NoError(os.RemoveAll(file)) + file := filepath.Join(tmpDir, ".processing") ok, err := ExistsProcessingFile(file) require.NoError(err) diff --git a/sql/index/pilosa/driver_test.go b/sql/index/pilosa/driver_test.go index cd6255854..0639ffc56 100644 --- a/sql/index/pilosa/driver_test.go +++ b/sql/index/pilosa/driver_test.go @@ -25,6 +25,8 @@ import ( var ( dockerIsRunning bool dockerCmdOutput string + + tmpDir string ) func init() { @@ -34,6 +36,22 @@ func init() { dockerCmdOutput, dockerIsRunning = string(b), (err == nil) } +func setup(t *testing.T) { + var err error + + tmpDir, err = ioutil.TempDir("", "pilosa") + if err != nil { + t.Fatal(err) + } +} + +func cleanup(t *testing.T) { + err := os.RemoveAll(tmpDir) + if err != nil { + t.Fatal(err) + } +} + func TestID(t *testing.T) { d := &Driver{} @@ -44,11 +62,10 @@ func TestID(t *testing.T) { func TestLoadAll(t *testing.T) { require := require.New(t) - path, err := ioutil.TempDir(os.TempDir(), "indexes-") - require.NoError(err) - defer os.RemoveAll(path) + setup(t) + defer cleanup(t) - d := NewIndexDriver(path) + d := NewIndexDriver(tmpDir) idx1, err := d.Create("db", "table", "id1", makeExpressions("hash1"), nil) require.NoError(err) @@ -91,14 +108,13 @@ func TestSaveAndLoad(t *testing.T) { t.Skipf("Skip TestSaveAndLoad: %s", dockerCmdOutput) } require := require.New(t) + setup(t) + defer cleanup(t) db, table, id := "db_name", "table_name", "index_id" expressions := makeExpressions("lang", "hash") - path, err := ioutil.TempDir("", "indexes") - require.NoError(err) - defer os.RemoveAll(path) - d := NewDriver(path, newClientWithTimeout(200*time.Millisecond)) + d := NewDriver(tmpDir, newClientWithTimeout(200*time.Millisecond)) sqlIdx, err := d.Create(db, table, id, expressions, nil) require.NoError(err) @@ -184,15 +200,13 @@ func TestSaveAndGetAll(t *testing.T) { if !dockerIsRunning { t.Skipf("Skip TestSaveAndGetAll: %s", dockerCmdOutput) } + setup(t) + defer cleanup(t) require := require.New(t) db, table, id := "db_name", "table_name", "index_id" expressions := makeExpressions("lang", "hash") - path, err := ioutil.TempDir("", "indexes") - require.NoError(err) - defer os.RemoveAll(path) - - d := NewDriver(path, newClientWithTimeout(200*time.Millisecond)) + d := NewDriver(tmpDir, newClientWithTimeout(200*time.Millisecond)) sqlIdx, err := d.Create(db, table, id, expressions, nil) require.NoError(err) @@ -218,12 +232,11 @@ func TestSaveAndGetAll(t *testing.T) { func TestLoadCorruptedIndex(t *testing.T) { require := require.New(t) - path, err := ioutil.TempDir("", "indexes") - require.NoError(err) - defer os.RemoveAll(path) + setup(t) + defer cleanup(t) - d := NewIndexDriver(path).(*Driver) - _, err = d.Create("db", "table", "id", nil, nil) + d := NewIndexDriver(tmpDir).(*Driver) + _, err := d.Create("db", "table", "id", nil, nil) require.NoError(err) require.NoError(index.CreateProcessingFile(d.processingFileName("db", "table", "id"))) @@ -242,11 +255,10 @@ func TestDelete(t *testing.T) { t.Skipf("Skip TestDelete: %s", dockerCmdOutput) } require := require.New(t) + setup(t) + defer cleanup(t) db, table, id := "db_name", "table_name", "index_id" - root, err := ioutil.TempDir("", "indexes") - require.NoError(err) - defer os.RemoveAll(root) h1 := sha1.Sum([]byte("lang")) exh1 := sql.ExpressionHash(h1[:]) @@ -256,7 +268,7 @@ func TestDelete(t *testing.T) { expressions := []sql.ExpressionHash{exh1, exh2} - d := NewIndexDriver(root) + d := NewIndexDriver(tmpDir) sqlIdx, err := d.Create(db, table, id, expressions, nil) require.NoError(err) @@ -266,12 +278,8 @@ func TestDelete(t *testing.T) { func TestLoadAllDirectoryDoesNotExist(t *testing.T) { require := require.New(t) - tmpDir, err := ioutil.TempDir("", "pilosa-") - require.NoError(err) - - defer func() { - require.NoError(os.RemoveAll(tmpDir)) - }() + setup(t) + defer cleanup(t) driver := &Driver{root: tmpDir} drivers, err := driver.LoadAll("foo", "bar") @@ -346,15 +354,11 @@ func TestNegateIndex(t *testing.T) { t.Skipf("Skip test: %s", dockerCmdOutput) } require := require.New(t) + setup(t) + defer cleanup(t) db, table := "db_name", "table_name" - path, err := mkdir(os.TempDir(), "indexes") - require.NoError(err) - defer func() { - require.NoError(os.RemoveAll(path)) - }() - - d := NewDriver(path, newClientWithTimeout(200*time.Millisecond)) + d := NewDriver(tmpDir, newClientWithTimeout(200*time.Millisecond)) idx, err := d.Create(db, table, "index_id", makeExpressions("a"), nil) require.NoError(err) @@ -420,16 +424,14 @@ func TestIntersection(t *testing.T) { t.Skipf("Skip TestUnion: %s", dockerCmdOutput) } require := require.New(t) + setup(t) + defer cleanup(t) db, table := "db_name", "table_name" idxLang, expLang := "idx_lang", makeExpressions("lang") idxPath, expPath := "idx_path", makeExpressions("path") - path, err := ioutil.TempDir(os.TempDir(), "indexes") - require.NoError(err) - defer os.RemoveAll(path) - - d := NewDriver(path, newClientWithTimeout(200*time.Millisecond)) + d := NewDriver(tmpDir, newClientWithTimeout(200*time.Millisecond)) sqlIdxLang, err := d.Create(db, table, idxLang, expLang, nil) require.NoError(err) @@ -498,16 +500,14 @@ func TestUnion(t *testing.T) { t.Skipf("Skip TestUnion: %s", dockerCmdOutput) } require := require.New(t) + setup(t) + defer cleanup(t) db, table := "db_name", "table_name" idxLang, expLang := "idx_lang", makeExpressions("lang") idxPath, expPath := "idx_path", makeExpressions("path") - path, err := ioutil.TempDir(os.TempDir(), "indexes") - require.NoError(err) - defer os.RemoveAll(path) - - d := NewDriver(path, newClientWithTimeout(200*time.Millisecond)) + d := NewDriver(tmpDir, newClientWithTimeout(200*time.Millisecond)) sqlIdxLang, err := d.Create(db, table, idxLang, expLang, nil) require.NoError(err) @@ -588,16 +588,14 @@ func TestDifference(t *testing.T) { t.Skipf("Skip TestUnion: %s", dockerCmdOutput) } require := require.New(t) + setup(t) + defer cleanup(t) db, table := "db_name", "table_name" idxLang, expLang := "idx_lang", makeExpressions("lang") idxPath, expPath := "idx_path", makeExpressions("path") - path, err := ioutil.TempDir(os.TempDir(), "indexes") - require.NoError(err) - defer os.RemoveAll(path) - - d := NewDriver(path, newClientWithTimeout(200*time.Millisecond)) + d := NewDriver(tmpDir, newClientWithTimeout(200*time.Millisecond)) sqlIdxLang, err := d.Create(db, table, idxLang, expLang, nil) require.NoError(err) @@ -661,15 +659,13 @@ func TestUnionDiffAsc(t *testing.T) { t.Skipf("Skip TestUnion: %s", dockerCmdOutput) } require := require.New(t) + setup(t) + defer cleanup(t) db, table := "db_name", "table_name" idx, exp := "idx_lang", makeExpressions("lang") - path, err := ioutil.TempDir(os.TempDir(), "indexes") - require.NoError(err) - defer os.RemoveAll(path) - - d := NewDriver(path, newClientWithTimeout(200*time.Millisecond)) + d := NewDriver(tmpDir, newClientWithTimeout(200*time.Millisecond)) sqlIdx, err := d.Create(db, table, idx, exp, nil) require.NoError(err) pilosaIdx, ok := sqlIdx.(*pilosaIndex) @@ -721,15 +717,13 @@ func TestInterRanges(t *testing.T) { t.Skipf("Skip TestUnion: %s", dockerCmdOutput) } require := require.New(t) + setup(t) + defer cleanup(t) db, table := "db_name", "table_name" idx, exp := "idx_lang", makeExpressions("lang") - path, err := ioutil.TempDir(os.TempDir(), "indexes") - require.NoError(err) - defer os.RemoveAll(path) - - d := NewDriver(path, newClientWithTimeout(200*time.Millisecond)) + d := NewDriver(tmpDir, newClientWithTimeout(200*time.Millisecond)) sqlIdx, err := d.Create(db, table, idx, exp, nil) require.NoError(err) pilosaIdx, ok := sqlIdx.(*pilosaIndex) @@ -782,10 +776,9 @@ func setupAscendDescend(t *testing.T) (*pilosaIndex, func()) { db, table, id := "db_name", "table_name", "index_id" expressions := makeExpressions("a", "b") - path, err := mkdir(os.TempDir(), "indexes") - require.NoError(err) + setup(t) - d := NewDriver(path, newClientWithTimeout(200*time.Millisecond)) + d := NewDriver(tmpDir, newClientWithTimeout(200*time.Millisecond)) sqlIdx, err := d.Create(db, table, id, expressions, nil) require.NoError(err) @@ -808,7 +801,7 @@ func setupAscendDescend(t *testing.T) (*pilosaIndex, func()) { require.NoError(err) return sqlIdx.(*pilosaIndex), func() { - require.NoError(os.RemoveAll(path)) + cleanup(t) } } diff --git a/sql/index/pilosa/mapping_test.go b/sql/index/pilosa/mapping_test.go index f2c297052..2eba39ae6 100644 --- a/sql/index/pilosa/mapping_test.go +++ b/sql/index/pilosa/mapping_test.go @@ -2,7 +2,6 @@ package pilosa import ( "encoding/binary" - "os" "path/filepath" "testing" @@ -11,12 +10,10 @@ import ( func TestRowID(t *testing.T) { require := require.New(t) + setup(t) + defer cleanup(t) - root, err := mkdir(os.TempDir(), "mapping_test") - require.NoError(err) - defer os.RemoveAll(root) - - m := newMapping(filepath.Join(root, "id.map")) + m := newMapping(filepath.Join(tmpDir, "id.map")) m.open() defer m.close() @@ -36,12 +33,10 @@ func TestRowID(t *testing.T) { func TestLocation(t *testing.T) { require := require.New(t) + setup(t) + defer cleanup(t) - root, err := mkdir(os.TempDir(), "mapping_test") - require.NoError(err) - defer os.RemoveAll(root) - - m := newMapping(filepath.Join(root, "id.map")) + m := newMapping(filepath.Join(tmpDir, "id.map")) m.open() defer m.close() @@ -54,7 +49,7 @@ func TestLocation(t *testing.T) { } for colID, loc := range cases { - err = m.putLocation("index name", colID, []byte(loc)) + err := m.putLocation("index name", colID, []byte(loc)) require.NoError(err) } @@ -67,12 +62,10 @@ func TestLocation(t *testing.T) { func TestGet(t *testing.T) { require := require.New(t) + setup(t) + defer cleanup(t) - root, err := mkdir(os.TempDir(), "mapping_test") - require.NoError(err) - defer os.RemoveAll(root) - - m := newMapping(filepath.Join(root, "id.map")) + m := newMapping(filepath.Join(tmpDir, "id.map")) m.open() defer m.close() diff --git a/sql/index/pilosalib/driver_test.go b/sql/index/pilosalib/driver_test.go index dd64b6fc5..19371c22c 100644 --- a/sql/index/pilosalib/driver_test.go +++ b/sql/index/pilosalib/driver_test.go @@ -19,6 +19,24 @@ import ( "gopkg.in/src-d/go-mysql-server.v0/test" ) +var tmpDir string + +func setup(t *testing.T) { + var err error + + tmpDir, err = ioutil.TempDir("", "pilosalib") + if err != nil { + t.Fatal(err) + } +} + +func cleanup(t *testing.T) { + err := os.RemoveAll(tmpDir) + if err != nil { + t.Fatal(err) + } +} + func TestID(t *testing.T) { d := &Driver{} @@ -28,12 +46,10 @@ func TestID(t *testing.T) { func TestLoadAll(t *testing.T) { require := require.New(t) + setup(t) + defer cleanup(t) - root, err := ioutil.TempDir("", "indexes") - require.NoError(err) - defer os.RemoveAll(root) - - d := NewDriver(root) + d := NewDriver(tmpDir) idx1, err := d.Create("db", "table", "id1", makeExpressions("hash1"), nil) require.NoError(err) @@ -59,14 +75,13 @@ type logLoc struct { func TestSaveAndLoad(t *testing.T) { require := require.New(t) + setup(t) + defer cleanup(t) db, table, id := "db_name", "table_name", "index_id" expressions := makeExpressions("lang", "hash") - root, err := ioutil.TempDir("", "indexes") - require.NoError(err) - defer os.RemoveAll(root) - d := NewDriver(root) + d := NewDriver(tmpDir) sqlIdx, err := d.Create(db, table, id, expressions, nil) require.NoError(err) @@ -149,14 +164,13 @@ func TestSaveAndLoad(t *testing.T) { func TestSaveAndGetAll(t *testing.T) { require := require.New(t) + setup(t) + defer cleanup(t) db, table, id := "db_name", "table_name", "index_id" expressions := makeExpressions("lang", "hash") - root, err := ioutil.TempDir("", "indexes") - require.NoError(err) - defer os.RemoveAll(root) - d := NewDriver(root) + d := NewDriver(tmpDir) sqlIdx, err := d.Create(db, table, id, expressions, nil) require.NoError(err) @@ -181,12 +195,11 @@ func TestSaveAndGetAll(t *testing.T) { func TestLoadCorruptedIndex(t *testing.T) { require := require.New(t) - root, err := ioutil.TempDir("", "indexes") - require.NoError(err) - defer os.RemoveAll(root) + setup(t) + defer cleanup(t) - d := NewDriver(root) - _, err = d.Create("db", "table", "id", nil, nil) + d := NewDriver(tmpDir) + _, err := d.Create("db", "table", "id", nil, nil) require.NoError(err) require.NoError(index.CreateProcessingFile(d.processingFileName("db", "table", "id"))) @@ -202,11 +215,10 @@ func TestLoadCorruptedIndex(t *testing.T) { func TestDelete(t *testing.T) { require := require.New(t) + setup(t) + defer cleanup(t) db, table, id := "db_name", "table_name", "index_id" - root, err := ioutil.TempDir("", "indexes") - require.NoError(err) - defer os.RemoveAll(root) h1 := sha1.Sum([]byte("lang")) exh1 := sql.ExpressionHash(h1[:]) @@ -216,7 +228,7 @@ func TestDelete(t *testing.T) { expressions := []sql.ExpressionHash{exh1, exh2} - d := NewDriver(root) + d := NewDriver(tmpDir) sqlIdx, err := d.Create(db, table, id, expressions, nil) require.NoError(err) @@ -226,14 +238,10 @@ func TestDelete(t *testing.T) { func TestLoadAllDirectoryDoesNotExist(t *testing.T) { require := require.New(t) - root, err := ioutil.TempDir("", "pilosa-") - require.NoError(err) + setup(t) + defer cleanup(t) - defer func() { - require.NoError(os.RemoveAll(root)) - }() - - driver := NewDriver(root) + driver := NewDriver(tmpDir) indexes, err := driver.LoadAll("foo", "bar") require.NoError(err) require.Len(indexes, 0) @@ -316,16 +324,14 @@ func TestAscendDescendIndex(t *testing.T) { func TestIntersection(t *testing.T) { ctx := sql.NewContext(context.Background()) require := require.New(t) + setup(t) + defer cleanup(t) db, table := "db_name", "table_name" idxLang, expLang := "idx_lang", makeExpressions("lang") idxPath, expPath := "idx_path", makeExpressions("path") - root, err := ioutil.TempDir("", "indexes-") - require.NoError(err) - defer os.RemoveAll(root) - - d := NewDriver(root) + d := NewDriver(tmpDir) sqlIdxLang, err := d.Create(db, table, idxLang, expLang, nil) require.NoError(err) @@ -391,16 +397,14 @@ func TestIntersection(t *testing.T) { func TestUnion(t *testing.T) { require := require.New(t) + setup(t) + defer cleanup(t) db, table := "db_name", "table_name" idxLang, expLang := "idx_lang", makeExpressions("lang") idxPath, expPath := "idx_path", makeExpressions("path") - root, err := ioutil.TempDir("", "indexes") - require.NoError(err) - defer os.RemoveAll(root) - - d := NewDriver(root) + d := NewDriver(tmpDir) sqlIdxLang, err := d.Create(db, table, idxLang, expLang, nil) require.NoError(err) @@ -478,16 +482,14 @@ func TestUnion(t *testing.T) { func TestDifference(t *testing.T) { require := require.New(t) + setup(t) + defer cleanup(t) db, table := "db_name", "table_name" idxLang, expLang := "idx_lang", makeExpressions("lang") idxPath, expPath := "idx_path", makeExpressions("path") - root, err := ioutil.TempDir("", "indexes") - require.NoError(err) - defer os.RemoveAll(root) - - d := NewDriver(root) + d := NewDriver(tmpDir) sqlIdxLang, err := d.Create(db, table, idxLang, expLang, nil) require.NoError(err) @@ -548,15 +550,13 @@ func TestDifference(t *testing.T) { func TestUnionDiffAsc(t *testing.T) { require := require.New(t) + setup(t) + defer cleanup(t) db, table := "db_name", "table_name" idx, exp := "idx_lang", makeExpressions("lang") - root, err := ioutil.TempDir("", "indexes") - require.NoError(err) - defer os.RemoveAll(root) - - d := NewDriver(root) + d := NewDriver(tmpDir) sqlIdx, err := d.Create(db, table, idx, exp, nil) require.NoError(err) pilosaIdx, ok := sqlIdx.(*pilosaIndex) @@ -605,15 +605,13 @@ func TestUnionDiffAsc(t *testing.T) { func TestInterRanges(t *testing.T) { require := require.New(t) + setup(t) + defer cleanup(t) db, table := "db_name", "table_name" idx, exp := "idx_lang", makeExpressions("lang") - root, err := ioutil.TempDir("", "indexes") - require.NoError(err) - defer os.RemoveAll(root) - - d := NewDriver(root) + d := NewDriver(tmpDir) sqlIdx, err := d.Create(db, table, idx, exp, nil) require.NoError(err) pilosaIdx, ok := sqlIdx.(*pilosaIndex) @@ -659,15 +657,12 @@ func TestInterRanges(t *testing.T) { func TestNegateIndex(t *testing.T) { require := require.New(t) + setup(t) + defer cleanup(t) db, table := "db_name", "table_name" - root, err := mkdir(os.TempDir(), "indexes") - require.NoError(err) - defer func() { - require.NoError(os.RemoveAll(root)) - }() - d := NewDriver(root) + d := NewDriver(tmpDir) idx, err := d.Create(db, table, "index_id", makeExpressions("a"), nil) require.NoError(err) @@ -730,15 +725,12 @@ func TestNegateIndex(t *testing.T) { func TestPilosaHolder(t *testing.T) { require := require.New(t) - - path, err := ioutil.TempDir("", "indexes-") - require.NoError(err) - defer os.RemoveAll(path) + setup(t) + defer cleanup(t) h := pilosa.NewHolder() - - h.Path = path - err = h.Open() + h.Path = tmpDir + err := h.Open() require.NoError(err) idx1, err := h.CreateIndexIfNotExists("idx", pilosa.IndexOptions{}) @@ -916,13 +908,12 @@ func (it *testIndexKeyValueIter) Close() error { func setupAscendDescend(t *testing.T) (*pilosaIndex, func()) { t.Helper() require := require.New(t) + setup(t) db, table, id := "db_name", "table_name", "index_id" expressions := makeExpressions("a", "b") - root, err := mkdir(os.TempDir(), "indexes") - require.NoError(err) - d := NewDriver(root) + d := NewDriver(tmpDir) sqlIdx, err := d.Create(db, table, id, expressions, nil) require.NoError(err) @@ -945,7 +936,7 @@ func setupAscendDescend(t *testing.T) (*pilosaIndex, func()) { require.NoError(err) return sqlIdx.(*pilosaIndex), func() { - require.NoError(os.RemoveAll(root)) + cleanup(t) } } diff --git a/sql/index/pilosalib/mapping_test.go b/sql/index/pilosalib/mapping_test.go index 68bf157b2..98bdb7d6f 100644 --- a/sql/index/pilosalib/mapping_test.go +++ b/sql/index/pilosalib/mapping_test.go @@ -2,7 +2,6 @@ package pilosalib import ( "encoding/binary" - "os" "path/filepath" "testing" @@ -11,12 +10,9 @@ import ( func TestRowID(t *testing.T) { require := require.New(t) - - root, err := mkdir(os.TempDir(), "mapping_test") - require.NoError(err) - defer os.RemoveAll(root) - - m := newMapping(filepath.Join(root, "id.map")) + setup(t) + defer cleanup(t) + m := newMapping(filepath.Join(tmpDir, "id.map")) m.open() defer m.close() @@ -36,12 +32,10 @@ func TestRowID(t *testing.T) { func TestLocation(t *testing.T) { require := require.New(t) + setup(t) + defer cleanup(t) - root, err := mkdir(os.TempDir(), "mapping_test") - require.NoError(err) - defer os.RemoveAll(root) - - m := newMapping(filepath.Join(root, "id.map")) + m := newMapping(filepath.Join(tmpDir, "id.map")) m.open() defer m.close() @@ -54,7 +48,7 @@ func TestLocation(t *testing.T) { } for colID, loc := range cases { - err = m.putLocation("index name", colID, []byte(loc)) + err := m.putLocation("index name", colID, []byte(loc)) require.NoError(err) } @@ -67,12 +61,10 @@ func TestLocation(t *testing.T) { func TestGet(t *testing.T) { require := require.New(t) + setup(t) + defer cleanup(t) - root, err := mkdir(os.TempDir(), "mapping_test") - require.NoError(err) - defer os.RemoveAll(root) - - m := newMapping(filepath.Join(root, "id.map")) + m := newMapping(filepath.Join(tmpDir, "id.map")) m.open() defer m.close() From 198abbe52cf04259f910afaa888c49a288aaaed0 Mon Sep 17 00:00:00 2001 From: kuba-- Date: Thu, 2 Aug 2018 11:08:57 +0200 Subject: [PATCH 5/5] Refactor index folder structure. Signed-off-by: kuba-- --- sql/index/config.go | 22 ++--- sql/index/pilosa/driver.go | 110 ++++++++++++------------- sql/index/pilosa/driver_test.go | 6 +- sql/index/pilosalib/driver.go | 124 ++++++++++++++++------------- sql/index/pilosalib/driver_test.go | 4 +- 5 files changed, 141 insertions(+), 125 deletions(-) diff --git a/sql/index/config.go b/sql/index/config.go index 277118b0c..fefa5f791 100644 --- a/sql/index/config.go +++ b/sql/index/config.go @@ -53,8 +53,8 @@ func WriteConfig(w io.Writer, cfg *Config) error { } // WriteConfigFile writes the configuration to file. -func WriteConfigFile(file string, cfg *Config) error { - f, err := os.Create(file) +func WriteConfigFile(path string, cfg *Config) error { + f, err := os.Create(path) if err != nil { return err } @@ -75,9 +75,9 @@ func ReadConfig(r io.Reader) (*Config, error) { return &cfg, err } -// ReadConfigFile reads an configuration from file. -func ReadConfigFile(file string) (*Config, error) { - f, err := os.Open(file) +// ReadConfigFile reads an configuration from file. +func ReadConfigFile(path string) (*Config, error) { + f, err := os.Open(path) if err != nil { return nil, err } @@ -87,8 +87,8 @@ func ReadConfigFile(file string) (*Config, error) { } // CreateProcessingFile creates a file saying whether the index is being created. -func CreateProcessingFile(file string) error { - f, err := os.Create(file) +func CreateProcessingFile(path string) error { + f, err := os.Create(path) if err != nil { return err } @@ -99,13 +99,13 @@ func CreateProcessingFile(file string) error { } // RemoveProcessingFile removes the file that says whether the index is still being created. -func RemoveProcessingFile(file string) error { - return os.Remove(file) +func RemoveProcessingFile(path string) error { + return os.Remove(path) } // ExistsProcessingFile returns whether the processing file exists. -func ExistsProcessingFile(file string) (bool, error) { - _, err := os.Stat(file) +func ExistsProcessingFile(path string) (bool, error) { + _, err := os.Stat(path) if err != nil { if os.IsNotExist(err) { return false, nil diff --git a/sql/index/pilosa/driver.go b/sql/index/pilosa/driver.go index d64fedd31..559cfcaf8 100644 --- a/sql/index/pilosa/driver.go +++ b/sql/index/pilosa/driver.go @@ -4,6 +4,7 @@ import ( "crypto/sha1" "fmt" "io" + "io/ioutil" "os" "path/filepath" "strings" @@ -25,14 +26,14 @@ const ( // FrameNamePrefix the pilosa's frames prefix FrameNamePrefix = "frm" - // ConfigFileExt is the extension of an index config file. - ConfigFileExt = ".cfg" + // ConfigFileName is the name of an index config file. + ConfigFileName = "config.yml" - // ProcessingFileExt is the extension of the lock/processing index file. - ProcessingFileExt = ".processing" + // ProcessingFileName is the name of the lock/processing index file. + ProcessingFileName = ".processing" - // MappingFileExt is the extension of the mapping file. - MappingFileExt = ".map" + // MappingFileName is the name of the mapping file. + MappingFileName = "mapping.db" ) var ( @@ -57,7 +58,7 @@ type Driver struct { // which satisfies sql.IndexDriver interface func NewDriver(root string, client *pilosa.Client) *Driver { return &Driver{ - root: filepath.Join(root, DriverID), + root: root, client: client, } } @@ -74,7 +75,7 @@ func (*Driver) ID() string { // Create a new index. func (d *Driver) Create(db, table, id string, expressions []sql.Expression, config map[string]string) (sql.Index, error) { - _, err := mkdir(d.root, db, table) + _, err := mkdir(d.root, db, table, id) if err != nil { return nil, err } @@ -85,79 +86,86 @@ func (d *Driver) Create(db, table, id string, expressions []sql.Expression, conf } cfg := index.NewConfig(db, table, id, exprs, d.ID(), config) - err = index.WriteConfigFile(d.configFileName(db, table, id), cfg) + err = index.WriteConfigFile(d.configFilePath(db, table, id), cfg) if err != nil { return nil, err } - return newPilosaIndex(d.mappingFileName(db, table, id), d.client, cfg), nil + return newPilosaIndex(d.mappingFilePath(db, table, id), d.client, cfg), nil } // LoadAll loads all indexes for given db and table func (d *Driver) LoadAll(db, table string) ([]sql.Index, error) { - path := filepath.Join(d.root, db, table) - var ( indexes []sql.Index errors []string + root = filepath.Join(d.root, db, table) ) - configfiles, err := filepath.Glob(filepath.Join(path, "*") + ConfigFileExt) + dirs, err := ioutil.ReadDir(root) if err != nil { + if os.IsNotExist(err) { + return indexes, nil + } return nil, err } - - for _, config := range configfiles { - idx, err := d.loadIndex(config) - if err != nil { - if !errCorruptedIndex.Is(err) { - errors = append(errors, err.Error()) + for _, info := range dirs { + if info.IsDir() && !strings.HasPrefix(info.Name(), ".") { + idx, err := d.loadIndex(db, table, info.Name()) + if err != nil { + if !errCorruptedIndex.Is(err) { + errors = append(errors, err.Error()) + } + continue } - continue + indexes = append(indexes, idx) } - - indexes = append(indexes, idx) } + if len(errors) > 0 { return nil, fmt.Errorf(strings.Join(errors, "\n")) } + return indexes, nil } -func (d *Driver) loadIndex(configfile string) (sql.Index, error) { - mapping := strings.Replace(configfile, ConfigFileExt, MappingFileExt, 1) - processing := strings.Replace(configfile, ConfigFileExt, ProcessingFileExt, 1) +func (d *Driver) loadIndex(db, table, id string) (sql.Index, error) { + dir := filepath.Join(d.root, db, table, id) + config := d.configFilePath(db, table, id) + if _, err := os.Stat(config); err != nil { + return nil, errCorruptedIndex.New(dir) + } + + mapping := d.mappingFilePath(db, table, id) + processing := d.processingFilePath(db, table, id) ok, err := index.ExistsProcessingFile(processing) if err != nil { return nil, err } - if ok { log := logrus.WithFields(logrus.Fields{ - "err": err, - "path": configfile, + "err": err, + "db": db, + "table": table, + "id": id, + "dir": dir, }) log.Warn("could not read index file, index is corrupt and will be deleted") - - if err := os.RemoveAll(configfile); err != nil { - log.Warn("unable to remove corrupted index: " + configfile) + if err := os.RemoveAll(dir); err != nil { + log.Warn("unable to remove corrupted index: " + dir) } - if err := os.RemoveAll(processing); err != nil { - log.Warn("unable to remove corrupted index: " + processing) - } - - if err := os.RemoveAll(mapping); err != nil { - log.Warn("unable to remove corrupted index: " + mapping) - } - return nil, errCorruptedIndex.New(configfile) + return nil, errCorruptedIndex.New(dir) } - cfg, err := index.ReadConfigFile(configfile) + cfg, err := index.ReadConfigFile(config) if err != nil { return nil, err } + if cfg.Driver(DriverID) == nil { + return nil, errCorruptedIndex.New(dir) + } idx := newPilosaIndex(mapping, d.client, cfg) return idx, nil @@ -177,7 +185,7 @@ func (d *Driver) Save( return errInvalidIndexType.New(i) } - processingFile := d.processingFileName(idx.Database(), idx.Table(), idx.ID()) + processingFile := d.processingFilePath(idx.Database(), idx.Table(), idx.ID()) if err = index.CreateProcessingFile(processingFile); err != nil { return err } @@ -297,13 +305,7 @@ func (d *Driver) Save( // Delete the index with the given path. func (d *Driver) Delete(idx sql.Index) error { - if err := os.RemoveAll(d.configFileName(idx.Database(), idx.Table(), idx.ID())); err != nil { - return err - } - if err := os.RemoveAll(d.mappingFileName(idx.Database(), idx.Table(), idx.ID())); err != nil { - return err - } - if err := os.RemoveAll(d.processingFileName(idx.Database(), idx.Table(), idx.ID())); err != nil { + if err := os.RemoveAll(filepath.Join(d.root, idx.Database(), idx.Table(), idx.ID())); err != nil { return err } @@ -439,14 +441,14 @@ func mkdir(elem ...string) (string, error) { return path, os.MkdirAll(path, 0750) } -func (d *Driver) configFileName(db, table, id string) string { - return filepath.Join(d.root, db, table, id) + ConfigFileExt +func (d *Driver) configFilePath(db, table, id string) string { + return filepath.Join(d.root, db, table, id, ConfigFileName) } -func (d *Driver) processingFileName(db, table, id string) string { - return filepath.Join(d.root, db, table, id) + ProcessingFileExt +func (d *Driver) processingFilePath(db, table, id string) string { + return filepath.Join(d.root, db, table, id, ProcessingFileName) } -func (d *Driver) mappingFileName(db, table, id string) string { - return filepath.Join(d.root, db, table, id) + MappingFileExt +func (d *Driver) mappingFilePath(db, table, id string) string { + return filepath.Join(d.root, db, table, id, MappingFileName) } diff --git a/sql/index/pilosa/driver_test.go b/sql/index/pilosa/driver_test.go index 0dd21f20e..dc66ff6c1 100644 --- a/sql/index/pilosa/driver_test.go +++ b/sql/index/pilosa/driver_test.go @@ -239,13 +239,13 @@ func TestLoadCorruptedIndex(t *testing.T) { _, err := d.Create("db", "table", "id", nil, nil) require.NoError(err) - require.NoError(index.CreateProcessingFile(d.processingFileName("db", "table", "id"))) + require.NoError(index.CreateProcessingFile(d.processingFilePath("db", "table", "id"))) - _, err = d.loadIndex(d.configFileName("db", "table", "id")) + _, err = d.loadIndex("db", "table", "id") require.Error(err) require.True(errCorruptedIndex.Is(err)) - _, err = os.Stat(d.processingFileName("db", "table", "id")) + _, err = os.Stat(d.processingFilePath("db", "table", "id")) require.Error(err) require.True(os.IsNotExist(err)) } diff --git a/sql/index/pilosalib/driver.go b/sql/index/pilosalib/driver.go index 46cdf8b67..09cc2bee0 100644 --- a/sql/index/pilosalib/driver.go +++ b/sql/index/pilosalib/driver.go @@ -4,13 +4,13 @@ import ( "crypto/sha1" "fmt" "io" + "io/ioutil" "os" "path/filepath" "strings" "time" opentracing "github.com/opentracing/opentracing-go" - "github.com/opentracing/opentracing-go/log" pilosa "github.com/pilosa/pilosa" "github.com/sirupsen/logrus" errors "gopkg.in/src-d/go-errors.v1" @@ -28,14 +28,14 @@ const ( // FieldNamePrefix the pilosa's field prefix FieldNamePrefix = "fld" - // ConfigFileExt is the extension of an index config file. - ConfigFileExt = ".cfg" + // ConfigFileName is the extension of an index config file. + ConfigFileName = "config.yml" - // ProcessingFileExt is the extension of the lock/processing index file. - ProcessingFileExt = ".processing" + // ProcessingFileName is the extension of the lock/processing index file. + ProcessingFileName = ".processing" - // MappingFileExt is the extension of the mapping file. - MappingFileExt = ".map" + // MappingFileName is the extension of the mapping file. + MappingFileName = "mapping.db" ) var ( @@ -70,7 +70,7 @@ type ( // which satisfies sql.IndexDriver interface func NewDriver(root string) *Driver { return &Driver{ - root: filepath.Join(root, DriverID), + root: root, holder: pilosa.NewHolder(), } } @@ -82,28 +82,34 @@ func (*Driver) ID() string { // Create a new index. func (d *Driver) Create(db, table, id string, expressions []sql.Expression, config map[string]string) (sql.Index, error) { - root, err := mkdir(d.root, db, table) + _, err := mkdir(d.root, db, table, id) if err != nil { return nil, err } name := indexName(db, table) + if config == nil { + config = make(map[string]string) + } exprs := make([]string, len(expressions)) for i, e := range expressions { - exprs[i] = e.String() + name := e.String() + + exprs[i] = name + config[fieldName(id, name)] = name } cfg := index.NewConfig(db, table, id, exprs, d.ID(), config) - err = index.WriteConfigFile(d.configFileName(db, table, id), cfg) + err = index.WriteConfigFile(d.configFilePath(db, table, id), cfg) if err != nil { return nil, err } - d.holder.Path = root + d.holder.Path = d.pilosaDirPath(db, table) idx, err := d.holder.CreateIndexIfNotExists(name, pilosa.IndexOptions{}) if err != nil { return nil, err } - mapping := newMapping(d.mappingFileName(db, table, id)) + mapping := newMapping(d.mappingFilePath(db, table, id)) return newPilosaIndex(idx, mapping, cfg), nil } @@ -113,38 +119,40 @@ func (d *Driver) LoadAll(db, table string) ([]sql.Index, error) { var ( indexes []sql.Index errors []string + root = filepath.Join(d.root, db, table) ) - d.holder.Path = filepath.Join(d.root, db, table) + d.holder.Path = d.pilosaDirPath(db, table) err := d.holder.Open() if err != nil { return nil, err } defer d.holder.Close() - configfiles, err := filepath.Glob(filepath.Join(d.holder.Path, "*") + ConfigFileExt) + dirs, err := ioutil.ReadDir(root) if err != nil { + if os.IsNotExist(err) { + return indexes, nil + } return nil, err } - for _, path := range configfiles { - filename := filepath.Base(path) - id := filename[:len(filename)-len(ConfigFileExt)] - idx, err := d.loadIndex(db, table, id) - if err != nil { - if errLoadingIndex.Is(err) { - return nil, err - } - if !errCorruptedIndex.Is(err) { - errors = append(errors, err.Error()) + for _, info := range dirs { + if info.IsDir() && !strings.HasPrefix(info.Name(), ".") { + idx, err := d.loadIndex(db, table, info.Name()) + if err != nil { + if !errCorruptedIndex.Is(err) { + errors = append(errors, err.Error()) + } + continue } - continue + + indexes = append(indexes, idx) } - indexes = append(indexes, idx) } - if len(errors) > 0 { return nil, fmt.Errorf(strings.Join(errors, "\n")) } + return indexes, nil } @@ -155,29 +163,43 @@ func (d *Driver) loadIndex(db, table, id string) (*pilosaIndex, error) { return nil, errLoadingIndex.New(name) } - processing := d.processingFileName(db, table, id) + dir := filepath.Join(d.root, db, table, id) + config := d.configFilePath(db, table, id) + if _, err := os.Stat(config); err != nil { + return nil, errCorruptedIndex.New(dir) + } + + mapping := d.mappingFilePath(db, table, id) + processing := d.processingFilePath(db, table, id) ok, err := index.ExistsProcessingFile(processing) if err != nil { return nil, err } if ok { log := logrus.WithFields(logrus.Fields{ + "err": err, "db": db, "table": table, "id": id, + "dir": dir, }) log.Warn("could not read index file, index is corrupt and will be deleted") - d.removeResources(db, table, id) - return nil, errCorruptedIndex.New(id) + if err := os.RemoveAll(dir); err != nil { + log.Warn("unable to remove corrupted index: " + dir) + } + + return nil, errCorruptedIndex.New(dir) } - cfg, err := index.ReadConfigFile(d.configFileName(db, table, id)) + cfg, err := index.ReadConfigFile(config) if err != nil { return nil, err } + if cfg.Driver(DriverID) == nil { + return nil, errCorruptedIndex.New(dir) + } - mapping := newMapping(d.mappingFileName(db, table, id)) - return newPilosaIndex(idx, mapping, cfg), nil + return newPilosaIndex(idx, newMapping(mapping), cfg), nil } // Save the given index (mapping and bitmap) @@ -190,7 +212,7 @@ func (d *Driver) Save(ctx *sql.Context, i sql.Index, iter sql.IndexKeyValueIter) return errInvalidIndexType.New(i) } - processingFile := d.processingFileName(idx.Database(), idx.Table(), idx.ID()) + processingFile := d.processingFilePath(idx.Database(), idx.Table(), idx.ID()) if err = index.CreateProcessingFile(processingFile); err != nil { return err } @@ -292,13 +314,15 @@ func (d *Driver) Save(ctx *sql.Context, i sql.Index, iter sql.IndexKeyValueIter) // Delete the index with the given path. func (d *Driver) Delete(i sql.Index) error { + if err := os.RemoveAll(filepath.Join(d.root, i.Database(), i.Table(), i.ID())); err != nil { + return err + } + idx, ok := i.(*pilosaIndex) if !ok { return errInvalidIndexType.New(i) } - d.removeResources(idx.Database(), idx.Table(), idx.ID()) - err := idx.index.Open() if err != nil { return err @@ -426,28 +450,18 @@ func mkdir(elem ...string) (string, error) { return path, os.MkdirAll(path, 0750) } -func (d *Driver) configFileName(db, table, id string) string { - return filepath.Join(d.root, db, table, id) + ConfigFileExt +func (d *Driver) pilosaDirPath(db, table string) string { + return filepath.Join(d.root, db, table, "."+DriverID) } -func (d *Driver) processingFileName(db, table, id string) string { - return filepath.Join(d.root, db, table, id) + ProcessingFileExt +func (d *Driver) configFilePath(db, table, id string) string { + return filepath.Join(d.root, db, table, id, ConfigFileName) } -func (d *Driver) mappingFileName(db, table, id string) string { - return filepath.Join(d.root, db, table, id) + MappingFileExt +func (d *Driver) processingFilePath(db, table, id string) string { + return filepath.Join(d.root, db, table, id, ProcessingFileName) } -func (d *Driver) removeResources(db, table, id string) { - if err := os.RemoveAll(d.configFileName(db, table, id)); err != nil { - log.Error(err) - } - - if err := os.RemoveAll(d.processingFileName(db, table, id)); err != nil { - log.Error(err) - } - - if err := os.RemoveAll(d.mappingFileName(db, table, id)); err != nil { - log.Error(err) - } +func (d *Driver) mappingFilePath(db, table, id string) string { + return filepath.Join(d.root, db, table, id, MappingFileName) } diff --git a/sql/index/pilosalib/driver_test.go b/sql/index/pilosalib/driver_test.go index 6e62bcce8..d7075ae60 100644 --- a/sql/index/pilosalib/driver_test.go +++ b/sql/index/pilosalib/driver_test.go @@ -202,13 +202,13 @@ func TestLoadCorruptedIndex(t *testing.T) { _, err := d.Create("db", "table", "id", nil, nil) require.NoError(err) - require.NoError(index.CreateProcessingFile(d.processingFileName("db", "table", "id"))) + require.NoError(index.CreateProcessingFile(d.processingFilePath("db", "table", "id"))) _, err = d.loadIndex("db", "table", "id") require.Error(err) require.True(errCorruptedIndex.Is(err)) - _, err = os.Stat(d.processingFileName("db", "table", "id")) + _, err = os.Stat(d.processingFilePath("db", "table", "id")) require.Error(err) require.True(os.IsNotExist(err)) } pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy