Skip to content
This repository was archived by the owner on Jan 28, 2021. It is now read-only.

Commit f813628

Browse files
ajnavarroajnavarro
authored
Merge pull request #308 from kuba--/pilosa-lib
Pilosa index driver as library
2 parents cf475f6 + 198abbe commit f813628

17 files changed

+3373
-185
lines changed

‎.travis.yml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,9 @@ before_install:
2323
- docker ps -a
2424

2525
install:
26-
- go get -u github.com/pilosa/go-pilosa
27-
- cd "$GOPATH/src/github.com/pilosa/go-pilosa" && git checkout v0.9.0 && cd "$TRAVIS_BUILD_DIR"
26+
- go get -u github.com/golang/dep/cmd/dep
27+
- touch Gopkg.toml
28+
- dep ensure -v -add "github.com/pilosa/go-pilosa@0.9.0" "github.com/pilosa/pilosa@1.0.2"
2829
- make dependencies
2930

3031
script:

‎server/context.go

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -60,11 +60,7 @@ func (s *SessionManager) NewContext(conn *mysql.Conn) (*sql.Context, DoneFunc, e
6060
sess := s.sessions[conn.ConnectionID]
6161
s.mu.Unlock()
6262
context := sql.NewContext(ctx, sql.WithSession(sess), sql.WithTracer(s.tracer))
63-
id, err := uuid.NewV4()
64-
if err != nil {
65-
cancel()
66-
return nil, nil, err
67-
}
63+
id := uuid.NewV4()
6864

6965
s.mu.Lock()
7066
s.sessionContexts[conn.ConnectionID] = append(s.sessionContexts[conn.ConnectionID], id)

‎sql/index/config.go

Lines changed: 13 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,10 @@ import (
44
"io"
55
"io/ioutil"
66
"os"
7-
"path/filepath"
87

98
yaml "gopkg.in/yaml.v2"
109
)
1110

12-
const (
13-
// ConfigFileName is the name of an index config file.
14-
ConfigFileName = "config.yml"
15-
// ProcessingFileName is the name of the processing index file.
16-
ProcessingFileName = ".processing"
17-
)
18-
1911
// Config represents index configuration
2012
type Config struct {
2113
DB string
@@ -60,9 +52,8 @@ func WriteConfig(w io.Writer, cfg *Config) error {
6052
return err
6153
}
6254

63-
// WriteConfigFile writes the configuration to dir/config.yml file.
64-
func WriteConfigFile(dir string, cfg *Config) error {
65-
path := filepath.Join(dir, ConfigFileName)
55+
// WriteConfigFile writes the configuration to file.
56+
func WriteConfigFile(path string, cfg *Config) error {
6657
f, err := os.Create(path)
6758
if err != nil {
6859
return err
@@ -84,9 +75,8 @@ func ReadConfig(r io.Reader) (*Config, error) {
8475
return &cfg, err
8576
}
8677

87-
// ReadConfigFile reads an configuration from dir/config.yml file.
88-
func ReadConfigFile(dir string) (*Config, error) {
89-
path := filepath.Join(dir, ConfigFileName)
78+
// ReadConfigFile reads an configuration from file.
79+
func ReadConfigFile(path string) (*Config, error) {
9080
f, err := os.Open(path)
9181
if err != nil {
9282
return nil, err
@@ -96,10 +86,9 @@ func ReadConfigFile(dir string) (*Config, error) {
9686
return ReadConfig(f)
9787
}
9888

99-
// CreateProcessingFile creates a file inside the directory saying whether
100-
// the index is being created.
101-
func CreateProcessingFile(dir string) error {
102-
f, err := os.Create(filepath.Join(dir, ProcessingFileName))
89+
// CreateProcessingFile creates a file saying whether the index is being created.
90+
func CreateProcessingFile(path string) error {
91+
f, err := os.Create(path)
10392
if err != nil {
10493
return err
10594
}
@@ -109,16 +98,14 @@ func CreateProcessingFile(dir string) error {
10998
return nil
11099
}
111100

112-
// RemoveProcessingFile removes the file that says whether the index is still
113-
// being created.
114-
func RemoveProcessingFile(dir string) error {
115-
return os.Remove(filepath.Join(dir, ProcessingFileName))
101+
// RemoveProcessingFile removes the file that says whether the index is still being created.
102+
func RemoveProcessingFile(path string) error {
103+
return os.Remove(path)
116104
}
117105

118-
// ExistsProcessingFile returns whether the processing file exists inside an
119-
// index directory.
120-
func ExistsProcessingFile(dir string) (bool, error) {
121-
_, err := os.Stat(filepath.Join(dir, ProcessingFileName))
106+
// ExistsProcessingFile returns whether the processing file exists.
107+
func ExistsProcessingFile(path string) (bool, error) {
108+
_, err := os.Stat(path)
122109
if err != nil {
123110
if os.IsNotExist(err) {
124111
return false, nil

‎sql/index/config_test.go

Lines changed: 19 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,17 @@ import (
1111

1212
func TestConfig(t *testing.T) {
1313
require := require.New(t)
14+
tmpDir, err := ioutil.TempDir("", "index")
15+
require.NoError(err)
16+
defer func() { require.NoError(os.RemoveAll(tmpDir)) }()
1417

18+
driver := "driver"
1519
db, table, id := "db_name", "table_name", "index_id"
16-
path := filepath.Join(os.TempDir(), db, table, id)
17-
err := os.MkdirAll(path, 0750)
18-
20+
dir := filepath.Join(tmpDir, driver)
21+
subdir := filepath.Join(dir, db, table)
22+
err = os.MkdirAll(subdir, 0750)
1923
require.NoError(err)
20-
defer os.RemoveAll(path)
24+
file := filepath.Join(subdir, id+".cfg")
2125

2226
cfg1 := NewConfig(
2327
db,
@@ -31,36 +35,35 @@ func TestConfig(t *testing.T) {
3135
},
3236
)
3337

34-
err = WriteConfigFile(path, cfg1)
38+
err = WriteConfigFile(file, cfg1)
3539
require.NoError(err)
3640

37-
cfg2, err := ReadConfigFile(path)
41+
cfg2, err := ReadConfigFile(file)
3842
require.NoError(err)
3943
require.Equal(cfg1, cfg2)
4044
}
4145

4246
func TestProcessingFile(t *testing.T) {
4347
require := require.New(t)
44-
45-
dir, err := ioutil.TempDir(os.TempDir(), "processing-file")
48+
tmpDir, err := ioutil.TempDir("", "index")
4649
require.NoError(err)
47-
defer func() {
48-
require.NoError(os.RemoveAll(dir))
49-
}()
50+
defer func() { require.NoError(os.RemoveAll(tmpDir)) }()
51+
52+
file := filepath.Join(tmpDir, ".processing")
5053

51-
ok, err := ExistsProcessingFile(dir)
54+
ok, err := ExistsProcessingFile(file)
5255
require.NoError(err)
5356
require.False(ok)
5457

55-
require.NoError(CreateProcessingFile(dir))
58+
require.NoError(CreateProcessingFile(file))
5659

57-
ok, err = ExistsProcessingFile(dir)
60+
ok, err = ExistsProcessingFile(file)
5861
require.NoError(err)
5962
require.True(ok)
6063

61-
require.NoError(RemoveProcessingFile(dir))
64+
require.NoError(RemoveProcessingFile(file))
6265

63-
ok, err = ExistsProcessingFile(dir)
66+
ok, err = ExistsProcessingFile(file)
6467
require.NoError(err)
6568
require.False(ok)
6669
}

‎sql/index/pilosa/driver.go

Lines changed: 68 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"crypto/sha1"
55
"fmt"
66
"io"
7+
"io/ioutil"
78
"os"
89
"path/filepath"
910
"strings"
@@ -24,6 +25,15 @@ const (
2425
IndexNamePrefix = "idx"
2526
// FrameNamePrefix the pilosa's frames prefix
2627
FrameNamePrefix = "frm"
28+
29+
// ConfigFileName is the name of an index config file.
30+
ConfigFileName = "config.yml"
31+
32+
// ProcessingFileName is the name of the lock/processing index file.
33+
ProcessingFileName = ".processing"
34+
35+
// MappingFileName is the name of the mapping file.
36+
MappingFileName = "mapping.db"
2737
)
2838

2939
var (
@@ -65,7 +75,7 @@ func (*Driver) ID() string {
6575

6676
// Create a new index.
6777
func (d *Driver) Create(db, table, id string, expressions []sql.Expression, config map[string]string) (sql.Index, error) {
68-
path, err := mkdir(d.root, db, table, id)
78+
_, err := mkdir(d.root, db, table, id)
6979
if err != nil {
7080
return nil, err
7181
}
@@ -76,79 +86,88 @@ func (d *Driver) Create(db, table, id string, expressions []sql.Expression, conf
7686
}
7787

7888
cfg := index.NewConfig(db, table, id, exprs, d.ID(), config)
79-
err = index.WriteConfigFile(path, cfg)
89+
err = index.WriteConfigFile(d.configFilePath(db, table, id), cfg)
8090
if err != nil {
8191
return nil, err
8292
}
8393

84-
return newPilosaIndex(path, d.client, cfg), nil
94+
return newPilosaIndex(d.mappingFilePath(db, table, id), d.client, cfg), nil
8595
}
8696

8797
// LoadAll loads all indexes for given db and table
8898
func (d *Driver) LoadAll(db, table string) ([]sql.Index, error) {
89-
root := filepath.Join(d.root, db, table)
90-
9199
var (
92100
indexes []sql.Index
93101
errors []string
94-
err error
102+
root = filepath.Join(d.root, db, table)
95103
)
96-
filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
97-
if err != nil {
98-
if path != root || !os.IsNotExist(err) {
99-
errors = append(errors, err.Error())
100-
}
101-
return filepath.SkipDir
102-
}
103104

104-
if info.IsDir() && path != root && info.Name() != "." && info.Name() != ".." {
105-
idx, err := d.loadIndex(path)
105+
dirs, err := ioutil.ReadDir(root)
106+
if err != nil {
107+
if os.IsNotExist(err) {
108+
return indexes, nil
109+
}
110+
return nil, err
111+
}
112+
for _, info := range dirs {
113+
if info.IsDir() && !strings.HasPrefix(info.Name(), ".") {
114+
idx, err := d.loadIndex(db, table, info.Name())
106115
if err != nil {
107116
if !errCorruptedIndex.Is(err) {
108117
errors = append(errors, err.Error())
109118
}
110-
111-
return filepath.SkipDir
119+
continue
112120
}
113121

114122
indexes = append(indexes, idx)
115123
}
116-
117-
return nil
118-
})
124+
}
119125

120126
if len(errors) > 0 {
121-
err = fmt.Errorf(strings.Join(errors, "\n"))
127+
return nil, fmt.Errorf(strings.Join(errors, "\n"))
122128
}
123-
return indexes, err
129+
130+
return indexes, nil
124131
}
125132

126-
func (d *Driver) loadIndex(path string) (sql.Index, error) {
127-
ok, err := index.ExistsProcessingFile(path)
133+
func (d *Driver) loadIndex(db, table, id string) (sql.Index, error) {
134+
dir := filepath.Join(d.root, db, table, id)
135+
config := d.configFilePath(db, table, id)
136+
if _, err := os.Stat(config); err != nil {
137+
return nil, errCorruptedIndex.New(dir)
138+
}
139+
140+
mapping := d.mappingFilePath(db, table, id)
141+
processing := d.processingFilePath(db, table, id)
142+
ok, err := index.ExistsProcessingFile(processing)
128143
if err != nil {
129144
return nil, err
130145
}
131-
132146
if ok {
133147
log := logrus.WithFields(logrus.Fields{
134-
"err": err,
135-
"path": path,
148+
"err": err,
149+
"db": db,
150+
"table": table,
151+
"id": id,
152+
"dir": dir,
136153
})
137154
log.Warn("could not read index file, index is corrupt and will be deleted")
138-
139-
if err := os.RemoveAll(path); err != nil {
140-
log.Warn("unable to remove folder of corrupted index")
155+
if err := os.RemoveAll(dir); err != nil {
156+
log.Warn("unable to remove corrupted index: " + dir)
141157
}
142158

143-
return nil, errCorruptedIndex.New(path)
159+
return nil, errCorruptedIndex.New(dir)
144160
}
145161

146-
cfg, err := index.ReadConfigFile(path)
162+
cfg, err := index.ReadConfigFile(config)
147163
if err != nil {
148164
return nil, err
149165
}
166+
if cfg.Driver(DriverID) == nil {
167+
return nil, errCorruptedIndex.New(dir)
168+
}
150169

151-
idx := newPilosaIndex(path, d.client, cfg)
170+
idx := newPilosaIndex(mapping, d.client, cfg)
152171
return idx, nil
153172
}
154173

@@ -166,12 +185,8 @@ func (d *Driver) Save(
166185
return errInvalidIndexType.New(i)
167186
}
168187

169-
path, err := mkdir(d.root, i.Database(), i.Table(), i.ID())
170-
if err != nil {
171-
return err
172-
}
173-
174-
if err = index.CreateProcessingFile(path); err != nil {
188+
processingFile := d.processingFilePath(idx.Database(), idx.Table(), idx.ID())
189+
if err = index.CreateProcessingFile(processingFile); err != nil {
175190
return err
176191
}
177192

@@ -285,13 +300,12 @@ func (d *Driver) Save(
285300
"id": i.ID(),
286301
}).Debugf("finished pilosa indexing")
287302

288-
return index.RemoveProcessingFile(path)
303+
return index.RemoveProcessingFile(processingFile)
289304
}
290305

291306
// Delete the index with the given path.
292307
func (d *Driver) Delete(idx sql.Index) error {
293-
path := filepath.Join(d.root, idx.Database(), idx.Table(), idx.ID())
294-
if err := os.RemoveAll(path); err != nil {
308+
if err := os.RemoveAll(filepath.Join(d.root, idx.Database(), idx.Table(), idx.ID())); err != nil {
295309
return err
296310
}
297311

@@ -426,3 +440,15 @@ func mkdir(elem ...string) (string, error) {
426440
path := filepath.Join(elem...)
427441
return path, os.MkdirAll(path, 0750)
428442
}
443+
444+
func (d *Driver) configFilePath(db, table, id string) string {
445+
return filepath.Join(d.root, db, table, id, ConfigFileName)
446+
}
447+
448+
func (d *Driver) processingFilePath(db, table, id string) string {
449+
return filepath.Join(d.root, db, table, id, ProcessingFileName)
450+
}
451+
452+
func (d *Driver) mappingFilePath(db, table, id string) string {
453+
return filepath.Join(d.root, db, table, id, MappingFileName)
454+
}

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy