Skip to content
This repository was archived by the owner on Jan 28, 2021. It is now read-only.

Commit 634630f

Browse files
ajnavarroajnavarro
authored
Make mapping per partition (#681)
Make mapping per partition
2 parents e98fa12 + 409e0be commit 634630f

File tree

6 files changed

+140
-71
lines changed

6 files changed

+140
-71
lines changed

‎sql/index/pilosa/driver.go

Lines changed: 54 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,10 @@ const (
3939
// ProcessingFileName is the extension of the lock/processing index file.
4040
ProcessingFileName = ".processing"
4141

42-
// MappingFileName is the extension of the mapping file.
43-
MappingFileName = "mapping.db"
42+
// MappingFileNamePrefix is the prefix in mapping file <prefix>-<mappingKey><extension>
43+
MappingFileNamePrefix = "map"
44+
// MappingFileNameExtension is the extension in mapping file <prefix>-<mappingKey><extension>
45+
MappingFileNameExtension = ".db"
4446
)
4547

4648
const (
@@ -124,7 +126,6 @@ func (d *Driver) Create(
124126
return nil, err
125127
}
126128

127-
mapping := newMapping(d.mappingFilePath(db, table, id))
128129
processingFile := d.processingFilePath(db, table, id)
129130
if err := index.WriteProcessingFile(
130131
processingFile,
@@ -133,7 +134,7 @@ func (d *Driver) Create(
133134
return nil, err
134135
}
135136

136-
return newPilosaIndex(idx, mapping, cfg), nil
137+
return newPilosaIndex(idx, cfg), nil
137138
}
138139

139140
// LoadAll loads all indexes for given db and table
@@ -187,7 +188,6 @@ func (d *Driver) loadIndex(db, table, id string) (*pilosaIndex, error) {
187188
return nil, errCorruptedIndex.New(dir)
188189
}
189190

190-
mapping := d.mappingFilePath(db, table, id)
191191
processing := d.processingFilePath(db, table, id)
192192
ok, err := index.ExistsProcessingFile(processing)
193193
if err != nil {
@@ -213,11 +213,23 @@ func (d *Driver) loadIndex(db, table, id string) (*pilosaIndex, error) {
213213
if err != nil {
214214
return nil, err
215215
}
216-
if cfg.Driver(DriverID) == nil {
216+
cfgDriver := cfg.Driver(DriverID)
217+
if cfgDriver == nil {
217218
return nil, errCorruptedIndex.New(dir)
218219
}
219220

220-
return newPilosaIndex(idx, newMapping(mapping), cfg), nil
221+
pilosaIndex := newPilosaIndex(idx, cfg)
222+
for k, v := range cfgDriver {
223+
if strings.HasPrefix(v, MappingFileNamePrefix) && strings.HasSuffix(v, MappingFileNameExtension) {
224+
path := d.mappingFilePath(db, table, id, k)
225+
if _, err := os.Stat(path); err != nil {
226+
continue
227+
}
228+
pilosaIndex.mapping[k] = newMapping(path)
229+
}
230+
}
231+
232+
return pilosaIndex, nil
221233
}
222234

223235
func (d *Driver) savePartition(
@@ -245,28 +257,33 @@ func (d *Driver) savePartition(
245257
}
246258

247259
rollback := true
248-
if err := idx.mapping.openCreate(true); err != nil {
260+
mk := mappingKey(p)
261+
mapping, ok := idx.mapping[mk]
262+
if !ok {
263+
return 0, errMappingNotFound.New(mk)
264+
}
265+
if err := mapping.openCreate(true); err != nil {
249266
return 0, err
250267
}
251268

252269
defer func() {
253270
if rollback {
254-
idx.mapping.rollback()
271+
mapping.rollback()
255272
} else {
256-
e := d.saveMapping(ctx, idx.mapping, colID, false, b)
273+
e := d.saveMapping(ctx, mapping, colID, false, b)
257274
if e != nil && err == nil {
258275
err = e
259276
}
260277
}
261278

262-
idx.mapping.close()
279+
mapping.close()
263280
kviter.Close()
264281
}()
265282

266283
for colID = 0; err == nil; colID++ {
267284
// commit each batch of objects (pilosa and boltdb)
268285
if colID%sql.IndexBatchSize == 0 && colID != 0 {
269-
if err = d.saveBatch(ctx, idx.mapping, colID, b); err != nil {
286+
if err = d.saveBatch(ctx, mapping, colID, b); err != nil {
270287
return 0, err
271288
}
272289
}
@@ -287,15 +304,15 @@ func (d *Driver) savePartition(
287304
continue
288305
}
289306

290-
rowID, err := idx.mapping.getRowID(field.Name(), values[i])
307+
rowID, err := mapping.getRowID(field.Name(), values[i])
291308
if err != nil {
292309
return 0, err
293310
}
294311

295312
b.bitBatches[i].Add(rowID, colID)
296313
}
297314

298-
err = idx.mapping.putLocation(pilosaIndex.Name(), p, colID, location)
315+
err = mapping.putLocation(pilosaIndex.Name(), colID, location)
299316
if err != nil {
300317
return 0, err
301318
}
@@ -352,6 +369,13 @@ func (d *Driver) Save(
352369
return err
353370
}
354371

372+
cfgPath := d.configFilePath(i.Database(), i.Table(), i.ID())
373+
cfg, err := index.ReadConfigFile(cfgPath)
374+
if err != nil {
375+
return err
376+
}
377+
driverCfg := cfg.Driver(DriverID)
378+
355379
defer iter.Close()
356380
pilosaIndex := idx.index
357381

@@ -382,6 +406,10 @@ func (d *Driver) Save(
382406
wg.Wait()
383407
return err
384408
}
409+
mk := mappingKey(p)
410+
driverCfg[mk] = mappingFileName(mk)
411+
mapping := newMapping(d.mappingFilePath(idx.Database(), idx.Table(), idx.ID(), mk))
412+
idx.mapping[mk] = mapping
385413

386414
wg.Add(1)
387415

@@ -417,6 +445,9 @@ func (d *Driver) Save(
417445
if len(errors) > 0 {
418446
return errors[0]
419447
}
448+
if err = index.WriteConfigFile(cfgPath, cfg); err != nil {
449+
return err
450+
}
420451

421452
logrus.WithFields(logrus.Fields{
422453
"duration": time.Since(start),
@@ -469,6 +500,8 @@ func (d *Driver) Delete(i sql.Index, partitions sql.PartitionIter) error {
469500
return err
470501
}
471502
}
503+
mk := mappingKey(p)
504+
delete(idx.mapping, mk)
472505
}
473506

474507
return partitions.Close()
@@ -581,8 +614,13 @@ func (d *Driver) processingFilePath(db, table, id string) string {
581614
return filepath.Join(d.root, db, table, id, ProcessingFileName)
582615
}
583616

584-
func (d *Driver) mappingFilePath(db, table, id string) string {
585-
return filepath.Join(d.root, db, table, id, MappingFileName)
617+
func mappingFileName(key string) string {
618+
h := sha1.New()
619+
io.WriteString(h, key)
620+
return fmt.Sprintf("%s-%x%s", MappingFileNamePrefix, h.Sum(nil), MappingFileNameExtension)
621+
}
622+
func (d *Driver) mappingFilePath(db, table, id string, key string) string {
623+
return filepath.Join(d.root, db, table, id, mappingFileName(key))
586624
}
587625

588626
func (d *Driver) newPilosaIndex(db, table string) (*pilosa.Index, error) {

‎sql/index/pilosa/index.go

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ var (
5959
// pilosaIndex is an pilosa implementation of sql.Index interface
6060
type pilosaIndex struct {
6161
index *concurrentPilosaIndex
62-
mapping *mapping
62+
mapping map[string]*mapping
6363
cancel context.CancelFunc
6464
wg sync.WaitGroup
6565

@@ -70,7 +70,7 @@ type pilosaIndex struct {
7070
checksum string
7171
}
7272

73-
func newPilosaIndex(idx *pilosa.Index, mapping *mapping, cfg *index.Config) *pilosaIndex {
73+
func newPilosaIndex(idx *pilosa.Index, cfg *index.Config) *pilosaIndex {
7474
var checksum string
7575
for _, c := range cfg.Drivers {
7676
if ch, ok := c[sql.ChecksumKey]; ok {
@@ -85,7 +85,7 @@ func newPilosaIndex(idx *pilosa.Index, mapping *mapping, cfg *index.Config) *pil
8585
table: cfg.Table,
8686
id: cfg.ID,
8787
expressions: cfg.Expressions,
88-
mapping: mapping,
88+
mapping: make(map[string]*mapping),
8989
checksum: checksum,
9090
}
9191
}
@@ -116,15 +116,21 @@ func (idx *pilosaIndex) Get(keys ...interface{}) (sql.IndexLookup, error) {
116116

117117
// Has checks if the given key is present in the index mapping
118118
func (idx *pilosaIndex) Has(p sql.Partition, key ...interface{}) (bool, error) {
119-
if err := idx.mapping.open(); err != nil {
119+
mk := mappingKey(p)
120+
m, ok := idx.mapping[mk]
121+
if !ok {
122+
return false, errMappingNotFound.New(mk)
123+
}
124+
125+
if err := m.open(); err != nil {
120126
return false, err
121127
}
122-
defer idx.mapping.close()
128+
defer m.close()
123129

124130
for i, expr := range idx.expressions {
125131
name := fieldName(idx.ID(), expr, p)
126132

127-
val, err := idx.mapping.get(name, key[i])
133+
val, err := m.get(name, key[i])
128134
if err != nil || val == nil {
129135
return false, err
130136
}

‎sql/index/pilosa/iterator.go

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import (
55

66
"github.com/sirupsen/logrus"
77
bolt "go.etcd.io/bbolt"
8-
"gopkg.in/src-d/go-mysql-server.v0/sql"
98
)
109

1110
type locationValueIter struct {
@@ -32,7 +31,6 @@ type indexValueIter struct {
3231
total uint64
3332
bits []uint64
3433
mapping *mapping
35-
partition sql.Partition
3634
indexName string
3735

3836
// share transaction and bucket on all getLocation calls
@@ -47,7 +45,7 @@ func (it *indexValueIter) Next() ([]byte, error) {
4745
return nil, err
4846
}
4947

50-
bucket, err := it.mapping.getBucket(it.indexName, it.partition, false)
48+
bucket, err := it.mapping.getBucket(it.indexName, false)
5149
if err != nil {
5250
_ = it.Close()
5351
return nil, err

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy