Skip to content
This repository was archived by the owner on Jan 28, 2021. It is now read-only.

Commit 345af2a

Browse files
erizocosmicoerizocosmico
committed
sql: implement struct types
This PR implements struct types, which will be serialized to JSON before being sent to the mysql client using the mysql wire proto. Internally, structs are just `map[string]interface{}`, but they can actually be anything that's convertible to that, because of the way the Convert method of the struct type is implemented. That means, the result of an UDF or a table that returns a struct may be an actual Go struct and it will then be transformed into the internal map. It does have a penalty, though, because structs require encoding to JSON and then decoding into `map[string]interface{}`. Structs have a schema, identical to a table schema, except their `Source` will always be empty. Resolution of columns has also been slightly change in order to resolve getting fields from structs using the `.` operator, which required some trade-offs in some rules, such as not erroring anymore in `qualify_columns` when the table is not found. That error was delegated to `resolve_columns` in order to make resolution possible, as the syntax is a bit ambiguous. The advantage of using dot is the fact that no changes have to be made to the parser in order for it to work. Signed-off-by: Miguel Molina <miguel@erizocosmi.co>
1 parent ab5656f commit 345af2a

File tree

10 files changed

+511
-17
lines changed

10 files changed

+511
-17
lines changed

‎engine_test.go

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2519,6 +2519,64 @@ func TestGenerators(t *testing.T) {
25192519
}
25202520
}
25212521

2522+
var structQueries = []struct {
2523+
query string
2524+
expected []sql.Row
2525+
}{
2526+
{
2527+
`SELECT s.i, t.s.t FROM t ORDER BY s.i`,
2528+
[]sql.Row{
2529+
{int64(1), "first"},
2530+
{int64(2), "second"},
2531+
{int64(3), "third"},
2532+
},
2533+
},
2534+
{
2535+
`SELECT s.i, s.t FROM t ORDER BY s.i`,
2536+
[]sql.Row{
2537+
{int64(1), "first"},
2538+
{int64(2), "second"},
2539+
{int64(3), "third"},
2540+
},
2541+
},
2542+
{
2543+
`SELECT s.i, COUNT(*) FROM t GROUP BY s.i`,
2544+
[]sql.Row{
2545+
{int64(1), int64(1)},
2546+
{int64(2), int64(1)},
2547+
{int64(3), int64(1)},
2548+
},
2549+
},
2550+
}
2551+
2552+
func TestStructs(t *testing.T) {
2553+
schema := sql.Schema{
2554+
{Name: "i", Type: sql.Int64},
2555+
{Name: "t", Type: sql.Text},
2556+
}
2557+
table := mem.NewPartitionedTable("t", sql.Schema{
2558+
{Name: "s", Type: sql.Struct(schema), Source: "t"},
2559+
}, testNumPartitions)
2560+
2561+
insertRows(
2562+
t, table,
2563+
sql.NewRow(map[string]interface{}{"i": int64(1), "t": "first"}),
2564+
sql.NewRow(map[string]interface{}{"i": int64(2), "t": "second"}),
2565+
sql.NewRow(map[string]interface{}{"i": int64(3), "t": "third"}),
2566+
)
2567+
2568+
db := mem.NewDatabase("db")
2569+
db.AddTable("t", table)
2570+
2571+
catalog := sql.NewCatalog()
2572+
catalog.AddDatabase(db)
2573+
e := sqle.New(catalog, analyzer.NewDefault(catalog), new(sqle.Config))
2574+
2575+
for _, q := range structQueries {
2576+
testQuery(t, e, q.query, q.expected)
2577+
}
2578+
}
2579+
25222580
func insertRows(t *testing.T, table sql.Inserter, rows ...sql.Row) {
25232581
t.Helper()
25242582

‎sql/analyzer/resolve_columns.go

Lines changed: 87 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,10 @@ import (
55
"sort"
66
"strings"
77

8-
"gopkg.in/src-d/go-errors.v1"
9-
"github.com/src-d/go-mysql-server/internal/similartext"
108
"github.com/src-d/go-mysql-server/sql"
119
"github.com/src-d/go-mysql-server/sql/expression"
1210
"github.com/src-d/go-mysql-server/sql/plan"
11+
"gopkg.in/src-d/go-errors.v1"
1312
"gopkg.in/src-d/go-vitess.v1/vt/sqlparser"
1413
)
1514

@@ -147,17 +146,24 @@ func qualifyExpression(
147146
return col, nil
148147
}
149148

150-
name, table := strings.ToLower(col.Name()), strings.ToLower(col.Table())
149+
name, tableName := strings.ToLower(col.Name()), strings.ToLower(col.Table())
151150
availableTables := dedupStrings(columns[name])
152-
if table != "" {
153-
table, ok := tables[table]
151+
if tableName != "" {
152+
table, ok := tables[tableName]
154153
if !ok {
155-
if len(tables) == 0 {
156-
return nil, sql.ErrTableNotFound.New(col.Table())
154+
// If the table does not exist but the column does, it may be a
155+
// struct field access.
156+
if columnExists(columns, tableName) {
157+
return expression.NewUnresolvedField(
158+
expression.NewUnresolvedColumn(col.Table()),
159+
col.Name(),
160+
), nil
157161
}
158162

159-
similar := similartext.FindFromMap(tables, col.Table())
160-
return nil, sql.ErrTableNotFound.New(col.Table() + similar)
163+
// If it cannot be resolved, then pass along and let it fail
164+
// somewhere else. Maybe we're missing some steps before this
165+
// can be resolved.
166+
return col, nil
161167
}
162168

163169
// If the table exists but it's not available for this node it
@@ -207,6 +213,15 @@ func qualifyExpression(
207213
}
208214
}
209215

216+
func columnExists(columns map[string][]string, col string) bool {
217+
for c := range columns {
218+
if strings.ToLower(c) == strings.ToLower(col) {
219+
return true
220+
}
221+
}
222+
return false
223+
}
224+
210225
func getNodeAvailableColumns(n sql.Node) map[string][]string {
211226
var columns = make(map[string][]string)
212227
getColumnsInNodes(n.Children(), columns)
@@ -369,7 +384,23 @@ func resolveColumnExpression(
369384
return &deferredColumn{uc}, nil
370385
default:
371386
if table != "" {
372-
return nil, ErrColumnTableNotFound.New(e.Table(), e.Name())
387+
if isStructField(uc, columns) {
388+
return expression.NewUnresolvedField(
389+
expression.NewUnresolvedColumn(uc.Table()),
390+
uc.Name(),
391+
), nil
392+
}
393+
394+
// If we manage to find any column with the given table, it's because
395+
// the column does not exist.
396+
for col := range columns {
397+
if col.table == table {
398+
return nil, ErrColumnTableNotFound.New(e.Table(), e.Name())
399+
}
400+
}
401+
402+
// In any other case, it's the table the one that does not exist.
403+
return nil, sql.ErrTableNotFound.New(e.Table())
373404
}
374405

375406
return nil, ErrColumnNotFound.New(e.Name())
@@ -385,6 +416,52 @@ func resolveColumnExpression(
385416
), nil
386417
}
387418

419+
func isStructField(c column, columns map[tableCol]indexedCol) bool {
420+
for _, col := range columns {
421+
if strings.ToLower(c.Table()) == strings.ToLower(col.Name) &&
422+
sql.Field(col.Type, c.Name()) != nil {
423+
return true
424+
}
425+
}
426+
return false
427+
}
428+
429+
var errFieldNotFound = errors.NewKind("field %s not found on struct %s")
430+
431+
func resolveStructFields(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) {
432+
span, _ := ctx.Span("resolve_struct_fields")
433+
defer span.Finish()
434+
435+
return n.TransformUp(func(n sql.Node) (sql.Node, error) {
436+
if n.Resolved() {
437+
return n, nil
438+
}
439+
440+
expressioner, ok := n.(sql.Expressioner)
441+
if !ok {
442+
return n, nil
443+
}
444+
445+
return expressioner.TransformExpressions(func(e sql.Expression) (sql.Expression, error) {
446+
f, ok := e.(*expression.UnresolvedField)
447+
if !ok {
448+
return e, nil
449+
}
450+
451+
if !f.Struct.Resolved() {
452+
return e, nil
453+
}
454+
455+
field := sql.Field(f.Struct.Type(), f.Name)
456+
if field == nil {
457+
return nil, errFieldNotFound.New(f.Name, f.Struct)
458+
}
459+
460+
return expression.NewGetStructField(f.Struct, f.Name), nil
461+
})
462+
})
463+
}
464+
388465
// resolveGroupingColumns reorders the aggregation in a groupby so aliases
389466
// defined in it can be resolved in the grouping of the groupby. To do so,
390467
// all aliases are pushed down to a projection node under the group by.

‎sql/analyzer/resolve_columns_test.go

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -186,14 +186,24 @@ func TestQualifyColumns(t *testing.T) {
186186

187187
node = plan.NewProject(
188188
[]sql.Expression{
189-
expression.NewUnresolvedQualifiedColumn("foo", "i"),
189+
expression.NewUnresolvedQualifiedColumn("i", "some_field"),
190190
},
191-
plan.NewTableAlias("a", plan.NewResolvedTable(table)),
191+
plan.NewResolvedTable(table),
192192
)
193193

194-
_, err = f.Apply(sql.NewEmptyContext(), nil, node)
195-
require.Error(err)
196-
require.True(sql.ErrTableNotFound.Is(err))
194+
expected = plan.NewProject(
195+
[]sql.Expression{
196+
expression.NewUnresolvedField(
197+
expression.NewUnresolvedColumn("i"),
198+
"some_field",
199+
),
200+
},
201+
plan.NewResolvedTable(table),
202+
)
203+
204+
result, err = f.Apply(sql.NewEmptyContext(), nil, node)
205+
require.NoError(err)
206+
require.Equal(expected, result)
197207

198208
node = plan.NewProject(
199209
[]sql.Expression{

‎sql/analyzer/rules.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ var DefaultRules = []Rule{
1212
{"resolve_grouping_columns", resolveGroupingColumns},
1313
{"qualify_columns", qualifyColumns},
1414
{"resolve_columns", resolveColumns},
15+
{"resolve_struct_fields", resolveStructFields},
1516
{"resolve_database", resolveDatabase},
1617
{"resolve_star", resolveStar},
1718
{"resolve_functions", resolveFunctions},

‎sql/expression/get_field.go

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,3 +128,74 @@ func (f *GetSessionField) String() string { return "@@" + f.name }
128128
func (f *GetSessionField) TransformUp(fn sql.TransformExprFunc) (sql.Expression, error) {
129129
return fn(f)
130130
}
131+
132+
// GetStructField is an expression to get a field from a struct column.
133+
type GetStructField struct {
134+
Struct sql.Expression
135+
Name string
136+
}
137+
138+
// NewGetStructField creates a new GetStructField expression.
139+
func NewGetStructField(s sql.Expression, fieldName string) *GetStructField {
140+
return &GetStructField{s, fieldName}
141+
}
142+
143+
// Children implements the Expression interface.
144+
func (p *GetStructField) Children() []sql.Expression {
145+
return []sql.Expression{p.Struct}
146+
}
147+
148+
// Resolved implements the Expression interface.
149+
func (p *GetStructField) Resolved() bool {
150+
return p.Struct.Resolved()
151+
}
152+
153+
func (p *GetStructField) column() *sql.Column {
154+
return sql.Field(p.Struct.Type(), p.Name)
155+
}
156+
157+
// IsNullable returns whether the field is nullable or not.
158+
func (p *GetStructField) IsNullable() bool {
159+
return p.Struct.IsNullable() || p.column().Nullable
160+
}
161+
162+
// Type returns the type of the field.
163+
func (p *GetStructField) Type() sql.Type {
164+
return p.column().Type
165+
}
166+
167+
// Eval implements the Expression interface.
168+
func (p *GetStructField) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) {
169+
s, err := p.Struct.Eval(ctx, row)
170+
if err != nil {
171+
return nil, err
172+
}
173+
174+
if s == nil {
175+
return nil, nil
176+
}
177+
178+
s, err = p.Struct.Type().Convert(s)
179+
if err != nil {
180+
return nil, err
181+
}
182+
183+
if val, ok := s.(map[string]interface{})[p.Name]; ok {
184+
return p.Type().Convert(val)
185+
}
186+
187+
return nil, nil
188+
}
189+
190+
// TransformUp implements the Expression interface.
191+
func (p *GetStructField) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) {
192+
s, err := p.Struct.TransformUp(f)
193+
if err != nil {
194+
return nil, err
195+
}
196+
return f(NewGetStructField(s, p.Name))
197+
}
198+
199+
func (p *GetStructField) String() string {
200+
return fmt.Sprintf("%s.%s", p.Struct, p.Name)
201+
}

‎sql/expression/unresolved.go

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,3 +139,52 @@ func (uf *UnresolvedFunction) TransformUp(f sql.TransformExprFunc) (sql.Expressi
139139

140140
return f(NewUnresolvedFunction(uf.name, uf.IsAggregate, rc...))
141141
}
142+
143+
// UnresolvedField is an unresolved expression to get a field from a struct column.
144+
type UnresolvedField struct {
145+
Struct sql.Expression
146+
Name string
147+
}
148+
149+
// NewUnresolvedField creates a new UnresolvedField expression.
150+
func NewUnresolvedField(s sql.Expression, fieldName string) *UnresolvedField {
151+
return &UnresolvedField{s, fieldName}
152+
}
153+
154+
// Children implements the Expression interface.
155+
func (p *UnresolvedField) Children() []sql.Expression {
156+
return []sql.Expression{p.Struct}
157+
}
158+
159+
// Resolved implements the Expression interface.
160+
func (p *UnresolvedField) Resolved() bool {
161+
return false
162+
}
163+
164+
// IsNullable returns whether the field is nullable or not.
165+
func (p *UnresolvedField) IsNullable() bool {
166+
panic("unresolved field is a placeholder node, but IsNullable was called")
167+
}
168+
169+
// Type returns the type of the field.
170+
func (p *UnresolvedField) Type() sql.Type {
171+
panic("unresolved field is a placeholder node, but Type was called")
172+
}
173+
174+
// Eval implements the Expression interface.
175+
func (p *UnresolvedField) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) {
176+
panic("unresolved field is a placeholder node, but Eval was called")
177+
}
178+
179+
// TransformUp implements the Expression interface.
180+
func (p *UnresolvedField) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) {
181+
s, err := p.Struct.TransformUp(f)
182+
if err != nil {
183+
return nil, err
184+
}
185+
return f(NewUnresolvedField(s, p.Name))
186+
}
187+
188+
func (p *UnresolvedField) String() string {
189+
return fmt.Sprintf("%s.%s", p.Struct, p.Name)
190+
}

‎sql/parse/parse.go

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -753,6 +753,20 @@ func exprToExpression(e sqlparser.Expr) (sql.Expression, error) {
753753
return expression.NewLiteral(nil, sql.Null), nil
754754
case *sqlparser.ColName:
755755
if !v.Qualifier.IsEmpty() {
756+
// If we find something of the form A.B.C we're going to treat it
757+
// as a struct field access.
758+
// TODO: this should be handled better when GetFields support being
759+
// qualified with the database.
760+
if !v.Qualifier.Qualifier.IsEmpty() {
761+
return expression.NewUnresolvedField(
762+
expression.NewUnresolvedQualifiedColumn(
763+
v.Qualifier.Qualifier.String(),
764+
v.Qualifier.Name.String(),
765+
),
766+
v.Name.String(),
767+
), nil
768+
}
769+
756770
return expression.NewUnresolvedQualifiedColumn(
757771
v.Qualifier.Name.String(),
758772
v.Name.String(),

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy