Skip to content
This repository was archived by the owner on Jan 28, 2021. It is now read-only.

Commit cf3c34d

Browse files
erizocosmicoerizocosmico
committed
sql: implement struct types
This PR implements struct types, which will be serialized to JSON before being sent to the mysql client using the mysql wire proto. Internally, structs are just `map[string]interface{}`, but they can actually be anything that's convertible to that, because of the way the Convert method of the struct type is implemented. That means, the result of an UDF or a table that returns a struct may be an actual Go struct and it will then be transformed into the internal map. It does have a penalty, though, because structs require encoding to JSON and then decoding into `map[string]interface{}`. Structs have a schema, identical to a table schema, except their `Source` will always be empty. Resolution of columns has also been slightly change in order to resolve getting fields from structs using the `.` operator, which required some trade-offs in some rules, such as not erroring anymore in `qualify_columns` when the table is not found. That error was delegated to `resolve_columns` in order to make resolution possible, as the syntax is a bit ambiguous. The advantage of using dot is the fact that no changes have to be made to the parser in order for it to work. Signed-off-by: Miguel Molina <miguel@erizocosmi.co>
1 parent 550cc54 commit cf3c34d

File tree

10 files changed

+503
-17
lines changed

10 files changed

+503
-17
lines changed

‎engine_test.go

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2786,6 +2786,64 @@ func TestGenerators(t *testing.T) {
27862786
}
27872787
}
27882788

2789+
var structQueries = []struct {
2790+
query string
2791+
expected []sql.Row
2792+
}{
2793+
{
2794+
`SELECT s.i, t.s.t FROM t ORDER BY s.i`,
2795+
[]sql.Row{
2796+
{int64(1), "first"},
2797+
{int64(2), "second"},
2798+
{int64(3), "third"},
2799+
},
2800+
},
2801+
{
2802+
`SELECT s.i, s.t FROM t ORDER BY s.i`,
2803+
[]sql.Row{
2804+
{int64(1), "first"},
2805+
{int64(2), "second"},
2806+
{int64(3), "third"},
2807+
},
2808+
},
2809+
{
2810+
`SELECT s.i, COUNT(*) FROM t GROUP BY s.i`,
2811+
[]sql.Row{
2812+
{int64(1), int64(1)},
2813+
{int64(2), int64(1)},
2814+
{int64(3), int64(1)},
2815+
},
2816+
},
2817+
}
2818+
2819+
func TestStructs(t *testing.T) {
2820+
schema := sql.Schema{
2821+
{Name: "i", Type: sql.Int64},
2822+
{Name: "t", Type: sql.Text},
2823+
}
2824+
table := mem.NewPartitionedTable("t", sql.Schema{
2825+
{Name: "s", Type: sql.Struct(schema), Source: "t"},
2826+
}, testNumPartitions)
2827+
2828+
insertRows(
2829+
t, table,
2830+
sql.NewRow(map[string]interface{}{"i": int64(1), "t": "first"}),
2831+
sql.NewRow(map[string]interface{}{"i": int64(2), "t": "second"}),
2832+
sql.NewRow(map[string]interface{}{"i": int64(3), "t": "third"}),
2833+
)
2834+
2835+
db := mem.NewDatabase("db")
2836+
db.AddTable("t", table)
2837+
2838+
catalog := sql.NewCatalog()
2839+
catalog.AddDatabase(db)
2840+
e := sqle.New(catalog, analyzer.NewDefault(catalog), new(sqle.Config))
2841+
2842+
for _, q := range structQueries {
2843+
testQuery(t, e, q.query, q.expected)
2844+
}
2845+
}
2846+
27892847
func insertRows(t *testing.T, table sql.Inserter, rows ...sql.Row) {
27902848
t.Helper()
27912849

‎sql/analyzer/resolve_columns.go

Lines changed: 85 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import (
55
"sort"
66
"strings"
77

8-
"github.com/src-d/go-mysql-server/internal/similartext"
98
"github.com/src-d/go-mysql-server/sql"
109
"github.com/src-d/go-mysql-server/sql/expression"
1110
"github.com/src-d/go-mysql-server/sql/plan"
@@ -153,17 +152,24 @@ func qualifyExpression(
153152
return col, nil
154153
}
155154

156-
name, table := strings.ToLower(col.Name()), strings.ToLower(col.Table())
155+
name, tableName := strings.ToLower(col.Name()), strings.ToLower(col.Table())
157156
availableTables := dedupStrings(columns[name])
158-
if table != "" {
159-
table, ok := tables[table]
157+
if tableName != "" {
158+
table, ok := tables[tableName]
160159
if !ok {
161-
if len(tables) == 0 {
162-
return nil, sql.ErrTableNotFound.New(col.Table())
160+
// If the table does not exist but the column does, it may be a
161+
// struct field access.
162+
if columnExists(columns, tableName) {
163+
return expression.NewUnresolvedField(
164+
expression.NewUnresolvedColumn(col.Table()),
165+
col.Name(),
166+
), nil
163167
}
164168

165-
similar := similartext.FindFromMap(tables, col.Table())
166-
return nil, sql.ErrTableNotFound.New(col.Table() + similar)
169+
// If it cannot be resolved, then pass along and let it fail
170+
// somewhere else. Maybe we're missing some steps before this
171+
// can be resolved.
172+
return col, nil
167173
}
168174

169175
// If the table exists but it's not available for this node it
@@ -213,6 +219,15 @@ func qualifyExpression(
213219
}
214220
}
215221

222+
func columnExists(columns map[string][]string, col string) bool {
223+
for c := range columns {
224+
if strings.ToLower(c) == strings.ToLower(col) {
225+
return true
226+
}
227+
}
228+
return false
229+
}
230+
216231
func getNodeAvailableColumns(n sql.Node) map[string][]string {
217232
var columns = make(map[string][]string)
218233
getColumnsInNodes(n.Children(), columns)
@@ -374,7 +389,23 @@ func resolveColumnExpression(
374389
return &deferredColumn{uc}, nil
375390
default:
376391
if table != "" {
377-
return nil, ErrColumnTableNotFound.New(e.Table(), e.Name())
392+
if isStructField(uc, columns) {
393+
return expression.NewUnresolvedField(
394+
expression.NewUnresolvedColumn(uc.Table()),
395+
uc.Name(),
396+
), nil
397+
}
398+
399+
// If we manage to find any column with the given table, it's because
400+
// the column does not exist.
401+
for col := range columns {
402+
if col.table == table {
403+
return nil, ErrColumnTableNotFound.New(e.Table(), e.Name())
404+
}
405+
}
406+
407+
// In any other case, it's the table the one that does not exist.
408+
return nil, sql.ErrTableNotFound.New(e.Table())
378409
}
379410

380411
return nil, ErrColumnNotFound.New(e.Name())
@@ -390,6 +421,51 @@ func resolveColumnExpression(
390421
), nil
391422
}
392423

424+
func isStructField(c column, columns map[tableCol]indexedCol) bool {
425+
for _, col := range columns {
426+
if strings.ToLower(c.Table()) == strings.ToLower(col.Name) &&
427+
sql.Field(col.Type, c.Name()) != nil {
428+
return true
429+
}
430+
}
431+
return false
432+
}
433+
434+
var errFieldNotFound = errors.NewKind("field %s not found on struct %s")
435+
436+
func resolveStructFields(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) {
437+
span, _ := ctx.Span("resolve_struct_fields")
438+
defer span.Finish()
439+
440+
return plan.TransformUp(n, func(n sql.Node) (sql.Node, error) {
441+
if n.Resolved() {
442+
return n, nil
443+
}
444+
445+
if _, ok := n.(sql.Expressioner); !ok {
446+
return n, nil
447+
}
448+
449+
return plan.TransformExpressions(n, func(e sql.Expression) (sql.Expression, error) {
450+
f, ok := e.(*expression.UnresolvedField)
451+
if !ok {
452+
return e, nil
453+
}
454+
455+
if !f.Struct.Resolved() {
456+
return e, nil
457+
}
458+
459+
field := sql.Field(f.Struct.Type(), f.Name)
460+
if field == nil {
461+
return nil, errFieldNotFound.New(f.Name, f.Struct)
462+
}
463+
464+
return expression.NewGetStructField(f.Struct, f.Name), nil
465+
})
466+
})
467+
}
468+
393469
// resolveGroupingColumns reorders the aggregation in a groupby so aliases
394470
// defined in it can be resolved in the grouping of the groupby. To do so,
395471
// all aliases are pushed down to a projection node under the group by.

‎sql/analyzer/resolve_columns_test.go

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -186,14 +186,24 @@ func TestQualifyColumns(t *testing.T) {
186186

187187
node = plan.NewProject(
188188
[]sql.Expression{
189-
expression.NewUnresolvedQualifiedColumn("foo", "i"),
189+
expression.NewUnresolvedQualifiedColumn("i", "some_field"),
190190
},
191-
plan.NewTableAlias("a", plan.NewResolvedTable(table)),
191+
plan.NewResolvedTable(table),
192192
)
193193

194-
_, err = f.Apply(sql.NewEmptyContext(), nil, node)
195-
require.Error(err)
196-
require.True(sql.ErrTableNotFound.Is(err))
194+
expected = plan.NewProject(
195+
[]sql.Expression{
196+
expression.NewUnresolvedField(
197+
expression.NewUnresolvedColumn("i"),
198+
"some_field",
199+
),
200+
},
201+
plan.NewResolvedTable(table),
202+
)
203+
204+
result, err = f.Apply(sql.NewEmptyContext(), nil, node)
205+
require.NoError(err)
206+
require.Equal(expected, result)
197207

198208
node = plan.NewProject(
199209
[]sql.Expression{

‎sql/analyzer/rules.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ var DefaultRules = []Rule{
1212
{"resolve_grouping_columns", resolveGroupingColumns},
1313
{"qualify_columns", qualifyColumns},
1414
{"resolve_columns", resolveColumns},
15+
{"resolve_struct_fields", resolveStructFields},
1516
{"resolve_database", resolveDatabase},
1617
{"resolve_star", resolveStar},
1718
{"resolve_functions", resolveFunctions},

‎sql/expression/get_field.go

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,3 +133,73 @@ func (f *GetSessionField) WithChildren(children ...sql.Expression) (sql.Expressi
133133
}
134134
return f, nil
135135
}
136+
137+
// GetStructField is an expression to get a field from a struct column.
138+
type GetStructField struct {
139+
Struct sql.Expression
140+
Name string
141+
}
142+
143+
// NewGetStructField creates a new GetStructField expression.
144+
func NewGetStructField(s sql.Expression, fieldName string) *GetStructField {
145+
return &GetStructField{s, fieldName}
146+
}
147+
148+
// Children implements the Expression interface.
149+
func (p *GetStructField) Children() []sql.Expression {
150+
return []sql.Expression{p.Struct}
151+
}
152+
153+
// Resolved implements the Expression interface.
154+
func (p *GetStructField) Resolved() bool {
155+
return p.Struct.Resolved()
156+
}
157+
158+
func (p *GetStructField) column() *sql.Column {
159+
return sql.Field(p.Struct.Type(), p.Name)
160+
}
161+
162+
// IsNullable returns whether the field is nullable or not.
163+
func (p *GetStructField) IsNullable() bool {
164+
return p.Struct.IsNullable() || p.column().Nullable
165+
}
166+
167+
// Type returns the type of the field.
168+
func (p *GetStructField) Type() sql.Type {
169+
return p.column().Type
170+
}
171+
172+
// Eval implements the Expression interface.
173+
func (p *GetStructField) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) {
174+
s, err := p.Struct.Eval(ctx, row)
175+
if err != nil {
176+
return nil, err
177+
}
178+
179+
if s == nil {
180+
return nil, nil
181+
}
182+
183+
s, err = p.Struct.Type().Convert(s)
184+
if err != nil {
185+
return nil, err
186+
}
187+
188+
if val, ok := s.(map[string]interface{})[p.Name]; ok {
189+
return p.Type().Convert(val)
190+
}
191+
192+
return nil, nil
193+
}
194+
195+
// WithChildren implements the Expression interface.
196+
func (p *GetStructField) WithChildren(children ...sql.Expression) (sql.Expression, error) {
197+
if len(children) != 1 {
198+
return nil, sql.ErrInvalidChildrenNumber.New(p, len(children), 1)
199+
}
200+
return &GetStructField{children[0], p.Name}, nil
201+
}
202+
203+
func (p *GetStructField) String() string {
204+
return fmt.Sprintf("%s.%s", p.Struct, p.Name)
205+
}

‎sql/expression/unresolved.go

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,3 +135,51 @@ func (uf *UnresolvedFunction) WithChildren(children ...sql.Expression) (sql.Expr
135135
}
136136
return NewUnresolvedFunction(uf.name, uf.IsAggregate, children...), nil
137137
}
138+
139+
// UnresolvedField is an unresolved expression to get a field from a struct column.
140+
type UnresolvedField struct {
141+
Struct sql.Expression
142+
Name string
143+
}
144+
145+
// NewUnresolvedField creates a new UnresolvedField expression.
146+
func NewUnresolvedField(s sql.Expression, fieldName string) *UnresolvedField {
147+
return &UnresolvedField{s, fieldName}
148+
}
149+
150+
// Children implements the Expression interface.
151+
func (p *UnresolvedField) Children() []sql.Expression {
152+
return []sql.Expression{p.Struct}
153+
}
154+
155+
// Resolved implements the Expression interface.
156+
func (p *UnresolvedField) Resolved() bool {
157+
return false
158+
}
159+
160+
// IsNullable returns whether the field is nullable or not.
161+
func (p *UnresolvedField) IsNullable() bool {
162+
panic("unresolved field is a placeholder node, but IsNullable was called")
163+
}
164+
165+
// Type returns the type of the field.
166+
func (p *UnresolvedField) Type() sql.Type {
167+
panic("unresolved field is a placeholder node, but Type was called")
168+
}
169+
170+
// Eval implements the Expression interface.
171+
func (p *UnresolvedField) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) {
172+
panic("unresolved field is a placeholder node, but Eval was called")
173+
}
174+
175+
// WithChildren implements the Expression interface.
176+
func (p *UnresolvedField) WithChildren(children ...sql.Expression) (sql.Expression, error) {
177+
if len(children) != 1 {
178+
return nil, sql.ErrInvalidChildrenNumber.New(p, len(children), 1)
179+
}
180+
return &UnresolvedField{children[0], p.Name}, nil
181+
}
182+
183+
func (p *UnresolvedField) String() string {
184+
return fmt.Sprintf("%s.%s", p.Struct, p.Name)
185+
}

‎sql/parse/parse.go

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -779,6 +779,20 @@ func exprToExpression(e sqlparser.Expr) (sql.Expression, error) {
779779
return expression.NewLiteral(nil, sql.Null), nil
780780
case *sqlparser.ColName:
781781
if !v.Qualifier.IsEmpty() {
782+
// If we find something of the form A.B.C we're going to treat it
783+
// as a struct field access.
784+
// TODO: this should be handled better when GetFields support being
785+
// qualified with the database.
786+
if !v.Qualifier.Qualifier.IsEmpty() {
787+
return expression.NewUnresolvedField(
788+
expression.NewUnresolvedQualifiedColumn(
789+
v.Qualifier.Qualifier.String(),
790+
v.Qualifier.Name.String(),
791+
),
792+
v.Name.String(),
793+
), nil
794+
}
795+
782796
return expression.NewUnresolvedQualifiedColumn(
783797
v.Qualifier.Name.String(),
784798
v.Name.String(),

‎sql/parse/parse_test.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1173,6 +1173,15 @@ var fixtures = map[string]sql.Node{
11731173
[]sql.Expression{},
11741174
plan.NewUnresolvedTable("foo", ""),
11751175
),
1176+
`SELECT a.b.c FROM a`: plan.NewProject(
1177+
[]sql.Expression{
1178+
expression.NewUnresolvedField(
1179+
expression.NewUnresolvedQualifiedColumn("a", "b"),
1180+
"c",
1181+
),
1182+
},
1183+
plan.NewUnresolvedTable("a", ""),
1184+
),
11761185
}
11771186

11781187
func TestParse(t *testing.T) {

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy